@ai-sdk/google 4.0.78 → 4.0.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.d.ts +13 -1
- package/dist/index.js +573 -397
- package/dist/index.js.map +1 -1
- package/dist/internal/index.d.ts +26 -2
- package/dist/internal/index.js +370 -219
- package/dist/internal/index.js.map +1 -1
- package/docs/15-google.mdx +85 -14
- package/package.json +2 -2
- package/src/convert-to-google-messages.ts +28 -0
- package/src/google-embedding-model.ts +39 -8
- package/src/google-json-accumulator.ts +0 -1
- package/src/google-prompt.ts +12 -0
- package/src/google-speech-api.ts +2 -1
- package/src/google-speech-input.ts +64 -0
- package/src/google-speech-model-options.ts +28 -1
- package/src/google-speech-model.ts +137 -25
- package/src/internal/index.ts +1 -0
- package/src/tool/code-execution.ts +14 -10
package/docs/15-google.mdx
CHANGED
|
@@ -2233,45 +2233,100 @@ console.log(result.providerMetadata?.google?.groundingMetadata);
|
|
|
2233
2233
|
You can create models that call the [Gemini text-to-speech API](https://ai.google.dev/gemini-api/docs/speech-generation)
|
|
2234
2234
|
using the `.speech()` factory method.
|
|
2235
2235
|
|
|
2236
|
-
The first argument is the model
|
|
2236
|
+
The first argument is the model ID, such as `gemini-3.8-flash-tts` or
|
|
2237
|
+
`gemini-3.8-flash-lite-tts`. Speech generation uses the `generateContent` API.
|
|
2237
2238
|
|
|
2238
2239
|
```ts
|
|
2239
|
-
const model = google.speech('gemini-
|
|
2240
|
+
const model = google.speech('gemini-3.8-flash-tts');
|
|
2240
2241
|
```
|
|
2241
2242
|
|
|
2242
2243
|
The `voice` argument can be set to one of Gemini's [30 prebuilt voices](https://ai.google.dev/gemini-api/docs/speech-generation#voices)
|
|
2243
|
-
e.g. `Kore`, `Puck`, `Zephyr`, or `Charon`. Voice names are case-sensitive.
|
|
2244
|
+
e.g. `Kore`, `Puck`, `Zephyr`, or `Charon`. Voice names are case-sensitive.
|
|
2245
|
+
The default voice is `Kore`. Custom voice IDs and replication keys are not supported.
|
|
2244
2246
|
|
|
2245
2247
|
```ts highlight="6"
|
|
2246
2248
|
import { generateSpeech } from 'ai';
|
|
2247
2249
|
import { google } from '@ai-sdk/google';
|
|
2248
2250
|
|
|
2249
2251
|
const result = await generateSpeech({
|
|
2250
|
-
model: google.speech('gemini-
|
|
2252
|
+
model: google.speech('gemini-3.8-flash-tts'),
|
|
2251
2253
|
text: 'Hello, world!',
|
|
2252
2254
|
voice: 'Kore', // Gemini voice name
|
|
2253
2255
|
});
|
|
2254
2256
|
```
|
|
2255
2257
|
|
|
2256
|
-
By default the generated audio is returned as a playable WAV file
|
|
2257
|
-
`audio/wav`).
|
|
2258
|
-
bytes
|
|
2258
|
+
By default the generated audio is returned as a playable WAV file
|
|
2259
|
+
(`result.audio.mediaType` is `audio/wav`). Gemini 3.8 returns a complete WAV file;
|
|
2260
|
+
the provider passes those bytes through unchanged. Older models return raw PCM,
|
|
2261
|
+
which the provider wraps in a WAV container. Write `result.audio.uint8Array`
|
|
2262
|
+
directly to disk without adding another WAV header.
|
|
2259
2263
|
|
|
2260
|
-
Gemini
|
|
2261
|
-
|
|
2264
|
+
Gemini 3.8 supports the following `outputFormat` values:
|
|
2265
|
+
|
|
2266
|
+
| Format | Values | Result |
|
|
2267
|
+
| ------ | ---------------------- | ---------------------------- |
|
|
2268
|
+
| WAV | `wav`, `audio/wav` | WAV with a RIFF header |
|
|
2269
|
+
| PCM | `pcm`, `audio/l16` | Headerless signed 16-bit PCM |
|
|
2270
|
+
| Mu-law | `mulaw`, `audio/mulaw` | Headerless mu-law audio |
|
|
2271
|
+
| A-law | `alaw`, `audio/alaw` | Headerless A-law audio |
|
|
2272
|
+
|
|
2273
|
+
These map to `generationConfig.responseFormat.audio.mimeType` in
|
|
2274
|
+
`generateContent`. Older models support `wav` and `pcm`. The source MIME type and
|
|
2275
|
+
sample rate are available in `result.providerMetadata.google`.
|
|
2276
|
+
|
|
2277
|
+
Gemini 2.5 and 3.1 model IDs retain the legacy request format. Other model IDs,
|
|
2278
|
+
including custom aliases, use the structured speech format.
|
|
2279
|
+
|
|
2280
|
+
### Speech directions in Gemini 3.8
|
|
2281
|
+
|
|
2282
|
+
Gemini 3.8 treats text as a verbatim transcript. The provider sends `instructions`
|
|
2283
|
+
as `speechMetadata.style`, leaving the text unchanged. You can also supply
|
|
2284
|
+
`providerOptions.google.speechMetadata` with `style` and `speaker` fields.
|
|
2285
|
+
An explicit style overrides `instructions`, including an empty style string.
|
|
2286
|
+
|
|
2287
|
+
```ts
|
|
2288
|
+
const result = await generateSpeech({
|
|
2289
|
+
model: google.speech('gemini-3.8-flash-tts'),
|
|
2290
|
+
text: 'Welcome back. <short pause> It is good to see you. <laugh>',
|
|
2291
|
+
instructions: 'Warm, relaxed, and speaking slowly',
|
|
2292
|
+
});
|
|
2293
|
+
```
|
|
2294
|
+
|
|
2295
|
+
Put sustained delivery directions such as whispering, speaking slowly, or being
|
|
2296
|
+
out of breath in metadata. Keep momentary vocal events and pauses such as
|
|
2297
|
+
`<laugh>`, `<sigh>`, `<cough>`, `<breath>`, and `<short pause>` in the transcript.
|
|
2298
|
+
Avoid sound-effect tags such as applause or thuds. The provider preserves inline
|
|
2299
|
+
tags as written.
|
|
2300
|
+
|
|
2301
|
+
For Gemini 3.1 and earlier, the provider retains the existing behavior of
|
|
2302
|
+
prepending `instructions` to single-speaker text. It ignores instructions with a
|
|
2303
|
+
warning for older multi-speaker requests. Structured `turns` and `speechMetadata`
|
|
2304
|
+
require Gemini 3.8.
|
|
2262
2305
|
|
|
2263
2306
|
### Multi-speaker audio
|
|
2264
2307
|
|
|
2265
|
-
For
|
|
2266
|
-
|
|
2308
|
+
For Gemini 3.8 dialogue, pass `multiSpeakerVoiceConfig` and `turns` through
|
|
2309
|
+
`providerOptions.google`. Every turn must include `speechMetadata.speaker`
|
|
2310
|
+
matching a configured speaker; missing or unknown speakers produce an error
|
|
2311
|
+
before the request is sent. Speaker labels belong in metadata, not the text.
|
|
2312
|
+
|
|
2313
|
+
The `turns` array replaces the top-level text, so pass `text: ''`. The combined
|
|
2314
|
+
turn text must be non-empty. Passing
|
|
2315
|
+
nonempty top-level text with turns produces a warning. Put metadata on each turn
|
|
2316
|
+
instead of combining turns with top-level `speechMetadata`. Each turn's style
|
|
2317
|
+
overrides `instructions`; turns without a style inherit `instructions`.
|
|
2318
|
+
|
|
2319
|
+
`multiSpeakerVoiceConfig` overrides the top-level `voice`. Each `voiceConfig`
|
|
2320
|
+
uses `prebuiltVoiceConfig: { voiceName: 'Kore' }`. Google supports up to two
|
|
2321
|
+
prebuilt voices in a multi-speaker request.
|
|
2267
2322
|
|
|
2268
|
-
```ts
|
|
2323
|
+
```ts
|
|
2269
2324
|
import { generateSpeech } from 'ai';
|
|
2270
2325
|
import { google, type GoogleSpeechModelOptions } from '@ai-sdk/google';
|
|
2271
2326
|
|
|
2272
2327
|
const result = await generateSpeech({
|
|
2273
|
-
model: google.speech('gemini-
|
|
2274
|
-
text: '
|
|
2328
|
+
model: google.speech('gemini-3.8-flash-lite-tts'),
|
|
2329
|
+
text: '',
|
|
2275
2330
|
providerOptions: {
|
|
2276
2331
|
google: {
|
|
2277
2332
|
multiSpeakerVoiceConfig: {
|
|
@@ -2286,11 +2341,25 @@ const result = await generateSpeech({
|
|
|
2286
2341
|
},
|
|
2287
2342
|
],
|
|
2288
2343
|
},
|
|
2344
|
+
turns: [
|
|
2345
|
+
{
|
|
2346
|
+
text: 'How are you?',
|
|
2347
|
+
speechMetadata: { speaker: 'Joe', style: 'curious' },
|
|
2348
|
+
},
|
|
2349
|
+
{
|
|
2350
|
+
text: '<laugh> Doing great, thanks!',
|
|
2351
|
+
speechMetadata: { speaker: 'Jane', style: 'cheerful' },
|
|
2352
|
+
},
|
|
2353
|
+
],
|
|
2289
2354
|
} satisfies GoogleSpeechModelOptions,
|
|
2290
2355
|
},
|
|
2291
2356
|
});
|
|
2292
2357
|
```
|
|
2293
2358
|
|
|
2359
|
+
Older models continue to use labelled transcripts, such as
|
|
2360
|
+
`text: 'Joe: How are you? Jane: Doing great, thanks!'`, with
|
|
2361
|
+
`multiSpeakerVoiceConfig` and without `turns`.
|
|
2362
|
+
|
|
2294
2363
|
<Note>
|
|
2295
2364
|
Gemini TTS models do not support the `speed` or `language` options; passing
|
|
2296
2365
|
them adds a warning to `result.warnings`. Language is detected automatically
|
|
@@ -2304,6 +2373,8 @@ const result = await generateSpeech({
|
|
|
2304
2373
|
| `gemini-2.5-flash-preview-tts` | <Check /> | <Check /> |
|
|
2305
2374
|
| `gemini-2.5-pro-preview-tts` | <Check /> | <Check /> |
|
|
2306
2375
|
| `gemini-3.1-flash-tts-preview` | <Check /> | <Check /> |
|
|
2376
|
+
| `gemini-3.8-flash-tts` | <Check /> | <Check /> |
|
|
2377
|
+
| `gemini-3.8-flash-lite-tts` | <Check /> | <Check /> |
|
|
2307
2378
|
|
|
2308
2379
|
## Evaluation Models
|
|
2309
2380
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ai-sdk/google",
|
|
3
|
-
"version": "4.0.
|
|
3
|
+
"version": "4.0.79",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
38
|
"@ai-sdk/provider": "4.0.18",
|
|
39
|
-
"@ai-sdk/provider-utils": "5.0.
|
|
39
|
+
"@ai-sdk/provider-utils": "5.0.47"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@ai-sdk/test-server": "2.0.1",
|
|
@@ -18,6 +18,10 @@ import type {
|
|
|
18
18
|
GoogleFunctionResponsePart,
|
|
19
19
|
GooglePrompt,
|
|
20
20
|
} from './google-prompt';
|
|
21
|
+
import {
|
|
22
|
+
codeExecutionInputSchema,
|
|
23
|
+
codeExecutionOutputSchema,
|
|
24
|
+
} from './tool/code-execution';
|
|
21
25
|
|
|
22
26
|
/**
|
|
23
27
|
* Sentinel value Google documents for replaying functionCall parts whose
|
|
@@ -465,6 +469,19 @@ export function convertToGoogleMessages(
|
|
|
465
469
|
}
|
|
466
470
|
|
|
467
471
|
case 'tool-call': {
|
|
472
|
+
if (
|
|
473
|
+
part.providerExecuted === true &&
|
|
474
|
+
part.toolName === 'code_execution'
|
|
475
|
+
) {
|
|
476
|
+
return {
|
|
477
|
+
executableCode: codeExecutionInputSchema.parse(
|
|
478
|
+
typeof part.input === 'string'
|
|
479
|
+
? secureJsonParse(part.input)
|
|
480
|
+
: part.input,
|
|
481
|
+
),
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
|
|
468
485
|
const serverToolCallId =
|
|
469
486
|
providerOpts?.serverToolCallId != null
|
|
470
487
|
? String(providerOpts.serverToolCallId)
|
|
@@ -520,6 +537,17 @@ export function convertToGoogleMessages(
|
|
|
520
537
|
}
|
|
521
538
|
|
|
522
539
|
case 'tool-result': {
|
|
540
|
+
if (
|
|
541
|
+
part.toolName === 'code_execution' &&
|
|
542
|
+
part.output.type === 'json'
|
|
543
|
+
) {
|
|
544
|
+
return {
|
|
545
|
+
codeExecutionResult: codeExecutionOutputSchema.parse(
|
|
546
|
+
part.output.value,
|
|
547
|
+
),
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
|
|
523
551
|
const serverToolCallId =
|
|
524
552
|
providerOpts?.serverToolCallId != null
|
|
525
553
|
? String(providerOpts.serverToolCallId)
|
|
@@ -5,6 +5,8 @@ import {
|
|
|
5
5
|
import {
|
|
6
6
|
combineHeaders,
|
|
7
7
|
createJsonResponseHandler,
|
|
8
|
+
type EmbeddingModelProviderOptionsTransformer,
|
|
9
|
+
EXPERIMENTAL_EMBEDDING_MODEL_PROVIDER_OPTIONS_TRANSFORMER,
|
|
8
10
|
lazySchema,
|
|
9
11
|
parseProviderOptions,
|
|
10
12
|
postJsonToApi,
|
|
@@ -33,6 +35,28 @@ export class GoogleEmbeddingModel implements EmbeddingModelV4 {
|
|
|
33
35
|
readonly modelId: GoogleEmbeddingModelId;
|
|
34
36
|
readonly maxEmbeddingsPerCall = 100;
|
|
35
37
|
readonly supportsParallelCalls = true;
|
|
38
|
+
readonly [EXPERIMENTAL_EMBEDDING_MODEL_PROVIDER_OPTIONS_TRANSFORMER]: EmbeddingModelProviderOptionsTransformer =
|
|
39
|
+
({ providerOptions, values, startIndex, endIndex }) => {
|
|
40
|
+
const multimodalContent = providerOptions?.google?.content;
|
|
41
|
+
|
|
42
|
+
// Leave schema validation to doEmbed, after middleware transforms options.
|
|
43
|
+
if (!Array.isArray(multimodalContent)) {
|
|
44
|
+
return providerOptions;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
validateMultimodalContentLength({
|
|
48
|
+
multimodalContent,
|
|
49
|
+
values,
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
...providerOptions,
|
|
54
|
+
google: {
|
|
55
|
+
...providerOptions?.google,
|
|
56
|
+
content: multimodalContent.slice(startIndex, endIndex),
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
};
|
|
36
60
|
|
|
37
61
|
private readonly config: GoogleEmbeddingConfig;
|
|
38
62
|
|
|
@@ -89,14 +113,7 @@ export class GoogleEmbeddingModel implements EmbeddingModelV4 {
|
|
|
89
113
|
|
|
90
114
|
const multimodalContent = googleOptions?.content;
|
|
91
115
|
|
|
92
|
-
|
|
93
|
-
multimodalContent != null &&
|
|
94
|
-
multimodalContent.length !== values.length
|
|
95
|
-
) {
|
|
96
|
-
throw new Error(
|
|
97
|
-
`The number of multimodal content entries (${multimodalContent.length}) must match the number of values (${values.length}).`,
|
|
98
|
-
);
|
|
99
|
-
}
|
|
116
|
+
validateMultimodalContentLength({ multimodalContent, values });
|
|
100
117
|
|
|
101
118
|
// For single embeddings, use the single endpoint
|
|
102
119
|
if (values.length === 1) {
|
|
@@ -199,3 +216,17 @@ const googleGenerativeAISingleEmbeddingResponseSchema = lazySchema(() =>
|
|
|
199
216
|
}),
|
|
200
217
|
),
|
|
201
218
|
);
|
|
219
|
+
|
|
220
|
+
function validateMultimodalContentLength({
|
|
221
|
+
multimodalContent,
|
|
222
|
+
values,
|
|
223
|
+
}: {
|
|
224
|
+
multimodalContent: Array<unknown> | undefined;
|
|
225
|
+
values: Array<string>;
|
|
226
|
+
}) {
|
|
227
|
+
if (multimodalContent != null && multimodalContent.length !== values.length) {
|
|
228
|
+
throw new Error(
|
|
229
|
+
`The number of multimodal content entries (${multimodalContent.length}) must match the number of values (${values.length}).`,
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
@@ -367,5 +367,4 @@ function resolvePartialArgValue(arg: {
|
|
|
367
367
|
const value = arg.stringValue ?? arg.numberValue ?? arg.boolValue;
|
|
368
368
|
if (value != null) return { value, json: JSON.stringify(value) };
|
|
369
369
|
if ('nullValue' in arg) return { value: null, json: 'null' };
|
|
370
|
-
return undefined;
|
|
371
370
|
}
|
package/src/google-prompt.ts
CHANGED
|
@@ -59,6 +59,18 @@ export type GoogleContentPart =
|
|
|
59
59
|
id: string;
|
|
60
60
|
};
|
|
61
61
|
thoughtSignature?: string;
|
|
62
|
+
}
|
|
63
|
+
| {
|
|
64
|
+
executableCode: {
|
|
65
|
+
language: string;
|
|
66
|
+
code: string;
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
| {
|
|
70
|
+
codeExecutionResult: {
|
|
71
|
+
outcome: string;
|
|
72
|
+
output: string;
|
|
73
|
+
};
|
|
62
74
|
};
|
|
63
75
|
|
|
64
76
|
export type GoogleFunctionResponsePart = {
|
package/src/google-speech-api.ts
CHANGED
|
@@ -4,7 +4,8 @@ import { z } from 'zod/v4';
|
|
|
4
4
|
/**
|
|
5
5
|
* Response schema for the Gemini `:generateContent` endpoint when called with
|
|
6
6
|
* `responseModalities: ['AUDIO']`. The generated audio is returned as base64
|
|
7
|
-
* encoded
|
|
7
|
+
* encoded audio in the first inline-data part (WAV by default on Gemini 3.8,
|
|
8
|
+
* raw PCM on older models).
|
|
8
9
|
*/
|
|
9
10
|
export const googleSpeechResponseSchema = lazySchema(() =>
|
|
10
11
|
zodSchema(
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inspects speech input without replacing provider option validation. This is
|
|
3
|
+
* also used by intermediaries that need the transcript or voice kind before
|
|
4
|
+
* invoking a model. Model support for structured turns is validated separately.
|
|
5
|
+
*/
|
|
6
|
+
export function getGoogleSpeechInput({
|
|
7
|
+
text,
|
|
8
|
+
voice,
|
|
9
|
+
providerOptions,
|
|
10
|
+
}: {
|
|
11
|
+
text: string;
|
|
12
|
+
voice?: string;
|
|
13
|
+
providerOptions?: Record<string, unknown>;
|
|
14
|
+
}): { text: string; usesCustomVoice: boolean } {
|
|
15
|
+
const google = providerOptions?.google;
|
|
16
|
+
const options =
|
|
17
|
+
google != null && typeof google === 'object' ? google : undefined;
|
|
18
|
+
const turns = options && 'turns' in options ? options.turns : undefined;
|
|
19
|
+
const turnTexts: string[] = [];
|
|
20
|
+
if (Array.isArray(turns) && turns.length > 0) {
|
|
21
|
+
for (const turn of turns as unknown[]) {
|
|
22
|
+
if (
|
|
23
|
+
turn == null ||
|
|
24
|
+
typeof turn !== 'object' ||
|
|
25
|
+
!('text' in turn) ||
|
|
26
|
+
typeof turn.text !== 'string'
|
|
27
|
+
) {
|
|
28
|
+
break;
|
|
29
|
+
}
|
|
30
|
+
turnTexts.push(turn.text);
|
|
31
|
+
}
|
|
32
|
+
if (turnTexts.length === turns.length) {
|
|
33
|
+
text = turnTexts.join('');
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const config =
|
|
38
|
+
options && 'multiSpeakerVoiceConfig' in options
|
|
39
|
+
? options.multiSpeakerVoiceConfig
|
|
40
|
+
: undefined;
|
|
41
|
+
const speakers =
|
|
42
|
+
config != null &&
|
|
43
|
+
typeof config === 'object' &&
|
|
44
|
+
'speakerVoiceConfigs' in config &&
|
|
45
|
+
Array.isArray(config.speakerVoiceConfigs)
|
|
46
|
+
? (config.speakerVoiceConfigs as unknown[])
|
|
47
|
+
: [];
|
|
48
|
+
|
|
49
|
+
return {
|
|
50
|
+
text,
|
|
51
|
+
usesCustomVoice:
|
|
52
|
+
voice?.startsWith('voice_') === true ||
|
|
53
|
+
voice?.startsWith('voicekey_') === true ||
|
|
54
|
+
speakers.some(
|
|
55
|
+
speaker =>
|
|
56
|
+
speaker != null &&
|
|
57
|
+
typeof speaker === 'object' &&
|
|
58
|
+
'voiceConfig' in speaker &&
|
|
59
|
+
speaker.voiceConfig != null &&
|
|
60
|
+
typeof speaker.voiceConfig === 'object' &&
|
|
61
|
+
'voice' in speaker.voiceConfig,
|
|
62
|
+
),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
@@ -9,6 +9,8 @@ export type GoogleSpeechModelId =
|
|
|
9
9
|
| 'gemini-2.5-flash-preview-tts'
|
|
10
10
|
| 'gemini-2.5-pro-preview-tts'
|
|
11
11
|
| 'gemini-3.1-flash-tts-preview'
|
|
12
|
+
| 'gemini-3.8-flash-tts'
|
|
13
|
+
| 'gemini-3.8-flash-lite-tts'
|
|
12
14
|
| (string & {});
|
|
13
15
|
|
|
14
16
|
const prebuiltVoiceConfigSchema = z.object({
|
|
@@ -17,15 +19,40 @@ const prebuiltVoiceConfigSchema = z.object({
|
|
|
17
19
|
|
|
18
20
|
const voiceConfigSchema = z.object({
|
|
19
21
|
prebuiltVoiceConfig: prebuiltVoiceConfigSchema,
|
|
22
|
+
voice: z.never().optional(),
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const speechMetadataSchema = z.object({
|
|
26
|
+
speaker: z.string().min(1).optional(),
|
|
27
|
+
style: z.string().optional(),
|
|
20
28
|
});
|
|
21
29
|
|
|
22
30
|
export const googleSpeechProviderOptionsSchema = lazySchema(() =>
|
|
23
31
|
zodSchema(
|
|
24
32
|
z.object({
|
|
33
|
+
/** Turn-level directions for the top-level text, for Gemini 3.8 TTS. */
|
|
34
|
+
speechMetadata: speechMetadataSchema.optional(),
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Structured transcript for Gemini 3.8 TTS. Replaces the top-level text;
|
|
38
|
+
* pass text: '' when using turns. Each multi-speaker turn must name a
|
|
39
|
+
* configured speaker in speechMetadata. Per-turn styles override instructions.
|
|
40
|
+
*/
|
|
41
|
+
turns: z
|
|
42
|
+
.array(
|
|
43
|
+
z.object({
|
|
44
|
+
text: z.string(),
|
|
45
|
+
speechMetadata: speechMetadataSchema.optional(),
|
|
46
|
+
}),
|
|
47
|
+
)
|
|
48
|
+
.min(1)
|
|
49
|
+
.optional(),
|
|
50
|
+
|
|
25
51
|
/**
|
|
26
52
|
* Multi-speaker configuration for dialogue audio. When provided, this
|
|
27
53
|
* overrides the top-level `voice`. The Gemini TTS API supports up to two
|
|
28
|
-
* speakers
|
|
54
|
+
* speakers. For Gemini 3.8, each turn's speechMetadata.speaker must match
|
|
55
|
+
* a configured speaker; older models use speaker labels in the text.
|
|
29
56
|
*
|
|
30
57
|
* https://ai.google.dev/gemini-api/docs/speech-generation#multi-speaker
|
|
31
58
|
*/
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import
|
|
1
|
+
import {
|
|
2
|
+
InvalidArgumentError,
|
|
3
|
+
type SpeechModelV4,
|
|
4
|
+
type SharedV4Warning,
|
|
5
|
+
} from '@ai-sdk/provider';
|
|
2
6
|
import {
|
|
3
7
|
combineHeaders,
|
|
4
8
|
convertBase64ToUint8Array,
|
|
@@ -14,6 +18,7 @@ import {
|
|
|
14
18
|
} from '@ai-sdk/provider-utils';
|
|
15
19
|
import { googleFailedResponseHandler } from './google-error';
|
|
16
20
|
import { googleSpeechResponseSchema } from './google-speech-api';
|
|
21
|
+
import { getGoogleSpeechInput } from './google-speech-input';
|
|
17
22
|
import {
|
|
18
23
|
googleSpeechProviderOptionsSchema,
|
|
19
24
|
type GoogleSpeechModelId,
|
|
@@ -101,18 +106,36 @@ export class GoogleSpeechModel implements SpeechModelV4 {
|
|
|
101
106
|
});
|
|
102
107
|
}
|
|
103
108
|
|
|
109
|
+
// Older Gemini families require prompt-based directions. Default newer and
|
|
110
|
+
// custom model IDs to structured speech without enumerating their aliases.
|
|
111
|
+
const usesStructuredSpeech =
|
|
112
|
+
!this.modelId.startsWith('gemini-2.5-') &&
|
|
113
|
+
!this.modelId.startsWith('gemini-3.1-');
|
|
114
|
+
|
|
115
|
+
const input = getGoogleSpeechInput({
|
|
116
|
+
text,
|
|
117
|
+
voice,
|
|
118
|
+
providerOptions: { google: googleOptions },
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
if (input.usesCustomVoice) {
|
|
122
|
+
throw new InvalidArgumentError({
|
|
123
|
+
argument: 'voice',
|
|
124
|
+
message:
|
|
125
|
+
'Custom voices are not supported. Use a prebuilt voice instead.',
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
104
129
|
// Multi-speaker (provider option) takes precedence over the single voice.
|
|
105
130
|
const multiSpeakerVoiceConfig = googleOptions?.multiSpeakerVoiceConfig;
|
|
106
131
|
const speechConfig = multiSpeakerVoiceConfig
|
|
107
132
|
? { multiSpeakerVoiceConfig }
|
|
108
133
|
: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } } };
|
|
109
134
|
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
// the transcript starts with speaker labels (e.g. `Joe: ...`), so prepending
|
|
113
|
-
// instructions would corrupt that parsing — ignore them there (with a warning).
|
|
135
|
+
// Older models expect directions in the prompt. Prepending them to a
|
|
136
|
+
// labelled multi-speaker transcript would break speaker parsing.
|
|
114
137
|
let promptText = text;
|
|
115
|
-
if (instructions != null) {
|
|
138
|
+
if (instructions != null && !usesStructuredSpeech) {
|
|
116
139
|
if (multiSpeakerVoiceConfig) {
|
|
117
140
|
warnings.push({
|
|
118
141
|
type: 'unsupported',
|
|
@@ -126,6 +149,65 @@ export class GoogleSpeechModel implements SpeechModelV4 {
|
|
|
126
149
|
}
|
|
127
150
|
}
|
|
128
151
|
|
|
152
|
+
let parts: Array<{
|
|
153
|
+
text: string;
|
|
154
|
+
speechMetadata?: { speaker?: string; style?: string };
|
|
155
|
+
}> = [{ text: promptText }];
|
|
156
|
+
|
|
157
|
+
if (usesStructuredSpeech) {
|
|
158
|
+
if (googleOptions?.turns && googleOptions.speechMetadata) {
|
|
159
|
+
throw new InvalidArgumentError({
|
|
160
|
+
argument: 'providerOptions',
|
|
161
|
+
message: 'Set speechMetadata on each turn when using turns.',
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
if (googleOptions?.turns && text !== '') {
|
|
165
|
+
warnings.push({
|
|
166
|
+
type: 'unsupported',
|
|
167
|
+
feature: 'text',
|
|
168
|
+
details: 'Google TTS turns replace the top-level text.',
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
parts = (
|
|
172
|
+
googleOptions?.turns ?? [
|
|
173
|
+
{ text, speechMetadata: googleOptions?.speechMetadata },
|
|
174
|
+
]
|
|
175
|
+
).map(part => {
|
|
176
|
+
const style = part.speechMetadata?.style ?? instructions;
|
|
177
|
+
const speaker = part.speechMetadata?.speaker;
|
|
178
|
+
if (
|
|
179
|
+
multiSpeakerVoiceConfig &&
|
|
180
|
+
!multiSpeakerVoiceConfig.speakerVoiceConfigs.some(
|
|
181
|
+
config => config.speaker === speaker,
|
|
182
|
+
)
|
|
183
|
+
) {
|
|
184
|
+
throw new InvalidArgumentError({
|
|
185
|
+
argument: 'speechMetadata.speaker',
|
|
186
|
+
message:
|
|
187
|
+
'Every multi-speaker turn must specify a speechMetadata.speaker matching a configured speaker.',
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
text: part.text,
|
|
192
|
+
...(style != null || speaker != null
|
|
193
|
+
? { speechMetadata: { style, speaker } }
|
|
194
|
+
: {}),
|
|
195
|
+
};
|
|
196
|
+
});
|
|
197
|
+
} else if (googleOptions?.turns || googleOptions?.speechMetadata) {
|
|
198
|
+
throw new InvalidArgumentError({
|
|
199
|
+
argument: 'providerOptions',
|
|
200
|
+
message: 'Structured speech metadata and turns require Gemini 3.8 TTS.',
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (input.text.length === 0) {
|
|
205
|
+
throw new InvalidArgumentError({
|
|
206
|
+
argument: 'text',
|
|
207
|
+
message: 'Speech input must contain a non-empty transcript.',
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
|
|
129
211
|
if (speed != null) {
|
|
130
212
|
warnings.push({
|
|
131
213
|
type: 'unsupported',
|
|
@@ -145,11 +227,25 @@ export class GoogleSpeechModel implements SpeechModelV4 {
|
|
|
145
227
|
});
|
|
146
228
|
}
|
|
147
229
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
230
|
+
const formats: Record<string, string> = usesStructuredSpeech
|
|
231
|
+
? {
|
|
232
|
+
wav: 'AUDIO_WAV',
|
|
233
|
+
'audio/wav': 'AUDIO_WAV',
|
|
234
|
+
pcm: 'AUDIO_L16',
|
|
235
|
+
'audio/l16': 'AUDIO_L16',
|
|
236
|
+
mulaw: 'AUDIO_MULAW',
|
|
237
|
+
'audio/mulaw': 'AUDIO_MULAW',
|
|
238
|
+
alaw: 'AUDIO_ALAW',
|
|
239
|
+
'audio/alaw': 'AUDIO_ALAW',
|
|
240
|
+
}
|
|
241
|
+
: { wav: 'AUDIO_WAV', pcm: 'AUDIO_L16' };
|
|
242
|
+
let resolvedOutputFormat = 'wav';
|
|
243
|
+
if (
|
|
244
|
+
outputFormat != null &&
|
|
245
|
+
Object.prototype.hasOwnProperty.call(formats, outputFormat)
|
|
246
|
+
) {
|
|
247
|
+
resolvedOutputFormat = outputFormat;
|
|
248
|
+
} else if (outputFormat != null) {
|
|
153
249
|
warnings.push({
|
|
154
250
|
type: 'unsupported',
|
|
155
251
|
feature: 'outputFormat',
|
|
@@ -158,21 +254,34 @@ export class GoogleSpeechModel implements SpeechModelV4 {
|
|
|
158
254
|
}
|
|
159
255
|
|
|
160
256
|
const requestBody = {
|
|
161
|
-
contents: [{ role: 'user', parts
|
|
257
|
+
contents: [{ role: 'user', parts }],
|
|
162
258
|
generationConfig: {
|
|
163
259
|
responseModalities: ['AUDIO'],
|
|
164
260
|
speechConfig,
|
|
261
|
+
...(usesStructuredSpeech && outputFormat != null
|
|
262
|
+
? {
|
|
263
|
+
responseFormat: {
|
|
264
|
+
audio: { mimeType: formats[resolvedOutputFormat] },
|
|
265
|
+
},
|
|
266
|
+
}
|
|
267
|
+
: {}),
|
|
165
268
|
},
|
|
166
269
|
};
|
|
167
270
|
|
|
168
|
-
return {
|
|
271
|
+
return {
|
|
272
|
+
requestBody,
|
|
273
|
+
warnings,
|
|
274
|
+
outputFormat: formats[resolvedOutputFormat],
|
|
275
|
+
usesStructuredSpeech,
|
|
276
|
+
};
|
|
169
277
|
}
|
|
170
278
|
|
|
171
279
|
async doGenerate(
|
|
172
280
|
options: Parameters<SpeechModelV4['doGenerate']>[0],
|
|
173
281
|
): Promise<Awaited<ReturnType<SpeechModelV4['doGenerate']>>> {
|
|
174
282
|
const currentDate = this.config._internal?.currentDate?.() ?? new Date();
|
|
175
|
-
const { requestBody, warnings, outputFormat } =
|
|
283
|
+
const { requestBody, warnings, outputFormat, usesStructuredSpeech } =
|
|
284
|
+
await this.getArgs(options);
|
|
176
285
|
|
|
177
286
|
const {
|
|
178
287
|
value: response,
|
|
@@ -211,23 +320,26 @@ export class GoogleSpeechModel implements SpeechModelV4 {
|
|
|
211
320
|
}
|
|
212
321
|
|
|
213
322
|
const sampleRate = parseSampleRate(mimeType) ?? DEFAULT_SAMPLE_RATE;
|
|
214
|
-
const
|
|
323
|
+
const bytes =
|
|
215
324
|
base64Audio != null
|
|
216
325
|
? convertBase64ToUint8Array(base64Audio)
|
|
217
326
|
: new Uint8Array(0);
|
|
218
327
|
|
|
219
|
-
//
|
|
220
|
-
//
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
// Empty audio is returned as-is so the core layer throws NoSpeechGeneratedError.
|
|
328
|
+
// Older models return PCM, which needs a container for default WAV output.
|
|
329
|
+
// Gemini 3.8 returns WAV itself; adding another header corrupts that audio.
|
|
330
|
+
const isPcm =
|
|
331
|
+
/^audio\/(?:l16|pcm)(?:;|$)/i.test(mimeType ?? '') ||
|
|
332
|
+
(mimeType == null && !usesStructuredSpeech);
|
|
225
333
|
const audio =
|
|
226
|
-
outputFormat === '
|
|
227
|
-
?
|
|
228
|
-
:
|
|
334
|
+
outputFormat === 'AUDIO_WAV' && isPcm && bytes.length > 0
|
|
335
|
+
? addWavHeader(bytes, sampleRate)
|
|
336
|
+
: bytes;
|
|
229
337
|
|
|
230
|
-
if (
|
|
338
|
+
if (
|
|
339
|
+
outputFormat === 'AUDIO_L16' &&
|
|
340
|
+
bytes.length > 0 &&
|
|
341
|
+
!usesStructuredSpeech
|
|
342
|
+
) {
|
|
231
343
|
warnings.push({
|
|
232
344
|
type: 'unsupported',
|
|
233
345
|
feature: 'outputFormat',
|
package/src/internal/index.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from '../google-language-model';
|
|
2
2
|
export * from '../google-speech-model';
|
|
3
|
+
export { getGoogleSpeechInput } from '../google-speech-input';
|
|
3
4
|
export { googleTools } from '../google-tools';
|
|
4
5
|
export type { GoogleModelId } from '../google-language-model-options';
|
|
5
6
|
export {
|