@tanstack/ai-gemini 0.0.3 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/esm/adapters/image.d.ts +83 -0
- package/dist/esm/adapters/image.js +60 -0
- package/dist/esm/adapters/image.js.map +1 -0
- package/dist/esm/adapters/summarize.d.ts +53 -0
- package/dist/esm/adapters/summarize.js +139 -0
- package/dist/esm/adapters/summarize.js.map +1 -0
- package/dist/esm/adapters/text.d.ts +63 -0
- package/dist/esm/{gemini-adapter.js → adapters/text.js} +86 -132
- package/dist/esm/adapters/text.js.map +1 -0
- package/dist/esm/adapters/tts.d.ts +129 -0
- package/dist/esm/adapters/tts.js +78 -0
- package/dist/esm/adapters/tts.js.map +1 -0
- package/dist/esm/image/image-provider-options.d.ts +139 -0
- package/dist/esm/image/image-provider-options.js +53 -0
- package/dist/esm/image/image-provider-options.js.map +1 -0
- package/dist/esm/index.d.ts +15 -2
- package/dist/esm/index.js +22 -4
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/model-meta.d.ts +128 -1
- package/dist/esm/model-meta.js +71 -5
- package/dist/esm/model-meta.js.map +1 -1
- package/dist/esm/tools/tool-converter.js +5 -3
- package/dist/esm/tools/tool-converter.js.map +1 -1
- package/dist/esm/utils/client.d.ts +17 -0
- package/dist/esm/utils/client.js +25 -0
- package/dist/esm/utils/client.js.map +1 -0
- package/dist/esm/utils/index.d.ts +1 -0
- package/package.json +5 -4
- package/src/adapters/image.ts +188 -0
- package/src/adapters/summarize.ts +242 -0
- package/src/{gemini-adapter.ts → adapters/text.ts} +196 -245
- package/src/adapters/tts.ts +223 -0
- package/src/image/image-provider-options.ts +239 -0
- package/src/index.ts +66 -2
- package/src/model-meta.ts +87 -62
- package/src/tools/tool-converter.ts +6 -5
- package/src/utils/client.ts +43 -0
- package/src/utils/index.ts +6 -0
- package/dist/esm/gemini-adapter.d.ts +0 -71
- package/dist/esm/gemini-adapter.js.map +0 -1
|
@@ -1,16 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
class
|
|
6
|
-
constructor(config) {
|
|
7
|
-
super(
|
|
1
|
+
import { FinishReason } from "@google/genai";
|
|
2
|
+
import { BaseTextAdapter } from "@tanstack/ai/adapters";
|
|
3
|
+
import { convertToolsToProviderFormat } from "../tools/tool-converter.js";
|
|
4
|
+
import { createGeminiClient, generateId, getGeminiApiKeyFromEnv } from "../utils/client.js";
|
|
5
|
+
class GeminiTextAdapter extends BaseTextAdapter {
|
|
6
|
+
constructor(config, model) {
|
|
7
|
+
super({}, model);
|
|
8
|
+
this.kind = "text";
|
|
8
9
|
this.name = "gemini";
|
|
9
|
-
this.
|
|
10
|
-
this.embeddingModels = GEMINI_EMBEDDING_MODELS;
|
|
11
|
-
this.client = new GoogleGenAI({
|
|
12
|
-
apiKey: config.apiKey
|
|
13
|
-
});
|
|
10
|
+
this.client = createGeminiClient(config);
|
|
14
11
|
}
|
|
15
12
|
async *chatStream(options) {
|
|
16
13
|
const mappedOptions = this.mapCommonOptionsToGemini(options);
|
|
@@ -21,7 +18,7 @@ class GeminiAdapter extends BaseAdapter {
|
|
|
21
18
|
const timestamp = Date.now();
|
|
22
19
|
yield {
|
|
23
20
|
type: "error",
|
|
24
|
-
id: this.
|
|
21
|
+
id: generateId(this.name),
|
|
25
22
|
model: options.model,
|
|
26
23
|
timestamp,
|
|
27
24
|
error: {
|
|
@@ -30,99 +27,55 @@ class GeminiAdapter extends BaseAdapter {
|
|
|
30
27
|
};
|
|
31
28
|
}
|
|
32
29
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
summary += part.text;
|
|
30
|
+
/**
|
|
31
|
+
* Generate structured output using Gemini's native JSON response format.
|
|
32
|
+
* Uses responseMimeType: 'application/json' and responseSchema for structured output.
|
|
33
|
+
* The outputSchema is already JSON Schema (converted in the ai layer).
|
|
34
|
+
*/
|
|
35
|
+
async structuredOutput(options) {
|
|
36
|
+
const { chatOptions, outputSchema } = options;
|
|
37
|
+
const mappedOptions = this.mapCommonOptionsToGemini(chatOptions);
|
|
38
|
+
try {
|
|
39
|
+
const result = await this.client.models.generateContent({
|
|
40
|
+
...mappedOptions,
|
|
41
|
+
config: {
|
|
42
|
+
...mappedOptions.config,
|
|
43
|
+
responseMimeType: "application/json",
|
|
44
|
+
responseSchema: outputSchema
|
|
49
45
|
}
|
|
46
|
+
});
|
|
47
|
+
const rawText = this.extractTextFromResponse(result);
|
|
48
|
+
let parsed;
|
|
49
|
+
try {
|
|
50
|
+
parsed = JSON.parse(rawText);
|
|
51
|
+
} catch {
|
|
52
|
+
throw new Error(
|
|
53
|
+
`Failed to parse structured output as JSON. Content: ${rawText.slice(0, 200)}${rawText.length > 200 ? "..." : ""}`
|
|
54
|
+
);
|
|
50
55
|
}
|
|
56
|
+
return {
|
|
57
|
+
data: parsed,
|
|
58
|
+
rawText
|
|
59
|
+
};
|
|
60
|
+
} catch (error) {
|
|
61
|
+
throw new Error(
|
|
62
|
+
error instanceof Error ? error.message : "An unknown error occurred during structured output generation."
|
|
63
|
+
);
|
|
51
64
|
}
|
|
52
|
-
if (!summary && typeof result.text === "string") {
|
|
53
|
-
summary = result.text;
|
|
54
|
-
}
|
|
55
|
-
const promptTokens = this.estimateTokens(prompt);
|
|
56
|
-
const completionTokens = this.estimateTokens(summary);
|
|
57
|
-
return {
|
|
58
|
-
id: this.generateId(),
|
|
59
|
-
model: options.model,
|
|
60
|
-
summary,
|
|
61
|
-
usage: {
|
|
62
|
-
promptTokens,
|
|
63
|
-
completionTokens,
|
|
64
|
-
totalTokens: promptTokens + completionTokens
|
|
65
|
-
}
|
|
66
|
-
};
|
|
67
65
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (embedding.values && Array.isArray(embedding.values)) {
|
|
78
|
-
embeddings.push(embedding.values);
|
|
79
|
-
} else if (Array.isArray(embedding)) {
|
|
80
|
-
embeddings.push(embedding);
|
|
66
|
+
/**
|
|
67
|
+
* Extract text content from a non-streaming response
|
|
68
|
+
*/
|
|
69
|
+
extractTextFromResponse(response) {
|
|
70
|
+
let textContent = "";
|
|
71
|
+
if (response.candidates?.[0]?.content?.parts) {
|
|
72
|
+
for (const part of response.candidates[0].content.parts) {
|
|
73
|
+
if (part.text) {
|
|
74
|
+
textContent += part.text;
|
|
81
75
|
}
|
|
82
76
|
}
|
|
83
77
|
}
|
|
84
|
-
|
|
85
|
-
(sum, input) => sum + this.estimateTokens(input),
|
|
86
|
-
0
|
|
87
|
-
);
|
|
88
|
-
return {
|
|
89
|
-
id: this.generateId(),
|
|
90
|
-
model: options.model || "gemini-embedding-001",
|
|
91
|
-
embeddings,
|
|
92
|
-
usage: {
|
|
93
|
-
promptTokens,
|
|
94
|
-
totalTokens: promptTokens
|
|
95
|
-
}
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
buildSummarizationPrompt(options, text) {
|
|
99
|
-
let prompt = "You are a professional summarizer. ";
|
|
100
|
-
switch (options.style) {
|
|
101
|
-
case "bullet-points":
|
|
102
|
-
prompt += "Provide a summary in bullet point format. ";
|
|
103
|
-
break;
|
|
104
|
-
case "paragraph":
|
|
105
|
-
prompt += "Provide a summary in paragraph format. ";
|
|
106
|
-
break;
|
|
107
|
-
case "concise":
|
|
108
|
-
prompt += "Provide a very concise summary in 1-2 sentences. ";
|
|
109
|
-
break;
|
|
110
|
-
default:
|
|
111
|
-
prompt += "Provide a clear and concise summary. ";
|
|
112
|
-
}
|
|
113
|
-
if (options.focus && options.focus.length > 0) {
|
|
114
|
-
prompt += `Focus on the following aspects: ${options.focus.join(", ")}. `;
|
|
115
|
-
}
|
|
116
|
-
prompt += `
|
|
117
|
-
|
|
118
|
-
Text to summarize:
|
|
119
|
-
${text}
|
|
120
|
-
|
|
121
|
-
Summary:`;
|
|
122
|
-
return prompt;
|
|
123
|
-
}
|
|
124
|
-
estimateTokens(text) {
|
|
125
|
-
return Math.ceil(text.length / 4);
|
|
78
|
+
return textContent;
|
|
126
79
|
}
|
|
127
80
|
async *processStreamChunks(result, model) {
|
|
128
81
|
const timestamp = Date.now();
|
|
@@ -137,7 +90,7 @@ Summary:`;
|
|
|
137
90
|
accumulatedContent += part.text;
|
|
138
91
|
yield {
|
|
139
92
|
type: "content",
|
|
140
|
-
id: this.
|
|
93
|
+
id: generateId(this.name),
|
|
141
94
|
model,
|
|
142
95
|
timestamp,
|
|
143
96
|
delta: part.text,
|
|
@@ -169,7 +122,7 @@ Summary:`;
|
|
|
169
122
|
}
|
|
170
123
|
yield {
|
|
171
124
|
type: "tool_call",
|
|
172
|
-
id: this.
|
|
125
|
+
id: generateId(this.name),
|
|
173
126
|
model,
|
|
174
127
|
timestamp,
|
|
175
128
|
toolCall: {
|
|
@@ -188,7 +141,7 @@ Summary:`;
|
|
|
188
141
|
accumulatedContent += chunk.data;
|
|
189
142
|
yield {
|
|
190
143
|
type: "content",
|
|
191
|
-
id: this.
|
|
144
|
+
id: generateId(this.name),
|
|
192
145
|
model,
|
|
193
146
|
timestamp,
|
|
194
147
|
delta: chunk.data,
|
|
@@ -212,7 +165,7 @@ Summary:`;
|
|
|
212
165
|
});
|
|
213
166
|
yield {
|
|
214
167
|
type: "tool_call",
|
|
215
|
-
id: this.
|
|
168
|
+
id: generateId(this.name),
|
|
216
169
|
model,
|
|
217
170
|
timestamp,
|
|
218
171
|
toolCall: {
|
|
@@ -232,7 +185,7 @@ Summary:`;
|
|
|
232
185
|
if (finishReason === FinishReason.MAX_TOKENS) {
|
|
233
186
|
yield {
|
|
234
187
|
type: "error",
|
|
235
|
-
id: this.
|
|
188
|
+
id: generateId(this.name),
|
|
236
189
|
model,
|
|
237
190
|
timestamp,
|
|
238
191
|
error: {
|
|
@@ -242,13 +195,13 @@ Summary:`;
|
|
|
242
195
|
}
|
|
243
196
|
yield {
|
|
244
197
|
type: "done",
|
|
245
|
-
id: this.
|
|
198
|
+
id: generateId(this.name),
|
|
246
199
|
model,
|
|
247
200
|
timestamp,
|
|
248
201
|
finishReason: toolCallMap.size > 0 ? "tool_calls" : "stop",
|
|
249
202
|
usage: chunk.usageMetadata ? {
|
|
250
203
|
promptTokens: chunk.usageMetadata.promptTokenCount ?? 0,
|
|
251
|
-
completionTokens: chunk.usageMetadata.
|
|
204
|
+
completionTokens: chunk.usageMetadata.candidatesTokenCount ?? 0,
|
|
252
205
|
totalTokens: chunk.usageMetadata.totalTokenCount ?? 0
|
|
253
206
|
} : void 0
|
|
254
207
|
};
|
|
@@ -256,6 +209,18 @@ Summary:`;
|
|
|
256
209
|
}
|
|
257
210
|
}
|
|
258
211
|
convertContentPartToGemini(part) {
|
|
212
|
+
const getDefaultFileType = (part2) => {
|
|
213
|
+
switch (part2) {
|
|
214
|
+
case "image":
|
|
215
|
+
return "image/jpeg";
|
|
216
|
+
case "audio":
|
|
217
|
+
return "audio/mp3";
|
|
218
|
+
case "video":
|
|
219
|
+
return "video/mp4";
|
|
220
|
+
case "document":
|
|
221
|
+
return "application/pdf";
|
|
222
|
+
}
|
|
223
|
+
};
|
|
259
224
|
switch (part.type) {
|
|
260
225
|
case "text":
|
|
261
226
|
return { text: part.content };
|
|
@@ -268,14 +233,14 @@ Summary:`;
|
|
|
268
233
|
return {
|
|
269
234
|
inlineData: {
|
|
270
235
|
data: part.source.value,
|
|
271
|
-
mimeType: metadata?.mimeType ??
|
|
236
|
+
mimeType: metadata?.mimeType ?? getDefaultFileType(part.type)
|
|
272
237
|
}
|
|
273
238
|
};
|
|
274
239
|
} else {
|
|
275
240
|
return {
|
|
276
241
|
fileData: {
|
|
277
242
|
fileUri: part.source.value,
|
|
278
|
-
mimeType: metadata?.mimeType ??
|
|
243
|
+
mimeType: metadata?.mimeType ?? getDefaultFileType(part.type)
|
|
279
244
|
}
|
|
280
245
|
};
|
|
281
246
|
}
|
|
@@ -319,7 +284,6 @@ Summary:`;
|
|
|
319
284
|
parts.push({
|
|
320
285
|
functionResponse: {
|
|
321
286
|
name: msg.toolCallId,
|
|
322
|
-
// Gemini uses function name here
|
|
323
287
|
response: {
|
|
324
288
|
content: msg.content || ""
|
|
325
289
|
}
|
|
@@ -332,20 +296,16 @@ Summary:`;
|
|
|
332
296
|
};
|
|
333
297
|
});
|
|
334
298
|
}
|
|
335
|
-
/**
|
|
336
|
-
* Maps common options to Gemini-specific format
|
|
337
|
-
* Handles translation of normalized options to Gemini's API format
|
|
338
|
-
*/
|
|
339
299
|
mapCommonOptionsToGemini(options) {
|
|
340
|
-
const providerOpts = options.
|
|
300
|
+
const providerOpts = options.modelOptions;
|
|
341
301
|
const requestOptions = {
|
|
342
302
|
model: options.model,
|
|
343
303
|
contents: this.formatMessages(options.messages),
|
|
344
304
|
config: {
|
|
345
305
|
...providerOpts,
|
|
346
|
-
temperature: options.
|
|
347
|
-
topP: options.
|
|
348
|
-
maxOutputTokens: options.
|
|
306
|
+
temperature: options.temperature,
|
|
307
|
+
topP: options.topP,
|
|
308
|
+
maxOutputTokens: options.maxTokens,
|
|
349
309
|
systemInstruction: options.systemPrompts?.join("\n"),
|
|
350
310
|
...providerOpts?.generationConfig,
|
|
351
311
|
tools: convertToolsToProviderFormat(options.tools)
|
|
@@ -354,22 +314,16 @@ Summary:`;
|
|
|
354
314
|
return requestOptions;
|
|
355
315
|
}
|
|
356
316
|
}
|
|
357
|
-
function
|
|
358
|
-
return new
|
|
317
|
+
function createGeminiChat(model, apiKey, config) {
|
|
318
|
+
return new GeminiTextAdapter({ apiKey, ...config }, model);
|
|
359
319
|
}
|
|
360
|
-
function
|
|
361
|
-
const
|
|
362
|
-
|
|
363
|
-
if (!key) {
|
|
364
|
-
throw new Error(
|
|
365
|
-
"GOOGLE_API_KEY or GEMINI_API_KEY is required. Please set it in your environment variables or use createGemini(apiKey, config) instead."
|
|
366
|
-
);
|
|
367
|
-
}
|
|
368
|
-
return createGemini(key, config);
|
|
320
|
+
function geminiText(model, config) {
|
|
321
|
+
const apiKey = getGeminiApiKeyFromEnv();
|
|
322
|
+
return createGeminiChat(model, apiKey, config);
|
|
369
323
|
}
|
|
370
324
|
export {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
325
|
+
GeminiTextAdapter,
|
|
326
|
+
createGeminiChat,
|
|
327
|
+
geminiText
|
|
374
328
|
};
|
|
375
|
-
//# sourceMappingURL=
|
|
329
|
+
//# sourceMappingURL=text.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"text.js","sources":["../../../src/adapters/text.ts"],"sourcesContent":["import { FinishReason } from '@google/genai'\nimport { BaseTextAdapter } from '@tanstack/ai/adapters'\nimport { convertToolsToProviderFormat } from '../tools/tool-converter'\nimport {\n createGeminiClient,\n generateId,\n getGeminiApiKeyFromEnv,\n} from '../utils'\nimport type {\n GEMINI_MODELS,\n GeminiChatModelProviderOptionsByName,\n GeminiModelInputModalitiesByName,\n} from '../model-meta'\nimport type {\n StructuredOutputOptions,\n StructuredOutputResult,\n} from '@tanstack/ai/adapters'\nimport type {\n GenerateContentParameters,\n GenerateContentResponse,\n GoogleGenAI,\n Part,\n} from '@google/genai'\nimport type {\n ContentPart,\n Modality,\n ModelMessage,\n StreamChunk,\n TextOptions,\n} from '@tanstack/ai'\nimport type { ExternalTextProviderOptions } from '../text/text-provider-options'\nimport type {\n GeminiAudioMetadata,\n GeminiDocumentMetadata,\n GeminiImageMetadata,\n GeminiMessageMetadataByModality,\n GeminiVideoMetadata,\n} from '../message-types'\nimport type { GeminiClientConfig } from '../utils'\n\n/**\n * Configuration for Gemini text adapter\n */\nexport interface GeminiTextConfig extends GeminiClientConfig {}\n\n/**\n * Gemini-specific provider options for text/chat\n */\nexport type GeminiTextProviderOptions = ExternalTextProviderOptions\n\n// ===========================\n// Type Resolution Helpers\n// ===========================\n\n/**\n * Resolve provider options for a specific model.\n * If the model has explicit options in the map, use those; otherwise use base options.\n */\ntype ResolveProviderOptions<TModel extends string> =\n TModel extends keyof GeminiChatModelProviderOptionsByName\n ? GeminiChatModelProviderOptionsByName[TModel]\n : GeminiTextProviderOptions\n\n/**\n * Resolve input modalities for a specific model.\n * If the model has explicit modalities in the map, use those; otherwise use all modalities.\n */\ntype ResolveInputModalities<TModel extends string> =\n TModel extends keyof GeminiModelInputModalitiesByName\n ? GeminiModelInputModalitiesByName[TModel]\n : readonly ['text', 'image', 'audio', 'video', 'document']\n\n// ===========================\n// Adapter Implementation\n// ===========================\n\n/**\n * Gemini Text (Chat) Adapter\n *\n * Tree-shakeable adapter for Gemini chat/text completion functionality.\n * Import only what you need for smaller bundle sizes.\n */\nexport class GeminiTextAdapter<\n TModel extends (typeof GEMINI_MODELS)[number],\n TProviderOptions extends object = ResolveProviderOptions<TModel>,\n TInputModalities extends ReadonlyArray<Modality> =\n ResolveInputModalities<TModel>,\n> extends BaseTextAdapter<\n TModel,\n TProviderOptions,\n TInputModalities,\n GeminiMessageMetadataByModality\n> {\n readonly kind = 'text' as const\n readonly name = 'gemini' as const\n\n private client: GoogleGenAI\n\n constructor(config: GeminiTextConfig, model: TModel) {\n super({}, model)\n this.client = createGeminiClient(config)\n }\n\n async *chatStream(\n options: TextOptions<GeminiTextProviderOptions>,\n ): AsyncIterable<StreamChunk> {\n const mappedOptions = this.mapCommonOptionsToGemini(options)\n\n try {\n const result =\n await this.client.models.generateContentStream(mappedOptions)\n\n yield* this.processStreamChunks(result, options.model)\n } catch (error) {\n const timestamp = Date.now()\n yield {\n type: 'error',\n id: generateId(this.name),\n model: options.model,\n timestamp,\n error: {\n message:\n error instanceof Error\n ? error.message\n : 'An unknown error occurred during the chat stream.',\n },\n }\n }\n }\n\n /**\n * Generate structured output using Gemini's native JSON response format.\n * Uses responseMimeType: 'application/json' and responseSchema for structured output.\n * The outputSchema is already JSON Schema (converted in the ai layer).\n */\n async structuredOutput(\n options: StructuredOutputOptions<GeminiTextProviderOptions>,\n ): Promise<StructuredOutputResult<unknown>> {\n const { chatOptions, outputSchema } = options\n\n const mappedOptions = this.mapCommonOptionsToGemini(chatOptions)\n\n try {\n // Add structured output configuration\n const result = await this.client.models.generateContent({\n ...mappedOptions,\n config: {\n ...mappedOptions.config,\n responseMimeType: 'application/json',\n responseSchema: outputSchema,\n },\n })\n\n // Extract text content from the response\n const rawText = this.extractTextFromResponse(result)\n\n // Parse the JSON response\n let parsed: unknown\n try {\n parsed = JSON.parse(rawText)\n } catch {\n throw new Error(\n `Failed to parse structured output as JSON. Content: ${rawText.slice(0, 200)}${rawText.length > 200 ? '...' : ''}`,\n )\n }\n\n return {\n data: parsed,\n rawText,\n }\n } catch (error) {\n throw new Error(\n error instanceof Error\n ? error.message\n : 'An unknown error occurred during structured output generation.',\n )\n }\n }\n\n /**\n * Extract text content from a non-streaming response\n */\n private extractTextFromResponse(response: GenerateContentResponse): string {\n let textContent = ''\n\n if (response.candidates?.[0]?.content?.parts) {\n for (const part of response.candidates[0].content.parts) {\n if (part.text) {\n textContent += part.text\n }\n }\n }\n\n return textContent\n }\n\n private async *processStreamChunks(\n result: AsyncGenerator<GenerateContentResponse, unknown, unknown>,\n model: string,\n ): AsyncIterable<StreamChunk> {\n const timestamp = Date.now()\n let accumulatedContent = ''\n const toolCallMap = new Map<\n string,\n { name: string; args: string; index: number }\n >()\n let nextToolIndex = 0\n\n for await (const chunk of result) {\n if (chunk.candidates?.[0]?.content?.parts) {\n const parts = chunk.candidates[0].content.parts\n\n for (const part of parts) {\n if (part.text) {\n accumulatedContent += part.text\n yield {\n type: 'content',\n id: generateId(this.name),\n model,\n timestamp,\n delta: part.text,\n content: accumulatedContent,\n role: 'assistant',\n }\n }\n\n const functionCall = part.functionCall\n if (functionCall) {\n const toolCallId =\n functionCall.name || `call_${Date.now()}_${nextToolIndex}`\n const functionArgs = functionCall.args || {}\n\n let toolCallData = toolCallMap.get(toolCallId)\n if (!toolCallData) {\n toolCallData = {\n name: functionCall.name || '',\n args:\n typeof functionArgs === 'string'\n ? functionArgs\n : JSON.stringify(functionArgs),\n index: nextToolIndex++,\n }\n toolCallMap.set(toolCallId, toolCallData)\n } else {\n try {\n const existingArgs = JSON.parse(toolCallData.args)\n const newArgs =\n typeof functionArgs === 'string'\n ? JSON.parse(functionArgs)\n : functionArgs\n const mergedArgs = { ...existingArgs, ...newArgs }\n toolCallData.args = JSON.stringify(mergedArgs)\n } catch {\n toolCallData.args =\n typeof functionArgs === 'string'\n ? functionArgs\n : JSON.stringify(functionArgs)\n }\n }\n\n yield {\n type: 'tool_call',\n id: generateId(this.name),\n model,\n timestamp,\n toolCall: {\n id: toolCallId,\n type: 'function',\n function: {\n name: toolCallData.name,\n arguments: toolCallData.args,\n },\n },\n index: toolCallData.index,\n }\n }\n }\n } else if (chunk.data) {\n accumulatedContent += chunk.data\n yield {\n type: 'content',\n id: generateId(this.name),\n model,\n timestamp,\n delta: chunk.data,\n content: accumulatedContent,\n role: 'assistant',\n }\n }\n\n if (chunk.candidates?.[0]?.finishReason) {\n const finishReason = chunk.candidates[0].finishReason\n\n if (finishReason === FinishReason.UNEXPECTED_TOOL_CALL) {\n if (chunk.candidates[0].content?.parts) {\n for (const part of chunk.candidates[0].content.parts) {\n const functionCall = part.functionCall\n if (functionCall) {\n const toolCallId =\n functionCall.name || `call_${Date.now()}_${nextToolIndex}`\n const functionArgs = functionCall.args || {}\n\n toolCallMap.set(toolCallId, {\n name: functionCall.name || '',\n args:\n typeof functionArgs === 'string'\n ? functionArgs\n : JSON.stringify(functionArgs),\n index: nextToolIndex++,\n })\n\n yield {\n type: 'tool_call',\n id: generateId(this.name),\n model,\n timestamp,\n toolCall: {\n id: toolCallId,\n type: 'function',\n function: {\n name: functionCall.name || '',\n arguments:\n typeof functionArgs === 'string'\n ? functionArgs\n : JSON.stringify(functionArgs),\n },\n },\n index: nextToolIndex - 1,\n }\n }\n }\n }\n }\n if (finishReason === FinishReason.MAX_TOKENS) {\n yield {\n type: 'error',\n id: generateId(this.name),\n model,\n timestamp,\n error: {\n message:\n 'The response was cut off because the maximum token limit was reached.',\n },\n }\n }\n\n yield {\n type: 'done',\n id: generateId(this.name),\n model,\n timestamp,\n finishReason: toolCallMap.size > 0 ? 'tool_calls' : 'stop',\n usage: chunk.usageMetadata\n ? {\n promptTokens: chunk.usageMetadata.promptTokenCount ?? 0,\n completionTokens: chunk.usageMetadata.candidatesTokenCount ?? 0,\n totalTokens: chunk.usageMetadata.totalTokenCount ?? 0,\n }\n : undefined,\n }\n }\n }\n }\n\n private convertContentPartToGemini(part: ContentPart): Part {\n const getDefaultFileType = (\n part: 'image' | 'audio' | 'video' | 'document',\n ) => {\n switch (part) {\n case 'image':\n return 'image/jpeg'\n case 'audio':\n return 'audio/mp3'\n case 'video':\n return 'video/mp4'\n case 'document':\n return 'application/pdf'\n }\n }\n switch (part.type) {\n case 'text':\n return { text: part.content }\n case 'image':\n case 'audio':\n case 'video':\n case 'document': {\n const metadata = part.metadata as\n | GeminiDocumentMetadata\n | GeminiImageMetadata\n | GeminiVideoMetadata\n | GeminiAudioMetadata\n | undefined\n if (part.source.type === 'data') {\n return {\n inlineData: {\n data: part.source.value,\n mimeType: metadata?.mimeType ?? getDefaultFileType(part.type),\n },\n }\n } else {\n return {\n fileData: {\n fileUri: part.source.value,\n mimeType: metadata?.mimeType ?? getDefaultFileType(part.type),\n },\n }\n }\n }\n default: {\n const _exhaustiveCheck: never = part\n throw new Error(\n `Unsupported content part type: ${(_exhaustiveCheck as ContentPart).type}`,\n )\n }\n }\n }\n\n private formatMessages(\n messages: Array<ModelMessage>,\n ): GenerateContentParameters['contents'] {\n return messages.map((msg) => {\n const role: 'user' | 'model' = msg.role === 'assistant' ? 'model' : 'user'\n const parts: Array<Part> = []\n\n if (Array.isArray(msg.content)) {\n for (const contentPart of msg.content) {\n parts.push(this.convertContentPartToGemini(contentPart))\n }\n } else if (msg.content) {\n parts.push({ text: msg.content })\n }\n\n if (msg.role === 'assistant' && msg.toolCalls?.length) {\n for (const toolCall of msg.toolCalls) {\n let parsedArgs: Record<string, unknown> = {}\n try {\n parsedArgs = toolCall.function.arguments\n ? (JSON.parse(toolCall.function.arguments) as Record<\n string,\n unknown\n >)\n : {}\n } catch {\n parsedArgs = toolCall.function.arguments as unknown as Record<\n string,\n unknown\n >\n }\n\n parts.push({\n functionCall: {\n name: toolCall.function.name,\n args: parsedArgs,\n },\n })\n }\n }\n\n if (msg.role === 'tool' && msg.toolCallId) {\n parts.push({\n functionResponse: {\n name: msg.toolCallId,\n response: {\n content: msg.content || '',\n },\n },\n })\n }\n\n return {\n role,\n parts: parts.length > 0 ? parts : [{ text: '' }],\n }\n })\n }\n\n private mapCommonOptionsToGemini(options: TextOptions) {\n const providerOpts = options.modelOptions\n const requestOptions: GenerateContentParameters = {\n model: options.model,\n contents: this.formatMessages(options.messages),\n config: {\n ...providerOpts,\n temperature: options.temperature,\n topP: options.topP,\n maxOutputTokens: options.maxTokens,\n systemInstruction: options.systemPrompts?.join('\\n'),\n ...((providerOpts as Record<string, unknown> | undefined)\n ?.generationConfig as Record<string, unknown> | undefined),\n tools: convertToolsToProviderFormat(options.tools),\n },\n }\n\n return requestOptions\n }\n}\n\n/**\n * Creates a Gemini text adapter with explicit API key.\n * Type resolution happens here at the call site.\n */\nexport function createGeminiChat<TModel extends (typeof GEMINI_MODELS)[number]>(\n model: TModel,\n apiKey: string,\n config?: Omit<GeminiTextConfig, 'apiKey'>,\n): GeminiTextAdapter<\n TModel,\n ResolveProviderOptions<TModel>,\n ResolveInputModalities<TModel>\n> {\n return new GeminiTextAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Gemini text adapter with automatic API key detection.\n * Type resolution happens here at the call site.\n */\nexport function geminiText<TModel extends (typeof GEMINI_MODELS)[number]>(\n model: TModel,\n config?: Omit<GeminiTextConfig, 'apiKey'>,\n): GeminiTextAdapter<\n TModel,\n ResolveProviderOptions<TModel>,\n ResolveInputModalities<TModel>\n> {\n const apiKey = getGeminiApiKeyFromEnv()\n return createGeminiChat(model, apiKey, config)\n}\n"],"names":["part"],"mappings":";;;;AAkFO,MAAM,0BAKH,gBAKR;AAAA,EAMA,YAAY,QAA0B,OAAe;AACnD,UAAM,CAAA,GAAI,KAAK;AANjB,SAAS,OAAO;AAChB,SAAS,OAAO;AAMd,SAAK,SAAS,mBAAmB,MAAM;AAAA,EACzC;AAAA,EAEA,OAAO,WACL,SAC4B;AAC5B,UAAM,gBAAgB,KAAK,yBAAyB,OAAO;AAE3D,QAAI;AACF,YAAM,SACJ,MAAM,KAAK,OAAO,OAAO,sBAAsB,aAAa;AAE9D,aAAO,KAAK,oBAAoB,QAAQ,QAAQ,KAAK;AAAA,IACvD,SAAS,OAAO;AACd,YAAM,YAAY,KAAK,IAAA;AACvB,YAAM;AAAA,QACJ,MAAM;AAAA,QACN,IAAI,WAAW,KAAK,IAAI;AAAA,QACxB,OAAO,QAAQ;AAAA,QACf;AAAA,QACA,OAAO;AAAA,UACL,SACE,iBAAiB,QACb,MAAM,UACN;AAAA,QAAA;AAAA,MACR;AAAA,IAEJ;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,iBACJ,SAC0C;AAC1C,UAAM,EAAE,aAAa,aAAA,IAAiB;AAEtC,UAAM,gBAAgB,KAAK,yBAAyB,WAAW;AAE/D,QAAI;AAEF,YAAM,SAAS,MAAM,KAAK,OAAO,OAAO,gBAAgB;AAAA,QACtD,GAAG;AAAA,QACH,QAAQ;AAAA,UACN,GAAG,cAAc;AAAA,UACjB,kBAAkB;AAAA,UAClB,gBAAgB;AAAA,QAAA;AAAA,MAClB,CACD;AAGD,YAAM,UAAU,KAAK,wBAAwB,MAAM;AAGnD,UAAI;AACJ,UAAI;AACF,iBAAS,KAAK,MAAM,OAAO;AAAA,MAC7B,QAAQ;AACN,cAAM,IAAI;AAAA,UACR,uDAAuD,QAAQ,MAAM,GAAG,GAAG,CAAC,GAAG,QAAQ,SAAS,MAAM,QAAQ,EAAE;AAAA,QAAA;AAAA,MAEpH;AAEA,aAAO;AAAA,QACL,MAAM;AAAA,QACN;AAAA,MAAA;AAAA,IAEJ,SAAS,OAAO;AACd,YAAM,IAAI;AAAA,QACR,iBAAiB,QACb,MAAM,UACN;AAAA,MAAA;AAAA,IAER;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,wBAAwB,UAA2C;AACzE,QAAI,cAAc;AAElB,QAAI,SAAS,aAAa,CAAC,GAAG,SAAS,OAAO;AAC5C,iBAAW,QAAQ,SAAS,WAAW,CAAC,EAAE,QAAQ,OAAO;AACvD,YAAI,KAAK,MAAM;AACb,yBAAe,KAAK;AAAA,QACtB;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,OAAe,oBACb,QACA,OAC4B;AAC5B,UAAM,YAAY,KAAK,IAAA;AACvB,QAAI,qBAAqB;AACzB,UAAM,kCAAkB,IAAA;AAIxB,QAAI,gBAAgB;AAEpB,qBAAiB,SAAS,QAAQ;AAChC,UAAI,MAAM,aAAa,CAAC,GAAG,SAAS,OAAO;AACzC,cAAM,QAAQ,MAAM,WAAW,CAAC,EAAE,QAAQ;AAE1C,mBAAW,QAAQ,OAAO;AACxB,cAAI,KAAK,MAAM;AACb,kCAAsB,KAAK;AAC3B,kBAAM;AAAA,cACJ,MAAM;AAAA,cACN,IAAI,WAAW,KAAK,IAAI;AAAA,cACxB;AAAA,cACA;AAAA,cACA,OAAO,KAAK;AAAA,cACZ,SAAS;AAAA,cACT,MAAM;AAAA,YAAA;AAAA,UAEV;AAEA,gBAAM,eAAe,KAAK;AAC1B,cAAI,cAAc;AAChB,kBAAM,aACJ,aAAa,QAAQ,QAAQ,KAAK,IAAA,CAAK,IAAI,aAAa;AAC1D,kBAAM,eAAe,aAAa,QAAQ,CAAA;AAE1C,gBAAI,eAAe,YAAY,IAAI,UAAU;AAC7C,gBAAI,CAAC,cAAc;AACjB,6BAAe;AAAA,gBACb,MAAM,aAAa,QAAQ;AAAA,gBAC3B,MACE,OAAO,iBAAiB,WACpB,eACA,KAAK,UAAU,YAAY;AAAA,gBACjC,OAAO;AAAA,cAAA;AAET,0BAAY,IAAI,YAAY,YAAY;AAAA,YAC1C,OAAO;AACL,kBAAI;AACF,sBAAM,eAAe,KAAK,MAAM,aAAa,IAAI;AACjD,sBAAM,UACJ,OAAO,iBAAiB,WACpB,KAAK,MAAM,YAAY,IACvB;AACN,sBAAM,aAAa,EAAE,GAAG,cAAc,GAAG,QAAA;AACzC,6BAAa,OAAO,KAAK,UAAU,UAAU;AAAA,cAC/C,QAAQ;AACN,6BAAa,OACX,OAAO,iBAAiB,WACpB,eACA,KAAK,UAAU,YAAY;AAAA,cACnC;AAAA,YACF;AAEA,kBAAM;AAAA,cACJ,MAAM;AAAA,cACN,IAAI,WAAW,KAAK,IAAI;AAAA,cACxB;AAAA,cACA;AAAA,cACA,UAAU;AAAA,gBACR,IAAI;AAAA,gBACJ,MAAM;AAAA,gBACN,UAAU;AAAA,kBACR,MAAM,aAAa;AAAA,kBACnB,WAAW,aAAa;AAAA,gBAAA;AAAA,cAC1B;AAAA,cAEF,OAAO,aAAa;AAAA,YAAA;AAAA,UAExB;AAAA,QACF;AAAA,MACF,WAAW,MAAM,MAAM;AACrB,8BAAsB,MAAM;AAC5B,cAAM;AAAA,UACJ,MAAM;AAAA,UACN,IAAI,WAAW,KAAK,IAAI;AAAA,UACxB;AAAA,UACA;AAAA,UACA,OAAO,MAAM;AAAA,UACb,SAAS;AAAA,UACT,MAAM;AAAA,QAAA;AAAA,MAEV;AAEA,UAAI,MAAM,aAAa,CAAC,GAAG,cAAc;AACvC,cAAM,eAAe,MAAM,WAAW,CAAC,EAAE;AAEzC,YAAI,iBAAiB,aAAa,sBAAsB;AACtD,cAAI,MAAM,WAAW,CAAC,EAAE,SAAS,OAAO;AACtC,uBAAW,QAAQ,MAAM,WAAW,CAAC,EAAE,QAAQ,OAAO;AACpD,oBAAM,eAAe,KAAK;AAC1B,kBAAI,cAAc;AAChB,sBAAM,aACJ,aAAa,QAAQ,QAAQ,KAAK,IAAA,CAAK,IAAI,aAAa;AAC1D,sBAAM,eAAe,aAAa,QAAQ,CAAA;AAE1C,4BAAY,IAAI,YAAY;AAAA,kBAC1B,MAAM,aAAa,QAAQ;AAAA,kBAC3B,MACE,OAAO,iBAAiB,WACpB,eACA,KAAK,UAAU,YAAY;AAAA,kBACjC,OAAO;AAAA,gBAAA,CACR;AAED,sBAAM;AAAA,kBACJ,MAAM;AAAA,kBACN,IAAI,WAAW,KAAK,IAAI;AAAA,kBACxB;AAAA,kBACA;AAAA,kBACA,UAAU;AAAA,oBACR,IAAI;AAAA,oBACJ,MAAM;AAAA,oBACN,UAAU;AAAA,sBACR,MAAM,aAAa,QAAQ;AAAA,sBAC3B,WACE,OAAO,iBAAiB,WACpB,eACA,KAAK,UAAU,YAAY;AAAA,oBAAA;AAAA,kBACnC;AAAA,kBAEF,OAAO,gBAAgB;AAAA,gBAAA;AAAA,cAE3B;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,iBAAiB,aAAa,YAAY;AAC5C,gBAAM;AAAA,YACJ,MAAM;AAAA,YACN,IAAI,WAAW,KAAK,IAAI;AAAA,YACxB;AAAA,YACA;AAAA,YACA,OAAO;AAAA,cACL,SACE;AAAA,YAAA;AAAA,UACJ;AAAA,QAEJ;AAEA,cAAM;AAAA,UACJ,MAAM;AAAA,UACN,IAAI,WAAW,KAAK,IAAI;AAAA,UACxB;AAAA,UACA;AAAA,UACA,cAAc,YAAY,OAAO,IAAI,eAAe;AAAA,UACpD,OAAO,MAAM,gBACT;AAAA,YACE,cAAc,MAAM,cAAc,oBAAoB;AAAA,YACtD,kBAAkB,MAAM,cAAc,wBAAwB;AAAA,YAC9D,aAAa,MAAM,cAAc,mBAAmB;AAAA,UAAA,IAEtD;AAAA,QAAA;AAAA,MAER;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,2BAA2B,MAAyB;AAC1D,UAAM,qBAAqB,CACzBA,UACG;AACH,cAAQA,OAAAA;AAAAA,QACN,KAAK;AACH,iBAAO;AAAA,QACT,KAAK;AACH,iBAAO;AAAA,QACT,KAAK;AACH,iBAAO;AAAA,QACT,KAAK;AACH,iBAAO;AAAA,MAAA;AAAA,IAEb;AACA,YAAQ,KAAK,MAAA;AAAA,MACX,KAAK;AACH,eAAO,EAAE,MAAM,KAAK,QAAA;AAAA,MACtB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK,YAAY;AACf,cAAM,WAAW,KAAK;AAMtB,YAAI,KAAK,OAAO,SAAS,QAAQ;AAC/B,iBAAO;AAAA,YACL,YAAY;AAAA,cACV,MAAM,KAAK,OAAO;AAAA,cAClB,UAAU,UAAU,YAAY,mBAAmB,KAAK,IAAI;AAAA,YAAA;AAAA,UAC9D;AAAA,QAEJ,OAAO;AACL,iBAAO;AAAA,YACL,UAAU;AAAA,cACR,SAAS,KAAK,OAAO;AAAA,cACrB,UAAU,UAAU,YAAY,mBAAmB,KAAK,IAAI;AAAA,YAAA;AAAA,UAC9D;AAAA,QAEJ;AAAA,MACF;AAAA,MACA,SAAS;AACP,cAAM,mBAA0B;AAChC,cAAM,IAAI;AAAA,UACR,kCAAmC,iBAAiC,IAAI;AAAA,QAAA;AAAA,MAE5E;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEQ,eACN,UACuC;AACvC,WAAO,SAAS,IAAI,CAAC,QAAQ;AAC3B,YAAM,OAAyB,IAAI,SAAS,cAAc,UAAU;AACpE,YAAM,QAAqB,CAAA;AAE3B,UAAI,MAAM,QAAQ,IAAI,OAAO,GAAG;AAC9B,mBAAW,eAAe,IAAI,SAAS;AACrC,gBAAM,KAAK,KAAK,2BAA2B,WAAW,CAAC;AAAA,QACzD;AAAA,MACF,WAAW,IAAI,SAAS;AACtB,cAAM,KAAK,EAAE,MAAM,IAAI,SAAS;AAAA,MAClC;AAEA,UAAI,IAAI,SAAS,eAAe,IAAI,WAAW,QAAQ;AACrD,mBAAW,YAAY,IAAI,WAAW;AACpC,cAAI,aAAsC,CAAA;AAC1C,cAAI;AACF,yBAAa,SAAS,SAAS,YAC1B,KAAK,MAAM,SAAS,SAAS,SAAS,IAIvC,CAAA;AAAA,UACN,QAAQ;AACN,yBAAa,SAAS,SAAS;AAAA,UAIjC;AAEA,gBAAM,KAAK;AAAA,YACT,cAAc;AAAA,cACZ,MAAM,SAAS,SAAS;AAAA,cACxB,MAAM;AAAA,YAAA;AAAA,UACR,CACD;AAAA,QACH;AAAA,MACF;AAEA,UAAI,IAAI,SAAS,UAAU,IAAI,YAAY;AACzC,cAAM,KAAK;AAAA,UACT,kBAAkB;AAAA,YAChB,MAAM,IAAI;AAAA,YACV,UAAU;AAAA,cACR,SAAS,IAAI,WAAW;AAAA,YAAA;AAAA,UAC1B;AAAA,QACF,CACD;AAAA,MACH;AAEA,aAAO;AAAA,QACL;AAAA,QACA,OAAO,MAAM,SAAS,IAAI,QAAQ,CAAC,EAAE,MAAM,GAAA,CAAI;AAAA,MAAA;AAAA,IAEnD,CAAC;AAAA,EACH;AAAA,EAEQ,yBAAyB,SAAsB;AACrD,UAAM,eAAe,QAAQ;AAC7B,UAAM,iBAA4C;AAAA,MAChD,OAAO,QAAQ;AAAA,MACf,UAAU,KAAK,eAAe,QAAQ,QAAQ;AAAA,MAC9C,QAAQ;AAAA,QACN,GAAG;AAAA,QACH,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,QACd,iBAAiB,QAAQ;AAAA,QACzB,mBAAmB,QAAQ,eAAe,KAAK,IAAI;AAAA,QACnD,GAAK,cACD;AAAA,QACJ,OAAO,6BAA6B,QAAQ,KAAK;AAAA,MAAA;AAAA,IACnD;AAGF,WAAO;AAAA,EACT;AACF;AAMO,SAAS,iBACd,OACA,QACA,QAKA;AACA,SAAO,IAAI,kBAAkB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AAC3D;AAMO,SAAS,WACd,OACA,QAKA;AACA,QAAM,SAAS,uBAAA;AACf,SAAO,iBAAiB,OAAO,QAAQ,MAAM;AAC/C;"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { BaseTTSAdapter } from '@tanstack/ai/adapters';
|
|
2
|
+
import { GEMINI_TTS_MODELS, GeminiTTSVoice } from '../model-meta.js';
|
|
3
|
+
import { TTSOptions, TTSResult } from '@tanstack/ai';
|
|
4
|
+
import { GeminiClientConfig } from '../utils.js';
|
|
5
|
+
/**
|
|
6
|
+
* Provider-specific options for Gemini TTS
|
|
7
|
+
*
|
|
8
|
+
* @experimental Gemini TTS is an experimental feature.
|
|
9
|
+
* @see https://ai.google.dev/gemini-api/docs/speech-generation
|
|
10
|
+
*/
|
|
11
|
+
export interface GeminiTTSProviderOptions {
|
|
12
|
+
/**
|
|
13
|
+
* Voice configuration for TTS.
|
|
14
|
+
* Choose from 30 available voices with different characteristics.
|
|
15
|
+
*/
|
|
16
|
+
voiceConfig?: {
|
|
17
|
+
prebuiltVoiceConfig?: {
|
|
18
|
+
/**
|
|
19
|
+
* The voice name to use for speech synthesis.
|
|
20
|
+
* @see https://ai.google.dev/gemini-api/docs/speech-generation#voices
|
|
21
|
+
*/
|
|
22
|
+
voiceName?: GeminiTTSVoice;
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* System instruction for controlling speech style.
|
|
27
|
+
* Use natural language to describe the desired speaking style,
|
|
28
|
+
* pace, tone, accent, or other characteristics.
|
|
29
|
+
*
|
|
30
|
+
* @example "Speak slowly and calmly, as if telling a bedtime story"
|
|
31
|
+
* @example "Use an upbeat, enthusiastic tone with moderate pace"
|
|
32
|
+
* @example "Speak with a British accent"
|
|
33
|
+
*/
|
|
34
|
+
systemInstruction?: string;
|
|
35
|
+
/**
|
|
36
|
+
* Language code hint for the speech synthesis.
|
|
37
|
+
* Gemini TTS supports 24 languages and can auto-detect,
|
|
38
|
+
* but you can provide a hint for better results.
|
|
39
|
+
*
|
|
40
|
+
* @example "en-US" for American English
|
|
41
|
+
* @example "es-ES" for Spanish (Spain)
|
|
42
|
+
* @example "ja-JP" for Japanese
|
|
43
|
+
*/
|
|
44
|
+
languageCode?: string;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Configuration for Gemini TTS adapter
|
|
48
|
+
*
|
|
49
|
+
* @experimental Gemini TTS is an experimental feature.
|
|
50
|
+
*/
|
|
51
|
+
export interface GeminiTTSConfig extends GeminiClientConfig {
|
|
52
|
+
}
|
|
53
|
+
/** Model type for Gemini TTS */
|
|
54
|
+
export type GeminiTTSModel = (typeof GEMINI_TTS_MODELS)[number];
|
|
55
|
+
/**
|
|
56
|
+
* Gemini Text-to-Speech Adapter
|
|
57
|
+
*
|
|
58
|
+
* Tree-shakeable adapter for Gemini TTS functionality.
|
|
59
|
+
*
|
|
60
|
+
* **IMPORTANT**: Gemini TTS uses the Live API (WebSocket-based) which requires
|
|
61
|
+
* different handling than traditional REST APIs. This adapter provides a
|
|
62
|
+
* simplified interface but may have limitations.
|
|
63
|
+
*
|
|
64
|
+
* @experimental Gemini TTS is an experimental feature and may change.
|
|
65
|
+
*
|
|
66
|
+
* Models:
|
|
67
|
+
* - gemini-2.5-flash-preview-tts
|
|
68
|
+
*/
|
|
69
|
+
export declare class GeminiTTSAdapter<TModel extends GeminiTTSModel> extends BaseTTSAdapter<TModel, GeminiTTSProviderOptions> {
|
|
70
|
+
readonly name: "gemini";
|
|
71
|
+
private client;
|
|
72
|
+
constructor(config: GeminiTTSConfig, model: TModel);
|
|
73
|
+
/**
|
|
74
|
+
* Generate speech from text using Gemini's TTS model.
|
|
75
|
+
*
|
|
76
|
+
* @experimental This implementation is experimental and may change.
|
|
77
|
+
* @see https://ai.google.dev/gemini-api/docs/speech-generation
|
|
78
|
+
*/
|
|
79
|
+
generateSpeech(options: TTSOptions<GeminiTTSProviderOptions>): Promise<TTSResult>;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Creates a Gemini TTS adapter with explicit API key.
|
|
83
|
+
* Type resolution happens here at the call site.
|
|
84
|
+
*
|
|
85
|
+
* @experimental Gemini TTS is an experimental feature and may change.
|
|
86
|
+
*
|
|
87
|
+
* @param model - The model name (e.g., 'gemini-2.5-flash-preview-tts')
|
|
88
|
+
* @param apiKey - Your Google API key
|
|
89
|
+
* @param config - Optional additional configuration
|
|
90
|
+
* @returns Configured Gemini TTS adapter instance with resolved types
|
|
91
|
+
*
|
|
92
|
+
* @example
|
|
93
|
+
* ```typescript
|
|
94
|
+
* const adapter = createGeminiSpeech('gemini-2.5-flash-preview-tts', "your-api-key");
|
|
95
|
+
*
|
|
96
|
+
* const result = await generateSpeech({
|
|
97
|
+
* adapter,
|
|
98
|
+
* text: 'Hello, world!'
|
|
99
|
+
* });
|
|
100
|
+
* ```
|
|
101
|
+
*/
|
|
102
|
+
export declare function createGeminiSpeech<TModel extends GeminiTTSModel>(model: TModel, apiKey: string, config?: Omit<GeminiTTSConfig, 'apiKey'>): GeminiTTSAdapter<TModel>;
|
|
103
|
+
/**
|
|
104
|
+
* Creates a Gemini speech adapter with automatic API key detection from environment variables.
|
|
105
|
+
* Type resolution happens here at the call site.
|
|
106
|
+
*
|
|
107
|
+
* @experimental Gemini TTS is an experimental feature and may change.
|
|
108
|
+
*
|
|
109
|
+
* Looks for `GOOGLE_API_KEY` or `GEMINI_API_KEY` in:
|
|
110
|
+
* - `process.env` (Node.js)
|
|
111
|
+
* - `window.env` (Browser with injected env)
|
|
112
|
+
*
|
|
113
|
+
* @param model - The model name (e.g., 'gemini-2.5-flash-preview-tts')
|
|
114
|
+
* @param config - Optional configuration (excluding apiKey which is auto-detected)
|
|
115
|
+
* @returns Configured Gemini speech adapter instance with resolved types
|
|
116
|
+
* @throws Error if GOOGLE_API_KEY or GEMINI_API_KEY is not found in environment
|
|
117
|
+
*
|
|
118
|
+
* @example
|
|
119
|
+
* ```typescript
|
|
120
|
+
* // Automatically uses GOOGLE_API_KEY from environment
|
|
121
|
+
* const adapter = geminiSpeech('gemini-2.5-flash-preview-tts');
|
|
122
|
+
*
|
|
123
|
+
* const result = await generateSpeech({
|
|
124
|
+
* adapter,
|
|
125
|
+
* text: 'Welcome to TanStack AI!'
|
|
126
|
+
* });
|
|
127
|
+
* ```
|
|
128
|
+
*/
|
|
129
|
+
export declare function geminiSpeech<TModel extends GeminiTTSModel>(model: TModel, config?: Omit<GeminiTTSConfig, 'apiKey'>): GeminiTTSAdapter<TModel>;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { BaseTTSAdapter } from "@tanstack/ai/adapters";
|
|
2
|
+
import { createGeminiClient, generateId, getGeminiApiKeyFromEnv } from "../utils/client.js";
|
|
3
|
+
class GeminiTTSAdapter extends BaseTTSAdapter {
|
|
4
|
+
constructor(config, model) {
|
|
5
|
+
super(config, model);
|
|
6
|
+
this.name = "gemini";
|
|
7
|
+
this.client = createGeminiClient(config);
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Generate speech from text using Gemini's TTS model.
|
|
11
|
+
*
|
|
12
|
+
* @experimental This implementation is experimental and may change.
|
|
13
|
+
* @see https://ai.google.dev/gemini-api/docs/speech-generation
|
|
14
|
+
*/
|
|
15
|
+
async generateSpeech(options) {
|
|
16
|
+
const { model, text, modelOptions } = options;
|
|
17
|
+
const voiceConfig = modelOptions?.voiceConfig || {
|
|
18
|
+
prebuiltVoiceConfig: {
|
|
19
|
+
voiceName: "Kore"
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
const response = await this.client.models.generateContent({
|
|
23
|
+
model,
|
|
24
|
+
contents: [
|
|
25
|
+
{
|
|
26
|
+
role: "user",
|
|
27
|
+
parts: [{ text }]
|
|
28
|
+
}
|
|
29
|
+
],
|
|
30
|
+
config: {
|
|
31
|
+
responseModalities: ["AUDIO"],
|
|
32
|
+
speechConfig: {
|
|
33
|
+
voiceConfig,
|
|
34
|
+
...modelOptions?.languageCode && {
|
|
35
|
+
languageCode: modelOptions.languageCode
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
...modelOptions?.systemInstruction && {
|
|
40
|
+
systemInstruction: modelOptions.systemInstruction
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
const candidate = response.candidates?.[0];
|
|
44
|
+
const parts = candidate?.content?.parts;
|
|
45
|
+
if (!parts || parts.length === 0) {
|
|
46
|
+
throw new Error("No audio output received from Gemini TTS");
|
|
47
|
+
}
|
|
48
|
+
const audioPart = parts.find(
|
|
49
|
+
(part) => part.inlineData?.mimeType?.startsWith("audio/")
|
|
50
|
+
);
|
|
51
|
+
if (!audioPart || !audioPart.inlineData || !audioPart.inlineData.data) {
|
|
52
|
+
throw new Error("No audio data in Gemini TTS response");
|
|
53
|
+
}
|
|
54
|
+
const audioBase64 = audioPart.inlineData.data;
|
|
55
|
+
const mimeType = audioPart.inlineData.mimeType || "audio/wav";
|
|
56
|
+
const format = mimeType.split("/")[1] || "wav";
|
|
57
|
+
return {
|
|
58
|
+
id: generateId(this.name),
|
|
59
|
+
model,
|
|
60
|
+
audio: audioBase64,
|
|
61
|
+
format,
|
|
62
|
+
contentType: mimeType
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function createGeminiSpeech(model, apiKey, config) {
|
|
67
|
+
return new GeminiTTSAdapter({ apiKey, ...config }, model);
|
|
68
|
+
}
|
|
69
|
+
function geminiSpeech(model, config) {
|
|
70
|
+
const apiKey = getGeminiApiKeyFromEnv();
|
|
71
|
+
return createGeminiSpeech(model, apiKey, config);
|
|
72
|
+
}
|
|
73
|
+
export {
|
|
74
|
+
GeminiTTSAdapter,
|
|
75
|
+
createGeminiSpeech,
|
|
76
|
+
geminiSpeech
|
|
77
|
+
};
|
|
78
|
+
//# sourceMappingURL=tts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tts.js","sources":["../../../src/adapters/tts.ts"],"sourcesContent":["import { BaseTTSAdapter } from '@tanstack/ai/adapters'\nimport {\n createGeminiClient,\n generateId,\n getGeminiApiKeyFromEnv,\n} from '../utils'\nimport type { GEMINI_TTS_MODELS, GeminiTTSVoice } from '../model-meta'\nimport type { TTSOptions, TTSResult } from '@tanstack/ai'\nimport type { GoogleGenAI } from '@google/genai'\nimport type { GeminiClientConfig } from '../utils'\n\n/**\n * Provider-specific options for Gemini TTS\n *\n * @experimental Gemini TTS is an experimental feature.\n * @see https://ai.google.dev/gemini-api/docs/speech-generation\n */\nexport interface GeminiTTSProviderOptions {\n /**\n * Voice configuration for TTS.\n * Choose from 30 available voices with different characteristics.\n */\n voiceConfig?: {\n prebuiltVoiceConfig?: {\n /**\n * The voice name to use for speech synthesis.\n * @see https://ai.google.dev/gemini-api/docs/speech-generation#voices\n */\n voiceName?: GeminiTTSVoice\n }\n }\n\n /**\n * System instruction for controlling speech style.\n * Use natural language to describe the desired speaking style,\n * pace, tone, accent, or other characteristics.\n *\n * @example \"Speak slowly and calmly, as if telling a bedtime story\"\n * @example \"Use an upbeat, enthusiastic tone with moderate pace\"\n * @example \"Speak with a British accent\"\n */\n systemInstruction?: string\n\n /**\n * Language code hint for the speech synthesis.\n * Gemini TTS supports 24 languages and can auto-detect,\n * but you can provide a hint for better results.\n *\n * @example \"en-US\" for American English\n * @example \"es-ES\" for Spanish (Spain)\n * @example \"ja-JP\" for Japanese\n */\n languageCode?: string\n}\n\n/**\n * Configuration for Gemini TTS adapter\n *\n * @experimental Gemini TTS is an experimental feature.\n */\nexport interface GeminiTTSConfig extends GeminiClientConfig {}\n\n/** Model type for Gemini TTS */\nexport type GeminiTTSModel = (typeof GEMINI_TTS_MODELS)[number]\n\n/**\n * Gemini Text-to-Speech Adapter\n *\n * Tree-shakeable adapter for Gemini TTS functionality.\n *\n * **IMPORTANT**: Gemini TTS uses the Live API (WebSocket-based) which requires\n * different handling than traditional REST APIs. This adapter provides a\n * simplified interface but may have limitations.\n *\n * @experimental Gemini TTS is an experimental feature and may change.\n *\n * Models:\n * - gemini-2.5-flash-preview-tts\n */\nexport class GeminiTTSAdapter<\n TModel extends GeminiTTSModel,\n> extends BaseTTSAdapter<TModel, GeminiTTSProviderOptions> {\n readonly name = 'gemini' as const\n\n private client: GoogleGenAI\n\n constructor(config: GeminiTTSConfig, model: TModel) {\n super(config, model)\n this.client = createGeminiClient(config)\n }\n\n /**\n * Generate speech from text using Gemini's TTS model.\n *\n * @experimental This implementation is experimental and may change.\n * @see https://ai.google.dev/gemini-api/docs/speech-generation\n */\n async generateSpeech(\n options: TTSOptions<GeminiTTSProviderOptions>,\n ): Promise<TTSResult> {\n const { model, text, modelOptions } = options\n\n const voiceConfig = modelOptions?.voiceConfig || {\n prebuiltVoiceConfig: {\n voiceName: 'Kore',\n },\n }\n\n const response = await this.client.models.generateContent({\n model,\n contents: [\n {\n role: 'user',\n parts: [{ text }],\n },\n ],\n config: {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig,\n ...(modelOptions?.languageCode && {\n languageCode: modelOptions.languageCode,\n }),\n },\n },\n ...(modelOptions?.systemInstruction && {\n systemInstruction: modelOptions.systemInstruction,\n }),\n })\n\n // Extract audio data from response\n const candidate = response.candidates?.[0]\n const parts = candidate?.content?.parts\n\n if (!parts || parts.length === 0) {\n throw new Error('No audio output received from Gemini TTS')\n }\n\n // Look for inline data (audio)\n const audioPart = parts.find((part: any) =>\n part.inlineData?.mimeType?.startsWith('audio/'),\n )\n\n if (!audioPart || !audioPart.inlineData || !audioPart.inlineData.data) {\n throw new Error('No audio data in Gemini TTS response')\n }\n\n const audioBase64 = audioPart.inlineData.data\n const mimeType = audioPart.inlineData.mimeType || 'audio/wav'\n const format = mimeType.split('/')[1] || 'wav'\n\n return {\n id: generateId(this.name),\n model,\n audio: audioBase64,\n format,\n contentType: mimeType,\n }\n }\n}\n\n/**\n * Creates a Gemini TTS adapter with explicit API key.\n * Type resolution happens here at the call site.\n *\n * @experimental Gemini TTS is an experimental feature and may change.\n *\n * @param model - The model name (e.g., 'gemini-2.5-flash-preview-tts')\n * @param apiKey - Your Google API key\n * @param config - Optional additional configuration\n * @returns Configured Gemini TTS adapter instance with resolved types\n *\n * @example\n * ```typescript\n * const adapter = createGeminiSpeech('gemini-2.5-flash-preview-tts', \"your-api-key\");\n *\n * const result = await generateSpeech({\n * adapter,\n * text: 'Hello, world!'\n * });\n * ```\n */\nexport function createGeminiSpeech<TModel extends GeminiTTSModel>(\n model: TModel,\n apiKey: string,\n config?: Omit<GeminiTTSConfig, 'apiKey'>,\n): GeminiTTSAdapter<TModel> {\n return new GeminiTTSAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Gemini speech adapter with automatic API key detection from environment variables.\n * Type resolution happens here at the call site.\n *\n * @experimental Gemini TTS is an experimental feature and may change.\n *\n * Looks for `GOOGLE_API_KEY` or `GEMINI_API_KEY` in:\n * - `process.env` (Node.js)\n * - `window.env` (Browser with injected env)\n *\n * @param model - The model name (e.g., 'gemini-2.5-flash-preview-tts')\n * @param config - Optional configuration (excluding apiKey which is auto-detected)\n * @returns Configured Gemini speech adapter instance with resolved types\n * @throws Error if GOOGLE_API_KEY or GEMINI_API_KEY is not found in environment\n *\n * @example\n * ```typescript\n * // Automatically uses GOOGLE_API_KEY from environment\n * const adapter = geminiSpeech('gemini-2.5-flash-preview-tts');\n *\n * const result = await generateSpeech({\n * adapter,\n * text: 'Welcome to TanStack AI!'\n * });\n * ```\n */\nexport function geminiSpeech<TModel extends GeminiTTSModel>(\n model: TModel,\n config?: Omit<GeminiTTSConfig, 'apiKey'>,\n): GeminiTTSAdapter<TModel> {\n const apiKey = getGeminiApiKeyFromEnv()\n return createGeminiSpeech(model, apiKey, config)\n}\n"],"names":[],"mappings":";;AA+EO,MAAM,yBAEH,eAAiD;AAAA,EAKzD,YAAY,QAAyB,OAAe;AAClD,UAAM,QAAQ,KAAK;AALrB,SAAS,OAAO;AAMd,SAAK,SAAS,mBAAmB,MAAM;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,eACJ,SACoB;AACpB,UAAM,EAAE,OAAO,MAAM,aAAA,IAAiB;AAEtC,UAAM,cAAc,cAAc,eAAe;AAAA,MAC/C,qBAAqB;AAAA,QACnB,WAAW;AAAA,MAAA;AAAA,IACb;AAGF,UAAM,WAAW,MAAM,KAAK,OAAO,OAAO,gBAAgB;AAAA,MACxD;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,OAAO,CAAC,EAAE,KAAA,CAAM;AAAA,QAAA;AAAA,MAClB;AAAA,MAEF,QAAQ;AAAA,QACN,oBAAoB,CAAC,OAAO;AAAA,QAC5B,cAAc;AAAA,UACZ;AAAA,UACA,GAAI,cAAc,gBAAgB;AAAA,YAChC,cAAc,aAAa;AAAA,UAAA;AAAA,QAC7B;AAAA,MACF;AAAA,MAEF,GAAI,cAAc,qBAAqB;AAAA,QACrC,mBAAmB,aAAa;AAAA,MAAA;AAAA,IAClC,CACD;AAGD,UAAM,YAAY,SAAS,aAAa,CAAC;AACzC,UAAM,QAAQ,WAAW,SAAS;AAElC,QAAI,CAAC,SAAS,MAAM,WAAW,GAAG;AAChC,YAAM,IAAI,MAAM,0CAA0C;AAAA,IAC5D;AAGA,UAAM,YAAY,MAAM;AAAA,MAAK,CAAC,SAC5B,KAAK,YAAY,UAAU,WAAW,QAAQ;AAAA,IAAA;AAGhD,QAAI,CAAC,aAAa,CAAC,UAAU,cAAc,CAAC,UAAU,WAAW,MAAM;AACrE,YAAM,IAAI,MAAM,sCAAsC;AAAA,IACxD;AAEA,UAAM,cAAc,UAAU,WAAW;AACzC,UAAM,WAAW,UAAU,WAAW,YAAY;AAClD,UAAM,SAAS,SAAS,MAAM,GAAG,EAAE,CAAC,KAAK;AAEzC,WAAO;AAAA,MACL,IAAI,WAAW,KAAK,IAAI;AAAA,MACxB;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA,aAAa;AAAA,IAAA;AAAA,EAEjB;AACF;AAuBO,SAAS,mBACd,OACA,QACA,QAC0B;AAC1B,SAAO,IAAI,iBAAiB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AAC1D;AA4BO,SAAS,aACd,OACA,QAC0B;AAC1B,QAAM,SAAS,uBAAA;AACf,SAAO,mBAAmB,OAAO,QAAQ,MAAM;AACjD;"}
|