@charivo/tts 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/dist/gemini/index.d.mts +47 -0
- package/dist/gemini/index.d.ts +47 -0
- package/dist/gemini/index.js +327 -0
- package/dist/gemini/index.mjs +304 -0
- package/package.json +7 -2
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@ Stateful TTS manager for Charivo.
|
|
|
4
4
|
|
|
5
5
|
This package coordinates a `TTSPlayer`, audio playback lifecycle, and lip-sync
|
|
6
6
|
events. It works with browser-native speech, remote TTS APIs, and direct OpenAI
|
|
7
|
-
players.
|
|
7
|
+
and Gemini players.
|
|
8
8
|
|
|
9
9
|
For `"audio"` playback mode, the manager analyzes the `<audio>` element it
|
|
10
10
|
creates from the generated bytes using the shared core lip-sync analyzer
|
|
@@ -48,6 +48,23 @@ await ttsManager.speak("Hello", { voice: "marin" });
|
|
|
48
48
|
- `@charivo/tts/openai`: `createOpenAITTSPlayer(config)` (browser player,
|
|
49
49
|
dev/testing only) and, for server-side use, `createOpenAITTSProvider(config)`,
|
|
50
50
|
`OpenAITTSProvider`, `type OpenAITTSConfig`
|
|
51
|
+
- `@charivo/tts/gemini`: `createGeminiTTSPlayer(config)` (browser player,
|
|
52
|
+
dev/testing only) and, for server-side use, `createGeminiTTSProvider(config)`,
|
|
53
|
+
`GeminiTTSProvider`, `type GeminiTTSConfig`. The provider wraps Gemini's raw
|
|
54
|
+
PCM (`audio/l16`) response as a 16-bit WAV file; `rate` and `pitch` are
|
|
55
|
+
ignored, since the API has no speed or pitch control; voices are Google's
|
|
56
|
+
prebuilt names (default `Kore`). The text is sent behind a fixed synthesis
|
|
57
|
+
preamble, and a 5xx or a text-only answer is retried once within the same
|
|
58
|
+
`timeoutMs`, per Google's documented mitigations. It calls
|
|
59
|
+
`models/{model}:generateContent` because that is the shape measured working
|
|
60
|
+
and the simpler one-shot request for one utterance per call, even though
|
|
61
|
+
Google now labels it legacy (still fully supported). `timeoutMs` defaults
|
|
62
|
+
to 90s, since synthesis takes roughly 0.55–0.7x the audio's length; that
|
|
63
|
+
default suits the direct player and callers that own their own deadline. A
|
|
64
|
+
route behind `@charivo/tts/remote` must pass a `timeoutMs` under that
|
|
65
|
+
player's fixed 30s (the demo's `TTS_GEMINI_ROUTE_TIMEOUT_MS` is 25,000) and
|
|
66
|
+
cap its text as the real latency control (the demo's
|
|
67
|
+
`TTS_GEMINI_MAX_TEXT_CHARS` is 400).
|
|
51
68
|
|
|
52
69
|
## Event Bridge
|
|
53
70
|
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { TTSProvider, TTSOptions, TTSPlayer } from '@charivo/core';
|
|
2
|
+
|
|
3
|
+
interface GeminiTTSConfig {
|
|
4
|
+
apiKey: string;
|
|
5
|
+
defaultVoice?: string;
|
|
6
|
+
defaultModel?: string;
|
|
7
|
+
baseUrl?: string;
|
|
8
|
+
timeoutMs?: number;
|
|
9
|
+
dangerouslyAllowBrowser?: boolean;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Server-side Gemini TTS over `models/{model}:generateContent`, returning WAV.
|
|
13
|
+
*
|
|
14
|
+
* `TTSOptions.rate` and `pitch` are ignored: Gemini TTS has no speed or pitch
|
|
15
|
+
* parameter, and prompt-steered pacing is unreliable, so neither is mapped into
|
|
16
|
+
* the prompt. The text is sent behind a fixed synthesis preamble, and the model
|
|
17
|
+
* caps its input at 8,192 tokens.
|
|
18
|
+
*
|
|
19
|
+
* A 5xx or a text-only answer is retried once inside the same `timeoutMs`, not
|
|
20
|
+
* a fresh one.
|
|
21
|
+
*
|
|
22
|
+
* Synthesis is not streamed: measured latency is ~0.55-0.7x the audio duration
|
|
23
|
+
* (120 chars ~ 6s, 600 ~ 20s, 1,800 ~ 68s), which is why the default budget is
|
|
24
|
+
* 90s. That default suits the direct player and callers that own their own
|
|
25
|
+
* deadline; a route behind `@charivo/tts/remote` must pass a `timeoutMs` under
|
|
26
|
+
* that player's fixed 30s (e.g. 25_000) so the server gives up first, and cap
|
|
27
|
+
* its text length on top of that as the real latency control.
|
|
28
|
+
*/
|
|
29
|
+
declare class GeminiTTSProvider implements TTSProvider {
|
|
30
|
+
private apiKey;
|
|
31
|
+
private baseUrl;
|
|
32
|
+
private voice;
|
|
33
|
+
private model;
|
|
34
|
+
private timeoutMs;
|
|
35
|
+
private timeoutMessage;
|
|
36
|
+
constructor(config: GeminiTTSConfig);
|
|
37
|
+
setVoice(voice: string): void;
|
|
38
|
+
setModel(model: string): void;
|
|
39
|
+
generateSpeech(text: string, options?: TTSOptions): Promise<ArrayBuffer>;
|
|
40
|
+
private requestOnce;
|
|
41
|
+
}
|
|
42
|
+
declare function createGeminiTTSProvider(config: GeminiTTSConfig): GeminiTTSProvider;
|
|
43
|
+
|
|
44
|
+
type GeminiTTSPlayerConfig = GeminiTTSConfig;
|
|
45
|
+
declare function createGeminiTTSPlayer(config: GeminiTTSPlayerConfig): TTSPlayer;
|
|
46
|
+
|
|
47
|
+
export { type GeminiTTSConfig, type GeminiTTSPlayerConfig, GeminiTTSProvider, createGeminiTTSPlayer, createGeminiTTSProvider };
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { TTSProvider, TTSOptions, TTSPlayer } from '@charivo/core';
|
|
2
|
+
|
|
3
|
+
interface GeminiTTSConfig {
|
|
4
|
+
apiKey: string;
|
|
5
|
+
defaultVoice?: string;
|
|
6
|
+
defaultModel?: string;
|
|
7
|
+
baseUrl?: string;
|
|
8
|
+
timeoutMs?: number;
|
|
9
|
+
dangerouslyAllowBrowser?: boolean;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Server-side Gemini TTS over `models/{model}:generateContent`, returning WAV.
|
|
13
|
+
*
|
|
14
|
+
* `TTSOptions.rate` and `pitch` are ignored: Gemini TTS has no speed or pitch
|
|
15
|
+
* parameter, and prompt-steered pacing is unreliable, so neither is mapped into
|
|
16
|
+
* the prompt. The text is sent behind a fixed synthesis preamble, and the model
|
|
17
|
+
* caps its input at 8,192 tokens.
|
|
18
|
+
*
|
|
19
|
+
* A 5xx or a text-only answer is retried once inside the same `timeoutMs`, not
|
|
20
|
+
* a fresh one.
|
|
21
|
+
*
|
|
22
|
+
* Synthesis is not streamed: measured latency is ~0.55-0.7x the audio duration
|
|
23
|
+
* (120 chars ~ 6s, 600 ~ 20s, 1,800 ~ 68s), which is why the default budget is
|
|
24
|
+
* 90s. That default suits the direct player and callers that own their own
|
|
25
|
+
* deadline; a route behind `@charivo/tts/remote` must pass a `timeoutMs` under
|
|
26
|
+
* that player's fixed 30s (e.g. 25_000) so the server gives up first, and cap
|
|
27
|
+
* its text length on top of that as the real latency control.
|
|
28
|
+
*/
|
|
29
|
+
declare class GeminiTTSProvider implements TTSProvider {
|
|
30
|
+
private apiKey;
|
|
31
|
+
private baseUrl;
|
|
32
|
+
private voice;
|
|
33
|
+
private model;
|
|
34
|
+
private timeoutMs;
|
|
35
|
+
private timeoutMessage;
|
|
36
|
+
constructor(config: GeminiTTSConfig);
|
|
37
|
+
setVoice(voice: string): void;
|
|
38
|
+
setModel(model: string): void;
|
|
39
|
+
generateSpeech(text: string, options?: TTSOptions): Promise<ArrayBuffer>;
|
|
40
|
+
private requestOnce;
|
|
41
|
+
}
|
|
42
|
+
declare function createGeminiTTSProvider(config: GeminiTTSConfig): GeminiTTSProvider;
|
|
43
|
+
|
|
44
|
+
type GeminiTTSPlayerConfig = GeminiTTSConfig;
|
|
45
|
+
declare function createGeminiTTSPlayer(config: GeminiTTSPlayerConfig): TTSPlayer;
|
|
46
|
+
|
|
47
|
+
export { type GeminiTTSConfig, type GeminiTTSPlayerConfig, GeminiTTSProvider, createGeminiTTSPlayer, createGeminiTTSProvider };
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/gemini/index.ts
|
|
21
|
+
var gemini_exports = {};
|
|
22
|
+
__export(gemini_exports, {
|
|
23
|
+
GeminiTTSProvider: () => GeminiTTSProvider,
|
|
24
|
+
createGeminiTTSPlayer: () => createGeminiTTSPlayer,
|
|
25
|
+
createGeminiTTSProvider: () => createGeminiTTSProvider
|
|
26
|
+
});
|
|
27
|
+
module.exports = __toCommonJS(gemini_exports);
|
|
28
|
+
|
|
29
|
+
// src/gemini/provider.ts
|
|
30
|
+
var import_core = require("@charivo/core");
|
|
31
|
+
var DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com";
|
|
32
|
+
var DEFAULT_MODEL = "gemini-3.1-flash-tts-preview";
|
|
33
|
+
var DEFAULT_VOICE = "Kore";
|
|
34
|
+
var DEFAULT_REQUEST_TIMEOUT_MS = 9e4;
|
|
35
|
+
var PROMPT_PREAMBLE = "TTS the following text:\n";
|
|
36
|
+
var DEFAULT_SAMPLE_RATE = 24e3;
|
|
37
|
+
var DEFAULT_CHANNELS = 1;
|
|
38
|
+
var BITS_PER_SAMPLE = 16;
|
|
39
|
+
var WAV_HEADER_BYTES = 44;
|
|
40
|
+
var GeminiTTSProvider = class {
|
|
41
|
+
apiKey;
|
|
42
|
+
baseUrl;
|
|
43
|
+
voice;
|
|
44
|
+
model;
|
|
45
|
+
timeoutMs;
|
|
46
|
+
timeoutMessage;
|
|
47
|
+
constructor(config) {
|
|
48
|
+
if (typeof window !== "undefined" && !config.dangerouslyAllowBrowser) {
|
|
49
|
+
throw new import_core.CharivoStateError(
|
|
50
|
+
"Gemini TTS provider is for server-side use only. Set dangerouslyAllowBrowser: true for testing"
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
this.apiKey = config.apiKey;
|
|
54
|
+
this.baseUrl = (config.baseUrl || DEFAULT_BASE_URL).replace(/\/$/, "");
|
|
55
|
+
this.voice = config.defaultVoice || DEFAULT_VOICE;
|
|
56
|
+
this.model = config.defaultModel || DEFAULT_MODEL;
|
|
57
|
+
this.timeoutMs = config.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
58
|
+
this.timeoutMessage = `Gemini TTS request timed out after ${this.timeoutMs}ms`;
|
|
59
|
+
}
|
|
60
|
+
setVoice(voice) {
|
|
61
|
+
this.voice = voice;
|
|
62
|
+
}
|
|
63
|
+
setModel(model) {
|
|
64
|
+
this.model = model;
|
|
65
|
+
}
|
|
66
|
+
async generateSpeech(text, options) {
|
|
67
|
+
const url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent`;
|
|
68
|
+
const body = JSON.stringify({
|
|
69
|
+
contents: [{ parts: [{ text: `${PROMPT_PREAMBLE}${text}` }] }],
|
|
70
|
+
generationConfig: {
|
|
71
|
+
responseModalities: ["AUDIO"],
|
|
72
|
+
speechConfig: {
|
|
73
|
+
voiceConfig: {
|
|
74
|
+
prebuiltVoiceConfig: { voiceName: options?.voice || this.voice }
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
try {
|
|
80
|
+
const deadline = Date.now() + this.timeoutMs;
|
|
81
|
+
const first = await this.requestOnce(url, body, this.timeoutMs);
|
|
82
|
+
if ("audio" in first) {
|
|
83
|
+
return first.audio;
|
|
84
|
+
}
|
|
85
|
+
const remainingMs = deadline - Date.now();
|
|
86
|
+
if (remainingMs <= 0) {
|
|
87
|
+
throw new import_core.CharivoTimeoutError(this.timeoutMessage, {
|
|
88
|
+
cause: first.retry
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
const second = await this.requestOnce(url, body, remainingMs);
|
|
92
|
+
if ("audio" in second) {
|
|
93
|
+
return second.audio;
|
|
94
|
+
}
|
|
95
|
+
throw second.retry;
|
|
96
|
+
} catch (error) {
|
|
97
|
+
throw (0, import_core.toCharivoError)("provider", error, "Gemini TTS request failed");
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
requestOnce(url, body, timeoutMs) {
|
|
101
|
+
return (0, import_core.fetchWithTimeout)(
|
|
102
|
+
url,
|
|
103
|
+
{
|
|
104
|
+
method: "POST",
|
|
105
|
+
headers: {
|
|
106
|
+
// Never in the URL: proxies and request logs capture query strings.
|
|
107
|
+
"x-goog-api-key": this.apiKey,
|
|
108
|
+
"Content-Type": "application/json"
|
|
109
|
+
},
|
|
110
|
+
body
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
timeoutMs,
|
|
114
|
+
timeoutMessage: this.timeoutMessage,
|
|
115
|
+
// DNS/TLS/connection failures land here as raw fetch errors — map them
|
|
116
|
+
// so every failure escaping this provider is a CharivoError.
|
|
117
|
+
mapError: (error) => (0, import_core.toCharivoError)("provider", error, "Gemini TTS request failed")
|
|
118
|
+
},
|
|
119
|
+
// Consumed inside the helper so the timeout also covers downloading and
|
|
120
|
+
// parsing the body, which carries the whole audio clip.
|
|
121
|
+
async (response) => {
|
|
122
|
+
if (response.status >= 500) {
|
|
123
|
+
return {
|
|
124
|
+
retry: new import_core.CharivoProviderError(
|
|
125
|
+
`Gemini TTS Error: ${await readResponseText(response)}`
|
|
126
|
+
)
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
if (!response.ok) {
|
|
130
|
+
throw new import_core.CharivoProviderError(
|
|
131
|
+
`Gemini TTS Error: ${await readResponseText(response)}`
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
const inlineData = extractInlineAudio(await readResponseJson(response));
|
|
135
|
+
if (!inlineData) {
|
|
136
|
+
return {
|
|
137
|
+
retry: new import_core.CharivoProviderError(
|
|
138
|
+
"Gemini TTS Error: response contained no audio"
|
|
139
|
+
)
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
const { sampleRate, channels } = parseL16MimeType(inlineData.mimeType);
|
|
143
|
+
return {
|
|
144
|
+
audio: toWavBuffer(
|
|
145
|
+
decodeBase64(inlineData.data),
|
|
146
|
+
sampleRate,
|
|
147
|
+
channels
|
|
148
|
+
)
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
function createGeminiTTSProvider(config) {
|
|
155
|
+
return new GeminiTTSProvider(config);
|
|
156
|
+
}
|
|
157
|
+
function extractInlineAudio(payload) {
|
|
158
|
+
if (!isRecord(payload)) {
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
const candidate = Array.isArray(payload.candidates) ? payload.candidates[0] : void 0;
|
|
162
|
+
if (!isRecord(candidate) || !isRecord(candidate.content)) {
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
const parts = Array.isArray(candidate.content.parts) ? candidate.content.parts : [];
|
|
166
|
+
const part = parts.find(
|
|
167
|
+
(entry) => isRecord(entry) && isRecord(entry.inlineData)
|
|
168
|
+
);
|
|
169
|
+
if (!isRecord(part) || !isRecord(part.inlineData)) {
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
const { mimeType, data } = part.inlineData;
|
|
173
|
+
return typeof mimeType === "string" && typeof data === "string" ? { mimeType, data } : null;
|
|
174
|
+
}
|
|
175
|
+
function parseL16MimeType(mimeType) {
|
|
176
|
+
const [mediaType, ...parameters] = mimeType.split(";");
|
|
177
|
+
if (mediaType?.trim().toLowerCase() !== "audio/l16") {
|
|
178
|
+
throw new import_core.CharivoProviderError(
|
|
179
|
+
`Gemini TTS Error: unsupported audio format "${mimeType}"`
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
let sampleRate = DEFAULT_SAMPLE_RATE;
|
|
183
|
+
let channels = DEFAULT_CHANNELS;
|
|
184
|
+
for (const parameter of parameters) {
|
|
185
|
+
const [rawName, rawValue] = parameter.split("=");
|
|
186
|
+
const name = rawName?.trim().toLowerCase();
|
|
187
|
+
const value = Number(rawValue?.trim());
|
|
188
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (name === "rate") {
|
|
192
|
+
sampleRate = value;
|
|
193
|
+
} else if (name === "channels") {
|
|
194
|
+
channels = value;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return { sampleRate, channels };
|
|
198
|
+
}
|
|
199
|
+
function decodeBase64(data) {
|
|
200
|
+
const binary = atob(data);
|
|
201
|
+
const bytes = new Uint8Array(binary.length);
|
|
202
|
+
for (let i = 0; i < binary.length; i++) {
|
|
203
|
+
bytes[i] = binary.charCodeAt(i);
|
|
204
|
+
}
|
|
205
|
+
return bytes;
|
|
206
|
+
}
|
|
207
|
+
function toWavBuffer(pcm, sampleRate, channels) {
|
|
208
|
+
const bytesPerSample = BITS_PER_SAMPLE / 8;
|
|
209
|
+
const buffer = new ArrayBuffer(WAV_HEADER_BYTES + pcm.length);
|
|
210
|
+
const view = new DataView(buffer);
|
|
211
|
+
writeAscii(view, 0, "RIFF");
|
|
212
|
+
view.setUint32(4, WAV_HEADER_BYTES - 8 + pcm.length, true);
|
|
213
|
+
writeAscii(view, 8, "WAVE");
|
|
214
|
+
writeAscii(view, 12, "fmt ");
|
|
215
|
+
view.setUint32(16, 16, true);
|
|
216
|
+
view.setUint16(20, 1, true);
|
|
217
|
+
view.setUint16(22, channels, true);
|
|
218
|
+
view.setUint32(24, sampleRate, true);
|
|
219
|
+
view.setUint32(28, sampleRate * channels * bytesPerSample, true);
|
|
220
|
+
view.setUint16(32, channels * bytesPerSample, true);
|
|
221
|
+
view.setUint16(34, BITS_PER_SAMPLE, true);
|
|
222
|
+
writeAscii(view, 36, "data");
|
|
223
|
+
view.setUint32(40, pcm.length, true);
|
|
224
|
+
new Uint8Array(buffer, WAV_HEADER_BYTES).set(pcm);
|
|
225
|
+
return buffer;
|
|
226
|
+
}
|
|
227
|
+
function writeAscii(view, offset, text) {
|
|
228
|
+
for (let i = 0; i < text.length; i++) {
|
|
229
|
+
view.setUint8(offset + i, text.charCodeAt(i));
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
function isRecord(value) {
|
|
233
|
+
return typeof value === "object" && value !== null;
|
|
234
|
+
}
|
|
235
|
+
async function readResponseText(response) {
|
|
236
|
+
try {
|
|
237
|
+
return await response.text();
|
|
238
|
+
} catch (error) {
|
|
239
|
+
if (isAbortError(error)) {
|
|
240
|
+
throw error;
|
|
241
|
+
}
|
|
242
|
+
throw (0, import_core.toCharivoError)(
|
|
243
|
+
"provider",
|
|
244
|
+
error,
|
|
245
|
+
"Failed to read Gemini TTS response body"
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
async function readResponseJson(response) {
|
|
250
|
+
try {
|
|
251
|
+
return await response.json();
|
|
252
|
+
} catch (error) {
|
|
253
|
+
if (isAbortError(error)) {
|
|
254
|
+
throw error;
|
|
255
|
+
}
|
|
256
|
+
throw (0, import_core.toCharivoError)(
|
|
257
|
+
"provider",
|
|
258
|
+
error,
|
|
259
|
+
"Failed to parse Gemini TTS response body"
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
function isAbortError(error) {
|
|
264
|
+
return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// src/gemini/index.ts
|
|
268
|
+
var GeminiTTSPlayer = class {
|
|
269
|
+
playbackMode = "audio";
|
|
270
|
+
audioMimeType = "audio/wav";
|
|
271
|
+
provider;
|
|
272
|
+
constructor(config) {
|
|
273
|
+
this.provider = createGeminiTTSProvider({
|
|
274
|
+
...config,
|
|
275
|
+
dangerouslyAllowBrowser: true
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Stateless audio generation (used by the TTS Manager)
|
|
280
|
+
*/
|
|
281
|
+
async generateAudio(text, options) {
|
|
282
|
+
return this.provider.generateSpeech(text, options);
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Legacy speak method (kept for compatibility)
|
|
286
|
+
*/
|
|
287
|
+
async speak(text, options) {
|
|
288
|
+
const audioBuffer = await this.generateAudio(text, options);
|
|
289
|
+
const blob = new Blob([audioBuffer], { type: "audio/wav" });
|
|
290
|
+
const audioUrl = URL.createObjectURL(blob);
|
|
291
|
+
return new Promise((resolve, reject) => {
|
|
292
|
+
const audio = new Audio(audioUrl);
|
|
293
|
+
if (options?.volume !== void 0) {
|
|
294
|
+
audio.volume = Math.max(0, Math.min(1, options.volume));
|
|
295
|
+
}
|
|
296
|
+
audio.onended = () => {
|
|
297
|
+
URL.revokeObjectURL(audioUrl);
|
|
298
|
+
resolve();
|
|
299
|
+
};
|
|
300
|
+
audio.onerror = () => {
|
|
301
|
+
URL.revokeObjectURL(audioUrl);
|
|
302
|
+
reject(new Error("Audio playback failed"));
|
|
303
|
+
};
|
|
304
|
+
audio.play().catch((error) => {
|
|
305
|
+
URL.revokeObjectURL(audioUrl);
|
|
306
|
+
reject(error);
|
|
307
|
+
});
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
async stop() {
|
|
311
|
+
}
|
|
312
|
+
setVoice(voice) {
|
|
313
|
+
this.provider.setVoice(voice);
|
|
314
|
+
}
|
|
315
|
+
isSupported() {
|
|
316
|
+
return typeof window !== "undefined" && typeof fetch !== "undefined";
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
function createGeminiTTSPlayer(config) {
|
|
320
|
+
return new GeminiTTSPlayer(config);
|
|
321
|
+
}
|
|
322
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
323
|
+
0 && (module.exports = {
|
|
324
|
+
GeminiTTSProvider,
|
|
325
|
+
createGeminiTTSPlayer,
|
|
326
|
+
createGeminiTTSProvider
|
|
327
|
+
});
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
// src/gemini/provider.ts
|
|
2
|
+
import {
|
|
3
|
+
CharivoProviderError,
|
|
4
|
+
CharivoStateError,
|
|
5
|
+
CharivoTimeoutError,
|
|
6
|
+
fetchWithTimeout,
|
|
7
|
+
toCharivoError
|
|
8
|
+
} from "@charivo/core";
|
|
9
|
+
var DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com";
|
|
10
|
+
var DEFAULT_MODEL = "gemini-3.1-flash-tts-preview";
|
|
11
|
+
var DEFAULT_VOICE = "Kore";
|
|
12
|
+
var DEFAULT_REQUEST_TIMEOUT_MS = 9e4;
|
|
13
|
+
var PROMPT_PREAMBLE = "TTS the following text:\n";
|
|
14
|
+
var DEFAULT_SAMPLE_RATE = 24e3;
|
|
15
|
+
var DEFAULT_CHANNELS = 1;
|
|
16
|
+
var BITS_PER_SAMPLE = 16;
|
|
17
|
+
var WAV_HEADER_BYTES = 44;
|
|
18
|
+
var GeminiTTSProvider = class {
|
|
19
|
+
apiKey;
|
|
20
|
+
baseUrl;
|
|
21
|
+
voice;
|
|
22
|
+
model;
|
|
23
|
+
timeoutMs;
|
|
24
|
+
timeoutMessage;
|
|
25
|
+
constructor(config) {
|
|
26
|
+
if (typeof window !== "undefined" && !config.dangerouslyAllowBrowser) {
|
|
27
|
+
throw new CharivoStateError(
|
|
28
|
+
"Gemini TTS provider is for server-side use only. Set dangerouslyAllowBrowser: true for testing"
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
this.apiKey = config.apiKey;
|
|
32
|
+
this.baseUrl = (config.baseUrl || DEFAULT_BASE_URL).replace(/\/$/, "");
|
|
33
|
+
this.voice = config.defaultVoice || DEFAULT_VOICE;
|
|
34
|
+
this.model = config.defaultModel || DEFAULT_MODEL;
|
|
35
|
+
this.timeoutMs = config.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
36
|
+
this.timeoutMessage = `Gemini TTS request timed out after ${this.timeoutMs}ms`;
|
|
37
|
+
}
|
|
38
|
+
setVoice(voice) {
|
|
39
|
+
this.voice = voice;
|
|
40
|
+
}
|
|
41
|
+
setModel(model) {
|
|
42
|
+
this.model = model;
|
|
43
|
+
}
|
|
44
|
+
async generateSpeech(text, options) {
|
|
45
|
+
const url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent`;
|
|
46
|
+
const body = JSON.stringify({
|
|
47
|
+
contents: [{ parts: [{ text: `${PROMPT_PREAMBLE}${text}` }] }],
|
|
48
|
+
generationConfig: {
|
|
49
|
+
responseModalities: ["AUDIO"],
|
|
50
|
+
speechConfig: {
|
|
51
|
+
voiceConfig: {
|
|
52
|
+
prebuiltVoiceConfig: { voiceName: options?.voice || this.voice }
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
try {
|
|
58
|
+
const deadline = Date.now() + this.timeoutMs;
|
|
59
|
+
const first = await this.requestOnce(url, body, this.timeoutMs);
|
|
60
|
+
if ("audio" in first) {
|
|
61
|
+
return first.audio;
|
|
62
|
+
}
|
|
63
|
+
const remainingMs = deadline - Date.now();
|
|
64
|
+
if (remainingMs <= 0) {
|
|
65
|
+
throw new CharivoTimeoutError(this.timeoutMessage, {
|
|
66
|
+
cause: first.retry
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
const second = await this.requestOnce(url, body, remainingMs);
|
|
70
|
+
if ("audio" in second) {
|
|
71
|
+
return second.audio;
|
|
72
|
+
}
|
|
73
|
+
throw second.retry;
|
|
74
|
+
} catch (error) {
|
|
75
|
+
throw toCharivoError("provider", error, "Gemini TTS request failed");
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
requestOnce(url, body, timeoutMs) {
|
|
79
|
+
return fetchWithTimeout(
|
|
80
|
+
url,
|
|
81
|
+
{
|
|
82
|
+
method: "POST",
|
|
83
|
+
headers: {
|
|
84
|
+
// Never in the URL: proxies and request logs capture query strings.
|
|
85
|
+
"x-goog-api-key": this.apiKey,
|
|
86
|
+
"Content-Type": "application/json"
|
|
87
|
+
},
|
|
88
|
+
body
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
timeoutMs,
|
|
92
|
+
timeoutMessage: this.timeoutMessage,
|
|
93
|
+
// DNS/TLS/connection failures land here as raw fetch errors — map them
|
|
94
|
+
// so every failure escaping this provider is a CharivoError.
|
|
95
|
+
mapError: (error) => toCharivoError("provider", error, "Gemini TTS request failed")
|
|
96
|
+
},
|
|
97
|
+
// Consumed inside the helper so the timeout also covers downloading and
|
|
98
|
+
// parsing the body, which carries the whole audio clip.
|
|
99
|
+
async (response) => {
|
|
100
|
+
if (response.status >= 500) {
|
|
101
|
+
return {
|
|
102
|
+
retry: new CharivoProviderError(
|
|
103
|
+
`Gemini TTS Error: ${await readResponseText(response)}`
|
|
104
|
+
)
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
if (!response.ok) {
|
|
108
|
+
throw new CharivoProviderError(
|
|
109
|
+
`Gemini TTS Error: ${await readResponseText(response)}`
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
const inlineData = extractInlineAudio(await readResponseJson(response));
|
|
113
|
+
if (!inlineData) {
|
|
114
|
+
return {
|
|
115
|
+
retry: new CharivoProviderError(
|
|
116
|
+
"Gemini TTS Error: response contained no audio"
|
|
117
|
+
)
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const { sampleRate, channels } = parseL16MimeType(inlineData.mimeType);
|
|
121
|
+
return {
|
|
122
|
+
audio: toWavBuffer(
|
|
123
|
+
decodeBase64(inlineData.data),
|
|
124
|
+
sampleRate,
|
|
125
|
+
channels
|
|
126
|
+
)
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
function createGeminiTTSProvider(config) {
|
|
133
|
+
return new GeminiTTSProvider(config);
|
|
134
|
+
}
|
|
135
|
+
function extractInlineAudio(payload) {
|
|
136
|
+
if (!isRecord(payload)) {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
const candidate = Array.isArray(payload.candidates) ? payload.candidates[0] : void 0;
|
|
140
|
+
if (!isRecord(candidate) || !isRecord(candidate.content)) {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
const parts = Array.isArray(candidate.content.parts) ? candidate.content.parts : [];
|
|
144
|
+
const part = parts.find(
|
|
145
|
+
(entry) => isRecord(entry) && isRecord(entry.inlineData)
|
|
146
|
+
);
|
|
147
|
+
if (!isRecord(part) || !isRecord(part.inlineData)) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const { mimeType, data } = part.inlineData;
|
|
151
|
+
return typeof mimeType === "string" && typeof data === "string" ? { mimeType, data } : null;
|
|
152
|
+
}
|
|
153
|
+
function parseL16MimeType(mimeType) {
|
|
154
|
+
const [mediaType, ...parameters] = mimeType.split(";");
|
|
155
|
+
if (mediaType?.trim().toLowerCase() !== "audio/l16") {
|
|
156
|
+
throw new CharivoProviderError(
|
|
157
|
+
`Gemini TTS Error: unsupported audio format "${mimeType}"`
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
let sampleRate = DEFAULT_SAMPLE_RATE;
|
|
161
|
+
let channels = DEFAULT_CHANNELS;
|
|
162
|
+
for (const parameter of parameters) {
|
|
163
|
+
const [rawName, rawValue] = parameter.split("=");
|
|
164
|
+
const name = rawName?.trim().toLowerCase();
|
|
165
|
+
const value = Number(rawValue?.trim());
|
|
166
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (name === "rate") {
|
|
170
|
+
sampleRate = value;
|
|
171
|
+
} else if (name === "channels") {
|
|
172
|
+
channels = value;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return { sampleRate, channels };
|
|
176
|
+
}
|
|
177
|
+
function decodeBase64(data) {
|
|
178
|
+
const binary = atob(data);
|
|
179
|
+
const bytes = new Uint8Array(binary.length);
|
|
180
|
+
for (let i = 0; i < binary.length; i++) {
|
|
181
|
+
bytes[i] = binary.charCodeAt(i);
|
|
182
|
+
}
|
|
183
|
+
return bytes;
|
|
184
|
+
}
|
|
185
|
+
function toWavBuffer(pcm, sampleRate, channels) {
|
|
186
|
+
const bytesPerSample = BITS_PER_SAMPLE / 8;
|
|
187
|
+
const buffer = new ArrayBuffer(WAV_HEADER_BYTES + pcm.length);
|
|
188
|
+
const view = new DataView(buffer);
|
|
189
|
+
writeAscii(view, 0, "RIFF");
|
|
190
|
+
view.setUint32(4, WAV_HEADER_BYTES - 8 + pcm.length, true);
|
|
191
|
+
writeAscii(view, 8, "WAVE");
|
|
192
|
+
writeAscii(view, 12, "fmt ");
|
|
193
|
+
view.setUint32(16, 16, true);
|
|
194
|
+
view.setUint16(20, 1, true);
|
|
195
|
+
view.setUint16(22, channels, true);
|
|
196
|
+
view.setUint32(24, sampleRate, true);
|
|
197
|
+
view.setUint32(28, sampleRate * channels * bytesPerSample, true);
|
|
198
|
+
view.setUint16(32, channels * bytesPerSample, true);
|
|
199
|
+
view.setUint16(34, BITS_PER_SAMPLE, true);
|
|
200
|
+
writeAscii(view, 36, "data");
|
|
201
|
+
view.setUint32(40, pcm.length, true);
|
|
202
|
+
new Uint8Array(buffer, WAV_HEADER_BYTES).set(pcm);
|
|
203
|
+
return buffer;
|
|
204
|
+
}
|
|
205
|
+
function writeAscii(view, offset, text) {
|
|
206
|
+
for (let i = 0; i < text.length; i++) {
|
|
207
|
+
view.setUint8(offset + i, text.charCodeAt(i));
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
function isRecord(value) {
|
|
211
|
+
return typeof value === "object" && value !== null;
|
|
212
|
+
}
|
|
213
|
+
async function readResponseText(response) {
|
|
214
|
+
try {
|
|
215
|
+
return await response.text();
|
|
216
|
+
} catch (error) {
|
|
217
|
+
if (isAbortError(error)) {
|
|
218
|
+
throw error;
|
|
219
|
+
}
|
|
220
|
+
throw toCharivoError(
|
|
221
|
+
"provider",
|
|
222
|
+
error,
|
|
223
|
+
"Failed to read Gemini TTS response body"
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
async function readResponseJson(response) {
|
|
228
|
+
try {
|
|
229
|
+
return await response.json();
|
|
230
|
+
} catch (error) {
|
|
231
|
+
if (isAbortError(error)) {
|
|
232
|
+
throw error;
|
|
233
|
+
}
|
|
234
|
+
throw toCharivoError(
|
|
235
|
+
"provider",
|
|
236
|
+
error,
|
|
237
|
+
"Failed to parse Gemini TTS response body"
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
function isAbortError(error) {
|
|
242
|
+
return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// src/gemini/index.ts
|
|
246
|
+
var GeminiTTSPlayer = class {
|
|
247
|
+
playbackMode = "audio";
|
|
248
|
+
audioMimeType = "audio/wav";
|
|
249
|
+
provider;
|
|
250
|
+
constructor(config) {
|
|
251
|
+
this.provider = createGeminiTTSProvider({
|
|
252
|
+
...config,
|
|
253
|
+
dangerouslyAllowBrowser: true
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Stateless audio generation (used by the TTS Manager)
|
|
258
|
+
*/
|
|
259
|
+
async generateAudio(text, options) {
|
|
260
|
+
return this.provider.generateSpeech(text, options);
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Legacy speak method (kept for compatibility)
|
|
264
|
+
*/
|
|
265
|
+
async speak(text, options) {
|
|
266
|
+
const audioBuffer = await this.generateAudio(text, options);
|
|
267
|
+
const blob = new Blob([audioBuffer], { type: "audio/wav" });
|
|
268
|
+
const audioUrl = URL.createObjectURL(blob);
|
|
269
|
+
return new Promise((resolve, reject) => {
|
|
270
|
+
const audio = new Audio(audioUrl);
|
|
271
|
+
if (options?.volume !== void 0) {
|
|
272
|
+
audio.volume = Math.max(0, Math.min(1, options.volume));
|
|
273
|
+
}
|
|
274
|
+
audio.onended = () => {
|
|
275
|
+
URL.revokeObjectURL(audioUrl);
|
|
276
|
+
resolve();
|
|
277
|
+
};
|
|
278
|
+
audio.onerror = () => {
|
|
279
|
+
URL.revokeObjectURL(audioUrl);
|
|
280
|
+
reject(new Error("Audio playback failed"));
|
|
281
|
+
};
|
|
282
|
+
audio.play().catch((error) => {
|
|
283
|
+
URL.revokeObjectURL(audioUrl);
|
|
284
|
+
reject(error);
|
|
285
|
+
});
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
async stop() {
|
|
289
|
+
}
|
|
290
|
+
setVoice(voice) {
|
|
291
|
+
this.provider.setVoice(voice);
|
|
292
|
+
}
|
|
293
|
+
isSupported() {
|
|
294
|
+
return typeof window !== "undefined" && typeof fetch !== "undefined";
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
function createGeminiTTSPlayer(config) {
|
|
298
|
+
return new GeminiTTSPlayer(config);
|
|
299
|
+
}
|
|
300
|
+
export {
|
|
301
|
+
GeminiTTSProvider,
|
|
302
|
+
createGeminiTTSPlayer,
|
|
303
|
+
createGeminiTTSProvider
|
|
304
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@charivo/tts",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "TTS manager and browser adapters for Charivo",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -21,6 +21,11 @@
|
|
|
21
21
|
"import": "./dist/openai/index.mjs",
|
|
22
22
|
"require": "./dist/openai/index.js"
|
|
23
23
|
},
|
|
24
|
+
"./gemini": {
|
|
25
|
+
"types": "./dist/gemini/index.d.ts",
|
|
26
|
+
"import": "./dist/gemini/index.mjs",
|
|
27
|
+
"require": "./dist/gemini/index.js"
|
|
28
|
+
},
|
|
24
29
|
"./web": {
|
|
25
30
|
"types": "./dist/web/index.d.ts",
|
|
26
31
|
"import": "./dist/web/index.mjs",
|
|
@@ -36,7 +41,7 @@
|
|
|
36
41
|
],
|
|
37
42
|
"dependencies": {
|
|
38
43
|
"openai": "^4.47.1",
|
|
39
|
-
"@charivo/core": "^0.
|
|
44
|
+
"@charivo/core": "^0.33.0"
|
|
40
45
|
},
|
|
41
46
|
"devDependencies": {
|
|
42
47
|
"tsup": "^8.5.0",
|