@charivo/tts 0.8.1 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -29
- package/dist/gemini/index.d.mts +43 -11
- package/dist/gemini/index.d.ts +43 -11
- package/dist/gemini/index.js +228 -17
- package/dist/gemini/index.mjs +228 -17
- package/dist/index.js +312 -18
- package/dist/index.mjs +312 -18
- package/dist/remote/index.d.mts +7 -0
- package/dist/remote/index.d.ts +7 -0
- package/dist/remote/index.js +156 -6
- package/dist/remote/index.mjs +157 -6
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -6,22 +6,25 @@ This package coordinates a `TTSPlayer`, audio playback lifecycle, and lip-sync
|
|
|
6
6
|
events. It works with browser-native speech, remote TTS APIs, and direct OpenAI
|
|
7
7
|
and Gemini players.
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
`"audio"` playback mode is satisfied by either audio path a player
|
|
10
|
+
implements. When a player has `generateAudioStream()`, the manager prefers
|
|
11
|
+
it: it schedules the incoming PCM into Web Audio as it arrives, tapping the
|
|
12
|
+
shared core lip-sync analyzer (`createLipSyncAnalyzer` from `@charivo/core`)
|
|
13
|
+
off that same playback graph. Otherwise it plays a `generateAudio()` buffer
|
|
14
|
+
through the `<audio>` element it creates and analyzes the same way. Either
|
|
15
|
+
path emits `tts:lipsync:update`. For `"web-speech"` mode, lip-sync is
|
|
16
|
+
simulated from the spoken text instead — there is no audio to analyze.
|
|
14
17
|
|
|
15
18
|
Concrete players should declare `playbackMode` (`"audio"` or `"web-speech"`)
|
|
16
19
|
and can optionally declare `audioMimeType` so the manager does not need to rely
|
|
17
20
|
on constructor-name inference.
|
|
18
21
|
|
|
19
|
-
`"audio"` playback mode requires the player to implement `generateAudio()
|
|
20
|
-
since the manager needs
|
|
21
|
-
analyzes; `createTTSManager(player)` throws an explicit error if a
|
|
22
|
-
declares `"audio"` mode
|
|
23
|
-
`generateAudio()`, e.g. the Web
|
|
24
|
-
instead.
|
|
22
|
+
`"audio"` playback mode requires the player to implement `generateAudio()` or
|
|
23
|
+
`generateAudioStream()`, since the manager needs one of them to build the
|
|
24
|
+
playback it analyzes; `createTTSManager(player)` throws an explicit error if a
|
|
25
|
+
player declares `"audio"` mode with neither. Players that only implement
|
|
26
|
+
`speak()` (no `generateAudio()` or `generateAudioStream()`, e.g. the Web
|
|
27
|
+
Speech API) must use `"web-speech"` mode instead.
|
|
25
28
|
|
|
26
29
|
## Install
|
|
27
30
|
|
|
@@ -45,6 +48,16 @@ await ttsManager.speak("Hello", { voice: "marin" });
|
|
|
45
48
|
## Exports
|
|
46
49
|
|
|
47
50
|
- `createTTSManager(player)`
|
|
51
|
+
- `@charivo/tts/remote`: `createRemoteTTSPlayer(config)`, the production
|
|
52
|
+
browser player that posts to your own API route. `config.streaming` is
|
|
53
|
+
opt-in: off by default, since an unconditionally streaming player would
|
|
54
|
+
break any route that only answers a finished container, and this player
|
|
55
|
+
cannot fall back to the buffered path after a non-PCM answer without paying
|
|
56
|
+
for a second synthesis. Enabled, it requests `audio/pcm` and expects
|
|
57
|
+
`pcm-s16le`, mono-only PCM back, delivered as it is produced rather than one
|
|
58
|
+
fixed-length payload; requests enforce a connect-and-headers deadline plus a
|
|
59
|
+
separate 10s inactivity deadline once the body starts, and deliberately no
|
|
60
|
+
total deadline, so a long reply is never cut for being long.
|
|
48
61
|
- `@charivo/tts/openai`: `createOpenAITTSPlayer(config)` (browser player,
|
|
49
62
|
dev/testing only) and, for server-side use, `createOpenAITTSProvider(config)`,
|
|
50
63
|
`OpenAITTSProvider`, `type OpenAITTSConfig`
|
|
@@ -58,13 +71,22 @@ await ttsManager.speak("Hello", { voice: "marin" });
|
|
|
58
71
|
`timeoutMs`, per Google's documented mitigations. It calls
|
|
59
72
|
`models/{model}:generateContent` because that is the shape measured working
|
|
60
73
|
and the simpler one-shot request for one utterance per call, even though
|
|
61
|
-
Google now labels it legacy (still fully supported). `timeoutMs`
|
|
62
|
-
to 90s, since synthesis takes roughly
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
cap its text as the
|
|
67
|
-
`TTS_GEMINI_MAX_TEXT_CHARS` is 400).
|
|
74
|
+
Google now labels it legacy (still fully supported). Its `timeoutMs`
|
|
75
|
+
defaults to 90s, since synthesis takes roughly one second of fixed startup
|
|
76
|
+
cost plus about 0.7x the audio's length; that default suits the direct
|
|
77
|
+
player and callers that own their own deadline. A route behind
|
|
78
|
+
`@charivo/tts/remote` must pass a `timeoutMs` under that player's fixed 30s
|
|
79
|
+
(the demo's `TTS_GEMINI_ROUTE_TIMEOUT_MS` is 25,000) and cap its text as the
|
|
80
|
+
real latency control (the demo's `TTS_GEMINI_MAX_TEXT_CHARS` is 400).
|
|
81
|
+
`generateSpeechStream` calls the `:streamGenerateContent?alt=sse` sibling of
|
|
82
|
+
the same endpoint and returns a `TTSPcmStream` instead, delivering its first
|
|
83
|
+
audio in roughly 1.1-1.4s regardless of text length; the same `timeoutMs`
|
|
84
|
+
bounds its whole streamed body, not just the connect-and-headers phase, so a
|
|
85
|
+
reply that runs past it errors mid-utterance rather than merely failing to
|
|
86
|
+
start. It is not retried once its first chunk has reached the caller, since
|
|
87
|
+
a retry would replay speech already played, and it shares the same text cap
|
|
88
|
+
because a long enough stream can end on a spurious `SAFETY` finish reason
|
|
89
|
+
instead of completing.
|
|
68
90
|
|
|
69
91
|
## Event Bridge
|
|
70
92
|
|
|
@@ -80,17 +102,27 @@ When connected, the manager emits:
|
|
|
80
102
|
|
|
81
103
|
## Audio Lifecycle
|
|
82
104
|
|
|
83
|
-
- `stop()` — stops active playback
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
105
|
+
- `stop()` — stops active playback: on the buffered path this pauses and
|
|
106
|
+
discards the `<audio>` element; on the streaming path it flushes whatever
|
|
107
|
+
PCM is already queued in Web Audio and aborts the upstream request. If it
|
|
108
|
+
interrupts an in-flight `speak()` call, that call's promise settles as part
|
|
109
|
+
of the stop instead of being left pending: a deliberate stop is treated as
|
|
110
|
+
a cancellation, not a failure, so it resolves rather than rejects. `stop()`
|
|
111
|
+
also cancels a `speak()` call still starting up — the pre-speech stop, or
|
|
112
|
+
audio synthesis. On the buffered path that cancelled call resolves silently
|
|
113
|
+
and never begins playback, whether or not it had already opened an audio
|
|
114
|
+
session, so a resolved `speak()` is not proof audio played there. Streamed
|
|
115
|
+
playback cannot promise that much: its audio session opens on the first
|
|
116
|
+
scheduled buffer, so a stop from that same `tts:audio:start` listener can
|
|
117
|
+
land after a few milliseconds of sound already played — the call still
|
|
118
|
+
resolves and emits exactly one `tts:audio:end`. A newer `speak()` cancels a
|
|
119
|
+
still-starting older one the same way.
|
|
120
|
+
- `prepareAudio()` — creates the lip-sync `AudioContext` up front, and, for a
|
|
121
|
+
player that streams, warms the streamed playback context too. Call it from
|
|
122
|
+
a user-gesture handler before the first `speak()` so mobile browsers that
|
|
123
|
+
require audio to start from a gesture do not block playback; on a browser
|
|
124
|
+
without sticky activation (Safari), skipping it before a streaming
|
|
125
|
+
`speak()` fails that call with `CharivoStateError` instead of playing.
|
|
94
126
|
- `dispose()` — releases lip-sync audio resources and unsubscribes browser
|
|
95
127
|
lifecycle listeners. Call `stop()` first if speech is in-flight; `dispose()`
|
|
96
128
|
does not stop playback. `Charivo.dispose()` calls this automatically for an
|
package/dist/gemini/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TTSProvider, TTSOptions, TTSPlayer } from '@charivo/core';
|
|
1
|
+
import { TTSProvider, TTSOptions, TTSPcmStream, TTSPlayer } from '@charivo/core';
|
|
2
2
|
|
|
3
3
|
interface GeminiTTSConfig {
|
|
4
4
|
apiKey: string;
|
|
@@ -9,22 +9,34 @@ interface GeminiTTSConfig {
|
|
|
9
9
|
dangerouslyAllowBrowser?: boolean;
|
|
10
10
|
}
|
|
11
11
|
/**
|
|
12
|
-
* Server-side Gemini TTS over `models/{model}:generateContent`, returning WAV
|
|
12
|
+
* Server-side Gemini TTS over `models/{model}:generateContent`, returning WAV,
|
|
13
|
+
* with `generateSpeechStream` over `:streamGenerateContent?alt=sse` handing
|
|
14
|
+
* back raw PCM as it is synthesized.
|
|
13
15
|
*
|
|
14
16
|
* `TTSOptions.rate` and `pitch` are ignored: Gemini TTS has no speed or pitch
|
|
15
17
|
* parameter, and prompt-steered pacing is unreliable, so neither is mapped into
|
|
16
18
|
* the prompt. The text is sent behind a fixed synthesis preamble, and the model
|
|
17
19
|
* caps its input at 8,192 tokens.
|
|
18
20
|
*
|
|
19
|
-
* A 5xx or
|
|
20
|
-
* a fresh one.
|
|
21
|
+
* A 5xx or an answer carrying no audio is retried once inside the same
|
|
22
|
+
* `timeoutMs`, not a fresh one.
|
|
21
23
|
*
|
|
22
|
-
*
|
|
23
|
-
* (
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* that
|
|
27
|
-
*
|
|
24
|
+
* `generateSpeech` does not stream: measured latency is a fixed startup cost
|
|
25
|
+
* plus ~0.75x the audio duration (56 chars ~ 3s, 120 ~ 6s, 600 ~ 20s, 1,800 ~
|
|
26
|
+
* 68s), so a short reply does not get proportionally cheaper. The long end is
|
|
27
|
+
* why the default budget is 90s. That default suits the direct player and
|
|
28
|
+
* callers that own their own deadline; a route behind `@charivo/tts/remote`
|
|
29
|
+
* must pass a `timeoutMs` under that player's fixed 30s (e.g. 25_000) so the
|
|
30
|
+
* server gives up first, and cap its text length on top of that as the real
|
|
31
|
+
* latency control.
|
|
32
|
+
*
|
|
33
|
+
* `generateSpeechStream` hands back the first audio in ~1.1-1.4s whatever the
|
|
34
|
+
* text length, and then delivers ~3.5x faster than realtime. It pays for that
|
|
35
|
+
* in completeness: a long text is truncated under a 200 and terminated with a
|
|
36
|
+
* non-`STOP` finish reason (measured: 2,358 chars ended `SAFETY` where 1,182
|
|
37
|
+
* completed), so any terminator but `STOP` is a failed stream. Nothing is
|
|
38
|
+
* retried once the first chunk has been handed out — a retry would replay
|
|
39
|
+
* speech the caller has already played.
|
|
28
40
|
*/
|
|
29
41
|
declare class GeminiTTSProvider implements TTSProvider {
|
|
30
42
|
private apiKey;
|
|
@@ -37,11 +49,31 @@ declare class GeminiTTSProvider implements TTSProvider {
|
|
|
37
49
|
setVoice(voice: string): void;
|
|
38
50
|
setModel(model: string): void;
|
|
39
51
|
generateSpeech(text: string, options?: TTSOptions): Promise<ArrayBuffer>;
|
|
52
|
+
generateSpeechStream(text: string, options?: TTSOptions, signal?: AbortSignal): Promise<TTSPcmStream>;
|
|
53
|
+
private buildRequestBody;
|
|
40
54
|
private requestOnce;
|
|
55
|
+
private openStreamOnce;
|
|
41
56
|
}
|
|
42
57
|
declare function createGeminiTTSProvider(config: GeminiTTSConfig): GeminiTTSProvider;
|
|
43
58
|
|
|
44
|
-
|
|
59
|
+
/**
|
|
60
|
+
* The provider's config without `dangerouslyAllowBrowser`, which this player
|
|
61
|
+
* forces on: accepting it would offer a switch the constructor ignores, and a
|
|
62
|
+
* caller passing `false` would read it as keeping credentials out of the
|
|
63
|
+
* browser. A type error says so while they can still act on it.
|
|
64
|
+
*/
|
|
65
|
+
type GeminiTTSPlayerConfig = Omit<GeminiTTSConfig, "dangerouslyAllowBrowser"> & {
|
|
66
|
+
/**
|
|
67
|
+
* Opt in to `generateAudioStream`. Off by default: the manager streams
|
|
68
|
+
* whenever the method exists, and streaming is not a free upgrade here.
|
|
69
|
+
* This player caps no text, and the streaming endpoint truncates a long one
|
|
70
|
+
* with a non-`STOP` finish reason -- which fails the utterance mid-sentence
|
|
71
|
+
* where the buffered path completes it -- while streamed playback also needs
|
|
72
|
+
* a running `AudioContext`, so `speak()` starts requiring a `prepareAudio()`
|
|
73
|
+
* from a user gesture.
|
|
74
|
+
*/
|
|
75
|
+
streaming?: boolean;
|
|
76
|
+
};
|
|
45
77
|
declare function createGeminiTTSPlayer(config: GeminiTTSPlayerConfig): TTSPlayer;
|
|
46
78
|
|
|
47
79
|
export { type GeminiTTSConfig, type GeminiTTSPlayerConfig, GeminiTTSProvider, createGeminiTTSPlayer, createGeminiTTSProvider };
|
package/dist/gemini/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TTSProvider, TTSOptions, TTSPlayer } from '@charivo/core';
|
|
1
|
+
import { TTSProvider, TTSOptions, TTSPcmStream, TTSPlayer } from '@charivo/core';
|
|
2
2
|
|
|
3
3
|
interface GeminiTTSConfig {
|
|
4
4
|
apiKey: string;
|
|
@@ -9,22 +9,34 @@ interface GeminiTTSConfig {
|
|
|
9
9
|
dangerouslyAllowBrowser?: boolean;
|
|
10
10
|
}
|
|
11
11
|
/**
|
|
12
|
-
* Server-side Gemini TTS over `models/{model}:generateContent`, returning WAV
|
|
12
|
+
* Server-side Gemini TTS over `models/{model}:generateContent`, returning WAV,
|
|
13
|
+
* with `generateSpeechStream` over `:streamGenerateContent?alt=sse` handing
|
|
14
|
+
* back raw PCM as it is synthesized.
|
|
13
15
|
*
|
|
14
16
|
* `TTSOptions.rate` and `pitch` are ignored: Gemini TTS has no speed or pitch
|
|
15
17
|
* parameter, and prompt-steered pacing is unreliable, so neither is mapped into
|
|
16
18
|
* the prompt. The text is sent behind a fixed synthesis preamble, and the model
|
|
17
19
|
* caps its input at 8,192 tokens.
|
|
18
20
|
*
|
|
19
|
-
* A 5xx or
|
|
20
|
-
* a fresh one.
|
|
21
|
+
* A 5xx or an answer carrying no audio is retried once inside the same
|
|
22
|
+
* `timeoutMs`, not a fresh one.
|
|
21
23
|
*
|
|
22
|
-
*
|
|
23
|
-
* (
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* that
|
|
27
|
-
*
|
|
24
|
+
* `generateSpeech` does not stream: measured latency is a fixed startup cost
|
|
25
|
+
* plus ~0.75x the audio duration (56 chars ~ 3s, 120 ~ 6s, 600 ~ 20s, 1,800 ~
|
|
26
|
+
* 68s), so a short reply does not get proportionally cheaper. The long end is
|
|
27
|
+
* why the default budget is 90s. That default suits the direct player and
|
|
28
|
+
* callers that own their own deadline; a route behind `@charivo/tts/remote`
|
|
29
|
+
* must pass a `timeoutMs` under that player's fixed 30s (e.g. 25_000) so the
|
|
30
|
+
* server gives up first, and cap its text length on top of that as the real
|
|
31
|
+
* latency control.
|
|
32
|
+
*
|
|
33
|
+
* `generateSpeechStream` hands back the first audio in ~1.1-1.4s whatever the
|
|
34
|
+
* text length, and then delivers ~3.5x faster than realtime. It pays for that
|
|
35
|
+
* in completeness: a long text is truncated under a 200 and terminated with a
|
|
36
|
+
* non-`STOP` finish reason (measured: 2,358 chars ended `SAFETY` where 1,182
|
|
37
|
+
* completed), so any terminator but `STOP` is a failed stream. Nothing is
|
|
38
|
+
* retried once the first chunk has been handed out — a retry would replay
|
|
39
|
+
* speech the caller has already played.
|
|
28
40
|
*/
|
|
29
41
|
declare class GeminiTTSProvider implements TTSProvider {
|
|
30
42
|
private apiKey;
|
|
@@ -37,11 +49,31 @@ declare class GeminiTTSProvider implements TTSProvider {
|
|
|
37
49
|
setVoice(voice: string): void;
|
|
38
50
|
setModel(model: string): void;
|
|
39
51
|
generateSpeech(text: string, options?: TTSOptions): Promise<ArrayBuffer>;
|
|
52
|
+
generateSpeechStream(text: string, options?: TTSOptions, signal?: AbortSignal): Promise<TTSPcmStream>;
|
|
53
|
+
private buildRequestBody;
|
|
40
54
|
private requestOnce;
|
|
55
|
+
private openStreamOnce;
|
|
41
56
|
}
|
|
42
57
|
declare function createGeminiTTSProvider(config: GeminiTTSConfig): GeminiTTSProvider;
|
|
43
58
|
|
|
44
|
-
|
|
59
|
+
/**
|
|
60
|
+
* The provider's config without `dangerouslyAllowBrowser`, which this player
|
|
61
|
+
* forces on: accepting it would offer a switch the constructor ignores, and a
|
|
62
|
+
* caller passing `false` would read it as keeping credentials out of the
|
|
63
|
+
* browser. A type error says so while they can still act on it.
|
|
64
|
+
*/
|
|
65
|
+
type GeminiTTSPlayerConfig = Omit<GeminiTTSConfig, "dangerouslyAllowBrowser"> & {
|
|
66
|
+
/**
|
|
67
|
+
* Opt in to `generateAudioStream`. Off by default: the manager streams
|
|
68
|
+
* whenever the method exists, and streaming is not a free upgrade here.
|
|
69
|
+
* This player caps no text, and the streaming endpoint truncates a long one
|
|
70
|
+
* with a non-`STOP` finish reason -- which fails the utterance mid-sentence
|
|
71
|
+
* where the buffered path completes it -- while streamed playback also needs
|
|
72
|
+
* a running `AudioContext`, so `speak()` starts requiring a `prepareAudio()`
|
|
73
|
+
* from a user gesture.
|
|
74
|
+
*/
|
|
75
|
+
streaming?: boolean;
|
|
76
|
+
};
|
|
45
77
|
declare function createGeminiTTSPlayer(config: GeminiTTSPlayerConfig): TTSPlayer;
|
|
46
78
|
|
|
47
79
|
export { type GeminiTTSConfig, type GeminiTTSPlayerConfig, GeminiTTSProvider, createGeminiTTSPlayer, createGeminiTTSProvider };
|
package/dist/gemini/index.js
CHANGED
|
@@ -37,6 +37,9 @@ var DEFAULT_SAMPLE_RATE = 24e3;
|
|
|
37
37
|
var DEFAULT_CHANNELS = 1;
|
|
38
38
|
var BITS_PER_SAMPLE = 16;
|
|
39
39
|
var WAV_HEADER_BYTES = 44;
|
|
40
|
+
var SSE_FRAME_SEPARATOR = /\r?\n\r?\n/;
|
|
41
|
+
var SSE_LINE_SEPARATOR = /\r?\n/;
|
|
42
|
+
var SSE_DATA_PREFIX = "data:";
|
|
40
43
|
var GeminiTTSProvider = class {
|
|
41
44
|
apiKey;
|
|
42
45
|
baseUrl;
|
|
@@ -65,17 +68,7 @@ var GeminiTTSProvider = class {
|
|
|
65
68
|
}
|
|
66
69
|
async generateSpeech(text, options) {
|
|
67
70
|
const url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent`;
|
|
68
|
-
const body =
|
|
69
|
-
contents: [{ parts: [{ text: `${PROMPT_PREAMBLE}${text}` }] }],
|
|
70
|
-
generationConfig: {
|
|
71
|
-
responseModalities: ["AUDIO"],
|
|
72
|
-
speechConfig: {
|
|
73
|
-
voiceConfig: {
|
|
74
|
-
prebuiltVoiceConfig: { voiceName: options?.voice || this.voice }
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
});
|
|
71
|
+
const body = this.buildRequestBody(text, options?.voice || this.voice);
|
|
79
72
|
try {
|
|
80
73
|
const deadline = Date.now() + this.timeoutMs;
|
|
81
74
|
const first = await this.requestOnce(url, body, this.timeoutMs);
|
|
@@ -97,6 +90,49 @@ var GeminiTTSProvider = class {
|
|
|
97
90
|
throw (0, import_core.toCharivoError)("provider", error, "Gemini TTS request failed");
|
|
98
91
|
}
|
|
99
92
|
}
|
|
93
|
+
async generateSpeechStream(text, options, signal) {
|
|
94
|
+
const url = `${this.baseUrl}/v1beta/models/${this.model}:streamGenerateContent?alt=sse`;
|
|
95
|
+
const body = this.buildRequestBody(text, options?.voice || this.voice);
|
|
96
|
+
try {
|
|
97
|
+
const deadline = Date.now() + this.timeoutMs;
|
|
98
|
+
const first = await this.openStreamOnce(
|
|
99
|
+
url,
|
|
100
|
+
body,
|
|
101
|
+
this.timeoutMs,
|
|
102
|
+
signal
|
|
103
|
+
);
|
|
104
|
+
if ("stream" in first) {
|
|
105
|
+
return first.stream;
|
|
106
|
+
}
|
|
107
|
+
const remainingMs = deadline - Date.now();
|
|
108
|
+
if (remainingMs <= 0) {
|
|
109
|
+
throw new import_core.CharivoTimeoutError(this.timeoutMessage, {
|
|
110
|
+
cause: first.retry
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
const second = await this.openStreamOnce(url, body, remainingMs, signal);
|
|
114
|
+
if ("stream" in second) {
|
|
115
|
+
return second.stream;
|
|
116
|
+
}
|
|
117
|
+
throw second.retry;
|
|
118
|
+
} catch (error) {
|
|
119
|
+
if (signal?.aborted && (error === signal.reason || isAbortError(error))) {
|
|
120
|
+
throw error;
|
|
121
|
+
}
|
|
122
|
+
throw (0, import_core.toCharivoError)("provider", error, "Gemini TTS request failed");
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
buildRequestBody(text, voice) {
|
|
126
|
+
return JSON.stringify({
|
|
127
|
+
contents: [{ parts: [{ text: `${PROMPT_PREAMBLE}${text}` }] }],
|
|
128
|
+
generationConfig: {
|
|
129
|
+
responseModalities: ["AUDIO"],
|
|
130
|
+
speechConfig: {
|
|
131
|
+
voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } }
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
}
|
|
100
136
|
requestOnce(url, body, timeoutMs) {
|
|
101
137
|
return (0, import_core.fetchWithTimeout)(
|
|
102
138
|
url,
|
|
@@ -150,16 +186,166 @@ var GeminiTTSProvider = class {
|
|
|
150
186
|
}
|
|
151
187
|
);
|
|
152
188
|
}
|
|
189
|
+
openStreamOnce(url, body, timeoutMs, signal) {
|
|
190
|
+
const attempt = new AbortController();
|
|
191
|
+
const onCallerAbort = () => attempt.abort(signal.reason);
|
|
192
|
+
if (signal?.aborted) {
|
|
193
|
+
attempt.abort(signal.reason);
|
|
194
|
+
} else {
|
|
195
|
+
signal?.addEventListener("abort", onCallerAbort);
|
|
196
|
+
}
|
|
197
|
+
let open = null;
|
|
198
|
+
let cancelledByConsumer = false;
|
|
199
|
+
let deliverStream;
|
|
200
|
+
const delivery = new Promise((resolve) => {
|
|
201
|
+
deliverStream = resolve;
|
|
202
|
+
});
|
|
203
|
+
const request = (0, import_core.fetchWithTimeout)(
|
|
204
|
+
url,
|
|
205
|
+
{
|
|
206
|
+
method: "POST",
|
|
207
|
+
headers: {
|
|
208
|
+
// Never in the URL: proxies and request logs capture query strings.
|
|
209
|
+
"x-goog-api-key": this.apiKey,
|
|
210
|
+
"Content-Type": "application/json"
|
|
211
|
+
},
|
|
212
|
+
body
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
timeoutMs,
|
|
216
|
+
timeoutMessage: this.timeoutMessage,
|
|
217
|
+
signal: attempt.signal,
|
|
218
|
+
mapError: (error) => (0, import_core.toCharivoError)("provider", error, "Gemini TTS request failed")
|
|
219
|
+
},
|
|
220
|
+
// The whole SSE pump runs in here, not after this helper resolves: the
|
|
221
|
+
// helper unwires its timer and the caller's signal the moment this
|
|
222
|
+
// callback settles, and an unwired request keeps downloading — keeps
|
|
223
|
+
// Gemini synthesizing, and billing — long after a stop().
|
|
224
|
+
async (response) => {
|
|
225
|
+
if (response.status >= 500) {
|
|
226
|
+
return {
|
|
227
|
+
retry: new import_core.CharivoProviderError(
|
|
228
|
+
`Gemini TTS Error: ${await readResponseText(response)}`
|
|
229
|
+
)
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
if (!response.ok) {
|
|
233
|
+
throw new import_core.CharivoProviderError(
|
|
234
|
+
`Gemini TTS Error: ${await readResponseText(response)}`
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
const noAudio = () => ({
|
|
238
|
+
retry: new import_core.CharivoProviderError(
|
|
239
|
+
"Gemini TTS Error: response contained no audio"
|
|
240
|
+
)
|
|
241
|
+
});
|
|
242
|
+
const fail = (message) => {
|
|
243
|
+
attempt.abort();
|
|
244
|
+
throw new import_core.CharivoProviderError(`Gemini TTS Error: ${message}`);
|
|
245
|
+
};
|
|
246
|
+
const openWith = (audio, pcm) => {
|
|
247
|
+
const { sampleRate, channels } = parseL16MimeType(audio.mimeType);
|
|
248
|
+
let controller;
|
|
249
|
+
const streamBody = new ReadableStream({
|
|
250
|
+
start(streamController) {
|
|
251
|
+
controller = streamController;
|
|
252
|
+
streamController.enqueue(pcm);
|
|
253
|
+
},
|
|
254
|
+
cancel() {
|
|
255
|
+
cancelledByConsumer = true;
|
|
256
|
+
attempt.abort();
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
return {
|
|
260
|
+
controller,
|
|
261
|
+
mimeType: audio.mimeType,
|
|
262
|
+
result: {
|
|
263
|
+
stream: {
|
|
264
|
+
body: streamBody,
|
|
265
|
+
format: { encoding: "pcm-s16le", sampleRate, channels }
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
};
|
|
270
|
+
const reader = response.body?.getReader();
|
|
271
|
+
if (!reader) {
|
|
272
|
+
return noAudio();
|
|
273
|
+
}
|
|
274
|
+
const decoder = new TextDecoder();
|
|
275
|
+
let buffer = "";
|
|
276
|
+
try {
|
|
277
|
+
for (; ; ) {
|
|
278
|
+
const { done, value } = await reader.read();
|
|
279
|
+
if (done) {
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
buffer += decoder.decode(value, { stream: true });
|
|
283
|
+
const frames = buffer.split(SSE_FRAME_SEPARATOR);
|
|
284
|
+
buffer = frames.pop() ?? "";
|
|
285
|
+
for (const frame of frames) {
|
|
286
|
+
const { audio, finishReason } = extractStreamEvent(
|
|
287
|
+
parseSseData(frame)
|
|
288
|
+
);
|
|
289
|
+
if (audio) {
|
|
290
|
+
const pcm = decodeBase64(audio.data);
|
|
291
|
+
if (!open) {
|
|
292
|
+
open = openWith(audio, pcm);
|
|
293
|
+
deliverStream(open.result);
|
|
294
|
+
} else if (audio.mimeType !== open.mimeType) {
|
|
295
|
+
fail(
|
|
296
|
+
`audio format changed to "${audio.mimeType}" mid-stream`
|
|
297
|
+
);
|
|
298
|
+
} else {
|
|
299
|
+
open.controller.enqueue(pcm);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
if (finishReason) {
|
|
303
|
+
void reader.cancel().catch(() => void 0);
|
|
304
|
+
if (!open) {
|
|
305
|
+
return noAudio();
|
|
306
|
+
}
|
|
307
|
+
if (finishReason !== "STOP") {
|
|
308
|
+
fail(`stream ended with finishReason "${finishReason}"`);
|
|
309
|
+
}
|
|
310
|
+
open.controller.close();
|
|
311
|
+
return open.result;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
return open ? fail("stream ended without a finish reason") : noAudio();
|
|
316
|
+
} catch (error) {
|
|
317
|
+
attempt.abort();
|
|
318
|
+
throw error;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
);
|
|
322
|
+
const outcome = request.then(
|
|
323
|
+
(result) => {
|
|
324
|
+
signal?.removeEventListener("abort", onCallerAbort);
|
|
325
|
+
return result;
|
|
326
|
+
},
|
|
327
|
+
(error) => {
|
|
328
|
+
signal?.removeEventListener("abort", onCallerAbort);
|
|
329
|
+
if (!open) {
|
|
330
|
+
throw error;
|
|
331
|
+
}
|
|
332
|
+
if (!cancelledByConsumer) {
|
|
333
|
+
open.controller.error(
|
|
334
|
+
isAbortError(error) ? error : (0, import_core.toCharivoError)("provider", error, "Gemini TTS request failed")
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
return open.result;
|
|
338
|
+
}
|
|
339
|
+
);
|
|
340
|
+
return Promise.race([delivery, outcome]);
|
|
341
|
+
}
|
|
153
342
|
};
|
|
154
343
|
function createGeminiTTSProvider(config) {
|
|
155
344
|
return new GeminiTTSProvider(config);
|
|
156
345
|
}
|
|
157
346
|
function extractInlineAudio(payload) {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
}
|
|
161
|
-
const candidate = Array.isArray(payload.candidates) ? payload.candidates[0] : void 0;
|
|
162
|
-
if (!isRecord(candidate) || !isRecord(candidate.content)) {
|
|
347
|
+
const candidate = firstCandidate(payload);
|
|
348
|
+
if (!candidate || !isRecord(candidate.content)) {
|
|
163
349
|
return null;
|
|
164
350
|
}
|
|
165
351
|
const parts = Array.isArray(candidate.content.parts) ? candidate.content.parts : [];
|
|
@@ -172,6 +358,22 @@ function extractInlineAudio(payload) {
|
|
|
172
358
|
const { mimeType, data } = part.inlineData;
|
|
173
359
|
return typeof mimeType === "string" && typeof data === "string" ? { mimeType, data } : null;
|
|
174
360
|
}
|
|
361
|
+
function extractStreamEvent(payload) {
|
|
362
|
+
const candidate = firstCandidate(payload);
|
|
363
|
+
const finishReason = candidate && typeof candidate.finishReason === "string" ? candidate.finishReason : null;
|
|
364
|
+
return { audio: extractInlineAudio(payload), finishReason };
|
|
365
|
+
}
|
|
366
|
+
function parseSseData(frame) {
|
|
367
|
+
const data = frame.split(SSE_LINE_SEPARATOR).filter((line) => line.startsWith(SSE_DATA_PREFIX)).map((line) => line.slice(SSE_DATA_PREFIX.length).trim()).join("\n");
|
|
368
|
+
return data ? JSON.parse(data) : null;
|
|
369
|
+
}
|
|
370
|
+
function firstCandidate(payload) {
|
|
371
|
+
if (!isRecord(payload) || !Array.isArray(payload.candidates)) {
|
|
372
|
+
return null;
|
|
373
|
+
}
|
|
374
|
+
const candidate = payload.candidates[0];
|
|
375
|
+
return isRecord(candidate) ? candidate : null;
|
|
376
|
+
}
|
|
175
377
|
function parseL16MimeType(mimeType) {
|
|
176
378
|
const [mediaType, ...parameters] = mimeType.split(";");
|
|
177
379
|
if (mediaType?.trim().toLowerCase() !== "audio/l16") {
|
|
@@ -268,12 +470,21 @@ function isAbortError(error) {
|
|
|
268
470
|
var GeminiTTSPlayer = class {
|
|
269
471
|
playbackMode = "audio";
|
|
270
472
|
audioMimeType = "audio/wav";
|
|
473
|
+
/**
|
|
474
|
+
* Present only on a streaming player: the manager selects the streaming
|
|
475
|
+
* path with `typeof player.generateAudioStream === "function"`, so an
|
|
476
|
+
* unflagged player must not carry the method at all.
|
|
477
|
+
*/
|
|
478
|
+
generateAudioStream;
|
|
271
479
|
provider;
|
|
272
|
-
constructor(config) {
|
|
480
|
+
constructor({ streaming, ...config }) {
|
|
273
481
|
this.provider = createGeminiTTSProvider({
|
|
274
482
|
...config,
|
|
275
483
|
dangerouslyAllowBrowser: true
|
|
276
484
|
});
|
|
485
|
+
if (streaming) {
|
|
486
|
+
this.generateAudioStream = (text, options, signal) => this.provider.generateSpeechStream(text, options, signal);
|
|
487
|
+
}
|
|
277
488
|
}
|
|
278
489
|
/**
|
|
279
490
|
* Stateless audio generation (used by the TTS Manager)
|