@framers/agentos-ext-streaming-tts-elevenlabs 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/SKILL.md +2 -0
- package/dist/ElevenLabsTTSSession.d.ts +24 -33
- package/dist/ElevenLabsTTSSession.d.ts.map +1 -1
- package/dist/ElevenLabsTTSSession.js +82 -100
- package/dist/ElevenLabsTTSSession.js.map +1 -1
- package/manifest.json +1 -1
- package/package.json +4 -1
- package/src/ElevenLabsTTSSession.ts +72 -129
package/README.md
CHANGED
|
@@ -48,14 +48,20 @@ Voice settings in `startSession({ providerOptions: ... })` override pack options
|
|
|
48
48
|
|
|
49
49
|
The pack registers an enabled `streaming-tts-provider` with provider id `elevenlabs-streaming-tts`. `createElevenLabsStreamingTTS` also creates a standalone provider from an API key and optional voice settings.
|
|
50
50
|
|
|
51
|
-
`startSession` opens a WebSocket
|
|
51
|
+
`startSession` opens a WebSocket. On opening, a live session sends the API key and voice settings, then drains queued messages. Each token is sent once; punctuation adds `flush: true` without resending earlier tokens. `flush` queues EOS and resolves without waiting for remote completion.
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
JSON base64 and retained binary messages emit core and legacy MP3 audio. A remote final emits completion once. Duration fields remain estimates, and alignment data is not consumed.
|
|
54
54
|
|
|
55
|
-
`cancel` closes the socket and emits `cancelled` with pending text. `close` closes the socket and emits `close`. The generic session `language` field is not forwarded. No reconnect or explicit timeout is implemented.
|
|
55
|
+
`cancel` closes the socket and emits `cancelled` with pending text. `close` closes the socket and emits `close`. A socket error emits `error`, as does a text message that is not a JSON object, carries an `error` field or holds an `audio` value that is not a base64 string of at most 8,388,608 characters. The generic session `language` field is not forwarded. No reconnect or explicit timeout is implemented.
|
|
56
56
|
|
|
57
57
|
The factory uses an empty key when the secret is absent, unless the runtime options spread overrides `apiKey`. It does not validate credentials or read environment variables.
|
|
58
58
|
|
|
59
|
+
## Core event compatibility
|
|
60
|
+
|
|
61
|
+
The session decodes base64 JSON audio envelopes and retains binary MP3 compatibility. Every decoded chunk emits `audio` and the legacy `audio_chunk` in wire order. Subscribe to one audio name. Text fragments are sent once, including when punctuation requests synthesis.
|
|
62
|
+
|
|
63
|
+
For compatibility, `flush()` resolves after EOS is queued. A remote final event emits `flush_complete` and the legacy `utterance_complete` once. EOS enqueue and socket closure are not completed synthesis. Cancellation discards queued text and blocks a subsequent socket-open event. Duration values remain estimates. This event-only adapter has no awaited output sink, and these aliases do not establish scoped hosted eligibility.
|
|
64
|
+
|
|
59
65
|
## License
|
|
60
66
|
|
|
61
67
|
Apache-2.0
|
package/SKILL.md
CHANGED
|
@@ -58,8 +58,10 @@ Provider-specific options via `providerOptions`:
|
|
|
58
58
|
|
|
59
59
|
| Event | Payload | Description |
|
|
60
60
|
|----------------------|-------------------------|----------------------------------------------------|
|
|
61
|
+
| `audio` | `EncodedAudioChunk` | Core audio event; same MP3 chunk as `audio_chunk` |
|
|
61
62
|
| `audio_chunk` | `EncodedAudioChunk` | MP3 audio buffer ready for playback |
|
|
62
63
|
| `utterance_complete` | `{ text, durationMs }` | ElevenLabs signalled final audio generation done |
|
|
64
|
+
| `flush_complete` | (none) | Remote final received; synthesis complete |
|
|
63
65
|
| `cancelled` | `{ remaining: string }` | Session was cancelled; remaining text not rendered |
|
|
64
66
|
| `error` | `Error` | WebSocket or synthesis error |
|
|
65
67
|
| `close` | — | Session fully terminated |
|
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
*
|
|
8
8
|
* | Event | Payload | Description |
|
|
9
9
|
* |----------------------|----------------------------|------------------------------------------------------|
|
|
10
|
+
* | `'audio'` | `EncodedAudioChunk` | Core audio event; same MP3 chunk as `audio_chunk` |
|
|
10
11
|
* | `'audio_chunk'` | `EncodedAudioChunk` | MP3 audio buffer from ElevenLabs, ready for playback |
|
|
11
12
|
* | `'utterance_complete'`| `{ text, durationMs }` | ElevenLabs signalled end of audio generation |
|
|
13
|
+
* | `'flush_complete'` | (none) | Remote final received; synthesis complete |
|
|
12
14
|
* | `'cancelled'` | `{ remaining: string }` | Session cancelled; remaining text not rendered |
|
|
13
15
|
* | `'error'` | `Error` | WebSocket or protocol error |
|
|
14
16
|
* | `'close'` | — | Session fully terminated |
|
|
@@ -22,9 +24,14 @@
|
|
|
22
24
|
* 3. Stream text tokens as `{ text: "..." }` JSON messages.
|
|
23
25
|
* 4. To generate audio for accumulated text, include `flush: true` in the text message.
|
|
24
26
|
* 5. To end the stream, send an EOS (End-Of-Stream) message: `{ text: "" }`.
|
|
25
|
-
* 6.
|
|
27
|
+
* 6. JSON audio envelopes carry base64 MP3. Binary audio remains accepted for compatibility.
|
|
26
28
|
* 7. When all audio has been generated the server sends `{ isFinal: true }`.
|
|
27
29
|
*
|
|
30
|
+
* `audio` and `audio_chunk` carry each decoded chunk in wire order. One remote
|
|
31
|
+
* final emits `utterance_complete` and `flush_complete`. Queuing EOS and closing
|
|
32
|
+
* the socket are not completion. Durations remain legacy estimates; this pack
|
|
33
|
+
* does not provide the awaited, bounded output required by the hosted profile.
|
|
34
|
+
*
|
|
28
35
|
* @module streaming-tts-elevenlabs/ElevenLabsTTSSession
|
|
29
36
|
*/
|
|
30
37
|
import { EventEmitter } from 'node:events';
|
|
@@ -74,6 +81,12 @@ export declare class ElevenLabsTTSSession extends EventEmitter {
|
|
|
74
81
|
private pendingText;
|
|
75
82
|
/** `true` after {@link close} or {@link cancel} is called. */
|
|
76
83
|
private closed;
|
|
84
|
+
/** `true` once {@link flush} has queued EOS; later tokens and flushes are ignored. */
|
|
85
|
+
private eosQueued;
|
|
86
|
+
/** `true` once the server's final message has been handled; later messages are ignored. */
|
|
87
|
+
private finalReceived;
|
|
88
|
+
/** All text accepted by {@link pushTokens}, reported by `utterance_complete`. */
|
|
89
|
+
private totalText;
|
|
77
90
|
/**
|
|
78
91
|
* Whether the BOS (beginning-of-stream) message has been sent.
|
|
79
92
|
*
|
|
@@ -93,52 +106,30 @@ export declare class ElevenLabsTTSSession extends EventEmitter {
|
|
|
93
106
|
* future `language` forwarding).
|
|
94
107
|
*/
|
|
95
108
|
constructor(providerConfig: ElevenLabsStreamingTTSConfig, _sessionConfig: StreamingTTSConfig);
|
|
96
|
-
/**
|
|
97
|
-
* Push LLM output token(s) into the stream.
|
|
98
|
-
*
|
|
99
|
-
* Text is accumulated in an internal buffer. When a sentence boundary
|
|
100
|
-
* (`.`, `?`, `!`) is detected the accumulated segment is sent to ElevenLabs
|
|
101
|
-
* with `flush: true` so the server generates audio immediately. Mid-sentence
|
|
102
|
-
* tokens are sent with a trailing space continuation to avoid falling intonation
|
|
103
|
-
* on partial phrases.
|
|
104
|
-
*
|
|
105
|
-
* @param text - One or more LLM token characters to append.
|
|
106
|
-
*/
|
|
109
|
+
/** Send each token once; punctuation requests synthesis without resending prior tokens. */
|
|
107
110
|
pushTokens(text: string): void;
|
|
108
|
-
/**
|
|
109
|
-
* Flush any remaining buffered text and signal EOS to ElevenLabs.
|
|
110
|
-
*
|
|
111
|
-
* Call this when the LLM stream has ended. Sends an EOS (end-of-stream)
|
|
112
|
-
* message (`{ text: "" }`) which instructs ElevenLabs to finalise audio
|
|
113
|
-
* generation for all remaining buffered text and close the stream.
|
|
114
|
-
*
|
|
115
|
-
* @returns Promise that resolves when the EOS message is queued for sending.
|
|
116
|
-
*/
|
|
111
|
+
/** Queue EOS once. Listen for flush_complete to observe actual remote synthesis completion. */
|
|
117
112
|
flush(): Promise<void>;
|
|
118
|
-
/**
|
|
119
|
-
* Cancel the session immediately.
|
|
120
|
-
*
|
|
121
|
-
* Closes the WebSocket, discards any unsent text, and emits a `'cancelled'`
|
|
122
|
-
* event carrying the remaining un-synthesised text.
|
|
123
|
-
*/
|
|
113
|
+
/** Discard queued messages and prevent an open event from admitting work after cancellation. */
|
|
124
114
|
cancel(): void;
|
|
125
|
-
/**
|
|
126
|
-
* Close the session cleanly.
|
|
127
|
-
*
|
|
128
|
-
* Sends a WebSocket close frame and emits a `'close'` event.
|
|
129
|
-
* Idempotent — safe to call multiple times.
|
|
130
|
-
*/
|
|
115
|
+
/** Close the socket once; socket closure alone never reports completed synthesis. */
|
|
131
116
|
close(): void;
|
|
132
117
|
/**
|
|
133
118
|
* Attach event listeners to the WebSocket instance and handle the full
|
|
134
119
|
* message lifecycle.
|
|
135
120
|
*/
|
|
136
121
|
private _wireWebSocket;
|
|
122
|
+
/**
|
|
123
|
+
* Emit one decoded MP3 chunk as `audio`, then as the legacy `audio_chunk`.
|
|
124
|
+
* Nothing is emitted after closure, after the remote final or for an empty buffer.
|
|
125
|
+
*/
|
|
126
|
+
private _emitAudio;
|
|
137
127
|
/**
|
|
138
128
|
* Send a JSON message over the WebSocket.
|
|
139
129
|
*
|
|
140
130
|
* If the socket is not yet open, the message is queued and will be sent once
|
|
141
131
|
* the `open` event fires (after the BOS message is dispatched).
|
|
132
|
+
* Nothing is sent or queued once the session is closed or cancelled.
|
|
142
133
|
*
|
|
143
134
|
* @param msg - The message object to serialise and send.
|
|
144
135
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ElevenLabsTTSSession.d.ts","sourceRoot":"","sources":["../src/ElevenLabsTTSSession.ts"],"names":[],"mappings":"AACA
|
|
1
|
+
{"version":3,"file":"ElevenLabsTTSSession.d.ts","sourceRoot":"","sources":["../src/ElevenLabsTTSSession.ts"],"names":[],"mappings":"AACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,4BAA4B,EAAE,kBAAkB,EAAqB,MAAM,YAAY,CAAC;AAoDtG;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,oBAAqB,SAAQ,YAAY;IA0ElD,OAAO,CAAC,QAAQ,CAAC,cAAc;IArEjC,2BAA2B;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IAEjC,2BAA2B;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IAEjC,mCAAmC;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,oCAAoC;IACpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IAEzC,sCAAsC;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAE/B,uCAAuC;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAU;IAM1C,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAE/B;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAAM;IAEzB,8DAA8D;IAC9D,OAAO,CAAC,MAAM,CAAS;IAEvB,sFAAsF;IACtF,OAAO,CAAC,SAAS,CAAS;IAE1B,2FAA2F;IAC3F,OAAO,CAAC,aAAa,CAAS;IAE9B,iFAAiF;IACjF,OAAO,CAAC,SAAS,CAAM;IAEvB;;;;OAIG;IACH,OAAO,CAAC,eAAe,CAAS;IAEhC;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAM/C;;;;OAIG;gBAEgB,cAAc,EAAE,4BAA4B,EAC7D,cAAc,EAAE,kBAAkB;IA0BpC,2FAA2F;IAC3F,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAQ9B,+FAA+F;IACzF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B,gGAAgG;IAChG,MAAM,IAAI,IAAI;IAQd,qFAAqF;IACrF,KAAK,IAAI,IAAI;IAUb;;;OAGG;IACH,OAAO,CAAC,cAAc;IA6CtB;;;OAGG;IACH,OAAO,CAAC,UAAU;IAOlB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK;CASd"}
|
|
@@ -8,8 +8,10 @@
|
|
|
8
8
|
*
|
|
9
9
|
* | Event | Payload | Description |
|
|
10
10
|
* |----------------------|----------------------------|------------------------------------------------------|
|
|
11
|
+
* | `'audio'` | `EncodedAudioChunk` | Core audio event; same MP3 chunk as `audio_chunk` |
|
|
11
12
|
* | `'audio_chunk'` | `EncodedAudioChunk` | MP3 audio buffer from ElevenLabs, ready for playback |
|
|
12
13
|
* | `'utterance_complete'`| `{ text, durationMs }` | ElevenLabs signalled end of audio generation |
|
|
14
|
+
* | `'flush_complete'` | (none) | Remote final received; synthesis complete |
|
|
13
15
|
* | `'cancelled'` | `{ remaining: string }` | Session cancelled; remaining text not rendered |
|
|
14
16
|
* | `'error'` | `Error` | WebSocket or protocol error |
|
|
15
17
|
* | `'close'` | — | Session fully terminated |
|
|
@@ -23,9 +25,14 @@
|
|
|
23
25
|
* 3. Stream text tokens as `{ text: "..." }` JSON messages.
|
|
24
26
|
* 4. To generate audio for accumulated text, include `flush: true` in the text message.
|
|
25
27
|
* 5. To end the stream, send an EOS (End-Of-Stream) message: `{ text: "" }`.
|
|
26
|
-
* 6.
|
|
28
|
+
* 6. JSON audio envelopes carry base64 MP3. Binary audio remains accepted for compatibility.
|
|
27
29
|
* 7. When all audio has been generated the server sends `{ isFinal: true }`.
|
|
28
30
|
*
|
|
31
|
+
* `audio` and `audio_chunk` carry each decoded chunk in wire order. One remote
|
|
32
|
+
* final emits `utterance_complete` and `flush_complete`. Queuing EOS and closing
|
|
33
|
+
* the socket are not completion. Durations remain legacy estimates; this pack
|
|
34
|
+
* does not provide the awaited, bounded output required by the hosted profile.
|
|
35
|
+
*
|
|
29
36
|
* @module streaming-tts-elevenlabs/ElevenLabsTTSSession
|
|
30
37
|
*/
|
|
31
38
|
import { EventEmitter } from 'node:events';
|
|
@@ -120,6 +127,12 @@ export class ElevenLabsTTSSession extends EventEmitter {
|
|
|
120
127
|
pendingText = '';
|
|
121
128
|
/** `true` after {@link close} or {@link cancel} is called. */
|
|
122
129
|
closed = false;
|
|
130
|
+
/** `true` once {@link flush} has queued EOS; later tokens and flushes are ignored. */
|
|
131
|
+
eosQueued = false;
|
|
132
|
+
/** `true` once the server's final message has been handled; later messages are ignored. */
|
|
133
|
+
finalReceived = false;
|
|
134
|
+
/** All text accepted by {@link pushTokens}, reported by `utterance_complete`. */
|
|
135
|
+
totalText = '';
|
|
123
136
|
/**
|
|
124
137
|
* Whether the BOS (beginning-of-stream) message has been sent.
|
|
125
138
|
*
|
|
@@ -160,81 +173,41 @@ export class ElevenLabsTTSSession extends EventEmitter {
|
|
|
160
173
|
// -------------------------------------------------------------------------
|
|
161
174
|
// StreamingTTSSession interface
|
|
162
175
|
// -------------------------------------------------------------------------
|
|
163
|
-
/**
|
|
164
|
-
* Push LLM output token(s) into the stream.
|
|
165
|
-
*
|
|
166
|
-
* Text is accumulated in an internal buffer. When a sentence boundary
|
|
167
|
-
* (`.`, `?`, `!`) is detected the accumulated segment is sent to ElevenLabs
|
|
168
|
-
* with `flush: true` so the server generates audio immediately. Mid-sentence
|
|
169
|
-
* tokens are sent with a trailing space continuation to avoid falling intonation
|
|
170
|
-
* on partial phrases.
|
|
171
|
-
*
|
|
172
|
-
* @param text - One or more LLM token characters to append.
|
|
173
|
-
*/
|
|
176
|
+
/** Send each token once; punctuation requests synthesis without resending prior tokens. */
|
|
174
177
|
pushTokens(text) {
|
|
175
|
-
if (this.closed)
|
|
178
|
+
if (this.closed || this.eosQueued || this.finalReceived || !text)
|
|
176
179
|
return;
|
|
177
180
|
this.pendingText += text;
|
|
178
|
-
|
|
179
|
-
const
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
const segment = this.pendingText;
|
|
181
|
+
this.totalText += text;
|
|
182
|
+
const boundary = SENTENCE_BOUNDARY_RE.test(text);
|
|
183
|
+
this._send(boundary ? { text, flush: true } : { text });
|
|
184
|
+
if (boundary)
|
|
183
185
|
this.pendingText = '';
|
|
184
|
-
this._send({ text: segment, flush: true });
|
|
185
|
-
}
|
|
186
|
-
else {
|
|
187
|
-
// Send the token chunk without flush; include a space continuation to hint
|
|
188
|
-
// that the phrase is mid-sentence (prevents falling prosody on partial text).
|
|
189
|
-
this._send({ text });
|
|
190
|
-
}
|
|
191
186
|
}
|
|
192
|
-
/**
|
|
193
|
-
* Flush any remaining buffered text and signal EOS to ElevenLabs.
|
|
194
|
-
*
|
|
195
|
-
* Call this when the LLM stream has ended. Sends an EOS (end-of-stream)
|
|
196
|
-
* message (`{ text: "" }`) which instructs ElevenLabs to finalise audio
|
|
197
|
-
* generation for all remaining buffered text and close the stream.
|
|
198
|
-
*
|
|
199
|
-
* @returns Promise that resolves when the EOS message is queued for sending.
|
|
200
|
-
*/
|
|
187
|
+
/** Queue EOS once. Listen for flush_complete to observe actual remote synthesis completion. */
|
|
201
188
|
async flush() {
|
|
202
|
-
if (this.closed)
|
|
189
|
+
if (this.closed || this.eosQueued || this.finalReceived)
|
|
203
190
|
return;
|
|
204
|
-
|
|
205
|
-
if (this.pendingText.trim().length > 0) {
|
|
206
|
-
const segment = this.pendingText;
|
|
207
|
-
this.pendingText = '';
|
|
208
|
-
this._send({ text: segment, flush: true });
|
|
209
|
-
}
|
|
210
|
-
// Send EOS — empty string signals end of stream to ElevenLabs.
|
|
191
|
+
this.eosQueued = true;
|
|
211
192
|
this._send({ text: '' });
|
|
212
193
|
}
|
|
213
|
-
/**
|
|
214
|
-
* Cancel the session immediately.
|
|
215
|
-
*
|
|
216
|
-
* Closes the WebSocket, discards any unsent text, and emits a `'cancelled'`
|
|
217
|
-
* event carrying the remaining un-synthesised text.
|
|
218
|
-
*/
|
|
194
|
+
/** Discard queued messages and prevent an open event from admitting work after cancellation. */
|
|
219
195
|
cancel() {
|
|
220
196
|
if (this.closed)
|
|
221
197
|
return;
|
|
222
198
|
this.closed = true;
|
|
223
199
|
const remaining = this.pendingText;
|
|
224
200
|
this.pendingText = '';
|
|
201
|
+
this.sendQueue.length = 0;
|
|
225
202
|
this.ws.close();
|
|
226
203
|
this.emit('cancelled', { remaining });
|
|
227
204
|
}
|
|
228
|
-
/**
|
|
229
|
-
* Close the session cleanly.
|
|
230
|
-
*
|
|
231
|
-
* Sends a WebSocket close frame and emits a `'close'` event.
|
|
232
|
-
* Idempotent — safe to call multiple times.
|
|
233
|
-
*/
|
|
205
|
+
/** Close the socket once; socket closure alone never reports completed synthesis. */
|
|
234
206
|
close() {
|
|
235
207
|
if (this.closed)
|
|
236
208
|
return;
|
|
237
209
|
this.closed = true;
|
|
210
|
+
this.sendQueue.length = 0;
|
|
238
211
|
this.ws.close();
|
|
239
212
|
this.emit('close');
|
|
240
213
|
}
|
|
@@ -247,82 +220,91 @@ export class ElevenLabsTTSSession extends EventEmitter {
|
|
|
247
220
|
*/
|
|
248
221
|
_wireWebSocket() {
|
|
249
222
|
this.ws.on('open', () => {
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
style: this.style,
|
|
257
|
-
use_speaker_boost: this.useSpeakerBoost,
|
|
258
|
-
},
|
|
259
|
-
xi_api_key: this.providerConfig.apiKey,
|
|
260
|
-
};
|
|
261
|
-
// Use the raw WS send to bypass the queue (BOS must be first).
|
|
262
|
-
this.ws.send(JSON.stringify(bosMessage));
|
|
223
|
+
if (this.closed)
|
|
224
|
+
return;
|
|
225
|
+
this.ws.send(JSON.stringify({ text: ' ', voice_settings: {
|
|
226
|
+
stability: this.stability, similarity_boost: this.similarityBoost,
|
|
227
|
+
style: this.style, use_speaker_boost: this.useSpeakerBoost,
|
|
228
|
+
}, xi_api_key: this.providerConfig.apiKey }));
|
|
263
229
|
this.bosAcknowledged = true;
|
|
264
|
-
// Drain any messages queued before the socket opened.
|
|
265
230
|
for (const queued of this.sendQueue) {
|
|
231
|
+
if (this.closed)
|
|
232
|
+
break;
|
|
266
233
|
this.ws.send(JSON.stringify(queued));
|
|
267
234
|
}
|
|
268
235
|
this.sendQueue.length = 0;
|
|
269
236
|
});
|
|
270
237
|
this.ws.on('message', (data, isBinary) => {
|
|
271
|
-
if (this.closed)
|
|
238
|
+
if (this.closed || this.finalReceived)
|
|
272
239
|
return;
|
|
273
240
|
if (isBinary) {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
: Buffer.from(data);
|
|
278
|
-
const durationMs = estimateDurationMs(this.pendingText);
|
|
279
|
-
const chunk = {
|
|
280
|
-
audio,
|
|
281
|
-
format: 'mp3',
|
|
282
|
-
sampleRate: SAMPLE_RATE_HZ,
|
|
283
|
-
durationMs,
|
|
284
|
-
text: '',
|
|
285
|
-
};
|
|
286
|
-
this.emit('audio_chunk', chunk);
|
|
241
|
+
const audio = Buffer.isBuffer(data) ? data : Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data);
|
|
242
|
+
this._emitAudio(audio);
|
|
243
|
+
return;
|
|
287
244
|
}
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
245
|
+
let msg;
|
|
246
|
+
try {
|
|
247
|
+
const parsed = JSON.parse(data.toString());
|
|
248
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed))
|
|
249
|
+
throw new Error('shape');
|
|
250
|
+
msg = parsed;
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
this.emit('error', new Error('ElevenLabs status envelope invalid'));
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
if (msg.error !== undefined) {
|
|
257
|
+
this.emit('error', new Error('ElevenLabs synthesis failed'));
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
if (msg.audio !== undefined && msg.audio !== null && msg.audio !== '') {
|
|
261
|
+
if (typeof msg.audio !== 'string' || msg.audio.length > 8388608 || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(msg.audio)) {
|
|
262
|
+
this.emit('error', new Error('ElevenLabs audio envelope invalid'));
|
|
263
|
+
return;
|
|
302
264
|
}
|
|
265
|
+
this._emitAudio(Buffer.from(msg.audio, 'base64'));
|
|
303
266
|
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
this.
|
|
267
|
+
if (msg.isFinal === true && !this.closed) {
|
|
268
|
+
this.finalReceived = true;
|
|
269
|
+
this.emit('utterance_complete', { text: this.totalText, durationMs: estimateDurationMs(this.totalText) });
|
|
270
|
+
if (!this.closed)
|
|
271
|
+
this.emit('flush_complete');
|
|
308
272
|
}
|
|
309
273
|
});
|
|
274
|
+
this.ws.on('error', () => { if (!this.closed)
|
|
275
|
+
this.emit('error', new Error('ElevenLabs connection failed')); });
|
|
310
276
|
this.ws.on('close', () => {
|
|
311
277
|
if (!this.closed) {
|
|
312
278
|
this.closed = true;
|
|
279
|
+
this.sendQueue.length = 0;
|
|
313
280
|
this.emit('close');
|
|
314
281
|
}
|
|
315
282
|
});
|
|
316
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* Emit one decoded MP3 chunk as `audio`, then as the legacy `audio_chunk`.
|
|
286
|
+
* Nothing is emitted after closure, after the remote final or for an empty buffer.
|
|
287
|
+
*/
|
|
288
|
+
_emitAudio(audio) {
|
|
289
|
+
if (this.closed || this.finalReceived || audio.length === 0)
|
|
290
|
+
return;
|
|
291
|
+
const chunk = { audio, format: 'mp3', sampleRate: SAMPLE_RATE_HZ, durationMs: estimateDurationMs(this.pendingText), text: '' };
|
|
292
|
+
this.emit('audio', chunk);
|
|
293
|
+
if (!this.closed)
|
|
294
|
+
this.emit('audio_chunk', chunk);
|
|
295
|
+
}
|
|
317
296
|
/**
|
|
318
297
|
* Send a JSON message over the WebSocket.
|
|
319
298
|
*
|
|
320
299
|
* If the socket is not yet open, the message is queued and will be sent once
|
|
321
300
|
* the `open` event fires (after the BOS message is dispatched).
|
|
301
|
+
* Nothing is sent or queued once the session is closed or cancelled.
|
|
322
302
|
*
|
|
323
303
|
* @param msg - The message object to serialise and send.
|
|
324
304
|
*/
|
|
325
305
|
_send(msg) {
|
|
306
|
+
if (this.closed)
|
|
307
|
+
return;
|
|
326
308
|
if (this.bosAcknowledged && this.ws.readyState === WebSocket.OPEN) {
|
|
327
309
|
this.ws.send(JSON.stringify(msg));
|
|
328
310
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ElevenLabsTTSSession.js","sourceRoot":"","sources":["../src/ElevenLabsTTSSession.ts"],"names":[],"mappings":"AAAA,cAAc;AACd
|
|
1
|
+
{"version":3,"file":"ElevenLabsTTSSession.js","sourceRoot":"","sources":["../src/ElevenLabsTTSSession.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,SAAS,MAAM,IAAI,CAAC;AAG3B,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,yCAAyC;AACzC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC;AAErD,kEAAkE;AAClE,MAAM,gBAAgB,GAAG,sBAAsB,CAAC;AAEhD,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC;AAE3C,gDAAgD;AAChD,MAAM,aAAa,GAAG,eAAe,CAAC;AAEtC,4DAA4D;AAC5D,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B;;;GAGG;AACH,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAC;AAElC,iDAAiD;AACjD,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAErC,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAClE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,OAAO,oBAAqB,SAAQ,YAAY;IA0EjC;IAzEnB,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,2BAA2B;IACV,OAAO,CAAS;IAEjC,2BAA2B;IACV,OAAO,CAAS;IAEjC,mCAAmC;IAClB,SAAS,CAAS;IAEnC,oCAAoC;IACnB,eAAe,CAAS;IAEzC,sCAAsC;IACrB,KAAK,CAAS;IAE/B,uCAAuC;IACtB,eAAe,CAAU;IAE1C,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,sEAAsE;IACrD,EAAE,CAAY;IAE/B;;;;OAIG;IACK,WAAW,GAAG,EAAE,CAAC;IAEzB,8DAA8D;IACtD,MAAM,GAAG,KAAK,CAAC;IAEvB,sFAAsF;IAC9E,SAAS,GAAG,KAAK,CAAC;IAE1B,2FAA2F;IACnF,aAAa,GAAG,KAAK,CAAC;IAE9B,iFAAiF;IACzE,SAAS,GAAG,EAAE,CAAC;IAEvB;;;;OAIG;IACK,eAAe,GAAG,KAAK,CAAC;IAEhC;;;;;OAKG;IACc,SAAS,GAAkB,EAAE,CAAC;IAE/C,4EAA4E;IAC5E,cAAc;IACd,4EAA4E;IAE5E;;;;OAIG;IACH,YACmB,cAA4C,EAC7D,cAAkC;QAElC,KAAK,EAAE,CAAC;QAHS,mBAAc,GAAd,cAAc,CAA8B;QAK7D,kBAAkB;QAClB,IAAI,CAAC,OAAO,GAAU,cAAc,CAAC,OAAO,IAAW,gBAAgB,CAAC;QACxE,IAAI,CAAC,OAAO,GAAU,cAAc,CAAC,OAAO,IAAW,gBAAgB,CAAC;QACxE,IAAI,CAAC,SAAS,GAAQ,cAAc,CAAC,SAAS,IAAS,GAAG,CAAC;QAC3D,IAAI,CAAC,eAAe,GAAE,cAAc,CAAC,eAAe,IAAI,IAAI,CAAC;QAC7D,IAAI,CAAC,KAAK,GAAY,cAAc,CAAC,KAAK,IAAa,GAAG,CAAC;QAC3D,IAAI,CAAC,eAAe,GAAE,cAAc,CAAC,eAAe,IAAI,IAAI,CAAC;QAE7D,oCAAoC;QACpC,MAAM,KAAK,GACT,GAAG,kBAAkB,sBAAsB,IAAI,CAAC,OAAO,eAAe;YACtE,aAAa,IAAI,CAAC,OAAO,kBAAkB,aAAa,EAAE,CAAC;QAE7D,IAAI,CAAC,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAED,4EAA4E;IAC5E,gCAAgC;IAChC,4EAA4E;IAE5E,2FAA2F;IAC3F,UAAU,CAAC,IAAY;QACrB,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI;YAAE,OAAO;QACzE,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC;QAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC;QACjD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,IAAI,QAAQ;YAAE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;IACtC,CAAC;IAED,+FAA+F;IAC/F,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO;QAChE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED,gGAAgG;IAChG,MAAM;QACJ,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;QACjD,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,qFAAqF;IACrF,KAAK;QACH,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9C,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,4EAA4E;IAC5E,8BAA8B;IAC9B,4EAA4E;IAE5E;;;OAGG;IACK,cAAc;QACpB,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACtB,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO;YACxB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,cAAc,EAAE;oBACvD,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,gBAAgB,EAAE,IAAI,CAAC,eAAe;oBACjE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,iBAAiB,EAAE,IAAI,CAAC,eAAe;iBAC3D,EAAE,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpC,IAAI,IAAI,CAAC,MAAM;oBAAE,MAAM;gBACvB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAuB,EAAE,QAAiB,EAAE,EAAE;YACnE,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa;gBAAE,OAAO;YAC9C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC3G,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAAC,OAAO;YACjC,CAAC;YACD,IAAI,GAA4B,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACpD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC7F,GAAG,GAAG,MAAiC,CAAC;YAC1C,CAAC;YAAC,MAAM,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACxF,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACtG,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,KAAK,EAAE,EAAE,CAAC;gBACtE,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,IAAI,CAAC,kEAAkE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACvJ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;oBAAC,OAAO;gBAC7E,CAAC;gBACD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACpD,CAAC;YACD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACzC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAC1G,IAAI,CAAC,IAAI,CAAC,MAAM;oBAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChD,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACvB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,CAAC;QAC1F,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,KAAa;QAC9B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACpE,MAAM,KAAK,GAAsB,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;QAClJ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,GAAW;QACvB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YAClE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,gCAAgC;YAChC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;CACF"}
|
package/manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@framers/agentos-ext-streaming-tts-elevenlabs",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5",
|
|
4
4
|
"description": "Streaming TTS via ElevenLabs WebSocket API for AgentOS voice pipeline",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -17,6 +17,9 @@
|
|
|
17
17
|
"SKILL.md",
|
|
18
18
|
"manifest.json"
|
|
19
19
|
],
|
|
20
|
+
"engines": {
|
|
21
|
+
"node": ">=22"
|
|
22
|
+
},
|
|
20
23
|
"peerDependencies": {
|
|
21
24
|
"@framers/agentos": ">=0.10.40"
|
|
22
25
|
},
|
|
@@ -8,8 +8,10 @@
|
|
|
8
8
|
*
|
|
9
9
|
* | Event | Payload | Description |
|
|
10
10
|
* |----------------------|----------------------------|------------------------------------------------------|
|
|
11
|
+
* | `'audio'` | `EncodedAudioChunk` | Core audio event; same MP3 chunk as `audio_chunk` |
|
|
11
12
|
* | `'audio_chunk'` | `EncodedAudioChunk` | MP3 audio buffer from ElevenLabs, ready for playback |
|
|
12
13
|
* | `'utterance_complete'`| `{ text, durationMs }` | ElevenLabs signalled end of audio generation |
|
|
14
|
+
* | `'flush_complete'` | (none) | Remote final received; synthesis complete |
|
|
13
15
|
* | `'cancelled'` | `{ remaining: string }` | Session cancelled; remaining text not rendered |
|
|
14
16
|
* | `'error'` | `Error` | WebSocket or protocol error |
|
|
15
17
|
* | `'close'` | — | Session fully terminated |
|
|
@@ -23,9 +25,14 @@
|
|
|
23
25
|
* 3. Stream text tokens as `{ text: "..." }` JSON messages.
|
|
24
26
|
* 4. To generate audio for accumulated text, include `flush: true` in the text message.
|
|
25
27
|
* 5. To end the stream, send an EOS (End-Of-Stream) message: `{ text: "" }`.
|
|
26
|
-
* 6.
|
|
28
|
+
* 6. JSON audio envelopes carry base64 MP3. Binary audio remains accepted for compatibility.
|
|
27
29
|
* 7. When all audio has been generated the server sends `{ isFinal: true }`.
|
|
28
30
|
*
|
|
31
|
+
* `audio` and `audio_chunk` carry each decoded chunk in wire order. One remote
|
|
32
|
+
* final emits `utterance_complete` and `flush_complete`. Queuing EOS and closing
|
|
33
|
+
* the socket are not completion. Durations remain legacy estimates; this pack
|
|
34
|
+
* does not provide the awaited, bounded output required by the hosted profile.
|
|
35
|
+
*
|
|
29
36
|
* @module streaming-tts-elevenlabs/ElevenLabsTTSSession
|
|
30
37
|
*/
|
|
31
38
|
|
|
@@ -144,6 +151,15 @@ export class ElevenLabsTTSSession extends EventEmitter {
|
|
|
144
151
|
/** `true` after {@link close} or {@link cancel} is called. */
|
|
145
152
|
private closed = false;
|
|
146
153
|
|
|
154
|
+
/** `true` once {@link flush} has queued EOS; later tokens and flushes are ignored. */
|
|
155
|
+
private eosQueued = false;
|
|
156
|
+
|
|
157
|
+
/** `true` once the server's final message has been handled; later messages are ignored. */
|
|
158
|
+
private finalReceived = false;
|
|
159
|
+
|
|
160
|
+
/** All text accepted by {@link pushTokens}, reported by `utterance_complete`. */
|
|
161
|
+
private totalText = '';
|
|
162
|
+
|
|
147
163
|
/**
|
|
148
164
|
* Whether the BOS (beginning-of-stream) message has been sent.
|
|
149
165
|
*
|
|
@@ -196,89 +212,36 @@ export class ElevenLabsTTSSession extends EventEmitter {
|
|
|
196
212
|
// StreamingTTSSession interface
|
|
197
213
|
// -------------------------------------------------------------------------
|
|
198
214
|
|
|
199
|
-
/**
|
|
200
|
-
* Push LLM output token(s) into the stream.
|
|
201
|
-
*
|
|
202
|
-
* Text is accumulated in an internal buffer. When a sentence boundary
|
|
203
|
-
* (`.`, `?`, `!`) is detected the accumulated segment is sent to ElevenLabs
|
|
204
|
-
* with `flush: true` so the server generates audio immediately. Mid-sentence
|
|
205
|
-
* tokens are sent with a trailing space continuation to avoid falling intonation
|
|
206
|
-
* on partial phrases.
|
|
207
|
-
*
|
|
208
|
-
* @param text - One or more LLM token characters to append.
|
|
209
|
-
*/
|
|
215
|
+
/** Send each token once; punctuation requests synthesis without resending prior tokens. */
|
|
210
216
|
pushTokens(text: string): void {
|
|
211
|
-
if (this.closed) return;
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const hasBoundary = SENTENCE_BOUNDARY_RE.test(text);
|
|
217
|
-
|
|
218
|
-
if (hasBoundary) {
|
|
219
|
-
// Send the accumulated text with flush: true so ElevenLabs generates audio.
|
|
220
|
-
const segment = this.pendingText;
|
|
221
|
-
this.pendingText = '';
|
|
222
|
-
this._send({ text: segment, flush: true });
|
|
223
|
-
} else {
|
|
224
|
-
// Send the token chunk without flush; include a space continuation to hint
|
|
225
|
-
// that the phrase is mid-sentence (prevents falling prosody on partial text).
|
|
226
|
-
this._send({ text });
|
|
227
|
-
}
|
|
217
|
+
if (this.closed || this.eosQueued || this.finalReceived || !text) return;
|
|
218
|
+
this.pendingText += text; this.totalText += text;
|
|
219
|
+
const boundary = SENTENCE_BOUNDARY_RE.test(text);
|
|
220
|
+
this._send(boundary ? { text, flush: true } : { text });
|
|
221
|
+
if (boundary) this.pendingText = '';
|
|
228
222
|
}
|
|
229
223
|
|
|
230
|
-
/**
|
|
231
|
-
* Flush any remaining buffered text and signal EOS to ElevenLabs.
|
|
232
|
-
*
|
|
233
|
-
* Call this when the LLM stream has ended. Sends an EOS (end-of-stream)
|
|
234
|
-
* message (`{ text: "" }`) which instructs ElevenLabs to finalise audio
|
|
235
|
-
* generation for all remaining buffered text and close the stream.
|
|
236
|
-
*
|
|
237
|
-
* @returns Promise that resolves when the EOS message is queued for sending.
|
|
238
|
-
*/
|
|
224
|
+
/** Queue EOS once. Listen for flush_complete to observe actual remote synthesis completion. */
|
|
239
225
|
async flush(): Promise<void> {
|
|
240
|
-
if (this.closed) return;
|
|
241
|
-
|
|
242
|
-
// If there is buffered text that has not been sent, send it with flush: true first.
|
|
243
|
-
if (this.pendingText.trim().length > 0) {
|
|
244
|
-
const segment = this.pendingText;
|
|
245
|
-
this.pendingText = '';
|
|
246
|
-
this._send({ text: segment, flush: true });
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// Send EOS — empty string signals end of stream to ElevenLabs.
|
|
226
|
+
if (this.closed || this.eosQueued || this.finalReceived) return;
|
|
227
|
+
this.eosQueued = true;
|
|
250
228
|
this._send({ text: '' });
|
|
251
229
|
}
|
|
252
230
|
|
|
253
|
-
/**
|
|
254
|
-
* Cancel the session immediately.
|
|
255
|
-
*
|
|
256
|
-
* Closes the WebSocket, discards any unsent text, and emits a `'cancelled'`
|
|
257
|
-
* event carrying the remaining un-synthesised text.
|
|
258
|
-
*/
|
|
231
|
+
/** Discard queued messages and prevent an open event from admitting work after cancellation. */
|
|
259
232
|
cancel(): void {
|
|
260
233
|
if (this.closed) return;
|
|
261
234
|
this.closed = true;
|
|
262
|
-
|
|
263
235
|
const remaining = this.pendingText;
|
|
264
|
-
this.pendingText = '';
|
|
265
|
-
|
|
266
|
-
this.ws.close();
|
|
267
|
-
this.emit('cancelled', { remaining });
|
|
236
|
+
this.pendingText = ''; this.sendQueue.length = 0;
|
|
237
|
+
this.ws.close(); this.emit('cancelled', { remaining });
|
|
268
238
|
}
|
|
269
239
|
|
|
270
|
-
/**
|
|
271
|
-
* Close the session cleanly.
|
|
272
|
-
*
|
|
273
|
-
* Sends a WebSocket close frame and emits a `'close'` event.
|
|
274
|
-
* Idempotent — safe to call multiple times.
|
|
275
|
-
*/
|
|
240
|
+
/** Close the socket once; socket closure alone never reports completed synthesis. */
|
|
276
241
|
close(): void {
|
|
277
242
|
if (this.closed) return;
|
|
278
|
-
this.closed = true;
|
|
279
|
-
|
|
280
|
-
this.ws.close();
|
|
281
|
-
this.emit('close');
|
|
243
|
+
this.closed = true; this.sendQueue.length = 0;
|
|
244
|
+
this.ws.close(); this.emit('close');
|
|
282
245
|
}
|
|
283
246
|
|
|
284
247
|
// -------------------------------------------------------------------------
|
|
@@ -291,91 +254,71 @@ export class ElevenLabsTTSSession extends EventEmitter {
|
|
|
291
254
|
*/
|
|
292
255
|
private _wireWebSocket(): void {
|
|
293
256
|
this.ws.on('open', () => {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
similarity_boost: this.similarityBoost,
|
|
300
|
-
style: this.style,
|
|
301
|
-
use_speaker_boost: this.useSpeakerBoost,
|
|
302
|
-
},
|
|
303
|
-
xi_api_key: this.providerConfig.apiKey,
|
|
304
|
-
};
|
|
305
|
-
|
|
306
|
-
// Use the raw WS send to bypass the queue (BOS must be first).
|
|
307
|
-
this.ws.send(JSON.stringify(bosMessage));
|
|
257
|
+
if (this.closed) return;
|
|
258
|
+
this.ws.send(JSON.stringify({ text: ' ', voice_settings: {
|
|
259
|
+
stability: this.stability, similarity_boost: this.similarityBoost,
|
|
260
|
+
style: this.style, use_speaker_boost: this.useSpeakerBoost,
|
|
261
|
+
}, xi_api_key: this.providerConfig.apiKey }));
|
|
308
262
|
this.bosAcknowledged = true;
|
|
309
|
-
|
|
310
|
-
// Drain any messages queued before the socket opened.
|
|
311
263
|
for (const queued of this.sendQueue) {
|
|
264
|
+
if (this.closed) break;
|
|
312
265
|
this.ws.send(JSON.stringify(queued));
|
|
313
266
|
}
|
|
314
267
|
this.sendQueue.length = 0;
|
|
315
268
|
});
|
|
316
|
-
|
|
317
269
|
this.ws.on('message', (data: WebSocket.RawData, isBinary: boolean) => {
|
|
318
|
-
if (this.closed) return;
|
|
319
|
-
|
|
270
|
+
if (this.closed || this.finalReceived) return;
|
|
320
271
|
if (isBinary) {
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
const
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
};
|
|
335
|
-
|
|
336
|
-
this.emit('audio_chunk', chunk);
|
|
337
|
-
} else {
|
|
338
|
-
// Text (JSON) message: parse for protocol signals.
|
|
339
|
-
try {
|
|
340
|
-
const msg = JSON.parse(data.toString()) as Record<string, unknown>;
|
|
341
|
-
|
|
342
|
-
if (msg['isFinal'] === true) {
|
|
343
|
-
// ElevenLabs has finished generating all audio for this stream.
|
|
344
|
-
const durationMs = estimateDurationMs(this.pendingText);
|
|
345
|
-
this.emit('utterance_complete', { text: this.pendingText, durationMs });
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
// Other JSON messages (e.g. alignment data) are silently ignored for now.
|
|
349
|
-
} catch {
|
|
350
|
-
// Malformed JSON from the server — emit a non-fatal error.
|
|
351
|
-
this.emit('error', new Error(`ElevenLabs: unexpected non-JSON text message: ${String(data)}`));
|
|
272
|
+
const audio = Buffer.isBuffer(data) ? data : Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data);
|
|
273
|
+
this._emitAudio(audio); return;
|
|
274
|
+
}
|
|
275
|
+
let msg: Record<string, unknown>;
|
|
276
|
+
try {
|
|
277
|
+
const parsed: unknown = JSON.parse(data.toString());
|
|
278
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('shape');
|
|
279
|
+
msg = parsed as Record<string, unknown>;
|
|
280
|
+
} catch { this.emit('error', new Error('ElevenLabs status envelope invalid')); return; }
|
|
281
|
+
if (msg.error !== undefined) { this.emit('error', new Error('ElevenLabs synthesis failed')); return; }
|
|
282
|
+
if (msg.audio !== undefined && msg.audio !== null && msg.audio !== '') {
|
|
283
|
+
if (typeof msg.audio !== 'string' || msg.audio.length > 8388608 || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(msg.audio)) {
|
|
284
|
+
this.emit('error', new Error('ElevenLabs audio envelope invalid')); return;
|
|
352
285
|
}
|
|
286
|
+
this._emitAudio(Buffer.from(msg.audio, 'base64'));
|
|
353
287
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
this.emit('error', err);
|
|
288
|
+
if (msg.isFinal === true && !this.closed) {
|
|
289
|
+
this.finalReceived = true;
|
|
290
|
+
this.emit('utterance_complete', { text: this.totalText, durationMs: estimateDurationMs(this.totalText) });
|
|
291
|
+
if (!this.closed) this.emit('flush_complete');
|
|
359
292
|
}
|
|
360
293
|
});
|
|
361
|
-
|
|
294
|
+
this.ws.on('error', () => { if (!this.closed) this.emit('error', new Error('ElevenLabs connection failed')); });
|
|
362
295
|
this.ws.on('close', () => {
|
|
363
|
-
if (!this.closed) {
|
|
364
|
-
this.closed = true;
|
|
365
|
-
this.emit('close');
|
|
366
|
-
}
|
|
296
|
+
if (!this.closed) { this.closed = true; this.sendQueue.length = 0; this.emit('close'); }
|
|
367
297
|
});
|
|
368
298
|
}
|
|
369
299
|
|
|
300
|
+
/**
|
|
301
|
+
* Emit one decoded MP3 chunk as `audio`, then as the legacy `audio_chunk`.
|
|
302
|
+
* Nothing is emitted after closure, after the remote final or for an empty buffer.
|
|
303
|
+
*/
|
|
304
|
+
private _emitAudio(audio: Buffer): void {
|
|
305
|
+
if (this.closed || this.finalReceived || audio.length === 0) return;
|
|
306
|
+
const chunk: EncodedAudioChunk = { audio, format: 'mp3', sampleRate: SAMPLE_RATE_HZ, durationMs: estimateDurationMs(this.pendingText), text: '' };
|
|
307
|
+
this.emit('audio', chunk);
|
|
308
|
+
if (!this.closed) this.emit('audio_chunk', chunk);
|
|
309
|
+
}
|
|
310
|
+
|
|
370
311
|
/**
|
|
371
312
|
* Send a JSON message over the WebSocket.
|
|
372
313
|
*
|
|
373
314
|
* If the socket is not yet open, the message is queued and will be sent once
|
|
374
315
|
* the `open` event fires (after the BOS message is dispatched).
|
|
316
|
+
* Nothing is sent or queued once the session is closed or cancelled.
|
|
375
317
|
*
|
|
376
318
|
* @param msg - The message object to serialise and send.
|
|
377
319
|
*/
|
|
378
320
|
private _send(msg: object): void {
|
|
321
|
+
if (this.closed) return;
|
|
379
322
|
if (this.bosAcknowledged && this.ws.readyState === WebSocket.OPEN) {
|
|
380
323
|
this.ws.send(JSON.stringify(msg));
|
|
381
324
|
} else {
|