@micdrop/elevenlabs 1.2.1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,53 +1,136 @@
1
1
  // src/ElevenLabsTTS.ts
2
- import { TTS } from "@micdrop/server";
3
-
4
- // src/DialogueWebSocket.ts
5
- import WebSocket2 from "ws";
2
+ import {
3
+ cachedModels,
4
+ cachedVoices,
5
+ TTS,
6
+ voiceGender,
7
+ voicesForLanguage
8
+ } from "@micdrop/server";
6
9
 
7
10
  // src/ElevenLabsWebSocket.ts
8
- import WebSocket from "ws";
9
- function closeSocket(socket) {
10
- socket.removeAllListeners();
11
- socket.on("error", () => {
12
- });
13
- if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
14
- socket.close(1e3);
11
+ import { ReconnectingSocket } from "@micdrop/server";
12
+ var DEFAULT_MAX_RETRY = 3;
13
+ var ElevenLabsConnection = class {
14
+ constructor(options) {
15
+ this.options = options;
16
+ // Whether the current connection carried audio or lived long enough for a
17
+ // keep-alive, which means the server accepted the key and the voice
18
+ this.proven = false;
19
+ this.failures = 0;
20
+ this.socket = this.createSocket();
15
21
  }
16
- }
22
+ /** Opens the connection, again after giving up, and does nothing otherwise */
23
+ connect() {
24
+ this.socket.connect();
25
+ }
26
+ /**
27
+ * Resolves to false when the connection gives up or is destroyed meanwhile,
28
+ * and right away once it gave up, until the next `connect()`: the stream
29
+ * that was speaking then has nothing left to wait for.
30
+ */
31
+ ready() {
32
+ if (this.socket.status === "idle") return Promise.resolve(false);
33
+ return this.socket.ready();
34
+ }
35
+ send(message) {
36
+ return this.socket.send(JSON.stringify(message));
37
+ }
38
+ /** Drops the connection, with the synthesis running on it, for a new one */
39
+ restart() {
40
+ this.socket.restart();
41
+ }
42
+ destroy() {
43
+ this.socket.destroy();
44
+ }
45
+ /** The current connection works, so its next drop is not a failure */
46
+ prove() {
47
+ this.proven = true;
48
+ this.failures = 0;
49
+ }
50
+ createSocket() {
51
+ const { tts, keepAlive, host } = this.options;
52
+ const socket = new ReconnectingSocket({
53
+ url: this.options.url,
54
+ clientOptions: { headers: { "xi-api-key": tts.apiKey } },
55
+ connectionTimeout: tts.connectionTimeout,
56
+ retryDelay: tts.retryDelay,
57
+ maxRetry: tts.maxRetry,
58
+ handshake: {
59
+ start: (ws) => ws.send(JSON.stringify(this.options.init))
60
+ },
61
+ keepAlive: {
62
+ interval: keepAlive.interval,
63
+ message: () => {
64
+ this.prove();
65
+ return JSON.stringify(keepAlive.message);
66
+ }
67
+ },
68
+ log: (...message) => host.log(...message)
69
+ });
70
+ socket.on("Open", ({ reconnected }) => {
71
+ this.proven = false;
72
+ this.options.onOpen(reconnected);
73
+ });
74
+ socket.on("Message", (data) => {
75
+ try {
76
+ this.options.onMessage(JSON.parse(data.toString()));
77
+ } catch (error) {
78
+ host.log("message" in error ? error.message : error);
79
+ host.log("Event data during error:", data.toString());
80
+ }
81
+ });
82
+ socket.on("Close", () => {
83
+ if (this.proven) return;
84
+ this.failures++;
85
+ if (this.failures > (tts.maxRetry ?? DEFAULT_MAX_RETRY)) this.giveUp();
86
+ });
87
+ socket.on("Failed", () => this.giveUp());
88
+ return socket;
89
+ }
90
+ giveUp() {
91
+ this.options.host.log("Max retries reached, giving up");
92
+ this.failures = 0;
93
+ this.socket.destroy();
94
+ this.socket = this.createSocket();
95
+ this.options.onFailed();
96
+ }
97
+ };
17
98
 
18
99
  // src/types.ts
19
100
  var DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
20
- var DEFAULT_OUTPUT_FORMAT = "pcm_16000";
101
+ var DEFAULT_SAMPLE_RATE = 48e3;
102
+ var DEFAULT_OUTPUT_FORMAT = "pcm_48000";
103
+ function getOutputFormat(options) {
104
+ if (options.sampleRate) return `pcm_${options.sampleRate}`;
105
+ return options.outputFormat ?? `pcm_${DEFAULT_SAMPLE_RATE}`;
106
+ }
107
+ function getSampleRate(options) {
108
+ const match = /^pcm_(\d+)$/.exec(getOutputFormat(options));
109
+ return match ? Number(match[1]) : DEFAULT_SAMPLE_RATE;
110
+ }
21
111
  function isDialogueModel(modelId) {
22
112
  return modelId.startsWith("eleven_v3") || modelId.startsWith("eleven_v4");
23
113
  }
24
114
 
25
115
  // src/DialogueWebSocket.ts
26
116
  var KEEP_ALIVE_INTERVAL = 15e3;
27
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
28
- var DEFAULT_RETRY_DELAY = 1e3;
29
- var DEFAULT_MAX_RETRY = 3;
30
117
  var SENTENCE_END = /(?:[!?]+|(?<!\.)\.(?!\.))\s+/g;
31
118
  var DialogueWebSocket = class {
32
119
  // Whether the connection received any text
33
120
  constructor(options, host) {
34
121
  this.options = options;
35
122
  this.host = host;
36
- this.retryCount = 0;
37
- this.destroyed = false;
123
+ this.state = "idle";
38
124
  // Bumped by every speak() and every cancel(), so a stream that starts late
39
125
  // can tell whether it is still the one that should be heard.
40
126
  this.generation = 0;
41
127
  // Generation of the stream being spoken. A speak() that never carries a
42
128
  // word leaves it in place, so the sentence it sends goes on to its end.
43
129
  this.speaking = 0;
44
- this.isProcessing = false;
45
- this.textEnded = false;
46
- // Whether the text stream has ended
47
130
  this.textSent = "";
48
131
  // Text of the utterance sent to ElevenLabs
49
132
  this.textBuffer = "";
50
- // End of the last chunk, held until its word is complete
133
+ // Text not sent yet, held until its word is complete
51
134
  this.unflushed = false;
52
135
  // Whether text was sent since the last flush
53
136
  this.pendingFlushes = 0;
@@ -55,7 +138,31 @@ var DialogueWebSocket = class {
55
138
  this.newTurn = false;
56
139
  // Whether the next input starts a new utterance
57
140
  this.hasSpoken = false;
58
- this.connect();
141
+ const params = new URLSearchParams();
142
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
143
+ params.append("output_format", getOutputFormat(this.options));
144
+ if (this.options.language) {
145
+ params.append("language_code", this.options.language);
146
+ }
147
+ const stability = this.options.voiceSettings?.stability;
148
+ const init = {
149
+ voices: [this.options.voiceId],
150
+ voice_settings: stability === void 0 ? void 0 : { stability }
151
+ };
152
+ this.connection = new ElevenLabsConnection({
153
+ url: `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
154
+ tts: options,
155
+ init,
156
+ keepAlive: {
157
+ interval: KEEP_ALIVE_INTERVAL,
158
+ message: { keep_alive: true }
159
+ },
160
+ host,
161
+ onOpen: () => this.onOpen(),
162
+ onMessage: (message) => this.onMessage(message),
163
+ onFailed: () => this.onFailed()
164
+ });
165
+ this.connection.connect();
59
166
  }
60
167
  speak(textStream) {
61
168
  const generation = ++this.generation;
@@ -64,171 +171,100 @@ var DialogueWebSocket = class {
64
171
  if (started) return this.speaking === generation;
65
172
  if (this.generation !== generation) return false;
66
173
  started = true;
67
- if (this.isProcessing) this.restart();
174
+ if (this.state !== "idle") this.connection.restart();
175
+ else this.connection.connect();
176
+ this.resetUtterance();
68
177
  this.speaking = generation;
69
- this.isProcessing = true;
70
- this.textEnded = false;
71
- this.textSent = "";
72
- this.textBuffer = "";
73
- this.unflushed = false;
74
- this.pendingFlushes = 0;
178
+ this.state = "streaming";
75
179
  this.newTurn = true;
76
180
  return true;
77
181
  };
78
182
  textStream.on("data", async (chunk) => {
79
183
  if (!start()) return;
80
- const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
81
- await this.initPromise;
184
+ this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
185
+ if (!await this.connection.ready()) return;
82
186
  if (this.speaking !== generation) return;
83
- const spaceIndex = text.lastIndexOf(" ");
84
- if (spaceIndex === -1) {
85
- this.textBuffer += text;
86
- } else {
87
- this.sendText(this.textBuffer + text.slice(0, spaceIndex + 1));
88
- this.textBuffer = text.slice(spaceIndex + 1);
89
- }
187
+ const spaceIndex = this.textBuffer.lastIndexOf(" ");
188
+ if (spaceIndex === -1) return;
189
+ this.sendText(this.textBuffer.slice(0, spaceIndex + 1));
190
+ this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
90
191
  });
91
192
  textStream.on("error", (error) => {
92
193
  if (!started || this.speaking !== generation) return;
93
194
  this.log("Error in text stream, ending audio stream", error);
94
- this.isProcessing = false;
195
+ this.speaking = 0;
196
+ this.textBuffer = "";
197
+ this.endText();
95
198
  });
96
199
  textStream.on("end", async () => {
97
200
  if (!started) return;
98
- await this.initPromise;
201
+ if (!await this.connection.ready()) return;
99
202
  if (this.speaking !== generation) return;
100
203
  if (this.textBuffer.trim()) {
101
204
  this.sendText(this.textBuffer + " ");
102
- this.textBuffer = "";
103
205
  }
104
- this.textEnded = true;
105
- this.flush();
106
- this.checkEnded();
206
+ this.textBuffer = "";
207
+ this.endText();
107
208
  });
108
209
  }
109
210
  cancel() {
110
211
  this.generation++;
111
212
  this.speaking = 0;
112
- if (!this.isProcessing) return;
213
+ if (this.state === "idle") return;
113
214
  this.log("Cancel");
114
- this.isProcessing = false;
115
- this.textSent = "";
116
- this.textBuffer = "";
117
- this.restart();
215
+ this.resetUtterance();
216
+ this.connection.restart();
118
217
  }
119
218
  destroy() {
120
- this.destroyed = true;
121
- if (this.reconnectTimeout) {
122
- clearTimeout(this.reconnectTimeout);
123
- this.reconnectTimeout = void 0;
219
+ this.generation++;
220
+ this.speaking = 0;
221
+ this.resetUtterance();
222
+ this.connection.destroy();
223
+ }
224
+ onOpen() {
225
+ this.hasSpoken = false;
226
+ if (this.state === "idle") return;
227
+ const text = this.textSent;
228
+ this.textSent = "";
229
+ this.unflushed = false;
230
+ this.pendingFlushes = 0;
231
+ if (text.trim()) {
232
+ this.log("Sending text again");
233
+ this.sendText(text);
124
234
  }
125
- this.dropSocket();
126
- this.isProcessing = false;
235
+ if (this.state === "draining") this.flush();
236
+ this.checkEnded();
127
237
  }
128
- connect() {
129
- this.initPromise = this.initWS().catch((error) => {
130
- console.error("[ElevenLabsTTS] Connection error:", error);
131
- this.reconnect();
132
- });
238
+ /** Every retry failed: hands the unspoken text over and drops the utterance */
239
+ onFailed() {
240
+ const unspoken = this.state === "idle" ? "" : this.textSent + this.textBuffer;
241
+ this.speaking = 0;
242
+ this.resetUtterance();
243
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
133
244
  }
134
- /** Drops the connection, with the synthesis running on it, for a new one */
135
- restart() {
136
- if (this.reconnectTimeout) {
137
- clearTimeout(this.reconnectTimeout);
138
- this.reconnectTimeout = void 0;
139
- }
140
- this.dropSocket();
141
- this.connect();
142
- }
143
- /** Closes the connection without reconnecting */
144
- dropSocket() {
145
- if (this.connectionTimeout) {
146
- clearTimeout(this.connectionTimeout);
147
- this.connectionTimeout = void 0;
148
- }
149
- if (this.keepAliveInterval) {
150
- clearInterval(this.keepAliveInterval);
151
- this.keepAliveInterval = void 0;
152
- }
153
- if (this.socket) closeSocket(this.socket);
154
- this.socket = void 0;
155
- }
156
- initWS() {
157
- return new Promise((resolve, reject) => {
158
- const params = new URLSearchParams();
159
- params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
160
- params.append(
161
- "output_format",
162
- this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
163
- );
164
- if (this.options.language) {
165
- params.append("language_code", this.options.language);
166
- }
167
- const socket = new WebSocket2(
168
- `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
169
- {
170
- headers: {
171
- "xi-api-key": this.options.apiKey
172
- }
173
- }
174
- );
175
- this.socket = socket;
176
- let opened = false;
177
- const timeout = setTimeout(() => {
178
- this.log("Connection timeout");
179
- closeSocket(socket);
180
- this.socket = void 0;
181
- reject(new Error("WebSocket connection timeout"));
182
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
183
- this.connectionTimeout = timeout;
184
- socket.addEventListener("open", () => {
185
- clearTimeout(timeout);
186
- opened = true;
187
- this.log("Connection opened");
188
- const stability = this.options.voiceSettings?.stability;
189
- this.send({
190
- voices: [this.options.voiceId],
191
- voice_settings: stability === void 0 ? void 0 : { stability }
192
- });
193
- this.hasSpoken = false;
194
- this.keepAliveInterval = setInterval(() => {
195
- this.send({ keep_alive: true });
196
- this.retryCount = 0;
197
- }, KEEP_ALIVE_INTERVAL);
198
- resolve();
199
- });
200
- socket.addEventListener("error", (error) => {
201
- clearTimeout(timeout);
202
- this.log("WebSocket error:", error);
203
- reject(new Error("WebSocket connection error"));
204
- });
205
- socket.addEventListener("message", (event) => {
206
- try {
207
- this.onMessage(JSON.parse(event.data.toString()));
208
- } catch (error) {
209
- this.log("message" in error ? error.message : error);
210
- this.log("Event data during error:", event.data);
211
- }
212
- });
213
- socket.addEventListener("close", ({ code, reason }) => {
214
- clearTimeout(timeout);
215
- if (this.keepAliveInterval) {
216
- clearInterval(this.keepAliveInterval);
217
- this.keepAliveInterval = void 0;
218
- }
219
- this.socket?.removeAllListeners();
220
- this.socket = void 0;
221
- this.log("Connection closed", { code, reason });
222
- if (opened) this.reconnect();
223
- });
224
- });
245
+ resetUtterance() {
246
+ this.state = "idle";
247
+ this.textSent = "";
248
+ this.textBuffer = "";
249
+ this.unflushed = false;
250
+ this.pendingFlushes = 0;
251
+ }
252
+ /** The text stream is over: flushes what the server holds */
253
+ endText() {
254
+ this.state = "draining";
255
+ this.flush();
256
+ this.checkEnded();
225
257
  }
226
258
  onMessage(message) {
227
259
  if ("audio" in message && message.audio) {
228
260
  const chunk = Buffer.from(message.audio, "base64");
229
- this.log(`Received audio chunk (${chunk.length} bytes)`);
230
- this.retryCount = 0;
231
- this.host.audio(chunk);
261
+ this.connection.prove();
262
+ if (this.state === "idle") {
263
+ this.log("Ignore audio chunk, nothing is being spoken");
264
+ } else {
265
+ this.log(`Received audio chunk (${chunk.length} bytes)`);
266
+ this.host.audio(chunk);
267
+ }
232
268
  }
233
269
  if ("is_final_audio_for_turn" in message && message.is_final_audio_for_turn) {
234
270
  this.pendingFlushes = Math.max(0, this.pendingFlushes - 1);
@@ -284,45 +320,15 @@ var DialogueWebSocket = class {
284
320
  }
285
321
  /** Ends the utterance once its text is over and all its audio came */
286
322
  checkEnded() {
287
- if (!this.isProcessing || !this.textEnded) return;
323
+ if (this.state !== "draining") return;
288
324
  if (this.unflushed || this.pendingFlushes > 0) return;
289
325
  this.log("Audio ended");
290
- this.isProcessing = false;
291
- this.textSent = "";
326
+ this.resetUtterance();
292
327
  }
328
+ // Lost when the connection just dropped, which is fine: the reconnection
329
+ // sends the utterance again from textSent
293
330
  send(message) {
294
- if (this.socket?.readyState !== WebSocket2.OPEN) return;
295
- this.socket.send(JSON.stringify(message));
296
- }
297
- reconnect() {
298
- if (this.destroyed) return;
299
- this.retryCount++;
300
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
301
- this.log("Max retries reached, giving up");
302
- const unspoken = this.isProcessing ? this.textSent + this.textBuffer : "";
303
- this.host.failed(unspoken.trim() ? [unspoken] : []);
304
- return;
305
- }
306
- this.initPromise = new Promise((resolve) => {
307
- this.log("Reconnecting...");
308
- this.reconnectTimeout = setTimeout(() => {
309
- this.reconnectTimeout = void 0;
310
- this.initWS().then(() => {
311
- if (this.isProcessing && this.textSent.trim()) {
312
- this.log("Sending text again");
313
- const text = this.textSent;
314
- this.textSent = "";
315
- this.unflushed = false;
316
- this.pendingFlushes = 0;
317
- this.sendText(text);
318
- if (this.textEnded) this.flush();
319
- }
320
- }).then(resolve).catch((error) => {
321
- this.log("Reconnection error:", error);
322
- this.reconnect();
323
- });
324
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
325
- });
331
+ this.connection.send(message);
326
332
  }
327
333
  log(...message) {
328
334
  this.host.log(...message);
@@ -330,197 +336,150 @@ var DialogueWebSocket = class {
330
336
  };
331
337
 
332
338
  // src/SpeechWebSocket.ts
333
- import WebSocket3 from "ws";
334
339
  var WS_INACTIVITY_TIMEOUT = 180;
335
- var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
336
- var DEFAULT_RETRY_DELAY2 = 1e3;
337
- var DEFAULT_MAX_RETRY2 = 3;
338
340
  var SpeechWebSocket = class {
341
+ // Index in textSent where the audio received ends
339
342
  constructor(options, host) {
340
343
  this.options = options;
341
344
  this.host = host;
342
- this.isProcessing = false;
343
- this.textEnded = false;
344
- // Whether the text stream has ended
345
- this.textSent = "";
346
- // Text sent to ElevenLabs
347
- this.textBuffer = "";
348
- // Buffer of text to send to ElevenLabs
349
- this.receivedAudioText = "";
350
- this.retryCount = 0;
351
- this.destroyed = false;
352
- this.canceled = false;
345
+ this.state = "idle";
353
346
  // Bumped by every speak() and every cancel(), so a stream that starts late
354
347
  // can tell whether it is still the one that should be heard.
355
348
  this.generation = 0;
356
- this.initPromise = this.initWS().catch((error) => {
357
- console.error("[ElevenLabsTTS] Connection error:", error);
358
- this.reconnect();
349
+ // Generation of the stream being spoken. A speak() that never carries a
350
+ // word leaves it in place, so the sentence it sends goes on to its end.
351
+ this.speaking = 0;
352
+ this.textSent = "";
353
+ // Text of the utterance sent to ElevenLabs
354
+ this.textBuffer = "";
355
+ // Text not sent yet, held until its word is complete
356
+ this.heardUntil = 0;
357
+ const params = new URLSearchParams();
358
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
359
+ params.append("output_format", getOutputFormat(this.options));
360
+ params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
361
+ const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
362
+ if (voiceSettings) {
363
+ params.append("voice_settings", JSON.stringify(voiceSettings));
364
+ }
365
+ if (this.options.language) {
366
+ params.append("language_code", this.options.language);
367
+ }
368
+ this.connection = new ElevenLabsConnection({
369
+ url: `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`,
370
+ tts: options,
371
+ // Initialization message with voice settings
372
+ init: { text: " ", voice_settings: voiceSettings },
373
+ keepAlive: {
374
+ interval: (WS_INACTIVITY_TIMEOUT - 1) * 1e3,
375
+ message: { text: " " }
376
+ },
377
+ host,
378
+ onOpen: () => this.resume(),
379
+ onMessage: (message) => this.onMessage(message),
380
+ onFailed: () => this.onFailed()
359
381
  });
382
+ this.connection.connect();
360
383
  }
361
384
  speak(textStream) {
362
385
  const generation = ++this.generation;
363
386
  let started = false;
364
387
  const start = () => {
365
- if (started) return true;
388
+ if (started) return this.speaking === generation;
366
389
  if (this.generation !== generation) return false;
367
390
  started = true;
368
- this.canceled = false;
369
- this.isProcessing = true;
370
- this.textEnded = false;
371
- this.textBuffer = "";
372
- this.textSent = "";
373
- this.receivedAudioText = "";
391
+ if (this.state !== "idle") this.connection.restart();
392
+ else this.connection.connect();
393
+ this.resetUtterance();
394
+ this.speaking = generation;
395
+ this.state = "speaking";
374
396
  return true;
375
397
  };
376
398
  textStream.on("data", async (chunk) => {
377
399
  if (!start()) return;
378
- if (this.canceled) return;
379
- const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
380
- this.textSent += text;
381
- await this.initPromise;
382
- const spaceIndex = text.lastIndexOf(" ");
383
- if (spaceIndex === -1) {
384
- this.textBuffer += text;
385
- } else {
386
- this.sendTranscript(this.textBuffer + text.slice(0, spaceIndex + 1));
387
- this.textBuffer = text.slice(spaceIndex + 1);
388
- }
400
+ this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
401
+ if (!await this.connection.ready()) return;
402
+ if (this.speaking !== generation) return;
403
+ const spaceIndex = this.textBuffer.lastIndexOf(" ");
404
+ if (spaceIndex === -1) return;
405
+ this.sendTranscript(this.textBuffer.slice(0, spaceIndex + 1));
406
+ this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
389
407
  });
390
408
  textStream.on("error", (error) => {
391
- if (!started) return;
409
+ if (!started || this.speaking !== generation) return;
392
410
  this.log("Error in text stream, ending audio stream", error);
393
- this.isProcessing = false;
411
+ this.speaking = 0;
412
+ this.textBuffer = "";
413
+ this.flush();
394
414
  });
395
415
  textStream.on("end", async () => {
396
- if (!started || this.canceled) return;
397
- await this.initPromise;
416
+ if (!started) return;
417
+ if (!await this.connection.ready()) return;
418
+ if (this.speaking !== generation) return;
398
419
  if (this.textBuffer.trim()) {
399
420
  this.sendTranscript(this.textBuffer + " ");
400
- this.textBuffer = "";
401
421
  }
402
- this.textEnded = true;
403
- this.socket?.send(JSON.stringify({ text: " ", flush: true }));
404
- this.log("Flushed text");
422
+ this.textBuffer = "";
423
+ this.flush();
405
424
  });
406
425
  }
407
426
  cancel() {
408
427
  this.generation++;
409
- if (!this.isProcessing) return;
428
+ this.speaking = 0;
429
+ if (this.state === "idle") return;
410
430
  this.log("Cancel");
411
- this.canceled = true;
412
- this.textSent = "";
413
- this.receivedAudioText = "";
414
- this.isProcessing = false;
415
- if (this.socket?.readyState === WebSocket3.OPEN) {
416
- this.socket.send(JSON.stringify({ text: " ", flush: true }));
417
- }
431
+ this.resetUtterance();
432
+ this.connection.restart();
418
433
  }
419
434
  destroy() {
420
- this.destroyed = true;
421
- if (this.reconnectTimeout) {
422
- clearTimeout(this.reconnectTimeout);
423
- this.reconnectTimeout = void 0;
424
- }
425
- if (this.connectionTimeout) {
426
- clearTimeout(this.connectionTimeout);
427
- this.connectionTimeout = void 0;
428
- }
429
- if (this.keepAliveInterval) {
430
- clearInterval(this.keepAliveInterval);
431
- this.keepAliveInterval = void 0;
435
+ this.generation++;
436
+ this.speaking = 0;
437
+ this.resetUtterance();
438
+ this.connection.destroy();
439
+ }
440
+ /** Flushes buffered text and marks the end of the utterance */
441
+ flush() {
442
+ this.state = "flushed";
443
+ this.connection.send({ text: " ", flush: true });
444
+ this.log("Flushed text");
445
+ this.checkEnded();
446
+ }
447
+ /**
448
+ * Sends the text of the utterance in flight again on a new connection. The
449
+ * server lost what it was generating, so it goes on from the audio received.
450
+ */
451
+ resume() {
452
+ if (this.state === "idle") return;
453
+ const wordStart = this.textSent.lastIndexOf(" ", this.heardUntil - 1) + 1;
454
+ const unheard = this.textSent.slice(wordStart);
455
+ this.textSent = "";
456
+ this.heardUntil = 0;
457
+ if (unheard.trim()) {
458
+ this.log("Sending text chunks again");
459
+ this.sendTranscript(unheard);
432
460
  }
433
- if (this.socket) closeSocket(this.socket);
434
- this.socket = void 0;
435
- this.isProcessing = false;
436
- }
437
- initWS() {
438
- return new Promise((resolve, reject) => {
439
- const params = new URLSearchParams();
440
- params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
441
- params.append(
442
- "output_format",
443
- this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
444
- );
445
- params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
446
- const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
447
- if (voiceSettings) {
448
- params.append("voice_settings", JSON.stringify(voiceSettings));
449
- }
450
- if (this.options.language) {
451
- params.append("language_code", this.options.language);
452
- }
453
- const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`;
454
- const socket = new WebSocket3(uri, {
455
- headers: {
456
- "xi-api-key": this.options.apiKey
457
- }
458
- });
459
- this.socket = socket;
460
- const timeout = setTimeout(() => {
461
- this.log("Connection timeout");
462
- closeSocket(socket);
463
- this.socket = void 0;
464
- reject(new Error("WebSocket connection timeout"));
465
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
466
- this.connectionTimeout = timeout;
467
- socket.addEventListener("open", () => {
468
- clearTimeout(timeout);
469
- this.log("Connection opened");
470
- this.socket?.send(
471
- JSON.stringify({
472
- text: " ",
473
- voice_settings: voiceSettings
474
- })
475
- );
476
- this.keepAliveInterval = setInterval(
477
- () => {
478
- if (this.socket?.readyState === WebSocket3.OPEN) {
479
- this.log("Sending keep-alive message");
480
- this.socket.send(JSON.stringify({ text: " " }));
481
- this.retryCount = 0;
482
- }
483
- },
484
- (WS_INACTIVITY_TIMEOUT - 1) * 1e3
485
- );
486
- resolve();
487
- });
488
- socket.addEventListener("error", (error) => {
489
- clearTimeout(timeout);
490
- this.log("WebSocket error:", error);
491
- reject(new Error("WebSocket connection error"));
492
- });
493
- socket.addEventListener("message", (event) => {
494
- if (this.canceled) return;
495
- try {
496
- this.onMessage(JSON.parse(event.data.toString()));
497
- } catch (error) {
498
- this.log("message" in error ? error.message : error);
499
- this.log("Event data during error:", event.data);
500
- }
501
- });
502
- socket.addEventListener("close", ({ code, reason }) => {
503
- clearTimeout(timeout);
504
- this.socket?.removeAllListeners();
505
- this.socket = void 0;
506
- this.isProcessing = false;
507
- if (this.keepAliveInterval) {
508
- clearInterval(this.keepAliveInterval);
509
- this.keepAliveInterval = void 0;
510
- }
511
- this.log("Connection closed", { code, reason });
512
- this.reconnect();
513
- });
514
- });
461
+ if (this.state === "flushed") this.flush();
462
+ }
463
+ /** Every retry failed: hands the unspoken text over and drops the utterance */
464
+ onFailed() {
465
+ const unspoken = this.state === "idle" ? "" : this.textSent.slice(this.heardUntil) + this.textBuffer;
466
+ this.speaking = 0;
467
+ this.resetUtterance();
468
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
469
+ }
470
+ resetUtterance() {
471
+ this.state = "idle";
472
+ this.textSent = "";
473
+ this.textBuffer = "";
474
+ this.heardUntil = 0;
515
475
  }
516
476
  onMessage(message) {
517
477
  if ("audio" in message && message.audio) {
518
478
  this.processAudioMessage(message);
519
479
  }
520
- if ("isFinal" in message && message.isFinal) {
480
+ if ("isFinal" in message && message.isFinal && this.state !== "idle") {
521
481
  this.log("Audio ended");
522
- this.isProcessing = false;
523
- this.textSent = "";
482
+ this.resetUtterance();
524
483
  }
525
484
  if ("error" in message) {
526
485
  console.error("[ElevenLabsTTS] Error:", message.error, message.message);
@@ -528,58 +487,59 @@ var SpeechWebSocket = class {
528
487
  }
529
488
  processAudioMessage(message) {
530
489
  const chunk = Buffer.from(message.audio, "base64");
531
- this.retryCount = 0;
490
+ this.connection.prove();
491
+ if (this.state === "idle") {
492
+ this.log("Ignore audio chunk, nothing is being spoken");
493
+ return;
494
+ }
532
495
  if (message.alignment) {
533
496
  const chunkText = message.alignment.chars.join("");
534
- const receivedAudioText = this.receivedAudioText + chunkText;
535
- if (this.textSent.trim().indexOf(receivedAudioText.trim()) === -1) {
536
- this.log(`Ignore audio chunk ("${chunkText}")`);
537
- return;
497
+ const end = findHeardEnd(this.textSent, this.heardUntil, chunkText);
498
+ if (end === -1) {
499
+ this.log(`Audio chunk off the text sent ("${chunkText}")`);
500
+ } else {
501
+ this.log(`Received audio chunk ("${chunkText}")`);
502
+ this.heardUntil = end;
538
503
  }
539
- this.log(`Received audio chunk ("${chunkText}")`);
540
- this.receivedAudioText = receivedAudioText;
541
- this.fixIsFinal(message);
542
504
  }
543
505
  this.host.audio(chunk);
506
+ this.checkEnded();
544
507
  }
545
- fixIsFinal(message) {
546
- if (message.isFinal || !this.textEnded) return;
547
- if (this.receivedAudioText.trim() === this.textSent.trim()) {
548
- message.isFinal = true;
549
- }
508
+ checkEnded() {
509
+ if (this.state !== "flushed") return;
510
+ if (this.textSent.slice(this.heardUntil).trim()) return;
511
+ this.log("Audio ended");
512
+ this.resetUtterance();
550
513
  }
551
514
  sendTranscript(text) {
552
- this.socket?.send(JSON.stringify({ text, try_trigger_generation: true }));
515
+ this.textSent += text;
516
+ if (!this.connection.send({ text, try_trigger_generation: true })) return;
553
517
  this.log(`Sent transcript: "${text}"`);
554
518
  }
555
- reconnect() {
556
- if (this.destroyed) return;
557
- this.retryCount++;
558
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
559
- this.log("Max retries reached, giving up");
560
- this.host.failed(this.textSent.trim() ? [this.textSent] : []);
561
- return;
562
- }
563
- this.initPromise = new Promise((resolve) => {
564
- this.log("Reconnecting...");
565
- this.reconnectTimeout = setTimeout(() => {
566
- this.reconnectTimeout = void 0;
567
- this.initWS().then(() => {
568
- if (this.textSent.length > 0) {
569
- this.log("Sending text chunks again");
570
- this.sendTranscript(this.textSent);
571
- }
572
- }).then(resolve).catch((error) => {
573
- this.log("Reconnection error:", error);
574
- this.reconnect();
575
- });
576
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
577
- });
578
- }
579
519
  log(...message) {
580
520
  this.host.log(...message);
581
521
  }
582
522
  };
523
+ function findHeardEnd(text, from, heard) {
524
+ for (let start = from; start < text.length; start++) {
525
+ const end = endOfPrefix(text, start, heard);
526
+ if (end !== -1) return end;
527
+ }
528
+ return -1;
529
+ }
530
+ function endOfPrefix(text, from, heard) {
531
+ let index = from;
532
+ for (let i = 0; i < heard.length; i++) {
533
+ if (isSpace(heard[i])) continue;
534
+ while (index < text.length && isSpace(text[index])) index++;
535
+ if (text[index] !== heard[i]) return -1;
536
+ index++;
537
+ }
538
+ return index;
539
+ }
540
+ function isSpace(char) {
541
+ return char === " " || char === " " || char === "\n" || char === "\r";
542
+ }
583
543
  function toWireVoiceSettings(settings) {
584
544
  if (!settings) return void 0;
585
545
  const { similarityBoost, useSpeakerBoost, ...rest } = settings;
@@ -591,9 +551,56 @@ function toWireVoiceSettings(settings) {
591
551
  }
592
552
 
593
553
  // src/ElevenLabsTTS.ts
554
+ var CUSTOM_CATEGORIES = ["cloned", "generated", "professional"];
594
555
  var ElevenLabsTTS = class extends TTS {
556
+ /**
557
+ * Lists the models that speak, with the languages each one reads.
558
+ *
559
+ * The list is fetched once and shared by every call of the process for a
560
+ * day. `refresh` fetches it again.
561
+ */
562
+ static listModels(options) {
563
+ return cachedModels(
564
+ ["elevenlabs"],
565
+ options.apiKey,
566
+ async () => {
567
+ const response = await fetch("https://api.elevenlabs.io/v1/models", {
568
+ headers: { "xi-api-key": options.apiKey }
569
+ });
570
+ if (!response.ok) {
571
+ throw new Error(`ElevenLabs answered ${response.status}`);
572
+ }
573
+ const models = await response.json();
574
+ return models.filter((model) => model.can_do_text_to_speech).map((model) => ({
575
+ id: model.model_id,
576
+ name: model.name,
577
+ description: model.description,
578
+ languages: model.languages?.map(({ language_id }) => language_id)
579
+ }));
580
+ },
581
+ options.refresh
582
+ );
583
+ }
584
+ /**
585
+ * Lists the voices the account can speak with: the premade ones, and the
586
+ * ones cloned, designed or added from the library.
587
+ *
588
+ * `language`, such as `fr` or `fr-FR`, keeps the voices that speak it.
589
+ *
590
+ * The list is fetched once and shared by every call of the process for a
591
+ * day. `refresh` fetches it again, for a voice added a moment ago.
592
+ */
593
+ static listVoices(options) {
594
+ return cachedVoices(
595
+ ["elevenlabs"],
596
+ options.apiKey,
597
+ () => fetchVoices(options.apiKey),
598
+ options.refresh
599
+ ).then((voices) => voicesForLanguage(voices, options.language));
600
+ }
595
601
  constructor(options) {
596
602
  super();
603
+ this.rate = getSampleRate(options);
597
604
  const host = {
598
605
  audio: (chunk) => this.emit("Audio", chunk),
599
606
  failed: (texts) => this.emit("Failed", texts),
@@ -601,6 +608,9 @@ var ElevenLabsTTS = class extends TTS {
601
608
  };
602
609
  this.webSocket = isDialogueModel(options.modelId ?? DEFAULT_MODEL_ID) ? new DialogueWebSocket(options, host) : new SpeechWebSocket(options, host);
603
610
  }
611
+ get sampleRate() {
612
+ return this.rate;
613
+ }
604
614
  speak(textStream) {
605
615
  this.webSocket.speak(textStream);
606
616
  }
@@ -612,10 +622,44 @@ var ElevenLabsTTS = class extends TTS {
612
622
  super.destroy();
613
623
  }
614
624
  };
625
+ async function fetchVoices(apiKey) {
626
+ const voices = [];
627
+ let pageToken;
628
+ do {
629
+ const params = new URLSearchParams({ page_size: "100" });
630
+ if (pageToken) params.set("next_page_token", pageToken);
631
+ const response = await fetch(
632
+ `https://api.elevenlabs.io/v2/voices?${params}`,
633
+ { headers: { "xi-api-key": apiKey } }
634
+ );
635
+ if (!response.ok) throw new Error(`ElevenLabs answered ${response.status}`);
636
+ const page = await response.json();
637
+ voices.push(...page.voices.map(toMicdropVoice));
638
+ pageToken = page.has_more ? page.next_page_token ?? void 0 : void 0;
639
+ } while (pageToken);
640
+ return voices;
641
+ }
642
+ function toMicdropVoice(voice) {
643
+ const verified = (voice.verified_languages ?? []).map(({ locale, language }) => locale || language).filter((code) => !!code);
644
+ const label = voice.labels?.language;
645
+ const languages = verified.length ? verified : label ? [label] : void 0;
646
+ return {
647
+ id: voice.voice_id,
648
+ name: voice.name,
649
+ languages: languages && [...new Set(languages)],
650
+ gender: voiceGender(voice.labels?.gender),
651
+ description: voice.description || void 0,
652
+ previewUrl: voice.preview_url || void 0,
653
+ custom: CUSTOM_CATEGORIES.includes(voice.category ?? "")
654
+ };
655
+ }
615
656
  export {
616
657
  DEFAULT_MODEL_ID,
617
658
  DEFAULT_OUTPUT_FORMAT,
659
+ DEFAULT_SAMPLE_RATE,
618
660
  ElevenLabsTTS,
661
+ getOutputFormat,
662
+ getSampleRate,
619
663
  isDialogueModel
620
664
  };
621
665
  //# sourceMappingURL=index.mjs.map