@micdrop/elevenlabs 1.1.3 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,20 +1,344 @@
1
1
  // src/ElevenLabsTTS.ts
2
2
  import { TTS } from "@micdrop/server";
3
+
4
+ // src/DialogueWebSocket.ts
5
+ import WebSocket2 from "ws";
6
+
7
+ // src/ElevenLabsWebSocket.ts
3
8
  import WebSocket from "ws";
9
+ function closeSocket(socket) {
10
+ socket.removeAllListeners();
11
+ socket.on("error", () => {
12
+ });
13
+ if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
14
+ socket.close(1e3);
15
+ }
16
+ }
4
17
 
5
18
  // src/types.ts
6
19
  var DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
7
20
  var DEFAULT_OUTPUT_FORMAT = "pcm_16000";
21
+ function isDialogueModel(modelId) {
22
+ return modelId.startsWith("eleven_v3") || modelId.startsWith("eleven_v4");
23
+ }
8
24
 
9
- // src/ElevenLabsTTS.ts
10
- var WS_INACTIVITY_TIMEOUT = 180;
25
+ // src/DialogueWebSocket.ts
26
+ var KEEP_ALIVE_INTERVAL = 15e3;
11
27
  var DEFAULT_CONNECTION_TIMEOUT = 5e3;
12
28
  var DEFAULT_RETRY_DELAY = 1e3;
13
29
  var DEFAULT_MAX_RETRY = 3;
14
- var ElevenLabsTTS = class extends TTS {
15
- constructor(options) {
16
- super();
30
+ var SENTENCE_END = /(?:[!?]+|(?<!\.)\.(?!\.))\s+/g;
31
+ var DialogueWebSocket = class {
32
+ // Whether the connection received any text
33
+ constructor(options, host) {
34
+ this.options = options;
35
+ this.host = host;
36
+ this.retryCount = 0;
37
+ this.destroyed = false;
38
+ // Bumped by every speak() and every cancel(), so a stream that starts late
39
+ // can tell whether it is still the one that should be heard.
40
+ this.generation = 0;
41
+ // Generation of the stream being spoken. A speak() that never carries a
42
+ // word leaves it in place, so the sentence it sends goes on to its end.
43
+ this.speaking = 0;
44
+ this.isProcessing = false;
45
+ this.textEnded = false;
46
+ // Whether the text stream has ended
47
+ this.textSent = "";
48
+ // Text of the utterance sent to ElevenLabs
49
+ this.textBuffer = "";
50
+ // End of the last chunk, held until its word is complete
51
+ this.unflushed = false;
52
+ // Whether text was sent since the last flush
53
+ this.pendingFlushes = 0;
54
+ // Flushes whose audio is not complete yet
55
+ this.newTurn = false;
56
+ // Whether the next input starts a new utterance
57
+ this.hasSpoken = false;
58
+ this.connect();
59
+ }
60
+ speak(textStream) {
61
+ const generation = ++this.generation;
62
+ let started = false;
63
+ const start = () => {
64
+ if (started) return this.speaking === generation;
65
+ if (this.generation !== generation) return false;
66
+ started = true;
67
+ if (this.isProcessing) this.restart();
68
+ this.speaking = generation;
69
+ this.isProcessing = true;
70
+ this.textEnded = false;
71
+ this.textSent = "";
72
+ this.textBuffer = "";
73
+ this.unflushed = false;
74
+ this.pendingFlushes = 0;
75
+ this.newTurn = true;
76
+ return true;
77
+ };
78
+ textStream.on("data", async (chunk) => {
79
+ if (!start()) return;
80
+ const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
81
+ await this.initPromise;
82
+ if (this.speaking !== generation) return;
83
+ const spaceIndex = text.lastIndexOf(" ");
84
+ if (spaceIndex === -1) {
85
+ this.textBuffer += text;
86
+ } else {
87
+ this.sendText(this.textBuffer + text.slice(0, spaceIndex + 1));
88
+ this.textBuffer = text.slice(spaceIndex + 1);
89
+ }
90
+ });
91
+ textStream.on("error", (error) => {
92
+ if (!started || this.speaking !== generation) return;
93
+ this.log("Error in text stream, ending audio stream", error);
94
+ this.isProcessing = false;
95
+ });
96
+ textStream.on("end", async () => {
97
+ if (!started) return;
98
+ await this.initPromise;
99
+ if (this.speaking !== generation) return;
100
+ if (this.textBuffer.trim()) {
101
+ this.sendText(this.textBuffer + " ");
102
+ this.textBuffer = "";
103
+ }
104
+ this.textEnded = true;
105
+ this.flush();
106
+ this.checkEnded();
107
+ });
108
+ }
109
+ cancel() {
110
+ this.generation++;
111
+ this.speaking = 0;
112
+ if (!this.isProcessing) return;
113
+ this.log("Cancel");
114
+ this.isProcessing = false;
115
+ this.textSent = "";
116
+ this.textBuffer = "";
117
+ this.restart();
118
+ }
119
+ destroy() {
120
+ this.destroyed = true;
121
+ if (this.reconnectTimeout) {
122
+ clearTimeout(this.reconnectTimeout);
123
+ this.reconnectTimeout = void 0;
124
+ }
125
+ this.dropSocket();
126
+ this.isProcessing = false;
127
+ }
128
+ connect() {
129
+ this.initPromise = this.initWS().catch((error) => {
130
+ console.error("[ElevenLabsTTS] Connection error:", error);
131
+ this.reconnect();
132
+ });
133
+ }
134
+ /** Drops the connection, with the synthesis running on it, for a new one */
135
+ restart() {
136
+ if (this.reconnectTimeout) {
137
+ clearTimeout(this.reconnectTimeout);
138
+ this.reconnectTimeout = void 0;
139
+ }
140
+ this.dropSocket();
141
+ this.connect();
142
+ }
143
+ /** Closes the connection without reconnecting */
144
+ dropSocket() {
145
+ if (this.connectionTimeout) {
146
+ clearTimeout(this.connectionTimeout);
147
+ this.connectionTimeout = void 0;
148
+ }
149
+ if (this.keepAliveInterval) {
150
+ clearInterval(this.keepAliveInterval);
151
+ this.keepAliveInterval = void 0;
152
+ }
153
+ if (this.socket) closeSocket(this.socket);
154
+ this.socket = void 0;
155
+ }
156
+ initWS() {
157
+ return new Promise((resolve, reject) => {
158
+ const params = new URLSearchParams();
159
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
160
+ params.append(
161
+ "output_format",
162
+ this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
163
+ );
164
+ if (this.options.language) {
165
+ params.append("language_code", this.options.language);
166
+ }
167
+ const socket = new WebSocket2(
168
+ `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
169
+ {
170
+ headers: {
171
+ "xi-api-key": this.options.apiKey
172
+ }
173
+ }
174
+ );
175
+ this.socket = socket;
176
+ let opened = false;
177
+ const timeout = setTimeout(() => {
178
+ this.log("Connection timeout");
179
+ closeSocket(socket);
180
+ this.socket = void 0;
181
+ reject(new Error("WebSocket connection timeout"));
182
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
183
+ this.connectionTimeout = timeout;
184
+ socket.addEventListener("open", () => {
185
+ clearTimeout(timeout);
186
+ opened = true;
187
+ this.log("Connection opened");
188
+ const stability = this.options.voiceSettings?.stability;
189
+ this.send({
190
+ voices: [this.options.voiceId],
191
+ voice_settings: stability === void 0 ? void 0 : { stability }
192
+ });
193
+ this.hasSpoken = false;
194
+ this.keepAliveInterval = setInterval(() => {
195
+ this.send({ keep_alive: true });
196
+ this.retryCount = 0;
197
+ }, KEEP_ALIVE_INTERVAL);
198
+ resolve();
199
+ });
200
+ socket.addEventListener("error", (error) => {
201
+ clearTimeout(timeout);
202
+ this.log("WebSocket error:", error);
203
+ reject(new Error("WebSocket connection error"));
204
+ });
205
+ socket.addEventListener("message", (event) => {
206
+ try {
207
+ this.onMessage(JSON.parse(event.data.toString()));
208
+ } catch (error) {
209
+ this.log("message" in error ? error.message : error);
210
+ this.log("Event data during error:", event.data);
211
+ }
212
+ });
213
+ socket.addEventListener("close", ({ code, reason }) => {
214
+ clearTimeout(timeout);
215
+ if (this.keepAliveInterval) {
216
+ clearInterval(this.keepAliveInterval);
217
+ this.keepAliveInterval = void 0;
218
+ }
219
+ this.socket?.removeAllListeners();
220
+ this.socket = void 0;
221
+ this.log("Connection closed", { code, reason });
222
+ if (opened) this.reconnect();
223
+ });
224
+ });
225
+ }
226
+ onMessage(message) {
227
+ if ("audio" in message && message.audio) {
228
+ const chunk = Buffer.from(message.audio, "base64");
229
+ this.log(`Received audio chunk (${chunk.length} bytes)`);
230
+ this.retryCount = 0;
231
+ this.host.audio(chunk);
232
+ }
233
+ if ("is_final_audio_for_turn" in message && message.is_final_audio_for_turn) {
234
+ this.pendingFlushes = Math.max(0, this.pendingFlushes - 1);
235
+ this.checkEnded();
236
+ }
237
+ if ("error" in message) {
238
+ console.error("[ElevenLabsTTS] Error:", message.error, message.message);
239
+ }
240
+ }
241
+ /** Sends text, flushing after each sentence it completes */
242
+ sendText(text) {
243
+ let start = 0;
244
+ for (const match of text.matchAll(SENTENCE_END)) {
245
+ const end = match.index + match[0].length;
246
+ this.sendInput(text.slice(start, end), true);
247
+ start = end;
248
+ }
249
+ if (start < text.length) {
250
+ this.sendInput(text.slice(start), false);
251
+ }
252
+ }
253
+ sendInput(text, flush) {
254
+ this.textSent += text;
255
+ if (!text.trim()) return;
256
+ this.send({
257
+ inputs: [
258
+ {
259
+ text,
260
+ voice_id: this.options.voiceId,
261
+ // Resets the prosody between two utterances
262
+ new_turn: this.newTurn && this.hasSpoken || void 0
263
+ }
264
+ ],
265
+ flush: flush || void 0
266
+ });
267
+ this.log(`Sent text${flush ? " and flushed" : ""}: "${text}"`);
268
+ this.newTurn = false;
269
+ this.hasSpoken = true;
270
+ if (flush) {
271
+ this.pendingFlushes++;
272
+ this.unflushed = false;
273
+ } else {
274
+ this.unflushed = true;
275
+ }
276
+ }
277
+ /** Generates the text held by the server, if any */
278
+ flush() {
279
+ if (!this.unflushed) return;
280
+ this.send({ flush: true });
281
+ this.pendingFlushes++;
282
+ this.unflushed = false;
283
+ this.log("Flushed text");
284
+ }
285
+ /** Ends the utterance once its text is over and all its audio came */
286
+ checkEnded() {
287
+ if (!this.isProcessing || !this.textEnded) return;
288
+ if (this.unflushed || this.pendingFlushes > 0) return;
289
+ this.log("Audio ended");
290
+ this.isProcessing = false;
291
+ this.textSent = "";
292
+ }
293
+ send(message) {
294
+ if (this.socket?.readyState !== WebSocket2.OPEN) return;
295
+ this.socket.send(JSON.stringify(message));
296
+ }
297
+ reconnect() {
298
+ if (this.destroyed) return;
299
+ this.retryCount++;
300
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
301
+ this.log("Max retries reached, giving up");
302
+ const unspoken = this.isProcessing ? this.textSent + this.textBuffer : "";
303
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
304
+ return;
305
+ }
306
+ this.initPromise = new Promise((resolve) => {
307
+ this.log("Reconnecting...");
308
+ this.reconnectTimeout = setTimeout(() => {
309
+ this.reconnectTimeout = void 0;
310
+ this.initWS().then(() => {
311
+ if (this.isProcessing && this.textSent.trim()) {
312
+ this.log("Sending text again");
313
+ const text = this.textSent;
314
+ this.textSent = "";
315
+ this.unflushed = false;
316
+ this.pendingFlushes = 0;
317
+ this.sendText(text);
318
+ if (this.textEnded) this.flush();
319
+ }
320
+ }).then(resolve).catch((error) => {
321
+ this.log("Reconnection error:", error);
322
+ this.reconnect();
323
+ });
324
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
325
+ });
326
+ }
327
+ log(...message) {
328
+ this.host.log(...message);
329
+ }
330
+ };
331
+
332
+ // src/SpeechWebSocket.ts
333
+ import WebSocket3 from "ws";
334
+ var WS_INACTIVITY_TIMEOUT = 180;
335
+ var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
336
+ var DEFAULT_RETRY_DELAY2 = 1e3;
337
+ var DEFAULT_MAX_RETRY2 = 3;
338
+ var SpeechWebSocket = class {
339
+ constructor(options, host) {
17
340
  this.options = options;
341
+ this.host = host;
18
342
  this.isProcessing = false;
19
343
  this.textEnded = false;
20
344
  // Whether the text stream has ended
@@ -24,20 +348,33 @@ var ElevenLabsTTS = class extends TTS {
24
348
  // Buffer of text to send to ElevenLabs
25
349
  this.receivedAudioText = "";
26
350
  this.retryCount = 0;
351
+ this.destroyed = false;
27
352
  this.canceled = false;
353
+ // Bumped by every speak() and every cancel(), so a stream that starts late
354
+ // can tell whether it is still the one that should be heard.
355
+ this.generation = 0;
28
356
  this.initPromise = this.initWS().catch((error) => {
29
357
  console.error("[ElevenLabsTTS] Connection error:", error);
30
358
  this.reconnect();
31
359
  });
32
360
  }
33
361
  speak(textStream) {
34
- this.canceled = false;
35
- this.isProcessing = true;
36
- this.textEnded = false;
37
- this.textBuffer = "";
38
- this.textSent = "";
39
- this.receivedAudioText = "";
362
+ const generation = ++this.generation;
363
+ let started = false;
364
+ const start = () => {
365
+ if (started) return true;
366
+ if (this.generation !== generation) return false;
367
+ started = true;
368
+ this.canceled = false;
369
+ this.isProcessing = true;
370
+ this.textEnded = false;
371
+ this.textBuffer = "";
372
+ this.textSent = "";
373
+ this.receivedAudioText = "";
374
+ return true;
375
+ };
40
376
  textStream.on("data", async (chunk) => {
377
+ if (!start()) return;
41
378
  if (this.canceled) return;
42
379
  const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
43
380
  this.textSent += text;
@@ -51,11 +388,12 @@ var ElevenLabsTTS = class extends TTS {
51
388
  }
52
389
  });
53
390
  textStream.on("error", (error) => {
391
+ if (!started) return;
54
392
  this.log("Error in text stream, ending audio stream", error);
55
393
  this.isProcessing = false;
56
394
  });
57
395
  textStream.on("end", async () => {
58
- if (this.canceled) return;
396
+ if (!started || this.canceled) return;
59
397
  await this.initPromise;
60
398
  if (this.textBuffer.trim()) {
61
399
  this.sendTranscript(this.textBuffer + " ");
@@ -67,32 +405,34 @@ var ElevenLabsTTS = class extends TTS {
67
405
  });
68
406
  }
69
407
  cancel() {
408
+ this.generation++;
70
409
  if (!this.isProcessing) return;
71
410
  this.log("Cancel");
72
411
  this.canceled = true;
73
412
  this.textSent = "";
74
413
  this.receivedAudioText = "";
75
414
  this.isProcessing = false;
76
- if (this.socket?.readyState === WebSocket.OPEN) {
415
+ if (this.socket?.readyState === WebSocket3.OPEN) {
77
416
  this.socket.send(JSON.stringify({ text: " ", flush: true }));
78
417
  }
79
418
  }
80
419
  destroy() {
420
+ this.destroyed = true;
81
421
  if (this.reconnectTimeout) {
82
422
  clearTimeout(this.reconnectTimeout);
83
423
  this.reconnectTimeout = void 0;
84
424
  }
425
+ if (this.connectionTimeout) {
426
+ clearTimeout(this.connectionTimeout);
427
+ this.connectionTimeout = void 0;
428
+ }
85
429
  if (this.keepAliveInterval) {
86
430
  clearInterval(this.keepAliveInterval);
87
431
  this.keepAliveInterval = void 0;
88
432
  }
89
- this.socket?.removeAllListeners();
90
- if (this.socket?.readyState === WebSocket.OPEN) {
91
- this.socket?.close(1e3);
92
- }
433
+ if (this.socket) closeSocket(this.socket);
93
434
  this.socket = void 0;
94
435
  this.isProcessing = false;
95
- super.destroy();
96
436
  }
97
437
  initWS() {
98
438
  return new Promise((resolve, reject) => {
@@ -103,15 +443,15 @@ var ElevenLabsTTS = class extends TTS {
103
443
  this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
104
444
  );
105
445
  params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
106
- params.append(
107
- "voice_settings",
108
- JSON.stringify(this.options.voiceSettings)
109
- );
446
+ const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
447
+ if (voiceSettings) {
448
+ params.append("voice_settings", JSON.stringify(voiceSettings));
449
+ }
110
450
  if (this.options.language) {
111
451
  params.append("language_code", this.options.language);
112
452
  }
113
453
  const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`;
114
- const socket = new WebSocket(uri, {
454
+ const socket = new WebSocket3(uri, {
115
455
  headers: {
116
456
  "xi-api-key": this.options.apiKey
117
457
  }
@@ -119,25 +459,26 @@ var ElevenLabsTTS = class extends TTS {
119
459
  this.socket = socket;
120
460
  const timeout = setTimeout(() => {
121
461
  this.log("Connection timeout");
122
- socket.removeAllListeners();
123
- socket.close();
462
+ closeSocket(socket);
124
463
  this.socket = void 0;
125
464
  reject(new Error("WebSocket connection timeout"));
126
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
465
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
466
+ this.connectionTimeout = timeout;
127
467
  socket.addEventListener("open", () => {
128
468
  clearTimeout(timeout);
129
469
  this.log("Connection opened");
130
470
  this.socket?.send(
131
471
  JSON.stringify({
132
472
  text: " ",
133
- voice_settings: this.options.voiceSettings
473
+ voice_settings: voiceSettings
134
474
  })
135
475
  );
136
476
  this.keepAliveInterval = setInterval(
137
477
  () => {
138
- if (this.socket?.readyState === WebSocket.OPEN) {
478
+ if (this.socket?.readyState === WebSocket3.OPEN) {
139
479
  this.log("Sending keep-alive message");
140
480
  this.socket.send(JSON.stringify({ text: " " }));
481
+ this.retryCount = 0;
141
482
  }
142
483
  },
143
484
  (WS_INACTIVITY_TIMEOUT - 1) * 1e3
@@ -182,11 +523,12 @@ var ElevenLabsTTS = class extends TTS {
182
523
  this.textSent = "";
183
524
  }
184
525
  if ("error" in message) {
185
- this.log("Error:", message.error);
526
+ console.error("[ElevenLabsTTS] Error:", message.error, message.message);
186
527
  }
187
528
  }
188
529
  processAudioMessage(message) {
189
530
  const chunk = Buffer.from(message.audio, "base64");
531
+ this.retryCount = 0;
190
532
  if (message.alignment) {
191
533
  const chunkText = message.alignment.chars.join("");
192
534
  const receivedAudioText = this.receivedAudioText + chunkText;
@@ -198,7 +540,7 @@ var ElevenLabsTTS = class extends TTS {
198
540
  this.receivedAudioText = receivedAudioText;
199
541
  this.fixIsFinal(message);
200
542
  }
201
- this.emit("Audio", chunk);
543
+ this.host.audio(chunk);
202
544
  }
203
545
  fixIsFinal(message) {
204
546
  if (message.isFinal || !this.textEnded) return;
@@ -211,10 +553,11 @@ var ElevenLabsTTS = class extends TTS {
211
553
  this.log(`Sent transcript: "${text}"`);
212
554
  }
213
555
  reconnect() {
556
+ if (this.destroyed) return;
214
557
  this.retryCount++;
215
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
558
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
216
559
  this.log("Max retries reached, giving up");
217
- this.emit("Failed", [this.textSent]);
560
+ this.host.failed(this.textSent.trim() ? [this.textSent] : []);
218
561
  return;
219
562
  }
220
563
  this.initPromise = new Promise((resolve) => {
@@ -222,7 +565,6 @@ var ElevenLabsTTS = class extends TTS {
222
565
  this.reconnectTimeout = setTimeout(() => {
223
566
  this.reconnectTimeout = void 0;
224
567
  this.initWS().then(() => {
225
- this.retryCount = 0;
226
568
  if (this.textSent.length > 0) {
227
569
  this.log("Sending text chunks again");
228
570
  this.sendTranscript(this.textSent);
@@ -231,13 +573,49 @@ var ElevenLabsTTS = class extends TTS {
231
573
  this.log("Reconnection error:", error);
232
574
  this.reconnect();
233
575
  });
234
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
576
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
235
577
  });
236
578
  }
579
+ log(...message) {
580
+ this.host.log(...message);
581
+ }
582
+ };
583
+ function toWireVoiceSettings(settings) {
584
+ if (!settings) return void 0;
585
+ const { similarityBoost, useSpeakerBoost, ...rest } = settings;
586
+ return {
587
+ similarity_boost: similarityBoost,
588
+ use_speaker_boost: useSpeakerBoost,
589
+ ...rest
590
+ };
591
+ }
592
+
593
+ // src/ElevenLabsTTS.ts
594
+ var ElevenLabsTTS = class extends TTS {
595
+ constructor(options) {
596
+ super();
597
+ const host = {
598
+ audio: (chunk) => this.emit("Audio", chunk),
599
+ failed: (texts) => this.emit("Failed", texts),
600
+ log: (...message) => this.log(...message)
601
+ };
602
+ this.webSocket = isDialogueModel(options.modelId ?? DEFAULT_MODEL_ID) ? new DialogueWebSocket(options, host) : new SpeechWebSocket(options, host);
603
+ }
604
+ speak(textStream) {
605
+ this.webSocket.speak(textStream);
606
+ }
607
+ cancel() {
608
+ this.webSocket.cancel();
609
+ }
610
+ destroy() {
611
+ this.webSocket.destroy();
612
+ super.destroy();
613
+ }
237
614
  };
238
615
  export {
239
616
  DEFAULT_MODEL_ID,
240
617
  DEFAULT_OUTPUT_FORMAT,
241
- ElevenLabsTTS
618
+ ElevenLabsTTS,
619
+ isDialogueModel
242
620
  };
243
621
  //# sourceMappingURL=index.mjs.map