@micdrop/elevenlabs 1.1.4 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,20 +1,344 @@
1
1
  // src/ElevenLabsTTS.ts
2
2
  import { TTS } from "@micdrop/server";
3
+
4
+ // src/DialogueWebSocket.ts
5
+ import WebSocket2 from "ws";
6
+
7
+ // src/ElevenLabsWebSocket.ts
3
8
  import WebSocket from "ws";
9
+ function closeSocket(socket) {
10
+ socket.removeAllListeners();
11
+ socket.on("error", () => {
12
+ });
13
+ if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
14
+ socket.close(1e3);
15
+ }
16
+ }
4
17
 
5
18
  // src/types.ts
6
19
  var DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
7
20
  var DEFAULT_OUTPUT_FORMAT = "pcm_16000";
21
+ function isDialogueModel(modelId) {
22
+ return modelId.startsWith("eleven_v3") || modelId.startsWith("eleven_v4");
23
+ }
8
24
 
9
- // src/ElevenLabsTTS.ts
10
- var WS_INACTIVITY_TIMEOUT = 180;
25
+ // src/DialogueWebSocket.ts
26
+ var KEEP_ALIVE_INTERVAL = 15e3;
11
27
  var DEFAULT_CONNECTION_TIMEOUT = 5e3;
12
28
  var DEFAULT_RETRY_DELAY = 1e3;
13
29
  var DEFAULT_MAX_RETRY = 3;
14
- var ElevenLabsTTS = class extends TTS {
15
- constructor(options) {
16
- super();
30
+ var SENTENCE_END = /(?:[!?]+|(?<!\.)\.(?!\.))\s+/g;
31
+ var DialogueWebSocket = class {
32
+ // Whether the connection received any text
33
+ constructor(options, host) {
34
+ this.options = options;
35
+ this.host = host;
36
+ this.retryCount = 0;
37
+ this.destroyed = false;
38
+ // Bumped by every speak() and every cancel(), so a stream that starts late
39
+ // can tell whether it is still the one that should be heard.
40
+ this.generation = 0;
41
+ // Generation of the stream being spoken. A speak() that never carries a
42
+ // word leaves it in place, so the sentence it sends goes on to its end.
43
+ this.speaking = 0;
44
+ this.isProcessing = false;
45
+ this.textEnded = false;
46
+ // Whether the text stream has ended
47
+ this.textSent = "";
48
+ // Text of the utterance sent to ElevenLabs
49
+ this.textBuffer = "";
50
+ // End of the last chunk, held until its word is complete
51
+ this.unflushed = false;
52
+ // Whether text was sent since the last flush
53
+ this.pendingFlushes = 0;
54
+ // Flushes whose audio is not complete yet
55
+ this.newTurn = false;
56
+ // Whether the next input starts a new utterance
57
+ this.hasSpoken = false;
58
+ this.connect();
59
+ }
60
+ speak(textStream) {
61
+ const generation = ++this.generation;
62
+ let started = false;
63
+ const start = () => {
64
+ if (started) return this.speaking === generation;
65
+ if (this.generation !== generation) return false;
66
+ started = true;
67
+ if (this.isProcessing) this.restart();
68
+ this.speaking = generation;
69
+ this.isProcessing = true;
70
+ this.textEnded = false;
71
+ this.textSent = "";
72
+ this.textBuffer = "";
73
+ this.unflushed = false;
74
+ this.pendingFlushes = 0;
75
+ this.newTurn = true;
76
+ return true;
77
+ };
78
+ textStream.on("data", async (chunk) => {
79
+ if (!start()) return;
80
+ const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
81
+ await this.initPromise;
82
+ if (this.speaking !== generation) return;
83
+ const spaceIndex = text.lastIndexOf(" ");
84
+ if (spaceIndex === -1) {
85
+ this.textBuffer += text;
86
+ } else {
87
+ this.sendText(this.textBuffer + text.slice(0, spaceIndex + 1));
88
+ this.textBuffer = text.slice(spaceIndex + 1);
89
+ }
90
+ });
91
+ textStream.on("error", (error) => {
92
+ if (!started || this.speaking !== generation) return;
93
+ this.log("Error in text stream, ending audio stream", error);
94
+ this.isProcessing = false;
95
+ });
96
+ textStream.on("end", async () => {
97
+ if (!started) return;
98
+ await this.initPromise;
99
+ if (this.speaking !== generation) return;
100
+ if (this.textBuffer.trim()) {
101
+ this.sendText(this.textBuffer + " ");
102
+ this.textBuffer = "";
103
+ }
104
+ this.textEnded = true;
105
+ this.flush();
106
+ this.checkEnded();
107
+ });
108
+ }
109
+ cancel() {
110
+ this.generation++;
111
+ this.speaking = 0;
112
+ if (!this.isProcessing) return;
113
+ this.log("Cancel");
114
+ this.isProcessing = false;
115
+ this.textSent = "";
116
+ this.textBuffer = "";
117
+ this.restart();
118
+ }
119
+ destroy() {
120
+ this.destroyed = true;
121
+ if (this.reconnectTimeout) {
122
+ clearTimeout(this.reconnectTimeout);
123
+ this.reconnectTimeout = void 0;
124
+ }
125
+ this.dropSocket();
126
+ this.isProcessing = false;
127
+ }
128
+ connect() {
129
+ this.initPromise = this.initWS().catch((error) => {
130
+ console.error("[ElevenLabsTTS] Connection error:", error);
131
+ this.reconnect();
132
+ });
133
+ }
134
+ /** Drops the connection, with the synthesis running on it, for a new one */
135
+ restart() {
136
+ if (this.reconnectTimeout) {
137
+ clearTimeout(this.reconnectTimeout);
138
+ this.reconnectTimeout = void 0;
139
+ }
140
+ this.dropSocket();
141
+ this.connect();
142
+ }
143
+ /** Closes the connection without reconnecting */
144
+ dropSocket() {
145
+ if (this.connectionTimeout) {
146
+ clearTimeout(this.connectionTimeout);
147
+ this.connectionTimeout = void 0;
148
+ }
149
+ if (this.keepAliveInterval) {
150
+ clearInterval(this.keepAliveInterval);
151
+ this.keepAliveInterval = void 0;
152
+ }
153
+ if (this.socket) closeSocket(this.socket);
154
+ this.socket = void 0;
155
+ }
156
+ initWS() {
157
+ return new Promise((resolve, reject) => {
158
+ const params = new URLSearchParams();
159
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
160
+ params.append(
161
+ "output_format",
162
+ this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
163
+ );
164
+ if (this.options.language) {
165
+ params.append("language_code", this.options.language);
166
+ }
167
+ const socket = new WebSocket2(
168
+ `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
169
+ {
170
+ headers: {
171
+ "xi-api-key": this.options.apiKey
172
+ }
173
+ }
174
+ );
175
+ this.socket = socket;
176
+ let opened = false;
177
+ const timeout = setTimeout(() => {
178
+ this.log("Connection timeout");
179
+ closeSocket(socket);
180
+ this.socket = void 0;
181
+ reject(new Error("WebSocket connection timeout"));
182
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
183
+ this.connectionTimeout = timeout;
184
+ socket.addEventListener("open", () => {
185
+ clearTimeout(timeout);
186
+ opened = true;
187
+ this.log("Connection opened");
188
+ const stability = this.options.voiceSettings?.stability;
189
+ this.send({
190
+ voices: [this.options.voiceId],
191
+ voice_settings: stability === void 0 ? void 0 : { stability }
192
+ });
193
+ this.hasSpoken = false;
194
+ this.keepAliveInterval = setInterval(() => {
195
+ this.send({ keep_alive: true });
196
+ this.retryCount = 0;
197
+ }, KEEP_ALIVE_INTERVAL);
198
+ resolve();
199
+ });
200
+ socket.addEventListener("error", (error) => {
201
+ clearTimeout(timeout);
202
+ this.log("WebSocket error:", error);
203
+ reject(new Error("WebSocket connection error"));
204
+ });
205
+ socket.addEventListener("message", (event) => {
206
+ try {
207
+ this.onMessage(JSON.parse(event.data.toString()));
208
+ } catch (error) {
209
+ this.log("message" in error ? error.message : error);
210
+ this.log("Event data during error:", event.data);
211
+ }
212
+ });
213
+ socket.addEventListener("close", ({ code, reason }) => {
214
+ clearTimeout(timeout);
215
+ if (this.keepAliveInterval) {
216
+ clearInterval(this.keepAliveInterval);
217
+ this.keepAliveInterval = void 0;
218
+ }
219
+ this.socket?.removeAllListeners();
220
+ this.socket = void 0;
221
+ this.log("Connection closed", { code, reason });
222
+ if (opened) this.reconnect();
223
+ });
224
+ });
225
+ }
226
+ onMessage(message) {
227
+ if ("audio" in message && message.audio) {
228
+ const chunk = Buffer.from(message.audio, "base64");
229
+ this.log(`Received audio chunk (${chunk.length} bytes)`);
230
+ this.retryCount = 0;
231
+ this.host.audio(chunk);
232
+ }
233
+ if ("is_final_audio_for_turn" in message && message.is_final_audio_for_turn) {
234
+ this.pendingFlushes = Math.max(0, this.pendingFlushes - 1);
235
+ this.checkEnded();
236
+ }
237
+ if ("error" in message) {
238
+ console.error("[ElevenLabsTTS] Error:", message.error, message.message);
239
+ }
240
+ }
241
+ /** Sends text, flushing after each sentence it completes */
242
+ sendText(text) {
243
+ let start = 0;
244
+ for (const match of text.matchAll(SENTENCE_END)) {
245
+ const end = match.index + match[0].length;
246
+ this.sendInput(text.slice(start, end), true);
247
+ start = end;
248
+ }
249
+ if (start < text.length) {
250
+ this.sendInput(text.slice(start), false);
251
+ }
252
+ }
253
+ sendInput(text, flush) {
254
+ this.textSent += text;
255
+ if (!text.trim()) return;
256
+ this.send({
257
+ inputs: [
258
+ {
259
+ text,
260
+ voice_id: this.options.voiceId,
261
+ // Resets the prosody between two utterances
262
+ new_turn: this.newTurn && this.hasSpoken || void 0
263
+ }
264
+ ],
265
+ flush: flush || void 0
266
+ });
267
+ this.log(`Sent text${flush ? " and flushed" : ""}: "${text}"`);
268
+ this.newTurn = false;
269
+ this.hasSpoken = true;
270
+ if (flush) {
271
+ this.pendingFlushes++;
272
+ this.unflushed = false;
273
+ } else {
274
+ this.unflushed = true;
275
+ }
276
+ }
277
+ /** Generates the text held by the server, if any */
278
+ flush() {
279
+ if (!this.unflushed) return;
280
+ this.send({ flush: true });
281
+ this.pendingFlushes++;
282
+ this.unflushed = false;
283
+ this.log("Flushed text");
284
+ }
285
+ /** Ends the utterance once its text is over and all its audio came */
286
+ checkEnded() {
287
+ if (!this.isProcessing || !this.textEnded) return;
288
+ if (this.unflushed || this.pendingFlushes > 0) return;
289
+ this.log("Audio ended");
290
+ this.isProcessing = false;
291
+ this.textSent = "";
292
+ }
293
+ send(message) {
294
+ if (this.socket?.readyState !== WebSocket2.OPEN) return;
295
+ this.socket.send(JSON.stringify(message));
296
+ }
297
+ reconnect() {
298
+ if (this.destroyed) return;
299
+ this.retryCount++;
300
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
301
+ this.log("Max retries reached, giving up");
302
+ const unspoken = this.isProcessing ? this.textSent + this.textBuffer : "";
303
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
304
+ return;
305
+ }
306
+ this.initPromise = new Promise((resolve) => {
307
+ this.log("Reconnecting...");
308
+ this.reconnectTimeout = setTimeout(() => {
309
+ this.reconnectTimeout = void 0;
310
+ this.initWS().then(() => {
311
+ if (this.isProcessing && this.textSent.trim()) {
312
+ this.log("Sending text again");
313
+ const text = this.textSent;
314
+ this.textSent = "";
315
+ this.unflushed = false;
316
+ this.pendingFlushes = 0;
317
+ this.sendText(text);
318
+ if (this.textEnded) this.flush();
319
+ }
320
+ }).then(resolve).catch((error) => {
321
+ this.log("Reconnection error:", error);
322
+ this.reconnect();
323
+ });
324
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
325
+ });
326
+ }
327
+ log(...message) {
328
+ this.host.log(...message);
329
+ }
330
+ };
331
+
332
+ // src/SpeechWebSocket.ts
333
+ import WebSocket3 from "ws";
334
+ var WS_INACTIVITY_TIMEOUT = 180;
335
+ var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
336
+ var DEFAULT_RETRY_DELAY2 = 1e3;
337
+ var DEFAULT_MAX_RETRY2 = 3;
338
+ var SpeechWebSocket = class {
339
+ constructor(options, host) {
17
340
  this.options = options;
341
+ this.host = host;
18
342
  this.isProcessing = false;
19
343
  this.textEnded = false;
20
344
  // Whether the text stream has ended
@@ -24,6 +348,7 @@ var ElevenLabsTTS = class extends TTS {
24
348
  // Buffer of text to send to ElevenLabs
25
349
  this.receivedAudioText = "";
26
350
  this.retryCount = 0;
351
+ this.destroyed = false;
27
352
  this.canceled = false;
28
353
  // Bumped by every speak() and every cancel(), so a stream that starts late
29
354
  // can tell whether it is still the one that should be heard.
@@ -87,26 +412,27 @@ var ElevenLabsTTS = class extends TTS {
87
412
  this.textSent = "";
88
413
  this.receivedAudioText = "";
89
414
  this.isProcessing = false;
90
- if (this.socket?.readyState === WebSocket.OPEN) {
415
+ if (this.socket?.readyState === WebSocket3.OPEN) {
91
416
  this.socket.send(JSON.stringify({ text: " ", flush: true }));
92
417
  }
93
418
  }
94
419
  destroy() {
420
+ this.destroyed = true;
95
421
  if (this.reconnectTimeout) {
96
422
  clearTimeout(this.reconnectTimeout);
97
423
  this.reconnectTimeout = void 0;
98
424
  }
425
+ if (this.connectionTimeout) {
426
+ clearTimeout(this.connectionTimeout);
427
+ this.connectionTimeout = void 0;
428
+ }
99
429
  if (this.keepAliveInterval) {
100
430
  clearInterval(this.keepAliveInterval);
101
431
  this.keepAliveInterval = void 0;
102
432
  }
103
- this.socket?.removeAllListeners();
104
- if (this.socket?.readyState === WebSocket.OPEN) {
105
- this.socket?.close(1e3);
106
- }
433
+ if (this.socket) closeSocket(this.socket);
107
434
  this.socket = void 0;
108
435
  this.isProcessing = false;
109
- super.destroy();
110
436
  }
111
437
  initWS() {
112
438
  return new Promise((resolve, reject) => {
@@ -117,15 +443,15 @@ var ElevenLabsTTS = class extends TTS {
117
443
  this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
118
444
  );
119
445
  params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
120
- params.append(
121
- "voice_settings",
122
- JSON.stringify(this.options.voiceSettings)
123
- );
446
+ const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
447
+ if (voiceSettings) {
448
+ params.append("voice_settings", JSON.stringify(voiceSettings));
449
+ }
124
450
  if (this.options.language) {
125
451
  params.append("language_code", this.options.language);
126
452
  }
127
453
  const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`;
128
- const socket = new WebSocket(uri, {
454
+ const socket = new WebSocket3(uri, {
129
455
  headers: {
130
456
  "xi-api-key": this.options.apiKey
131
457
  }
@@ -133,25 +459,26 @@ var ElevenLabsTTS = class extends TTS {
133
459
  this.socket = socket;
134
460
  const timeout = setTimeout(() => {
135
461
  this.log("Connection timeout");
136
- socket.removeAllListeners();
137
- socket.close();
462
+ closeSocket(socket);
138
463
  this.socket = void 0;
139
464
  reject(new Error("WebSocket connection timeout"));
140
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
465
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
466
+ this.connectionTimeout = timeout;
141
467
  socket.addEventListener("open", () => {
142
468
  clearTimeout(timeout);
143
469
  this.log("Connection opened");
144
470
  this.socket?.send(
145
471
  JSON.stringify({
146
472
  text: " ",
147
- voice_settings: this.options.voiceSettings
473
+ voice_settings: voiceSettings
148
474
  })
149
475
  );
150
476
  this.keepAliveInterval = setInterval(
151
477
  () => {
152
- if (this.socket?.readyState === WebSocket.OPEN) {
478
+ if (this.socket?.readyState === WebSocket3.OPEN) {
153
479
  this.log("Sending keep-alive message");
154
480
  this.socket.send(JSON.stringify({ text: " " }));
481
+ this.retryCount = 0;
155
482
  }
156
483
  },
157
484
  (WS_INACTIVITY_TIMEOUT - 1) * 1e3
@@ -196,11 +523,12 @@ var ElevenLabsTTS = class extends TTS {
196
523
  this.textSent = "";
197
524
  }
198
525
  if ("error" in message) {
199
- this.log("Error:", message.error);
526
+ console.error("[ElevenLabsTTS] Error:", message.error, message.message);
200
527
  }
201
528
  }
202
529
  processAudioMessage(message) {
203
530
  const chunk = Buffer.from(message.audio, "base64");
531
+ this.retryCount = 0;
204
532
  if (message.alignment) {
205
533
  const chunkText = message.alignment.chars.join("");
206
534
  const receivedAudioText = this.receivedAudioText + chunkText;
@@ -212,7 +540,7 @@ var ElevenLabsTTS = class extends TTS {
212
540
  this.receivedAudioText = receivedAudioText;
213
541
  this.fixIsFinal(message);
214
542
  }
215
- this.emit("Audio", chunk);
543
+ this.host.audio(chunk);
216
544
  }
217
545
  fixIsFinal(message) {
218
546
  if (message.isFinal || !this.textEnded) return;
@@ -225,10 +553,11 @@ var ElevenLabsTTS = class extends TTS {
225
553
  this.log(`Sent transcript: "${text}"`);
226
554
  }
227
555
  reconnect() {
556
+ if (this.destroyed) return;
228
557
  this.retryCount++;
229
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
558
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
230
559
  this.log("Max retries reached, giving up");
231
- this.emit("Failed", [this.textSent]);
560
+ this.host.failed(this.textSent.trim() ? [this.textSent] : []);
232
561
  return;
233
562
  }
234
563
  this.initPromise = new Promise((resolve) => {
@@ -236,7 +565,6 @@ var ElevenLabsTTS = class extends TTS {
236
565
  this.reconnectTimeout = setTimeout(() => {
237
566
  this.reconnectTimeout = void 0;
238
567
  this.initWS().then(() => {
239
- this.retryCount = 0;
240
568
  if (this.textSent.length > 0) {
241
569
  this.log("Sending text chunks again");
242
570
  this.sendTranscript(this.textSent);
@@ -245,13 +573,49 @@ var ElevenLabsTTS = class extends TTS {
245
573
  this.log("Reconnection error:", error);
246
574
  this.reconnect();
247
575
  });
248
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
576
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
249
577
  });
250
578
  }
579
+ log(...message) {
580
+ this.host.log(...message);
581
+ }
582
+ };
583
+ function toWireVoiceSettings(settings) {
584
+ if (!settings) return void 0;
585
+ const { similarityBoost, useSpeakerBoost, ...rest } = settings;
586
+ return {
587
+ similarity_boost: similarityBoost,
588
+ use_speaker_boost: useSpeakerBoost,
589
+ ...rest
590
+ };
591
+ }
592
+
593
+ // src/ElevenLabsTTS.ts
594
+ var ElevenLabsTTS = class extends TTS {
595
+ constructor(options) {
596
+ super();
597
+ const host = {
598
+ audio: (chunk) => this.emit("Audio", chunk),
599
+ failed: (texts) => this.emit("Failed", texts),
600
+ log: (...message) => this.log(...message)
601
+ };
602
+ this.webSocket = isDialogueModel(options.modelId ?? DEFAULT_MODEL_ID) ? new DialogueWebSocket(options, host) : new SpeechWebSocket(options, host);
603
+ }
604
+ speak(textStream) {
605
+ this.webSocket.speak(textStream);
606
+ }
607
+ cancel() {
608
+ this.webSocket.cancel();
609
+ }
610
+ destroy() {
611
+ this.webSocket.destroy();
612
+ super.destroy();
613
+ }
251
614
  };
252
615
  export {
253
616
  DEFAULT_MODEL_ID,
254
617
  DEFAULT_OUTPUT_FORMAT,
255
- ElevenLabsTTS
618
+ ElevenLabsTTS,
619
+ isDialogueModel
256
620
  };
257
621
  //# sourceMappingURL=index.mjs.map