@micdrop/elevenlabs 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,19 +1,94 @@
1
1
  // src/ElevenLabsTTS.ts
2
2
  import { TTS } from "@micdrop/server";
3
3
 
4
- // src/DialogueWebSocket.ts
5
- import WebSocket2 from "ws";
6
-
7
4
  // src/ElevenLabsWebSocket.ts
8
- import WebSocket from "ws";
9
- function closeSocket(socket) {
10
- socket.removeAllListeners();
11
- socket.on("error", () => {
12
- });
13
- if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
14
- socket.close(1e3);
5
+ import { ReconnectingSocket } from "@micdrop/server";
6
+ var DEFAULT_MAX_RETRY = 3;
7
+ var ElevenLabsConnection = class {
8
+ constructor(options) {
9
+ this.options = options;
10
+ // Whether the current connection carried audio or lived long enough for a
11
+ // keep-alive, which means the server accepted the key and the voice
12
+ this.proven = false;
13
+ this.failures = 0;
14
+ this.socket = this.createSocket();
15
15
  }
16
- }
16
+ /** Opens the connection, again after giving up, and does nothing otherwise */
17
+ connect() {
18
+ this.socket.connect();
19
+ }
20
+ /**
21
+ * Resolves to false when the connection gives up or is destroyed meanwhile,
22
+ * and right away once it gave up, until the next `connect()`: the stream
23
+ * that was speaking then has nothing left to wait for.
24
+ */
25
+ ready() {
26
+ if (this.socket.status === "idle") return Promise.resolve(false);
27
+ return this.socket.ready();
28
+ }
29
+ send(message) {
30
+ return this.socket.send(JSON.stringify(message));
31
+ }
32
+ /** Drops the connection, with the synthesis running on it, for a new one */
33
+ restart() {
34
+ this.socket.restart();
35
+ }
36
+ destroy() {
37
+ this.socket.destroy();
38
+ }
39
+ /** The current connection works, so its next drop is not a failure */
40
+ prove() {
41
+ this.proven = true;
42
+ this.failures = 0;
43
+ }
44
+ createSocket() {
45
+ const { tts, keepAlive, host } = this.options;
46
+ const socket = new ReconnectingSocket({
47
+ url: this.options.url,
48
+ clientOptions: { headers: { "xi-api-key": tts.apiKey } },
49
+ connectionTimeout: tts.connectionTimeout,
50
+ retryDelay: tts.retryDelay,
51
+ maxRetry: tts.maxRetry,
52
+ handshake: {
53
+ start: (ws) => ws.send(JSON.stringify(this.options.init))
54
+ },
55
+ keepAlive: {
56
+ interval: keepAlive.interval,
57
+ message: () => {
58
+ this.prove();
59
+ return JSON.stringify(keepAlive.message);
60
+ }
61
+ },
62
+ log: (...message) => host.log(...message)
63
+ });
64
+ socket.on("Open", ({ reconnected }) => {
65
+ this.proven = false;
66
+ this.options.onOpen(reconnected);
67
+ });
68
+ socket.on("Message", (data) => {
69
+ try {
70
+ this.options.onMessage(JSON.parse(data.toString()));
71
+ } catch (error) {
72
+ host.log("message" in error ? error.message : error);
73
+ host.log("Event data during error:", data.toString());
74
+ }
75
+ });
76
+ socket.on("Close", () => {
77
+ if (this.proven) return;
78
+ this.failures++;
79
+ if (this.failures > (tts.maxRetry ?? DEFAULT_MAX_RETRY)) this.giveUp();
80
+ });
81
+ socket.on("Failed", () => this.giveUp());
82
+ return socket;
83
+ }
84
+ giveUp() {
85
+ this.options.host.log("Max retries reached, giving up");
86
+ this.failures = 0;
87
+ this.socket.destroy();
88
+ this.socket = this.createSocket();
89
+ this.options.onFailed();
90
+ }
91
+ };
17
92
 
18
93
  // src/types.ts
19
94
  var DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
@@ -24,30 +99,23 @@ function isDialogueModel(modelId) {
24
99
 
25
100
  // src/DialogueWebSocket.ts
26
101
  var KEEP_ALIVE_INTERVAL = 15e3;
27
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
28
- var DEFAULT_RETRY_DELAY = 1e3;
29
- var DEFAULT_MAX_RETRY = 3;
30
102
  var SENTENCE_END = /(?:[!?]+|(?<!\.)\.(?!\.))\s+/g;
31
103
  var DialogueWebSocket = class {
32
104
  // Whether the connection received any text
33
105
  constructor(options, host) {
34
106
  this.options = options;
35
107
  this.host = host;
36
- this.retryCount = 0;
37
- this.destroyed = false;
108
+ this.state = "idle";
38
109
  // Bumped by every speak() and every cancel(), so a stream that starts late
39
110
  // can tell whether it is still the one that should be heard.
40
111
  this.generation = 0;
41
112
  // Generation of the stream being spoken. A speak() that never carries a
42
113
  // word leaves it in place, so the sentence it sends goes on to its end.
43
114
  this.speaking = 0;
44
- this.isProcessing = false;
45
- this.textEnded = false;
46
- // Whether the text stream has ended
47
115
  this.textSent = "";
48
116
  // Text of the utterance sent to ElevenLabs
49
117
  this.textBuffer = "";
50
- // End of the last chunk, held until its word is complete
118
+ // Text not sent yet, held until its word is complete
51
119
  this.unflushed = false;
52
120
  // Whether text was sent since the last flush
53
121
  this.pendingFlushes = 0;
@@ -55,7 +123,34 @@ var DialogueWebSocket = class {
55
123
  this.newTurn = false;
56
124
  // Whether the next input starts a new utterance
57
125
  this.hasSpoken = false;
58
- this.connect();
126
+ const params = new URLSearchParams();
127
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
128
+ params.append(
129
+ "output_format",
130
+ this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
131
+ );
132
+ if (this.options.language) {
133
+ params.append("language_code", this.options.language);
134
+ }
135
+ const stability = this.options.voiceSettings?.stability;
136
+ const init = {
137
+ voices: [this.options.voiceId],
138
+ voice_settings: stability === void 0 ? void 0 : { stability }
139
+ };
140
+ this.connection = new ElevenLabsConnection({
141
+ url: `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
142
+ tts: options,
143
+ init,
144
+ keepAlive: {
145
+ interval: KEEP_ALIVE_INTERVAL,
146
+ message: { keep_alive: true }
147
+ },
148
+ host,
149
+ onOpen: () => this.onOpen(),
150
+ onMessage: (message) => this.onMessage(message),
151
+ onFailed: () => this.onFailed()
152
+ });
153
+ this.connection.connect();
59
154
  }
60
155
  speak(textStream) {
61
156
  const generation = ++this.generation;
@@ -64,171 +159,100 @@ var DialogueWebSocket = class {
64
159
  if (started) return this.speaking === generation;
65
160
  if (this.generation !== generation) return false;
66
161
  started = true;
67
- if (this.isProcessing) this.restart();
162
+ if (this.state !== "idle") this.connection.restart();
163
+ else this.connection.connect();
164
+ this.resetUtterance();
68
165
  this.speaking = generation;
69
- this.isProcessing = true;
70
- this.textEnded = false;
71
- this.textSent = "";
72
- this.textBuffer = "";
73
- this.unflushed = false;
74
- this.pendingFlushes = 0;
166
+ this.state = "streaming";
75
167
  this.newTurn = true;
76
168
  return true;
77
169
  };
78
170
  textStream.on("data", async (chunk) => {
79
171
  if (!start()) return;
80
- const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
81
- await this.initPromise;
172
+ this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
173
+ if (!await this.connection.ready()) return;
82
174
  if (this.speaking !== generation) return;
83
- const spaceIndex = text.lastIndexOf(" ");
84
- if (spaceIndex === -1) {
85
- this.textBuffer += text;
86
- } else {
87
- this.sendText(this.textBuffer + text.slice(0, spaceIndex + 1));
88
- this.textBuffer = text.slice(spaceIndex + 1);
89
- }
175
+ const spaceIndex = this.textBuffer.lastIndexOf(" ");
176
+ if (spaceIndex === -1) return;
177
+ this.sendText(this.textBuffer.slice(0, spaceIndex + 1));
178
+ this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
90
179
  });
91
180
  textStream.on("error", (error) => {
92
181
  if (!started || this.speaking !== generation) return;
93
182
  this.log("Error in text stream, ending audio stream", error);
94
- this.isProcessing = false;
183
+ this.speaking = 0;
184
+ this.textBuffer = "";
185
+ this.endText();
95
186
  });
96
187
  textStream.on("end", async () => {
97
188
  if (!started) return;
98
- await this.initPromise;
189
+ if (!await this.connection.ready()) return;
99
190
  if (this.speaking !== generation) return;
100
191
  if (this.textBuffer.trim()) {
101
192
  this.sendText(this.textBuffer + " ");
102
- this.textBuffer = "";
103
193
  }
104
- this.textEnded = true;
105
- this.flush();
106
- this.checkEnded();
194
+ this.textBuffer = "";
195
+ this.endText();
107
196
  });
108
197
  }
109
198
  cancel() {
110
199
  this.generation++;
111
200
  this.speaking = 0;
112
- if (!this.isProcessing) return;
201
+ if (this.state === "idle") return;
113
202
  this.log("Cancel");
114
- this.isProcessing = false;
115
- this.textSent = "";
116
- this.textBuffer = "";
117
- this.restart();
203
+ this.resetUtterance();
204
+ this.connection.restart();
118
205
  }
119
206
  destroy() {
120
- this.destroyed = true;
121
- if (this.reconnectTimeout) {
122
- clearTimeout(this.reconnectTimeout);
123
- this.reconnectTimeout = void 0;
207
+ this.generation++;
208
+ this.speaking = 0;
209
+ this.resetUtterance();
210
+ this.connection.destroy();
211
+ }
212
+ onOpen() {
213
+ this.hasSpoken = false;
214
+ if (this.state === "idle") return;
215
+ const text = this.textSent;
216
+ this.textSent = "";
217
+ this.unflushed = false;
218
+ this.pendingFlushes = 0;
219
+ if (text.trim()) {
220
+ this.log("Sending text again");
221
+ this.sendText(text);
124
222
  }
125
- this.dropSocket();
126
- this.isProcessing = false;
223
+ if (this.state === "draining") this.flush();
224
+ this.checkEnded();
127
225
  }
128
- connect() {
129
- this.initPromise = this.initWS().catch((error) => {
130
- console.error("[ElevenLabsTTS] Connection error:", error);
131
- this.reconnect();
132
- });
226
+ /** Every retry failed: hands the unspoken text over and drops the utterance */
227
+ onFailed() {
228
+ const unspoken = this.state === "idle" ? "" : this.textSent + this.textBuffer;
229
+ this.speaking = 0;
230
+ this.resetUtterance();
231
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
133
232
  }
134
- /** Drops the connection, with the synthesis running on it, for a new one */
135
- restart() {
136
- if (this.reconnectTimeout) {
137
- clearTimeout(this.reconnectTimeout);
138
- this.reconnectTimeout = void 0;
139
- }
140
- this.dropSocket();
141
- this.connect();
142
- }
143
- /** Closes the connection without reconnecting */
144
- dropSocket() {
145
- if (this.connectionTimeout) {
146
- clearTimeout(this.connectionTimeout);
147
- this.connectionTimeout = void 0;
148
- }
149
- if (this.keepAliveInterval) {
150
- clearInterval(this.keepAliveInterval);
151
- this.keepAliveInterval = void 0;
152
- }
153
- if (this.socket) closeSocket(this.socket);
154
- this.socket = void 0;
155
- }
156
- initWS() {
157
- return new Promise((resolve, reject) => {
158
- const params = new URLSearchParams();
159
- params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
160
- params.append(
161
- "output_format",
162
- this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
163
- );
164
- if (this.options.language) {
165
- params.append("language_code", this.options.language);
166
- }
167
- const socket = new WebSocket2(
168
- `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
169
- {
170
- headers: {
171
- "xi-api-key": this.options.apiKey
172
- }
173
- }
174
- );
175
- this.socket = socket;
176
- let opened = false;
177
- const timeout = setTimeout(() => {
178
- this.log("Connection timeout");
179
- closeSocket(socket);
180
- this.socket = void 0;
181
- reject(new Error("WebSocket connection timeout"));
182
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
183
- this.connectionTimeout = timeout;
184
- socket.addEventListener("open", () => {
185
- clearTimeout(timeout);
186
- opened = true;
187
- this.log("Connection opened");
188
- const stability = this.options.voiceSettings?.stability;
189
- this.send({
190
- voices: [this.options.voiceId],
191
- voice_settings: stability === void 0 ? void 0 : { stability }
192
- });
193
- this.hasSpoken = false;
194
- this.keepAliveInterval = setInterval(() => {
195
- this.send({ keep_alive: true });
196
- this.retryCount = 0;
197
- }, KEEP_ALIVE_INTERVAL);
198
- resolve();
199
- });
200
- socket.addEventListener("error", (error) => {
201
- clearTimeout(timeout);
202
- this.log("WebSocket error:", error);
203
- reject(new Error("WebSocket connection error"));
204
- });
205
- socket.addEventListener("message", (event) => {
206
- try {
207
- this.onMessage(JSON.parse(event.data.toString()));
208
- } catch (error) {
209
- this.log("message" in error ? error.message : error);
210
- this.log("Event data during error:", event.data);
211
- }
212
- });
213
- socket.addEventListener("close", ({ code, reason }) => {
214
- clearTimeout(timeout);
215
- if (this.keepAliveInterval) {
216
- clearInterval(this.keepAliveInterval);
217
- this.keepAliveInterval = void 0;
218
- }
219
- this.socket?.removeAllListeners();
220
- this.socket = void 0;
221
- this.log("Connection closed", { code, reason });
222
- if (opened) this.reconnect();
223
- });
224
- });
233
+ resetUtterance() {
234
+ this.state = "idle";
235
+ this.textSent = "";
236
+ this.textBuffer = "";
237
+ this.unflushed = false;
238
+ this.pendingFlushes = 0;
239
+ }
240
+ /** The text stream is over: flushes what the server holds */
241
+ endText() {
242
+ this.state = "draining";
243
+ this.flush();
244
+ this.checkEnded();
225
245
  }
226
246
  onMessage(message) {
227
247
  if ("audio" in message && message.audio) {
228
248
  const chunk = Buffer.from(message.audio, "base64");
229
- this.log(`Received audio chunk (${chunk.length} bytes)`);
230
- this.retryCount = 0;
231
- this.host.audio(chunk);
249
+ this.connection.prove();
250
+ if (this.state === "idle") {
251
+ this.log("Ignore audio chunk, nothing is being spoken");
252
+ } else {
253
+ this.log(`Received audio chunk (${chunk.length} bytes)`);
254
+ this.host.audio(chunk);
255
+ }
232
256
  }
233
257
  if ("is_final_audio_for_turn" in message && message.is_final_audio_for_turn) {
234
258
  this.pendingFlushes = Math.max(0, this.pendingFlushes - 1);
@@ -284,45 +308,15 @@ var DialogueWebSocket = class {
284
308
  }
285
309
  /** Ends the utterance once its text is over and all its audio came */
286
310
  checkEnded() {
287
- if (!this.isProcessing || !this.textEnded) return;
311
+ if (this.state !== "draining") return;
288
312
  if (this.unflushed || this.pendingFlushes > 0) return;
289
313
  this.log("Audio ended");
290
- this.isProcessing = false;
291
- this.textSent = "";
314
+ this.resetUtterance();
292
315
  }
316
+ // Lost when the connection just dropped, which is fine: the reconnection
317
+ // sends the utterance again from textSent
293
318
  send(message) {
294
- if (this.socket?.readyState !== WebSocket2.OPEN) return;
295
- this.socket.send(JSON.stringify(message));
296
- }
297
- reconnect() {
298
- if (this.destroyed) return;
299
- this.retryCount++;
300
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
301
- this.log("Max retries reached, giving up");
302
- const unspoken = this.isProcessing ? this.textSent + this.textBuffer : "";
303
- this.host.failed(unspoken.trim() ? [unspoken] : []);
304
- return;
305
- }
306
- this.initPromise = new Promise((resolve) => {
307
- this.log("Reconnecting...");
308
- this.reconnectTimeout = setTimeout(() => {
309
- this.reconnectTimeout = void 0;
310
- this.initWS().then(() => {
311
- if (this.isProcessing && this.textSent.trim()) {
312
- this.log("Sending text again");
313
- const text = this.textSent;
314
- this.textSent = "";
315
- this.unflushed = false;
316
- this.pendingFlushes = 0;
317
- this.sendText(text);
318
- if (this.textEnded) this.flush();
319
- }
320
- }).then(resolve).catch((error) => {
321
- this.log("Reconnection error:", error);
322
- this.reconnect();
323
- });
324
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
325
- });
319
+ this.connection.send(message);
326
320
  }
327
321
  log(...message) {
328
322
  this.host.log(...message);
@@ -330,197 +324,153 @@ var DialogueWebSocket = class {
330
324
  };
331
325
 
332
326
  // src/SpeechWebSocket.ts
333
- import WebSocket3 from "ws";
334
327
  var WS_INACTIVITY_TIMEOUT = 180;
335
- var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
336
- var DEFAULT_RETRY_DELAY2 = 1e3;
337
- var DEFAULT_MAX_RETRY2 = 3;
338
328
  var SpeechWebSocket = class {
329
+ // Index in textSent where the audio received ends
339
330
  constructor(options, host) {
340
331
  this.options = options;
341
332
  this.host = host;
342
- this.isProcessing = false;
343
- this.textEnded = false;
344
- // Whether the text stream has ended
345
- this.textSent = "";
346
- // Text sent to ElevenLabs
347
- this.textBuffer = "";
348
- // Buffer of text to send to ElevenLabs
349
- this.receivedAudioText = "";
350
- this.retryCount = 0;
351
- this.destroyed = false;
352
- this.canceled = false;
333
+ this.state = "idle";
353
334
  // Bumped by every speak() and every cancel(), so a stream that starts late
354
335
  // can tell whether it is still the one that should be heard.
355
336
  this.generation = 0;
356
- this.initPromise = this.initWS().catch((error) => {
357
- console.error("[ElevenLabsTTS] Connection error:", error);
358
- this.reconnect();
337
+ // Generation of the stream being spoken. A speak() that never carries a
338
+ // word leaves it in place, so the sentence it sends goes on to its end.
339
+ this.speaking = 0;
340
+ this.textSent = "";
341
+ // Text of the utterance sent to ElevenLabs
342
+ this.textBuffer = "";
343
+ // Text not sent yet, held until its word is complete
344
+ this.heardUntil = 0;
345
+ const params = new URLSearchParams();
346
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
347
+ params.append(
348
+ "output_format",
349
+ this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
350
+ );
351
+ params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
352
+ const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
353
+ if (voiceSettings) {
354
+ params.append("voice_settings", JSON.stringify(voiceSettings));
355
+ }
356
+ if (this.options.language) {
357
+ params.append("language_code", this.options.language);
358
+ }
359
+ this.connection = new ElevenLabsConnection({
360
+ url: `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`,
361
+ tts: options,
362
+ // Initialization message with voice settings
363
+ init: { text: " ", voice_settings: voiceSettings },
364
+ keepAlive: {
365
+ interval: (WS_INACTIVITY_TIMEOUT - 1) * 1e3,
366
+ message: { text: " " }
367
+ },
368
+ host,
369
+ onOpen: () => this.resume(),
370
+ onMessage: (message) => this.onMessage(message),
371
+ onFailed: () => this.onFailed()
359
372
  });
373
+ this.connection.connect();
360
374
  }
361
375
  speak(textStream) {
362
376
  const generation = ++this.generation;
363
377
  let started = false;
364
378
  const start = () => {
365
- if (started) return true;
379
+ if (started) return this.speaking === generation;
366
380
  if (this.generation !== generation) return false;
367
381
  started = true;
368
- this.canceled = false;
369
- this.isProcessing = true;
370
- this.textEnded = false;
371
- this.textBuffer = "";
372
- this.textSent = "";
373
- this.receivedAudioText = "";
382
+ if (this.state !== "idle") this.connection.restart();
383
+ else this.connection.connect();
384
+ this.resetUtterance();
385
+ this.speaking = generation;
386
+ this.state = "speaking";
374
387
  return true;
375
388
  };
376
389
  textStream.on("data", async (chunk) => {
377
390
  if (!start()) return;
378
- if (this.canceled) return;
379
- const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
380
- this.textSent += text;
381
- await this.initPromise;
382
- const spaceIndex = text.lastIndexOf(" ");
383
- if (spaceIndex === -1) {
384
- this.textBuffer += text;
385
- } else {
386
- this.sendTranscript(this.textBuffer + text.slice(0, spaceIndex + 1));
387
- this.textBuffer = text.slice(spaceIndex + 1);
388
- }
391
+ this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
392
+ if (!await this.connection.ready()) return;
393
+ if (this.speaking !== generation) return;
394
+ const spaceIndex = this.textBuffer.lastIndexOf(" ");
395
+ if (spaceIndex === -1) return;
396
+ this.sendTranscript(this.textBuffer.slice(0, spaceIndex + 1));
397
+ this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
389
398
  });
390
399
  textStream.on("error", (error) => {
391
- if (!started) return;
400
+ if (!started || this.speaking !== generation) return;
392
401
  this.log("Error in text stream, ending audio stream", error);
393
- this.isProcessing = false;
402
+ this.speaking = 0;
403
+ this.textBuffer = "";
404
+ this.flush();
394
405
  });
395
406
  textStream.on("end", async () => {
396
- if (!started || this.canceled) return;
397
- await this.initPromise;
407
+ if (!started) return;
408
+ if (!await this.connection.ready()) return;
409
+ if (this.speaking !== generation) return;
398
410
  if (this.textBuffer.trim()) {
399
411
  this.sendTranscript(this.textBuffer + " ");
400
- this.textBuffer = "";
401
412
  }
402
- this.textEnded = true;
403
- this.socket?.send(JSON.stringify({ text: " ", flush: true }));
404
- this.log("Flushed text");
413
+ this.textBuffer = "";
414
+ this.flush();
405
415
  });
406
416
  }
407
417
  cancel() {
408
418
  this.generation++;
409
- if (!this.isProcessing) return;
419
+ this.speaking = 0;
420
+ if (this.state === "idle") return;
410
421
  this.log("Cancel");
411
- this.canceled = true;
412
- this.textSent = "";
413
- this.receivedAudioText = "";
414
- this.isProcessing = false;
415
- if (this.socket?.readyState === WebSocket3.OPEN) {
416
- this.socket.send(JSON.stringify({ text: " ", flush: true }));
417
- }
422
+ this.resetUtterance();
423
+ this.connection.restart();
418
424
  }
419
425
  destroy() {
420
- this.destroyed = true;
421
- if (this.reconnectTimeout) {
422
- clearTimeout(this.reconnectTimeout);
423
- this.reconnectTimeout = void 0;
424
- }
425
- if (this.connectionTimeout) {
426
- clearTimeout(this.connectionTimeout);
427
- this.connectionTimeout = void 0;
428
- }
429
- if (this.keepAliveInterval) {
430
- clearInterval(this.keepAliveInterval);
431
- this.keepAliveInterval = void 0;
426
+ this.generation++;
427
+ this.speaking = 0;
428
+ this.resetUtterance();
429
+ this.connection.destroy();
430
+ }
431
+ /** Flushes buffered text and marks the end of the utterance */
432
+ flush() {
433
+ this.state = "flushed";
434
+ this.connection.send({ text: " ", flush: true });
435
+ this.log("Flushed text");
436
+ this.checkEnded();
437
+ }
438
+ /**
439
+ * Sends the text of the utterance in flight again on a new connection. The
440
+ * server lost what it was generating, so it goes on from the audio received.
441
+ */
442
+ resume() {
443
+ if (this.state === "idle") return;
444
+ const wordStart = this.textSent.lastIndexOf(" ", this.heardUntil - 1) + 1;
445
+ const unheard = this.textSent.slice(wordStart);
446
+ this.textSent = "";
447
+ this.heardUntil = 0;
448
+ if (unheard.trim()) {
449
+ this.log("Sending text chunks again");
450
+ this.sendTranscript(unheard);
432
451
  }
433
- if (this.socket) closeSocket(this.socket);
434
- this.socket = void 0;
435
- this.isProcessing = false;
436
- }
437
- initWS() {
438
- return new Promise((resolve, reject) => {
439
- const params = new URLSearchParams();
440
- params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
441
- params.append(
442
- "output_format",
443
- this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
444
- );
445
- params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
446
- const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
447
- if (voiceSettings) {
448
- params.append("voice_settings", JSON.stringify(voiceSettings));
449
- }
450
- if (this.options.language) {
451
- params.append("language_code", this.options.language);
452
- }
453
- const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`;
454
- const socket = new WebSocket3(uri, {
455
- headers: {
456
- "xi-api-key": this.options.apiKey
457
- }
458
- });
459
- this.socket = socket;
460
- const timeout = setTimeout(() => {
461
- this.log("Connection timeout");
462
- closeSocket(socket);
463
- this.socket = void 0;
464
- reject(new Error("WebSocket connection timeout"));
465
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
466
- this.connectionTimeout = timeout;
467
- socket.addEventListener("open", () => {
468
- clearTimeout(timeout);
469
- this.log("Connection opened");
470
- this.socket?.send(
471
- JSON.stringify({
472
- text: " ",
473
- voice_settings: voiceSettings
474
- })
475
- );
476
- this.keepAliveInterval = setInterval(
477
- () => {
478
- if (this.socket?.readyState === WebSocket3.OPEN) {
479
- this.log("Sending keep-alive message");
480
- this.socket.send(JSON.stringify({ text: " " }));
481
- this.retryCount = 0;
482
- }
483
- },
484
- (WS_INACTIVITY_TIMEOUT - 1) * 1e3
485
- );
486
- resolve();
487
- });
488
- socket.addEventListener("error", (error) => {
489
- clearTimeout(timeout);
490
- this.log("WebSocket error:", error);
491
- reject(new Error("WebSocket connection error"));
492
- });
493
- socket.addEventListener("message", (event) => {
494
- if (this.canceled) return;
495
- try {
496
- this.onMessage(JSON.parse(event.data.toString()));
497
- } catch (error) {
498
- this.log("message" in error ? error.message : error);
499
- this.log("Event data during error:", event.data);
500
- }
501
- });
502
- socket.addEventListener("close", ({ code, reason }) => {
503
- clearTimeout(timeout);
504
- this.socket?.removeAllListeners();
505
- this.socket = void 0;
506
- this.isProcessing = false;
507
- if (this.keepAliveInterval) {
508
- clearInterval(this.keepAliveInterval);
509
- this.keepAliveInterval = void 0;
510
- }
511
- this.log("Connection closed", { code, reason });
512
- this.reconnect();
513
- });
514
- });
452
+ if (this.state === "flushed") this.flush();
453
+ }
454
+ /** Every retry failed: hands the unspoken text over and drops the utterance */
455
+ onFailed() {
456
+ const unspoken = this.state === "idle" ? "" : this.textSent.slice(this.heardUntil) + this.textBuffer;
457
+ this.speaking = 0;
458
+ this.resetUtterance();
459
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
460
+ }
461
+ resetUtterance() {
462
+ this.state = "idle";
463
+ this.textSent = "";
464
+ this.textBuffer = "";
465
+ this.heardUntil = 0;
515
466
  }
516
467
  onMessage(message) {
517
468
  if ("audio" in message && message.audio) {
518
469
  this.processAudioMessage(message);
519
470
  }
520
- if ("isFinal" in message && message.isFinal) {
471
+ if ("isFinal" in message && message.isFinal && this.state !== "idle") {
521
472
  this.log("Audio ended");
522
- this.isProcessing = false;
523
- this.textSent = "";
473
+ this.resetUtterance();
524
474
  }
525
475
  if ("error" in message) {
526
476
  console.error("[ElevenLabsTTS] Error:", message.error, message.message);
@@ -528,58 +478,59 @@ var SpeechWebSocket = class {
528
478
  }
529
479
  processAudioMessage(message) {
530
480
  const chunk = Buffer.from(message.audio, "base64");
531
- this.retryCount = 0;
481
+ this.connection.prove();
482
+ if (this.state === "idle") {
483
+ this.log("Ignore audio chunk, nothing is being spoken");
484
+ return;
485
+ }
532
486
  if (message.alignment) {
533
487
  const chunkText = message.alignment.chars.join("");
534
- const receivedAudioText = this.receivedAudioText + chunkText;
535
- if (this.textSent.trim().indexOf(receivedAudioText.trim()) === -1) {
536
- this.log(`Ignore audio chunk ("${chunkText}")`);
537
- return;
488
+ const end = findHeardEnd(this.textSent, this.heardUntil, chunkText);
489
+ if (end === -1) {
490
+ this.log(`Audio chunk off the text sent ("${chunkText}")`);
491
+ } else {
492
+ this.log(`Received audio chunk ("${chunkText}")`);
493
+ this.heardUntil = end;
538
494
  }
539
- this.log(`Received audio chunk ("${chunkText}")`);
540
- this.receivedAudioText = receivedAudioText;
541
- this.fixIsFinal(message);
542
495
  }
543
496
  this.host.audio(chunk);
497
+ this.checkEnded();
544
498
  }
545
- fixIsFinal(message) {
546
- if (message.isFinal || !this.textEnded) return;
547
- if (this.receivedAudioText.trim() === this.textSent.trim()) {
548
- message.isFinal = true;
549
- }
499
+ checkEnded() {
500
+ if (this.state !== "flushed") return;
501
+ if (this.textSent.slice(this.heardUntil).trim()) return;
502
+ this.log("Audio ended");
503
+ this.resetUtterance();
550
504
  }
551
505
  sendTranscript(text) {
552
- this.socket?.send(JSON.stringify({ text, try_trigger_generation: true }));
506
+ this.textSent += text;
507
+ if (!this.connection.send({ text, try_trigger_generation: true })) return;
553
508
  this.log(`Sent transcript: "${text}"`);
554
509
  }
555
- reconnect() {
556
- if (this.destroyed) return;
557
- this.retryCount++;
558
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
559
- this.log("Max retries reached, giving up");
560
- this.host.failed(this.textSent.trim() ? [this.textSent] : []);
561
- return;
562
- }
563
- this.initPromise = new Promise((resolve) => {
564
- this.log("Reconnecting...");
565
- this.reconnectTimeout = setTimeout(() => {
566
- this.reconnectTimeout = void 0;
567
- this.initWS().then(() => {
568
- if (this.textSent.length > 0) {
569
- this.log("Sending text chunks again");
570
- this.sendTranscript(this.textSent);
571
- }
572
- }).then(resolve).catch((error) => {
573
- this.log("Reconnection error:", error);
574
- this.reconnect();
575
- });
576
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
577
- });
578
- }
579
510
  log(...message) {
580
511
  this.host.log(...message);
581
512
  }
582
513
  };
514
+ function findHeardEnd(text, from, heard) {
515
+ for (let start = from; start < text.length; start++) {
516
+ const end = endOfPrefix(text, start, heard);
517
+ if (end !== -1) return end;
518
+ }
519
+ return -1;
520
+ }
521
+ function endOfPrefix(text, from, heard) {
522
+ let index = from;
523
+ for (let i = 0; i < heard.length; i++) {
524
+ if (isSpace(heard[i])) continue;
525
+ while (index < text.length && isSpace(text[index])) index++;
526
+ if (text[index] !== heard[i]) return -1;
527
+ index++;
528
+ }
529
+ return index;
530
+ }
531
+ function isSpace(char) {
532
+ return char === " " || char === " " || char === "\n" || char === "\r";
533
+ }
583
534
  function toWireVoiceSettings(settings) {
584
535
  if (!settings) return void 0;
585
536
  const { similarityBoost, useSpeakerBoost, ...rest } = settings;