@micdrop/elevenlabs 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,9 +1,7 @@
1
1
  "use strict";
2
- var __create = Object.create;
3
2
  var __defProp = Object.defineProperty;
4
3
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
4
  var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
5
  var __hasOwnProp = Object.prototype.hasOwnProperty;
8
6
  var __export = (target, all) => {
9
7
  for (var name in all)
@@ -17,14 +15,6 @@ var __copyProps = (to, from, except, desc) => {
17
15
  }
18
16
  return to;
19
17
  };
20
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
- // If the importer is in node compatibility mode or this is not an ESM
22
- // file that has been converted to a CommonJS file using a Babel-
23
- // compatible transform (i.e. "__esModule" has not been set), then set
24
- // "default" to the CommonJS "module.exports" for node compatibility.
25
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
- mod
27
- ));
28
18
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
19
 
30
20
  // src/index.ts
@@ -38,21 +28,96 @@ __export(index_exports, {
38
28
  module.exports = __toCommonJS(index_exports);
39
29
 
40
30
  // src/ElevenLabsTTS.ts
41
- var import_server = require("@micdrop/server");
42
-
43
- // src/DialogueWebSocket.ts
44
- var import_ws2 = __toESM(require("ws"));
31
+ var import_server2 = require("@micdrop/server");
45
32
 
46
33
  // src/ElevenLabsWebSocket.ts
47
- var import_ws = __toESM(require("ws"));
48
- function closeSocket(socket) {
49
- socket.removeAllListeners();
50
- socket.on("error", () => {
51
- });
52
- if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
53
- socket.close(1e3);
34
+ var import_server = require("@micdrop/server");
35
+ var DEFAULT_MAX_RETRY = 3;
36
+ var ElevenLabsConnection = class {
37
+ constructor(options) {
38
+ this.options = options;
39
+ // Whether the current connection carried audio or lived long enough for a
40
+ // keep-alive, which means the server accepted the key and the voice
41
+ this.proven = false;
42
+ this.failures = 0;
43
+ this.socket = this.createSocket();
54
44
  }
55
- }
45
+ /** Opens the connection, again after giving up, and does nothing otherwise */
46
+ connect() {
47
+ this.socket.connect();
48
+ }
49
+ /**
50
+ * Resolves to false when the connection gives up or is destroyed meanwhile,
51
+ * and right away once it gave up, until the next `connect()`: the stream
52
+ * that was speaking then has nothing left to wait for.
53
+ */
54
+ ready() {
55
+ if (this.socket.status === "idle") return Promise.resolve(false);
56
+ return this.socket.ready();
57
+ }
58
+ send(message) {
59
+ return this.socket.send(JSON.stringify(message));
60
+ }
61
+ /** Drops the connection, with the synthesis running on it, for a new one */
62
+ restart() {
63
+ this.socket.restart();
64
+ }
65
+ destroy() {
66
+ this.socket.destroy();
67
+ }
68
+ /** The current connection works, so its next drop is not a failure */
69
+ prove() {
70
+ this.proven = true;
71
+ this.failures = 0;
72
+ }
73
+ createSocket() {
74
+ const { tts, keepAlive, host } = this.options;
75
+ const socket = new import_server.ReconnectingSocket({
76
+ url: this.options.url,
77
+ clientOptions: { headers: { "xi-api-key": tts.apiKey } },
78
+ connectionTimeout: tts.connectionTimeout,
79
+ retryDelay: tts.retryDelay,
80
+ maxRetry: tts.maxRetry,
81
+ handshake: {
82
+ start: (ws) => ws.send(JSON.stringify(this.options.init))
83
+ },
84
+ keepAlive: {
85
+ interval: keepAlive.interval,
86
+ message: () => {
87
+ this.prove();
88
+ return JSON.stringify(keepAlive.message);
89
+ }
90
+ },
91
+ log: (...message) => host.log(...message)
92
+ });
93
+ socket.on("Open", ({ reconnected }) => {
94
+ this.proven = false;
95
+ this.options.onOpen(reconnected);
96
+ });
97
+ socket.on("Message", (data) => {
98
+ try {
99
+ this.options.onMessage(JSON.parse(data.toString()));
100
+ } catch (error) {
101
+ host.log("message" in error ? error.message : error);
102
+ host.log("Event data during error:", data.toString());
103
+ }
104
+ });
105
+ socket.on("Close", () => {
106
+ if (this.proven) return;
107
+ this.failures++;
108
+ if (this.failures > (tts.maxRetry ?? DEFAULT_MAX_RETRY)) this.giveUp();
109
+ });
110
+ socket.on("Failed", () => this.giveUp());
111
+ return socket;
112
+ }
113
+ giveUp() {
114
+ this.options.host.log("Max retries reached, giving up");
115
+ this.failures = 0;
116
+ this.socket.destroy();
117
+ this.socket = this.createSocket();
118
+ this.options.onFailed();
119
+ }
120
+ };
56
121
 
57
122
  // src/types.ts
58
123
  var DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
@@ -63,30 +128,23 @@ function isDialogueModel(modelId) {
63
128
 
64
129
  // src/DialogueWebSocket.ts
65
130
  var KEEP_ALIVE_INTERVAL = 15e3;
66
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
67
- var DEFAULT_RETRY_DELAY = 1e3;
68
- var DEFAULT_MAX_RETRY = 3;
69
131
  var SENTENCE_END = /(?:[!?]+|(?<!\.)\.(?!\.))\s+/g;
70
132
  var DialogueWebSocket = class {
71
133
  // Whether the connection received any text
72
134
  constructor(options, host) {
73
135
  this.options = options;
74
136
  this.host = host;
75
- this.retryCount = 0;
76
- this.destroyed = false;
137
+ this.state = "idle";
77
138
  // Bumped by every speak() and every cancel(), so a stream that starts late
78
139
  // can tell whether it is still the one that should be heard.
79
140
  this.generation = 0;
80
141
  // Generation of the stream being spoken. A speak() that never carries a
81
142
  // word leaves it in place, so the sentence it sends goes on to its end.
82
143
  this.speaking = 0;
83
- this.isProcessing = false;
84
- this.textEnded = false;
85
- // Whether the text stream has ended
86
144
  this.textSent = "";
87
145
  // Text of the utterance sent to ElevenLabs
88
146
  this.textBuffer = "";
89
- // End of the last chunk, held until its word is complete
147
+ // Text not sent yet, held until its word is complete
90
148
  this.unflushed = false;
91
149
  // Whether text was sent since the last flush
92
150
  this.pendingFlushes = 0;
@@ -94,7 +152,34 @@ var DialogueWebSocket = class {
94
152
  this.newTurn = false;
95
153
  // Whether the next input starts a new utterance
96
154
  this.hasSpoken = false;
97
- this.connect();
155
+ const params = new URLSearchParams();
156
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
157
+ params.append(
158
+ "output_format",
159
+ this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
160
+ );
161
+ if (this.options.language) {
162
+ params.append("language_code", this.options.language);
163
+ }
164
+ const stability = this.options.voiceSettings?.stability;
165
+ const init = {
166
+ voices: [this.options.voiceId],
167
+ voice_settings: stability === void 0 ? void 0 : { stability }
168
+ };
169
+ this.connection = new ElevenLabsConnection({
170
+ url: `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
171
+ tts: options,
172
+ init,
173
+ keepAlive: {
174
+ interval: KEEP_ALIVE_INTERVAL,
175
+ message: { keep_alive: true }
176
+ },
177
+ host,
178
+ onOpen: () => this.onOpen(),
179
+ onMessage: (message) => this.onMessage(message),
180
+ onFailed: () => this.onFailed()
181
+ });
182
+ this.connection.connect();
98
183
  }
99
184
  speak(textStream) {
100
185
  const generation = ++this.generation;
@@ -103,171 +188,100 @@ var DialogueWebSocket = class {
103
188
  if (started) return this.speaking === generation;
104
189
  if (this.generation !== generation) return false;
105
190
  started = true;
106
- if (this.isProcessing) this.restart();
191
+ if (this.state !== "idle") this.connection.restart();
192
+ else this.connection.connect();
193
+ this.resetUtterance();
107
194
  this.speaking = generation;
108
- this.isProcessing = true;
109
- this.textEnded = false;
110
- this.textSent = "";
111
- this.textBuffer = "";
112
- this.unflushed = false;
113
- this.pendingFlushes = 0;
195
+ this.state = "streaming";
114
196
  this.newTurn = true;
115
197
  return true;
116
198
  };
117
199
  textStream.on("data", async (chunk) => {
118
200
  if (!start()) return;
119
- const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
120
- await this.initPromise;
201
+ this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
202
+ if (!await this.connection.ready()) return;
121
203
  if (this.speaking !== generation) return;
122
- const spaceIndex = text.lastIndexOf(" ");
123
- if (spaceIndex === -1) {
124
- this.textBuffer += text;
125
- } else {
126
- this.sendText(this.textBuffer + text.slice(0, spaceIndex + 1));
127
- this.textBuffer = text.slice(spaceIndex + 1);
128
- }
204
+ const spaceIndex = this.textBuffer.lastIndexOf(" ");
205
+ if (spaceIndex === -1) return;
206
+ this.sendText(this.textBuffer.slice(0, spaceIndex + 1));
207
+ this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
129
208
  });
130
209
  textStream.on("error", (error) => {
131
210
  if (!started || this.speaking !== generation) return;
132
211
  this.log("Error in text stream, ending audio stream", error);
133
- this.isProcessing = false;
212
+ this.speaking = 0;
213
+ this.textBuffer = "";
214
+ this.endText();
134
215
  });
135
216
  textStream.on("end", async () => {
136
217
  if (!started) return;
137
- await this.initPromise;
218
+ if (!await this.connection.ready()) return;
138
219
  if (this.speaking !== generation) return;
139
220
  if (this.textBuffer.trim()) {
140
221
  this.sendText(this.textBuffer + " ");
141
- this.textBuffer = "";
142
222
  }
143
- this.textEnded = true;
144
- this.flush();
145
- this.checkEnded();
223
+ this.textBuffer = "";
224
+ this.endText();
146
225
  });
147
226
  }
148
227
  cancel() {
149
228
  this.generation++;
150
229
  this.speaking = 0;
151
- if (!this.isProcessing) return;
230
+ if (this.state === "idle") return;
152
231
  this.log("Cancel");
153
- this.isProcessing = false;
154
- this.textSent = "";
155
- this.textBuffer = "";
156
- this.restart();
232
+ this.resetUtterance();
233
+ this.connection.restart();
157
234
  }
158
235
  destroy() {
159
- this.destroyed = true;
160
- if (this.reconnectTimeout) {
161
- clearTimeout(this.reconnectTimeout);
162
- this.reconnectTimeout = void 0;
236
+ this.generation++;
237
+ this.speaking = 0;
238
+ this.resetUtterance();
239
+ this.connection.destroy();
240
+ }
241
+ onOpen() {
242
+ this.hasSpoken = false;
243
+ if (this.state === "idle") return;
244
+ const text = this.textSent;
245
+ this.textSent = "";
246
+ this.unflushed = false;
247
+ this.pendingFlushes = 0;
248
+ if (text.trim()) {
249
+ this.log("Sending text again");
250
+ this.sendText(text);
163
251
  }
164
- this.dropSocket();
165
- this.isProcessing = false;
252
+ if (this.state === "draining") this.flush();
253
+ this.checkEnded();
166
254
  }
167
- connect() {
168
- this.initPromise = this.initWS().catch((error) => {
169
- console.error("[ElevenLabsTTS] Connection error:", error);
170
- this.reconnect();
171
- });
255
+ /** Every retry failed: hands the unspoken text over and drops the utterance */
256
+ onFailed() {
257
+ const unspoken = this.state === "idle" ? "" : this.textSent + this.textBuffer;
258
+ this.speaking = 0;
259
+ this.resetUtterance();
260
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
172
261
  }
173
- /** Drops the connection, with the synthesis running on it, for a new one */
174
- restart() {
175
- if (this.reconnectTimeout) {
176
- clearTimeout(this.reconnectTimeout);
177
- this.reconnectTimeout = void 0;
178
- }
179
- this.dropSocket();
180
- this.connect();
181
- }
182
- /** Closes the connection without reconnecting */
183
- dropSocket() {
184
- if (this.connectionTimeout) {
185
- clearTimeout(this.connectionTimeout);
186
- this.connectionTimeout = void 0;
187
- }
188
- if (this.keepAliveInterval) {
189
- clearInterval(this.keepAliveInterval);
190
- this.keepAliveInterval = void 0;
191
- }
192
- if (this.socket) closeSocket(this.socket);
193
- this.socket = void 0;
194
- }
195
- initWS() {
196
- return new Promise((resolve, reject) => {
197
- const params = new URLSearchParams();
198
- params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
199
- params.append(
200
- "output_format",
201
- this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
202
- );
203
- if (this.options.language) {
204
- params.append("language_code", this.options.language);
205
- }
206
- const socket = new import_ws2.default(
207
- `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
208
- {
209
- headers: {
210
- "xi-api-key": this.options.apiKey
211
- }
212
- }
213
- );
214
- this.socket = socket;
215
- let opened = false;
216
- const timeout = setTimeout(() => {
217
- this.log("Connection timeout");
218
- closeSocket(socket);
219
- this.socket = void 0;
220
- reject(new Error("WebSocket connection timeout"));
221
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
222
- this.connectionTimeout = timeout;
223
- socket.addEventListener("open", () => {
224
- clearTimeout(timeout);
225
- opened = true;
226
- this.log("Connection opened");
227
- const stability = this.options.voiceSettings?.stability;
228
- this.send({
229
- voices: [this.options.voiceId],
230
- voice_settings: stability === void 0 ? void 0 : { stability }
231
- });
232
- this.hasSpoken = false;
233
- this.keepAliveInterval = setInterval(() => {
234
- this.send({ keep_alive: true });
235
- this.retryCount = 0;
236
- }, KEEP_ALIVE_INTERVAL);
237
- resolve();
238
- });
239
- socket.addEventListener("error", (error) => {
240
- clearTimeout(timeout);
241
- this.log("WebSocket error:", error);
242
- reject(new Error("WebSocket connection error"));
243
- });
244
- socket.addEventListener("message", (event) => {
245
- try {
246
- this.onMessage(JSON.parse(event.data.toString()));
247
- } catch (error) {
248
- this.log("message" in error ? error.message : error);
249
- this.log("Event data during error:", event.data);
250
- }
251
- });
252
- socket.addEventListener("close", ({ code, reason }) => {
253
- clearTimeout(timeout);
254
- if (this.keepAliveInterval) {
255
- clearInterval(this.keepAliveInterval);
256
- this.keepAliveInterval = void 0;
257
- }
258
- this.socket?.removeAllListeners();
259
- this.socket = void 0;
260
- this.log("Connection closed", { code, reason });
261
- if (opened) this.reconnect();
262
- });
263
- });
262
+ resetUtterance() {
263
+ this.state = "idle";
264
+ this.textSent = "";
265
+ this.textBuffer = "";
266
+ this.unflushed = false;
267
+ this.pendingFlushes = 0;
268
+ }
269
+ /** The text stream is over: flushes what the server holds */
270
+ endText() {
271
+ this.state = "draining";
272
+ this.flush();
273
+ this.checkEnded();
264
274
  }
265
275
  onMessage(message) {
266
276
  if ("audio" in message && message.audio) {
267
277
  const chunk = Buffer.from(message.audio, "base64");
268
- this.log(`Received audio chunk (${chunk.length} bytes)`);
269
- this.retryCount = 0;
270
- this.host.audio(chunk);
278
+ this.connection.prove();
279
+ if (this.state === "idle") {
280
+ this.log("Ignore audio chunk, nothing is being spoken");
281
+ } else {
282
+ this.log(`Received audio chunk (${chunk.length} bytes)`);
283
+ this.host.audio(chunk);
284
+ }
271
285
  }
272
286
  if ("is_final_audio_for_turn" in message && message.is_final_audio_for_turn) {
273
287
  this.pendingFlushes = Math.max(0, this.pendingFlushes - 1);
@@ -323,45 +337,15 @@ var DialogueWebSocket = class {
323
337
  }
324
338
  /** Ends the utterance once its text is over and all its audio came */
325
339
  checkEnded() {
326
- if (!this.isProcessing || !this.textEnded) return;
340
+ if (this.state !== "draining") return;
327
341
  if (this.unflushed || this.pendingFlushes > 0) return;
328
342
  this.log("Audio ended");
329
- this.isProcessing = false;
330
- this.textSent = "";
343
+ this.resetUtterance();
331
344
  }
345
+ // Lost when the connection just dropped, which is fine: the reconnection
346
+ // sends the utterance again from textSent
332
347
  send(message) {
333
- if (this.socket?.readyState !== import_ws2.default.OPEN) return;
334
- this.socket.send(JSON.stringify(message));
335
- }
336
- reconnect() {
337
- if (this.destroyed) return;
338
- this.retryCount++;
339
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
340
- this.log("Max retries reached, giving up");
341
- const unspoken = this.isProcessing ? this.textSent + this.textBuffer : "";
342
- this.host.failed(unspoken.trim() ? [unspoken] : []);
343
- return;
344
- }
345
- this.initPromise = new Promise((resolve) => {
346
- this.log("Reconnecting...");
347
- this.reconnectTimeout = setTimeout(() => {
348
- this.reconnectTimeout = void 0;
349
- this.initWS().then(() => {
350
- if (this.isProcessing && this.textSent.trim()) {
351
- this.log("Sending text again");
352
- const text = this.textSent;
353
- this.textSent = "";
354
- this.unflushed = false;
355
- this.pendingFlushes = 0;
356
- this.sendText(text);
357
- if (this.textEnded) this.flush();
358
- }
359
- }).then(resolve).catch((error) => {
360
- this.log("Reconnection error:", error);
361
- this.reconnect();
362
- });
363
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
364
- });
348
+ this.connection.send(message);
365
349
  }
366
350
  log(...message) {
367
351
  this.host.log(...message);
@@ -369,197 +353,153 @@ var DialogueWebSocket = class {
369
353
  };
370
354
 
371
355
  // src/SpeechWebSocket.ts
372
- var import_ws3 = __toESM(require("ws"));
373
356
  var WS_INACTIVITY_TIMEOUT = 180;
374
- var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
375
- var DEFAULT_RETRY_DELAY2 = 1e3;
376
- var DEFAULT_MAX_RETRY2 = 3;
377
357
  var SpeechWebSocket = class {
358
+ // Index in textSent where the audio received ends
378
359
  constructor(options, host) {
379
360
  this.options = options;
380
361
  this.host = host;
381
- this.isProcessing = false;
382
- this.textEnded = false;
383
- // Whether the text stream has ended
384
- this.textSent = "";
385
- // Text sent to ElevenLabs
386
- this.textBuffer = "";
387
- // Buffer of text to send to ElevenLabs
388
- this.receivedAudioText = "";
389
- this.retryCount = 0;
390
- this.destroyed = false;
391
- this.canceled = false;
362
+ this.state = "idle";
392
363
  // Bumped by every speak() and every cancel(), so a stream that starts late
393
364
  // can tell whether it is still the one that should be heard.
394
365
  this.generation = 0;
395
- this.initPromise = this.initWS().catch((error) => {
396
- console.error("[ElevenLabsTTS] Connection error:", error);
397
- this.reconnect();
366
+ // Generation of the stream being spoken. A speak() that never carries a
367
+ // word leaves it in place, so the sentence it sends goes on to its end.
368
+ this.speaking = 0;
369
+ this.textSent = "";
370
+ // Text of the utterance sent to ElevenLabs
371
+ this.textBuffer = "";
372
+ // Text not sent yet, held until its word is complete
373
+ this.heardUntil = 0;
374
+ const params = new URLSearchParams();
375
+ params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
376
+ params.append(
377
+ "output_format",
378
+ this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
379
+ );
380
+ params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
381
+ const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
382
+ if (voiceSettings) {
383
+ params.append("voice_settings", JSON.stringify(voiceSettings));
384
+ }
385
+ if (this.options.language) {
386
+ params.append("language_code", this.options.language);
387
+ }
388
+ this.connection = new ElevenLabsConnection({
389
+ url: `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`,
390
+ tts: options,
391
+ // Initialization message with voice settings
392
+ init: { text: " ", voice_settings: voiceSettings },
393
+ keepAlive: {
394
+ interval: (WS_INACTIVITY_TIMEOUT - 1) * 1e3,
395
+ message: { text: " " }
396
+ },
397
+ host,
398
+ onOpen: () => this.resume(),
399
+ onMessage: (message) => this.onMessage(message),
400
+ onFailed: () => this.onFailed()
398
401
  });
402
+ this.connection.connect();
399
403
  }
400
404
  speak(textStream) {
401
405
  const generation = ++this.generation;
402
406
  let started = false;
403
407
  const start = () => {
404
- if (started) return true;
408
+ if (started) return this.speaking === generation;
405
409
  if (this.generation !== generation) return false;
406
410
  started = true;
407
- this.canceled = false;
408
- this.isProcessing = true;
409
- this.textEnded = false;
410
- this.textBuffer = "";
411
- this.textSent = "";
412
- this.receivedAudioText = "";
411
+ if (this.state !== "idle") this.connection.restart();
412
+ else this.connection.connect();
413
+ this.resetUtterance();
414
+ this.speaking = generation;
415
+ this.state = "speaking";
413
416
  return true;
414
417
  };
415
418
  textStream.on("data", async (chunk) => {
416
419
  if (!start()) return;
417
- if (this.canceled) return;
418
- const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
419
- this.textSent += text;
420
- await this.initPromise;
421
- const spaceIndex = text.lastIndexOf(" ");
422
- if (spaceIndex === -1) {
423
- this.textBuffer += text;
424
- } else {
425
- this.sendTranscript(this.textBuffer + text.slice(0, spaceIndex + 1));
426
- this.textBuffer = text.slice(spaceIndex + 1);
427
- }
420
+ this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
421
+ if (!await this.connection.ready()) return;
422
+ if (this.speaking !== generation) return;
423
+ const spaceIndex = this.textBuffer.lastIndexOf(" ");
424
+ if (spaceIndex === -1) return;
425
+ this.sendTranscript(this.textBuffer.slice(0, spaceIndex + 1));
426
+ this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
428
427
  });
429
428
  textStream.on("error", (error) => {
430
- if (!started) return;
429
+ if (!started || this.speaking !== generation) return;
431
430
  this.log("Error in text stream, ending audio stream", error);
432
- this.isProcessing = false;
431
+ this.speaking = 0;
432
+ this.textBuffer = "";
433
+ this.flush();
433
434
  });
434
435
  textStream.on("end", async () => {
435
- if (!started || this.canceled) return;
436
- await this.initPromise;
436
+ if (!started) return;
437
+ if (!await this.connection.ready()) return;
438
+ if (this.speaking !== generation) return;
437
439
  if (this.textBuffer.trim()) {
438
440
  this.sendTranscript(this.textBuffer + " ");
439
- this.textBuffer = "";
440
441
  }
441
- this.textEnded = true;
442
- this.socket?.send(JSON.stringify({ text: " ", flush: true }));
443
- this.log("Flushed text");
442
+ this.textBuffer = "";
443
+ this.flush();
444
444
  });
445
445
  }
446
446
  cancel() {
447
447
  this.generation++;
448
- if (!this.isProcessing) return;
448
+ this.speaking = 0;
449
+ if (this.state === "idle") return;
449
450
  this.log("Cancel");
450
- this.canceled = true;
451
- this.textSent = "";
452
- this.receivedAudioText = "";
453
- this.isProcessing = false;
454
- if (this.socket?.readyState === import_ws3.default.OPEN) {
455
- this.socket.send(JSON.stringify({ text: " ", flush: true }));
456
- }
451
+ this.resetUtterance();
452
+ this.connection.restart();
457
453
  }
458
454
  destroy() {
459
- this.destroyed = true;
460
- if (this.reconnectTimeout) {
461
- clearTimeout(this.reconnectTimeout);
462
- this.reconnectTimeout = void 0;
463
- }
464
- if (this.connectionTimeout) {
465
- clearTimeout(this.connectionTimeout);
466
- this.connectionTimeout = void 0;
467
- }
468
- if (this.keepAliveInterval) {
469
- clearInterval(this.keepAliveInterval);
470
- this.keepAliveInterval = void 0;
455
+ this.generation++;
456
+ this.speaking = 0;
457
+ this.resetUtterance();
458
+ this.connection.destroy();
459
+ }
460
+ /** Flushes buffered text and marks the end of the utterance */
461
+ flush() {
462
+ this.state = "flushed";
463
+ this.connection.send({ text: " ", flush: true });
464
+ this.log("Flushed text");
465
+ this.checkEnded();
466
+ }
467
+ /**
468
+ * Sends the text of the utterance in flight again on a new connection. The
469
+ * server lost what it was generating, so it goes on from the audio received.
470
+ */
471
+ resume() {
472
+ if (this.state === "idle") return;
473
+ const wordStart = this.textSent.lastIndexOf(" ", this.heardUntil - 1) + 1;
474
+ const unheard = this.textSent.slice(wordStart);
475
+ this.textSent = "";
476
+ this.heardUntil = 0;
477
+ if (unheard.trim()) {
478
+ this.log("Sending text chunks again");
479
+ this.sendTranscript(unheard);
471
480
  }
472
- if (this.socket) closeSocket(this.socket);
473
- this.socket = void 0;
474
- this.isProcessing = false;
475
- }
476
- initWS() {
477
- return new Promise((resolve, reject) => {
478
- const params = new URLSearchParams();
479
- params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
480
- params.append(
481
- "output_format",
482
- this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
483
- );
484
- params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
485
- const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
486
- if (voiceSettings) {
487
- params.append("voice_settings", JSON.stringify(voiceSettings));
488
- }
489
- if (this.options.language) {
490
- params.append("language_code", this.options.language);
491
- }
492
- const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`;
493
- const socket = new import_ws3.default(uri, {
494
- headers: {
495
- "xi-api-key": this.options.apiKey
496
- }
497
- });
498
- this.socket = socket;
499
- const timeout = setTimeout(() => {
500
- this.log("Connection timeout");
501
- closeSocket(socket);
502
- this.socket = void 0;
503
- reject(new Error("WebSocket connection timeout"));
504
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
505
- this.connectionTimeout = timeout;
506
- socket.addEventListener("open", () => {
507
- clearTimeout(timeout);
508
- this.log("Connection opened");
509
- this.socket?.send(
510
- JSON.stringify({
511
- text: " ",
512
- voice_settings: voiceSettings
513
- })
514
- );
515
- this.keepAliveInterval = setInterval(
516
- () => {
517
- if (this.socket?.readyState === import_ws3.default.OPEN) {
518
- this.log("Sending keep-alive message");
519
- this.socket.send(JSON.stringify({ text: " " }));
520
- this.retryCount = 0;
521
- }
522
- },
523
- (WS_INACTIVITY_TIMEOUT - 1) * 1e3
524
- );
525
- resolve();
526
- });
527
- socket.addEventListener("error", (error) => {
528
- clearTimeout(timeout);
529
- this.log("WebSocket error:", error);
530
- reject(new Error("WebSocket connection error"));
531
- });
532
- socket.addEventListener("message", (event) => {
533
- if (this.canceled) return;
534
- try {
535
- this.onMessage(JSON.parse(event.data.toString()));
536
- } catch (error) {
537
- this.log("message" in error ? error.message : error);
538
- this.log("Event data during error:", event.data);
539
- }
540
- });
541
- socket.addEventListener("close", ({ code, reason }) => {
542
- clearTimeout(timeout);
543
- this.socket?.removeAllListeners();
544
- this.socket = void 0;
545
- this.isProcessing = false;
546
- if (this.keepAliveInterval) {
547
- clearInterval(this.keepAliveInterval);
548
- this.keepAliveInterval = void 0;
549
- }
550
- this.log("Connection closed", { code, reason });
551
- this.reconnect();
552
- });
553
- });
481
+ if (this.state === "flushed") this.flush();
482
+ }
483
+ /** Every retry failed: hands the unspoken text over and drops the utterance */
484
+ onFailed() {
485
+ const unspoken = this.state === "idle" ? "" : this.textSent.slice(this.heardUntil) + this.textBuffer;
486
+ this.speaking = 0;
487
+ this.resetUtterance();
488
+ this.host.failed(unspoken.trim() ? [unspoken] : []);
489
+ }
490
+ resetUtterance() {
491
+ this.state = "idle";
492
+ this.textSent = "";
493
+ this.textBuffer = "";
494
+ this.heardUntil = 0;
554
495
  }
555
496
  onMessage(message) {
556
497
  if ("audio" in message && message.audio) {
557
498
  this.processAudioMessage(message);
558
499
  }
559
- if ("isFinal" in message && message.isFinal) {
500
+ if ("isFinal" in message && message.isFinal && this.state !== "idle") {
560
501
  this.log("Audio ended");
561
- this.isProcessing = false;
562
- this.textSent = "";
502
+ this.resetUtterance();
563
503
  }
564
504
  if ("error" in message) {
565
505
  console.error("[ElevenLabsTTS] Error:", message.error, message.message);
@@ -567,58 +507,59 @@ var SpeechWebSocket = class {
567
507
  }
568
508
  processAudioMessage(message) {
569
509
  const chunk = Buffer.from(message.audio, "base64");
570
- this.retryCount = 0;
510
+ this.connection.prove();
511
+ if (this.state === "idle") {
512
+ this.log("Ignore audio chunk, nothing is being spoken");
513
+ return;
514
+ }
571
515
  if (message.alignment) {
572
516
  const chunkText = message.alignment.chars.join("");
573
- const receivedAudioText = this.receivedAudioText + chunkText;
574
- if (this.textSent.trim().indexOf(receivedAudioText.trim()) === -1) {
575
- this.log(`Ignore audio chunk ("${chunkText}")`);
576
- return;
517
+ const end = findHeardEnd(this.textSent, this.heardUntil, chunkText);
518
+ if (end === -1) {
519
+ this.log(`Audio chunk off the text sent ("${chunkText}")`);
520
+ } else {
521
+ this.log(`Received audio chunk ("${chunkText}")`);
522
+ this.heardUntil = end;
577
523
  }
578
- this.log(`Received audio chunk ("${chunkText}")`);
579
- this.receivedAudioText = receivedAudioText;
580
- this.fixIsFinal(message);
581
524
  }
582
525
  this.host.audio(chunk);
526
+ this.checkEnded();
583
527
  }
584
- fixIsFinal(message) {
585
- if (message.isFinal || !this.textEnded) return;
586
- if (this.receivedAudioText.trim() === this.textSent.trim()) {
587
- message.isFinal = true;
588
- }
528
+ checkEnded() {
529
+ if (this.state !== "flushed") return;
530
+ if (this.textSent.slice(this.heardUntil).trim()) return;
531
+ this.log("Audio ended");
532
+ this.resetUtterance();
589
533
  }
590
534
  sendTranscript(text) {
591
- this.socket?.send(JSON.stringify({ text, try_trigger_generation: true }));
535
+ this.textSent += text;
536
+ if (!this.connection.send({ text, try_trigger_generation: true })) return;
592
537
  this.log(`Sent transcript: "${text}"`);
593
538
  }
594
- reconnect() {
595
- if (this.destroyed) return;
596
- this.retryCount++;
597
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
598
- this.log("Max retries reached, giving up");
599
- this.host.failed(this.textSent.trim() ? [this.textSent] : []);
600
- return;
601
- }
602
- this.initPromise = new Promise((resolve) => {
603
- this.log("Reconnecting...");
604
- this.reconnectTimeout = setTimeout(() => {
605
- this.reconnectTimeout = void 0;
606
- this.initWS().then(() => {
607
- if (this.textSent.length > 0) {
608
- this.log("Sending text chunks again");
609
- this.sendTranscript(this.textSent);
610
- }
611
- }).then(resolve).catch((error) => {
612
- this.log("Reconnection error:", error);
613
- this.reconnect();
614
- });
615
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
616
- });
617
- }
618
539
  log(...message) {
619
540
  this.host.log(...message);
620
541
  }
621
542
  };
543
+ function findHeardEnd(text, from, heard) {
544
+ for (let start = from; start < text.length; start++) {
545
+ const end = endOfPrefix(text, start, heard);
546
+ if (end !== -1) return end;
547
+ }
548
+ return -1;
549
+ }
550
+ function endOfPrefix(text, from, heard) {
551
+ let index = from;
552
+ for (let i = 0; i < heard.length; i++) {
553
+ if (isSpace(heard[i])) continue;
554
+ while (index < text.length && isSpace(text[index])) index++;
555
+ if (text[index] !== heard[i]) return -1;
556
+ index++;
557
+ }
558
+ return index;
559
+ }
560
+ function isSpace(char) {
561
+ return char === " " || char === " " || char === "\n" || char === "\r";
562
+ }
622
563
  function toWireVoiceSettings(settings) {
623
564
  if (!settings) return void 0;
624
565
  const { similarityBoost, useSpeakerBoost, ...rest } = settings;
@@ -630,7 +571,7 @@ function toWireVoiceSettings(settings) {
630
571
  }
631
572
 
632
573
  // src/ElevenLabsTTS.ts
633
- var ElevenLabsTTS = class extends import_server.TTS {
574
+ var ElevenLabsTTS = class extends import_server2.TTS {
634
575
  constructor(options) {
635
576
  super();
636
577
  const host = {