@micdrop/elevenlabs 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +328 -387
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +326 -375
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1,19 +1,94 @@
|
|
|
1
1
|
// src/ElevenLabsTTS.ts
|
|
2
2
|
import { TTS } from "@micdrop/server";
|
|
3
3
|
|
|
4
|
-
// src/DialogueWebSocket.ts
|
|
5
|
-
import WebSocket2 from "ws";
|
|
6
|
-
|
|
7
4
|
// src/ElevenLabsWebSocket.ts
|
|
8
|
-
import
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
5
|
+
import { ReconnectingSocket } from "@micdrop/server";
|
|
6
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
7
|
+
var ElevenLabsConnection = class {
|
|
8
|
+
constructor(options) {
|
|
9
|
+
this.options = options;
|
|
10
|
+
// Whether the current connection carried audio or lived long enough for a
|
|
11
|
+
// keep-alive, which means the server accepted the key and the voice
|
|
12
|
+
this.proven = false;
|
|
13
|
+
this.failures = 0;
|
|
14
|
+
this.socket = this.createSocket();
|
|
15
15
|
}
|
|
16
|
-
|
|
16
|
+
/** Opens the connection, again after giving up, and does nothing otherwise */
|
|
17
|
+
connect() {
|
|
18
|
+
this.socket.connect();
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Resolves to false when the connection gives up or is destroyed meanwhile,
|
|
22
|
+
* and right away once it gave up, until the next `connect()`: the stream
|
|
23
|
+
* that was speaking then has nothing left to wait for.
|
|
24
|
+
*/
|
|
25
|
+
ready() {
|
|
26
|
+
if (this.socket.status === "idle") return Promise.resolve(false);
|
|
27
|
+
return this.socket.ready();
|
|
28
|
+
}
|
|
29
|
+
send(message) {
|
|
30
|
+
return this.socket.send(JSON.stringify(message));
|
|
31
|
+
}
|
|
32
|
+
/** Drops the connection, with the synthesis running on it, for a new one */
|
|
33
|
+
restart() {
|
|
34
|
+
this.socket.restart();
|
|
35
|
+
}
|
|
36
|
+
destroy() {
|
|
37
|
+
this.socket.destroy();
|
|
38
|
+
}
|
|
39
|
+
/** The current connection works, so its next drop is not a failure */
|
|
40
|
+
prove() {
|
|
41
|
+
this.proven = true;
|
|
42
|
+
this.failures = 0;
|
|
43
|
+
}
|
|
44
|
+
createSocket() {
|
|
45
|
+
const { tts, keepAlive, host } = this.options;
|
|
46
|
+
const socket = new ReconnectingSocket({
|
|
47
|
+
url: this.options.url,
|
|
48
|
+
clientOptions: { headers: { "xi-api-key": tts.apiKey } },
|
|
49
|
+
connectionTimeout: tts.connectionTimeout,
|
|
50
|
+
retryDelay: tts.retryDelay,
|
|
51
|
+
maxRetry: tts.maxRetry,
|
|
52
|
+
handshake: {
|
|
53
|
+
start: (ws) => ws.send(JSON.stringify(this.options.init))
|
|
54
|
+
},
|
|
55
|
+
keepAlive: {
|
|
56
|
+
interval: keepAlive.interval,
|
|
57
|
+
message: () => {
|
|
58
|
+
this.prove();
|
|
59
|
+
return JSON.stringify(keepAlive.message);
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
log: (...message) => host.log(...message)
|
|
63
|
+
});
|
|
64
|
+
socket.on("Open", ({ reconnected }) => {
|
|
65
|
+
this.proven = false;
|
|
66
|
+
this.options.onOpen(reconnected);
|
|
67
|
+
});
|
|
68
|
+
socket.on("Message", (data) => {
|
|
69
|
+
try {
|
|
70
|
+
this.options.onMessage(JSON.parse(data.toString()));
|
|
71
|
+
} catch (error) {
|
|
72
|
+
host.log("message" in error ? error.message : error);
|
|
73
|
+
host.log("Event data during error:", data.toString());
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
socket.on("Close", () => {
|
|
77
|
+
if (this.proven) return;
|
|
78
|
+
this.failures++;
|
|
79
|
+
if (this.failures > (tts.maxRetry ?? DEFAULT_MAX_RETRY)) this.giveUp();
|
|
80
|
+
});
|
|
81
|
+
socket.on("Failed", () => this.giveUp());
|
|
82
|
+
return socket;
|
|
83
|
+
}
|
|
84
|
+
giveUp() {
|
|
85
|
+
this.options.host.log("Max retries reached, giving up");
|
|
86
|
+
this.failures = 0;
|
|
87
|
+
this.socket.destroy();
|
|
88
|
+
this.socket = this.createSocket();
|
|
89
|
+
this.options.onFailed();
|
|
90
|
+
}
|
|
91
|
+
};
|
|
17
92
|
|
|
18
93
|
// src/types.ts
|
|
19
94
|
var DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
|
|
@@ -24,30 +99,23 @@ function isDialogueModel(modelId) {
|
|
|
24
99
|
|
|
25
100
|
// src/DialogueWebSocket.ts
|
|
26
101
|
var KEEP_ALIVE_INTERVAL = 15e3;
|
|
27
|
-
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
28
|
-
var DEFAULT_RETRY_DELAY = 1e3;
|
|
29
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
30
102
|
var SENTENCE_END = /(?:[!?]+|(?<!\.)\.(?!\.))\s+/g;
|
|
31
103
|
var DialogueWebSocket = class {
|
|
32
104
|
// Whether the connection received any text
|
|
33
105
|
constructor(options, host) {
|
|
34
106
|
this.options = options;
|
|
35
107
|
this.host = host;
|
|
36
|
-
this.
|
|
37
|
-
this.destroyed = false;
|
|
108
|
+
this.state = "idle";
|
|
38
109
|
// Bumped by every speak() and every cancel(), so a stream that starts late
|
|
39
110
|
// can tell whether it is still the one that should be heard.
|
|
40
111
|
this.generation = 0;
|
|
41
112
|
// Generation of the stream being spoken. A speak() that never carries a
|
|
42
113
|
// word leaves it in place, so the sentence it sends goes on to its end.
|
|
43
114
|
this.speaking = 0;
|
|
44
|
-
this.isProcessing = false;
|
|
45
|
-
this.textEnded = false;
|
|
46
|
-
// Whether the text stream has ended
|
|
47
115
|
this.textSent = "";
|
|
48
116
|
// Text of the utterance sent to ElevenLabs
|
|
49
117
|
this.textBuffer = "";
|
|
50
|
-
//
|
|
118
|
+
// Text not sent yet, held until its word is complete
|
|
51
119
|
this.unflushed = false;
|
|
52
120
|
// Whether text was sent since the last flush
|
|
53
121
|
this.pendingFlushes = 0;
|
|
@@ -55,7 +123,34 @@ var DialogueWebSocket = class {
|
|
|
55
123
|
this.newTurn = false;
|
|
56
124
|
// Whether the next input starts a new utterance
|
|
57
125
|
this.hasSpoken = false;
|
|
58
|
-
|
|
126
|
+
const params = new URLSearchParams();
|
|
127
|
+
params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
|
|
128
|
+
params.append(
|
|
129
|
+
"output_format",
|
|
130
|
+
this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
|
|
131
|
+
);
|
|
132
|
+
if (this.options.language) {
|
|
133
|
+
params.append("language_code", this.options.language);
|
|
134
|
+
}
|
|
135
|
+
const stability = this.options.voiceSettings?.stability;
|
|
136
|
+
const init = {
|
|
137
|
+
voices: [this.options.voiceId],
|
|
138
|
+
voice_settings: stability === void 0 ? void 0 : { stability }
|
|
139
|
+
};
|
|
140
|
+
this.connection = new ElevenLabsConnection({
|
|
141
|
+
url: `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
|
|
142
|
+
tts: options,
|
|
143
|
+
init,
|
|
144
|
+
keepAlive: {
|
|
145
|
+
interval: KEEP_ALIVE_INTERVAL,
|
|
146
|
+
message: { keep_alive: true }
|
|
147
|
+
},
|
|
148
|
+
host,
|
|
149
|
+
onOpen: () => this.onOpen(),
|
|
150
|
+
onMessage: (message) => this.onMessage(message),
|
|
151
|
+
onFailed: () => this.onFailed()
|
|
152
|
+
});
|
|
153
|
+
this.connection.connect();
|
|
59
154
|
}
|
|
60
155
|
speak(textStream) {
|
|
61
156
|
const generation = ++this.generation;
|
|
@@ -64,171 +159,100 @@ var DialogueWebSocket = class {
|
|
|
64
159
|
if (started) return this.speaking === generation;
|
|
65
160
|
if (this.generation !== generation) return false;
|
|
66
161
|
started = true;
|
|
67
|
-
if (this.
|
|
162
|
+
if (this.state !== "idle") this.connection.restart();
|
|
163
|
+
else this.connection.connect();
|
|
164
|
+
this.resetUtterance();
|
|
68
165
|
this.speaking = generation;
|
|
69
|
-
this.
|
|
70
|
-
this.textEnded = false;
|
|
71
|
-
this.textSent = "";
|
|
72
|
-
this.textBuffer = "";
|
|
73
|
-
this.unflushed = false;
|
|
74
|
-
this.pendingFlushes = 0;
|
|
166
|
+
this.state = "streaming";
|
|
75
167
|
this.newTurn = true;
|
|
76
168
|
return true;
|
|
77
169
|
};
|
|
78
170
|
textStream.on("data", async (chunk) => {
|
|
79
171
|
if (!start()) return;
|
|
80
|
-
|
|
81
|
-
await this.
|
|
172
|
+
this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
|
|
173
|
+
if (!await this.connection.ready()) return;
|
|
82
174
|
if (this.speaking !== generation) return;
|
|
83
|
-
const spaceIndex =
|
|
84
|
-
if (spaceIndex === -1)
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
this.sendText(this.textBuffer + text.slice(0, spaceIndex + 1));
|
|
88
|
-
this.textBuffer = text.slice(spaceIndex + 1);
|
|
89
|
-
}
|
|
175
|
+
const spaceIndex = this.textBuffer.lastIndexOf(" ");
|
|
176
|
+
if (spaceIndex === -1) return;
|
|
177
|
+
this.sendText(this.textBuffer.slice(0, spaceIndex + 1));
|
|
178
|
+
this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
|
|
90
179
|
});
|
|
91
180
|
textStream.on("error", (error) => {
|
|
92
181
|
if (!started || this.speaking !== generation) return;
|
|
93
182
|
this.log("Error in text stream, ending audio stream", error);
|
|
94
|
-
this.
|
|
183
|
+
this.speaking = 0;
|
|
184
|
+
this.textBuffer = "";
|
|
185
|
+
this.endText();
|
|
95
186
|
});
|
|
96
187
|
textStream.on("end", async () => {
|
|
97
188
|
if (!started) return;
|
|
98
|
-
await this.
|
|
189
|
+
if (!await this.connection.ready()) return;
|
|
99
190
|
if (this.speaking !== generation) return;
|
|
100
191
|
if (this.textBuffer.trim()) {
|
|
101
192
|
this.sendText(this.textBuffer + " ");
|
|
102
|
-
this.textBuffer = "";
|
|
103
193
|
}
|
|
104
|
-
this.
|
|
105
|
-
this.
|
|
106
|
-
this.checkEnded();
|
|
194
|
+
this.textBuffer = "";
|
|
195
|
+
this.endText();
|
|
107
196
|
});
|
|
108
197
|
}
|
|
109
198
|
cancel() {
|
|
110
199
|
this.generation++;
|
|
111
200
|
this.speaking = 0;
|
|
112
|
-
if (
|
|
201
|
+
if (this.state === "idle") return;
|
|
113
202
|
this.log("Cancel");
|
|
114
|
-
this.
|
|
115
|
-
this.
|
|
116
|
-
this.textBuffer = "";
|
|
117
|
-
this.restart();
|
|
203
|
+
this.resetUtterance();
|
|
204
|
+
this.connection.restart();
|
|
118
205
|
}
|
|
119
206
|
destroy() {
|
|
120
|
-
this.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
207
|
+
this.generation++;
|
|
208
|
+
this.speaking = 0;
|
|
209
|
+
this.resetUtterance();
|
|
210
|
+
this.connection.destroy();
|
|
211
|
+
}
|
|
212
|
+
onOpen() {
|
|
213
|
+
this.hasSpoken = false;
|
|
214
|
+
if (this.state === "idle") return;
|
|
215
|
+
const text = this.textSent;
|
|
216
|
+
this.textSent = "";
|
|
217
|
+
this.unflushed = false;
|
|
218
|
+
this.pendingFlushes = 0;
|
|
219
|
+
if (text.trim()) {
|
|
220
|
+
this.log("Sending text again");
|
|
221
|
+
this.sendText(text);
|
|
124
222
|
}
|
|
125
|
-
this.
|
|
126
|
-
this.
|
|
223
|
+
if (this.state === "draining") this.flush();
|
|
224
|
+
this.checkEnded();
|
|
127
225
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
226
|
+
/** Every retry failed: hands the unspoken text over and drops the utterance */
|
|
227
|
+
onFailed() {
|
|
228
|
+
const unspoken = this.state === "idle" ? "" : this.textSent + this.textBuffer;
|
|
229
|
+
this.speaking = 0;
|
|
230
|
+
this.resetUtterance();
|
|
231
|
+
this.host.failed(unspoken.trim() ? [unspoken] : []);
|
|
133
232
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
clearTimeout(this.connectionTimeout);
|
|
147
|
-
this.connectionTimeout = void 0;
|
|
148
|
-
}
|
|
149
|
-
if (this.keepAliveInterval) {
|
|
150
|
-
clearInterval(this.keepAliveInterval);
|
|
151
|
-
this.keepAliveInterval = void 0;
|
|
152
|
-
}
|
|
153
|
-
if (this.socket) closeSocket(this.socket);
|
|
154
|
-
this.socket = void 0;
|
|
155
|
-
}
|
|
156
|
-
initWS() {
|
|
157
|
-
return new Promise((resolve, reject) => {
|
|
158
|
-
const params = new URLSearchParams();
|
|
159
|
-
params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
|
|
160
|
-
params.append(
|
|
161
|
-
"output_format",
|
|
162
|
-
this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
|
|
163
|
-
);
|
|
164
|
-
if (this.options.language) {
|
|
165
|
-
params.append("language_code", this.options.language);
|
|
166
|
-
}
|
|
167
|
-
const socket = new WebSocket2(
|
|
168
|
-
`wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
|
|
169
|
-
{
|
|
170
|
-
headers: {
|
|
171
|
-
"xi-api-key": this.options.apiKey
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
);
|
|
175
|
-
this.socket = socket;
|
|
176
|
-
let opened = false;
|
|
177
|
-
const timeout = setTimeout(() => {
|
|
178
|
-
this.log("Connection timeout");
|
|
179
|
-
closeSocket(socket);
|
|
180
|
-
this.socket = void 0;
|
|
181
|
-
reject(new Error("WebSocket connection timeout"));
|
|
182
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
183
|
-
this.connectionTimeout = timeout;
|
|
184
|
-
socket.addEventListener("open", () => {
|
|
185
|
-
clearTimeout(timeout);
|
|
186
|
-
opened = true;
|
|
187
|
-
this.log("Connection opened");
|
|
188
|
-
const stability = this.options.voiceSettings?.stability;
|
|
189
|
-
this.send({
|
|
190
|
-
voices: [this.options.voiceId],
|
|
191
|
-
voice_settings: stability === void 0 ? void 0 : { stability }
|
|
192
|
-
});
|
|
193
|
-
this.hasSpoken = false;
|
|
194
|
-
this.keepAliveInterval = setInterval(() => {
|
|
195
|
-
this.send({ keep_alive: true });
|
|
196
|
-
this.retryCount = 0;
|
|
197
|
-
}, KEEP_ALIVE_INTERVAL);
|
|
198
|
-
resolve();
|
|
199
|
-
});
|
|
200
|
-
socket.addEventListener("error", (error) => {
|
|
201
|
-
clearTimeout(timeout);
|
|
202
|
-
this.log("WebSocket error:", error);
|
|
203
|
-
reject(new Error("WebSocket connection error"));
|
|
204
|
-
});
|
|
205
|
-
socket.addEventListener("message", (event) => {
|
|
206
|
-
try {
|
|
207
|
-
this.onMessage(JSON.parse(event.data.toString()));
|
|
208
|
-
} catch (error) {
|
|
209
|
-
this.log("message" in error ? error.message : error);
|
|
210
|
-
this.log("Event data during error:", event.data);
|
|
211
|
-
}
|
|
212
|
-
});
|
|
213
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
214
|
-
clearTimeout(timeout);
|
|
215
|
-
if (this.keepAliveInterval) {
|
|
216
|
-
clearInterval(this.keepAliveInterval);
|
|
217
|
-
this.keepAliveInterval = void 0;
|
|
218
|
-
}
|
|
219
|
-
this.socket?.removeAllListeners();
|
|
220
|
-
this.socket = void 0;
|
|
221
|
-
this.log("Connection closed", { code, reason });
|
|
222
|
-
if (opened) this.reconnect();
|
|
223
|
-
});
|
|
224
|
-
});
|
|
233
|
+
resetUtterance() {
|
|
234
|
+
this.state = "idle";
|
|
235
|
+
this.textSent = "";
|
|
236
|
+
this.textBuffer = "";
|
|
237
|
+
this.unflushed = false;
|
|
238
|
+
this.pendingFlushes = 0;
|
|
239
|
+
}
|
|
240
|
+
/** The text stream is over: flushes what the server holds */
|
|
241
|
+
endText() {
|
|
242
|
+
this.state = "draining";
|
|
243
|
+
this.flush();
|
|
244
|
+
this.checkEnded();
|
|
225
245
|
}
|
|
226
246
|
onMessage(message) {
|
|
227
247
|
if ("audio" in message && message.audio) {
|
|
228
248
|
const chunk = Buffer.from(message.audio, "base64");
|
|
229
|
-
this.
|
|
230
|
-
this.
|
|
231
|
-
|
|
249
|
+
this.connection.prove();
|
|
250
|
+
if (this.state === "idle") {
|
|
251
|
+
this.log("Ignore audio chunk, nothing is being spoken");
|
|
252
|
+
} else {
|
|
253
|
+
this.log(`Received audio chunk (${chunk.length} bytes)`);
|
|
254
|
+
this.host.audio(chunk);
|
|
255
|
+
}
|
|
232
256
|
}
|
|
233
257
|
if ("is_final_audio_for_turn" in message && message.is_final_audio_for_turn) {
|
|
234
258
|
this.pendingFlushes = Math.max(0, this.pendingFlushes - 1);
|
|
@@ -284,45 +308,15 @@ var DialogueWebSocket = class {
|
|
|
284
308
|
}
|
|
285
309
|
/** Ends the utterance once its text is over and all its audio came */
|
|
286
310
|
checkEnded() {
|
|
287
|
-
if (
|
|
311
|
+
if (this.state !== "draining") return;
|
|
288
312
|
if (this.unflushed || this.pendingFlushes > 0) return;
|
|
289
313
|
this.log("Audio ended");
|
|
290
|
-
this.
|
|
291
|
-
this.textSent = "";
|
|
314
|
+
this.resetUtterance();
|
|
292
315
|
}
|
|
316
|
+
// Lost when the connection just dropped, which is fine: the reconnection
|
|
317
|
+
// sends the utterance again from textSent
|
|
293
318
|
send(message) {
|
|
294
|
-
|
|
295
|
-
this.socket.send(JSON.stringify(message));
|
|
296
|
-
}
|
|
297
|
-
reconnect() {
|
|
298
|
-
if (this.destroyed) return;
|
|
299
|
-
this.retryCount++;
|
|
300
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
301
|
-
this.log("Max retries reached, giving up");
|
|
302
|
-
const unspoken = this.isProcessing ? this.textSent + this.textBuffer : "";
|
|
303
|
-
this.host.failed(unspoken.trim() ? [unspoken] : []);
|
|
304
|
-
return;
|
|
305
|
-
}
|
|
306
|
-
this.initPromise = new Promise((resolve) => {
|
|
307
|
-
this.log("Reconnecting...");
|
|
308
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
309
|
-
this.reconnectTimeout = void 0;
|
|
310
|
-
this.initWS().then(() => {
|
|
311
|
-
if (this.isProcessing && this.textSent.trim()) {
|
|
312
|
-
this.log("Sending text again");
|
|
313
|
-
const text = this.textSent;
|
|
314
|
-
this.textSent = "";
|
|
315
|
-
this.unflushed = false;
|
|
316
|
-
this.pendingFlushes = 0;
|
|
317
|
-
this.sendText(text);
|
|
318
|
-
if (this.textEnded) this.flush();
|
|
319
|
-
}
|
|
320
|
-
}).then(resolve).catch((error) => {
|
|
321
|
-
this.log("Reconnection error:", error);
|
|
322
|
-
this.reconnect();
|
|
323
|
-
});
|
|
324
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
325
|
-
});
|
|
319
|
+
this.connection.send(message);
|
|
326
320
|
}
|
|
327
321
|
log(...message) {
|
|
328
322
|
this.host.log(...message);
|
|
@@ -330,197 +324,153 @@ var DialogueWebSocket = class {
|
|
|
330
324
|
};
|
|
331
325
|
|
|
332
326
|
// src/SpeechWebSocket.ts
|
|
333
|
-
import WebSocket3 from "ws";
|
|
334
327
|
var WS_INACTIVITY_TIMEOUT = 180;
|
|
335
|
-
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
336
|
-
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
337
|
-
var DEFAULT_MAX_RETRY2 = 3;
|
|
338
328
|
var SpeechWebSocket = class {
|
|
329
|
+
// Index in textSent where the audio received ends
|
|
339
330
|
constructor(options, host) {
|
|
340
331
|
this.options = options;
|
|
341
332
|
this.host = host;
|
|
342
|
-
this.
|
|
343
|
-
this.textEnded = false;
|
|
344
|
-
// Whether the text stream has ended
|
|
345
|
-
this.textSent = "";
|
|
346
|
-
// Text sent to ElevenLabs
|
|
347
|
-
this.textBuffer = "";
|
|
348
|
-
// Buffer of text to send to ElevenLabs
|
|
349
|
-
this.receivedAudioText = "";
|
|
350
|
-
this.retryCount = 0;
|
|
351
|
-
this.destroyed = false;
|
|
352
|
-
this.canceled = false;
|
|
333
|
+
this.state = "idle";
|
|
353
334
|
// Bumped by every speak() and every cancel(), so a stream that starts late
|
|
354
335
|
// can tell whether it is still the one that should be heard.
|
|
355
336
|
this.generation = 0;
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
337
|
+
// Generation of the stream being spoken. A speak() that never carries a
|
|
338
|
+
// word leaves it in place, so the sentence it sends goes on to its end.
|
|
339
|
+
this.speaking = 0;
|
|
340
|
+
this.textSent = "";
|
|
341
|
+
// Text of the utterance sent to ElevenLabs
|
|
342
|
+
this.textBuffer = "";
|
|
343
|
+
// Text not sent yet, held until its word is complete
|
|
344
|
+
this.heardUntil = 0;
|
|
345
|
+
const params = new URLSearchParams();
|
|
346
|
+
params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
|
|
347
|
+
params.append(
|
|
348
|
+
"output_format",
|
|
349
|
+
this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
|
|
350
|
+
);
|
|
351
|
+
params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
|
|
352
|
+
const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
|
|
353
|
+
if (voiceSettings) {
|
|
354
|
+
params.append("voice_settings", JSON.stringify(voiceSettings));
|
|
355
|
+
}
|
|
356
|
+
if (this.options.language) {
|
|
357
|
+
params.append("language_code", this.options.language);
|
|
358
|
+
}
|
|
359
|
+
this.connection = new ElevenLabsConnection({
|
|
360
|
+
url: `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`,
|
|
361
|
+
tts: options,
|
|
362
|
+
// Initialization message with voice settings
|
|
363
|
+
init: { text: " ", voice_settings: voiceSettings },
|
|
364
|
+
keepAlive: {
|
|
365
|
+
interval: (WS_INACTIVITY_TIMEOUT - 1) * 1e3,
|
|
366
|
+
message: { text: " " }
|
|
367
|
+
},
|
|
368
|
+
host,
|
|
369
|
+
onOpen: () => this.resume(),
|
|
370
|
+
onMessage: (message) => this.onMessage(message),
|
|
371
|
+
onFailed: () => this.onFailed()
|
|
359
372
|
});
|
|
373
|
+
this.connection.connect();
|
|
360
374
|
}
|
|
361
375
|
speak(textStream) {
|
|
362
376
|
const generation = ++this.generation;
|
|
363
377
|
let started = false;
|
|
364
378
|
const start = () => {
|
|
365
|
-
if (started) return
|
|
379
|
+
if (started) return this.speaking === generation;
|
|
366
380
|
if (this.generation !== generation) return false;
|
|
367
381
|
started = true;
|
|
368
|
-
this.
|
|
369
|
-
this.
|
|
370
|
-
this.
|
|
371
|
-
this.
|
|
372
|
-
this.
|
|
373
|
-
this.receivedAudioText = "";
|
|
382
|
+
if (this.state !== "idle") this.connection.restart();
|
|
383
|
+
else this.connection.connect();
|
|
384
|
+
this.resetUtterance();
|
|
385
|
+
this.speaking = generation;
|
|
386
|
+
this.state = "speaking";
|
|
374
387
|
return true;
|
|
375
388
|
};
|
|
376
389
|
textStream.on("data", async (chunk) => {
|
|
377
390
|
if (!start()) return;
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
this.
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
} else {
|
|
386
|
-
this.sendTranscript(this.textBuffer + text.slice(0, spaceIndex + 1));
|
|
387
|
-
this.textBuffer = text.slice(spaceIndex + 1);
|
|
388
|
-
}
|
|
391
|
+
this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
|
|
392
|
+
if (!await this.connection.ready()) return;
|
|
393
|
+
if (this.speaking !== generation) return;
|
|
394
|
+
const spaceIndex = this.textBuffer.lastIndexOf(" ");
|
|
395
|
+
if (spaceIndex === -1) return;
|
|
396
|
+
this.sendTranscript(this.textBuffer.slice(0, spaceIndex + 1));
|
|
397
|
+
this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
|
|
389
398
|
});
|
|
390
399
|
textStream.on("error", (error) => {
|
|
391
|
-
if (!started) return;
|
|
400
|
+
if (!started || this.speaking !== generation) return;
|
|
392
401
|
this.log("Error in text stream, ending audio stream", error);
|
|
393
|
-
this.
|
|
402
|
+
this.speaking = 0;
|
|
403
|
+
this.textBuffer = "";
|
|
404
|
+
this.flush();
|
|
394
405
|
});
|
|
395
406
|
textStream.on("end", async () => {
|
|
396
|
-
if (!started
|
|
397
|
-
await this.
|
|
407
|
+
if (!started) return;
|
|
408
|
+
if (!await this.connection.ready()) return;
|
|
409
|
+
if (this.speaking !== generation) return;
|
|
398
410
|
if (this.textBuffer.trim()) {
|
|
399
411
|
this.sendTranscript(this.textBuffer + " ");
|
|
400
|
-
this.textBuffer = "";
|
|
401
412
|
}
|
|
402
|
-
this.
|
|
403
|
-
this.
|
|
404
|
-
this.log("Flushed text");
|
|
413
|
+
this.textBuffer = "";
|
|
414
|
+
this.flush();
|
|
405
415
|
});
|
|
406
416
|
}
|
|
407
417
|
cancel() {
|
|
408
418
|
this.generation++;
|
|
409
|
-
|
|
419
|
+
this.speaking = 0;
|
|
420
|
+
if (this.state === "idle") return;
|
|
410
421
|
this.log("Cancel");
|
|
411
|
-
this.
|
|
412
|
-
this.
|
|
413
|
-
this.receivedAudioText = "";
|
|
414
|
-
this.isProcessing = false;
|
|
415
|
-
if (this.socket?.readyState === WebSocket3.OPEN) {
|
|
416
|
-
this.socket.send(JSON.stringify({ text: " ", flush: true }));
|
|
417
|
-
}
|
|
422
|
+
this.resetUtterance();
|
|
423
|
+
this.connection.restart();
|
|
418
424
|
}
|
|
419
425
|
destroy() {
|
|
420
|
-
this.
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
426
|
+
this.generation++;
|
|
427
|
+
this.speaking = 0;
|
|
428
|
+
this.resetUtterance();
|
|
429
|
+
this.connection.destroy();
|
|
430
|
+
}
|
|
431
|
+
/** Flushes buffered text and marks the end of the utterance */
|
|
432
|
+
flush() {
|
|
433
|
+
this.state = "flushed";
|
|
434
|
+
this.connection.send({ text: " ", flush: true });
|
|
435
|
+
this.log("Flushed text");
|
|
436
|
+
this.checkEnded();
|
|
437
|
+
}
|
|
438
|
+
/**
|
|
439
|
+
* Sends the text of the utterance in flight again on a new connection. The
|
|
440
|
+
* server lost what it was generating, so it goes on from the audio received.
|
|
441
|
+
*/
|
|
442
|
+
resume() {
|
|
443
|
+
if (this.state === "idle") return;
|
|
444
|
+
const wordStart = this.textSent.lastIndexOf(" ", this.heardUntil - 1) + 1;
|
|
445
|
+
const unheard = this.textSent.slice(wordStart);
|
|
446
|
+
this.textSent = "";
|
|
447
|
+
this.heardUntil = 0;
|
|
448
|
+
if (unheard.trim()) {
|
|
449
|
+
this.log("Sending text chunks again");
|
|
450
|
+
this.sendTranscript(unheard);
|
|
432
451
|
}
|
|
433
|
-
if (this.
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
if (voiceSettings) {
|
|
448
|
-
params.append("voice_settings", JSON.stringify(voiceSettings));
|
|
449
|
-
}
|
|
450
|
-
if (this.options.language) {
|
|
451
|
-
params.append("language_code", this.options.language);
|
|
452
|
-
}
|
|
453
|
-
const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`;
|
|
454
|
-
const socket = new WebSocket3(uri, {
|
|
455
|
-
headers: {
|
|
456
|
-
"xi-api-key": this.options.apiKey
|
|
457
|
-
}
|
|
458
|
-
});
|
|
459
|
-
this.socket = socket;
|
|
460
|
-
const timeout = setTimeout(() => {
|
|
461
|
-
this.log("Connection timeout");
|
|
462
|
-
closeSocket(socket);
|
|
463
|
-
this.socket = void 0;
|
|
464
|
-
reject(new Error("WebSocket connection timeout"));
|
|
465
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
|
|
466
|
-
this.connectionTimeout = timeout;
|
|
467
|
-
socket.addEventListener("open", () => {
|
|
468
|
-
clearTimeout(timeout);
|
|
469
|
-
this.log("Connection opened");
|
|
470
|
-
this.socket?.send(
|
|
471
|
-
JSON.stringify({
|
|
472
|
-
text: " ",
|
|
473
|
-
voice_settings: voiceSettings
|
|
474
|
-
})
|
|
475
|
-
);
|
|
476
|
-
this.keepAliveInterval = setInterval(
|
|
477
|
-
() => {
|
|
478
|
-
if (this.socket?.readyState === WebSocket3.OPEN) {
|
|
479
|
-
this.log("Sending keep-alive message");
|
|
480
|
-
this.socket.send(JSON.stringify({ text: " " }));
|
|
481
|
-
this.retryCount = 0;
|
|
482
|
-
}
|
|
483
|
-
},
|
|
484
|
-
(WS_INACTIVITY_TIMEOUT - 1) * 1e3
|
|
485
|
-
);
|
|
486
|
-
resolve();
|
|
487
|
-
});
|
|
488
|
-
socket.addEventListener("error", (error) => {
|
|
489
|
-
clearTimeout(timeout);
|
|
490
|
-
this.log("WebSocket error:", error);
|
|
491
|
-
reject(new Error("WebSocket connection error"));
|
|
492
|
-
});
|
|
493
|
-
socket.addEventListener("message", (event) => {
|
|
494
|
-
if (this.canceled) return;
|
|
495
|
-
try {
|
|
496
|
-
this.onMessage(JSON.parse(event.data.toString()));
|
|
497
|
-
} catch (error) {
|
|
498
|
-
this.log("message" in error ? error.message : error);
|
|
499
|
-
this.log("Event data during error:", event.data);
|
|
500
|
-
}
|
|
501
|
-
});
|
|
502
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
503
|
-
clearTimeout(timeout);
|
|
504
|
-
this.socket?.removeAllListeners();
|
|
505
|
-
this.socket = void 0;
|
|
506
|
-
this.isProcessing = false;
|
|
507
|
-
if (this.keepAliveInterval) {
|
|
508
|
-
clearInterval(this.keepAliveInterval);
|
|
509
|
-
this.keepAliveInterval = void 0;
|
|
510
|
-
}
|
|
511
|
-
this.log("Connection closed", { code, reason });
|
|
512
|
-
this.reconnect();
|
|
513
|
-
});
|
|
514
|
-
});
|
|
452
|
+
if (this.state === "flushed") this.flush();
|
|
453
|
+
}
|
|
454
|
+
/** Every retry failed: hands the unspoken text over and drops the utterance */
|
|
455
|
+
onFailed() {
|
|
456
|
+
const unspoken = this.state === "idle" ? "" : this.textSent.slice(this.heardUntil) + this.textBuffer;
|
|
457
|
+
this.speaking = 0;
|
|
458
|
+
this.resetUtterance();
|
|
459
|
+
this.host.failed(unspoken.trim() ? [unspoken] : []);
|
|
460
|
+
}
|
|
461
|
+
resetUtterance() {
|
|
462
|
+
this.state = "idle";
|
|
463
|
+
this.textSent = "";
|
|
464
|
+
this.textBuffer = "";
|
|
465
|
+
this.heardUntil = 0;
|
|
515
466
|
}
|
|
516
467
|
onMessage(message) {
|
|
517
468
|
if ("audio" in message && message.audio) {
|
|
518
469
|
this.processAudioMessage(message);
|
|
519
470
|
}
|
|
520
|
-
if ("isFinal" in message && message.isFinal) {
|
|
471
|
+
if ("isFinal" in message && message.isFinal && this.state !== "idle") {
|
|
521
472
|
this.log("Audio ended");
|
|
522
|
-
this.
|
|
523
|
-
this.textSent = "";
|
|
473
|
+
this.resetUtterance();
|
|
524
474
|
}
|
|
525
475
|
if ("error" in message) {
|
|
526
476
|
console.error("[ElevenLabsTTS] Error:", message.error, message.message);
|
|
@@ -528,58 +478,59 @@ var SpeechWebSocket = class {
|
|
|
528
478
|
}
|
|
529
479
|
processAudioMessage(message) {
|
|
530
480
|
const chunk = Buffer.from(message.audio, "base64");
|
|
531
|
-
this.
|
|
481
|
+
this.connection.prove();
|
|
482
|
+
if (this.state === "idle") {
|
|
483
|
+
this.log("Ignore audio chunk, nothing is being spoken");
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
532
486
|
if (message.alignment) {
|
|
533
487
|
const chunkText = message.alignment.chars.join("");
|
|
534
|
-
const
|
|
535
|
-
if (
|
|
536
|
-
this.log(`
|
|
537
|
-
|
|
488
|
+
const end = findHeardEnd(this.textSent, this.heardUntil, chunkText);
|
|
489
|
+
if (end === -1) {
|
|
490
|
+
this.log(`Audio chunk off the text sent ("${chunkText}")`);
|
|
491
|
+
} else {
|
|
492
|
+
this.log(`Received audio chunk ("${chunkText}")`);
|
|
493
|
+
this.heardUntil = end;
|
|
538
494
|
}
|
|
539
|
-
this.log(`Received audio chunk ("${chunkText}")`);
|
|
540
|
-
this.receivedAudioText = receivedAudioText;
|
|
541
|
-
this.fixIsFinal(message);
|
|
542
495
|
}
|
|
543
496
|
this.host.audio(chunk);
|
|
497
|
+
this.checkEnded();
|
|
544
498
|
}
|
|
545
|
-
|
|
546
|
-
if (
|
|
547
|
-
if (this.
|
|
548
|
-
|
|
549
|
-
|
|
499
|
+
checkEnded() {
|
|
500
|
+
if (this.state !== "flushed") return;
|
|
501
|
+
if (this.textSent.slice(this.heardUntil).trim()) return;
|
|
502
|
+
this.log("Audio ended");
|
|
503
|
+
this.resetUtterance();
|
|
550
504
|
}
|
|
551
505
|
sendTranscript(text) {
|
|
552
|
-
this.
|
|
506
|
+
this.textSent += text;
|
|
507
|
+
if (!this.connection.send({ text, try_trigger_generation: true })) return;
|
|
553
508
|
this.log(`Sent transcript: "${text}"`);
|
|
554
509
|
}
|
|
555
|
-
reconnect() {
|
|
556
|
-
if (this.destroyed) return;
|
|
557
|
-
this.retryCount++;
|
|
558
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
559
|
-
this.log("Max retries reached, giving up");
|
|
560
|
-
this.host.failed(this.textSent.trim() ? [this.textSent] : []);
|
|
561
|
-
return;
|
|
562
|
-
}
|
|
563
|
-
this.initPromise = new Promise((resolve) => {
|
|
564
|
-
this.log("Reconnecting...");
|
|
565
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
566
|
-
this.reconnectTimeout = void 0;
|
|
567
|
-
this.initWS().then(() => {
|
|
568
|
-
if (this.textSent.length > 0) {
|
|
569
|
-
this.log("Sending text chunks again");
|
|
570
|
-
this.sendTranscript(this.textSent);
|
|
571
|
-
}
|
|
572
|
-
}).then(resolve).catch((error) => {
|
|
573
|
-
this.log("Reconnection error:", error);
|
|
574
|
-
this.reconnect();
|
|
575
|
-
});
|
|
576
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
577
|
-
});
|
|
578
|
-
}
|
|
579
510
|
log(...message) {
|
|
580
511
|
this.host.log(...message);
|
|
581
512
|
}
|
|
582
513
|
};
|
|
514
|
+
function findHeardEnd(text, from, heard) {
|
|
515
|
+
for (let start = from; start < text.length; start++) {
|
|
516
|
+
const end = endOfPrefix(text, start, heard);
|
|
517
|
+
if (end !== -1) return end;
|
|
518
|
+
}
|
|
519
|
+
return -1;
|
|
520
|
+
}
|
|
521
|
+
function endOfPrefix(text, from, heard) {
|
|
522
|
+
let index = from;
|
|
523
|
+
for (let i = 0; i < heard.length; i++) {
|
|
524
|
+
if (isSpace(heard[i])) continue;
|
|
525
|
+
while (index < text.length && isSpace(text[index])) index++;
|
|
526
|
+
if (text[index] !== heard[i]) return -1;
|
|
527
|
+
index++;
|
|
528
|
+
}
|
|
529
|
+
return index;
|
|
530
|
+
}
|
|
531
|
+
function isSpace(char) {
|
|
532
|
+
return char === " " || char === " " || char === "\n" || char === "\r";
|
|
533
|
+
}
|
|
583
534
|
function toWireVoiceSettings(settings) {
|
|
584
535
|
if (!settings) return void 0;
|
|
585
536
|
const { similarityBoost, useSpeakerBoost, ...rest } = settings;
|