@micdrop/elevenlabs 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +328 -387
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +326 -375
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
2
|
var __defProp = Object.defineProperty;
|
|
4
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
6
|
var __export = (target, all) => {
|
|
9
7
|
for (var name in all)
|
|
@@ -17,14 +15,6 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
17
15
|
}
|
|
18
16
|
return to;
|
|
19
17
|
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
19
|
|
|
30
20
|
// src/index.ts
|
|
@@ -38,21 +28,96 @@ __export(index_exports, {
|
|
|
38
28
|
module.exports = __toCommonJS(index_exports);
|
|
39
29
|
|
|
40
30
|
// src/ElevenLabsTTS.ts
|
|
41
|
-
var
|
|
42
|
-
|
|
43
|
-
// src/DialogueWebSocket.ts
|
|
44
|
-
var import_ws2 = __toESM(require("ws"));
|
|
31
|
+
var import_server2 = require("@micdrop/server");
|
|
45
32
|
|
|
46
33
|
// src/ElevenLabsWebSocket.ts
|
|
47
|
-
var
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
34
|
+
var import_server = require("@micdrop/server");
|
|
35
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
36
|
+
var ElevenLabsConnection = class {
|
|
37
|
+
constructor(options) {
|
|
38
|
+
this.options = options;
|
|
39
|
+
// Whether the current connection carried audio or lived long enough for a
|
|
40
|
+
// keep-alive, which means the server accepted the key and the voice
|
|
41
|
+
this.proven = false;
|
|
42
|
+
this.failures = 0;
|
|
43
|
+
this.socket = this.createSocket();
|
|
54
44
|
}
|
|
55
|
-
|
|
45
|
+
/** Opens the connection, again after giving up, and does nothing otherwise */
|
|
46
|
+
connect() {
|
|
47
|
+
this.socket.connect();
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Resolves to false when the connection gives up or is destroyed meanwhile,
|
|
51
|
+
* and right away once it gave up, until the next `connect()`: the stream
|
|
52
|
+
* that was speaking then has nothing left to wait for.
|
|
53
|
+
*/
|
|
54
|
+
ready() {
|
|
55
|
+
if (this.socket.status === "idle") return Promise.resolve(false);
|
|
56
|
+
return this.socket.ready();
|
|
57
|
+
}
|
|
58
|
+
send(message) {
|
|
59
|
+
return this.socket.send(JSON.stringify(message));
|
|
60
|
+
}
|
|
61
|
+
/** Drops the connection, with the synthesis running on it, for a new one */
|
|
62
|
+
restart() {
|
|
63
|
+
this.socket.restart();
|
|
64
|
+
}
|
|
65
|
+
destroy() {
|
|
66
|
+
this.socket.destroy();
|
|
67
|
+
}
|
|
68
|
+
/** The current connection works, so its next drop is not a failure */
|
|
69
|
+
prove() {
|
|
70
|
+
this.proven = true;
|
|
71
|
+
this.failures = 0;
|
|
72
|
+
}
|
|
73
|
+
createSocket() {
|
|
74
|
+
const { tts, keepAlive, host } = this.options;
|
|
75
|
+
const socket = new import_server.ReconnectingSocket({
|
|
76
|
+
url: this.options.url,
|
|
77
|
+
clientOptions: { headers: { "xi-api-key": tts.apiKey } },
|
|
78
|
+
connectionTimeout: tts.connectionTimeout,
|
|
79
|
+
retryDelay: tts.retryDelay,
|
|
80
|
+
maxRetry: tts.maxRetry,
|
|
81
|
+
handshake: {
|
|
82
|
+
start: (ws) => ws.send(JSON.stringify(this.options.init))
|
|
83
|
+
},
|
|
84
|
+
keepAlive: {
|
|
85
|
+
interval: keepAlive.interval,
|
|
86
|
+
message: () => {
|
|
87
|
+
this.prove();
|
|
88
|
+
return JSON.stringify(keepAlive.message);
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
log: (...message) => host.log(...message)
|
|
92
|
+
});
|
|
93
|
+
socket.on("Open", ({ reconnected }) => {
|
|
94
|
+
this.proven = false;
|
|
95
|
+
this.options.onOpen(reconnected);
|
|
96
|
+
});
|
|
97
|
+
socket.on("Message", (data) => {
|
|
98
|
+
try {
|
|
99
|
+
this.options.onMessage(JSON.parse(data.toString()));
|
|
100
|
+
} catch (error) {
|
|
101
|
+
host.log("message" in error ? error.message : error);
|
|
102
|
+
host.log("Event data during error:", data.toString());
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
socket.on("Close", () => {
|
|
106
|
+
if (this.proven) return;
|
|
107
|
+
this.failures++;
|
|
108
|
+
if (this.failures > (tts.maxRetry ?? DEFAULT_MAX_RETRY)) this.giveUp();
|
|
109
|
+
});
|
|
110
|
+
socket.on("Failed", () => this.giveUp());
|
|
111
|
+
return socket;
|
|
112
|
+
}
|
|
113
|
+
giveUp() {
|
|
114
|
+
this.options.host.log("Max retries reached, giving up");
|
|
115
|
+
this.failures = 0;
|
|
116
|
+
this.socket.destroy();
|
|
117
|
+
this.socket = this.createSocket();
|
|
118
|
+
this.options.onFailed();
|
|
119
|
+
}
|
|
120
|
+
};
|
|
56
121
|
|
|
57
122
|
// src/types.ts
|
|
58
123
|
var DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
|
|
@@ -63,30 +128,23 @@ function isDialogueModel(modelId) {
|
|
|
63
128
|
|
|
64
129
|
// src/DialogueWebSocket.ts
|
|
65
130
|
var KEEP_ALIVE_INTERVAL = 15e3;
|
|
66
|
-
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
67
|
-
var DEFAULT_RETRY_DELAY = 1e3;
|
|
68
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
69
131
|
var SENTENCE_END = /(?:[!?]+|(?<!\.)\.(?!\.))\s+/g;
|
|
70
132
|
var DialogueWebSocket = class {
|
|
71
133
|
// Whether the connection received any text
|
|
72
134
|
constructor(options, host) {
|
|
73
135
|
this.options = options;
|
|
74
136
|
this.host = host;
|
|
75
|
-
this.
|
|
76
|
-
this.destroyed = false;
|
|
137
|
+
this.state = "idle";
|
|
77
138
|
// Bumped by every speak() and every cancel(), so a stream that starts late
|
|
78
139
|
// can tell whether it is still the one that should be heard.
|
|
79
140
|
this.generation = 0;
|
|
80
141
|
// Generation of the stream being spoken. A speak() that never carries a
|
|
81
142
|
// word leaves it in place, so the sentence it sends goes on to its end.
|
|
82
143
|
this.speaking = 0;
|
|
83
|
-
this.isProcessing = false;
|
|
84
|
-
this.textEnded = false;
|
|
85
|
-
// Whether the text stream has ended
|
|
86
144
|
this.textSent = "";
|
|
87
145
|
// Text of the utterance sent to ElevenLabs
|
|
88
146
|
this.textBuffer = "";
|
|
89
|
-
//
|
|
147
|
+
// Text not sent yet, held until its word is complete
|
|
90
148
|
this.unflushed = false;
|
|
91
149
|
// Whether text was sent since the last flush
|
|
92
150
|
this.pendingFlushes = 0;
|
|
@@ -94,7 +152,34 @@ var DialogueWebSocket = class {
|
|
|
94
152
|
this.newTurn = false;
|
|
95
153
|
// Whether the next input starts a new utterance
|
|
96
154
|
this.hasSpoken = false;
|
|
97
|
-
|
|
155
|
+
const params = new URLSearchParams();
|
|
156
|
+
params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
|
|
157
|
+
params.append(
|
|
158
|
+
"output_format",
|
|
159
|
+
this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
|
|
160
|
+
);
|
|
161
|
+
if (this.options.language) {
|
|
162
|
+
params.append("language_code", this.options.language);
|
|
163
|
+
}
|
|
164
|
+
const stability = this.options.voiceSettings?.stability;
|
|
165
|
+
const init = {
|
|
166
|
+
voices: [this.options.voiceId],
|
|
167
|
+
voice_settings: stability === void 0 ? void 0 : { stability }
|
|
168
|
+
};
|
|
169
|
+
this.connection = new ElevenLabsConnection({
|
|
170
|
+
url: `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
|
|
171
|
+
tts: options,
|
|
172
|
+
init,
|
|
173
|
+
keepAlive: {
|
|
174
|
+
interval: KEEP_ALIVE_INTERVAL,
|
|
175
|
+
message: { keep_alive: true }
|
|
176
|
+
},
|
|
177
|
+
host,
|
|
178
|
+
onOpen: () => this.onOpen(),
|
|
179
|
+
onMessage: (message) => this.onMessage(message),
|
|
180
|
+
onFailed: () => this.onFailed()
|
|
181
|
+
});
|
|
182
|
+
this.connection.connect();
|
|
98
183
|
}
|
|
99
184
|
speak(textStream) {
|
|
100
185
|
const generation = ++this.generation;
|
|
@@ -103,171 +188,100 @@ var DialogueWebSocket = class {
|
|
|
103
188
|
if (started) return this.speaking === generation;
|
|
104
189
|
if (this.generation !== generation) return false;
|
|
105
190
|
started = true;
|
|
106
|
-
if (this.
|
|
191
|
+
if (this.state !== "idle") this.connection.restart();
|
|
192
|
+
else this.connection.connect();
|
|
193
|
+
this.resetUtterance();
|
|
107
194
|
this.speaking = generation;
|
|
108
|
-
this.
|
|
109
|
-
this.textEnded = false;
|
|
110
|
-
this.textSent = "";
|
|
111
|
-
this.textBuffer = "";
|
|
112
|
-
this.unflushed = false;
|
|
113
|
-
this.pendingFlushes = 0;
|
|
195
|
+
this.state = "streaming";
|
|
114
196
|
this.newTurn = true;
|
|
115
197
|
return true;
|
|
116
198
|
};
|
|
117
199
|
textStream.on("data", async (chunk) => {
|
|
118
200
|
if (!start()) return;
|
|
119
|
-
|
|
120
|
-
await this.
|
|
201
|
+
this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
|
|
202
|
+
if (!await this.connection.ready()) return;
|
|
121
203
|
if (this.speaking !== generation) return;
|
|
122
|
-
const spaceIndex =
|
|
123
|
-
if (spaceIndex === -1)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
this.sendText(this.textBuffer + text.slice(0, spaceIndex + 1));
|
|
127
|
-
this.textBuffer = text.slice(spaceIndex + 1);
|
|
128
|
-
}
|
|
204
|
+
const spaceIndex = this.textBuffer.lastIndexOf(" ");
|
|
205
|
+
if (spaceIndex === -1) return;
|
|
206
|
+
this.sendText(this.textBuffer.slice(0, spaceIndex + 1));
|
|
207
|
+
this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
|
|
129
208
|
});
|
|
130
209
|
textStream.on("error", (error) => {
|
|
131
210
|
if (!started || this.speaking !== generation) return;
|
|
132
211
|
this.log("Error in text stream, ending audio stream", error);
|
|
133
|
-
this.
|
|
212
|
+
this.speaking = 0;
|
|
213
|
+
this.textBuffer = "";
|
|
214
|
+
this.endText();
|
|
134
215
|
});
|
|
135
216
|
textStream.on("end", async () => {
|
|
136
217
|
if (!started) return;
|
|
137
|
-
await this.
|
|
218
|
+
if (!await this.connection.ready()) return;
|
|
138
219
|
if (this.speaking !== generation) return;
|
|
139
220
|
if (this.textBuffer.trim()) {
|
|
140
221
|
this.sendText(this.textBuffer + " ");
|
|
141
|
-
this.textBuffer = "";
|
|
142
222
|
}
|
|
143
|
-
this.
|
|
144
|
-
this.
|
|
145
|
-
this.checkEnded();
|
|
223
|
+
this.textBuffer = "";
|
|
224
|
+
this.endText();
|
|
146
225
|
});
|
|
147
226
|
}
|
|
148
227
|
cancel() {
|
|
149
228
|
this.generation++;
|
|
150
229
|
this.speaking = 0;
|
|
151
|
-
if (
|
|
230
|
+
if (this.state === "idle") return;
|
|
152
231
|
this.log("Cancel");
|
|
153
|
-
this.
|
|
154
|
-
this.
|
|
155
|
-
this.textBuffer = "";
|
|
156
|
-
this.restart();
|
|
232
|
+
this.resetUtterance();
|
|
233
|
+
this.connection.restart();
|
|
157
234
|
}
|
|
158
235
|
destroy() {
|
|
159
|
-
this.
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
236
|
+
this.generation++;
|
|
237
|
+
this.speaking = 0;
|
|
238
|
+
this.resetUtterance();
|
|
239
|
+
this.connection.destroy();
|
|
240
|
+
}
|
|
241
|
+
onOpen() {
|
|
242
|
+
this.hasSpoken = false;
|
|
243
|
+
if (this.state === "idle") return;
|
|
244
|
+
const text = this.textSent;
|
|
245
|
+
this.textSent = "";
|
|
246
|
+
this.unflushed = false;
|
|
247
|
+
this.pendingFlushes = 0;
|
|
248
|
+
if (text.trim()) {
|
|
249
|
+
this.log("Sending text again");
|
|
250
|
+
this.sendText(text);
|
|
163
251
|
}
|
|
164
|
-
this.
|
|
165
|
-
this.
|
|
252
|
+
if (this.state === "draining") this.flush();
|
|
253
|
+
this.checkEnded();
|
|
166
254
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
255
|
+
/** Every retry failed: hands the unspoken text over and drops the utterance */
|
|
256
|
+
onFailed() {
|
|
257
|
+
const unspoken = this.state === "idle" ? "" : this.textSent + this.textBuffer;
|
|
258
|
+
this.speaking = 0;
|
|
259
|
+
this.resetUtterance();
|
|
260
|
+
this.host.failed(unspoken.trim() ? [unspoken] : []);
|
|
172
261
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
clearTimeout(this.connectionTimeout);
|
|
186
|
-
this.connectionTimeout = void 0;
|
|
187
|
-
}
|
|
188
|
-
if (this.keepAliveInterval) {
|
|
189
|
-
clearInterval(this.keepAliveInterval);
|
|
190
|
-
this.keepAliveInterval = void 0;
|
|
191
|
-
}
|
|
192
|
-
if (this.socket) closeSocket(this.socket);
|
|
193
|
-
this.socket = void 0;
|
|
194
|
-
}
|
|
195
|
-
initWS() {
|
|
196
|
-
return new Promise((resolve, reject) => {
|
|
197
|
-
const params = new URLSearchParams();
|
|
198
|
-
params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
|
|
199
|
-
params.append(
|
|
200
|
-
"output_format",
|
|
201
|
-
this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
|
|
202
|
-
);
|
|
203
|
-
if (this.options.language) {
|
|
204
|
-
params.append("language_code", this.options.language);
|
|
205
|
-
}
|
|
206
|
-
const socket = new import_ws2.default(
|
|
207
|
-
`wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?${params.toString()}`,
|
|
208
|
-
{
|
|
209
|
-
headers: {
|
|
210
|
-
"xi-api-key": this.options.apiKey
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
);
|
|
214
|
-
this.socket = socket;
|
|
215
|
-
let opened = false;
|
|
216
|
-
const timeout = setTimeout(() => {
|
|
217
|
-
this.log("Connection timeout");
|
|
218
|
-
closeSocket(socket);
|
|
219
|
-
this.socket = void 0;
|
|
220
|
-
reject(new Error("WebSocket connection timeout"));
|
|
221
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
222
|
-
this.connectionTimeout = timeout;
|
|
223
|
-
socket.addEventListener("open", () => {
|
|
224
|
-
clearTimeout(timeout);
|
|
225
|
-
opened = true;
|
|
226
|
-
this.log("Connection opened");
|
|
227
|
-
const stability = this.options.voiceSettings?.stability;
|
|
228
|
-
this.send({
|
|
229
|
-
voices: [this.options.voiceId],
|
|
230
|
-
voice_settings: stability === void 0 ? void 0 : { stability }
|
|
231
|
-
});
|
|
232
|
-
this.hasSpoken = false;
|
|
233
|
-
this.keepAliveInterval = setInterval(() => {
|
|
234
|
-
this.send({ keep_alive: true });
|
|
235
|
-
this.retryCount = 0;
|
|
236
|
-
}, KEEP_ALIVE_INTERVAL);
|
|
237
|
-
resolve();
|
|
238
|
-
});
|
|
239
|
-
socket.addEventListener("error", (error) => {
|
|
240
|
-
clearTimeout(timeout);
|
|
241
|
-
this.log("WebSocket error:", error);
|
|
242
|
-
reject(new Error("WebSocket connection error"));
|
|
243
|
-
});
|
|
244
|
-
socket.addEventListener("message", (event) => {
|
|
245
|
-
try {
|
|
246
|
-
this.onMessage(JSON.parse(event.data.toString()));
|
|
247
|
-
} catch (error) {
|
|
248
|
-
this.log("message" in error ? error.message : error);
|
|
249
|
-
this.log("Event data during error:", event.data);
|
|
250
|
-
}
|
|
251
|
-
});
|
|
252
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
253
|
-
clearTimeout(timeout);
|
|
254
|
-
if (this.keepAliveInterval) {
|
|
255
|
-
clearInterval(this.keepAliveInterval);
|
|
256
|
-
this.keepAliveInterval = void 0;
|
|
257
|
-
}
|
|
258
|
-
this.socket?.removeAllListeners();
|
|
259
|
-
this.socket = void 0;
|
|
260
|
-
this.log("Connection closed", { code, reason });
|
|
261
|
-
if (opened) this.reconnect();
|
|
262
|
-
});
|
|
263
|
-
});
|
|
262
|
+
resetUtterance() {
|
|
263
|
+
this.state = "idle";
|
|
264
|
+
this.textSent = "";
|
|
265
|
+
this.textBuffer = "";
|
|
266
|
+
this.unflushed = false;
|
|
267
|
+
this.pendingFlushes = 0;
|
|
268
|
+
}
|
|
269
|
+
/** The text stream is over: flushes what the server holds */
|
|
270
|
+
endText() {
|
|
271
|
+
this.state = "draining";
|
|
272
|
+
this.flush();
|
|
273
|
+
this.checkEnded();
|
|
264
274
|
}
|
|
265
275
|
onMessage(message) {
|
|
266
276
|
if ("audio" in message && message.audio) {
|
|
267
277
|
const chunk = Buffer.from(message.audio, "base64");
|
|
268
|
-
this.
|
|
269
|
-
this.
|
|
270
|
-
|
|
278
|
+
this.connection.prove();
|
|
279
|
+
if (this.state === "idle") {
|
|
280
|
+
this.log("Ignore audio chunk, nothing is being spoken");
|
|
281
|
+
} else {
|
|
282
|
+
this.log(`Received audio chunk (${chunk.length} bytes)`);
|
|
283
|
+
this.host.audio(chunk);
|
|
284
|
+
}
|
|
271
285
|
}
|
|
272
286
|
if ("is_final_audio_for_turn" in message && message.is_final_audio_for_turn) {
|
|
273
287
|
this.pendingFlushes = Math.max(0, this.pendingFlushes - 1);
|
|
@@ -323,45 +337,15 @@ var DialogueWebSocket = class {
|
|
|
323
337
|
}
|
|
324
338
|
/** Ends the utterance once its text is over and all its audio came */
|
|
325
339
|
checkEnded() {
|
|
326
|
-
if (
|
|
340
|
+
if (this.state !== "draining") return;
|
|
327
341
|
if (this.unflushed || this.pendingFlushes > 0) return;
|
|
328
342
|
this.log("Audio ended");
|
|
329
|
-
this.
|
|
330
|
-
this.textSent = "";
|
|
343
|
+
this.resetUtterance();
|
|
331
344
|
}
|
|
345
|
+
// Lost when the connection just dropped, which is fine: the reconnection
|
|
346
|
+
// sends the utterance again from textSent
|
|
332
347
|
send(message) {
|
|
333
|
-
|
|
334
|
-
this.socket.send(JSON.stringify(message));
|
|
335
|
-
}
|
|
336
|
-
reconnect() {
|
|
337
|
-
if (this.destroyed) return;
|
|
338
|
-
this.retryCount++;
|
|
339
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
340
|
-
this.log("Max retries reached, giving up");
|
|
341
|
-
const unspoken = this.isProcessing ? this.textSent + this.textBuffer : "";
|
|
342
|
-
this.host.failed(unspoken.trim() ? [unspoken] : []);
|
|
343
|
-
return;
|
|
344
|
-
}
|
|
345
|
-
this.initPromise = new Promise((resolve) => {
|
|
346
|
-
this.log("Reconnecting...");
|
|
347
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
348
|
-
this.reconnectTimeout = void 0;
|
|
349
|
-
this.initWS().then(() => {
|
|
350
|
-
if (this.isProcessing && this.textSent.trim()) {
|
|
351
|
-
this.log("Sending text again");
|
|
352
|
-
const text = this.textSent;
|
|
353
|
-
this.textSent = "";
|
|
354
|
-
this.unflushed = false;
|
|
355
|
-
this.pendingFlushes = 0;
|
|
356
|
-
this.sendText(text);
|
|
357
|
-
if (this.textEnded) this.flush();
|
|
358
|
-
}
|
|
359
|
-
}).then(resolve).catch((error) => {
|
|
360
|
-
this.log("Reconnection error:", error);
|
|
361
|
-
this.reconnect();
|
|
362
|
-
});
|
|
363
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
364
|
-
});
|
|
348
|
+
this.connection.send(message);
|
|
365
349
|
}
|
|
366
350
|
log(...message) {
|
|
367
351
|
this.host.log(...message);
|
|
@@ -369,197 +353,153 @@ var DialogueWebSocket = class {
|
|
|
369
353
|
};
|
|
370
354
|
|
|
371
355
|
// src/SpeechWebSocket.ts
|
|
372
|
-
var import_ws3 = __toESM(require("ws"));
|
|
373
356
|
var WS_INACTIVITY_TIMEOUT = 180;
|
|
374
|
-
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
375
|
-
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
376
|
-
var DEFAULT_MAX_RETRY2 = 3;
|
|
377
357
|
var SpeechWebSocket = class {
|
|
358
|
+
// Index in textSent where the audio received ends
|
|
378
359
|
constructor(options, host) {
|
|
379
360
|
this.options = options;
|
|
380
361
|
this.host = host;
|
|
381
|
-
this.
|
|
382
|
-
this.textEnded = false;
|
|
383
|
-
// Whether the text stream has ended
|
|
384
|
-
this.textSent = "";
|
|
385
|
-
// Text sent to ElevenLabs
|
|
386
|
-
this.textBuffer = "";
|
|
387
|
-
// Buffer of text to send to ElevenLabs
|
|
388
|
-
this.receivedAudioText = "";
|
|
389
|
-
this.retryCount = 0;
|
|
390
|
-
this.destroyed = false;
|
|
391
|
-
this.canceled = false;
|
|
362
|
+
this.state = "idle";
|
|
392
363
|
// Bumped by every speak() and every cancel(), so a stream that starts late
|
|
393
364
|
// can tell whether it is still the one that should be heard.
|
|
394
365
|
this.generation = 0;
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
366
|
+
// Generation of the stream being spoken. A speak() that never carries a
|
|
367
|
+
// word leaves it in place, so the sentence it sends goes on to its end.
|
|
368
|
+
this.speaking = 0;
|
|
369
|
+
this.textSent = "";
|
|
370
|
+
// Text of the utterance sent to ElevenLabs
|
|
371
|
+
this.textBuffer = "";
|
|
372
|
+
// Text not sent yet, held until its word is complete
|
|
373
|
+
this.heardUntil = 0;
|
|
374
|
+
const params = new URLSearchParams();
|
|
375
|
+
params.append("model_id", this.options.modelId ?? DEFAULT_MODEL_ID);
|
|
376
|
+
params.append(
|
|
377
|
+
"output_format",
|
|
378
|
+
this.options.outputFormat ?? DEFAULT_OUTPUT_FORMAT
|
|
379
|
+
);
|
|
380
|
+
params.append("inactivity_timeout", WS_INACTIVITY_TIMEOUT.toString());
|
|
381
|
+
const voiceSettings = toWireVoiceSettings(this.options.voiceSettings);
|
|
382
|
+
if (voiceSettings) {
|
|
383
|
+
params.append("voice_settings", JSON.stringify(voiceSettings));
|
|
384
|
+
}
|
|
385
|
+
if (this.options.language) {
|
|
386
|
+
params.append("language_code", this.options.language);
|
|
387
|
+
}
|
|
388
|
+
this.connection = new ElevenLabsConnection({
|
|
389
|
+
url: `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`,
|
|
390
|
+
tts: options,
|
|
391
|
+
// Initialization message with voice settings
|
|
392
|
+
init: { text: " ", voice_settings: voiceSettings },
|
|
393
|
+
keepAlive: {
|
|
394
|
+
interval: (WS_INACTIVITY_TIMEOUT - 1) * 1e3,
|
|
395
|
+
message: { text: " " }
|
|
396
|
+
},
|
|
397
|
+
host,
|
|
398
|
+
onOpen: () => this.resume(),
|
|
399
|
+
onMessage: (message) => this.onMessage(message),
|
|
400
|
+
onFailed: () => this.onFailed()
|
|
398
401
|
});
|
|
402
|
+
this.connection.connect();
|
|
399
403
|
}
|
|
400
404
|
speak(textStream) {
|
|
401
405
|
const generation = ++this.generation;
|
|
402
406
|
let started = false;
|
|
403
407
|
const start = () => {
|
|
404
|
-
if (started) return
|
|
408
|
+
if (started) return this.speaking === generation;
|
|
405
409
|
if (this.generation !== generation) return false;
|
|
406
410
|
started = true;
|
|
407
|
-
this.
|
|
408
|
-
this.
|
|
409
|
-
this.
|
|
410
|
-
this.
|
|
411
|
-
this.
|
|
412
|
-
this.receivedAudioText = "";
|
|
411
|
+
if (this.state !== "idle") this.connection.restart();
|
|
412
|
+
else this.connection.connect();
|
|
413
|
+
this.resetUtterance();
|
|
414
|
+
this.speaking = generation;
|
|
415
|
+
this.state = "speaking";
|
|
413
416
|
return true;
|
|
414
417
|
};
|
|
415
418
|
textStream.on("data", async (chunk) => {
|
|
416
419
|
if (!start()) return;
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
this.
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
} else {
|
|
425
|
-
this.sendTranscript(this.textBuffer + text.slice(0, spaceIndex + 1));
|
|
426
|
-
this.textBuffer = text.slice(spaceIndex + 1);
|
|
427
|
-
}
|
|
420
|
+
this.textBuffer += chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
|
|
421
|
+
if (!await this.connection.ready()) return;
|
|
422
|
+
if (this.speaking !== generation) return;
|
|
423
|
+
const spaceIndex = this.textBuffer.lastIndexOf(" ");
|
|
424
|
+
if (spaceIndex === -1) return;
|
|
425
|
+
this.sendTranscript(this.textBuffer.slice(0, spaceIndex + 1));
|
|
426
|
+
this.textBuffer = this.textBuffer.slice(spaceIndex + 1);
|
|
428
427
|
});
|
|
429
428
|
textStream.on("error", (error) => {
|
|
430
|
-
if (!started) return;
|
|
429
|
+
if (!started || this.speaking !== generation) return;
|
|
431
430
|
this.log("Error in text stream, ending audio stream", error);
|
|
432
|
-
this.
|
|
431
|
+
this.speaking = 0;
|
|
432
|
+
this.textBuffer = "";
|
|
433
|
+
this.flush();
|
|
433
434
|
});
|
|
434
435
|
textStream.on("end", async () => {
|
|
435
|
-
if (!started
|
|
436
|
-
await this.
|
|
436
|
+
if (!started) return;
|
|
437
|
+
if (!await this.connection.ready()) return;
|
|
438
|
+
if (this.speaking !== generation) return;
|
|
437
439
|
if (this.textBuffer.trim()) {
|
|
438
440
|
this.sendTranscript(this.textBuffer + " ");
|
|
439
|
-
this.textBuffer = "";
|
|
440
441
|
}
|
|
441
|
-
this.
|
|
442
|
-
this.
|
|
443
|
-
this.log("Flushed text");
|
|
442
|
+
this.textBuffer = "";
|
|
443
|
+
this.flush();
|
|
444
444
|
});
|
|
445
445
|
}
|
|
446
446
|
cancel() {
|
|
447
447
|
this.generation++;
|
|
448
|
-
|
|
448
|
+
this.speaking = 0;
|
|
449
|
+
if (this.state === "idle") return;
|
|
449
450
|
this.log("Cancel");
|
|
450
|
-
this.
|
|
451
|
-
this.
|
|
452
|
-
this.receivedAudioText = "";
|
|
453
|
-
this.isProcessing = false;
|
|
454
|
-
if (this.socket?.readyState === import_ws3.default.OPEN) {
|
|
455
|
-
this.socket.send(JSON.stringify({ text: " ", flush: true }));
|
|
456
|
-
}
|
|
451
|
+
this.resetUtterance();
|
|
452
|
+
this.connection.restart();
|
|
457
453
|
}
|
|
458
454
|
destroy() {
|
|
459
|
-
this.
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
455
|
+
this.generation++;
|
|
456
|
+
this.speaking = 0;
|
|
457
|
+
this.resetUtterance();
|
|
458
|
+
this.connection.destroy();
|
|
459
|
+
}
|
|
460
|
+
/** Flushes buffered text and marks the end of the utterance */
|
|
461
|
+
flush() {
|
|
462
|
+
this.state = "flushed";
|
|
463
|
+
this.connection.send({ text: " ", flush: true });
|
|
464
|
+
this.log("Flushed text");
|
|
465
|
+
this.checkEnded();
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Sends the text of the utterance in flight again on a new connection. The
|
|
469
|
+
* server lost what it was generating, so it goes on from the audio received.
|
|
470
|
+
*/
|
|
471
|
+
resume() {
|
|
472
|
+
if (this.state === "idle") return;
|
|
473
|
+
const wordStart = this.textSent.lastIndexOf(" ", this.heardUntil - 1) + 1;
|
|
474
|
+
const unheard = this.textSent.slice(wordStart);
|
|
475
|
+
this.textSent = "";
|
|
476
|
+
this.heardUntil = 0;
|
|
477
|
+
if (unheard.trim()) {
|
|
478
|
+
this.log("Sending text chunks again");
|
|
479
|
+
this.sendTranscript(unheard);
|
|
471
480
|
}
|
|
472
|
-
if (this.
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
if (voiceSettings) {
|
|
487
|
-
params.append("voice_settings", JSON.stringify(voiceSettings));
|
|
488
|
-
}
|
|
489
|
-
if (this.options.language) {
|
|
490
|
-
params.append("language_code", this.options.language);
|
|
491
|
-
}
|
|
492
|
-
const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.options.voiceId}/stream-input?${params.toString()}`;
|
|
493
|
-
const socket = new import_ws3.default(uri, {
|
|
494
|
-
headers: {
|
|
495
|
-
"xi-api-key": this.options.apiKey
|
|
496
|
-
}
|
|
497
|
-
});
|
|
498
|
-
this.socket = socket;
|
|
499
|
-
const timeout = setTimeout(() => {
|
|
500
|
-
this.log("Connection timeout");
|
|
501
|
-
closeSocket(socket);
|
|
502
|
-
this.socket = void 0;
|
|
503
|
-
reject(new Error("WebSocket connection timeout"));
|
|
504
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
|
|
505
|
-
this.connectionTimeout = timeout;
|
|
506
|
-
socket.addEventListener("open", () => {
|
|
507
|
-
clearTimeout(timeout);
|
|
508
|
-
this.log("Connection opened");
|
|
509
|
-
this.socket?.send(
|
|
510
|
-
JSON.stringify({
|
|
511
|
-
text: " ",
|
|
512
|
-
voice_settings: voiceSettings
|
|
513
|
-
})
|
|
514
|
-
);
|
|
515
|
-
this.keepAliveInterval = setInterval(
|
|
516
|
-
() => {
|
|
517
|
-
if (this.socket?.readyState === import_ws3.default.OPEN) {
|
|
518
|
-
this.log("Sending keep-alive message");
|
|
519
|
-
this.socket.send(JSON.stringify({ text: " " }));
|
|
520
|
-
this.retryCount = 0;
|
|
521
|
-
}
|
|
522
|
-
},
|
|
523
|
-
(WS_INACTIVITY_TIMEOUT - 1) * 1e3
|
|
524
|
-
);
|
|
525
|
-
resolve();
|
|
526
|
-
});
|
|
527
|
-
socket.addEventListener("error", (error) => {
|
|
528
|
-
clearTimeout(timeout);
|
|
529
|
-
this.log("WebSocket error:", error);
|
|
530
|
-
reject(new Error("WebSocket connection error"));
|
|
531
|
-
});
|
|
532
|
-
socket.addEventListener("message", (event) => {
|
|
533
|
-
if (this.canceled) return;
|
|
534
|
-
try {
|
|
535
|
-
this.onMessage(JSON.parse(event.data.toString()));
|
|
536
|
-
} catch (error) {
|
|
537
|
-
this.log("message" in error ? error.message : error);
|
|
538
|
-
this.log("Event data during error:", event.data);
|
|
539
|
-
}
|
|
540
|
-
});
|
|
541
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
542
|
-
clearTimeout(timeout);
|
|
543
|
-
this.socket?.removeAllListeners();
|
|
544
|
-
this.socket = void 0;
|
|
545
|
-
this.isProcessing = false;
|
|
546
|
-
if (this.keepAliveInterval) {
|
|
547
|
-
clearInterval(this.keepAliveInterval);
|
|
548
|
-
this.keepAliveInterval = void 0;
|
|
549
|
-
}
|
|
550
|
-
this.log("Connection closed", { code, reason });
|
|
551
|
-
this.reconnect();
|
|
552
|
-
});
|
|
553
|
-
});
|
|
481
|
+
if (this.state === "flushed") this.flush();
|
|
482
|
+
}
|
|
483
|
+
/** Every retry failed: hands the unspoken text over and drops the utterance */
|
|
484
|
+
onFailed() {
|
|
485
|
+
const unspoken = this.state === "idle" ? "" : this.textSent.slice(this.heardUntil) + this.textBuffer;
|
|
486
|
+
this.speaking = 0;
|
|
487
|
+
this.resetUtterance();
|
|
488
|
+
this.host.failed(unspoken.trim() ? [unspoken] : []);
|
|
489
|
+
}
|
|
490
|
+
resetUtterance() {
|
|
491
|
+
this.state = "idle";
|
|
492
|
+
this.textSent = "";
|
|
493
|
+
this.textBuffer = "";
|
|
494
|
+
this.heardUntil = 0;
|
|
554
495
|
}
|
|
555
496
|
onMessage(message) {
|
|
556
497
|
if ("audio" in message && message.audio) {
|
|
557
498
|
this.processAudioMessage(message);
|
|
558
499
|
}
|
|
559
|
-
if ("isFinal" in message && message.isFinal) {
|
|
500
|
+
if ("isFinal" in message && message.isFinal && this.state !== "idle") {
|
|
560
501
|
this.log("Audio ended");
|
|
561
|
-
this.
|
|
562
|
-
this.textSent = "";
|
|
502
|
+
this.resetUtterance();
|
|
563
503
|
}
|
|
564
504
|
if ("error" in message) {
|
|
565
505
|
console.error("[ElevenLabsTTS] Error:", message.error, message.message);
|
|
@@ -567,58 +507,59 @@ var SpeechWebSocket = class {
|
|
|
567
507
|
}
|
|
568
508
|
processAudioMessage(message) {
|
|
569
509
|
const chunk = Buffer.from(message.audio, "base64");
|
|
570
|
-
this.
|
|
510
|
+
this.connection.prove();
|
|
511
|
+
if (this.state === "idle") {
|
|
512
|
+
this.log("Ignore audio chunk, nothing is being spoken");
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
571
515
|
if (message.alignment) {
|
|
572
516
|
const chunkText = message.alignment.chars.join("");
|
|
573
|
-
const
|
|
574
|
-
if (
|
|
575
|
-
this.log(`
|
|
576
|
-
|
|
517
|
+
const end = findHeardEnd(this.textSent, this.heardUntil, chunkText);
|
|
518
|
+
if (end === -1) {
|
|
519
|
+
this.log(`Audio chunk off the text sent ("${chunkText}")`);
|
|
520
|
+
} else {
|
|
521
|
+
this.log(`Received audio chunk ("${chunkText}")`);
|
|
522
|
+
this.heardUntil = end;
|
|
577
523
|
}
|
|
578
|
-
this.log(`Received audio chunk ("${chunkText}")`);
|
|
579
|
-
this.receivedAudioText = receivedAudioText;
|
|
580
|
-
this.fixIsFinal(message);
|
|
581
524
|
}
|
|
582
525
|
this.host.audio(chunk);
|
|
526
|
+
this.checkEnded();
|
|
583
527
|
}
|
|
584
|
-
|
|
585
|
-
if (
|
|
586
|
-
if (this.
|
|
587
|
-
|
|
588
|
-
|
|
528
|
+
checkEnded() {
|
|
529
|
+
if (this.state !== "flushed") return;
|
|
530
|
+
if (this.textSent.slice(this.heardUntil).trim()) return;
|
|
531
|
+
this.log("Audio ended");
|
|
532
|
+
this.resetUtterance();
|
|
589
533
|
}
|
|
590
534
|
sendTranscript(text) {
|
|
591
|
-
this.
|
|
535
|
+
this.textSent += text;
|
|
536
|
+
if (!this.connection.send({ text, try_trigger_generation: true })) return;
|
|
592
537
|
this.log(`Sent transcript: "${text}"`);
|
|
593
538
|
}
|
|
594
|
-
reconnect() {
|
|
595
|
-
if (this.destroyed) return;
|
|
596
|
-
this.retryCount++;
|
|
597
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
598
|
-
this.log("Max retries reached, giving up");
|
|
599
|
-
this.host.failed(this.textSent.trim() ? [this.textSent] : []);
|
|
600
|
-
return;
|
|
601
|
-
}
|
|
602
|
-
this.initPromise = new Promise((resolve) => {
|
|
603
|
-
this.log("Reconnecting...");
|
|
604
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
605
|
-
this.reconnectTimeout = void 0;
|
|
606
|
-
this.initWS().then(() => {
|
|
607
|
-
if (this.textSent.length > 0) {
|
|
608
|
-
this.log("Sending text chunks again");
|
|
609
|
-
this.sendTranscript(this.textSent);
|
|
610
|
-
}
|
|
611
|
-
}).then(resolve).catch((error) => {
|
|
612
|
-
this.log("Reconnection error:", error);
|
|
613
|
-
this.reconnect();
|
|
614
|
-
});
|
|
615
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
616
|
-
});
|
|
617
|
-
}
|
|
618
539
|
log(...message) {
|
|
619
540
|
this.host.log(...message);
|
|
620
541
|
}
|
|
621
542
|
};
|
|
543
|
+
function findHeardEnd(text, from, heard) {
|
|
544
|
+
for (let start = from; start < text.length; start++) {
|
|
545
|
+
const end = endOfPrefix(text, start, heard);
|
|
546
|
+
if (end !== -1) return end;
|
|
547
|
+
}
|
|
548
|
+
return -1;
|
|
549
|
+
}
|
|
550
|
+
function endOfPrefix(text, from, heard) {
|
|
551
|
+
let index = from;
|
|
552
|
+
for (let i = 0; i < heard.length; i++) {
|
|
553
|
+
if (isSpace(heard[i])) continue;
|
|
554
|
+
while (index < text.length && isSpace(text[index])) index++;
|
|
555
|
+
if (text[index] !== heard[i]) return -1;
|
|
556
|
+
index++;
|
|
557
|
+
}
|
|
558
|
+
return index;
|
|
559
|
+
}
|
|
560
|
+
function isSpace(char) {
|
|
561
|
+
return char === " " || char === " " || char === "\n" || char === "\r";
|
|
562
|
+
}
|
|
622
563
|
function toWireVoiceSettings(settings) {
|
|
623
564
|
if (!settings) return void 0;
|
|
624
565
|
const { similarityBoost, useSpeakerBoost, ...rest } = settings;
|
|
@@ -630,7 +571,7 @@ function toWireVoiceSettings(settings) {
|
|
|
630
571
|
}
|
|
631
572
|
|
|
632
573
|
// src/ElevenLabsTTS.ts
|
|
633
|
-
var ElevenLabsTTS = class extends
|
|
574
|
+
var ElevenLabsTTS = class extends import_server2.TTS {
|
|
634
575
|
constructor(options) {
|
|
635
576
|
super();
|
|
636
577
|
const host = {
|