kugelaudio 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +30 -1
- package/dist/chunk-CB3B6T7F.mjs +391 -0
- package/dist/chunk-D57AKD2S.mjs +391 -0
- package/dist/index.d.mts +84 -11
- package/dist/index.d.ts +84 -11
- package/dist/index.js +193 -7
- package/dist/index.mjs +186 -334
- package/dist/livekit/index.d.mts +222 -0
- package/dist/livekit/index.d.ts +222 -0
- package/dist/livekit/index.js +994 -0
- package/dist/livekit/index.mjs +746 -0
- package/package.json +23 -3
- package/src/cfg-scale.test.ts +33 -0
- package/src/client.test.ts +110 -0
- package/src/client.ts +213 -5
- package/src/index.ts +2 -0
- package/src/livekit/index.ts +32 -0
- package/src/livekit/models.test.ts +30 -0
- package/src/livekit/models.ts +84 -0
- package/src/livekit/tts.functional.test.ts +348 -0
- package/src/livekit/tts.ts +833 -0
- package/src/livekit/wire.test.ts +112 -0
- package/src/livekit/wire.ts +126 -0
- package/src/types.ts +47 -7
- package/src/utils.ts +16 -0
|
@@ -0,0 +1,746 @@
|
|
|
1
|
+
import {
|
|
2
|
+
__privateAdd,
|
|
3
|
+
__privateGet,
|
|
4
|
+
__privateMethod,
|
|
5
|
+
__privateSet,
|
|
6
|
+
clampCfgScale,
|
|
7
|
+
classifyWsFrame,
|
|
8
|
+
classifyWsHandshakeError,
|
|
9
|
+
package_default
|
|
10
|
+
} from "../chunk-CB3B6T7F.mjs";
|
|
11
|
+
|
|
12
|
+
// src/livekit/tts.ts
|
|
13
|
+
import {
|
|
14
|
+
APIConnectionError,
|
|
15
|
+
APIStatusError,
|
|
16
|
+
APITimeoutError,
|
|
17
|
+
AudioByteStream,
|
|
18
|
+
createTimedString,
|
|
19
|
+
log,
|
|
20
|
+
shortuuid,
|
|
21
|
+
tts
|
|
22
|
+
} from "@livekit/agents";
|
|
23
|
+
import { WebSocket } from "ws";
|
|
24
|
+
|
|
25
|
+
// src/livekit/models.ts
|
|
26
|
+
var DEFAULT_MODEL = "kugel-3";
|
|
27
|
+
var SUPPORTED_SAMPLE_RATES = [24e3, 22050, 16e3, 8e3];
|
|
28
|
+
var DEFAULT_SAMPLE_RATE = 24e3;
|
|
29
|
+
var DEFAULT_VOICE_ID = null;
|
|
30
|
+
var DEFAULT_CFG_SCALE = 2;
|
|
31
|
+
var DEFAULT_MAX_NEW_TOKENS = 2048;
|
|
32
|
+
var SUPPORTED_LANGUAGES = /* @__PURE__ */ new Set([
|
|
33
|
+
// Germanic
|
|
34
|
+
"de",
|
|
35
|
+
"en",
|
|
36
|
+
"nl",
|
|
37
|
+
"sv",
|
|
38
|
+
"da",
|
|
39
|
+
"no",
|
|
40
|
+
// Romance
|
|
41
|
+
"fr",
|
|
42
|
+
"es",
|
|
43
|
+
"it",
|
|
44
|
+
"pt",
|
|
45
|
+
"ro",
|
|
46
|
+
// Slavic
|
|
47
|
+
"pl",
|
|
48
|
+
"cs",
|
|
49
|
+
"uk",
|
|
50
|
+
"bg",
|
|
51
|
+
"sk",
|
|
52
|
+
"sl",
|
|
53
|
+
"hr",
|
|
54
|
+
"sr",
|
|
55
|
+
// Uralic
|
|
56
|
+
"fi",
|
|
57
|
+
"hu",
|
|
58
|
+
// Other European
|
|
59
|
+
"el",
|
|
60
|
+
"tr",
|
|
61
|
+
"ru",
|
|
62
|
+
// CJK + SEA
|
|
63
|
+
"zh",
|
|
64
|
+
"ja",
|
|
65
|
+
"ko",
|
|
66
|
+
"vi",
|
|
67
|
+
"yue",
|
|
68
|
+
"th",
|
|
69
|
+
"id",
|
|
70
|
+
"ms",
|
|
71
|
+
// Semitic + Indic + Other
|
|
72
|
+
"ar",
|
|
73
|
+
"hi",
|
|
74
|
+
"he",
|
|
75
|
+
"fa",
|
|
76
|
+
"ur",
|
|
77
|
+
"bn",
|
|
78
|
+
"ta"
|
|
79
|
+
]);
|
|
80
|
+
function validateLanguage(language) {
|
|
81
|
+
if (language === null || language === void 0) return void 0;
|
|
82
|
+
if (!SUPPORTED_LANGUAGES.has(language)) {
|
|
83
|
+
const supported = [...SUPPORTED_LANGUAGES].sort().join(", ");
|
|
84
|
+
throw new Error(
|
|
85
|
+
`language must be a supported ISO 639-1 code (${supported}), got '${language}'. Note: BCP 47 tags like 'de-DE' are not accepted \u2014 use 'de' instead.`
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
return language;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// src/livekit/wire.ts
|
|
92
|
+
var SDK_NAME = "js";
|
|
93
|
+
var SDK_VERSION = package_default.version;
|
|
94
|
+
function appendSdkQuery(url) {
|
|
95
|
+
const separator = url.includes("?") ? "&" : "?";
|
|
96
|
+
return `${url}${separator}sdk=${encodeURIComponent(SDK_NAME)}&sdk_version=${encodeURIComponent(SDK_VERSION)}`;
|
|
97
|
+
}
|
|
98
|
+
function buildMultiWsUrl(baseUrl, apiKey) {
|
|
99
|
+
const wsBase = baseUrl.replace("https://", "wss://").replace("http://", "ws://");
|
|
100
|
+
return appendSdkQuery(`${wsBase}/ws/tts/multi?api_key=${apiKey}`);
|
|
101
|
+
}
|
|
102
|
+
function buildTextPayload(contextId, text, opts, { flush = false, includeConfig = false } = {}) {
|
|
103
|
+
const msg = { text, context_id: contextId };
|
|
104
|
+
if (flush) msg.flush = true;
|
|
105
|
+
if (includeConfig) {
|
|
106
|
+
msg.model_id = opts.model;
|
|
107
|
+
msg.sample_rate = opts.sampleRate;
|
|
108
|
+
msg.word_timestamps = opts.wordTimestamps;
|
|
109
|
+
msg.normalize = opts.normalize;
|
|
110
|
+
if (opts.language !== void 0) msg.language = opts.language;
|
|
111
|
+
const voiceSettings = {};
|
|
112
|
+
if (opts.voiceId !== null && opts.voiceId !== void 0) {
|
|
113
|
+
voiceSettings.voice_id = opts.voiceId;
|
|
114
|
+
}
|
|
115
|
+
if (opts.cfgScale !== 2) voiceSettings.cfg_scale = opts.cfgScale;
|
|
116
|
+
if (opts.maxNewTokens !== 2048) voiceSettings.max_new_tokens = opts.maxNewTokens;
|
|
117
|
+
if (Object.keys(voiceSettings).length > 0) msg.voice_settings = voiceSettings;
|
|
118
|
+
}
|
|
119
|
+
return msg;
|
|
120
|
+
}
|
|
121
|
+
function buildClosePayload(contextId, immediate = false) {
|
|
122
|
+
const msg = {
|
|
123
|
+
close_context: true,
|
|
124
|
+
context_id: contextId
|
|
125
|
+
};
|
|
126
|
+
if (immediate) msg.immediate = true;
|
|
127
|
+
return msg;
|
|
128
|
+
}
|
|
129
|
+
function wordTimestampsToTimed(timestamps) {
|
|
130
|
+
return timestamps.map((ts) => ({
|
|
131
|
+
text: ts.word,
|
|
132
|
+
startTime: ts.start_ms / 1e3,
|
|
133
|
+
endTime: ts.end_ms / 1e3
|
|
134
|
+
}));
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// src/livekit/tts.ts
|
|
138
|
+
var NUM_CHANNELS = 1;
|
|
139
|
+
var DEFAULT_BASE_URL = "https://api.kugelaudio.com";
|
|
140
|
+
var EU_BASE_URL = "https://api.eu.kugelaudio.com";
|
|
141
|
+
function parseApiKey(apiKey) {
|
|
142
|
+
for (const prefix of ["eu-", "us-", "global-"]) {
|
|
143
|
+
if (apiKey.startsWith(prefix)) {
|
|
144
|
+
return { cleanKey: apiKey.slice(prefix.length), region: prefix.slice(0, -1) };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return { cleanKey: apiKey };
|
|
148
|
+
}
|
|
149
|
+
function apiStatusErrorFromFrame(data, requestId) {
|
|
150
|
+
const err = classifyWsFrame(data);
|
|
151
|
+
const statusCode = err.statusCode ?? (typeof data.code === "number" ? data.code : 500);
|
|
152
|
+
return new APIStatusError({
|
|
153
|
+
message: err.message,
|
|
154
|
+
options: { statusCode, requestId, body: data }
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
function toLiveKitConnError(err) {
|
|
158
|
+
if (err instanceof Error && /handshake has timed out/i.test(err.message)) {
|
|
159
|
+
return new APITimeoutError({ message: "Timed out connecting to /ws/tts/multi" });
|
|
160
|
+
}
|
|
161
|
+
const typed = classifyWsHandshakeError(err);
|
|
162
|
+
if (typed && typed.statusCode !== void 0) {
|
|
163
|
+
return new APIStatusError({
|
|
164
|
+
message: typed.message,
|
|
165
|
+
options: { statusCode: typed.statusCode }
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
169
|
+
return new APIConnectionError({ message: `Failed to connect to /ws/tts/multi: ${message}` });
|
|
170
|
+
}
|
|
171
|
+
function bufferToArrayBuffer(buf) {
|
|
172
|
+
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
173
|
+
}
|
|
174
|
+
function deferred() {
|
|
175
|
+
const d = { settled: false };
|
|
176
|
+
d.promise = new Promise((resolve, reject) => {
|
|
177
|
+
d.resolve = () => {
|
|
178
|
+
if (d.settled) return;
|
|
179
|
+
d.settled = true;
|
|
180
|
+
resolve();
|
|
181
|
+
};
|
|
182
|
+
d.reject = (err) => {
|
|
183
|
+
if (d.settled) return;
|
|
184
|
+
d.settled = true;
|
|
185
|
+
reject(err);
|
|
186
|
+
};
|
|
187
|
+
});
|
|
188
|
+
d.promise.catch(() => {
|
|
189
|
+
});
|
|
190
|
+
return d;
|
|
191
|
+
}
|
|
192
|
+
function delay(ms) {
|
|
193
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
194
|
+
}
|
|
195
|
+
var _tail;
|
|
196
|
+
var Mutex = class {
|
|
197
|
+
constructor() {
|
|
198
|
+
__privateAdd(this, _tail, Promise.resolve());
|
|
199
|
+
}
|
|
200
|
+
async lock() {
|
|
201
|
+
let release;
|
|
202
|
+
const next = new Promise((resolve) => {
|
|
203
|
+
release = resolve;
|
|
204
|
+
});
|
|
205
|
+
const prev = __privateGet(this, _tail);
|
|
206
|
+
__privateSet(this, _tail, __privateGet(this, _tail).then(() => next));
|
|
207
|
+
await prev;
|
|
208
|
+
return release;
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
_tail = new WeakMap();
|
|
212
|
+
var _bstream, _lastFrame, _pendingTimed, _ended, _AudioSink_instances, emit_fn;
|
|
213
|
+
var AudioSink = class {
|
|
214
|
+
constructor(sampleRate, put, requestId, segmentId) {
|
|
215
|
+
this.put = put;
|
|
216
|
+
this.requestId = requestId;
|
|
217
|
+
this.segmentId = segmentId;
|
|
218
|
+
__privateAdd(this, _AudioSink_instances);
|
|
219
|
+
__privateAdd(this, _bstream);
|
|
220
|
+
__privateAdd(this, _lastFrame);
|
|
221
|
+
__privateAdd(this, _pendingTimed, []);
|
|
222
|
+
__privateAdd(this, _ended, false);
|
|
223
|
+
__privateSet(this, _bstream, new AudioByteStream(sampleRate, NUM_CHANNELS));
|
|
224
|
+
}
|
|
225
|
+
pushAudio(bytes) {
|
|
226
|
+
if (__privateGet(this, _ended)) return;
|
|
227
|
+
for (const frame of __privateGet(this, _bstream).write(bufferToArrayBuffer(bytes))) {
|
|
228
|
+
__privateMethod(this, _AudioSink_instances, emit_fn).call(this, false);
|
|
229
|
+
__privateSet(this, _lastFrame, frame);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
pushTimed(words) {
|
|
233
|
+
if (__privateGet(this, _ended)) return;
|
|
234
|
+
__privateGet(this, _pendingTimed).push(...words);
|
|
235
|
+
}
|
|
236
|
+
/** Flush any buffered audio and emit the terminal (`final`) frame. */
|
|
237
|
+
end() {
|
|
238
|
+
if (__privateGet(this, _ended)) return;
|
|
239
|
+
__privateSet(this, _ended, true);
|
|
240
|
+
for (const frame of __privateGet(this, _bstream).flush()) {
|
|
241
|
+
__privateMethod(this, _AudioSink_instances, emit_fn).call(this, false);
|
|
242
|
+
__privateSet(this, _lastFrame, frame);
|
|
243
|
+
}
|
|
244
|
+
__privateMethod(this, _AudioSink_instances, emit_fn).call(this, true);
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
_bstream = new WeakMap();
|
|
248
|
+
_lastFrame = new WeakMap();
|
|
249
|
+
_pendingTimed = new WeakMap();
|
|
250
|
+
_ended = new WeakMap();
|
|
251
|
+
_AudioSink_instances = new WeakSet();
|
|
252
|
+
emit_fn = function(final) {
|
|
253
|
+
if (!__privateGet(this, _lastFrame)) return;
|
|
254
|
+
this.put({
|
|
255
|
+
requestId: this.requestId,
|
|
256
|
+
segmentId: this.segmentId,
|
|
257
|
+
frame: __privateGet(this, _lastFrame),
|
|
258
|
+
final,
|
|
259
|
+
timedTranscripts: __privateGet(this, _pendingTimed).length > 0 ? __privateGet(this, _pendingTimed) : void 0
|
|
260
|
+
});
|
|
261
|
+
__privateSet(this, _lastFrame, void 0);
|
|
262
|
+
__privateSet(this, _pendingTimed, []);
|
|
263
|
+
};
|
|
264
|
+
var _opts, _ws, _isCurrent, _closed, _activeContexts, _contexts, _inputQueue, _inputResolver, _sendTask, _recvTask, _logger, _Connection_instances, enqueue_fn, sendLoop_fn, recvLoop_fn, handleMessage_fn;
|
|
265
|
+
var Connection = class {
|
|
266
|
+
constructor(opts) {
|
|
267
|
+
__privateAdd(this, _Connection_instances);
|
|
268
|
+
__privateAdd(this, _opts);
|
|
269
|
+
__privateAdd(this, _ws, null);
|
|
270
|
+
__privateAdd(this, _isCurrent, true);
|
|
271
|
+
__privateAdd(this, _closed, false);
|
|
272
|
+
__privateAdd(this, _activeContexts, /* @__PURE__ */ new Set());
|
|
273
|
+
__privateAdd(this, _contexts, /* @__PURE__ */ new Map());
|
|
274
|
+
__privateAdd(this, _inputQueue, []);
|
|
275
|
+
__privateAdd(this, _inputResolver, null);
|
|
276
|
+
__privateAdd(this, _sendTask, null);
|
|
277
|
+
__privateAdd(this, _recvTask, null);
|
|
278
|
+
__privateAdd(this, _logger, log());
|
|
279
|
+
__privateSet(this, _opts, opts);
|
|
280
|
+
}
|
|
281
|
+
get isCurrent() {
|
|
282
|
+
return __privateGet(this, _isCurrent);
|
|
283
|
+
}
|
|
284
|
+
get closed() {
|
|
285
|
+
return __privateGet(this, _closed);
|
|
286
|
+
}
|
|
287
|
+
markNonCurrent() {
|
|
288
|
+
__privateSet(this, _isCurrent, false);
|
|
289
|
+
}
|
|
290
|
+
async connect(timeoutMs = 1e4) {
|
|
291
|
+
const url = buildMultiWsUrl(__privateGet(this, _opts).baseURL, __privateGet(this, _opts).apiKey);
|
|
292
|
+
await new Promise((resolve, reject) => {
|
|
293
|
+
let opened = false;
|
|
294
|
+
const ws = new WebSocket(url, { handshakeTimeout: timeoutMs });
|
|
295
|
+
__privateSet(this, _ws, ws);
|
|
296
|
+
ws.on("open", () => {
|
|
297
|
+
opened = true;
|
|
298
|
+
__privateSet(this, _sendTask, __privateMethod(this, _Connection_instances, sendLoop_fn).call(this));
|
|
299
|
+
__privateSet(this, _recvTask, __privateMethod(this, _Connection_instances, recvLoop_fn).call(this));
|
|
300
|
+
resolve();
|
|
301
|
+
});
|
|
302
|
+
ws.on("unexpected-response", (_req, res) => {
|
|
303
|
+
if (!opened) {
|
|
304
|
+
reject(toLiveKitConnError({ statusCode: res.statusCode, message: res.statusMessage }));
|
|
305
|
+
}
|
|
306
|
+
});
|
|
307
|
+
ws.on("error", (err) => {
|
|
308
|
+
if (!opened) reject(toLiveKitConnError(err));
|
|
309
|
+
});
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
registerContext(contextId, sink, waiter, isStreaming) {
|
|
313
|
+
if (__privateGet(this, _closed)) {
|
|
314
|
+
waiter.reject(
|
|
315
|
+
new APIConnectionError({ message: "Connection closed before request started" })
|
|
316
|
+
);
|
|
317
|
+
return null;
|
|
318
|
+
}
|
|
319
|
+
const ctx = {
|
|
320
|
+
sink,
|
|
321
|
+
waiter,
|
|
322
|
+
isStreaming,
|
|
323
|
+
lastActivityAt: Date.now()
|
|
324
|
+
};
|
|
325
|
+
__privateGet(this, _contexts).set(contextId, ctx);
|
|
326
|
+
return ctx;
|
|
327
|
+
}
|
|
328
|
+
sendText(contextId, text, flush) {
|
|
329
|
+
const includeConfig = !__privateGet(this, _activeContexts).has(contextId);
|
|
330
|
+
if (includeConfig) __privateGet(this, _activeContexts).add(contextId);
|
|
331
|
+
__privateMethod(this, _Connection_instances, enqueue_fn).call(this, buildTextPayload(contextId, text, __privateGet(this, _opts), { flush, includeConfig }));
|
|
332
|
+
}
|
|
333
|
+
closeContext(contextId, immediate) {
|
|
334
|
+
__privateMethod(this, _Connection_instances, enqueue_fn).call(this, buildClosePayload(contextId, immediate));
|
|
335
|
+
}
|
|
336
|
+
cleanupContext(contextId) {
|
|
337
|
+
__privateGet(this, _contexts).delete(contextId);
|
|
338
|
+
__privateGet(this, _activeContexts).delete(contextId);
|
|
339
|
+
}
|
|
340
|
+
async close() {
|
|
341
|
+
var _a;
|
|
342
|
+
if (__privateGet(this, _closed)) return;
|
|
343
|
+
__privateSet(this, _closed, true);
|
|
344
|
+
(_a = __privateGet(this, _inputResolver)) == null ? void 0 : _a.call(this);
|
|
345
|
+
for (const ctx of __privateGet(this, _contexts).values()) {
|
|
346
|
+
ctx.waiter.reject(new APIConnectionError({ message: "Connection closed" }));
|
|
347
|
+
}
|
|
348
|
+
__privateGet(this, _contexts).clear();
|
|
349
|
+
if (__privateGet(this, _ws)) {
|
|
350
|
+
try {
|
|
351
|
+
if (__privateGet(this, _ws).readyState === WebSocket.OPEN) {
|
|
352
|
+
__privateGet(this, _ws).send(JSON.stringify({ close_socket: true }));
|
|
353
|
+
}
|
|
354
|
+
__privateGet(this, _ws).close();
|
|
355
|
+
} catch {
|
|
356
|
+
}
|
|
357
|
+
__privateSet(this, _ws, null);
|
|
358
|
+
}
|
|
359
|
+
await __privateGet(this, _sendTask)?.catch(() => {
|
|
360
|
+
});
|
|
361
|
+
await __privateGet(this, _recvTask)?.catch(() => {
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
};
|
|
365
|
+
_opts = new WeakMap();
|
|
366
|
+
_ws = new WeakMap();
|
|
367
|
+
_isCurrent = new WeakMap();
|
|
368
|
+
_closed = new WeakMap();
|
|
369
|
+
_activeContexts = new WeakMap();
|
|
370
|
+
_contexts = new WeakMap();
|
|
371
|
+
_inputQueue = new WeakMap();
|
|
372
|
+
_inputResolver = new WeakMap();
|
|
373
|
+
_sendTask = new WeakMap();
|
|
374
|
+
_recvTask = new WeakMap();
|
|
375
|
+
_logger = new WeakMap();
|
|
376
|
+
_Connection_instances = new WeakSet();
|
|
377
|
+
enqueue_fn = function(payload) {
|
|
378
|
+
var _a;
|
|
379
|
+
if (__privateGet(this, _closed)) return;
|
|
380
|
+
__privateGet(this, _inputQueue).push(payload);
|
|
381
|
+
(_a = __privateGet(this, _inputResolver)) == null ? void 0 : _a.call(this);
|
|
382
|
+
};
|
|
383
|
+
sendLoop_fn = async function() {
|
|
384
|
+
try {
|
|
385
|
+
while (!__privateGet(this, _closed)) {
|
|
386
|
+
if (__privateGet(this, _inputQueue).length === 0) {
|
|
387
|
+
await new Promise((resolve) => {
|
|
388
|
+
__privateSet(this, _inputResolver, resolve);
|
|
389
|
+
});
|
|
390
|
+
__privateSet(this, _inputResolver, null);
|
|
391
|
+
}
|
|
392
|
+
if (__privateGet(this, _closed)) break;
|
|
393
|
+
const payload = __privateGet(this, _inputQueue).shift();
|
|
394
|
+
if (!payload) continue;
|
|
395
|
+
if (!__privateGet(this, _ws) || __privateGet(this, _ws).readyState !== WebSocket.OPEN) break;
|
|
396
|
+
__privateGet(this, _ws).send(JSON.stringify(payload));
|
|
397
|
+
}
|
|
398
|
+
} catch (err) {
|
|
399
|
+
__privateGet(this, _logger).warn({ error: err }, "kugelaudio livekit send loop error");
|
|
400
|
+
__privateSet(this, _isCurrent, false);
|
|
401
|
+
if (__privateGet(this, _ws) && __privateGet(this, _ws).readyState === WebSocket.OPEN) __privateGet(this, _ws).close();
|
|
402
|
+
}
|
|
403
|
+
};
|
|
404
|
+
recvLoop_fn = async function() {
|
|
405
|
+
const ws = __privateGet(this, _ws);
|
|
406
|
+
if (!ws) return;
|
|
407
|
+
try {
|
|
408
|
+
await new Promise((resolve) => {
|
|
409
|
+
const onMessage = (raw) => {
|
|
410
|
+
const text = typeof raw === "string" ? raw : Array.isArray(raw) ? Buffer.concat(raw).toString() : raw instanceof ArrayBuffer ? Buffer.from(raw).toString() : raw.toString();
|
|
411
|
+
try {
|
|
412
|
+
__privateMethod(this, _Connection_instances, handleMessage_fn).call(this, JSON.parse(text));
|
|
413
|
+
} catch (err) {
|
|
414
|
+
__privateGet(this, _logger).warn({ error: err }, "failed to parse /ws/tts/multi message");
|
|
415
|
+
}
|
|
416
|
+
};
|
|
417
|
+
const finish = () => {
|
|
418
|
+
ws.off("message", onMessage);
|
|
419
|
+
ws.off("close", finish);
|
|
420
|
+
ws.off("error", finish);
|
|
421
|
+
resolve();
|
|
422
|
+
};
|
|
423
|
+
ws.on("message", onMessage);
|
|
424
|
+
ws.on("close", finish);
|
|
425
|
+
ws.on("error", finish);
|
|
426
|
+
});
|
|
427
|
+
} finally {
|
|
428
|
+
__privateSet(this, _isCurrent, false);
|
|
429
|
+
for (const ctx of __privateGet(this, _contexts).values()) {
|
|
430
|
+
ctx.waiter.reject(
|
|
431
|
+
new APIConnectionError({ message: "WebSocket connection closed unexpectedly" })
|
|
432
|
+
);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
};
|
|
436
|
+
handleMessage_fn = function(data) {
|
|
437
|
+
if (data.session_closed) {
|
|
438
|
+
__privateSet(this, _isCurrent, false);
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
const contextId = typeof data.context_id === "string" ? data.context_id : void 0;
|
|
442
|
+
if (data.error) {
|
|
443
|
+
if (contextId) {
|
|
444
|
+
const ctx2 = __privateGet(this, _contexts).get(contextId);
|
|
445
|
+
ctx2?.waiter.reject(apiStatusErrorFromFrame(data, contextId));
|
|
446
|
+
this.cleanupContext(contextId);
|
|
447
|
+
}
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
const ctx = contextId ? __privateGet(this, _contexts).get(contextId) : void 0;
|
|
451
|
+
if (!ctx) return;
|
|
452
|
+
ctx.lastActivityAt = Date.now();
|
|
453
|
+
if (data.context_timeout) {
|
|
454
|
+
ctx.waiter.reject(new APITimeoutError({ message: "context timed out" }));
|
|
455
|
+
this.cleanupContext(contextId);
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
if (typeof data.audio === "string") {
|
|
459
|
+
ctx.sink.pushAudio(Buffer.from(data.audio, "base64"));
|
|
460
|
+
}
|
|
461
|
+
if (Array.isArray(data.word_timestamps)) {
|
|
462
|
+
ctx.sink.pushTimed(
|
|
463
|
+
wordTimestampsToTimed(data.word_timestamps).map((w) => createTimedString(w))
|
|
464
|
+
);
|
|
465
|
+
}
|
|
466
|
+
if (data.chunk_complete && !ctx.isStreaming) {
|
|
467
|
+
this.closeContext(contextId, false);
|
|
468
|
+
}
|
|
469
|
+
if (data.context_closed) {
|
|
470
|
+
ctx.sink.end();
|
|
471
|
+
ctx.waiter.resolve();
|
|
472
|
+
this.cleanupContext(contextId);
|
|
473
|
+
}
|
|
474
|
+
};
|
|
475
|
+
async function waitForContextIdle(ctx, idleThresholdMs) {
|
|
476
|
+
const tick = Math.min(1e3, Math.max(100, idleThresholdMs / 4));
|
|
477
|
+
for (; ; ) {
|
|
478
|
+
const result = await Promise.race([
|
|
479
|
+
ctx.waiter.promise.then(() => "done"),
|
|
480
|
+
delay(tick).then(() => "tick")
|
|
481
|
+
]);
|
|
482
|
+
if (result === "done") return;
|
|
483
|
+
if (Date.now() - ctx.lastActivityAt >= idleThresholdMs) {
|
|
484
|
+
throw new APITimeoutError({ message: "timed out waiting for KugelAudio audio" });
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
var _opts2, _streams, _currentConnection, _connectionLock, _logger2;
|
|
489
|
+
var TTS = class extends tts.TTS {
|
|
490
|
+
constructor(opts = {}) {
|
|
491
|
+
const sampleRate = opts.sampleRate ?? DEFAULT_SAMPLE_RATE;
|
|
492
|
+
const wordTimestamps = opts.wordTimestamps ?? false;
|
|
493
|
+
super(sampleRate, NUM_CHANNELS, {
|
|
494
|
+
streaming: true,
|
|
495
|
+
alignedTranscript: wordTimestamps
|
|
496
|
+
});
|
|
497
|
+
__privateAdd(this, _opts2);
|
|
498
|
+
__privateAdd(this, _streams, /* @__PURE__ */ new Set());
|
|
499
|
+
__privateAdd(this, _currentConnection, null);
|
|
500
|
+
__privateAdd(this, _connectionLock, new Mutex());
|
|
501
|
+
__privateAdd(this, _logger2, log());
|
|
502
|
+
this.label = "kugelaudio.TTS";
|
|
503
|
+
const apiKey = opts.apiKey ?? process.env.KUGELAUDIO_API_KEY;
|
|
504
|
+
if (!apiKey) {
|
|
505
|
+
throw new Error(
|
|
506
|
+
"KUGELAUDIO_API_KEY must be set or apiKey must be provided to the TTS constructor."
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
const { cleanKey } = parseApiKey(apiKey);
|
|
510
|
+
const language = validateLanguage(opts.language);
|
|
511
|
+
let baseURL;
|
|
512
|
+
if (opts.baseURL) {
|
|
513
|
+
baseURL = opts.baseURL.replace(/\/$/, "");
|
|
514
|
+
} else {
|
|
515
|
+
const { region } = parseApiKey(apiKey);
|
|
516
|
+
baseURL = region === "eu" ? EU_BASE_URL : DEFAULT_BASE_URL;
|
|
517
|
+
}
|
|
518
|
+
__privateSet(this, _opts2, {
|
|
519
|
+
apiKey: cleanKey,
|
|
520
|
+
baseURL,
|
|
521
|
+
model: opts.model ?? DEFAULT_MODEL,
|
|
522
|
+
voiceId: opts.voiceId ?? DEFAULT_VOICE_ID,
|
|
523
|
+
sampleRate,
|
|
524
|
+
cfgScale: clampCfgScale(opts.cfgScale ?? DEFAULT_CFG_SCALE) ?? DEFAULT_CFG_SCALE,
|
|
525
|
+
maxNewTokens: opts.maxNewTokens ?? DEFAULT_MAX_NEW_TOKENS,
|
|
526
|
+
wordTimestamps,
|
|
527
|
+
normalize: opts.normalize ?? true,
|
|
528
|
+
language
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
get model() {
|
|
532
|
+
return __privateGet(this, _opts2).model;
|
|
533
|
+
}
|
|
534
|
+
get provider() {
|
|
535
|
+
return "KugelAudio";
|
|
536
|
+
}
|
|
537
|
+
/** Get or create the persistent `/ws/tts/multi` connection. */
|
|
538
|
+
async currentConnection(timeoutMs = 1e4) {
|
|
539
|
+
const unlock = await __privateGet(this, _connectionLock).lock();
|
|
540
|
+
try {
|
|
541
|
+
if (__privateGet(this, _currentConnection) && __privateGet(this, _currentConnection).isCurrent && !__privateGet(this, _currentConnection).closed) {
|
|
542
|
+
return __privateGet(this, _currentConnection);
|
|
543
|
+
}
|
|
544
|
+
const conn = new Connection({ ...__privateGet(this, _opts2) });
|
|
545
|
+
await conn.connect(timeoutMs);
|
|
546
|
+
__privateSet(this, _currentConnection, conn);
|
|
547
|
+
return conn;
|
|
548
|
+
} finally {
|
|
549
|
+
unlock();
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Eagerly establish the WebSocket connection to remove handshake latency
|
|
554
|
+
* from the first synthesis. Errors are logged and retried on first use.
|
|
555
|
+
*/
|
|
556
|
+
prewarm() {
|
|
557
|
+
this.currentConnection().then(() => __privateGet(this, _logger2).info("KugelAudio TTS connection pre-warmed")).catch(
|
|
558
|
+
(err) => __privateGet(this, _logger2).warn(
|
|
559
|
+
{ error: err },
|
|
560
|
+
"Failed to prewarm KugelAudio connection; will retry on first synthesis"
|
|
561
|
+
)
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Update TTS options at runtime. Changing any option marks the current
|
|
566
|
+
* connection non-current so the next synthesis opens a fresh one.
|
|
567
|
+
*/
|
|
568
|
+
updateOptions(opts) {
|
|
569
|
+
let changed = false;
|
|
570
|
+
const set = (key, value) => {
|
|
571
|
+
if (__privateGet(this, _opts2)[key] !== value) {
|
|
572
|
+
__privateGet(this, _opts2)[key] = value;
|
|
573
|
+
changed = true;
|
|
574
|
+
}
|
|
575
|
+
};
|
|
576
|
+
if (opts.model !== void 0) set("model", opts.model);
|
|
577
|
+
if (opts.voiceId !== void 0) set("voiceId", opts.voiceId);
|
|
578
|
+
if (opts.cfgScale !== void 0) {
|
|
579
|
+
set("cfgScale", clampCfgScale(opts.cfgScale) ?? DEFAULT_CFG_SCALE);
|
|
580
|
+
}
|
|
581
|
+
if (opts.maxNewTokens !== void 0) set("maxNewTokens", opts.maxNewTokens);
|
|
582
|
+
if (opts.normalize !== void 0) set("normalize", opts.normalize);
|
|
583
|
+
if (opts.wordTimestamps !== void 0) set("wordTimestamps", opts.wordTimestamps);
|
|
584
|
+
if (opts.language !== void 0) set("language", validateLanguage(opts.language));
|
|
585
|
+
if (changed && __privateGet(this, _currentConnection)) {
|
|
586
|
+
const old = __privateGet(this, _currentConnection);
|
|
587
|
+
old.markNonCurrent();
|
|
588
|
+
__privateSet(this, _currentConnection, null);
|
|
589
|
+
old.close().catch(() => {
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
synthesize(text, connOptions, abortSignal) {
|
|
594
|
+
return new ChunkedStream(this, text, { ...__privateGet(this, _opts2) }, connOptions, abortSignal);
|
|
595
|
+
}
|
|
596
|
+
stream(options) {
|
|
597
|
+
const stream = new SynthesizeStream(this, { ...__privateGet(this, _opts2) }, options?.connOptions);
|
|
598
|
+
__privateGet(this, _streams).add(stream);
|
|
599
|
+
stream.abortSignal.addEventListener("abort", () => __privateGet(this, _streams).delete(stream), {
|
|
600
|
+
once: true
|
|
601
|
+
});
|
|
602
|
+
return stream;
|
|
603
|
+
}
|
|
604
|
+
async close() {
|
|
605
|
+
for (const stream of __privateGet(this, _streams)) stream.close();
|
|
606
|
+
__privateGet(this, _streams).clear();
|
|
607
|
+
if (__privateGet(this, _currentConnection)) {
|
|
608
|
+
await __privateGet(this, _currentConnection).close();
|
|
609
|
+
__privateSet(this, _currentConnection, null);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
};
|
|
613
|
+
_opts2 = new WeakMap();
|
|
614
|
+
_streams = new WeakMap();
|
|
615
|
+
_currentConnection = new WeakMap();
|
|
616
|
+
_connectionLock = new WeakMap();
|
|
617
|
+
_logger2 = new WeakMap();
|
|
618
|
+
var _tts, _opts3, _connOptions;
|
|
619
|
+
var ChunkedStream = class extends tts.ChunkedStream {
|
|
620
|
+
constructor(ttsInstance, text, opts, connOptions, abortSignal) {
|
|
621
|
+
super(text, ttsInstance, connOptions, abortSignal);
|
|
622
|
+
__privateAdd(this, _tts);
|
|
623
|
+
__privateAdd(this, _opts3);
|
|
624
|
+
__privateAdd(this, _connOptions);
|
|
625
|
+
this.label = "kugelaudio.ChunkedStream";
|
|
626
|
+
__privateSet(this, _tts, ttsInstance);
|
|
627
|
+
__privateSet(this, _opts3, opts);
|
|
628
|
+
__privateSet(this, _connOptions, connOptions ?? { maxRetry: 3, retryIntervalMs: 2e3, timeoutMs: 1e4 });
|
|
629
|
+
}
|
|
630
|
+
async run() {
|
|
631
|
+
const contextId = shortuuid();
|
|
632
|
+
const connection = await __privateGet(this, _tts).currentConnection(__privateGet(this, _connOptions).timeoutMs);
|
|
633
|
+
const sink = new AudioSink(__privateGet(this, _opts3).sampleRate, (a) => this.queue.put(a), contextId, contextId);
|
|
634
|
+
const waiter = deferred();
|
|
635
|
+
const ctx = connection.registerContext(contextId, sink, waiter, false);
|
|
636
|
+
if (!ctx) {
|
|
637
|
+
await waiter.promise;
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
const onAbort = () => waiter.reject(new Error("aborted"));
|
|
641
|
+
this.abortController.signal.addEventListener("abort", onAbort, { once: true });
|
|
642
|
+
try {
|
|
643
|
+
connection.sendText(contextId, this.inputText, true);
|
|
644
|
+
await waitForContextIdle(ctx, __privateGet(this, _connOptions).timeoutMs);
|
|
645
|
+
} catch (err) {
|
|
646
|
+
connection.closeContext(contextId, true);
|
|
647
|
+
connection.cleanupContext(contextId);
|
|
648
|
+
if (this.abortController.signal.aborted) return;
|
|
649
|
+
throw err;
|
|
650
|
+
} finally {
|
|
651
|
+
this.abortController.signal.removeEventListener("abort", onAbort);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
};
|
|
655
|
+
_tts = new WeakMap();
|
|
656
|
+
_opts3 = new WeakMap();
|
|
657
|
+
_connOptions = new WeakMap();
|
|
658
|
+
var _tts2, _opts4, _contextId;
|
|
659
|
+
var _SynthesizeStream = class _SynthesizeStream extends tts.SynthesizeStream {
|
|
660
|
+
constructor(ttsInstance, opts, connOptions) {
|
|
661
|
+
super(ttsInstance, connOptions);
|
|
662
|
+
__privateAdd(this, _tts2);
|
|
663
|
+
__privateAdd(this, _opts4);
|
|
664
|
+
__privateAdd(this, _contextId);
|
|
665
|
+
this.label = "kugelaudio.SynthesizeStream";
|
|
666
|
+
__privateSet(this, _tts2, ttsInstance);
|
|
667
|
+
__privateSet(this, _opts4, opts);
|
|
668
|
+
}
|
|
669
|
+
/** The server-side context id of the current (latest) run attempt. */
|
|
670
|
+
get contextId() {
|
|
671
|
+
return __privateGet(this, _contextId);
|
|
672
|
+
}
|
|
673
|
+
async run() {
|
|
674
|
+
const contextId = shortuuid();
|
|
675
|
+
__privateSet(this, _contextId, contextId);
|
|
676
|
+
const timeoutMs = this.connOptions.timeoutMs;
|
|
677
|
+
const connection = await __privateGet(this, _tts2).currentConnection(timeoutMs);
|
|
678
|
+
const sink = new AudioSink(__privateGet(this, _opts4).sampleRate, (a) => this.queue.put(a), contextId, contextId);
|
|
679
|
+
const waiter = deferred();
|
|
680
|
+
const ctx = connection.registerContext(contextId, sink, waiter, true);
|
|
681
|
+
if (!ctx) {
|
|
682
|
+
await waiter.promise;
|
|
683
|
+
return;
|
|
684
|
+
}
|
|
685
|
+
const onAbort = () => waiter.reject(new Error("aborted"));
|
|
686
|
+
this.abortController.signal.addEventListener("abort", onAbort, { once: true });
|
|
687
|
+
const STOP = /* @__PURE__ */ Symbol("stop");
|
|
688
|
+
const stopInput = deferred();
|
|
689
|
+
let failed = false;
|
|
690
|
+
const inputTask = async () => {
|
|
691
|
+
for (; ; ) {
|
|
692
|
+
const result = await Promise.race([
|
|
693
|
+
this.input.next(),
|
|
694
|
+
stopInput.promise.then(() => STOP)
|
|
695
|
+
]);
|
|
696
|
+
if (typeof result === "symbol") return;
|
|
697
|
+
if (result.done) break;
|
|
698
|
+
if (this.abortController.signal.aborted || failed) return;
|
|
699
|
+
const data = result.value;
|
|
700
|
+
if (data === _SynthesizeStream.FLUSH_SENTINEL) {
|
|
701
|
+
connection.sendText(contextId, "", true);
|
|
702
|
+
continue;
|
|
703
|
+
}
|
|
704
|
+
if (!data) continue;
|
|
705
|
+
connection.sendText(contextId, data, false);
|
|
706
|
+
}
|
|
707
|
+
if (!this.abortController.signal.aborted && !failed) {
|
|
708
|
+
connection.sendText(contextId, "", true);
|
|
709
|
+
connection.closeContext(contextId, false);
|
|
710
|
+
}
|
|
711
|
+
};
|
|
712
|
+
const input = inputTask();
|
|
713
|
+
input.catch((err) => waiter.reject(err instanceof Error ? err : new Error(String(err))));
|
|
714
|
+
try {
|
|
715
|
+
await waitForContextIdle(ctx, timeoutMs);
|
|
716
|
+
} catch (err) {
|
|
717
|
+
failed = true;
|
|
718
|
+
connection.closeContext(contextId, true);
|
|
719
|
+
connection.cleanupContext(contextId);
|
|
720
|
+
if (this.abortController.signal.aborted) return;
|
|
721
|
+
throw err;
|
|
722
|
+
} finally {
|
|
723
|
+
this.abortController.signal.removeEventListener("abort", onAbort);
|
|
724
|
+
stopInput.resolve();
|
|
725
|
+
await input.catch(() => {
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
};
|
|
730
|
+
_tts2 = new WeakMap();
|
|
731
|
+
_opts4 = new WeakMap();
|
|
732
|
+
_contextId = new WeakMap();
|
|
733
|
+
var SynthesizeStream = _SynthesizeStream;
|
|
734
|
+
export {
|
|
735
|
+
ChunkedStream,
|
|
736
|
+
DEFAULT_CFG_SCALE,
|
|
737
|
+
DEFAULT_MAX_NEW_TOKENS,
|
|
738
|
+
DEFAULT_MODEL,
|
|
739
|
+
DEFAULT_SAMPLE_RATE,
|
|
740
|
+
DEFAULT_VOICE_ID,
|
|
741
|
+
SUPPORTED_LANGUAGES,
|
|
742
|
+
SUPPORTED_SAMPLE_RATES,
|
|
743
|
+
SynthesizeStream,
|
|
744
|
+
TTS,
|
|
745
|
+
validateLanguage
|
|
746
|
+
};
|