kugelaudio 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +41 -3
- package/dist/chunk-CB3B6T7F.mjs +391 -0
- package/dist/chunk-D57AKD2S.mjs +391 -0
- package/dist/index.d.mts +200 -16
- package/dist/index.d.ts +200 -16
- package/dist/index.js +287 -23
- package/dist/index.mjs +276 -347
- package/dist/livekit/index.d.mts +222 -0
- package/dist/livekit/index.d.ts +222 -0
- package/dist/livekit/index.js +994 -0
- package/dist/livekit/index.mjs +746 -0
- package/package.json +23 -3
- package/src/cfg-scale.test.ts +33 -0
- package/src/client.test.ts +466 -2
- package/src/client.ts +308 -17
- package/src/index.ts +4 -0
- package/src/livekit/index.ts +32 -0
- package/src/livekit/models.test.ts +30 -0
- package/src/livekit/models.ts +84 -0
- package/src/livekit/tts.functional.test.ts +348 -0
- package/src/livekit/tts.ts +833 -0
- package/src/livekit/wire.test.ts +112 -0
- package/src/livekit/wire.ts +126 -0
- package/src/types.ts +171 -13
- package/src/utils.ts +16 -0
package/src/client.ts
CHANGED
|
@@ -16,10 +16,12 @@ import type {
|
|
|
16
16
|
AudioChunk,
|
|
17
17
|
AudioResponse,
|
|
18
18
|
CreateVoiceOptions,
|
|
19
|
+
EffectiveSettings,
|
|
19
20
|
GenerateOptions,
|
|
20
21
|
GenerationStats,
|
|
21
22
|
KugelAudioOptions,
|
|
22
23
|
Model,
|
|
24
|
+
SettingsUpdate,
|
|
23
25
|
StreamCallbacks,
|
|
24
26
|
StreamConfig,
|
|
25
27
|
StreamingSessionCallbacks,
|
|
@@ -29,7 +31,8 @@ import type {
|
|
|
29
31
|
VoiceReference,
|
|
30
32
|
WordTimestamp
|
|
31
33
|
} from './types';
|
|
32
|
-
import {
|
|
34
|
+
import { parseSessionUsage } from './types';
|
|
35
|
+
import { base64ToArrayBuffer, clampCfgScale } from './utils';
|
|
33
36
|
import { getWebSocket } from './websocket';
|
|
34
37
|
|
|
35
38
|
import type { Region } from './types';
|
|
@@ -77,6 +80,145 @@ function createWs(url: string): WebSocket {
|
|
|
77
80
|
/** WebSocket OPEN readyState constant. */
|
|
78
81
|
const WS_OPEN = 1;
|
|
79
82
|
|
|
83
|
+
/**
|
|
84
|
+
* camelCase SDK field → snake_case wire field for the generation parameters
|
|
85
|
+
* changeable mid-connection via `update_settings` (KUG-1166). The key set is
|
|
86
|
+
* authoritative: anything not here is fixed for the connection's lifetime.
|
|
87
|
+
*/
|
|
88
|
+
const UPDATABLE_SETTINGS: Record<keyof SettingsUpdate, string> = {
|
|
89
|
+
cfgScale: 'cfg_scale',
|
|
90
|
+
temperature: 'temperature',
|
|
91
|
+
speed: 'speed',
|
|
92
|
+
maxNewTokens: 'max_new_tokens',
|
|
93
|
+
language: 'language',
|
|
94
|
+
normalize: 'normalize',
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/** Build the snake_case `update_settings` body; throws when empty (KUG-1166). */
|
|
98
|
+
function buildSettingsUpdateBody(settings: SettingsUpdate): Record<string, unknown> {
|
|
99
|
+
const body: Record<string, unknown> = {};
|
|
100
|
+
for (const camel of Object.keys(UPDATABLE_SETTINGS) as (keyof SettingsUpdate)[]) {
|
|
101
|
+
const value = settings[camel];
|
|
102
|
+
if (value !== undefined) body[UPDATABLE_SETTINGS[camel]] = value;
|
|
103
|
+
}
|
|
104
|
+
if (Object.keys(body).length === 0) {
|
|
105
|
+
throw new KugelAudioError(
|
|
106
|
+
'updateSettings requires at least one parameter to change ' +
|
|
107
|
+
`(one of ${Object.keys(UPDATABLE_SETTINGS).join(', ')})`,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return body;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Map the server's snake_case `settings` echo back to camelCase. */
|
|
114
|
+
function parseEffectiveSettings(raw: unknown): EffectiveSettings {
|
|
115
|
+
const s = (raw && typeof raw === 'object' ? raw : {}) as Record<string, unknown>;
|
|
116
|
+
return {
|
|
117
|
+
cfgScale: s.cfg_scale as number | undefined,
|
|
118
|
+
temperature: s.temperature as number | null | undefined,
|
|
119
|
+
speed: s.speed as number | undefined,
|
|
120
|
+
maxNewTokens: s.max_new_tokens as number | undefined,
|
|
121
|
+
language: s.language as string | null | undefined,
|
|
122
|
+
normalize: s.normalize as boolean | undefined,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Mirror an accepted settings update onto an SDK config object. */
|
|
127
|
+
function applyAcceptedSettingsUpdate(
|
|
128
|
+
config: Record<string, unknown>,
|
|
129
|
+
settings: SettingsUpdate,
|
|
130
|
+
): void {
|
|
131
|
+
for (const camel of Object.keys(UPDATABLE_SETTINGS) as (keyof SettingsUpdate)[]) {
|
|
132
|
+
if (settings[camel] !== undefined) {
|
|
133
|
+
config[camel] = settings[camel];
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Send an `update_settings` message on an open socket and resolve with the
|
|
140
|
+
* server's `settings_updated` echo (KUG-1166). Audio frames that arrive while
|
|
141
|
+
* waiting are passed through to the session's normal handler (not dropped); an
|
|
142
|
+
* `error` frame rejects, as does a quiet timeout or a socket close.
|
|
143
|
+
*/
|
|
144
|
+
function sendUpdateSettings(
|
|
145
|
+
ws: WebSocket,
|
|
146
|
+
body: Record<string, unknown>,
|
|
147
|
+
): Promise<EffectiveSettings> {
|
|
148
|
+
const QUIET_TIMEOUT_MS = 15_000;
|
|
149
|
+
return new Promise<EffectiveSettings>((resolve, reject) => {
|
|
150
|
+
let settled = false;
|
|
151
|
+
let timer: ReturnType<typeof setTimeout>;
|
|
152
|
+
const prevMessage = ws.onmessage;
|
|
153
|
+
const prevClose = ws.onclose;
|
|
154
|
+
|
|
155
|
+
const finish = (action: () => void): void => {
|
|
156
|
+
if (settled) return;
|
|
157
|
+
settled = true;
|
|
158
|
+
clearTimeout(timer);
|
|
159
|
+
ws.onmessage = prevMessage;
|
|
160
|
+
ws.onclose = prevClose;
|
|
161
|
+
action();
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const armQuietTimer = (): void => {
|
|
165
|
+
clearTimeout(timer);
|
|
166
|
+
timer = setTimeout(
|
|
167
|
+
() =>
|
|
168
|
+
finish(() =>
|
|
169
|
+
reject(
|
|
170
|
+
new KugelAudioError(
|
|
171
|
+
'Timed out waiting for settings_updated acknowledgement',
|
|
172
|
+
),
|
|
173
|
+
),
|
|
174
|
+
),
|
|
175
|
+
QUIET_TIMEOUT_MS,
|
|
176
|
+
);
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
armQuietTimer();
|
|
180
|
+
|
|
181
|
+
ws.onmessage = (event: MessageEvent) => {
|
|
182
|
+
armQuietTimer();
|
|
183
|
+
let data: Record<string, unknown> | null = null;
|
|
184
|
+
try {
|
|
185
|
+
const raw = typeof event.data === 'string'
|
|
186
|
+
? event.data
|
|
187
|
+
: event.data instanceof Buffer
|
|
188
|
+
? event.data.toString()
|
|
189
|
+
: String(event.data);
|
|
190
|
+
data = JSON.parse(raw) as Record<string, unknown>;
|
|
191
|
+
} catch {
|
|
192
|
+
/* ignore parse errors */
|
|
193
|
+
}
|
|
194
|
+
if (data?.settings_updated) {
|
|
195
|
+
finish(() => resolve(parseEffectiveSettings(data!.settings)));
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (data?.error) {
|
|
199
|
+
finish(() => reject(new KugelAudioError(String(data!.error))));
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
// Unrelated frame (e.g. stray audio from a draining turn): deliver it
|
|
203
|
+
// through the session's normal handler rather than dropping it.
|
|
204
|
+
if (prevMessage) prevMessage.call(ws, event);
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
ws.onclose = (event: CloseEvent) => {
|
|
208
|
+
if (prevClose) prevClose.call(ws, event);
|
|
209
|
+
finish(() =>
|
|
210
|
+
reject(
|
|
211
|
+
new ConnectionError(
|
|
212
|
+
'WebSocket closed before settings_updated acknowledgement',
|
|
213
|
+
),
|
|
214
|
+
),
|
|
215
|
+
);
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
ws.send(JSON.stringify({ update_settings: body }));
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
|
|
80
222
|
let _languageWarningLogged = false;
|
|
81
223
|
|
|
82
224
|
function warnIfNoLanguage(
|
|
@@ -151,6 +293,7 @@ class VoicesResource {
|
|
|
151
293
|
category: v.category,
|
|
152
294
|
sex: v.sex,
|
|
153
295
|
age: v.age,
|
|
296
|
+
quality: v.quality,
|
|
154
297
|
supportedLanguages: v.supported_languages || [],
|
|
155
298
|
sampleText: v.sample_text,
|
|
156
299
|
avatarUrl: v.avatar_url,
|
|
@@ -596,6 +739,7 @@ class TTSResource {
|
|
|
596
739
|
generationMs: data.gen_ms,
|
|
597
740
|
rtf: data.rtf,
|
|
598
741
|
error: data.error,
|
|
742
|
+
usage: parseSessionUsage(data) ?? undefined,
|
|
599
743
|
};
|
|
600
744
|
pending.callbacks.onFinal?.(stats);
|
|
601
745
|
this.pendingRequests.delete(requestId);
|
|
@@ -709,15 +853,19 @@ class TTSResource {
|
|
|
709
853
|
text: options.text,
|
|
710
854
|
model_id: options.modelId || 'kugel-3',
|
|
711
855
|
voice_id: options.voiceId,
|
|
712
|
-
cfg_scale: options.cfgScale ?? 2.0,
|
|
856
|
+
cfg_scale: clampCfgScale(options.cfgScale) ?? 2.0,
|
|
713
857
|
...(options.temperature !== undefined && { temperature: options.temperature }),
|
|
714
858
|
max_new_tokens: options.maxNewTokens ?? 2048,
|
|
715
859
|
sample_rate: options.sampleRate ?? 24000,
|
|
860
|
+
...(options.outputFormat && { output_format: options.outputFormat }),
|
|
716
861
|
normalize: options.normalize ?? true,
|
|
717
862
|
...(options.language && { language: options.language }),
|
|
718
863
|
...(options.wordTimestamps && { word_timestamps: true }),
|
|
719
864
|
...(options.speed !== undefined && { speed: options.speed }),
|
|
720
865
|
...(options.projectId !== undefined && { project_id: options.projectId }),
|
|
866
|
+
// [] is meaningful (explicit opt-out) and must be sent; only
|
|
867
|
+
// undefined (use the project default) is omitted.
|
|
868
|
+
...(options.dictionaryIds !== undefined && { dictionary_ids: options.dictionaryIds }),
|
|
721
869
|
}));
|
|
722
870
|
});
|
|
723
871
|
}
|
|
@@ -741,14 +889,18 @@ class TTSResource {
|
|
|
741
889
|
text: options.text,
|
|
742
890
|
model_id: options.modelId || 'kugel-3',
|
|
743
891
|
voice_id: options.voiceId,
|
|
744
|
-
cfg_scale: options.cfgScale ?? 2.0,
|
|
892
|
+
cfg_scale: clampCfgScale(options.cfgScale) ?? 2.0,
|
|
745
893
|
max_new_tokens: options.maxNewTokens ?? 2048,
|
|
746
894
|
sample_rate: options.sampleRate ?? 24000,
|
|
895
|
+
...(options.outputFormat && { output_format: options.outputFormat }),
|
|
747
896
|
normalize: options.normalize ?? true,
|
|
748
897
|
...(options.language && { language: options.language }),
|
|
749
898
|
...(options.wordTimestamps && { word_timestamps: true }),
|
|
750
899
|
...(options.speed !== undefined && { speed: options.speed }),
|
|
751
900
|
...(options.projectId !== undefined && { project_id: options.projectId }),
|
|
901
|
+
// [] is meaningful (explicit opt-out) and must be sent; only
|
|
902
|
+
// undefined (use the project default) is omitted.
|
|
903
|
+
...(options.dictionaryIds !== undefined && { dictionary_ids: options.dictionaryIds }),
|
|
752
904
|
}));
|
|
753
905
|
};
|
|
754
906
|
|
|
@@ -779,6 +931,7 @@ class TTSResource {
|
|
|
779
931
|
generationMs: data.gen_ms,
|
|
780
932
|
rtf: data.rtf,
|
|
781
933
|
error: data.error,
|
|
934
|
+
usage: parseSessionUsage(data) ?? undefined,
|
|
782
935
|
};
|
|
783
936
|
callbacks.onFinal?.(stats);
|
|
784
937
|
ws.close();
|
|
@@ -977,7 +1130,11 @@ class MultiContextSession {
|
|
|
977
1130
|
private config: import('./types').MultiContextConfig;
|
|
978
1131
|
private callbacks: import('./types').MultiContextCallbacks = {};
|
|
979
1132
|
private contexts: Set<string> = new Set();
|
|
1133
|
+
/** Contexts a create message has been sent for (not yet necessarily
|
|
1134
|
+
* confirmed by the server via context_created). */
|
|
1135
|
+
private requestedContexts: Set<string> = new Set();
|
|
980
1136
|
private _sessionId: string | null = null;
|
|
1137
|
+
private _contextUsage: Map<string, import('./types').SessionUsage> = new Map();
|
|
981
1138
|
private isStarted = false;
|
|
982
1139
|
|
|
983
1140
|
constructor(
|
|
@@ -994,6 +1151,20 @@ class MultiContextSession {
|
|
|
994
1151
|
return this._sessionId;
|
|
995
1152
|
}
|
|
996
1153
|
|
|
1154
|
+
/**
|
|
1155
|
+
* Per-context usage (audio time + amount charged) for a closed context, or
|
|
1156
|
+
* null if that context hasn't closed yet. Each context is its own
|
|
1157
|
+
* conversation — use this to bill per conversation. See {@link SessionUsage}.
|
|
1158
|
+
*/
|
|
1159
|
+
usageFor(contextId: string): import('./types').SessionUsage | null {
|
|
1160
|
+
return this._contextUsage.get(contextId) ?? null;
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
/** Map of context_id → per-context usage for all closed contexts. */
|
|
1164
|
+
get contextUsage(): Map<string, import('./types').SessionUsage> {
|
|
1165
|
+
return new Map(this._contextUsage);
|
|
1166
|
+
}
|
|
1167
|
+
|
|
997
1168
|
/**
|
|
998
1169
|
* Connect to the multi-context WebSocket endpoint.
|
|
999
1170
|
*
|
|
@@ -1063,13 +1234,25 @@ class MultiContextSession {
|
|
|
1063
1234
|
this.callbacks.onChunk?.(chunk);
|
|
1064
1235
|
}
|
|
1065
1236
|
|
|
1237
|
+
if (data.final && data.context_id) {
|
|
1238
|
+
// Per-context end-of-audio marker (KUG-1238): all audio admitted
|
|
1239
|
+
// before the client's flush has been delivered; also precedes
|
|
1240
|
+
// context_closed on a graceful close.
|
|
1241
|
+
this.callbacks.onFinal?.(data.context_id);
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1066
1244
|
if (data.context_closed) {
|
|
1067
1245
|
this.contexts.delete(data.context_id);
|
|
1068
|
-
this.
|
|
1246
|
+
this.requestedContexts.delete(data.context_id);
|
|
1247
|
+
// Per-context (per-conversation) usage rides on context_closed.
|
|
1248
|
+
const ctxUsage = parseSessionUsage(data) ?? undefined;
|
|
1249
|
+
if (ctxUsage) this._contextUsage.set(data.context_id, ctxUsage);
|
|
1250
|
+
this.callbacks.onContextClosed?.(data.context_id, ctxUsage);
|
|
1069
1251
|
}
|
|
1070
1252
|
|
|
1071
1253
|
if (data.context_timeout) {
|
|
1072
1254
|
this.contexts.delete(data.context_id);
|
|
1255
|
+
this.requestedContexts.delete(data.context_id);
|
|
1073
1256
|
this.callbacks.onContextTimeout?.(data.context_id);
|
|
1074
1257
|
}
|
|
1075
1258
|
|
|
@@ -1124,6 +1307,7 @@ class MultiContextSession {
|
|
|
1124
1307
|
this.ws = null;
|
|
1125
1308
|
this.isStarted = false;
|
|
1126
1309
|
this.contexts.clear();
|
|
1310
|
+
this.requestedContexts.clear();
|
|
1127
1311
|
};
|
|
1128
1312
|
});
|
|
1129
1313
|
}
|
|
@@ -1141,6 +1325,7 @@ class MultiContextSession {
|
|
|
1141
1325
|
if (!this.ws || this.ws.readyState !== WS_OPEN) {
|
|
1142
1326
|
throw new KugelAudioError('WebSocket not connected');
|
|
1143
1327
|
}
|
|
1328
|
+
this.requestedContexts.add(contextId);
|
|
1144
1329
|
|
|
1145
1330
|
const msg: Record<string, unknown> = {
|
|
1146
1331
|
text: ' ',
|
|
@@ -1151,26 +1336,36 @@ class MultiContextSession {
|
|
|
1151
1336
|
if (!this.isStarted) {
|
|
1152
1337
|
warnIfNoLanguage(this.config.language, this.config.normalize);
|
|
1153
1338
|
if (this.config.sampleRate) msg.sample_rate = this.config.sampleRate;
|
|
1154
|
-
if (this.config.
|
|
1339
|
+
if (this.config.outputFormat) msg.output_format = this.config.outputFormat;
|
|
1340
|
+
if (this.config.cfgScale !== undefined) msg.cfg_scale = clampCfgScale(this.config.cfgScale);
|
|
1155
1341
|
if (this.config.temperature !== undefined) msg.temperature = this.config.temperature;
|
|
1156
1342
|
if (this.config.maxNewTokens) msg.max_new_tokens = this.config.maxNewTokens;
|
|
1157
1343
|
if (this.config.normalize !== undefined) msg.normalize = this.config.normalize;
|
|
1158
1344
|
if (this.config.language) msg.language = this.config.language;
|
|
1345
|
+
// [] is meaningful (explicit opt-out) and must be sent; only
|
|
1346
|
+
// undefined (use the project default) is omitted.
|
|
1347
|
+
if (this.config.dictionaryIds !== undefined) msg.dictionary_ids = this.config.dictionaryIds;
|
|
1159
1348
|
if (this.config.inactivityTimeout) msg.inactivity_timeout = this.config.inactivityTimeout;
|
|
1160
1349
|
}
|
|
1161
1350
|
|
|
1162
|
-
// Per-context voice
|
|
1351
|
+
// Per-context voice. The server binds a context's voice ONLY from
|
|
1352
|
+
// voice_settings.voice_id at context creation — a top-level voice_id
|
|
1353
|
+
// merely updates the session config and leaves the context voiceless,
|
|
1354
|
+
// which the server rejects with MISSING_VOICE_ID on the first text
|
|
1355
|
+
// (KUG-1233). This matches the Python SDK's wire format.
|
|
1356
|
+
const voiceSettings: Record<string, unknown> = {};
|
|
1163
1357
|
const voiceId = options?.voiceId || this.config.defaultVoiceId;
|
|
1164
|
-
if (voiceId)
|
|
1358
|
+
if (voiceId) voiceSettings.voice_id = voiceId;
|
|
1165
1359
|
|
|
1166
1360
|
if (options?.voiceSettings) {
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1361
|
+
voiceSettings.stability = options.voiceSettings.stability;
|
|
1362
|
+
voiceSettings.similarity_boost = options.voiceSettings.similarityBoost;
|
|
1363
|
+
voiceSettings.style = options.voiceSettings.style;
|
|
1364
|
+
voiceSettings.use_speaker_boost = options.voiceSettings.useSpeakerBoost;
|
|
1365
|
+
voiceSettings.speed = options.voiceSettings.speed;
|
|
1366
|
+
}
|
|
1367
|
+
if (Object.keys(voiceSettings).length > 0) {
|
|
1368
|
+
msg.voice_settings = voiceSettings;
|
|
1174
1369
|
}
|
|
1175
1370
|
|
|
1176
1371
|
this.ws.send(JSON.stringify(msg));
|
|
@@ -1184,8 +1379,12 @@ class MultiContextSession {
|
|
|
1184
1379
|
throw new KugelAudioError('WebSocket not connected');
|
|
1185
1380
|
}
|
|
1186
1381
|
|
|
1187
|
-
// Auto-create context if needed
|
|
1188
|
-
|
|
1382
|
+
// Auto-create context if needed. Tracked via requestedContexts (sent
|
|
1383
|
+
// creates, not yet necessarily confirmed) rather than this.contexts
|
|
1384
|
+
// (server-confirmed) — otherwise a send() to a new context after the
|
|
1385
|
+
// session started goes out bare, and the server auto-creates the
|
|
1386
|
+
// context without voice_settings → MISSING_VOICE_ID (KUG-1233).
|
|
1387
|
+
if (!this.requestedContexts.has(contextId) && !this.contexts.has(contextId)) {
|
|
1189
1388
|
this.createContext(contextId);
|
|
1190
1389
|
}
|
|
1191
1390
|
|
|
@@ -1240,6 +1439,36 @@ class MultiContextSession {
|
|
|
1240
1439
|
}));
|
|
1241
1440
|
}
|
|
1242
1441
|
|
|
1442
|
+
/**
|
|
1443
|
+
* Change the session's generation parameters mid-connection (KUG-1166).
|
|
1444
|
+
*
|
|
1445
|
+
* Session-scoped (there is no `contextId`): the update applies to contexts
|
|
1446
|
+
* started **after** it — a context already streaming keeps the settings it
|
|
1447
|
+
* began with, since generation parameters are bound when a context's engine
|
|
1448
|
+
* session opens. With the common one-context-per-turn pattern that means it
|
|
1449
|
+
* takes effect on the next turn. Per-context `cfgScale` / `maxNewTokens`
|
|
1450
|
+
* passed to {@link createContext} still win for that context.
|
|
1451
|
+
*
|
|
1452
|
+
* Resolves with the generation parameters now in effect (the server's echo).
|
|
1453
|
+
*
|
|
1454
|
+
* @throws {KugelAudioError} if no field is provided, the socket is not open,
|
|
1455
|
+
* or the server rejects the update.
|
|
1456
|
+
*/
|
|
1457
|
+
updateSettings(settings: SettingsUpdate): Promise<EffectiveSettings> {
|
|
1458
|
+
if (!this.ws || this.ws.readyState !== WS_OPEN) {
|
|
1459
|
+
throw new KugelAudioError(
|
|
1460
|
+
'MultiContextSession not connected. Call connect() first.',
|
|
1461
|
+
);
|
|
1462
|
+
}
|
|
1463
|
+
const body = buildSettingsUpdateBody(settings);
|
|
1464
|
+
return sendUpdateSettings(this.ws, body).then((effective) => {
|
|
1465
|
+
// Mirror onto local config only after the server accepts the update, so
|
|
1466
|
+
// rejected values do not leak into future context creation.
|
|
1467
|
+
applyAcceptedSettingsUpdate(this.config as Record<string, unknown>, settings);
|
|
1468
|
+
return effective;
|
|
1469
|
+
});
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1243
1472
|
/**
|
|
1244
1473
|
* Close the session and all contexts.
|
|
1245
1474
|
*/
|
|
@@ -1251,6 +1480,7 @@ class MultiContextSession {
|
|
|
1251
1480
|
this.ws = null;
|
|
1252
1481
|
this.isStarted = false;
|
|
1253
1482
|
this.contexts.clear();
|
|
1483
|
+
this.requestedContexts.clear();
|
|
1254
1484
|
}
|
|
1255
1485
|
|
|
1256
1486
|
/**
|
|
@@ -1303,6 +1533,7 @@ class StreamingSession {
|
|
|
1303
1533
|
private callbacks: StreamingSessionCallbacks;
|
|
1304
1534
|
private client: KugelAudio;
|
|
1305
1535
|
private configSent = false;
|
|
1536
|
+
private _lastUsage: import('./types').SessionUsage | null = null;
|
|
1306
1537
|
|
|
1307
1538
|
constructor(client: KugelAudio, config: StreamConfig, callbacks: StreamingSessionCallbacks) {
|
|
1308
1539
|
this.client = client;
|
|
@@ -1310,6 +1541,15 @@ class StreamingSession {
|
|
|
1310
1541
|
this.callbacks = callbacks;
|
|
1311
1542
|
}
|
|
1312
1543
|
|
|
1544
|
+
/**
|
|
1545
|
+
* Per-session usage from the most recently closed session, or null before
|
|
1546
|
+
* the first session closes. Use this to bill your own customers per
|
|
1547
|
+
* conversation. See {@link SessionUsage}.
|
|
1548
|
+
*/
|
|
1549
|
+
get lastUsage(): import('./types').SessionUsage | null {
|
|
1550
|
+
return this._lastUsage;
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1313
1553
|
/**
|
|
1314
1554
|
* Open the WebSocket connection and authenticate.
|
|
1315
1555
|
*
|
|
@@ -1389,7 +1629,18 @@ class StreamingSession {
|
|
|
1389
1629
|
this.callbacks.onInterrupted?.();
|
|
1390
1630
|
}
|
|
1391
1631
|
|
|
1632
|
+
if (data.final) {
|
|
1633
|
+
// End-of-audio marker for the turn (KUG-1238) — arrives after
|
|
1634
|
+
// the last audio frame and before session_closed.
|
|
1635
|
+
this.callbacks.onFinal?.(
|
|
1636
|
+
data.total_audio_seconds ?? 0,
|
|
1637
|
+
data.total_text_chunks ?? 0,
|
|
1638
|
+
data.total_audio_chunks ?? 0,
|
|
1639
|
+
);
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1392
1642
|
if (data.session_closed) {
|
|
1643
|
+
this._lastUsage = parseSessionUsage(data);
|
|
1393
1644
|
this.callbacks.onSessionClosed?.(
|
|
1394
1645
|
data.total_audio_seconds ?? 0,
|
|
1395
1646
|
data.total_text_chunks ?? 0,
|
|
@@ -1470,10 +1721,11 @@ class StreamingSession {
|
|
|
1470
1721
|
if (!this.configSent) {
|
|
1471
1722
|
if (this.config.voiceId !== undefined) msg.voice_id = this.config.voiceId;
|
|
1472
1723
|
if (this.config.modelId !== undefined) msg.model_id = this.config.modelId;
|
|
1473
|
-
if (this.config.cfgScale !== undefined) msg.cfg_scale = this.config.cfgScale;
|
|
1724
|
+
if (this.config.cfgScale !== undefined) msg.cfg_scale = clampCfgScale(this.config.cfgScale);
|
|
1474
1725
|
if (this.config.temperature !== undefined) msg.temperature = this.config.temperature;
|
|
1475
1726
|
if (this.config.maxNewTokens !== undefined) msg.max_new_tokens = this.config.maxNewTokens;
|
|
1476
1727
|
if (this.config.sampleRate !== undefined) msg.sample_rate = this.config.sampleRate;
|
|
1728
|
+
if (this.config.outputFormat !== undefined) msg.output_format = this.config.outputFormat;
|
|
1477
1729
|
if (this.config.flushTimeoutMs !== undefined) msg.flush_timeout_ms = this.config.flushTimeoutMs;
|
|
1478
1730
|
if (this.config.maxBufferLength !== undefined) msg.max_buffer_length = this.config.maxBufferLength;
|
|
1479
1731
|
if (this.config.normalize !== undefined) msg.normalize = this.config.normalize;
|
|
@@ -1482,6 +1734,9 @@ class StreamingSession {
|
|
|
1482
1734
|
if (this.config.autoMode !== undefined) msg.auto_mode = this.config.autoMode;
|
|
1483
1735
|
if (this.config.chunkLengthSchedule?.length) msg.chunk_length_schedule = this.config.chunkLengthSchedule;
|
|
1484
1736
|
if (this.config.speed !== undefined) msg.speed = this.config.speed;
|
|
1737
|
+
// [] is meaningful (explicit opt-out) and must be sent; only
|
|
1738
|
+
// undefined (use the project default) is omitted.
|
|
1739
|
+
if (this.config.dictionaryIds !== undefined) msg.dictionary_ids = this.config.dictionaryIds;
|
|
1485
1740
|
this.configSent = true;
|
|
1486
1741
|
}
|
|
1487
1742
|
|
|
@@ -1666,6 +1921,42 @@ class StreamingSession {
|
|
|
1666
1921
|
this.configSent = false;
|
|
1667
1922
|
}
|
|
1668
1923
|
|
|
1924
|
+
/**
|
|
1925
|
+
* Change generation parameters mid-connection without reconnecting (KUG-1166).
|
|
1926
|
+
*
|
|
1927
|
+
* Sends an `update_settings` message and resolves with the generation
|
|
1928
|
+
* parameters now in effect (the server's echo). Only the six fields of
|
|
1929
|
+
* {@link SettingsUpdate} are updatable; identity / audio-format fields
|
|
1930
|
+
* (`voiceId`, `modelId`, `sampleRate`, `outputFormat`, `dictionaryIds`) are
|
|
1931
|
+
* fixed for the connection — change those with {@link updateConfig} after
|
|
1932
|
+
* {@link endSession} instead.
|
|
1933
|
+
*
|
|
1934
|
+
* The change applies to the **next turn**: a turn already streaming keeps the
|
|
1935
|
+
* settings it started with, so call this between turns.
|
|
1936
|
+
*
|
|
1937
|
+
* @example
|
|
1938
|
+
* ```typescript
|
|
1939
|
+
* const effective = await session.updateSettings({ cfgScale: 1.5, speed: 1.1 });
|
|
1940
|
+
* ```
|
|
1941
|
+
*
|
|
1942
|
+
* @throws {KugelAudioError} if no field is provided, the socket is not open,
|
|
1943
|
+
* or the server rejects the update (e.g. a value out of range).
|
|
1944
|
+
*/
|
|
1945
|
+
updateSettings(settings: SettingsUpdate): Promise<EffectiveSettings> {
|
|
1946
|
+
if (!this.ws || this.ws.readyState !== WS_OPEN) {
|
|
1947
|
+
throw new KugelAudioError(
|
|
1948
|
+
'StreamingSession not connected. Call connect() first.',
|
|
1949
|
+
);
|
|
1950
|
+
}
|
|
1951
|
+
const body = buildSettingsUpdateBody(settings);
|
|
1952
|
+
return sendUpdateSettings(this.ws, body).then((effective) => {
|
|
1953
|
+
// Keep the local config in sync only after the server accepts the
|
|
1954
|
+
// change; rejected updates must not poison later config re-sends.
|
|
1955
|
+
applyAcceptedSettingsUpdate(this.config as Record<string, unknown>, settings);
|
|
1956
|
+
return effective;
|
|
1957
|
+
});
|
|
1958
|
+
}
|
|
1959
|
+
|
|
1669
1960
|
/**
|
|
1670
1961
|
* Close the session and the WebSocket connection.
|
|
1671
1962
|
*
|
package/src/index.ts
CHANGED
|
@@ -54,6 +54,7 @@ export type {
|
|
|
54
54
|
DictionaryEntry,
|
|
55
55
|
DictionaryEntryInput,
|
|
56
56
|
DictionaryEntryListResponse,
|
|
57
|
+
EffectiveSettings,
|
|
57
58
|
GenerateOptions,
|
|
58
59
|
GenerationStats,
|
|
59
60
|
KugelAudioOptions,
|
|
@@ -62,6 +63,8 @@ export type {
|
|
|
62
63
|
MultiContextAudioChunk,
|
|
63
64
|
MultiContextCallbacks,
|
|
64
65
|
MultiContextConfig,
|
|
66
|
+
SessionUsage,
|
|
67
|
+
SettingsUpdate,
|
|
65
68
|
StreamCallbacks,
|
|
66
69
|
StreamConfig,
|
|
67
70
|
StreamingSessionCallbacks,
|
|
@@ -78,6 +81,7 @@ export type {
|
|
|
78
81
|
VoiceSex,
|
|
79
82
|
WordTimestamp
|
|
80
83
|
} from './types';
|
|
84
|
+
export { parseSessionUsage } from './types';
|
|
81
85
|
|
|
82
86
|
export { DictionariesResource, DictionaryEntriesResource } from './dictionaries';
|
|
83
87
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KugelAudio TTS plugin for the LiveKit Agents (Node.js) framework.
|
|
3
|
+
*
|
|
4
|
+
* Import from `kugelaudio/livekit`. Requires `@livekit/agents` and its peer
|
|
5
|
+
* `@livekit/rtc-node` to be installed (they are optional peer dependencies of
|
|
6
|
+
* this SDK). This is the TypeScript counterpart to `kugelaudio.livekit` in the
|
|
7
|
+
* Python SDK.
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```ts
|
|
11
|
+
* import { TTS } from 'kugelaudio/livekit';
|
|
12
|
+
*
|
|
13
|
+
* const tts = new TTS({ voiceId: 1071, model: 'kugel-3', language: 'en' });
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* @packageDocumentation
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export { TTS, ChunkedStream, SynthesizeStream } from './tts';
|
|
20
|
+
export type { TTSOptions } from './tts';
|
|
21
|
+
|
|
22
|
+
export {
|
|
23
|
+
DEFAULT_CFG_SCALE,
|
|
24
|
+
DEFAULT_MAX_NEW_TOKENS,
|
|
25
|
+
DEFAULT_MODEL,
|
|
26
|
+
DEFAULT_SAMPLE_RATE,
|
|
27
|
+
DEFAULT_VOICE_ID,
|
|
28
|
+
SUPPORTED_LANGUAGES,
|
|
29
|
+
SUPPORTED_SAMPLE_RATES,
|
|
30
|
+
validateLanguage,
|
|
31
|
+
} from './models';
|
|
32
|
+
export type { TTSModels } from './models';
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { SUPPORTED_LANGUAGES, validateLanguage } from './models';
|
|
3
|
+
|
|
4
|
+
describe('validateLanguage', () => {
|
|
5
|
+
it('returns undefined for null/undefined', () => {
|
|
6
|
+
expect(validateLanguage(undefined)).toBeUndefined();
|
|
7
|
+
expect(validateLanguage(null)).toBeUndefined();
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
it('accepts supported ISO 639-1 codes', () => {
|
|
11
|
+
expect(validateLanguage('de')).toBe('de');
|
|
12
|
+
expect(validateLanguage('en')).toBe('en');
|
|
13
|
+
expect(validateLanguage('yue')).toBe('yue');
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it('rejects BCP 47 locale tags with a helpful message', () => {
|
|
17
|
+
expect(() => validateLanguage('de-DE')).toThrow(/de-DE/);
|
|
18
|
+
expect(() => validateLanguage('de-DE')).toThrow(/use 'de' instead/);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('rejects unknown codes', () => {
|
|
22
|
+
expect(() => validateLanguage('xx')).toThrow();
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('has a non-empty language set covering common languages', () => {
|
|
26
|
+
for (const code of ['de', 'en', 'fr', 'es', 'it', 'zh', 'ja', 'ko', 'ar', 'hi']) {
|
|
27
|
+
expect(SUPPORTED_LANGUAGES.has(code)).toBe(true);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
});
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Models, constants, and language validation for the KugelAudio LiveKit
|
|
3
|
+
* Agents TTS plugin.
|
|
4
|
+
*
|
|
5
|
+
* This module has NO dependency on `@livekit/agents`, so it can be imported
|
|
6
|
+
* and unit-tested without the (heavy, native) LiveKit runtime.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Available TTS models. Legacy ids (`kugel-2.5`, `kugel-2-turbo`,
|
|
11
|
+
* `kugel-1-turbo`, `kugel-1`) remain accepted for back-compat — server-side
|
|
12
|
+
* they all alias to `kugel-3` — but new code should use `'kugel-3'`.
|
|
13
|
+
*/
|
|
14
|
+
export type TTSModels =
|
|
15
|
+
| 'kugel-3'
|
|
16
|
+
| 'kugel-2.5'
|
|
17
|
+
| 'kugel-2-turbo'
|
|
18
|
+
| 'kugel-1-turbo'
|
|
19
|
+
| 'kugel-1';
|
|
20
|
+
|
|
21
|
+
/** Default model (matches the public API default; server-side `is_default=kugel-3`). */
|
|
22
|
+
export const DEFAULT_MODEL: TTSModels = 'kugel-3';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Supported output sample rates. The model generates at 24 kHz natively;
|
|
26
|
+
* other rates use server-side resampling.
|
|
27
|
+
*/
|
|
28
|
+
export const SUPPORTED_SAMPLE_RATES = [24000, 22050, 16000, 8000] as const;
|
|
29
|
+
|
|
30
|
+
/** Default output sample rate. */
|
|
31
|
+
export const DEFAULT_SAMPLE_RATE = 24000;
|
|
32
|
+
|
|
33
|
+
/** Default voice id (`null` means use the server default voice). */
|
|
34
|
+
export const DEFAULT_VOICE_ID: number | null = null;
|
|
35
|
+
|
|
36
|
+
/** Default classifier-free guidance scale. */
|
|
37
|
+
export const DEFAULT_CFG_SCALE = 2.0;
|
|
38
|
+
|
|
39
|
+
/** Default maximum number of tokens to generate. */
|
|
40
|
+
export const DEFAULT_MAX_NEW_TOKENS = 2048;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* ISO 639-1 language codes accepted by the API for text normalization.
|
|
44
|
+
* Mirrors `kugelaudio.livekit.tts.SUPPORTED_LANGUAGES` in the Python SDK.
|
|
45
|
+
*/
|
|
46
|
+
export const SUPPORTED_LANGUAGES: ReadonlySet<string> = new Set([
|
|
47
|
+
// Germanic
|
|
48
|
+
'de', 'en', 'nl', 'sv', 'da', 'no',
|
|
49
|
+
// Romance
|
|
50
|
+
'fr', 'es', 'it', 'pt', 'ro',
|
|
51
|
+
// Slavic
|
|
52
|
+
'pl', 'cs', 'uk', 'bg', 'sk', 'sl', 'hr', 'sr',
|
|
53
|
+
// Uralic
|
|
54
|
+
'fi', 'hu',
|
|
55
|
+
// Other European
|
|
56
|
+
'el', 'tr', 'ru',
|
|
57
|
+
// CJK + SEA
|
|
58
|
+
'zh', 'ja', 'ko', 'vi', 'yue', 'th', 'id', 'ms',
|
|
59
|
+
// Semitic + Indic + Other
|
|
60
|
+
'ar', 'hi', 'he', 'fa', 'ur', 'bn', 'ta',
|
|
61
|
+
]);
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Validate that `language` is an ISO 639-1 code supported by the API.
|
|
65
|
+
*
|
|
66
|
+
* @param language - The language code, or `undefined`/`null` to leave unset.
|
|
67
|
+
* @returns The validated code, or `undefined` when none was supplied.
|
|
68
|
+
* @throws {Error} If `language` is not a supported ISO 639-1 code. A common
|
|
69
|
+
* mistake is passing a BCP 47 locale tag such as `'de-DE'` instead of `'de'`.
|
|
70
|
+
*/
|
|
71
|
+
export function validateLanguage(
|
|
72
|
+
language: string | null | undefined,
|
|
73
|
+
): string | undefined {
|
|
74
|
+
if (language === null || language === undefined) return undefined;
|
|
75
|
+
if (!SUPPORTED_LANGUAGES.has(language)) {
|
|
76
|
+
const supported = [...SUPPORTED_LANGUAGES].sort().join(', ');
|
|
77
|
+
throw new Error(
|
|
78
|
+
`language must be a supported ISO 639-1 code (${supported}), got ` +
|
|
79
|
+
`'${language}'. Note: BCP 47 tags like 'de-DE' are not accepted — ` +
|
|
80
|
+
`use 'de' instead.`,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
return language;
|
|
84
|
+
}
|