@lokutor/sdk 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -2
- package/dist/index.d.mts +70 -25
- package/dist/index.d.ts +70 -25
- package/dist/index.js +267 -106
- package/dist/index.mjs +265 -106
- package/package.json +1 -1
- package/src/browser-audio.ts +82 -13
- package/src/client.ts +185 -99
- package/src/conversational-panel.ts +61 -15
- package/src/index.ts +1 -1
- package/src/types.ts +38 -17
package/dist/index.mjs
CHANGED
|
@@ -80,13 +80,19 @@ function isRetryableCode(code) {
|
|
|
80
80
|
"auth.invalid_key",
|
|
81
81
|
"auth.time_limited",
|
|
82
82
|
"session.ended",
|
|
83
|
+
"audio.microphone_unavailable",
|
|
83
84
|
"validation.invalid_voice",
|
|
84
85
|
"validation.invalid_language",
|
|
85
86
|
"validation.text_too_long",
|
|
86
87
|
"validation.speed_out_of_range",
|
|
87
88
|
"validation.steps_out_of_range",
|
|
88
89
|
"validation.invalid_request_format",
|
|
89
|
-
"internal.cancelled"
|
|
90
|
+
"internal.cancelled",
|
|
91
|
+
"agent.not_found",
|
|
92
|
+
"billing.insufficient_balance",
|
|
93
|
+
"billing.topup_required",
|
|
94
|
+
"billing.limit_reached",
|
|
95
|
+
"billing.account_disputed"
|
|
90
96
|
];
|
|
91
97
|
return !fatal.includes(code);
|
|
92
98
|
}
|
|
@@ -234,6 +240,36 @@ var StreamResampler = class {
|
|
|
234
240
|
};
|
|
235
241
|
|
|
236
242
|
// src/browser-audio.ts
|
|
243
|
+
function requestMicrophone(constraints = {}) {
|
|
244
|
+
if (typeof navigator === "undefined") {
|
|
245
|
+
return Promise.reject(new Error("Microphone capture needs a browser"));
|
|
246
|
+
}
|
|
247
|
+
const session = navigator.audioSession;
|
|
248
|
+
if (session && session.type !== "auto" && session.type !== "play-and-record") {
|
|
249
|
+
try {
|
|
250
|
+
session.type = "auto";
|
|
251
|
+
} catch (_) {
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (!navigator.mediaDevices?.getUserMedia) {
|
|
255
|
+
return Promise.reject(new Error("Microphone capture is not available here (it needs HTTPS)"));
|
|
256
|
+
}
|
|
257
|
+
return navigator.mediaDevices.getUserMedia({
|
|
258
|
+
audio: {
|
|
259
|
+
autoGainControl: constraints.autoGainControl ?? true,
|
|
260
|
+
echoCancellation: constraints.echoCancellation ?? true,
|
|
261
|
+
noiseSuppression: constraints.noiseSuppression ?? true
|
|
262
|
+
}
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
async function resumeWithin(ctx, ms) {
|
|
266
|
+
if (ctx.state === "running") return;
|
|
267
|
+
await Promise.race([
|
|
268
|
+
ctx.resume().catch(() => {
|
|
269
|
+
}),
|
|
270
|
+
new Promise((r) => setTimeout(r, ms))
|
|
271
|
+
]);
|
|
272
|
+
}
|
|
237
273
|
var BrowserAudioManager = class {
|
|
238
274
|
audioContext = null;
|
|
239
275
|
mediaStreamAudioSourceNode = null;
|
|
@@ -256,6 +292,7 @@ var BrowserAudioManager = class {
|
|
|
256
292
|
echoCancellation;
|
|
257
293
|
noiseSuppression;
|
|
258
294
|
keepAwake;
|
|
295
|
+
providedMicrophone;
|
|
259
296
|
// Screen wake lock, held while the microphone is live. Browsers release it whenever the page is
|
|
260
297
|
// hidden, so it is taken again when the page becomes visible.
|
|
261
298
|
wakeLock = null;
|
|
@@ -273,6 +310,9 @@ var BrowserAudioManager = class {
|
|
|
273
310
|
this.echoCancellation = config.echoCancellation ?? true;
|
|
274
311
|
this.noiseSuppression = config.noiseSuppression ?? true;
|
|
275
312
|
this.keepAwake = config.keepAwake ?? true;
|
|
313
|
+
this.providedMicrophone = config.microphone ?? null;
|
|
314
|
+
if (this.providedMicrophone instanceof Promise) this.providedMicrophone.catch(() => {
|
|
315
|
+
});
|
|
276
316
|
this.onInputError = config.onInputError;
|
|
277
317
|
}
|
|
278
318
|
/**
|
|
@@ -288,9 +328,8 @@ var BrowserAudioManager = class {
|
|
|
288
328
|
if (!this.audioContext) {
|
|
289
329
|
throw new Error("Failed to initialize AudioContext");
|
|
290
330
|
}
|
|
291
|
-
if (this.audioContext.state
|
|
292
|
-
await this.audioContext
|
|
293
|
-
console.log("\u{1F442} AudioContext resumed");
|
|
331
|
+
if (this.audioContext.state !== "running") {
|
|
332
|
+
await resumeWithin(this.audioContext, 500);
|
|
294
333
|
}
|
|
295
334
|
if (analyserConfig?.enabled !== false) {
|
|
296
335
|
this.analyserNode = this.audioContext.createAnalyser();
|
|
@@ -311,7 +350,10 @@ var BrowserAudioManager = class {
|
|
|
311
350
|
try {
|
|
312
351
|
this.onAudioInput = onAudioInput;
|
|
313
352
|
this.isListening = true;
|
|
314
|
-
|
|
353
|
+
const provided = this.providedMicrophone;
|
|
354
|
+
this.providedMicrophone = null;
|
|
355
|
+
const stream = provided ? await provided : null;
|
|
356
|
+
this.mediaStream = stream && stream.getAudioTracks().some((t) => t.readyState === "live") ? stream : await navigator.mediaDevices.getUserMedia({
|
|
315
357
|
audio: {
|
|
316
358
|
autoGainControl: this.autoGainControl,
|
|
317
359
|
echoCancellation: this.echoCancellation,
|
|
@@ -344,6 +386,7 @@ var BrowserAudioManager = class {
|
|
|
344
386
|
console.log("\u{1F3A4} Microphone started");
|
|
345
387
|
this.holdScreenAwake();
|
|
346
388
|
} catch (error) {
|
|
389
|
+
this.isListening = false;
|
|
347
390
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
348
391
|
if (this.onInputError) this.onInputError(err);
|
|
349
392
|
throw err;
|
|
@@ -578,6 +621,13 @@ var BrowserAudioManager = class {
|
|
|
578
621
|
this.analyserNode.disconnect();
|
|
579
622
|
this.analyserNode = null;
|
|
580
623
|
}
|
|
624
|
+
if (this.audioContext) {
|
|
625
|
+
const ctx = this.audioContext;
|
|
626
|
+
this.audioContext = null;
|
|
627
|
+
this.nextPlaybackTime = 0;
|
|
628
|
+
ctx.close().catch(() => {
|
|
629
|
+
});
|
|
630
|
+
}
|
|
581
631
|
}
|
|
582
632
|
/**
|
|
583
633
|
* Get current audio context state
|
|
@@ -617,6 +667,9 @@ function nowMs() {
|
|
|
617
667
|
function wsToHttp(url) {
|
|
618
668
|
return url.replace(/^wss:/, "https:").replace(/^ws:/, "http:");
|
|
619
669
|
}
|
|
670
|
+
function apiOrigin(base) {
|
|
671
|
+
return new URL(wsToHttp(base || DEFAULT_URLS.VOICE_AGENT)).origin;
|
|
672
|
+
}
|
|
620
673
|
async function fetchJson(url, timeoutMs = 1e4) {
|
|
621
674
|
const controller = new AbortController();
|
|
622
675
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
@@ -729,10 +782,16 @@ var VoiceAgentClient = class {
|
|
|
729
782
|
serverPlaybackRate = 44100;
|
|
730
783
|
serverInputRate = 16e3;
|
|
731
784
|
serverUrl;
|
|
785
|
+
authProvider;
|
|
786
|
+
voiceExplicit = false;
|
|
787
|
+
languageExplicit = false;
|
|
732
788
|
constructor(config) {
|
|
733
|
-
this.apiKey = config.apiKey;
|
|
789
|
+
this.apiKey = config.apiKey || "";
|
|
790
|
+
this.authProvider = config.auth;
|
|
734
791
|
this.agentId = config.agentId || "";
|
|
735
|
-
this.prompt = config.prompt;
|
|
792
|
+
this.prompt = config.prompt ?? "";
|
|
793
|
+
this.voiceExplicit = config.voice !== void 0;
|
|
794
|
+
this.languageExplicit = config.language !== void 0;
|
|
736
795
|
this.voice = config.voice || "F1" /* F1 */;
|
|
737
796
|
this.language = config.language || "en" /* ENGLISH */;
|
|
738
797
|
this.serverUrl = config.serverUrl || DEFAULT_URLS.VOICE_AGENT;
|
|
@@ -752,6 +811,15 @@ var VoiceAgentClient = class {
|
|
|
752
811
|
*/
|
|
753
812
|
async connect(customAudioManager) {
|
|
754
813
|
this.isUserDisconnect = false;
|
|
814
|
+
if (this.authProvider) {
|
|
815
|
+
try {
|
|
816
|
+
this.apiKey = await this.authProvider();
|
|
817
|
+
} catch (err) {
|
|
818
|
+
const error = new LokutorError("auth.missing_key", "The auth callback failed to return a credential", { original: err, retryable: true });
|
|
819
|
+
this.emit("error", error);
|
|
820
|
+
throw error;
|
|
821
|
+
}
|
|
822
|
+
}
|
|
755
823
|
if (this.enableAudio || customAudioManager) {
|
|
756
824
|
if (customAudioManager) {
|
|
757
825
|
this.audioManager = customAudioManager;
|
|
@@ -774,11 +842,11 @@ var VoiceAgentClient = class {
|
|
|
774
842
|
let url = this.serverUrl;
|
|
775
843
|
if (this.apiKey) {
|
|
776
844
|
const separator = url.includes("?") ? "&" : "?";
|
|
777
|
-
url += `${separator}api_key=${this.apiKey}`;
|
|
845
|
+
url += `${separator}api_key=${encodeURIComponent(this.apiKey)}`;
|
|
778
846
|
}
|
|
779
847
|
if (this.agentId) {
|
|
780
848
|
const separator = url.includes("?") ? "&" : "?";
|
|
781
|
-
url += `${separator}agent_id=${this.agentId}`;
|
|
849
|
+
url += `${separator}agent_id=${encodeURIComponent(this.agentId)}`;
|
|
782
850
|
}
|
|
783
851
|
const redactedUrl = url.replace(/api_key=[^&]+/, "api_key=***");
|
|
784
852
|
sdkTrace("ws.connect", {
|
|
@@ -800,11 +868,29 @@ var VoiceAgentClient = class {
|
|
|
800
868
|
sdkTrace("ws.open");
|
|
801
869
|
this.sendConfig();
|
|
802
870
|
if (this.audioManager && !this.micStarted) {
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
this.
|
|
871
|
+
try {
|
|
872
|
+
await this.audioManager.startMicrophone((data) => {
|
|
873
|
+
if (this.isConnected) {
|
|
874
|
+
this.sendAudio(data);
|
|
875
|
+
}
|
|
876
|
+
});
|
|
877
|
+
} catch (e) {
|
|
878
|
+
const name = e?.name || "Error";
|
|
879
|
+
const error = new LokutorError(
|
|
880
|
+
"audio.microphone_unavailable",
|
|
881
|
+
name === "NotAllowedError" || name === "SecurityError" ? "Microphone access was not allowed." : "The microphone could not be started.",
|
|
882
|
+
{ detail: `${name}: ${e?.message ?? e}`, original: e, retryable: false }
|
|
883
|
+
);
|
|
884
|
+
sdkTrace("mic.failed", { name });
|
|
885
|
+
settle(() => reject(error));
|
|
886
|
+
this.isUserDisconnect = true;
|
|
887
|
+
try {
|
|
888
|
+
this.ws?.close();
|
|
889
|
+
} catch (_) {
|
|
806
890
|
}
|
|
807
|
-
|
|
891
|
+
this.emit("error", error);
|
|
892
|
+
return;
|
|
893
|
+
}
|
|
808
894
|
this.micStarted = true;
|
|
809
895
|
}
|
|
810
896
|
settle(() => resolve(true));
|
|
@@ -826,7 +912,7 @@ var VoiceAgentClient = class {
|
|
|
826
912
|
});
|
|
827
913
|
console.error("\u274C WebSocket error:", error.message);
|
|
828
914
|
sdkTrace("ws.error", { code: error.code, message: error.message });
|
|
829
|
-
|
|
915
|
+
this.emit("error", error);
|
|
830
916
|
if (!this.isConnected) {
|
|
831
917
|
settle(() => reject(error));
|
|
832
918
|
}
|
|
@@ -863,8 +949,8 @@ var VoiceAgentClient = class {
|
|
|
863
949
|
});
|
|
864
950
|
console.log(`Session ended by the server (code ${event.code})`);
|
|
865
951
|
this.emit("ended", { code: event.code, reason: event.reason || "" });
|
|
866
|
-
|
|
867
|
-
|
|
952
|
+
this.emit("error", ended);
|
|
953
|
+
this.emit("status", "disconnected");
|
|
868
954
|
return;
|
|
869
955
|
}
|
|
870
956
|
if (!this.isUserDisconnect && this.reconnectAttempts < this.maxReconnectAttempts) {
|
|
@@ -872,7 +958,7 @@ var VoiceAgentClient = class {
|
|
|
872
958
|
this.reconnectAttempts++;
|
|
873
959
|
const backoffDelay = Math.min(1e3 * Math.pow(2, this.reconnectAttempts), 1e4);
|
|
874
960
|
console.warn(`Connection lost. Reconnecting in ${backoffDelay}ms (attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})`);
|
|
875
|
-
|
|
961
|
+
this.emit("status", "reconnecting");
|
|
876
962
|
if (this.reconnectTimer) clearTimeout(this.reconnectTimer);
|
|
877
963
|
this.reconnectTimer = setTimeout(() => {
|
|
878
964
|
this.reconnectTimer = null;
|
|
@@ -882,15 +968,15 @@ var VoiceAgentClient = class {
|
|
|
882
968
|
}, backoffDelay);
|
|
883
969
|
} else if (this.isUserDisconnect) {
|
|
884
970
|
console.log("Disconnected");
|
|
885
|
-
|
|
971
|
+
this.emit("status", "disconnected");
|
|
886
972
|
} else {
|
|
887
973
|
console.log("Max reconnection attempts reached");
|
|
888
|
-
|
|
974
|
+
this.emit("status", "disconnected");
|
|
889
975
|
}
|
|
890
976
|
};
|
|
891
977
|
} catch (err) {
|
|
892
978
|
const error = err instanceof LokutorError ? err : new LokutorError("internal.error", "Failed to create WebSocket connection", { original: err });
|
|
893
|
-
|
|
979
|
+
this.emit("error", error);
|
|
894
980
|
settle(() => reject(error));
|
|
895
981
|
}
|
|
896
982
|
});
|
|
@@ -918,13 +1004,14 @@ var VoiceAgentClient = class {
|
|
|
918
1004
|
sendConfig() {
|
|
919
1005
|
if (!this.ws || !this.isConnected) return;
|
|
920
1006
|
this.configConfirmed = false;
|
|
921
|
-
|
|
922
|
-
this.ws.send(JSON.stringify({ type: "
|
|
1007
|
+
const agentSession = Boolean(this.agentId);
|
|
1008
|
+
if (!agentSession || this.voiceExplicit) this.ws.send(JSON.stringify({ type: "voice", data: this.voice }));
|
|
1009
|
+
if (!agentSession || this.languageExplicit) this.ws.send(JSON.stringify({ type: "language", data: this.language }));
|
|
923
1010
|
if (this.tools && this.tools.length > 0) {
|
|
924
1011
|
this.ws.send(JSON.stringify({ type: "tools", data: this.tools }));
|
|
925
1012
|
}
|
|
926
1013
|
this.ws.send(JSON.stringify({ type: "visemes", data: this.wantVisemes }));
|
|
927
|
-
this.ws.send(JSON.stringify({ type: "prompt", data: this.prompt }));
|
|
1014
|
+
if (!agentSession || this.prompt) this.ws.send(JSON.stringify({ type: "prompt", data: this.prompt }));
|
|
928
1015
|
this.ws.send(JSON.stringify({ type: "rates", playback: 44100, input: 16e3 }));
|
|
929
1016
|
sdkTrace("ws.send.config", {
|
|
930
1017
|
promptLen: this.prompt?.length || 0,
|
|
@@ -985,6 +1072,10 @@ var VoiceAgentClient = class {
|
|
|
985
1072
|
sdkTrace("server_rates", { playback: this.serverPlaybackRate, input: this.serverInputRate });
|
|
986
1073
|
break;
|
|
987
1074
|
case "transcript":
|
|
1075
|
+
if (msg.partial) {
|
|
1076
|
+
this.emit("transcription_partial", msg.data);
|
|
1077
|
+
break;
|
|
1078
|
+
}
|
|
988
1079
|
const role = msg.role === "user" ? "user" : "agent";
|
|
989
1080
|
this.messages.push({
|
|
990
1081
|
role,
|
|
@@ -992,11 +1083,9 @@ var VoiceAgentClient = class {
|
|
|
992
1083
|
timestamp: nowMs()
|
|
993
1084
|
});
|
|
994
1085
|
if (msg.role === "user") {
|
|
995
|
-
if (this.onTranscription) this.onTranscription(msg.data);
|
|
996
1086
|
this.emit("transcription", msg.data);
|
|
997
1087
|
console.log(`\u{1F4AC} You: ${msg.data}`);
|
|
998
1088
|
} else {
|
|
999
|
-
if (this.onResponse) this.onResponse(msg.data);
|
|
1000
1089
|
this.emit("response", msg.data);
|
|
1001
1090
|
console.log(`\u{1F916} Agent: ${msg.data}`);
|
|
1002
1091
|
}
|
|
@@ -1018,7 +1107,6 @@ var VoiceAgentClient = class {
|
|
|
1018
1107
|
if (msg.data === "interrupted" && this.audioManager) {
|
|
1019
1108
|
this.audioManager.stopPlayback();
|
|
1020
1109
|
}
|
|
1021
|
-
if (this.onStatus) this.onStatus(msg.data);
|
|
1022
1110
|
this.emit("status", msg.data);
|
|
1023
1111
|
const icons = {
|
|
1024
1112
|
"interrupted": "\u26A1",
|
|
@@ -1056,7 +1144,6 @@ var VoiceAgentClient = class {
|
|
|
1056
1144
|
detail: backendDetail,
|
|
1057
1145
|
retryable: backendRetryable
|
|
1058
1146
|
});
|
|
1059
|
-
if (this.onError) this.onError(error);
|
|
1060
1147
|
this.emit("error", error);
|
|
1061
1148
|
console.error(`\u274C Server error: [${error.code}] ${error.message}`);
|
|
1062
1149
|
break;
|
|
@@ -1069,11 +1156,12 @@ var VoiceAgentClient = class {
|
|
|
1069
1156
|
}
|
|
1070
1157
|
break;
|
|
1071
1158
|
}
|
|
1072
|
-
case "tool_call":
|
|
1073
|
-
|
|
1159
|
+
case "tool_call": {
|
|
1160
|
+
const call = { name: msg.name, arguments: msg.arguments, call_id: msg.call_id };
|
|
1074
1161
|
console.log(`\u{1F6E0}\uFE0F Tool Call: ${msg.name}(${msg.arguments})`);
|
|
1075
|
-
this.emit("tool_call",
|
|
1162
|
+
this.emit("tool_call", call);
|
|
1076
1163
|
break;
|
|
1164
|
+
}
|
|
1077
1165
|
}
|
|
1078
1166
|
} catch (e) {
|
|
1079
1167
|
sdkTrace("ws.recv.parse_error", { preview: text?.slice(0, 120) });
|
|
@@ -1187,7 +1275,7 @@ var VoiceAgentClient = class {
|
|
|
1187
1275
|
* No authentication required.
|
|
1188
1276
|
*/
|
|
1189
1277
|
static async fetchVoices(baseUrl) {
|
|
1190
|
-
const url =
|
|
1278
|
+
const url = apiOrigin(baseUrl);
|
|
1191
1279
|
const data = await fetchJson(`${url}/voices`);
|
|
1192
1280
|
return data.voices || [];
|
|
1193
1281
|
}
|
|
@@ -1196,7 +1284,7 @@ var VoiceAgentClient = class {
|
|
|
1196
1284
|
* No authentication required.
|
|
1197
1285
|
*/
|
|
1198
1286
|
static async fetchLanguages(baseUrl) {
|
|
1199
|
-
const url =
|
|
1287
|
+
const url = apiOrigin(baseUrl);
|
|
1200
1288
|
const data = await fetchJson(`${url}/languages`);
|
|
1201
1289
|
return data.languages || [];
|
|
1202
1290
|
}
|
|
@@ -1205,7 +1293,7 @@ var VoiceAgentClient = class {
|
|
|
1205
1293
|
* No authentication required.
|
|
1206
1294
|
*/
|
|
1207
1295
|
static async fetchModels(baseUrl) {
|
|
1208
|
-
const url =
|
|
1296
|
+
const url = apiOrigin(baseUrl);
|
|
1209
1297
|
const data = await fetchJson(`${url}/models`);
|
|
1210
1298
|
return data.models || [];
|
|
1211
1299
|
}
|
|
@@ -1214,7 +1302,7 @@ var VoiceAgentClient = class {
|
|
|
1214
1302
|
* No authentication required.
|
|
1215
1303
|
*/
|
|
1216
1304
|
static async fetchConfig(baseUrl) {
|
|
1217
|
-
const url =
|
|
1305
|
+
const url = apiOrigin(baseUrl);
|
|
1218
1306
|
return fetchJson(`${url}/config`);
|
|
1219
1307
|
}
|
|
1220
1308
|
/**
|
|
@@ -1222,16 +1310,16 @@ var VoiceAgentClient = class {
|
|
|
1222
1310
|
* No authentication required.
|
|
1223
1311
|
*/
|
|
1224
1312
|
static async fetchStatus(baseUrl) {
|
|
1225
|
-
const url =
|
|
1313
|
+
const url = apiOrigin(baseUrl);
|
|
1226
1314
|
return fetchJson(`${url}/status`);
|
|
1227
1315
|
}
|
|
1228
1316
|
/**
|
|
1229
|
-
* Fetch
|
|
1230
|
-
* No authentication required.
|
|
1317
|
+
* Fetch readiness from the server (GET /ready: 200 once it can serve).
|
|
1318
|
+
* No authentication required. (/health is the operator dashboard's and needs its password.)
|
|
1231
1319
|
*/
|
|
1232
1320
|
static async fetchHealth(baseUrl) {
|
|
1233
|
-
const url =
|
|
1234
|
-
return fetchJson(`${url}/
|
|
1321
|
+
const url = apiOrigin(baseUrl);
|
|
1322
|
+
return fetchJson(`${url}/ready`);
|
|
1235
1323
|
}
|
|
1236
1324
|
/**
|
|
1237
1325
|
* Update the system prompt mid-conversation
|
|
@@ -1256,6 +1344,7 @@ var VoiceAgentClient = class {
|
|
|
1256
1344
|
*/
|
|
1257
1345
|
updateVoice(voice) {
|
|
1258
1346
|
this.voice = voice;
|
|
1347
|
+
this.voiceExplicit = true;
|
|
1259
1348
|
if (this.ws && this.ws.readyState === WebSocket.OPEN && this.isConnected) {
|
|
1260
1349
|
this.ws.send(JSON.stringify({ type: "voice", data: voice }));
|
|
1261
1350
|
console.log(`\u2699\uFE0F Updated voice: ${voice}`);
|
|
@@ -1266,6 +1355,7 @@ var VoiceAgentClient = class {
|
|
|
1266
1355
|
*/
|
|
1267
1356
|
updateLanguage(language) {
|
|
1268
1357
|
this.language = language;
|
|
1358
|
+
this.languageExplicit = true;
|
|
1269
1359
|
if (this.ws && this.ws.readyState === WebSocket.OPEN && this.isConnected) {
|
|
1270
1360
|
this.ws.send(JSON.stringify({ type: "language", data: language }));
|
|
1271
1361
|
console.log(`\u2699\uFE0F Updated language: ${language}`);
|
|
@@ -1325,98 +1415,120 @@ var TTSClient = class {
|
|
|
1325
1415
|
*/
|
|
1326
1416
|
synthesize(options) {
|
|
1327
1417
|
return new Promise((resolve, reject) => {
|
|
1328
|
-
let
|
|
1418
|
+
let timer;
|
|
1329
1419
|
let ws;
|
|
1330
1420
|
let startTime;
|
|
1331
1421
|
let firstByteReceived = false;
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1422
|
+
let finished = false;
|
|
1423
|
+
const finish = (err) => {
|
|
1424
|
+
if (finished) return;
|
|
1425
|
+
finished = true;
|
|
1426
|
+
if (timer) clearTimeout(timer);
|
|
1427
|
+
try {
|
|
1428
|
+
ws?.close();
|
|
1429
|
+
} catch {
|
|
1430
|
+
}
|
|
1431
|
+
if (err) {
|
|
1432
|
+
if (options.onError) options.onError(err);
|
|
1433
|
+
reject(err);
|
|
1434
|
+
} else {
|
|
1337
1435
|
resolve();
|
|
1338
|
-
}
|
|
1436
|
+
}
|
|
1437
|
+
};
|
|
1438
|
+
const arm = () => {
|
|
1439
|
+
if (timer) clearTimeout(timer);
|
|
1440
|
+
timer = setTimeout(
|
|
1441
|
+
() => finish(new LokutorError(
|
|
1442
|
+
"internal.timeout",
|
|
1443
|
+
firstByteReceived ? "TTS stream stalled" : "No audio within 30 s",
|
|
1444
|
+
{ retryable: true }
|
|
1445
|
+
)),
|
|
1446
|
+
firstByteReceived ? 15e3 : 3e4
|
|
1447
|
+
);
|
|
1339
1448
|
};
|
|
1340
1449
|
try {
|
|
1341
1450
|
let url = DEFAULT_URLS.TTS;
|
|
1342
1451
|
if (this.apiKey) {
|
|
1343
1452
|
const separator = url.includes("?") ? "&" : "?";
|
|
1344
|
-
url += `${separator}api_key=${this.apiKey}`;
|
|
1453
|
+
url += `${separator}api_key=${encodeURIComponent(this.apiKey)}`;
|
|
1345
1454
|
}
|
|
1346
1455
|
ws = new WebSocket(url);
|
|
1347
1456
|
ws.binaryType = "arraybuffer";
|
|
1348
1457
|
ws.onopen = () => {
|
|
1349
|
-
|
|
1458
|
+
arm();
|
|
1350
1459
|
const req = {
|
|
1351
1460
|
text: options.text,
|
|
1352
1461
|
voice: options.voice || "F1" /* F1 */,
|
|
1353
1462
|
language: options.language || "en" /* ENGLISH */,
|
|
1354
|
-
speed: options.speed || 1.05,
|
|
1355
|
-
steps: options.steps || 24,
|
|
1356
1463
|
visemes: options.visemes || false
|
|
1357
1464
|
};
|
|
1465
|
+
if (options.speed !== void 0) req.speed = options.speed;
|
|
1466
|
+
if (options.steps !== void 0) req.steps = options.steps;
|
|
1358
1467
|
ws.send(JSON.stringify(req));
|
|
1359
1468
|
startTime = nowMs();
|
|
1360
1469
|
};
|
|
1361
1470
|
ws.onmessage = async (event) => {
|
|
1362
|
-
refreshTimeout();
|
|
1363
1471
|
if (event.data instanceof ArrayBuffer) {
|
|
1364
1472
|
if (!firstByteReceived) {
|
|
1365
1473
|
const ttfb = nowMs() - startTime;
|
|
1366
1474
|
if (options.onTTFB) options.onTTFB(ttfb);
|
|
1367
1475
|
firstByteReceived = true;
|
|
1368
1476
|
}
|
|
1477
|
+
arm();
|
|
1369
1478
|
if (options.onAudio) options.onAudio(new Uint8Array(event.data));
|
|
1370
1479
|
return;
|
|
1371
1480
|
}
|
|
1372
1481
|
const text = event.data.toString();
|
|
1373
1482
|
if (text === "EOS") {
|
|
1374
|
-
|
|
1375
|
-
ws.close();
|
|
1376
|
-
resolve();
|
|
1483
|
+
finish();
|
|
1377
1484
|
return;
|
|
1378
1485
|
}
|
|
1486
|
+
let msg;
|
|
1379
1487
|
try {
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
options.
|
|
1397
|
-
|
|
1398
|
-
}
|
|
1399
|
-
if (msg.type === "eos") {
|
|
1400
|
-
if (activityTimeout) clearTimeout(activityTimeout);
|
|
1401
|
-
ws.close();
|
|
1402
|
-
resolve();
|
|
1488
|
+
msg = JSON.parse(text);
|
|
1489
|
+
} catch {
|
|
1490
|
+
return;
|
|
1491
|
+
}
|
|
1492
|
+
if (msg.type === "error") {
|
|
1493
|
+
const data = typeof msg.data === "object" && msg.data ? msg.data : { message: msg.data };
|
|
1494
|
+
finish(new LokutorError(data.code ?? "internal.error", data.message ?? "TTS failed", {
|
|
1495
|
+
detail: data.detail,
|
|
1496
|
+
retryable: data.retryable ?? false
|
|
1497
|
+
}));
|
|
1498
|
+
return;
|
|
1499
|
+
}
|
|
1500
|
+
if (msg.type === "audio" && msg.data) {
|
|
1501
|
+
const audioBuffer = base64ToUint8Array(msg.data);
|
|
1502
|
+
if (!firstByteReceived) {
|
|
1503
|
+
const ttfb = nowMs() - startTime;
|
|
1504
|
+
if (options.onTTFB) options.onTTFB(ttfb);
|
|
1505
|
+
firstByteReceived = true;
|
|
1403
1506
|
}
|
|
1404
|
-
|
|
1507
|
+
arm();
|
|
1508
|
+
if (options.onAudio) options.onAudio(audioBuffer);
|
|
1509
|
+
return;
|
|
1510
|
+
}
|
|
1511
|
+
if (msg.type === "visemes" && Array.isArray(msg.data) && options.onVisemes) {
|
|
1512
|
+
options.onVisemes(normalizeVisemes(msg.data));
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
if (Array.isArray(msg) && options.onVisemes) {
|
|
1516
|
+
options.onVisemes(normalizeVisemes(msg));
|
|
1517
|
+
return;
|
|
1518
|
+
}
|
|
1519
|
+
if (msg.type === "eos") {
|
|
1520
|
+
finish();
|
|
1405
1521
|
}
|
|
1406
1522
|
};
|
|
1407
|
-
ws.onerror = (err) =>
|
|
1408
|
-
|
|
1409
|
-
if (
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
if (activityTimeout) clearTimeout(activityTimeout);
|
|
1414
|
-
resolve();
|
|
1523
|
+
ws.onerror = (err) => finish(err);
|
|
1524
|
+
ws.onclose = (ev) => {
|
|
1525
|
+
if (!finished) {
|
|
1526
|
+
if (firstByteReceived && ev.code === 1e3) finish();
|
|
1527
|
+
else finish(new LokutorError("internal.error", `TTS socket closed (${ev.code}${ev.reason ? `: ${ev.reason}` : ""})`, { retryable: true }));
|
|
1528
|
+
}
|
|
1415
1529
|
};
|
|
1416
1530
|
} catch (err) {
|
|
1417
|
-
|
|
1418
|
-
if (options.onError) options.onError(err);
|
|
1419
|
-
reject(err);
|
|
1531
|
+
finish(err);
|
|
1420
1532
|
}
|
|
1421
1533
|
});
|
|
1422
1534
|
}
|
|
@@ -1522,17 +1634,28 @@ var SpeechToTextClient = class {
|
|
|
1522
1634
|
try {
|
|
1523
1635
|
let url = this.serverUrl;
|
|
1524
1636
|
const separator = url.includes("?") ? "&" : "?";
|
|
1525
|
-
url += `${separator}api_key=${this.apiKey}`;
|
|
1637
|
+
url += `${separator}api_key=${encodeURIComponent(this.apiKey)}`;
|
|
1526
1638
|
this.ws = new WebSocket(url);
|
|
1527
1639
|
this.ws.onopen = async () => {
|
|
1528
1640
|
this.isConnected = true;
|
|
1529
1641
|
this.onStatusChange?.("connected");
|
|
1530
1642
|
this.ws.send(JSON.stringify({ language: this.language || "en" /* ENGLISH */, vad: this.vad }));
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
this.ws.
|
|
1643
|
+
try {
|
|
1644
|
+
await this.audioManager.startMicrophone((data) => {
|
|
1645
|
+
if (this.isConnected && this.ws?.readyState === WebSocket.OPEN) {
|
|
1646
|
+
this.ws.send(data);
|
|
1647
|
+
}
|
|
1648
|
+
});
|
|
1649
|
+
} catch (err) {
|
|
1650
|
+
const error = err instanceof LokutorError ? err : new LokutorError("audio.microphone_unavailable", "Could not start the microphone", { original: err });
|
|
1651
|
+
this.onError?.(error);
|
|
1652
|
+
try {
|
|
1653
|
+
this.ws?.close();
|
|
1654
|
+
} catch {
|
|
1534
1655
|
}
|
|
1535
|
-
|
|
1656
|
+
settle(() => reject(error));
|
|
1657
|
+
return;
|
|
1658
|
+
}
|
|
1536
1659
|
settle(() => resolve(true));
|
|
1537
1660
|
};
|
|
1538
1661
|
this.ws.onmessage = (event) => {
|
|
@@ -1546,7 +1669,11 @@ var SpeechToTextClient = class {
|
|
|
1546
1669
|
this.onPartialTranscript?.(msg.data ?? "");
|
|
1547
1670
|
}
|
|
1548
1671
|
} else if (msg.type === "error") {
|
|
1549
|
-
|
|
1672
|
+
const data = typeof msg.data === "object" && msg.data ? msg.data : { message: msg.data };
|
|
1673
|
+
this.onError?.(new LokutorError(data.code ?? "internal.error", data.message ?? "STT error", {
|
|
1674
|
+
detail: data.detail,
|
|
1675
|
+
retryable: data.retryable ?? false
|
|
1676
|
+
}));
|
|
1550
1677
|
}
|
|
1551
1678
|
} catch {
|
|
1552
1679
|
}
|
|
@@ -2018,6 +2145,17 @@ function injectStyles() {
|
|
|
2018
2145
|
document.head.appendChild(style);
|
|
2019
2146
|
styleInjected = true;
|
|
2020
2147
|
}
|
|
2148
|
+
function microphoneMessage(err) {
|
|
2149
|
+
const name = String(err?.original?.name ?? err?.name ?? "");
|
|
2150
|
+
const detail = String(err?.detail ?? err?.message ?? "");
|
|
2151
|
+
if (/NotAllowedError|SecurityError/.test(name + " " + detail)) {
|
|
2152
|
+
return "Microphone access is blocked. Allow it for this site in your browser settings, then try again.";
|
|
2153
|
+
}
|
|
2154
|
+
if (/NotFoundError|OverconstrainedError/.test(name + " " + detail)) {
|
|
2155
|
+
return "No microphone was found on this device.";
|
|
2156
|
+
}
|
|
2157
|
+
return "The microphone could not start. Please try again.";
|
|
2158
|
+
}
|
|
2021
2159
|
var ConversationalPanel = class {
|
|
2022
2160
|
cfg;
|
|
2023
2161
|
container;
|
|
@@ -2047,6 +2185,8 @@ var ConversationalPanel = class {
|
|
|
2047
2185
|
countdownEl;
|
|
2048
2186
|
noteEl;
|
|
2049
2187
|
countdownTicker = null;
|
|
2188
|
+
microphone = null;
|
|
2189
|
+
micError = null;
|
|
2050
2190
|
// Callbacks
|
|
2051
2191
|
onTranscription;
|
|
2052
2192
|
onResponse;
|
|
@@ -2148,14 +2288,23 @@ var ConversationalPanel = class {
|
|
|
2148
2288
|
d.textContent = s;
|
|
2149
2289
|
return d.innerHTML;
|
|
2150
2290
|
}
|
|
2151
|
-
/**
|
|
2291
|
+
/**
|
|
2292
|
+
* Start the conversation (called when user clicks "Start Conversation" or externally). Call it from
|
|
2293
|
+
* inside the click or tap, before any await: it asks for the microphone there. Resolves true once the
|
|
2294
|
+
* call is live, false when it could not start (the reason is shown, and passed to onError).
|
|
2295
|
+
*/
|
|
2152
2296
|
async start() {
|
|
2153
|
-
if (this.isRunning) return;
|
|
2297
|
+
if (this.isRunning) return false;
|
|
2154
2298
|
if (this._locked) {
|
|
2155
2299
|
this.showError("This session has ended. Refresh the page to start a new one.");
|
|
2156
|
-
return;
|
|
2300
|
+
return false;
|
|
2157
2301
|
}
|
|
2158
2302
|
this.isRunning = true;
|
|
2303
|
+
this.microphone = requestMicrophone();
|
|
2304
|
+
this.micError = null;
|
|
2305
|
+
this.microphone.catch((e) => {
|
|
2306
|
+
this.micError = e;
|
|
2307
|
+
});
|
|
2159
2308
|
this.startBtn.disabled = true;
|
|
2160
2309
|
this.errorEl.classList.remove("is-visible");
|
|
2161
2310
|
this.playConnectingSound();
|
|
@@ -2206,8 +2355,9 @@ var ConversationalPanel = class {
|
|
|
2206
2355
|
onError: (err) => {
|
|
2207
2356
|
if (!this.isRunning) return;
|
|
2208
2357
|
this.onError?.(err);
|
|
2209
|
-
this.end(err?.code === "session.ended" && err?.message ? err.message : "Connection issue. Try again in a moment.");
|
|
2210
|
-
}
|
|
2358
|
+
this.end(err?.code === "audio.microphone_unavailable" ? microphoneMessage(err) : err?.code === "session.ended" && err?.message ? err.message : "Connection issue. Try again in a moment.");
|
|
2359
|
+
},
|
|
2360
|
+
microphone: this.microphone
|
|
2211
2361
|
});
|
|
2212
2362
|
this.agent = agentHandle;
|
|
2213
2363
|
visualizer.setAudioClient(agentHandle);
|
|
@@ -2217,6 +2367,7 @@ var ConversationalPanel = class {
|
|
|
2217
2367
|
} catch (_) {
|
|
2218
2368
|
}
|
|
2219
2369
|
const ok = await agentHandle.connect();
|
|
2370
|
+
if (!this.isRunning) return false;
|
|
2220
2371
|
if (ok) {
|
|
2221
2372
|
this.fadeOutConnectingSound();
|
|
2222
2373
|
const wrap = this.visualizerWrap;
|
|
@@ -2230,19 +2381,21 @@ var ConversationalPanel = class {
|
|
|
2230
2381
|
this.startTimer();
|
|
2231
2382
|
this.onStart?.();
|
|
2232
2383
|
if (window.lucide) window.lucide.createIcons();
|
|
2233
|
-
|
|
2234
|
-
this.fadeOutConnectingSound();
|
|
2235
|
-
this.playErrorTone();
|
|
2236
|
-
this.curtain.classList.remove("is-up");
|
|
2237
|
-
this.showError("Could not connect. Please try again.");
|
|
2238
|
-
this.isRunning = false;
|
|
2384
|
+
return true;
|
|
2239
2385
|
}
|
|
2386
|
+
this.fadeOutConnectingSound();
|
|
2387
|
+
this.playErrorTone();
|
|
2388
|
+
const micError = this.micError;
|
|
2389
|
+
this.stop();
|
|
2390
|
+
this.showError(micError ? microphoneMessage(micError) : "Could not connect. Please try again.");
|
|
2391
|
+
return false;
|
|
2240
2392
|
} catch (err) {
|
|
2241
2393
|
this.fadeOutConnectingSound();
|
|
2242
2394
|
this.playErrorTone();
|
|
2243
2395
|
console.error("ConversationalPanel start error:", err);
|
|
2244
2396
|
this.stop();
|
|
2245
2397
|
this.showError("Something went wrong. Please try again.");
|
|
2398
|
+
return false;
|
|
2246
2399
|
} finally {
|
|
2247
2400
|
this.startBtn.disabled = false;
|
|
2248
2401
|
}
|
|
@@ -2260,6 +2413,10 @@ var ConversationalPanel = class {
|
|
|
2260
2413
|
}
|
|
2261
2414
|
this.agent = null;
|
|
2262
2415
|
}
|
|
2416
|
+
const mic = this.microphone;
|
|
2417
|
+
this.microphone = null;
|
|
2418
|
+
mic?.then((s) => s.getTracks().forEach((t) => t.stop()), () => {
|
|
2419
|
+
});
|
|
2263
2420
|
if (this.timerTicker) {
|
|
2264
2421
|
clearInterval(this.timerTicker);
|
|
2265
2422
|
this.timerTicker = null;
|
|
@@ -2483,9 +2640,11 @@ export {
|
|
|
2483
2640
|
calculateRMS,
|
|
2484
2641
|
float32ToPcm16,
|
|
2485
2642
|
isRetryable,
|
|
2643
|
+
microphoneMessage,
|
|
2486
2644
|
normalizeAudio,
|
|
2487
2645
|
pcm16ToBytes,
|
|
2488
2646
|
pcm16ToFloat32,
|
|
2647
|
+
requestMicrophone,
|
|
2489
2648
|
resample,
|
|
2490
2649
|
resampleWithAntiAliasing,
|
|
2491
2650
|
simpleConversation,
|