@lokutor/sdk 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -2
- package/dist/index.d.mts +70 -25
- package/dist/index.d.ts +70 -25
- package/dist/index.js +267 -106
- package/dist/index.mjs +265 -106
- package/package.json +1 -1
- package/src/browser-audio.ts +82 -13
- package/src/client.ts +185 -99
- package/src/conversational-panel.ts +61 -15
- package/src/index.ts +1 -1
- package/src/types.ts +38 -17
package/dist/index.js
CHANGED
|
@@ -38,9 +38,11 @@ __export(index_exports, {
|
|
|
38
38
|
calculateRMS: () => calculateRMS,
|
|
39
39
|
float32ToPcm16: () => float32ToPcm16,
|
|
40
40
|
isRetryable: () => isRetryable,
|
|
41
|
+
microphoneMessage: () => microphoneMessage,
|
|
41
42
|
normalizeAudio: () => normalizeAudio,
|
|
42
43
|
pcm16ToBytes: () => pcm16ToBytes,
|
|
43
44
|
pcm16ToFloat32: () => pcm16ToFloat32,
|
|
45
|
+
requestMicrophone: () => requestMicrophone,
|
|
44
46
|
resample: () => resample,
|
|
45
47
|
resampleWithAntiAliasing: () => resampleWithAntiAliasing,
|
|
46
48
|
simpleConversation: () => simpleConversation,
|
|
@@ -131,13 +133,19 @@ function isRetryableCode(code) {
|
|
|
131
133
|
"auth.invalid_key",
|
|
132
134
|
"auth.time_limited",
|
|
133
135
|
"session.ended",
|
|
136
|
+
"audio.microphone_unavailable",
|
|
134
137
|
"validation.invalid_voice",
|
|
135
138
|
"validation.invalid_language",
|
|
136
139
|
"validation.text_too_long",
|
|
137
140
|
"validation.speed_out_of_range",
|
|
138
141
|
"validation.steps_out_of_range",
|
|
139
142
|
"validation.invalid_request_format",
|
|
140
|
-
"internal.cancelled"
|
|
143
|
+
"internal.cancelled",
|
|
144
|
+
"agent.not_found",
|
|
145
|
+
"billing.insufficient_balance",
|
|
146
|
+
"billing.topup_required",
|
|
147
|
+
"billing.limit_reached",
|
|
148
|
+
"billing.account_disputed"
|
|
141
149
|
];
|
|
142
150
|
return !fatal.includes(code);
|
|
143
151
|
}
|
|
@@ -285,6 +293,36 @@ var StreamResampler = class {
|
|
|
285
293
|
};
|
|
286
294
|
|
|
287
295
|
// src/browser-audio.ts
|
|
296
|
+
function requestMicrophone(constraints = {}) {
|
|
297
|
+
if (typeof navigator === "undefined") {
|
|
298
|
+
return Promise.reject(new Error("Microphone capture needs a browser"));
|
|
299
|
+
}
|
|
300
|
+
const session = navigator.audioSession;
|
|
301
|
+
if (session && session.type !== "auto" && session.type !== "play-and-record") {
|
|
302
|
+
try {
|
|
303
|
+
session.type = "auto";
|
|
304
|
+
} catch (_) {
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (!navigator.mediaDevices?.getUserMedia) {
|
|
308
|
+
return Promise.reject(new Error("Microphone capture is not available here (it needs HTTPS)"));
|
|
309
|
+
}
|
|
310
|
+
return navigator.mediaDevices.getUserMedia({
|
|
311
|
+
audio: {
|
|
312
|
+
autoGainControl: constraints.autoGainControl ?? true,
|
|
313
|
+
echoCancellation: constraints.echoCancellation ?? true,
|
|
314
|
+
noiseSuppression: constraints.noiseSuppression ?? true
|
|
315
|
+
}
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
async function resumeWithin(ctx, ms) {
|
|
319
|
+
if (ctx.state === "running") return;
|
|
320
|
+
await Promise.race([
|
|
321
|
+
ctx.resume().catch(() => {
|
|
322
|
+
}),
|
|
323
|
+
new Promise((r) => setTimeout(r, ms))
|
|
324
|
+
]);
|
|
325
|
+
}
|
|
288
326
|
var BrowserAudioManager = class {
|
|
289
327
|
audioContext = null;
|
|
290
328
|
mediaStreamAudioSourceNode = null;
|
|
@@ -307,6 +345,7 @@ var BrowserAudioManager = class {
|
|
|
307
345
|
echoCancellation;
|
|
308
346
|
noiseSuppression;
|
|
309
347
|
keepAwake;
|
|
348
|
+
providedMicrophone;
|
|
310
349
|
// Screen wake lock, held while the microphone is live. Browsers release it whenever the page is
|
|
311
350
|
// hidden, so it is taken again when the page becomes visible.
|
|
312
351
|
wakeLock = null;
|
|
@@ -324,6 +363,9 @@ var BrowserAudioManager = class {
|
|
|
324
363
|
this.echoCancellation = config.echoCancellation ?? true;
|
|
325
364
|
this.noiseSuppression = config.noiseSuppression ?? true;
|
|
326
365
|
this.keepAwake = config.keepAwake ?? true;
|
|
366
|
+
this.providedMicrophone = config.microphone ?? null;
|
|
367
|
+
if (this.providedMicrophone instanceof Promise) this.providedMicrophone.catch(() => {
|
|
368
|
+
});
|
|
327
369
|
this.onInputError = config.onInputError;
|
|
328
370
|
}
|
|
329
371
|
/**
|
|
@@ -339,9 +381,8 @@ var BrowserAudioManager = class {
|
|
|
339
381
|
if (!this.audioContext) {
|
|
340
382
|
throw new Error("Failed to initialize AudioContext");
|
|
341
383
|
}
|
|
342
|
-
if (this.audioContext.state
|
|
343
|
-
await this.audioContext
|
|
344
|
-
console.log("\u{1F442} AudioContext resumed");
|
|
384
|
+
if (this.audioContext.state !== "running") {
|
|
385
|
+
await resumeWithin(this.audioContext, 500);
|
|
345
386
|
}
|
|
346
387
|
if (analyserConfig?.enabled !== false) {
|
|
347
388
|
this.analyserNode = this.audioContext.createAnalyser();
|
|
@@ -362,7 +403,10 @@ var BrowserAudioManager = class {
|
|
|
362
403
|
try {
|
|
363
404
|
this.onAudioInput = onAudioInput;
|
|
364
405
|
this.isListening = true;
|
|
365
|
-
|
|
406
|
+
const provided = this.providedMicrophone;
|
|
407
|
+
this.providedMicrophone = null;
|
|
408
|
+
const stream = provided ? await provided : null;
|
|
409
|
+
this.mediaStream = stream && stream.getAudioTracks().some((t) => t.readyState === "live") ? stream : await navigator.mediaDevices.getUserMedia({
|
|
366
410
|
audio: {
|
|
367
411
|
autoGainControl: this.autoGainControl,
|
|
368
412
|
echoCancellation: this.echoCancellation,
|
|
@@ -395,6 +439,7 @@ var BrowserAudioManager = class {
|
|
|
395
439
|
console.log("\u{1F3A4} Microphone started");
|
|
396
440
|
this.holdScreenAwake();
|
|
397
441
|
} catch (error) {
|
|
442
|
+
this.isListening = false;
|
|
398
443
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
399
444
|
if (this.onInputError) this.onInputError(err);
|
|
400
445
|
throw err;
|
|
@@ -629,6 +674,13 @@ var BrowserAudioManager = class {
|
|
|
629
674
|
this.analyserNode.disconnect();
|
|
630
675
|
this.analyserNode = null;
|
|
631
676
|
}
|
|
677
|
+
if (this.audioContext) {
|
|
678
|
+
const ctx = this.audioContext;
|
|
679
|
+
this.audioContext = null;
|
|
680
|
+
this.nextPlaybackTime = 0;
|
|
681
|
+
ctx.close().catch(() => {
|
|
682
|
+
});
|
|
683
|
+
}
|
|
632
684
|
}
|
|
633
685
|
/**
|
|
634
686
|
* Get current audio context state
|
|
@@ -668,6 +720,9 @@ function nowMs() {
|
|
|
668
720
|
function wsToHttp(url) {
|
|
669
721
|
return url.replace(/^wss:/, "https:").replace(/^ws:/, "http:");
|
|
670
722
|
}
|
|
723
|
+
function apiOrigin(base) {
|
|
724
|
+
return new URL(wsToHttp(base || DEFAULT_URLS.VOICE_AGENT)).origin;
|
|
725
|
+
}
|
|
671
726
|
async function fetchJson(url, timeoutMs = 1e4) {
|
|
672
727
|
const controller = new AbortController();
|
|
673
728
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
@@ -780,10 +835,16 @@ var VoiceAgentClient = class {
|
|
|
780
835
|
serverPlaybackRate = 44100;
|
|
781
836
|
serverInputRate = 16e3;
|
|
782
837
|
serverUrl;
|
|
838
|
+
authProvider;
|
|
839
|
+
voiceExplicit = false;
|
|
840
|
+
languageExplicit = false;
|
|
783
841
|
constructor(config) {
|
|
784
|
-
this.apiKey = config.apiKey;
|
|
842
|
+
this.apiKey = config.apiKey || "";
|
|
843
|
+
this.authProvider = config.auth;
|
|
785
844
|
this.agentId = config.agentId || "";
|
|
786
|
-
this.prompt = config.prompt;
|
|
845
|
+
this.prompt = config.prompt ?? "";
|
|
846
|
+
this.voiceExplicit = config.voice !== void 0;
|
|
847
|
+
this.languageExplicit = config.language !== void 0;
|
|
787
848
|
this.voice = config.voice || "F1" /* F1 */;
|
|
788
849
|
this.language = config.language || "en" /* ENGLISH */;
|
|
789
850
|
this.serverUrl = config.serverUrl || DEFAULT_URLS.VOICE_AGENT;
|
|
@@ -803,6 +864,15 @@ var VoiceAgentClient = class {
|
|
|
803
864
|
*/
|
|
804
865
|
async connect(customAudioManager) {
|
|
805
866
|
this.isUserDisconnect = false;
|
|
867
|
+
if (this.authProvider) {
|
|
868
|
+
try {
|
|
869
|
+
this.apiKey = await this.authProvider();
|
|
870
|
+
} catch (err) {
|
|
871
|
+
const error = new LokutorError("auth.missing_key", "The auth callback failed to return a credential", { original: err, retryable: true });
|
|
872
|
+
this.emit("error", error);
|
|
873
|
+
throw error;
|
|
874
|
+
}
|
|
875
|
+
}
|
|
806
876
|
if (this.enableAudio || customAudioManager) {
|
|
807
877
|
if (customAudioManager) {
|
|
808
878
|
this.audioManager = customAudioManager;
|
|
@@ -825,11 +895,11 @@ var VoiceAgentClient = class {
|
|
|
825
895
|
let url = this.serverUrl;
|
|
826
896
|
if (this.apiKey) {
|
|
827
897
|
const separator = url.includes("?") ? "&" : "?";
|
|
828
|
-
url += `${separator}api_key=${this.apiKey}`;
|
|
898
|
+
url += `${separator}api_key=${encodeURIComponent(this.apiKey)}`;
|
|
829
899
|
}
|
|
830
900
|
if (this.agentId) {
|
|
831
901
|
const separator = url.includes("?") ? "&" : "?";
|
|
832
|
-
url += `${separator}agent_id=${this.agentId}`;
|
|
902
|
+
url += `${separator}agent_id=${encodeURIComponent(this.agentId)}`;
|
|
833
903
|
}
|
|
834
904
|
const redactedUrl = url.replace(/api_key=[^&]+/, "api_key=***");
|
|
835
905
|
sdkTrace("ws.connect", {
|
|
@@ -851,11 +921,29 @@ var VoiceAgentClient = class {
|
|
|
851
921
|
sdkTrace("ws.open");
|
|
852
922
|
this.sendConfig();
|
|
853
923
|
if (this.audioManager && !this.micStarted) {
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
this.
|
|
924
|
+
try {
|
|
925
|
+
await this.audioManager.startMicrophone((data) => {
|
|
926
|
+
if (this.isConnected) {
|
|
927
|
+
this.sendAudio(data);
|
|
928
|
+
}
|
|
929
|
+
});
|
|
930
|
+
} catch (e) {
|
|
931
|
+
const name = e?.name || "Error";
|
|
932
|
+
const error = new LokutorError(
|
|
933
|
+
"audio.microphone_unavailable",
|
|
934
|
+
name === "NotAllowedError" || name === "SecurityError" ? "Microphone access was not allowed." : "The microphone could not be started.",
|
|
935
|
+
{ detail: `${name}: ${e?.message ?? e}`, original: e, retryable: false }
|
|
936
|
+
);
|
|
937
|
+
sdkTrace("mic.failed", { name });
|
|
938
|
+
settle(() => reject(error));
|
|
939
|
+
this.isUserDisconnect = true;
|
|
940
|
+
try {
|
|
941
|
+
this.ws?.close();
|
|
942
|
+
} catch (_) {
|
|
857
943
|
}
|
|
858
|
-
|
|
944
|
+
this.emit("error", error);
|
|
945
|
+
return;
|
|
946
|
+
}
|
|
859
947
|
this.micStarted = true;
|
|
860
948
|
}
|
|
861
949
|
settle(() => resolve(true));
|
|
@@ -877,7 +965,7 @@ var VoiceAgentClient = class {
|
|
|
877
965
|
});
|
|
878
966
|
console.error("\u274C WebSocket error:", error.message);
|
|
879
967
|
sdkTrace("ws.error", { code: error.code, message: error.message });
|
|
880
|
-
|
|
968
|
+
this.emit("error", error);
|
|
881
969
|
if (!this.isConnected) {
|
|
882
970
|
settle(() => reject(error));
|
|
883
971
|
}
|
|
@@ -914,8 +1002,8 @@ var VoiceAgentClient = class {
|
|
|
914
1002
|
});
|
|
915
1003
|
console.log(`Session ended by the server (code ${event.code})`);
|
|
916
1004
|
this.emit("ended", { code: event.code, reason: event.reason || "" });
|
|
917
|
-
|
|
918
|
-
|
|
1005
|
+
this.emit("error", ended);
|
|
1006
|
+
this.emit("status", "disconnected");
|
|
919
1007
|
return;
|
|
920
1008
|
}
|
|
921
1009
|
if (!this.isUserDisconnect && this.reconnectAttempts < this.maxReconnectAttempts) {
|
|
@@ -923,7 +1011,7 @@ var VoiceAgentClient = class {
|
|
|
923
1011
|
this.reconnectAttempts++;
|
|
924
1012
|
const backoffDelay = Math.min(1e3 * Math.pow(2, this.reconnectAttempts), 1e4);
|
|
925
1013
|
console.warn(`Connection lost. Reconnecting in ${backoffDelay}ms (attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})`);
|
|
926
|
-
|
|
1014
|
+
this.emit("status", "reconnecting");
|
|
927
1015
|
if (this.reconnectTimer) clearTimeout(this.reconnectTimer);
|
|
928
1016
|
this.reconnectTimer = setTimeout(() => {
|
|
929
1017
|
this.reconnectTimer = null;
|
|
@@ -933,15 +1021,15 @@ var VoiceAgentClient = class {
|
|
|
933
1021
|
}, backoffDelay);
|
|
934
1022
|
} else if (this.isUserDisconnect) {
|
|
935
1023
|
console.log("Disconnected");
|
|
936
|
-
|
|
1024
|
+
this.emit("status", "disconnected");
|
|
937
1025
|
} else {
|
|
938
1026
|
console.log("Max reconnection attempts reached");
|
|
939
|
-
|
|
1027
|
+
this.emit("status", "disconnected");
|
|
940
1028
|
}
|
|
941
1029
|
};
|
|
942
1030
|
} catch (err) {
|
|
943
1031
|
const error = err instanceof LokutorError ? err : new LokutorError("internal.error", "Failed to create WebSocket connection", { original: err });
|
|
944
|
-
|
|
1032
|
+
this.emit("error", error);
|
|
945
1033
|
settle(() => reject(error));
|
|
946
1034
|
}
|
|
947
1035
|
});
|
|
@@ -969,13 +1057,14 @@ var VoiceAgentClient = class {
|
|
|
969
1057
|
sendConfig() {
|
|
970
1058
|
if (!this.ws || !this.isConnected) return;
|
|
971
1059
|
this.configConfirmed = false;
|
|
972
|
-
|
|
973
|
-
this.ws.send(JSON.stringify({ type: "
|
|
1060
|
+
const agentSession = Boolean(this.agentId);
|
|
1061
|
+
if (!agentSession || this.voiceExplicit) this.ws.send(JSON.stringify({ type: "voice", data: this.voice }));
|
|
1062
|
+
if (!agentSession || this.languageExplicit) this.ws.send(JSON.stringify({ type: "language", data: this.language }));
|
|
974
1063
|
if (this.tools && this.tools.length > 0) {
|
|
975
1064
|
this.ws.send(JSON.stringify({ type: "tools", data: this.tools }));
|
|
976
1065
|
}
|
|
977
1066
|
this.ws.send(JSON.stringify({ type: "visemes", data: this.wantVisemes }));
|
|
978
|
-
this.ws.send(JSON.stringify({ type: "prompt", data: this.prompt }));
|
|
1067
|
+
if (!agentSession || this.prompt) this.ws.send(JSON.stringify({ type: "prompt", data: this.prompt }));
|
|
979
1068
|
this.ws.send(JSON.stringify({ type: "rates", playback: 44100, input: 16e3 }));
|
|
980
1069
|
sdkTrace("ws.send.config", {
|
|
981
1070
|
promptLen: this.prompt?.length || 0,
|
|
@@ -1036,6 +1125,10 @@ var VoiceAgentClient = class {
|
|
|
1036
1125
|
sdkTrace("server_rates", { playback: this.serverPlaybackRate, input: this.serverInputRate });
|
|
1037
1126
|
break;
|
|
1038
1127
|
case "transcript":
|
|
1128
|
+
if (msg.partial) {
|
|
1129
|
+
this.emit("transcription_partial", msg.data);
|
|
1130
|
+
break;
|
|
1131
|
+
}
|
|
1039
1132
|
const role = msg.role === "user" ? "user" : "agent";
|
|
1040
1133
|
this.messages.push({
|
|
1041
1134
|
role,
|
|
@@ -1043,11 +1136,9 @@ var VoiceAgentClient = class {
|
|
|
1043
1136
|
timestamp: nowMs()
|
|
1044
1137
|
});
|
|
1045
1138
|
if (msg.role === "user") {
|
|
1046
|
-
if (this.onTranscription) this.onTranscription(msg.data);
|
|
1047
1139
|
this.emit("transcription", msg.data);
|
|
1048
1140
|
console.log(`\u{1F4AC} You: ${msg.data}`);
|
|
1049
1141
|
} else {
|
|
1050
|
-
if (this.onResponse) this.onResponse(msg.data);
|
|
1051
1142
|
this.emit("response", msg.data);
|
|
1052
1143
|
console.log(`\u{1F916} Agent: ${msg.data}`);
|
|
1053
1144
|
}
|
|
@@ -1069,7 +1160,6 @@ var VoiceAgentClient = class {
|
|
|
1069
1160
|
if (msg.data === "interrupted" && this.audioManager) {
|
|
1070
1161
|
this.audioManager.stopPlayback();
|
|
1071
1162
|
}
|
|
1072
|
-
if (this.onStatus) this.onStatus(msg.data);
|
|
1073
1163
|
this.emit("status", msg.data);
|
|
1074
1164
|
const icons = {
|
|
1075
1165
|
"interrupted": "\u26A1",
|
|
@@ -1107,7 +1197,6 @@ var VoiceAgentClient = class {
|
|
|
1107
1197
|
detail: backendDetail,
|
|
1108
1198
|
retryable: backendRetryable
|
|
1109
1199
|
});
|
|
1110
|
-
if (this.onError) this.onError(error);
|
|
1111
1200
|
this.emit("error", error);
|
|
1112
1201
|
console.error(`\u274C Server error: [${error.code}] ${error.message}`);
|
|
1113
1202
|
break;
|
|
@@ -1120,11 +1209,12 @@ var VoiceAgentClient = class {
|
|
|
1120
1209
|
}
|
|
1121
1210
|
break;
|
|
1122
1211
|
}
|
|
1123
|
-
case "tool_call":
|
|
1124
|
-
|
|
1212
|
+
case "tool_call": {
|
|
1213
|
+
const call = { name: msg.name, arguments: msg.arguments, call_id: msg.call_id };
|
|
1125
1214
|
console.log(`\u{1F6E0}\uFE0F Tool Call: ${msg.name}(${msg.arguments})`);
|
|
1126
|
-
this.emit("tool_call",
|
|
1215
|
+
this.emit("tool_call", call);
|
|
1127
1216
|
break;
|
|
1217
|
+
}
|
|
1128
1218
|
}
|
|
1129
1219
|
} catch (e) {
|
|
1130
1220
|
sdkTrace("ws.recv.parse_error", { preview: text?.slice(0, 120) });
|
|
@@ -1238,7 +1328,7 @@ var VoiceAgentClient = class {
|
|
|
1238
1328
|
* No authentication required.
|
|
1239
1329
|
*/
|
|
1240
1330
|
static async fetchVoices(baseUrl) {
|
|
1241
|
-
const url =
|
|
1331
|
+
const url = apiOrigin(baseUrl);
|
|
1242
1332
|
const data = await fetchJson(`${url}/voices`);
|
|
1243
1333
|
return data.voices || [];
|
|
1244
1334
|
}
|
|
@@ -1247,7 +1337,7 @@ var VoiceAgentClient = class {
|
|
|
1247
1337
|
* No authentication required.
|
|
1248
1338
|
*/
|
|
1249
1339
|
static async fetchLanguages(baseUrl) {
|
|
1250
|
-
const url =
|
|
1340
|
+
const url = apiOrigin(baseUrl);
|
|
1251
1341
|
const data = await fetchJson(`${url}/languages`);
|
|
1252
1342
|
return data.languages || [];
|
|
1253
1343
|
}
|
|
@@ -1256,7 +1346,7 @@ var VoiceAgentClient = class {
|
|
|
1256
1346
|
* No authentication required.
|
|
1257
1347
|
*/
|
|
1258
1348
|
static async fetchModels(baseUrl) {
|
|
1259
|
-
const url =
|
|
1349
|
+
const url = apiOrigin(baseUrl);
|
|
1260
1350
|
const data = await fetchJson(`${url}/models`);
|
|
1261
1351
|
return data.models || [];
|
|
1262
1352
|
}
|
|
@@ -1265,7 +1355,7 @@ var VoiceAgentClient = class {
|
|
|
1265
1355
|
* No authentication required.
|
|
1266
1356
|
*/
|
|
1267
1357
|
static async fetchConfig(baseUrl) {
|
|
1268
|
-
const url =
|
|
1358
|
+
const url = apiOrigin(baseUrl);
|
|
1269
1359
|
return fetchJson(`${url}/config`);
|
|
1270
1360
|
}
|
|
1271
1361
|
/**
|
|
@@ -1273,16 +1363,16 @@ var VoiceAgentClient = class {
|
|
|
1273
1363
|
* No authentication required.
|
|
1274
1364
|
*/
|
|
1275
1365
|
static async fetchStatus(baseUrl) {
|
|
1276
|
-
const url =
|
|
1366
|
+
const url = apiOrigin(baseUrl);
|
|
1277
1367
|
return fetchJson(`${url}/status`);
|
|
1278
1368
|
}
|
|
1279
1369
|
/**
|
|
1280
|
-
* Fetch
|
|
1281
|
-
* No authentication required.
|
|
1370
|
+
* Fetch readiness from the server (GET /ready: 200 once it can serve).
|
|
1371
|
+
* No authentication required. (/health is the operator dashboard's and needs its password.)
|
|
1282
1372
|
*/
|
|
1283
1373
|
static async fetchHealth(baseUrl) {
|
|
1284
|
-
const url =
|
|
1285
|
-
return fetchJson(`${url}/
|
|
1374
|
+
const url = apiOrigin(baseUrl);
|
|
1375
|
+
return fetchJson(`${url}/ready`);
|
|
1286
1376
|
}
|
|
1287
1377
|
/**
|
|
1288
1378
|
* Update the system prompt mid-conversation
|
|
@@ -1307,6 +1397,7 @@ var VoiceAgentClient = class {
|
|
|
1307
1397
|
*/
|
|
1308
1398
|
updateVoice(voice) {
|
|
1309
1399
|
this.voice = voice;
|
|
1400
|
+
this.voiceExplicit = true;
|
|
1310
1401
|
if (this.ws && this.ws.readyState === WebSocket.OPEN && this.isConnected) {
|
|
1311
1402
|
this.ws.send(JSON.stringify({ type: "voice", data: voice }));
|
|
1312
1403
|
console.log(`\u2699\uFE0F Updated voice: ${voice}`);
|
|
@@ -1317,6 +1408,7 @@ var VoiceAgentClient = class {
|
|
|
1317
1408
|
*/
|
|
1318
1409
|
updateLanguage(language) {
|
|
1319
1410
|
this.language = language;
|
|
1411
|
+
this.languageExplicit = true;
|
|
1320
1412
|
if (this.ws && this.ws.readyState === WebSocket.OPEN && this.isConnected) {
|
|
1321
1413
|
this.ws.send(JSON.stringify({ type: "language", data: language }));
|
|
1322
1414
|
console.log(`\u2699\uFE0F Updated language: ${language}`);
|
|
@@ -1376,98 +1468,120 @@ var TTSClient = class {
|
|
|
1376
1468
|
*/
|
|
1377
1469
|
synthesize(options) {
|
|
1378
1470
|
return new Promise((resolve, reject) => {
|
|
1379
|
-
let
|
|
1471
|
+
let timer;
|
|
1380
1472
|
let ws;
|
|
1381
1473
|
let startTime;
|
|
1382
1474
|
let firstByteReceived = false;
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1475
|
+
let finished = false;
|
|
1476
|
+
const finish = (err) => {
|
|
1477
|
+
if (finished) return;
|
|
1478
|
+
finished = true;
|
|
1479
|
+
if (timer) clearTimeout(timer);
|
|
1480
|
+
try {
|
|
1481
|
+
ws?.close();
|
|
1482
|
+
} catch {
|
|
1483
|
+
}
|
|
1484
|
+
if (err) {
|
|
1485
|
+
if (options.onError) options.onError(err);
|
|
1486
|
+
reject(err);
|
|
1487
|
+
} else {
|
|
1388
1488
|
resolve();
|
|
1389
|
-
}
|
|
1489
|
+
}
|
|
1490
|
+
};
|
|
1491
|
+
const arm = () => {
|
|
1492
|
+
if (timer) clearTimeout(timer);
|
|
1493
|
+
timer = setTimeout(
|
|
1494
|
+
() => finish(new LokutorError(
|
|
1495
|
+
"internal.timeout",
|
|
1496
|
+
firstByteReceived ? "TTS stream stalled" : "No audio within 30 s",
|
|
1497
|
+
{ retryable: true }
|
|
1498
|
+
)),
|
|
1499
|
+
firstByteReceived ? 15e3 : 3e4
|
|
1500
|
+
);
|
|
1390
1501
|
};
|
|
1391
1502
|
try {
|
|
1392
1503
|
let url = DEFAULT_URLS.TTS;
|
|
1393
1504
|
if (this.apiKey) {
|
|
1394
1505
|
const separator = url.includes("?") ? "&" : "?";
|
|
1395
|
-
url += `${separator}api_key=${this.apiKey}`;
|
|
1506
|
+
url += `${separator}api_key=${encodeURIComponent(this.apiKey)}`;
|
|
1396
1507
|
}
|
|
1397
1508
|
ws = new WebSocket(url);
|
|
1398
1509
|
ws.binaryType = "arraybuffer";
|
|
1399
1510
|
ws.onopen = () => {
|
|
1400
|
-
|
|
1511
|
+
arm();
|
|
1401
1512
|
const req = {
|
|
1402
1513
|
text: options.text,
|
|
1403
1514
|
voice: options.voice || "F1" /* F1 */,
|
|
1404
1515
|
language: options.language || "en" /* ENGLISH */,
|
|
1405
|
-
speed: options.speed || 1.05,
|
|
1406
|
-
steps: options.steps || 24,
|
|
1407
1516
|
visemes: options.visemes || false
|
|
1408
1517
|
};
|
|
1518
|
+
if (options.speed !== void 0) req.speed = options.speed;
|
|
1519
|
+
if (options.steps !== void 0) req.steps = options.steps;
|
|
1409
1520
|
ws.send(JSON.stringify(req));
|
|
1410
1521
|
startTime = nowMs();
|
|
1411
1522
|
};
|
|
1412
1523
|
ws.onmessage = async (event) => {
|
|
1413
|
-
refreshTimeout();
|
|
1414
1524
|
if (event.data instanceof ArrayBuffer) {
|
|
1415
1525
|
if (!firstByteReceived) {
|
|
1416
1526
|
const ttfb = nowMs() - startTime;
|
|
1417
1527
|
if (options.onTTFB) options.onTTFB(ttfb);
|
|
1418
1528
|
firstByteReceived = true;
|
|
1419
1529
|
}
|
|
1530
|
+
arm();
|
|
1420
1531
|
if (options.onAudio) options.onAudio(new Uint8Array(event.data));
|
|
1421
1532
|
return;
|
|
1422
1533
|
}
|
|
1423
1534
|
const text = event.data.toString();
|
|
1424
1535
|
if (text === "EOS") {
|
|
1425
|
-
|
|
1426
|
-
ws.close();
|
|
1427
|
-
resolve();
|
|
1536
|
+
finish();
|
|
1428
1537
|
return;
|
|
1429
1538
|
}
|
|
1539
|
+
let msg;
|
|
1430
1540
|
try {
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
options.
|
|
1448
|
-
|
|
1449
|
-
}
|
|
1450
|
-
if (msg.type === "eos") {
|
|
1451
|
-
if (activityTimeout) clearTimeout(activityTimeout);
|
|
1452
|
-
ws.close();
|
|
1453
|
-
resolve();
|
|
1541
|
+
msg = JSON.parse(text);
|
|
1542
|
+
} catch {
|
|
1543
|
+
return;
|
|
1544
|
+
}
|
|
1545
|
+
if (msg.type === "error") {
|
|
1546
|
+
const data = typeof msg.data === "object" && msg.data ? msg.data : { message: msg.data };
|
|
1547
|
+
finish(new LokutorError(data.code ?? "internal.error", data.message ?? "TTS failed", {
|
|
1548
|
+
detail: data.detail,
|
|
1549
|
+
retryable: data.retryable ?? false
|
|
1550
|
+
}));
|
|
1551
|
+
return;
|
|
1552
|
+
}
|
|
1553
|
+
if (msg.type === "audio" && msg.data) {
|
|
1554
|
+
const audioBuffer = base64ToUint8Array(msg.data);
|
|
1555
|
+
if (!firstByteReceived) {
|
|
1556
|
+
const ttfb = nowMs() - startTime;
|
|
1557
|
+
if (options.onTTFB) options.onTTFB(ttfb);
|
|
1558
|
+
firstByteReceived = true;
|
|
1454
1559
|
}
|
|
1455
|
-
|
|
1560
|
+
arm();
|
|
1561
|
+
if (options.onAudio) options.onAudio(audioBuffer);
|
|
1562
|
+
return;
|
|
1563
|
+
}
|
|
1564
|
+
if (msg.type === "visemes" && Array.isArray(msg.data) && options.onVisemes) {
|
|
1565
|
+
options.onVisemes(normalizeVisemes(msg.data));
|
|
1566
|
+
return;
|
|
1567
|
+
}
|
|
1568
|
+
if (Array.isArray(msg) && options.onVisemes) {
|
|
1569
|
+
options.onVisemes(normalizeVisemes(msg));
|
|
1570
|
+
return;
|
|
1571
|
+
}
|
|
1572
|
+
if (msg.type === "eos") {
|
|
1573
|
+
finish();
|
|
1456
1574
|
}
|
|
1457
1575
|
};
|
|
1458
|
-
ws.onerror = (err) =>
|
|
1459
|
-
|
|
1460
|
-
if (
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
if (activityTimeout) clearTimeout(activityTimeout);
|
|
1465
|
-
resolve();
|
|
1576
|
+
ws.onerror = (err) => finish(err);
|
|
1577
|
+
ws.onclose = (ev) => {
|
|
1578
|
+
if (!finished) {
|
|
1579
|
+
if (firstByteReceived && ev.code === 1e3) finish();
|
|
1580
|
+
else finish(new LokutorError("internal.error", `TTS socket closed (${ev.code}${ev.reason ? `: ${ev.reason}` : ""})`, { retryable: true }));
|
|
1581
|
+
}
|
|
1466
1582
|
};
|
|
1467
1583
|
} catch (err) {
|
|
1468
|
-
|
|
1469
|
-
if (options.onError) options.onError(err);
|
|
1470
|
-
reject(err);
|
|
1584
|
+
finish(err);
|
|
1471
1585
|
}
|
|
1472
1586
|
});
|
|
1473
1587
|
}
|
|
@@ -1573,17 +1687,28 @@ var SpeechToTextClient = class {
|
|
|
1573
1687
|
try {
|
|
1574
1688
|
let url = this.serverUrl;
|
|
1575
1689
|
const separator = url.includes("?") ? "&" : "?";
|
|
1576
|
-
url += `${separator}api_key=${this.apiKey}`;
|
|
1690
|
+
url += `${separator}api_key=${encodeURIComponent(this.apiKey)}`;
|
|
1577
1691
|
this.ws = new WebSocket(url);
|
|
1578
1692
|
this.ws.onopen = async () => {
|
|
1579
1693
|
this.isConnected = true;
|
|
1580
1694
|
this.onStatusChange?.("connected");
|
|
1581
1695
|
this.ws.send(JSON.stringify({ language: this.language || "en" /* ENGLISH */, vad: this.vad }));
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
this.ws.
|
|
1696
|
+
try {
|
|
1697
|
+
await this.audioManager.startMicrophone((data) => {
|
|
1698
|
+
if (this.isConnected && this.ws?.readyState === WebSocket.OPEN) {
|
|
1699
|
+
this.ws.send(data);
|
|
1700
|
+
}
|
|
1701
|
+
});
|
|
1702
|
+
} catch (err) {
|
|
1703
|
+
const error = err instanceof LokutorError ? err : new LokutorError("audio.microphone_unavailable", "Could not start the microphone", { original: err });
|
|
1704
|
+
this.onError?.(error);
|
|
1705
|
+
try {
|
|
1706
|
+
this.ws?.close();
|
|
1707
|
+
} catch {
|
|
1585
1708
|
}
|
|
1586
|
-
|
|
1709
|
+
settle(() => reject(error));
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1587
1712
|
settle(() => resolve(true));
|
|
1588
1713
|
};
|
|
1589
1714
|
this.ws.onmessage = (event) => {
|
|
@@ -1597,7 +1722,11 @@ var SpeechToTextClient = class {
|
|
|
1597
1722
|
this.onPartialTranscript?.(msg.data ?? "");
|
|
1598
1723
|
}
|
|
1599
1724
|
} else if (msg.type === "error") {
|
|
1600
|
-
|
|
1725
|
+
const data = typeof msg.data === "object" && msg.data ? msg.data : { message: msg.data };
|
|
1726
|
+
this.onError?.(new LokutorError(data.code ?? "internal.error", data.message ?? "STT error", {
|
|
1727
|
+
detail: data.detail,
|
|
1728
|
+
retryable: data.retryable ?? false
|
|
1729
|
+
}));
|
|
1601
1730
|
}
|
|
1602
1731
|
} catch {
|
|
1603
1732
|
}
|
|
@@ -2069,6 +2198,17 @@ function injectStyles() {
|
|
|
2069
2198
|
document.head.appendChild(style);
|
|
2070
2199
|
styleInjected = true;
|
|
2071
2200
|
}
|
|
2201
|
+
function microphoneMessage(err) {
|
|
2202
|
+
const name = String(err?.original?.name ?? err?.name ?? "");
|
|
2203
|
+
const detail = String(err?.detail ?? err?.message ?? "");
|
|
2204
|
+
if (/NotAllowedError|SecurityError/.test(name + " " + detail)) {
|
|
2205
|
+
return "Microphone access is blocked. Allow it for this site in your browser settings, then try again.";
|
|
2206
|
+
}
|
|
2207
|
+
if (/NotFoundError|OverconstrainedError/.test(name + " " + detail)) {
|
|
2208
|
+
return "No microphone was found on this device.";
|
|
2209
|
+
}
|
|
2210
|
+
return "The microphone could not start. Please try again.";
|
|
2211
|
+
}
|
|
2072
2212
|
var ConversationalPanel = class {
|
|
2073
2213
|
cfg;
|
|
2074
2214
|
container;
|
|
@@ -2098,6 +2238,8 @@ var ConversationalPanel = class {
|
|
|
2098
2238
|
countdownEl;
|
|
2099
2239
|
noteEl;
|
|
2100
2240
|
countdownTicker = null;
|
|
2241
|
+
microphone = null;
|
|
2242
|
+
micError = null;
|
|
2101
2243
|
// Callbacks
|
|
2102
2244
|
onTranscription;
|
|
2103
2245
|
onResponse;
|
|
@@ -2199,14 +2341,23 @@ var ConversationalPanel = class {
|
|
|
2199
2341
|
d.textContent = s;
|
|
2200
2342
|
return d.innerHTML;
|
|
2201
2343
|
}
|
|
2202
|
-
/**
|
|
2344
|
+
/**
|
|
2345
|
+
* Start the conversation (called when user clicks "Start Conversation" or externally). Call it from
|
|
2346
|
+
* inside the click or tap, before any await: it asks for the microphone there. Resolves true once the
|
|
2347
|
+
* call is live, false when it could not start (the reason is shown, and passed to onError).
|
|
2348
|
+
*/
|
|
2203
2349
|
async start() {
|
|
2204
|
-
if (this.isRunning) return;
|
|
2350
|
+
if (this.isRunning) return false;
|
|
2205
2351
|
if (this._locked) {
|
|
2206
2352
|
this.showError("This session has ended. Refresh the page to start a new one.");
|
|
2207
|
-
return;
|
|
2353
|
+
return false;
|
|
2208
2354
|
}
|
|
2209
2355
|
this.isRunning = true;
|
|
2356
|
+
this.microphone = requestMicrophone();
|
|
2357
|
+
this.micError = null;
|
|
2358
|
+
this.microphone.catch((e) => {
|
|
2359
|
+
this.micError = e;
|
|
2360
|
+
});
|
|
2210
2361
|
this.startBtn.disabled = true;
|
|
2211
2362
|
this.errorEl.classList.remove("is-visible");
|
|
2212
2363
|
this.playConnectingSound();
|
|
@@ -2257,8 +2408,9 @@ var ConversationalPanel = class {
|
|
|
2257
2408
|
onError: (err) => {
|
|
2258
2409
|
if (!this.isRunning) return;
|
|
2259
2410
|
this.onError?.(err);
|
|
2260
|
-
this.end(err?.code === "session.ended" && err?.message ? err.message : "Connection issue. Try again in a moment.");
|
|
2261
|
-
}
|
|
2411
|
+
this.end(err?.code === "audio.microphone_unavailable" ? microphoneMessage(err) : err?.code === "session.ended" && err?.message ? err.message : "Connection issue. Try again in a moment.");
|
|
2412
|
+
},
|
|
2413
|
+
microphone: this.microphone
|
|
2262
2414
|
});
|
|
2263
2415
|
this.agent = agentHandle;
|
|
2264
2416
|
visualizer.setAudioClient(agentHandle);
|
|
@@ -2268,6 +2420,7 @@ var ConversationalPanel = class {
|
|
|
2268
2420
|
} catch (_) {
|
|
2269
2421
|
}
|
|
2270
2422
|
const ok = await agentHandle.connect();
|
|
2423
|
+
if (!this.isRunning) return false;
|
|
2271
2424
|
if (ok) {
|
|
2272
2425
|
this.fadeOutConnectingSound();
|
|
2273
2426
|
const wrap = this.visualizerWrap;
|
|
@@ -2281,19 +2434,21 @@ var ConversationalPanel = class {
|
|
|
2281
2434
|
this.startTimer();
|
|
2282
2435
|
this.onStart?.();
|
|
2283
2436
|
if (window.lucide) window.lucide.createIcons();
|
|
2284
|
-
|
|
2285
|
-
this.fadeOutConnectingSound();
|
|
2286
|
-
this.playErrorTone();
|
|
2287
|
-
this.curtain.classList.remove("is-up");
|
|
2288
|
-
this.showError("Could not connect. Please try again.");
|
|
2289
|
-
this.isRunning = false;
|
|
2437
|
+
return true;
|
|
2290
2438
|
}
|
|
2439
|
+
this.fadeOutConnectingSound();
|
|
2440
|
+
this.playErrorTone();
|
|
2441
|
+
const micError = this.micError;
|
|
2442
|
+
this.stop();
|
|
2443
|
+
this.showError(micError ? microphoneMessage(micError) : "Could not connect. Please try again.");
|
|
2444
|
+
return false;
|
|
2291
2445
|
} catch (err) {
|
|
2292
2446
|
this.fadeOutConnectingSound();
|
|
2293
2447
|
this.playErrorTone();
|
|
2294
2448
|
console.error("ConversationalPanel start error:", err);
|
|
2295
2449
|
this.stop();
|
|
2296
2450
|
this.showError("Something went wrong. Please try again.");
|
|
2451
|
+
return false;
|
|
2297
2452
|
} finally {
|
|
2298
2453
|
this.startBtn.disabled = false;
|
|
2299
2454
|
}
|
|
@@ -2311,6 +2466,10 @@ var ConversationalPanel = class {
|
|
|
2311
2466
|
}
|
|
2312
2467
|
this.agent = null;
|
|
2313
2468
|
}
|
|
2469
|
+
const mic = this.microphone;
|
|
2470
|
+
this.microphone = null;
|
|
2471
|
+
mic?.then((s) => s.getTracks().forEach((t) => t.stop()), () => {
|
|
2472
|
+
});
|
|
2314
2473
|
if (this.timerTicker) {
|
|
2315
2474
|
clearInterval(this.timerTicker);
|
|
2316
2475
|
this.timerTicker = null;
|
|
@@ -2535,9 +2694,11 @@ var ConversationalPanel = class {
|
|
|
2535
2694
|
calculateRMS,
|
|
2536
2695
|
float32ToPcm16,
|
|
2537
2696
|
isRetryable,
|
|
2697
|
+
microphoneMessage,
|
|
2538
2698
|
normalizeAudio,
|
|
2539
2699
|
pcm16ToBytes,
|
|
2540
2700
|
pcm16ToFloat32,
|
|
2701
|
+
requestMicrophone,
|
|
2541
2702
|
resample,
|
|
2542
2703
|
resampleWithAntiAliasing,
|
|
2543
2704
|
simpleConversation,
|