@lokutor/sdk 1.1.43 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +150 -7
- package/dist/index.d.ts +150 -7
- package/dist/index.js +340 -36
- package/dist/index.mjs +335 -35
- package/package.json +1 -1
- package/src/browser-audio.ts +32 -5
- package/src/client.ts +219 -0
- package/src/conversational-panel.ts +59 -34
- package/src/index.ts +15 -1
- package/src/node-audio.ts +15 -3
- package/src/types.ts +59 -0
package/dist/index.mjs
CHANGED
|
@@ -60,7 +60,8 @@ var AUDIO_CONFIG = {
|
|
|
60
60
|
};
|
|
61
61
|
var DEFAULT_URLS = {
|
|
62
62
|
VOICE_AGENT: "wss://api.lokutor.com/ws/agent",
|
|
63
|
-
TTS: "wss://api.lokutor.com/ws/tts"
|
|
63
|
+
TTS: "wss://api.lokutor.com/ws/tts",
|
|
64
|
+
STT: "wss://api.lokutor.com/ws/stt"
|
|
64
65
|
};
|
|
65
66
|
var LokutorError = class extends Error {
|
|
66
67
|
code;
|
|
@@ -240,6 +241,10 @@ var BrowserAudioManager = class {
|
|
|
240
241
|
mediaStreamAudioSourceNode = null;
|
|
241
242
|
scriptProcessor = null;
|
|
242
243
|
analyserNode = null;
|
|
244
|
+
// Reused across getAmplitude() calls instead of allocating a new
|
|
245
|
+
// Uint8Array on every call — this runs once per animation frame (~60/sec)
|
|
246
|
+
// from the visualizer, on the same main thread as audio capture.
|
|
247
|
+
amplitudeBuffer = null;
|
|
243
248
|
mediaStream = null;
|
|
244
249
|
resampler = null;
|
|
245
250
|
// Playback scheduling
|
|
@@ -306,7 +311,7 @@ var BrowserAudioManager = class {
|
|
|
306
311
|
}
|
|
307
312
|
});
|
|
308
313
|
this.mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.mediaStream);
|
|
309
|
-
const bufferSize =
|
|
314
|
+
const bufferSize = 1024;
|
|
310
315
|
this.scriptProcessor = this.audioContext.createScriptProcessor(
|
|
311
316
|
bufferSize,
|
|
312
317
|
1,
|
|
@@ -340,7 +345,6 @@ var BrowserAudioManager = class {
|
|
|
340
345
|
*/
|
|
341
346
|
_processAudioInput(event) {
|
|
342
347
|
if (!this.onAudioInput || !this.audioContext || !this.isListening) return;
|
|
343
|
-
if (this.isMuted) return;
|
|
344
348
|
const inputBuffer = event.inputBuffer;
|
|
345
349
|
const inputData = inputBuffer.getChannelData(0);
|
|
346
350
|
const outputBuffer = event.outputBuffer;
|
|
@@ -352,6 +356,9 @@ var BrowserAudioManager = class {
|
|
|
352
356
|
processedData = this.resampler.process(processedData);
|
|
353
357
|
}
|
|
354
358
|
if (processedData.length === 0) return;
|
|
359
|
+
if (this.isMuted) {
|
|
360
|
+
processedData = new Float32Array(processedData.length);
|
|
361
|
+
}
|
|
355
362
|
const int16Data = float32ToPcm16(processedData);
|
|
356
363
|
const uint8Data = new Uint8Array(
|
|
357
364
|
int16Data.buffer,
|
|
@@ -475,9 +482,11 @@ var BrowserAudioManager = class {
|
|
|
475
482
|
*/
|
|
476
483
|
getAmplitude() {
|
|
477
484
|
if (!this.analyserNode) return 0;
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
485
|
+
if (!this.amplitudeBuffer || this.amplitudeBuffer.length !== this.analyserNode.frequencyBinCount) {
|
|
486
|
+
this.amplitudeBuffer = new Uint8Array(this.analyserNode.frequencyBinCount);
|
|
487
|
+
}
|
|
488
|
+
this.analyserNode.getByteTimeDomainData(this.amplitudeBuffer);
|
|
489
|
+
const rms = calculateRMS(this.amplitudeBuffer);
|
|
481
490
|
return Math.min(rms * 10, 1);
|
|
482
491
|
}
|
|
483
492
|
/**
|
|
@@ -584,6 +593,14 @@ function base64ToUint8Array(base64) {
|
|
|
584
593
|
}
|
|
585
594
|
return bytes;
|
|
586
595
|
}
|
|
596
|
+
function uint8ArrayToBase64(bytes) {
|
|
597
|
+
let binaryString = "";
|
|
598
|
+
const chunkSize = 32768;
|
|
599
|
+
for (let i = 0; i < bytes.length; i += chunkSize) {
|
|
600
|
+
binaryString += String.fromCharCode(...bytes.subarray(i, i + chunkSize));
|
|
601
|
+
}
|
|
602
|
+
return btoa(binaryString);
|
|
603
|
+
}
|
|
587
604
|
function normalizeVisemes(payload) {
|
|
588
605
|
if (!Array.isArray(payload)) return [];
|
|
589
606
|
const normalized = [];
|
|
@@ -620,6 +637,7 @@ function extractVisemePayload(msg) {
|
|
|
620
637
|
var VoiceAgentClient = class {
|
|
621
638
|
ws = null;
|
|
622
639
|
apiKey;
|
|
640
|
+
agentId = "";
|
|
623
641
|
prompt;
|
|
624
642
|
voice;
|
|
625
643
|
language;
|
|
@@ -652,6 +670,7 @@ var VoiceAgentClient = class {
|
|
|
652
670
|
serverUrl;
|
|
653
671
|
constructor(config) {
|
|
654
672
|
this.apiKey = config.apiKey;
|
|
673
|
+
this.agentId = config.agentId || "";
|
|
655
674
|
this.prompt = config.prompt;
|
|
656
675
|
this.voice = config.voice || "F1" /* F1 */;
|
|
657
676
|
this.language = config.language || "en" /* ENGLISH */;
|
|
@@ -696,6 +715,10 @@ var VoiceAgentClient = class {
|
|
|
696
715
|
const separator = url.includes("?") ? "&" : "?";
|
|
697
716
|
url += `${separator}api_key=${this.apiKey}`;
|
|
698
717
|
}
|
|
718
|
+
if (this.agentId) {
|
|
719
|
+
const separator = url.includes("?") ? "&" : "?";
|
|
720
|
+
url += `${separator}agent_id=${this.agentId}`;
|
|
721
|
+
}
|
|
699
722
|
const redactedUrl = url.replace(/api_key=[^&]+/, "api_key=***");
|
|
700
723
|
sdkTrace("ws.connect", {
|
|
701
724
|
endpoint: this.serverUrl,
|
|
@@ -1315,6 +1338,164 @@ var TTSClient = class {
|
|
|
1315
1338
|
});
|
|
1316
1339
|
}
|
|
1317
1340
|
};
|
|
1341
|
+
var STTClient = class {
|
|
1342
|
+
apiKey;
|
|
1343
|
+
baseUrl;
|
|
1344
|
+
constructor(config) {
|
|
1345
|
+
this.apiKey = config.apiKey;
|
|
1346
|
+
this.baseUrl = wsToHttp(config.serverUrl || DEFAULT_URLS.STT).replace(/\/ws\/stt\/?$/, "");
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* Transcribe a complete audio clip. Pass a Blob/File (e.g. from a file
|
|
1350
|
+
* input or MediaRecorder) for WAV/compressed audio, or a raw PCM16
|
|
1351
|
+
* buffer with `format: "pcm16"` and `sampleRate` set.
|
|
1352
|
+
*/
|
|
1353
|
+
async transcribe(options) {
|
|
1354
|
+
const url = `${this.baseUrl}/stt/transcribe`;
|
|
1355
|
+
let res;
|
|
1356
|
+
if (typeof Blob !== "undefined" && options.audio instanceof Blob) {
|
|
1357
|
+
const form = new FormData();
|
|
1358
|
+
form.append("audio", options.audio, "audio.wav");
|
|
1359
|
+
if (options.language) form.append("lang", options.language);
|
|
1360
|
+
if (options.sampleRate) form.append("sample_rate", String(options.sampleRate));
|
|
1361
|
+
res = await fetch(url, {
|
|
1362
|
+
method: "POST",
|
|
1363
|
+
headers: { "X-API-Key": this.apiKey },
|
|
1364
|
+
body: form
|
|
1365
|
+
});
|
|
1366
|
+
} else {
|
|
1367
|
+
const bytes = options.audio instanceof Uint8Array ? options.audio : new Uint8Array(options.audio);
|
|
1368
|
+
res = await fetch(url, {
|
|
1369
|
+
method: "POST",
|
|
1370
|
+
headers: { "X-API-Key": this.apiKey, "Content-Type": "application/json" },
|
|
1371
|
+
body: JSON.stringify({
|
|
1372
|
+
audio: uint8ArrayToBase64(bytes),
|
|
1373
|
+
format: options.format || "pcm16",
|
|
1374
|
+
sample_rate: options.sampleRate,
|
|
1375
|
+
lang: options.language
|
|
1376
|
+
})
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1379
|
+
if (!res.ok) {
|
|
1380
|
+
const detail = await res.text().catch(() => "");
|
|
1381
|
+
throw new LokutorError("internal.error", `HTTP ${res.status} from ${url}`, {
|
|
1382
|
+
detail,
|
|
1383
|
+
retryable: res.status >= 500
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
1386
|
+
const data = await res.json();
|
|
1387
|
+
return {
|
|
1388
|
+
text: data.text ?? "",
|
|
1389
|
+
latencyMs: data.latency_ms ?? 0,
|
|
1390
|
+
engine: data.engine ?? "",
|
|
1391
|
+
sampleRate: data.sample_rate ?? 0,
|
|
1392
|
+
language: data.language ?? "",
|
|
1393
|
+
durationSeconds: data.duration_seconds ?? 0,
|
|
1394
|
+
segments: data.segments
|
|
1395
|
+
};
|
|
1396
|
+
}
|
|
1397
|
+
};
|
|
1398
|
+
var SpeechToTextClient = class {
|
|
1399
|
+
apiKey;
|
|
1400
|
+
serverUrl;
|
|
1401
|
+
language;
|
|
1402
|
+
vad;
|
|
1403
|
+
onPartialTranscript;
|
|
1404
|
+
onFinalTranscript;
|
|
1405
|
+
onError;
|
|
1406
|
+
onStatusChange;
|
|
1407
|
+
ws = null;
|
|
1408
|
+
audioManager = null;
|
|
1409
|
+
isConnected = false;
|
|
1410
|
+
constructor(config) {
|
|
1411
|
+
this.apiKey = config.apiKey;
|
|
1412
|
+
this.serverUrl = config.serverUrl || DEFAULT_URLS.STT;
|
|
1413
|
+
this.language = config.language;
|
|
1414
|
+
this.vad = config.vad || "silero";
|
|
1415
|
+
this.onPartialTranscript = config.onPartialTranscript;
|
|
1416
|
+
this.onFinalTranscript = config.onFinalTranscript;
|
|
1417
|
+
this.onError = config.onError;
|
|
1418
|
+
this.onStatusChange = config.onStatusChange;
|
|
1419
|
+
}
|
|
1420
|
+
/**
|
|
1421
|
+
* Connect and start streaming microphone audio.
|
|
1422
|
+
* @param customAudioManager Optional replacement for the default audio hardware handler (e.g. NodeAudioManager for CLI use)
|
|
1423
|
+
*/
|
|
1424
|
+
async connect(customAudioManager) {
|
|
1425
|
+
this.audioManager = customAudioManager || (typeof window !== "undefined" ? new BrowserAudioManager() : null);
|
|
1426
|
+
if (!this.audioManager) {
|
|
1427
|
+
throw new LokutorError("internal.error", "No audio manager available \u2014 pass one explicitly outside the browser (e.g. NodeAudioManager).");
|
|
1428
|
+
}
|
|
1429
|
+
await this.audioManager.init();
|
|
1430
|
+
this.onStatusChange?.("connecting");
|
|
1431
|
+
return new Promise((resolve, reject) => {
|
|
1432
|
+
let settled = false;
|
|
1433
|
+
const settle = (fn) => {
|
|
1434
|
+
if (!settled) {
|
|
1435
|
+
settled = true;
|
|
1436
|
+
fn();
|
|
1437
|
+
}
|
|
1438
|
+
};
|
|
1439
|
+
try {
|
|
1440
|
+
let url = this.serverUrl;
|
|
1441
|
+
const separator = url.includes("?") ? "&" : "?";
|
|
1442
|
+
url += `${separator}api_key=${this.apiKey}`;
|
|
1443
|
+
this.ws = new WebSocket(url);
|
|
1444
|
+
this.ws.onopen = async () => {
|
|
1445
|
+
this.isConnected = true;
|
|
1446
|
+
this.onStatusChange?.("connected");
|
|
1447
|
+
this.ws.send(JSON.stringify({ lang: this.language || "en" /* ENGLISH */, vad: this.vad }));
|
|
1448
|
+
await this.audioManager.startMicrophone((data) => {
|
|
1449
|
+
if (this.isConnected && this.ws?.readyState === WebSocket.OPEN) {
|
|
1450
|
+
this.ws.send(data);
|
|
1451
|
+
}
|
|
1452
|
+
});
|
|
1453
|
+
settle(() => resolve(true));
|
|
1454
|
+
};
|
|
1455
|
+
this.ws.onmessage = (event) => {
|
|
1456
|
+
if (typeof event.data !== "string") return;
|
|
1457
|
+
try {
|
|
1458
|
+
const msg = JSON.parse(event.data);
|
|
1459
|
+
if (msg.type === "transcript") {
|
|
1460
|
+
if (msg.isFinal) {
|
|
1461
|
+
this.onFinalTranscript?.(msg.data ?? "");
|
|
1462
|
+
} else {
|
|
1463
|
+
this.onPartialTranscript?.(msg.data ?? "");
|
|
1464
|
+
}
|
|
1465
|
+
} else if (msg.type === "error") {
|
|
1466
|
+
this.onError?.(new LokutorError("internal.error", msg.data ?? "STT error"));
|
|
1467
|
+
}
|
|
1468
|
+
} catch {
|
|
1469
|
+
}
|
|
1470
|
+
};
|
|
1471
|
+
this.ws.onerror = (err) => {
|
|
1472
|
+
const lokutorErr = new LokutorError("internal.error", "WebSocket error", { original: err });
|
|
1473
|
+
this.onError?.(lokutorErr);
|
|
1474
|
+
settle(() => reject(lokutorErr));
|
|
1475
|
+
};
|
|
1476
|
+
this.ws.onclose = () => {
|
|
1477
|
+
this.isConnected = false;
|
|
1478
|
+
this.onStatusChange?.("disconnected");
|
|
1479
|
+
};
|
|
1480
|
+
} catch (err) {
|
|
1481
|
+
settle(() => reject(err));
|
|
1482
|
+
}
|
|
1483
|
+
});
|
|
1484
|
+
}
|
|
1485
|
+
/** Force-finalize whatever utterance is currently in progress. */
|
|
1486
|
+
endUtterance() {
|
|
1487
|
+
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
1488
|
+
this.ws.send(JSON.stringify({ type: "end" }));
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
disconnect() {
|
|
1492
|
+
this.isConnected = false;
|
|
1493
|
+
this.audioManager?.stopMicrophone();
|
|
1494
|
+
this.audioManager?.cleanup();
|
|
1495
|
+
this.ws?.close();
|
|
1496
|
+
this.ws = null;
|
|
1497
|
+
}
|
|
1498
|
+
};
|
|
1318
1499
|
async function simpleConversation(config) {
|
|
1319
1500
|
const client = new VoiceAgentClient(config);
|
|
1320
1501
|
await client.connect();
|
|
@@ -1324,6 +1505,102 @@ async function simpleTTS(options) {
|
|
|
1324
1505
|
const client = new TTSClient({ apiKey: options.apiKey });
|
|
1325
1506
|
return client.synthesize(options);
|
|
1326
1507
|
}
|
|
1508
|
+
async function simpleTranscribe(options) {
|
|
1509
|
+
const client = new STTClient({ apiKey: options.apiKey, serverUrl: options.serverUrl });
|
|
1510
|
+
return client.transcribe(options);
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
// src/node-audio.ts
|
|
1514
|
+
function optionalImport(moduleName) {
|
|
1515
|
+
return import(moduleName);
|
|
1516
|
+
}
|
|
1517
|
+
var NodeAudioManager = class {
|
|
1518
|
+
speaker = null;
|
|
1519
|
+
recorder = null;
|
|
1520
|
+
recordingStream = null;
|
|
1521
|
+
isMuted = false;
|
|
1522
|
+
isListening = false;
|
|
1523
|
+
constructor() {
|
|
1524
|
+
}
|
|
1525
|
+
async init() {
|
|
1526
|
+
try {
|
|
1527
|
+
const Speaker = await optionalImport("speaker").catch(() => null);
|
|
1528
|
+
if (!Speaker) {
|
|
1529
|
+
console.warn('\u26A0\uFE0F Package "speaker" is missing. Hardware output will be disabled.');
|
|
1530
|
+
console.warn("\u{1F449} Run: npm install speaker");
|
|
1531
|
+
}
|
|
1532
|
+
} catch (e) {
|
|
1533
|
+
console.error("Error initializing Node audio:", e);
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
async startMicrophone(onAudioInput) {
|
|
1537
|
+
if (this.isListening) return;
|
|
1538
|
+
try {
|
|
1539
|
+
const recorder = await optionalImport("node-record-lpcm16").catch(() => null);
|
|
1540
|
+
if (!recorder) {
|
|
1541
|
+
throw new Error('Package "node-record-lpcm16" is missing. Microphone input failed.\n\u{1F449} Run: npm install node-record-lpcm16');
|
|
1542
|
+
}
|
|
1543
|
+
console.log("\u{1F3A4} Starting microphone (Node.js)...");
|
|
1544
|
+
this.recordingStream = recorder.record({
|
|
1545
|
+
sampleRate: AUDIO_CONFIG.SAMPLE_RATE,
|
|
1546
|
+
threshold: 0,
|
|
1547
|
+
verbose: false,
|
|
1548
|
+
recordProgram: "sox"
|
|
1549
|
+
// default
|
|
1550
|
+
});
|
|
1551
|
+
this.recordingStream.stream().on("data", (chunk) => {
|
|
1552
|
+
if (!this.isMuted && onAudioInput) {
|
|
1553
|
+
onAudioInput(new Uint8Array(chunk));
|
|
1554
|
+
}
|
|
1555
|
+
});
|
|
1556
|
+
this.isListening = true;
|
|
1557
|
+
} catch (e) {
|
|
1558
|
+
console.error("Failed to start microphone:", e.message);
|
|
1559
|
+
throw e;
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
stopMicrophone() {
|
|
1563
|
+
if (this.recordingStream) {
|
|
1564
|
+
this.recordingStream.stop();
|
|
1565
|
+
this.recordingStream = null;
|
|
1566
|
+
}
|
|
1567
|
+
this.isListening = false;
|
|
1568
|
+
}
|
|
1569
|
+
async playAudio(pcm16Data) {
|
|
1570
|
+
try {
|
|
1571
|
+
if (!this.speaker) {
|
|
1572
|
+
const Speaker = (await optionalImport("speaker")).default;
|
|
1573
|
+
this.speaker = new Speaker({
|
|
1574
|
+
channels: AUDIO_CONFIG.CHANNELS,
|
|
1575
|
+
bitDepth: 16,
|
|
1576
|
+
sampleRate: AUDIO_CONFIG.SPEAKER_SAMPLE_RATE
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
this.speaker.write(Buffer.from(pcm16Data));
|
|
1580
|
+
} catch (e) {
|
|
1581
|
+
console.error("NodeAudioManager: speaker playback failed:", e);
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
stopPlayback() {
|
|
1585
|
+
if (this.speaker) {
|
|
1586
|
+
this.speaker.end();
|
|
1587
|
+
this.speaker = null;
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
cleanup() {
|
|
1591
|
+
this.stopMicrophone();
|
|
1592
|
+
this.stopPlayback();
|
|
1593
|
+
}
|
|
1594
|
+
isMicMuted() {
|
|
1595
|
+
return this.isMuted;
|
|
1596
|
+
}
|
|
1597
|
+
setMuted(muted) {
|
|
1598
|
+
this.isMuted = muted;
|
|
1599
|
+
}
|
|
1600
|
+
getAmplitude() {
|
|
1601
|
+
return 0;
|
|
1602
|
+
}
|
|
1603
|
+
};
|
|
1327
1604
|
|
|
1328
1605
|
// src/conversational-panel.ts
|
|
1329
1606
|
var PANEL_CSS = (
|
|
@@ -1419,7 +1696,7 @@ var PANEL_CSS = (
|
|
|
1419
1696
|
}
|
|
1420
1697
|
.cv-curtain-btn svg { transition: transform 0.3s ease; }
|
|
1421
1698
|
.cv-curtain-btn:hover svg { transform: translateX(4px); }
|
|
1422
|
-
.cv-
|
|
1699
|
+
.cv-error {
|
|
1423
1700
|
display: flex;
|
|
1424
1701
|
flex-direction: column;
|
|
1425
1702
|
align-items: center;
|
|
@@ -1437,12 +1714,6 @@ var PANEL_CSS = (
|
|
|
1437
1714
|
gap: 1rem;
|
|
1438
1715
|
letter-spacing: -0.02em;
|
|
1439
1716
|
}
|
|
1440
|
-
.cv-title .cv-timer {
|
|
1441
|
-
font-variant-numeric: tabular-nums;
|
|
1442
|
-
color: var(--cv-accent);
|
|
1443
|
-
font-weight: 400;
|
|
1444
|
-
opacity: 0.8;
|
|
1445
|
-
}
|
|
1446
1717
|
.cv-visualizer-wrap {
|
|
1447
1718
|
position: absolute;
|
|
1448
1719
|
top: 50%;
|
|
@@ -1545,18 +1816,52 @@ var PANEL_CSS = (
|
|
|
1545
1816
|
}
|
|
1546
1817
|
.cv-error.is-visible { display: flex; }
|
|
1547
1818
|
.cv-error-icon { color: var(--cv-accent); flex-shrink: 0; }
|
|
1819
|
+
.cv-voice-picker {
|
|
1820
|
+
display: flex;
|
|
1821
|
+
flex-wrap: wrap;
|
|
1822
|
+
justify-content: center;
|
|
1823
|
+
gap: 0.35rem;
|
|
1824
|
+
z-index: 2;
|
|
1825
|
+
}
|
|
1826
|
+
.cv-voice-btn {
|
|
1827
|
+
padding: 0.25rem 0.5rem;
|
|
1828
|
+
border-radius: 100px;
|
|
1829
|
+
border: 1px solid rgba(255,255,255,0.15);
|
|
1830
|
+
background: rgba(255,255,255,0.05);
|
|
1831
|
+
color: rgba(255,255,255,0.6);
|
|
1832
|
+
font-size: 0.65rem;
|
|
1833
|
+
font-weight: 600;
|
|
1834
|
+
cursor: pointer;
|
|
1835
|
+
transition: all 0.2s ease;
|
|
1836
|
+
}
|
|
1837
|
+
.cv-voice-btn:hover {
|
|
1838
|
+
background: rgba(255,255,255,0.12);
|
|
1839
|
+
color: #fff;
|
|
1840
|
+
}
|
|
1841
|
+
.cv-voice-btn.is-selected {
|
|
1842
|
+
background: #fff;
|
|
1843
|
+
color: #000;
|
|
1844
|
+
border-color: #fff;
|
|
1845
|
+
}
|
|
1846
|
+
.cv-voice-label {
|
|
1847
|
+
font-size: 0.55rem;
|
|
1848
|
+
text-transform: uppercase;
|
|
1849
|
+
letter-spacing: 0.08em;
|
|
1850
|
+
color: rgba(255,255,255,0.35);
|
|
1851
|
+
z-index: 2;
|
|
1852
|
+
}
|
|
1548
1853
|
|
|
1549
1854
|
/* === COMPACT MODE: < 300px width === */
|
|
1550
1855
|
@container (max-width: 299px) {
|
|
1551
1856
|
.cv-curtain-content { padding: 0.5rem; gap: 0.3rem; }
|
|
1552
1857
|
.cv-curtain-title { font-size: 0.8rem; }
|
|
1553
1858
|
.cv-curtain-desc { display: none; }
|
|
1859
|
+
.cv-voice-picker { display: none; }
|
|
1554
1860
|
.cv-curtain-btn { padding: 0.3rem 0.6rem; font-size: 0.6rem; gap: 0.3rem; }
|
|
1555
1861
|
.cv-curtain-btn svg { display: none; }
|
|
1556
1862
|
.cv-visualizer-wrap { width: 60px; height: 60px; }
|
|
1557
1863
|
.cv-header { padding-top: 0.75rem; }
|
|
1558
1864
|
.cv-title { font-size: 0.7rem; gap: 0.3rem; }
|
|
1559
|
-
.cv-title .cv-timer { display: none; }
|
|
1560
1865
|
.cv-controls { gap: 0.6rem; padding-bottom: 0.6rem; }
|
|
1561
1866
|
.cv-pill { padding: 0.15rem 0.4rem; gap: 0.3rem; }
|
|
1562
1867
|
.cv-btn { padding: 0.2rem 0.4rem; font-size: 0.6rem; gap: 0; }
|
|
@@ -1574,7 +1879,6 @@ var PANEL_CSS = (
|
|
|
1574
1879
|
.cv-visualizer-wrap { width: clamp(80px, 35cqw, 140px); height: clamp(80px, 35cqw, 140px); }
|
|
1575
1880
|
.cv-header { padding-top: 0.8rem; }
|
|
1576
1881
|
.cv-title { font-size: clamp(0.7rem, 2.5cqw, 1rem); gap: 0.4rem; }
|
|
1577
|
-
.cv-title .cv-timer { font-size: 0.65em; }
|
|
1578
1882
|
.cv-controls { gap: clamp(0.6rem, 1.5cqw, 1rem); padding-bottom: clamp(0.6rem, 1.5cqw, 1rem); }
|
|
1579
1883
|
.cv-pill { padding: clamp(0.15rem, 0.5cqw, 0.25rem) clamp(0.35rem, 1cqw, 0.6rem); font-size: 0.65rem; gap: 0.3rem; }
|
|
1580
1884
|
.cv-btn { padding: clamp(0.2rem, 0.5cqw, 0.3rem) clamp(0.35rem, 1cqw, 0.6rem); font-size: clamp(0.6rem, 1.2cqw, 0.7rem); }
|
|
@@ -1592,7 +1896,6 @@ var PANEL_CSS = (
|
|
|
1592
1896
|
.cv-visualizer-wrap { width: clamp(100px, 40cqw, 200px); height: clamp(100px, 40cqw, 200px); }
|
|
1593
1897
|
.cv-header { padding-top: clamp(1rem, 1.5cqw, 1.5rem); }
|
|
1594
1898
|
.cv-title { font-size: clamp(0.9rem, 3cqw, 1.3rem); gap: clamp(0.4rem, 1cqw, 0.75rem); }
|
|
1595
|
-
.cv-title .cv-timer { font-size: 0.85em; }
|
|
1596
1899
|
.cv-controls { gap: clamp(0.8rem, 1.5cqw, 1.2rem); padding-bottom: clamp(0.8rem, 1.5cqw, 1.2rem); }
|
|
1597
1900
|
.cv-pill { padding: clamp(0.2rem, 0.75cqw, 0.3rem) clamp(0.5rem, 1.2cqw, 0.75rem); font-size: 0.7rem; }
|
|
1598
1901
|
.cv-btn { padding: clamp(0.25rem, 0.75cqw, 0.35rem) clamp(0.5rem, 1.2cqw, 0.75rem); font-size: clamp(0.65rem, 1.2cqw, 0.8rem); }
|
|
@@ -1608,7 +1911,6 @@ var PANEL_CSS = (
|
|
|
1608
1911
|
.cv-visualizer-wrap { width: clamp(140px, 45cqw, 300px); height: clamp(140px, 45cqw, 300px); }
|
|
1609
1912
|
.cv-header { padding-top: clamp(1.5rem, 2cqw, 2rem); }
|
|
1610
1913
|
.cv-title { font-size: clamp(1rem, 3cqw, 1.8rem); gap: clamp(0.75rem, 1.5cqw, 1.2rem); }
|
|
1611
|
-
.cv-title .cv-timer { font-size: 0.9em; }
|
|
1612
1914
|
.cv-controls { gap: clamp(1.2rem, 2cqw, 1.8rem); padding-bottom: clamp(1.2rem, 2cqw, 1.8rem); }
|
|
1613
1915
|
.cv-pill { padding: clamp(0.3rem, 0.8cqw, 0.4rem) clamp(0.75rem, 1.2cqw, 1rem); font-size: 0.85rem; gap: 0.75rem; }
|
|
1614
1916
|
.cv-btn { padding: clamp(0.35rem, 0.8cqw, 0.4rem) clamp(0.75rem, 1.2cqw, 1rem); font-size: clamp(0.75rem, 1.2cqw, 0.9rem); }
|
|
@@ -1635,15 +1937,16 @@ var ConversationalPanel = class {
|
|
|
1635
1937
|
isRunning = false;
|
|
1636
1938
|
_locked = false;
|
|
1637
1939
|
_lastSpeechTime = 0;
|
|
1638
|
-
|
|
1940
|
+
selectedVoice = "M1";
|
|
1639
1941
|
el;
|
|
1640
1942
|
curtain;
|
|
1641
1943
|
curtainTitle;
|
|
1642
1944
|
curtainDesc;
|
|
1945
|
+
curtainBg;
|
|
1946
|
+
curtainOverlay;
|
|
1643
1947
|
startBtn;
|
|
1644
1948
|
errorEl;
|
|
1645
1949
|
errorText;
|
|
1646
|
-
timerEl;
|
|
1647
1950
|
canvas;
|
|
1648
1951
|
muteBtn;
|
|
1649
1952
|
muteSvg;
|
|
@@ -1659,6 +1962,7 @@ var ConversationalPanel = class {
|
|
|
1659
1962
|
constructor(cfg) {
|
|
1660
1963
|
this.cfg = cfg;
|
|
1661
1964
|
this.container = cfg.container;
|
|
1965
|
+
this.selectedVoice = cfg.voice || "M1";
|
|
1662
1966
|
injectStyles();
|
|
1663
1967
|
this.buildDOM();
|
|
1664
1968
|
}
|
|
@@ -1674,9 +1978,9 @@ var ConversationalPanel = class {
|
|
|
1674
1978
|
<div class="cv-curtain-bg"></div>
|
|
1675
1979
|
<div class="cv-curtain-overlay"></div>
|
|
1676
1980
|
<div class="cv-curtain-content">
|
|
1677
|
-
<h3 class="cv-curtain-title">${this.esc(this.cfg.title)}</h3>
|
|
1678
|
-
<p class="cv-curtain-desc">${this.esc(this.cfg.description)}</p>
|
|
1679
|
-
|
|
1981
|
+
<h3 class="cv-curtain-title">${this.esc(this.cfg.title || "Voice Chat")}</h3>
|
|
1982
|
+
<p class="cv-curtain-desc">${this.esc(this.cfg.description || "")}</p>
|
|
1983
|
+
<button class="cv-curtain-btn">
|
|
1680
1984
|
<span>Start Conversation</span>
|
|
1681
1985
|
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" width="20" height="20">
|
|
1682
1986
|
<path d="M5 12h14M12 5l7 7-7 7"/>
|
|
@@ -1694,7 +1998,7 @@ var ConversationalPanel = class {
|
|
|
1694
1998
|
</div>
|
|
1695
1999
|
</div>
|
|
1696
2000
|
<div class="cv-header">
|
|
1697
|
-
<h2 class="cv-title">${this.esc(this.cfg.title
|
|
2001
|
+
<h2 class="cv-title">${this.esc(this.cfg.title || "Voice Chat")}</h2>
|
|
1698
2002
|
</div>
|
|
1699
2003
|
<div class="cv-visualizer-wrap">
|
|
1700
2004
|
<canvas class="cv-canvas"></canvas>
|
|
@@ -1725,7 +2029,6 @@ var ConversationalPanel = class {
|
|
|
1725
2029
|
this.startBtn = this.el.querySelector(".cv-curtain-btn");
|
|
1726
2030
|
this.errorEl = this.el.querySelector(".cv-error");
|
|
1727
2031
|
this.errorText = this.el.querySelector(".cv-error-text");
|
|
1728
|
-
this.timerEl = this.el.querySelector(".cv-timer");
|
|
1729
2032
|
this.canvas = this.el.querySelector(".cv-canvas");
|
|
1730
2033
|
this.muteBtn = this.el.querySelector(".cv-btn--mute");
|
|
1731
2034
|
this.muteSvg = this.muteBtn.querySelector("svg");
|
|
@@ -1772,7 +2075,7 @@ var ConversationalPanel = class {
|
|
|
1772
2075
|
apiKey: this.cfg.apiKey,
|
|
1773
2076
|
tools: this.cfg.tools,
|
|
1774
2077
|
prompt: this.cfg.prompt,
|
|
1775
|
-
voice: this.
|
|
2078
|
+
voice: this.selectedVoice,
|
|
1776
2079
|
language: this.cfg.language || "en",
|
|
1777
2080
|
onStatusChange: (status) => {
|
|
1778
2081
|
this.el.classList.remove("cv-is-speaking", "cv-is-thinking");
|
|
@@ -1903,8 +2206,9 @@ var ConversationalPanel = class {
|
|
|
1903
2206
|
this.cfg.title = title;
|
|
1904
2207
|
this.curtainTitle.textContent = title;
|
|
1905
2208
|
const h2 = this.el.querySelector(".cv-title");
|
|
1906
|
-
if (h2) h2.
|
|
2209
|
+
if (h2) h2.textContent = title;
|
|
1907
2210
|
}
|
|
2211
|
+
/** Select a voice from the picker */
|
|
1908
2212
|
/** Update description text */
|
|
1909
2213
|
setDescription(desc) {
|
|
1910
2214
|
this.cfg.description = desc;
|
|
@@ -1932,21 +2236,13 @@ var ConversationalPanel = class {
|
|
|
1932
2236
|
this.el.remove();
|
|
1933
2237
|
}
|
|
1934
2238
|
// ─── internal helpers ────────────────────────────────────
|
|
1935
|
-
fmt(s) {
|
|
1936
|
-
const m = Math.floor(s / 60).toString().padStart(2, "0");
|
|
1937
|
-
const sec = (s % 60).toString().padStart(2, "0");
|
|
1938
|
-
return `${m}:${sec}`;
|
|
1939
|
-
}
|
|
1940
2239
|
startTimer() {
|
|
1941
2240
|
let elapsed = 0;
|
|
1942
2241
|
const maxDur = this.cfg.maxDuration || 300;
|
|
1943
2242
|
const silentMax = this.cfg.silenceTimeout || 60;
|
|
1944
2243
|
this._lastSpeechTime = Date.now();
|
|
1945
|
-
this.timerEl.textContent = this.fmt(maxDur);
|
|
1946
2244
|
this.timerTicker = window.setInterval(() => {
|
|
1947
2245
|
elapsed++;
|
|
1948
|
-
const remaining = Math.max(0, maxDur - elapsed);
|
|
1949
|
-
this.timerEl.textContent = this.fmt(remaining);
|
|
1950
2246
|
if (elapsed >= maxDur) {
|
|
1951
2247
|
this._locked = true;
|
|
1952
2248
|
this.stop();
|
|
@@ -2034,6 +2330,9 @@ export {
|
|
|
2034
2330
|
DEFAULT_URLS,
|
|
2035
2331
|
Language,
|
|
2036
2332
|
LokutorError,
|
|
2333
|
+
NodeAudioManager,
|
|
2334
|
+
STTClient,
|
|
2335
|
+
SpeechToTextClient,
|
|
2037
2336
|
StreamResampler,
|
|
2038
2337
|
TTSClient,
|
|
2039
2338
|
VoiceAgentClient,
|
|
@@ -2049,5 +2348,6 @@ export {
|
|
|
2049
2348
|
resample,
|
|
2050
2349
|
resampleWithAntiAliasing,
|
|
2051
2350
|
simpleConversation,
|
|
2052
|
-
simpleTTS
|
|
2351
|
+
simpleTTS,
|
|
2352
|
+
simpleTranscribe
|
|
2053
2353
|
};
|
package/package.json
CHANGED
package/src/browser-audio.ts
CHANGED
|
@@ -30,6 +30,10 @@ export class BrowserAudioManager {
|
|
|
30
30
|
private mediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null;
|
|
31
31
|
private scriptProcessor: ScriptProcessorNode | null = null;
|
|
32
32
|
private analyserNode: AnalyserNode | null = null;
|
|
33
|
+
// Reused across getAmplitude() calls instead of allocating a new
|
|
34
|
+
// Uint8Array on every call — this runs once per animation frame (~60/sec)
|
|
35
|
+
// from the visualizer, on the same main thread as audio capture.
|
|
36
|
+
private amplitudeBuffer: Uint8Array<ArrayBuffer> | null = null;
|
|
33
37
|
private mediaStream: MediaStream | null = null;
|
|
34
38
|
private resampler: StreamResampler | null = null;
|
|
35
39
|
|
|
@@ -123,7 +127,17 @@ export class BrowserAudioManager {
|
|
|
123
127
|
// Create script processor for PCM extraction
|
|
124
128
|
// Note: ScriptProcessorNode is deprecated but widely supported.
|
|
125
129
|
// AudioWorklet would be better but requires additional setup.
|
|
126
|
-
|
|
130
|
+
//
|
|
131
|
+
// 1024 samples is ~21ms at a typical 48kHz hardware rate — matched to
|
|
132
|
+
// AUDIO_CONFIG.CHUNK_DURATION_MS (20ms), the granularity the backend's
|
|
133
|
+
// VAD is tuned around. The previous 4096 was ~85ms of audio buffered
|
|
134
|
+
// before a single byte reached the server: on top of adding raw
|
|
135
|
+
// latency, ScriptProcessorNode callbacks run on the main thread, so
|
|
136
|
+
// that much buffering meant audio could arrive in large, uneven
|
|
137
|
+
// bursts whenever the main thread was briefly busy (a re-render, the
|
|
138
|
+
// visualizer) instead of a steady stream — exactly the kind of
|
|
139
|
+
// mistimed input that can trip server-side VAD into a false barge-in.
|
|
140
|
+
const bufferSize = 1024;
|
|
127
141
|
this.scriptProcessor = this.audioContext!.createScriptProcessor(
|
|
128
142
|
bufferSize,
|
|
129
143
|
1, // input channels
|
|
@@ -165,7 +179,6 @@ export class BrowserAudioManager {
|
|
|
165
179
|
*/
|
|
166
180
|
private _processAudioInput(event: AudioProcessingEvent): void {
|
|
167
181
|
if (!this.onAudioInput || !this.audioContext || !this.isListening) return;
|
|
168
|
-
if (this.isMuted) return;
|
|
169
182
|
|
|
170
183
|
const inputBuffer = event.inputBuffer;
|
|
171
184
|
const inputData = inputBuffer.getChannelData(0);
|
|
@@ -185,6 +198,18 @@ export class BrowserAudioManager {
|
|
|
185
198
|
|
|
186
199
|
if (processedData.length === 0) return; // Need more data for resampler
|
|
187
200
|
|
|
201
|
+
// While muted, keep sending — silent — chunks instead of sending
|
|
202
|
+
// nothing at all (the old `if (this.isMuted) return` above this).
|
|
203
|
+
// The server's turn-taking/VAD is purely reactive to incoming chunks:
|
|
204
|
+
// if the client stops sending entirely mid-utterance, the server never
|
|
205
|
+
// observes the silence it needs to close out the turn, so muting
|
|
206
|
+
// mid-sentence left the conversation stuck instead of handing off to
|
|
207
|
+
// the agent. Explicitly zeroed here rather than relying solely on the
|
|
208
|
+
// disabled MediaStreamTrack to already read as silence.
|
|
209
|
+
if (this.isMuted) {
|
|
210
|
+
processedData = new Float32Array(processedData.length);
|
|
211
|
+
}
|
|
212
|
+
|
|
188
213
|
// Convert Float32 to Int16 PCM
|
|
189
214
|
const int16Data = float32ToPcm16(processedData);
|
|
190
215
|
const uint8Data = new Uint8Array(
|
|
@@ -339,10 +364,12 @@ export class BrowserAudioManager {
|
|
|
339
364
|
getAmplitude(): number {
|
|
340
365
|
if (!this.analyserNode) return 0;
|
|
341
366
|
|
|
342
|
-
|
|
343
|
-
|
|
367
|
+
if (!this.amplitudeBuffer || this.amplitudeBuffer.length !== this.analyserNode.frequencyBinCount) {
|
|
368
|
+
this.amplitudeBuffer = new Uint8Array(this.analyserNode.frequencyBinCount);
|
|
369
|
+
}
|
|
370
|
+
this.analyserNode.getByteTimeDomainData(this.amplitudeBuffer);
|
|
344
371
|
|
|
345
|
-
const rms = calculateRMS(
|
|
372
|
+
const rms = calculateRMS(this.amplitudeBuffer);
|
|
346
373
|
return Math.min(rms * 10, 1); // Boost for visualization
|
|
347
374
|
}
|
|
348
375
|
|