@lokutor/sdk 1.4.1 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +76 -5
- package/dist/index.d.ts +76 -5
- package/dist/index.js +273 -26
- package/dist/index.mjs +271 -26
- package/package.json +1 -1
- package/src/browser-audio.ts +149 -13
- package/src/client.ts +76 -8
- package/src/conversational-panel.ts +137 -18
- package/src/index.ts +1 -1
- package/src/types.ts +4 -0
package/src/browser-audio.ts
CHANGED
|
@@ -10,9 +10,67 @@ export interface BrowserAudioConfig {
|
|
|
10
10
|
autoGainControl?: boolean;
|
|
11
11
|
echoCancellation?: boolean;
|
|
12
12
|
noiseSuppression?: boolean;
|
|
13
|
+
/**
|
|
14
|
+
* Keep the screen awake while the microphone is live (Screen Wake Lock API), as a video player does.
|
|
15
|
+
* Default true: a phone that dims and locks mid-call suspends the page's microphone and audio, and
|
|
16
|
+
* the call goes silent in both directions. Set false if your app manages this itself.
|
|
17
|
+
*/
|
|
18
|
+
keepAwake?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* A microphone stream the page already asked for, ideally with requestMicrophone() inside the tap that
|
|
21
|
+
* started the call. startMicrophone uses it instead of calling getUserMedia again later, after the
|
|
22
|
+
* connection opens -- which iOS Safari can refuse without showing its prompt.
|
|
23
|
+
*/
|
|
24
|
+
microphone?: MediaStream | Promise<MediaStream>;
|
|
13
25
|
onInputError?: (error: Error) => void;
|
|
14
26
|
}
|
|
15
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Ask for the microphone now, from inside the tap or click that starts a call, and return the pending stream.
|
|
30
|
+
*
|
|
31
|
+
* Call it synchronously in the gesture handler, before any await, and pass the result to BrowserAudioManager
|
|
32
|
+
* as `microphone` (ConversationalPanel does this itself). Two iOS Safari behaviours made a later request fail
|
|
33
|
+
* with no prompt at all -- the call connected, the agent spoke, and nothing the caller said was heard:
|
|
34
|
+
* - a request outside a user gesture may be refused instead of prompting;
|
|
35
|
+
* - while the page's audio session is in 'playback' mode (set by pages so audio plays through the mute
|
|
36
|
+
* switch), capture is refused outright. It is put back to 'auto', under which Safari switches to
|
|
37
|
+
* play-and-record itself when capture starts and keeps the loudspeaker.
|
|
38
|
+
*/
|
|
39
|
+
export function requestMicrophone(
|
|
40
|
+
constraints: Pick<BrowserAudioConfig, 'autoGainControl' | 'echoCancellation' | 'noiseSuppression'> = {}
|
|
41
|
+
): Promise<MediaStream> {
|
|
42
|
+
if (typeof navigator === 'undefined') {
|
|
43
|
+
return Promise.reject(new Error('Microphone capture needs a browser'));
|
|
44
|
+
}
|
|
45
|
+
const session = (navigator as any).audioSession;
|
|
46
|
+
if (session && session.type !== 'auto' && session.type !== 'play-and-record') {
|
|
47
|
+
try { session.type = 'auto'; } catch (_) { /* read-only in this browser */ }
|
|
48
|
+
}
|
|
49
|
+
if (!navigator.mediaDevices?.getUserMedia) {
|
|
50
|
+
return Promise.reject(new Error('Microphone capture is not available here (it needs HTTPS)'));
|
|
51
|
+
}
|
|
52
|
+
return navigator.mediaDevices.getUserMedia({
|
|
53
|
+
audio: {
|
|
54
|
+
autoGainControl: constraints.autoGainControl ?? true,
|
|
55
|
+
echoCancellation: constraints.echoCancellation ?? true,
|
|
56
|
+
noiseSuppression: constraints.noiseSuppression ?? true,
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Resume an AudioContext without waiting on it forever. Outside a user gesture iOS Safari leaves resume()'s
|
|
63
|
+
* promise pending, which stalled the whole call start; the context starts on its own once capture is live,
|
|
64
|
+
* and playback resumes it again anyway.
|
|
65
|
+
*/
|
|
66
|
+
async function resumeWithin(ctx: AudioContext, ms: number): Promise<void> {
|
|
67
|
+
if (ctx.state === 'running') return;
|
|
68
|
+
await Promise.race([
|
|
69
|
+
ctx.resume().catch(() => {}),
|
|
70
|
+
new Promise<void>((r) => setTimeout(r, ms)),
|
|
71
|
+
]);
|
|
72
|
+
}
|
|
73
|
+
|
|
16
74
|
/**
|
|
17
75
|
* Analyser configuration for audio visualization
|
|
18
76
|
*/
|
|
@@ -48,6 +106,13 @@ export class BrowserAudioManager {
|
|
|
48
106
|
private autoGainControl: boolean;
|
|
49
107
|
private echoCancellation: boolean;
|
|
50
108
|
private noiseSuppression: boolean;
|
|
109
|
+
private keepAwake: boolean;
|
|
110
|
+
private providedMicrophone: MediaStream | Promise<MediaStream> | null;
|
|
111
|
+
|
|
112
|
+
// Screen wake lock, held while the microphone is live. Browsers release it whenever the page is
|
|
113
|
+
// hidden, so it is taken again when the page becomes visible.
|
|
114
|
+
private wakeLock: { release(): Promise<void> } | null = null;
|
|
115
|
+
private visibilityHandler: (() => void) | null = null;
|
|
51
116
|
|
|
52
117
|
// Callbacks
|
|
53
118
|
private onAudioInput?: (pcm16Data: Uint8Array) => void;
|
|
@@ -63,6 +128,10 @@ export class BrowserAudioManager {
|
|
|
63
128
|
this.autoGainControl = config.autoGainControl ?? true;
|
|
64
129
|
this.echoCancellation = config.echoCancellation ?? true;
|
|
65
130
|
this.noiseSuppression = config.noiseSuppression ?? true;
|
|
131
|
+
this.keepAwake = config.keepAwake ?? true;
|
|
132
|
+
this.providedMicrophone = config.microphone ?? null;
|
|
133
|
+
// Never an unhandled rejection: startMicrophone reports it.
|
|
134
|
+
if (this.providedMicrophone instanceof Promise) this.providedMicrophone.catch(() => {});
|
|
66
135
|
this.onInputError = config.onInputError;
|
|
67
136
|
}
|
|
68
137
|
|
|
@@ -85,9 +154,8 @@ export class BrowserAudioManager {
|
|
|
85
154
|
throw new Error('Failed to initialize AudioContext');
|
|
86
155
|
}
|
|
87
156
|
|
|
88
|
-
if (this.audioContext.state
|
|
89
|
-
await this.audioContext
|
|
90
|
-
console.log('👂 AudioContext resumed');
|
|
157
|
+
if (this.audioContext.state !== 'running') {
|
|
158
|
+
await resumeWithin(this.audioContext, 500);
|
|
91
159
|
}
|
|
92
160
|
|
|
93
161
|
// Setup analyser for visualization if enabled
|
|
@@ -107,18 +175,32 @@ export class BrowserAudioManager {
|
|
|
107
175
|
await this.init();
|
|
108
176
|
}
|
|
109
177
|
|
|
178
|
+
// Already capturing: take the new callback and keep the one pipeline. A second getUserMedia +
|
|
179
|
+
// ScriptProcessor here used to run two captures into the same resampler, doubling and scrambling
|
|
180
|
+
// what the server heard.
|
|
181
|
+
if (this.isListening && this.mediaStream) {
|
|
182
|
+
this.onAudioInput = onAudioInput;
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
110
186
|
try {
|
|
111
187
|
this.onAudioInput = onAudioInput;
|
|
112
188
|
this.isListening = true;
|
|
113
189
|
|
|
114
|
-
//
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
190
|
+
// The page's own request (made inside the tap), when it has one and its track is still live;
|
|
191
|
+
// otherwise ask now.
|
|
192
|
+
const provided = this.providedMicrophone;
|
|
193
|
+
this.providedMicrophone = null;
|
|
194
|
+
const stream = provided ? await provided : null;
|
|
195
|
+
this.mediaStream = stream && stream.getAudioTracks().some((t) => t.readyState === 'live')
|
|
196
|
+
? stream
|
|
197
|
+
: await navigator.mediaDevices.getUserMedia({
|
|
198
|
+
audio: {
|
|
199
|
+
autoGainControl: this.autoGainControl,
|
|
200
|
+
echoCancellation: this.echoCancellation,
|
|
201
|
+
noiseSuppression: this.noiseSuppression,
|
|
202
|
+
},
|
|
203
|
+
});
|
|
122
204
|
|
|
123
205
|
// Create source from microphone stream
|
|
124
206
|
this.mediaStreamAudioSourceNode =
|
|
@@ -167,7 +249,9 @@ export class BrowserAudioManager {
|
|
|
167
249
|
};
|
|
168
250
|
|
|
169
251
|
console.log('🎤 Microphone started');
|
|
252
|
+
this.holdScreenAwake();
|
|
170
253
|
} catch (error) {
|
|
254
|
+
this.isListening = false;
|
|
171
255
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
172
256
|
if (this.onInputError) this.onInputError(err);
|
|
173
257
|
throw err;
|
|
@@ -226,6 +310,7 @@ export class BrowserAudioManager {
|
|
|
226
310
|
*/
|
|
227
311
|
stopMicrophone(): void {
|
|
228
312
|
this.isListening = false;
|
|
313
|
+
this.releaseScreenAwake();
|
|
229
314
|
|
|
230
315
|
if (this.mediaStream) {
|
|
231
316
|
this.mediaStream.getTracks().forEach((track) => track.stop());
|
|
@@ -245,6 +330,50 @@ export class BrowserAudioManager {
|
|
|
245
330
|
console.log('🎤 Microphone stopped');
|
|
246
331
|
}
|
|
247
332
|
|
|
333
|
+
/**
|
|
334
|
+
* Keep the screen on while the microphone is live, and bring audio back when the page returns
|
|
335
|
+
* to the foreground (iOS leaves the context "interrupted" after the screen locks).
|
|
336
|
+
*/
|
|
337
|
+
private holdScreenAwake(): void {
|
|
338
|
+
if (typeof document === 'undefined') return;
|
|
339
|
+
if (!this.visibilityHandler) {
|
|
340
|
+
this.visibilityHandler = () => {
|
|
341
|
+
if (document.visibilityState !== 'visible' || !this.isListening) return;
|
|
342
|
+
if (this.audioContext && this.audioContext.state !== 'running') {
|
|
343
|
+
this.audioContext.resume().catch(() => {});
|
|
344
|
+
}
|
|
345
|
+
void this.requestWakeLock();
|
|
346
|
+
};
|
|
347
|
+
document.addEventListener('visibilitychange', this.visibilityHandler);
|
|
348
|
+
}
|
|
349
|
+
void this.requestWakeLock();
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
private async requestWakeLock(): Promise<void> {
|
|
353
|
+
if (!this.keepAwake || this.wakeLock || typeof navigator === 'undefined') return;
|
|
354
|
+
const api = (navigator as any).wakeLock;
|
|
355
|
+
if (!api?.request || document.visibilityState !== 'visible') return;
|
|
356
|
+
try {
|
|
357
|
+
const lock = await api.request('screen');
|
|
358
|
+
if (!this.isListening) { await lock.release().catch(() => {}); return; }
|
|
359
|
+
this.wakeLock = lock;
|
|
360
|
+
// The browser drops the lock when the page is hidden; forget it so it is taken again on return.
|
|
361
|
+
lock.addEventListener?.('release', () => { if (this.wakeLock === lock) this.wakeLock = null; });
|
|
362
|
+
} catch (_) {
|
|
363
|
+
// Not allowed right now (battery saver, permissions policy): the call works, the screen may dim.
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
private releaseScreenAwake(): void {
|
|
368
|
+
if (this.visibilityHandler && typeof document !== 'undefined') {
|
|
369
|
+
document.removeEventListener('visibilitychange', this.visibilityHandler);
|
|
370
|
+
this.visibilityHandler = null;
|
|
371
|
+
}
|
|
372
|
+
const lock = this.wakeLock;
|
|
373
|
+
this.wakeLock = null;
|
|
374
|
+
lock?.release().catch(() => {});
|
|
375
|
+
}
|
|
376
|
+
|
|
248
377
|
/**
|
|
249
378
|
* Play back audio received from the server
|
|
250
379
|
* @param pcm16Data Int16 PCM audio data at SPEAKER_SAMPLE_RATE
|
|
@@ -411,8 +540,15 @@ export class BrowserAudioManager {
|
|
|
411
540
|
this.analyserNode = null;
|
|
412
541
|
}
|
|
413
542
|
|
|
414
|
-
//
|
|
415
|
-
//
|
|
543
|
+
// Closed, not left open: a page makes a new manager per call, so every open context was one more
|
|
544
|
+
// left running for the life of the page (iOS Safari showed five at the start of a second call, which
|
|
545
|
+
// then hung). init() makes a fresh one if this manager is used again.
|
|
546
|
+
if (this.audioContext) {
|
|
547
|
+
const ctx = this.audioContext;
|
|
548
|
+
this.audioContext = null;
|
|
549
|
+
this.nextPlaybackTime = 0;
|
|
550
|
+
ctx.close().catch(() => {});
|
|
551
|
+
}
|
|
416
552
|
}
|
|
417
553
|
|
|
418
554
|
/**
|
package/src/client.ts
CHANGED
|
@@ -91,6 +91,24 @@ export interface AudioManager {
|
|
|
91
91
|
getAmplitude(): number;
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
/**
|
|
95
|
+
* Close codes after which the client reconnects: the connection dropped or the server went away
|
|
96
|
+
* (a deploy, a restart, a network change), so another attempt can succeed. Every other close is
|
|
97
|
+
* the server ending the session on purpose -- 1000 when the agent ends the call, 1008 when a limit
|
|
98
|
+
* is reached (the landing demo's time allowance), 4xxx for application reasons -- and reconnecting
|
|
99
|
+
* after one of those used to open a fresh session behind the caller's back: a new conversation
|
|
100
|
+
* with no memory of the old one, a second microphone pipeline, and every reply discarded as stale.
|
|
101
|
+
*/
|
|
102
|
+
const RECONNECTABLE_CLOSE_CODES = new Set([1001, 1006, 1011, 1012, 1013, 1014]);
|
|
103
|
+
|
|
104
|
+
/** What the server announced about how long this session may run (landing-page demo sessions). */
|
|
105
|
+
export interface SessionLimit {
|
|
106
|
+
/** Seconds the session may still run, from when the message arrived. */
|
|
107
|
+
seconds: number;
|
|
108
|
+
/** Why there is a limit, e.g. "demo". */
|
|
109
|
+
reason?: string;
|
|
110
|
+
}
|
|
111
|
+
|
|
94
112
|
// Browser-compatible base64 to Uint8Array
|
|
95
113
|
function base64ToUint8Array(base64: string): Uint8Array {
|
|
96
114
|
const binaryString = atob(base64);
|
|
@@ -189,6 +207,9 @@ export class VoiceAgentClient {
|
|
|
189
207
|
private reconnectAttempts: number = 0;
|
|
190
208
|
private maxReconnectAttempts: number = 5;
|
|
191
209
|
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
|
210
|
+
// The microphone is started once per session of this client, not once per socket: a reconnect
|
|
211
|
+
// reuses it. Starting it again on every open ran two capture pipelines into one resampler.
|
|
212
|
+
private micStarted: boolean = false;
|
|
192
213
|
|
|
193
214
|
// Runtime state
|
|
194
215
|
private serverPlaybackRate: number = 44100;
|
|
@@ -284,16 +305,39 @@ export class VoiceAgentClient {
|
|
|
284
305
|
this.isConnected = true;
|
|
285
306
|
this.reconnectAttempts = 0;
|
|
286
307
|
this.reconnecting = false;
|
|
308
|
+
// Every socket is a new server session, and a new session numbers its replies from 1.
|
|
309
|
+
// Keeping the previous session's count here discarded every reply after a reconnect as
|
|
310
|
+
// "ghost audio" -- the agent answered and the caller heard nothing.
|
|
311
|
+
this.currentGeneration = 0;
|
|
287
312
|
console.log('✅ Connected to voice agent!');
|
|
288
313
|
sdkTrace('ws.open');
|
|
289
314
|
this.sendConfig();
|
|
290
315
|
|
|
291
|
-
if (this.audioManager) {
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
this.
|
|
295
|
-
|
|
296
|
-
|
|
316
|
+
if (this.audioManager && !this.micStarted) {
|
|
317
|
+
try {
|
|
318
|
+
await this.audioManager.startMicrophone((data) => {
|
|
319
|
+
if (this.isConnected) {
|
|
320
|
+
this.sendAudio(data);
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
} catch (e) {
|
|
324
|
+
// No microphone, no call. Uncaught here, this left connect() pending forever while the session
|
|
325
|
+
// it had opened went on talking: the caller heard the agent, nothing they said reached it, and
|
|
326
|
+
// the page never learned why (iOS refusing capture with no prompt, 2026-09-28).
|
|
327
|
+
const name = (e as { name?: string } | null)?.name || 'Error';
|
|
328
|
+
const error = new LokutorError('audio.microphone_unavailable',
|
|
329
|
+
name === 'NotAllowedError' || name === 'SecurityError'
|
|
330
|
+
? 'Microphone access was not allowed.'
|
|
331
|
+
: 'The microphone could not be started.',
|
|
332
|
+
{ detail: `${name}: ${(e as Error)?.message ?? e}`, original: e, retryable: false });
|
|
333
|
+
sdkTrace('mic.failed', { name });
|
|
334
|
+
settle(() => reject(error));
|
|
335
|
+
this.isUserDisconnect = true;
|
|
336
|
+
try { this.ws?.close(); } catch (_) { /* already closing */ }
|
|
337
|
+
this.emit('error', error); // reaches onError too
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
this.micStarted = true;
|
|
297
341
|
}
|
|
298
342
|
|
|
299
343
|
settle(() => resolve(true));
|
|
@@ -352,6 +396,19 @@ export class VoiceAgentClient {
|
|
|
352
396
|
|
|
353
397
|
console.log(`🔌 WebSocket closed — code: ${event.code}, reason: "${event.reason || 'none'}", clean: ${event.wasClean}`);
|
|
354
398
|
|
|
399
|
+
if (!this.isUserDisconnect && !RECONNECTABLE_CLOSE_CODES.has(event.code)) {
|
|
400
|
+
// The server ended the session on purpose. Say so, with its reason, and stay closed.
|
|
401
|
+
const ended = new LokutorError('session.ended', event.reason || 'The session was ended by the server.', {
|
|
402
|
+
detail: `close code ${event.code}`,
|
|
403
|
+
retryable: false,
|
|
404
|
+
});
|
|
405
|
+
console.log(`Session ended by the server (code ${event.code})`);
|
|
406
|
+
this.emit('ended', { code: event.code, reason: event.reason || '' });
|
|
407
|
+
if (this.onError) this.onError(ended);
|
|
408
|
+
if (this.onStatus) this.onStatus('disconnected');
|
|
409
|
+
return;
|
|
410
|
+
}
|
|
411
|
+
|
|
355
412
|
if (!this.isUserDisconnect && this.reconnectAttempts < this.maxReconnectAttempts) {
|
|
356
413
|
this.reconnecting = true;
|
|
357
414
|
this.reconnectAttempts++;
|
|
@@ -561,11 +618,21 @@ export class VoiceAgentClient {
|
|
|
561
618
|
detail: backendDetail,
|
|
562
619
|
retryable: backendRetryable,
|
|
563
620
|
});
|
|
564
|
-
|
|
565
|
-
this.emit('error', error);
|
|
621
|
+
this.emit('error', error); // reaches onError too (calling it here as well delivered every server error twice)
|
|
566
622
|
console.error(`❌ Server error: [${error.code}] ${error.message}`);
|
|
567
623
|
break;
|
|
568
624
|
}
|
|
625
|
+
case 'session_limit': {
|
|
626
|
+
// How long this session may run, sent by the server when there is a limit (the landing
|
|
627
|
+
// page's key-less demo). A UI can show it as a countdown instead of the call just ending.
|
|
628
|
+
const seconds = Number(msg.data?.seconds ?? msg.seconds);
|
|
629
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
630
|
+
const limit: SessionLimit = { seconds, reason: msg.data?.reason ?? msg.reason };
|
|
631
|
+
this.emit('session_limit', limit);
|
|
632
|
+
}
|
|
633
|
+
break;
|
|
634
|
+
}
|
|
635
|
+
|
|
569
636
|
case 'tool_call':
|
|
570
637
|
this.emit('tool_call', { name: msg.name, arguments: msg.arguments });
|
|
571
638
|
console.log(`🛠️ Tool Call: ${msg.name}(${msg.arguments})`);
|
|
@@ -648,6 +715,7 @@ export class VoiceAgentClient {
|
|
|
648
715
|
if (this.audioManager) {
|
|
649
716
|
this.audioManager.cleanup();
|
|
650
717
|
}
|
|
718
|
+
this.micStarted = false;
|
|
651
719
|
this.isConnected = false;
|
|
652
720
|
this.configConfirmed = false;
|
|
653
721
|
this.reconnecting = false;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { requestMicrophone } from './browser-audio';
|
|
2
|
+
|
|
1
3
|
const PANEL_CSS = /*css*/ `
|
|
2
4
|
.cv-panel {
|
|
3
5
|
position: relative;
|
|
@@ -102,6 +104,22 @@ const PANEL_CSS = /*css*/ `
|
|
|
102
104
|
margin-bottom: auto;
|
|
103
105
|
padding-top: 1.5rem;
|
|
104
106
|
}
|
|
107
|
+
.cv-countdown, .cv-note {
|
|
108
|
+
display: none;
|
|
109
|
+
margin-top: 0.4rem;
|
|
110
|
+
padding: 0.15rem 0.6rem;
|
|
111
|
+
border-radius: 999px;
|
|
112
|
+
border: 1px solid var(--cv-ui-border, rgba(255,255,255,0.1));
|
|
113
|
+
background: var(--cv-ui-bg, rgba(255,255,255,0.05));
|
|
114
|
+
color: var(--cv-text-dim, rgba(255,255,255,0.6));
|
|
115
|
+
font-size: 0.75rem;
|
|
116
|
+
font-variant-numeric: tabular-nums;
|
|
117
|
+
width: fit-content;
|
|
118
|
+
margin-left: auto;
|
|
119
|
+
margin-right: auto;
|
|
120
|
+
}
|
|
121
|
+
.cv-countdown.is-visible, .cv-note.is-visible { display: block; }
|
|
122
|
+
.cv-countdown.is-low { color: var(--cv-accent); border-color: var(--cv-accent); }
|
|
105
123
|
.cv-title {
|
|
106
124
|
font-size: clamp(1rem, 2.5vw, 1.75rem);
|
|
107
125
|
font-weight: 700;
|
|
@@ -333,7 +351,11 @@ export interface ConversationalPanelConfig {
|
|
|
333
351
|
backgroundColor?: string;
|
|
334
352
|
/** API key for connecting to the voice agent */
|
|
335
353
|
apiKey: string;
|
|
336
|
-
/**
|
|
354
|
+
/**
|
|
355
|
+
* Reference to the ConvoAgent class. Its constructor options include `microphone`, a
|
|
356
|
+
* Promise<MediaStream> requested inside the Start tap (see requestMicrophone); pass it on to
|
|
357
|
+
* BrowserAudioManager as `microphone` so the call uses it instead of asking again later.
|
|
358
|
+
*/
|
|
337
359
|
ConvoAgent: any;
|
|
338
360
|
/** Reference to the SphereVisualizer class */
|
|
339
361
|
SphereVisualizer: any;
|
|
@@ -349,6 +371,20 @@ export interface ConversationalPanelConfig {
|
|
|
349
371
|
silenceTimeout?: number;
|
|
350
372
|
}
|
|
351
373
|
|
|
374
|
+
/** What to tell the caller when the microphone did not start: a DOMException from getUserMedia, or the
|
|
375
|
+
* SDK's 'audio.microphone_unavailable' error wrapping one. */
|
|
376
|
+
export function microphoneMessage(err: any): string {
|
|
377
|
+
const name = String(err?.original?.name ?? err?.name ?? '');
|
|
378
|
+
const detail = String(err?.detail ?? err?.message ?? '');
|
|
379
|
+
if (/NotAllowedError|SecurityError/.test(name + ' ' + detail)) {
|
|
380
|
+
return 'Microphone access is blocked. Allow it for this site in your browser settings, then try again.';
|
|
381
|
+
}
|
|
382
|
+
if (/NotFoundError|OverconstrainedError/.test(name + ' ' + detail)) {
|
|
383
|
+
return 'No microphone was found on this device.';
|
|
384
|
+
}
|
|
385
|
+
return 'The microphone could not start. Please try again.';
|
|
386
|
+
}
|
|
387
|
+
|
|
352
388
|
export class ConversationalPanel {
|
|
353
389
|
private cfg: ConversationalPanelConfig;
|
|
354
390
|
private container: HTMLElement;
|
|
@@ -375,6 +411,11 @@ export class ConversationalPanel {
|
|
|
375
411
|
private muteSvg!: SVGElement;
|
|
376
412
|
private stopBtn!: HTMLElement;
|
|
377
413
|
private visualizerWrap!: HTMLElement;
|
|
414
|
+
private countdownEl!: HTMLElement;
|
|
415
|
+
private noteEl!: HTMLElement;
|
|
416
|
+
private countdownTicker: number | null = null;
|
|
417
|
+
private microphone: Promise<MediaStream> | null = null;
|
|
418
|
+
private micError: unknown = null;
|
|
378
419
|
|
|
379
420
|
// Callbacks
|
|
380
421
|
onTranscription?: (text: string) => void;
|
|
@@ -383,6 +424,12 @@ export class ConversationalPanel {
|
|
|
383
424
|
onStop?: () => void;
|
|
384
425
|
onError?: (err: any) => void;
|
|
385
426
|
onToolCall?: (tool: any) => void;
|
|
427
|
+
/**
|
|
428
|
+
* The call ended without the visitor pressing End: a time limit, the silence timeout, the server
|
|
429
|
+
* closing the session, or the connection being lost. Receives the message the panel shows, so a
|
|
430
|
+
* page that renders its own chrome (and hides this panel's curtain) can show it too.
|
|
431
|
+
*/
|
|
432
|
+
onEnded?: (message: string) => void;
|
|
386
433
|
|
|
387
434
|
constructor(cfg: ConversationalPanelConfig) {
|
|
388
435
|
this.cfg = cfg;
|
|
@@ -427,6 +474,8 @@ export class ConversationalPanel {
|
|
|
427
474
|
</div>
|
|
428
475
|
<div class="cv-header">
|
|
429
476
|
<h2 class="cv-title">${this.esc(this.cfg.title || "Voice Chat")}</h2>
|
|
477
|
+
<div class="cv-countdown" aria-live="off"></div>
|
|
478
|
+
<div class="cv-note" role="status"></div>
|
|
430
479
|
</div>
|
|
431
480
|
<div class="cv-visualizer-wrap">
|
|
432
481
|
<canvas class="cv-canvas"></canvas>
|
|
@@ -465,6 +514,8 @@ export class ConversationalPanel {
|
|
|
465
514
|
this.muteSvg = this.muteBtn.querySelector('svg')!;
|
|
466
515
|
this.stopBtn = this.el.querySelector('.cv-btn--end')!;
|
|
467
516
|
this.visualizerWrap = this.el.querySelector('.cv-visualizer-wrap')!;
|
|
517
|
+
this.countdownEl = this.el.querySelector('.cv-countdown')!;
|
|
518
|
+
this.noteEl = this.el.querySelector('.cv-note')!;
|
|
468
519
|
|
|
469
520
|
// Events
|
|
470
521
|
this.startBtn.addEventListener('click', () => this.start());
|
|
@@ -478,14 +529,25 @@ export class ConversationalPanel {
|
|
|
478
529
|
return d.innerHTML;
|
|
479
530
|
}
|
|
480
531
|
|
|
481
|
-
/**
|
|
482
|
-
|
|
483
|
-
|
|
532
|
+
/**
|
|
533
|
+
* Start the conversation (called when user clicks "Start Conversation" or externally). Call it from
|
|
534
|
+
* inside the click or tap, before any await: it asks for the microphone there. Resolves true once the
|
|
535
|
+
* call is live, false when it could not start (the reason is shown, and passed to onError).
|
|
536
|
+
*/
|
|
537
|
+
async start(): Promise<boolean> {
|
|
538
|
+
if (this.isRunning) return false;
|
|
484
539
|
if (this._locked) {
|
|
485
540
|
this.showError('This session has ended. Refresh the page to start a new one.');
|
|
486
|
-
return;
|
|
541
|
+
return false;
|
|
487
542
|
}
|
|
488
543
|
this.isRunning = true;
|
|
544
|
+
// First, while this is still the tap: iOS Safari only reliably shows its microphone prompt for a request
|
|
545
|
+
// made inside a user gesture, and refuses capture with no prompt at all while the page's audio session
|
|
546
|
+
// is in playback mode. Asked for later, after the connection opened, the call went live with the agent
|
|
547
|
+
// talking and the caller's microphone never captured (2026-09-28).
|
|
548
|
+
this.microphone = requestMicrophone();
|
|
549
|
+
this.micError = null;
|
|
550
|
+
this.microphone.catch((e) => { this.micError = e; });
|
|
489
551
|
this.startBtn.disabled = true;
|
|
490
552
|
this.errorEl.classList.remove('is-visible');
|
|
491
553
|
this.playConnectingSound();
|
|
@@ -523,7 +585,15 @@ export class ConversationalPanel {
|
|
|
523
585
|
this.el.classList.remove('cv-is-speaking', 'cv-is-thinking');
|
|
524
586
|
if (status === 'speaking') this.el.classList.add('cv-is-speaking');
|
|
525
587
|
else if (status === 'thinking') this.el.classList.add('cv-is-thinking');
|
|
588
|
+
// A dropped connection is retried by the client; say so instead of looking live.
|
|
589
|
+
this.setNote(status === 'reconnecting' ? 'Reconnecting…' : '');
|
|
590
|
+
// The client gave up, or the server ended the session: the call is over. Never leave a
|
|
591
|
+
// panel that looks live while nothing can answer.
|
|
592
|
+
if (status === 'disconnected' && this.isRunning) {
|
|
593
|
+
this.end('The call has ended.');
|
|
594
|
+
}
|
|
526
595
|
},
|
|
596
|
+
onSessionLimit: (seconds: number) => this.startCountdown(seconds),
|
|
527
597
|
onTranscription: (text: string) => {
|
|
528
598
|
this._lastSpeechTime = Date.now();
|
|
529
599
|
this.onTranscription?.(text);
|
|
@@ -538,9 +608,15 @@ export class ConversationalPanel {
|
|
|
538
608
|
onError: (err: any) => {
|
|
539
609
|
if (!this.isRunning) return;
|
|
540
610
|
this.onError?.(err);
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
611
|
+
// The server's own words when it ended the session on purpose (e.g. "Demo time limit
|
|
612
|
+
// reached..."); a generic line for anything else.
|
|
613
|
+
this.end(err?.code === 'audio.microphone_unavailable'
|
|
614
|
+
? microphoneMessage(err)
|
|
615
|
+
: err?.code === 'session.ended' && err?.message
|
|
616
|
+
? err.message
|
|
617
|
+
: 'Connection issue. Try again in a moment.');
|
|
618
|
+
},
|
|
619
|
+
microphone: this.microphone,
|
|
544
620
|
});
|
|
545
621
|
|
|
546
622
|
this.agent = agentHandle;
|
|
@@ -551,6 +627,8 @@ export class ConversationalPanel {
|
|
|
551
627
|
try { agentHandle.unlockAudioForMobile?.(); } catch (_) { /* ignore */ }
|
|
552
628
|
|
|
553
629
|
const ok = await agentHandle.connect();
|
|
630
|
+
// Ended while connecting (the microphone was refused, the server closed): the reason is already shown.
|
|
631
|
+
if (!this.isRunning) return false;
|
|
554
632
|
if (ok) {
|
|
555
633
|
this.fadeOutConnectingSound();
|
|
556
634
|
// Resize canvas to container
|
|
@@ -565,19 +643,22 @@ export class ConversationalPanel {
|
|
|
565
643
|
this.startTimer();
|
|
566
644
|
this.onStart?.();
|
|
567
645
|
if ((window as any).lucide) (window as any).lucide.createIcons();
|
|
568
|
-
|
|
569
|
-
this.fadeOutConnectingSound();
|
|
570
|
-
this.playErrorTone();
|
|
571
|
-
this.curtain.classList.remove('is-up');
|
|
572
|
-
this.showError('Could not connect. Please try again.');
|
|
573
|
-
this.isRunning = false;
|
|
646
|
+
return true;
|
|
574
647
|
}
|
|
648
|
+
this.fadeOutConnectingSound();
|
|
649
|
+
this.playErrorTone();
|
|
650
|
+
// The microphone's own failure when that is what stopped the call, not a generic connection line.
|
|
651
|
+
const micError = this.micError;
|
|
652
|
+
this.stop();
|
|
653
|
+
this.showError(micError ? microphoneMessage(micError) : 'Could not connect. Please try again.');
|
|
654
|
+
return false;
|
|
575
655
|
} catch (err: any) {
|
|
576
656
|
this.fadeOutConnectingSound();
|
|
577
657
|
this.playErrorTone();
|
|
578
658
|
console.error('ConversationalPanel start error:', err);
|
|
579
659
|
this.stop();
|
|
580
660
|
this.showError('Something went wrong. Please try again.');
|
|
661
|
+
return false;
|
|
581
662
|
} finally {
|
|
582
663
|
this.startBtn.disabled = false;
|
|
583
664
|
}
|
|
@@ -593,7 +674,14 @@ export class ConversationalPanel {
|
|
|
593
674
|
try { this.agent.disconnect(); } catch (_) {}
|
|
594
675
|
this.agent = null;
|
|
595
676
|
}
|
|
677
|
+
// Release the stream asked for at Start even if the agent never took it over (a ConvoAgent that
|
|
678
|
+
// ignores `microphone` asks again itself); stopping tracks the agent already stopped is harmless.
|
|
679
|
+
const mic = this.microphone;
|
|
680
|
+
this.microphone = null;
|
|
681
|
+
mic?.then((s) => s.getTracks().forEach((t) => t.stop()), () => {});
|
|
596
682
|
if (this.timerTicker) { clearInterval(this.timerTicker); this.timerTicker = null; }
|
|
683
|
+
this.stopCountdown();
|
|
684
|
+
this.setNote('');
|
|
597
685
|
if (this.visualizer) { this.visualizer.stop(); this.visualizer = null; }
|
|
598
686
|
this.curtain.classList.remove('is-up');
|
|
599
687
|
this.el.classList.remove('cv-is-speaking', 'cv-is-thinking');
|
|
@@ -683,6 +771,39 @@ export class ConversationalPanel {
|
|
|
683
771
|
}, 5000);
|
|
684
772
|
}
|
|
685
773
|
|
|
774
|
+
/** End the call for a reason other than the End button, and say why. */
|
|
775
|
+
private end(message: string) {
|
|
776
|
+
this.stop();
|
|
777
|
+
this.showError(message);
|
|
778
|
+
this.onEnded?.(message);
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/** Show how long the session may still run, as announced by the server. */
|
|
782
|
+
private startCountdown(seconds: number) {
|
|
783
|
+
this.stopCountdown();
|
|
784
|
+
const endsAt = Date.now() + seconds * 1000;
|
|
785
|
+
const render = () => {
|
|
786
|
+
const left = Math.max(0, Math.round((endsAt - Date.now()) / 1000));
|
|
787
|
+
this.countdownEl.textContent = `${Math.floor(left / 60)}:${String(left % 60).padStart(2, '0')} left`;
|
|
788
|
+
this.countdownEl.classList.toggle('is-low', left <= 30);
|
|
789
|
+
if (left === 0) this.stopCountdown(false);
|
|
790
|
+
};
|
|
791
|
+
this.countdownEl.classList.add('is-visible');
|
|
792
|
+
render();
|
|
793
|
+
this.countdownTicker = window.setInterval(render, 1000);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
private stopCountdown(hide = true) {
|
|
797
|
+
if (this.countdownTicker) { clearInterval(this.countdownTicker); this.countdownTicker = null; }
|
|
798
|
+
if (hide) this.countdownEl?.classList.remove('is-visible', 'is-low');
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
private setNote(text: string) {
|
|
802
|
+
if (!this.noteEl) return;
|
|
803
|
+
this.noteEl.textContent = text;
|
|
804
|
+
this.noteEl.classList.toggle('is-visible', Boolean(text));
|
|
805
|
+
}
|
|
806
|
+
|
|
686
807
|
/** Destroy the component, removing all DOM and stopping any active session */
|
|
687
808
|
destroy() {
|
|
688
809
|
this.stop();
|
|
@@ -703,15 +824,13 @@ export class ConversationalPanel {
|
|
|
703
824
|
|
|
704
825
|
if (elapsed >= maxDur) {
|
|
705
826
|
this._locked = true;
|
|
706
|
-
this.
|
|
707
|
-
this.showError('Tiempo límite alcanzado.');
|
|
827
|
+
this.end('Time limit reached.');
|
|
708
828
|
return;
|
|
709
829
|
}
|
|
710
830
|
|
|
711
831
|
if (Date.now() - this._lastSpeechTime > silentMax * 1000) {
|
|
712
832
|
this._locked = true;
|
|
713
|
-
this.
|
|
714
|
-
this.showError('Sesión finalizada por inactividad.');
|
|
833
|
+
this.end('The call ended after a long silence.');
|
|
715
834
|
}
|
|
716
835
|
}, 1000);
|
|
717
836
|
}
|
package/src/index.ts
CHANGED
|
@@ -14,7 +14,7 @@ export {
|
|
|
14
14
|
} from './client';
|
|
15
15
|
export { BrowserAudioManager } from './browser-audio';
|
|
16
16
|
export { NodeAudioManager } from './node-audio';
|
|
17
|
-
export { ConversationalPanel } from './conversational-panel';
|
|
17
|
+
export { ConversationalPanel, microphoneMessage } from './conversational-panel';
|
|
18
18
|
export type { ConversationalPanelConfig } from './conversational-panel';
|
|
19
19
|
export {
|
|
20
20
|
pcm16ToFloat32,
|
package/src/types.ts
CHANGED
|
@@ -338,6 +338,8 @@ export type ErrorCode =
|
|
|
338
338
|
| 'stt.language_not_supported'
|
|
339
339
|
| 'agent.session_failed'
|
|
340
340
|
| 'agent.provider_error'
|
|
341
|
+
| 'session.ended'
|
|
342
|
+
| 'audio.microphone_unavailable'
|
|
341
343
|
| 'internal.error'
|
|
342
344
|
| 'internal.timeout'
|
|
343
345
|
| 'internal.cancelled'
|
|
@@ -369,6 +371,8 @@ function isRetryableCode(code: ErrorCode): boolean {
|
|
|
369
371
|
'auth.missing_key',
|
|
370
372
|
'auth.invalid_key',
|
|
371
373
|
'auth.time_limited',
|
|
374
|
+
'session.ended',
|
|
375
|
+
'audio.microphone_unavailable',
|
|
372
376
|
'validation.invalid_voice',
|
|
373
377
|
'validation.invalid_language',
|
|
374
378
|
'validation.text_too_long',
|