@alexkroman1/aai-ui 1.14.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/audio.d.ts +43 -5
  2. package/dist/audio.js +39 -20
  3. package/dist/chat-view-DKFhxMAT.js +129 -0
  4. package/dist/client-config.d.ts +7 -6
  5. package/dist/components/chat-view.d.ts +1 -8
  6. package/dist/components/chat-view.js +2 -2
  7. package/dist/components/console-shell.d.ts +37 -0
  8. package/dist/components/controls.js +1 -1
  9. package/dist/components/url-chips.d.ts +1 -1
  10. package/dist/components/workflow-view.d.ts +13 -0
  11. package/dist/{controls-BbZcmnJf.js → controls-4OJoekj6.js} +1 -1
  12. package/dist/default-client/assets/audio-Cgviqo9t.js +1 -0
  13. package/dist/default-client/assets/capture-processor-DLHxAIfT.js +90 -0
  14. package/dist/default-client/assets/index-BgbIWnfG.css +2 -0
  15. package/dist/default-client/assets/index-DbKKR3UE.js +62 -0
  16. package/dist/default-client/assets/playback-processor-bMTdFp-8.js +271 -0
  17. package/dist/default-client/assets/rolldown-runtime-BpQH8Ho1.js +1 -0
  18. package/dist/default-client/assets/types-Bpg3ZIZK.js +64 -0
  19. package/dist/default-client/index.html +5 -2
  20. package/dist/define-client-yoGybEYR.js +710 -0
  21. package/dist/define-client.d.ts +0 -9
  22. package/dist/define-client.js +1 -1
  23. package/dist/index.d.ts +3 -4
  24. package/dist/index.js +6 -5
  25. package/dist/{session-core-B64kau_v.js → session-core-BLhiQ18c.js} +6 -0
  26. package/dist/session-core.js +1 -1
  27. package/dist/sync-mic.d.ts +30 -43
  28. package/dist/types.d.ts +24 -1
  29. package/dist/types.js +32 -2
  30. package/dist/worklets/capture-processor.d.ts +1 -1
  31. package/dist/worklets/capture-processor.js +32 -49
  32. package/dist/worklets/playback-processor.d.ts +1 -1
  33. package/dist/worklets/playback-processor.js +125 -9
  34. package/package.json +2 -2
  35. package/dist/chat-view-gi6FccZq.js +0 -193
  36. package/dist/components/sync-chat-view.d.ts +0 -18
  37. package/dist/components/text-controls.d.ts +0 -14
  38. package/dist/default-client/assets/audio-CTPVKBQ_.js +0 -1
  39. package/dist/default-client/assets/capture-processor-UlKEKyIW.js +0 -108
  40. package/dist/default-client/assets/index-CF3RKVUo.css +0 -2
  41. package/dist/default-client/assets/index-I5mZ1vB1.js +0 -121
  42. package/dist/default-client/assets/playback-processor-C5HVRVbu.js +0 -156
  43. package/dist/define-client-Dx4JaWzL.js +0 -881
  44. package/dist/sync-vad.d.ts +0 -54
@@ -1,881 +0,0 @@
1
- import { SessionProvider, ThemeProvider, useTheme } from "./context.js";
2
- import { Button } from "./components/button.js";
3
- import { i as TEXT_MUTED, r as TEXT_FAINT, t as ERROR_COLOR } from "./_colors-DYX7XRTr.js";
4
- import { t as AaiLogo } from "./aai-logo-B8lDmsut.js";
5
- import { a as UrlChip } from "./controls-BbZcmnJf.js";
6
- import { t as Eyebrow } from "./eyebrow-C6ZFuiz6.js";
7
- import { n as ToolConfigContext } from "./tool-call-block-DIxpG8GM.js";
8
- import { ThinkingDots } from "./components/message-list.js";
9
- import { n as stateColor, t as ChatView } from "./chat-view-gi6FccZq.js";
10
- import { SidebarLayout } from "./components/sidebar-layout.js";
11
- import { StartScreen } from "./components/start-screen.js";
12
- import { t as createSessionCore } from "./session-core-B64kau_v.js";
13
- import { CLIENT_CONFIG_PATH, ClientConfigResponseSchema, SyncTurnResponseSchema } from "@alexkroman1/aai/protocol";
14
- import clsx from "clsx";
15
- import { createElement, useEffect, useRef, useState } from "react";
16
- import { jsx, jsxs } from "react/jsx-runtime";
17
- import { errorMessage, safeJsonParse } from "@alexkroman1/aai";
18
- import { flushSync } from "react-dom";
19
- import { createRoot } from "react-dom/client";
20
- //#region client-config.ts
21
- /**
22
- * Pre-connection client-config lookup.
23
- *
24
- * `GET client-config` (relative to the agent's base URL — see
25
- * `sdk/client-config.ts` in `@alexkroman1/aai`) tells the default client how
26
- * to talk to the agent before any connection exists, most importantly which
27
- * transport `agent({ transport })` declared. Every failure path — network
28
- * error, 404 from an older server, malformed body — degrades to the
29
- * WebSocket default, so this lookup can never break an existing agent.
30
- */
31
- /** Resolve a relative endpoint path against the agent's base URL. */
32
- function buildAgentUrl(platformUrl, endpointPath) {
33
- return new URL(endpointPath, platformUrl.endsWith("/") ? platformUrl : `${platformUrl}/`);
34
- }
35
- const WEBSOCKET_DEFAULT = { transport: "websocket" };
36
- /** Fetch the agent's client config; any failure yields the WebSocket default. */
37
- async function fetchClientConfig(platformUrl, fetchFn) {
38
- const doFetch = fetchFn ?? ((input, init) => globalThis.fetch(input, init));
39
- try {
40
- const resp = await doFetch(buildAgentUrl(platformUrl, CLIENT_CONFIG_PATH).href);
41
- if (!resp.ok) return WEBSOCKET_DEFAULT;
42
- const parsed = ClientConfigResponseSchema.safeParse(await resp.json());
43
- return parsed.success ? parsed.data : WEBSOCKET_DEFAULT;
44
- } catch {
45
- return WEBSOCKET_DEFAULT;
46
- }
47
- }
48
- //#endregion
49
- //#region sync-vad.ts
50
- const DEFAULT_SPEECH_RMS = .015;
51
- const DEFAULT_MAX_UTTERANCE_MS = 3e4;
52
- function rmsOf(frame) {
53
- if (frame.length === 0) return 0;
54
- let sum = 0;
55
- for (const s of frame) {
56
- const f = s / 32768;
57
- sum += f * f;
58
- }
59
- return Math.sqrt(sum / frame.length);
60
- }
61
- function concat(frames) {
62
- let total = 0;
63
- for (const f of frames) total += f.length;
64
- const out = new Int16Array(total);
65
- let offset = 0;
66
- for (const f of frames) {
67
- out.set(f, offset);
68
- offset += f.length;
69
- }
70
- return out;
71
- }
72
- /** Create an energy-based {@link UtteranceDetector}. */
73
- function createUtteranceDetector(opts) {
74
- const { sampleRate, speechRms = DEFAULT_SPEECH_RMS, minSpeechMs = 150, hangoverMs = 700, prerollMs = 300, maxUtteranceMs = DEFAULT_MAX_UTTERANCE_MS } = opts;
75
- const msOf = (frame) => frame.length / sampleRate * 1e3;
76
- let preroll = [];
77
- let prerollTotalMs = 0;
78
- let utterance = [];
79
- let utteranceMs = 0;
80
- let speaking = false;
81
- /** Consecutive voiced ms while still a candidate (below minSpeechMs). */
82
- let candidateVoicedMs = 0;
83
- /** Consecutive silent ms while speaking (toward hangoverMs). */
84
- let silenceMs = 0;
85
- function toIdle() {
86
- utterance = [];
87
- utteranceMs = 0;
88
- speaking = false;
89
- candidateVoicedMs = 0;
90
- silenceMs = 0;
91
- }
92
- function pushPreroll(frame) {
93
- preroll.push(frame);
94
- prerollTotalMs += msOf(frame);
95
- while (preroll.length > 0 && prerollTotalMs - msOf(preroll[0]) >= prerollMs) prerollTotalMs -= msOf(preroll.shift());
96
- }
97
- function beginCandidate(frame) {
98
- utterance = [...preroll, frame];
99
- utteranceMs = prerollTotalMs + msOf(frame);
100
- preroll = [];
101
- prerollTotalMs = 0;
102
- candidateVoicedMs = msOf(frame);
103
- }
104
- function finalize() {
105
- const out = concat(utterance);
106
- toIdle();
107
- return out;
108
- }
109
- function pushIdle(frame, voiced) {
110
- if (!voiced) {
111
- pushPreroll(frame);
112
- return;
113
- }
114
- beginCandidate(frame);
115
- }
116
- function pushCandidate(frame, voiced) {
117
- if (!voiced) {
118
- for (const f of utterance) pushPreroll(f);
119
- toIdle();
120
- return;
121
- }
122
- utterance.push(frame);
123
- utteranceMs += msOf(frame);
124
- candidateVoicedMs += msOf(frame);
125
- }
126
- function pushSpeaking(frame, voiced) {
127
- utterance.push(frame);
128
- utteranceMs += msOf(frame);
129
- silenceMs = voiced ? 0 : silenceMs + msOf(frame);
130
- return silenceMs >= hangoverMs ? finalize() : null;
131
- }
132
- return {
133
- get speaking() {
134
- return speaking;
135
- },
136
- push(frame) {
137
- if (frame.length === 0) return null;
138
- const voiced = rmsOf(frame) >= speechRms;
139
- let closed = null;
140
- if (speaking) closed = pushSpeaking(frame, voiced);
141
- else if (utterance.length > 0) pushCandidate(frame, voiced);
142
- else pushIdle(frame, voiced);
143
- if (closed) return closed;
144
- if (!speaking && candidateVoicedMs >= minSpeechMs) speaking = true;
145
- if (speaking && utteranceMs >= maxUtteranceMs) return finalize();
146
- return null;
147
- },
148
- flush() {
149
- const out = speaking ? finalize() : null;
150
- this.reset();
151
- return out;
152
- },
153
- reset() {
154
- preroll = [];
155
- prerollTotalMs = 0;
156
- toIdle();
157
- }
158
- };
159
- }
160
- //#endregion
161
- //#region sync-mic.ts
162
- /**
163
- * WebRTC microphone capture for sync mode.
164
- *
165
- * Captures voice through `getUserMedia` with the WebRTC voice-processing
166
- * constraints (echo cancellation, noise suppression, auto gain — the
167
- * processing that makes the energy VAD in `sync-vad.ts` reliable), runs an
168
- * AudioWorklet that batches raw frames to the main thread, feeds them
169
- * through the utterance detector, and hands each completed utterance to
170
- * the sync session as one HTTP turn. No WebSocket anywhere on the path.
171
- *
172
- * The worklet module ships inline as a blob URL (same pattern as the
173
- * WebSocket path's worklets), so sync mode needs no separately-served
174
- * processor file. A blob URL rather than a data URI because the agent
175
- * page's CSP allows `script-src blob:` but not `data:` — a data-URI
176
- * module fails `addModule` with "Unable to load a worklet's module".
177
- */
178
- /** Default capture rate — what the STT providers expect. */
179
- const DEFAULT_SYNC_MIC_SAMPLE_RATE = 16e3;
180
- /** ~128 ms at 16 kHz: few messages per second, fine-enough VAD granularity. */
181
- const CAPTURE_BATCH_SAMPLES = 2048;
182
- /**
183
- * The capture processor: coalesces 128-sample render quanta into
184
- * {@link CAPTURE_BATCH_SAMPLES} batches and posts them (transferred, so no
185
- * per-batch copy). Inlined as source because it must be stringified into a
186
- * blob URL.
187
- */
188
- const CAPTURE_PROCESSOR_SRC = `
189
- registerProcessor("aai-sync-capture", class extends AudioWorkletProcessor {
190
- constructor(options) {
191
- super();
192
- const batch = (options && options.processorOptions && options.processorOptions.batchSamples) || ${CAPTURE_BATCH_SAMPLES};
193
- this.buf = new Float32Array(batch);
194
- this.len = 0;
195
- }
196
- process(inputs) {
197
- const ch = inputs[0] && inputs[0][0];
198
- if (!ch) return true;
199
- let read = 0;
200
- while (read < ch.length) {
201
- const n = Math.min(ch.length - read, this.buf.length - this.len);
202
- this.buf.set(ch.subarray(read, read + n), this.len);
203
- this.len += n;
204
- read += n;
205
- if (this.len === this.buf.length) {
206
- const out = this.buf;
207
- this.port.postMessage({ event: "chunk", samples: out }, [out.buffer]);
208
- this.buf = new Float32Array(out.length);
209
- this.len = 0;
210
- }
211
- }
212
- return true;
213
- }
214
- });
215
- `;
216
- /**
217
- * Blob-URL module for the capture processor (no served asset). Satisfies the
218
- * agent page's `script-src blob:` CSP, which rejects data-URI modules.
219
- */
220
- const CAPTURE_WORKLET_MODULE_URL = URL.createObjectURL(new Blob([CAPTURE_PROCESSOR_SRC], { type: "application/javascript" }));
221
- /** Clamp-and-convert one Float32 capture batch to PCM16. */
222
- function floatToPcm16(samples) {
223
- const pcm = new Int16Array(samples.length);
224
- let i = 0;
225
- for (const sample of samples) {
226
- const s = Math.max(-1, Math.min(1, sample));
227
- pcm[i++] = s < 0 ? s * 32768 : s * 32767;
228
- }
229
- return pcm;
230
- }
231
- /**
232
- * Push-to-talk recorder on the same WebRTC capture pipeline as
233
- * {@link startSyncMicrophone} — `getUserMedia` voice processing feeding the
234
- * capture worklet — minus the VAD: the caller's button is the endpointing.
235
- * Recording runs exactly between `start()` and `stop()`; the mic stays open
236
- * across presses until `close()`.
237
- *
238
- * @public
239
- */
240
- function createPttRecorder(sampleRate = DEFAULT_SYNC_MIC_SAMPLE_RATE) {
241
- let ctx = null;
242
- let stream = null;
243
- let node = null;
244
- let chunks = [];
245
- let recording = false;
246
- async function ensureOpen() {
247
- if (ctx) return;
248
- const streamPromise = navigator.mediaDevices.getUserMedia({ audio: {
249
- echoCancellation: true,
250
- noiseSuppression: true,
251
- autoGainControl: true
252
- } });
253
- const audioCtx = new AudioContext({
254
- sampleRate,
255
- latencyHint: "interactive"
256
- });
257
- try {
258
- const [media] = await Promise.all([
259
- streamPromise,
260
- audioCtx.resume(),
261
- audioCtx.audioWorklet.addModule(CAPTURE_WORKLET_MODULE_URL)
262
- ]);
263
- stream = media;
264
- } catch (err) {
265
- streamPromise.then((s) => {
266
- for (const t of s.getTracks()) t.stop();
267
- }).catch(() => {});
268
- await audioCtx.close().catch(() => {});
269
- throw err;
270
- }
271
- const workletNode = new AudioWorkletNode(audioCtx, "aai-sync-capture", {
272
- channelCount: 1,
273
- channelCountMode: "explicit",
274
- processorOptions: { batchSamples: CAPTURE_BATCH_SAMPLES }
275
- });
276
- workletNode.port.onmessage = (e) => {
277
- const data = e.data;
278
- if (recording && data.event === "chunk" && data.samples) chunks.push(data.samples);
279
- };
280
- audioCtx.createMediaStreamSource(stream).connect(workletNode);
281
- ctx = audioCtx;
282
- node = workletNode;
283
- }
284
- return {
285
- async start() {
286
- await ensureOpen();
287
- chunks = [];
288
- recording = true;
289
- },
290
- async stop() {
291
- await new Promise((r) => setTimeout(r, 150));
292
- recording = false;
293
- const total = chunks.reduce((n, c) => n + c.length, 0);
294
- const all = new Float32Array(total);
295
- let offset = 0;
296
- for (const c of chunks) {
297
- all.set(c, offset);
298
- offset += c.length;
299
- }
300
- chunks = [];
301
- return floatToPcm16(all);
302
- },
303
- async close() {
304
- recording = false;
305
- node?.disconnect();
306
- if (stream) for (const t of stream.getTracks()) t.stop();
307
- await ctx?.close().catch(() => {});
308
- ctx = null;
309
- node = null;
310
- stream = null;
311
- }
312
- };
313
- }
314
- /**
315
- * Open the microphone and stream endpointed utterances into a sync session.
316
- *
317
- * @throws If microphone access is denied or worklet registration fails.
318
- */
319
- async function startSyncMicrophone(opts) {
320
- const sampleRate = opts.sampleRate ?? 16e3;
321
- const detector = createUtteranceDetector({
322
- sampleRate,
323
- ...opts.vad
324
- });
325
- const fail = (err) => {
326
- opts.onError?.(err instanceof Error ? err : new Error(errorMessage(err)));
327
- };
328
- const streamPromise = navigator.mediaDevices.getUserMedia({ audio: {
329
- echoCancellation: true,
330
- noiseSuppression: true,
331
- autoGainControl: true
332
- } });
333
- const ctx = new AudioContext({
334
- sampleRate,
335
- latencyHint: "interactive"
336
- });
337
- let stream;
338
- try {
339
- [stream] = await Promise.all([
340
- streamPromise,
341
- ctx.resume(),
342
- ctx.audioWorklet.addModule(CAPTURE_WORKLET_MODULE_URL)
343
- ]);
344
- } catch (err) {
345
- streamPromise.then((s) => {
346
- for (const t of s.getTracks()) t.stop();
347
- }).catch(() => {});
348
- await ctx.close().catch(() => {});
349
- throw err;
350
- }
351
- const mic = ctx.createMediaStreamSource(stream);
352
- const node = new AudioWorkletNode(ctx, "aai-sync-capture", {
353
- channelCount: 1,
354
- channelCountMode: "explicit",
355
- processorOptions: { batchSamples: CAPTURE_BATCH_SAMPLES }
356
- });
357
- mic.connect(node);
358
- let stopped = false;
359
- let wasSpeaking = false;
360
- function dispatch(utterance) {
361
- if (!utterance) return;
362
- opts.onSpeechEnd?.();
363
- opts.session.sendPcm16(utterance, sampleRate).catch(fail);
364
- }
365
- function handleFrame(samples) {
366
- if (stopped) return;
367
- dispatch(detector.push(floatToPcm16(samples)));
368
- if (detector.speaking && !wasSpeaking) opts.onSpeechStart?.();
369
- wasSpeaking = detector.speaking;
370
- }
371
- node.port.onmessage = (e) => {
372
- const data = e.data;
373
- if (data.event === "chunk" && data.samples) handleFrame(data.samples);
374
- };
375
- node.onprocessorerror = () => fail(/* @__PURE__ */ new Error("Sync capture worklet crashed"));
376
- return {
377
- get speaking() {
378
- return detector.speaking;
379
- },
380
- async stop() {
381
- if (stopped) return;
382
- stopped = true;
383
- dispatch(detector.flush());
384
- mic.disconnect();
385
- node.disconnect();
386
- for (const t of stream.getTracks()) t.stop();
387
- await ctx.close().catch(() => {});
388
- }
389
- };
390
- }
391
- //#endregion
392
- //#region sync-session.ts
393
- /**
394
- * Sync-mode browser session — HTTP turns, no WebSocket.
395
- *
396
- * The client half of the server's `POST /sync` endpoint (see
397
- * `host/sync-turn.ts` in `@alexkroman1/aai`): each turn is one request
398
- * carrying committed text or one endpointed utterance of PCM16 audio plus
399
- * the conversation history, answered with the transcript, the reply text,
400
- * and (when the agent's TTS provider supports one-shot synthesis) the
401
- * spoken reply. The server holds no session state — this object owns the
402
- * history and replays it every turn.
403
- *
404
- * Microphone capture and utterance endpointing live in `sync-mic.ts` /
405
- * `sync-vad.ts`; this module is transport only, so it also runs in
406
- * non-browser clients that bring their own audio.
407
- */
408
- /** Base64-encode PCM16 samples (chunked — `btoa` takes a binary string). */
409
- function pcm16ToBase64(pcm) {
410
- const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
411
- let binary = "";
412
- const CHUNK = 32768;
413
- for (let i = 0; i < bytes.length; i += CHUNK) binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
414
- return btoa(binary);
415
- }
416
- /** Decode base64 PCM16LE (the sync response's `audio` field) into samples. */
417
- function base64ToPcm16(base64) {
418
- const binary = atob(base64);
419
- const samples = binary.length >> 1;
420
- const bytes = new Uint8Array(samples * 2);
421
- for (let i = 0; i < bytes.length; i++) bytes[i] = binary.charCodeAt(i);
422
- return new Int16Array(bytes.buffer, 0, samples);
423
- }
424
- /** Create a {@link SyncSession} against a sync-mode agent server. */
425
- function createSyncSession(opts) {
426
- const fetchFn = opts.fetch ?? ((input, init) => globalThis.fetch(input, init));
427
- const history = [];
428
- let queue = Promise.resolve();
429
- async function parseTurn(resp) {
430
- if (!resp.ok) {
431
- const detail = safeJsonParse(await resp.text().catch(() => "")) ?? {};
432
- throw new Error(`Sync turn failed: HTTP ${resp.status}${detail.error ? ` (${detail.error})` : ""}`);
433
- }
434
- const parsed = SyncTurnResponseSchema.safeParse(await resp.json());
435
- if (!parsed.success) throw new Error("Sync turn failed: malformed server response");
436
- return parsed.data;
437
- }
438
- async function runTurn(body) {
439
- try {
440
- const turn = await parseTurn(await fetchFn(opts.url, {
441
- method: "POST",
442
- headers: { "Content-Type": "application/json" },
443
- body: JSON.stringify({
444
- ...body,
445
- history: [...history]
446
- })
447
- }));
448
- history.push({
449
- role: "user",
450
- content: turn.transcript
451
- });
452
- if (turn.reply.length > 0) history.push({
453
- role: "assistant",
454
- content: turn.reply
455
- });
456
- const result = {
457
- ...turn,
458
- pcm: turn.audio !== void 0 ? base64ToPcm16(turn.audio) : null
459
- };
460
- opts.onTurn?.(result);
461
- return result;
462
- } catch (err) {
463
- const wrapped = err instanceof Error ? err : new Error(errorMessage(err));
464
- opts.onError?.(wrapped);
465
- throw wrapped;
466
- }
467
- }
468
- function enqueue(body) {
469
- const next = queue.then(() => runTurn(body), () => runTurn(body));
470
- queue = next.catch(() => void 0);
471
- return next;
472
- }
473
- return {
474
- get history() {
475
- return history;
476
- },
477
- sendText(text) {
478
- return enqueue({ text });
479
- },
480
- sendPcm16(pcm, sampleRate) {
481
- return enqueue({
482
- audio: pcm16ToBase64(pcm),
483
- sampleRate
484
- });
485
- },
486
- reset() {
487
- history.length = 0;
488
- }
489
- };
490
- }
491
- //#endregion
492
- //#region components/sync-chat-view.tsx
493
- /** @jsxImportSource react */
494
- /**
495
- * Default shell for sync-transport agents (`agent({ transport: "sync" })`).
496
- *
497
- * A hands-free voice agent: one toggle starts the conversation, and from
498
- * then on the mic stays open — `startSyncMicrophone` runs the WebRTC
499
- * voice-processing capture through the energy VAD (`sync-vad.ts`), which
500
- * endpoints each utterance automatically and sends it as one `POST /sync`
501
- * request through `createSyncSession`. No button per turn: speak, pause,
502
- * and the reply comes back and plays. The view shows what was heard, the
503
- * agent's reply, and — via the endpoint chip next to the toggle — exactly
504
- * where each utterance is being sent.
505
- *
506
- * Visually it is the same "voice agent console" as the WebSocket
507
- * {@link ChatView}: header with logo + live-status eyebrow, the output on a
508
- * raised card, controls beneath — built from the same shared pieces
509
- * ({@link ThinkingDots}, {@link Eyebrow}, {@link Button}, {@link UrlChip})
510
- * so the two transports are indistinguishable at a glance.
511
- *
512
- * Rendered by `client()` when the agent's `GET /client-config` declares
513
- * `transport: "sync"`; also exported for custom clients that want the stock
514
- * sync UI with their own chrome around it.
515
- */
516
- /**
517
- * Play one reply's PCM16 through a shared AudioContext. `onPlaying`
518
- * tracks playback so the eyebrow can show "speaking" while the reply is
519
- * audible.
520
- */
521
- function playReply(ctxRef, turn, onPlaying) {
522
- if (!(turn.pcm && turn.sampleRate)) return;
523
- ctxRef.current ??= new AudioContext();
524
- const ctx = ctxRef.current;
525
- const buffer = ctx.createBuffer(1, turn.pcm.length, turn.sampleRate);
526
- const channel = buffer.getChannelData(0);
527
- turn.pcm.forEach((sample, i) => {
528
- channel[i] = sample / 32768;
529
- });
530
- const source = ctx.createBufferSource();
531
- source.buffer = buffer;
532
- source.connect(ctx.destination);
533
- source.onended = () => onPlaying(false);
534
- onPlaying(true);
535
- source.start();
536
- }
537
- /**
538
- * Map the hands-free conversation lifecycle onto the same states the
539
- * WebSocket eyebrow shows, so the status chip reads identically across
540
- * transports: an endpointed utterance in flight is "thinking", an audible
541
- * reply is "speaking", a live mic is "listening".
542
- */
543
- function syncState(opts) {
544
- if (opts.pending > 0) return "thinking";
545
- if (opts.agentSpeaking) return "speaking";
546
- if (opts.live) return "listening";
547
- if (opts.error) return "error";
548
- return "ready";
549
- }
550
- /**
551
- * Sync-transport view: a hands-free VAD-endpointed conversation, transcript
552
- * + reply output, and the endpoint each utterance is POSTed to — one HTTP
553
- * request per turn.
554
- *
555
- * @public
556
- */
557
- function SyncChatView({ syncUrl, title, greeting }) {
558
- const theme = useTheme();
559
- const [exchanges, setExchanges] = useState([]);
560
- const [live, setLive] = useState(false);
561
- const [userSpeaking, setUserSpeaking] = useState(false);
562
- const [pending, setPending] = useState(0);
563
- const [agentSpeaking, setAgentSpeaking] = useState(false);
564
- const [error, setError] = useState(null);
565
- const playbackCtx = useRef(null);
566
- const mic = useRef(null);
567
- const toggling = useRef(false);
568
- const anchorRef = useRef(null);
569
- const sessionRef = useRef(createSyncSession({
570
- url: syncUrl,
571
- onTurn: (turn) => {
572
- setExchanges((prev) => [...prev, {
573
- id: prev.length,
574
- heard: turn.transcript,
575
- reply: turn.reply
576
- }]);
577
- setPending((n) => Math.max(0, n - 1));
578
- setError(turn.ttsError ? `TTS unavailable: ${turn.ttsError}` : null);
579
- playReply(playbackCtx, turn, setAgentSpeaking);
580
- },
581
- onError: (err) => {
582
- setPending((n) => Math.max(0, n - 1));
583
- setError(err.message);
584
- }
585
- }));
586
- useEffect(() => () => {
587
- mic.current?.stop();
588
- playbackCtx.current?.close();
589
- }, []);
590
- const contentCount = exchanges.length + (pending > 0 ? 1 : 0);
591
- useEffect(() => {
592
- if (contentCount === 0) return;
593
- anchorRef.current?.scrollIntoView({
594
- behavior: "smooth",
595
- block: "end"
596
- });
597
- }, [contentCount]);
598
- async function toggleConversation() {
599
- if (toggling.current) return;
600
- toggling.current = true;
601
- try {
602
- if (mic.current) {
603
- const handle = mic.current;
604
- mic.current = null;
605
- setLive(false);
606
- setUserSpeaking(false);
607
- await handle.stop();
608
- return;
609
- }
610
- mic.current = await startSyncMicrophone({
611
- session: sessionRef.current,
612
- onSpeechStart: () => setUserSpeaking(true),
613
- onSpeechEnd: () => {
614
- setUserSpeaking(false);
615
- setPending((n) => n + 1);
616
- },
617
- onError: (err) => setError(err.message)
618
- });
619
- setLive(true);
620
- setError(null);
621
- } catch (err) {
622
- setError(err instanceof Error ? err.message : String(err));
623
- } finally {
624
- toggling.current = false;
625
- }
626
- }
627
- const state = syncState({
628
- error,
629
- live,
630
- pending,
631
- agentSpeaking
632
- });
633
- const buttonLabel = live ? "End conversation" : "Start conversation";
634
- const pulsing = userSpeaking || agentSpeaking;
635
- return /* @__PURE__ */ jsxs("div", {
636
- className: "flex flex-col h-screen w-full max-w-190 mx-auto box-border px-6 py-8 gap-5 font-aai text-sm",
637
- style: {
638
- background: theme.bg,
639
- color: theme.text
640
- },
641
- children: [
642
- /* @__PURE__ */ jsxs("div", {
643
- className: "flex items-center justify-between shrink-0",
644
- children: [/* @__PURE__ */ jsxs("div", {
645
- className: "flex items-center gap-3 min-w-0",
646
- children: [/* @__PURE__ */ jsx(AaiLogo, { size: 22 }), /* @__PURE__ */ jsx("span", {
647
- className: "font-aai-serif text-[22px] leading-[1.2] font-normal truncate",
648
- style: { color: theme.text },
649
- children: title ?? "Voice Agent"
650
- })]
651
- }), /* @__PURE__ */ jsxs(Eyebrow, {
652
- className: "shrink-0",
653
- "data-state": state,
654
- children: [/* @__PURE__ */ jsx("span", {
655
- className: "w-[7px] h-[7px] rounded-full",
656
- style: {
657
- background: stateColor(state, theme.primary),
658
- animation: pulsing ? "aai-pulse 1.6s ease-in-out infinite" : "none"
659
- }
660
- }), state]
661
- })]
662
- }),
663
- error && /* @__PURE__ */ jsx("div", {
664
- className: "px-3.5 py-2.5 rounded-aai border text-[13px] leading-[130%] shrink-0",
665
- style: {
666
- borderColor: "rgba(179,38,30,0.35)",
667
- background: "rgba(179,38,30,0.06)",
668
- color: "#B3261E"
669
- },
670
- children: error
671
- }),
672
- /* @__PURE__ */ jsx("div", {
673
- className: "flex flex-col flex-1 min-h-0 border rounded-lg overflow-hidden",
674
- style: {
675
- background: theme.surface,
676
- borderColor: theme.border,
677
- boxShadow: "0 1px 3px 0 rgb(20 18 12 / 0.06)"
678
- },
679
- children: /* @__PURE__ */ jsx("div", {
680
- role: "log",
681
- className: "flex-1 overflow-y-auto [scrollbar-width:none]",
682
- style: { background: theme.surface },
683
- children: /* @__PURE__ */ jsxs("div", {
684
- className: "flex flex-col gap-5 p-7",
685
- children: [
686
- greeting !== void 0 && greeting.length > 0 && /* @__PURE__ */ jsx("p", {
687
- className: "text-[15px] leading-[23px]",
688
- style: { color: theme.text },
689
- children: greeting
690
- }),
691
- exchanges.length === 0 && /* @__PURE__ */ jsx("p", {
692
- className: "text-sm",
693
- style: { color: "#57534B" },
694
- children: "Start the conversation and just talk — each pause endpoints an utterance, which goes out as one HTTP request to the endpoint below."
695
- }),
696
- exchanges.map((ex) => /* @__PURE__ */ jsxs("div", {
697
- className: "flex flex-col gap-1.5",
698
- children: [
699
- /* @__PURE__ */ jsx("span", {
700
- className: "text-[10px] font-medium tracking-[1.2px] uppercase leading-none",
701
- style: { color: TEXT_FAINT },
702
- children: "Heard"
703
- }),
704
- /* @__PURE__ */ jsx("p", {
705
- className: "text-[15px] leading-[22px]",
706
- style: { color: TEXT_MUTED },
707
- children: ex.heard
708
- }),
709
- /* @__PURE__ */ jsx("span", {
710
- className: "text-[10px] font-medium tracking-[1.2px] uppercase leading-none mt-1.5",
711
- style: { color: TEXT_FAINT },
712
- children: "Agent"
713
- }),
714
- /* @__PURE__ */ jsx("p", {
715
- className: "whitespace-pre-wrap wrap-break-word text-[15px] font-normal leading-[23px]",
716
- style: { color: theme.text },
717
- children: ex.reply
718
- })
719
- ]
720
- }, ex.id)),
721
- pending > 0 && /* @__PURE__ */ jsx("div", {
722
- "data-testid": "thinking",
723
- children: /* @__PURE__ */ jsx(ThinkingDots, {})
724
- }),
725
- /* @__PURE__ */ jsx("div", { ref: anchorRef })
726
- ]
727
- })
728
- })
729
- }),
730
- /* @__PURE__ */ jsxs("div", {
731
- className: "flex items-center gap-2 shrink-0",
732
- children: [/* @__PURE__ */ jsxs(Button, {
733
- size: "lg",
734
- variant: live ? "default" : "secondary",
735
- className: "select-none",
736
- style: live ? {
737
- background: ERROR_COLOR,
738
- borderColor: "transparent"
739
- } : void 0,
740
- onClick: () => void toggleConversation(),
741
- "aria-pressed": live,
742
- title: "Start or end the conversation",
743
- children: [/* @__PURE__ */ jsx("span", {
744
- className: clsx("w-2 h-2 rounded-full mr-2", pulsing && "animate-pulse"),
745
- style: { background: live ? "#fff" : ERROR_COLOR }
746
- }), buttonLabel]
747
- }), /* @__PURE__ */ jsx(UrlChip, {
748
- label: "Sync",
749
- url: syncUrl,
750
- hint: "Each utterance is one POST to this endpoint",
751
- testId: "sync-url-chip",
752
- className: "ml-auto min-w-0 max-w-[55%]"
753
- })]
754
- })
755
- ]
756
- });
757
- }
758
- //#endregion
759
- //#region define-client.tsx
760
- /** @jsxImportSource react */
761
- function resolveContainer(target = "#app") {
762
- if (typeof target !== "string") return target;
763
- const el = document.querySelector(target);
764
- if (!el) throw new Error(`Element not found: ${target}`);
765
- return el;
766
- }
767
- /**
768
- * Default shell rendered in config tier.
769
- * Wraps StartScreen → (SidebarLayout →) ChatView.
770
- */
771
- function DefaultShell({ name, Sidebar, sidebarWidth }) {
772
- const chat = /* @__PURE__ */ jsx(ChatView, { title: name });
773
- return /* @__PURE__ */ jsx(StartScreen, {
774
- title: name,
775
- children: Sidebar ? /* @__PURE__ */ jsx(SidebarLayout, {
776
- sidebar: /* @__PURE__ */ jsx(Sidebar, {}),
777
- sidebarWidth,
778
- children: chat
779
- }) : chat
780
- });
781
- }
782
- /**
783
- * Config-tier root: resolves the transport (explicit option, else the
784
- * server's `GET client-config`) and renders the WebSocket shell or the
785
- * sync shell.
786
- *
787
- * The WebSocket shell renders immediately — optimistically — while the
788
- * lookup is in flight, then swaps if the agent declared `"sync"`. That
789
- * keeps mounting synchronous, keeps agents on servers without the endpoint
790
- * (every lookup failure resolves to `"websocket"`) exactly as before, and
791
- * the worst-case race — Start clicked on a sync agent before the lookup
792
- * lands — still yields a working session: sync agents answer WebSocket
793
- * sessions too.
794
- */
795
- function DefaultRoot({ platformUrl, transport, name, Sidebar, sidebarWidth }) {
796
- const [resolved, setResolved] = useState(transport !== void 0 ? { transport } : null);
797
- useEffect(() => {
798
- if (transport !== void 0) return;
799
- let cancelled = false;
800
- fetchClientConfig(platformUrl).then((cfg) => {
801
- if (!cancelled) setResolved(cfg);
802
- });
803
- return () => {
804
- cancelled = true;
805
- };
806
- }, [platformUrl, transport]);
807
- if (resolved?.transport === "sync") return /* @__PURE__ */ jsx(SyncChatView, {
808
- syncUrl: buildAgentUrl(platformUrl, "sync").href,
809
- title: name ?? resolved.name,
810
- greeting: resolved.greeting
811
- });
812
- return /* @__PURE__ */ jsx(DefaultShell, {
813
- name,
814
- Sidebar,
815
- sidebarWidth
816
- });
817
- }
818
- /**
819
- * Define and mount a client UI for a voice agent.
820
- *
821
- * **Tier 1 (config-only):** Pass options without `component` to get the
822
- * default shell (StartScreen + ChatView, optional sidebar).
823
- *
824
- * **Tier 2 (custom component):** Pass `component` to render a fully custom
825
- * root component inside the providers.
826
- *
827
- * @example Tier 1
828
- * ```tsx
829
- * client({
830
- * name: "Pizza Ordering",
831
- * theme: { bg: "#1a1a1a", primary: "#e55" },
832
- * sidebar: OrderPanel,
833
- * tools: { add_pizza: { icon: "🍕", label: "Adding pizza" } },
834
- * });
835
- * ```
836
- *
837
- * @example Tier 2
838
- * ```tsx
839
- * client({ component: MyCustomApp });
840
- * ```
841
- *
842
- * @returns A {@link ClientHandle} for cleanup.
843
- * @throws If the target element is not found in the DOM.
844
- *
845
- * @public
846
- */
847
- function client(config) {
848
- const container = resolveContainer(config.target);
849
- const platformUrl = config.platformUrl ?? globalThis.location.origin + globalThis.location.pathname;
850
- const session = createSessionCore({
851
- platformUrl,
852
- onSessionId: config.onSessionId,
853
- resumeSessionId: config.resumeSessionId,
854
- WebSocket: config.WebSocket
855
- });
856
- const rootNode = config.component ? createElement(config.component) : createElement(DefaultRoot, {
857
- platformUrl,
858
- transport: config.transport,
859
- name: config.name,
860
- Sidebar: config.sidebar,
861
- sidebarWidth: config.sidebarWidth
862
- });
863
- const toolConfig = config.tools ?? {};
864
- const root = createRoot(container);
865
- flushSync(() => {
866
- root.render(createElement(ToolConfigContext.Provider, { value: toolConfig }, createElement(ThemeProvider, { value: config.theme }, createElement(SessionProvider, { value: session }, rootNode))));
867
- });
868
- const handle = {
869
- session,
870
- dispose() {
871
- root.unmount();
872
- session[Symbol.dispose]();
873
- },
874
- [Symbol.dispose]() {
875
- handle.dispose();
876
- }
877
- };
878
- return handle;
879
- }
880
- //#endregion
881
- export { pcm16ToBase64 as a, createPttRecorder as c, createUtteranceDetector as d, buildAgentUrl as f, createSyncSession as i, floatToPcm16 as l, SyncChatView as n, CAPTURE_WORKLET_MODULE_URL as o, fetchClientConfig as p, base64ToPcm16 as r, DEFAULT_SYNC_MIC_SAMPLE_RATE as s, client as t, startSyncMicrophone as u };