@alexkroman1/aai-ui 1.16.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_module-url-C_4gRVL0.js +15 -0
- package/dist/audio.d.ts +85 -11
- package/dist/audio.js +112 -83
- package/dist/chat-view-C1oJxsWz.js +129 -0
- package/dist/client-config.d.ts +6 -6
- package/dist/components/chat-view.d.ts +1 -8
- package/dist/components/chat-view.js +2 -2
- package/dist/components/console-shell.d.ts +37 -0
- package/dist/components/controls.js +1 -1
- package/dist/components/url-chips.d.ts +1 -4
- package/dist/context.d.ts +1 -1
- package/dist/context.js +1 -4
- package/dist/{controls-BbZcmnJf.js → controls-DV368uhb.js} +2 -5
- package/dist/default-client/assets/_module-url-BX0RuRU2.js +1 -0
- package/dist/default-client/assets/audio-BGHiiDY_.js +1 -0
- package/dist/default-client/assets/capture-processor-BLPsqnGl.js +90 -0
- package/dist/default-client/assets/{index-BunwIXSP.js → index-DXf7-M0Q.js} +156 -36
- package/dist/default-client/assets/index-Dv-Q5VRL.css +2 -0
- package/dist/default-client/assets/playback-processor-B_T_1KQP.js +278 -0
- package/dist/default-client/index.html +2 -2
- package/dist/define-client-Cyf1jqAl.js +150 -0
- package/dist/define-client.d.ts +0 -9
- package/dist/define-client.js +1 -1
- package/dist/index.d.ts +2 -5
- package/dist/index.js +7 -5
- package/dist/{session-core-B64kau_v.js → session-core-BM3WHeoY.js} +36 -128
- package/dist/session-core-messages.d.ts +0 -4
- package/dist/session-core-types.d.ts +1 -27
- package/dist/session-core.js +1 -1
- package/dist/types.d.ts +24 -1
- package/dist/types.js +32 -2
- package/dist/worklets/_module-url.d.ts +10 -0
- package/dist/worklets/capture-processor.d.ts +3 -3
- package/dist/worklets/capture-processor.js +35 -52
- package/dist/worklets/playback-processor.d.ts +3 -3
- package/dist/worklets/playback-processor.js +149 -26
- package/package.json +2 -2
- package/dist/chat-view-gi6FccZq.js +0 -193
- package/dist/components/sync-chat-view.d.ts +0 -18
- package/dist/components/text-controls.d.ts +0 -14
- package/dist/default-client/assets/audio-DNDZgEZp.js +0 -1
- package/dist/default-client/assets/capture-processor-UlKEKyIW.js +0 -108
- package/dist/default-client/assets/index-BogmeUln.css +0 -2
- package/dist/default-client/assets/playback-processor-C5HVRVbu.js +0 -156
- package/dist/define-client-DdpijqAu.js +0 -906
- package/dist/session-core-upload.d.ts +0 -16
- package/dist/sync-mic.d.ts +0 -93
- package/dist/sync-session.d.ts +0 -49
- package/dist/sync-vad.d.ts +0 -54
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
//#region worklets/_module-url.ts
|
|
2
|
+
/**
|
|
3
|
+
* Blob-URL module for an inlined AudioWorklet processor source.
|
|
4
|
+
*
|
|
5
|
+
* A blob URL rather than a data URI because the agent page's CSP allows
|
|
6
|
+
* `script-src blob:` but not `data:` — a data-URI module fails `addModule`
|
|
7
|
+
* with the opaque "Unable to load a worklet's module". Every inline worklet
|
|
8
|
+
* in this package must load through this helper so that constraint lives in
|
|
9
|
+
* one place.
|
|
10
|
+
*/
|
|
11
|
+
function workletModuleUrl(source) {
|
|
12
|
+
return URL.createObjectURL(new Blob([source], { type: "application/javascript" }));
|
|
13
|
+
}
|
|
14
|
+
//#endregion
|
|
15
|
+
export { workletModuleUrl as t };
|
package/dist/audio.d.ts
CHANGED
|
@@ -1,12 +1,35 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* Clamp-and-convert Float32 samples to PCM16. The one main-thread home of the
|
|
3
|
+
* asymmetric-rounding convention (negative × 0x8000, positive × 0x7fff) —
|
|
4
|
+
* the capture worklet's `accumulate` embeds the same math, which cannot
|
|
5
|
+
* import it (worklet source is a string).
|
|
6
6
|
*
|
|
7
|
-
* @
|
|
7
|
+
* @public
|
|
8
8
|
*/
|
|
9
|
-
export declare function
|
|
9
|
+
export declare function floatToPcm16(samples: Float32Array): Int16Array;
|
|
10
|
+
/**
|
|
11
|
+
* How much of one turn's playback was covered by concealment rather than
|
|
12
|
+
* received audio — the playback worklet's underrun report, in the shape
|
|
13
|
+
* WebRTC's `inbound-rtp` audio stats use, so the numbers mean the same thing
|
|
14
|
+
* here as in a `getStats()` dump.
|
|
15
|
+
*
|
|
16
|
+
* A turn with `concealmentEvents: 0` never needed its jitter buffer; a turn
|
|
17
|
+
* with a high `silentConcealedSamples` share starved for longer than
|
|
18
|
+
* concealment can plausibly cover, which is a bandwidth problem rather than a
|
|
19
|
+
* buffer-tuning one.
|
|
20
|
+
*
|
|
21
|
+
* @public
|
|
22
|
+
*/
|
|
23
|
+
export type PlaybackStats = {
|
|
24
|
+
/** Samples emitted to cover a gap, including the silent ones. */
|
|
25
|
+
concealedSamples: number;
|
|
26
|
+
/** The subset of {@link PlaybackStats.concealedSamples} that were silence. */
|
|
27
|
+
silentConcealedSamples: number;
|
|
28
|
+
/** Distinct underrun episodes, however many render quanta each spanned. */
|
|
29
|
+
concealmentEvents: number;
|
|
30
|
+
/** Episodes that lasted long enough to decay to silence. */
|
|
31
|
+
silentConcealmentEvents: number;
|
|
32
|
+
};
|
|
10
33
|
/** Configuration for creating a {@link VoiceIO} instance. */
|
|
11
34
|
export type VoiceIOOptions = {
|
|
12
35
|
/** Sample rate in Hz expected by the STT engine (e.g. 16000). */
|
|
@@ -25,13 +48,24 @@ export type VoiceIOOptions = {
|
|
|
25
48
|
* transition out of listening/speaking instead of looking healthy forever.
|
|
26
49
|
*/
|
|
27
50
|
onError?: ((err: Error) => void) | undefined;
|
|
51
|
+
/**
|
|
52
|
+
* Called at the end of any turn whose playback had to conceal a gap. Never
|
|
53
|
+
* called for a clean turn, so it can be wired straight to a warning.
|
|
54
|
+
*/
|
|
55
|
+
onPlaybackStats?: ((stats: PlaybackStats) => void) | undefined;
|
|
56
|
+
/**
|
|
57
|
+
* Called once if the microphone delivers nothing but digital silence for
|
|
58
|
+
* the first {@link MIC_SILENCE_PROBE_MS} of capture — a muted or wrong input
|
|
59
|
+
* device, which otherwise looks exactly like a user who hasn't spoken.
|
|
60
|
+
*/
|
|
61
|
+
onMicSilent?: (() => void) | undefined;
|
|
28
62
|
};
|
|
29
63
|
/**
|
|
30
64
|
* Audio I/O interface for voice capture and playback.
|
|
31
65
|
*
|
|
32
|
-
* Manages microphone capture via an AudioWorklet
|
|
33
|
-
*
|
|
34
|
-
*
|
|
66
|
+
* Manages microphone capture via an AudioWorklet and TTS audio playback
|
|
67
|
+
* through a second AudioWorklet. Implements {@link AsyncDisposable} for
|
|
68
|
+
* resource cleanup.
|
|
35
69
|
*/
|
|
36
70
|
export type VoiceIO = AsyncDisposable & {
|
|
37
71
|
/** Enqueue a PCM16 audio buffer for playback through the TTS pipeline. */
|
|
@@ -44,12 +78,52 @@ export type VoiceIO = AsyncDisposable & {
|
|
|
44
78
|
/** Release all audio resources (microphone, AudioContext, worklets). */
|
|
45
79
|
close(): Promise<void>;
|
|
46
80
|
};
|
|
81
|
+
/**
|
|
82
|
+
* Throw unless the browser honored a requested context sample rate. The
|
|
83
|
+
* requested rates are never advisory: captured audio is tagged with the
|
|
84
|
+
* requested rate on the wire, so a context running at some other rate ships
|
|
85
|
+
* audio that only sounds like speech to the wrong decoder. Every capture
|
|
86
|
+
* path must call this after context creation.
|
|
87
|
+
*/
|
|
88
|
+
export declare function assertGranted(granted: number, requested: number, side: string): void;
|
|
89
|
+
/**
|
|
90
|
+
* Release a microphone that was (or later gets) granted while another init
|
|
91
|
+
* step failed; if `getUserMedia` itself rejected, this is a no-op. Without
|
|
92
|
+
* it, a mic granted after a failed init keeps the browser's recording
|
|
93
|
+
* indicator lit with no way to turn it off.
|
|
94
|
+
*/
|
|
95
|
+
export declare function releaseStreamOnFailure(streamPromise: Promise<MediaStream>): void;
|
|
96
|
+
/** Handle to one capture worklet node (`worklets/capture-processor.ts`). */
|
|
97
|
+
export type CaptureNode = {
|
|
98
|
+
node: AudioWorkletNode;
|
|
99
|
+
/** Begin accumulating — the worklet gates capture on its start/stop protocol. */
|
|
100
|
+
start(): void;
|
|
101
|
+
/**
|
|
102
|
+
* Stop accumulating and wait (bounded by {@link CAPTURE_STOP_ACK_TIMEOUT_MS})
|
|
103
|
+
* for the 'stopped' ack that follows the worklet's final flush, so the tail
|
|
104
|
+
* of speech reaches `onChunk` before the node is torn down.
|
|
105
|
+
*/
|
|
106
|
+
stop(): Promise<void>;
|
|
107
|
+
};
|
|
108
|
+
/**
|
|
109
|
+
* Wire one capture worklet node: node construction, the chunk/silent/stopped
|
|
110
|
+
* port protocol, and the stop→ack handshake — one spelling for both the
|
|
111
|
+
* WebSocket mic and the push-to-talk recorder. No `processorOptions`: the
|
|
112
|
+
* worklet reads the context rate from its global scope (callers assert the
|
|
113
|
+
* granted rate first) and owns its own batching default — re-spelling
|
|
114
|
+
* defaults caller-side is drift.
|
|
115
|
+
*/
|
|
116
|
+
export declare function createCaptureNode(ctx: AudioContext, onChunk: (pcm16: ArrayBuffer) => void, onSilent?: () => void): CaptureNode;
|
|
47
117
|
/**
|
|
48
118
|
* Create a {@link VoiceIO} instance that captures microphone audio and
|
|
49
119
|
* plays back TTS audio using the Web Audio API.
|
|
50
120
|
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
121
|
+
* Playback runs on a context at the TTS sample rate for fidelity, and capture
|
|
122
|
+
* on its own context at the STT rate so the *browser* performs the rate
|
|
123
|
+
* conversion with its band-limited resampler. The two collapse into one
|
|
124
|
+
* context when the rates match. A browser that declines either requested rate
|
|
125
|
+
* fails init rather than falling back to converting in the worklet, which
|
|
126
|
+
* would alias.
|
|
53
127
|
*
|
|
54
128
|
* @param opts - Voice I/O configuration options.
|
|
55
129
|
* @returns A promise that resolves to a {@link VoiceIO} handle.
|
package/dist/audio.js
CHANGED
|
@@ -1,120 +1,156 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CAPTURE_STOP_ACK_TIMEOUT_MS, PLAYBACK_DONE_MAX_WAIT_MS, PLAYBACK_DONE_POLL_MS, VOICE_CAPTURE_CONSTRAINTS } from "./types.js";
|
|
2
2
|
//#region audio.ts
|
|
3
|
-
/** How often {@link VoiceIO.done} checks that the AudioContext is still rendering. */
|
|
4
|
-
const DONE_POLL_INTERVAL_MS = 1e3;
|
|
5
3
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
|
|
10
|
-
const DONE_MAX_WAIT_MS = 65e3;
|
|
11
|
-
/**
|
|
12
|
-
* Bounded wait for the capture worklet's 'stopped' ack during close(). The
|
|
13
|
-
* ack follows the final flush, so waiting for it keeps the tail of speech
|
|
14
|
-
* from being dropped; the timeout covers a dead worklet.
|
|
15
|
-
*/
|
|
16
|
-
const CAPTURE_STOP_ACK_TIMEOUT_MS = 250;
|
|
17
|
-
/**
|
|
18
|
-
* Decode an audio file (any container/codec the browser can decode) and
|
|
19
|
-
* resample it to mono PCM16 at `targetRate` — the format the server's STT
|
|
20
|
-
* side expects on the wire. Returns the raw clip; any endpointing padding
|
|
21
|
-
* is the caller's concern (the one-shot upload path needs none).
|
|
4
|
+
* Clamp-and-convert Float32 samples to PCM16. The one main-thread home of the
|
|
5
|
+
* asymmetric-rounding convention (negative × 0x8000, positive × 0x7fff) —
|
|
6
|
+
* the capture worklet's `accumulate` embeds the same math, which cannot
|
|
7
|
+
* import it (worklet source is a string).
|
|
22
8
|
*
|
|
23
|
-
* @
|
|
9
|
+
* @public
|
|
24
10
|
*/
|
|
25
|
-
|
|
26
|
-
const
|
|
27
|
-
const frames = Math.ceil(decoded.duration * targetRate);
|
|
28
|
-
const offline = new OfflineAudioContext(1, frames, targetRate);
|
|
29
|
-
const source = offline.createBufferSource();
|
|
30
|
-
source.buffer = decoded;
|
|
31
|
-
source.connect(offline.destination);
|
|
32
|
-
source.start();
|
|
33
|
-
const f32 = (await offline.startRendering()).getChannelData(0);
|
|
34
|
-
const pcm = new Int16Array(f32.length);
|
|
11
|
+
function floatToPcm16(samples) {
|
|
12
|
+
const pcm = new Int16Array(samples.length);
|
|
35
13
|
let i = 0;
|
|
36
|
-
for (const sample of
|
|
14
|
+
for (const sample of samples) {
|
|
37
15
|
const s = Math.max(-1, Math.min(1, sample));
|
|
38
16
|
pcm[i++] = s < 0 ? s * 32768 : s * 32767;
|
|
39
17
|
}
|
|
40
18
|
return pcm;
|
|
41
19
|
}
|
|
42
20
|
/**
|
|
21
|
+
* Throw unless the browser honored a requested context sample rate. The
|
|
22
|
+
* requested rates are never advisory: captured audio is tagged with the
|
|
23
|
+
* requested rate on the wire, so a context running at some other rate ships
|
|
24
|
+
* audio that only sounds like speech to the wrong decoder. Every capture
|
|
25
|
+
* path must call this after context creation.
|
|
26
|
+
*/
|
|
27
|
+
function assertGranted(granted, requested, side) {
|
|
28
|
+
if (granted === requested) return;
|
|
29
|
+
throw new Error(`Browser refused the ${side} sample rate: asked for ${requested} Hz, got ${granted} Hz`);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Release a microphone that was (or later gets) granted while another init
|
|
33
|
+
* step failed; if `getUserMedia` itself rejected, this is a no-op. Without
|
|
34
|
+
* it, a mic granted after a failed init keeps the browser's recording
|
|
35
|
+
* indicator lit with no way to turn it off.
|
|
36
|
+
*/
|
|
37
|
+
function releaseStreamOnFailure(streamPromise) {
|
|
38
|
+
streamPromise.then((s) => {
|
|
39
|
+
for (const t of s.getTracks()) t.stop();
|
|
40
|
+
}).catch(() => {});
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Wire one capture worklet node: node construction, the chunk/silent/stopped
|
|
44
|
+
* port protocol, and the stop→ack handshake — one spelling for both the
|
|
45
|
+
* WebSocket mic and the push-to-talk recorder. No `processorOptions`: the
|
|
46
|
+
* worklet reads the context rate from its global scope (callers assert the
|
|
47
|
+
* granted rate first) and owns its own batching default — re-spelling
|
|
48
|
+
* defaults caller-side is drift.
|
|
49
|
+
*/
|
|
50
|
+
function createCaptureNode(ctx, onChunk, onSilent) {
|
|
51
|
+
const node = new AudioWorkletNode(ctx, "capture-processor", {
|
|
52
|
+
channelCount: 1,
|
|
53
|
+
channelCountMode: "explicit"
|
|
54
|
+
});
|
|
55
|
+
let onStopped = null;
|
|
56
|
+
node.port.onmessage = (e) => {
|
|
57
|
+
const d = e.data;
|
|
58
|
+
if (d.event === "chunk" && d.buffer) onChunk(d.buffer);
|
|
59
|
+
else if (d.event === "silent") onSilent?.();
|
|
60
|
+
else if (d.event === "stopped") {
|
|
61
|
+
onStopped?.();
|
|
62
|
+
onStopped = null;
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
node,
|
|
67
|
+
start() {
|
|
68
|
+
node.port.postMessage({ event: "start" });
|
|
69
|
+
},
|
|
70
|
+
stop() {
|
|
71
|
+
return new Promise((resolve) => {
|
|
72
|
+
const cap = setTimeout(resolve, CAPTURE_STOP_ACK_TIMEOUT_MS);
|
|
73
|
+
onStopped = () => {
|
|
74
|
+
clearTimeout(cap);
|
|
75
|
+
resolve();
|
|
76
|
+
};
|
|
77
|
+
node.port.postMessage({ event: "stop" });
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
43
83
|
* Create a {@link VoiceIO} instance that captures microphone audio and
|
|
44
84
|
* plays back TTS audio using the Web Audio API.
|
|
45
85
|
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
86
|
+
* Playback runs on a context at the TTS sample rate for fidelity, and capture
|
|
87
|
+
* on its own context at the STT rate so the *browser* performs the rate
|
|
88
|
+
* conversion with its band-limited resampler. The two collapse into one
|
|
89
|
+
* context when the rates match. A browser that declines either requested rate
|
|
90
|
+
* fails init rather than falling back to converting in the worklet, which
|
|
91
|
+
* would alias.
|
|
48
92
|
*
|
|
49
93
|
* @param opts - Voice I/O configuration options.
|
|
50
94
|
* @returns A promise that resolves to a {@link VoiceIO} handle.
|
|
51
95
|
* @throws If microphone access is denied or AudioWorklet registration fails.
|
|
52
96
|
*/
|
|
53
97
|
async function createVoiceIO(opts) {
|
|
54
|
-
const { sttSampleRate, ttsSampleRate, captureWorkletSrc, playbackWorkletSrc, onMicData, onError } = opts;
|
|
55
|
-
const contextRate = ttsSampleRate;
|
|
98
|
+
const { sttSampleRate, ttsSampleRate, captureWorkletSrc, playbackWorkletSrc, onMicData, onError, onPlaybackStats, onMicSilent } = opts;
|
|
56
99
|
const ctx = new AudioContext({
|
|
57
|
-
sampleRate:
|
|
100
|
+
sampleRate: ttsSampleRate,
|
|
58
101
|
latencyHint: "playback"
|
|
59
102
|
});
|
|
103
|
+
const sharesContext = sttSampleRate === ttsSampleRate;
|
|
104
|
+
const capCtx = sharesContext ? ctx : new AudioContext({
|
|
105
|
+
sampleRate: sttSampleRate,
|
|
106
|
+
latencyHint: "interactive"
|
|
107
|
+
});
|
|
108
|
+
async function closeContexts() {
|
|
109
|
+
const contexts = sharesContext ? [ctx] : [ctx, capCtx];
|
|
110
|
+
await Promise.all(contexts.map((c) => c.close().catch((err) => {
|
|
111
|
+
console.warn("AudioContext close failed:", err);
|
|
112
|
+
})));
|
|
113
|
+
}
|
|
60
114
|
const streamPromise = navigator.mediaDevices.getUserMedia({ audio: {
|
|
61
115
|
deviceId: { ideal: "default" },
|
|
62
|
-
|
|
63
|
-
noiseSuppression: true,
|
|
64
|
-
autoGainControl: true,
|
|
65
|
-
voiceIsolation: true
|
|
116
|
+
...VOICE_CAPTURE_CONSTRAINTS
|
|
66
117
|
} });
|
|
67
118
|
let stream;
|
|
68
119
|
try {
|
|
69
120
|
[stream] = await Promise.all([
|
|
70
121
|
streamPromise,
|
|
71
122
|
ctx.resume(),
|
|
72
|
-
|
|
123
|
+
capCtx.resume(),
|
|
124
|
+
capCtx.audioWorklet.addModule(captureWorkletSrc),
|
|
73
125
|
ctx.audioWorklet.addModule(playbackWorkletSrc)
|
|
74
126
|
]);
|
|
127
|
+
assertGranted(capCtx.sampleRate, sttSampleRate, "capture");
|
|
128
|
+
assertGranted(ctx.sampleRate, ttsSampleRate, "playback");
|
|
75
129
|
} catch (err) {
|
|
76
|
-
streamPromise
|
|
77
|
-
|
|
78
|
-
}).catch(() => {});
|
|
79
|
-
await ctx.close().catch((err) => {
|
|
80
|
-
console.warn("AudioContext close failed:", err);
|
|
81
|
-
});
|
|
130
|
+
releaseStreamOnFailure(streamPromise);
|
|
131
|
+
await closeContexts();
|
|
82
132
|
throw err;
|
|
83
133
|
}
|
|
84
|
-
const mic =
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
processorOptions: {
|
|
89
|
-
contextRate,
|
|
90
|
-
sttSampleRate,
|
|
91
|
-
bufferSeconds: MIC_BUFFER_SECONDS
|
|
92
|
-
}
|
|
93
|
-
});
|
|
94
|
-
mic.connect(capNode);
|
|
95
|
-
capNode.onprocessorerror = () => {
|
|
134
|
+
const mic = capCtx.createMediaStreamSource(stream);
|
|
135
|
+
const capture = createCaptureNode(capCtx, onMicData, onMicSilent);
|
|
136
|
+
mic.connect(capture.node);
|
|
137
|
+
capture.node.onprocessorerror = () => {
|
|
96
138
|
const err = /* @__PURE__ */ new Error("Audio capture worklet crashed");
|
|
97
139
|
console.error("[aai-ui]", err.message);
|
|
98
140
|
onError?.(err);
|
|
99
141
|
};
|
|
100
|
-
|
|
101
|
-
let onCaptureStopped = null;
|
|
102
|
-
capNode.port.onmessage = (e) => {
|
|
103
|
-
if (e.data.event === "chunk") onMicData(e.data.buffer);
|
|
104
|
-
else if (e.data.event === "stopped") {
|
|
105
|
-
onCaptureStopped?.();
|
|
106
|
-
onCaptureStopped = null;
|
|
107
|
-
}
|
|
108
|
-
};
|
|
142
|
+
capture.start();
|
|
109
143
|
let playNode = null;
|
|
110
144
|
let onPlaybackStop = null;
|
|
111
145
|
const lifecycle = new AbortController();
|
|
112
146
|
function ensurePlayNode() {
|
|
113
147
|
if (playNode) return playNode;
|
|
114
|
-
const node = new AudioWorkletNode(ctx, "playback-processor"
|
|
148
|
+
const node = new AudioWorkletNode(ctx, "playback-processor");
|
|
115
149
|
node.connect(ctx.destination);
|
|
116
150
|
node.port.onmessage = (e) => {
|
|
117
151
|
if (e.data.event === "stop") {
|
|
152
|
+
const stats = e.data.stats;
|
|
153
|
+
if (stats && stats.concealedSamples > 0) onPlaybackStats?.(stats);
|
|
118
154
|
if (e.data.reason === "interrupt") return;
|
|
119
155
|
onPlaybackStop?.();
|
|
120
156
|
onPlaybackStop = null;
|
|
@@ -141,6 +177,7 @@ async function createVoiceIO(opts) {
|
|
|
141
177
|
},
|
|
142
178
|
done() {
|
|
143
179
|
if (!playNode) return Promise.resolve();
|
|
180
|
+
playNode.port.postMessage({ event: "done" });
|
|
144
181
|
if (ctx.state !== "running") return Promise.resolve();
|
|
145
182
|
return new Promise((resolve) => {
|
|
146
183
|
onPlaybackStop?.();
|
|
@@ -152,10 +189,9 @@ async function createVoiceIO(opts) {
|
|
|
152
189
|
};
|
|
153
190
|
const poll = setInterval(() => {
|
|
154
191
|
if (ctx.state !== "running") settle();
|
|
155
|
-
},
|
|
156
|
-
const cap = setTimeout(settle,
|
|
192
|
+
}, PLAYBACK_DONE_POLL_MS);
|
|
193
|
+
const cap = setTimeout(settle, PLAYBACK_DONE_MAX_WAIT_MS);
|
|
157
194
|
onPlaybackStop = settle;
|
|
158
|
-
playNode?.port.postMessage({ event: "done" });
|
|
159
195
|
});
|
|
160
196
|
},
|
|
161
197
|
flush() {
|
|
@@ -167,19 +203,12 @@ async function createVoiceIO(opts) {
|
|
|
167
203
|
async close() {
|
|
168
204
|
if (lifecycle.signal.aborted) return;
|
|
169
205
|
lifecycle.abort();
|
|
170
|
-
await
|
|
171
|
-
const cap = setTimeout(resolve, CAPTURE_STOP_ACK_TIMEOUT_MS);
|
|
172
|
-
onCaptureStopped = () => {
|
|
173
|
-
clearTimeout(cap);
|
|
174
|
-
resolve();
|
|
175
|
-
};
|
|
176
|
-
capNode.port.postMessage({ event: "stop" });
|
|
177
|
-
});
|
|
206
|
+
await capture.stop();
|
|
178
207
|
mic.disconnect();
|
|
179
|
-
|
|
208
|
+
capture.node.disconnect();
|
|
180
209
|
if (playNode) playNode.disconnect();
|
|
181
210
|
for (const t of stream.getTracks()) t.stop();
|
|
182
|
-
await
|
|
211
|
+
await closeContexts();
|
|
183
212
|
},
|
|
184
213
|
async [Symbol.asyncDispose]() {
|
|
185
214
|
await io.close();
|
|
@@ -188,4 +217,4 @@ async function createVoiceIO(opts) {
|
|
|
188
217
|
return io;
|
|
189
218
|
}
|
|
190
219
|
//#endregion
|
|
191
|
-
export { createVoiceIO,
|
|
220
|
+
export { assertGranted, createCaptureNode, createVoiceIO, floatToPcm16, releaseStreamOnFailure };
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { useSessionSelector, useTheme } from "./context.js";
|
|
2
|
+
import { r as TEXT_FAINT, t as ERROR_COLOR } from "./_colors-DYX7XRTr.js";
|
|
3
|
+
import { t as AaiLogo } from "./aai-logo-B8lDmsut.js";
|
|
4
|
+
import { t as Eyebrow } from "./eyebrow-C6ZFuiz6.js";
|
|
5
|
+
import { t as Controls } from "./controls-DV368uhb.js";
|
|
6
|
+
import { MessageList } from "./components/message-list.js";
|
|
7
|
+
import clsx from "clsx";
|
|
8
|
+
import { jsx, jsxs } from "react/jsx-runtime";
|
|
9
|
+
//#region components/console-shell.tsx
|
|
10
|
+
/** @jsxImportSource react */
|
|
11
|
+
/**
|
|
12
|
+
* Indicator dot color per state, on the light refresh palette.
|
|
13
|
+
*
|
|
14
|
+
* @internal
|
|
15
|
+
*/
|
|
16
|
+
function stateColor(state, primary) {
|
|
17
|
+
switch (state) {
|
|
18
|
+
case "listening":
|
|
19
|
+
case "speaking":
|
|
20
|
+
case "ready": return primary;
|
|
21
|
+
case "thinking": return "#B98900";
|
|
22
|
+
case "error": return ERROR_COLOR;
|
|
23
|
+
default: return TEXT_FAINT;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The design-system "console" chrome for the chat shell:
|
|
28
|
+
* a 760px column on the cream page with a header
|
|
29
|
+
* (logo + live-status eyebrow), an optional error banner, the main content
|
|
30
|
+
* on a raised white card, and a footer row beneath it.
|
|
31
|
+
*
|
|
32
|
+
* Extracted so the two default surfaces stay visually identical by
|
|
33
|
+
* construction — they used to be hand-copied down to the same `boxShadow`
|
|
34
|
+
* literal, and drifted.
|
|
35
|
+
*
|
|
36
|
+
* @internal
|
|
37
|
+
*/
|
|
38
|
+
function ConsoleShell({ icon, title, state, pulsing, error, children, footer, className }) {
|
|
39
|
+
const theme = useTheme();
|
|
40
|
+
return /* @__PURE__ */ jsxs("div", {
|
|
41
|
+
className: clsx("flex flex-col h-screen w-full max-w-190 mx-auto box-border px-6 py-8 gap-5 font-aai text-sm", className),
|
|
42
|
+
style: {
|
|
43
|
+
background: theme.bg,
|
|
44
|
+
color: theme.text
|
|
45
|
+
},
|
|
46
|
+
children: [
|
|
47
|
+
/* @__PURE__ */ jsxs("div", {
|
|
48
|
+
className: "flex items-center justify-between shrink-0",
|
|
49
|
+
children: [/* @__PURE__ */ jsxs("div", {
|
|
50
|
+
className: "flex items-center gap-3 min-w-0",
|
|
51
|
+
children: [icon ?? /* @__PURE__ */ jsx(AaiLogo, { size: 22 }), title && /* @__PURE__ */ jsx("span", {
|
|
52
|
+
className: "font-aai-serif text-[22px] leading-[1.2] font-normal truncate",
|
|
53
|
+
style: { color: theme.text },
|
|
54
|
+
children: title
|
|
55
|
+
})]
|
|
56
|
+
}), /* @__PURE__ */ jsxs(Eyebrow, {
|
|
57
|
+
className: "shrink-0",
|
|
58
|
+
"data-state": state,
|
|
59
|
+
children: [/* @__PURE__ */ jsx("span", {
|
|
60
|
+
className: "w-[7px] h-[7px] rounded-full",
|
|
61
|
+
style: {
|
|
62
|
+
background: stateColor(state, theme.primary),
|
|
63
|
+
animation: pulsing ? "aai-pulse 1.6s ease-in-out infinite" : "none"
|
|
64
|
+
}
|
|
65
|
+
}), state]
|
|
66
|
+
})]
|
|
67
|
+
}),
|
|
68
|
+
error && /* @__PURE__ */ jsx("div", {
|
|
69
|
+
className: "px-3.5 py-2.5 rounded-aai border text-[13px] leading-[130%] shrink-0",
|
|
70
|
+
style: {
|
|
71
|
+
borderColor: "rgba(179,38,30,0.35)",
|
|
72
|
+
background: "rgba(179,38,30,0.06)",
|
|
73
|
+
color: "#B3261E"
|
|
74
|
+
},
|
|
75
|
+
children: error
|
|
76
|
+
}),
|
|
77
|
+
/* @__PURE__ */ jsx("div", {
|
|
78
|
+
className: "flex flex-col flex-1 min-h-0 border rounded-lg overflow-hidden",
|
|
79
|
+
style: {
|
|
80
|
+
background: theme.surface,
|
|
81
|
+
borderColor: theme.border,
|
|
82
|
+
boxShadow: "0 1px 3px 0 rgb(20 18 12 / 0.06)"
|
|
83
|
+
},
|
|
84
|
+
children
|
|
85
|
+
}),
|
|
86
|
+
footer
|
|
87
|
+
]
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
//#endregion
|
|
91
|
+
//#region components/chat-view.tsx
|
|
92
|
+
const PULSING_STATES = /* @__PURE__ */ new Set(["listening", "speaking"]);
|
|
93
|
+
/**
|
|
94
|
+
* The main chat interface for a voice agent session — the design-system
|
|
95
|
+
* "voice agent console": a 760px column on the cream page with a header
|
|
96
|
+
* (logo + live-status eyebrow), the conversation on a raised white card,
|
|
97
|
+
* and the session controls beneath it.
|
|
98
|
+
*
|
|
99
|
+
* Must be rendered inside a {@link SessionProvider}.
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* ```tsx
|
|
103
|
+
* <StartScreen icon="🍕" title="Pizza Palace">
|
|
104
|
+
* <ChatView />
|
|
105
|
+
* </StartScreen>
|
|
106
|
+
* ```
|
|
107
|
+
*
|
|
108
|
+
* @param icon - Optional element rendered in place of the logo in the header.
|
|
109
|
+
* @param title - Optional title string for the header.
|
|
110
|
+
* @param className - Additional CSS class names applied to the root element.
|
|
111
|
+
*
|
|
112
|
+
* @public
|
|
113
|
+
*/
|
|
114
|
+
function ChatView({ icon, title, className }) {
|
|
115
|
+
const state = useSessionSelector((s) => s.state);
|
|
116
|
+
const error = useSessionSelector((s) => s.error);
|
|
117
|
+
return /* @__PURE__ */ jsx(ConsoleShell, {
|
|
118
|
+
icon,
|
|
119
|
+
title,
|
|
120
|
+
state,
|
|
121
|
+
pulsing: PULSING_STATES.has(state),
|
|
122
|
+
error: error?.message,
|
|
123
|
+
className,
|
|
124
|
+
footer: /* @__PURE__ */ jsx(Controls, {}),
|
|
125
|
+
children: /* @__PURE__ */ jsx(MessageList, {})
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
//#endregion
|
|
129
|
+
export { ChatView as t };
|
package/dist/client-config.d.ts
CHANGED
|
@@ -2,15 +2,15 @@
|
|
|
2
2
|
* Pre-connection client-config lookup.
|
|
3
3
|
*
|
|
4
4
|
* `GET client-config` (relative to the agent's base URL — see
|
|
5
|
-
* `sdk/client-config.ts` in `@alexkroman1/aai`)
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
5
|
+
* `sdk/client-config.ts` in `@alexkroman1/aai`) gives the default client the
|
|
6
|
+
* agent's display name and greeting before any connection exists. Every
|
|
7
|
+
* failure path — network error, 404 from an older server, malformed body —
|
|
8
|
+
* degrades to the empty default, so this lookup can never break an existing
|
|
9
|
+
* agent.
|
|
10
10
|
*/
|
|
11
11
|
import { type ClientConfigResponse } from "@alexkroman1/aai/protocol";
|
|
12
12
|
export type { ClientConfigResponse } from "@alexkroman1/aai/protocol";
|
|
13
13
|
/** Resolve a relative endpoint path against the agent's base URL. */
|
|
14
14
|
export declare function buildAgentUrl(platformUrl: string, endpointPath: string): URL;
|
|
15
|
-
/** Fetch the agent's client config; any failure yields the
|
|
15
|
+
/** Fetch the agent's client config; any failure yields the agent default. */
|
|
16
16
|
export declare function fetchClientConfig(platformUrl: string, fetchFn?: typeof globalThis.fetch): Promise<ClientConfigResponse>;
|
|
@@ -1,12 +1,5 @@
|
|
|
1
|
+
/** @jsxImportSource react */
|
|
1
2
|
import type { ReactNode } from "react";
|
|
2
|
-
import type { AgentState } from "../types.ts";
|
|
3
|
-
/**
|
|
4
|
-
* Indicator dot color per state, on the light refresh palette. Shared with
|
|
5
|
-
* the sync-transport chat shell's status eyebrow.
|
|
6
|
-
*
|
|
7
|
-
* @internal
|
|
8
|
-
*/
|
|
9
|
-
export declare function stateColor(state: AgentState, primary: string): string;
|
|
10
3
|
/**
|
|
11
4
|
* The main chat interface for a voice agent session — the design-system
|
|
12
5
|
* "voice agent console": a 760px column on the cream page with a header
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import "../context.js";
|
|
2
|
-
import {
|
|
3
|
-
export { ChatView
|
|
2
|
+
import { t as ChatView } from "../chat-view-C1oJxsWz.js";
|
|
3
|
+
export { ChatView };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { ReactNode } from "react";
|
|
2
|
+
import type { AgentState } from "../types.ts";
|
|
3
|
+
/**
|
|
4
|
+
* Indicator dot color per state, on the light refresh palette.
|
|
5
|
+
*
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export declare function stateColor(state: AgentState, primary: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* The design-system "console" chrome for the chat shell:
|
|
11
|
+
* a 760px column on the cream page with a header
|
|
12
|
+
* (logo + live-status eyebrow), an optional error banner, the main content
|
|
13
|
+
* on a raised white card, and a footer row beneath it.
|
|
14
|
+
*
|
|
15
|
+
* Extracted so the two default surfaces stay visually identical by
|
|
16
|
+
* construction — they used to be hand-copied down to the same `boxShadow`
|
|
17
|
+
* literal, and drifted.
|
|
18
|
+
*
|
|
19
|
+
* @internal
|
|
20
|
+
*/
|
|
21
|
+
export declare function ConsoleShell({ icon, title, state, pulsing, error, children, footer, className, }: {
|
|
22
|
+
/** Element rendered in place of the logo in the header. */
|
|
23
|
+
icon?: ReactNode | undefined;
|
|
24
|
+
/** Title string for the header. */
|
|
25
|
+
title?: string | undefined;
|
|
26
|
+
/** Live status shown in the header eyebrow. */
|
|
27
|
+
state: AgentState;
|
|
28
|
+
/** Whether the status dot pulses. */
|
|
29
|
+
pulsing: boolean;
|
|
30
|
+
/** Error banner text; `null`/`undefined` hides the banner. */
|
|
31
|
+
error?: string | null | undefined;
|
|
32
|
+
/** Card content. */
|
|
33
|
+
children: ReactNode;
|
|
34
|
+
/** Row rendered beneath the card (controls). */
|
|
35
|
+
footer: ReactNode;
|
|
36
|
+
className?: string | undefined;
|
|
37
|
+
}): ReactNode;
|
|
@@ -4,9 +4,6 @@
|
|
|
4
4
|
* The label is what makes a pair of these readable — on its own a bare URL
|
|
5
5
|
* leaves you guessing whether it's the page or the socket.
|
|
6
6
|
*
|
|
7
|
-
* Exported for the sync-transport shell, which labels its HTTP endpoint
|
|
8
|
-
* without a session snapshot to read from.
|
|
9
|
-
*
|
|
10
7
|
* @internal
|
|
11
8
|
*/
|
|
12
9
|
export declare function UrlChip({ label, url, hint, testId, className, }: {
|
|
@@ -37,7 +34,7 @@ export declare function ApiUrlChip({ className }: {
|
|
|
37
34
|
* The UI and API URLs side by side. They answer the same question — "how do I
|
|
38
35
|
* reach this agent?" — so they belong together and each needs its label to be
|
|
39
36
|
* told apart. Rendered by the default shell in every session mode (S2S,
|
|
40
|
-
* pipeline
|
|
37
|
+
* pipeline).
|
|
41
38
|
*
|
|
42
39
|
* @public
|
|
43
40
|
*/
|
package/dist/context.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ export declare function SessionProvider({ value, children }: {
|
|
|
7
7
|
}): import("react").FunctionComponentElement<import("react").ProviderProps<SessionCore | null>>;
|
|
8
8
|
/** The session snapshot merged with the core's control methods. Method
|
|
9
9
|
* signatures come from {@link SessionCore} — one source of truth. */
|
|
10
|
-
export type Session = SessionSnapshot & Pick<SessionCore, "start" | "cancel" | "resetState" | "reset" | "disconnect" | "toggle"
|
|
10
|
+
export type Session = SessionSnapshot & Pick<SessionCore, "start" | "cancel" | "resetState" | "reset" | "disconnect" | "toggle">;
|
|
11
11
|
/**
|
|
12
12
|
* Return the raw {@link SessionCore} from context without subscribing to
|
|
13
13
|
* snapshot changes. Useful for accessing stable methods (`start`, `toggle`,
|