@alexkroman1/aai-ui 1.9.2 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{_colors-DJUordGv.js → _colors-DYX7XRTr.js} +8 -0
- package/dist/{_utils-5cs73OrA.js → _utils-CN1yVgYS.js} +2 -5
- package/dist/audio.d.ts +6 -0
- package/dist/audio.js +62 -9
- package/dist/{chat-view-C1XbqDk0.js → chat-view-u3yBAlig.js} +16 -14
- package/dist/components/_colors.d.ts +0 -4
- package/dist/components/chat-view.js +1 -1
- package/dist/components/controls.d.ts +2 -2
- package/dist/components/controls.js +1 -1
- package/dist/components/message-list.d.ts +2 -2
- package/dist/components/message-list.js +70 -64
- package/dist/components/start-screen.js +1 -1
- package/dist/components/text-controls.d.ts +2 -2
- package/dist/components/tool-call-block.d.ts +6 -2
- package/dist/components/tool-call-block.js +1 -1
- package/dist/context.d.ts +3 -11
- package/dist/context.js +9 -22
- package/dist/{controls-BngrPbOC.js → controls-B2EPUJDU.js} +12 -9
- package/dist/default-client/assets/audio-COt0_Zvp.js +1 -0
- package/dist/default-client/assets/{capture-processor-C19oBn4L.js → capture-processor-UlKEKyIW.js} +4 -1
- package/dist/default-client/assets/index-B2bISVbm.css +2 -0
- package/dist/default-client/assets/index-DtsWxxc_.js +94 -0
- package/dist/default-client/assets/{playback-processor-BtlzAH78.js → playback-processor-C5HVRVbu.js} +11 -4
- package/dist/default-client/index.html +2 -2
- package/dist/define-client.js +3 -3
- package/dist/hooks.js +5 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +5 -5
- package/dist/{session-core-D3NDaySY.js → session-core-Cfz7PPhN.js} +248 -111
- package/dist/session-core-audio-setup.d.ts +36 -0
- package/dist/session-core-messages.d.ts +12 -0
- package/dist/session-core-reconnect.d.ts +19 -0
- package/dist/session-core-url.d.ts +2 -0
- package/dist/session-core.js +1 -1
- package/dist/{tool-call-block-7f1GTG-P.js → tool-call-block-DIxpG8GM.js} +11 -6
- package/dist/types.d.ts +11 -3
- package/dist/worklets/capture-processor.d.ts +1 -1
- package/dist/worklets/capture-processor.js +4 -1
- package/dist/worklets/playback-processor.d.ts +1 -1
- package/dist/worklets/playback-processor.js +11 -4
- package/package.json +8 -5
- package/dist/default-client/assets/audio-Cs-6t_Wd.js +0 -1
- package/dist/default-client/assets/index-Bf4ZTNcx.js +0 -73
- package/dist/default-client/assets/index-Bzlh9i7w.css +0 -2
|
@@ -26,6 +26,10 @@ type MessageHandlerDeps = {
|
|
|
26
26
|
getSnapshot: () => SessionSnapshot;
|
|
27
27
|
updateState: (partial: Partial<SessionSnapshot>) => void;
|
|
28
28
|
conn: ConnState;
|
|
29
|
+
/** Invalidate any in-flight file upload (the upload sender's `discard`). */
|
|
30
|
+
discardUpload: () => void;
|
|
31
|
+
/** Release the microphone/VoiceIO (the session core's `cleanupAudio`). */
|
|
32
|
+
cleanupAudio: () => void;
|
|
29
33
|
};
|
|
30
34
|
type MessageHandlers = {
|
|
31
35
|
/**
|
|
@@ -38,6 +42,14 @@ type MessageHandlers = {
|
|
|
38
42
|
* otherwise `undefined`.
|
|
39
43
|
*/
|
|
40
44
|
handleMessage(data: unknown): SessionConfigMessage | undefined;
|
|
45
|
+
/**
|
|
46
|
+
* Wait for `io`'s playback queue to drain, then transition to `"listening"`
|
|
47
|
+
* — guarded by the same turn-boundary generation the live `audio_done`
|
|
48
|
+
* path uses. `initAudioCapture` routes the pre-init greeting replay
|
|
49
|
+
* through this so a barge-in mid-greeting can't be stomped by the
|
|
50
|
+
* replayed completion resolving late.
|
|
51
|
+
*/
|
|
52
|
+
settleWhenAudioDrained(io: NonNullable<ConnState["voiceIO"]>): void;
|
|
41
53
|
};
|
|
42
54
|
/**
|
|
43
55
|
* Create the server→client message handlers for one session core.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Automatic reconnection for the browser session socket, built on
|
|
3
|
+
* partysocket's `ReconnectingWebSocket`. Kept out of `session-core.ts` so
|
|
4
|
+
* the state machine there reads as protocol logic, not socket plumbing.
|
|
5
|
+
*/
|
|
6
|
+
import ReconnectingWebSocket from "partysocket/ws";
|
|
7
|
+
/**
|
|
8
|
+
* Open partysocket's reconnecting WebSocket. The URL is a *provider*,
|
|
9
|
+
* re-evaluated on every attempt, so each retry picks up the current resume
|
|
10
|
+
* URL rather than the one the session started with.
|
|
11
|
+
*/
|
|
12
|
+
export declare function openReconnectingSocket(urlProvider: () => string): ReconnectingWebSocket;
|
|
13
|
+
/**
|
|
14
|
+
* True while `socket` is a reconnecting socket that will retry after the
|
|
15
|
+
* close event currently being handled. partysocket schedules the retry
|
|
16
|
+
* *before* dispatching `close`, so `retryCount` already names the attempt
|
|
17
|
+
* just scheduled — at `maxRetries` it has given up.
|
|
18
|
+
*/
|
|
19
|
+
export declare function reconnectPending(socket: unknown): boolean;
|
package/dist/session-core.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createSessionCore } from "./session-core-
|
|
1
|
+
import { t as createSessionCore } from "./session-core-Cfz7PPhN.js";
|
|
2
2
|
export { createSessionCore };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { useTheme } from "./context.js";
|
|
2
|
-
import { r as TEXT_FAINT } from "./_colors-
|
|
2
|
+
import { r as TEXT_FAINT } from "./_colors-DYX7XRTr.js";
|
|
3
3
|
import { t as Eyebrow } from "./eyebrow-C6ZFuiz6.js";
|
|
4
|
-
import { n as tryParseJSON, t as truncate } from "./_utils-
|
|
4
|
+
import { n as tryParseJSON, t as truncate } from "./_utils-CN1yVgYS.js";
|
|
5
5
|
import clsx from "clsx";
|
|
6
|
-
import { createContext, useContext, useMemo, useState } from "react";
|
|
6
|
+
import { createContext, memo, useContext, useMemo, useState } from "react";
|
|
7
7
|
import { jsx, jsxs } from "react/jsx-runtime";
|
|
8
8
|
//#region components/tool-config-context.ts
|
|
9
9
|
/**
|
|
@@ -40,6 +40,10 @@ function formatResult(result) {
|
|
|
40
40
|
* While the tool call is pending a shimmer animation is shown. Once
|
|
41
41
|
* complete, clicking the row expands the formatted JSON result.
|
|
42
42
|
*
|
|
43
|
+
* Memoized: tool-call objects are referentially stable across session
|
|
44
|
+
* snapshots and rows are keyed on the stable `callId`, so a list update only
|
|
45
|
+
* re-renders the rows whose tool call actually changed.
|
|
46
|
+
*
|
|
43
47
|
* @example
|
|
44
48
|
* ```tsx
|
|
45
49
|
* <ToolCallBlock toolCall={toolCall} />
|
|
@@ -50,10 +54,11 @@ function formatResult(result) {
|
|
|
50
54
|
*
|
|
51
55
|
* @public
|
|
52
56
|
*/
|
|
53
|
-
function ToolCallBlock({ toolCall, className }) {
|
|
57
|
+
const ToolCallBlock = memo(function ToolCallBlock({ toolCall, className }) {
|
|
54
58
|
const [isOpen, setIsOpen] = useState(false);
|
|
55
59
|
const theme = useTheme();
|
|
56
|
-
const
|
|
60
|
+
const toolConfig = useToolConfig();
|
|
61
|
+
const config = Object.hasOwn(toolConfig, toolCall.name) ? toolConfig[toolCall.name] : void 0;
|
|
57
62
|
const isPending = toolCall.status === "pending";
|
|
58
63
|
const title = config?.label || toolCall.name;
|
|
59
64
|
const icon = config?.icon;
|
|
@@ -128,6 +133,6 @@ function ToolCallBlock({ toolCall, className }) {
|
|
|
128
133
|
})]
|
|
129
134
|
})]
|
|
130
135
|
});
|
|
131
|
-
}
|
|
136
|
+
});
|
|
132
137
|
//#endregion
|
|
133
138
|
export { ToolConfigContext as n, ToolCallBlock as t };
|
package/dist/types.d.ts
CHANGED
|
@@ -81,17 +81,25 @@ export type VoiceSessionOptions = {
|
|
|
81
81
|
* Called when the server sends a session ID in the config message.
|
|
82
82
|
* Use this to store the ID (e.g. in localStorage) for reconnection
|
|
83
83
|
* via `resumeSessionId`.
|
|
84
|
+
*
|
|
85
|
+
* Treat session IDs as sensitive: whoever holds one can resume the
|
|
86
|
+
* session and read its replayed history. They travel as a WebSocket
|
|
87
|
+
* query parameter (browsers cannot set WS headers), so they may appear
|
|
88
|
+
* in proxy and server access logs — don't put them in shared URLs.
|
|
84
89
|
*/
|
|
85
90
|
onSessionId?: ((sessionId: string) => void) | undefined;
|
|
86
91
|
/**
|
|
87
92
|
* Session ID from a previous connection. When set, the server will
|
|
88
93
|
* attempt to restore persisted session state (if the agent has
|
|
89
|
-
* `persistence` enabled).
|
|
94
|
+
* `persistence` enabled). Sensitive — see {@link onSessionId}.
|
|
90
95
|
*/
|
|
91
96
|
resumeSessionId?: string | undefined;
|
|
92
97
|
/**
|
|
93
|
-
* WebSocket constructor override.
|
|
94
|
-
*
|
|
98
|
+
* WebSocket constructor override. Primarily useful for testing with a mock
|
|
99
|
+
* WebSocket. When omitted, the session uses a reconnecting WebSocket
|
|
100
|
+
* (partysocket) that retries with exponential backoff after an unexpected
|
|
101
|
+
* close and resumes the session; an injected constructor is used as-is and
|
|
102
|
+
* never reconnects on its own.
|
|
95
103
|
*/
|
|
96
104
|
WebSocket?: WebSocketConstructor | undefined;
|
|
97
105
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
/** Raw worklet source — exported so tests can evaluate the processor directly. */
|
|
2
|
-
export declare const captureProcessorSource = "\nclass CaptureProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n this.recording = false;\n const opts = options.processorOptions || {};\n this.fromRate = opts.contextRate || sampleRate;\n this.toRate = opts.sttSampleRate || sampleRate;\n this.ratio = this.fromRate / this.toRate;\n this.needsResample = this.fromRate !== this.toRate;\n // Streaming-resampler state carried across process() blocks so the output\n // clock doesn't drift and there's no discontinuity at 128-sample block\n // boundaries. `prev` is the previous block's last input sample (used to\n // interpolate across the boundary); `pos` is the fractional read position,\n // in input samples, relative to an extended [prev, ...input] frame. Start\n // at 1 so the first output is input[0] (not the bogus initial prev).\n this.prev = 0;\n this.pos = 1;\n // Output buffer reused across process() calls -- the render quantum is a\n // fixed size, so allocating per call would churn the realtime audio thread.\n this.resampleBuf = null;\n // Int16 accumulation buffer: flushed to the main thread as one transferred\n // ArrayBuffer once ~bufferSeconds of output samples are batched. Sized 2x\n // the flush target so a whole render quantum always fits before flushing.\n this.targetSamples = Math.max(1, Math.round(this.toRate * (opts.bufferSeconds || 0.1)));\n this.pending = new Int16Array(this.targetSamples * 2);\n this.pendingLen = 0;\n this.port.onmessage = (e) => {\n if (e.data.event === 'start') this.recording = true;\n else if (e.data.event === 'stop') {\n // Final flush so the tail of speech isn't dropped on close.\n this.flush();\n this.recording = false;\n }\n };\n }\n\n resample(input) {\n const ratio = this.ratio;\n const n = input.length;\n if (n === 0) return new Float32Array(0);\n // At most ceil(n / ratio) + 1 outputs per block (pos starts in [0, ratio)).\n const maxLen = Math.ceil(n / ratio) + 1;\n if (this.resampleBuf === null || this.resampleBuf.length < maxLen) {\n this.resampleBuf = new Float32Array(maxLen);\n }\n const out = this.resampleBuf;\n // Extended frame: index 0 = prev (previous block's last sample),\n // index k>=1 = input[k-1]. Interpolate at fractional positions stepping by\n // ratio, carrying `pos` across calls to preserve the sample clock.\n let count = 0;\n let pos = this.pos;\n while (pos < n) {\n const idx = pos | 0;\n const frac = pos - idx;\n const a = idx === 0 ? this.prev : input[idx - 1];\n const b = input[idx];\n out[count++] = a + frac * (b - a);\n pos += ratio;\n }\n // Shift the origin to this block's last sample for the next call.\n this.prev = input[n - 1];\n this.pos = pos - n;\n return out.subarray(0, count);\n }\n\n // Convert Float32 -> Int16 and append to the pending batch. Writes through\n // an Int16Array directly (assignment truncates like DataView.setInt16).\n accumulate(samples) {\n let buf = this.pending;\n if (this.pendingLen + samples.length > buf.length) {\n // Defensive: only reachable if a render quantum outproduces the 1x\n // headroom above the flush target (never with 128-sample quanta).\n const grown = new Int16Array((this.pendingLen + samples.length) * 2);\n grown.set(buf.subarray(0, this.pendingLen));\n this.pending = grown;\n buf = grown;\n }\n for (let i = 0; i < samples.length; i++) {\n const s = Math.max(-1, Math.min(1, samples[i]));\n buf[this.pendingLen++] = s < 0 ? s * 0x8000 : s * 0x7fff;\n }\n }\n\n // Post the batched samples as one transferred ArrayBuffer and reset.\n flush() {\n if (this.pendingLen === 0) return;\n const buffer = this.pending.buffer.slice(0, this.pendingLen * 2);\n this.pendingLen = 0;\n this.port.postMessage({ event: 'chunk', buffer }, [buffer]);\n }\n\n process(inputs) {\n const input = inputs[0];\n if (!input || !input[0] || !this.recording) return true;\n\n const samples = this.needsResample ? this.resample(input[0]) : input[0];\n this.accumulate(samples);\n if (this.pendingLen >= this.targetSamples) this.flush();\n return true;\n }\n}\n\nregisterProcessor('capture-processor', CaptureProcessor);\n";
|
|
2
|
+
export declare const captureProcessorSource = "\nclass CaptureProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n this.recording = false;\n const opts = options.processorOptions || {};\n this.fromRate = opts.contextRate || sampleRate;\n this.toRate = opts.sttSampleRate || sampleRate;\n this.ratio = this.fromRate / this.toRate;\n this.needsResample = this.fromRate !== this.toRate;\n // Streaming-resampler state carried across process() blocks so the output\n // clock doesn't drift and there's no discontinuity at 128-sample block\n // boundaries. `prev` is the previous block's last input sample (used to\n // interpolate across the boundary); `pos` is the fractional read position,\n // in input samples, relative to an extended [prev, ...input] frame. Start\n // at 1 so the first output is input[0] (not the bogus initial prev).\n this.prev = 0;\n this.pos = 1;\n // Output buffer reused across process() calls -- the render quantum is a\n // fixed size, so allocating per call would churn the realtime audio thread.\n this.resampleBuf = null;\n // Int16 accumulation buffer: flushed to the main thread as one transferred\n // ArrayBuffer once ~bufferSeconds of output samples are batched. Sized 2x\n // the flush target so a whole render quantum always fits before flushing.\n this.targetSamples = Math.max(1, Math.round(this.toRate * (opts.bufferSeconds || 0.1)));\n this.pending = new Int16Array(this.targetSamples * 2);\n this.pendingLen = 0;\n this.port.onmessage = (e) => {\n if (e.data.event === 'start') this.recording = true;\n else if (e.data.event === 'stop') {\n // Final flush so the tail of speech isn't dropped on close, then ack\n // so the host knows the tail chunk (if any) has been posted and it is\n // safe to tear the context down.\n this.flush();\n this.recording = false;\n this.port.postMessage({ event: 'stopped' });\n }\n };\n }\n\n resample(input) {\n const ratio = this.ratio;\n const n = input.length;\n if (n === 0) return new Float32Array(0);\n // At most ceil(n / ratio) + 1 outputs per block (pos starts in [0, ratio)).\n const maxLen = Math.ceil(n / ratio) + 1;\n if (this.resampleBuf === null || this.resampleBuf.length < maxLen) {\n this.resampleBuf = new Float32Array(maxLen);\n }\n const out = this.resampleBuf;\n // Extended frame: index 0 = prev (previous block's last sample),\n // index k>=1 = input[k-1]. Interpolate at fractional positions stepping by\n // ratio, carrying `pos` across calls to preserve the sample clock.\n let count = 0;\n let pos = this.pos;\n while (pos < n) {\n const idx = pos | 0;\n const frac = pos - idx;\n const a = idx === 0 ? this.prev : input[idx - 1];\n const b = input[idx];\n out[count++] = a + frac * (b - a);\n pos += ratio;\n }\n // Shift the origin to this block's last sample for the next call.\n this.prev = input[n - 1];\n this.pos = pos - n;\n return out.subarray(0, count);\n }\n\n // Convert Float32 -> Int16 and append to the pending batch. Writes through\n // an Int16Array directly (assignment truncates like DataView.setInt16).\n accumulate(samples) {\n let buf = this.pending;\n if (this.pendingLen + samples.length > buf.length) {\n // Defensive: only reachable if a render quantum outproduces the 1x\n // headroom above the flush target (never with 128-sample quanta).\n const grown = new Int16Array((this.pendingLen + samples.length) * 2);\n grown.set(buf.subarray(0, this.pendingLen));\n this.pending = grown;\n buf = grown;\n }\n for (let i = 0; i < samples.length; i++) {\n const s = Math.max(-1, Math.min(1, samples[i]));\n buf[this.pendingLen++] = s < 0 ? s * 0x8000 : s * 0x7fff;\n }\n }\n\n // Post the batched samples as one transferred ArrayBuffer and reset.\n flush() {\n if (this.pendingLen === 0) return;\n const buffer = this.pending.buffer.slice(0, this.pendingLen * 2);\n this.pendingLen = 0;\n this.port.postMessage({ event: 'chunk', buffer }, [buffer]);\n }\n\n process(inputs) {\n const input = inputs[0];\n if (!input || !input[0] || !this.recording) return true;\n\n const samples = this.needsResample ? this.resample(input[0]) : input[0];\n this.accumulate(samples);\n if (this.pendingLen >= this.targetSamples) this.flush();\n return true;\n }\n}\n\nregisterProcessor('capture-processor', CaptureProcessor);\n";
|
|
3
3
|
declare const src: string;
|
|
4
4
|
export default src;
|
|
@@ -29,9 +29,12 @@ class CaptureProcessor extends AudioWorkletProcessor {
|
|
|
29
29
|
this.port.onmessage = (e) => {
|
|
30
30
|
if (e.data.event === 'start') this.recording = true;
|
|
31
31
|
else if (e.data.event === 'stop') {
|
|
32
|
-
// Final flush so the tail of speech isn't dropped on close
|
|
32
|
+
// Final flush so the tail of speech isn't dropped on close, then ack
|
|
33
|
+
// so the host knows the tail chunk (if any) has been posted and it is
|
|
34
|
+
// safe to tear the context down.
|
|
33
35
|
this.flush();
|
|
34
36
|
this.recording = false;
|
|
37
|
+
this.port.postMessage({ event: 'stopped' });
|
|
35
38
|
}
|
|
36
39
|
};
|
|
37
40
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
/** Raw worklet source — exported so tests can evaluate the processor directly. */
|
|
2
|
-
export declare const playbackProcessorSource = "\nclass PlaybackProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n const rate = options.processorOptions?.sampleRate ?? 24000;\n // Wait for ~400ms of audio before starting.\n // If 'done' arrives first (short utterance), start immediately.\n this.jitterSamples = Math.floor(rate * 0.4);\n // Float32 ring buffer \u2014 60s at the context sample rate. Allocated once for\n // the node's lifetime; per-turn state resets via resetTurn(). writePos and\n // readPos are absolute (monotonic) sample counts; the buffer is indexed\n // modulo capacity so a reply longer than 60s keeps playing instead of\n // writing past the end and going silent.\n this.capacity = rate * 60;\n this.samples = new Float32Array(this.capacity);\n // Platform endianness probe: the wire format is PCM16 little-endian, so\n // the Int16Array fast path in ingestBytes is only valid on LE hosts\n // (every shipping browser target; the DataView path is the fallback).\n this.littleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;\n this.resetTurn();\n\n this.port.onmessage = (e) => {\n const d = e.data;\n if (d.event === 'write') {\n this.ingestBytes(d.buffer);\n } else if (d.event === 'interrupt') {\n this.interrupted = true;\n } else if (d.event === 'done') {\n this.isDone = true;\n }\n };\n }\n\n // Reset per-turn state so the node is reusable across replies without\n // reallocating the sample buffer or re-instantiating the worklet.\n resetTurn() {\n this.interrupted = false;\n this.isDone = false;\n this.playing = false;\n // Carry-over byte for split samples across chunks\n this.carry = null;\n this.writePos = 0;\n this.readPos = 0;\n }\n\n // End the current turn: notify the host and rearm for the next reply.\n // Must NOT return false from process() \u2014 a processor that stops is dead\n // for good, forcing a new node (and buffer) per reply.\n stopTurn() {\n this.port.postMessage({ event: 'stop' });\n this.resetTurn();\n }\n\n ingestBytes(uint8) {\n let bytes = uint8;\n\n if (this.carry !== null) {\n const merged = new Uint8Array(1 + bytes.length);\n merged[0] = this.carry;\n merged.set(bytes, 1);\n bytes = merged;\n this.carry = null;\n }\n\n if (bytes.length % 2 !== 0) {\n this.carry = bytes[bytes.length - 1];\n bytes = bytes.subarray(0, bytes.length - 1);\n }\n\n if (bytes.length === 0) return;\n const numSamples = bytes.length / 2;\n const cap = this.capacity;\n const samples = this.samples;\n if (this.littleEndian && (bytes.byteOffset & 1) === 0) {\n // Fast path: 2-byte-aligned LE bytes wrap directly as an Int16Array;\n // copy wrap-aware in at most two runs with no per-sample DataView call\n // or modulo. This runs on the realtime audio thread.\n const int16 = new Int16Array(bytes.buffer, bytes.byteOffset, numSamples);\n let src = 0;\n let dst = this.writePos % cap;\n while (src < numSamples) {\n const run = Math.min(numSamples - src, cap - dst);\n for (let j = 0; j < run; j++) {\n samples[dst + j] = int16[src + j] / 0x8000;\n }\n src += run;\n dst = 0;\n }\n } else {\n // Odd byte offset (or big-endian host): fall back to per-sample reads.\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);\n let dst = this.writePos % cap;\n for (let i = 0; i < numSamples; i++) {\n samples[dst] = view.getInt16(i * 2, true) / 0x8000;\n dst++;\n if (dst === cap) dst = 0;\n }\n }\n this.writePos += numSamples;\n // If the producer outran the consumer by more than the buffer holds, drop\n // the oldest unplayed audio rather than reading samples we've overwritten.\n if (this.writePos - this.readPos > this.capacity) {\n this.readPos = this.writePos - this.capacity;\n }\n }\n\n process(inputs, outputs) {\n const out = outputs[0][0];\n if (this.interrupted) {\n out.fill(0);\n this.stopTurn();\n return true;\n }\n\n const avail = this.writePos - this.readPos;\n\n // Wait for jitter buffer to fill, unless done (short utterance)\n if (!this.playing) {\n if (avail >= this.jitterSamples || this.isDone) {\n this.playing = true;\n } else {\n out.fill(0);\n return true;\n }\n }\n\n if (avail > 0) {\n const n = Math.min(avail, out.length);\n // Copy from the ring buffer, splitting across the wrap boundary.\n const start = this.readPos % this.capacity;\n const first = Math.min(n, this.capacity - start);\n out.set(this.samples.subarray(start, start + first), 0);\n if (n > first) out.set(this.samples.subarray(0, n - first), first);\n this.readPos += n;\n out.fill(0, n);\n return true;\n }\n\n // No data: output silence, end the turn only when done\n out.fill(0);\n if (this.isDone) {\n this.stopTurn();\n }\n return true;\n }\n}\n\nregisterProcessor('playback-processor', PlaybackProcessor);\n";
|
|
2
|
+
export declare const playbackProcessorSource = "\nclass PlaybackProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n const rate = options.processorOptions?.sampleRate ?? 24000;\n // Wait for ~400ms of audio before starting.\n // If 'done' arrives first (short utterance), start immediately.\n this.jitterSamples = Math.floor(rate * 0.4);\n // Float32 ring buffer \u2014 60s at the context sample rate. Allocated once for\n // the node's lifetime; per-turn state resets via resetTurn(). writePos and\n // readPos are absolute (monotonic) sample counts; the buffer is indexed\n // modulo capacity so a reply longer than 60s keeps playing instead of\n // writing past the end and going silent.\n this.capacity = rate * 60;\n this.samples = new Float32Array(this.capacity);\n // Platform endianness probe: the wire format is PCM16 little-endian, so\n // the Int16Array fast path in ingestBytes is only valid on LE hosts\n // (every shipping browser target; the DataView path is the fallback).\n this.littleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;\n this.resetTurn();\n\n this.port.onmessage = (e) => {\n const d = e.data;\n if (d.event === 'write') {\n this.ingestBytes(d.buffer);\n } else if (d.event === 'interrupt') {\n this.interrupted = true;\n } else if (d.event === 'done') {\n this.isDone = true;\n }\n };\n }\n\n // Reset per-turn state so the node is reusable across replies without\n // reallocating the sample buffer or re-instantiating the worklet.\n resetTurn() {\n this.interrupted = false;\n this.isDone = false;\n this.playing = false;\n // Carry-over byte for split samples across chunks\n this.carry = null;\n this.writePos = 0;\n this.readPos = 0;\n }\n\n // End the current turn: notify the host and rearm for the next reply.\n // Must NOT return false from process() \u2014 a processor that stops is dead\n // for good, forcing a new node (and buffer) per reply.\n // `reason` ('interrupt' | 'done') tells the host which turn boundary this\n // stop belongs to: interrupt-stops are dropped host-side (flush() already\n // settled that turn), so they can never resolve a later turn's done() early.\n stopTurn(reason) {\n this.port.postMessage({ event: 'stop', reason });\n this.resetTurn();\n }\n\n ingestBytes(uint8) {\n let bytes = uint8;\n\n if (this.carry !== null) {\n const merged = new Uint8Array(1 + bytes.length);\n merged[0] = this.carry;\n merged.set(bytes, 1);\n bytes = merged;\n this.carry = null;\n }\n\n if (bytes.length % 2 !== 0) {\n this.carry = bytes[bytes.length - 1];\n bytes = bytes.subarray(0, bytes.length - 1);\n }\n\n if (bytes.length === 0) return;\n const numSamples = bytes.length / 2;\n const cap = this.capacity;\n const samples = this.samples;\n if (this.littleEndian && (bytes.byteOffset & 1) === 0) {\n // Fast path: 2-byte-aligned LE bytes wrap directly as an Int16Array;\n // copy wrap-aware in at most two runs with no per-sample DataView call\n // or modulo. This runs on the realtime audio thread.\n const int16 = new Int16Array(bytes.buffer, bytes.byteOffset, numSamples);\n let src = 0;\n let dst = this.writePos % cap;\n while (src < numSamples) {\n const run = Math.min(numSamples - src, cap - dst);\n for (let j = 0; j < run; j++) {\n samples[dst + j] = int16[src + j] / 0x8000;\n }\n src += run;\n dst = 0;\n }\n } else {\n // Odd byte offset (or big-endian host): fall back to per-sample reads.\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);\n let dst = this.writePos % cap;\n for (let i = 0; i < numSamples; i++) {\n samples[dst] = view.getInt16(i * 2, true) / 0x8000;\n dst++;\n if (dst === cap) dst = 0;\n }\n }\n this.writePos += numSamples;\n // If the producer outran the consumer by more than the buffer holds, drop\n // the oldest unplayed audio rather than reading samples we've overwritten.\n if (this.writePos - this.readPos > this.capacity) {\n this.readPos = this.writePos - this.capacity;\n }\n }\n\n process(inputs, outputs) {\n // No output wired up yet \u2014 nothing to render this quantum. Throwing here\n // would permanently kill the processor (the node is persistent per\n // session), so guard like the capture processor does.\n if (!outputs[0] || !outputs[0][0]) return true;\n const out = outputs[0][0];\n if (this.interrupted) {\n out.fill(0);\n this.stopTurn('interrupt');\n return true;\n }\n\n const avail = this.writePos - this.readPos;\n\n // Wait for jitter buffer to fill, unless done (short utterance)\n if (!this.playing) {\n if (avail >= this.jitterSamples || this.isDone) {\n this.playing = true;\n } else {\n out.fill(0);\n return true;\n }\n }\n\n if (avail > 0) {\n const n = Math.min(avail, out.length);\n // Copy from the ring buffer, splitting across the wrap boundary.\n const start = this.readPos % this.capacity;\n const first = Math.min(n, this.capacity - start);\n out.set(this.samples.subarray(start, start + first), 0);\n if (n > first) out.set(this.samples.subarray(0, n - first), first);\n this.readPos += n;\n out.fill(0, n);\n return true;\n }\n\n // No data: output silence, end the turn only when done\n out.fill(0);\n if (this.isDone) {\n this.stopTurn('done');\n }\n return true;\n }\n}\n\nregisterProcessor('playback-processor', PlaybackProcessor);\n";
|
|
3
3
|
declare const src: string;
|
|
4
4
|
export default src;
|
|
@@ -47,8 +47,11 @@ class PlaybackProcessor extends AudioWorkletProcessor {
|
|
|
47
47
|
// End the current turn: notify the host and rearm for the next reply.
|
|
48
48
|
// Must NOT return false from process() — a processor that stops is dead
|
|
49
49
|
// for good, forcing a new node (and buffer) per reply.
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
// \`reason\` ('interrupt' | 'done') tells the host which turn boundary this
|
|
51
|
+
// stop belongs to: interrupt-stops are dropped host-side (flush() already
|
|
52
|
+
// settled that turn), so they can never resolve a later turn's done() early.
|
|
53
|
+
stopTurn(reason) {
|
|
54
|
+
this.port.postMessage({ event: 'stop', reason });
|
|
52
55
|
this.resetTurn();
|
|
53
56
|
}
|
|
54
57
|
|
|
@@ -106,10 +109,14 @@ class PlaybackProcessor extends AudioWorkletProcessor {
|
|
|
106
109
|
}
|
|
107
110
|
|
|
108
111
|
process(inputs, outputs) {
|
|
112
|
+
// No output wired up yet — nothing to render this quantum. Throwing here
|
|
113
|
+
// would permanently kill the processor (the node is persistent per
|
|
114
|
+
// session), so guard like the capture processor does.
|
|
115
|
+
if (!outputs[0] || !outputs[0][0]) return true;
|
|
109
116
|
const out = outputs[0][0];
|
|
110
117
|
if (this.interrupted) {
|
|
111
118
|
out.fill(0);
|
|
112
|
-
this.stopTurn();
|
|
119
|
+
this.stopTurn('interrupt');
|
|
113
120
|
return true;
|
|
114
121
|
}
|
|
115
122
|
|
|
@@ -140,7 +147,7 @@ class PlaybackProcessor extends AudioWorkletProcessor {
|
|
|
140
147
|
// No data: output silence, end the turn only when done
|
|
141
148
|
out.fill(0);
|
|
142
149
|
if (this.isDone) {
|
|
143
|
-
this.stopTurn();
|
|
150
|
+
this.stopTurn('done');
|
|
144
151
|
}
|
|
145
152
|
return true;
|
|
146
153
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alexkroman1/aai-ui",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"files": [
|
|
6
6
|
"dist",
|
|
@@ -18,12 +18,14 @@
|
|
|
18
18
|
},
|
|
19
19
|
"dependencies": {
|
|
20
20
|
"clsx": "^2.1.1",
|
|
21
|
-
"
|
|
21
|
+
"partysocket": "^1.3.0",
|
|
22
|
+
"use-sync-external-store": "^1.6.0",
|
|
23
|
+
"@alexkroman1/aai": "1.10.0"
|
|
22
24
|
},
|
|
23
25
|
"peerDependencies": {
|
|
24
|
-
"react": "^19.
|
|
25
|
-
"react-dom": "^19.
|
|
26
|
-
"tailwindcss": "^4.
|
|
26
|
+
"react": "^19.0.0",
|
|
27
|
+
"react-dom": "^19.0.0",
|
|
28
|
+
"tailwindcss": "^4.0.0"
|
|
27
29
|
},
|
|
28
30
|
"peerDependenciesMeta": {
|
|
29
31
|
"tailwindcss": {
|
|
@@ -36,6 +38,7 @@
|
|
|
36
38
|
"@testing-library/react": "^16.3.2",
|
|
37
39
|
"@types/react": "^19.2.17",
|
|
38
40
|
"@types/react-dom": "^19.2.3",
|
|
41
|
+
"@types/use-sync-external-store": "^1.5.0",
|
|
39
42
|
"@vitejs/plugin-react": "^6.0.4",
|
|
40
43
|
"jsdom": "^29.1.1",
|
|
41
44
|
"react": "^19.2.8",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{t as e}from"./index-Bf4ZTNcx.js";async function t(e,t){let n=await new OfflineAudioContext(1,1,t).decodeAudioData(e),r=Math.ceil(n.duration*t),i=new OfflineAudioContext(1,r,t),a=i.createBufferSource();a.buffer=n,a.connect(i.destination),a.start();let o=(await i.startRendering()).getChannelData(0),s=new Int16Array(o.length);for(let e=0;e<o.length;e++){let t=Math.max(-1,Math.min(1,o[e]??0));s[e]=t<0?t*32768:t*32767}return s}async function n(t){let{sttSampleRate:n,ttsSampleRate:r,captureWorkletSrc:i,playbackWorkletSrc:a,onMicData:o}=t,s=r,c=new AudioContext({sampleRate:s,latencyHint:`playback`}),l=navigator.mediaDevices.getUserMedia({audio:{deviceId:{ideal:`default`},echoCancellation:!0,noiseSuppression:!0,autoGainControl:!0,voiceIsolation:!0}}),u;try{[u]=await Promise.all([l,c.resume(),c.audioWorklet.addModule(i),c.audioWorklet.addModule(a)])}catch(e){throw l.then(e=>{for(let t of e.getTracks())t.stop()}).catch(()=>{}),await c.close().catch(e=>{console.warn(`AudioContext close failed:`,e)}),e}let d=c.createMediaStreamSource(u),f=new AudioWorkletNode(c,`capture-processor`,{channelCount:1,channelCountMode:`explicit`,processorOptions:{contextRate:s,sttSampleRate:n,bufferSeconds:e}});d.connect(f),f.port.postMessage({event:`start`}),f.port.onmessage=e=>{e.data.event===`chunk`&&o(e.data.buffer)};let p=null,m=null,h=new AbortController;function g(){if(p)return p;let e=new AudioWorkletNode(c,`playback-processor`,{processorOptions:{sampleRate:s}});return e.connect(c.destination),e.port.onmessage=e=>{e.data.event===`stop`&&(m?.(),m=null)},p=e,e}let _={enqueue(e){h.signal.aborted||e.byteLength!==0&&g().port.postMessage({event:`write`,buffer:new Uint8Array(e)},[e])},done(){return!p||c.state!==`running`?Promise.resolve():new Promise(e=>{m=e,p?.port.postMessage({event:`done`})})},flush(){p&&p.port.postMessage({event:`interrupt`})},async close(){if(!h.signal.aborted){h.abort(),f.port.postMessage({event:`stop`}),d.disconnect(),f.disconnect(),p&&p.disconnect();for(let e of u.getTracks())e.stop();await c.close().catch(()=>{})}},async[Symbol.asyncDispose](){await _.close()}};return _}export{n as createVoiceIO,t as decodeAudioToPcm16};
|