@alexkroman1/aai-ui 1.9.2 → 1.11.0

package/dist/default-client/assets/{playback-processor-BtlzAH78.js → playback-processor-C5HVRVbu.js}

@@ -46,8 +46,11 @@ class PlaybackProcessor extends AudioWorkletProcessor {
   // End the current turn: notify the host and rearm for the next reply.
   // Must NOT return false from process() — a processor that stops is dead
   // for good, forcing a new node (and buffer) per reply.
-  stopTurn() {
-    this.port.postMessage({ event: 'stop' });
+  // \`reason\` ('interrupt' | 'done') tells the host which turn boundary this
+  // stop belongs to: interrupt-stops are dropped host-side (flush() already
+  // settled that turn), so they can never resolve a later turn's done() early.
+  stopTurn(reason) {
+    this.port.postMessage({ event: 'stop', reason });
     this.resetTurn();
   }
 
@@ -105,10 +108,14 @@ class PlaybackProcessor extends AudioWorkletProcessor {
   }
 
   process(inputs, outputs) {
+    // No output wired up yet — nothing to render this quantum. Throwing here
+    // would permanently kill the processor (the node is persistent per
+    // session), so guard like the capture processor does.
+    if (!outputs[0] || !outputs[0][0]) return true;
     const out = outputs[0][0];
     if (this.interrupted) {
       out.fill(0);
-      this.stopTurn();
+      this.stopTurn('interrupt');
       return true;
     }
 
@@ -139,7 +146,7 @@ class PlaybackProcessor extends AudioWorkletProcessor {
     // No data: output silence, end the turn only when done
     out.fill(0);
     if (this.isDone) {
-      this.stopTurn();
+      this.stopTurn('done');
     }
     return true;
   }

package/dist/default-client/index.html

@@ -6,8 +6,8 @@
     <title>aai</title>
     <link rel="icon" href="data:," />
     <style>html, body { background: #FBF8F2; margin: 0; }</style>
-    <script type="module" crossorigin src="./assets/index-Bf4ZTNcx.js"></script>
-    <link rel="stylesheet" crossorigin href="./assets/index-Bzlh9i7w.css">
+    <script type="module" crossorigin src="./assets/index-BTqSueyz.js"></script>
+    <link rel="stylesheet" crossorigin href="./assets/index-B2bISVbm.css">
   </head>
   <body>
     <main id="app"></main>

package/dist/define-client.js

@@ -1,9 +1,9 @@
 import { SessionProvider, ThemeProvider } from "./context.js";
-import { n as ToolConfigContext } from "./tool-call-block-7f1GTG-P.js";
-import { t as ChatView } from "./chat-view-C1XbqDk0.js";
+import { n as ToolConfigContext } from "./tool-call-block-DIxpG8GM.js";
+import { t as ChatView } from "./chat-view-u3yBAlig.js";
 import { SidebarLayout } from "./components/sidebar-layout.js";
 import { StartScreen } from "./components/start-screen.js";
-import { t as createSessionCore } from "./session-core-D3NDaySY.js";
+import { t as createSessionCore } from "./session-core-KUoY-0JO.js";
 import { createElement } from "react";
 import { jsx } from "react/jsx-runtime";
 import { flushSync } from "react-dom";

package/dist/hooks.js

@@ -1,5 +1,5 @@
 import { useSessionSelector } from "./context.js";
-import { n as tryParseJSON } from "./_utils-5cs73OrA.js";
+import { n as tryParseJSON } from "./_utils-CN1yVgYS.js";
 import { useEffect, useRef } from "react";
 //#region hooks.ts
 /** Index of the first item whose sequence number is above the watermark (tail scan from the end). */
@@ -51,13 +51,16 @@ function useToolCallEffect(status, args, fire) {
 		seq: 0,
 		fired: /* @__PURE__ */ new Set()
 	});
+	const mountedRef = useRef(false);
 	const callbackRef = useRef(callback);
 	callbackRef.current = callback;
 	const fireRef = useRef(fire);
 	fireRef.current = fire;
 	useEffect(() => {
+		const firstRun = !mountedRef.current;
+		mountedRef.current = true;
 		processToolCallTail(toolCalls, cursorRef.current, (tc) => status === "pending" || tc.status === status, (tc) => {
-			if (tc.status !== status) return;
+			if (tc.status !== status && firstRun) return;
 			if (filterName && tc.name !== filterName) return;
 			fireRef.current(callbackRef.current, tc, filterName !== null);
 		});

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export { Button } from "./components/button.tsx";
 export { ChatView } from "./components/chat-view.tsx";
+export { Controls } from "./components/controls.tsx";
 export { MessageList } from "./components/message-list.tsx";
 export { SidebarLayout } from "./components/sidebar-layout.tsx";
 export { StartScreen } from "./components/start-screen.tsx";
@@ -14,4 +15,7 @@ export { client } from "./define-client.tsx";
 export { useEvent, useToolCallStart, useToolResult } from "./hooks.ts";
 export type { CustomEvent, SessionCore, SessionCoreOptions, SessionSnapshot, } from "./session-core.ts";
 export { createSessionCore } from "./session-core.ts";
+export { CAPTURE_WORKLET_DATA_URI, DEFAULT_SYNC_MIC_SAMPLE_RATE, floatToPcm16, type SyncMicrophone, type SyncMicrophoneOptions, startSyncMicrophone, } from "./sync-mic.ts";
+export { base64ToPcm16, createSyncSession, pcm16ToBase64, type SyncSession, type SyncSessionOptions, type SyncTurnResult, } from "./sync-session.ts";
+export { createUtteranceDetector, type UtteranceDetector, type UtteranceDetectorOptions, } from "./sync-vad.ts";
 export type { AgentState, ChatMessage, ClientTheme, SessionError, SessionErrorCode, ToolCallInfo, VoiceSessionOptions, WebSocketConstructor, } from "./types.ts";

package/dist/index.js

@@ -1,12 +1,361 @@
 import { SessionProvider, ThemeProvider, useSession, useSessionSelector, useTheme } from "./context.js";
 import { Button } from "./components/button.js";
-import { i as UiUrlChip, n as ApiUrlChip, r as SessionUrlChips } from "./controls-BngrPbOC.js";
-import { n as ToolConfigContext } from "./tool-call-block-7f1GTG-P.js";
+import { i as UiUrlChip, n as ApiUrlChip, r as SessionUrlChips, t as Controls } from "./controls-B2EPUJDU.js";
+import { n as ToolConfigContext } from "./tool-call-block-DIxpG8GM.js";
 import { MessageList } from "./components/message-list.js";
-import { n as TextControls, t as ChatView } from "./chat-view-C1XbqDk0.js";
+import { n as TextControls, t as ChatView } from "./chat-view-u3yBAlig.js";
 import { SidebarLayout } from "./components/sidebar-layout.js";
 import { StartScreen } from "./components/start-screen.js";
-import { t as createSessionCore } from "./session-core-D3NDaySY.js";
+import { t as createSessionCore } from "./session-core-KUoY-0JO.js";
 import { client } from "./define-client.js";
 import { useEvent, useToolCallStart, useToolResult } from "./hooks.js";
-export { ApiUrlChip, Button, ChatView, MessageList, SessionProvider, SessionUrlChips, SidebarLayout, StartScreen, TextControls, ThemeProvider, ToolConfigContext, UiUrlChip, client, createSessionCore, useEvent, useSession, useSessionSelector, useTheme, useToolCallStart, useToolResult };
+import { errorMessage, safeJsonParse } from "@alexkroman1/aai";
+import { SyncTurnResponseSchema } from "@alexkroman1/aai/protocol";
+//#region sync-vad.ts
+const DEFAULT_SPEECH_RMS = .015;
+const DEFAULT_MAX_UTTERANCE_MS = 3e4;
+function rmsOf(frame) {
+	if (frame.length === 0) return 0;
+	let sum = 0;
+	for (const s of frame) {
+		const f = s / 32768;
+		sum += f * f;
+	}
+	return Math.sqrt(sum / frame.length);
+}
+function concat(frames) {
+	let total = 0;
+	for (const f of frames) total += f.length;
+	const out = new Int16Array(total);
+	let offset = 0;
+	for (const f of frames) {
+		out.set(f, offset);
+		offset += f.length;
+	}
+	return out;
+}
+/** Create an energy-based {@link UtteranceDetector}. */
+function createUtteranceDetector(opts) {
+	const { sampleRate, speechRms = DEFAULT_SPEECH_RMS, minSpeechMs = 150, hangoverMs = 700, prerollMs = 300, maxUtteranceMs = DEFAULT_MAX_UTTERANCE_MS } = opts;
+	const msOf = (frame) => frame.length / sampleRate * 1e3;
+	let preroll = [];
+	let prerollTotalMs = 0;
+	let utterance = [];
+	let utteranceMs = 0;
+	let speaking = false;
+	/** Consecutive voiced ms while still a candidate (below minSpeechMs). */
+	let candidateVoicedMs = 0;
+	/** Consecutive silent ms while speaking (toward hangoverMs). */
+	let silenceMs = 0;
+	function toIdle() {
+		utterance = [];
+		utteranceMs = 0;
+		speaking = false;
+		candidateVoicedMs = 0;
+		silenceMs = 0;
+	}
+	function pushPreroll(frame) {
+		preroll.push(frame);
+		prerollTotalMs += msOf(frame);
+		while (preroll.length > 0 && prerollTotalMs - msOf(preroll[0]) >= prerollMs) prerollTotalMs -= msOf(preroll.shift());
+	}
+	function beginCandidate(frame) {
+		utterance = [...preroll, frame];
+		utteranceMs = prerollTotalMs + msOf(frame);
+		preroll = [];
+		prerollTotalMs = 0;
+		candidateVoicedMs = msOf(frame);
+	}
+	function finalize() {
+		const out = concat(utterance);
+		toIdle();
+		return out;
+	}
+	function pushIdle(frame, voiced) {
+		if (!voiced) {
+			pushPreroll(frame);
+			return;
+		}
+		beginCandidate(frame);
+	}
+	function pushCandidate(frame, voiced) {
+		if (!voiced) {
+			for (const f of utterance) pushPreroll(f);
+			toIdle();
+			return;
+		}
+		utterance.push(frame);
+		utteranceMs += msOf(frame);
+		candidateVoicedMs += msOf(frame);
+	}
+	function pushSpeaking(frame, voiced) {
+		utterance.push(frame);
+		utteranceMs += msOf(frame);
+		silenceMs = voiced ? 0 : silenceMs + msOf(frame);
+		return silenceMs >= hangoverMs ? finalize() : null;
+	}
+	return {
+		get speaking() {
+			return speaking;
+		},
+		push(frame) {
+			if (frame.length === 0) return null;
+			const voiced = rmsOf(frame) >= speechRms;
+			let closed = null;
+			if (speaking) closed = pushSpeaking(frame, voiced);
+			else if (utterance.length > 0) pushCandidate(frame, voiced);
+			else pushIdle(frame, voiced);
+			if (closed) return closed;
+			if (!speaking && candidateVoicedMs >= minSpeechMs) speaking = true;
+			if (speaking && utteranceMs >= maxUtteranceMs) return finalize();
+			return null;
+		},
+		flush() {
+			const out = speaking ? finalize() : null;
+			this.reset();
+			return out;
+		},
+		reset() {
+			preroll = [];
+			prerollTotalMs = 0;
+			toIdle();
+		}
+	};
+}
+//#endregion
+//#region sync-mic.ts
+/**
+* WebRTC microphone capture for sync mode.
+*
+* Captures voice through `getUserMedia` with the WebRTC voice-processing
+* constraints (echo cancellation, noise suppression, auto gain — the
+* processing that makes the energy VAD in `sync-vad.ts` reliable), runs an
+* AudioWorklet that batches raw frames to the main thread, feeds them
+* through the utterance detector, and hands each completed utterance to
+* the sync session as one HTTP turn. No WebSocket anywhere on the path.
+*
+* The worklet module ships inline as a data URI, so sync mode needs no
+* separately-served processor file.
+*/
+/** Default capture rate — what the STT providers expect. */
+const DEFAULT_SYNC_MIC_SAMPLE_RATE = 16e3;
+/** ~128 ms at 16 kHz: few messages per second, fine-enough VAD granularity. */
+const CAPTURE_BATCH_SAMPLES = 2048;
+/** Data URI form of the capture processor (no served asset, no blob URL). */
+const CAPTURE_WORKLET_DATA_URI = `data:application/javascript;charset=utf-8,${encodeURIComponent(`
+registerProcessor("aai-sync-capture", class extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    const batch = (options && options.processorOptions && options.processorOptions.batchSamples) || ${CAPTURE_BATCH_SAMPLES};
+    this.buf = new Float32Array(batch);
+    this.len = 0;
+  }
+  process(inputs) {
+    const ch = inputs[0] && inputs[0][0];
+    if (!ch) return true;
+    let read = 0;
+    while (read < ch.length) {
+      const n = Math.min(ch.length - read, this.buf.length - this.len);
+      this.buf.set(ch.subarray(read, read + n), this.len);
+      this.len += n;
+      read += n;
+      if (this.len === this.buf.length) {
+        const out = this.buf;
+        this.port.postMessage({ event: "chunk", samples: out }, [out.buffer]);
+        this.buf = new Float32Array(out.length);
+        this.len = 0;
+      }
+    }
+    return true;
+  }
+});
+`)}`;
+/** Clamp-and-convert one Float32 capture batch to PCM16. */
+function floatToPcm16(samples) {
+	const pcm = new Int16Array(samples.length);
+	let i = 0;
+	for (const sample of samples) {
+		const s = Math.max(-1, Math.min(1, sample));
+		pcm[i++] = s < 0 ? s * 32768 : s * 32767;
+	}
+	return pcm;
+}
+/**
+* Open the microphone and stream endpointed utterances into a sync session.
+*
+* @throws If microphone access is denied or worklet registration fails.
+*/
+async function startSyncMicrophone(opts) {
+	const sampleRate = opts.sampleRate ?? 16e3;
+	const detector = createUtteranceDetector({
+		sampleRate,
+		...opts.vad
+	});
+	const fail = (err) => {
+		opts.onError?.(err instanceof Error ? err : new Error(errorMessage(err)));
+	};
+	const streamPromise = navigator.mediaDevices.getUserMedia({ audio: {
+		echoCancellation: true,
+		noiseSuppression: true,
+		autoGainControl: true
+	} });
+	const ctx = new AudioContext({
+		sampleRate,
+		latencyHint: "interactive"
+	});
+	let stream;
+	try {
+		[stream] = await Promise.all([
+			streamPromise,
+			ctx.resume(),
+			ctx.audioWorklet.addModule(CAPTURE_WORKLET_DATA_URI)
+		]);
+	} catch (err) {
+		streamPromise.then((s) => {
+			for (const t of s.getTracks()) t.stop();
+		}).catch(() => {});
+		await ctx.close().catch(() => {});
+		throw err;
+	}
+	const mic = ctx.createMediaStreamSource(stream);
+	const node = new AudioWorkletNode(ctx, "aai-sync-capture", {
+		channelCount: 1,
+		channelCountMode: "explicit",
+		processorOptions: { batchSamples: CAPTURE_BATCH_SAMPLES }
+	});
+	mic.connect(node);
+	let stopped = false;
+	let wasSpeaking = false;
+	function dispatch(utterance) {
+		if (!utterance) return;
+		opts.onSpeechEnd?.();
+		opts.session.sendPcm16(utterance, sampleRate).catch(fail);
+	}
+	function handleFrame(samples) {
+		if (stopped) return;
+		dispatch(detector.push(floatToPcm16(samples)));
+		if (detector.speaking && !wasSpeaking) opts.onSpeechStart?.();
+		wasSpeaking = detector.speaking;
+	}
+	node.port.onmessage = (e) => {
+		const data = e.data;
+		if (data.event === "chunk" && data.samples) handleFrame(data.samples);
+	};
+	node.onprocessorerror = () => fail(/* @__PURE__ */ new Error("Sync capture worklet crashed"));
+	return {
+		get speaking() {
+			return detector.speaking;
+		},
+		async stop() {
+			if (stopped) return;
+			stopped = true;
+			dispatch(detector.flush());
+			mic.disconnect();
+			node.disconnect();
+			for (const t of stream.getTracks()) t.stop();
+			await ctx.close().catch(() => {});
+		}
+	};
+}
+//#endregion
+//#region sync-session.ts
+/**
+* Sync-mode browser session — HTTP turns, no WebSocket.
+*
+* The client half of the server's `POST /sync` endpoint (see
+* `host/sync-turn.ts` in `@alexkroman1/aai`): each turn is one request
+* carrying committed text or one endpointed utterance of PCM16 audio plus
+* the conversation history, answered with the transcript, the reply text,
+* and (when the agent's TTS provider supports one-shot synthesis) the
+* spoken reply. The server holds no session state — this object owns the
+* history and replays it every turn.
+*
+* Microphone capture and utterance endpointing live in `sync-mic.ts` /
+* `sync-vad.ts`; this module is transport only, so it also runs in
+* non-browser clients that bring their own audio.
+*/
+/** Base64-encode PCM16 samples (chunked — `btoa` takes a binary string). */
+function pcm16ToBase64(pcm) {
+	const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+	let binary = "";
+	const CHUNK = 32768;
+	for (let i = 0; i < bytes.length; i += CHUNK) binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
+	return btoa(binary);
+}
+/** Decode base64 PCM16LE (the sync response's `audio` field) into samples. */
+function base64ToPcm16(base64) {
+	const binary = atob(base64);
+	const samples = binary.length >> 1;
+	const bytes = new Uint8Array(samples * 2);
+	for (let i = 0; i < bytes.length; i++) bytes[i] = binary.charCodeAt(i);
+	return new Int16Array(bytes.buffer, 0, samples);
+}
+/** Create a {@link SyncSession} against a sync-mode agent server. */
+function createSyncSession(opts) {
+	const fetchFn = opts.fetch ?? ((input, init) => globalThis.fetch(input, init));
+	const history = [];
+	let queue = Promise.resolve();
+	async function parseTurn(resp) {
+		if (!resp.ok) {
+			const detail = safeJsonParse(await resp.text().catch(() => "")) ?? {};
+			throw new Error(`Sync turn failed: HTTP ${resp.status}${detail.error ? ` (${detail.error})` : ""}`);
+		}
+		const parsed = SyncTurnResponseSchema.safeParse(await resp.json());
+		if (!parsed.success) throw new Error("Sync turn failed: malformed server response");
+		return parsed.data;
+	}
+	async function runTurn(body) {
+		try {
+			const turn = await parseTurn(await fetchFn(opts.url, {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					...body,
+					history: [...history]
+				})
+			}));
+			history.push({
+				role: "user",
+				content: turn.transcript
+			});
+			if (turn.reply.length > 0) history.push({
+				role: "assistant",
+				content: turn.reply
+			});
+			const result = {
+				...turn,
+				pcm: turn.audio !== void 0 ? base64ToPcm16(turn.audio) : null
+			};
+			opts.onTurn?.(result);
+			return result;
+		} catch (err) {
+			const wrapped = err instanceof Error ? err : new Error(errorMessage(err));
+			opts.onError?.(wrapped);
+			throw wrapped;
+		}
+	}
+	function enqueue(body) {
+		const next = queue.then(() => runTurn(body), () => runTurn(body));
+		queue = next.catch(() => void 0);
+		return next;
+	}
+	return {
+		get history() {
+			return history;
+		},
+		sendText(text) {
+			return enqueue({ text });
+		},
+		sendPcm16(pcm, sampleRate) {
+			return enqueue({
+				audio: pcm16ToBase64(pcm),
+				sampleRate
+			});
+		},
+		reset() {
+			history.length = 0;
+		}
+	};
+}
+//#endregion
+export { ApiUrlChip, Button, CAPTURE_WORKLET_DATA_URI, ChatView, Controls, DEFAULT_SYNC_MIC_SAMPLE_RATE, MessageList, SessionProvider, SessionUrlChips, SidebarLayout, StartScreen, TextControls, ThemeProvider, ToolConfigContext, UiUrlChip, base64ToPcm16, client, createSessionCore, createSyncSession, createUtteranceDetector, floatToPcm16, pcm16ToBase64, startSyncMicrophone, useEvent, useSession, useSessionSelector, useTheme, useToolCallStart, useToolResult };