@missionsquad/browserai 0.1.0

package/LICENSE

@@ -0,0 +1,15 @@
+ISC License
+
+Copyright (c) 2026 Mission Squad
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -0,0 +1,229 @@
+# @missionsquad/browserai
+
+Run small open AI models **entirely in the browser** with WebGPU — no inference server — as an
+embeddable, UI-free TypeScript SDK. BrowserAI packages a curated 47-model catalog, model
+loading/downloading with progress, capability-slot management, three inference runtimes, streaming,
+schema-constrained JSON generation, run metrics, browser-cache management, and a complete
+press-to-talk voice-assistant pipeline. Drop it into any frontend site and build your own UI on top.
+
+Three inference runtimes:
+
+- **WebLLM (MLC)** for quantized MLC text models, with schema-constrained JSON decoding and detailed
+  per-token runtime telemetry.
+- **Transformers.js + ONNX/WebGPU** for browser-ready ONNX models: text LLMs, Whisper/Moonshine/
+  Parakeet transcription, Supertonic speech, SmolVLM/Qwen-VL/OCR vision, Gemma 4 multimodal, and
+  audio-LLM ASR (Granite Speech, Voxtral realtime).
+- **kokoro-js** (loaded from its CDN bundle by default) for Kokoro 82M text-to-speech, including
+  sentence-by-sentence streaming synthesis.
+
+## Install
+
+```bash
+npm install @missionsquad/browserai
+```
+
+Requirements: a WebGPU-capable browser (lightweight speech models — Whisper/Moonshine/Parakeet
+transcription and the TTS engines — can fall back to wasm; text, vision, and audio-LLM models
+require WebGPU), Node.js 20+ for development. `@mlc-ai/web-llm` and `@huggingface/transformers` are regular
+dependencies, dynamically imported only when a model that needs them loads — your initial bundle
+stays small.
+
+## Quickstart
+
+```ts
+import { BrowserAI } from "@missionsquad/browserai";
+
+const ai = new BrowserAI();
+ai.on("loadprogress", ({ progress, status }) => console.log(progress, status));
+
+await ai.load("Qwen3.5-0.8B-q4f16_1-MLC"); // downloads once, then serves from browser cache
+
+const { text, stats } = await ai.generateText(
+  [
+    { role: "system", content: "You are terse." },
+    { role: "user", content: "Why is the sky blue?" },
+  ],
+  { runtime: { jsonMode: "none" }, onDelta: (full) => render(full) },
+);
+console.log(text, stats.extra?.decode_tokens_per_s);
+```
+
+### Capability slots
+
+Models load into one of four **capability slots** — `text`, `vision`, `stt` (transcription), `tts`
+(speech) — with at most one model per slot, so up to four models can be resident at once. Loading a
+model whose slots are occupied replaces the conflicting owner. Loads/unloads are single-flight, and
+runs are serialized per loaded model, so concurrent calls queue instead of driving one engine from
+two places.
+
+```ts
+await ai.load("onnx-community/whisper-base");          // stt slot
+await ai.load("onnx-community/Kokoro-82M-v1.0-ONNX");  // tts slot
+
+const { text } = await ai.transcribe(pcm16k);           // Float32Array @ 16 kHz mono
+const clip = await ai.synthesizeToClip("Hello!", { voice: "af_heart" });
+ai.occupiedSlots();                                     // Set { "text", "stt", "tts" }
+```
+
+### Schema-constrained JSON
+
+```ts
+import { buildExtractionMessages, parseJsonFromModel, validateAgainstJsonSchema } from "@missionsquad/browserai";
+
+const messages = buildExtractionMessages(inputText, mySchema);
+const { text } = await ai.generateText(messages, { schema: mySchema }); // WebLLM: real constrained decoding
+const parsed = parseJsonFromModel(text);
+if (parsed.ok) console.log(validateAgainstJsonSchema(parsed.value, mySchema)); // [] when valid
+```
+
+WebLLM backends enforce the schema with a grammar-constrained decoder; Transformers.js backends are
+prompt-guided, so validate the output either way.
+
+### Voice pipeline (press-to-talk assistant)
+
+`VoicePipeline` chains mic → STT → text LLM → TTS → speakers as a headless state machine with typed
+events. With Kokoro in the speech slot, replies are spoken **while the text model is still
+generating** (audio starts after the first sentence); other TTS models speak the finished reply as
+one clip. Pressing talk during playback stops it (barge-in).
+
+```ts
+import { BrowserAI, VoicePipeline } from "@missionsquad/browserai";
+
+const ai = new BrowserAI();
+const voice = new VoicePipeline(ai);
+
+await voice.loadRecommendedModels(); // Whisper Base + Qwen3.5 0.8B + Kokoro 82M (~2.5 GB VRAM)
+
+voice.on("stagechange", ({ stage }) => setIndicator(stage)); // idle/listening/transcribing/thinking/speaking/error
+voice.on("transcript", ({ text }) => addUserBubble(text));
+voice.on("replydelta", ({ text }) => updateAssistantBubble(text));
+voice.on("turncommitted", ({ messages }) => renderHistory(messages));
+voice.on("error", ({ label, message }) => showError(`${label}: ${message}`));
+
+talkButton.onclick = () => voice.toggle(); // press to talk, press again to send
+newChatButton.onclick = () => voice.reset();
+```
+
+The turn commits to history only on success; discarded takes (`turndiscarded`: too short, no speech,
+empty reply, contention with other work) leave history untouched. For input-level visualization,
+subscribe to `micstream` (attach an AnalyserNode) and `micstreamended`; for output, pass
+`streamingAudio.onGraphReady` to tap the streaming player's gain node.
+
+### Vision
+
+```ts
+await ai.load("HuggingFaceTB/SmolVLM-256M-Instruct");
+const answer = await ai.describeImage(imageDataUrl, "What does this chart show?", (partial) => render(partial));
+```
+
+### Metrics + run history
+
+Every generation returns `RuntimeStats` (tokens, decode tok/s, TTFT, per-token latency breakdown for
+WebLLM, measured telemetry for Transformers.js). Persist and aggregate runs with `RunHistoryStore`:
+
+```ts
+import { RunHistoryStore, runRecordFromStats } from "@missionsquad/browserai";
+
+const history = new RunHistoryStore(); // localStorage-backed; pass { storage: null } for in-memory
+const record = runRecordFromStats("chat", stats);
+if (record) history.append(record);
+history.aggregateByModel("chat"); // decode/ttft averages per model
+```
+
+### Cache management
+
+Models download once (hundreds of MB to a few GB) and are cached in browser storage — WebLLM weights
+in IndexedDB by default (configurable via `cacheBackend`), Transformers.js artifacts always in the
+Cache API bucket `transformers-cache`.
+
+```ts
+await ai.cacheStatus("onnx-community/whisper-base"); // { downloaded, matchedEntries, … }
+await ai.deleteModelArtifacts("onnx-community/whisper-base");
+await ai.deleteAllModelArtifacts();                  // releases loaded models first
+await ai.estimateStorage();                          // origin-wide usage/quota
+```
+
+Cleanup targets model artifact storage only — never cookies, localStorage, or unrelated site data.
+
+## Configuration
+
+```ts
+const ai = new BrowserAI({
+  modelSource: "direct",          // "direct" (default, straight from Hugging Face) | "proxy"
+  proxyOrigin: undefined,          // proxy Worker origin; defaults to the page origin in proxy mode
+  verifyProxy: true,               // probe the proxy before each proxied load
+  cacheBackend: "indexeddb",       // WebLLM artifact cache: "indexeddb" | "cache" | "opfs" | "cross-origin"
+  webllm: {
+    worker: () => new Worker(new URL("./llm.worker.ts", import.meta.url), { type: "module" }),
+    logLevel: "INFO",
+  },
+  kokoro: { url: "https://cdn.jsdelivr.net/npm/kokoro-js@1.2.1/+esm" }, // or load: () => import("kokoro-js")
+  tts: { voice: "af_heart", speed: 1, pitch: 1 },
+  history: { key: "browserai:v1:history", maxRecords: 200 },
+});
+```
+
+### Running WebLLM in a worker
+
+By default WebLLM engines run on the main thread. To keep loading and token generation off it,
+create a two-line worker entry in your app and pass its factory:
+
+```ts
+// llm.worker.ts — bundled by YOUR build (Vite/webpack understand this pattern)
+export * from "@missionsquad/browserai/worker";
+```
+
+```ts
+const ai = new BrowserAI({
+  webllm: { worker: () => new Worker(new URL("./llm.worker.ts", import.meta.url), { type: "module" }) },
+});
+```
+
+### Model-artifact proxy (optional)
+
+Direct downloads work on any site. If a hosted deployment hits CORS or cache-storage edge cases with
+cross-origin model downloads, deploy the Cloudflare Worker in [`worker-template/`](worker-template/)
+and configure `{ modelSource: "proxy" }`. The Worker proxies `/hf/*` (WebLLM weights),
+`/hf-transformers/*` (Transformers.js artifacts), and `/gh-raw/*` (WebLLM wasm libraries), restricted
+to an allowlist derived from the SDK's model catalog. Kokoro is the one exception: kokoro-js ships
+its own runtime and fetches its weights directly.
+
+## Model catalog
+
+47 presets across four capability types (see `MODEL_PRESETS`; each entry carries GPU-memory
+estimates, download size, stability rating, and per-model default runtime parameters):
+
+- **Text (WebLLM/MLC):** Qwen3.5 0.8B/2B/4B, Gemma 3 1B, Gemma 4 E2B/E4B (experimental custom
+  records), Llama 3.2 1B/3B, Hermes 3, OLMo 2, Phi-4 mini, Ministral 3, Mistral 7B.
+- **Text (Transformers.js/ONNX):** Nemotron 3 Nano, LFM2.5 230M/350M/1.2B-Thinking, Falcon H1 Tiny,
+  the Granite 4.0 family, MiniCPM5, SmolLM3, Gemma 4 E2B/E4B ONNX (multimodal: text + image + audio).
+- **Transcription:** Whisper Base/Tiny/Medium/Medium-EN, Moonshine Base/Tiny, Cohere Transcribe,
+  Parakeet CTC 0.6B, Granite Speech 2B, Voxtral Mini 4B Realtime.
+- **Speech:** Kokoro 82M (streams), Supertonic 1/2.
+- **Vision:** SmolVLM 256M/500M, Qwen3-VL 2B, Qwen2.5-VL 3B, GLM-OCR, LightOnOCR-2.
+
+## Browser constraints worth knowing
+
+- **WebGPU** is required for text, vision, Gemma-multimodal, and audio-LLM models; ASR-pipeline
+  (Whisper/Moonshine/Parakeet) and TTS presets fall back to wasm. `probeHardware()` reports
+  adapter/features/limits; other loads throw `WebGPUUnavailableError` without it.
+- `decodeAudioTo16kMono`, `capSrc` (image downscaling), `MicRecorder`, and the audio players use
+  main-thread browser APIs (`AudioContext`, `Image`, canvas, `MediaRecorder`) — decode/capture on
+  the page even if you move inference elsewhere.
+- If you attach an analyzer to an `<audio>` element playing clips, remember a media element allows
+  exactly **one** `MediaElementSourceNode` for its whole life — create the analyzer once and never
+  close its context while the element is in use.
+- All SDK errors are `BrowserAIError` subclasses (`ModelLoadError`, `MicrophoneError`,
+  `MissingModelError`, `VoiceTurnError`, …) — branch with `instanceof`.
+
+## Development
+
+```bash
+npm ci
+npm run build   # type-check + emit dist (ESM + .d.ts)
+npm test        # vitest unit tests
+```
+
+## License
+
+ISC

package/dist/audio/clip.d.ts

@@ -0,0 +1,51 @@
+import type { MultimodalHandle, RawAudio } from "../multimodal.js";
+/** A synthesized, playable speech clip. `samples` is the ORIGINAL (un-pitched) PCM. */
+export type SynthClip = {
+    /** Object URL for the (pitch-baked) blob; revoke it when replacing the clip. Absent when `createUrl: false`. */
+    url: string | null;
+    blob: Blob;
+    /** Original un-pitched PCM, kept so a pitch change can re-render without re-running the model. */
+    samples: Float32Array;
+    sampleRate: number;
+    durationSec: number;
+};
+export type SynthesizeClipOptions = {
+    voice: string;
+    speed?: number;
+    /** Pitch ratio baked into the blob (duration-preserving). Default 1 (blob is the model's own output). */
+    pitch?: number;
+    /** Create an object URL for the blob (default true). Pass false when only PCM/Blob is needed. */
+    createUrl?: boolean;
+};
+/**
+ * Synthesize `text` with a loaded TTS handle and package it as a playable clip. The pitch shift is
+ * baked into the blob (duration-preserving) while `samples` keeps the original PCM so a pitch slider
+ * can re-shift without re-synthesis (see {@link repitchClip}).
+ */
+export declare function synthesizeClip(handle: MultimodalHandle, text: string, options: SynthesizeClipOptions): Promise<SynthClip>;
+/** Package an already-synthesized RawAudio as a clip (same pitch-baking rules as synthesizeClip). */
+export declare function clipFromRawAudio(audio: RawAudio, pitch: number, createUrl?: boolean): SynthClip;
+/**
+ * Re-render a clip from its stored original PCM at a new pitch, revoking the previous object URL.
+ * Lets a pitch control take effect immediately on an existing clip without re-running the model.
+ */
+export declare function repitchClip(clip: SynthClip, pitch: number): SynthClip;
+/** Revoke a clip's object URL (call when discarding or replacing a clip). Safe on URL-less clips. */
+export declare function releaseClip(clip: SynthClip | null | undefined): void;
+/**
+ * Playback rate that applies the speed setting exactly once per engine: Kokoro bakes speed into
+ * synthesis (the audio is already time-scaled), so it plays at 1×; the Supertonic pipeline ignores
+ * generation-time speed, so the speed setting is applied at playback instead.
+ */
+export declare function clipPlaybackRate(handle: MultimodalHandle | null | undefined, speed: number): number;
+/**
+ * Play a clip through an HTMLAudioElement. Resolves when playback STARTS (autoplay rejection is
+ * swallowed), not when it ends — await the element's "ended" event for drain semantics.
+ *
+ * Visualization caveat for hosts: a media element can be captured by exactly ONE
+ * MediaElementAudioSourceNode for its entire life, and closing the AudioContext that source routes
+ * through mutes the element permanently. If you attach an analyzer to `audioEl`, create it once per
+ * element and never dispose it (or its context) while the element is still in use.
+ */
+export declare function playClipThroughElement(audioEl: HTMLAudioElement, clip: SynthClip, playbackRate?: number): Promise<void>;
+//# sourceMappingURL=clip.d.ts.map

package/dist/audio/clip.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clip.d.ts","sourceRoot":"","sources":["../../src/audio/clip.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAInE,uFAAuF;AACvF,MAAM,MAAM,SAAS,GAAG;IACtB,gHAAgH;IAChH,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,IAAI,EAAE,IAAI,CAAC;IACX,kGAAkG;IAClG,OAAO,EAAE,YAAY,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yGAAyG;IACzG,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iGAAiG;IACjG,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,SAAS,CAAC,CAG/H;AAED,qGAAqG;AACrG,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,UAAO,GAAG,SAAS,CAY5F;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAUrE;AAED,qGAAqG;AACrG,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CAEpE;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnG;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAKxH"}

package/dist/audio/clip.js

@@ -0,0 +1,71 @@
+import { synthesize } from "../multimodal.js";
+import { encodeWavBlob, pitchShiftBuffer } from "./pcm.js";
+/**
+ * Synthesize `text` with a loaded TTS handle and package it as a playable clip. The pitch shift is
+ * baked into the blob (duration-preserving) while `samples` keeps the original PCM so a pitch slider
+ * can re-shift without re-synthesis (see {@link repitchClip}).
+ */
+export async function synthesizeClip(handle, text, options) {
+    const audio = await synthesize(handle, text, { voice: options.voice, speed: options.speed ?? 1 });
+    return clipFromRawAudio(audio, options.pitch ?? 1, options.createUrl ?? true);
+}
+/** Package an already-synthesized RawAudio as a clip (same pitch-baking rules as synthesizeClip). */
+export function clipFromRawAudio(audio, pitch, createUrl = true) {
+    const samples = audio.audio;
+    const sampleRate = audio.sampling_rate;
+    const shifted = pitchShiftBuffer(samples, pitch);
+    const blob = pitch === 1 ? audio.toBlob() : encodeWavBlob(shifted, sampleRate);
+    return {
+        url: createUrl ? URL.createObjectURL(blob) : null,
+        blob,
+        samples,
+        sampleRate,
+        durationSec: shifted.length / sampleRate,
+    };
+}
+/**
+ * Re-render a clip from its stored original PCM at a new pitch, revoking the previous object URL.
+ * Lets a pitch control take effect immediately on an existing clip without re-running the model.
+ */
+export function repitchClip(clip, pitch) {
+    const shifted = pitchShiftBuffer(clip.samples, pitch);
+    const blob = encodeWavBlob(shifted, clip.sampleRate);
+    if (clip.url)
+        URL.revokeObjectURL(clip.url);
+    return {
+        ...clip,
+        url: clip.url === null ? null : URL.createObjectURL(blob),
+        blob,
+        durationSec: shifted.length / clip.sampleRate,
+    };
+}
+/** Revoke a clip's object URL (call when discarding or replacing a clip). Safe on URL-less clips. */
+export function releaseClip(clip) {
+    if (clip?.url)
+        URL.revokeObjectURL(clip.url);
+}
+/**
+ * Playback rate that applies the speed setting exactly once per engine: Kokoro bakes speed into
+ * synthesis (the audio is already time-scaled), so it plays at 1×; the Supertonic pipeline ignores
+ * generation-time speed, so the speed setting is applied at playback instead.
+ */
+export function clipPlaybackRate(handle, speed) {
+    return handle?.kind === "tts-kokoro" ? 1 : speed;
+}
+/**
+ * Play a clip through an HTMLAudioElement. Resolves when playback STARTS (autoplay rejection is
+ * swallowed), not when it ends — await the element's "ended" event for drain semantics.
+ *
+ * Visualization caveat for hosts: a media element can be captured by exactly ONE
+ * MediaElementAudioSourceNode for its entire life, and closing the AudioContext that source routes
+ * through mutes the element permanently. If you attach an analyzer to `audioEl`, create it once per
+ * element and never dispose it (or its context) while the element is still in use.
+ */
+export async function playClipThroughElement(audioEl, clip, playbackRate = 1) {
+    if (!clip.url)
+        throw new Error("Clip has no object URL (synthesized with createUrl: false).");
+    audioEl.src = clip.url;
+    audioEl.playbackRate = playbackRate;
+    await audioEl.play().catch(() => undefined);
+}
+//# sourceMappingURL=clip.js.map

package/dist/audio/clip.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clip.js","sourceRoot":"","sources":["../../src/audio/clip.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAsB3D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAwB,EAAE,IAAY,EAAE,OAA8B;IACzG,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;IAClG,OAAO,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;AAChF,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,gBAAgB,CAAC,KAAe,EAAE,KAAa,EAAE,SAAS,GAAG,IAAI;IAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC;IAC5B,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC;IACvC,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAC/E,OAAO;QACL,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACjD,IAAI;QACJ,OAAO;QACP,UAAU;QACV,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,UAAU;KACzC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,IAAe,EAAE,KAAa;IACxD,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACrD,IAAI,IAAI,CAAC,GAAG;QAAE,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,IAAI;QACP,GAAG,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC;QACzD,IAAI;QACJ,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU;KAC9C,CAAC;AACJ,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,WAAW,CAAC,IAAkC;IAC5D,IAAI,IAAI,EAAE,GAAG;QAAE,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAA2C,EAAE,KAAa;IACzF,OAAO,MAAM,EAAE,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACnD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAAyB,EAAE,IAAe,EAAE,YAAY,GAAG,CAAC;IACvG,IAAI,CAAC,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IAC9F,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;IACvB,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;IACpC,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;AAC9C,CAAC"}

package/dist/audio/decode.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Decode an audio file (ArrayBuffer) to a 16 kHz mono Float32Array, matching Transformers.js
+ * `read_audio` output. Browser main-thread only: uses `AudioContext`/`OfflineAudioContext`, which do
+ * not exist in workers or Node — decode on the page and pass the PCM into worker-side consumers.
+ */
+export declare function decodeAudioTo16kMono(buffer: ArrayBuffer): Promise<Float32Array>;
+//# sourceMappingURL=decode.d.ts.map

package/dist/audio/decode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decode.d.ts","sourceRoot":"","sources":["../../src/audio/decode.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAWrF"}

package/dist/audio/decode.js

@@ -0,0 +1,19 @@
+import { AUDIO_SR } from "./pcm.js";
+/**
+ * Decode an audio file (ArrayBuffer) to a 16 kHz mono Float32Array, matching Transformers.js
+ * `read_audio` output. Browser main-thread only: uses `AudioContext`/`OfflineAudioContext`, which do
+ * not exist in workers or Node — decode on the page and pass the PCM into worker-side consumers.
+ */
+export async function decodeAudioTo16kMono(buffer) {
+    const ctx = new AudioContext();
+    // Close the context even when decode rejects (e.g. a header-only blob from an instant start/stop).
+    const decoded = await ctx.decodeAudioData(buffer).finally(() => ctx.close().catch(() => undefined));
+    const offline = new OfflineAudioContext(1, Math.ceil(decoded.duration * AUDIO_SR), AUDIO_SR);
+    const source = offline.createBufferSource();
+    source.buffer = decoded;
+    source.connect(offline.destination);
+    source.start();
+    const rendered = await offline.startRendering();
+    return rendered.getChannelData(0);
+}
+//# sourceMappingURL=decode.js.map

package/dist/audio/decode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decode.js","sourceRoot":"","sources":["../../src/audio/decode.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,MAAmB;IAC5D,MAAM,GAAG,GAAG,IAAI,YAAY,EAAE,CAAC;IAC/B,mGAAmG;IACnG,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IACpG,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAC5C,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC;IACxB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACpC,MAAM,CAAC,KAAK,EAAE,CAAC;IACf,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC;IAChD,OAAO,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/dist/audio/pcm.d.ts

@@ -0,0 +1,13 @@
+/** Sample rate every STT/audio-LLM recipe in the SDK consumes (Whisper-style 16 kHz mono). */
+export declare const AUDIO_SR = 16000;
+/**
+ * Pitch-shift a mono PCM buffer by `ratio` (2 = up one octave, 0.5 = down one octave) while preserving
+ * duration. Time-domain granular overlap-add: Hann-windowed grains hop through the output at a fixed
+ * rate (so timing is unchanged) while each grain is read from the input at `ratio×` speed via linear
+ * interpolation (so pitch scales). A pure Float32Array→Float32Array function — no Web Audio — so pitch
+ * can be applied offline (baked into a clip) and unit-tested independently of a browser.
+ */
+export declare function pitchShiftBuffer(input: Float32Array, ratio: number): Float32Array;
+/** Encode a mono Float32 PCM buffer as a 16-bit PCM WAV Blob (used when pitch is baked into a clip). */
+export declare function encodeWavBlob(samples: Float32Array, sampleRate: number): Blob;
+//# sourceMappingURL=pcm.d.ts.map

package/dist/audio/pcm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pcm.d.ts","sourceRoot":"","sources":["../../src/audio/pcm.ts"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,eAAO,MAAM,QAAQ,QAAQ,CAAC;AAE9B;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAsBjF;AAED,wGAAwG;AACxG,wBAAgB,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CA0B7E"}

package/dist/audio/pcm.js

@@ -0,0 +1,66 @@
+/** Sample rate every STT/audio-LLM recipe in the SDK consumes (Whisper-style 16 kHz mono). */
+export const AUDIO_SR = 16000;
+/**
+ * Pitch-shift a mono PCM buffer by `ratio` (2 = up one octave, 0.5 = down one octave) while preserving
+ * duration. Time-domain granular overlap-add: Hann-windowed grains hop through the output at a fixed
+ * rate (so timing is unchanged) while each grain is read from the input at `ratio×` speed via linear
+ * interpolation (so pitch scales). A pure Float32Array→Float32Array function — no Web Audio — so pitch
+ * can be applied offline (baked into a clip) and unit-tested independently of a browser.
+ */
+export function pitchShiftBuffer(input, ratio) {
+    if (ratio === 1 || input.length === 0)
+        return input;
+    const grain = 1024;
+    const hop = grain / 4; // 75% overlap
+    const output = new Float32Array(input.length);
+    const weight = new Float32Array(input.length);
+    const hann = new Float32Array(grain);
+    for (let i = 0; i < grain; i += 1)
+        hann[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (grain - 1));
+    for (let out = 0; out + grain <= output.length; out += hop) {
+        for (let i = 0; i < grain; i += 1) {
+            const src = out + i * ratio; // grain anchored at `out` (time preserved), read at ratio× (pitch)
+            const i0 = Math.floor(src);
+            if (i0 + 1 >= input.length)
+                break;
+            const frac = src - i0;
+            const sample = input[i0] * (1 - frac) + input[i0 + 1] * frac;
+            output[out + i] += sample * hann[i];
+            weight[out + i] += hann[i];
+        }
+    }
+    for (let i = 0; i < output.length; i += 1)
+        if (weight[i] > 1e-6)
+            output[i] /= weight[i]; // OLA gain normalization
+    return output;
+}
+/** Encode a mono Float32 PCM buffer as a 16-bit PCM WAV Blob (used when pitch is baked into a clip). */
+export function encodeWavBlob(samples, sampleRate) {
+    const buffer = new ArrayBuffer(44 + samples.length * 2);
+    const view = new DataView(buffer);
+    const writeStr = (offset, text) => {
+        for (let i = 0; i < text.length; i += 1)
+            view.setUint8(offset + i, text.charCodeAt(i));
+    };
+    writeStr(0, "RIFF");
+    view.setUint32(4, 36 + samples.length * 2, true);
+    writeStr(8, "WAVE");
+    writeStr(12, "fmt ");
+    view.setUint32(16, 16, true); // PCM fmt chunk size
+    view.setUint16(20, 1, true); // PCM format
+    view.setUint16(22, 1, true); // mono
+    view.setUint32(24, sampleRate, true);
+    view.setUint32(28, sampleRate * 2, true); // byte rate
+    view.setUint16(32, 2, true); // block align
+    view.setUint16(34, 16, true); // bits per sample
+    writeStr(36, "data");
+    view.setUint32(40, samples.length * 2, true);
+    let offset = 44;
+    for (let i = 0; i < samples.length; i += 1) {
+        const clamped = Math.max(-1, Math.min(1, samples[i]));
+        view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
+        offset += 2;
+    }
+    return new Blob([buffer], { type: "audio/wav" });
+}
+//# sourceMappingURL=pcm.js.map

package/dist/audio/pcm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pcm.js","sourceRoot":"","sources":["../../src/audio/pcm.ts"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,CAAC;AAE9B;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAmB,EAAE,KAAa;IACjE,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC;IACnB,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,cAAc;IACrC,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,CAAC;QAAE,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAEnG,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAE,CAAC;QAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,mEAAmE;YAChG,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,EAAE,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,MAAM;YAClC,MAAM,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;YAC7D,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;QAAE,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI;YAAE,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,yBAAyB;IAClH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,aAAa,CAAC,OAAqB,EAAE,UAAkB;IACrE,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,IAAY,EAAQ,EAAE;QACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;YAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC;IACF,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACpB,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACjD,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACrB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,qBAAqB;IACnD,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,aAAa;IAC1C,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO;IACpC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY;IACtD,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,cAAc;IAC3C,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB;IAChD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACrB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/E,MAAM,IAAI,CAAC,CAAC;IACd,CAAC;IACD,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AACnD,CAAC"}

package/dist/audio/recorder.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Press-to-talk microphone recorder: getUserMedia → MediaRecorder with no timeslice (one blob at
+ * stop) → 16 kHz mono PCM. Hardened lifecycle ported from small-ai's pipeline recorder:
+ * - a synchronous pending claim so a double-press cannot start two recorders;
+ * - an inactive-recorder guard (the recorder auto-stops when its track ends — device unplugged,
+ *   permission revoked — and `stop()` would then be a silent no-op whose `onstop` never fires);
+ * - try/finally teardown that always releases the mic tracks and closes the AudioContext.
+ *
+ * The recorder creates an AudioContext + MediaStreamSource so hosts can visualize input levels:
+ * `onStreamReady` receives them (attach an AnalyserNode there). The SDK never renders anything.
+ */
+export type MicRecorderOptions = {
+    /** Called once per take after the mic is live — attach analyzers/visualizers here. */
+    onStreamReady?: (graph: {
+        context: AudioContext;
+        source: MediaStreamAudioSourceNode;
+        stream: MediaStream;
+    }) => void;
+    /** Called during teardown, before the context closes — dispose whatever onStreamReady created. */
+    onStreamEnded?: () => void;
+};
+export declare class MicRecorder {
+    #private;
+    constructor(options?: MicRecorderOptions);
+    /** Whether a take is currently being captured (true between a successful start() and stop()). */
+    get recording(): boolean;
+    /**
+     * Start capturing. No-op (returns false) when already recording or a start is mid-flight.
+     * @throws {MicrophoneError} when the mic cannot be acquired (`permissionDenied` set for denials).
+     */
+    start(): Promise<boolean>;
+    /**
+     * Stop the take and decode it to 16 kHz mono PCM. Returns null when nothing was recorded (no
+     * take in flight, or an empty capture). Always releases the mic, even if stopping throws.
+     */
+    stop(): Promise<Float32Array | null>;
+    /** Abandon a take without decoding it (releases the mic; the audio is discarded). Also aborts a start() still awaiting the permission prompt. */
+    cancel(): Promise<void>;
+}
+//# sourceMappingURL=recorder.d.ts.map

package/dist/audio/recorder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recorder.d.ts","sourceRoot":"","sources":["../../src/audio/recorder.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;GAUG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,sFAAsF;IACtF,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,OAAO,EAAE,YAAY,CAAC;QAAC,MAAM,EAAE,0BAA0B,CAAC;QAAC,MAAM,EAAE,WAAW,CAAA;KAAE,KAAK,IAAI,CAAC;IACpH,kGAAkG;IAClG,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;CAC5B,CAAC;AASF,qBAAa,WAAW;;gBAMV,OAAO,GAAE,kBAAuB;IAI5C,iGAAiG;IACjG,IAAI,SAAS,IAAI,OAAO,CAEvB;IAUD;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;IAqC/B;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IA2B1C,iJAAiJ;IAC3I,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAqB9B"}