@missionsquad/browserai 0.1.0

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@missionsquad/browserai",
+  "version": "0.1.0",
+  "description": "Run small open AI models entirely in the browser with WebGPU: model catalog, capability slots, WebLLM + Transformers.js + Kokoro inference, streaming, schema-constrained JSON, metrics, cache management, and a press-to-talk voice pipeline — as an embeddable, UI-free SDK.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./worker": {
+      "types": "./dist/webllm.worker.d.ts",
+      "default": "./dist/webllm.worker.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "worker-template",
+    "README.md"
+  ],
+  "sideEffects": [
+    "./dist/webllm.worker.js"
+  ],
+  "scripts": {
+    "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"",
+    "build": "npm run clean && tsc --noEmit && tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "prepublishOnly": "npm run build && npm test"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "keywords": [
+    "webgpu",
+    "webllm",
+    "transformers.js",
+    "onnx",
+    "browser",
+    "llm",
+    "stt",
+    "tts",
+    "kokoro",
+    "whisper",
+    "voice-assistant",
+    "local-ai"
+  ],
+  "author": "Mission Squad",
+  "license": "ISC",
+  "dependencies": {
+    "@huggingface/transformers": "4.2.0",
+    "@mlc-ai/web-llm": "0.2.84"
+  },
+  "devDependencies": {
+    "@types/node": "24.10.4",
+    "typescript": "6.0.3",
+    "vitest": "4.1.9"
+  }
+}

package/src/audio/clip.ts

@@ -0,0 +1,94 @@
+import type { MultimodalHandle, RawAudio } from "../multimodal.js";
+import { synthesize } from "../multimodal.js";
+import { encodeWavBlob, pitchShiftBuffer } from "./pcm.js";
+
+/** A synthesized, playable speech clip. `samples` is the ORIGINAL (un-pitched) PCM. */
+export type SynthClip = {
+  /** Object URL for the (pitch-baked) blob; revoke it when replacing the clip. Absent when `createUrl: false`. */
+  url: string | null;
+  blob: Blob;
+  /** Original un-pitched PCM, kept so a pitch change can re-render without re-running the model. */
+  samples: Float32Array;
+  sampleRate: number;
+  durationSec: number;
+};
+
+export type SynthesizeClipOptions = {
+  voice: string;
+  speed?: number;
+  /** Pitch ratio baked into the blob (duration-preserving). Default 1 (blob is the model's own output). */
+  pitch?: number;
+  /** Create an object URL for the blob (default true). Pass false when only PCM/Blob is needed. */
+  createUrl?: boolean;
+};
+
+/**
+ * Synthesize `text` with a loaded TTS handle and package it as a playable clip. The pitch shift is
+ * baked into the blob (duration-preserving) while `samples` keeps the original PCM so a pitch slider
+ * can re-shift without re-synthesis (see {@link repitchClip}).
+ */
+export async function synthesizeClip(handle: MultimodalHandle, text: string, options: SynthesizeClipOptions): Promise<SynthClip> {
+  const audio = await synthesize(handle, text, { voice: options.voice, speed: options.speed ?? 1 });
+  return clipFromRawAudio(audio, options.pitch ?? 1, options.createUrl ?? true);
+}
+
+/** Package an already-synthesized RawAudio as a clip (same pitch-baking rules as synthesizeClip). */
+export function clipFromRawAudio(audio: RawAudio, pitch: number, createUrl = true): SynthClip {
+  const samples = audio.audio;
+  const sampleRate = audio.sampling_rate;
+  const shifted = pitchShiftBuffer(samples, pitch);
+  const blob = pitch === 1 ? audio.toBlob() : encodeWavBlob(shifted, sampleRate);
+  return {
+    url: createUrl ? URL.createObjectURL(blob) : null,
+    blob,
+    samples,
+    sampleRate,
+    durationSec: shifted.length / sampleRate,
+  };
+}
+
+/**
+ * Re-render a clip from its stored original PCM at a new pitch, revoking the previous object URL.
+ * Lets a pitch control take effect immediately on an existing clip without re-running the model.
+ */
+export function repitchClip(clip: SynthClip, pitch: number): SynthClip {
+  const shifted = pitchShiftBuffer(clip.samples, pitch);
+  const blob = encodeWavBlob(shifted, clip.sampleRate);
+  if (clip.url) URL.revokeObjectURL(clip.url);
+  return {
+    ...clip,
+    url: clip.url === null ? null : URL.createObjectURL(blob),
+    blob,
+    durationSec: shifted.length / clip.sampleRate,
+  };
+}
+
+/** Revoke a clip's object URL (call when discarding or replacing a clip). Safe on URL-less clips. */
+export function releaseClip(clip: SynthClip | null | undefined): void {
+  if (clip?.url) URL.revokeObjectURL(clip.url);
+}
+
+/**
+ * Playback rate that applies the speed setting exactly once per engine: Kokoro bakes speed into
+ * synthesis (the audio is already time-scaled), so it plays at 1×; the Supertonic pipeline ignores
+ * generation-time speed, so the speed setting is applied at playback instead.
+ */
+export function clipPlaybackRate(handle: MultimodalHandle | null | undefined, speed: number): number {
+  return handle?.kind === "tts-kokoro" ? 1 : speed;
+}
+
+/**
+ * Play a clip through an HTMLAudioElement. Resolves when playback STARTS (autoplay rejection is
+ * swallowed), not when it ends — await the element's "ended" event for drain semantics.
+ *
+ * Visualization caveat for hosts: a media element can be captured by exactly ONE
+ * MediaElementAudioSourceNode for its entire life, and closing the AudioContext that source routes
+ * through mutes the element permanently. If you attach an analyzer to `audioEl`, create it once per
+ * element and never dispose it (or its context) while the element is still in use.
+ */
+export async function playClipThroughElement(audioEl: HTMLAudioElement, clip: SynthClip, playbackRate = 1): Promise<void> {
+  if (!clip.url) throw new Error("Clip has no object URL (synthesized with createUrl: false).");
+  audioEl.src = clip.url;
+  audioEl.playbackRate = playbackRate;
+  await audioEl.play().catch(() => undefined);
+}

package/src/audio/decode.ts

@@ -0,0 +1,19 @@
+import { AUDIO_SR } from "./pcm.js";
+
+/**
+ * Decode an audio file (ArrayBuffer) to a 16 kHz mono Float32Array, matching Transformers.js
+ * `read_audio` output. Browser main-thread only: uses `AudioContext`/`OfflineAudioContext`, which do
+ * not exist in workers or Node — decode on the page and pass the PCM into worker-side consumers.
+ */
+export async function decodeAudioTo16kMono(buffer: ArrayBuffer): Promise<Float32Array> {
+  const ctx = new AudioContext();
+  // Close the context even when decode rejects (e.g. a header-only blob from an instant start/stop).
+  const decoded = await ctx.decodeAudioData(buffer).finally(() => ctx.close().catch(() => undefined));
+  const offline = new OfflineAudioContext(1, Math.ceil(decoded.duration * AUDIO_SR), AUDIO_SR);
+  const source = offline.createBufferSource();
+  source.buffer = decoded;
+  source.connect(offline.destination);
+  source.start();
+  const rendered = await offline.startRendering();
+  return rendered.getChannelData(0);
+}

package/src/audio/pcm.ts

@@ -0,0 +1,62 @@
+/** Sample rate every STT/audio-LLM recipe in the SDK consumes (Whisper-style 16 kHz mono). */
+export const AUDIO_SR = 16000;
+
+/**
+ * Pitch-shift a mono PCM buffer by `ratio` (2 = up one octave, 0.5 = down one octave) while preserving
+ * duration. Time-domain granular overlap-add: Hann-windowed grains hop through the output at a fixed
+ * rate (so timing is unchanged) while each grain is read from the input at `ratio×` speed via linear
+ * interpolation (so pitch scales). A pure Float32Array→Float32Array function — no Web Audio — so pitch
+ * can be applied offline (baked into a clip) and unit-tested independently of a browser.
+ */
+export function pitchShiftBuffer(input: Float32Array, ratio: number): Float32Array {
+  if (ratio === 1 || input.length === 0) return input;
+  const grain = 1024;
+  const hop = grain / 4; // 75% overlap
+  const output = new Float32Array(input.length);
+  const weight = new Float32Array(input.length);
+  const hann = new Float32Array(grain);
+  for (let i = 0; i < grain; i += 1) hann[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (grain - 1));
+
+  for (let out = 0; out + grain <= output.length; out += hop) {
+    for (let i = 0; i < grain; i += 1) {
+      const src = out + i * ratio; // grain anchored at `out` (time preserved), read at ratio× (pitch)
+      const i0 = Math.floor(src);
+      if (i0 + 1 >= input.length) break;
+      const frac = src - i0;
+      const sample = input[i0] * (1 - frac) + input[i0 + 1] * frac;
+      output[out + i] += sample * hann[i];
+      weight[out + i] += hann[i];
+    }
+  }
+  for (let i = 0; i < output.length; i += 1) if (weight[i] > 1e-6) output[i] /= weight[i]; // OLA gain normalization
+  return output;
+}
+
+/** Encode a mono Float32 PCM buffer as a 16-bit PCM WAV Blob (used when pitch is baked into a clip). */
+export function encodeWavBlob(samples: Float32Array, sampleRate: number): Blob {
+  const buffer = new ArrayBuffer(44 + samples.length * 2);
+  const view = new DataView(buffer);
+  const writeStr = (offset: number, text: string): void => {
+    for (let i = 0; i < text.length; i += 1) view.setUint8(offset + i, text.charCodeAt(i));
+  };
+  writeStr(0, "RIFF");
+  view.setUint32(4, 36 + samples.length * 2, true);
+  writeStr(8, "WAVE");
+  writeStr(12, "fmt ");
+  view.setUint32(16, 16, true); // PCM fmt chunk size
+  view.setUint16(20, 1, true); // PCM format
+  view.setUint16(22, 1, true); // mono
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * 2, true); // byte rate
+  view.setUint16(32, 2, true); // block align
+  view.setUint16(34, 16, true); // bits per sample
+  writeStr(36, "data");
+  view.setUint32(40, samples.length * 2, true);
+  let offset = 44;
+  for (let i = 0; i < samples.length; i += 1) {
+    const clamped = Math.max(-1, Math.min(1, samples[i]));
+    view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
+    offset += 2;
+  }
+  return new Blob([buffer], { type: "audio/wav" });
+}

package/src/audio/recorder.ts

@@ -0,0 +1,146 @@
+import { MicrophoneError } from "../errors.js";
+import { decodeAudioTo16kMono } from "./decode.js";
+
+/**
+ * Press-to-talk microphone recorder: getUserMedia → MediaRecorder with no timeslice (one blob at
+ * stop) → 16 kHz mono PCM. Hardened lifecycle ported from small-ai's pipeline recorder:
+ * - a synchronous pending claim so a double-press cannot start two recorders;
+ * - an inactive-recorder guard (the recorder auto-stops when its track ends — device unplugged,
+ *   permission revoked — and `stop()` would then be a silent no-op whose `onstop` never fires);
+ * - try/finally teardown that always releases the mic tracks and closes the AudioContext.
+ *
+ * The recorder creates an AudioContext + MediaStreamSource so hosts can visualize input levels:
+ * `onStreamReady` receives them (attach an AnalyserNode there). The SDK never renders anything.
+ */
+export type MicRecorderOptions = {
+  /** Called once per take after the mic is live — attach analyzers/visualizers here. */
+  onStreamReady?: (graph: { context: AudioContext; source: MediaStreamAudioSourceNode; stream: MediaStream }) => void;
+  /** Called during teardown, before the context closes — dispose whatever onStreamReady created. */
+  onStreamEnded?: () => void;
+};
+
+type ActiveTake = {
+  ctx: AudioContext;
+  stream: MediaStream;
+  recorder: MediaRecorder;
+  chunks: Blob[];
+};
+
+export class MicRecorder {
+  readonly #options: MicRecorderOptions;
+  #pending = false;
+  #abortPending = false;
+  #take: ActiveTake | null = null;
+
+  constructor(options: MicRecorderOptions = {}) {
+    this.#options = options;
+  }
+
+  /** Whether a take is currently being captured (true between a successful start() and stop()). */
+  get recording(): boolean {
+    return this.#take !== null;
+  }
+
+  #streamEnded(): void {
+    try {
+      this.#options.onStreamEnded?.();
+    } catch {
+      // A throwing host disposal hook must not interrupt mic/context teardown.
+    }
+  }
+
+  /**
+   * Start capturing. No-op (returns false) when already recording or a start is mid-flight.
+   * @throws {MicrophoneError} when the mic cannot be acquired (`permissionDenied` set for denials).
+   */
+  async start(): Promise<boolean> {
+    if (this.#pending || this.#take) return false;
+    this.#pending = true; // claimed synchronously before the getUserMedia await (double-press guard)
+    let stream: MediaStream | null = null;
+    let ctx: AudioContext | null = null;
+    try {
+      stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      if (this.#abortPending) {
+        // cancel() ran while the permission prompt / acquisition was pending — release and bail.
+        this.#abortPending = false;
+        stream.getTracks().forEach((track) => track.stop());
+        return false;
+      }
+      ctx = new AudioContext();
+      const source = ctx.createMediaStreamSource(stream);
+      try {
+        this.#options.onStreamReady?.({ context: ctx, source, stream });
+      } catch {
+        // A throwing host visualization hook must not abort the take.
+      }
+      const recorder = new MediaRecorder(stream);
+      const chunks: Blob[] = [];
+      recorder.ondataavailable = (event) => chunks.push(event.data);
+      recorder.start();
+      this.#take = { ctx, stream, recorder, chunks };
+      return true;
+    } catch (error) {
+      // Release anything acquired before the failure — otherwise the mic indicator stays lit.
+      stream?.getTracks().forEach((track) => track.stop());
+      this.#streamEnded();
+      if (ctx) void ctx.close().catch(() => undefined);
+      throw new MicrophoneError(error);
+    } finally {
+      this.#pending = false;
+    }
+  }
+
+  /**
+   * Stop the take and decode it to 16 kHz mono PCM. Returns null when nothing was recorded (no
+   * take in flight, or an empty capture). Always releases the mic, even if stopping throws.
+   */
+  async stop(): Promise<Float32Array | null> {
+    const take = this.#take;
+    this.#take = null; // claim synchronously so a re-entrant stop can't double-clean
+    if (!take) return null;
+    const { recorder, stream, ctx, chunks } = take;
+    try {
+      await new Promise<void>((resolve) => {
+        // The recorder auto-stops when its track ends (device unplugged, permission revoked). stop()
+        // would then be a silent no-op and onstop would never fire — resolve immediately instead.
+        if (recorder.state === "inactive") {
+          resolve();
+          return;
+        }
+        recorder.onstop = () => resolve();
+        recorder.stop();
+      });
+    } finally {
+      // Always release the mic and context — even if stop() threw (legacy InvalidStateError).
+      stream.getTracks().forEach((track) => track.stop());
+      this.#streamEnded();
+      await ctx.close().catch(() => undefined);
+    }
+    const blob = new Blob(chunks, { type: recorder.mimeType || "audio/webm" });
+    if (blob.size === 0) return null;
+    return decodeAudioTo16kMono(await blob.arrayBuffer());
+  }
+
+  /** Abandon a take without decoding it (releases the mic; the audio is discarded). Also aborts a start() still awaiting the permission prompt. */
+  async cancel(): Promise<void> {
+    if (this.#pending && !this.#take) this.#abortPending = true; // start() releases the mic when getUserMedia resolves
+    const take = this.#take;
+    this.#take = null;
+    if (!take) return;
+    const { recorder, stream, ctx } = take;
+    try {
+      await new Promise<void>((resolve) => {
+        if (recorder.state === "inactive") {
+          resolve();
+          return;
+        }
+        recorder.onstop = () => resolve();
+        recorder.stop();
+      });
+    } finally {
+      stream.getTracks().forEach((track) => track.stop());
+      this.#streamEnded();
+      await ctx.close().catch(() => undefined);
+    }
+  }
+}

package/src/audio/streaming-player.ts

@@ -0,0 +1,99 @@
+export type StreamingAudioPlayer = {
+  /** Schedule one PCM chunk to play back-to-back after everything already scheduled. */
+  push(samples: Float32Array, sampleRate: number): void;
+  /** No more chunks; resolves (and tears down) once everything scheduled has finished playing. */
+  end(): Promise<void>;
+  /** Barge-in/error: stop playback now and tear down. Idempotent. */
+  stop(): void;
+};
+
+export type StreamingAudioPlayerOptions = {
+  /**
+   * Called once with the player's audio graph so hosts can tap it for visualization (e.g. connect an
+   * AnalyserNode to `gain`). The audible path is gain → destination only — do not connect the tap to
+   * the destination or audio doubles.
+   */
+  onGraphReady?: (graph: { context: AudioContext; gain: GainNode }) => void;
+  /** Called during teardown, before the context closes — dispose whatever onGraphReady created. */
+  onTeardown?: () => void;
+};
+
+/**
+ * Gapless player for streamed TTS: schedules each chunk's PCM into one AudioContext end-to-end.
+ * Chunks are butt-joined at a running cursor; if synthesis falls behind playback this yields a small
+ * silent gap rather than overlap. Teardown (close ctx) is idempotent, and `stop()` resolves any
+ * in-flight `end()` promise so a barge-in can never hang a waiting turn.
+ */
+export function createStreamingAudioPlayer(options: StreamingAudioPlayerOptions = {}): StreamingAudioPlayer {
+  const ctx = new AudioContext();
+  const gain = ctx.createGain();
+  gain.connect(ctx.destination);
+  try {
+    options.onGraphReady?.({ context: ctx, gain });
+  } catch {
+    // A throwing host visualization tap must not break (or leak) the player.
+  }
+
+  let nextTime = ctx.currentTime;
+  let pending = 0;
+  let ended = false;
+  let torn = false;
+  let onDrained: (() => void) | null = null;
+
+  const teardown = (): void => {
+    if (torn) return;
+    torn = true;
+    try {
+      options.onTeardown?.();
+    } catch {
+      // Host tap disposal errors must not prevent the context from closing.
+    }
+    void ctx.close().catch(() => undefined);
+  };
+  const maybeDrain = (): void => {
+    if (ended && pending === 0) {
+      teardown();
+      onDrained?.();
+      onDrained = null;
+    }
+  };
+
+  return {
+    push(samples, sampleRate) {
+      if (torn || samples.length === 0) return;
+      // The context is often created outside a click call stack — resume defensively (autoplay policy).
+      if (ctx.state === "suspended") void ctx.resume().catch(() => undefined);
+      const buffer = ctx.createBuffer(1, samples.length, sampleRate);
+      buffer.getChannelData(0).set(samples);
+      const source = ctx.createBufferSource();
+      source.buffer = buffer;
+      source.connect(gain);
+      // If synthesis is slower than playback this yields a small silent gap rather than overlap.
+      const startAt = Math.max(nextTime, ctx.currentTime);
+      pending += 1;
+      source.onended = () => {
+        pending -= 1;
+        maybeDrain();
+      };
+      source.start(startAt);
+      nextTime = startAt + buffer.duration;
+    },
+    end() {
+      ended = true;
+      return new Promise<void>((resolve) => {
+        if (pending === 0 || torn) {
+          teardown();
+          resolve();
+          return;
+        }
+        onDrained = resolve;
+      });
+    },
+    stop() {
+      ended = true;
+      teardown();
+      onDrained?.();
+      onDrained = null;
+    },
+  };
+}