@djangocfg/ui-tools 2.1.381 → 2.1.382

package/src/tools/SpeechRecognition/core/engine/http.ts

@@ -0,0 +1,147 @@
+/**
+ * HTTP engine — records audio with MediaRecorder and POSTs each chunk to
+ * a host-supplied URL. The host owns response parsing via `parse()`, so
+ * this engine works with OpenAI Whisper REST, custom Django/FastAPI
+ * endpoints, or anything else that takes audio and returns text.
+ */
+
+import { newSegmentId } from '../ids';
+import { sttLogger } from '../logger';
+import { createEngineBus } from './index';
+import { startMicCapture, type MicCaptureHandle } from './mediarecorder';
+import type {
+  EngineStartOptions,
+  RecognitionEngine,
+  RecognitionError,
+  Unsub,
+} from '../../types';
+
+export interface HttpEngineParseResult {
+  text: string;
+  isFinal: boolean;
+  /** Optional engine-provided confidence 0..1. */
+  confidence?: number;
+}
+
+export interface HttpEngineOptions {
+  /** Endpoint URL. Receives `POST` with the audio chunk as the body. */
+  url: string | ((language: string) => string);
+  /** Per-request headers, awaited each chunk so tokens can be refreshed. */
+  headers?: () => Promise<Record<string, string>> | Record<string, string>;
+  /** Chunk emission interval, ms. Default 750 — long enough for useful audio. */
+  chunkMs?: number;
+  /** Preferred MIME for the encoder. Probed against `MediaRecorder` support. */
+  mime?: string;
+  /** Parse the engine response — return null/undefined to skip emit. */
+  parse: (
+    resp: Response,
+  ) => Promise<HttpEngineParseResult | null | undefined> | HttpEngineParseResult | null | undefined;
+  /** Stable engine id for telemetry / UI badge. Default 'http'. */
+  id?: string;
+}
+
+export function createHttpEngine(opts: HttpEngineOptions): RecognitionEngine {
+  const bus = createEngineBus();
+  let capture: MicCaptureHandle | null = null;
+  let currentSegmentId: string | null = null;
+  let ctrl: AbortController | null = null;
+  let stopping = false;
+
+  async function sendChunk(blob: Blob, language: string): Promise<void> {
+    if (stopping) return;
+    const url = typeof opts.url === 'function' ? opts.url(language) : opts.url;
+    const headers = (await opts.headers?.()) ?? {};
+    try {
+      const resp = await fetch(url, {
+        method: 'POST',
+        headers,
+        body: blob,
+        signal: ctrl?.signal,
+      });
+      if (!resp.ok) {
+        bus.emit('error', {
+          code: 'network',
+          message: `STT endpoint returned ${resp.status}`,
+        });
+        return;
+      }
+      const parsed = await opts.parse(resp);
+      if (!parsed || !parsed.text) return;
+      if (!currentSegmentId) currentSegmentId = newSegmentId();
+      if (parsed.isFinal) {
+        bus.emit('final', parsed.text, currentSegmentId, parsed.confidence);
+        currentSegmentId = null;
+      } else {
+        bus.emit('partial', parsed.text, currentSegmentId);
+      }
+    } catch (cause) {
+      if ((cause as { name?: string })?.name === 'AbortError') return;
+      sttLogger.warn('[http] chunk send failed', cause);
+      bus.emit('error', {
+        code: 'network',
+        message: 'Failed to deliver audio chunk to STT endpoint.',
+        cause,
+      });
+    }
+  }
+
+  return {
+    id: opts.id ?? 'http',
+    isSupported:
+      typeof navigator !== 'undefined' &&
+      !!navigator.mediaDevices?.getUserMedia &&
+      typeof MediaRecorder !== 'undefined',
+    on(event, cb): Unsub {
+      return bus.on(event, cb);
+    },
+    async start(start: EngineStartOptions): Promise<void> {
+      if (capture) return;
+      stopping = false;
+      ctrl = new AbortController();
+      bus.emit('state', 'connecting');
+      try {
+        capture = await startMicCapture({
+          deviceId: start.deviceId,
+          mime: opts.mime,
+          chunkMs: opts.chunkMs ?? 750,
+          onChunk: (chunk) => {
+            void sendChunk(chunk, start.language);
+          },
+          onError: (err) => bus.emit('error', err),
+        });
+        bus.emit('state', 'listening');
+      } catch (cause) {
+        const err = cause as RecognitionError;
+        bus.emit('error', err);
+        bus.emit('state', 'error');
+        throw err;
+      }
+
+      start.signal?.addEventListener('abort', () => {
+        void this.stop();
+      });
+    },
+    async stop(): Promise<void> {
+      stopping = true;
+      bus.emit('state', 'closing');
+      ctrl?.abort();
+      ctrl = null;
+      await capture?.stop();
+      capture = null;
+      currentSegmentId = null;
+      bus.emit('state', 'closed');
+    },
+    abort(): void {
+      stopping = true;
+      ctrl?.abort();
+      ctrl = null;
+      capture?.stop().catch(() => undefined);
+      capture = null;
+      currentSegmentId = null;
+      bus.emit('state', 'closed');
+    },
+    getStream(): MediaStream | null {
+      return capture?.stream ?? null;
+    },
+  };
+}

package/src/tools/SpeechRecognition/core/engine/index.ts

@@ -0,0 +1,52 @@
+/**
+ * Tiny event-bus helper shared by every engine. Lets engine authors avoid
+ * re-implementing add/remove listener bookkeeping while keeping the
+ * public `RecognitionEngine.on(...)` contract identical across engines.
+ */
+
+import type { EngineEventMap, Unsub } from '../../types';
+
+type Listeners = {
+  [K in keyof EngineEventMap]: Set<EngineEventMap[K]>;
+};
+
+export function createEngineBus(): {
+  on: <K extends keyof EngineEventMap>(event: K, cb: EngineEventMap[K]) => Unsub;
+  emit: <K extends keyof EngineEventMap>(
+    event: K,
+    ...args: Parameters<EngineEventMap[K]>
+  ) => void;
+  clear: () => void;
+} {
+  const listeners: Listeners = {
+    partial: new Set(),
+    final: new Set(),
+    error: new Set(),
+    state: new Set(),
+  };
+
+  return {
+    on(event, cb) {
+      const set = listeners[event] as Set<typeof cb>;
+      set.add(cb);
+      return () => {
+        set.delete(cb);
+      };
+    },
+    emit(event, ...args) {
+      const set = listeners[event];
+      for (const cb of set) {
+        try {
+          (cb as (...a: unknown[]) => void)(...(args as unknown[]));
+        } catch {
+          // listener errors are isolated — never break the engine loop
+        }
+      }
+    },
+    clear() {
+      for (const key of Object.keys(listeners) as Array<keyof Listeners>) {
+        listeners[key].clear();
+      }
+    },
+  };
+}

package/src/tools/SpeechRecognition/core/engine/mediarecorder.ts

@@ -0,0 +1,105 @@
+/**
+ * Shared mic capture used by the HTTP and WebSocket engines.
+ *
+ * Probes the browser for a working `MediaRecorder` MIME type and emits
+ * `Blob` chunks on a steady interval. Picks the first supported MIME in
+ * order: `audio/webm;codecs=opus` → `audio/ogg;codecs=opus` →
+ * `audio/mp4;codecs=mp4a`. Falls back to engine default if none match.
+ *
+ * The capture also exposes the raw `MediaStream` so callers can wire up
+ * an `AnalyserNode` for the level meter without owning a second copy.
+ */
+
+import { sttLogger } from '../logger';
+import type { RecognitionError } from '../../types';
+
+const PREFERRED_MIMES = [
+  'audio/webm;codecs=opus',
+  'audio/ogg;codecs=opus',
+  'audio/mp4;codecs=mp4a',
+  'audio/webm',
+];
+
+export function pickMime(preferred?: string): string | undefined {
+  if (typeof MediaRecorder === 'undefined') return undefined;
+  const candidates = preferred ? [preferred, ...PREFERRED_MIMES] : PREFERRED_MIMES;
+  for (const mime of candidates) {
+    if (MediaRecorder.isTypeSupported(mime)) return mime;
+  }
+  return undefined;
+}
+
+export interface MicCaptureOptions {
+  deviceId?: string;
+  /** Override probed MIME — useful when the backend expects a specific codec. */
+  mime?: string;
+  /** Chunk emission interval, ms. Default 250. */
+  chunkMs?: number;
+  onChunk: (chunk: Blob) => void;
+  onError?: (err: RecognitionError) => void;
+}
+
+export interface MicCaptureHandle {
+  readonly stream: MediaStream;
+  readonly mime: string | undefined;
+  stop(): Promise<void>;
+}
+
+function toErr(code: RecognitionError['code'], message: string, cause?: unknown): RecognitionError {
+  return { code, message, cause };
+}
+
+export async function startMicCapture(
+  opts: MicCaptureOptions,
+): Promise<MicCaptureHandle> {
+  if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
+    throw toErr('unsupported', 'getUserMedia is not available in this environment.');
+  }
+  if (typeof MediaRecorder === 'undefined') {
+    throw toErr('unsupported', 'MediaRecorder is not available in this environment.');
+  }
+
+  let stream: MediaStream;
+  try {
+    stream = await navigator.mediaDevices.getUserMedia({
+      audio: opts.deviceId ? { deviceId: { exact: opts.deviceId } } : true,
+      video: false,
+    });
+  } catch (cause) {
+    const name = (cause as { name?: string })?.name;
+    if (name === 'NotAllowedError' || name === 'SecurityError') {
+      throw toErr('permission-denied', 'Microphone permission denied.', cause);
+    }
+    if (name === 'NotFoundError' || name === 'OverconstrainedError') {
+      throw toErr('no-microphone', 'No microphone found matching the constraints.', cause);
+    }
+    throw toErr('unknown', 'Failed to access microphone.', cause);
+  }
+
+  const mime = pickMime(opts.mime);
+  const rec = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
+
+  rec.ondataavailable = (e) => {
+    if (e.data && e.data.size > 0) opts.onChunk(e.data);
+  };
+  rec.onerror = (e) => {
+    const err = toErr('engine', 'MediaRecorder error.', e);
+    sttLogger.warn('[capture] recorder error', e);
+    opts.onError?.(err);
+  };
+
+  rec.start(opts.chunkMs ?? 250);
+
+  return {
+    stream,
+    mime: mime ?? rec.mimeType,
+    async stop() {
+      const done = new Promise<void>((resolve) => {
+        rec.addEventListener('stop', () => resolve(), { once: true });
+      });
+      if (rec.state !== 'inactive') rec.stop();
+      stream.getTracks().forEach((t) => t.stop());
+      await done;
+    },
+  };
+}

package/src/tools/SpeechRecognition/core/engine/websocket.ts

@@ -0,0 +1,211 @@
+/**
+ * WebSocket engine — pushes recorded audio frames over a persistent socket
+ * and parses server responses through a host-supplied `parseMessage`
+ * callback. Works with Deepgram / AssemblyAI realtime endpoints or any
+ * custom gateway that speaks JSON or binary frames.
+ *
+ * Reconnect: simple exponential backoff capped at 5 s; the engine emits
+ * `state: 'connecting'` between attempts so UIs can show "reconnecting…".
+ */
+
+import { newSegmentId } from '../ids';
+import { sttLogger } from '../logger';
+import { createEngineBus } from './index';
+import { startMicCapture, type MicCaptureHandle } from './mediarecorder';
+import type {
+  EngineStartOptions,
+  RecognitionEngine,
+  RecognitionError,
+  Unsub,
+} from '../../types';
+
+export type WsParsedEvent =
+  | { kind: 'partial'; text: string; segmentId?: string; confidence?: number }
+  | { kind: 'final'; text: string; segmentId?: string; confidence?: number }
+  | { kind: 'error'; error: RecognitionError }
+  | { kind: 'ignore' };
+
+export interface WebSocketEngineOptions {
+  url: string | ((language: string) => Promise<string> | string);
+  protocols?: string[];
+  /** Chunk emission interval, ms. Default 250 for realtime feel. */
+  chunkMs?: number;
+  mime?: string;
+  /** Parse one frame (string or binary) into our normalised event shape. */
+  parseMessage: (data: string | ArrayBuffer) => WsParsedEvent;
+  /** Stable engine id for telemetry / UI badge. Default 'websocket'. */
+  id?: string;
+  /** Max reconnect attempts before giving up. Default 5. */
+  maxReconnect?: number;
+}
+
+const MIN_BACKOFF = 250;
+const MAX_BACKOFF = 5000;
+
+export function createWebSocketEngine(
+  opts: WebSocketEngineOptions,
+): RecognitionEngine {
+  const bus = createEngineBus();
+  let socket: WebSocket | null = null;
+  let capture: MicCaptureHandle | null = null;
+  let currentSegmentId: string | null = null;
+  let stopping = false;
+  let attempts = 0;
+
+  function emitParsed(parsed: WsParsedEvent): void {
+    switch (parsed.kind) {
+      case 'partial': {
+        const id = parsed.segmentId ?? currentSegmentId ?? newSegmentId();
+        currentSegmentId = id;
+        bus.emit('partial', parsed.text, id);
+        return;
+      }
+      case 'final': {
+        const id = parsed.segmentId ?? currentSegmentId ?? newSegmentId();
+        bus.emit('final', parsed.text, id, parsed.confidence);
+        currentSegmentId = null;
+        return;
+      }
+      case 'error':
+        bus.emit('error', parsed.error);
+        return;
+      case 'ignore':
+      default:
+        return;
+    }
+  }
+
+  async function openSocket(language: string): Promise<WebSocket> {
+    const url =
+      typeof opts.url === 'function' ? await opts.url(language) : opts.url;
+    const ws = new WebSocket(url, opts.protocols);
+    ws.binaryType = 'arraybuffer';
+    return ws;
+  }
+
+  async function connect(start: EngineStartOptions): Promise<void> {
+    if (stopping) return;
+    bus.emit('state', 'connecting');
+    let ws: WebSocket;
+    try {
+      ws = await openSocket(start.language);
+    } catch (cause) {
+      bus.emit('error', {
+        code: 'network',
+        message: 'Failed to open STT socket.',
+        cause,
+      });
+      return;
+    }
+    socket = ws;
+
+    ws.onopen = () => {
+      attempts = 0;
+      bus.emit('state', 'listening');
+    };
+    ws.onmessage = (e) => {
+      try {
+        const parsed = opts.parseMessage(e.data as string | ArrayBuffer);
+        emitParsed(parsed);
+      } catch (cause) {
+        sttLogger.warn('[ws] parseMessage threw', cause);
+      }
+    };
+    ws.onerror = () => {
+      bus.emit('error', { code: 'network', message: 'STT socket error.' });
+    };
+    ws.onclose = () => {
+      socket = null;
+      if (stopping) {
+        bus.emit('state', 'closed');
+        return;
+      }
+      attempts += 1;
+      const max = opts.maxReconnect ?? 5;
+      if (attempts > max) {
+        bus.emit('error', {
+          code: 'network',
+          message: `STT socket closed; gave up after ${max} attempts.`,
+        });
+        bus.emit('state', 'closed');
+        return;
+      }
+      const delay = Math.min(MIN_BACKOFF * 2 ** (attempts - 1), MAX_BACKOFF);
+      setTimeout(() => {
+        void connect(start);
+      }, delay);
+    };
+  }
+
+  return {
+    id: opts.id ?? 'websocket',
+    isSupported:
+      typeof WebSocket !== 'undefined' &&
+      typeof navigator !== 'undefined' &&
+      !!navigator.mediaDevices?.getUserMedia &&
+      typeof MediaRecorder !== 'undefined',
+    on(event, cb): Unsub {
+      return bus.on(event, cb);
+    },
+    async start(start: EngineStartOptions): Promise<void> {
+      if (capture) return;
+      stopping = false;
+      attempts = 0;
+      try {
+        capture = await startMicCapture({
+          deviceId: start.deviceId,
+          mime: opts.mime,
+          chunkMs: opts.chunkMs ?? 250,
+          onChunk: (chunk) => {
+            if (socket?.readyState === WebSocket.OPEN) {
+              chunk
+                .arrayBuffer()
+                .then((buf) => socket?.send(buf))
+                .catch((cause) => sttLogger.warn('[ws] send failed', cause));
+            }
+          },
+          onError: (err) => bus.emit('error', err),
+        });
+      } catch (cause) {
+        const err = cause as RecognitionError;
+        bus.emit('error', err);
+        bus.emit('state', 'error');
+        throw err;
+      }
+      await connect(start);
+      start.signal?.addEventListener('abort', () => {
+        void this.stop();
+      });
+    },
+    async stop(): Promise<void> {
+      stopping = true;
+      bus.emit('state', 'closing');
+      try {
+        socket?.close(1000, 'client-stop');
+      } catch {
+        // ignore
+      }
+      socket = null;
+      await capture?.stop();
+      capture = null;
+      currentSegmentId = null;
+      bus.emit('state', 'closed');
+    },
+    abort(): void {
+      stopping = true;
+      try {
+        socket?.close(4000, 'client-abort');
+      } catch {
+        // ignore
+      }
+      socket = null;
+      capture?.stop().catch(() => undefined);
+      capture = null;
+      currentSegmentId = null;
+      bus.emit('state', 'closed');
+    },
+    getStream(): MediaStream | null {
+      return capture?.stream ?? null;
+    },
+  };
+}

package/src/tools/SpeechRecognition/core/engine/webspeech.ts

@@ -0,0 +1,188 @@
+/**
+ * Default engine — wraps the browser's `SpeechRecognition` API.
+ *
+ * Lives behind the same `RecognitionEngine` contract every other engine
+ * implements. When the browser doesn't expose `SpeechRecognition`
+ * (Firefox, some mobile WebViews) `isSupported` is `false` and `start()`
+ * throws an `unsupported` error.
+ */
+
+import { newSegmentId } from '../ids';
+import { sttLogger } from '../logger';
+import { createEngineBus } from './index';
+import type {
+  EngineStartOptions,
+  RecognitionEngine,
+  RecognitionError,
+  RecognitionErrorCode,
+  Unsub,
+} from '../../types';
+
+// Minimal subset of the Web Speech API we actually rely on. Browsers
+// expose either `SpeechRecognition` (Edge / Safari new) or the older
+// `webkitSpeechRecognition` (Chrome). Both share the same shape.
+interface BrowserSpeechRecognition extends EventTarget {
+  lang: string;
+  interimResults: boolean;
+  continuous: boolean;
+  maxAlternatives: number;
+  start(): void;
+  stop(): void;
+  abort(): void;
+  onresult: ((e: BrowserSpeechRecognitionEvent) => void) | null;
+  onerror: ((e: BrowserSpeechRecognitionError) => void) | null;
+  onstart: (() => void) | null;
+  onend: (() => void) | null;
+}
+
+interface BrowserSpeechRecognitionResult {
+  isFinal: boolean;
+  0: { transcript: string; confidence: number };
+}
+
+interface BrowserSpeechRecognitionEvent extends Event {
+  resultIndex: number;
+  results: ArrayLike<BrowserSpeechRecognitionResult>;
+}
+
+interface BrowserSpeechRecognitionError extends Event {
+  error: string;
+  message?: string;
+}
+
+type Ctor = new () => BrowserSpeechRecognition;
+
+function resolveCtor(): Ctor | null {
+  if (typeof window === 'undefined') return null;
+  const w = window as unknown as {
+    SpeechRecognition?: Ctor;
+    webkitSpeechRecognition?: Ctor;
+  };
+  return w.SpeechRecognition ?? w.webkitSpeechRecognition ?? null;
+}
+
+const ERROR_MAP: Record<string, RecognitionErrorCode> = {
+  'no-speech': 'no-speech',
+  aborted: 'aborted',
+  'audio-capture': 'no-microphone',
+  network: 'network',
+  'not-allowed': 'permission-denied',
+  'service-not-allowed': 'permission-denied',
+  'bad-grammar': 'engine',
+  'language-not-supported': 'language',
+};
+
+export interface WebSpeechEngineOptions {
+  /** Whether the underlying recognition should be continuous. Default true. */
+  continuous?: boolean;
+  /** Max alternatives the engine should request. Default 1. */
+  maxAlternatives?: number;
+}
+
+export function createWebSpeechEngine(
+  opts: WebSpeechEngineOptions = {},
+): RecognitionEngine {
+  const Ctor = resolveCtor();
+  const bus = createEngineBus();
+  let instance: BrowserSpeechRecognition | null = null;
+  let currentSegmentId: string | null = null;
+
+  function teardown(): void {
+    if (!instance) return;
+    instance.onresult = null;
+    instance.onerror = null;
+    instance.onstart = null;
+    instance.onend = null;
+    instance = null;
+    currentSegmentId = null;
+  }
+
+  return {
+    id: 'webspeech',
+    isSupported: Ctor !== null,
+    on(event, cb): Unsub {
+      return bus.on(event, cb);
+    },
+    async start(start: EngineStartOptions): Promise<void> {
+      if (!Ctor) {
+        const err: RecognitionError = {
+          code: 'unsupported',
+          message: 'Web Speech API is not available in this browser.',
+        };
+        bus.emit('error', err);
+        throw err;
+      }
+      if (instance) {
+        sttLogger.debug('[webspeech] start() called while running — ignoring');
+        return;
+      }
+
+      bus.emit('state', 'connecting');
+
+      const rec = new Ctor();
+      rec.lang = start.language;
+      rec.interimResults = start.interim;
+      rec.continuous = opts.continuous ?? true;
+      rec.maxAlternatives = opts.maxAlternatives ?? 1;
+
+      rec.onstart = () => {
+        bus.emit('state', 'listening');
+      };
+      rec.onend = () => {
+        bus.emit('state', 'closed');
+        teardown();
+      };
+      rec.onerror = (e) => {
+        const code = ERROR_MAP[e.error] ?? 'engine';
+        const err: RecognitionError = {
+          code,
+          message: e.message || `Web Speech error: ${e.error}`,
+        };
+        bus.emit('error', err);
+      };
+      rec.onresult = (e) => {
+        for (let i = e.resultIndex; i < e.results.length; i += 1) {
+          const res = e.results[i];
+          const alt = res[0];
+          const text = alt.transcript;
+          if (!currentSegmentId) currentSegmentId = newSegmentId();
+          if (res.isFinal) {
+            bus.emit('final', text, currentSegmentId, alt.confidence);
+            currentSegmentId = null;
+          } else {
+            bus.emit('partial', text, currentSegmentId);
+          }
+        }
+      };
+
+      if (start.signal) {
+        start.signal.addEventListener('abort', () => {
+          rec.abort();
+        });
+      }
+
+      instance = rec;
+      try {
+        rec.start();
+      } catch (cause) {
+        const err: RecognitionError = {
+          code: 'engine',
+          message: 'Failed to start Web Speech recognition.',
+          cause,
+        };
+        bus.emit('error', err);
+        teardown();
+        throw err;
+      }
+    },
+    async stop(): Promise<void> {
+      if (!instance) return;
+      bus.emit('state', 'closing');
+      instance.stop();
+    },
+    abort(): void {
+      if (!instance) return;
+      instance.abort();
+    },
+  };
+}