@djangocfg/ui-tools 2.1.381 → 2.1.383

package/src/tools/SpeechRecognition/core/engine/external.ts

@@ -0,0 +1,222 @@
+/**
+ * External-controlled engine — for backends that own the entire
+ * record / decode / transcribe pipeline (Wails / Tauri / native
+ * sidecar). The frontend exposes "start" and "stop" verbs to the
+ * native layer, which then pushes a single `final` (or rolling
+ * `partial`s + `final`) back through events.
+ *
+ * Use this when:
+ *   - Audio capture lives outside the browser (cmdop_go OS-wide
+ *     hotkey, system audio device claims, etc.).
+ *   - Transcription runs on the backend (whisper.cpp, Vosk, custom
+ *     ONNX) and the browser never sees the raw audio.
+ *
+ * Compared to `createHttpEngine`/`createWebSocketEngine`:
+ *   - No `MediaRecorder` / `getUserMedia` involvement.
+ *   - `isSupported` defaults to `true` (the host knows whether the
+ *     native side is present — let it gate via `supported`).
+ *   - No `MediaStream` to expose, so the VU meter falls through to
+ *     the host's own level event (cmdop wires it via a separate hook).
+ */
+
+import { newSegmentId } from '../ids';
+import { sttLogger } from '../logger';
+import { createEngineBus } from './index';
+import type {
+  EngineStartOptions,
+  RecognitionEngine,
+  RecognitionError,
+  Unsub,
+} from '../../types';
+
+export interface ExternalEngineHandle {
+  /**
+   * Push an interim transcript fragment. Wrapped into a `partial`
+   * event with a generated segment id. Subsequent calls before
+   * `emitFinal` mutate the same interim segment.
+   */
+  emitPartial(text: string): void;
+  /**
+   * Push the final transcript. Closes the current segment; the engine
+   * emits `final` then transitions to `closed`.
+   */
+  emitFinal(text: string, confidence?: number): void;
+  /** Surface a backend error. Engine transitions to `closed`. */
+  emitError(err: RecognitionError): void;
+  /**
+   * Notify the engine that the native side actually started capturing.
+   * Flips status to `listening`. Call from a backend `recording`
+   * event. Optional — see `autoMarkListening`.
+   */
+  markListening(): void;
+}
+
+export interface ExternalEngineOptions {
+  /** Stable engine id for telemetry / UI badge. */
+  id?: string;
+  /**
+   * Whether the host believes the native side is available. Wire to
+   * a Wails ping / Tauri capability check. Defaults to `true`.
+   */
+  supported?: boolean;
+  /** Ask the backend to start capture. */
+  onStart: (opts: EngineStartOptions) => Promise<void> | void;
+  /** Ask the backend to stop capture (and finalise the buffer). */
+  onStop: () => Promise<void> | void;
+  /**
+   * Optional hard cancel — `onStop` may finalise, while `onAbort`
+   * discards. Falls back to `onStop` when omitted.
+   */
+  onAbort?: () => Promise<void> | void;
+  /**
+   * Subscribe to backend events. Called once per `start()`. The host
+   * wires its native event source (Wails `EventsOn`, Tauri
+   * `appWindow.listen`, …) and uses the supplied `handle` to push
+   * transcript fragments through the engine bus.
+   *
+   * Must return an unsubscribe function so the engine can detach on
+   * teardown.
+   */
+  subscribe: (handle: ExternalEngineHandle) => Unsub;
+  /**
+   * If `true` (default), the engine flips state to `listening` right
+   * after `onStart` resolves. Set `false` and call
+   * `handle.markListening()` explicitly when you want to wait for the
+   * native side to confirm the capture session opened.
+   */
+  autoMarkListening?: boolean;
+}
+
+/**
+ * Wraps a backend-driven STT pipeline into the standard
+ * `RecognitionEngine` shape so it works with `useSpeechRecognition`,
+ * `VoiceComposerSlot`, and every other piece of the SpeechRecognition
+ * tool.
+ *
+ * Example (cmdop Wails):
+ *
+ * ```ts
+ * import { EventsOn } from '@runtime';
+ * import * as VoiceService from '@bindings/desktop/services/voice/service';
+ *
+ * const engine = createExternalEngine({
+ *   id: 'wails-whisper',
+ *   onStart: () => VoiceService.StartRecordingForChat(),
+ *   onStop: () => VoiceService.StopRecordingForChat(),
+ *   subscribe: (handle) => {
+ *     const offText = EventsOn('voice:chat-text', (p) => {
+ *       if (p?.error) handle.emitError({ code: 'engine', message: p.error });
+ *       else if (p?.text) handle.emitFinal(p.text);
+ *       else handle.emitError({ code: 'no-speech', message: '' });
+ *     });
+ *     const offState = EventsOn('voice:state', (s) => {
+ *       if (s.state === 'recording' || s.state === 'streaming') {
+ *         handle.markListening();
+ *       }
+ *       if (s.partial) handle.emitPartial(s.partial);
+ *     });
+ *     return () => { offText(); offState(); };
+ *   },
+ * });
+ * ```
+ */
+export function createExternalEngine(
+  opts: ExternalEngineOptions,
+): RecognitionEngine {
+  const bus = createEngineBus();
+  let currentSegmentId: string | null = null;
+  let unsubscribe: Unsub | null = null;
+  let running = false;
+
+  function teardown(): void {
+    unsubscribe?.();
+    unsubscribe = null;
+    currentSegmentId = null;
+    running = false;
+  }
+
+  const handle: ExternalEngineHandle = {
+    emitPartial(text: string): void {
+      if (!running) return;
+      if (!currentSegmentId) currentSegmentId = newSegmentId();
+      bus.emit('partial', text, currentSegmentId);
+    },
+    emitFinal(text: string, confidence?: number): void {
+      if (!running) return;
+      const id = currentSegmentId ?? newSegmentId();
+      bus.emit('final', text, id, confidence);
+      // External engines almost always go idle right after their
+      // final — close the session so consumers' `onStop` fires
+      // without requiring a separate `stop()` call.
+      bus.emit('state', 'closed');
+      teardown();
+    },
+    emitError(err: RecognitionError): void {
+      bus.emit('error', err);
+      bus.emit('state', 'closed');
+      teardown();
+    },
+    markListening(): void {
+      if (!running) return;
+      bus.emit('state', 'listening');
+    },
+  };
+
+  return {
+    id: opts.id ?? 'external',
+    isSupported: opts.supported ?? true,
+    on(event, cb): Unsub {
+      return bus.on(event, cb);
+    },
+    async start(start: EngineStartOptions): Promise<void> {
+      if (running) return;
+      running = true;
+      bus.emit('state', 'connecting');
+      // Subscribe before the native side starts so we never miss the
+      // first event (some backends emit `recording` synchronously).
+      unsubscribe = opts.subscribe(handle);
+      try {
+        await opts.onStart(start);
+      } catch (cause) {
+        const err: RecognitionError = {
+          code: 'engine',
+          message: 'External engine failed to start.',
+          cause,
+        };
+        bus.emit('error', err);
+        bus.emit('state', 'closed');
+        teardown();
+        throw err;
+      }
+      if (opts.autoMarkListening !== false) {
+        bus.emit('state', 'listening');
+      }
+      start.signal?.addEventListener('abort', () => {
+        this.abort();
+      });
+    },
+    async stop(): Promise<void> {
+      if (!running) return;
+      bus.emit('state', 'closing');
+      try {
+        await opts.onStop();
+      } catch (cause) {
+        sttLogger.warn('[external] onStop threw', cause);
+      }
+      // Note: we DO NOT flip to `closed` here — most external engines
+      // need a roundtrip (transcribe + LLM rewrite) before the final
+      // text arrives. `emitFinal` / `emitError` are responsible for
+      // closing the session.
+    },
+    abort(): void {
+      if (!running) return;
+      bus.emit('state', 'closing');
+      const stopper = opts.onAbort ?? opts.onStop;
+      Promise.resolve(stopper()).catch((cause) => {
+        sttLogger.warn('[external] abort hook threw', cause);
+      });
+      bus.emit('state', 'closed');
+      teardown();
+    },
+  };
+}

package/src/tools/SpeechRecognition/core/engine/http.ts

@@ -0,0 +1,147 @@
+/**
+ * HTTP engine — records audio with MediaRecorder and POSTs each chunk to
+ * a host-supplied URL. The host owns response parsing via `parse()`, so
+ * this engine works with OpenAI Whisper REST, custom Django/FastAPI
+ * endpoints, or anything else that takes audio and returns text.
+ */
+
+import { newSegmentId } from '../ids';
+import { sttLogger } from '../logger';
+import { createEngineBus } from './index';
+import { startMicCapture, type MicCaptureHandle } from './mediarecorder';
+import type {
+  EngineStartOptions,
+  RecognitionEngine,
+  RecognitionError,
+  Unsub,
+} from '../../types';
+
+export interface HttpEngineParseResult {
+  text: string;
+  isFinal: boolean;
+  /** Optional engine-provided confidence 0..1. */
+  confidence?: number;
+}
+
+export interface HttpEngineOptions {
+  /** Endpoint URL. Receives `POST` with the audio chunk as the body. */
+  url: string | ((language: string) => string);
+  /** Per-request headers, awaited each chunk so tokens can be refreshed. */
+  headers?: () => Promise<Record<string, string>> | Record<string, string>;
+  /** Chunk emission interval, ms. Default 750 — long enough for useful audio. */
+  chunkMs?: number;
+  /** Preferred MIME for the encoder. Probed against `MediaRecorder` support. */
+  mime?: string;
+  /** Parse the engine response — return null/undefined to skip emit. */
+  parse: (
+    resp: Response,
+  ) => Promise<HttpEngineParseResult | null | undefined> | HttpEngineParseResult | null | undefined;
+  /** Stable engine id for telemetry / UI badge. Default 'http'. */
+  id?: string;
+}
+
+export function createHttpEngine(opts: HttpEngineOptions): RecognitionEngine {
+  const bus = createEngineBus();
+  let capture: MicCaptureHandle | null = null;
+  let currentSegmentId: string | null = null;
+  let ctrl: AbortController | null = null;
+  let stopping = false;
+
+  async function sendChunk(blob: Blob, language: string): Promise<void> {
+    if (stopping) return;
+    const url = typeof opts.url === 'function' ? opts.url(language) : opts.url;
+    const headers = (await opts.headers?.()) ?? {};
+    try {
+      const resp = await fetch(url, {
+        method: 'POST',
+        headers,
+        body: blob,
+        signal: ctrl?.signal,
+      });
+      if (!resp.ok) {
+        bus.emit('error', {
+          code: 'network',
+          message: `STT endpoint returned ${resp.status}`,
+        });
+        return;
+      }
+      const parsed = await opts.parse(resp);
+      if (!parsed || !parsed.text) return;
+      if (!currentSegmentId) currentSegmentId = newSegmentId();
+      if (parsed.isFinal) {
+        bus.emit('final', parsed.text, currentSegmentId, parsed.confidence);
+        currentSegmentId = null;
+      } else {
+        bus.emit('partial', parsed.text, currentSegmentId);
+      }
+    } catch (cause) {
+      if ((cause as { name?: string })?.name === 'AbortError') return;
+      sttLogger.warn('[http] chunk send failed', cause);
+      bus.emit('error', {
+        code: 'network',
+        message: 'Failed to deliver audio chunk to STT endpoint.',
+        cause,
+      });
+    }
+  }
+
+  return {
+    id: opts.id ?? 'http',
+    isSupported:
+      typeof navigator !== 'undefined' &&
+      !!navigator.mediaDevices?.getUserMedia &&
+      typeof MediaRecorder !== 'undefined',
+    on(event, cb): Unsub {
+      return bus.on(event, cb);
+    },
+    async start(start: EngineStartOptions): Promise<void> {
+      if (capture) return;
+      stopping = false;
+      ctrl = new AbortController();
+      bus.emit('state', 'connecting');
+      try {
+        capture = await startMicCapture({
+          deviceId: start.deviceId,
+          mime: opts.mime,
+          chunkMs: opts.chunkMs ?? 750,
+          onChunk: (chunk) => {
+            void sendChunk(chunk, start.language);
+          },
+          onError: (err) => bus.emit('error', err),
+        });
+        bus.emit('state', 'listening');
+      } catch (cause) {
+        const err = cause as RecognitionError;
+        bus.emit('error', err);
+        bus.emit('state', 'error');
+        throw err;
+      }
+
+      start.signal?.addEventListener('abort', () => {
+        void this.stop();
+      });
+    },
+    async stop(): Promise<void> {
+      stopping = true;
+      bus.emit('state', 'closing');
+      ctrl?.abort();
+      ctrl = null;
+      await capture?.stop();
+      capture = null;
+      currentSegmentId = null;
+      bus.emit('state', 'closed');
+    },
+    abort(): void {
+      stopping = true;
+      ctrl?.abort();
+      ctrl = null;
+      capture?.stop().catch(() => undefined);
+      capture = null;
+      currentSegmentId = null;
+      bus.emit('state', 'closed');
+    },
+    getStream(): MediaStream | null {
+      return capture?.stream ?? null;
+    },
+  };
+}

package/src/tools/SpeechRecognition/core/engine/index.ts

@@ -0,0 +1,52 @@
+/**
+ * Tiny event-bus helper shared by every engine. Lets engine authors avoid
+ * re-implementing add/remove listener bookkeeping while keeping the
+ * public `RecognitionEngine.on(...)` contract identical across engines.
+ */
+
+import type { EngineEventMap, Unsub } from '../../types';
+
+type Listeners = {
+  [K in keyof EngineEventMap]: Set<EngineEventMap[K]>;
+};
+
+export function createEngineBus(): {
+  on: <K extends keyof EngineEventMap>(event: K, cb: EngineEventMap[K]) => Unsub;
+  emit: <K extends keyof EngineEventMap>(
+    event: K,
+    ...args: Parameters<EngineEventMap[K]>
+  ) => void;
+  clear: () => void;
+} {
+  const listeners: Listeners = {
+    partial: new Set(),
+    final: new Set(),
+    error: new Set(),
+    state: new Set(),
+  };
+
+  return {
+    on(event, cb) {
+      const set = listeners[event] as Set<typeof cb>;
+      set.add(cb);
+      return () => {
+        set.delete(cb);
+      };
+    },
+    emit(event, ...args) {
+      const set = listeners[event];
+      for (const cb of set) {
+        try {
+          (cb as (...a: unknown[]) => void)(...(args as unknown[]));
+        } catch {
+          // listener errors are isolated — never break the engine loop
+        }
+      }
+    },
+    clear() {
+      for (const key of Object.keys(listeners) as Array<keyof Listeners>) {
+        listeners[key].clear();
+      }
+    },
+  };
+}

package/src/tools/SpeechRecognition/core/engine/mediarecorder.ts

@@ -0,0 +1,105 @@
+/**
+ * Shared mic capture used by the HTTP and WebSocket engines.
+ *
+ * Probes the browser for a working `MediaRecorder` MIME type and emits
+ * `Blob` chunks on a steady interval. Picks the first supported MIME in
+ * order: `audio/webm;codecs=opus` → `audio/ogg;codecs=opus` →
+ * `audio/mp4;codecs=mp4a`. Falls back to engine default if none match.
+ *
+ * The capture also exposes the raw `MediaStream` so callers can wire up
+ * an `AnalyserNode` for the level meter without owning a second copy.
+ */
+
+import { sttLogger } from '../logger';
+import type { RecognitionError } from '../../types';
+
+const PREFERRED_MIMES = [
+  'audio/webm;codecs=opus',
+  'audio/ogg;codecs=opus',
+  'audio/mp4;codecs=mp4a',
+  'audio/webm',
+];
+
+export function pickMime(preferred?: string): string | undefined {
+  if (typeof MediaRecorder === 'undefined') return undefined;
+  const candidates = preferred ? [preferred, ...PREFERRED_MIMES] : PREFERRED_MIMES;
+  for (const mime of candidates) {
+    if (MediaRecorder.isTypeSupported(mime)) return mime;
+  }
+  return undefined;
+}
+
+export interface MicCaptureOptions {
+  deviceId?: string;
+  /** Override probed MIME — useful when the backend expects a specific codec. */
+  mime?: string;
+  /** Chunk emission interval, ms. Default 250. */
+  chunkMs?: number;
+  onChunk: (chunk: Blob) => void;
+  onError?: (err: RecognitionError) => void;
+}
+
+export interface MicCaptureHandle {
+  readonly stream: MediaStream;
+  readonly mime: string | undefined;
+  stop(): Promise<void>;
+}
+
+function toErr(code: RecognitionError['code'], message: string, cause?: unknown): RecognitionError {
+  return { code, message, cause };
+}
+
+export async function startMicCapture(
+  opts: MicCaptureOptions,
+): Promise<MicCaptureHandle> {
+  if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
+    throw toErr('unsupported', 'getUserMedia is not available in this environment.');
+  }
+  if (typeof MediaRecorder === 'undefined') {
+    throw toErr('unsupported', 'MediaRecorder is not available in this environment.');
+  }
+
+  let stream: MediaStream;
+  try {
+    stream = await navigator.mediaDevices.getUserMedia({
+      audio: opts.deviceId ? { deviceId: { exact: opts.deviceId } } : true,
+      video: false,
+    });
+  } catch (cause) {
+    const name = (cause as { name?: string })?.name;
+    if (name === 'NotAllowedError' || name === 'SecurityError') {
+      throw toErr('permission-denied', 'Microphone permission denied.', cause);
+    }
+    if (name === 'NotFoundError' || name === 'OverconstrainedError') {
+      throw toErr('no-microphone', 'No microphone found matching the constraints.', cause);
+    }
+    throw toErr('unknown', 'Failed to access microphone.', cause);
+  }
+
+  const mime = pickMime(opts.mime);
+  const rec = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
+
+  rec.ondataavailable = (e) => {
+    if (e.data && e.data.size > 0) opts.onChunk(e.data);
+  };
+  rec.onerror = (e) => {
+    const err = toErr('engine', 'MediaRecorder error.', e);
+    sttLogger.warn('[capture] recorder error', e);
+    opts.onError?.(err);
+  };
+
+  rec.start(opts.chunkMs ?? 250);
+
+  return {
+    stream,
+    mime: mime ?? rec.mimeType,
+    async stop() {
+      const done = new Promise<void>((resolve) => {
+        rec.addEventListener('stop', () => resolve(), { once: true });
+      });
+      if (rec.state !== 'inactive') rec.stop();
+      stream.getTracks().forEach((t) => t.stop());
+      await done;
+    },
+  };
+}