even-toolkit 1.1.0 → 1.1.2

package/stt/engine.ts

@@ -0,0 +1,274 @@
+import type {
+  STTEngineConfig,
+  STTProvider,
+  STTState,
+  STTTranscript,
+  STTError,
+  AudioSource,
+} from './types';
+import { createProvider } from './registry';
+import { MicrophoneSource } from './sources/microphone';
+import { resample } from './audio/resample';
+import { createVAD } from './audio/vad';
+import { createAudioBuffer } from './audio/buffer';
+
+/**
+ * STTEngine orchestrates source -> processing -> provider.
+ *
+ * For `web-speech` provider: skips audio source (it handles its own mic).
+ * For other providers: starts audio source, pipes through optional resample
+ * and VAD, buffers audio, and calls provider.transcribe() on speech end.
+ */
+export class STTEngine {
+  private config: STTEngineConfig;
+  private provider: STTProvider | null = null;
+  private source: AudioSource | null = null;
+  private sourceUnsub: (() => void) | null = null;
+
+  private transcriptListeners: Array<(t: STTTranscript) => void> = [];
+  private stateListeners: Array<(s: STTState) => void> = [];
+  private errorListeners: Array<(e: STTError) => void> = [];
+
+  private providerUnsubs: Array<() => void> = [];
+
+  private vad: ReturnType<typeof createVAD> | null = null;
+  private buffer: ReturnType<typeof createAudioBuffer> | null = null;
+  private targetSampleRate: number;
+
+  constructor(config: STTEngineConfig) {
+    this.config = config;
+    this.targetSampleRate = config.sampleRate ?? 16000;
+  }
+
+  // ── Event subscriptions ──
+
+  onTranscript(cb: (t: STTTranscript) => void): () => void {
+    this.transcriptListeners.push(cb);
+    return () => {
+      const idx = this.transcriptListeners.indexOf(cb);
+      if (idx >= 0) this.transcriptListeners.splice(idx, 1);
+    };
+  }
+
+  onStateChange(cb: (s: STTState) => void): () => void {
+    this.stateListeners.push(cb);
+    return () => {
+      const idx = this.stateListeners.indexOf(cb);
+      if (idx >= 0) this.stateListeners.splice(idx, 1);
+    };
+  }
+
+  onError(cb: (e: STTError) => void): () => void {
+    this.errorListeners.push(cb);
+    return () => {
+      const idx = this.errorListeners.indexOf(cb);
+      if (idx >= 0) this.errorListeners.splice(idx, 1);
+    };
+  }
+
+  private emitTranscript(t: STTTranscript): void {
+    for (const cb of this.transcriptListeners) cb(t);
+  }
+
+  private emitState(s: STTState): void {
+    for (const cb of this.stateListeners) cb(s);
+  }
+
+  private emitError(e: STTError): void {
+    for (const cb of this.errorListeners) cb(e);
+  }
+
+  // ── Lifecycle ──
+
+  async start(): Promise<void> {
+    this.emitState('loading');
+
+    try {
+      // Create and init provider
+      this.provider = await createProvider(this.config.provider);
+      this.subscribeProvider(this.provider);
+
+      await this.provider.init({
+        language: this.config.language,
+        mode: this.config.mode,
+        apiKey: this.config.apiKey,
+        modelId: this.config.modelId,
+        continuous: this.config.continuous,
+        vadEnabled: typeof this.config.vad === 'boolean' ? this.config.vad : !!this.config.vad,
+        vadSilenceMs: typeof this.config.vad === 'object' ? this.config.vad.silenceMs : undefined,
+        sampleRate: this.targetSampleRate,
+      });
+
+      // web-speech handles its own microphone
+      if (this.config.provider === 'web-speech') {
+        this.provider.start();
+        return;
+      }
+
+      // Set up audio source
+      this.source = this.resolveSource();
+      await this.source.start();
+
+      // Set up VAD if enabled
+      if (this.config.vad) {
+        const vadConfig = typeof this.config.vad === 'object' ? {
+          silenceThresholdMs: this.config.vad.silenceMs,
+          speechThresholdDb: this.config.vad.thresholdDb,
+        } : undefined;
+        this.vad = createVAD(vadConfig);
+      }
+
+      // Set up audio buffer for batch mode
+      this.buffer = createAudioBuffer({ sampleRate: this.targetSampleRate });
+
+      // Wire audio pipeline
+      this.sourceUnsub = this.source.onAudioData((pcm, sampleRate) => {
+        this.processAudio(pcm, sampleRate);
+      });
+
+      this.provider.start();
+    } catch (err) {
+      const error: STTError = {
+        code: 'unknown',
+        message: err instanceof Error ? err.message : String(err),
+        provider: this.config.provider,
+      };
+      this.emitError(error);
+      this.emitState('error');
+
+      // Attempt fallback
+      if (this.config.fallback) {
+        await this.switchToFallback();
+      }
+    }
+  }
+
+  stop(): void {
+    this.provider?.stop();
+    this.sourceUnsub?.();
+    this.sourceUnsub = null;
+    this.source?.stop();
+    this.vad?.reset();
+    this.buffer?.clear();
+  }
+
+  abort(): void {
+    this.provider?.abort();
+    this.sourceUnsub?.();
+    this.sourceUnsub = null;
+    this.source?.stop();
+    this.vad?.reset();
+    this.buffer?.clear();
+  }
+
+  dispose(): void {
+    this.abort();
+    for (const unsub of this.providerUnsubs) unsub();
+    this.providerUnsubs.length = 0;
+    this.provider?.dispose();
+    this.provider = null;
+    this.source?.dispose();
+    this.source = null;
+    this.transcriptListeners.length = 0;
+    this.stateListeners.length = 0;
+    this.errorListeners.length = 0;
+  }
+
+  // ── Internal ──
+
+  private resolveSource(): AudioSource {
+    const src = this.config.source;
+    if (!src || src === 'microphone') {
+      return new MicrophoneSource();
+    }
+    if (src === 'glass-bridge') {
+      throw new Error(
+        'glass-bridge source requires a GlassBridgeSource instance. ' +
+        'Pass an AudioSource object directly via config.source.'
+      );
+    }
+    // Custom AudioSource instance
+    return src;
+  }
+
+  private processAudio(pcm: Float32Array, sampleRate: number): void {
+    // Resample if needed
+    let samples = sampleRate !== this.targetSampleRate
+      ? resample(pcm, sampleRate, this.targetSampleRate)
+      : pcm;
+
+    if (!this.buffer) return;
+
+    // If VAD is enabled, check for speech boundaries
+    if (this.vad) {
+      const result = this.vad.process(samples);
+
+      if (result.isSpeech || result.speechEnded) {
+        this.buffer.append(samples);
+      }
+
+      if (result.speechEnded) {
+        this.flushBuffer();
+      }
+    } else {
+      // No VAD: accumulate everything, provider handles streaming
+      this.buffer.append(samples);
+    }
+  }
+
+  private async flushBuffer(): Promise<void> {
+    if (!this.buffer || !this.provider) return;
+
+    const audio = this.buffer.getAll();
+    this.buffer.clear();
+
+    if (audio.length === 0) return;
+
+    // If provider supports batch transcription
+    if (this.provider.transcribe) {
+      try {
+        const transcript = await this.provider.transcribe(audio, this.targetSampleRate);
+        this.emitTranscript(transcript);
+      } catch (err) {
+        this.emitError({
+          code: 'unknown',
+          message: err instanceof Error ? err.message : String(err),
+          provider: this.config.provider,
+        });
+      }
+    }
+  }
+
+  private subscribeProvider(provider: STTProvider): void {
+    this.providerUnsubs.push(
+      provider.onTranscript((t) => this.emitTranscript(t)),
+      provider.onStateChange((s) => this.emitState(s)),
+      provider.onError((e) => {
+        this.emitError(e);
+        if (this.config.fallback) {
+          this.switchToFallback();
+        }
+      }),
+    );
+  }
+
+  private async switchToFallback(): Promise<void> {
+    if (!this.config.fallback) return;
+
+    // Clean up current provider
+    for (const unsub of this.providerUnsubs) unsub();
+    this.providerUnsubs.length = 0;
+    this.provider?.dispose();
+    this.provider = null;
+
+    // Switch to fallback
+    const fallbackType = this.config.fallback;
+    this.config = { ...this.config, provider: fallbackType, fallback: undefined };
+
+    try {
+      await this.start();
+    } catch {
+      // Fallback also failed — nothing more to do
+    }
+  }
+}

package/stt/i18n.ts

@@ -0,0 +1,39 @@
+/** Language mapping utilities for STT providers */
+
+/** BCP 47 → Whisper ISO 639-1 */
+export function toWhisperLang(bcp47: string): string {
+  return bcp47.split('-')[0].toLowerCase();
+}
+
+/** Short code → BCP 47 (best guess) */
+export function toWebSpeechLang(lang: string): string {
+  const map: Record<string, string> = {
+    en: 'en-US', it: 'it-IT', es: 'es-ES', fr: 'fr-FR',
+    de: 'de-DE', pt: 'pt-BR', zh: 'zh-CN', ja: 'ja-JP',
+    ko: 'ko-KR', ru: 'ru-RU', ar: 'ar-SA', hi: 'hi-IN',
+  };
+  if (lang.includes('-')) return lang;
+  return map[lang.toLowerCase()] ?? `${lang}-${lang.toUpperCase()}`;
+}
+
+export interface SupportedLanguage {
+  code: string;
+  name: string;
+  whisper: boolean;
+  webSpeech: boolean;
+}
+
+export const SUPPORTED_LANGUAGES: SupportedLanguage[] = [
+  { code: 'en-US', name: 'English', whisper: true, webSpeech: true },
+  { code: 'it-IT', name: 'Italian', whisper: true, webSpeech: true },
+  { code: 'es-ES', name: 'Spanish', whisper: true, webSpeech: true },
+  { code: 'fr-FR', name: 'French', whisper: true, webSpeech: true },
+  { code: 'de-DE', name: 'German', whisper: true, webSpeech: true },
+  { code: 'pt-BR', name: 'Portuguese', whisper: true, webSpeech: true },
+  { code: 'zh-CN', name: 'Chinese', whisper: true, webSpeech: true },
+  { code: 'ja-JP', name: 'Japanese', whisper: true, webSpeech: true },
+  { code: 'ko-KR', name: 'Korean', whisper: true, webSpeech: true },
+  { code: 'ru-RU', name: 'Russian', whisper: true, webSpeech: true },
+  { code: 'ar-SA', name: 'Arabic', whisper: true, webSpeech: true },
+  { code: 'hi-IN', name: 'Hindi', whisper: true, webSpeech: true },
+];

package/stt/index.ts

@@ -0,0 +1,10 @@
+export * from './types';
+export { STTEngine } from './engine';
+export { createProvider } from './registry';
+export * from './i18n';
+export { MicrophoneSource } from './sources/microphone';
+export { GlassBridgeSource } from './sources/glass-bridge';
+export * from './audio/pcm-utils';
+export * from './audio/resample';
+export * from './audio/vad';
+export { createAudioBuffer } from './audio/buffer';

package/stt/providers/deepgram.ts

@@ -0,0 +1,178 @@
+import type {
+  STTProvider,
+  STTProviderConfig,
+  STTMode,
+  STTState,
+  STTTranscript,
+  STTError,
+} from '../types';
+
+interface DeepgramResult {
+  channel?: {
+    alternatives?: Array<{
+      transcript?: string;
+      confidence?: number;
+    }>;
+  };
+  is_final?: boolean;
+  speech_final?: boolean;
+}
+
+export class DeepgramProvider implements STTProvider {
+  readonly type = 'deepgram' as const;
+  readonly supportedModes: STTMode[] = ['streaming'];
+
+  private _state: STTState = 'idle';
+  private apiKey = '';
+  private language = 'en';
+  private modelId = 'nova-2';
+  private ws: WebSocket | null = null;
+
+  private transcriptCbs: Array<(t: STTTranscript) => void> = [];
+  private stateCbs: Array<(s: STTState) => void> = [];
+  private errorCbs: Array<(e: STTError) => void> = [];
+
+  get state(): STTState {
+    return this._state;
+  }
+
+  async init(config: STTProviderConfig): Promise<void> {
+    this.apiKey = config.apiKey ?? '';
+    this.language = config.language ?? 'en';
+    this.modelId = config.modelId ?? 'nova-2';
+
+    if (!this.apiKey) {
+      const err: STTError = { code: 'not-allowed', message: 'Deepgram API key is required', provider: this.type };
+      this.emitError(err);
+      throw new Error(err.message);
+    }
+  }
+
+  start(): void {
+    if (this.ws) {
+      this.closeSocket();
+    }
+
+    const params = new URLSearchParams({
+      model: this.modelId,
+      language: this.language,
+      interim_results: 'true',
+      punctuate: 'true',
+      encoding: 'linear16',
+      sample_rate: '16000',
+    });
+
+    const url = `wss://api.deepgram.com/v1/listen?${params.toString()}`;
+
+    this.ws = new WebSocket(url, ['token', this.apiKey]);
+    this.ws.binaryType = 'arraybuffer';
+
+    this.ws.onopen = () => {
+      this.setState('listening');
+    };
+
+    this.ws.onmessage = (event: MessageEvent) => {
+      try {
+        const data = JSON.parse(event.data as string) as DeepgramResult;
+        const alt = data.channel?.alternatives?.[0];
+        if (!alt?.transcript) return;
+
+        const transcript: STTTranscript = {
+          text: alt.transcript,
+          isFinal: data.is_final ?? false,
+          confidence: alt.confidence ?? 0,
+          timestamp: Date.now(),
+        };
+        this.emitTranscript(transcript);
+      } catch {
+        // Non-JSON message, ignore
+      }
+    };
+
+    this.ws.onerror = () => {
+      const err: STTError = {
+        code: 'network',
+        message: 'Deepgram WebSocket error',
+        provider: this.type,
+      };
+      this.emitError(err);
+      this.setState('error');
+    };
+
+    this.ws.onclose = () => {
+      this.ws = null;
+      if (this._state === 'listening') {
+        this.setState('idle');
+      }
+    };
+  }
+
+  /** Send raw audio data (PCM Int16 or Float32 as ArrayBuffer) to the Deepgram stream. */
+  sendAudio(data: ArrayBuffer | Int16Array | Float32Array): void {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
+
+    if (data instanceof ArrayBuffer) {
+      this.ws.send(data);
+    } else {
+      this.ws.send(data.buffer);
+    }
+  }
+
+  stop(): void {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+      // Send close message per Deepgram protocol
+      this.ws.send(JSON.stringify({ type: 'CloseStream' }));
+    }
+    this.closeSocket();
+  }
+
+  abort(): void {
+    this.closeSocket();
+  }
+
+  dispose(): void {
+    this.closeSocket();
+    this.transcriptCbs = [];
+    this.stateCbs = [];
+    this.errorCbs = [];
+  }
+
+  onTranscript(cb: (t: STTTranscript) => void): () => void {
+    this.transcriptCbs.push(cb);
+    return () => { this.transcriptCbs = this.transcriptCbs.filter((c) => c !== cb); };
+  }
+
+  onStateChange(cb: (s: STTState) => void): () => void {
+    this.stateCbs.push(cb);
+    return () => { this.stateCbs = this.stateCbs.filter((c) => c !== cb); };
+  }
+
+  onError(cb: (e: STTError) => void): () => void {
+    this.errorCbs.push(cb);
+    return () => { this.errorCbs = this.errorCbs.filter((c) => c !== cb); };
+  }
+
+  // ── Private ──
+
+  private closeSocket(): void {
+    if (this.ws) {
+      try { this.ws.close(); } catch { /* ignore */ }
+      this.ws = null;
+    }
+    this.setState('idle');
+  }
+
+  private setState(s: STTState): void {
+    if (this._state === s) return;
+    this._state = s;
+    for (const cb of this.stateCbs) cb(s);
+  }
+
+  private emitTranscript(t: STTTranscript): void {
+    for (const cb of this.transcriptCbs) cb(t);
+  }
+
+  private emitError(e: STTError): void {
+    for (const cb of this.errorCbs) cb(e);
+  }
+}