@happyvertical/smrt-svelte 0.30.0

package/dist/browser-ai/adapters/stt/whisper-wasm.js

@@ -0,0 +1,380 @@
+/**
+ * Whisper adapter using @xenova/transformers (v2)
+ *
+ * Uses Whisper models running in the browser via WASM.
+ * V2 uses a simpler, more stable pipeline API.
+ * Requires downloading model files (~40MB-150MB depending on model size).
+ */
+import { importOptional } from '@happyvertical/smrt-ui/utils/import-optional.js';
+import { CapabilityNotAvailableError, InitializationError, } from '../../core/errors.js';
+// Model IDs - using Xenova's quantized models for v2
+const MODEL_IDS = {
+    tiny: 'Xenova/whisper-tiny.en',
+    base: 'Xenova/whisper-base.en',
+    small: 'Xenova/whisper-small.en',
+};
+// Approximate download sizes
+const MODEL_SIZES = {
+    tiny: 40 * 1024 * 1024, // ~40MB
+    base: 75 * 1024 * 1024, // ~75MB
+    small: 150 * 1024 * 1024, // ~150MB
+};
+/**
+ * Whisper adapter for high-accuracy speech recognition
+ * Uses transformers.js v2 pipeline API
+ */
+export class WhisperWasmSTTAdapter {
+    type = 'whisper-wasm';
+    _initState = 'uninitialized';
+    options;
+    _isListening = false;
+    // Audio recording
+    mediaRecorder = null;
+    audioChunks = [];
+    stream = null;
+    // Event listeners
+    resultListeners = new Set();
+    errorListeners = new Set();
+    startListeners = new Set();
+    endListeners = new Set();
+    // Transformers.js v2 pipeline
+    transcriber = null;
+    // Promise that resolves when transcription is complete
+    processingPromise = null;
+    processingResolve = null;
+    constructor(options = {}) {
+        this.options = {
+            type: 'whisper-wasm',
+            modelSize: 'tiny',
+            defaultLanguage: 'en',
+            ...options,
+        };
+    }
+    get initState() {
+        return this._initState;
+    }
+    async ensureInitialized(onProgress) {
+        if (this._initState === 'ready')
+            return;
+        if (this._initState === 'initializing') {
+            return new Promise((resolve, reject) => {
+                const check = () => {
+                    if (this._initState === 'ready')
+                        resolve();
+                    else if (this._initState === 'error')
+                        reject(new Error('Initialization failed'));
+                    else
+                        setTimeout(check, 50);
+                };
+                check();
+            });
+        }
+        this._initState = 'initializing';
+        try {
+            // Check WASM support
+            if (typeof WebAssembly === 'undefined') {
+                throw new CapabilityNotAvailableError('WebAssembly', 'whisper-wasm');
+            }
+            const modelSize = this.options.modelSize || 'tiny';
+            const modelId = MODEL_IDS[modelSize] || MODEL_IDS.tiny;
+            const totalSize = MODEL_SIZES[modelSize] || MODEL_SIZES.tiny;
+            onProgress?.({
+                state: 'downloading',
+                bytesLoaded: 0,
+                bytesTotal: totalSize,
+                percent: 0,
+                currentFile: `${modelId}`,
+            });
+            // Dynamically import transformers.js v2
+            const transformers = await this.importTransformers();
+            const { pipeline } = transformers;
+            console.log('[Whisper v2] Loading pipeline...');
+            // Create ASR pipeline - v2 uses simpler API
+            this.transcriber = await pipeline('automatic-speech-recognition', modelId, {
+                progress_callback: (progress) => {
+                    if (progress.status === 'progress' &&
+                        progress.progress !== undefined) {
+                        const percent = Math.round(progress.progress);
+                        onProgress?.({
+                            state: 'downloading',
+                            bytesLoaded: Math.round((totalSize * percent) / 100),
+                            bytesTotal: totalSize,
+                            percent,
+                            currentFile: progress.file || modelId,
+                        });
+                    }
+                },
+            });
+            console.log('[Whisper v2] Pipeline loaded successfully');
+            onProgress?.({
+                state: 'complete',
+                bytesLoaded: totalSize,
+                bytesTotal: totalSize,
+                percent: 100,
+            });
+            this._initState = 'ready';
+        }
+        catch (error) {
+            this._initState = 'error';
+            console.error('[Whisper v2] Initialization error:', error);
+            throw error instanceof Error
+                ? error
+                : new InitializationError('whisper-wasm', String(error));
+        }
+    }
+    // Store the transformers module for read_audio utility
+    transformersModule = null;
+    async importTransformers() {
+        try {
+            // Try v2 (@xenova/transformers) first
+            this.transformersModule = await importOptional('@xenova/transformers');
+            this.configureTransformersEnv(this.transformersModule);
+            return this.transformersModule;
+        }
+        catch {
+            try {
+                // Fall back to v3 (@huggingface/transformers)
+                this.transformersModule = await importOptional('@huggingface/transformers');
+                this.configureTransformersEnv(this.transformersModule);
+                return this.transformersModule;
+            }
+            catch {
+                throw new InitializationError('whisper-wasm', 'Neither @xenova/transformers nor @huggingface/transformers installed.');
+            }
+        }
+    }
+    /**
+     * Configure transformers.js environment settings
+     */
+    configureTransformersEnv(transformers) {
+        if (transformers.env) {
+            // Default to remote models unless explicitly allowing local
+            const allowLocal = this.options.allowLocalModels ?? false;
+            transformers.env.allowLocalModels = allowLocal;
+            // Always use browser cache for better performance
+            transformers.env.useBrowserCache = true;
+            console.log('[Whisper v2] Configured env:', {
+                allowLocalModels: allowLocal,
+                useBrowserCache: true,
+            });
+        }
+    }
+    getCapabilities() {
+        const modelSize = this.options.modelSize || 'tiny';
+        return {
+            continuous: false, // Whisper processes chunks, not continuous
+            interimResults: false, // No interim results in standard Whisper
+            languages: [
+                'en',
+                'es',
+                'fr',
+                'de',
+                'it',
+                'pt',
+                'nl',
+                'pl',
+                'ru',
+                'zh',
+                'ja',
+                'ko',
+                'ar',
+                'hi',
+                'tr',
+                'vi',
+                'th',
+            ],
+            accuracy: modelSize === 'tiny' ? 'medium' : 'high',
+            requiresDownload: true,
+            downloadSize: MODEL_SIZES[modelSize] || MODEL_SIZES.tiny,
+        };
+    }
+    async start(_options = {}) {
+        await this.ensureInitialized();
+        if (this._isListening)
+            return;
+        try {
+            // Get microphone access
+            this.stream = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    channelCount: 1,
+                    sampleRate: 16000, // Whisper expects 16kHz
+                },
+            });
+            this.audioChunks = [];
+            // Create MediaRecorder
+            this.mediaRecorder = new MediaRecorder(this.stream, {
+                mimeType: this.getSupportedMimeType(),
+            });
+            this.mediaRecorder.ondataavailable = (event) => {
+                if (event.data.size > 0) {
+                    this.audioChunks.push(event.data);
+                }
+            };
+            this.mediaRecorder.onstop = async () => {
+                await this.processRecording();
+                this.processingResolve?.();
+            };
+            // Create promise that resolves when transcription is done
+            this.processingPromise = new Promise((resolve) => {
+                this.processingResolve = resolve;
+            });
+            // Start with timeslice to capture audio in chunks (more reliable)
+            this.mediaRecorder.start(500);
+            this._isListening = true;
+            for (const cb of this.startListeners) {
+                cb();
+            }
+        }
+        catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            for (const cb of this.errorListeners) {
+                cb(err);
+            }
+            throw err;
+        }
+    }
+    getSupportedMimeType() {
+        const types = [
+            'audio/webm;codecs=opus',
+            'audio/webm',
+            'audio/ogg;codecs=opus',
+            'audio/mp4',
+        ];
+        for (const type of types) {
+            if (MediaRecorder.isTypeSupported(type)) {
+                return type;
+            }
+        }
+        return 'audio/webm'; // Fallback
+    }
+    async processRecording() {
+        console.log('[Whisper v2] processRecording called, chunks:', this.audioChunks.length);
+        if (this.audioChunks.length === 0) {
+            console.log('[Whisper v2] No audio chunks captured');
+            return;
+        }
+        // Use the actual MIME type from the recorder
+        const mimeType = this.mediaRecorder?.mimeType || 'audio/webm';
+        const audioBlob = new Blob(this.audioChunks, { type: mimeType });
+        console.log('[Whisper v2] Audio blob size:', audioBlob.size, 'type:', mimeType);
+        try {
+            if (!this.transcriber || !this.transformersModule) {
+                throw new Error('Whisper not initialized');
+            }
+            // Decode audio using Web Audio API for better format support
+            console.log('[Whisper v2] Decoding audio with Web Audio API...');
+            const arrayBuffer = await audioBlob.arrayBuffer();
+            const audioContext = new AudioContext({ sampleRate: 16000 });
+            const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+            // Get mono audio data
+            const audioData = audioBuffer.getChannelData(0);
+            console.log('[Whisper v2] Audio decoded - length:', audioData.length, 'samples, duration:', audioBuffer.duration.toFixed(2), 's');
+            // Debug: check audio levels
+            let maxLevel = 0;
+            let sumSquares = 0;
+            for (let i = 0; i < audioData.length; i++) {
+                const abs = Math.abs(audioData[i]);
+                if (abs > maxLevel)
+                    maxLevel = abs;
+                sumSquares += audioData[i] * audioData[i];
+            }
+            const rms = Math.sqrt(sumSquares / audioData.length);
+            console.log('[Whisper v2] Audio levels - max:', maxLevel.toFixed(4), 'RMS:', rms.toFixed(4));
+            if (maxLevel < 0.01) {
+                console.warn('[Whisper v2] Audio appears to be silent or very quiet!');
+            }
+            // Close audio context
+            await audioContext.close();
+            console.log('[Whisper v2] Starting transcription...');
+            const startTime = performance.now();
+            // Pass the Float32Array audio data
+            const result = await this.transcriber(audioData);
+            const elapsedTime = performance.now() - startTime;
+            console.log('[Whisper v2] Transcription completed in', Math.round(elapsedTime), 'ms');
+            console.log('[Whisper v2] Result:', result);
+            const text = result.text?.trim() || '';
+            console.log('[Whisper v2] Transcription result:', text);
+            const sttResult = {
+                text,
+                confidence: 0.95,
+                isFinal: true,
+                timestamp: Date.now(),
+            };
+            for (const cb of this.resultListeners) {
+                cb(sttResult);
+            }
+        }
+        catch (error) {
+            console.error('[Whisper v2] Processing error:', error);
+            const err = error instanceof Error ? error : new Error(String(error));
+            for (const cb of this.errorListeners) {
+                cb(err);
+            }
+        }
+    }
+    async stop() {
+        if (this.mediaRecorder && this._isListening) {
+            this.mediaRecorder.stop();
+            if (this.stream) {
+                for (const track of this.stream.getTracks()) {
+                    track.stop();
+                }
+            }
+            this.stream = null;
+            if (this.processingPromise) {
+                await this.processingPromise;
+                this.processingPromise = null;
+                this.processingResolve = null;
+            }
+            this._isListening = false;
+            for (const cb of this.endListeners) {
+                cb();
+            }
+        }
+    }
+    abort() {
+        if (this.mediaRecorder && this._isListening) {
+            this.mediaRecorder.stop();
+            this._isListening = false;
+            this.audioChunks = [];
+            if (this.stream) {
+                for (const track of this.stream.getTracks()) {
+                    track.stop();
+                }
+            }
+            this.stream = null;
+            for (const cb of this.endListeners) {
+                cb();
+            }
+        }
+    }
+    isListening() {
+        return this._isListening;
+    }
+    onResult(callback) {
+        this.resultListeners.add(callback);
+        return () => this.resultListeners.delete(callback);
+    }
+    onError(callback) {
+        this.errorListeners.add(callback);
+        return () => this.errorListeners.delete(callback);
+    }
+    onStart(callback) {
+        this.startListeners.add(callback);
+        return () => this.startListeners.delete(callback);
+    }
+    onEnd(callback) {
+        this.endListeners.add(callback);
+        return () => this.endListeners.delete(callback);
+    }
+    async dispose() {
+        this.abort();
+        this.transcriber = null;
+        this.transformersModule = null;
+        this.resultListeners.clear();
+        this.errorListeners.clear();
+        this.startListeners.clear();
+        this.endListeners.clear();
+        this._initState = 'uninitialized';
+    }
+}

package/dist/browser-ai/adapters/tts/browser-synthesis.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Browser Speech Synthesis API adapter
+ *
+ * Uses the Web Speech API (SpeechSynthesis) available in most modern browsers.
+ * No download required, works immediately.
+ */
+import type { InitState, OnProgress } from '../../core/types.js';
+import type { BrowserSynthesisTTSOptions, TTSAdapter, TTSCapabilities, TTSOptions, TTSVoice } from './types.js';
+/**
+ * Browser Speech Synthesis adapter
+ */
+export declare class BrowserSynthesisTTSAdapter implements TTSAdapter {
+    readonly type: "browser-synthesis";
+    private _initState;
+    private synthesis;
+    private options;
+    private voices;
+    private currentUtterance;
+    private startListeners;
+    private endListeners;
+    private errorListeners;
+    private boundaryListeners;
+    constructor(options?: BrowserSynthesisTTSOptions);
+    get initState(): InitState;
+    ensureInitialized(_onProgress?: OnProgress): Promise<void>;
+    private loadVoices;
+    getCapabilities(): TTSCapabilities;
+    getVoices(): TTSVoice[];
+    private findVoice;
+    speak(text: string, options?: TTSOptions): Promise<void>;
+    stop(): void;
+    pause(): void;
+    resume(): void;
+    isSpeaking(): boolean;
+    isPaused(): boolean;
+    onStart(callback: () => void): () => void;
+    onEnd(callback: () => void): () => void;
+    onError(callback: (error: Error) => void): () => void;
+    onBoundary(callback: (charIndex: number, charLength: number) => void): () => void;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=browser-synthesis.d.ts.map

package/dist/browser-ai/adapters/tts/browser-synthesis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-synthesis.d.ts","sourceRoot":"","sources":["../../../../src/browser-ai/adapters/tts/browser-synthesis.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EACV,0BAA0B,EAC1B,UAAU,EACV,eAAe,EACf,UAAU,EACV,QAAQ,EACT,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,qBAAa,0BAA2B,YAAW,UAAU;IAC3D,QAAQ,CAAC,IAAI,EAAG,mBAAmB,CAAU;IAE7C,OAAO,CAAC,UAAU,CAA8B;IAChD,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,MAAM,CAA8B;IAE5C,OAAO,CAAC,gBAAgB,CAAyC;IAGjE,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,YAAY,CAAyB;IAC7C,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAErB;gBAEQ,OAAO,GAAE,0BAA+B;IAUpD,IAAI,SAAS,IAAI,SAAS,CAEzB;IAEK,iBAAiB,CAAC,WAAW,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;YAqClD,UAAU;IA+BxB,eAAe,IAAI,eAAe;IAUlC,SAAS,IAAI,QAAQ,EAAE;IAUvB,OAAO,CAAC,SAAS;IAkCX,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAiElE,IAAI,IAAI,IAAI;IAOZ,KAAK,IAAI,IAAI;IAMb,MAAM,IAAI,IAAI;IAMd,UAAU,IAAI,OAAO;IAIrB,QAAQ,IAAI,OAAO;IAInB,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAKzC,KAAK,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAKvC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,MAAM,IAAI;IAKrD,UAAU,CACR,QAAQ,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,IAAI,GACxD,MAAM,IAAI;IAKP,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAU/B"}

package/dist/browser-ai/adapters/tts/browser-synthesis.js

@@ -0,0 +1,235 @@
+/**
+ * Browser Speech Synthesis API adapter
+ *
+ * Uses the Web Speech API (SpeechSynthesis) available in most modern browsers.
+ * No download required, works immediately.
+ */
+import { CapabilityNotAvailableError } from '../../core/errors.js';
+/**
+ * Browser Speech Synthesis adapter
+ */
+export class BrowserSynthesisTTSAdapter {
+    type = 'browser-synthesis';
+    _initState = 'uninitialized';
+    synthesis = null;
+    options;
+    voices = [];
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: Used in speak/stop/pause
+    currentUtterance = null;
+    // Event listeners
+    startListeners = new Set();
+    endListeners = new Set();
+    errorListeners = new Set();
+    boundaryListeners = new Set();
+    constructor(options = {}) {
+        this.options = {
+            defaultLanguage: 'en-US',
+            defaultRate: 1,
+            defaultPitch: 1,
+            defaultVolume: 1,
+            ...options,
+        };
+    }
+    get initState() {
+        return this._initState;
+    }
+    async ensureInitialized(_onProgress) {
+        if (this._initState === 'ready')
+            return;
+        if (this._initState === 'initializing') {
+            // Wait for existing initialization
+            return new Promise((resolve, reject) => {
+                const check = () => {
+                    if (this._initState === 'ready')
+                        resolve();
+                    else if (this._initState === 'error')
+                        reject(new Error('Initialization failed'));
+                    else
+                        setTimeout(check, 50);
+                };
+                check();
+            });
+        }
+        this._initState = 'initializing';
+        try {
+            if (typeof window === 'undefined' || !window.speechSynthesis) {
+                throw new CapabilityNotAvailableError('Web Speech Synthesis API', 'browser-synthesis');
+            }
+            this.synthesis = window.speechSynthesis;
+            // Load voices - they may load asynchronously
+            await this.loadVoices();
+            this._initState = 'ready';
+        }
+        catch (error) {
+            this._initState = 'error';
+            throw error;
+        }
+    }
+    async loadVoices() {
+        if (!this.synthesis)
+            return;
+        // Get voices - may be empty initially in some browsers
+        this.voices = this.synthesis.getVoices();
+        if (this.voices.length === 0) {
+            // Wait for voices to load
+            await new Promise((resolve) => {
+                const handleVoicesChanged = () => {
+                    this.voices = this.synthesis?.getVoices() ?? [];
+                    if (this.voices.length > 0) {
+                        this.synthesis?.removeEventListener('voiceschanged', handleVoicesChanged);
+                        resolve();
+                    }
+                };
+                this.synthesis?.addEventListener('voiceschanged', handleVoicesChanged);
+                // Timeout after 2 seconds - some browsers may not fire the event
+                setTimeout(() => {
+                    this.voices = this.synthesis?.getVoices() ?? [];
+                    resolve();
+                }, 2000);
+            });
+        }
+    }
+    getCapabilities() {
+        return {
+            voices: this.getVoices(),
+            ssml: false, // Browser API doesn't support SSML
+            rateRange: { min: 0.1, max: 10 },
+            pitchRange: { min: 0, max: 2 },
+            requiresDownload: false,
+        };
+    }
+    getVoices() {
+        return this.voices.map((voice) => ({
+            id: voice.voiceURI,
+            name: voice.name,
+            language: voice.lang,
+            local: voice.localService,
+            default: voice.default,
+        }));
+    }
+    findVoice(voiceNameOrId, language) {
+        if (!voiceNameOrId && !language) {
+            // Use default voice
+            return this.voices.find((v) => v.default) || this.voices[0];
+        }
+        if (voiceNameOrId) {
+            // Try to find by name or URI
+            const voice = this.voices.find((v) => v.name === voiceNameOrId || v.voiceURI === voiceNameOrId);
+            if (voice)
+                return voice;
+        }
+        if (language) {
+            // Find voice matching language
+            const exactMatch = this.voices.find((v) => v.lang === language);
+            if (exactMatch)
+                return exactMatch;
+            // Try language prefix match (e.g., 'en' matches 'en-US')
+            const langPrefix = language.split('-')[0];
+            const prefixMatch = this.voices.find((v) => v.lang.startsWith(langPrefix));
+            if (prefixMatch)
+                return prefixMatch;
+        }
+        // Fallback to default
+        return this.voices.find((v) => v.default) || this.voices[0];
+    }
+    async speak(text, options = {}) {
+        await this.ensureInitialized();
+        if (!this.synthesis) {
+            throw new Error('Synthesis not initialized');
+        }
+        // Cancel any current speech
+        this.stop();
+        return new Promise((resolve, reject) => {
+            const utterance = new SpeechSynthesisUtterance(text);
+            // Set voice
+            const voice = this.findVoice(options.voice || this.options.defaultVoice, options.language || this.options.defaultLanguage);
+            if (voice) {
+                utterance.voice = voice;
+                utterance.lang = voice.lang;
+            }
+            else if (options.language) {
+                utterance.lang = options.language;
+            }
+            // Set parameters
+            utterance.rate = options.rate ?? this.options.defaultRate ?? 1;
+            utterance.pitch = options.pitch ?? this.options.defaultPitch ?? 1;
+            utterance.volume = options.volume ?? this.options.defaultVolume ?? 1;
+            // Set up event handlers
+            utterance.onstart = () => {
+                for (const cb of this.startListeners) {
+                    cb();
+                }
+            };
+            utterance.onend = () => {
+                this.currentUtterance = null;
+                for (const cb of this.endListeners) {
+                    cb();
+                }
+                resolve();
+            };
+            utterance.onerror = (event) => {
+                this.currentUtterance = null;
+                const error = new Error(`Speech synthesis error: ${event.error}`);
+                for (const cb of this.errorListeners) {
+                    cb(error);
+                }
+                reject(error);
+            };
+            utterance.onboundary = (event) => {
+                for (const cb of this.boundaryListeners) {
+                    cb(event.charIndex, event.charLength || 1);
+                }
+            };
+            this.currentUtterance = utterance;
+            this.synthesis?.speak(utterance);
+        });
+    }
+    stop() {
+        if (this.synthesis) {
+            this.synthesis.cancel();
+            this.currentUtterance = null;
+        }
+    }
+    pause() {
+        if (this.synthesis) {
+            this.synthesis.pause();
+        }
+    }
+    resume() {
+        if (this.synthesis) {
+            this.synthesis.resume();
+        }
+    }
+    isSpeaking() {
+        return this.synthesis?.speaking ?? false;
+    }
+    isPaused() {
+        return this.synthesis?.paused ?? false;
+    }
+    onStart(callback) {
+        this.startListeners.add(callback);
+        return () => this.startListeners.delete(callback);
+    }
+    onEnd(callback) {
+        this.endListeners.add(callback);
+        return () => this.endListeners.delete(callback);
+    }
+    onError(callback) {
+        this.errorListeners.add(callback);
+        return () => this.errorListeners.delete(callback);
+    }
+    onBoundary(callback) {
+        this.boundaryListeners.add(callback);
+        return () => this.boundaryListeners.delete(callback);
+    }
+    async dispose() {
+        this.stop();
+        this.synthesis = null;
+        this.voices = [];
+        this.startListeners.clear();
+        this.endListeners.clear();
+        this.errorListeners.clear();
+        this.boundaryListeners.clear();
+        this._initState = 'uninitialized';
+    }
+}