@happyvertical/smrt-svelte 0.30.0

package/dist/browser-ai/adapters/stt/whisper-cpp.js

@@ -0,0 +1,348 @@
+/**
+ * Whisper.cpp adapter using @remotion/whisper-web
+ *
+ * Uses whisper.cpp compiled to WASM for high-accuracy speech recognition.
+ * More stable than transformers.js implementation.
+ */
+import { importOptional } from '@happyvertical/smrt-ui/utils/import-optional.js';
+import { CapabilityNotAvailableError, InitializationError, } from '../../core/errors.js';
+// Model download sizes (approximate)
+const MODEL_SIZES = {
+    'tiny.en': 75 * 1024 * 1024,
+    'base.en': 142 * 1024 * 1024,
+    'small.en': 466 * 1024 * 1024,
+    tiny: 75 * 1024 * 1024,
+    base: 142 * 1024 * 1024,
+    small: 466 * 1024 * 1024,
+};
+/**
+ * Whisper.cpp adapter for high-accuracy speech recognition
+ */
+export class WhisperCppSTTAdapter {
+    type = 'whisper-cpp';
+    _initState = 'uninitialized';
+    options;
+    _isListening = false;
+    // Audio recording
+    mediaRecorder = null;
+    audioChunks = [];
+    stream = null;
+    // Event listeners
+    resultListeners = new Set();
+    errorListeners = new Set();
+    startListeners = new Set();
+    endListeners = new Set();
+    // @remotion/whisper-web module
+    whisperWeb = null;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: Used in init/dispose
+    modelDownloaded = false;
+    // Promise that resolves when transcription is complete
+    processingPromise = null;
+    processingResolve = null;
+    constructor(options = {}) {
+        this.options = {
+            type: 'whisper-cpp',
+            model: 'tiny.en',
+            ...options,
+        };
+    }
+    get initState() {
+        return this._initState;
+    }
+    get model() {
+        return this.options.model || 'tiny.en';
+    }
+    async ensureInitialized(onProgress) {
+        if (this._initState === 'ready')
+            return;
+        if (this._initState === 'initializing') {
+            return new Promise((resolve, reject) => {
+                const check = () => {
+                    if (this._initState === 'ready')
+                        resolve();
+                    else if (this._initState === 'error')
+                        reject(new Error('Initialization failed'));
+                    else
+                        setTimeout(check, 50);
+                };
+                check();
+            });
+        }
+        this._initState = 'initializing';
+        try {
+            // Check WASM support
+            if (typeof WebAssembly === 'undefined') {
+                throw new CapabilityNotAvailableError('WebAssembly', 'whisper-cpp');
+            }
+            const totalSize = MODEL_SIZES[this.model] || MODEL_SIZES['tiny.en'];
+            onProgress?.({
+                state: 'downloading',
+                bytesLoaded: 0,
+                bytesTotal: totalSize,
+                percent: 0,
+                currentFile: `whisper-${this.model}`,
+            });
+            // Dynamically import @remotion/whisper-web
+            this.whisperWeb = await this.importWhisperWeb();
+            // Check cross-origin isolation first
+            const isCrossOriginIsolated = typeof crossOriginIsolated !== 'undefined' && crossOriginIsolated;
+            console.log('[WhisperCpp] crossOriginIsolated:', isCrossOriginIsolated);
+            if (!isCrossOriginIsolated) {
+                throw new CapabilityNotAvailableError('Cross-origin isolation required. Add headers: Cross-Origin-Opener-Policy: same-origin, Cross-Origin-Embedder-Policy: require-corp', 'whisper-cpp');
+            }
+            // Check if whisper-web is supported
+            const { supported, error } = await this.whisperWeb.canUseWhisperWeb(this.model);
+            console.log('[WhisperCpp] canUseWhisperWeb:', { supported, error });
+            if (!supported) {
+                throw new CapabilityNotAvailableError(error || 'whisper-web not supported in this browser', 'whisper-cpp');
+            }
+            // Download the model
+            await this.whisperWeb.downloadWhisperModel({
+                model: this.model,
+                onProgress: (progress) => {
+                    const percent = Math.round(progress.progress * 100);
+                    onProgress?.({
+                        state: 'downloading',
+                        bytesLoaded: Math.round((totalSize * percent) / 100),
+                        bytesTotal: totalSize,
+                        percent,
+                        currentFile: `whisper-${this.model}`,
+                    });
+                },
+            });
+            this.modelDownloaded = true;
+            onProgress?.({
+                state: 'complete',
+                bytesLoaded: totalSize,
+                bytesTotal: totalSize,
+                percent: 100,
+            });
+            this._initState = 'ready';
+            console.log('[WhisperCpp] Initialized successfully');
+        }
+        catch (error) {
+            this._initState = 'error';
+            console.error('[WhisperCpp] Initialization error:', error);
+            throw error instanceof Error
+                ? error
+                : new InitializationError('whisper-cpp', String(error));
+        }
+    }
+    async importWhisperWeb() {
+        try {
+            return await importOptional('@remotion/whisper-web');
+        }
+        catch {
+            throw new InitializationError('whisper-cpp', '@remotion/whisper-web not installed. Run: npm install @remotion/whisper-web');
+        }
+    }
+    getCapabilities() {
+        return {
+            continuous: false,
+            interimResults: false,
+            languages: this.model.endsWith('.en')
+                ? ['en']
+                : [
+                    'en',
+                    'es',
+                    'fr',
+                    'de',
+                    'it',
+                    'pt',
+                    'nl',
+                    'pl',
+                    'ru',
+                    'zh',
+                    'ja',
+                    'ko',
+                    'ar',
+                    'hi',
+                    'tr',
+                    'vi',
+                    'th',
+                ],
+            accuracy: this.model.includes('small') ? 'high' : 'medium',
+            requiresDownload: true,
+            downloadSize: MODEL_SIZES[this.model] || MODEL_SIZES['tiny.en'],
+        };
+    }
+    async start(_options = {}) {
+        await this.ensureInitialized();
+        if (this._isListening)
+            return;
+        try {
+            // Get microphone access
+            this.stream = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    channelCount: 1,
+                    sampleRate: 16000,
+                },
+            });
+            this.audioChunks = [];
+            // Create MediaRecorder
+            this.mediaRecorder = new MediaRecorder(this.stream, {
+                mimeType: this.getSupportedMimeType(),
+            });
+            this.mediaRecorder.ondataavailable = (event) => {
+                if (event.data.size > 0) {
+                    this.audioChunks.push(event.data);
+                }
+            };
+            this.mediaRecorder.onstop = async () => {
+                await this.processRecording();
+                this.processingResolve?.();
+            };
+            this.processingPromise = new Promise((resolve) => {
+                this.processingResolve = resolve;
+            });
+            this.mediaRecorder.start();
+            this._isListening = true;
+            for (const cb of this.startListeners) {
+                cb();
+            }
+            console.log('[WhisperCpp] Started recording');
+        }
+        catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            for (const cb of this.errorListeners) {
+                cb(err);
+            }
+            throw err;
+        }
+    }
+    getSupportedMimeType() {
+        const types = [
+            'audio/webm;codecs=opus',
+            'audio/webm',
+            'audio/ogg;codecs=opus',
+            'audio/mp4',
+        ];
+        for (const type of types) {
+            if (MediaRecorder.isTypeSupported(type)) {
+                return type;
+            }
+        }
+        return 'audio/webm';
+    }
+    async processRecording() {
+        console.log('[WhisperCpp] Processing recording, chunks:', this.audioChunks.length);
+        if (this.audioChunks.length === 0) {
+            console.log('[WhisperCpp] No audio chunks captured');
+            return;
+        }
+        const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
+        console.log('[WhisperCpp] Audio blob size:', audioBlob.size);
+        try {
+            // Convert blob to File for resampleTo16Khz
+            const audioFile = new File([audioBlob], 'recording.webm', {
+                type: audioBlob.type,
+            });
+            // Resample to 16kHz as required by Whisper
+            console.log('[WhisperCpp] Resampling audio...');
+            const resampleResult = await this.whisperWeb.resampleTo16Khz({
+                file: audioFile,
+                onProgress: (p) => console.log('[WhisperCpp] Resample progress:', p),
+            });
+            console.log('[WhisperCpp] Resample result:', resampleResult);
+            console.log('[WhisperCpp] Resample result keys:', Object.keys(resampleResult));
+            // Handle different possible return structures
+            const channelWaveform = resampleResult.channelWaveform ?? resampleResult;
+            console.log('[WhisperCpp] Resampled, samples:', channelWaveform?.length);
+            // Transcribe
+            console.log('[WhisperCpp] Starting transcription...');
+            const { transcription } = await this.whisperWeb.transcribe({
+                channelWaveform,
+                model: this.model,
+                onProgress: (p) => console.log('[WhisperCpp] Transcribe progress:', p),
+            });
+            console.log('[WhisperCpp] Transcription result:', transcription);
+            // Extract text from transcription segments
+            const text = transcription
+                .map((segment) => segment.text)
+                .join(' ')
+                .trim();
+            const result = {
+                text,
+                confidence: 0.95,
+                isFinal: true,
+                timestamp: Date.now(),
+            };
+            for (const cb of this.resultListeners) {
+                cb(result);
+            }
+        }
+        catch (error) {
+            console.error('[WhisperCpp] Processing error:', error);
+            const err = error instanceof Error ? error : new Error(String(error));
+            for (const cb of this.errorListeners) {
+                cb(err);
+            }
+        }
+    }
+    async stop() {
+        if (this.mediaRecorder && this._isListening) {
+            this.mediaRecorder.stop();
+            if (this.stream) {
+                for (const track of this.stream.getTracks()) {
+                    track.stop();
+                }
+            }
+            this.stream = null;
+            if (this.processingPromise) {
+                await this.processingPromise;
+                this.processingPromise = null;
+                this.processingResolve = null;
+            }
+            this._isListening = false;
+            for (const cb of this.endListeners) {
+                cb();
+            }
+            console.log('[WhisperCpp] Stopped recording');
+        }
+    }
+    abort() {
+        if (this.mediaRecorder && this._isListening) {
+            this.mediaRecorder.stop();
+            this._isListening = false;
+            this.audioChunks = [];
+            if (this.stream) {
+                for (const track of this.stream.getTracks()) {
+                    track.stop();
+                }
+            }
+            this.stream = null;
+            for (const cb of this.endListeners) {
+                cb();
+            }
+        }
+    }
+    isListening() {
+        return this._isListening;
+    }
+    onResult(callback) {
+        this.resultListeners.add(callback);
+        return () => this.resultListeners.delete(callback);
+    }
+    onError(callback) {
+        this.errorListeners.add(callback);
+        return () => this.errorListeners.delete(callback);
+    }
+    onStart(callback) {
+        this.startListeners.add(callback);
+        return () => this.startListeners.delete(callback);
+    }
+    onEnd(callback) {
+        this.endListeners.add(callback);
+        return () => this.endListeners.delete(callback);
+    }
+    async dispose() {
+        this.abort();
+        this.whisperWeb = null;
+        this.modelDownloaded = false;
+        this.resultListeners.clear();
+        this.errorListeners.clear();
+        this.startListeners.clear();
+        this.endListeners.clear();
+        this._initState = 'uninitialized';
+    }
+}

package/dist/browser-ai/adapters/stt/whisper-wasm.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Whisper adapter using @xenova/transformers (v2)
+ *
+ * Uses Whisper models running in the browser via WASM.
+ * V2 uses a simpler, more stable pipeline API.
+ * Requires downloading model files (~40MB-150MB depending on model size).
+ */
+import type { InitState, OnProgress } from '../../core/types.js';
+import type { STTAdapter, STTCapabilities, STTOptions, STTResult, WhisperWasmSTTOptions } from './types.js';
+/**
+ * Whisper adapter for high-accuracy speech recognition
+ * Uses transformers.js v2 pipeline API
+ */
+export declare class WhisperWasmSTTAdapter implements STTAdapter {
+    readonly type: "whisper-wasm";
+    private _initState;
+    private options;
+    private _isListening;
+    private mediaRecorder;
+    private audioChunks;
+    private stream;
+    private resultListeners;
+    private errorListeners;
+    private startListeners;
+    private endListeners;
+    private transcriber;
+    private processingPromise;
+    private processingResolve;
+    constructor(options?: Partial<WhisperWasmSTTOptions>);
+    get initState(): InitState;
+    ensureInitialized(onProgress?: OnProgress): Promise<void>;
+    private transformersModule;
+    private importTransformers;
+    /**
+     * Configure transformers.js environment settings
+     */
+    private configureTransformersEnv;
+    getCapabilities(): STTCapabilities;
+    start(_options?: STTOptions): Promise<void>;
+    private getSupportedMimeType;
+    private processRecording;
+    stop(): Promise<void>;
+    abort(): void;
+    isListening(): boolean;
+    onResult(callback: (result: STTResult) => void): () => void;
+    onError(callback: (error: Error) => void): () => void;
+    onStart(callback: () => void): () => void;
+    onEnd(callback: () => void): () => void;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=whisper-wasm.d.ts.map

package/dist/browser-ai/adapters/stt/whisper-wasm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"whisper-wasm.d.ts","sourceRoot":"","sources":["../../../../src/browser-ai/adapters/stt/whisper-wasm.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAOH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,UAAU,EACV,SAAS,EACT,qBAAqB,EACtB,MAAM,YAAY,CAAC;AAgBpB;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,UAAU;IACtD,QAAQ,CAAC,IAAI,EAAG,cAAc,CAAU;IAExC,OAAO,CAAC,UAAU,CAA8B;IAChD,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,YAAY,CAAS;IAG7B,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,MAAM,CAA4B;IAG1C,OAAO,CAAC,eAAe,CAA0C;IACjE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,cAAc,CAAyB;IAC/C,OAAO,CAAC,YAAY,CAAyB;IAG7C,OAAO,CAAC,WAAW,CAAa;IAGhC,OAAO,CAAC,iBAAiB,CAA8B;IACvD,OAAO,CAAC,iBAAiB,CAA6B;gBAE1C,OAAO,GAAE,OAAO,CAAC,qBAAqB,CAAM;IASxD,IAAI,SAAS,IAAI,SAAS,CAEzB;IAEK,iBAAiB,CAAC,UAAU,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAmF/D,OAAO,CAAC,kBAAkB,CAAa;YAEzB,kBAAkB;IAuBhC;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAgBhC,eAAe,IAAI,eAAe;IA+B5B,KAAK,CAAC,QAAQ,GAAE,UAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAoDrD,OAAO,CAAC,oBAAoB;YAiBd,gBAAgB;IAqGxB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwB3B,KAAK,IAAI,IAAI;IAmBb,WAAW,IAAI,OAAO;IAItB,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,GAAG,MAAM,IAAI;IAK3D,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,MAAM,IAAI;IAKrD,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAKzC,KAAK,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAKjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAU/B"}