node-av-lgpl 0.0.0 → 7.1.0-beta.1

package/dist/api/whisper.js

@@ -0,0 +1,402 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+import { AV_NOPTS_VALUE } from '../constants/constants.js';
+import { FilterPreset } from './filter-presets.js';
+import { FilterAPI } from './filter.js';
+import { WhisperDownloader } from './utilities/whisper-model.js';
+/**
+ * High-level Whisper transcriber for automatic speech recognition.
+ *
+ * Provides streaming audio transcription using OpenAI's Whisper model via whisper.cpp.
+ * Supports GPU acceleration, VAD (Voice Activity Detection), and real-time processing.
+ * Built on FFmpeg's whisper filter with automatic frame metadata extraction.
+ *
+ * Features:
+ * - Real-time streaming transcription
+ * - GPU acceleration (CUDA/Vulkan/Metal)
+ * - Voice Activity Detection for better segmentation
+ * - Automatic language detection
+ * - Type-safe transcription segments
+ * - Frame-based API for flexible integration
+ *
+ * @example
+ * ```typescript
+ * import { Demuxer, Decoder, WhisperTranscriber } from 'node-av/api';
+ * import { WhisperDownloader } from 'node-av/api/utilities/whisper-model';
+ *
+ * // Download model
+ * const modelPath = await WhisperDownloader.downloadModel({
+ *   model: 'base.en',
+ *   outputPath: './models'
+ * });
+ *
+ * // Open audio and create decoder
+ * await using input = await Demuxer.open('podcast.mp3');
+ * using decoder = await Decoder.create(input.audio());
+ *
+ * // Create transcriber
+ * await using transcriber = await WhisperTranscriber.create({
+ *   model: modelPath,
+ *   language: 'en'
+ * });
+ *
+ * // Transcribe using decoded frames
+ * for await (const segment of transcriber.transcribe(decoder.frames(input.packets()))) {
+ *   const timestamp = `[${(segment.start / 1000).toFixed(1)}s - ${(segment.end / 1000).toFixed(1)}s]`;
+ *   console.log(`${timestamp}: ${segment.text}`);
+ * }
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Real-time microphone transcription with VAD
+ * import { Demuxer, Decoder, WhisperTranscriber } from 'node-av/api';
+ * import { WhisperDownloader } from 'node-av/api/utilities/whisper-model';
+ *
+ * // Download VAD model
+ * const vadPath = await WhisperDownloader.downloadVADModel('silero-v5.1.2', './models');
+ *
+ * // Setup transcriber with VAD
+ * await using transcriber = await WhisperTranscriber.create({
+ *   model: './models/ggml-medium.bin',
+ *   language: 'en',
+ *   queue: 10,
+ *   vadModel: vadPath,
+ *   vadThreshold: 0.5
+ * });
+ *
+ * // Live transcription from decoded audio frames
+ * using decoder = await Decoder.create(microphoneStream);
+ * for await (const segment of transcriber.transcribe(decoder.frames(microphonePackets))) {
+ *   if (segment.turn) {
+ *     console.log('\n--- New speaker turn ---');
+ *   }
+ *   console.log(segment.text);
+ * }
+ * ```
+ *
+ * @see {@link WhisperDownloader} For downloading Whisper and VAD models
+ * @see {@link Decoder} For audio decoding
+ * @see {@link Demuxer} For reading media files
+ */
+export class WhisperTranscriber {
+    options;
+    isClosed = false;
+    /**
+     * @param options - Transcriber configuration
+     *
+     * Use {@link create} factory method instead
+     *
+     * @internal
+     */
+    constructor(options) {
+        this.options = options;
+    }
+    /**
+     * Create a Whisper transcriber instance.
+     *
+     * Initializes the transcriber with the specified model and configuration.
+     * The transcriber can then process audio frames from any source.
+     *
+     * @param options - Transcriber configuration
+     *
+     * @returns Configured transcriber instance
+     *
+     * @throws {Error} If model file does not exist
+     *
+     * @throws {Error} If VAD model file does not exist (when vadModel specified)
+     *
+     * @example
+     * ```typescript
+     * import { WhisperTranscriber } from 'node-av/api';
+     *
+     * // Create transcriber with basic options
+     * await using transcriber = await WhisperTranscriber.create({
+     *   model: './models/ggml-base.en.bin',
+     *   language: 'en'
+     * });
+     * ```
+     *
+     * @example
+     * ```typescript
+     * // Create transcriber with GPU and VAD support
+     * await using transcriber = await WhisperTranscriber.create({
+     *   model: './models/ggml-base.bin',
+     *   language: 'auto',
+     *   useGpu: true,
+     *   gpuDevice: 0,
+     *   vadModel: './models/ggml-silero-v5.1.2.bin',
+     *   vadThreshold: 0.5,
+     *   queue: 10
+     * });
+     * ```
+     */
+    static async create(options) {
+        const modelsToDownload = [options.model, options.vadModel].filter(Boolean);
+        const [modelPath, vadModelPath] = await WhisperDownloader.downloadModels(modelsToDownload, options.modelDir);
+        const fullOptions = {
+            model: modelPath,
+            vadModel: vadModelPath,
+            modelDir: options.modelDir ?? WhisperDownloader.DEFAULT_MODEL_PATH,
+            language: options.language ?? 'auto',
+            queue: options.queue ?? 3,
+            useGpu: options.useGpu ?? true,
+            gpuDevice: options.gpuDevice ?? 0,
+            vadThreshold: options.vadThreshold ?? 0.5,
+            vadMinSpeechDuration: options.vadMinSpeechDuration ?? 0.1,
+            vadMinSilenceDuration: options.vadMinSilenceDuration ?? 0.5,
+        };
+        return new WhisperTranscriber(fullOptions);
+    }
+    /**
+     * Transcribe audio frames to text segments.
+     *
+     * Processes audio frames through the Whisper filter and yields transcribed segments.
+     * Each segment contains start/end timestamps and the transcribed text.
+     * Reads metadata directly from frame metadata tags (lavfi.whisper.text, lavfi.whisper.duration).
+     *
+     * The generator continues until the input stream ends or close() is called.
+     * When the input ends, the filter is flushed so audio still buffered in the
+     * whisper queue is transcribed before the generator finishes.
+     * Always use with `for await...of` to properly handle async iteration.
+     *
+     * @param frames - Audio frames (from Decoder.frames()) or single frame to transcribe
+     *
+     * @yields {WhisperSegment} Transcribed audio segments with timing and text
+     *
+     * @throws {FFmpegError} If filter initialization fails
+     *
+     * @example
+     * ```typescript
+     * import { Demuxer, Decoder, WhisperTranscriber } from 'node-av/api';
+     *
+     * await using input = await Demuxer.open('podcast.mp3');
+     * using decoder = await Decoder.create(input.audio());
+     * await using transcriber = await WhisperTranscriber.create({
+     *   model: './models/ggml-base.en.bin',
+     *   language: 'en'
+     * });
+     *
+     * // Transcribe decoded frames
+     * for await (const segment of transcriber.transcribe(decoder.frames(input.packets()))) {
+     *   console.log(`[${segment.start}ms]: ${segment.text}`);
+     * }
+     * ```
+     *
+     * @example
+     * ```typescript
+     * // With custom timing format
+     * const audioFrames = decoder.frames(input.packets());
+     * for await (const segment of transcriber.transcribe(audioFrames)) {
+     *   const startSec = (segment.start / 1000).toFixed(2);
+     *   const endSec = (segment.end / 1000).toFixed(2);
+     *   console.log(`[${startSec}s - ${endSec}s]: ${segment.text}`);
+     * }
+     * ```
+     *
+     * @example
+     * ```typescript
+     * // Process single frame
+     * using frame = decoder.decodeSync(packet);
+     * for await (const segment of transcriber.transcribe(frame)) {
+     *   console.log(`Transcribed: ${segment.text}`);
+     * }
+     * ```
+     */
+    async *transcribe(frames) {
+        const env_1 = { stack: [], error: void 0, hasError: false };
+        try {
+            const chain = FilterPreset.chain()
+                .whisper({
+                model: this.options.model,
+                language: this.options.language,
+                queue: this.options.queue,
+                useGpu: this.options.useGpu,
+                gpuDevice: this.options.gpuDevice,
+                vadModel: this.options.vadModel,
+                vadThreshold: this.options.vadThreshold,
+                vadMinSpeechDuration: this.options.vadMinSpeechDuration,
+                vadMinSilenceDuration: this.options.vadMinSilenceDuration,
+            })
+                .build();
+            // Create filter API
+            const filter = __addDisposableResource(env_1, FilterAPI.create(chain, {
+                allowReinit: true,
+                dropOnChange: false,
+            }), false);
+            // Fallback clock for frames without usable timestamps (in milliseconds)
+            let fallbackEnd = 0;
+            const toSegment = (frame) => {
+                const metadata = frame.getMetadata();
+                const text = metadata.get('lavfi.whisper.text');
+                const durationStr = metadata.get('lavfi.whisper.duration');
+                if (!text?.trim()) {
+                    return null;
+                }
+                // Parse duration (in seconds)
+                const duration = durationStr ? parseFloat(durationStr) * 1000 : 0;
+                // The whisper filter attaches its result to the frame that triggered the
+                // transcription, so that frame's timestamp marks (approximately) the end
+                // of the transcribed chunk. Deriving start/end from the frame PTS keeps
+                // segments aligned with the actual audio position; a counter that only
+                // advances on chunks that produced text would silently swallow every
+                // silent span and drift (e.g. a 60s intro would put the first segment
+                // at start 0).
+                const tb = frame.timeBase;
+                const frameTime = frame.pts !== AV_NOPTS_VALUE && tb.den !== 0 ? (Number(frame.pts) * 1000 * tb.num) / tb.den : null;
+                const end = frameTime ?? fallbackEnd + duration;
+                fallbackEnd = end;
+                return {
+                    start: Math.max(0, end - duration),
+                    end,
+                    text: text.trim(),
+                };
+            };
+            // Decode and process frames through filter
+            for await (const frame_1 of filter.frames(frames)) {
+                const env_2 = { stack: [], error: void 0, hasError: false };
+                try {
+                    const frame = __addDisposableResource(env_2, frame_1, false);
+                    if (this.isClosed) {
+                        break;
+                    }
+                    if (!frame?.isAudio()) {
+                        continue;
+                    }
+                    const segment = toSegment(frame);
+                    if (segment) {
+                        yield segment;
+                    }
+                }
+                catch (e_1) {
+                    env_2.error = e_1;
+                    env_2.hasError = true;
+                }
+                finally {
+                    __disposeResources(env_2);
+                }
+            }
+            // Inputs that end without a trailing null never flush the filter, and the
+            // whisper queue (up to `queue` seconds of buffered audio) would be
+            // destroyed with it. Drain explicitly; after an upstream EOF this second
+            // flush is a no-op.
+            if (!this.isClosed) {
+                for await (const frame_2 of filter.flushFrames()) {
+                    const env_3 = { stack: [], error: void 0, hasError: false };
+                    try {
+                        const frame = __addDisposableResource(env_3, frame_2, false);
+                        if (!frame.isAudio()) {
+                            continue;
+                        }
+                        const segment = toSegment(frame);
+                        if (segment) {
+                            yield segment;
+                        }
+                    }
+                    catch (e_2) {
+                        env_3.error = e_2;
+                        env_3.hasError = true;
+                    }
+                    finally {
+                        __disposeResources(env_3);
+                    }
+                }
+            }
+        }
+        catch (e_3) {
+            env_1.error = e_3;
+            env_1.hasError = true;
+        }
+        finally {
+            __disposeResources(env_1);
+        }
+    }
+    /**
+     * Close transcriber and clean up resources.
+     *
+     * Releases filter graph and stops frame processing.
+     * Called automatically when using `await using` syntax.
+     *
+     * @example
+     * ```typescript
+     * // Automatic cleanup
+     * {
+     *   await using transcriber = await WhisperTranscriber.create(options);
+     *   // Use transcriber
+     * } // Automatically calls close()
+     *
+     * // Manual cleanup
+     * const transcriber = await WhisperTranscriber.create(options);
+     * try {
+     *   // Use transcriber
+     * } finally {
+     *   await transcriber.close();
+     * }
+     * ```
+     */
+    close() {
+        if (this.isClosed) {
+            return;
+        }
+        this.isClosed = true;
+    }
+    /**
+     * Symbol.asyncDispose implementation for `await using` syntax.
+     *
+     * @internal
+     */
+    [Symbol.dispose]() {
+        this.close();
+    }
+}
+//# sourceMappingURL=whisper.js.map

package/dist/api/whisper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"whisper.js","sourceRoot":"","sources":["../../src/api/whisper.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAkIjE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0EG;AACH,MAAM,OAAO,kBAAkB;IACrB,OAAO,CAAsC;IAC7C,QAAQ,GAAG,KAAK,CAAC;IAEzB;;;;;;OAMG;IACH,YAAoB,OAA4C;QAC9D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAsCG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAkC;QACpD,MAAM,gBAAgB,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAA+C,CAAC;QACzH,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,MAAM,iBAAiB,CAAC,cAAc,CAAC,gBAAgB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE7G,MAAM,WAAW,GAAwC;YACvD,KAAK,EAAE,SAA6B;YACpC,QAAQ,EAAE,YAAmC;YAC7C,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC,kBAAkB;YAClE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM;YACpC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;YACzB,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI;YAC9B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC;YACjC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,GAAG;YACzC,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,IAAI,GAAG;YACzD,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,IAAI,GAAG;SAC5D,CAAC;QAEF,OAAO,IAAI,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAsDG;IACH,KAAK,CAAC,CAAC,UAAU,CAAC,MAAkD;;;YAClE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE;iBAC/B,OAAO,CAAC;gBACP,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;gBACzB,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAC/B,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;gBACzB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC3B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;gBACjC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAC/B,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY;gBACvC,oBAAoB,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;gBACvD,qBAAqB,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB;aAC1D,CAAC;iBACD,KAAK,EAAE,CAAC;YAEX,oBAAoB;YACpB,MAAM,MAAM,kCAAG,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE;gBACrC,WAAW,EAAE,IAAI;gBACjB,YAAY,EAAE,KAAK;aACpB,CAAC,QAAA,CAAC;YAEH,wEAAwE;YACxE,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,MAAM,SAAS,GAAG,CAAC,KAAY,EAAyB,EAAE;gBACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;gBACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;gBAChD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;gBAE3D,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBAClB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,8BAA8B;gBAC9B,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBAElE,yEAAyE;gBACzE,yEAAyE;gBACzE,wEAAwE;gBACxE,uEAAuE;gBACvE,qEAAqE;gBACrE,sEAAsE;gBACtE,eAAe;gBACf,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAC1B,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,KAAK,cAAc,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;gBACrH,MAAM,GAAG,GAAG,SAAS,IAAI,WAAW,GAAG,QAAQ,CAAC;gBAChD,WAAW,GAAG,GAAG,CAAC;gBAElB,OAAO;oBACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC;oBAClC,GAAG;oBACH,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;iBAClB,CAAC;YACJ,CAAC,CAAC;YAEF,2CAA2C;YAC3C,IAAI,KAAK,mBAAiB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;;;0BAAjC,KAAK,iDAAA;oBACpB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;oBAED,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC;wBACtB,SAAS;oBACX,CAAC;oBAED,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACjC,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,OAAO,CAAC;oBAChB,CAAC;;;;;;;;;aACF;YAED,0EAA0E;YAC1E,mEAAmE;YACnE,yEAAyE;YACzE,oBAAoB;YACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,IAAI,KAAK,mBAAiB,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;;;8BAAhC,KAAK,iDAAA;wBACpB,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;4BACrB,SAAS;wBACX,CAAC;wBAED,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;wBACjC,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,OAAO,CAAC;wBAChB,CAAC;;;;;;;;;iBACF;YACH,CAAC;;;;;;;;;KACF;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACH,CAAC,MAAM,CAAC,OAAO,CAAC;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;CACF"}