@qvac/decoder-audio 0.1.1 → 0.2.1

package/index.js

@@ -15,7 +15,9 @@ class GSTDecoder extends BaseInference {
   /**
    * Creates an instance of GSTDecoder.
    * @constructor
-   * @param {Object} config - environment-specific inference setup configuration
+   * @param {Object} options - Constructor options
+   * @param {Object} options.config - environment-specific inference setup configuration
+   * @param {*} [args] - Additional arguments passed to BaseInference
    */
 
   constructor ({ config, ...args }) {
@@ -101,6 +103,9 @@ class GSTDecoder extends BaseInference {
   }
 }
 
+const FFmpegDecoder = require('./lib/ffmpeg/ffmpeg-decoder')
+
 module.exports = GSTDecoder
 module.exports.FORMATS_NEEDING_DECODE = FORMATS_NEEDING_DECODE
 module.exports.SUPPORTED_AUDIO_FORMATS = SUPPORTED_AUDIO_FORMATS
+module.exports.FFmpegDecoder = FFmpegDecoder

package/lib/ffmpeg/channel-port-stream.js

@@ -0,0 +1,98 @@
+'use strict'
+
+const QvacLogger = require('@qvac/logging')
+
+class ChannelPortStream {
+  constructor (channelPort, logger) {
+    this.channelPort = channelPort
+    this.logger = new QvacLogger(logger)
+
+    this.channelEnd = false
+    this.channelClose = false
+
+    this.channelPort.on('end', () => {
+      this.logger.debug('ChannelPortStream Channel port ended')
+      this.channelEnd = true
+    })
+    this.channelPort.on('close', () => {
+      this.logger.debug('ChannelPortStream Channel port closed')
+      this.channelClose = true
+    })
+
+    this.bufferArray = []
+    this.bufferOneOffset = 0
+  }
+
+  portHasData () {
+    const hasData = (this.channelEnd || this.channelClose) && this.channelPort._queue.length
+    return hasData !== 0
+  }
+
+  _readFromChannelPort () {
+    let portHasDataCount = 0
+    while (this.portHasData()) {
+      portHasDataCount++
+      const chunk = this.channelPort.readSync()
+      if (!chunk || chunk.length === 0) {
+        this.logger.debug('Read chunk of length 0 from channelPort')
+        break
+      }
+      this.logger.debug(`Read chunk of length ${chunk.length} from channelPort`)
+      this.bufferArray.push(chunk)
+    }
+    this.logger.debug(`portHasData() loop ran ${portHasDataCount} times, bufferArray.length=${this.bufferArray.length}`)
+  }
+
+  _readFromBufferArray (buffer, len) {
+    let finalLen = 0
+    let bytesCopied = 0
+    if (this.bufferArray.length > 0) {
+      while (finalLen < len) {
+        // Get pending bytes in first buffer
+        const firstBufferPending = this.bufferArray[0].length - this.bufferOneOffset
+
+        // Get bytes to copy in case pending len is less than firstBufferPending
+        const toCopy = Math.min(len - finalLen, firstBufferPending)
+        this.logger.debug(`Copying ${toCopy} bytes from bufferArray[0] (pending: ${firstBufferPending}), bufferOneOffset=${this.bufferOneOffset}, finalLen=${finalLen}`)
+
+        // Copy bytes to buffer
+        bytesCopied += this.bufferArray[0].copy(buffer, finalLen, this.bufferOneOffset, this.bufferOneOffset + toCopy)
+        this.bufferOneOffset += toCopy
+
+        // Shift out buffer if it's empty
+        if (this.bufferOneOffset >= this.bufferArray[0].length) {
+          this.logger.debug('Finished bufferArray[0], shifting it out')
+          this.bufferArray.shift()
+          this.bufferOneOffset = 0
+        }
+
+        // Update finalLen
+        finalLen += toCopy
+
+        // Break inner loop if bufferArray has no more buffers
+        if (this.bufferArray.length === 0) {
+          this.logger.debug('bufferArray is empty, breaking inner loop')
+          break
+        }
+      }
+    } else {
+      this.logger.debug('bufferArray is empty, nothing to copy')
+    }
+
+    this.logger.debug(`_readFromBufferArray bytesCopied=${bytesCopied}`)
+    return finalLen
+  }
+
+  read (buffer, len) {
+    this.logger.debug(`Called with len=${len}`)
+
+    this._readFromChannelPort()
+
+    const finalLen = this._readFromBufferArray(buffer, len)
+
+    this.logger.debug(`Returning finalLen=${finalLen}`)
+    return finalLen
+  }
+}
+
+module.exports = ChannelPortStream

package/lib/ffmpeg/constants.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const FFmpegDecoderMessages = {
+  // Decoder messages
+  INIT: 'init', // Initialization message
+  STREAM_START: 'stream-start', // Start streaming message
+  CLEANUP: 'cleanup' // Cleanup message
+}
+
+const FFmpegWorkerMessages = {
+  // Worker messages
+  READY: 'ready', // Ready message
+  INIT_RESULT: 'init-result', // Initialization result message
+  RESULT_CHUNK: 'result-chunk', // Result chunk message
+  RESULT_END: 'result-end', // End streaming message
+  CLEANUP_DONE: 'cleanup-done', // Cleanup done message
+  ERROR: 'error' // Error message
+}
+
+module.exports = {
+  FFmpegDecoderMessages,
+  FFmpegWorkerMessages
+}

package/lib/ffmpeg/ffmpeg-decoder.js

@@ -0,0 +1,314 @@
+'use strict'
+
+const { Worker } = require('bare-worker')
+const QvacResponse = require('@qvac/response')
+const QvacLogger = require('@qvac/logging')
+const path = require('bare-path')
+const Channel = require('bare-channel')
+const { FFmpegDecoderMessages, FFmpegWorkerMessages } = require('./constants')
+const BaseInference = require('@qvac/infer-base/WeightsProvider/BaseInference')
+
+/**
+ * FFmpeg-based audio decoder that runs in a worker thread
+ */
+class FFmpegDecoder extends BaseInference {
+  /**
+   * Creates an instance of FFmpegDecoder.
+   * @param {Object} config - Configuration options
+   * @param logger - Logger instance
+   * @param streamIndex - Index of the stream to decode. Default: 0
+   * @param inputBitrate - Input audio bitrate. Default: 192000
+   * @param audioFormat - Output audio format. Default: 's16le'
+   * @param args - Additional arguments passed to BaseInference
+   * @param {Object} [config.streamIndex] - Index of the stream to decode (default: 0)
+   * @param {number} [config.inputBitrate] - Input audio bitrate (default: 192000)
+   * @param {string} [config.audioFormat] - Output audio format (default: 'f32le')
+   * @param {number} [config.sampleRate] - Output sample rate (default: 16000)
+   * @param {Object} [config.logger] - Logger instance
+   */
+  constructor ({
+    config = {},
+    logger = null,
+    streamIndex = 0,
+    inputBitrate = 192000,
+    audioFormat = 's16le',
+    ...args
+  }) {
+    super({ ...args, logger })
+
+    this.config = {
+      streamIndex,
+      inputBitrate,
+      audioFormat
+    }
+
+    this.logger = new QvacLogger(logger)
+    this.worker = null
+    this.isLoaded = false
+    this.currentJob = null
+  }
+
+  /**
+   * Load and initialize the decoder
+   */
+  async load () {
+    if (this.isLoaded) {
+      this.logger.info('FFmpegDecoder already loaded')
+      return
+    }
+
+    this.logger.info('Loading FFmpegDecoder with config:', this.config)
+
+    // Create worker thread
+    const workerPath = path.join(__dirname, 'ffmpeg-worker.js')
+    this.worker = new Worker(workerPath)
+
+    // Wait for worker to be ready
+    await new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        reject(new Error('Worker initialization timeout'))
+      }, 5000)
+
+      this.worker.once('message', (msg) => {
+        clearTimeout(timeout)
+        if (msg.type === FFmpegWorkerMessages.READY) {
+          resolve()
+        } else {
+          reject(new Error('Unexpected worker message during init'))
+        }
+      })
+    })
+
+    this.isLoaded = true
+    this.logger.info('FFmpegDecoder loaded successfully')
+  }
+
+  /**
+   * Unload the decoder and clean up resources
+   */
+  async unload () {
+    if (!this.isLoaded) {
+      return
+    }
+
+    this.logger.info('Unloading FFmpegDecoder')
+
+    if (this.worker) {
+      await new Promise((resolve) => {
+        this.worker.postMessage({ type: FFmpegDecoderMessages.CLEANUP })
+        this.worker.once('message', (msg) => {
+          if (msg.type === FFmpegWorkerMessages.CLEANUP_DONE) {
+            resolve()
+          }
+        })
+
+        // Fallback timeout
+        setTimeout(resolve, 1000)
+      })
+
+      await this.worker.terminate()
+      this.worker = null
+    }
+
+    this.isLoaded = false
+    this.currentJob = null
+    this.logger.info('FFmpegDecoder unloaded')
+  }
+
+  /**
+   * Run the decoder on an audio stream
+   * @param {Readable} audioStream - Input audio stream
+   * @returns {QvacResponse} Response with decoded audio
+   */
+  async run (audioStream) {
+    if (!this.isLoaded) {
+      throw new Error('Decoder not loaded. Call load() first.')
+    }
+
+    this.logger.info('Starting new audio stream processing')
+
+    const response = new QvacResponse({
+      cancelHandler: () => this.stop(),
+      pauseHandler: () => this.pause(),
+      continueHandler: () => this.unpause()
+    })
+
+    this.currentJob = {
+      response,
+      audioChunks: [],
+      isActive: true,
+      isPaused: false
+    }
+
+    // Process the audio stream
+    this._processStream(audioStream).catch(err => {
+      this.logger.error('Error processing audio stream:', err)
+      response.failed(err)
+    })
+
+    return response
+  }
+
+  _handleWorkerMessage (job) {
+    return new Promise((resolve, reject) => {
+      const messageHandler = async (msg) => {
+        this.logger.debug('[FFmpegDecoder] Received message from worker:', msg)
+        switch (msg.type) {
+          case FFmpegWorkerMessages.INIT_RESULT: {
+            if (msg.success) {
+              this.logger.info('[FFmpegDecoder] Decoder initialized successfully')
+              this.worker.postMessage({
+                type: FFmpegDecoderMessages.STREAM_START
+              })
+            } else {
+              this.worker.off('message', messageHandler)
+              this.logger.error('[FFmpegDecoder] Failed to initialize decoder:', msg.error)
+            }
+            break
+          }
+
+          case FFmpegWorkerMessages.RESULT_CHUNK: {
+            const decodedBuffer = Buffer.from(msg.data)
+            this.logger.debug('[FFmpegDecoder] Result chunk', decodedBuffer.length)
+            job.response.updateOutput({ outputArray: decodedBuffer })
+            break
+          }
+
+          case FFmpegWorkerMessages.RESULT_END: {
+            this.logger.debug('[FFmpegDecoder] Result end')
+            job.response.ended()
+            this.worker.off('message', messageHandler)
+            resolve()
+            break
+          }
+
+          case FFmpegWorkerMessages.ERROR: {
+            const errorMessage = msg.error || msg.data?.error || 'Unknown error'
+            this.logger.error('[FFmpegDecoder] Error from worker:', errorMessage)
+            job.response.failed(errorMessage)
+            this.worker.off('message', messageHandler)
+            reject(new Error(errorMessage))
+            break
+          }
+        }
+      }
+
+      this.worker.on('message', messageHandler)
+      this.worker.on('error', (err) => {
+        this.logger.error('[FFmpegDecoder] Error from worker:', err)
+        job.response.failed(err)
+        this.worker.off('message', messageHandler)
+        reject(err)
+      })
+    })
+  }
+
+  async _streamAudioToWorker (audioStream, port, job) {
+    let chunkCount = 0
+    for await (const chunk of audioStream) {
+      chunkCount++
+
+      this.logger.debug(`[FFmpegDecoder] Received audio chunk #${chunkCount}, size: ${chunk.length} bytes`)
+      if (!job.isActive) {
+        this.logger.info('[FFmpegDecoder] Job cancelled, stopping stream collection at chunk', chunkCount)
+        return
+      }
+
+      while (job.isPaused) {
+        this.logger.debug('[FFmpegDecoder] Job is paused, waiting to resume...')
+        await new Promise(resolve => setTimeout(resolve, 100))
+      }
+
+      port.writeSync(chunk)
+    }
+    return chunkCount
+  }
+
+  async _processStream (audioStream) {
+    const job = this.currentJob
+    if (!job.isActive) {
+      return
+    }
+
+    try {
+      const channel = new Channel()
+      const port = channel.connect()
+
+      this.logger.debug('[FFmpegDecoder] Processing audio stream with channel handle:', channel.handle)
+
+      // Worker promise to handle worker messages
+      const workerCompletionPromise = this._handleWorkerMessage(job)
+
+      // Initialize ffmpeg worker
+      this.worker.postMessage({
+        type: FFmpegDecoderMessages.INIT,
+        data: {
+          channelHandle: channel.handle,
+          config: this.config
+        }
+      })
+
+      // Stream all audio chunks to ffmpeg worker
+      const chunkCount = await this._streamAudioToWorker(audioStream, port, job)
+      this.logger.info(`[FFmpegDecoder] Finished streaming audio. Total chunks sent: ${chunkCount}`)
+
+      // Signal to worker that we're done sending audio
+      await port.close()
+
+      // Wait for worker to finish
+      await workerCompletionPromise
+    } catch (err) {
+      this.logger.error('Error processing audio stream:', err)
+      job.response.failed(err)
+    }
+
+    this.logger.info('Audio _processStream completed')
+  }
+
+  /**
+   * Pause the current job
+   */
+  pause () {
+    if (this.currentJob) {
+      this.currentJob.isPaused = true
+      this.logger.debug('Decoder paused')
+    }
+    return Promise.resolve()
+  }
+
+  /**
+   * Unpause the current job
+   */
+  unpause () {
+    if (this.currentJob) {
+      this.currentJob.isPaused = false
+      this.logger.debug('Decoder unpaused')
+    }
+    return Promise.resolve()
+  }
+
+  /**
+   * Stop the current job
+   */
+  stop () {
+    if (this.currentJob) {
+      this.currentJob.isActive = false
+      this.currentJob.response.finish()
+      this.logger.debug('Decoder stopped')
+    }
+    return Promise.resolve()
+  }
+
+  /**
+   * Get the current status
+   */
+  status () {
+    return {
+      loaded: this.isLoaded,
+      active: this.currentJob?.isActive || false,
+      paused: this.currentJob?.isPaused || false
+    }
+  }
+}
+
+module.exports = FFmpegDecoder

package/lib/ffmpeg/ffmpeg-worker.js

@@ -0,0 +1,293 @@
+'use strict'
+
+const { parentPort } = require('bare-worker')
+const ffmpeg = require('bare-ffmpeg')
+const QvacLogger = require('@qvac/logging')
+const Channel = require('bare-channel')
+const ChannelPortStream = require('./channel-port-stream')
+const { FFmpegDecoderMessages, FFmpegWorkerMessages } = require('./constants')
+
+class FFmpegWorker {
+  SUPPORTED_AUDIO_FORMATS = {
+    s16le: {
+      format: ffmpeg.constants.sampleFormats.S16,
+      byteLength: 2
+    },
+    f32le: {
+      format: ffmpeg.constants.sampleFormats.FLT,
+      byteLength: 4
+    }
+  }
+
+  // Whisper addon expectations
+  OUTPUT_FORMAT_BYTE_LENGTH = 2 // S16LE uses 16 bits = 2 bytes
+  OUTPUT_FORMAT = ffmpeg.constants.sampleFormats.S16
+  OUTPUT_SAMPLE_RATE = 16000
+  OUTPUT_CHANNEL_LAYOUT = ffmpeg.constants.channelLayouts.MONO
+
+  constructor (parentPort, logger = new QvacLogger(console)) {
+    this.parentPort = parentPort
+    this.logger = logger
+
+    this.channel = null
+    this.channelPort = null
+    this.channelPortStream = null
+
+    this.config = null
+    this.totalReadLen = 0
+
+    // Encoder delay handling
+    this.samplesSkipped = 0
+    this.totalSkipSamples = 0
+  }
+
+  sendMessage (type, data) {
+    this.parentPort.postMessage({ type, data })
+  }
+
+  sendResultChunk (data) {
+    this.sendMessage(FFmpegWorkerMessages.RESULT_CHUNK, data)
+  }
+
+  sendEndMessage () {
+    this.logger.debug('[ffmpeg-worker] Sending end message')
+    this.sendMessage(FFmpegWorkerMessages.RESULT_END)
+  }
+
+  initializeDecoder (data) {
+    const { channelHandle, config } = data
+    this.config = config
+
+    if (!this.SUPPORTED_AUDIO_FORMATS[config.audioFormat]) {
+      this.logger.error('[ffmpeg-worker] Unsupported audio format:', config.audioFormat)
+      return { success: false, error: 'Unsupported audio format' }
+    }
+
+    this.OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[config.audioFormat].format
+    this.OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[config.audioFormat].byteLength
+
+    this.logger.info('[ffmpeg-worker] Initializing decoder...')
+    try {
+      this.channel = Channel.from(channelHandle)
+      this.channelPort = this.channel.connect()
+      this.channelPortStream = new ChannelPortStream(this.channelPort, this.logger)
+      return { success: true }
+    } catch (err) {
+      this.logger.error('[ffmpeg-worker] Failed to initialize decoder:', err)
+      return { success: false, error: err.message }
+    }
+  }
+
+  startStream () {
+    this.decodeStream()
+  }
+
+  _processFrame (decoder, raw, resampler) {
+    while (decoder.receiveFrame(raw)) {
+      const output = new ffmpeg.Frame()
+      output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
+      output.format = this.OUTPUT_FORMAT
+      output.sampleRate = this.OUTPUT_SAMPLE_RATE
+      output.nbSamples = raw.nbSamples
+
+      const samples = new ffmpeg.Samples(
+        output.format,
+        output.channelLayout.nbChannels,
+        output.nbSamples
+      )
+      samples.fill(output)
+
+      // samples.data.length is always fixed, like 8192 bytes. So we need to
+      // extract only the data we need through `subarray`. `resampler.convert`
+      // gives the number of samples converted per channel. So
+      // `bytesPerSample` * `samplesPerChannel` * `channelsPerSample` gives
+      // total amount of bytes.
+      const count = resampler.convert(raw, output)
+
+      // Handle encoder delay by skipping initial samples
+      if (this.samplesSkipped < this.totalSkipSamples) {
+        const samplesToSkip = Math.min(count, this.totalSkipSamples - this.samplesSkipped)
+        this.samplesSkipped += samplesToSkip
+        if (samplesToSkip >= count) continue // Skip entire frame
+
+        // Skip partial frame
+        const skipBytes = this.OUTPUT_FORMAT_BYTE_LENGTH * samplesToSkip * output.channelLayout.nbChannels
+        const length = this.OUTPUT_FORMAT_BYTE_LENGTH * (count - samplesToSkip) * output.channelLayout.nbChannels
+        this.sendResultChunk(Buffer.from(samples.data.subarray(skipBytes, skipBytes + length)))
+      } else {
+        const length = this.OUTPUT_FORMAT_BYTE_LENGTH * count * output.channelLayout.nbChannels
+        this.sendResultChunk(Buffer.from(samples.data.subarray(0, length)))
+      }
+    }
+  }
+
+  _processPacket (format, packet, raw, decoder, resampler) {
+    while (format.readFrame(packet)) {
+      decoder.sendPacket(packet)
+      this._processFrame(decoder, raw, resampler)
+      packet.unref()
+    }
+  }
+
+  _processStream (format, stream) {
+    console.log(stream.codec, stream.codecParameters)
+    const packet = new ffmpeg.Packet()
+    const raw = new ffmpeg.Frame()
+
+    const resampler = new ffmpeg.Resampler(
+      stream.codecParameters.sampleRate,
+      stream.codecParameters.channelLayout,
+      stream.codecParameters.format,
+      this.OUTPUT_SAMPLE_RATE,
+      this.OUTPUT_CHANNEL_LAYOUT,
+      this.OUTPUT_FORMAT
+    )
+
+    const decoder = stream.decoder()
+    decoder.open()
+
+    // Auto-detect encoder delay: lossy codecs need ~400ms skipped to remove artifacts
+    // Lossless codecs (WAV, FLAC) need no skipping
+    const codecName = stream.codec.name.toLowerCase()
+    const SKIP_MS = {
+      mp3: 400,
+      vorbis: 400,
+      opus: 150,
+      aac: 300
+    }
+
+    const skipMs = SKIP_MS[codecName] || 0
+    this.samplesSkipped = 0
+    this.totalSkipSamples = Math.floor((this.OUTPUT_SAMPLE_RATE * skipMs) / 1000)
+
+    if (this.totalSkipSamples > 0) {
+      console.log(`[ffmpeg-worker] Skipping ${skipMs}ms (${this.totalSkipSamples} samples) for ${codecName} to remove encoder artifacts`)
+      this.logger.info(`[ffmpeg-worker] Skipping ${skipMs}ms for ${codecName} to remove encoder artifacts`)
+    }
+
+    this._processPacket(format, packet, raw, decoder, resampler)
+
+    const output = new ffmpeg.Frame()
+    output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
+    output.format = this.OUTPUT_FORMAT
+    output.sampleRate = this.OUTPUT_SAMPLE_RATE
+    output.nbSamples = 1024
+
+    const samples = new ffmpeg.Samples(
+      output.format,
+      output.channelLayout.nbChannels,
+      output.nbSamples
+    )
+    samples.fill(output)
+
+    while (resampler.flush(output) > 0) {
+      this.sendResultChunk(Buffer.from(samples.data))
+    }
+
+    decoder.destroy()
+    this.sendEndMessage()
+  }
+
+  _getBufferSize (inputBitrate) {
+    const maxBufferSize = 1024 * 1024 // 1MB max
+    return Math.min((inputBitrate / 8) * 4, maxBufferSize)
+  }
+
+  decodeStream () {
+    const {
+      streamIndex = 0,
+      inputBitrate
+    } = this.config
+
+    this.logger.debug('[ffmpeg-worker] Starting deocde with config', this.config)
+
+    if (!this.channelPortStream) {
+      this.logger.error('[ffmpeg-worker] Channel port stream not initialized')
+      return this.sendMessage(FFmpegWorkerMessages.ERROR, { error: 'Channel port stream not initialized' })
+    }
+
+    try {
+      const bufferSize = this._getBufferSize(inputBitrate)
+
+      const io = new ffmpeg.IOContext(bufferSize, {
+        onread: (buffer, requestedLen) => {
+          return this.channelPortStream.read(buffer, requestedLen)
+        }
+      })
+
+      this.logger.debug('[ffmpeg-worker] IOContext created')
+      const format = new ffmpeg.InputFormatContext(io)
+      this.logger.debug('[ffmpeg-worker] InputFormatContext created')
+
+      if (format.streams[streamIndex] === undefined) {
+        this.logger.error('[ffmpeg-worker] Stream index out of bounds')
+        return this.sendMessage(FFmpegWorkerMessages.ERROR, { error: 'Stream index out of bounds' })
+      }
+
+      this._processStream(format, format.streams[streamIndex])
+    } catch (err) {
+      this.logger.error('[ffmpeg-worker] Error during decodeChunk:', err)
+      return this.sendMessage(FFmpegWorkerMessages.ERROR, { error: err.message })
+    }
+  }
+
+  cleanup () {
+    this.logger.info('[ffmpeg-worker] Cleaning up decoder resources...')
+    this.logger.info('[ffmpeg-worker] Cleanup complete')
+    if (this.channelPort) {
+      this.channelPort.close()
+      this.logger.debug('[ffmpeg-worker] Channel port disposed')
+    }
+    if (this.channel) {
+      this.channel = null
+      this.logger.debug('[ffmpeg-worker] Channel disposed')
+    }
+  }
+}
+
+const logger = new QvacLogger()
+const ffmpegWorkerInstance = new FFmpegWorker(parentPort, logger)
+
+// Handle messages from main thread
+parentPort.on('message', ({ type, data }) => {
+  logger.debug(`[ffmpeg-worker] Received message: ${type}`)
+  switch (type) {
+    case FFmpegDecoderMessages.INIT: {
+      logger.info('[ffmpeg-worker] Handling init message with channel handle:', data)
+      const initResult = ffmpegWorkerInstance.initializeDecoder(data)
+      parentPort.postMessage({ type: FFmpegWorkerMessages.INIT_RESULT, ...initResult })
+      break
+    }
+
+    case FFmpegDecoderMessages.STREAM_START: {
+      logger.info('[ffmpeg-worker] Handling decode message')
+      ffmpegWorkerInstance.startStream()
+      break
+    }
+
+    case FFmpegDecoderMessages.CLEANUP: {
+      logger.info('[ffmpeg-worker] Handling cleanup message')
+      ffmpegWorkerInstance.cleanup()
+      parentPort.postMessage({ type: FFmpegWorkerMessages.CLEANUP_DONE, success: true })
+      break
+    }
+
+    default: {
+      logger.warn(`[ffmpeg-worker] Unknown message type: ${type}`)
+      parentPort.postMessage({ type: FFmpegWorkerMessages.ERROR, error: 'Unknown message type' })
+    }
+  }
+})
+
+parentPort.on('close', () => {
+  logger.info('[ffmpeg-worker] Worker exiting')
+})
+
+parentPort.on('error', (err) => {
+  logger.error('[ffmpeg-worker] Worker error:', err)
+  parentPort.postMessage({ type: FFmpegWorkerMessages.ERROR, error: err })
+})
+
+// Notify main thread that worker is ready
+logger.info('[ffmpeg-worker] Worker ready')
+parentPort.postMessage({ type: 'ready' })

package/lib/ffmpeg/index.d.ts

@@ -0,0 +1,98 @@
+import { Readable } from "stream";
+import { QvacResponse } from "@qvac/infer-base";
+import type { Loader, WhisperConfig } from "..";
+
+declare interface FFmpegDecoderConfig {
+  streamIndex?: number;
+  inputBitrate?: number;
+}
+
+declare interface TranscriptionAddonArgs {
+  loader: Loader;
+  params?: {
+    decoder?: FFmpegDecoderConfig;
+    [key: string]: unknown;
+  };
+  logger?: any;
+  modelName?: string;
+  vadModelName?: string;
+  diskPath?: string;
+  [key: string]: unknown;
+}
+
+declare interface TranscriptionAddonConfig {
+  path?: string;
+  opts?: {
+    stats?: boolean;
+    [key: string]: unknown;
+  };
+  whisperConfig?: WhisperConfig;
+  [key: string]: unknown;
+}
+
+/**
+ * TranscriptionAddon with FFmpeg decoder support
+ *
+ * Provides a complete transcription pipeline that:
+ * - Decodes various audio formats (WAV, MP3, M4A, OGG, Opus, FLAC, etc.)
+ * - Resamples to 16kHz mono
+ * - Transcribes using Whisper model
+ */
+declare class TranscriptionAddon {
+  /**
+   * Creates an instance of TranscriptionAddon
+   * @param args - Configuration arguments including loader, model paths, and decoder config
+   * @param config - Whisper configuration including VAD settings
+   */
+  constructor(args: TranscriptionAddonArgs, config?: TranscriptionAddonConfig);
+
+  /**
+   * Load model, decoder, and activate addon
+   * @param closeLoader - Whether to close the loader after loading
+   * @param reportProgress - Callback function to report loading progress
+   */
+  load(
+    closeLoader?: boolean,
+    reportProgress?: (data: any) => void
+  ): Promise<void>;
+
+  /**
+   * Unload and clean up resources
+   */
+  unload(): Promise<void>;
+
+  /**
+   * Run transcription on an audio stream
+   * @param audioStream - Audio stream in any format supported by FFmpeg
+   * @returns Promise that resolves to a QvacResponse with transcription results
+   */
+  run(audioStream: Readable): Promise<QvacResponse>;
+
+  /**
+   * Download model files
+   * @param progressReport - Optional progress report instance
+   */
+  download(progressReport?: any): Promise<any>;
+
+  /**
+   * Delete local model files
+   */
+  delete(): Promise<any>;
+
+  /**
+   * Pause inference
+   */
+  pause(): void;
+
+  /**
+   * Resume inference
+   */
+  resume(): void;
+
+  /**
+   * Destroy and clean up all resources
+   */
+  destroy(): Promise<void>;
+}
+
+export = TranscriptionAddon;

package/lib/ffmpeg/index.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const TranscriptionPipeline = require('@qvac/util-transcription')
+const TranscriptionWhispercpp = require('../index')
+const FFmpegDecoder = require('./ffmpeg-decoder')
+
+class TranscriptionAddon extends TranscriptionPipeline {
+  constructor ({ loader, params, logger, ...args }, config = {}) {
+    const whisperAddon = new TranscriptionWhispercpp(
+      {
+        loader,
+        params,
+        logger,
+        ...args
+      },
+      { ...config }
+    )
+
+    // Create FFmpeg decoder instance
+    const decoder = new FFmpegDecoder({
+      config: { ...params.decoder, audio_format: config?.audio_format },
+      logger
+    })
+
+    super({ whisperAddon, decoder }, params.decoder)
+  }
+}
+
+module.exports = TranscriptionAddon

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qvac/decoder-audio",
-  "version": "0.1.1",
+  "version": "0.2.1",
   "description": "",
   "license": "Apache-2.0",
   "author": "Tether",
@@ -13,10 +13,10 @@
     "test": "npm run test:unit && npm run test:integration",
     "coverage:unit": "brittle-bare --coverage --cov-dir=coverage/unit test/unit/*.test.js && npx istanbul report html --include=coverage/unit/coverage-final.json",
     "coverage": "brittle-bare --coverage --cov-dir=coverage/unit test/unit/*.test.js && npx istanbul report html --include=coverage/unit/coverage-final.json",
-    "lint": "standard --ignore 'addon/**'",
-    "lint:fix": "standard --fix --ignore 'addon/**'",
+    "lint": "standard \"test/**/*.js\" \"*.js\"",
+    "lint:fix": "standard --fix \"test/**/*.js\" \"*.js\"",
     "test:unit:generate": "brittle -r test/unit/all.js test/unit/*.test.js",
-    "test:integration:generate": "brittle -r test/integration/all.js test/integration/*.test.js",
+    "test:integration:generate": "brittle -r test/integration/all.js test/integration/ffmpeg-decoder.test.js",
     "test:integration": "npm run test:integration:generate && bare test/integration/all.js"
   },
   "files": [
@@ -27,19 +27,18 @@
     "constants.d.ts",
     "prebuilds",
     "utils",
-    "index.d.ts"
+    "lib",
+    "index.d.ts",
+    "test/mobile"
   ],
   "exports": {
     "./package": "./package.json",
-    ".": "./index.js",
-    "./constants": "./constants.js",
-    "./constants.js": "./constants.js"
+    ".": "./index.js"
   },
   "devDependencies": {
     "@types/node": "^22.14.1",
-    "bare-process": "^4.2.1",
     "brittle": "^3.13.1",
-    "cmake-bare": "^1.5.2",
+    "cmake-bare": "^1.7.5",
     "cmake-vcpkg": "^1.1.0",
     "istanbul": "^0.4.5",
     "standard": "^17.1.2",
@@ -48,9 +47,16 @@
   "dependencies": {
     "@qvac/infer-base": "^0.1.0",
     "@qvac/error": "^0.1.0",
+    "@qvac/logging": "^0.1.0",
     "@qvac/response": "^0.1.0",
-    "bare-assert": "^1.0.2",
-    "bare-fs": "^4.0.2"
+    "bare-assert": "^1.1.0",
+    "bare-channel": "^5.2.2",
+    "bare-ffmpeg": "^1.0.0-32",
+    "bare-fs": "^4.5.1",
+    "bare-path": "^3.0.0",
+    "bare-process": "^4.2.2",
+    "process": "npm:bare-process@^4.2.2",
+    "bare-worker": "^4.1.0"
   },
   "bugs": "https://github.com/tetherto/qvac-lib-decoder-audio/issues",
   "types": "index.d.ts"

package/prebuilds/darwin-arm64/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.1.1"
+        "version": "0.2.1"
       }
     ],
     "exported_symbols": [

package/prebuilds/darwin-x64/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.1.1"
+        "version": "0.2.1"
       }
     ],
     "exported_symbols": [

package/prebuilds/ios-arm64/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.1.1"
+        "version": "0.2.1"
       }
     ],
     "exported_symbols": [

package/prebuilds/ios-arm64-simulator/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.1.1"
+        "version": "0.2.1"
       }
     ],
     "exported_symbols": [

package/prebuilds/ios-x64-simulator/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.1.1"
+        "version": "0.2.1"
       }
     ],
     "exported_symbols": [

package/test/mobile/test.cjs

@@ -0,0 +1,103 @@
+const GSTDecoder = require('@qvac/decoder-audio')
+const fs = require('bare-fs')
+
+async function testDecodeMp3() {
+  const decoder = new GSTDecoder({
+    config: {
+      audioFormat: 'f32le',
+      sampleRate: 16000
+    }
+  })
+
+  try {
+    await decoder.load()
+    
+    const audioPath = getAssetPath('sample.mp3')
+    const audioStream = fs.createReadStream(audioPath)
+    const response = await decoder.run(audioStream)
+
+    let totalBytes = 0
+    let updateCount = 0
+
+    await response
+      .onUpdate(output => {
+        if (output && output.outputArray) {
+          const bytes = new Uint8Array(output.outputArray)
+          totalBytes += bytes.length
+          updateCount++
+        }
+      })
+      .await()
+
+    if (totalBytes === 0) {
+      throw new Error('No audio data decoded')
+    }
+
+    if (updateCount === 0) {
+      throw new Error('No decoder updates received')
+    }
+
+    console.log(`MP3 decode complete: ${totalBytes} bytes, ${updateCount} updates`)
+
+    return { 
+      success: true,
+      totalBytes,
+      updateCount
+    }
+  } catch (error) {
+    console.error('Error during testDecodeMp3:', error)
+    throw error
+  }
+}
+
+async function testDecodeWav() {
+  const decoder = new GSTDecoder({
+    config: {
+      audioFormat: 'f32le',
+      sampleRate: 16000
+    }
+  })
+
+  try {
+    await decoder.load()
+    
+    const audioPath = getAssetPath('sample.wav')
+    const audioStream = fs.createReadStream(audioPath)
+    const response = await decoder.run(audioStream)
+
+    let totalBytes = 0
+    let updateCount = 0
+
+    await response
+      .onUpdate(output => {
+        if (output && output.outputArray) {
+          const bytes = new Uint8Array(output.outputArray)
+          totalBytes += bytes.length
+          updateCount++
+        }
+      })
+      .await()
+
+    if (totalBytes === 0) {
+      throw new Error('No audio data decoded')
+    }
+
+    if (updateCount === 0) {
+      throw new Error('No decoder updates received')
+    }
+
+    console.log(`WAV decode complete: ${totalBytes} bytes, ${updateCount} updates`)
+
+    return { 
+      success: true,
+      totalBytes,
+      updateCount
+    }
+  } catch (error) {
+    console.error('Error during testDecodeWav:', error)
+    throw error
+  }
+}
+
+module.exports = { testDecodeMp3, testDecodeWav }
+