@qvac/decoder-audio 0.2.10 → 0.3.1

package/index.js

@@ -1,119 +1,413 @@
 'use strict'
 
 const QvacResponse = require('@qvac/response')
+const QvacLogger = require('@qvac/logging')
+const ffmpeg = require('bare-ffmpeg')
 const BaseInference = require('@qvac/infer-base/WeightsProvider/BaseInference')
-const { GSTDecoderInterface } = require('./gstreamer')
-const createStreamAccumulator = require('./utils/createStreamAccumulator')
-const { FORMATS_NEEDING_DECODE, SUPPORTED_AUDIO_FORMATS } = require('./constants')
-
-const END_OF_INPUT = 'end of job'
 
 /**
- * GSTDecoder client implementation for the Whisper transcription model
+ * FFmpeg-based audio decoder (single-threaded)
  */
-class GSTDecoder extends BaseInference {
+class FFmpegDecoder extends BaseInference {
+  SUPPORTED_AUDIO_FORMATS = {
+    s16le: {
+      format: null, // Will be set to ffmpeg.constants.sampleFormats.S16
+      byteLength: 2
+    },
+    f32le: {
+      format: null, // Will be set to ffmpeg.constants.sampleFormats.FLT
+      byteLength: 4
+    }
+  }
+
+  OUTPUT_CHANNEL_LAYOUT = null // Will be set to ffmpeg.constants.channelLayouts.MONO
   /**
-   * Creates an instance of GSTDecoder.
-   * @constructor
-   * @param {Object} options - Constructor options
-   * @param {Object} options.config - environment-specific inference setup configuration
-   * @param {*} [args] - Additional arguments passed to BaseInference
+   * Creates an instance of FFmpegDecoder.
+   * @param {Object} config - Configuration options
+   * @param logger - Logger instance
+   * @param streamIndex - Index of the stream to decode. Default: 0
+   * @param inputBitrate - Input audio bitrate. Default: 192000
+   * @param audioFormat - Output audio format. Default: 's16le'
+   * @param args - Additional arguments passed to BaseInference
+   * @param {Object} [config.streamIndex] - Index of the stream to decode (default: 0)
+   * @param {number} [config.inputBitrate] - Input audio bitrate (default: 192000)
+   * @param {string} [config.audioFormat] - Output audio format (default: 'f32le')
+   * @param {number} [config.sampleRate] - Output sample rate (default: 16000)
+   * @param {Object} [config.logger] - Logger instance
    */
+  constructor ({
+    config = {},
+    logger = null,
+    streamIndex = 0,
+    inputBitrate = 192000,
+    audioFormat = 's16le',
+    ...args
+  }) {
+    super({ ...args, logger })
+
+    this.config = {
+      streamIndex: config.streamIndex || streamIndex,
+      inputBitrate: config.inputBitrate || inputBitrate,
+      audioFormat: config.audioFormat || audioFormat,
+      sampleRate: config.sampleRate || 16000
+    }
+
+    this.logger = new QvacLogger(logger)
+    this.isLoaded = false
+    this.currentJob = null
 
-  constructor ({ config, ...args }) {
-    super(args)
-    this.config = config
+    // Encoder delay handling
+    this.samplesSkipped = 0
+    this.totalSkipSamples = 0
   }
 
+  /**
+   * Load and initialize the decoder
+   */
   async load () {
-    const config = {
-      audioFormat: this.config.audioFormat || 'encoded',
-      sampleRate: this.config.sampleRate || 16000
+    if (this.isLoaded) {
+      this.logger.info('FFmpegDecoder already loaded')
+      return
     }
 
-    this.logger.info('Loading GSTDecoder with config:', config)
-    this.addon = this.createAddon(config)
+    this.logger.info('Loading FFmpegDecoder with config:', this.config)
 
-    await this.addon.activate()
-    this.logger.info('GSTDecoder activated successfully')
+    // Initialize format constants
+    this.SUPPORTED_AUDIO_FORMATS.s16le.format = ffmpeg.constants.sampleFormats.S16
+    this.SUPPORTED_AUDIO_FORMATS.f32le.format = ffmpeg.constants.sampleFormats.FLT
+    this.OUTPUT_CHANNEL_LAYOUT = ffmpeg.constants.channelLayouts.MONO
+
+    // Validate audio format
+    if (!this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat]) {
+      throw new Error(`Unsupported audio format: ${this.config.audioFormat}`)
+    }
+
+    this.isLoaded = true
+    this.logger.info('FFmpegDecoder loaded successfully')
   }
 
-  async _runInternal (audioStream) {
+  /**
+   * Unload the decoder and clean up resources
+   */
+  async unload () {
+    if (!this.isLoaded) {
+      return
+    }
+
+    this.logger.info('Unloading FFmpegDecoder')
+
+    this.isLoaded = false
+    this.currentJob = null
+    this.logger.info('FFmpegDecoder unloaded')
+  }
+
+  /**
+   * Run the decoder on an audio stream
+   * @param {Readable} audioStream - Input audio stream
+   * @returns {QvacResponse} Response with decoded audio
+   */
+  async run (audioStream) {
+    if (!this.isLoaded) {
+      throw new Error('Decoder not loaded. Call load() first.')
+    }
+
     this.logger.info('Starting new audio stream processing')
-    const response = new QvacResponse({
-      cancelHandler: () => this.addon.cancel(jobId),
-      pauseHandler: () => this.addon.pause(),
-      continueHandler: () => this.addon.continue()
-    })
 
-    const jobId = await this.addon.append({
-      type: 'audio',
-      input: new Uint8Array().buffer,
-      priority: 1
+    const response = new QvacResponse({
+      cancelHandler: () => this.stop(),
+      pauseHandler: () => this.pause(),
+      continueHandler: () => this.unpause()
     })
 
-    this.logger.info('Created new job with ID:', jobId)
-    this._saveJobToResponseMapping(jobId, response)
+    this.currentJob = {
+      response,
+      audioChunks: [],
+      isActive: true,
+      isPaused: false
+    }
 
-    this._handleAudioStream(audioStream).catch(err => {
+    // Process the audio stream
+    this._processStream(audioStream).catch(err => {
       this.logger.error('Error processing audio stream:', err)
       response.failed(err)
     })
+
     return response
   }
 
-  async _handleAudioStream (audioStream) {
-    this.logger.info('Starting audio stream handling')
-    const streamAccumulator = createStreamAccumulator({
-      onChunk: async chunk => {
-        this.logger.debug('Processing audio chunk of size:', chunk.byteLength)
-        await this.addon.append({
-          type: 'audio',
-          input: chunk.buffer,
-          priority: 1
-        })
-      },
-      onFinish: async () => {
-        this.logger.info('Audio stream finished, sending end of input')
-        await this.addon.append({ type: END_OF_INPUT })
+  _getBufferSize (inputBitrate) {
+    const maxBufferSize = 1024 * 1024 // 1MB max
+    return Math.min((inputBitrate / 8) * 4, maxBufferSize)
+  }
+
+  _processFrame (decoder, raw, resampler, job) {
+    const OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].format
+    const OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].byteLength
+    const OUTPUT_SAMPLE_RATE = this.config.sampleRate
+
+    while (decoder.receiveFrame(raw)) {
+      const output = new ffmpeg.Frame()
+      output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
+      output.format = OUTPUT_FORMAT
+      output.sampleRate = OUTPUT_SAMPLE_RATE
+      output.nbSamples = raw.nbSamples
+
+      const samples = new ffmpeg.Samples(
+        output.format,
+        output.channelLayout.nbChannels,
+        output.nbSamples
+      )
+      samples.fill(output)
+
+      const count = resampler.convert(raw, output)
+
+      // Handle encoder delay by skipping initial samples
+      if (this.samplesSkipped < this.totalSkipSamples) {
+        const samplesToSkip = Math.min(count, this.totalSkipSamples - this.samplesSkipped)
+        this.samplesSkipped += samplesToSkip
+        if (samplesToSkip >= count) continue // Skip entire frame
+
+        // Skip partial frame
+        const skipBytes = OUTPUT_FORMAT_BYTE_LENGTH * samplesToSkip * output.channelLayout.nbChannels
+        const length = OUTPUT_FORMAT_BYTE_LENGTH * (count - samplesToSkip) * output.channelLayout.nbChannels
+        const chunk = Buffer.from(samples.data.subarray(skipBytes, skipBytes + length))
+        job.response.updateOutput({ outputArray: chunk })
+      } else {
+        const length = OUTPUT_FORMAT_BYTE_LENGTH * count * output.channelLayout.nbChannels
+        const chunk = Buffer.from(samples.data.subarray(0, length))
+        job.response.updateOutput({ outputArray: chunk })
       }
-    })
+    }
+  }
+
+  _processPacket (format, packet, raw, decoder, resampler, job) {
+    while (format.readFrame(packet)) {
+      decoder.sendPacket(packet)
+      this._processFrame(decoder, raw, resampler, job)
+      packet.unref()
+    }
+  }
+
+  _processFFmpegStream (format, stream, job) {
+    const OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].format
+    const OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].byteLength
+    const OUTPUT_SAMPLE_RATE = this.config.sampleRate
+
+    this.logger.debug('[FFmpegDecoder] Stream codec:', stream.codec, stream.codecParameters)
+
+    const packet = new ffmpeg.Packet()
+    const raw = new ffmpeg.Frame()
+
+    const resampler = new ffmpeg.Resampler(
+      stream.codecParameters.sampleRate,
+      stream.codecParameters.channelLayout,
+      stream.codecParameters.format,
+      OUTPUT_SAMPLE_RATE,
+      this.OUTPUT_CHANNEL_LAYOUT,
+      OUTPUT_FORMAT
+    )
+
+    const decoder = stream.decoder()
+    decoder.open()
+
+    // Auto-detect encoder delay: lossy codecs need ~400ms skipped to remove artifacts
+    const codecName = stream.codec.name.toLowerCase()
+    const SKIP_MS = {
+      mp3: 400,
+      vorbis: 400,
+      opus: 150,
+      aac: 300
+    }
+
+    const skipMs = SKIP_MS[codecName] || 0
+    this.samplesSkipped = 0
+    this.totalSkipSamples = Math.floor((OUTPUT_SAMPLE_RATE * skipMs) / 1000)
+
+    if (this.totalSkipSamples > 0) {
+      this.logger.info(`[FFmpegDecoder] Skipping ${skipMs}ms (${this.totalSkipSamples} samples) for ${codecName} to remove encoder artifacts`)
+    }
+
+    this._processPacket(format, packet, raw, decoder, resampler, job)
+
+    // Flush resampler
+    const output = new ffmpeg.Frame()
+    output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
+    output.format = OUTPUT_FORMAT
+    output.sampleRate = OUTPUT_SAMPLE_RATE
+    output.nbSamples = 1024
+
+    const samples = new ffmpeg.Samples(
+      output.format,
+      output.channelLayout.nbChannels,
+      output.nbSamples
+    )
+    samples.fill(output)
+
+    let flushCount
+    while ((flushCount = resampler.flush(output)) > 0) {
+      const actualLength = OUTPUT_FORMAT_BYTE_LENGTH * flushCount * output.channelLayout.nbChannels
+      const chunk = Buffer.from(samples.data.subarray(0, actualLength))
+      job.response.updateOutput({ outputArray: chunk })
+    }
+
+    decoder.destroy()
+  }
+
+  async _collectStreamData (audioStream, job) {
+    const chunks = []
+    let totalBytes = 0
 
     for await (const chunk of audioStream) {
-      await streamAccumulator.processData(chunk)
+      if (!job.isActive) {
+        this.logger.info('[FFmpegDecoder] Job cancelled, stopping stream collection')
+        break
+      }
+
+      while (job.isPaused) {
+        this.logger.debug('[FFmpegDecoder] Job is paused, waiting to resume...')
+        await new Promise(resolve => setTimeout(resolve, 100))
+      }
+
+      chunks.push(chunk)
+      totalBytes += chunk.length
+      this.logger.debug(`[FFmpegDecoder] Collected chunk, total bytes: ${totalBytes}`)
     }
 
-    await streamAccumulator.finish()
-    this.logger.info('Audio stream handling completed')
+    return Buffer.concat(chunks)
   }
 
-  async getStatus () {
-    const status = await this.addon.status()
-    this.logger.debug('Current status:', status)
-    return status
+  async _processStream (audioStream) {
+    const job = this.currentJob
+    if (!job.isActive) {
+      return
+    }
+
+    try {
+      this.logger.info('[FFmpegDecoder] Starting stream processing')
+
+      // Collect all audio data from stream
+      const audioBuffer = await this._collectStreamData(audioStream, job)
+      this.logger.info(`[FFmpegDecoder] Collected ${audioBuffer.length} bytes of audio data`)
+
+      if (!job.isActive) {
+        this.logger.info('[FFmpegDecoder] Job cancelled after data collection')
+        return
+      }
+
+      // Create FFmpeg IO context with the buffer
+      const bufferSize = this._getBufferSize(this.config.inputBitrate)
+      let bufferOffset = 0
+
+      const io = new ffmpeg.IOContext(bufferSize, {
+        onread: (buffer, requestedLen) => {
+          const remainingBytes = audioBuffer.length - bufferOffset
+          const bytesToRead = Math.min(requestedLen, remainingBytes)
+
+          if (bytesToRead <= 0) {
+            return 0 // EOF
+          }
+
+          audioBuffer.copy(buffer, 0, bufferOffset, bufferOffset + bytesToRead)
+          bufferOffset += bytesToRead
+
+          this.logger.debug(`[FFmpegDecoder] Read ${bytesToRead} bytes from buffer, offset now: ${bufferOffset}`)
+          return bytesToRead
+        },
+        onseek: (offset, whence) => {
+          const AVSEEK_SIZE = 0x10000
+
+          if (whence === AVSEEK_SIZE) {
+            return audioBuffer.length
+          }
+
+          let newOffset
+          if (whence === 0) {
+            newOffset = offset
+          } else if (whence === 1) {
+            newOffset = bufferOffset + offset
+          } else if (whence === 2) {
+            newOffset = audioBuffer.length + offset
+          } else {
+            return -1
+          }
+
+          if (newOffset < 0 || newOffset > audioBuffer.length) {
+            return -1
+          }
+
+          bufferOffset = newOffset
+          this.logger.debug(`[FFmpegDecoder] Seek to offset: ${bufferOffset}`)
+          return bufferOffset
+        }
+      })
+
+      this.logger.debug('[FFmpegDecoder] IOContext created')
+      const format = new ffmpeg.InputFormatContext(io)
+      this.logger.debug('[FFmpegDecoder] InputFormatContext created')
+
+      const streamIndex = this.config.streamIndex || 0
+      if (format.streams[streamIndex] === undefined) {
+        throw new Error('Stream index out of bounds')
+      }
+
+      // Process the stream and generate decoded output
+      this._processFFmpegStream(format, format.streams[streamIndex], job)
+
+      // Mark as complete
+      job.response.ended()
+      this.logger.info('[FFmpegDecoder] Stream processing completed successfully')
+    } catch (err) {
+      this.logger.error('Error processing audio stream:', err)
+      job.response.failed(err)
+    }
+
+    this.logger.info('Audio _processStream completed')
   }
 
-  async unload () {
-    if (this.addon) {
-      this.logger.info('Destroying GSTDecoder addon')
-      await this.addon.destroy()
-      this.addon = null
+  /**
+   * Pause the current job
+   */
+  pause () {
+    if (this.currentJob) {
+      this.currentJob.isPaused = true
+      this.logger.debug('Decoder paused')
     }
+    return Promise.resolve()
   }
 
-  createAddon (config) {
-    this.logger.info('Creating new GSTDecoderInterface instance')
-    return new GSTDecoderInterface(
-      config,
-      this._outputCallback.bind(this),
-      this.logger.info.bind(this.logger)
-    )
+  /**
+   * Unpause the current job
+   */
+  unpause () {
+    if (this.currentJob) {
+      this.currentJob.isPaused = false
+      this.logger.debug('Decoder unpaused')
+    }
+    return Promise.resolve()
   }
-}
 
-const FFmpegDecoder = require('./lib/ffmpeg/ffmpeg-decoder')
+  /**
+   * Stop the current job
+   */
+  stop () {
+    if (this.currentJob) {
+      this.currentJob.isActive = false
+      this.currentJob.response.finish()
+      this.logger.debug('Decoder stopped')
+    }
+    return Promise.resolve()
+  }
+
+  /**
+   * Get the current status
+   */
+  status () {
+    return {
+      loaded: this.isLoaded,
+      active: this.currentJob?.isActive || false,
+      paused: this.currentJob?.isPaused || false
+    }
+  }
+}
 
-module.exports = GSTDecoder
-module.exports.FORMATS_NEEDING_DECODE = FORMATS_NEEDING_DECODE
-module.exports.SUPPORTED_AUDIO_FORMATS = SUPPORTED_AUDIO_FORMATS
-module.exports.FFmpegDecoder = FFmpegDecoder
+module.exports = { FFmpegDecoder }

package/package.json

@@ -1,12 +1,11 @@
 {
   "name": "@qvac/decoder-audio",
-  "version": "0.2.10",
+  "version": "0.3.1",
   "description": "",
   "license": "Apache-2.0",
   "author": "Tether",
   "type": "commonjs",
   "main": "index.js",
-  "addon": true,
   "scripts": {
     "test:dts": "tsc index.d.ts --noEmit --esModuleInterop --skipLibCheck",
     "test:unit": "brittle-bare test/unit/*.test.js",
@@ -18,14 +17,10 @@
     "test:integration": "brittle-bare test/integration/*.test.js"
   },
   "files": [
-    "binding.js",
     "index.js",
-    "gstreamer.js",
     "constants.js",
     "constants.d.ts",
-    "prebuilds",
     "utils",
-    "lib",
     "index.d.ts",
     "test/mobile"
   ],
@@ -38,8 +33,6 @@
   "devDependencies": {
     "@types/node": "^22.14.1",
     "brittle": "^3.13.1",
-    "cmake-bare": "^1.7.5",
-    "cmake-vcpkg": "^1.1.0",
     "istanbul": "^0.4.5",
     "standard": "^17.1.2",
     "typescript": "^5.3.0"

package/utils/error.js

@@ -40,7 +40,7 @@ addCodes({
   },
   [ERR_CODES.FAILED_TO_GET_STATUS]: {
     name: 'FAILED_TO_GET_STATUS',
-    message: (message) => `Failed to get addon status, error: ${message}`
+    message: (message) => `Failed to get decoder status, error: ${message}`
   },
   [ERR_CODES.FAILED_TO_DESTROY]: {
     name: 'FAILED_TO_DESTROY',

package/binding.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require.addon()

package/gstreamer.js

@@ -1,131 +0,0 @@
-'use strict'
-
-const binding = require('./binding')
-const { QvacErrorDecoderAudio, ERR_CODES } = require('./utils/error')
-
-/**
- * An interface between Bare addon in C++ and JS runtime.
- */
-class GSTDecoderInterface {
-  /**
-   *
-   * @param {Object} configurationParams - all the required configuration for inference setup
-   * @param {Function} outputCb - to be called on any inference event ( started, new output, error, etc )
-   * @param {Function} transitionCb - to be called on addon state changes (LISTENING, IDLE, STOPPED, etc )
-   */
-  constructor (configurationParams, outputCb, transitionCb = null) {
-    this._handle = binding.createInstance(this, configurationParams, outputCb, transitionCb)
-  }
-
-  /**
-   *
-   * @param {Object} weightsData
-   * @param {String} weightsData.filename
-   * @param {Uint8Array} weightsData.contents
-   * @param {Boolean} weightsData.completed
-   */
-  async loadWeights (weightsData) {
-    try {
-      binding.loadWeights(this._handle, weightsData)
-    } catch (err) {
-      throw new QvacErrorDecoderAudio(
-        ERR_CODES.FAILED_TO_LOAD_WEIGHTS,
-        err.message
-      )
-    }
-  }
-
-  /**
-   * Moves addon to the LISTENING state after all the initialization is done
-   */
-  async activate () {
-    try {
-      binding.activate(this._handle)
-    } catch (err) {
-      throw new QvacErrorDecoderAudio(
-        ERR_CODES.FAILED_TO_ACTIVATE,
-        err.message
-      )
-    }
-  }
-
-  /**
-   * Pauses current inference process
-   */
-  async pause () {
-    try {
-      binding.pause(this._handle)
-    } catch (err) {
-      throw new QvacErrorDecoderAudio(
-        ERR_CODES.FAILED_TO_PAUSE,
-        err.message
-      )
-    }
-  }
-
-  /**
-   * Cancel a inference process by jobId, if no jobId is provided it cancel the whole queue
-   */
-  async cancel (jobId) {
-    try {
-      binding.cancel(this._handle, jobId)
-    } catch (err) {
-      throw new QvacErrorDecoderAudio(
-        ERR_CODES.FAILED_TO_CANCEL,
-        err.message
-      )
-    }
-  }
-
-  /**
-   * Adds new input to the processing queue
-   * @param {Object} data
-   * @param {String} data.type
-   * @param {String} data.input
-   * @returns {Number} - job ID
-   */
-  async append (data) {
-    try {
-      return binding.append(this._handle, data)
-    } catch (err) {
-      throw new QvacErrorDecoderAudio(
-        ERR_CODES.FAILED_TO_APPEND,
-        err.message
-      )
-    }
-  }
-
-  /**
-   * Addon process status
-   * @returns {String}
-   */
-  async status () {
-    try {
-      return binding.status(this._handle)
-    } catch (err) {
-      throw new QvacErrorDecoderAudio(
-        ERR_CODES.FAILED_TO_GET_STATUS,
-        err.message
-      )
-    }
-  }
-
-  /**
-   * Stops addon process and clears resources (including memory).
-   */
-  async destroy () {
-    try {
-      binding.destroyInstance(this._handle)
-      this._handle = null
-    } catch (err) {
-      throw new QvacErrorDecoderAudio(
-        ERR_CODES.FAILED_TO_DESTROY,
-        err.message
-      )
-    }
-  }
-}
-
-module.exports = {
-  GSTDecoderInterface
-}