@qvac/decoder-audio 0.2.4 → 0.2.6

package/lib/ffmpeg/ffmpeg-decoder.js

@@ -1,17 +1,26 @@
 'use strict'
 
-const { Worker } = require('bare-worker')
 const QvacResponse = require('@qvac/response')
 const QvacLogger = require('@qvac/logging')
-const path = require('bare-path')
-const Channel = require('bare-channel')
-const { FFmpegDecoderMessages, FFmpegWorkerMessages } = require('./constants')
+const ffmpeg = require('bare-ffmpeg')
 const BaseInference = require('@qvac/infer-base/WeightsProvider/BaseInference')
 
 /**
- * FFmpeg-based audio decoder that runs in a worker thread
+ * FFmpeg-based audio decoder (single-threaded)
  */
 class FFmpegDecoder extends BaseInference {
+  SUPPORTED_AUDIO_FORMATS = {
+    s16le: {
+      format: null, // Will be set to ffmpeg.constants.sampleFormats.S16
+      byteLength: 2
+    },
+    f32le: {
+      format: null, // Will be set to ffmpeg.constants.sampleFormats.FLT
+      byteLength: 4
+    }
+  }
+
+  OUTPUT_CHANNEL_LAYOUT = null // Will be set to ffmpeg.constants.channelLayouts.MONO
   /**
    * Creates an instance of FFmpegDecoder.
    * @param {Object} config - Configuration options
@@ -37,15 +46,19 @@ class FFmpegDecoder extends BaseInference {
     super({ ...args, logger })
 
     this.config = {
-      streamIndex,
-      inputBitrate,
-      audioFormat
+      streamIndex: config.streamIndex || streamIndex,
+      inputBitrate: config.inputBitrate || inputBitrate,
+      audioFormat: config.audioFormat || audioFormat,
+      sampleRate: config.sampleRate || 16000
     }
 
     this.logger = new QvacLogger(logger)
-    this.worker = null
     this.isLoaded = false
     this.currentJob = null
+    
+    // Encoder delay handling
+    this.samplesSkipped = 0
+    this.totalSkipSamples = 0
   }
 
   /**
@@ -59,25 +72,15 @@ class FFmpegDecoder extends BaseInference {
 
     this.logger.info('Loading FFmpegDecoder with config:', this.config)
 
-    // Create worker thread
-    const workerPath = path.join(__dirname, 'ffmpeg-worker.js')
-    this.worker = new Worker(workerPath)
-
-    // Wait for worker to be ready
-    await new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => {
-        reject(new Error('Worker initialization timeout'))
-      }, 5000)
-
-      this.worker.once('message', (msg) => {
-        clearTimeout(timeout)
-        if (msg.type === FFmpegWorkerMessages.READY) {
-          resolve()
-        } else {
-          reject(new Error('Unexpected worker message during init'))
-        }
-      })
-    })
+    // Initialize format constants
+    this.SUPPORTED_AUDIO_FORMATS.s16le.format = ffmpeg.constants.sampleFormats.S16
+    this.SUPPORTED_AUDIO_FORMATS.f32le.format = ffmpeg.constants.sampleFormats.FLT
+    this.OUTPUT_CHANNEL_LAYOUT = ffmpeg.constants.channelLayouts.MONO
+
+    // Validate audio format
+    if (!this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat]) {
+      throw new Error(`Unsupported audio format: ${this.config.audioFormat}`)
+    }
 
     this.isLoaded = true
     this.logger.info('FFmpegDecoder loaded successfully')
@@ -92,24 +95,7 @@ class FFmpegDecoder extends BaseInference {
     }
 
     this.logger.info('Unloading FFmpegDecoder')
-
-    if (this.worker) {
-      await new Promise((resolve) => {
-        this.worker.postMessage({ type: FFmpegDecoderMessages.CLEANUP })
-        this.worker.once('message', (msg) => {
-          if (msg.type === FFmpegWorkerMessages.CLEANUP_DONE) {
-            resolve()
-          }
-        })
-
-        // Fallback timeout
-        setTimeout(resolve, 1000)
-      })
-
-      await this.worker.terminate()
-      this.worker = null
-    }
-
+    
     this.isLoaded = false
     this.currentJob = null
     this.logger.info('FFmpegDecoder unloaded')
@@ -149,69 +135,130 @@ class FFmpegDecoder extends BaseInference {
     return response
   }
 
-  _handleWorkerMessage (job) {
-    return new Promise((resolve, reject) => {
-      const messageHandler = async (msg) => {
-        this.logger.debug('[FFmpegDecoder] Received message from worker:', msg)
-        switch (msg.type) {
-          case FFmpegWorkerMessages.INIT_RESULT: {
-            if (msg.success) {
-              this.logger.info('[FFmpegDecoder] Decoder initialized successfully')
-              this.worker.postMessage({
-                type: FFmpegDecoderMessages.STREAM_START
-              })
-            } else {
-              this.worker.off('message', messageHandler)
-              this.logger.error('[FFmpegDecoder] Failed to initialize decoder:', msg.error)
-            }
-            break
-          }
+  _getBufferSize (inputBitrate) {
+    const maxBufferSize = 1024 * 1024 // 1MB max
+    return Math.min((inputBitrate / 8) * 4, maxBufferSize)
+  }
 
-          case FFmpegWorkerMessages.RESULT_CHUNK: {
-            const decodedBuffer = Buffer.from(msg.data)
-            this.logger.debug('[FFmpegDecoder] Result chunk', decodedBuffer.length)
-            job.response.updateOutput({ outputArray: decodedBuffer })
-            break
-          }
+  _processFrame (decoder, raw, resampler, job) {
+    const OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].format
+    const OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].byteLength
+    const OUTPUT_SAMPLE_RATE = this.config.sampleRate
+
+    while (decoder.receiveFrame(raw)) {
+      const output = new ffmpeg.Frame()
+      output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
+      output.format = OUTPUT_FORMAT
+      output.sampleRate = OUTPUT_SAMPLE_RATE
+      output.nbSamples = raw.nbSamples
+
+      const samples = new ffmpeg.Samples(
+        output.format,
+        output.channelLayout.nbChannels,
+        output.nbSamples
+      )
+      samples.fill(output)
+
+      const count = resampler.convert(raw, output)
+
+      // Handle encoder delay by skipping initial samples
+      if (this.samplesSkipped < this.totalSkipSamples) {
+        const samplesToSkip = Math.min(count, this.totalSkipSamples - this.samplesSkipped)
+        this.samplesSkipped += samplesToSkip
+        if (samplesToSkip >= count) continue // Skip entire frame
+
+        // Skip partial frame
+        const skipBytes = OUTPUT_FORMAT_BYTE_LENGTH * samplesToSkip * output.channelLayout.nbChannels
+        const length = OUTPUT_FORMAT_BYTE_LENGTH * (count - samplesToSkip) * output.channelLayout.nbChannels
+        const chunk = Buffer.from(samples.data.subarray(skipBytes, skipBytes + length))
+        job.response.updateOutput({ outputArray: chunk })
+      } else {
+        const length = OUTPUT_FORMAT_BYTE_LENGTH * count * output.channelLayout.nbChannels
+        const chunk = Buffer.from(samples.data.subarray(0, length))
+        job.response.updateOutput({ outputArray: chunk })
+      }
+    }
+  }
 
-          case FFmpegWorkerMessages.RESULT_END: {
-            this.logger.debug('[FFmpegDecoder] Result end')
-            job.response.ended()
-            this.worker.off('message', messageHandler)
-            resolve()
-            break
-          }
+  _processPacket (format, packet, raw, decoder, resampler, job) {
+    while (format.readFrame(packet)) {
+      decoder.sendPacket(packet)
+      this._processFrame(decoder, raw, resampler, job)
+      packet.unref()
+    }
+  }
 
-          case FFmpegWorkerMessages.ERROR: {
-            const errorMessage = msg.error || msg.data?.error || 'Unknown error'
-            this.logger.error('[FFmpegDecoder] Error from worker:', errorMessage)
-            job.response.failed(errorMessage)
-            this.worker.off('message', messageHandler)
-            reject(new Error(errorMessage))
-            break
-          }
-        }
-      }
+  _processFFmpegStream (format, stream, job) {
+    const OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].format
+    const OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].byteLength
+    const OUTPUT_SAMPLE_RATE = this.config.sampleRate
+
+    this.logger.debug('[FFmpegDecoder] Stream codec:', stream.codec, stream.codecParameters)
+    
+    const packet = new ffmpeg.Packet()
+    const raw = new ffmpeg.Frame()
+
+    const resampler = new ffmpeg.Resampler(
+      stream.codecParameters.sampleRate,
+      stream.codecParameters.channelLayout,
+      stream.codecParameters.format,
+      OUTPUT_SAMPLE_RATE,
+      this.OUTPUT_CHANNEL_LAYOUT,
+      OUTPUT_FORMAT
+    )
+
+    const decoder = stream.decoder()
+    decoder.open()
+
+    // Auto-detect encoder delay: lossy codecs need ~400ms skipped to remove artifacts
+    const codecName = stream.codec.name.toLowerCase()
+    const SKIP_MS = {
+      mp3: 400,
+      vorbis: 400,
+      opus: 150,
+      aac: 300
+    }
 
-      this.worker.on('message', messageHandler)
-      this.worker.on('error', (err) => {
-        this.logger.error('[FFmpegDecoder] Error from worker:', err)
-        job.response.failed(err)
-        this.worker.off('message', messageHandler)
-        reject(err)
-      })
-    })
+    const skipMs = SKIP_MS[codecName] || 0
+    this.samplesSkipped = 0
+    this.totalSkipSamples = Math.floor((OUTPUT_SAMPLE_RATE * skipMs) / 1000)
+
+    if (this.totalSkipSamples > 0) {
+      this.logger.info(`[FFmpegDecoder] Skipping ${skipMs}ms (${this.totalSkipSamples} samples) for ${codecName} to remove encoder artifacts`)
+    }
+
+    this._processPacket(format, packet, raw, decoder, resampler, job)
+
+    // Flush resampler
+    const output = new ffmpeg.Frame()
+    output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
+    output.format = OUTPUT_FORMAT
+    output.sampleRate = OUTPUT_SAMPLE_RATE
+    output.nbSamples = 1024
+
+    const samples = new ffmpeg.Samples(
+      output.format,
+      output.channelLayout.nbChannels,
+      output.nbSamples
+    )
+    samples.fill(output)
+
+    while (resampler.flush(output) > 0) {
+      const chunk = Buffer.from(samples.data)
+      job.response.updateOutput({ outputArray: chunk })
+    }
+
+    decoder.destroy()
   }
 
-  async _streamAudioToWorker (audioStream, port, job) {
-    let chunkCount = 0
+  async _collectStreamData (audioStream, job) {
+    const chunks = []
+    let totalBytes = 0
+    
     for await (const chunk of audioStream) {
-      chunkCount++
-
-      this.logger.debug(`[FFmpegDecoder] Received audio chunk #${chunkCount}, size: ${chunk.length} bytes`)
       if (!job.isActive) {
-        this.logger.info('[FFmpegDecoder] Job cancelled, stopping stream collection at chunk', chunkCount)
-        return
+        this.logger.info('[FFmpegDecoder] Job cancelled, stopping stream collection')
+        break
       }
 
       while (job.isPaused) {
@@ -219,9 +266,12 @@ class FFmpegDecoder extends BaseInference {
         await new Promise(resolve => setTimeout(resolve, 100))
       }
 
-      port.writeSync(chunk)
+      chunks.push(chunk)
+      totalBytes += chunk.length
+      this.logger.debug(`[FFmpegDecoder] Collected chunk, total bytes: ${totalBytes}`)
     }
-    return chunkCount
+
+    return Buffer.concat(chunks)
   }
 
   async _processStream (audioStream) {
@@ -231,32 +281,53 @@ class FFmpegDecoder extends BaseInference {
     }
 
     try {
-      const channel = new Channel()
-      const port = channel.connect()
+      this.logger.info('[FFmpegDecoder] Starting stream processing')
 
-      this.logger.debug('[FFmpegDecoder] Processing audio stream with channel handle:', channel.handle)
+      // Collect all audio data from stream
+      const audioBuffer = await this._collectStreamData(audioStream, job)
+      this.logger.info(`[FFmpegDecoder] Collected ${audioBuffer.length} bytes of audio data`)
 
-      // Worker promise to handle worker messages
-      const workerCompletionPromise = this._handleWorkerMessage(job)
+      if (!job.isActive) {
+        this.logger.info('[FFmpegDecoder] Job cancelled after data collection')
+        return
+      }
+
+      // Create FFmpeg IO context with the buffer
+      const bufferSize = this._getBufferSize(this.config.inputBitrate)
+      let bufferOffset = 0
+
+      const io = new ffmpeg.IOContext(bufferSize, {
+        onread: (buffer, requestedLen) => {
+          const remainingBytes = audioBuffer.length - bufferOffset
+          const bytesToRead = Math.min(requestedLen, remainingBytes)
+          
+          if (bytesToRead <= 0) {
+            return 0 // EOF
+          }
 
-      // Initialize ffmpeg worker
-      this.worker.postMessage({
-        type: FFmpegDecoderMessages.INIT,
-        data: {
-          channelHandle: channel.handle,
-          config: this.config
+          audioBuffer.copy(buffer, 0, bufferOffset, bufferOffset + bytesToRead)
+          bufferOffset += bytesToRead
+          
+          this.logger.debug(`[FFmpegDecoder] Read ${bytesToRead} bytes from buffer, offset now: ${bufferOffset}`)
+          return bytesToRead
         }
       })
 
-      // Stream all audio chunks to ffmpeg worker
-      const chunkCount = await this._streamAudioToWorker(audioStream, port, job)
-      this.logger.info(`[FFmpegDecoder] Finished streaming audio. Total chunks sent: ${chunkCount}`)
+      this.logger.debug('[FFmpegDecoder] IOContext created')
+      const format = new ffmpeg.InputFormatContext(io)
+      this.logger.debug('[FFmpegDecoder] InputFormatContext created')
+
+      const streamIndex = this.config.streamIndex || 0
+      if (format.streams[streamIndex] === undefined) {
+        throw new Error('Stream index out of bounds')
+      }
 
-      // Signal to worker that we're done sending audio
-      await port.close()
+      // Process the stream and generate decoded output
+      this._processFFmpegStream(format, format.streams[streamIndex], job)
 
-      // Wait for worker to finish
-      await workerCompletionPromise
+      // Mark as complete
+      job.response.ended()
+      this.logger.info('[FFmpegDecoder] Stream processing completed successfully')
     } catch (err) {
       this.logger.error('Error processing audio stream:', err)
       job.response.failed(err)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qvac/decoder-audio",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "",
   "license": "Apache-2.0",
   "author": "Tether",
@@ -47,8 +47,8 @@
     "typescript": "^5.3.0"
   },
   "dependencies": {
-    "@qvac/infer-base": "^0.1.0",
     "@qvac/error": "^0.1.0",
+    "@qvac/infer-base": "^0.1.0",
     "@qvac/logging": "^0.1.0",
     "@qvac/response": "^0.1.0",
     "bare-assert": "^1.1.0",
@@ -57,8 +57,7 @@
     "bare-fs": "^4.5.1",
     "bare-path": "^3.0.0",
     "bare-process": "^4.2.2",
-    "process": "npm:bare-process@^4.2.2",
-    "bare-worker": "^4.1.0"
+    "process": "npm:bare-process@^4.2.2"
   },
   "bugs": "https://github.com/tetherto/qvac-lib-decoder-audio/issues",
   "types": "index.d.ts"

package/prebuilds/darwin-arm64/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.2.4"
+        "version": "0.2.6"
       }
     ],
     "exported_symbols": [

package/prebuilds/ios-arm64/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.2.4"
+        "version": "0.2.6"
       }
     ],
     "exported_symbols": [

package/prebuilds/ios-arm64-simulator/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.2.4"
+        "version": "0.2.6"
       }
     ],
     "exported_symbols": [

package/prebuilds/ios-x64-simulator/qvac__decoder-audio.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.2.4"
+        "version": "0.2.6"
       }
     ],
     "exported_symbols": [

package/test/mobile/ffmpeg-test.cjs

@@ -0,0 +1,105 @@
+const { FFmpegDecoder } = require('@qvac/decoder-audio')
+
+async function testFFmpegDecodeMp3() {
+    try {
+        const decoder = new FFmpegDecoder({
+            config: {
+                audioFormat: 's16le',
+                sampleRate: 16000
+            }
+        })
+    
+        await decoder.load()
+    
+        const audioPath = getAssetPath('sample.mp3')
+        const audioStream = fs.createReadStream(audioPath)
+    
+        const response = await decoder.run(audioStream)
+    
+        let totalBytes = 0
+        let updateCount = 0
+    
+        await response
+          .onUpdate(output => {
+            if (output && output.outputArray) {
+              const bytes = new Uint8Array(output.outputArray)
+              totalBytes += bytes.length
+              updateCount++
+            }
+          })
+          .await()
+    
+        if (totalBytes === 0) {
+          throw new Error('No audio data decoded')
+        }
+    
+        if (updateCount === 0) {
+          throw new Error('No decoder updates received')
+        }
+    
+        console.log(`MP3 decode complete: ${totalBytes} bytes, ${updateCount} updates`)
+    
+        await decoder.unload()
+
+        return { 
+          success: true,
+          totalBytes,
+          updateCount
+        }
+    } catch (error) {
+        console.error('Error during testFFmpegDecodeMp3:', error)
+        throw error
+    }
+}
+
+async function testFFmpegDecodeWav() {
+  try {
+      const decoder = new FFmpegDecoder({
+          config: {
+              audioFormat: 's16le',
+              sampleRate: 16000
+          }
+      })
+  
+      await decoder.load()
+  
+      const audioPath = getAssetPath('sample.wav')
+      const audioStream = fs.createReadStream(audioPath)
+  
+      const response = await decoder.run(audioStream)
+  
+      let totalBytes = 0
+      let updateCount = 0
+  
+      await response
+        .onUpdate(output => {
+          if (output && output.outputArray) {
+            const bytes = new Uint8Array(output.outputArray)
+            totalBytes += bytes.length
+            updateCount++
+          }
+        })
+        .await()
+  
+      if (totalBytes === 0) {
+        throw new Error('No audio data decoded')
+      }
+  
+      if (updateCount === 0) {
+        throw new Error('No decoder updates received')
+      }
+  
+      console.log(`MP3 decode complete: ${totalBytes} bytes, ${updateCount} updates`)
+  
+      await decoder.unload()
+
+      return { 
+        success: true,
+        totalBytes,
+        updateCount
+      }
+  } catch (error) {
+      console.error('Error during testFFmpegDecodeMp3:', error)
+      throw error
+  }
+}