@qvac/decoder-audio 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,26 @@
1
1
  'use strict'
2
2
 
3
- const { Worker } = require('bare-worker')
4
3
  const QvacResponse = require('@qvac/response')
5
4
  const QvacLogger = require('@qvac/logging')
6
- const path = require('bare-path')
7
- const Channel = require('bare-channel')
8
- const { FFmpegDecoderMessages, FFmpegWorkerMessages } = require('./constants')
5
+ const ffmpeg = require('bare-ffmpeg')
9
6
  const BaseInference = require('@qvac/infer-base/WeightsProvider/BaseInference')
10
7
 
11
8
  /**
12
- * FFmpeg-based audio decoder that runs in a worker thread
9
+ * FFmpeg-based audio decoder (single-threaded)
13
10
  */
14
11
  class FFmpegDecoder extends BaseInference {
12
+ SUPPORTED_AUDIO_FORMATS = {
13
+ s16le: {
14
+ format: null, // Will be set to ffmpeg.constants.sampleFormats.S16
15
+ byteLength: 2
16
+ },
17
+ f32le: {
18
+ format: null, // Will be set to ffmpeg.constants.sampleFormats.FLT
19
+ byteLength: 4
20
+ }
21
+ }
22
+
23
+ OUTPUT_CHANNEL_LAYOUT = null // Will be set to ffmpeg.constants.channelLayouts.MONO
15
24
  /**
16
25
  * Creates an instance of FFmpegDecoder.
17
26
  * @param {Object} config - Configuration options
@@ -37,15 +46,19 @@ class FFmpegDecoder extends BaseInference {
37
46
  super({ ...args, logger })
38
47
 
39
48
  this.config = {
40
- streamIndex,
41
- inputBitrate,
42
- audioFormat
49
+ streamIndex: config.streamIndex || streamIndex,
50
+ inputBitrate: config.inputBitrate || inputBitrate,
51
+ audioFormat: config.audioFormat || audioFormat,
52
+ sampleRate: config.sampleRate || 16000
43
53
  }
44
54
 
45
55
  this.logger = new QvacLogger(logger)
46
- this.worker = null
47
56
  this.isLoaded = false
48
57
  this.currentJob = null
58
+
59
+ // Encoder delay handling
60
+ this.samplesSkipped = 0
61
+ this.totalSkipSamples = 0
49
62
  }
50
63
 
51
64
  /**
@@ -59,25 +72,15 @@ class FFmpegDecoder extends BaseInference {
59
72
 
60
73
  this.logger.info('Loading FFmpegDecoder with config:', this.config)
61
74
 
62
- // Create worker thread
63
- const workerPath = path.join(__dirname, 'ffmpeg-worker.js')
64
- this.worker = new Worker(workerPath)
65
-
66
- // Wait for worker to be ready
67
- await new Promise((resolve, reject) => {
68
- const timeout = setTimeout(() => {
69
- reject(new Error('Worker initialization timeout'))
70
- }, 5000)
71
-
72
- this.worker.once('message', (msg) => {
73
- clearTimeout(timeout)
74
- if (msg.type === FFmpegWorkerMessages.READY) {
75
- resolve()
76
- } else {
77
- reject(new Error('Unexpected worker message during init'))
78
- }
79
- })
80
- })
75
+ // Initialize format constants
76
+ this.SUPPORTED_AUDIO_FORMATS.s16le.format = ffmpeg.constants.sampleFormats.S16
77
+ this.SUPPORTED_AUDIO_FORMATS.f32le.format = ffmpeg.constants.sampleFormats.FLT
78
+ this.OUTPUT_CHANNEL_LAYOUT = ffmpeg.constants.channelLayouts.MONO
79
+
80
+ // Validate audio format
81
+ if (!this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat]) {
82
+ throw new Error(`Unsupported audio format: ${this.config.audioFormat}`)
83
+ }
81
84
 
82
85
  this.isLoaded = true
83
86
  this.logger.info('FFmpegDecoder loaded successfully')
@@ -92,24 +95,7 @@ class FFmpegDecoder extends BaseInference {
92
95
  }
93
96
 
94
97
  this.logger.info('Unloading FFmpegDecoder')
95
-
96
- if (this.worker) {
97
- await new Promise((resolve) => {
98
- this.worker.postMessage({ type: FFmpegDecoderMessages.CLEANUP })
99
- this.worker.once('message', (msg) => {
100
- if (msg.type === FFmpegWorkerMessages.CLEANUP_DONE) {
101
- resolve()
102
- }
103
- })
104
-
105
- // Fallback timeout
106
- setTimeout(resolve, 1000)
107
- })
108
-
109
- await this.worker.terminate()
110
- this.worker = null
111
- }
112
-
98
+
113
99
  this.isLoaded = false
114
100
  this.currentJob = null
115
101
  this.logger.info('FFmpegDecoder unloaded')
@@ -149,69 +135,130 @@ class FFmpegDecoder extends BaseInference {
149
135
  return response
150
136
  }
151
137
 
152
- _handleWorkerMessage (job) {
153
- return new Promise((resolve, reject) => {
154
- const messageHandler = async (msg) => {
155
- this.logger.debug('[FFmpegDecoder] Received message from worker:', msg)
156
- switch (msg.type) {
157
- case FFmpegWorkerMessages.INIT_RESULT: {
158
- if (msg.success) {
159
- this.logger.info('[FFmpegDecoder] Decoder initialized successfully')
160
- this.worker.postMessage({
161
- type: FFmpegDecoderMessages.STREAM_START
162
- })
163
- } else {
164
- this.worker.off('message', messageHandler)
165
- this.logger.error('[FFmpegDecoder] Failed to initialize decoder:', msg.error)
166
- }
167
- break
168
- }
138
+ _getBufferSize (inputBitrate) {
139
+ const maxBufferSize = 1024 * 1024 // 1MB max
140
+ return Math.min((inputBitrate / 8) * 4, maxBufferSize)
141
+ }
169
142
 
170
- case FFmpegWorkerMessages.RESULT_CHUNK: {
171
- const decodedBuffer = Buffer.from(msg.data)
172
- this.logger.debug('[FFmpegDecoder] Result chunk', decodedBuffer.length)
173
- job.response.updateOutput({ outputArray: decodedBuffer })
174
- break
175
- }
143
+ _processFrame (decoder, raw, resampler, job) {
144
+ const OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].format
145
+ const OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].byteLength
146
+ const OUTPUT_SAMPLE_RATE = this.config.sampleRate
147
+
148
+ while (decoder.receiveFrame(raw)) {
149
+ const output = new ffmpeg.Frame()
150
+ output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
151
+ output.format = OUTPUT_FORMAT
152
+ output.sampleRate = OUTPUT_SAMPLE_RATE
153
+ output.nbSamples = raw.nbSamples
154
+
155
+ const samples = new ffmpeg.Samples(
156
+ output.format,
157
+ output.channelLayout.nbChannels,
158
+ output.nbSamples
159
+ )
160
+ samples.fill(output)
161
+
162
+ const count = resampler.convert(raw, output)
163
+
164
+ // Handle encoder delay by skipping initial samples
165
+ if (this.samplesSkipped < this.totalSkipSamples) {
166
+ const samplesToSkip = Math.min(count, this.totalSkipSamples - this.samplesSkipped)
167
+ this.samplesSkipped += samplesToSkip
168
+ if (samplesToSkip >= count) continue // Skip entire frame
169
+
170
+ // Skip partial frame
171
+ const skipBytes = OUTPUT_FORMAT_BYTE_LENGTH * samplesToSkip * output.channelLayout.nbChannels
172
+ const length = OUTPUT_FORMAT_BYTE_LENGTH * (count - samplesToSkip) * output.channelLayout.nbChannels
173
+ const chunk = Buffer.from(samples.data.subarray(skipBytes, skipBytes + length))
174
+ job.response.updateOutput({ outputArray: chunk })
175
+ } else {
176
+ const length = OUTPUT_FORMAT_BYTE_LENGTH * count * output.channelLayout.nbChannels
177
+ const chunk = Buffer.from(samples.data.subarray(0, length))
178
+ job.response.updateOutput({ outputArray: chunk })
179
+ }
180
+ }
181
+ }
176
182
 
177
- case FFmpegWorkerMessages.RESULT_END: {
178
- this.logger.debug('[FFmpegDecoder] Result end')
179
- job.response.ended()
180
- this.worker.off('message', messageHandler)
181
- resolve()
182
- break
183
- }
183
+ _processPacket (format, packet, raw, decoder, resampler, job) {
184
+ while (format.readFrame(packet)) {
185
+ decoder.sendPacket(packet)
186
+ this._processFrame(decoder, raw, resampler, job)
187
+ packet.unref()
188
+ }
189
+ }
184
190
 
185
- case FFmpegWorkerMessages.ERROR: {
186
- const errorMessage = msg.error || msg.data?.error || 'Unknown error'
187
- this.logger.error('[FFmpegDecoder] Error from worker:', errorMessage)
188
- job.response.failed(errorMessage)
189
- this.worker.off('message', messageHandler)
190
- reject(new Error(errorMessage))
191
- break
192
- }
193
- }
194
- }
191
+ _processFFmpegStream (format, stream, job) {
192
+ const OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].format
193
+ const OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[this.config.audioFormat].byteLength
194
+ const OUTPUT_SAMPLE_RATE = this.config.sampleRate
195
+
196
+ this.logger.debug('[FFmpegDecoder] Stream codec:', stream.codec, stream.codecParameters)
197
+
198
+ const packet = new ffmpeg.Packet()
199
+ const raw = new ffmpeg.Frame()
200
+
201
+ const resampler = new ffmpeg.Resampler(
202
+ stream.codecParameters.sampleRate,
203
+ stream.codecParameters.channelLayout,
204
+ stream.codecParameters.format,
205
+ OUTPUT_SAMPLE_RATE,
206
+ this.OUTPUT_CHANNEL_LAYOUT,
207
+ OUTPUT_FORMAT
208
+ )
209
+
210
+ const decoder = stream.decoder()
211
+ decoder.open()
212
+
213
+ // Auto-detect encoder delay: lossy codecs need ~400ms skipped to remove artifacts
214
+ const codecName = stream.codec.name.toLowerCase()
215
+ const SKIP_MS = {
216
+ mp3: 400,
217
+ vorbis: 400,
218
+ opus: 150,
219
+ aac: 300
220
+ }
195
221
 
196
- this.worker.on('message', messageHandler)
197
- this.worker.on('error', (err) => {
198
- this.logger.error('[FFmpegDecoder] Error from worker:', err)
199
- job.response.failed(err)
200
- this.worker.off('message', messageHandler)
201
- reject(err)
202
- })
203
- })
222
+ const skipMs = SKIP_MS[codecName] || 0
223
+ this.samplesSkipped = 0
224
+ this.totalSkipSamples = Math.floor((OUTPUT_SAMPLE_RATE * skipMs) / 1000)
225
+
226
+ if (this.totalSkipSamples > 0) {
227
+ this.logger.info(`[FFmpegDecoder] Skipping ${skipMs}ms (${this.totalSkipSamples} samples) for ${codecName} to remove encoder artifacts`)
228
+ }
229
+
230
+ this._processPacket(format, packet, raw, decoder, resampler, job)
231
+
232
+ // Flush resampler
233
+ const output = new ffmpeg.Frame()
234
+ output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
235
+ output.format = OUTPUT_FORMAT
236
+ output.sampleRate = OUTPUT_SAMPLE_RATE
237
+ output.nbSamples = 1024
238
+
239
+ const samples = new ffmpeg.Samples(
240
+ output.format,
241
+ output.channelLayout.nbChannels,
242
+ output.nbSamples
243
+ )
244
+ samples.fill(output)
245
+
246
+ while (resampler.flush(output) > 0) {
247
+ const chunk = Buffer.from(samples.data)
248
+ job.response.updateOutput({ outputArray: chunk })
249
+ }
250
+
251
+ decoder.destroy()
204
252
  }
205
253
 
206
- async _streamAudioToWorker (audioStream, port, job) {
207
- let chunkCount = 0
254
+ async _collectStreamData (audioStream, job) {
255
+ const chunks = []
256
+ let totalBytes = 0
257
+
208
258
  for await (const chunk of audioStream) {
209
- chunkCount++
210
-
211
- this.logger.debug(`[FFmpegDecoder] Received audio chunk #${chunkCount}, size: ${chunk.length} bytes`)
212
259
  if (!job.isActive) {
213
- this.logger.info('[FFmpegDecoder] Job cancelled, stopping stream collection at chunk', chunkCount)
214
- return
260
+ this.logger.info('[FFmpegDecoder] Job cancelled, stopping stream collection')
261
+ break
215
262
  }
216
263
 
217
264
  while (job.isPaused) {
@@ -219,9 +266,12 @@ class FFmpegDecoder extends BaseInference {
219
266
  await new Promise(resolve => setTimeout(resolve, 100))
220
267
  }
221
268
 
222
- port.writeSync(chunk)
269
+ chunks.push(chunk)
270
+ totalBytes += chunk.length
271
+ this.logger.debug(`[FFmpegDecoder] Collected chunk, total bytes: ${totalBytes}`)
223
272
  }
224
- return chunkCount
273
+
274
+ return Buffer.concat(chunks)
225
275
  }
226
276
 
227
277
  async _processStream (audioStream) {
@@ -231,32 +281,53 @@ class FFmpegDecoder extends BaseInference {
231
281
  }
232
282
 
233
283
  try {
234
- const channel = new Channel()
235
- const port = channel.connect()
284
+ this.logger.info('[FFmpegDecoder] Starting stream processing')
236
285
 
237
- this.logger.debug('[FFmpegDecoder] Processing audio stream with channel handle:', channel.handle)
286
+ // Collect all audio data from stream
287
+ const audioBuffer = await this._collectStreamData(audioStream, job)
288
+ this.logger.info(`[FFmpegDecoder] Collected ${audioBuffer.length} bytes of audio data`)
238
289
 
239
- // Worker promise to handle worker messages
240
- const workerCompletionPromise = this._handleWorkerMessage(job)
290
+ if (!job.isActive) {
291
+ this.logger.info('[FFmpegDecoder] Job cancelled after data collection')
292
+ return
293
+ }
294
+
295
+ // Create FFmpeg IO context with the buffer
296
+ const bufferSize = this._getBufferSize(this.config.inputBitrate)
297
+ let bufferOffset = 0
298
+
299
+ const io = new ffmpeg.IOContext(bufferSize, {
300
+ onread: (buffer, requestedLen) => {
301
+ const remainingBytes = audioBuffer.length - bufferOffset
302
+ const bytesToRead = Math.min(requestedLen, remainingBytes)
303
+
304
+ if (bytesToRead <= 0) {
305
+ return 0 // EOF
306
+ }
241
307
 
242
- // Initialize ffmpeg worker
243
- this.worker.postMessage({
244
- type: FFmpegDecoderMessages.INIT,
245
- data: {
246
- channelHandle: channel.handle,
247
- config: this.config
308
+ audioBuffer.copy(buffer, 0, bufferOffset, bufferOffset + bytesToRead)
309
+ bufferOffset += bytesToRead
310
+
311
+ this.logger.debug(`[FFmpegDecoder] Read ${bytesToRead} bytes from buffer, offset now: ${bufferOffset}`)
312
+ return bytesToRead
248
313
  }
249
314
  })
250
315
 
251
- // Stream all audio chunks to ffmpeg worker
252
- const chunkCount = await this._streamAudioToWorker(audioStream, port, job)
253
- this.logger.info(`[FFmpegDecoder] Finished streaming audio. Total chunks sent: ${chunkCount}`)
316
+ this.logger.debug('[FFmpegDecoder] IOContext created')
317
+ const format = new ffmpeg.InputFormatContext(io)
318
+ this.logger.debug('[FFmpegDecoder] InputFormatContext created')
319
+
320
+ const streamIndex = this.config.streamIndex || 0
321
+ if (format.streams[streamIndex] === undefined) {
322
+ throw new Error('Stream index out of bounds')
323
+ }
254
324
 
255
- // Signal to worker that we're done sending audio
256
- await port.close()
325
+ // Process the stream and generate decoded output
326
+ this._processFFmpegStream(format, format.streams[streamIndex], job)
257
327
 
258
- // Wait for worker to finish
259
- await workerCompletionPromise
328
+ // Mark as complete
329
+ job.response.ended()
330
+ this.logger.info('[FFmpegDecoder] Stream processing completed successfully')
260
331
  } catch (err) {
261
332
  this.logger.error('Error processing audio stream:', err)
262
333
  job.response.failed(err)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@qvac/decoder-audio",
3
- "version": "0.2.3",
3
+ "version": "0.2.5",
4
4
  "description": "",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Tether",
@@ -47,8 +47,8 @@
47
47
  "typescript": "^5.3.0"
48
48
  },
49
49
  "dependencies": {
50
- "@qvac/infer-base": "^0.1.0",
51
50
  "@qvac/error": "^0.1.0",
51
+ "@qvac/infer-base": "^0.1.0",
52
52
  "@qvac/logging": "^0.1.0",
53
53
  "@qvac/response": "^0.1.0",
54
54
  "bare-assert": "^1.1.0",
@@ -57,8 +57,7 @@
57
57
  "bare-fs": "^4.5.1",
58
58
  "bare-path": "^3.0.0",
59
59
  "bare-process": "^4.2.2",
60
- "process": "npm:bare-process@^4.2.2",
61
- "bare-worker": "^4.1.0"
60
+ "process": "npm:bare-process@^4.2.2"
62
61
  },
63
62
  "bugs": "https://github.com/tetherto/qvac-lib-decoder-audio/issues",
64
63
  "types": "index.d.ts"
@@ -7,7 +7,7 @@
7
7
  ],
8
8
  "current_versions": [
9
9
  {
10
- "version": "0.2.3"
10
+ "version": "0.2.5"
11
11
  }
12
12
  ],
13
13
  "exported_symbols": [
@@ -7,7 +7,7 @@
7
7
  ],
8
8
  "current_versions": [
9
9
  {
10
- "version": "0.2.3"
10
+ "version": "0.2.5"
11
11
  }
12
12
  ],
13
13
  "exported_symbols": [
@@ -7,7 +7,7 @@
7
7
  ],
8
8
  "current_versions": [
9
9
  {
10
- "version": "0.2.3"
10
+ "version": "0.2.5"
11
11
  }
12
12
  ],
13
13
  "exported_symbols": [
@@ -7,7 +7,7 @@
7
7
  ],
8
8
  "current_versions": [
9
9
  {
10
- "version": "0.2.3"
10
+ "version": "0.2.5"
11
11
  }
12
12
  ],
13
13
  "exported_symbols": [
@@ -7,7 +7,7 @@
7
7
  ],
8
8
  "current_versions": [
9
9
  {
10
- "version": "0.2.3"
10
+ "version": "0.2.5"
11
11
  }
12
12
  ],
13
13
  "exported_symbols": [
@@ -0,0 +1,105 @@
1
+ const { FFmpegDecoder } = require('@qvac/decoder-audio')
2
+
3
+ async function testFFmpegDecodeMp3() {
4
+ try {
5
+ const decoder = new FFmpegDecoder({
6
+ config: {
7
+ audioFormat: 's16le',
8
+ sampleRate: 16000
9
+ }
10
+ })
11
+
12
+ await decoder.load()
13
+
14
+ const audioPath = getAssetPath('sample.mp3')
15
+ const audioStream = fs.createReadStream(audioPath)
16
+
17
+ const response = await decoder.run(audioStream)
18
+
19
+ let totalBytes = 0
20
+ let updateCount = 0
21
+
22
+ await response
23
+ .onUpdate(output => {
24
+ if (output && output.outputArray) {
25
+ const bytes = new Uint8Array(output.outputArray)
26
+ totalBytes += bytes.length
27
+ updateCount++
28
+ }
29
+ })
30
+ .await()
31
+
32
+ if (totalBytes === 0) {
33
+ throw new Error('No audio data decoded')
34
+ }
35
+
36
+ if (updateCount === 0) {
37
+ throw new Error('No decoder updates received')
38
+ }
39
+
40
+ console.log(`MP3 decode complete: ${totalBytes} bytes, ${updateCount} updates`)
41
+
42
+ await decoder.unload()
43
+
44
+ return {
45
+ success: true,
46
+ totalBytes,
47
+ updateCount
48
+ }
49
+ } catch (error) {
50
+ console.error('Error during testFFmpegDecodeMp3:', error)
51
+ throw error
52
+ }
53
+ }
54
+
55
+ async function testFFmpegDecodeWav() {
56
+ try {
57
+ const decoder = new FFmpegDecoder({
58
+ config: {
59
+ audioFormat: 's16le',
60
+ sampleRate: 16000
61
+ }
62
+ })
63
+
64
+ await decoder.load()
65
+
66
+ const audioPath = getAssetPath('sample.wav')
67
+ const audioStream = fs.createReadStream(audioPath)
68
+
69
+ const response = await decoder.run(audioStream)
70
+
71
+ let totalBytes = 0
72
+ let updateCount = 0
73
+
74
+ await response
75
+ .onUpdate(output => {
76
+ if (output && output.outputArray) {
77
+ const bytes = new Uint8Array(output.outputArray)
78
+ totalBytes += bytes.length
79
+ updateCount++
80
+ }
81
+ })
82
+ .await()
83
+
84
+ if (totalBytes === 0) {
85
+ throw new Error('No audio data decoded')
86
+ }
87
+
88
+ if (updateCount === 0) {
89
+ throw new Error('No decoder updates received')
90
+ }
91
+
92
+ console.log(`MP3 decode complete: ${totalBytes} bytes, ${updateCount} updates`)
93
+
94
+ await decoder.unload()
95
+
96
+ return {
97
+ success: true,
98
+ totalBytes,
99
+ updateCount
100
+ }
101
+ } catch (error) {
102
+ console.error('Error during testFFmpegDecodeMp3:', error)
103
+ throw error
104
+ }
105
+ }
@@ -1,98 +0,0 @@
1
- 'use strict'
2
-
3
- const QvacLogger = require('@qvac/logging')
4
-
5
- class ChannelPortStream {
6
- constructor (channelPort, logger) {
7
- this.channelPort = channelPort
8
- this.logger = new QvacLogger(logger)
9
-
10
- this.channelEnd = false
11
- this.channelClose = false
12
-
13
- this.channelPort.on('end', () => {
14
- this.logger.debug('ChannelPortStream Channel port ended')
15
- this.channelEnd = true
16
- })
17
- this.channelPort.on('close', () => {
18
- this.logger.debug('ChannelPortStream Channel port closed')
19
- this.channelClose = true
20
- })
21
-
22
- this.bufferArray = []
23
- this.bufferOneOffset = 0
24
- }
25
-
26
- portHasData () {
27
- const hasData = (this.channelEnd || this.channelClose) && this.channelPort._queue.length
28
- return hasData !== 0
29
- }
30
-
31
- _readFromChannelPort () {
32
- let portHasDataCount = 0
33
- while (this.portHasData()) {
34
- portHasDataCount++
35
- const chunk = this.channelPort.readSync()
36
- if (!chunk || chunk.length === 0) {
37
- this.logger.debug('Read chunk of length 0 from channelPort')
38
- break
39
- }
40
- this.logger.debug(`Read chunk of length ${chunk.length} from channelPort`)
41
- this.bufferArray.push(chunk)
42
- }
43
- this.logger.debug(`portHasData() loop ran ${portHasDataCount} times, bufferArray.length=${this.bufferArray.length}`)
44
- }
45
-
46
- _readFromBufferArray (buffer, len) {
47
- let finalLen = 0
48
- let bytesCopied = 0
49
- if (this.bufferArray.length > 0) {
50
- while (finalLen < len) {
51
- // Get pending bytes in first buffer
52
- const firstBufferPending = this.bufferArray[0].length - this.bufferOneOffset
53
-
54
- // Get bytes to copy in case pending len is less than firstBufferPending
55
- const toCopy = Math.min(len - finalLen, firstBufferPending)
56
- this.logger.debug(`Copying ${toCopy} bytes from bufferArray[0] (pending: ${firstBufferPending}), bufferOneOffset=${this.bufferOneOffset}, finalLen=${finalLen}`)
57
-
58
- // Copy bytes to buffer
59
- bytesCopied += this.bufferArray[0].copy(buffer, finalLen, this.bufferOneOffset, this.bufferOneOffset + toCopy)
60
- this.bufferOneOffset += toCopy
61
-
62
- // Shift out buffer if it's empty
63
- if (this.bufferOneOffset >= this.bufferArray[0].length) {
64
- this.logger.debug('Finished bufferArray[0], shifting it out')
65
- this.bufferArray.shift()
66
- this.bufferOneOffset = 0
67
- }
68
-
69
- // Update finalLen
70
- finalLen += toCopy
71
-
72
- // Break inner loop if bufferArray has no more buffers
73
- if (this.bufferArray.length === 0) {
74
- this.logger.debug('bufferArray is empty, breaking inner loop')
75
- break
76
- }
77
- }
78
- } else {
79
- this.logger.debug('bufferArray is empty, nothing to copy')
80
- }
81
-
82
- this.logger.debug(`_readFromBufferArray bytesCopied=${bytesCopied}`)
83
- return finalLen
84
- }
85
-
86
- read (buffer, len) {
87
- this.logger.debug(`Called with len=${len}`)
88
-
89
- this._readFromChannelPort()
90
-
91
- const finalLen = this._readFromBufferArray(buffer, len)
92
-
93
- this.logger.debug(`Returning finalLen=${finalLen}`)
94
- return finalLen
95
- }
96
- }
97
-
98
- module.exports = ChannelPortStream
@@ -1,23 +0,0 @@
1
- 'use strict'
2
-
3
- const FFmpegDecoderMessages = {
4
- // Decoder messages
5
- INIT: 'init', // Initialization message
6
- STREAM_START: 'stream-start', // Start streaming message
7
- CLEANUP: 'cleanup' // Cleanup message
8
- }
9
-
10
- const FFmpegWorkerMessages = {
11
- // Worker messages
12
- READY: 'ready', // Ready message
13
- INIT_RESULT: 'init-result', // Initialization result message
14
- RESULT_CHUNK: 'result-chunk', // Result chunk message
15
- RESULT_END: 'result-end', // End streaming message
16
- CLEANUP_DONE: 'cleanup-done', // Cleanup done message
17
- ERROR: 'error' // Error message
18
- }
19
-
20
- module.exports = {
21
- FFmpegDecoderMessages,
22
- FFmpegWorkerMessages
23
- }
@@ -1,293 +0,0 @@
1
- 'use strict'
2
-
3
- const { parentPort } = require('bare-worker')
4
- const ffmpeg = require('bare-ffmpeg')
5
- const QvacLogger = require('@qvac/logging')
6
- const Channel = require('bare-channel')
7
- const ChannelPortStream = require('./channel-port-stream')
8
- const { FFmpegDecoderMessages, FFmpegWorkerMessages } = require('./constants')
9
-
10
- class FFmpegWorker {
11
- SUPPORTED_AUDIO_FORMATS = {
12
- s16le: {
13
- format: ffmpeg.constants.sampleFormats.S16,
14
- byteLength: 2
15
- },
16
- f32le: {
17
- format: ffmpeg.constants.sampleFormats.FLT,
18
- byteLength: 4
19
- }
20
- }
21
-
22
- // Whisper addon expectations
23
- OUTPUT_FORMAT_BYTE_LENGTH = 2 // S16LE uses 16 bits = 2 bytes
24
- OUTPUT_FORMAT = ffmpeg.constants.sampleFormats.S16
25
- OUTPUT_SAMPLE_RATE = 16000
26
- OUTPUT_CHANNEL_LAYOUT = ffmpeg.constants.channelLayouts.MONO
27
-
28
- constructor (parentPort, logger = new QvacLogger(console)) {
29
- this.parentPort = parentPort
30
- this.logger = logger
31
-
32
- this.channel = null
33
- this.channelPort = null
34
- this.channelPortStream = null
35
-
36
- this.config = null
37
- this.totalReadLen = 0
38
-
39
- // Encoder delay handling
40
- this.samplesSkipped = 0
41
- this.totalSkipSamples = 0
42
- }
43
-
44
- sendMessage (type, data) {
45
- this.parentPort.postMessage({ type, data })
46
- }
47
-
48
- sendResultChunk (data) {
49
- this.sendMessage(FFmpegWorkerMessages.RESULT_CHUNK, data)
50
- }
51
-
52
- sendEndMessage () {
53
- this.logger.debug('[ffmpeg-worker] Sending end message')
54
- this.sendMessage(FFmpegWorkerMessages.RESULT_END)
55
- }
56
-
57
- initializeDecoder (data) {
58
- const { channelHandle, config } = data
59
- this.config = config
60
-
61
- if (!this.SUPPORTED_AUDIO_FORMATS[config.audioFormat]) {
62
- this.logger.error('[ffmpeg-worker] Unsupported audio format:', config.audioFormat)
63
- return { success: false, error: 'Unsupported audio format' }
64
- }
65
-
66
- this.OUTPUT_FORMAT = this.SUPPORTED_AUDIO_FORMATS[config.audioFormat].format
67
- this.OUTPUT_FORMAT_BYTE_LENGTH = this.SUPPORTED_AUDIO_FORMATS[config.audioFormat].byteLength
68
-
69
- this.logger.info('[ffmpeg-worker] Initializing decoder...')
70
- try {
71
- this.channel = Channel.from(channelHandle)
72
- this.channelPort = this.channel.connect()
73
- this.channelPortStream = new ChannelPortStream(this.channelPort, this.logger)
74
- return { success: true }
75
- } catch (err) {
76
- this.logger.error('[ffmpeg-worker] Failed to initialize decoder:', err)
77
- return { success: false, error: err.message }
78
- }
79
- }
80
-
81
- startStream () {
82
- this.decodeStream()
83
- }
84
-
85
- _processFrame (decoder, raw, resampler) {
86
- while (decoder.receiveFrame(raw)) {
87
- const output = new ffmpeg.Frame()
88
- output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
89
- output.format = this.OUTPUT_FORMAT
90
- output.sampleRate = this.OUTPUT_SAMPLE_RATE
91
- output.nbSamples = raw.nbSamples
92
-
93
- const samples = new ffmpeg.Samples(
94
- output.format,
95
- output.channelLayout.nbChannels,
96
- output.nbSamples
97
- )
98
- samples.fill(output)
99
-
100
- // samples.data.length is always fixed, like 8192 bytes. So we need to
101
- // extract only the data we need through `subarray`. `resampler.convert`
102
- // gives the number of samples converted per channel. So
103
- // `bytesPerSample` * `samplesPerChannel` * `channelsPerSample` gives
104
- // total amount of bytes.
105
- const count = resampler.convert(raw, output)
106
-
107
- // Handle encoder delay by skipping initial samples
108
- if (this.samplesSkipped < this.totalSkipSamples) {
109
- const samplesToSkip = Math.min(count, this.totalSkipSamples - this.samplesSkipped)
110
- this.samplesSkipped += samplesToSkip
111
- if (samplesToSkip >= count) continue // Skip entire frame
112
-
113
- // Skip partial frame
114
- const skipBytes = this.OUTPUT_FORMAT_BYTE_LENGTH * samplesToSkip * output.channelLayout.nbChannels
115
- const length = this.OUTPUT_FORMAT_BYTE_LENGTH * (count - samplesToSkip) * output.channelLayout.nbChannels
116
- this.sendResultChunk(Buffer.from(samples.data.subarray(skipBytes, skipBytes + length)))
117
- } else {
118
- const length = this.OUTPUT_FORMAT_BYTE_LENGTH * count * output.channelLayout.nbChannels
119
- this.sendResultChunk(Buffer.from(samples.data.subarray(0, length)))
120
- }
121
- }
122
- }
123
-
124
- _processPacket (format, packet, raw, decoder, resampler) {
125
- while (format.readFrame(packet)) {
126
- decoder.sendPacket(packet)
127
- this._processFrame(decoder, raw, resampler)
128
- packet.unref()
129
- }
130
- }
131
-
132
- _processStream (format, stream) {
133
- console.log(stream.codec, stream.codecParameters)
134
- const packet = new ffmpeg.Packet()
135
- const raw = new ffmpeg.Frame()
136
-
137
- const resampler = new ffmpeg.Resampler(
138
- stream.codecParameters.sampleRate,
139
- stream.codecParameters.channelLayout,
140
- stream.codecParameters.format,
141
- this.OUTPUT_SAMPLE_RATE,
142
- this.OUTPUT_CHANNEL_LAYOUT,
143
- this.OUTPUT_FORMAT
144
- )
145
-
146
- const decoder = stream.decoder()
147
- decoder.open()
148
-
149
- // Auto-detect encoder delay: lossy codecs need ~400ms skipped to remove artifacts
150
- // Lossless codecs (WAV, FLAC) need no skipping
151
- const codecName = stream.codec.name.toLowerCase()
152
- const SKIP_MS = {
153
- mp3: 400,
154
- vorbis: 400,
155
- opus: 150,
156
- aac: 300
157
- }
158
-
159
- const skipMs = SKIP_MS[codecName] || 0
160
- this.samplesSkipped = 0
161
- this.totalSkipSamples = Math.floor((this.OUTPUT_SAMPLE_RATE * skipMs) / 1000)
162
-
163
- if (this.totalSkipSamples > 0) {
164
- console.log(`[ffmpeg-worker] Skipping ${skipMs}ms (${this.totalSkipSamples} samples) for ${codecName} to remove encoder artifacts`)
165
- this.logger.info(`[ffmpeg-worker] Skipping ${skipMs}ms for ${codecName} to remove encoder artifacts`)
166
- }
167
-
168
- this._processPacket(format, packet, raw, decoder, resampler)
169
-
170
- const output = new ffmpeg.Frame()
171
- output.channelLayout = this.OUTPUT_CHANNEL_LAYOUT
172
- output.format = this.OUTPUT_FORMAT
173
- output.sampleRate = this.OUTPUT_SAMPLE_RATE
174
- output.nbSamples = 1024
175
-
176
- const samples = new ffmpeg.Samples(
177
- output.format,
178
- output.channelLayout.nbChannels,
179
- output.nbSamples
180
- )
181
- samples.fill(output)
182
-
183
- while (resampler.flush(output) > 0) {
184
- this.sendResultChunk(Buffer.from(samples.data))
185
- }
186
-
187
- decoder.destroy()
188
- this.sendEndMessage()
189
- }
190
-
191
- _getBufferSize (inputBitrate) {
192
- const maxBufferSize = 1024 * 1024 // 1MB max
193
- return Math.min((inputBitrate / 8) * 4, maxBufferSize)
194
- }
195
-
196
- decodeStream () {
197
- const {
198
- streamIndex = 0,
199
- inputBitrate
200
- } = this.config
201
-
202
- this.logger.debug('[ffmpeg-worker] Starting deocde with config', this.config)
203
-
204
- if (!this.channelPortStream) {
205
- this.logger.error('[ffmpeg-worker] Channel port stream not initialized')
206
- return this.sendMessage(FFmpegWorkerMessages.ERROR, { error: 'Channel port stream not initialized' })
207
- }
208
-
209
- try {
210
- const bufferSize = this._getBufferSize(inputBitrate)
211
-
212
- const io = new ffmpeg.IOContext(bufferSize, {
213
- onread: (buffer, requestedLen) => {
214
- return this.channelPortStream.read(buffer, requestedLen)
215
- }
216
- })
217
-
218
- this.logger.debug('[ffmpeg-worker] IOContext created')
219
- const format = new ffmpeg.InputFormatContext(io)
220
- this.logger.debug('[ffmpeg-worker] InputFormatContext created')
221
-
222
- if (format.streams[streamIndex] === undefined) {
223
- this.logger.error('[ffmpeg-worker] Stream index out of bounds')
224
- return this.sendMessage(FFmpegWorkerMessages.ERROR, { error: 'Stream index out of bounds' })
225
- }
226
-
227
- this._processStream(format, format.streams[streamIndex])
228
- } catch (err) {
229
- this.logger.error('[ffmpeg-worker] Error during decodeChunk:', err)
230
- return this.sendMessage(FFmpegWorkerMessages.ERROR, { error: err.message })
231
- }
232
- }
233
-
234
- cleanup () {
235
- this.logger.info('[ffmpeg-worker] Cleaning up decoder resources...')
236
- this.logger.info('[ffmpeg-worker] Cleanup complete')
237
- if (this.channelPort) {
238
- this.channelPort.close()
239
- this.logger.debug('[ffmpeg-worker] Channel port disposed')
240
- }
241
- if (this.channel) {
242
- this.channel = null
243
- this.logger.debug('[ffmpeg-worker] Channel disposed')
244
- }
245
- }
246
- }
247
-
248
- const logger = new QvacLogger()
249
- const ffmpegWorkerInstance = new FFmpegWorker(parentPort, logger)
250
-
251
- // Handle messages from main thread
252
- parentPort.on('message', ({ type, data }) => {
253
- logger.debug(`[ffmpeg-worker] Received message: ${type}`)
254
- switch (type) {
255
- case FFmpegDecoderMessages.INIT: {
256
- logger.info('[ffmpeg-worker] Handling init message with channel handle:', data)
257
- const initResult = ffmpegWorkerInstance.initializeDecoder(data)
258
- parentPort.postMessage({ type: FFmpegWorkerMessages.INIT_RESULT, ...initResult })
259
- break
260
- }
261
-
262
- case FFmpegDecoderMessages.STREAM_START: {
263
- logger.info('[ffmpeg-worker] Handling decode message')
264
- ffmpegWorkerInstance.startStream()
265
- break
266
- }
267
-
268
- case FFmpegDecoderMessages.CLEANUP: {
269
- logger.info('[ffmpeg-worker] Handling cleanup message')
270
- ffmpegWorkerInstance.cleanup()
271
- parentPort.postMessage({ type: FFmpegWorkerMessages.CLEANUP_DONE, success: true })
272
- break
273
- }
274
-
275
- default: {
276
- logger.warn(`[ffmpeg-worker] Unknown message type: ${type}`)
277
- parentPort.postMessage({ type: FFmpegWorkerMessages.ERROR, error: 'Unknown message type' })
278
- }
279
- }
280
- })
281
-
282
- parentPort.on('close', () => {
283
- logger.info('[ffmpeg-worker] Worker exiting')
284
- })
285
-
286
- parentPort.on('error', (err) => {
287
- logger.error('[ffmpeg-worker] Worker error:', err)
288
- parentPort.postMessage({ type: FFmpegWorkerMessages.ERROR, error: err })
289
- })
290
-
291
- // Notify main thread that worker is ready
292
- logger.info('[ffmpeg-worker] Worker ready')
293
- parentPort.postMessage({ type: 'ready' })
File without changes