@siteed/audio-studio 3.1.1 → 3.2.0-beta.1

package/CHANGELOG.md

@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `streamAudioData(options, callbacks)` — progressive Float32 file decode for
+  long compressed recordings without full-PCM materialization. Supports
+  `AbortSignal` cancellation, async `onChunk` back-pressure, target sample
+  rate, channel downmix, and stable error codes via the new
+  `AudioStreamError` class.
+- `getAudioDecodeCapabilities()` for platform feature discovery.
+- iOS: `AVAssetReader`-based decoder with per-`requestId` cancellation.
+- Android: `MediaExtractor` + `MediaCodec` decoder with linear resampling that
+  preserves continuity across codec buffer boundaries.
+- Docs: `docs/STREAM_AUDIO_DATA.md`.
+
+### Fixed
+
+- iOS `extractRawAudioData` no longer traps on non-finite or out-of-range
+  decoded samples. Float-to-`Int16`/`Int32` conversion now replaces NaN/Inf
+  with `0` and clamps the scaled value into the integer range before
+  conversion.
+
 ## [3.1.1] - 2026-05-08
 ### Fixed
 - Trust Android final decoded PCM metadata for range extraction and trimming, including JS `Number` options bridged as Kotlin `Double`.

package/android/src/main/java/net/siteed/audiostudio/AudioStreamDecoder.kt

@@ -0,0 +1,640 @@
+// packages/audio-studio/android/src/main/java/net/siteed/audiostudio/AudioStreamDecoder.kt
+//
+// Progressive file decoder for `streamAudioData`. Uses MediaExtractor +
+// MediaCodec to decode a stored audio file into bounded Float32 PCM chunks
+// without loading the full PCM range into memory. Output is sanitized
+// (NaN/Inf -> 0, clamp to [-1, 1] when normalizeAudio=true) and supports
+// per-request cancellation + back-pressure ack.
+package net.siteed.audiostudio
+
+import android.content.Context
+import android.media.MediaCodec
+import android.media.MediaExtractor
+import android.media.MediaFormat
+import android.net.Uri
+import android.os.Bundle
+import androidx.core.os.bundleOf
+import java.io.File
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicInteger
+import kotlin.concurrent.thread
+
+interface AudioStreamDecoderDelegate {
+    fun streamDecoderEmit(eventName: String, payload: Bundle)
+}
+
+class AudioStreamDecoder(
+    private val context: Context,
+    private val options: Options,
+    private val delegate: AudioStreamDecoderDelegate,
+) {
+    data class Options(
+        val requestId: String,
+        val fileUri: String,
+        val startTimeMs: Long?,
+        val endTimeMs: Long?,
+        val targetSampleRate: Int?,
+        val channels: Int?,
+        val normalizeAudio: Boolean,
+        val chunkDurationMs: Int,
+        val maxChunkBytes: Int?,
+        val maxBufferedChunks: Int,
+    )
+
+    companion object {
+        private const val TAG = "AudioStreamDecoder"
+        private const val TIMEOUT_US = 10_000L
+    }
+
+    private val cancelled = AtomicBoolean(false)
+    private val lastAckedIndex = AtomicInteger(-1)
+    private val ackLock = Object()
+    private var workerThread: Thread? = null
+
+    fun start() {
+        workerThread = thread(
+            isDaemon = true,
+            name = "AudioStreamDecoder-${options.requestId}"
+        ) {
+            run()
+        }
+    }
+
+    fun cancel() {
+        cancelled.set(true)
+        synchronized(ackLock) { ackLock.notifyAll() }
+    }
+
+    fun acknowledgeChunk(index: Int) {
+        synchronized(ackLock) {
+            if (index > lastAckedIndex.get()) {
+                lastAckedIndex.set(index)
+            }
+            ackLock.notifyAll()
+        }
+    }
+
+    private fun run() {
+        val path = resolveFilePath(options.fileUri)
+        if (path == null) {
+            emitError("ERR_AUDIO_STREAM_FILE_NOT_FOUND", "Cannot resolve file: ${options.fileUri}")
+            return
+        }
+        if (!File(path).exists()) {
+            emitError("ERR_AUDIO_STREAM_FILE_NOT_FOUND", "File not found: $path")
+            return
+        }
+
+        val extractor = MediaExtractor()
+        var codec: MediaCodec? = null
+        var emittedChunks = 0
+        var emittedSamples = 0L
+        var totalDurationUs: Long = -1
+        var outputSampleRate = options.targetSampleRate ?: 0
+        var outputChannels = options.channels ?: 0
+
+        try {
+            try {
+                extractor.setDataSource(path)
+            } catch (e: Exception) {
+                emitError(
+                    "ERR_AUDIO_STREAM_DECODE_FAILED",
+                    "setDataSource failed: ${e.message}"
+                )
+                return
+            }
+
+            val trackIndex = (0 until extractor.trackCount).firstOrNull { idx ->
+                val format = extractor.getTrackFormat(idx)
+                format.getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true
+            }
+            if (trackIndex == null) {
+                emitError(
+                    "ERR_AUDIO_STREAM_UNSUPPORTED_FORMAT",
+                    "No audio track found"
+                )
+                return
+            }
+            extractor.selectTrack(trackIndex)
+            val format = extractor.getTrackFormat(trackIndex)
+            totalDurationUs = if (format.containsKey(MediaFormat.KEY_DURATION)) {
+                format.getLong(MediaFormat.KEY_DURATION)
+            } else {
+                -1
+            }
+
+            val sourceSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE)
+            val sourceChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
+            if (outputSampleRate <= 0) outputSampleRate = sourceSampleRate
+            if (outputChannels <= 0) outputChannels = minOf(2, maxOf(1, sourceChannels))
+
+            options.startTimeMs?.let {
+                extractor.seekTo(it * 1000L, MediaExtractor.SEEK_TO_CLOSEST_SYNC)
+            }
+
+            val mime = format.getString(MediaFormat.KEY_MIME)
+                ?: run {
+                    emitError(
+                        "ERR_AUDIO_STREAM_UNSUPPORTED_FORMAT",
+                        "Track has no MIME"
+                    )
+                    return
+                }
+            codec = try {
+                MediaCodec.createDecoderByType(mime)
+            } catch (e: Exception) {
+                emitError(
+                    "ERR_AUDIO_STREAM_UNSUPPORTED_FORMAT",
+                    "No decoder for $mime: ${e.message}"
+                )
+                return
+            }
+            codec.configure(format, null, null, 0)
+            codec.start()
+
+            val endTimeUs = options.endTimeMs?.let { it * 1000L } ?: Long.MAX_VALUE
+            val rangeStartMs = options.startTimeMs ?: 0L
+            val samplesPerChunk = run {
+                val byTime = (options.chunkDurationMs.toLong() *
+                    outputSampleRate.toLong() / 1000L).toInt() * outputChannels
+                val byBytes = options.maxChunkBytes?.let { it / 4 } ?: Int.MAX_VALUE
+                maxOf(1, minOf(byTime, byBytes))
+            }
+            val pending = FloatArray(samplesPerChunk * 2)
+            var pendingLen = 0
+            val info = MediaCodec.BufferInfo()
+            var sawInputEOS = false
+            var sawOutputEOS = false
+
+            // Resampling state across output buffers. We keep the last source
+            // sample to interpolate across buffer boundaries.
+            val resampler = LinearResampler(outputChannels)
+
+            while (!sawOutputEOS) {
+                if (cancelled.get()) break
+
+                if (!sawInputEOS) {
+                    val inputBufferIndex = codec.dequeueInputBuffer(TIMEOUT_US)
+                    if (inputBufferIndex >= 0) {
+                        val inputBuffer = codec.getInputBuffer(inputBufferIndex)
+                        if (inputBuffer != null) {
+                            val sampleSize = extractor.readSampleData(inputBuffer, 0)
+                            val sampleTime = extractor.sampleTime
+                            if (sampleSize < 0 || sampleTime > endTimeUs) {
+                                codec.queueInputBuffer(
+                                    inputBufferIndex,
+                                    0,
+                                    0,
+                                    0,
+                                    MediaCodec.BUFFER_FLAG_END_OF_STREAM
+                                )
+                                sawInputEOS = true
+                            } else {
+                                codec.queueInputBuffer(
+                                    inputBufferIndex,
+                                    0,
+                                    sampleSize,
+                                    sampleTime,
+                                    0
+                                )
+                                extractor.advance()
+                            }
+                        }
+                    }
+                }
+
+                val outputIndex = codec.dequeueOutputBuffer(info, TIMEOUT_US)
+                if (outputIndex >= 0) {
+                    val outputBuffer = codec.getOutputBuffer(outputIndex)
+                    if (outputBuffer != null && info.size > 0) {
+                        outputBuffer.position(info.offset)
+                        outputBuffer.limit(info.offset + info.size)
+                        val pcm = ByteArray(info.size)
+                        outputBuffer.get(pcm)
+
+                        val converted = pcmToFloat(
+                            pcm,
+                            sourceChannels,
+                            outputChannels,
+                            options.normalizeAudio
+                        )
+                        val resampled = resampler.process(
+                            converted,
+                            sourceSampleRate,
+                            outputSampleRate
+                        )
+
+                        // Append into pending, grow if needed
+                        val newLen = pendingLen + resampled.size
+                        val target = if (newLen > pending.size) {
+                            val grown = FloatArray(newLen.coerceAtLeast(pending.size * 2))
+                            System.arraycopy(pending, 0, grown, 0, pendingLen)
+                            grown.also {
+                                System.arraycopy(it, 0, pending, 0, pendingLen)
+                            }
+                        } else {
+                            pending
+                        }
+                        // Note: when grown, we use a transient buffer that we
+                        // process and never write back into the original.
+                        val workBuffer: FloatArray
+                        if (target !== pending) {
+                            System.arraycopy(resampled, 0, target, pendingLen, resampled.size)
+                            pendingLen += resampled.size
+                            workBuffer = target
+                        } else {
+                            System.arraycopy(resampled, 0, pending, pendingLen, resampled.size)
+                            pendingLen += resampled.size
+                            workBuffer = pending
+                        }
+
+                        while (pendingLen >= samplesPerChunk) {
+                            if (cancelled.get()) break
+                            val chunk = FloatArray(samplesPerChunk)
+                            System.arraycopy(workBuffer, 0, chunk, 0, samplesPerChunk)
+                            // Shift remainder forward.
+                            val remainder = pendingLen - samplesPerChunk
+                            if (remainder > 0) {
+                                System.arraycopy(
+                                    workBuffer,
+                                    samplesPerChunk,
+                                    workBuffer,
+                                    0,
+                                    remainder
+                                )
+                            }
+                            pendingLen = remainder
+
+                            val chunkDurationMs =
+                                (chunk.size.toDouble() /
+                                    (outputSampleRate.toDouble() * outputChannels.toDouble())) *
+                                    1000.0
+                            val startMs = rangeStartMs +
+                                (emittedSamples.toDouble() /
+                                    (outputSampleRate.toDouble() * outputChannels.toDouble())) *
+                                    1000.0
+                            emitChunk(
+                                index = emittedChunks,
+                                startTimeMs = startMs,
+                                endTimeMs = startMs + chunkDurationMs,
+                                startSample = (emittedSamples / outputChannels).toInt(),
+                                sampleRate = outputSampleRate,
+                                channels = outputChannels,
+                                samples = chunk,
+                                isFinal = false
+                            )
+                            emittedChunks += 1
+                            emittedSamples += chunk.size
+                            emitProgress(
+                                processedMs = startMs + chunkDurationMs,
+                                durationMs = if (totalDurationUs > 0) totalDurationUs / 1000.0 else 0.0,
+                                emittedChunks = emittedChunks
+                            )
+                            if (waitForAckOrCancel(emittedChunks - 1)) {
+                                cancelled.set(true)
+                                break
+                            }
+                        }
+                    }
+                    codec.releaseOutputBuffer(outputIndex, false)
+                    if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
+                        sawOutputEOS = true
+                    }
+                } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+                    val newFormat = codec.outputFormat
+                    if (newFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE)) {
+                        // Source rate from codec output overrides extractor's
+                        // format when the codec exposes a different value.
+                        @Suppress("UNUSED_VARIABLE")
+                        val updatedRate = newFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)
+                    }
+                }
+            }
+
+            if (cancelled.get()) {
+                emitError("ERR_AUDIO_STREAM_CANCELLED", "Stream cancelled")
+                emitComplete(
+                    durationMs = if (totalDurationUs > 0) totalDurationUs / 1000.0 else 0.0,
+                    sampleRate = outputSampleRate,
+                    channels = outputChannels,
+                    chunks = emittedChunks,
+                    samples = emittedSamples.toInt(),
+                    cancelled = true
+                )
+                return
+            }
+
+            // Flush remainder as final chunk
+            if (pendingLen > 0) {
+                val tail = FloatArray(pendingLen)
+                System.arraycopy(pending, 0, tail, 0, pendingLen)
+                val tailDurationMs =
+                    (tail.size.toDouble() /
+                        (outputSampleRate.toDouble() * outputChannels.toDouble())) *
+                        1000.0
+                val startMs = rangeStartMs +
+                    (emittedSamples.toDouble() /
+                        (outputSampleRate.toDouble() * outputChannels.toDouble())) *
+                        1000.0
+                emitChunk(
+                    index = emittedChunks,
+                    startTimeMs = startMs,
+                    endTimeMs = startMs + tailDurationMs,
+                    startSample = (emittedSamples / outputChannels).toInt(),
+                    sampleRate = outputSampleRate,
+                    channels = outputChannels,
+                    samples = tail,
+                    isFinal = true
+                )
+                emittedChunks += 1
+                emittedSamples += tail.size
+            } else if (emittedChunks > 0) {
+                emitChunk(
+                    index = emittedChunks,
+                    startTimeMs = rangeStartMs +
+                        (emittedSamples.toDouble() /
+                            (outputSampleRate.toDouble() * outputChannels.toDouble())) *
+                            1000.0,
+                    endTimeMs = rangeStartMs +
+                        (emittedSamples.toDouble() /
+                            (outputSampleRate.toDouble() * outputChannels.toDouble())) *
+                            1000.0,
+                    startSample = (emittedSamples / outputChannels).toInt(),
+                    sampleRate = outputSampleRate,
+                    channels = outputChannels,
+                    samples = FloatArray(0),
+                    isFinal = true
+                )
+                emittedChunks += 1
+            }
+
+            emitComplete(
+                durationMs = if (totalDurationUs > 0) totalDurationUs / 1000.0 else 0.0,
+                sampleRate = outputSampleRate,
+                channels = outputChannels,
+                chunks = emittedChunks,
+                samples = emittedSamples.toInt(),
+                cancelled = false
+            )
+        } catch (e: Exception) {
+            LogUtils.e(TAG, "Decode failed: ${e.message}", e)
+            emitError(
+                "ERR_AUDIO_STREAM_DECODE_FAILED",
+                e.message ?: "Unknown decode error"
+            )
+        } finally {
+            try {
+                codec?.stop()
+            } catch (_: Exception) { /* noop */ }
+            try {
+                codec?.release()
+            } catch (_: Exception) { /* noop */ }
+            try {
+                extractor.release()
+            } catch (_: Exception) { /* noop */ }
+        }
+    }
+
+    private fun resolveFilePath(uri: String): String? {
+        if (uri.startsWith("/")) return uri
+        return try {
+            val parsed = Uri.parse(uri)
+            when (parsed.scheme) {
+                "file" -> parsed.path
+                "content" -> {
+                    // Copy to cache so MediaExtractor can read by path.
+                    val temp = File.createTempFile(
+                        "audiostream_${options.requestId}_",
+                        null,
+                        context.cacheDir
+                    )
+                    context.contentResolver.openInputStream(parsed)?.use { input ->
+                        temp.outputStream().use { out -> input.copyTo(out) }
+                    }
+                    temp.absolutePath
+                }
+                null -> uri
+                else -> null
+            }
+        } catch (e: Exception) {
+            LogUtils.e(TAG, "resolveFilePath failed: ${e.message}", e)
+            null
+        }
+    }
+
+    private fun pcmToFloat(
+        pcm: ByteArray,
+        sourceChannels: Int,
+        targetChannels: Int,
+        normalize: Boolean,
+    ): FloatArray {
+        if (pcm.isEmpty() || sourceChannels <= 0) return FloatArray(0)
+        val buffer = ByteBuffer.wrap(pcm).order(ByteOrder.LITTLE_ENDIAN)
+        val totalShorts = pcm.size / 2
+        val srcFrames = totalShorts / sourceChannels
+        val out = FloatArray(srcFrames * targetChannels)
+        var outIdx = 0
+        val tempFrame = FloatArray(sourceChannels)
+        for (frame in 0 until srcFrames) {
+            for (c in 0 until sourceChannels) {
+                val s = buffer.short.toFloat() / 32768.0f
+                val safe = if (s.isFinite()) s else 0f
+                tempFrame[c] = if (normalize) {
+                    when {
+                        safe > 1f -> 1f
+                        safe < -1f -> -1f
+                        else -> safe
+                    }
+                } else {
+                    safe
+                }
+            }
+            when {
+                sourceChannels == targetChannels -> {
+                    for (c in 0 until targetChannels) {
+                        out[outIdx++] = tempFrame[c]
+                    }
+                }
+                targetChannels == 1 -> {
+                    // Downmix average
+                    var sum = 0f
+                    for (c in 0 until sourceChannels) sum += tempFrame[c]
+                    out[outIdx++] = sum / sourceChannels
+                }
+                sourceChannels == 1 -> {
+                    // Upmix mono -> N (duplicate)
+                    for (c in 0 until targetChannels) {
+                        out[outIdx++] = tempFrame[0]
+                    }
+                }
+                else -> {
+                    // Drop or duplicate channels; for simplicity take first N or zero-pad.
+                    for (c in 0 until targetChannels) {
+                        out[outIdx++] = if (c < sourceChannels) tempFrame[c] else 0f
+                    }
+                }
+            }
+        }
+        return out
+    }
+
+    private fun waitForAckOrCancel(emittedIndex: Int): Boolean {
+        synchronized(ackLock) {
+            while (true) {
+                if (cancelled.get()) return true
+                val inFlight = emittedIndex - lastAckedIndex.get()
+                if (inFlight < options.maxBufferedChunks) return false
+                ackLock.wait(50L)
+            }
+            @Suppress("UNREACHABLE_CODE")
+            return false
+        }
+    }
+
+    private fun emitChunk(
+        index: Int,
+        startTimeMs: Double,
+        endTimeMs: Double,
+        startSample: Int,
+        sampleRate: Int,
+        channels: Int,
+        samples: FloatArray,
+        isFinal: Boolean,
+    ) {
+        // Pass FloatArray directly; expo-modules-core converts to JS array.
+        val payload = bundleOf(
+            "requestId" to options.requestId,
+            "chunkIndex" to index,
+            "startTimeMs" to startTimeMs,
+            "endTimeMs" to endTimeMs,
+            "startSample" to startSample,
+            "sampleCount" to samples.size,
+            "sampleRate" to sampleRate,
+            "channels" to channels,
+            "samples" to samples,
+            "isFinal" to isFinal,
+        )
+        delegate.streamDecoderEmit(Constants.AUDIO_STREAM_CHUNK_EVENT, payload)
+    }
+
+    private fun emitProgress(
+        processedMs: Double,
+        durationMs: Double,
+        emittedChunks: Int,
+    ) {
+        val progress = if (durationMs > 0) {
+            (processedMs / durationMs).coerceIn(0.0, 1.0)
+        } else {
+            0.0
+        }
+        val payload = bundleOf(
+            "requestId" to options.requestId,
+            "processedMs" to processedMs,
+            "durationMs" to durationMs,
+            "progress" to progress,
+            "emittedChunks" to emittedChunks,
+        )
+        delegate.streamDecoderEmit(Constants.AUDIO_STREAM_PROGRESS_EVENT, payload)
+    }
+
+    private fun emitComplete(
+        durationMs: Double,
+        sampleRate: Int,
+        channels: Int,
+        chunks: Int,
+        samples: Int,
+        cancelled: Boolean,
+    ) {
+        val payload = bundleOf(
+            "requestId" to options.requestId,
+            "durationMs" to durationMs,
+            "sampleRate" to sampleRate,
+            "channels" to channels,
+            "chunks" to chunks,
+            "samples" to samples,
+            "cancelled" to cancelled,
+        )
+        delegate.streamDecoderEmit(Constants.AUDIO_STREAM_COMPLETE_EVENT, payload)
+    }
+
+    private fun emitError(code: String, message: String) {
+        val payload = bundleOf(
+            "requestId" to options.requestId,
+            "code" to code,
+            "message" to message,
+        )
+        delegate.streamDecoderEmit(Constants.AUDIO_STREAM_ERROR_EVENT, payload)
+    }
+}
+
+/**
+ * Per-stream linear resampler keyed by output channel count. Maintains the
+ * last source frame across buffer boundaries so resampling is continuous.
+ */
+private class LinearResampler(private val channels: Int) {
+    private var lastSrcFrame: FloatArray? = null
+    private var fractional: Double = 0.0
+
+    fun process(input: FloatArray, sourceRate: Int, targetRate: Int): FloatArray {
+        if (input.isEmpty() || channels <= 0) return FloatArray(0)
+        if (sourceRate == targetRate) return input
+
+        val ratio = sourceRate.toDouble() / targetRate.toDouble()
+        val srcFrames = input.size / channels
+        val previousFrame = lastSrcFrame
+        val totalSrcFrames = srcFrames + if (previousFrame != null) 1 else 0
+        if (totalSrcFrames < 2) {
+            // Not enough to interpolate yet; stash and return empty.
+            lastSrcFrame = FloatArray(channels).also { dst ->
+                if (srcFrames >= 1) {
+                    System.arraycopy(input, (srcFrames - 1) * channels, dst, 0, channels)
+                }
+            }
+            return FloatArray(0)
+        }
+
+        // Build virtual stream: [previousFrame?, input...].
+        val virtualLen = totalSrcFrames
+        val out = ArrayList<Float>(((virtualLen * targetRate) / sourceRate + 1) * channels)
+        var srcPos = fractional
+        while (srcPos < virtualLen - 1) {
+            val i = srcPos.toInt()
+            val frac = (srcPos - i).toFloat()
+            for (c in 0 until channels) {
+                val a = sampleAt(i, c, previousFrame, input)
+                val b = sampleAt(i + 1, c, previousFrame, input)
+                out.add(a + (b - a) * frac)
+            }
+            srcPos += ratio
+        }
+        fractional = srcPos - (virtualLen - 1)
+
+        // Stash last source frame for the next call.
+        lastSrcFrame = FloatArray(channels).also { dst ->
+            System.arraycopy(input, (srcFrames - 1) * channels, dst, 0, channels)
+        }
+
+        val arr = FloatArray(out.size)
+        for (i in out.indices) arr[i] = out[i]
+        return arr
+    }
+
+    private fun sampleAt(
+        frame: Int,
+        channel: Int,
+        previous: FloatArray?,
+        input: FloatArray,
+    ): Float {
+        return if (previous != null) {
+            if (frame == 0) {
+                previous[channel]
+            } else {
+                input[(frame - 1) * channels + channel]
+            }
+        } else {
+            input[frame * channels + channel]
+        }
+    }
+}