opuslib 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +107 -0
- package/README.md +140 -9
- package/android/src/main/cpp/opus_jni_wrapper.cpp +34 -0
- package/android/src/main/java/expo/modules/opuslib/AudioProcessor.kt +273 -0
- package/android/src/main/java/expo/modules/opuslib/AudioRecordManager.kt +66 -149
- package/android/src/main/java/expo/modules/opuslib/OpusEncoder.kt +14 -0
- package/android/src/main/java/expo/modules/opuslib/OpuslibModule.kt +47 -5
- package/build/Opuslib.types.d.ts +96 -2
- package/build/Opuslib.types.d.ts.map +1 -1
- package/build/Opuslib.types.js.map +1 -1
- package/build/OpuslibModule.d.ts +28 -1
- package/build/OpuslibModule.d.ts.map +1 -1
- package/build/OpuslibModule.js +25 -0
- package/build/OpuslibModule.js.map +1 -1
- package/build/OpuslibModule.web.d.ts +6 -0
- package/build/OpuslibModule.web.d.ts.map +1 -1
- package/build/OpuslibModule.web.js +6 -0
- package/build/OpuslibModule.web.js.map +1 -1
- package/ios/AudioEngineManager.swift +137 -168
- package/ios/AudioProcessor.swift +246 -0
- package/ios/OpusCtlHelpers.h +8 -0
- package/ios/OpusCtlHelpers.m +4 -0
- package/ios/OpusEncoder.swift +13 -0
- package/ios/OpuslibModule.swift +55 -6
- package/package.json +1 -1
- package/src/Opuslib.types.ts +106 -2
- package/src/OpuslibModule.ts +55 -2
- package/src/OpuslibModule.web.ts +8 -0
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* AudioProcessor - Dedicated encoding thread for Opus encoding and dispatch.
|
|
5
|
+
*
|
|
6
|
+
* The real-time audio tap must never block, and the Opus encoder is not safe to
|
|
7
|
+
* touch from multiple threads. This class isolates all encoder work onto a
|
|
8
|
+
* single serial queue:
|
|
9
|
+
* - The capture thread calls `pushSamples()`, which copies the PCM and posts it
|
|
10
|
+
* to the encoding queue.
|
|
11
|
+
* - All mutable state (pendingSamples, encoder, sequenceNumber, level) is only
|
|
12
|
+
* accessed on the encoding queue, so no locks are needed.
|
|
13
|
+
* - `audioStarted` / `audioEnd` events are emitted from the encoding queue, so
|
|
14
|
+
* `preSkip` / `sequenceNumber` are read without cross-thread risk.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/// A single encoded Opus frame with optional per-frame audio level
|
|
18
|
+
struct EncodedFrame {
|
|
19
|
+
let data: Data
|
|
20
|
+
let audioLevel: Float? // nil when enableAudioLevel is false
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
class AudioProcessor {
|
|
24
|
+
// Dedicated serial queue — all encoder access happens here
|
|
25
|
+
private let queue = DispatchQueue(label: "com.opuslib.encoding", qos: .userInitiated)
|
|
26
|
+
|
|
27
|
+
// All fields below are only accessed on `queue` — no locks needed
|
|
28
|
+
private var opusEncoder: OpusEncoder?
|
|
29
|
+
private var pendingSamples: [Int16] = []
|
|
30
|
+
private let samplesPerFrame: Int
|
|
31
|
+
private let framesPerPacket: Int // how many frames to batch before emitting
|
|
32
|
+
private var packetFrames: [EncodedFrame] = [] // independent Opus packets with per-frame level
|
|
33
|
+
private var sequenceNumber: Int = 0
|
|
34
|
+
private var startTime: Double = 0
|
|
35
|
+
|
|
36
|
+
// Whether to compute per-frame audio level
|
|
37
|
+
private let enableAudioLevel: Bool
|
|
38
|
+
|
|
39
|
+
// Debug file
|
|
40
|
+
private var pcmFileHandle: FileHandle?
|
|
41
|
+
|
|
42
|
+
// Event callbacks (all invoked on the encoding queue)
|
|
43
|
+
// onAudioChunk: (frames, timestamp, sequenceNumber, duration, frameCount)
|
|
44
|
+
private var onAudioChunk: (([EncodedFrame], Double, Int, Double, Int) -> Void)?
|
|
45
|
+
private var onStarted: ((_ timestamp: Double, _ sampleRate: Int, _ channels: Int, _ bitrate: Int, _ frameSize: Double, _ preSkip: Int) -> Void)?
|
|
46
|
+
private var onEnd: ((_ timestamp: Double, _ totalDuration: Double, _ totalPackets: Int) -> Void)?
|
|
47
|
+
|
|
48
|
+
// Configuration (immutable)
|
|
49
|
+
private let config: AudioConfig
|
|
50
|
+
|
|
51
|
+
init(config: AudioConfig) {
|
|
52
|
+
self.config = config
|
|
53
|
+
self.samplesPerFrame = Int(Double(config.sampleRate) * config.frameSize / 1000.0)
|
|
54
|
+
let framesPerCallback = config.framesPerCallback ?? 1
|
|
55
|
+
self.framesPerPacket = max(1, framesPerCallback)
|
|
56
|
+
self.enableAudioLevel = config.enableAudioLevel ?? false
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// MARK: - Public API
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Start: create the Opus encoder on the encoding queue and emit audioStarted.
|
|
63
|
+
* All encoder init + preSkip read happen on the same thread — no cross-thread risk.
|
|
64
|
+
*/
|
|
65
|
+
func start(debugFileURL: URL? = nil) throws {
|
|
66
|
+
var initError: Error?
|
|
67
|
+
queue.sync {
|
|
68
|
+
do {
|
|
69
|
+
self.opusEncoder = try OpusEncoder(
|
|
70
|
+
sampleRate: self.config.sampleRate,
|
|
71
|
+
channels: self.config.channels,
|
|
72
|
+
bitrate: self.config.bitrate,
|
|
73
|
+
frameSizeMs: self.config.frameSize,
|
|
74
|
+
dredDurationMs: self.config.dredDuration ?? 100
|
|
75
|
+
)
|
|
76
|
+
} catch {
|
|
77
|
+
initError = error
|
|
78
|
+
return
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Debug file
|
|
82
|
+
if let url = debugFileURL {
|
|
83
|
+
FileManager.default.createFile(atPath: url.path, contents: nil, attributes: nil)
|
|
84
|
+
self.pcmFileHandle = try? FileHandle(forWritingTo: url)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Emit audioStarted on the encoding queue — preSkip read is safe here
|
|
88
|
+
self.startTime = Date().timeIntervalSince1970 * 1000
|
|
89
|
+
let preSkip = self.opusEncoder?.preSkip ?? 0
|
|
90
|
+
self.onStarted?(
|
|
91
|
+
self.startTime,
|
|
92
|
+
self.config.sampleRate,
|
|
93
|
+
self.config.channels,
|
|
94
|
+
self.config.bitrate,
|
|
95
|
+
self.config.frameSize,
|
|
96
|
+
preSkip
|
|
97
|
+
)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if let error = initError { throw error }
|
|
101
|
+
|
|
102
|
+
print("[AudioProcessor] Started: \(config.sampleRate)Hz, frame=\(samplesPerFrame) samples")
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Push raw PCM samples from the capture thread (copy + post, no shared state).
|
|
107
|
+
*/
|
|
108
|
+
func pushSamples(_ samples: [Int16]) {
|
|
109
|
+
queue.async { [weak self] in
|
|
110
|
+
guard let self = self else { return }
|
|
111
|
+
self.pendingSamples.append(contentsOf: samples)
|
|
112
|
+
self._processFrames()
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Synchronously flush remaining audio, emit audioEnd, and destroy the encoder.
|
|
118
|
+
* All sequenceNumber/encoder access on the encoding queue — no cross-thread risk.
|
|
119
|
+
*/
|
|
120
|
+
func flushAndStop() {
|
|
121
|
+
queue.sync {
|
|
122
|
+
self._flushRemainingFrames()
|
|
123
|
+
|
|
124
|
+
// Emit audioEnd on the encoding queue — sequenceNumber read is safe here
|
|
125
|
+
let stopTime = Date().timeIntervalSince1970 * 1000
|
|
126
|
+
let totalDuration = stopTime - self.startTime
|
|
127
|
+
self.onEnd?(stopTime, totalDuration, self.sequenceNumber)
|
|
128
|
+
|
|
129
|
+
// Destroy the encoder on the same thread that used it
|
|
130
|
+
self.opusEncoder = nil
|
|
131
|
+
self.pendingSamples.removeAll()
|
|
132
|
+
self.pcmFileHandle?.closeFile()
|
|
133
|
+
self.pcmFileHandle = nil
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
print("[AudioProcessor] Flushed and stopped")
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// MARK: - Event callbacks
|
|
140
|
+
|
|
141
|
+
func setOnAudioChunk(_ callback: @escaping ([EncodedFrame], Double, Int, Double, Int) -> Void) {
|
|
142
|
+
self.onAudioChunk = callback
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
func setOnStarted(_ callback: @escaping (_ timestamp: Double, _ sampleRate: Int, _ channels: Int, _ bitrate: Int, _ frameSize: Double, _ preSkip: Int) -> Void) {
|
|
146
|
+
self.onStarted = callback
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
func setOnEnd(_ callback: @escaping (_ timestamp: Double, _ totalDuration: Double, _ totalPackets: Int) -> Void) {
|
|
150
|
+
self.onEnd = callback
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// MARK: - Encoding queue internals (all below only called on `queue`)
|
|
154
|
+
|
|
155
|
+
private func _processFrames() {
|
|
156
|
+
guard let opusEncoder = opusEncoder else { return }
|
|
157
|
+
|
|
158
|
+
while pendingSamples.count >= samplesPerFrame {
|
|
159
|
+
let frameData = Array(pendingSamples.prefix(samplesPerFrame))
|
|
160
|
+
pendingSamples.removeFirst(samplesPerFrame)
|
|
161
|
+
|
|
162
|
+
// Debug PCM file
|
|
163
|
+
if let fileHandle = pcmFileHandle {
|
|
164
|
+
frameData.withUnsafeBufferPointer { ptr in
|
|
165
|
+
let data = Data(bytes: ptr.baseAddress!, count: frameData.count * MemoryLayout<Int16>.size)
|
|
166
|
+
fileHandle.write(data)
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Encode single frame to Opus
|
|
171
|
+
var encodedPacket: Data?
|
|
172
|
+
frameData.withUnsafeBufferPointer { bufferPointer in
|
|
173
|
+
guard let baseAddress = bufferPointer.baseAddress else { return }
|
|
174
|
+
encodedPacket = opusEncoder.encode(pcm: baseAddress, frameSize: samplesPerFrame)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
guard let opusData = encodedPacket, !opusData.isEmpty else {
|
|
178
|
+
print("[AudioProcessor] Failed to encode Opus packet")
|
|
179
|
+
continue
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Per-frame audio level (RMS → dBFS → 0~1)
|
|
183
|
+
var frameLevel: Float? = nil
|
|
184
|
+
if enableAudioLevel {
|
|
185
|
+
var sumSquares: Double = 0.0
|
|
186
|
+
for sample in frameData {
|
|
187
|
+
let s = Double(sample) / 32768.0
|
|
188
|
+
sumSquares += s * s
|
|
189
|
+
}
|
|
190
|
+
let rms = sqrt(sumSquares / Double(frameData.count))
|
|
191
|
+
let dB = 20.0 * log10(max(rms, 1e-10))
|
|
192
|
+
let dbFloor = -35.0
|
|
193
|
+
let dbCeiling = -6.0
|
|
194
|
+
frameLevel = Float(max(0.0, min(1.0, (dB - dbFloor) / (dbCeiling - dbFloor))))
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// Accumulate the encoded frame as an independent packet (no byte concatenation)
|
|
198
|
+
packetFrames.append(EncodedFrame(data: opusData, audioLevel: frameLevel))
|
|
199
|
+
|
|
200
|
+
// Emit once we have enough frames (framesPerCallback)
|
|
201
|
+
if packetFrames.count >= framesPerPacket {
|
|
202
|
+
let timestampMs = Date().timeIntervalSince1970 * 1000
|
|
203
|
+
let frameCount = packetFrames.count
|
|
204
|
+
let duration = Double(frameCount) * config.frameSize
|
|
205
|
+
onAudioChunk?(packetFrames, timestampMs, sequenceNumber, duration, frameCount)
|
|
206
|
+
sequenceNumber += 1
|
|
207
|
+
packetFrames.removeAll()
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private func _flushRemainingFrames() {
|
|
213
|
+
guard let opusEncoder = opusEncoder else { return }
|
|
214
|
+
|
|
215
|
+
// Pad remaining PCM with silence to fill the last frame
|
|
216
|
+
if !pendingSamples.isEmpty && pendingSamples.count < samplesPerFrame {
|
|
217
|
+
pendingSamples.append(contentsOf: [Int16](repeating: 0, count: samplesPerFrame - pendingSamples.count))
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Encode remaining frames
|
|
221
|
+
while pendingSamples.count >= samplesPerFrame {
|
|
222
|
+
let frameData = Array(pendingSamples.prefix(samplesPerFrame))
|
|
223
|
+
pendingSamples.removeFirst(samplesPerFrame)
|
|
224
|
+
|
|
225
|
+
var encodedPacket: Data?
|
|
226
|
+
frameData.withUnsafeBufferPointer { bufferPointer in
|
|
227
|
+
guard let baseAddress = bufferPointer.baseAddress else { return }
|
|
228
|
+
encodedPacket = opusEncoder.encode(pcm: baseAddress, frameSize: samplesPerFrame)
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
guard let opusData = encodedPacket, !opusData.isEmpty else { continue }
|
|
232
|
+
// Flush frames get level 0 (silence-padded)
|
|
233
|
+
packetFrames.append(EncodedFrame(data: opusData, audioLevel: enableAudioLevel ? 0.0 : nil))
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// Flush any remaining frames (even if fewer than framesPerPacket)
|
|
237
|
+
if !packetFrames.isEmpty {
|
|
238
|
+
let timestampMs = Date().timeIntervalSince1970 * 1000
|
|
239
|
+
let frameCount = packetFrames.count
|
|
240
|
+
let duration = Double(frameCount) * config.frameSize
|
|
241
|
+
onAudioChunk?(packetFrames, timestampMs, sequenceNumber, duration, frameCount)
|
|
242
|
+
sequenceNumber += 1
|
|
243
|
+
packetFrames.removeAll()
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
package/ios/OpusCtlHelpers.h
CHANGED
|
@@ -56,6 +56,14 @@ NS_ASSUME_NONNULL_BEGIN
|
|
|
56
56
|
*/
|
|
57
57
|
+ (int)setDtx:(void *)encoder dtx:(int)dtx;
|
|
58
58
|
|
|
59
|
+
/**
|
|
60
|
+
* Get encoder lookahead (pre-skip samples)
|
|
61
|
+
* @param encoder Pointer to OpusEncoder (as void*)
|
|
62
|
+
* @param lookahead Output pointer for lookahead value
|
|
63
|
+
* @return OPUS_OK on success, or negative error code
|
|
64
|
+
*/
|
|
65
|
+
+ (int)getLookahead:(void *)encoder lookahead:(int *)lookahead;
|
|
66
|
+
|
|
59
67
|
@end
|
|
60
68
|
|
|
61
69
|
NS_ASSUME_NONNULL_END
|
package/ios/OpusCtlHelpers.m
CHANGED
package/ios/OpusEncoder.swift
CHANGED
|
@@ -31,6 +31,9 @@ class OpusEncoder {
|
|
|
31
31
|
// Buffer for encoded output
|
|
32
32
|
private let maxPacketSize = 4000 // bytes
|
|
33
33
|
|
|
34
|
+
// Opus lookahead (pre-skip samples) — decoders should skip this many samples at the start
|
|
35
|
+
private(set) var preSkip: Int = 0
|
|
36
|
+
|
|
34
37
|
/**
|
|
35
38
|
* Initialize Opus 1.6.1 encoder with DRED support
|
|
36
39
|
*
|
|
@@ -125,6 +128,15 @@ class OpusEncoder {
|
|
|
125
128
|
print("[OpusEncoder] Warning: Failed to set DTX (error \(result))")
|
|
126
129
|
}
|
|
127
130
|
|
|
131
|
+
// Get encoder lookahead (pre-skip)
|
|
132
|
+
var lookahead: Int32 = 0
|
|
133
|
+
result = Int32(OpusCtlHelpers.getLookahead(encoderPtr, lookahead: &lookahead))
|
|
134
|
+
if result == OPUS_OK {
|
|
135
|
+
self.preSkip = Int(lookahead)
|
|
136
|
+
} else {
|
|
137
|
+
print("[OpusEncoder] Warning: Failed to get lookahead (error \(result))")
|
|
138
|
+
}
|
|
139
|
+
|
|
128
140
|
// Enable DRED (Opus 1.6.1 feature)
|
|
129
141
|
if dredDurationMs > 0 {
|
|
130
142
|
result = Int32(OpusCtlHelpers.setDredDuration(encoderPtr, durationMs: Int32(dredDurationMs)))
|
|
@@ -147,6 +159,7 @@ class OpusEncoder {
|
|
|
147
159
|
- In-band FEC: \(inbandFec)
|
|
148
160
|
- DTX: \(dtx)
|
|
149
161
|
- DRED: \(dredDurationMs)ms
|
|
162
|
+
- Pre-skip: \(preSkip) samples
|
|
150
163
|
""")
|
|
151
164
|
}
|
|
152
165
|
|
package/ios/OpuslibModule.swift
CHANGED
|
@@ -15,7 +15,7 @@ public class OpuslibModule: Module {
|
|
|
15
15
|
Name("Opuslib")
|
|
16
16
|
|
|
17
17
|
// Events
|
|
18
|
-
Events("audioChunk", "amplitude", "error")
|
|
18
|
+
Events("audioChunk", "amplitude", "audioStarted", "audioEnd", "error")
|
|
19
19
|
|
|
20
20
|
// Start streaming method
|
|
21
21
|
AsyncFunction("startStreaming") { (config: AudioConfig) in
|
|
@@ -57,13 +57,46 @@ public class OpuslibModule: Module {
|
|
|
57
57
|
let manager = AudioEngineManager(config: config)
|
|
58
58
|
print("[OpuslibModule] ✅ AudioEngineManager created")
|
|
59
59
|
|
|
60
|
-
// Set up event callbacks
|
|
60
|
+
// Set up event callbacks — audioStarted/audioEnd are emitted from the encoding thread
|
|
61
61
|
print("[OpuslibModule] 🔗 Setting up event callbacks...")
|
|
62
|
-
manager.setOnAudioChunk { [weak self]
|
|
62
|
+
manager.setOnAudioChunk { [weak self] frames, timestamp, sequenceNumber, duration, frameCount in
|
|
63
|
+
// Each frame is an independent Opus packet wrapped in { data, audioLevel? }
|
|
64
|
+
let frameObjects: [[String: Any]] = frames.map { frame in
|
|
65
|
+
var obj: [String: Any] = ["data": frame.data]
|
|
66
|
+
if let level = frame.audioLevel {
|
|
67
|
+
obj["audioLevel"] = level
|
|
68
|
+
}
|
|
69
|
+
return obj
|
|
70
|
+
}
|
|
71
|
+
// `data` is the first frame's packet, kept for backward compatibility with
|
|
72
|
+
// consumers that read a single Opus packet per event.
|
|
73
|
+
let firstData = frames.first?.data ?? Data()
|
|
63
74
|
self?.sendEvent("audioChunk", [
|
|
64
|
-
"data":
|
|
75
|
+
"data": firstData,
|
|
76
|
+
"frames": frameObjects,
|
|
65
77
|
"timestamp": timestamp,
|
|
66
|
-
"sequenceNumber": sequenceNumber
|
|
78
|
+
"sequenceNumber": sequenceNumber,
|
|
79
|
+
"duration": duration,
|
|
80
|
+
"frameCount": frameCount
|
|
81
|
+
])
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
manager.setOnStarted { [weak self] timestamp, sampleRate, channels, bitrate, frameSize, preSkip in
|
|
85
|
+
self?.sendEvent("audioStarted", [
|
|
86
|
+
"timestamp": timestamp,
|
|
87
|
+
"sampleRate": sampleRate,
|
|
88
|
+
"channels": channels,
|
|
89
|
+
"bitrate": bitrate,
|
|
90
|
+
"frameSize": frameSize,
|
|
91
|
+
"preSkip": preSkip
|
|
92
|
+
])
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
manager.setOnEnd { [weak self] timestamp, totalDuration, totalPackets in
|
|
96
|
+
self?.sendEvent("audioEnd", [
|
|
97
|
+
"timestamp": timestamp,
|
|
98
|
+
"totalDuration": totalDuration,
|
|
99
|
+
"totalPackets": totalPackets
|
|
67
100
|
])
|
|
68
101
|
}
|
|
69
102
|
|
|
@@ -83,7 +116,7 @@ public class OpuslibModule: Module {
|
|
|
83
116
|
])
|
|
84
117
|
}
|
|
85
118
|
|
|
86
|
-
// Start audio capture
|
|
119
|
+
// Start audio capture + encoding
|
|
87
120
|
print("[OpuslibModule] 🚀 Calling manager.start()...")
|
|
88
121
|
try manager.start()
|
|
89
122
|
print("[OpuslibModule] ✅ manager.start() completed")
|
|
@@ -99,6 +132,7 @@ public class OpuslibModule: Module {
|
|
|
99
132
|
return
|
|
100
133
|
}
|
|
101
134
|
|
|
135
|
+
// stop() triggers flushAndStop(), which emits audioEnd from the encoding thread
|
|
102
136
|
audioEngineManager?.stop()
|
|
103
137
|
audioEngineManager = nil
|
|
104
138
|
isStreaming = false
|
|
@@ -167,11 +201,26 @@ struct AudioConfig: Record {
|
|
|
167
201
|
@Field var channels: Int = 1
|
|
168
202
|
@Field var bitrate: Int = 24000
|
|
169
203
|
@Field var frameSize: Double = 20.0
|
|
204
|
+
// Retained for backward compatibility; framesPerCallback supersedes it for batching.
|
|
170
205
|
@Field var packetDuration: Double = 20.0
|
|
206
|
+
// Number of Opus frames to batch per audioChunk event (default 1 = one packet per event).
|
|
207
|
+
@Field var framesPerCallback: Int? = 1
|
|
171
208
|
@Field var dredDuration: Int? = 100 // NEW: DRED recovery duration in ms
|
|
172
209
|
@Field var enableAmplitudeEvents: Bool? = false
|
|
173
210
|
@Field var amplitudeEventInterval: Double? = 16.0
|
|
211
|
+
@Field var enableAudioLevel: Bool? = false // Enable per-frame audio level calculation
|
|
174
212
|
@Field var saveDebugAudio: Bool? = false
|
|
213
|
+
@Field var iosAudioSession: IOSAudioSessionConfig? = nil
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* iOS AudioSession configuration (category, mode, options).
|
|
218
|
+
* Maps JS string values to AVAudioSession enums.
|
|
219
|
+
*/
|
|
220
|
+
struct IOSAudioSessionConfig: Record {
|
|
221
|
+
@Field var category: String = "record"
|
|
222
|
+
@Field var mode: String = "measurement"
|
|
223
|
+
@Field var options: [String]? = nil
|
|
175
224
|
}
|
|
176
225
|
|
|
177
226
|
// MARK: - Errors
|
package/package.json
CHANGED
package/src/Opuslib.types.ts
CHANGED
|
@@ -12,26 +12,99 @@ export interface AudioConfig {
|
|
|
12
12
|
frameSize: number
|
|
13
13
|
/** Packet duration in milliseconds (typically 20-100ms) */
|
|
14
14
|
packetDuration: number
|
|
15
|
+
/**
|
|
16
|
+
* Number of independently-encoded Opus frames to batch into a single
|
|
17
|
+
* `audioChunk` event (default 1). Each entry in `AudioChunkEvent.frames` is a
|
|
18
|
+
* complete, independently decodable Opus packet — frames are never
|
|
19
|
+
* concatenated. Batching reduces the number of bridge calls.
|
|
20
|
+
*/
|
|
21
|
+
framesPerCallback?: number
|
|
15
22
|
/** DRED recovery duration in milliseconds (0-100, default 100) - NEW in Opus 1.6 */
|
|
16
23
|
dredDuration?: number
|
|
17
24
|
/** Enable amplitude events for waveform visualization */
|
|
18
25
|
enableAmplitudeEvents?: boolean
|
|
19
26
|
/** Amplitude event interval in milliseconds (default 16) */
|
|
20
27
|
amplitudeEventInterval?: number
|
|
28
|
+
/**
|
|
29
|
+
* Enable per-frame audio level calculation (default false). When enabled,
|
|
30
|
+
* each `OpusFrame` carries an `audioLevel` (0.0 - 1.0) derived from RMS.
|
|
31
|
+
* Disabled by default to save computation.
|
|
32
|
+
*/
|
|
33
|
+
enableAudioLevel?: boolean
|
|
21
34
|
/** Save debug PCM audio to file (development only) */
|
|
22
35
|
saveDebugAudio?: boolean
|
|
36
|
+
/**
|
|
37
|
+
* iOS AudioSession configuration (iOS only; ignored on Android and web).
|
|
38
|
+
* Omit to keep the default recording session
|
|
39
|
+
* (`record` category, `measurement` mode, no options).
|
|
40
|
+
*/
|
|
41
|
+
iosAudioSession?: IOSAudioSessionConfig
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* iOS `AVAudioSession` configuration. iOS only — ignored on Android and web.
|
|
46
|
+
*/
|
|
47
|
+
export interface IOSAudioSessionConfig {
|
|
48
|
+
/**
|
|
49
|
+
* `AVAudioSession.Category`
|
|
50
|
+
* - `record`: pure recording (default)
|
|
51
|
+
* - `playAndRecord`: record and play simultaneously
|
|
52
|
+
* - `playback`: playback only
|
|
53
|
+
* - `ambient`: mix with other audio without interrupting it
|
|
54
|
+
*/
|
|
55
|
+
category: 'record' | 'playAndRecord' | 'playback' | 'ambient'
|
|
56
|
+
/**
|
|
57
|
+
* `AVAudioSession.Mode`
|
|
58
|
+
* - `measurement`: disable system audio processing (default)
|
|
59
|
+
* - `default`: enable system audio processing (AGC, echo cancellation)
|
|
60
|
+
* - `voiceChat`: optimized for voice calls
|
|
61
|
+
* - `spokenAudio`: optimized for spoken content
|
|
62
|
+
*/
|
|
63
|
+
mode: 'default' | 'voiceChat' | 'measurement' | 'spokenAudio'
|
|
64
|
+
/** `AVAudioSession.CategoryOptions` (combinable) */
|
|
65
|
+
options?: (
|
|
66
|
+
| 'mixWithOthers'
|
|
67
|
+
| 'defaultToSpeaker'
|
|
68
|
+
| 'allowBluetooth'
|
|
69
|
+
| 'allowAirPlay'
|
|
70
|
+
| 'allowBluetoothA2DP'
|
|
71
|
+
)[]
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* A single Opus frame — one complete `opus_encode()` output with its own TOC
|
|
76
|
+
* byte, decodable on its own.
|
|
77
|
+
*/
|
|
78
|
+
export interface OpusFrame {
|
|
79
|
+
/** Opus-encoded packet data (independent, decodable) */
|
|
80
|
+
data: ArrayBuffer
|
|
81
|
+
/** Per-frame audio level (0.0 - 1.0). Present only when `enableAudioLevel` is true. */
|
|
82
|
+
audioLevel?: number
|
|
23
83
|
}
|
|
24
84
|
|
|
25
85
|
/**
|
|
26
86
|
* Audio chunk event payload (Opus-encoded data)
|
|
27
87
|
*/
|
|
28
88
|
export interface AudioChunkEvent {
|
|
29
|
-
/**
|
|
89
|
+
/**
|
|
90
|
+
* The first frame's Opus packet, kept for backward compatibility — equivalent
|
|
91
|
+
* to `frames[0].data`. With the default `framesPerCallback` of 1 this is the
|
|
92
|
+
* single packet for the event. Prefer `frames` for new code.
|
|
93
|
+
*/
|
|
30
94
|
data: ArrayBuffer
|
|
95
|
+
/**
|
|
96
|
+
* Independently decodable Opus packets in this event (one per encoded frame).
|
|
97
|
+
* Contains a single entry unless `framesPerCallback` > 1.
|
|
98
|
+
*/
|
|
99
|
+
frames: OpusFrame[]
|
|
31
100
|
/** Timestamp in milliseconds */
|
|
32
101
|
timestamp: number
|
|
33
|
-
/** Sequence number (increments with each
|
|
102
|
+
/** Sequence number (increments with each event) */
|
|
34
103
|
sequenceNumber: number
|
|
104
|
+
/** Duration of all frames in milliseconds (`frameSize * frameCount`) */
|
|
105
|
+
duration: number
|
|
106
|
+
/** Number of Opus frames in this event (= `frames.length`) */
|
|
107
|
+
frameCount: number
|
|
35
108
|
}
|
|
36
109
|
|
|
37
110
|
/**
|
|
@@ -46,6 +119,37 @@ export interface AmplitudeEvent {
|
|
|
46
119
|
timestamp: number
|
|
47
120
|
}
|
|
48
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Audio started event payload. Emitted once when streaming begins.
|
|
124
|
+
*/
|
|
125
|
+
export interface AudioStartedEvent {
|
|
126
|
+
/** Timestamp in milliseconds when streaming started */
|
|
127
|
+
timestamp: number
|
|
128
|
+
/** Actual sample rate in Hz */
|
|
129
|
+
sampleRate: number
|
|
130
|
+
/** Number of channels */
|
|
131
|
+
channels: number
|
|
132
|
+
/** Configured bitrate in bits/second */
|
|
133
|
+
bitrate: number
|
|
134
|
+
/** Frame duration in milliseconds */
|
|
135
|
+
frameSize: number
|
|
136
|
+
/** Opus encoder lookahead in samples — decoders should skip this many samples at the start */
|
|
137
|
+
preSkip: number
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Audio end event payload. Emitted once when streaming stops, after the final
|
|
142
|
+
* buffered audio has been flushed.
|
|
143
|
+
*/
|
|
144
|
+
export interface AudioEndEvent {
|
|
145
|
+
/** Timestamp in milliseconds when streaming stopped */
|
|
146
|
+
timestamp: number
|
|
147
|
+
/** Total session duration in milliseconds */
|
|
148
|
+
totalDuration: number
|
|
149
|
+
/** Total number of packets (audioChunk events) emitted during the session */
|
|
150
|
+
totalPackets: number
|
|
151
|
+
}
|
|
152
|
+
|
|
49
153
|
/**
|
|
50
154
|
* Error event payload
|
|
51
155
|
*/
|
package/src/OpuslibModule.ts
CHANGED
|
@@ -4,6 +4,8 @@ import type {
|
|
|
4
4
|
AudioConfig,
|
|
5
5
|
AudioChunkEvent,
|
|
6
6
|
AmplitudeEvent,
|
|
7
|
+
AudioStartedEvent,
|
|
8
|
+
AudioEndEvent,
|
|
7
9
|
ErrorEvent,
|
|
8
10
|
Subscription,
|
|
9
11
|
} from './Opuslib.types'
|
|
@@ -96,6 +98,14 @@ export default {
|
|
|
96
98
|
* websocket.send(event.data)
|
|
97
99
|
* })
|
|
98
100
|
*
|
|
101
|
+
* // Listen for session lifecycle
|
|
102
|
+
* Opuslib.addListener('audioStarted', (event) => {
|
|
103
|
+
* console.log('Started; decoder pre-skip:', event.preSkip)
|
|
104
|
+
* })
|
|
105
|
+
* Opuslib.addListener('audioEnd', (event) => {
|
|
106
|
+
* console.log(`Ended: ${event.totalPackets} packets in ${event.totalDuration}ms`)
|
|
107
|
+
* })
|
|
108
|
+
*
|
|
99
109
|
* // Listen for errors
|
|
100
110
|
* const errorSub = Opuslib.addListener('error', (event) => {
|
|
101
111
|
* console.error('Error:', event.message)
|
|
@@ -107,8 +117,20 @@ export default {
|
|
|
107
117
|
* ```
|
|
108
118
|
*/
|
|
109
119
|
addListener: ((
|
|
110
|
-
eventName:
|
|
111
|
-
|
|
120
|
+
eventName:
|
|
121
|
+
| 'audioChunk'
|
|
122
|
+
| 'amplitude'
|
|
123
|
+
| 'audioStarted'
|
|
124
|
+
| 'audioEnd'
|
|
125
|
+
| 'error',
|
|
126
|
+
listener: (
|
|
127
|
+
event:
|
|
128
|
+
| AudioChunkEvent
|
|
129
|
+
| AmplitudeEvent
|
|
130
|
+
| AudioStartedEvent
|
|
131
|
+
| AudioEndEvent
|
|
132
|
+
| ErrorEvent,
|
|
133
|
+
) => void,
|
|
112
134
|
): Subscription => (emitter as any).addListener(eventName, listener)) as {
|
|
113
135
|
(
|
|
114
136
|
eventName: 'audioChunk',
|
|
@@ -118,6 +140,14 @@ export default {
|
|
|
118
140
|
eventName: 'amplitude',
|
|
119
141
|
listener: (event: AmplitudeEvent) => void,
|
|
120
142
|
): Subscription
|
|
143
|
+
(
|
|
144
|
+
eventName: 'audioStarted',
|
|
145
|
+
listener: (event: AudioStartedEvent) => void,
|
|
146
|
+
): Subscription
|
|
147
|
+
(
|
|
148
|
+
eventName: 'audioEnd',
|
|
149
|
+
listener: (event: AudioEndEvent) => void,
|
|
150
|
+
): Subscription
|
|
121
151
|
(eventName: 'error', listener: (event: ErrorEvent) => void): Subscription
|
|
122
152
|
},
|
|
123
153
|
|
|
@@ -131,6 +161,29 @@ export default {
|
|
|
131
161
|
listener: (event: AmplitudeEvent) => void,
|
|
132
162
|
): Subscription => (emitter as any).addListener('amplitude', listener),
|
|
133
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Listen for the `audioStarted` event, emitted once when streaming begins.
|
|
166
|
+
* Carries the active config and the Opus encoder `preSkip` (lookahead) so a
|
|
167
|
+
* decoder knows how many samples to skip at the start of the stream.
|
|
168
|
+
*
|
|
169
|
+
* @param listener Event listener callback
|
|
170
|
+
* @returns Subscription object with remove() method
|
|
171
|
+
*/
|
|
172
|
+
addAudioStartedListener: (
|
|
173
|
+
listener: (event: AudioStartedEvent) => void,
|
|
174
|
+
): Subscription => (emitter as any).addListener('audioStarted', listener),
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Listen for the `audioEnd` event, emitted once when streaming stops (after
|
|
178
|
+
* the final buffered audio has been flushed). Carries the session summary.
|
|
179
|
+
*
|
|
180
|
+
* @param listener Event listener callback
|
|
181
|
+
* @returns Subscription object with remove() method
|
|
182
|
+
*/
|
|
183
|
+
addAudioEndListener: (
|
|
184
|
+
listener: (event: AudioEndEvent) => void,
|
|
185
|
+
): Subscription => (emitter as any).addListener('audioEnd', listener),
|
|
186
|
+
|
|
134
187
|
/**
|
|
135
188
|
* Listen for error events
|
|
136
189
|
*
|
package/src/OpuslibModule.web.ts
CHANGED