opuslib 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +107 -0
- package/README.md +140 -9
- package/android/src/main/cpp/opus_jni_wrapper.cpp +34 -0
- package/android/src/main/java/expo/modules/opuslib/AudioProcessor.kt +273 -0
- package/android/src/main/java/expo/modules/opuslib/AudioRecordManager.kt +66 -149
- package/android/src/main/java/expo/modules/opuslib/OpusEncoder.kt +14 -0
- package/android/src/main/java/expo/modules/opuslib/OpuslibModule.kt +47 -5
- package/build/Opuslib.types.d.ts +96 -2
- package/build/Opuslib.types.d.ts.map +1 -1
- package/build/Opuslib.types.js.map +1 -1
- package/build/OpuslibModule.d.ts +28 -1
- package/build/OpuslibModule.d.ts.map +1 -1
- package/build/OpuslibModule.js +25 -0
- package/build/OpuslibModule.js.map +1 -1
- package/build/OpuslibModule.web.d.ts +6 -0
- package/build/OpuslibModule.web.d.ts.map +1 -1
- package/build/OpuslibModule.web.js +6 -0
- package/build/OpuslibModule.web.js.map +1 -1
- package/ios/AudioEngineManager.swift +137 -168
- package/ios/AudioProcessor.swift +246 -0
- package/ios/OpusCtlHelpers.h +8 -0
- package/ios/OpusCtlHelpers.m +4 -0
- package/ios/OpusEncoder.swift +13 -0
- package/ios/OpuslibModule.swift +55 -6
- package/package.json +1 -1
- package/src/Opuslib.types.ts +106 -2
- package/src/OpuslibModule.ts +55 -2
- package/src/OpuslibModule.web.ts +8 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format is based on
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
|
|
5
|
+
follows [Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [0.2.0]
|
|
8
|
+
|
|
9
|
+
This release is **fully backward compatible** — everything below is additive.
|
|
10
|
+
Existing code keeps working unchanged: `audioChunk` events still carry `data`,
|
|
11
|
+
and all the new config fields are optional.
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- **Encoding no longer runs on the real-time audio thread.** Capture and Opus
|
|
16
|
+
encoding now run on separate threads — the capture callback only converts and
|
|
17
|
+
copies PCM, then hands it to a dedicated serial encoding thread that owns all
|
|
18
|
+
encoder state. This removes the iOS crash that could occur when encoding on
|
|
19
|
+
the audio render thread, and keeps the real-time audio path unblocked.
|
|
20
|
+
(iOS: a serial `DispatchQueue`; Android: a `HandlerThread`.)
|
|
21
|
+
- **Flush on stop.** When you call `stopStreaming()`, any buffered tail of audio
|
|
22
|
+
is padded with silence, encoded, and emitted before teardown — so the end of a
|
|
23
|
+
session is no longer dropped.
|
|
24
|
+
|
|
25
|
+
### Added
|
|
26
|
+
|
|
27
|
+
- **`audioStarted` event** — fired once when streaming begins, from the encoding
|
|
28
|
+
thread. Includes the active config and the Opus encoder `preSkip`
|
|
29
|
+
(`OPUS_GET_LOOKAHEAD`) so a decoder knows how many samples to skip at the start
|
|
30
|
+
of the stream.
|
|
31
|
+
- **`audioEnd` event** — fired once when streaming stops (after the flush above),
|
|
32
|
+
with a session summary (`totalDuration`, `totalPackets`).
|
|
33
|
+
- **`framesPerCallback` config** — batch N independently-encoded Opus frames into
|
|
34
|
+
a single `audioChunk` event to reduce bridge calls. Defaults to `1`.
|
|
35
|
+
- **`audioChunk.frames` / `duration` / `frameCount`** — `frames` is an array of
|
|
36
|
+
independent, individually-decodable Opus packets (each with its own TOC byte;
|
|
37
|
+
never concatenated). `audioChunk.data` is retained and equals `frames[0].data`.
|
|
38
|
+
- **`enableAudioLevel` config + `OpusFrame.audioLevel`** — opt-in per-frame audio
|
|
39
|
+
level in `0.0–1.0` (RMS mapped through dBFS). Off by default.
|
|
40
|
+
- **`iosAudioSession` config (iOS only)** — customize the `AVAudioSession`
|
|
41
|
+
category / mode / options (e.g. `playAndRecord`, `defaultToSpeaker`,
|
|
42
|
+
`allowBluetooth`). Ignored on Android and web; omit to keep the default
|
|
43
|
+
recording session.
|
|
44
|
+
- New helper methods `addAudioStartedListener()` and `addAudioEndListener()`,
|
|
45
|
+
plus `audioStarted` / `audioEnd` overloads on `addListener()`.
|
|
46
|
+
- New exported types: `OpusFrame`, `AudioStartedEvent`, `AudioEndEvent`,
|
|
47
|
+
`IOSAudioSessionConfig`.
|
|
48
|
+
|
|
49
|
+
### How to use the new features
|
|
50
|
+
|
|
51
|
+
```typescript
|
|
52
|
+
import Opuslib from 'opuslib'
|
|
53
|
+
|
|
54
|
+
// Lifecycle events
|
|
55
|
+
Opuslib.addAudioStartedListener((e) => {
|
|
56
|
+
// e.preSkip — samples a decoder should skip at the start of the stream
|
|
57
|
+
console.log(`started @ ${e.sampleRate}Hz, preSkip=${e.preSkip}`)
|
|
58
|
+
})
|
|
59
|
+
Opuslib.addAudioEndListener((e) => {
|
|
60
|
+
console.log(`ended: ${e.totalPackets} packets over ${e.totalDuration}ms`)
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
// Per-frame audio level + frame batching
|
|
64
|
+
Opuslib.addListener('audioChunk', (e) => {
|
|
65
|
+
// e.data still works (back-compat) === e.frames[0].data
|
|
66
|
+
for (const frame of e.frames) {
|
|
67
|
+
websocket.send(frame.data) // each frame is an independent Opus packet
|
|
68
|
+
meter(frame.audioLevel) // present only when enableAudioLevel: true
|
|
69
|
+
}
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
await Opuslib.startStreaming({
|
|
73
|
+
sampleRate: 16000,
|
|
74
|
+
channels: 1,
|
|
75
|
+
bitrate: 24000,
|
|
76
|
+
frameSize: 20,
|
|
77
|
+
packetDuration: 100,
|
|
78
|
+
framesPerCallback: 5, // 5 independent frames per audioChunk (80% fewer bridge calls)
|
|
79
|
+
enableAudioLevel: true, // populate frame.audioLevel
|
|
80
|
+
// iOS-only: record + play, route to speaker
|
|
81
|
+
iosAudioSession: {
|
|
82
|
+
category: 'playAndRecord',
|
|
83
|
+
mode: 'default',
|
|
84
|
+
options: ['defaultToSpeaker', 'allowBluetooth'],
|
|
85
|
+
},
|
|
86
|
+
})
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Migration notes
|
|
90
|
+
|
|
91
|
+
Nothing is required. To adopt the new behavior:
|
|
92
|
+
|
|
93
|
+
- Reading `event.data` keeps working. To consume batched frames, switch to
|
|
94
|
+
iterating `event.frames` (with `framesPerCallback: 1`, the default, `frames`
|
|
95
|
+
has exactly one entry equal to `data`).
|
|
96
|
+
- `framesPerCallback` supersedes `packetDuration` for deciding how many frames
|
|
97
|
+
are grouped per event. `packetDuration` remains accepted for compatibility.
|
|
98
|
+
|
|
99
|
+
## [0.1.4]
|
|
100
|
+
|
|
101
|
+
### Changed
|
|
102
|
+
|
|
103
|
+
- Updated the vendored Opus codec from 1.6 to **1.6.1**.
|
|
104
|
+
|
|
105
|
+
## [0.1.2]
|
|
106
|
+
|
|
107
|
+
- Earlier published releases. See the Git history for details.
|
package/README.md
CHANGED
|
@@ -20,10 +20,13 @@ Created as I had a need for real-time voice communication in a React Native app.
|
|
|
20
20
|
- **Opus 1.6.1** - Latest codec version compiled from the [official source](https://opus-codec.org/downloads/)
|
|
21
21
|
- **Low Latency** - Real-time encoding with minimal overhead
|
|
22
22
|
- **Native Performance** - Direct C/C++ integration, no JavaScript encoding
|
|
23
|
+
- **Thread-safe encoding** - Capture and Opus encoding run on separate threads, so the real-time audio thread is never blocked
|
|
24
|
+
- **Audio level metering** - Optional per-frame level (0.0–1.0) via `enableAudioLevel`
|
|
25
|
+
- **Lifecycle events** - `audioStarted` / `audioEnd` with session metadata, plus flush-on-stop so no trailing audio is lost
|
|
23
26
|
- **High Quality** - 24kbps achieves excellent speech quality
|
|
24
27
|
- **Cross-Platform** - iOS and Android with a consistent API
|
|
25
28
|
- **Zero Dependencies** - Self-contained with vendored Opus source
|
|
26
|
-
- **Configurable** - Bitrate, sample rate, frame size
|
|
29
|
+
- **Configurable** - Bitrate, sample rate, frame size, frame batching
|
|
27
30
|
- **Event-Based** - Stream encoded audio chunks via events
|
|
28
31
|
|
|
29
32
|
### Why Opus 1.6.1?
|
|
@@ -141,13 +144,25 @@ interface AudioConfig {
|
|
|
141
144
|
bitrate: number; // Target bitrate in bits/second (e.g., 24000)
|
|
142
145
|
frameSize: number; // Frame duration in ms (2.5, 5, 10, 20, 40, 60)
|
|
143
146
|
packetDuration: number; // Packet duration in ms (multiple of frameSize)
|
|
147
|
+
framesPerCallback?: number; // Opus frames batched per audioChunk event (default: 1)
|
|
144
148
|
dredDuration?: number; // Reserved for future DRED support (default: 0)
|
|
145
149
|
enableAmplitudeEvents?: boolean; // Enable amplitude monitoring (default: false)
|
|
146
150
|
amplitudeEventInterval?: number; // Amplitude update interval in ms (default: 16)
|
|
151
|
+
enableAudioLevel?: boolean; // Per-frame audio level (0.0-1.0) on each OpusFrame (default: false)
|
|
147
152
|
saveDebugAudio?: boolean; // Save raw PCM to a file for debugging (development only)
|
|
153
|
+
iosAudioSession?: { // iOS AVAudioSession config (iOS only; ignored elsewhere)
|
|
154
|
+
category: 'record' | 'playAndRecord' | 'playback' | 'ambient';
|
|
155
|
+
mode: 'default' | 'voiceChat' | 'measurement' | 'spokenAudio';
|
|
156
|
+
options?: Array<'mixWithOthers' | 'defaultToSpeaker' | 'allowBluetooth' | 'allowAirPlay' | 'allowBluetoothA2DP'>;
|
|
157
|
+
};
|
|
148
158
|
}
|
|
149
159
|
```
|
|
150
160
|
|
|
161
|
+
> **Backward compatibility:** `framesPerCallback`, `enableAudioLevel`, and
|
|
162
|
+
> `iosAudioSession` are all optional. Omitting them preserves the previous
|
|
163
|
+
> behavior (one Opus packet per `audioChunk`, no level metering, default iOS
|
|
164
|
+
> recording session).
|
|
165
|
+
|
|
151
166
|
**Recommended Settings for Speech:**
|
|
152
167
|
```typescript
|
|
153
168
|
{
|
|
@@ -189,18 +204,89 @@ Emitted when an encoded Opus packet is ready.
|
|
|
189
204
|
|
|
190
205
|
```typescript
|
|
191
206
|
Opuslib.addListener('audioChunk', (event: AudioChunkEvent) => {
|
|
192
|
-
// event.data: ArrayBuffer -
|
|
207
|
+
// event.data: ArrayBuffer - First frame's Opus packet (back-compat; = frames[0].data)
|
|
208
|
+
// event.frames: OpusFrame[] - Independent Opus packets (one per encoded frame)
|
|
193
209
|
// event.timestamp: number - Capture timestamp in milliseconds
|
|
194
|
-
// event.sequenceNumber: number -
|
|
210
|
+
// event.sequenceNumber: number - Event sequence number (starts at 0)
|
|
211
|
+
// event.duration: number - Total duration in ms (frameSize * frameCount)
|
|
212
|
+
// event.frameCount: number - Number of frames in this event
|
|
213
|
+
|
|
214
|
+
// Each frame is an independent, decodable Opus packet:
|
|
215
|
+
for (const frame of event.frames) {
|
|
216
|
+
websocket.send(frame.data);
|
|
217
|
+
// frame.audioLevel?: number - present only when enableAudioLevel is true
|
|
218
|
+
}
|
|
195
219
|
});
|
|
196
220
|
```
|
|
197
221
|
|
|
198
222
|
**Event Data:**
|
|
199
223
|
```typescript
|
|
224
|
+
interface OpusFrame {
|
|
225
|
+
data: ArrayBuffer; // Independent Opus packet (one opus_encode() output, own TOC byte)
|
|
226
|
+
audioLevel?: number; // Per-frame level 0.0-1.0 (only when enableAudioLevel is true)
|
|
227
|
+
}
|
|
228
|
+
|
|
200
229
|
interface AudioChunkEvent {
|
|
201
|
-
data: ArrayBuffer; //
|
|
230
|
+
data: ArrayBuffer; // First frame's packet — kept for backward compatibility (= frames[0].data)
|
|
231
|
+
frames: OpusFrame[]; // Independent Opus packets (single entry unless framesPerCallback > 1)
|
|
202
232
|
timestamp: number; // Milliseconds since epoch
|
|
203
|
-
sequenceNumber: number; // Incrementing
|
|
233
|
+
sequenceNumber: number; // Incrementing event counter
|
|
234
|
+
duration: number; // Total duration in ms (frameSize * frameCount)
|
|
235
|
+
frameCount: number; // Number of Opus frames (= frames.length)
|
|
236
|
+
}
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
> With the default `framesPerCallback` of 1, `frames` has a single entry and
|
|
240
|
+
> `data === frames[0].data`, so existing `event.data` consumers are unaffected.
|
|
241
|
+
> Frames are **never** concatenated — each is independently decodable.
|
|
242
|
+
|
|
243
|
+
---
|
|
244
|
+
|
|
245
|
+
#### `audioStarted`
|
|
246
|
+
|
|
247
|
+
Emitted once when streaming starts. Carries the active config and the Opus
|
|
248
|
+
encoder `preSkip` (lookahead) so a decoder knows how many samples to skip at the
|
|
249
|
+
beginning of the stream.
|
|
250
|
+
|
|
251
|
+
```typescript
|
|
252
|
+
Opuslib.addAudioStartedListener((event: AudioStartedEvent) => {
|
|
253
|
+
console.log(`Started: ${event.sampleRate}Hz, preSkip=${event.preSkip}`);
|
|
254
|
+
});
|
|
255
|
+
// or: Opuslib.addListener('audioStarted', (event) => { ... })
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
**Event Data:**
|
|
259
|
+
```typescript
|
|
260
|
+
interface AudioStartedEvent {
|
|
261
|
+
timestamp: number; // Milliseconds since epoch
|
|
262
|
+
sampleRate: number; // Actual sample rate in Hz
|
|
263
|
+
channels: number; // Number of channels
|
|
264
|
+
bitrate: number; // Configured bitrate in bits/second
|
|
265
|
+
frameSize: number; // Frame duration in milliseconds
|
|
266
|
+
preSkip: number; // Encoder lookahead in samples (decoder should skip these)
|
|
267
|
+
}
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
#### `audioEnd`
|
|
273
|
+
|
|
274
|
+
Emitted once when streaming stops, after the final buffered audio has been
|
|
275
|
+
flushed (the trailing partial frame is padded with silence so no audio is lost).
|
|
276
|
+
|
|
277
|
+
```typescript
|
|
278
|
+
Opuslib.addAudioEndListener((event: AudioEndEvent) => {
|
|
279
|
+
console.log(`Ended: ${event.totalDuration}ms, ${event.totalPackets} packets`);
|
|
280
|
+
});
|
|
281
|
+
// or: Opuslib.addListener('audioEnd', (event) => { ... })
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
**Event Data:**
|
|
285
|
+
```typescript
|
|
286
|
+
interface AudioEndEvent {
|
|
287
|
+
timestamp: number; // Milliseconds since epoch
|
|
288
|
+
totalDuration: number; // Total session duration in milliseconds
|
|
289
|
+
totalPackets: number; // Total audioChunk events emitted during the session
|
|
204
290
|
}
|
|
205
291
|
```
|
|
206
292
|
|
|
@@ -254,7 +340,39 @@ interface ErrorEvent {
|
|
|
254
340
|
### iOS
|
|
255
341
|
|
|
256
342
|
- **Minimum iOS Version:** 15.1+
|
|
257
|
-
- **Audio Session:**
|
|
343
|
+
- **Audio Session:** Defaults to the `record` category with `measurement` mode (pure recording, system audio processing disabled). Override it per-session with the optional `iosAudioSession` config — e.g. for simultaneous playback or Bluetooth/speaker routing:
|
|
344
|
+
```typescript
|
|
345
|
+
await Opuslib.startStreaming({
|
|
346
|
+
sampleRate: 24000, channels: 1, bitrate: 24000, frameSize: 20, packetDuration: 100,
|
|
347
|
+
iosAudioSession: {
|
|
348
|
+
category: 'playAndRecord', // record + play at the same time
|
|
349
|
+
mode: 'default', // enable AGC / echo cancellation
|
|
350
|
+
options: ['defaultToSpeaker', 'allowBluetooth'],
|
|
351
|
+
},
|
|
352
|
+
});
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
| `category` | Behavior |
|
|
356
|
+
|------------|----------|
|
|
357
|
+
| `record` | Pure recording (default) |
|
|
358
|
+
| `playAndRecord` | Record and play simultaneously |
|
|
359
|
+
| `playback` | Playback only |
|
|
360
|
+
| `ambient` | Mix with other audio without interrupting it |
|
|
361
|
+
|
|
362
|
+
| `mode` | Behavior |
|
|
363
|
+
|--------|----------|
|
|
364
|
+
| `measurement` | Disable system audio processing (default) |
|
|
365
|
+
| `default` | Enable AGC, echo cancellation, etc. |
|
|
366
|
+
| `voiceChat` | Optimized for voice calls |
|
|
367
|
+
| `spokenAudio` | Optimized for spoken content |
|
|
368
|
+
|
|
369
|
+
| `options[]` | Behavior |
|
|
370
|
+
|-------------|----------|
|
|
371
|
+
| `mixWithOthers` | Allow mixing with other audio apps |
|
|
372
|
+
| `defaultToSpeaker` | Route to speaker instead of earpiece |
|
|
373
|
+
| `allowBluetooth` | Allow Bluetooth HFP devices |
|
|
374
|
+
| `allowAirPlay` | Allow AirPlay output |
|
|
375
|
+
| `allowBluetoothA2DP` | Allow Bluetooth A2DP (high-quality audio) |
|
|
258
376
|
- **Permissions:** Add to `app.json`:
|
|
259
377
|
```json
|
|
260
378
|
{
|
|
@@ -355,20 +473,33 @@ cd ..
|
|
|
355
473
|
|
|
356
474
|
## Technical Details
|
|
357
475
|
|
|
358
|
-
|
|
476
|
+
Capture and encoding run on **separate threads**. The capture thread only reads
|
|
477
|
+
PCM, converts it, and copies the samples onto a dedicated serial encoding
|
|
478
|
+
thread; all Opus encoder state and `opus_encode()` calls happen there. This
|
|
479
|
+
keeps the real-time audio thread unblocked and avoids encoding on it.
|
|
480
|
+
|
|
481
|
+
```
|
|
482
|
+
Capture thread Encoding thread (serial)
|
|
483
|
+
| read PCM (AVAudioEngine / AudioRecord)
|
|
484
|
+
| convert + copy ----- post ----> append to pending buffer
|
|
485
|
+
| while (>= one frame) opus_encode() -> frame
|
|
486
|
+
| per-frame audioLevel (if enabled)
|
|
487
|
+
| batch framesPerCallback -> emit audioChunk
|
|
488
|
+
| (stop) ------------- flush ---> pad silence + encode tail -> emit audioEnd
|
|
489
|
+
```
|
|
359
490
|
|
|
360
491
|
**iOS:**
|
|
361
492
|
- AVAudioEngine for audio capture (48kHz PCM)
|
|
362
493
|
- Custom resampler (48kHz → 16kHz)
|
|
494
|
+
- Dedicated serial `DispatchQueue` for Opus encoding and event dispatch
|
|
363
495
|
- Opus 1.6.1 encoder (native C via Swift)
|
|
364
496
|
- Objective-C wrapper for CTL operations
|
|
365
497
|
- Event emission via Expo modules
|
|
366
498
|
|
|
367
499
|
**Android:**
|
|
368
500
|
- AudioRecord for audio capture (16kHz PCM)
|
|
501
|
+
- Dedicated `HandlerThread` for Opus encoding and event dispatch
|
|
369
502
|
- JNI wrapper for Opus 1.6.1 C library
|
|
370
|
-
- Background thread for recording loop
|
|
371
|
-
- Kotlin coroutines for async operations
|
|
372
503
|
- Event emission via Expo modules
|
|
373
504
|
|
|
374
505
|
### Opus Build Configuration
|
|
@@ -144,6 +144,40 @@ Java_expo_modules_opuslib_OpusEncoder_nativeEncode(
|
|
|
144
144
|
return result;
|
|
145
145
|
}
|
|
146
146
|
|
|
147
|
+
/**
|
|
148
|
+
* Get Opus encoder lookahead (pre-skip samples)
|
|
149
|
+
*
|
|
150
|
+
* Decoders should skip this many samples at the start of the stream to account
|
|
151
|
+
* for the encoder's algorithmic delay.
|
|
152
|
+
*
|
|
153
|
+
* @param env JNI environment
|
|
154
|
+
* @param thiz Java object instance
|
|
155
|
+
* @param encoder_ptr Encoder pointer from nativeCreate
|
|
156
|
+
* @return Lookahead in samples, or 0 on failure
|
|
157
|
+
*/
|
|
158
|
+
JNIEXPORT jint JNICALL
|
|
159
|
+
Java_expo_modules_opuslib_OpusEncoder_nativeGetLookahead(
|
|
160
|
+
JNIEnv *env,
|
|
161
|
+
jobject thiz,
|
|
162
|
+
jlong encoder_ptr
|
|
163
|
+
) {
|
|
164
|
+
OpusEncoder *encoder = reinterpret_cast<OpusEncoder*>(encoder_ptr);
|
|
165
|
+
if (!encoder) {
|
|
166
|
+
LOGE("Encoder pointer is null");
|
|
167
|
+
return 0;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
opus_int32 lookahead = 0;
|
|
171
|
+
int result = opus_encoder_ctl(encoder, OPUS_GET_LOOKAHEAD(&lookahead));
|
|
172
|
+
if (result != OPUS_OK) {
|
|
173
|
+
LOGE("Failed to get lookahead: error %d", result);
|
|
174
|
+
return 0;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
LOGI("Opus lookahead (pre-skip): %d samples", lookahead);
|
|
178
|
+
return static_cast<jint>(lookahead);
|
|
179
|
+
}
|
|
180
|
+
|
|
147
181
|
/**
|
|
148
182
|
* Destroy Opus encoder and free resources
|
|
149
183
|
*
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
package expo.modules.opuslib
|
|
2
|
+
|
|
3
|
+
import android.os.Handler
|
|
4
|
+
import android.os.HandlerThread
|
|
5
|
+
import android.util.Log
|
|
6
|
+
import java.io.File
|
|
7
|
+
import java.io.FileOutputStream
|
|
8
|
+
import java.util.concurrent.CountDownLatch
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* A single encoded Opus frame with optional per-frame audio level.
|
|
12
|
+
*/
|
|
13
|
+
data class EncodedFrame(
|
|
14
|
+
val data: ByteArray,
|
|
15
|
+
val audioLevel: Float? // null when enableAudioLevel is false
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* AudioProcessor - Dedicated encoding thread for Opus 1.6.1 encoding and dispatch.
|
|
20
|
+
*
|
|
21
|
+
* The capture thread (AudioRecordManager) only reads PCM; this class owns the
|
|
22
|
+
* encoder and runs all encoding work on a single dedicated thread:
|
|
23
|
+
* - Owns a HandlerThread + Handler (a serial work queue)
|
|
24
|
+
* - The capture thread calls pushSamples() which copies the data and posts it
|
|
25
|
+
* to this thread, so the capture loop never blocks on encoding
|
|
26
|
+
* - All mutable state (pendingSamples, encoder, sequenceNumber, packet buffer)
|
|
27
|
+
* is only touched on the HandlerThread — no locks needed
|
|
28
|
+
* - audioStarted/audioEnd events are emitted from this thread too, so preSkip
|
|
29
|
+
* and sequenceNumber are read without any cross-thread risk
|
|
30
|
+
*/
|
|
31
|
+
class AudioProcessor(private val config: AudioConfig) {
|
|
32
|
+
companion object {
|
|
33
|
+
private const val TAG = "AudioProcessor"
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Encoding thread + its serial work queue
|
|
37
|
+
private var handlerThread: HandlerThread? = null
|
|
38
|
+
private var handler: Handler? = null
|
|
39
|
+
|
|
40
|
+
// All fields below are only accessed on handlerThread — no locks needed
|
|
41
|
+
private var opusEncoder: OpusEncoder? = null
|
|
42
|
+
private val pendingSamples = mutableListOf<Short>()
|
|
43
|
+
private val samplesPerFrame: Int = (config.sampleRate * config.frameSize / 1000.0).toInt()
|
|
44
|
+
private val framesPerPacket: Int = Math.max(1, config.framesPerCallback)
|
|
45
|
+
private var packetFrames = mutableListOf<EncodedFrame>() // independent Opus packets with per-frame level
|
|
46
|
+
private var sequenceNumber: Int = 0
|
|
47
|
+
private var startTime: Double = 0.0
|
|
48
|
+
|
|
49
|
+
// Whether to compute per-frame audio level
|
|
50
|
+
private val enableAudioLevel: Boolean = config.enableAudioLevel
|
|
51
|
+
|
|
52
|
+
// Debug file output
|
|
53
|
+
private var pcmFileOutputStream: FileOutputStream? = null
|
|
54
|
+
|
|
55
|
+
// Event callbacks (all invoked on encoding thread)
|
|
56
|
+
// onAudioChunk: (frames, timestamp, sequenceNumber, duration, frameCount)
|
|
57
|
+
private var onAudioChunk: ((List<EncodedFrame>, Double, Int, Double, Int) -> Unit)? = null
|
|
58
|
+
private var onStarted: ((timestamp: Double, sampleRate: Int, channels: Int, bitrate: Int, frameSize: Double, preSkip: Int) -> Unit)? = null
|
|
59
|
+
private var onEnd: ((timestamp: Double, totalDuration: Double, totalPackets: Int) -> Unit)? = null
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Start the encoding thread, create the Opus encoder, emit audioStarted.
|
|
63
|
+
* Encoder init + preSkip read happen on the same thread — no cross-thread risk.
|
|
64
|
+
*/
|
|
65
|
+
fun start(debugFile: File? = null) {
|
|
66
|
+
val thread = HandlerThread("OpusEncodingThread").apply { start() }
|
|
67
|
+
handlerThread = thread
|
|
68
|
+
handler = Handler(thread.looper)
|
|
69
|
+
|
|
70
|
+
val ready = CountDownLatch(1)
|
|
71
|
+
handler!!.post {
|
|
72
|
+
// Init encoder on encoding thread
|
|
73
|
+
_initEncoder()
|
|
74
|
+
|
|
75
|
+
// Debug file
|
|
76
|
+
if (debugFile != null) {
|
|
77
|
+
try {
|
|
78
|
+
pcmFileOutputStream = FileOutputStream(debugFile)
|
|
79
|
+
Log.d(TAG, "Debug PCM file: ${debugFile.absolutePath}")
|
|
80
|
+
} catch (e: Exception) {
|
|
81
|
+
Log.e(TAG, "Failed to create debug file: ${e.message}")
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Emit audioStarted on encoding thread — preSkip read is safe here
|
|
86
|
+
startTime = System.currentTimeMillis().toDouble()
|
|
87
|
+
val preSkip = opusEncoder?.preSkip ?: 0
|
|
88
|
+
onStarted?.invoke(
|
|
89
|
+
startTime,
|
|
90
|
+
config.sampleRate,
|
|
91
|
+
config.channels,
|
|
92
|
+
config.bitrate,
|
|
93
|
+
config.frameSize,
|
|
94
|
+
preSkip
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
ready.countDown()
|
|
98
|
+
}
|
|
99
|
+
ready.await()
|
|
100
|
+
|
|
101
|
+
Log.d(TAG, "Started: ${config.sampleRate}Hz, ${config.channels}ch, frame=$samplesPerFrame samples")
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Push raw PCM samples from the capture thread (copy + post, no shared state).
|
|
106
|
+
*/
|
|
107
|
+
fun pushSamples(samples: ShortArray, count: Int) {
|
|
108
|
+
val buf = samples.copyOf(count)
|
|
109
|
+
handler?.post {
|
|
110
|
+
for (s in buf) {
|
|
111
|
+
pendingSamples.add(s)
|
|
112
|
+
}
|
|
113
|
+
_processFrames()
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Synchronously flush remaining audio, emit audioEnd, destroy encoder, stop thread.
|
|
119
|
+
* All sequenceNumber/encoder access happens on the encoding thread — no cross-thread risk.
|
|
120
|
+
*/
|
|
121
|
+
fun flushAndStop() {
|
|
122
|
+
val h = handler ?: return
|
|
123
|
+
val done = CountDownLatch(1)
|
|
124
|
+
h.post {
|
|
125
|
+
_flushRemainingFrames()
|
|
126
|
+
|
|
127
|
+
// Emit audioEnd on encoding thread — sequenceNumber read is safe here
|
|
128
|
+
val stopTime = System.currentTimeMillis().toDouble()
|
|
129
|
+
val totalDuration = stopTime - startTime
|
|
130
|
+
onEnd?.invoke(stopTime, totalDuration, sequenceNumber)
|
|
131
|
+
|
|
132
|
+
// Destroy encoder on the same thread that used it
|
|
133
|
+
opusEncoder?.destroy()
|
|
134
|
+
opusEncoder = null
|
|
135
|
+
pendingSamples.clear()
|
|
136
|
+
pcmFileOutputStream?.close()
|
|
137
|
+
pcmFileOutputStream = null
|
|
138
|
+
done.countDown()
|
|
139
|
+
}
|
|
140
|
+
done.await()
|
|
141
|
+
|
|
142
|
+
handlerThread?.quitSafely()
|
|
143
|
+
handlerThread = null
|
|
144
|
+
handler = null
|
|
145
|
+
|
|
146
|
+
Log.d(TAG, "Flushed and stopped")
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// MARK: - Event callback setters
|
|
150
|
+
|
|
151
|
+
fun setOnAudioChunk(callback: (List<EncodedFrame>, Double, Int, Double, Int) -> Unit) {
|
|
152
|
+
this.onAudioChunk = callback
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
fun setOnStarted(callback: (timestamp: Double, sampleRate: Int, channels: Int, bitrate: Int, frameSize: Double, preSkip: Int) -> Unit) {
|
|
156
|
+
this.onStarted = callback
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
fun setOnEnd(callback: (timestamp: Double, totalDuration: Double, totalPackets: Int) -> Unit) {
|
|
160
|
+
this.onEnd = callback
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// MARK: - Encoding thread internals (all below only called on HandlerThread)
|
|
164
|
+
|
|
165
|
+
private fun _initEncoder() {
|
|
166
|
+
opusEncoder = OpusEncoder(
|
|
167
|
+
sampleRate = config.sampleRate,
|
|
168
|
+
channels = config.channels,
|
|
169
|
+
bitrate = config.bitrate,
|
|
170
|
+
frameSizeMs = config.frameSize,
|
|
171
|
+
dredDurationMs = config.dredDuration
|
|
172
|
+
)
|
|
173
|
+
Log.d(TAG, "Opus encoder created, preSkip=${opusEncoder?.preSkip}")
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private fun _processFrames() {
|
|
177
|
+
val encoder = opusEncoder ?: return
|
|
178
|
+
|
|
179
|
+
while (pendingSamples.size >= samplesPerFrame) {
|
|
180
|
+
val frameData = ShortArray(samplesPerFrame)
|
|
181
|
+
for (i in 0 until samplesPerFrame) {
|
|
182
|
+
frameData[i] = pendingSamples.removeAt(0)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Debug PCM file
|
|
186
|
+
pcmFileOutputStream?.let { fos ->
|
|
187
|
+
val bytes = ByteArray(frameData.size * 2)
|
|
188
|
+
java.nio.ByteBuffer.wrap(bytes).order(java.nio.ByteOrder.LITTLE_ENDIAN).asShortBuffer().put(frameData)
|
|
189
|
+
fos.write(bytes)
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Encode single frame to Opus
|
|
193
|
+
val opusData = try {
|
|
194
|
+
encoder.encode(frameData, samplesPerFrame)
|
|
195
|
+
} catch (e: Exception) {
|
|
196
|
+
Log.e(TAG, "Opus encode error: ${e.message}")
|
|
197
|
+
continue
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (opusData == null || opusData.isEmpty()) {
|
|
201
|
+
Log.w(TAG, "Opus encode returned null/empty")
|
|
202
|
+
continue
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Per-frame audio level (RMS → dBFS → 0~1)
|
|
206
|
+
var frameLevel: Float? = null
|
|
207
|
+
if (enableAudioLevel) {
|
|
208
|
+
var sumSquares = 0.0
|
|
209
|
+
for (sample in frameData) {
|
|
210
|
+
val s = sample.toDouble() / 32768.0
|
|
211
|
+
sumSquares += s * s
|
|
212
|
+
}
|
|
213
|
+
val rms = Math.sqrt(sumSquares / frameData.size)
|
|
214
|
+
val dB = 20.0 * Math.log10(Math.max(rms, 1e-10))
|
|
215
|
+
val dbFloor = -35.0
|
|
216
|
+
val dbCeiling = -6.0
|
|
217
|
+
frameLevel = Math.max(0.0, Math.min(1.0, (dB - dbFloor) / (dbCeiling - dbFloor))).toFloat()
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Accumulate encoded frame as an independent packet (no byte concatenation)
|
|
221
|
+
packetFrames.add(EncodedFrame(data = opusData, audioLevel = frameLevel))
|
|
222
|
+
|
|
223
|
+
// Emit when we have enough frames (framesPerCallback)
|
|
224
|
+
if (packetFrames.size >= framesPerPacket) {
|
|
225
|
+
val timestampMs = System.currentTimeMillis().toDouble()
|
|
226
|
+
val frameCount = packetFrames.size
|
|
227
|
+
val duration = frameCount * config.frameSize
|
|
228
|
+
onAudioChunk?.invoke(packetFrames.toList(), timestampMs, sequenceNumber, duration, frameCount)
|
|
229
|
+
sequenceNumber++
|
|
230
|
+
packetFrames.clear()
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private fun _flushRemainingFrames() {
|
|
236
|
+
val encoder = opusEncoder ?: return
|
|
237
|
+
|
|
238
|
+
// Pad remaining PCM with silence to fill the last frame
|
|
239
|
+
if (pendingSamples.isNotEmpty() && pendingSamples.size < samplesPerFrame) {
|
|
240
|
+
while (pendingSamples.size < samplesPerFrame) {
|
|
241
|
+
pendingSamples.add(0)
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Encode remaining frames
|
|
246
|
+
while (pendingSamples.size >= samplesPerFrame) {
|
|
247
|
+
val frameData = ShortArray(samplesPerFrame)
|
|
248
|
+
for (i in 0 until samplesPerFrame) {
|
|
249
|
+
frameData[i] = pendingSamples.removeAt(0)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
val opusData = try {
|
|
253
|
+
encoder.encode(frameData, samplesPerFrame)
|
|
254
|
+
} catch (e: Exception) {
|
|
255
|
+
continue
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
if (opusData == null || opusData.isEmpty()) continue
|
|
259
|
+
// Flush frames get level 0 (silence-padded)
|
|
260
|
+
packetFrames.add(EncodedFrame(data = opusData, audioLevel = if (enableAudioLevel) 0.0f else null))
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Flush any remaining frames (even if fewer than framesPerPacket)
|
|
264
|
+
if (packetFrames.isNotEmpty()) {
|
|
265
|
+
val timestampMs = System.currentTimeMillis().toDouble()
|
|
266
|
+
val frameCount = packetFrames.size
|
|
267
|
+
val duration = frameCount * config.frameSize
|
|
268
|
+
onAudioChunk?.invoke(packetFrames.toList(), timestampMs, sequenceNumber, duration, frameCount)
|
|
269
|
+
sequenceNumber++
|
|
270
|
+
packetFrames.clear()
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|