@edkimmel/expo-audio-stream 0.2.0

package/.eslintrc.js

@@ -0,0 +1,5 @@
+module.exports = {
+  root: true,
+  extends: ['universe/native', 'universe/web'],
+  ignorePatterns: ['build'],
+};

package/.yarnrc.yml

@@ -0,0 +1,8 @@
+yarnPath: .yarn/releases/yarn-4.1.1.cjs
+nodeLinker: node-modules
+nmMode: hardlinks-local
+
+npmScopes:
+  edkimmel:
+    npmPublishRegistry: ${NPM_PUBLISH_REGISTRY:-https://npm.pkg.github.com}
+    npmAuthToken: ${NODE_AUTH_TOKEN:-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}

package/NATIVE_EVENTS.md

@@ -0,0 +1,270 @@
+# Native-to-JS Events Reference
+
+Complete catalog of events emitted from native code (Android/iOS) to JavaScript,
+including payload shapes, trigger conditions, and recommended JS responses.
+
+---
+
+## Recording Events
+
+### `AudioData`
+
+Emitted at the configured interval during microphone recording. Also doubles as
+the recording error channel.
+
+| Field | Type | Notes |
+|---|---|---|
+| `encoded` | `string` | Base64-encoded PCM audio |
+| `deltaSize` | `number` | Bytes in this chunk |
+| `position` | `number` | Position in ms from start of recording |
+| `totalSize` | `number` | Cumulative bytes recorded |
+| `soundLevel` | `number?` | Power level in dB (-160 when silent) |
+| `streamUuid` | `string` | Unique ID for this recording stream |
+| `fileUri` | `string` | Always `""` (file I/O removed) |
+| `lastEmittedSize` | `number` | Previous `totalSize` value |
+| `mimeType` | `string` | e.g. `"audio/wav"` |
+
+**Error variant** (same event name, different shape):
+
+| Field | Type | Notes |
+|---|---|---|
+| `error` | `string` | Error code: `READ_ERROR`, `RECORDING_CRASH` |
+| `errorMessage` | `string` | Human-readable description |
+| `streamUuid` | `string` | Stream that errored |
+
+**Platform:** Android, iOS
+
+**JS response:**
+- Forward the base64 PCM to your STT pipeline or WebSocket.
+- Use `soundLevel` for VAD or UI visualisation.
+- Check for the `error` field before assuming the payload is audio data.
+  On error, stop the conversation turn or retry.
+
+---
+
+### `DeviceReconnected`
+
+Fired when an audio device is connected or disconnected (Bluetooth SCO/A2DP,
+wired headset/headphones, USB headset).
+
+| Field | Type | Notes |
+|---|---|---|
+| `reason` | `string` | `"newDeviceAvailable"` \| `"oldDeviceUnavailable"` \| `"unknown"` |
+
+**Platform:** Android, iOS
+
+**JS response:**
+- `oldDeviceUnavailable`: headset/BT just disconnected. Decide whether to stop
+  recording, switch to speaker, or show a UI prompt.
+- `newDeviceAvailable`: a device was plugged in. May want to re-route audio or
+  update UI to reflect the new output device.
+
+---
+
+## Legacy Playback Events (AudioPlaybackManager)
+
+### `SoundStarted`
+
+Fired once per turn when the first audio chunk begins playback.
+
+| Field | Type | Notes |
+|---|---|---|
+| *(empty payload)* | | |
+
+**Platform:** Android, iOS
+
+**JS response:**
+- Show a "speaking" indicator in the UI.
+- If half-duplex, mute the microphone.
+
+---
+
+### `SoundChunkPlayed`
+
+Fired after each audio chunk finishes playback.
+
+| Field | Type | Notes |
+|---|---|---|
+| `isFinal` | `boolean` | `true` if this was the last chunk in the sequence |
+
+**Platform:** Android, iOS
+
+**JS response:**
+- `isFinal === false`: progress notification. Use for UI or ignore.
+- `isFinal === true`: the turn is done playing. Resume mic recording, send next
+  turn, update UI to "listening".
+
+---
+
+## Pipeline Events (Android only)
+
+These events are emitted by `AudioPipeline` via `PipelineIntegration`. They are
+not available on iOS, which still uses the legacy `SoundStarted`/`SoundChunkPlayed`
+path.
+
+### `PipelineStateChanged`
+
+Fired on every pipeline state transition.
+
+| Field | Type | Notes |
+|---|---|---|
+| `state` | `string` | `"idle"` \| `"connecting"` \| `"streaming"` \| `"draining"` \| `"error"` |
+
+**JS response:**
+- Drive your UI state machine. `streaming` = show playback indicator.
+  `idle` = ready for next turn. `error` = surface to user or trigger reconnect.
+
+---
+
+### `PipelinePlaybackStarted`
+
+Fired once per turn when the jitter buffer has primed and real audio is hitting
+the speaker.
+
+| Field | Type | Notes |
+|---|---|---|
+| `turnId` | `string` | Conversation turn identifier |
+
+**JS response:**
+- This is the "time-to-first-audio" measurement point.
+- Update UI to "speaking".
+
+---
+
+### `PipelineError`
+
+Fired on any pipeline error condition.
+
+| Field | Type | Notes |
+|---|---|---|
+| `code` | `string` | See table below |
+| `message` | `string` | Human-readable description |
+
+| Code | Meaning | Recommended action |
+|---|---|---|
+| `CONNECT_FAILED` | AudioTrack or JitterBuffer creation failed | Retry `pipelineConnect()` |
+| `DECODE_ERROR` | Base64 decode failed on incoming chunk | Bad data from server -- log and drop the chunk |
+| `WRITE_ERROR` | `AudioTrack.write()` returned an error code | `pipelineDisconnect()` then `pipelineConnect()` |
+| `NOT_CONNECTED` | `pushAudio` called before `connect` | Ensure connection is established before pushing |
+
+---
+
+### `PipelineZombieDetected`
+
+Fired when the AudioTrack's `playbackHeadPosition` has not moved for 5+ seconds
+during `streaming` or `draining` state.
+
+| Field | Type | Notes |
+|---|---|---|
+| `playbackHead` | `number` | Last known playback head position |
+| `stalledMs` | `number` | Milliseconds since head last moved |
+
+**JS response:**
+- The AudioTrack is stuck. Tear down and reconnect:
+  `pipelineDisconnect()` then `pipelineConnect()`.
+- This is a device-level issue (some OEMs, Bluetooth routing glitches).
+
+---
+
+### `PipelineUnderrun`
+
+Fired when the jitter buffer runs dry during playback. Debounced: only fires
+when the cumulative count increases, not on every silence frame.
+
+| Field | Type | Notes |
+|---|---|---|
+| `count` | `number` | Total underrun count for this turn |
+
+**JS response:**
+- A single underrun during initial priming is normal.
+- Repeated underruns during `streaming` mean the server or JS bridge is not
+  delivering audio fast enough. Log for diagnostics. If `count` climbs quickly,
+  consider increasing `targetBufferMs`.
+- No immediate corrective action is typically needed.
+
+---
+
+### `PipelineDrained`
+
+Fired when the pipeline has played all buffered audio after `markEndOfStream`.
+The pipeline transitions from `draining` to `idle`.
+
+| Field | Type | Notes |
+|---|---|---|
+| `turnId` | `string` | The turn that just finished playing |
+
+**JS response:**
+- This is your "turn complete" signal. Resume mic recording, send next request,
+  update UI to "listening".
+- Equivalent to `SoundChunkPlayed` with `isFinal=true` but for the pipeline path.
+
+---
+
+### `PipelineAudioFocusLost`
+
+Fired when another app takes audio focus (phone call, navigation, music).
+The pipeline continues writing silence to keep the AudioTrack alive.
+
+| Field | Type | Notes |
+|---|---|---|
+| *(empty payload)* | | |
+
+**JS response:**
+- Decide whether to pause the conversation, mute the mic via `toggleSilence(isSilent)`,
+  or show a "paused" UI.
+- The pipeline will resume playing real audio automatically when focus returns.
+
+---
+
+### `PipelineAudioFocusResumed`
+
+Fired when audio focus is regained (`AUDIOFOCUS_GAIN`).
+
+| Field | Type | Notes |
+|---|---|---|
+| *(empty payload)* | | |
+
+**JS response:**
+- If you paused the conversation or muted the mic on focus loss, undo that now.
+- Audio that arrived during the focus-loss window was still buffered in the
+  jitter buffer (up to its capacity) and will play out normally.
+
+---
+
+## System Event Handling Gaps
+
+The following system events are **not currently handled** by native code. JS
+cannot respond to them because no event is emitted.
+
+| Gap | Impact | Notes |
+|---|---|---|
+| Phone call interruptions | No `TelephonyManager` / `READ_PHONE_STATE` listener on Android. Partially covered by audio focus loss on the pipeline side; recording side is unprotected. | On iOS, phone calls trigger `interruptionNotification`, but the `.began` handler is a no-op. |
+| App lifecycle (background/foreground) | No `OnPause`/`OnResume`/`OnStop` handling on Android. No `willResignActive`/`didBecomeActive` on iOS. | Recording continues in background until the system kills it. No foreground service. |
+| `ACTION_AUDIO_BECOMING_NOISY` | Headphones unplugged mid-playback could route audio to the speaker unexpectedly. | Android only. Not registered. |
+| Runtime permission revocation | If mic permission is revoked while recording, `AudioRecord.read()` returns errors. Detected by consecutive error counting but no proactive event. | Both platforms. |
+| Media button events | No `MediaSession` or remote command center integration. | Both platforms. |
+| Do Not Disturb | No detection or adaptation. | Both platforms. Low priority. |
+
+---
+
+## TypeScript Types
+
+All event payload types are defined in `src/events.ts` and
+`src/pipeline/types.ts`. Pipeline events can be subscribed to via:
+
+```typescript
+import { Pipeline } from "expo-audio-stream";
+
+Pipeline.subscribe("PipelineDrained", (event) => {
+  console.log("Turn complete:", event.turnId);
+});
+```
+
+Recording and legacy playback events use the helpers in `src/events.ts`:
+
+```typescript
+import { addAudioEventListener, addSoundChunkPlayedListener } from "expo-audio-stream";
+
+addAudioEventListener(async (event) => { /* ... */ });
+addSoundChunkPlayedListener(async (event) => { /* ... */ });
+```

package/README.md

@@ -0,0 +1,289 @@
+# @edkimmel/expo-audio-stream
+
+Native audio recording and low-latency playback for Expo/React Native. Designed for real-time voice AI applications: microphone capture, chunked PCM playback, and a jitter-buffered native pipeline for streaming audio from AI backends.
+
+## Install
+
+```bash
+npx expo install @edkimmel/expo-audio-stream
+```
+
+## Quick Start
+
+### Microphone Recording
+
+```typescript
+import { ExpoPlayAudioStream } from "@edkimmel/expo-audio-stream";
+
+const { recordingResult, subscription } =
+  await ExpoPlayAudioStream.startMicrophone({
+    sampleRate: 16000,
+    channels: 1,
+    encoding: "pcm_16bit",
+    interval: 100,
+    onAudioStream: async (event) => {
+      // event.data: base64-encoded PCM chunk
+      // event.soundLevel: current mic level (dB)
+      sendToBackend(event.data);
+    },
+  });
+
+// Later:
+await ExpoPlayAudioStream.stopMicrophone();
+subscription?.remove();
+```
+
+### Chunked Playback (playSound)
+
+For playing base64-encoded PCM audio in a queue with turn management:
+
+```typescript
+import {
+  ExpoPlayAudioStream,
+  EncodingTypes,
+} from "@edkimmel/expo-audio-stream";
+
+await ExpoPlayAudioStream.setSoundConfig({
+  sampleRate: 24000,
+  playbackMode: "conversation",
+});
+
+// Enqueue chunks as they arrive
+await ExpoPlayAudioStream.playSound(
+  base64Chunk,
+  "turn-1",
+  EncodingTypes.PCM_S16LE
+);
+
+// Listen for playback completion
+const sub = ExpoPlayAudioStream.subscribeToSoundChunkPlayed(async (e) => {
+  if (e.isFinal) console.log("Turn finished playing");
+});
+```
+
+### Native Pipeline (recommended for AI voice streaming)
+
+The `Pipeline` class provides jitter-buffered, low-latency playback with a native write thread. Use this for streaming audio from AI backends over WebSockets.
+
+```typescript
+import { Pipeline } from "@edkimmel/expo-audio-stream";
+
+// Connect with desired config
+const result = await Pipeline.connect({
+  sampleRate: 24000,
+  channelCount: 1,
+  targetBufferMs: 80,
+});
+
+// Subscribe to events
+const errorSub = Pipeline.onError((err) => {
+  console.error(`Pipeline error: ${err.code} - ${err.message}`);
+});
+
+const focusSub = Pipeline.onAudioFocus(({ focused }) => {
+  if (!focused) {
+    // Another app took audio focus; re-request audio on regain
+  }
+});
+
+// Hot path: push audio synchronously from WebSocket handler
+ws.onmessage = (msg) => {
+  Pipeline.pushAudioSync({
+    audio: msg.data, // base64 PCM16 LE
+    turnId: currentTurnId,
+    isFirstChunk: isFirst,
+    isLastChunk: isLast,
+  });
+};
+
+// On new turn, invalidate stale audio
+Pipeline.invalidateTurn({ turnId: newTurnId });
+
+// Tear down
+await Pipeline.disconnect();
+errorSub.remove();
+focusSub.remove();
+```
+
+## API Reference
+
+### ExpoPlayAudioStream
+
+All methods are static.
+
+#### Lifecycle
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `destroy()` | `void` | Release all resources. Resets internal state on both platforms. |
+
+#### Permissions
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `requestPermissionsAsync()` | `Promise<PermissionResult>` | Prompt the user for microphone permission. |
+| `getPermissionsAsync()` | `Promise<PermissionResult>` | Check the current microphone permission status. |
+
+#### Microphone
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `startMicrophone(config)` | `Promise<{ recordingResult, subscription? }>` | Start mic capture. Audio is delivered as base64 PCM via `onAudioStream` or `subscribeToAudioEvents`. |
+| `stopMicrophone()` | `Promise<AudioRecording \| null>` | Stop mic capture and return recording metadata. |
+| `toggleSilence(isSilent)` | `void` | Mute/unmute the mic stream without stopping the session. Silenced frames are zero-filled. |
+| `promptMicrophoneModes()` | `void` | (iOS only) Show the system voice isolation picker (iOS 15+). |
+
+#### Sound Playback
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `playSound(audio, turnId, encoding?)` | `Promise<void>` | Enqueue a base64 PCM chunk for playback. |
+| `stopSound()` | `Promise<void>` | Stop playback and clear the queue. |
+| `setSoundConfig(config)` | `Promise<void>` | Update playback sample rate and mode. |
+
+#### Event Subscriptions
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `subscribeToAudioEvents(callback)` | `Subscription` | Receive `AudioDataEvent` during mic capture. |
+| `subscribeToSoundChunkPlayed(callback)` | `Subscription` | Notified when a chunk finishes playing. `isFinal` is true when the queue drains. |
+| `subscribe(eventName, callback)` | `Subscription` | Generic event listener for any module event. |
+
+### Pipeline
+
+All methods are static. The pipeline manages its own native write thread, jitter buffer, and audio focus.
+
+#### Lifecycle
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `connect(options?)` | `Promise<ConnectPipelineResult>` | Create the native audio track, jitter buffer, and write thread. Config is immutable per session. |
+| `disconnect()` | `Promise<void>` | Tear down the pipeline and release all native resources. |
+
+#### Audio Push
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `pushAudio(options)` | `Promise<void>` | Push base64 PCM16 LE audio (async, with error propagation). |
+| `pushAudioSync(options)` | `boolean` | Push audio synchronously. No Promise overhead -- use in WebSocket `onmessage` for minimum latency. Returns `false` on failure. |
+
+#### Turn Management
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `invalidateTurn(options)` | `Promise<void>` | Discard buffered audio for the old turn. The jitter buffer is reset. |
+
+#### State & Telemetry
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `getState()` | `PipelineState` | Current state: `idle`, `connecting`, `streaming`, `draining`, or `error`. |
+| `getTelemetry()` | `PipelineTelemetry` | Snapshot of buffer levels, push counts, write loops, underruns, etc. |
+
+#### Event Subscriptions
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `subscribe(eventName, listener)` | `EventSubscription` | Type-safe subscription to any pipeline event. |
+| `onError(listener)` | `{ remove }` | Convenience: handles both `PipelineError` and `PipelineZombieDetected`. |
+| `onAudioFocus(listener)` | `{ remove }` | Convenience: `{ focused: true/false }` on audio focus changes. |
+
+## Configuration Types
+
+### RecordingConfig
+
+```typescript
+interface RecordingConfig {
+  sampleRate?: 16000 | 24000 | 44100 | 48000;
+  channels?: 1 | 2;
+  encoding?: "pcm_32bit" | "pcm_16bit" | "pcm_8bit";
+  interval?: number; // ms between audio data emissions (default 1000)
+  onAudioStream?: (event: AudioDataEvent) => Promise<void>;
+}
+```
+
+### SoundConfig
+
+```typescript
+interface SoundConfig {
+  sampleRate?: 16000 | 24000 | 44100 | 48000;
+  playbackMode?: "regular" | "voiceProcessing" | "conversation";
+  useDefault?: boolean; // reset to defaults
+}
+```
+
+### ConnectPipelineOptions
+
+```typescript
+interface ConnectPipelineOptions {
+  sampleRate?: number;     // default 24000
+  channelCount?: number;   // default 1 (mono)
+  targetBufferMs?: number; // ms to buffer before priming gate opens (default 80)
+}
+```
+
+### PushPipelineAudioOptions
+
+```typescript
+interface PushPipelineAudioOptions {
+  audio: string;           // base64-encoded PCM 16-bit signed LE
+  turnId: string;
+  isFirstChunk?: boolean;  // resets jitter buffer
+  isLastChunk?: boolean;   // marks end-of-stream, begins drain
+}
+```
+
+## Events
+
+### Core Events
+
+| Event | Payload | Description |
+|-------|---------|-------------|
+| `AudioData` | `{ encoded, position, deltaSize, totalSize, soundLevel, ... }` | Emitted during mic capture at the configured interval. |
+| `SoundChunkPlayed` | `{ isFinal: boolean }` | A queued chunk finished playing. `isFinal` when the queue is empty. |
+| `SoundStarted` | (none) | Playback began for a new turn. |
+| `DeviceReconnected` | `{ reason }` | Audio route changed (headphones, Bluetooth, etc). |
+
+### Pipeline Events
+
+| Event | Payload | Description |
+|-------|---------|-------------|
+| `PipelineStateChanged` | `{ state }` | Pipeline state transition. |
+| `PipelinePlaybackStarted` | `{ turnId }` | Priming gate opened, audio is now audible. |
+| `PipelineError` | `{ code, message }` | Non-recoverable error. |
+| `PipelineZombieDetected` | `{ playbackHead, stalledMs }` | Audio track stalled. |
+| `PipelineUnderrun` | `{ count }` | Jitter buffer underrun (silence inserted). |
+| `PipelineDrained` | `{ turnId }` | All buffered audio for the turn has been played. |
+| `PipelineAudioFocusLost` | (empty) | Another app took audio focus. |
+| `PipelineAudioFocusResumed` | (empty) | Audio focus regained. |
+
+## Constants
+
+```typescript
+import {
+  EncodingTypes,           // { PCM_F32LE: "pcm_f32le", PCM_S16LE: "pcm_s16le" }
+  PlaybackModes,           // { REGULAR, VOICE_PROCESSING, CONVERSATION }
+  AudioEvents,             // { AudioData, SoundChunkPlayed, SoundStarted, DeviceReconnected }
+  SuspendSoundEventTurnId, // "suspend-sound-events" -- suppresses playback events
+} from "@edkimmel/expo-audio-stream";
+```
+
+## Platform Notes
+
+### iOS
+
+- Uses `AVAudioEngine` with `AVAudioPlayerNode` for sound playback and pipeline audio.
+- Microphone capture via `AVAudioEngine.inputNode` tap.
+- Audio session configured as `.playAndRecord` with `.voiceChat` mode.
+- Voice processing (AEC/noise reduction) available via `voiceProcessing` and `conversation` playback modes.
+- `promptMicrophoneModes()` exposes the iOS 15+ system voice isolation picker.
+
+### Android
+
+- Uses `AudioTrack` (float PCM, `MODE_STREAM`) for sound playback.
+- Microphone capture via `AudioRecord` with `VOICE_RECOGNITION` source for far-field mic gain.
+- AEC, noise suppression, and AGC applied via `AudioEffectsManager`.
+
+## License
+
+MIT

package/android/build.gradle

@@ -0,0 +1,92 @@
+apply plugin: 'com.android.library'
+apply plugin: 'kotlin-android'
+apply plugin: 'maven-publish'
+
+group = 'expo.modules.audiostream'
+version = '0.1.0'
+
+buildscript {
+  def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle")
+  if (expoModulesCorePlugin.exists()) {
+    apply from: expoModulesCorePlugin
+    applyKotlinExpoModulesCorePlugin()
+  }
+
+  // Simple helper that allows the root project to override versions declared by this library.
+  ext.safeExtGet = { prop, fallback ->
+    rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
+  }
+
+  // Ensures backward compatibility
+  ext.getKotlinVersion = {
+    if (ext.has("kotlinVersion")) {
+      ext.kotlinVersion()
+    } else {
+      ext.safeExtGet("kotlinVersion", "2.0.21")
+    }
+  }
+
+  repositories {
+    mavenCentral()
+  }
+
+  dependencies {
+    classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:${getKotlinVersion()}")
+  }
+}
+
+afterEvaluate {
+  publishing {
+    publications {
+      release(MavenPublication) {
+        from components.release
+      }
+    }
+    repositories {
+      maven {
+        url = mavenLocal().url
+      }
+    }
+  }
+}
+
+android {
+  compileSdkVersion safeExtGet("compileSdkVersion", 35)
+
+  def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION
+  if (agpVersion.tokenize('.')[0].toInteger() < 8) {
+    compileOptions {
+      sourceCompatibility JavaVersion.VERSION_11
+      targetCompatibility JavaVersion.VERSION_11
+    }
+
+    kotlinOptions {
+      jvmTarget = JavaVersion.VERSION_11.majorVersion
+    }
+  }
+
+  namespace "expo.modules.audiostream"
+  defaultConfig {
+    minSdkVersion safeExtGet("minSdkVersion", 24)
+    targetSdkVersion safeExtGet("targetSdkVersion", 35)
+    versionCode 1
+    versionName "0.1.0"
+  }
+  lint {
+    abortOnError false
+  }
+  publishing {
+    singleVariant("release") {
+      withSourcesJar()
+    }
+  }
+}
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  implementation project(':expo-modules-core')
+  implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${getKotlinVersion()}"
+}

package/android/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <uses-permission android:name="android.permission.RECORD_AUDIO"/>
+    <uses-permission android:name="android.permission.INTERNET"/>
+</manifest>