@parastud/react-native-vban 0.1.0

package/README.md

@@ -0,0 +1,87 @@
+# @parastud/react-native-vban
+
+VBAN (VB-Audio Network) audio-over-IP for **React Native / Expo**. Stream audio
+to and from a PC running **Voicemeeter** (or any VBAN endpoint) over your local
+network — no server, just UDP.
+
+- **Native receive** — a UDP socket + `AudioTrack` playback run entirely on a
+  native thread (with a large socket buffer), so you don't drop packets waiting
+  on the JS bridge.
+- **Native send** — `AudioRecord` capture → VBAN packetization → UDP, also fully
+  native and paced by the audio clock, so packets are evenly spaced.
+- **Device (system) audio capture** — capture the phone's own playback via
+  Android `MediaProjection` + `AudioPlaybackCapture` (Android 10+).
+- **Pure-TypeScript VBAN codec** — encode/decode packet headers and convert PCM,
+  usable on its own.
+
+> **Platform:** Android only for now. 16-bit PCM audio streams.
+
+## Install
+
+```sh
+npm install @parastud/react-native-vban
+```
+
+This is an [Expo module](https://docs.expo.dev/modules/) — it autolinks. You need
+a **development build** (not Expo Go). After installing, rebuild the native app:
+
+```sh
+npx expo run:android
+```
+
+The module adds the `FOREGROUND_SERVICE_MEDIA_PROJECTION` permission and a
+`mediaProjection` foreground service. Make sure your app also declares
+`RECORD_AUDIO` and `INTERNET`.
+
+## Usage
+
+### Receive a stream (play it on the phone)
+
+```ts
+import { NativeVbanReceiver } from '@parastud/react-native-vban';
+
+const rx = new NativeVbanReceiver();
+await rx.start({ port: 6980 }, (stats) => {
+  console.log(stats.streamName, stats.level, 'dropped', stats.dropped);
+});
+// ... later
+await rx.stop();
+```
+
+Point Voicemeeter's outgoing VBAN stream at the phone's IP on that port.
+
+### Send audio to the PC
+
+```ts
+import { NativeVbanSender, isDeviceCaptureSupported } from '@parastud/react-native-vban';
+
+const tx = new NativeVbanSender();
+await tx.start(
+  {
+    host: '192.168.0.119', // the PC's IP
+    port: 6980,
+    streamName: 'Stream1',
+    sampleRate: 48000,
+    channels: 2,
+    source: 'mic', // or 'device' for system audio (shows a capture consent dialog)
+  },
+  (level) => console.log('level', level),
+);
+// ... later
+await tx.stop();
+```
+
+`source: 'device'` requires Android's MediaProjection consent (the OS
+"start recording/casting" dialog). Only audio is captured — nothing of the
+screen is read or sent — but the consent prompt is mandatory and cannot be
+suppressed.
+
+### Pure VBAN codec
+
+```ts
+import { encodePacket, decodeHeader, float32ToPcm, VbanDataType } from '@parastud/react-native-vban';
+```
+
+## License
+
+MIT

package/android/build.gradle

@@ -0,0 +1,15 @@
+plugins {
+  id 'com.android.library'
+  id 'expo-module-gradle-plugin'
+}
+
+group = 'expo.modules.systemaudiocapture'
+version = '0.0.1'
+
+android {
+  namespace "expo.modules.systemaudiocapture"
+  defaultConfig {
+    versionCode 1
+    versionName '0.0.1'
+  }
+}

package/android/src/main/AndroidManifest.xml

@@ -0,0 +1,12 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+  <!-- Required (API 34+) to run a mediaProjection foreground service; ignored on older versions. -->
+  <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
+
+  <application>
+    <service
+      android:name=".SystemAudioCaptureService"
+      android:exported="false"
+      android:foregroundServiceType="mediaProjection" />
+  </application>
+</manifest>

package/android/src/main/java/expo/modules/systemaudiocapture/SystemAudioCaptureModule.kt

@@ -0,0 +1,57 @@
+package expo.modules.systemaudiocapture
+
+import android.app.Activity
+import android.content.Context
+import android.media.projection.MediaProjectionManager
+import android.os.Build
+import expo.modules.kotlin.Promise
+import expo.modules.kotlin.modules.Module
+import expo.modules.kotlin.modules.ModuleDefinition
+
+/**
+ * Handles the MediaProjection consent flow for capturing device (playback)
+ * audio. The granted projection token is stashed on
+ * [SystemAudioCaptureService], which VbanSenderModule then uses to start a
+ * native device-audio VBAN send. This module does no capturing itself.
+ */
+class SystemAudioCaptureModule : Module() {
+  private val requestCode = 0xB1D0
+  private var permissionPromise: Promise? = null
+
+  override fun definition() = ModuleDefinition {
+    Name("SystemAudioCapture")
+
+    Function("isSupported") {
+      Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q
+    }
+
+    AsyncFunction("requestPermission") { promise: Promise ->
+      val activity = appContext.currentActivity
+      if (activity == null) {
+        promise.reject("NO_ACTIVITY", "No current activity to request permission from", null)
+        return@AsyncFunction
+      }
+      if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
+        promise.reject("UNSUPPORTED", "System audio capture requires Android 10+", null)
+        return@AsyncFunction
+      }
+      permissionPromise = promise
+      val mpm = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+      activity.startActivityForResult(mpm.createScreenCaptureIntent(), requestCode)
+    }
+
+    OnActivityResult { _, payload ->
+      if (payload.requestCode == requestCode) {
+        val promise = permissionPromise
+        permissionPromise = null
+        if (payload.resultCode == Activity.RESULT_OK && payload.data != null) {
+          SystemAudioCaptureService.resultCode = payload.resultCode
+          SystemAudioCaptureService.resultData = payload.data
+          promise?.resolve(true)
+        } else {
+          promise?.resolve(false)
+        }
+      }
+    }
+  }
+}

package/android/src/main/java/expo/modules/systemaudiocapture/SystemAudioCaptureService.kt

@@ -0,0 +1,151 @@
+package expo.modules.systemaudiocapture
+
+import android.app.Notification
+import android.app.NotificationChannel
+import android.app.NotificationManager
+import android.app.Service
+import android.content.Context
+import android.content.Intent
+import android.content.pm.ServiceInfo
+import android.media.AudioAttributes
+import android.media.AudioFormat
+import android.media.AudioPlaybackCaptureConfiguration
+import android.media.AudioRecord
+import android.media.projection.MediaProjection
+import android.media.projection.MediaProjectionManager
+import android.os.Build
+import android.os.IBinder
+
+/**
+ * Foreground service that captures the device's playback audio via
+ * MediaProjection + AudioPlaybackCapture (Android 10+) and streams it out as
+ * VBAN through a [VbanSendEngine] — entirely in native code. A foreground
+ * service of type `mediaProjection` must be running for the projection to stay
+ * valid, which is why the capture + send live here.
+ */
+class SystemAudioCaptureService : Service() {
+  private var mediaProjection: MediaProjection? = null
+  private var engine: VbanSendEngine? = null
+
+  companion object {
+    const val ACTION_START = "expo.modules.systemaudiocapture.START"
+    const val ACTION_STOP = "expo.modules.systemaudiocapture.STOP"
+
+    // MediaProjection consent result (set by SystemAudioCaptureModule).
+    var resultCode: Int = 0
+    var resultData: Intent? = null
+
+    // VBAN send config (set by VbanSenderModule before starting).
+    var host: String = ""
+    var port: Int = 6980
+    var streamName: String = "Stream1"
+    var sampleRate: Int = 48000
+    var channels: Int = 2
+
+    // Callbacks into the module while a session is active.
+    var onStats: ((Float, Long) -> Unit)? = null
+    var onErr: ((String) -> Unit)? = null
+
+    private const val CHANNEL_ID = "vban_system_audio"
+    private const val NOTIF_ID = 8123
+  }
+
+  override fun onBind(intent: Intent?): IBinder? = null
+
+  override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
+    if (intent?.action == ACTION_STOP) {
+      stopCapture()
+      stopSelf()
+      return START_NOT_STICKY
+    }
+    startForegroundNotification()
+    try {
+      startCapture()
+    } catch (e: Exception) {
+      onErr?.invoke(e.message ?: "System audio capture failed to start")
+      stopCapture()
+      stopSelf()
+    }
+    return START_NOT_STICKY
+  }
+
+  private fun startForegroundNotification() {
+    val nm = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+      nm.createNotificationChannel(
+        NotificationChannel(CHANNEL_ID, "VBAN audio capture", NotificationManager.IMPORTANCE_LOW)
+      )
+    }
+    val notification: Notification = Notification.Builder(this, CHANNEL_ID)
+      .setContentTitle("VBAN")
+      .setContentText("Streaming device audio over the network")
+      .setSmallIcon(android.R.drawable.ic_btn_speak_now)
+      .build()
+
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+      startForeground(NOTIF_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION)
+    } else {
+      startForeground(NOTIF_ID, notification)
+    }
+  }
+
+  private fun startCapture() {
+    if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
+      throw IllegalStateException("System audio capture requires Android 10 or newer")
+    }
+    val data = resultData ?: throw IllegalStateException("No screen-capture permission granted")
+
+    val mpm = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+    val projection = mpm.getMediaProjection(resultCode, data)
+      ?: throw IllegalStateException("Could not obtain MediaProjection")
+    projection.registerCallback(object : MediaProjection.Callback() {
+      override fun onStop() {
+        stopCapture()
+        stopSelf()
+      }
+    }, null)
+    mediaProjection = projection
+
+    val config = AudioPlaybackCaptureConfiguration.Builder(projection)
+      .addMatchingUsage(AudioAttributes.USAGE_MEDIA)
+      .addMatchingUsage(AudioAttributes.USAGE_GAME)
+      .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN)
+      .build()
+
+    val channelMask =
+      if (channels == 1) AudioFormat.CHANNEL_IN_MONO else AudioFormat.CHANNEL_IN_STEREO
+    val format = AudioFormat.Builder()
+      .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+      .setSampleRate(sampleRate)
+      .setChannelMask(channelMask)
+      .build()
+
+    val minBuf = AudioRecord.getMinBufferSize(sampleRate, channelMask, AudioFormat.ENCODING_PCM_16BIT)
+    val bufSize = if (minBuf > 0) minBuf * 2 else sampleRate * channels
+
+    val record = AudioRecord.Builder()
+      .setAudioFormat(format)
+      .setBufferSizeInBytes(bufSize)
+      .setAudioPlaybackCaptureConfig(config)
+      .build()
+
+    engine = VbanSendEngine(
+      record, host, port, streamName, sampleRate, channels,
+      onStats = { level, packets -> onStats?.invoke(level, packets) },
+      onError = { msg -> onErr?.invoke(msg) },
+    )
+    engine!!.start()
+  }
+
+  private fun stopCapture() {
+    engine?.stop()
+    engine = null
+    try { mediaProjection?.stop() } catch (_: Exception) {}
+    mediaProjection = null
+  }
+
+  override fun onDestroy() {
+    stopCapture()
+    super.onDestroy()
+  }
+}

package/android/src/main/java/expo/modules/systemaudiocapture/VbanReceiverModule.kt

@@ -0,0 +1,216 @@
+package expo.modules.systemaudiocapture
+
+import android.media.AudioAttributes
+import android.media.AudioFormat
+import android.media.AudioTrack
+import expo.modules.kotlin.Promise
+import expo.modules.kotlin.modules.Module
+import expo.modules.kotlin.modules.ModuleDefinition
+import java.net.DatagramPacket
+import java.net.DatagramSocket
+import java.net.InetSocketAddress
+
+/**
+ * Native VBAN receiver: binds a UDP socket (with a large receive buffer), parses
+ * VBAN audio packets, and writes 16-bit PCM straight to an AudioTrack — all on a
+ * background thread. This keeps the whole hot path in native code so we never
+ * lose packets to a slow JS bridge. Only throttled stats reach JS for the meter.
+ *
+ * The blocking AudioTrack write paces the loop: audio arrives and plays at the
+ * same sample rate, so the socket's buffer only has to absorb network jitter.
+ */
+class VbanReceiverModule : Module() {
+  @Volatile private var running = false
+  private var thread: Thread? = null
+  private var socket: DatagramSocket? = null
+
+  companion object {
+    private val SR = intArrayOf(
+      6000, 12000, 24000, 48000, 96000, 192000, 384000,
+      8000, 16000, 32000, 64000, 128000, 256000, 512000,
+      11025, 22050, 44100, 88200, 176400, 352800, 705600,
+    )
+  }
+
+  override fun definition() = ModuleDefinition {
+    Name("VbanNativeReceiver")
+    Events("onStats", "onError")
+
+    AsyncFunction("start") { port: Int, filter: String?, promise: Promise ->
+      if (running) {
+        promise.resolve(null)
+        return@AsyncFunction
+      }
+      try {
+        startReceive(port, filter)
+        promise.resolve(null)
+      } catch (e: Exception) {
+        promise.reject("START_FAILED", e.message, e)
+      }
+    }
+
+    AsyncFunction("stop") { promise: Promise ->
+      stopReceive()
+      promise.resolve(null)
+    }
+
+    OnDestroy { stopReceive() }
+  }
+
+  private fun startReceive(port: Int, filter: String?) {
+    val sock = DatagramSocket(null)
+    sock.reuseAddress = true
+    try { sock.receiveBufferSize = 4 * 1024 * 1024 } catch (_: Exception) {}
+    sock.bind(InetSocketAddress(port))
+    socket = sock
+    running = true
+
+    thread = Thread {
+      val buf = ByteArray(2048)
+      val packet = DatagramPacket(buf, buf.size)
+      var track: AudioTrack? = null
+      var trackSr = 0
+      var trackCh = 0
+      var lastFrame = -1L
+      var dropped = 0L
+      var packets = 0L
+      var peak = 0f
+      var lastEmit = System.currentTimeMillis()
+      var streamName = ""
+      var address = ""
+
+      try {
+        while (running) {
+          packet.length = buf.size
+          try {
+            sock.receive(packet)
+          } catch (e: Exception) {
+            if (running) continue else break
+          }
+          val len = packet.length
+          if (len < 28) continue
+
+          // 'VBAN' magic
+          if (buf[0].toInt() != 0x56 || buf[1].toInt() != 0x42 ||
+            buf[2].toInt() != 0x41 || buf[3].toInt() != 0x4E
+          ) continue
+
+          val formatSR = buf[4].toInt() and 0xFF
+          if ((formatSR and 0xE0) != 0x00) continue // audio sub-protocol only
+          val srIndex = formatSR and 0x1F
+          if (srIndex >= SR.size) continue
+          val sampleRate = SR[srIndex]
+          val samplesPerFrame = (buf[5].toInt() and 0xFF) + 1
+          val channels = (buf[6].toInt() and 0xFF) + 1
+          val formatBit = buf[7].toInt() and 0xFF
+          if ((formatBit and 0xF0) != 0x00) continue // PCM codec only
+          if ((formatBit and 0x07) != 0x01) continue // 16-bit PCM only
+
+          val sb = StringBuilder()
+          for (i in 8 until 24) {
+            val c = buf[i].toInt() and 0xFF
+            if (c == 0) break
+            sb.append(c.toChar())
+          }
+          val name = sb.toString()
+          if (!filter.isNullOrEmpty() && name != filter) continue
+
+          val frameCount = (buf[24].toInt() and 0xFF).toLong() or
+            ((buf[25].toInt() and 0xFF).toLong() shl 8) or
+            ((buf[26].toInt() and 0xFF).toLong() shl 16) or
+            ((buf[27].toInt() and 0xFF).toLong() shl 24)
+
+          val payloadBytes = samplesPerFrame * channels * 2
+          if (28 + payloadBytes > len) continue
+
+          if (track == null || trackSr != sampleRate || trackCh != channels) {
+            track?.let { try { it.stop(); it.release() } catch (_: Exception) {} }
+            val channelMask =
+              if (channels == 1) AudioFormat.CHANNEL_OUT_MONO else AudioFormat.CHANNEL_OUT_STEREO
+            val minBuf = AudioTrack.getMinBufferSize(
+              sampleRate, channelMask, AudioFormat.ENCODING_PCM_16BIT
+            )
+            val bufSize = if (minBuf > 0) minBuf * 4 else sampleRate * channels
+            track = AudioTrack.Builder()
+              .setAudioAttributes(
+                AudioAttributes.Builder()
+                  .setUsage(AudioAttributes.USAGE_MEDIA)
+                  .setContentType(AudioAttributes.CONTENT_TYPE_MUSIC)
+                  .build()
+              )
+              .setAudioFormat(
+                AudioFormat.Builder()
+                  .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+                  .setSampleRate(sampleRate)
+                  .setChannelMask(channelMask)
+                  .build()
+              )
+              .setBufferSizeInBytes(bufSize)
+              .setTransferMode(AudioTrack.MODE_STREAM)
+              .build()
+            track!!.play()
+            trackSr = sampleRate
+            trackCh = channels
+            lastFrame = -1L
+          }
+
+          if (lastFrame >= 0) {
+            val gap = (frameCount - lastFrame - 1) and 0xFFFFFFFFL
+            if (gap in 1..999) dropped += gap
+          }
+          lastFrame = frameCount
+          packets++
+          streamName = name
+          address = packet.address?.hostAddress ?: ""
+
+          // Peak level over the packet's samples for the meter.
+          var i = 28
+          val end = 28 + payloadBytes
+          var localPeak = 0
+          while (i + 1 < end) {
+            val raw = (buf[i].toInt() and 0xFF) or (buf[i + 1].toInt() shl 8)
+            val v = if (raw > 32767) raw - 65536 else raw
+            val a = if (v < 0) -v else v
+            if (a > localPeak) localPeak = a
+            i += 2
+          }
+          val pk = localPeak / 32768f
+          if (pk > peak) peak = pk
+
+          // Blocking write — paces the loop to real time.
+          track!!.write(buf, 28, payloadBytes)
+
+          val now = System.currentTimeMillis()
+          if (now - lastEmit >= 100) {
+            sendEvent(
+              "onStats",
+              mapOf(
+                "streamName" to streamName,
+                "address" to address,
+                "sampleRate" to trackSr,
+                "channels" to trackCh,
+                "level" to peak,
+                "dropped" to dropped,
+                "packets" to packets,
+              ),
+            )
+            lastEmit = now
+            peak = 0f
+          }
+        }
+      } catch (e: Exception) {
+        sendEvent("onError", mapOf("message" to (e.message ?: "receive error")))
+      } finally {
+        track?.let { try { it.stop(); it.release() } catch (_: Exception) {} }
+      }
+    }.also { it.start() }
+  }
+
+  private fun stopReceive() {
+    running = false
+    try { socket?.close() } catch (_: Exception) {}
+    socket = null
+    try { thread?.join(500) } catch (_: Exception) {}
+    thread = null
+  }
+}

package/android/src/main/java/expo/modules/systemaudiocapture/VbanSendEngine.kt

@@ -0,0 +1,141 @@
+package expo.modules.systemaudiocapture
+
+import android.media.AudioRecord
+import java.net.DatagramPacket
+import java.net.DatagramSocket
+import java.net.InetAddress
+
+/**
+ * Reads 16-bit PCM from an already-configured AudioRecord, packetizes it into
+ * VBAN datagrams (256 frames/packet), and sends them to a host/port — all on a
+ * single background thread. The blocking AudioRecord.read paces the loop to real
+ * time, so packets go out evenly spaced (no bursts for the receiver to choke on).
+ *
+ * Shared by the mic sender (module) and the device-audio sender (service).
+ */
+class VbanSendEngine(
+  private val record: AudioRecord,
+  private val host: String,
+  private val port: Int,
+  private val streamName: String,
+  private val sampleRate: Int,
+  private val channels: Int,
+  private val onStats: (level: Float, packets: Long) -> Unit,
+  private val onError: (String) -> Unit,
+) {
+  @Volatile private var running = false
+  private var thread: Thread? = null
+
+  companion object {
+    private val SR = intArrayOf(
+      6000, 12000, 24000, 48000, 96000, 192000, 384000,
+      8000, 16000, 32000, 64000, 128000, 256000, 512000,
+      11025, 22050, 44100, 88200, 176400, 352800, 705600,
+    )
+
+    fun srIndex(sr: Int): Int {
+      for (i in SR.indices) if (SR[i] == sr) return i
+      return -1
+    }
+  }
+
+  fun start() {
+    if (running) return
+    val srIdx = srIndex(sampleRate)
+    if (srIdx < 0) {
+      onError("Unsupported VBAN sample rate: $sampleRate")
+      return
+    }
+
+    val socket = DatagramSocket()
+    val addr = InetAddress.getByName(host)
+    running = true
+    try {
+      record.startRecording()
+    } catch (e: Exception) {
+      onError("Failed to start capture: ${e.message}")
+      running = false
+      return
+    }
+
+    thread = Thread {
+      val framesPerPacket = 256
+      val shortBuf = ShortArray(framesPerPacket * channels)
+      val packet = ByteArray(28 + framesPerPacket * channels * 2)
+
+      // Static header fields.
+      packet[0] = 0x56; packet[1] = 0x42; packet[2] = 0x41; packet[3] = 0x4E // 'VBAN'
+      packet[4] = (srIdx and 0x1F).toByte() // audio protocol (0) | sr index
+      packet[6] = ((channels - 1) and 0xFF).toByte()
+      packet[7] = 0x01 // 16-bit PCM
+      for (i in 0 until 16) {
+        packet[8 + i] = if (i < streamName.length) (streamName[i].code and 0x7F).toByte() else 0
+      }
+
+      var frameCount = 0L
+      var peak = 0f
+      var packets = 0L
+      var lastEmit = System.currentTimeMillis()
+
+      try {
+        while (running) {
+          // Fill one packet's worth of samples.
+          var read = 0
+          val need = shortBuf.size
+          while (read < need && running) {
+            val n = record.read(shortBuf, read, need - read)
+            if (n <= 0) {
+              if (n < 0) onError("AudioRecord read error: $n")
+              break
+            }
+            read += n
+          }
+          if (read < channels) continue
+          val framesInPacket = read / channels
+
+          packet[5] = ((framesInPacket - 1) and 0xFF).toByte()
+          packet[24] = (frameCount and 0xFF).toByte()
+          packet[25] = ((frameCount shr 8) and 0xFF).toByte()
+          packet[26] = ((frameCount shr 16) and 0xFF).toByte()
+          packet[27] = ((frameCount shr 24) and 0xFF).toByte()
+
+          var bi = 28
+          var localPeak = 0
+          val total = framesInPacket * channels
+          for (i in 0 until total) {
+            val s = shortBuf[i].toInt()
+            packet[bi++] = (s and 0xFF).toByte()
+            packet[bi++] = ((s shr 8) and 0xFF).toByte()
+            val a = if (s < 0) -s else s
+            if (a > localPeak) localPeak = a
+          }
+
+          socket.send(DatagramPacket(packet, 28 + total * 2, addr, port))
+          frameCount = (frameCount + 1) and 0xFFFFFFFFL
+          packets++
+
+          val pk = localPeak / 32768f
+          if (pk > peak) peak = pk
+          val now = System.currentTimeMillis()
+          if (now - lastEmit >= 100) {
+            onStats(peak, packets)
+            lastEmit = now
+            peak = 0f
+          }
+        }
+      } catch (e: Exception) {
+        onError(e.message ?: "send error")
+      } finally {
+        try { socket.close() } catch (_: Exception) {}
+        try { record.stop() } catch (_: Exception) {}
+        try { record.release() } catch (_: Exception) {}
+      }
+    }.also { it.start() }
+  }
+
+  fun stop() {
+    running = false
+    try { thread?.join(400) } catch (_: Exception) {}
+    thread = null
+  }
+}