@mentra/bluetooth-sdk 0.1.0

package/README.md

@@ -0,0 +1,278 @@
+# @mentra/bluetooth-sdk
+
+React Native and Expo SDK for connecting mobile apps directly to supported Mentra smart glasses over Bluetooth.
+
+The package includes:
+
+- A React Native / Expo module API exposed as `BluetoothSdk`.
+- Native Android code published as `com.mentra:bluetooth-sdk`.
+- Native iOS code published as the `MentraBluetoothSDK` CocoaPod.
+- An Expo config plugin that wires the native dependencies into generated Android and iOS projects.
+
+Use a development build or production native build. Expo Go cannot load this package because the SDK contains native code.
+
+## Requirements
+
+- React Native `0.72+`.
+- Expo `49+` when using Expo.
+- Android min SDK `28+`.
+- iOS deployment target `15.1+`.
+- A physical phone for real Bluetooth testing.
+- Bluetooth permissions, plus Android location permission where BLE scanning requires it.
+
+## Install
+
+```sh
+npm install @mentra/bluetooth-sdk
+npx expo install expo-build-properties
+```
+
+For Expo apps, add the plugin to `app.json` or `app.config.ts`:
+
+```json
+{
+  "expo": {
+    "plugins": [
+      [
+        "@mentra/bluetooth-sdk",
+        {
+          "node": true
+        }
+      ],
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "minSdkVersion": 28,
+            "packagingOptions": {
+              "pickFirst": [
+                "**/libc++_shared.so",
+                "**/libonnxruntime.so",
+                "**/libonnxruntime4j_jni.so"
+              ]
+            }
+          }
+        }
+      ]
+    ]
+  }
+}
+```
+
+Then regenerate native projects and run a development build:
+
+```sh
+npx expo prebuild
+npx expo run:ios
+# or
+npx expo run:android
+```
+
+## Permissions
+
+Android apps should request the permissions required by the features they use:
+
+```json
+{
+  "android": {
+    "permissions": [
+      "android.permission.BLUETOOTH",
+      "android.permission.BLUETOOTH_ADMIN",
+      "android.permission.BLUETOOTH_SCAN",
+      "android.permission.BLUETOOTH_CONNECT",
+      "android.permission.ACCESS_FINE_LOCATION",
+      "android.permission.RECORD_AUDIO",
+      "android.permission.INTERNET",
+      "android.permission.POST_NOTIFICATIONS"
+    ]
+  }
+}
+```
+
+Some Android 12+ devices still require Location permission and Location services before BLE scan callbacks are delivered.
+
+iOS apps should include usage descriptions:
+
+```json
+{
+  "ios": {
+    "infoPlist": {
+      "NSBluetoothAlwaysUsageDescription": "This app connects to your smart glasses over Bluetooth.",
+      "NSMicrophoneUsageDescription": "This app uses the microphone when you enable audio features.",
+      "NSLocalNetworkUsageDescription": "This app connects to local photo and streaming helpers during development."
+    }
+  }
+}
+```
+
+## Minimal Usage
+
+`startScan()` starts the scan. Discovered devices arrive asynchronously through `onBluetoothStatus()` updates and `device_discovered` events.
+
+```ts
+import BluetoothSdk, {
+  isReadyGlassesConnectionStatus,
+  type Device,
+} from '@mentra/bluetooth-sdk'
+
+const firstDevice = new Promise<Device>((resolve) => {
+  let removeBluetoothListener = () => {}
+  removeBluetoothListener = BluetoothSdk.onBluetoothStatus((status) => {
+    const device = status.searchResults?.[0]
+    if (device) {
+      removeBluetoothListener()
+      resolve(device)
+    }
+  })
+})
+
+const removeGlassesListener = BluetoothSdk.onGlassesStatus((status) => {
+  console.log('Glasses status changed', status)
+})
+
+await BluetoothSdk.startScan({model: 'Mentra Live'})
+
+await BluetoothSdk.connect(await firstDevice)
+
+const glasses = await BluetoothSdk.getGlassesStatus()
+if (isReadyGlassesConnectionStatus(glasses.connection)) {
+  await BluetoothSdk.displayText({
+    text: 'Hello from Mentra',
+    x: 0,
+    y: 0,
+    size: 24,
+  })
+}
+
+removeGlassesListener()
+```
+
+React Native status exposes `GlassesStatus.connection` as a discriminated union:
+
+```ts
+type GlassesConnectionStatus =
+  | {state: 'disconnected'}
+  | {state: 'scanning'}
+  | {state: 'connecting'}
+  | {state: 'bonding'}
+  | {state: 'connected'; fullyBooted: boolean}
+```
+
+Use `connection.state` for link progress. `fullyBooted` only exists when `state === 'connected'`. Android and iOS native APIs also keep `connectionState`, `connected`, and `fullyBooted` as native status properties for Kotlin and Swift callers.
+
+## Default Device
+
+`connectDefault()` connects to the default glasses target currently stored in the SDK. Apps that want this target to survive app restarts should persist the scanned `Device` in app storage and restore it with `setDefaultDevice()` before calling `connectDefault()`.
+
+```ts
+const savedDevice = await loadSavedDeviceFromYourAppStorage()
+if (savedDevice) {
+  await BluetoothSdk.setDefaultDevice(savedDevice)
+  await BluetoothSdk.connectDefault()
+}
+
+const discoveredDevice = await scanAndChooseDevice()
+await BluetoothSdk.connect(discoveredDevice)
+await saveDeviceToYourAppStorage(discoveredDevice)
+
+await BluetoothSdk.clearDefaultDevice()
+await saveDeviceToYourAppStorage(null)
+```
+
+## Common Commands
+
+```ts
+await BluetoothSdk.clearDisplay()
+await BluetoothSdk.showDashboard()
+
+await BluetoothSdk.requestWifiScan()
+await BluetoothSdk.sendWifiCredentials('Office WiFi', 'secret')
+await BluetoothSdk.forgetWifiNetwork('Office WiFi')
+await BluetoothSdk.setHotspotState(true)
+
+await BluetoothSdk.setGalleryMode('auto')
+await BluetoothSdk.setGalleryMode('manual')
+
+await BluetoothSdk.setMicState(true, true, false)
+await BluetoothSdk.setOwnAppAudioPlaying(false)
+
+await BluetoothSdk.rgbLedControl(
+  `led-${Date.now()}`,
+  'com.example.app',
+  'on',
+  'green',
+  500,
+  500,
+  3,
+)
+```
+
+## Photo Upload
+
+```ts
+await BluetoothSdk.photoRequest(
+  `photo-${Date.now()}`,
+  'com.example.app',
+  'medium',
+  'https://api.example.com/mentra/photo',
+  'optional-token',
+  'medium',
+  false,
+  true,
+)
+```
+
+The webhook should accept multipart form data with a `photo` file and `requestId`. If `authToken` is provided, the uploader adds `Authorization: Bearer <token>`.
+
+## Streaming
+
+```ts
+const streamId = `stream-${Date.now()}`
+
+await BluetoothSdk.startStream({
+  type: 'start_stream',
+  streamUrl: 'http://192.168.1.42:8889/mentra-live/whip',
+  streamId,
+})
+
+await BluetoothSdk.keepStreamAlive({
+  type: 'keep_stream_alive',
+  streamId,
+  ackId: `ack-${Date.now()}`,
+})
+
+await BluetoothSdk.stopStream()
+```
+
+Use `rtmp://` or `rtmps://` for RTMP, `srt://` for SRT, and `http://` or `https://` for WHIP/WebRTC ingest. Send keep-alives about every 15 seconds while streaming.
+
+## Events
+
+```ts
+const subscriptions = [
+  BluetoothSdk.addListener('button_press', (event) => console.log(event)),
+  BluetoothSdk.addListener('touch_event', (event) => console.log(event)),
+  BluetoothSdk.addListener('photo_response', (event) => console.log(event)),
+  BluetoothSdk.addListener('stream_status', (event) => console.log(event)),
+  BluetoothSdk.addListener('mic_pcm', (event) => console.log(event.pcm)),
+]
+
+subscriptions.forEach((subscription) => subscription.remove())
+```
+
+Common event names include `button_press`, `touch_event`, `head_up`, `battery_status`, `wifi_status_change`, `hotspot_status_change`, `photo_response`, `gallery_status`, `stream_status`, `keep_alive_ack`, `mic_pcm`, `mic_lc3`, `local_transcription`, `rgb_led_control_response`, `audio_connected`, `audio_disconnected`, `log`, `send_command_to_ble`, and `receive_command_from_ble`.
+
+## Local SDK Development
+
+For normal app development, install the published npm package. For SDK development before a package release, install a local checkout and point Metro/native resolution at the same path:
+
+```sh
+npm install --no-save /path/to/MentraOS/mobile/modules/bluetooth-sdk
+MENTRA_BLUETOOTH_SDK_PACKAGE_PATH=/path/to/MentraOS/mobile/modules/bluetooth-sdk npx expo run:ios
+```
+
+Use `npx expo run:android` for Android. Keep local paths in your shell or CI environment, not in committed app config.
+
+## Example App
+
+The partner example app lives in `Mentra-Bluetooth-SDK-Partner-Kit/examples/react-native`. It demonstrates scan/connect, display, camera photo upload, RTMP/SRT/WebRTC streaming, Wi-Fi/hotspot, microphone PCM, RGB LED, gallery-button mode, and console event inspection.

package/android/build.gradle

@@ -0,0 +1,167 @@
+import groovy.json.JsonSlurper
+
+apply plugin: 'com.android.library'
+apply plugin: 'maven-publish'
+apply plugin: 'signing'
+
+def packageJson = new JsonSlurper().parse(file("../package.json"))
+
+group = 'com.mentra'
+version = packageJson.version
+
+def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle")
+apply from: expoModulesCorePlugin
+applyKotlinExpoModulesCorePlugin()
+useExpoPublishing()
+
+// If you want to use the managed Android SDK versions from expo-modules-core, set this to true.
+// The Android SDK versions will be bumped from time to time in SDK releases and may introduce breaking changes in your module code.
+// Most of the time, you may like to manage the Android SDK versions yourself.
+def useManagedAndroidSdkVersions = false
+if (useManagedAndroidSdkVersions) {
+  useDefaultAndroidSdkVersions()
+} else {
+  buildscript {
+    // Simple helper that allows the root project to override versions declared by this library.
+    ext.safeExtGet = { prop, fallback ->
+      rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
+    }
+  }
+  project.android {
+    compileSdkVersion safeExtGet("compileSdkVersion", 36)
+    defaultConfig {
+      minSdkVersion safeExtGet("minSdkVersion", 28)
+      targetSdkVersion safeExtGet("targetSdkVersion", 36)
+    }
+  }
+}
+
+android {
+  namespace "com.mentra.bluetoothsdk"
+  defaultConfig {
+    versionCode 1
+    versionName packageJson.version
+  }
+  publishing {
+    singleVariant("release") {
+      withSourcesJar()
+      withJavadocJar()
+    }
+  }
+  lintOptions {
+    abortOnError false
+  }
+  packagingOptions {
+    pickFirst '**/libjsc.so'
+    pickFirst '**/libc++_shared.so'
+    pickFirst '**/libonnxruntime.so'
+    pickFirst '**/libonnxruntime4j_jni.so'
+  }
+}
+
+dependencies {
+  // Expo adapter class is compiled into the MentraOS module, but the bare SDK
+  // artifact should not expose Expo as a runtime dependency.
+  compileOnly project(':expo-modules-core')
+
+  // Kotlin standard library (explicitly added for IDE support)
+  // Version is managed by Expo modules, but explicit dependency helps IDE indexing
+  implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7"
+  implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
+  
+  implementation "androidx.lifecycle:lifecycle-service:2.6.1"
+  implementation "org.greenrobot:eventbus:3.3.1"
+  implementation 'com.google.code.gson:gson:2.10.1'
+
+  // Google Play Services for location settings dialog
+  implementation 'com.google.android.gms:play-services-location:21.0.1'
+  implementation 'com.google.android.gms:play-services-base:18.2.0'
+
+  // Apache Commons Compress for tar.bz2 extraction
+  implementation 'org.apache.commons:commons-compress:1.24.0'
+
+  // OkHttp for HTTP requests
+  implementation 'com.squareup.okhttp3:okhttp:4.12.0'
+
+  // LC3 audio codec library
+  implementation project(':lc3Lib')
+
+  // Sherpa-ONNX for local transcription
+  implementation 'com.github.k2-fsa:sherpa-onnx:v1.12.8'
+
+  // Silero VAD (with onnxruntime exclusions to avoid conflicts with sherpa-onnx)
+  implementation ('com.github.gkonovalov:android-vad:2.0.9') {
+    exclude group: 'com.microsoft.onnxruntime', module: 'onnxruntime'
+    exclude group: 'com.microsoft.onnxruntime', module: 'onnxruntime4j'
+    exclude group: 'com.microsoft.onnxruntime', module: 'onnxruntime4j-jni'
+  }
+
+  // Vuzix Z100
+  implementation 'com.vuzix:ultralite-sdk-android:1.8'
+  implementation 'com.squareup.picasso:picasso:2.8'
+
+  implementation 'com.google.protobuf:protobuf-java:4.32.0'
+  implementation 'com.google.protobuf:protobuf-java-util:4.32.0'
+}
+
+// Force override onnxruntime version to 1.17.1
+configurations.all {
+  resolutionStrategy {
+    eachDependency { DependencyResolveDetails details ->
+      if (details.requested.group == 'com.microsoft.onnxruntime') {
+        details.useVersion '1.17.1'
+        details.because('Force override vad\'s transitive version')
+      }
+      if (details.requested.name == 'onnxruntime-android') {
+        details.useVersion '1.17.1'
+        details.because('Force override vad\'s transitive version')
+      }
+    }
+  }
+}
+
+afterEvaluate {
+  publishing {
+    publications {
+      release(MavenPublication) {
+        groupId = 'com.mentra'
+        artifactId = 'bluetooth-sdk'
+        version = project.version
+
+        from components.release
+
+        pom {
+          name = 'Mentra Bluetooth SDK'
+          description = 'SDK for communicating with smart glasses'
+          url = 'https://github.com/Mentra-Community/MentraOS'
+          licenses {
+            license {
+              name = 'MIT License'
+              url = 'https://opensource.org/licenses/MIT'
+            }
+          }
+          developers {
+            developer {
+              id = 'mentra'
+              name = 'Mentra'
+              email = 'dev@mentra.glass'
+            }
+          }
+          scm {
+            connection = 'scm:git:https://github.com/Mentra-Community/MentraOS.git'
+            developerConnection = 'scm:git:ssh://git@github.com:Mentra-Community/MentraOS.git'
+            url = 'https://github.com/Mentra-Community/MentraOS'
+          }
+        }
+      }
+    }
+  }
+
+  signing {
+    required {
+      gradle.taskGraph.allTasks.any { it.name.toLowerCase().contains("publish") } &&
+        (findProperty("signing.keyId") || findProperty("signingKey"))
+    }
+    sign publishing.publications.release
+  }
+}

package/android/lc3Lib/CMakeLists.txt

@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.22.1)
+
+project("some-name-needed-here")
+
+add_subdirectory(src/main/cpp)
+add_subdirectory(src/main/cpp/jni)

package/android/lc3Lib/build.gradle

@@ -0,0 +1,109 @@
+import groovy.json.JsonSlurper
+
+plugins {
+    id 'com.android.library'
+    id 'maven-publish'
+    id 'signing'
+}
+
+def packageJson = new JsonSlurper().parse(file("../../package.json"))
+
+group = 'com.mentra'
+version = packageJson.version
+
+def isPublishTask = {
+    gradle.taskGraph.allTasks.any { it.name.toLowerCase().contains("publish") }
+}
+
+android {
+    namespace "com.mentra.lc3Lib"
+    compileSdk 33
+
+    publishing {
+        singleVariant("release") {
+            withSourcesJar()
+            withJavadocJar()
+        }
+    }
+
+    packagingOptions {
+        exclude 'META-INF/proguard/androidx-annotations.pro'
+        resources {
+            excludes += '/META-INF/{AL2.0,LGPL2.1,DEPENDENCIES}'
+            excludes += '/META-INF/INDEX.LIST'
+        }
+    }
+
+    defaultConfig {
+        minSdk 28
+        targetSdk 32
+        versionCode 3
+        versionName "0.1"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+
+        externalNativeBuild {
+            cmake {
+                arguments '-DCMAKE_OBJECT_PATH_MAX=600', '-DANDROID_STL=c++_static'
+            }
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path file('src/main/cpp/liblc3/CMakeLists.txt')
+            version '3.22.1'
+        }
+    }
+
+}
+
+dependencies {
+    testImplementation 'junit:junit:4.13.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.3'
+}
+
+afterEvaluate {
+    publishing {
+        publications {
+            release(MavenPublication) {
+                groupId = 'com.mentra'
+                artifactId = 'lc3Lib'
+                version = project.version
+
+                from components.release
+
+                pom {
+                    name = 'Mentra LC3 Android Library'
+                    description = 'LC3 audio codec support for Mentra Bluetooth SDK'
+                    url = 'https://github.com/Mentra-Community/MentraOS'
+                    licenses {
+                        license {
+                            name = 'MIT License'
+                            url = 'https://opensource.org/licenses/MIT'
+                        }
+                    }
+                    developers {
+                        developer {
+                            id = 'mentra'
+                            name = 'Mentra'
+                            email = 'dev@mentra.glass'
+                        }
+                    }
+                    scm {
+                        connection = 'scm:git:https://github.com/Mentra-Community/MentraOS.git'
+                        developerConnection = 'scm:git:ssh://git@github.com:Mentra-Community/MentraOS.git'
+                        url = 'https://github.com/Mentra-Community/MentraOS'
+                    }
+                }
+            }
+        }
+    }
+
+    signing {
+        required {
+            isPublishTask() && (findProperty("signing.keyId") || findProperty("signingKey"))
+        }
+        sign publishing.publications.release
+    }
+}

package/android/lc3Lib/proguard-rules.pro

@@ -0,0 +1,33 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile
+
+#don't leave logs in release build:
+-dontskipnonpubliclibraryclasses
+-dontobfuscate
+-forceprocessing
+-optimizationpasses 5
+
+-keep class * extends android.app.Activity
+-assumenosideeffects class android.util.Log {
+    public static *** d(...);
+    public static *** v(...);
+}

package/android/lc3Lib/src/main/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"/>

package/android/lc3Lib/src/main/cpp/CMakeLists.txt

@@ -0,0 +1,8 @@
+cmake_minimum_required(VERSION 3.18.1)
+
+# Include Project 1
+#add_subdirectory(${CMAKE_SOURCE_DIR}/google_opus_stuff)
+
+# Include Project 2
+add_subdirectory(${CMAKE_SOURCE_DIR}/liblc3)
+

package/android/lc3Lib/src/main/cpp/google_opus_stuff/CMakeLists.txt

@@ -0,0 +1,30 @@
+# Builds the ogg and opus encoder library which used third party module.
+cmake_minimum_required(VERSION 3.22.1)
+
+# set related path of third party libraries.
+set(third_party_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party)
+
+# libogg library refer from https://github.com/xiph/ogg.
+set(libogg_INCLUDE ${third_party_DIR}/libogg/include)
+set(libogg_LIB ${third_party_DIR}/libogg/lib/${ANDROID_ABI})
+add_library(lib_ogg STATIC IMPORTED)
+set_target_properties(lib_ogg PROPERTIES IMPORTED_LOCATION ${libogg_LIB}/libogg.a)
+
+# libopus library refer from https://github.com/xiph/opus.
+set(libopus_INCLUDE ${third_party_DIR}/libopus/include)
+set(libopus_LIB ${third_party_DIR}/libopus/lib/${ANDROID_ABI})
+add_library(lib_opus STATIC IMPORTED)
+set_target_properties(lib_opus PROPERTIES IMPORTED_LOCATION ${libopus_LIB}/libopus.a)
+
+# opus-tools https://github.com/xiph/opus-tools.
+set(opus_tools_INCLUDE ${third_party_DIR}/opus_tools/include)
+set(opus_tools_LIB ${third_party_DIR}/opus_tools/lib/${ANDROID_ABI})
+add_library(lib_opus_header STATIC IMPORTED)
+set_target_properties(lib_opus_header PROPERTIES IMPORTED_LOCATION
+        ${opus_tools_LIB}/libopus_header.a)
+
+# Build ogg_opus_encoder_tool shared lib.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++11")
+add_library(ogg_opus_encoder_tool SHARED ogg_opus_encoder.cc)
+target_include_directories(ogg_opus_encoder_tool PRIVATE ${libogg_INCLUDE})
+target_link_libraries(ogg_opus_encoder_tool lib_opus lib_ogg lib_opus_header)

package/android/lc3Lib/src/main/cpp/google_opus_stuff/jni/CMakeLists.txt

@@ -0,0 +1,7 @@
+#build for ogg_opus_encoder shared library
+cmake_minimum_required(VERSION 3.22.1)
+
+add_library(ogg_opus_encoder SHARED ogg_opus_encoder.cc ../ogg_opus_encoder.cc)
+
+# Include libraries needed for ogg_opus_encoder
+target_link_libraries(ogg_opus_encoder ogg_opus_encoder_tool)