@mentra/bluetooth-sdk 0.1.20 → 0.1.21-beta.0

package/ios/Source/status/WifiHotspotStatus.swift

@@ -133,20 +133,39 @@ public enum WifiStatus: CustomStringConvertible, Equatable {
 public struct WifiStatusEvent: CustomStringConvertible {
     public let status: WifiStatus
 
-    public init(status: WifiStatus) {
+    /// Glasses-reported provisioning failure reason when THIS event is the verdict of a
+    /// failed connect attempt; nil for routine link-state updates. An attempt property,
+    /// not a link property — which is why it lives on the event and not on `WifiStatus`:
+    /// "connect_timeout" arrives with a disconnected status (never associated), while
+    /// "connected_to_other_network" arrives with a *connected* status — the attempt
+    /// failed and Android's auto-join left the glasses on (or returned them to) a
+    /// different SSID than requested, so the link is genuinely up while the request
+    /// genuinely failed. Sent by ASG client builds that include the WiFi error
+    /// surfacing (v40+); older glasses never set it.
+    public let error: String?
+
+    public init(status: WifiStatus, error: String? = nil) {
         self.status = status
+        self.error = error
     }
 
     init(connected: Bool, ssid: String?, localIp: String?) {
         status = WifiStatus.fromStoreFields(connected: connected, ssid: ssid, localIp: localIp) ?? .disconnected
+        error = nil
     }
 
     init(values: [String: Any]) {
         status = WifiStatus(values: values) ?? .disconnected
+        let rawError = stringValue(values, "error")
+        error = rawError?.isEmpty == false ? rawError : nil
     }
 
     public var values: [String: Any] {
-        status.values.merging(["type": "wifi_status_change"]) { _, new in new }
+        var body = status.values.merging(["type": "wifi_status_change"]) { _, new in new }
+        if let error {
+            body["error"] = error
+        }
+        return body
     }
 
     public var description: String {

package/ios/Source/streaming/StreamModels.swift

@@ -553,6 +553,11 @@ public enum StreamStatus: CustomStringConvertible, Equatable {
 
 public struct StreamStatusEvent: CustomStringConvertible {
     public let status: StreamStatus
+    // True when the glasses will retry the failed publisher themselves
+    // (emitting side lands in PR #3488); absent on older firmware and on
+    // events not parsed from a glasses status map. Carried here instead of
+    // as an `.error` associated value so the public case stays unchanged.
+    public private(set) var willRetry: Bool?
 
     public init(status: StreamStatus) {
         self.status = status
@@ -560,6 +565,7 @@ public struct StreamStatusEvent: CustomStringConvertible {
 
     public init(values: [String: Any]) {
         status = StreamStatus(values: values)
+        willRetry = boolValue(values, "willRetry")
     }
 
     public var state: StreamState {
@@ -577,6 +583,9 @@ public struct StreamStatusEvent: CustomStringConvertible {
     public var values: [String: Any] {
         var values = status.values
         values["type"] = "stream_status"
+        if let willRetry {
+            values["willRetry"] = willRetry
+        }
         return values
     }
 

package/ios/Source/tts/TTSTools.swift

@@ -111,7 +111,7 @@ class TTSTools {
                 voiceStyle: (modelPath as NSString).appendingPathComponent(kFileVoiceStyle)
             )
             var modelConfig = sherpaOnnxOfflineTtsModelConfig(
-                numThreads: 2,
+                numThreads: 8,
                 supertonic: supertonic
             )
             var config = sherpaOnnxOfflineTtsConfig(model: modelConfig, maxNumSentences: 1)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mentra/bluetooth-sdk",
-  "version": "0.1.20",
+  "version": "0.1.21-beta.0",
   "description": "SDK for communicating with smart glasses",
   "main": "build/index.js",
   "react-native": "src/index.ts",
@@ -105,7 +105,6 @@
     "access": "public",
     "registry": "https://registry.npmjs.org/"
   },
-  "dependencies": {},
   "devDependencies": {
     "@types/node": "^25.9.3",
     "expo-module-scripts": "^55.0.2",

package/src/BluetoothSdk.types.ts

@@ -116,6 +116,16 @@ export function isConnectedWifiStatus(status: WifiStatus): status is ConnectedWi
 
 export type WifiStatusChangeEvent = WifiStatus & {
   type: "wifi_status_change"
+  /**
+   * Glasses-reported provisioning failure reason when THIS event is the verdict of a
+   * failed connect attempt; absent on routine link-state updates. An attempt property,
+   * not a link property — which is why it lives on the event, not on WifiStatus:
+   * "connect_timeout" arrives on a disconnected status (never associated), while
+   * "connected_to_other_network" arrives on a *connected* status (the attempt failed
+   * and the glasses ended up on / fell back to a different SSID than requested).
+   * Requires ASG client v40+ — older glasses never send it.
+   */
+  error?: string
 }
 
 export type HotspotStatus = {state: "disabled"} | {state: "enabled"; ssid: string; password: string; localIp: string}
@@ -405,6 +415,7 @@ export type SettingsAckSetting =
   | "button_video_recording"
   | "button_max_recording_time"
   | "camera_fov"
+  | "camera_fov_override"
   | "camera_tuning"
 
 export type SettingsAckEvent = {
@@ -416,6 +427,7 @@ export type SettingsAckEvent = {
   fov?: number
   roiPosition?: CameraRoiPositionValue
   hardwareApplied?: boolean
+  leaseId?: string
   active?: boolean
   size?: ButtonPhotoSize | string
   width?: number
@@ -533,6 +545,13 @@ export type CameraFovResult = {
   timestamp: number
 }
 
+export type CameraFovOverrideRequest = CameraFovRequest & {
+  /** Phone-owned lease used to make delayed releases safe. */
+  leaseId: string
+  /** Safety TTL; refresh the same lease/configuration to extend without a HAL restart. */
+  ttlMs?: number
+}
+
 export type CameraFovSetting = {
   fov: number
   roiPosition: CameraRoiPositionValue
@@ -551,6 +570,8 @@ export type PhotoRequestParams = {
   appId?: string
   size: PhotoSize
   mode?: PhotoMode
+  /** `ble` skips direct upload and forces phone-relayed BLE image transfer. */
+  transferMethod?: "auto" | "ble"
   webhookUrl: string | null
   authToken: string | null
   compress: PhotoCompression
@@ -573,9 +594,12 @@ export type PhotoRequestParams = {
 }
 
 export type WarmUpCameraParams = {
+  /** Supply this when the owner needs to call stopCameraWarmUp during teardown. */
   requestId?: string
   size: PhotoSize
+  mode?: PhotoMode
   exposureTimeNs?: number | null
+  /** Ready-state hold; defaults to 15 seconds and is capped at 60 seconds by ASG. */
   durationMs?: number
 }
 
@@ -838,6 +862,8 @@ export type BluetoothSdkModuleEvents = {
   speaking_status: (event: SpeakingStatusEvent) => void
   battery_status: (event: BatteryStatusEvent) => void
   local_transcription: (event: LocalTranscriptionEvent) => void
+  phone_notification: (event: PhoneNotificationEvent) => void
+  phone_notification_dismissed: (event: PhoneNotificationDismissedEvent) => void
   wifi_status_change: (event: WifiStatusChangeEvent) => void
   wifi_scan_result: (event: WifiScanResultEvent) => void
   hotspot_status_change: (event: HotspotStatusChangeEvent) => void
@@ -884,6 +910,23 @@ export interface ExtractionProgressEvent {
   totalBytes: number
 }
 
+export interface PhoneNotificationEvent {
+  notificationId: string
+  app: string
+  title: string
+  content: string
+  priority: string
+  timestamp: number
+  packageName: string
+}
+
+export interface PhoneNotificationDismissedEvent {
+  notificationId: string
+  notificationKey: string
+  packageName: string
+  timestamp: number
+}
+
 export type PublicGlassesStatus = Omit<
   GlassesStatus,
   "otaUpdateAvailable" | "otaProgress" | "otaInProgress" | "otaVersionUrl"
@@ -1002,6 +1045,10 @@ export interface BluetoothSdkPublicModule {
   setVideoRecordingDefaults(settings: VideoRecordingDefaults): Promise<SettingsAckSuccessEvent>
   setMaxVideoRecordingDuration(minutes: number): Promise<SettingsAckSuccessEvent>
   setCameraFov(request: CameraFovRequest): Promise<CameraFovResult>
+  /** One-way FOV command for legacy ASG clients that do not send settings acknowledgements. */
+  setLegacyCameraFov(request: CameraFovRequest): Promise<CameraFovResult>
+  setCameraFovOverride(request: CameraFovOverrideRequest): Promise<CameraFovResult>
+  releaseCameraFovOverride(leaseId: string): Promise<SettingsAckSuccessEvent>
   /**
    * Configure camera HAL tuning (ANR / gain) on Mentra Live glasses.
    *
@@ -1019,6 +1066,8 @@ export interface BluetoothSdkPublicModule {
   queryGalleryStatus(): Promise<GalleryStatusEvent>
   requestPhoto(params: PhotoRequestParams): Promise<PhotoSuccessResponseEvent>
   warmUpCamera(params: WarmUpCameraParams): Promise<CameraStatusEvent>
+  /** Release one request-owned warm-up. Opening requests reject with camera_warm_up_cancelled. */
+  stopCameraWarmUp(requestId: string): Promise<void>
   startVideoRecording(
     requestId: string,
     save: boolean,

package/src/_internal.ts

@@ -9,3 +9,5 @@
 export {default} from "./_private/BluetoothSdkModule"
 export type {BluetoothSdkInternalModule} from "./_private/BluetoothSdkModule"
 export * from "./BluetoothSdk.types"
+export {default as MentraLocalNetwork} from "./_private/MentraLocalNetworkModule"
+export * from "./_private/MentraLocalNetworkModule"

package/src/_private/BluetoothSdkModule.ts

@@ -11,6 +11,7 @@ import {
   CAMERA_FOV_MIN,
   CameraFovPreset,
   CameraFovRequest,
+  CameraFovOverrideRequest,
   CameraFovResult,
   CameraFovSetting,
   CameraRoiPosition,
@@ -116,7 +117,7 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
   sendWifiCredentials(ssid: string, password: string): Promise<WifiStatusChangeEvent>
   forgetWifiNetwork(ssid: string): Promise<WifiStatusChangeEvent>
   setHotspotState(enabled: boolean): Promise<HotspotStatusChangeEvent>
-  /** Set glasses system clock (Mentra Live only) when phone detects clock skew. */
+  /** Set the glasses system clock when phone detects clock skew (Mentra Live and G2). */
   setSystemTime(timestampMs: number): Promise<void>
   /** Logs current WiFi frequency (MHz) and 5 GHz band to Android logcat. */
   logCurrentWifiFrequency(): Promise<void>
@@ -132,6 +133,10 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
   setVideoRecordingDefaults(width: number, height: number, fps: number): Promise<SettingsAckSuccessEvent>
   setMaxVideoRecordingDuration(minutes: number): Promise<SettingsAckSuccessEvent>
   setCameraFov(request: CameraFovRequest): Promise<CameraFovResult>
+  setLegacyCameraFov(request: CameraFovRequest): Promise<CameraFovResult>
+  restoreLegacyCameraFov(): Promise<void>
+  setCameraFovOverride(request: CameraFovOverrideRequest): Promise<CameraFovResult>
+  releaseCameraFovOverride(leaseId: string): Promise<SettingsAckSuccessEvent>
   /**
    * Configure camera HAL tuning (ANR / gain) on Mentra Live glasses.
    * Sends a {@code camera_tuning_config} BLE message; the ASG client relays it as a
@@ -144,6 +149,7 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
   queryGalleryStatus(): Promise<GalleryStatusEvent>
   requestPhoto(params: PhotoRequestParams): Promise<PhotoSuccessResponseEvent>
   warmUpCamera(params: WarmUpCameraParams): Promise<CameraStatusEvent>
+  stopCameraWarmUp(requestId: string): Promise<void>
 
   // OTA Commands
   setOtaVersionUrl(otaVersionUrl: string): void
@@ -184,6 +190,23 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
   // Used to suspend LC3 mic during audio playback to avoid MCU overload
   setOwnAppAudioPlaying(playing: boolean): Promise<void>
 
+  // Live PCM output stream (miniapp speaker.createStream). MentraOS-internal
+  // — 16-bit LE PCM chunks into a streaming AudioTrack (USAGE_MEDIA, so it
+  // follows the phone's media route, e.g. A2DP to glasses). Implemented with
+  // AudioTrack on Android and AVAudioEngine on iOS.
+  /** Open a PCM stream session. One AudioTrack per id; caller manages ids. */
+  pcmStreamOpen(streamId: string, sampleRate: number, channels: number, volume: number): Promise<void>
+  /**
+   * Append base64 PCM. Resolves with the queued-but-unplayed backlog in ms;
+   * blocks (on a background dispatcher) while the backlog is above the
+   * backpressure ceiling, so awaited writes self-throttle to realtime.
+   */
+  pcmStreamWrite(streamId: string, base64: string): Promise<{bufferedMs: number}>
+  /** Drain the backlog, then stop. Resolves with the total played duration. */
+  pcmStreamClose(streamId: string): Promise<{durationMs: number}>
+  /** Stop immediately, dropping any backlog. Idempotent. */
+  pcmStreamAbort(streamId: string): Promise<void>
+
   /** Mentra Live only: K900 `cs_getvol` / `sr_getvol`. */
   getGlassesMediaVolume(): Promise<GlassesMediaVolumeGetResult>
   /** Mentra Live only: K900 `cs_vol` / `sr_vol`; level clamped 0–15 on native. */
@@ -244,10 +267,7 @@ const DEFAULT_CONNECT_OPTIONS: Required<ConnectOptions> = {
 
 const DEFAULT_SCAN_TIMEOUT_MS = 15_000
 
-function bindNativeMethod<T extends (...args: never[]) => unknown>(
-  module: Record<string, unknown>,
-  name: string,
-): T {
+function bindNativeMethod<T extends (...args: never[]) => unknown>(module: Record<string, unknown>, name: string): T {
   const method = module[name]
   if (typeof method !== "function") {
     console.warn(`[BluetoothSdk] Native method "${name}" is unavailable — rebuild the app (bun android / bun ios)`)
@@ -286,11 +306,7 @@ function normalizeCameraFov(request: CameraFovRequest): CameraFovSetting {
   const roiPosition = request.roiPosition ?? "center"
 
   return {
-    fov: clampInteger(
-      Number.isFinite(request.fov) ? request.fov : CAMERA_FOV_DEFAULT,
-      CAMERA_FOV_MIN,
-      CAMERA_FOV_MAX,
-    ),
+    fov: clampInteger(Number.isFinite(request.fov) ? request.fov : CAMERA_FOV_DEFAULT, CAMERA_FOV_MIN, CAMERA_FOV_MAX),
     roiPosition: clampInteger(CAMERA_ROI_POSITION_VALUES[roiPosition] ?? 0, CAMERA_ROI_MIN, CAMERA_ROI_MAX) as
       | 0
       | 1
@@ -470,14 +486,49 @@ NativeBluetoothSdkModule.setVoiceActivityDetectionEnabled = function (enabled: b
   return this.updateBluetoothSettings({voice_activity_detection_enabled: enabled})
 }
 
-const nativeSetCameraFov = bindNativeMethod<
-  (fov: CameraFovSetting) => MaybePromise<CameraFovResult>
->(NativeBluetoothSdkModule as unknown as Record<string, unknown>, "setCameraFov")
+const nativeSetCameraFov = bindNativeMethod<(fov: CameraFovSetting) => MaybePromise<CameraFovResult>>(
+  NativeBluetoothSdkModule as unknown as Record<string, unknown>,
+  "setCameraFov",
+)
 NativeBluetoothSdkModule.setCameraFov = function (request: CameraFovRequest) {
   const setting = normalizeCameraFov(request)
   return Promise.resolve(nativeSetCameraFov(setting))
 }
 
+const nativeSetLegacyCameraFov = bindNativeMethod<(fov: CameraFovSetting) => MaybePromise<CameraFovResult>>(
+  NativeBluetoothSdkModule as unknown as Record<string, unknown>,
+  "setLegacyCameraFov",
+)
+NativeBluetoothSdkModule.setLegacyCameraFov = function (request: CameraFovRequest) {
+  return Promise.resolve(nativeSetLegacyCameraFov(normalizeCameraFov(request)))
+}
+
+const nativeRestoreLegacyCameraFov = bindNativeMethod<() => MaybePromise<void>>(
+  NativeBluetoothSdkModule as unknown as Record<string, unknown>,
+  "restoreLegacyCameraFov",
+)
+NativeBluetoothSdkModule.restoreLegacyCameraFov = function () {
+  return Promise.resolve(nativeRestoreLegacyCameraFov())
+}
+
+const nativeSetCameraFovOverride = bindNativeMethod<
+  (request: CameraFovSetting & {leaseId: string; ttlMs: number}) => MaybePromise<CameraFovResult>
+>(NativeBluetoothSdkModule as unknown as Record<string, unknown>, "setCameraFovOverride")
+NativeBluetoothSdkModule.setCameraFovOverride = function (request: CameraFovOverrideRequest) {
+  const setting = normalizeCameraFov(request)
+  const requestedTtl =
+    typeof request.ttlMs === "number" && Number.isFinite(request.ttlMs) && request.ttlMs > 0
+      ? Math.round(request.ttlMs)
+      : 300_000
+  return Promise.resolve(
+    nativeSetCameraFovOverride({
+      ...setting,
+      leaseId: request.leaseId,
+      ttlMs: Math.min(requestedTtl, 600_000),
+    }),
+  )
+}
+
 NativeBluetoothSdkModule.setMicState = function (
   enabled: boolean,
   useGlassesMic?: boolean,

package/src/_private/MentraLocalNetworkModule.ts

@@ -0,0 +1,50 @@
+import {NativeModule, requireOptionalNativeModule} from "expo"
+
+export type LocalNetworkResponse = {
+  status: number
+  headers: Record<string, string>
+  bodyBase64: string
+}
+
+export type LocalNetworkDownloadResult = {
+  statusCode: number
+  bytesWritten: number
+  headers: Record<string, string>
+}
+
+export type LocalNetworkDownloadProgress = {
+  requestId: string
+  bytesWritten: number
+  contentLength: number
+  statusCode?: number
+  headers?: Record<string, string>
+}
+
+type LocalNetworkEvents = {
+  downloadProgress: (event: LocalNetworkDownloadProgress) => void
+  networkLost: (event: {ssid: string}) => void
+}
+
+declare class MentraLocalNetworkNativeModule extends NativeModule<LocalNetworkEvents> {
+  connect(ssid: string, password: string): Promise<{connected: boolean; ssid: string}>
+  request(
+    requestId: string,
+    url: string,
+    method: string,
+    headers: Record<string, string>,
+    body: string | null,
+    timeoutMs: number,
+  ): Promise<LocalNetworkResponse>
+  download(
+    requestId: string,
+    url: string,
+    destination: string,
+    headers: Record<string, string>,
+    connectionTimeoutMs: number,
+    readTimeoutMs: number,
+  ): Promise<LocalNetworkDownloadResult>
+  cancel(requestId: string): Promise<void>
+  disconnect(): Promise<void>
+}
+
+export default requireOptionalNativeModule<MentraLocalNetworkNativeModule>("MentraLocalNetwork")

package/src/_private/cameraRequestPayload.ts

@@ -12,6 +12,7 @@ export function warmUpCameraParamsForNative(
 ): Record<string, string | number> {
   const payload: Record<string, string | number> = {
     size: normalizePhotoSizeTier(params.size),
+    mode: params.mode ?? "photo",
   }
   const requestId = nonBlankString(params.requestId)
   if (requestId != null) {

package/src/_private/photoRequestPayload.ts

@@ -25,12 +25,11 @@ function nonBlankString(value?: string | null): string | undefined {
 }
 
 /** Expo Android bridge rejects null values in Map<String, Any> — omit optional nullish fields. */
-export function photoRequestParamsForNative(
-  params: PhotoRequestParams,
-): Record<string, string | number | boolean> {
+export function photoRequestParamsForNative(params: PhotoRequestParams): Record<string, string | number | boolean> {
   const payload: Record<string, string | number | boolean> = {
     size: normalizePhotoSizeTier(params.size),
     mode: params.mode ?? "photo",
+    transferMethod: params.transferMethod ?? "auto",
     webhookUrl: params.webhookUrl ?? "",
     compress: params.compress,
     sound: params.sound,

package/src/index.ts

@@ -113,10 +113,14 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
   },
   setMaxVideoRecordingDuration: bindPublicMethod("setMaxVideoRecordingDuration"),
   setCameraFov: bindPublicMethod("setCameraFov"),
+  setLegacyCameraFov: bindPublicMethod("setLegacyCameraFov"),
+  setCameraFovOverride: bindPublicMethod("setCameraFovOverride"),
+  releaseCameraFovOverride: bindPublicMethod("releaseCameraFovOverride"),
   setCameraTuningConfig: bindPublicMethod("setCameraTuningConfig"),
   queryGalleryStatus: bindPublicMethod("queryGalleryStatus"),
   requestPhoto: bindPublicMethod("requestPhoto"),
   warmUpCamera: bindPublicMethod("warmUpCamera"),
+  stopCameraWarmUp: bindPublicMethod("stopCameraWarmUp"),
   startVideoRecording: bindPublicMethod("startVideoRecording"),
   stopVideoRecording: bindPublicMethod("stopVideoRecording"),
   startStream: bindPublicMethod("startStream"),