@mentra/bluetooth-sdk 0.1.5 → 0.1.7

package/src/BluetoothSdk.types.ts

@@ -22,9 +22,15 @@ export type HeadUpEvent = {
   up: boolean
 }
 
-export type VadStatusEvent = {
-  type: "vad_status"
-  status: boolean
+export type VoiceActivityDetectionStatusEvent = {
+  type: "voice_activity_detection_status"
+  voiceActivityDetectionEnabled: boolean
+}
+
+export type SpeakingStatusEvent = {
+  type: "speaking_status"
+  speaking: boolean
+  timestamp: number
 }
 
 export type BatteryStatusEvent = {
@@ -61,6 +67,7 @@ export function createDisconnectedGlassesStatus(): Partial<GlassesStatus> {
   return {
     connection: {state: 'disconnected'},
     hotspot: {state: 'disabled'},
+    voiceActivityDetectionEnabled: true,
     wifi: {state: 'disconnected'},
   }
 }
@@ -190,8 +197,6 @@ export type RgbLedControlResponseEvent =
 
 export type RgbLedAction = "on" | "off"
 export type RgbLedColor = "red" | "green" | "blue" | "orange" | "white"
-/** `"auto"` enables local button photo/video capture; `"manual"` reports button events without local gallery capture. */
-export type GalleryMode = "auto" | "manual"
 export type PhotoSize = "small" | "medium" | "large" | "full"
 export type ButtonPhotoSize = "small" | "medium" | "large"
 export type PhotoCompression = "none" | "medium" | "heavy"
@@ -223,14 +228,30 @@ export type CameraFovSetting = {
   roiPosition: number
 }
 
+type NativeCameraFovSetting = {
+  fov: number
+  roi_position: number
+}
+
 export type MicPreference = "auto" | "phone" | "glasses" | "bluetooth"
 export type MicMode = "phone" | "glasses" | "bluetoothClassic" | "bluetooth"
 
+export type PhotoRequestParams = {
+  requestId: string
+  appId: string
+  size: PhotoSize
+  webhookUrl: string | null
+  authToken: string | null
+  compress: PhotoCompression
+  sound: boolean
+  exposureTimeNs?: number | null
+}
+
 export type StreamVideoConfig = {
   width?: number
   height?: number
   bitrate?: number
-  frameRate?: number
+  fps?: number
 }
 
 export type StreamAudioConfig = {
@@ -299,7 +320,7 @@ export type MicPcmEvent = {
   bitsPerSample: 16
   channels: 1
   encoding: "pcm_s16le"
-  vadGated: boolean
+  voiceActivityDetectionEnabled: boolean
 }
 
 export type MicLc3Event = {
@@ -312,7 +333,7 @@ export type MicLc3Event = {
   frameSizeBytes: number
   bitrate: number
   packetizedFromGlasses: boolean
-  vadGated: boolean
+  voiceActivityDetectionEnabled: boolean
 }
 
 export type StreamStatusLifecycleState = "initializing" | "streaming" | "stopping" | "stopped"
@@ -450,7 +471,8 @@ export type BluetoothSdkModuleEvents = {
   button_press: (event: ButtonPressEvent) => void
   touch_event: (event: TouchEvent) => void
   head_up: (event: HeadUpEvent) => void
-  vad_status: (event: VadStatusEvent) => void
+  voice_activity_detection_status: (event: VoiceActivityDetectionStatusEvent) => void
+  speaking_status: (event: SpeakingStatusEvent) => void
   battery_status: (event: BatteryStatusEvent) => void
   local_transcription: (event: LocalTranscriptionEvent) => void
   wifi_status_change: (event: WifiStatusChangeEvent) => void
@@ -500,7 +522,7 @@ export type PublicBluetoothStatus = Pick<
   | "wifiScanResults"
   | "lastLog"
   | "otherBtConnected"
-  | "galleryModeAuto"
+  | "galleryModeEnabled"
 >
 
 export type BluetoothSdkEventMap = {
@@ -511,7 +533,8 @@ export type BluetoothSdkEventMap = {
   button_press: ButtonPressEvent
   touch_event: TouchEvent
   head_up: HeadUpEvent
-  vad_status: VadStatusEvent
+  voice_activity_detection_status: VoiceActivityDetectionStatusEvent
+  speaking_status: SpeakingStatusEvent
   battery_status: BatteryStatusEvent
   local_transcription: LocalTranscriptionEvent
   wifi_status_change: WifiStatusChangeEvent
@@ -577,22 +600,15 @@ export interface BluetoothSdkPublicModule {
   forgetWifiNetwork(ssid: string): Promise<void>
   setHotspotState(enabled: boolean): Promise<void>
 
-  setGalleryMode(mode: GalleryMode): Promise<void>
+  setGalleryModeEnabled(enabled: boolean): Promise<void>
+  setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
   setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
-  setButtonVideoRecordingSettings(width: number, height: number, frameRate: number): Promise<void>
+  setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
   setButtonCameraLed(enabled: boolean): Promise<void>
   setButtonMaxRecordingTime(minutes: number): Promise<void>
   setCameraFov(fov: CameraFov): Promise<void>
   queryGalleryStatus(): Promise<void>
-  requestPhoto(
-    requestId: string,
-    appId: string,
-    size: PhotoSize,
-    webhookUrl: string | null,
-    authToken: string | null,
-    compress: PhotoCompression,
-    sound: boolean,
-  ): Promise<void>
+  requestPhoto(params: PhotoRequestParams): Promise<void>
   startVideoRecording(requestId: string, save: boolean, sound: boolean): Promise<void>
   stopVideoRecording(requestId: string): Promise<void>
 
@@ -603,7 +619,6 @@ export interface BluetoothSdkPublicModule {
   setMicState(
     enabled: boolean,
     useGlassesMic?: boolean,
-    bypassVad?: boolean,
     sendTranscript?: boolean,
     sendLc3Data?: boolean,
   ): Promise<void>
@@ -664,6 +679,7 @@ export interface GlassesStatus {
   // state:
   connection: GlassesConnectionStatus
   micEnabled: boolean
+  voiceActivityDetectionEnabled: boolean
   bluetoothClassicConnected: boolean
   signalStrength: number
   /** Milliseconds since epoch when signalStrength was last refreshed by the phone BLE stack. */
@@ -728,6 +744,10 @@ export interface Device {
   name: string
   /** Platform address/identifier when available: Android Bluetooth address, iOS CoreBluetooth identifier. */
   address?: string
+  /**
+   * Optional scan signal strength. It may be undefined at first discovery and
+   * appear in a later scan update when the platform reports RSSI metadata.
+   */
   rssi?: number
 }
 
@@ -783,7 +803,7 @@ export interface BluetoothStatus {
   lastLog: string[]
   otherBtConnected: boolean
   // desired settings the SDK sends to compatible connected glasses:
-  galleryModeAuto: boolean
+  galleryModeEnabled: boolean
 }
 
 export type BluetoothSettingsUpdate = Partial<{
@@ -801,19 +821,19 @@ export type BluetoothSettingsUpdate = Partial<{
   dashboard_height: number
   dashboard_depth: number
   menu_apps: DashboardMenuItem[] | CoreDashboardMenuItem[] | Array<Record<string, unknown>> | null
-  galleryModeAuto: boolean
+  gallery_mode: boolean
+  voice_activity_detection_enabled: boolean
   button_photo_size: ButtonPhotoSize
-  button_video_settings: {width: number; height: number; frameRate: number}
+  button_video_settings: {width: number; height: number; fps: number}
   button_video_width: number
   button_video_height: number
   button_video_fps: number
   button_camera_led: boolean
   button_max_recording_time: number
-  camera_fov: CameraFovSetting
+  camera_fov: NativeCameraFovSetting
   should_send_pcm: boolean
   should_send_lc3: boolean
   should_send_transcript: boolean
-  bypass_vad: boolean
   offline_mode: boolean
   offline_captions_running: boolean
   local_stt_fallback_active: boolean

package/src/_private/BluetoothSdkModule.ts

@@ -12,14 +12,12 @@ import {
   DashboardMenuItem,
   Device,
   DeviceModel,
-  GalleryMode,
   GlassesMediaVolumeGetResult,
   GlassesMediaVolumeSetResult,
   GlassesStatus,
   MicPreference,
   ObservableStoreCategory,
-  PhotoCompression,
-  PhotoSize,
+  PhotoRequestParams,
   PublicBluetoothStatus,
   RgbLedAction,
   RgbLedColor,
@@ -96,22 +94,15 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
   logCurrentWifiFrequency(): Promise<void>
 
   // Gallery Commands
-  setGalleryMode(mode: GalleryMode): Promise<void>
+  setGalleryModeEnabled(enabled: boolean): Promise<void>
+  setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
   setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
-  setButtonVideoRecordingSettings(width: number, height: number, frameRate: number): Promise<void>
+  setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
   setButtonCameraLed(enabled: boolean): Promise<void>
   setButtonMaxRecordingTime(minutes: number): Promise<void>
   setCameraFov(fov: CameraFov): Promise<void>
   queryGalleryStatus(): Promise<void>
-  requestPhoto(
-    requestId: string,
-    appId: string,
-    size: PhotoSize,
-    webhookUrl: string | null,
-    authToken: string | null,
-    compress: PhotoCompression,
-    sound: boolean,
-  ): Promise<void>
+  requestPhoto(params: PhotoRequestParams): Promise<void>
 
   // OTA Commands
   sendOtaStart(): Promise<void>
@@ -133,7 +124,6 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
   setMicState(
     enabled: boolean,
     useGlassesMic?: boolean,
-    bypassVad?: boolean,
     sendTranscript?: boolean,
     sendLc3Data?: boolean,
   ): Promise<void>
@@ -301,7 +291,6 @@ NativeBluetoothSdkModule.getDefaultDevice = function () {
 const nativeSetMicState = NativeBluetoothSdkModule.setMicState.bind(NativeBluetoothSdkModule) as (
   enabled: boolean,
   useGlassesMic: boolean,
-  bypassVad: boolean,
   sendTranscript: boolean,
   sendLc3Data: boolean,
 ) => MaybePromise<void>
@@ -355,15 +344,23 @@ NativeBluetoothSdkModule.setScreenDisabled = function (disabled: boolean) {
   return this.updateBluetoothSettings({screen_disabled: disabled})
 }
 
+NativeBluetoothSdkModule.setGalleryModeEnabled = function (enabled: boolean) {
+  return this.updateBluetoothSettings({gallery_mode: enabled})
+}
+
+NativeBluetoothSdkModule.setVoiceActivityDetectionEnabled = function (enabled: boolean) {
+  return this.updateBluetoothSettings({voice_activity_detection_enabled: enabled})
+}
+
 NativeBluetoothSdkModule.setButtonPhotoSettings = function (size: ButtonPhotoSize) {
   return this.updateBluetoothSettings({button_photo_size: size})
 }
 
-NativeBluetoothSdkModule.setButtonVideoRecordingSettings = function (width: number, height: number, frameRate: number) {
+NativeBluetoothSdkModule.setButtonVideoRecordingSettings = function (width: number, height: number, fps: number) {
   return this.updateBluetoothSettings({
     button_video_width: width,
     button_video_height: height,
-    button_video_fps: frameRate,
+    button_video_fps: fps,
   })
 }
 
@@ -376,13 +373,15 @@ NativeBluetoothSdkModule.setButtonMaxRecordingTime = function (minutes: number)
 }
 
 NativeBluetoothSdkModule.setCameraFov = function (fov: CameraFov) {
-  return this.updateBluetoothSettings({camera_fov: CAMERA_FOV_SETTINGS[fov]})
+  const setting = CAMERA_FOV_SETTINGS[fov]
+  return this.updateBluetoothSettings({
+    camera_fov: {fov: setting.fov, roi_position: setting.roiPosition},
+  })
 }
 
 NativeBluetoothSdkModule.setMicState = function (
   enabled: boolean,
   useGlassesMic?: boolean,
-  bypassVad?: boolean,
   sendTranscript?: boolean,
   sendLc3Data?: boolean,
 ) {
@@ -390,7 +389,6 @@ NativeBluetoothSdkModule.setMicState = function (
     nativeSetMicState(
       enabled,
       useGlassesMic ?? true,
-      bypassVad ?? true,
       sendTranscript ?? false,
       sendLc3Data ?? false,
     ),
@@ -466,6 +464,9 @@ NativeBluetoothSdkModule.scan = async function (
     }
 
     const handleBluetoothStatus = (status: Partial<BluetoothStatus>) => {
+      if (!Array.isArray(status.searchResults)) {
+        return
+      }
       emitResults(searchResultsForModel(status, scanOptions.model))
     }
 
@@ -484,5 +485,30 @@ NativeBluetoothSdkModule.scan = async function (
   })
 }
 
+/** Expo Android bridge rejects null values in Map<String, Any> — omit optional nullish fields. */
+function photoRequestParamsForNative(params: PhotoRequestParams): Record<string, string | number | boolean> {
+  const payload: Record<string, string | number | boolean> = {
+    requestId: params.requestId,
+    appId: params.appId,
+    size: params.size,
+    webhookUrl: params.webhookUrl ?? "",
+    compress: params.compress,
+    flash: true,
+    sound: params.sound,
+  }
+  if (params.authToken != null && params.authToken.length > 0) {
+    payload.authToken = params.authToken
+  }
+  if (params.exposureTimeNs != null && Number.isFinite(params.exposureTimeNs) && params.exposureTimeNs > 0) {
+    payload.exposureTimeNs = params.exposureTimeNs
+  }
+  return payload
+}
+
+const nativeRequestPhoto = NativeBluetoothSdkModule.requestPhoto.bind(NativeBluetoothSdkModule)
+NativeBluetoothSdkModule.requestPhoto = function (params: PhotoRequestParams) {
+  return nativeRequestPhoto(photoRequestParamsForNative(params) as unknown as PhotoRequestParams)
+}
+
 export default NativeBluetoothSdkModule
 export const BluetoothSdk = NativeBluetoothSdkModule as BluetoothSdkPublicModule

package/src/index.ts

@@ -13,7 +13,8 @@ const PUBLIC_EVENT_NAMES = new Set<BluetoothSdkEventName>([
   "button_press",
   "touch_event",
   "head_up",
-  "vad_status",
+  "voice_activity_detection_status",
+  "speaking_status",
   "battery_status",
   "local_transcription",
   "wifi_status_change",
@@ -70,7 +71,8 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
   sendWifiCredentials: PrivateBluetoothSdkModule.sendWifiCredentials.bind(PrivateBluetoothSdkModule),
   forgetWifiNetwork: PrivateBluetoothSdkModule.forgetWifiNetwork.bind(PrivateBluetoothSdkModule),
   setHotspotState: PrivateBluetoothSdkModule.setHotspotState.bind(PrivateBluetoothSdkModule),
-  setGalleryMode: PrivateBluetoothSdkModule.setGalleryMode.bind(PrivateBluetoothSdkModule),
+  setGalleryModeEnabled: PrivateBluetoothSdkModule.setGalleryModeEnabled.bind(PrivateBluetoothSdkModule),
+  setVoiceActivityDetectionEnabled: PrivateBluetoothSdkModule.setVoiceActivityDetectionEnabled.bind(PrivateBluetoothSdkModule),
   setButtonPhotoSettings: PrivateBluetoothSdkModule.setButtonPhotoSettings.bind(PrivateBluetoothSdkModule),
   setButtonVideoRecordingSettings: PrivateBluetoothSdkModule.setButtonVideoRecordingSettings.bind(PrivateBluetoothSdkModule),
   setButtonCameraLed: PrivateBluetoothSdkModule.setButtonCameraLed.bind(PrivateBluetoothSdkModule),
@@ -124,7 +126,6 @@ export type {
   Device,
   DeviceModel,
   EnabledHotspotStatus,
-  GalleryMode,
   GalleryStatusEvent,
   GlassesConnectionStatus,
   GlassesMediaVolumeGetResult,
@@ -151,6 +152,7 @@ export type {
   ScanModelOptions,
   ScanOptions,
   ScanResultsCallback,
+  SpeakingStatusEvent,
   StreamAudioConfig,
   StreamKeepAliveRequest,
   StreamStartRequest,
@@ -162,7 +164,7 @@ export type {
   SwipeVolumeStatusEvent,
   SwitchStatusEvent,
   TouchEvent,
-  VadStatusEvent,
+  VoiceActivityDetectionStatusEvent,
   WifiSearchResult,
   WifiStatus,
   WifiStatusChangeEvent,

package/src/react/useMentraBluetooth.ts

@@ -11,7 +11,6 @@ import type {
   ConnectOptions,
   Device,
   DeviceModel,
-  GalleryMode,
   GlassesConnectionStatus,
   HotspotStatus,
   PublicBluetoothStatus,
@@ -78,12 +77,13 @@ export type GlassesRuntimeState =
       hotspot: HotspotStatus
       ready: boolean
       signal: SignalState
+      voiceActivityDetectionEnabled: boolean
       wifi: WifiStatus
     }
 
 export type GalleryModeState = {
   applying: boolean
-  desired: GalleryMode
+  enabled: boolean
   error: unknown | null
 }
 
@@ -134,7 +134,8 @@ export type MentraBluetoothSession = {
   scan: ScanController
   sdk: PhoneSdkRuntimeState
   setDefaultDevice: (device: Device | null) => Promise<void>
-  setGalleryMode: (mode: GalleryMode) => Promise<void>
+  setGalleryModeEnabled: (enabled: boolean) => Promise<void>
+  setVoiceActivityDetectionEnabled: (enabled: boolean) => Promise<void>
 }
 
 function stringValue(value: unknown): string | undefined {
@@ -220,6 +221,7 @@ function runtimeGlassesState(status: Partial<PublicGlassesStatus>): GlassesRunti
       strengthDbm: numberValue((status as Record<string, unknown>).signalStrength),
       updatedAt: numberValue((status as Record<string, unknown>).signalStrengthUpdatedAt),
     },
+    voiceActivityDetectionEnabled: status.voiceActivityDetectionEnabled ?? true,
     wifi: status.wifi ?? {state: "disconnected"},
   }
 }
@@ -269,11 +271,11 @@ export function useMentraBluetooth(options: UseMentraBluetoothOptions = {}): Men
   const [galleryModeApplying, setGalleryModeApplying] = useState(false)
   const [galleryModeError, setGalleryModeError] = useState<unknown | null>(null)
 
-  async function setGalleryMode(mode: GalleryMode) {
+  async function setGalleryModeEnabled(enabled: boolean) {
     setGalleryModeApplying(true)
     setGalleryModeError(null)
     try {
-      await BluetoothSdk.setGalleryMode(mode)
+      await BluetoothSdk.setGalleryModeEnabled(enabled)
     } catch (error) {
       setGalleryModeError(error)
       options.onError?.(error)
@@ -283,9 +285,18 @@ export function useMentraBluetooth(options: UseMentraBluetoothOptions = {}): Men
     }
   }
 
+  async function setVoiceActivityDetectionEnabled(enabled: boolean) {
+    try {
+      await BluetoothSdk.setVoiceActivityDetectionEnabled(enabled)
+    } catch (error) {
+      options.onError?.(error)
+      throw error
+    }
+  }
+
   const galleryMode: GalleryModeState = {
     applying: galleryModeApplying,
-    desired: connection.bluetoothStatus.galleryModeAuto === false ? "manual" : "auto",
+    enabled: connection.bluetoothStatus.galleryModeEnabled !== false,
     error: galleryModeError,
   }
 
@@ -302,6 +313,7 @@ export function useMentraBluetooth(options: UseMentraBluetoothOptions = {}): Men
     scan: scanController(connection),
     sdk: phoneSdkState(connection.bluetoothStatus, connection.defaultDevice, galleryMode),
     setDefaultDevice: connection.setDefaultDevice,
-    setGalleryMode,
+    setGalleryModeEnabled,
+    setVoiceActivityDetectionEnabled,
   }
 }