@mentra/bluetooth-sdk 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +26 -22
  2. package/android/.project +28 -0
  3. package/android/build.gradle +5 -9
  4. package/android/lc3Lib/.project +28 -0
  5. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
  6. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
  7. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
  8. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
  9. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
  10. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
  11. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
  12. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
  13. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
  14. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
  15. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
  16. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
  17. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
  18. package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
  19. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
  20. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
  21. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
  22. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
  23. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
  24. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
  25. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
  26. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
  27. package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
  29. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
  30. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
  31. package/build/BluetoothSdk.types.d.ts +107 -48
  32. package/build/BluetoothSdk.types.d.ts.map +1 -1
  33. package/build/BluetoothSdk.types.js +4 -0
  34. package/build/BluetoothSdk.types.js.map +1 -1
  35. package/build/_private/BluetoothSdkModule.d.ts +8 -6
  36. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  37. package/build/_private/BluetoothSdkModule.js +58 -9
  38. package/build/_private/BluetoothSdkModule.js.map +1 -1
  39. package/build/index.d.ts +2 -2
  40. package/build/index.d.ts.map +1 -1
  41. package/build/index.js +6 -5
  42. package/build/index.js.map +1 -1
  43. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
  44. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
  45. package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
  46. package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
  47. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
  48. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
  49. package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
  50. package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
  51. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
  52. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
  53. package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
  54. package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
  55. package/build/photo-receiver/index.d.ts +3 -0
  56. package/build/photo-receiver/index.d.ts.map +1 -0
  57. package/build/photo-receiver/index.js +3 -0
  58. package/build/photo-receiver/index.js.map +1 -0
  59. package/build/react/useMentraBluetooth.d.ts +5 -3
  60. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  61. package/build/react/useMentraBluetooth.js +15 -4
  62. package/build/react/useMentraBluetooth.js.map +1 -1
  63. package/expo-module.config.json +5 -2
  64. package/ios/BluetoothSdkModule.swift +59 -26
  65. package/ios/LocalPhotoUploadServer.swift +411 -0
  66. package/ios/MentraBluetoothSDK.podspec +8 -5
  67. package/ios/MentraPhotoReceiverModule.swift +147 -0
  68. package/ios/Source/Audio/AudioModels.swift +10 -10
  69. package/ios/Source/Bridge.swift +42 -12
  70. package/ios/Source/Camera/CameraModels.swift +74 -19
  71. package/ios/Source/DeviceManager.swift +23 -98
  72. package/ios/Source/DeviceStore.swift +12 -4
  73. package/ios/Source/Events/BluetoothEvents.swift +37 -0
  74. package/ios/Source/MentraBluetoothSDK.swift +135 -14
  75. package/ios/Source/Status/DeviceStatus.swift +12 -4
  76. package/ios/Source/Status/RuntimeState.swift +22 -13
  77. package/ios/Source/Streaming/StreamModels.swift +239 -45
  78. package/ios/Source/controllers/ControllerManager.swift +3 -3
  79. package/ios/Source/controllers/R1.swift +9 -3
  80. package/ios/Source/sgcs/G1.swift +5 -4
  81. package/ios/Source/sgcs/G2.swift +5 -6
  82. package/ios/Source/sgcs/Mach1.swift +1 -1
  83. package/ios/Source/sgcs/MentraLive.swift +183 -22
  84. package/ios/Source/sgcs/MentraNex.swift +1 -1
  85. package/ios/Source/sgcs/SGCManager.swift +11 -3
  86. package/ios/Source/sgcs/Simulated.swift +2 -2
  87. package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
  88. package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
  89. package/package.json +6 -1
  90. package/src/BluetoothSdk.types.ts +133 -62
  91. package/src/_private/BluetoothSdkModule.ts +79 -26
  92. package/src/index.ts +15 -8
  93. package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
  94. package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
  95. package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
  96. package/src/photo-receiver/index.ts +2 -0
  97. package/src/react/useMentraBluetooth.ts +19 -7
  98. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
  99. package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
  100. package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
  101. package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
  102. package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
  103. package/ios/Packages/VAD/Data/Configuration.swift +0 -68
  104. package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
  105. package/ios/Packages/VAD/Data/Record.swift +0 -13
  106. package/ios/Packages/VAD/Data/Result.swift +0 -22
  107. package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
  108. package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
  109. package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
  110. package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
@@ -22,9 +22,15 @@ export type HeadUpEvent = {
22
22
  up: boolean
23
23
  }
24
24
 
25
- export type VadStatusEvent = {
26
- type: "vad_status"
27
- status: boolean
25
+ export type VoiceActivityDetectionStatusEvent = {
26
+ type: "voice_activity_detection_status"
27
+ voiceActivityDetectionEnabled: boolean
28
+ }
29
+
30
+ export type SpeakingStatusEvent = {
31
+ type: "speaking_status"
32
+ speaking: boolean
33
+ timestamp: number
28
34
  }
29
35
 
30
36
  export type BatteryStatusEvent = {
@@ -61,6 +67,7 @@ export function createDisconnectedGlassesStatus(): Partial<GlassesStatus> {
61
67
  return {
62
68
  connection: {state: 'disconnected'},
63
69
  hotspot: {state: 'disabled'},
70
+ voiceActivityDetectionEnabled: true,
64
71
  wifi: {state: 'disconnected'},
65
72
  }
66
73
  }
@@ -133,6 +140,43 @@ export type PhotoResponseEvent =
133
140
  errorMessage: string
134
141
  }
135
142
 
143
+ export type PhotoStatusState =
144
+ | "accepted"
145
+ | "queued"
146
+ | "configuring"
147
+ | "capturing"
148
+ | "captured"
149
+ | "compressing"
150
+ | "uploading"
151
+ | "uploaded"
152
+ | "ready_for_transfer"
153
+ | "transferring"
154
+ | "failed"
155
+
156
+ export type PhotoResolvedConfig = {
157
+ format?: "jpeg" | string
158
+ width?: number
159
+ height?: number
160
+ quality?: number
161
+ requestedSize?: PhotoSize | string
162
+ source?: "sdk" | "button" | string
163
+ transferMethod?: "webhook" | "ble" | "local" | string
164
+ compression?: PhotoCompression | string
165
+ saveToGallery?: boolean
166
+ exposureTimeNs?: number
167
+ iso?: number
168
+ }
169
+
170
+ export type PhotoStatusEvent = {
171
+ type: "photo_status"
172
+ requestId: string
173
+ status: PhotoStatusState | string
174
+ timestamp: number
175
+ resolvedConfig?: PhotoResolvedConfig
176
+ errorCode?: string
177
+ errorMessage?: string
178
+ }
179
+
136
180
  export type GalleryStatusEvent = {
137
181
  type: "gallery_status"
138
182
  photos: number
@@ -190,8 +234,6 @@ export type RgbLedControlResponseEvent =
190
234
 
191
235
  export type RgbLedAction = "on" | "off"
192
236
  export type RgbLedColor = "red" | "green" | "blue" | "orange" | "white"
193
- /** `"auto"` enables local button photo/video capture; `"manual"` reports button events without local gallery capture. */
194
- export type GalleryMode = "auto" | "manual"
195
237
  export type PhotoSize = "small" | "medium" | "large" | "full"
196
238
  export type ButtonPhotoSize = "small" | "medium" | "large"
197
239
  export type PhotoCompression = "none" | "medium" | "heavy"
@@ -216,21 +258,48 @@ export type DashboardMenuItem = {
216
258
  values?: Record<string, unknown>
217
259
  }
218
260
 
219
- export type CameraFov = "standard" | "wide"
261
+ export const CAMERA_FOV_MIN = 82
262
+ export const CAMERA_FOV_MAX = 118
263
+ export const CAMERA_FOV_DEFAULT = 102
264
+
265
+ export type CameraRoiPosition = 0 | 1 | 2
266
+
267
+ export type CameraFov = {
268
+ fov: number
269
+ roiPosition?: CameraRoiPosition
270
+ }
220
271
 
221
272
  export type CameraFovSetting = {
222
273
  fov: number
223
- roiPosition: number
274
+ roiPosition: CameraRoiPosition
275
+ }
276
+
277
+ type NativeCameraFovSetting = {
278
+ fov: number
279
+ roi_position: CameraRoiPosition
224
280
  }
225
281
 
226
282
  export type MicPreference = "auto" | "phone" | "glasses" | "bluetooth"
227
283
  export type MicMode = "phone" | "glasses" | "bluetoothClassic" | "bluetooth"
228
284
 
285
+ export type PhotoRequestParams = {
286
+ requestId: string
287
+ appId: string
288
+ size: PhotoSize
289
+ webhookUrl: string | null
290
+ authToken: string | null
291
+ compress: PhotoCompression
292
+ sound: boolean
293
+ exposureTimeNs?: number | null
294
+ /** Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure. */
295
+ iso?: number | null
296
+ }
297
+
229
298
  export type StreamVideoConfig = {
230
299
  width?: number
231
300
  height?: number
232
301
  bitrate?: number
233
- frameRate?: number
302
+ fps?: number
234
303
  }
235
304
 
236
305
  export type StreamAudioConfig = {
@@ -299,7 +368,7 @@ export type MicPcmEvent = {
299
368
  bitsPerSample: 16
300
369
  channels: 1
301
370
  encoding: "pcm_s16le"
302
- vadGated: boolean
371
+ voiceActivityDetectionEnabled: boolean
303
372
  }
304
373
 
305
374
  export type MicLc3Event = {
@@ -312,65 +381,72 @@ export type MicLc3Event = {
312
381
  frameSizeBytes: number
313
382
  bitrate: number
314
383
  packetizedFromGlasses: boolean
315
- vadGated: boolean
384
+ voiceActivityDetectionEnabled: boolean
316
385
  }
317
386
 
318
387
  export type StreamStatusLifecycleState = "initializing" | "streaming" | "stopping" | "stopped"
319
388
  export type StreamStatusReconnectState = "reconnecting" | "reconnected" | "reconnect_failed"
320
389
  export type StreamStatusState = StreamStatusLifecycleState | StreamStatusReconnectState | "error"
321
390
 
391
+ export type StreamResolvedConfig = {
392
+ transport?: "rtmp" | "srt" | "whip"
393
+ video?: {
394
+ width: number
395
+ height: number
396
+ captureWidth?: number
397
+ captureHeight?: number
398
+ bitrate: number
399
+ fps: number
400
+ }
401
+ audio?: {
402
+ bitrate?: number
403
+ sampleRate?: number
404
+ echoCancellation?: boolean
405
+ noiseSuppression?: boolean
406
+ }
407
+ }
408
+
409
+ type StreamStatusCommon = {
410
+ type: "stream_status"
411
+ streamId?: string
412
+ timestamp?: number
413
+ resolvedConfig?: StreamResolvedConfig
414
+ }
415
+
322
416
  export type StreamStatusEvent =
323
- | {
324
- type: "stream_status"
417
+ | (StreamStatusCommon & {
325
418
  kind: "lifecycle"
326
419
  status: StreamStatusLifecycleState
327
- streamId?: string
328
- timestamp?: number
329
- }
330
- | {
331
- type: "stream_status"
420
+ })
421
+ | (StreamStatusCommon & {
332
422
  kind: "reconnect"
333
423
  status: "reconnecting"
334
- streamId?: string
335
424
  attempt: number
336
425
  maxAttempts: number
337
426
  reason: string
338
- timestamp?: number
339
- }
340
- | {
341
- type: "stream_status"
427
+ })
428
+ | (StreamStatusCommon & {
342
429
  kind: "reconnect"
343
430
  status: "reconnected"
344
- streamId?: string
345
431
  attempt: number
346
- timestamp?: number
347
- }
348
- | {
349
- type: "stream_status"
432
+ })
433
+ | (StreamStatusCommon & {
350
434
  kind: "reconnect"
351
435
  status: "reconnect_failed"
352
- streamId?: string
353
436
  maxAttempts: number
354
- timestamp?: number
355
- }
356
- | {
357
- type: "stream_status"
437
+ })
438
+ | (StreamStatusCommon & {
358
439
  kind: "error"
359
440
  status: "error"
360
- streamId?: string
361
441
  errorDetails: string
362
- timestamp?: number
363
- }
364
- | {
365
- type: "stream_status"
442
+ })
443
+ | (StreamStatusCommon & {
366
444
  kind: "snapshot"
367
445
  status: "streaming" | "reconnecting" | "stopped"
368
446
  streaming: boolean
369
447
  reconnecting: boolean
370
- streamId?: string
371
448
  attempt?: number
372
- timestamp?: number
373
- }
449
+ })
374
450
 
375
451
  export type KeepAliveAckEvent = {
376
452
  type: "keep_alive_ack"
@@ -450,13 +526,15 @@ export type BluetoothSdkModuleEvents = {
450
526
  button_press: (event: ButtonPressEvent) => void
451
527
  touch_event: (event: TouchEvent) => void
452
528
  head_up: (event: HeadUpEvent) => void
453
- vad_status: (event: VadStatusEvent) => void
529
+ voice_activity_detection_status: (event: VoiceActivityDetectionStatusEvent) => void
530
+ speaking_status: (event: SpeakingStatusEvent) => void
454
531
  battery_status: (event: BatteryStatusEvent) => void
455
532
  local_transcription: (event: LocalTranscriptionEvent) => void
456
533
  wifi_status_change: (event: WifiStatusChangeEvent) => void
457
534
  hotspot_status_change: (event: HotspotStatusChangeEvent) => void
458
535
  hotspot_error: (event: HotspotErrorEvent) => void
459
536
  photo_response: (event: PhotoResponseEvent) => void
537
+ photo_status: (event: PhotoStatusEvent) => void
460
538
  gallery_status: (event: GalleryStatusEvent) => void
461
539
  compatible_glasses_search_stop: (event: CompatibleGlassesSearchStopEvent) => void
462
540
  heartbeat_sent: (event: HeartbeatSentEvent) => void
@@ -500,7 +578,7 @@ export type PublicBluetoothStatus = Pick<
500
578
  | "wifiScanResults"
501
579
  | "lastLog"
502
580
  | "otherBtConnected"
503
- | "galleryModeAuto"
581
+ | "galleryModeEnabled"
504
582
  >
505
583
 
506
584
  export type BluetoothSdkEventMap = {
@@ -511,13 +589,15 @@ export type BluetoothSdkEventMap = {
511
589
  button_press: ButtonPressEvent
512
590
  touch_event: TouchEvent
513
591
  head_up: HeadUpEvent
514
- vad_status: VadStatusEvent
592
+ voice_activity_detection_status: VoiceActivityDetectionStatusEvent
593
+ speaking_status: SpeakingStatusEvent
515
594
  battery_status: BatteryStatusEvent
516
595
  local_transcription: LocalTranscriptionEvent
517
596
  wifi_status_change: WifiStatusChangeEvent
518
597
  hotspot_status_change: HotspotStatusChangeEvent
519
598
  hotspot_error: HotspotErrorEvent
520
599
  photo_response: PhotoResponseEvent
600
+ photo_status: PhotoStatusEvent
521
601
  gallery_status: GalleryStatusEvent
522
602
  compatible_glasses_search_stop: CompatibleGlassesSearchStopEvent
523
603
  swipe_volume_status: SwipeVolumeStatusEvent
@@ -530,7 +610,6 @@ export type BluetoothSdkEventMap = {
530
610
  mic_pcm: MicPcmEvent
531
611
  mic_lc3: MicLc3Event
532
612
  stream_status: StreamStatusEvent
533
- keep_alive_ack: KeepAliveAckEvent
534
613
  }
535
614
 
536
615
  export type BluetoothSdkEventName = keyof BluetoothSdkEventMap
@@ -577,33 +656,24 @@ export interface BluetoothSdkPublicModule {
577
656
  forgetWifiNetwork(ssid: string): Promise<void>
578
657
  setHotspotState(enabled: boolean): Promise<void>
579
658
 
580
- setGalleryMode(mode: GalleryMode): Promise<void>
659
+ setGalleryModeEnabled(enabled: boolean): Promise<void>
660
+ setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
581
661
  setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
582
- setButtonVideoRecordingSettings(width: number, height: number, frameRate: number): Promise<void>
662
+ setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
583
663
  setButtonCameraLed(enabled: boolean): Promise<void>
584
664
  setButtonMaxRecordingTime(minutes: number): Promise<void>
585
665
  setCameraFov(fov: CameraFov): Promise<void>
586
666
  queryGalleryStatus(): Promise<void>
587
- requestPhoto(
588
- requestId: string,
589
- appId: string,
590
- size: PhotoSize,
591
- webhookUrl: string | null,
592
- authToken: string | null,
593
- compress: PhotoCompression,
594
- sound: boolean,
595
- ): Promise<void>
667
+ requestPhoto(params: PhotoRequestParams): Promise<void>
596
668
  startVideoRecording(requestId: string, save: boolean, sound: boolean): Promise<void>
597
669
  stopVideoRecording(requestId: string): Promise<void>
598
670
 
599
671
  startStream(params: StreamStartRequest): Promise<void>
600
672
  stopStream(): Promise<void>
601
- keepStreamAlive(params: StreamKeepAliveRequest): Promise<void>
602
673
 
603
674
  setMicState(
604
675
  enabled: boolean,
605
676
  useGlassesMic?: boolean,
606
- bypassVad?: boolean,
607
677
  sendTranscript?: boolean,
608
678
  sendLc3Data?: boolean,
609
679
  ): Promise<void>
@@ -664,6 +734,7 @@ export interface GlassesStatus {
664
734
  // state:
665
735
  connection: GlassesConnectionStatus
666
736
  micEnabled: boolean
737
+ voiceActivityDetectionEnabled: boolean
667
738
  bluetoothClassicConnected: boolean
668
739
  signalStrength: number
669
740
  /** Milliseconds since epoch when signalStrength was last refreshed by the phone BLE stack. */
@@ -787,7 +858,7 @@ export interface BluetoothStatus {
787
858
  lastLog: string[]
788
859
  otherBtConnected: boolean
789
860
  // desired settings the SDK sends to compatible connected glasses:
790
- galleryModeAuto: boolean
861
+ galleryModeEnabled: boolean
791
862
  }
792
863
 
793
864
  export type BluetoothSettingsUpdate = Partial<{
@@ -805,19 +876,19 @@ export type BluetoothSettingsUpdate = Partial<{
805
876
  dashboard_height: number
806
877
  dashboard_depth: number
807
878
  menu_apps: DashboardMenuItem[] | CoreDashboardMenuItem[] | Array<Record<string, unknown>> | null
808
- galleryModeAuto: boolean
879
+ gallery_mode: boolean
880
+ voice_activity_detection_enabled: boolean
809
881
  button_photo_size: ButtonPhotoSize
810
- button_video_settings: {width: number; height: number; frameRate: number}
882
+ button_video_settings: {width: number; height: number; fps: number}
811
883
  button_video_width: number
812
884
  button_video_height: number
813
885
  button_video_fps: number
814
886
  button_camera_led: boolean
815
887
  button_max_recording_time: number
816
- camera_fov: CameraFovSetting
888
+ camera_fov: NativeCameraFovSetting
817
889
  should_send_pcm: boolean
818
890
  should_send_lc3: boolean
819
891
  should_send_transcript: boolean
820
- bypass_vad: boolean
821
892
  offline_mode: boolean
822
893
  offline_captions_running: boolean
823
894
  local_stt_fallback_active: boolean
@@ -6,20 +6,20 @@ import {
6
6
  BluetoothSdkModuleEvents,
7
7
  BluetoothStatus,
8
8
  ButtonPhotoSize,
9
+ CAMERA_FOV_DEFAULT,
10
+ CAMERA_FOV_MAX,
11
+ CAMERA_FOV_MIN,
9
12
  CameraFov,
10
- CameraFovSetting,
11
13
  ConnectOptions,
12
14
  DashboardMenuItem,
13
15
  Device,
14
16
  DeviceModel,
15
- GalleryMode,
16
17
  GlassesMediaVolumeGetResult,
17
18
  GlassesMediaVolumeSetResult,
18
19
  GlassesStatus,
19
20
  MicPreference,
20
21
  ObservableStoreCategory,
21
- PhotoCompression,
22
- PhotoSize,
22
+ PhotoRequestParams,
23
23
  PublicBluetoothStatus,
24
24
  RgbLedAction,
25
25
  RgbLedColor,
@@ -96,22 +96,15 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
96
96
  logCurrentWifiFrequency(): Promise<void>
97
97
 
98
98
  // Gallery Commands
99
- setGalleryMode(mode: GalleryMode): Promise<void>
99
+ setGalleryModeEnabled(enabled: boolean): Promise<void>
100
+ setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
100
101
  setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
101
- setButtonVideoRecordingSettings(width: number, height: number, frameRate: number): Promise<void>
102
+ setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
102
103
  setButtonCameraLed(enabled: boolean): Promise<void>
103
104
  setButtonMaxRecordingTime(minutes: number): Promise<void>
104
105
  setCameraFov(fov: CameraFov): Promise<void>
105
106
  queryGalleryStatus(): Promise<void>
106
- requestPhoto(
107
- requestId: string,
108
- appId: string,
109
- size: PhotoSize,
110
- webhookUrl: string | null,
111
- authToken: string | null,
112
- compress: PhotoCompression,
113
- sound: boolean,
114
- ): Promise<void>
107
+ requestPhoto(params: PhotoRequestParams): Promise<void>
115
108
 
116
109
  // OTA Commands
117
110
  sendOtaStart(): Promise<void>
@@ -126,14 +119,14 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
126
119
 
127
120
  // Stream Commands
128
121
  startStream(params: StreamStartRequest): Promise<void>
122
+ startCloudManagedStream(params: StreamStartRequest): Promise<void>
129
123
  stopStream(): Promise<void>
130
- keepStreamAlive(params: StreamKeepAliveRequest): Promise<void>
124
+ sendCloudStreamKeepAlive(params: StreamKeepAliveRequest): Promise<void>
131
125
 
132
126
  // Microphone Commands
133
127
  setMicState(
134
128
  enabled: boolean,
135
129
  useGlassesMic?: boolean,
136
- bypassVad?: boolean,
137
130
  sendTranscript?: boolean,
138
131
  sendLc3Data?: boolean,
139
132
  ): Promise<void>
@@ -191,9 +184,26 @@ const DEFAULT_CONNECT_OPTIONS: Required<ConnectOptions> = {
191
184
 
192
185
  const DEFAULT_SCAN_TIMEOUT_MS = 15_000
193
186
 
194
- const CAMERA_FOV_SETTINGS: Record<CameraFov, CameraFovSetting> = {
195
- standard: {fov: 118, roiPosition: 0},
196
- wide: {fov: 118, roiPosition: 0},
187
+ const CAMERA_ROI_MIN = 0
188
+ const CAMERA_ROI_MAX = 2
189
+
190
+ function clampInteger(value: number, min: number, max: number): number {
191
+ return Math.max(min, Math.min(max, Math.round(value)))
192
+ }
193
+
194
+ function normalizeCameraFov(setting: CameraFov): Required<CameraFov> {
195
+ return {
196
+ fov: clampInteger(
197
+ Number.isFinite(setting.fov) ? setting.fov : CAMERA_FOV_DEFAULT,
198
+ CAMERA_FOV_MIN,
199
+ CAMERA_FOV_MAX,
200
+ ),
201
+ roiPosition: clampInteger(
202
+ Number.isFinite(setting.roiPosition ?? 0) ? (setting.roiPosition ?? 0) : 0,
203
+ CAMERA_ROI_MIN,
204
+ CAMERA_ROI_MAX,
205
+ ) as Required<CameraFov>["roiPosition"],
206
+ }
197
207
  }
198
208
 
199
209
  function searchResultsForModel(status: Partial<PublicBluetoothStatus>, model: DeviceModel): Device[] {
@@ -301,7 +311,6 @@ NativeBluetoothSdkModule.getDefaultDevice = function () {
301
311
  const nativeSetMicState = NativeBluetoothSdkModule.setMicState.bind(NativeBluetoothSdkModule) as (
302
312
  enabled: boolean,
303
313
  useGlassesMic: boolean,
304
- bypassVad: boolean,
305
314
  sendTranscript: boolean,
306
315
  sendLc3Data: boolean,
307
316
  ) => MaybePromise<void>
@@ -355,15 +364,23 @@ NativeBluetoothSdkModule.setScreenDisabled = function (disabled: boolean) {
355
364
  return this.updateBluetoothSettings({screen_disabled: disabled})
356
365
  }
357
366
 
367
+ NativeBluetoothSdkModule.setGalleryModeEnabled = function (enabled: boolean) {
368
+ return this.updateBluetoothSettings({gallery_mode: enabled})
369
+ }
370
+
371
+ NativeBluetoothSdkModule.setVoiceActivityDetectionEnabled = function (enabled: boolean) {
372
+ return this.updateBluetoothSettings({voice_activity_detection_enabled: enabled})
373
+ }
374
+
358
375
  NativeBluetoothSdkModule.setButtonPhotoSettings = function (size: ButtonPhotoSize) {
359
376
  return this.updateBluetoothSettings({button_photo_size: size})
360
377
  }
361
378
 
362
- NativeBluetoothSdkModule.setButtonVideoRecordingSettings = function (width: number, height: number, frameRate: number) {
379
+ NativeBluetoothSdkModule.setButtonVideoRecordingSettings = function (width: number, height: number, fps: number) {
363
380
  return this.updateBluetoothSettings({
364
381
  button_video_width: width,
365
382
  button_video_height: height,
366
- button_video_fps: frameRate,
383
+ button_video_fps: fps,
367
384
  })
368
385
  }
369
386
 
@@ -376,13 +393,15 @@ NativeBluetoothSdkModule.setButtonMaxRecordingTime = function (minutes: number)
376
393
  }
377
394
 
378
395
  NativeBluetoothSdkModule.setCameraFov = function (fov: CameraFov) {
379
- return this.updateBluetoothSettings({camera_fov: CAMERA_FOV_SETTINGS[fov]})
396
+ const setting = normalizeCameraFov(fov)
397
+ return this.updateBluetoothSettings({
398
+ camera_fov: {fov: setting.fov, roi_position: setting.roiPosition},
399
+ })
380
400
  }
381
401
 
382
402
  NativeBluetoothSdkModule.setMicState = function (
383
403
  enabled: boolean,
384
404
  useGlassesMic?: boolean,
385
- bypassVad?: boolean,
386
405
  sendTranscript?: boolean,
387
406
  sendLc3Data?: boolean,
388
407
  ) {
@@ -390,7 +409,6 @@ NativeBluetoothSdkModule.setMicState = function (
390
409
  nativeSetMicState(
391
410
  enabled,
392
411
  useGlassesMic ?? true,
393
- bypassVad ?? true,
394
412
  sendTranscript ?? false,
395
413
  sendLc3Data ?? false,
396
414
  ),
@@ -487,5 +505,40 @@ NativeBluetoothSdkModule.scan = async function (
487
505
  })
488
506
  }
489
507
 
508
+ /** Expo Android bridge rejects null values in Map<String, Any> — omit optional nullish fields. */
509
+ function photoRequestParamsForNative(params: PhotoRequestParams): Record<string, string | number | boolean> {
510
+ const payload: Record<string, string | number | boolean> = {
511
+ requestId: params.requestId,
512
+ appId: params.appId,
513
+ size: params.size,
514
+ webhookUrl: params.webhookUrl ?? "",
515
+ compress: params.compress,
516
+ flash: true,
517
+ sound: params.sound,
518
+ }
519
+ if (params.authToken != null && params.authToken.length > 0) {
520
+ payload.authToken = params.authToken
521
+ }
522
+ const exposureTimeNs = params.exposureTimeNs
523
+ const hasManualExposure = exposureTimeNs != null && Number.isFinite(exposureTimeNs) && exposureTimeNs > 0
524
+ if (hasManualExposure) {
525
+ payload.exposureTimeNs = exposureTimeNs
526
+ }
527
+ if (hasManualExposure && params.iso != null && Number.isFinite(params.iso) && params.iso > 0) {
528
+ payload.iso = Math.round(params.iso)
529
+ }
530
+ return payload
531
+ }
532
+
533
+ const nativeRequestPhoto = NativeBluetoothSdkModule.requestPhoto.bind(NativeBluetoothSdkModule)
534
+ NativeBluetoothSdkModule.requestPhoto = function (params: PhotoRequestParams) {
535
+ return nativeRequestPhoto(photoRequestParamsForNative(params) as unknown as PhotoRequestParams)
536
+ }
537
+
538
+ const nativeStartStream = NativeBluetoothSdkModule.startStream.bind(NativeBluetoothSdkModule)
539
+ NativeBluetoothSdkModule.startCloudManagedStream = function (params: StreamStartRequest) {
540
+ return nativeStartStream({...params, keepAliveMode: "external"} as StreamStartRequest)
541
+ }
542
+
490
543
  export default NativeBluetoothSdkModule
491
544
  export const BluetoothSdk = NativeBluetoothSdkModule as BluetoothSdkPublicModule
package/src/index.ts CHANGED
@@ -13,13 +13,15 @@ const PUBLIC_EVENT_NAMES = new Set<BluetoothSdkEventName>([
13
13
  "button_press",
14
14
  "touch_event",
15
15
  "head_up",
16
- "vad_status",
16
+ "voice_activity_detection_status",
17
+ "speaking_status",
17
18
  "battery_status",
18
19
  "local_transcription",
19
20
  "wifi_status_change",
20
21
  "hotspot_status_change",
21
22
  "hotspot_error",
22
23
  "photo_response",
24
+ "photo_status",
23
25
  "gallery_status",
24
26
  "compatible_glasses_search_stop",
25
27
  "swipe_volume_status",
@@ -32,7 +34,6 @@ const PUBLIC_EVENT_NAMES = new Set<BluetoothSdkEventName>([
32
34
  "mic_pcm",
33
35
  "mic_lc3",
34
36
  "stream_status",
35
- "keep_alive_ack",
36
37
  ])
37
38
 
38
39
  const addListener: BluetoothSdkPublicModule["addListener"] = (eventName, listener) => {
@@ -70,7 +71,8 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
70
71
  sendWifiCredentials: PrivateBluetoothSdkModule.sendWifiCredentials.bind(PrivateBluetoothSdkModule),
71
72
  forgetWifiNetwork: PrivateBluetoothSdkModule.forgetWifiNetwork.bind(PrivateBluetoothSdkModule),
72
73
  setHotspotState: PrivateBluetoothSdkModule.setHotspotState.bind(PrivateBluetoothSdkModule),
73
- setGalleryMode: PrivateBluetoothSdkModule.setGalleryMode.bind(PrivateBluetoothSdkModule),
74
+ setGalleryModeEnabled: PrivateBluetoothSdkModule.setGalleryModeEnabled.bind(PrivateBluetoothSdkModule),
75
+ setVoiceActivityDetectionEnabled: PrivateBluetoothSdkModule.setVoiceActivityDetectionEnabled.bind(PrivateBluetoothSdkModule),
74
76
  setButtonPhotoSettings: PrivateBluetoothSdkModule.setButtonPhotoSettings.bind(PrivateBluetoothSdkModule),
75
77
  setButtonVideoRecordingSettings: PrivateBluetoothSdkModule.setButtonVideoRecordingSettings.bind(PrivateBluetoothSdkModule),
76
78
  setButtonCameraLed: PrivateBluetoothSdkModule.setButtonCameraLed.bind(PrivateBluetoothSdkModule),
@@ -82,7 +84,6 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
82
84
  stopVideoRecording: PrivateBluetoothSdkModule.stopVideoRecording.bind(PrivateBluetoothSdkModule),
83
85
  startStream: PrivateBluetoothSdkModule.startStream.bind(PrivateBluetoothSdkModule),
84
86
  stopStream: PrivateBluetoothSdkModule.stopStream.bind(PrivateBluetoothSdkModule),
85
- keepStreamAlive: PrivateBluetoothSdkModule.keepStreamAlive.bind(PrivateBluetoothSdkModule),
86
87
  setMicState: PrivateBluetoothSdkModule.setMicState.bind(PrivateBluetoothSdkModule),
87
88
  setPreferredMic: PrivateBluetoothSdkModule.setPreferredMic.bind(PrivateBluetoothSdkModule),
88
89
  setOwnAppAudioPlaying: PrivateBluetoothSdkModule.setOwnAppAudioPlaying.bind(PrivateBluetoothSdkModule),
@@ -95,6 +96,9 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
95
96
  export default BluetoothSdk
96
97
 
97
98
  export {
99
+ CAMERA_FOV_DEFAULT,
100
+ CAMERA_FOV_MAX,
101
+ CAMERA_FOV_MIN,
98
102
  DeviceModels,
99
103
  isBusyGlassesConnectionStatus,
100
104
  isConnectedGlassesConnectionStatus,
@@ -117,6 +121,7 @@ export type {
117
121
  ButtonPhotoSize,
118
122
  ButtonPressEvent,
119
123
  CameraFov,
124
+ CameraRoiPosition,
120
125
  CompatibleGlassesSearchStopEvent,
121
126
  ConnectOptions,
122
127
  ConnectedGlassesConnectionStatus,
@@ -124,7 +129,6 @@ export type {
124
129
  Device,
125
130
  DeviceModel,
126
131
  EnabledHotspotStatus,
127
- GalleryMode,
128
132
  GalleryStatusEvent,
129
133
  GlassesConnectionStatus,
130
134
  GlassesMediaVolumeGetResult,
@@ -134,7 +138,6 @@ export type {
134
138
  HotspotErrorEvent,
135
139
  HotspotStatus,
136
140
  HotspotStatusChangeEvent,
137
- KeepAliveAckEvent,
138
141
  LocalTranscriptionEvent,
139
142
  LogEvent,
140
143
  MicLc3Event,
@@ -142,17 +145,21 @@ export type {
142
145
  MicPcmEvent,
143
146
  MicPreference,
144
147
  PairFailureEvent,
148
+ PhotoResolvedConfig,
145
149
  PhotoCompression,
146
150
  PhotoResponseEvent,
147
151
  PhotoSize,
152
+ PhotoStatusEvent,
153
+ PhotoStatusState,
148
154
  RgbLedAction,
149
155
  RgbLedColor,
150
156
  RgbLedControlResponseEvent,
151
157
  ScanModelOptions,
152
158
  ScanOptions,
153
159
  ScanResultsCallback,
160
+ SpeakingStatusEvent,
154
161
  StreamAudioConfig,
155
- StreamKeepAliveRequest,
162
+ StreamResolvedConfig,
156
163
  StreamStartRequest,
157
164
  StreamStatusEvent,
158
165
  StreamStatusLifecycleState,
@@ -162,7 +169,7 @@ export type {
162
169
  SwipeVolumeStatusEvent,
163
170
  SwitchStatusEvent,
164
171
  TouchEvent,
165
- VadStatusEvent,
172
+ VoiceActivityDetectionStatusEvent,
166
173
  WifiSearchResult,
167
174
  WifiStatus,
168
175
  WifiStatusChangeEvent,
@@ -0,0 +1,20 @@
1
+ export type PhotoReceiverUploadEvent = {
2
+ byteCount: number
3
+ fileUri: string
4
+ requestId?: string | null
5
+ }
6
+
7
+ export type PhotoReceiverStatusEvent = {
8
+ message: string
9
+ }
10
+
11
+ export type PhotoReceiverResult = {
12
+ host: string
13
+ port: number
14
+ uploadUrl: string
15
+ }
16
+
17
+ export type MentraPhotoReceiverModuleEvents = {
18
+ photoUpload: (event: PhotoReceiverUploadEvent) => void
19
+ receiverStatus: (event: PhotoReceiverStatusEvent) => void
20
+ }