@mentra/bluetooth-sdk 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +8 -2
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -4,8 +4,11 @@ export type GlassesNotReadyEvent = {
4
4
  message: string
5
5
  }
6
6
 
7
+ // NOTE: unlike most events below, the native module does NOT include a `type`
8
+ // field on the button_press payload — it sends only {buttonId, pressType,
9
+ // timestamp} (see BluetoothSdkModule on both iOS and Android). Consumers must
10
+ // filter on `pressType` / the "button_press" listener name, never `event.type`.
7
11
  export type ButtonPressEvent = {
8
- type: "button_press"
9
12
  buttonId: string
10
13
  pressType: "long" | "short"
11
14
  timestamp: number
@@ -27,6 +30,8 @@ export type VoiceActivityDetectionStatusEvent = {
27
30
  voiceActivityDetectionEnabled: boolean
28
31
  }
29
32
 
33
+ export const DEFAULT_VOICE_ACTIVITY_DETECTION_ENABLED = false
34
+
30
35
  export type SpeakingStatusEvent = {
31
36
  type: "speaking_status"
32
37
  speaking: boolean
@@ -41,34 +46,34 @@ export type BatteryStatusEvent = {
41
46
  }
42
47
 
43
48
  export type GlassesConnectionStatus =
44
- | {state: 'disconnected'}
45
- | {state: 'scanning'}
46
- | {state: 'connecting'}
47
- | {state: 'bonding'}
48
- | {state: 'connected'; fullyBooted: boolean}
49
+ | {state: "disconnected"}
50
+ | {state: "scanning"}
51
+ | {state: "connecting"}
52
+ | {state: "bonding"}
53
+ | {state: "connected"; fullyBooted: boolean}
49
54
 
50
- export type ConnectedGlassesConnectionStatus = Extract<GlassesConnectionStatus, {state: 'connected'}>
55
+ export type ConnectedGlassesConnectionStatus = Extract<GlassesConnectionStatus, {state: "connected"}>
51
56
 
52
57
  export function isConnectedGlassesConnectionStatus(
53
58
  status: GlassesConnectionStatus,
54
59
  ): status is ConnectedGlassesConnectionStatus {
55
- return status.state === 'connected'
60
+ return status.state === "connected"
56
61
  }
57
62
 
58
63
  export function isReadyGlassesConnectionStatus(status: GlassesConnectionStatus): boolean {
59
- return status.state === 'connected' && status.fullyBooted
64
+ return status.state === "connected" && status.fullyBooted
60
65
  }
61
66
 
62
67
  export function isBusyGlassesConnectionStatus(status: GlassesConnectionStatus): boolean {
63
- return status.state === 'scanning' || status.state === 'connecting' || status.state === 'bonding'
68
+ return status.state === "scanning" || status.state === "connecting" || status.state === "bonding"
64
69
  }
65
70
 
66
71
  export function createDisconnectedGlassesStatus(): Partial<GlassesStatus> {
67
72
  return {
68
- connection: {state: 'disconnected'},
69
- hotspot: {state: 'disabled'},
70
- voiceActivityDetectionEnabled: true,
71
- wifi: {state: 'disconnected'},
73
+ connection: {state: "disconnected"},
74
+ hotspot: {state: "disabled"},
75
+ voiceActivityDetectionEnabled: DEFAULT_VOICE_ACTIVITY_DETECTION_ENABLED,
76
+ wifi: {state: "disconnected"},
72
77
  }
73
78
  }
74
79
 
@@ -93,24 +98,24 @@ export type LogEvent = {
93
98
  message: string
94
99
  }
95
100
 
96
- export type WifiStatus = {state: 'disconnected'} | {state: 'connected'; ssid: string; localIp?: string}
101
+ export type WifiStatus = {state: "disconnected"} | {state: "connected"; ssid: string; localIp?: string}
97
102
 
98
- export type ConnectedWifiStatus = Extract<WifiStatus, {state: 'connected'}>
103
+ export type ConnectedWifiStatus = Extract<WifiStatus, {state: "connected"}>
99
104
 
100
105
  export function isConnectedWifiStatus(status: WifiStatus): status is ConnectedWifiStatus {
101
- return status.state === 'connected'
106
+ return status.state === "connected"
102
107
  }
103
108
 
104
109
  export type WifiStatusChangeEvent = WifiStatus & {
105
110
  type: "wifi_status_change"
106
111
  }
107
112
 
108
- export type HotspotStatus = {state: 'disabled'} | {state: 'enabled'; ssid: string; password: string; localIp: string}
113
+ export type HotspotStatus = {state: "disabled"} | {state: "enabled"; ssid: string; password: string; localIp: string}
109
114
 
110
- export type EnabledHotspotStatus = Extract<HotspotStatus, {state: 'enabled'}>
115
+ export type EnabledHotspotStatus = Extract<HotspotStatus, {state: "enabled"}>
111
116
 
112
117
  export function isEnabledHotspotStatus(status: HotspotStatus): status is EnabledHotspotStatus {
113
- return status.state === 'enabled'
118
+ return status.state === "enabled"
114
119
  }
115
120
 
116
121
  export type HotspotStatusChangeEvent = HotspotStatus & {
@@ -123,12 +128,37 @@ export type HotspotErrorEvent = {
123
128
  timestamp: number
124
129
  }
125
130
 
131
+ export type VersionInfoResult = {
132
+ androidVersion: string
133
+ firmwareVersion: string
134
+ besFirmwareVersion: string
135
+ mtkFirmwareVersion: string
136
+ buildNumber: string
137
+ systemTimeMs?: number
138
+ otaVersionUrl: string
139
+ appVersion: string
140
+ }
141
+
142
+ export type VersionInfoEvent = VersionInfoResult & {
143
+ type: "version_info"
144
+ }
145
+
146
+ export type WifiScanResultEvent = {
147
+ type: "wifi_scan_result"
148
+ networks: WifiSearchResult[]
149
+ scanComplete?: boolean
150
+ }
151
+
126
152
  export type PhotoResponseEvent =
127
153
  | {
128
154
  type: "photo_response"
129
155
  state: "success"
130
156
  requestId: string
131
157
  uploadUrl: string
158
+ photoUrl?: string
159
+ statusUrl?: string
160
+ contentType?: string
161
+ fileSizeBytes?: number
132
162
  timestamp: number
133
163
  }
134
164
  | {
@@ -140,6 +170,8 @@ export type PhotoResponseEvent =
140
170
  errorMessage: string
141
171
  }
142
172
 
173
+ export type PhotoSuccessResponseEvent = Extract<PhotoResponseEvent, {state: "success"}>
174
+
143
175
  export type PhotoStatusState =
144
176
  | "accepted"
145
177
  | "queued"
@@ -147,6 +179,7 @@ export type PhotoStatusState =
147
179
  | "capturing"
148
180
  | "captured"
149
181
  | "compressing"
182
+ | "ble_fallback_compression"
150
183
  | "uploading"
151
184
  | "uploaded"
152
185
  | "ready_for_transfer"
@@ -167,16 +200,105 @@ export type PhotoResolvedConfig = {
167
200
  iso?: number
168
201
  }
169
202
 
203
+ export type PhotoFpsRange = {
204
+ min?: number
205
+ max?: number
206
+ }
207
+
208
+ export type PhotoRequestedCaptureConfig = {
209
+ manual?: boolean
210
+ exposureTimeNs?: number
211
+ iso?: number
212
+ frameDurationNs?: number
213
+ aeMode?: number
214
+ aeLock?: boolean
215
+ aeExposureCompensation?: number
216
+ aeTargetFpsRange?: PhotoFpsRange
217
+ noiseReductionMode?: number
218
+ edgeMode?: number
219
+ afMode?: number
220
+ zsl?: boolean
221
+ }
222
+
223
+ export type PhotoMeteredPreview = {
224
+ exposureTimeNs?: number
225
+ iso?: number
226
+ totalLightProxy?: number
227
+ }
228
+
229
+ export type PhotoCaptureMetadata = {
230
+ manual?: boolean
231
+ exposureTimeNs?: number
232
+ iso?: number
233
+ frameDurationNs?: number
234
+ aeMode?: number
235
+ aeState?: number
236
+ aeStateName?: string
237
+ noiseReductionMode?: number
238
+ edgeMode?: number
239
+ zsl?: boolean
240
+ sensorTimestampNs?: number
241
+ totalLightProxy?: number
242
+ mfnrLikely?: boolean
243
+ }
244
+
170
245
  export type PhotoStatusEvent = {
171
246
  type: "photo_status"
172
247
  requestId: string
173
248
  status: PhotoStatusState | string
174
249
  timestamp: number
175
250
  resolvedConfig?: PhotoResolvedConfig
251
+ requestedCaptureConfig?: PhotoRequestedCaptureConfig
252
+ meteredPreview?: PhotoMeteredPreview
253
+ captureMetadata?: PhotoCaptureMetadata
176
254
  errorCode?: string
177
255
  errorMessage?: string
178
256
  }
179
257
 
258
+ export type VideoRecordingStatusEvent = {
259
+ type: "video_recording_status"
260
+ requestId?: string
261
+ success: boolean
262
+ status: VideoRecordingStatusState
263
+ details?: string | null
264
+ timestamp: number
265
+ data?: {
266
+ recording?: boolean
267
+ duration_ms?: number
268
+ duration_formatted?: string
269
+ [key: string]: unknown
270
+ }
271
+ }
272
+
273
+ export type VideoRecordingStatusState =
274
+ | "recording_started"
275
+ | "recording_status"
276
+ | "already_recording"
277
+ | "recording_stopped"
278
+ | "not_recording"
279
+ | "request_id_mismatch"
280
+ | "service_unavailable"
281
+ | "json_error"
282
+ | "battery_low"
283
+ | "camera_busy"
284
+ | "storage_unavailable"
285
+ | "integrity_failed"
286
+ | "error"
287
+
288
+ export type VideoRecordingStartedStatusEvent = Omit<VideoRecordingStatusEvent, "success" | "status"> & {
289
+ success: true
290
+ status: "recording_started"
291
+ }
292
+
293
+ export type VideoRecordingStoppedStatusEvent = Omit<VideoRecordingStatusEvent, "success" | "status"> & {
294
+ success: true
295
+ status: "recording_stopped"
296
+ }
297
+
298
+ export type VideoRecordingSuccessStatusEvent =
299
+ | VideoRecordingStartedStatusEvent
300
+ | VideoRecordingStoppedStatusEvent
301
+
180
302
  export type GalleryStatusEvent = {
181
303
  type: "gallery_status"
182
304
  photos: number
@@ -185,6 +307,7 @@ export type GalleryStatusEvent = {
185
307
  totalSize?: number
186
308
  hasContent: boolean
187
309
  cameraBusy: boolean
310
+ cameraBusyReason?: "video" | "stream" | (string & {})
188
311
  }
189
312
 
190
313
  export type CompatibleGlassesSearchStopEvent = {
@@ -232,11 +355,60 @@ export type RgbLedControlResponseEvent =
232
355
  errorCode: string
233
356
  }
234
357
 
358
+ export type RgbLedControlSuccessResponseEvent = Extract<RgbLedControlResponseEvent, {state: "success"}>
359
+
360
+ export type SettingsAckStatus = "applied" | "ready" | "error" | "failed" | "failure" | "rejected"
361
+
362
+ export type SettingsAckSetting =
363
+ | "gallery_mode"
364
+ | "button_photo"
365
+ | "button_video_recording"
366
+ | "button_camera_led"
367
+ | "button_max_recording_time"
368
+ | "camera_fov"
369
+
370
+ export type SettingsAckEvent = {
371
+ type: "settings_ack"
372
+ requestId: string
373
+ setting: SettingsAckSetting
374
+ status: SettingsAckStatus
375
+ timestamp: number
376
+ fov?: number
377
+ roiPosition?: CameraRoiPositionValue
378
+ hardwareApplied?: boolean
379
+ active?: boolean
380
+ size?: ButtonPhotoSize | string
381
+ width?: number
382
+ height?: number
383
+ fps?: number
384
+ enabled?: boolean
385
+ minutes?: number
386
+ errorCode?: string
387
+ errorMessage?: string
388
+ }
389
+
390
+ export type SettingsAckSuccessStatus = Exclude<SettingsAckStatus, "error" | "failed" | "failure" | "rejected">
391
+
392
+ export type SettingsAckSuccessEvent = Omit<SettingsAckEvent, "status"> & {
393
+ status: SettingsAckSuccessStatus
394
+ }
395
+
235
396
  export type RgbLedAction = "on" | "off"
236
397
  export type RgbLedColor = "red" | "green" | "blue" | "orange" | "white"
237
398
  export type PhotoSize = "small" | "medium" | "large" | "full"
238
399
  export type ButtonPhotoSize = "small" | "medium" | "large"
239
400
  export type PhotoCompression = "none" | "medium" | "heavy"
401
+
402
+ /**
403
+ * Optional per-recording video settings for {@link startVideoRecording}. When
404
+ * omitted, the glasses fall back to their saved button-video settings. Any
405
+ * field left undefined is omitted from the BLE command (glasses default applies).
406
+ */
407
+ export interface VideoRecordingSettings {
408
+ width?: number
409
+ height?: number
410
+ fps?: number
411
+ }
240
412
  export const DeviceModels = {
241
413
  Simulated: "Simulated Glasses",
242
414
  G1: "Even Realities G1",
@@ -258,25 +430,38 @@ export type DashboardMenuItem = {
258
430
  values?: Record<string, unknown>
259
431
  }
260
432
 
261
- export const CAMERA_FOV_MIN = 82
433
+ export const CAMERA_FOV_MIN = 62
262
434
  export const CAMERA_FOV_MAX = 118
263
435
  export const CAMERA_FOV_DEFAULT = 102
264
436
 
265
- export type CameraRoiPosition = 0 | 1 | 2
437
+ export type CameraRoiPosition = "center" | "bottom" | "top"
438
+ export type CameraRoiPositionValue = 0 | 1 | 2
439
+ export type CameraFovPreset = "narrow" | "standard" | "wide"
440
+
441
+ export type CameraFovRequest =
442
+ | {
443
+ fov: number
444
+ roiPosition?: CameraRoiPosition
445
+ }
446
+ | {
447
+ preset: CameraFovPreset
448
+ }
266
449
 
267
- export type CameraFov = {
450
+ export type CameraFovResult = {
451
+ requestId: string
268
452
  fov: number
269
- roiPosition?: CameraRoiPosition
453
+ roiPosition: CameraRoiPosition
454
+ timestamp: number
270
455
  }
271
456
 
272
457
  export type CameraFovSetting = {
273
458
  fov: number
274
- roiPosition: CameraRoiPosition
459
+ roiPosition: CameraRoiPositionValue
275
460
  }
276
461
 
277
462
  type NativeCameraFovSetting = {
278
463
  fov: number
279
- roi_position: CameraRoiPosition
464
+ roi_position: CameraRoiPositionValue
280
465
  }
281
466
 
282
467
  export type MicPreference = "auto" | "phone" | "glasses" | "bluetooth"
@@ -289,6 +474,7 @@ export type PhotoRequestParams = {
289
474
  webhookUrl: string | null
290
475
  authToken: string | null
291
476
  compress: PhotoCompression
477
+ save?: boolean
292
478
  sound: boolean
293
479
  exposureTimeNs?: number | null
294
480
  /** Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure. */
@@ -388,18 +574,27 @@ export type StreamStatusLifecycleState = "initializing" | "streaming" | "stoppin
388
574
  export type StreamStatusReconnectState = "reconnecting" | "reconnected" | "reconnect_failed"
389
575
  export type StreamStatusState = StreamStatusLifecycleState | StreamStatusReconnectState | "error"
390
576
 
577
+ /** Effective stream settings reported by the glasses after defaults and clamps. */
391
578
  export type StreamResolvedConfig = {
392
579
  transport?: "rtmp" | "srt" | "whip"
393
580
  video?: {
581
+ /** Encoded output width sent to the stream endpoint. */
394
582
  width: number
583
+ /** Encoded output height sent to the stream endpoint. */
395
584
  height: number
585
+ /** Native camera buffer width selected before crop/downscale. */
396
586
  captureWidth?: number
587
+ /** Native camera buffer height selected before crop/downscale. */
397
588
  captureHeight?: number
589
+ /** Encoded video bitrate in bits per second. */
398
590
  bitrate: number
591
+ /** Resolved capture/encode frame rate. */
399
592
  fps: number
400
593
  }
401
594
  audio?: {
595
+ /** Encoded audio bitrate in bits per second. */
402
596
  bitrate?: number
597
+ /** Audio sample rate in Hz. */
403
598
  sampleRate?: number
404
599
  echoCancellation?: boolean
405
600
  noiseSuppression?: boolean
@@ -492,14 +687,16 @@ export type OtaStatusEvent = {
492
687
  session_id: string
493
688
  total_steps: number
494
689
  current_step: number
495
- step_type: 'apk' | 'mtk' | 'bes'
496
- phase: 'download' | 'install'
690
+ step_type: "apk" | "mtk" | "bes"
691
+ phase: "download" | "install"
497
692
  step_percent: number
498
693
  overall_percent: number
499
- status: 'in_progress' | 'step_complete' | 'complete' | 'failed' | 'idle'
694
+ status: "in_progress" | "step_complete" | "complete" | "failed" | "idle"
500
695
  error_message?: string
501
696
  }
502
697
 
698
+ export type OtaQueryResult = OtaUpdateAvailableEvent | OtaStatusEvent
699
+
503
700
  /** Nex BLE protobuf trace (NexEventUtils); payload matches native Map keys. */
504
701
  export type BleCommandTraceEvent = {
505
702
  command: string
@@ -531,10 +728,12 @@ export type BluetoothSdkModuleEvents = {
531
728
  battery_status: (event: BatteryStatusEvent) => void
532
729
  local_transcription: (event: LocalTranscriptionEvent) => void
533
730
  wifi_status_change: (event: WifiStatusChangeEvent) => void
731
+ wifi_scan_result: (event: WifiScanResultEvent) => void
534
732
  hotspot_status_change: (event: HotspotStatusChangeEvent) => void
535
733
  hotspot_error: (event: HotspotErrorEvent) => void
536
734
  photo_response: (event: PhotoResponseEvent) => void
537
735
  photo_status: (event: PhotoStatusEvent) => void
736
+ video_recording_status: (event: VideoRecordingStatusEvent) => void
538
737
  gallery_status: (event: GalleryStatusEvent) => void
539
738
  compatible_glasses_search_stop: (event: CompatibleGlassesSearchStopEvent) => void
540
739
  heartbeat_sent: (event: HeartbeatSentEvent) => void
@@ -542,6 +741,7 @@ export type BluetoothSdkModuleEvents = {
542
741
  swipe_volume_status: (event: SwipeVolumeStatusEvent) => void
543
742
  switch_status: (event: SwitchStatusEvent) => void
544
743
  rgb_led_control_response: (event: RgbLedControlResponseEvent) => void
744
+ settings_ack: (event: SettingsAckEvent) => void
545
745
  pair_failure: (event: PairFailureEvent) => void
546
746
  audio_pairing_needed: (event: AudioPairingNeededEvent) => void
547
747
  audio_connected: (event: AudioConnectedEvent) => void
@@ -557,9 +757,17 @@ export type BluetoothSdkModuleEvents = {
557
757
  ota_update_available: (event: OtaUpdateAvailableEvent) => void
558
758
  ota_start_ack: (event: OtaStartAckEvent) => void
559
759
  ota_status: (event: OtaStatusEvent) => void
760
+ version_info: (event: VersionInfoEvent) => void
560
761
  send_command_to_ble: (event: BleCommandTraceEvent) => void
561
762
  receive_command_from_ble: (event: BleCommandTraceEvent) => void
562
763
  miniapp_selected: (event: MiniappSelectedEvent) => void
764
+ extraction_progress: (event: ExtractionProgressEvent) => void
765
+ }
766
+
767
+ export interface ExtractionProgressEvent {
768
+ percentage: number
769
+ bytesRead: number
770
+ totalBytes: number
563
771
  }
564
772
 
565
773
  export type PublicGlassesStatus = Omit<
@@ -594,15 +802,18 @@ export type BluetoothSdkEventMap = {
594
802
  battery_status: BatteryStatusEvent
595
803
  local_transcription: LocalTranscriptionEvent
596
804
  wifi_status_change: WifiStatusChangeEvent
805
+ wifi_scan_result: WifiScanResultEvent
597
806
  hotspot_status_change: HotspotStatusChangeEvent
598
807
  hotspot_error: HotspotErrorEvent
599
808
  photo_response: PhotoResponseEvent
600
809
  photo_status: PhotoStatusEvent
810
+ video_recording_status: VideoRecordingStatusEvent
601
811
  gallery_status: GalleryStatusEvent
602
812
  compatible_glasses_search_stop: CompatibleGlassesSearchStopEvent
603
813
  swipe_volume_status: SwipeVolumeStatusEvent
604
814
  switch_status: SwitchStatusEvent
605
815
  rgb_led_control_response: RgbLedControlResponseEvent
816
+ settings_ack: SettingsAckEvent
606
817
  pair_failure: PairFailureEvent
607
818
  audio_pairing_needed: AudioPairingNeededEvent
608
819
  audio_connected: AudioConnectedEvent
@@ -610,6 +821,11 @@ export type BluetoothSdkEventMap = {
610
821
  mic_pcm: MicPcmEvent
611
822
  mic_lc3: MicLc3Event
612
823
  stream_status: StreamStatusEvent
824
+ ota_update_available: OtaUpdateAvailableEvent
825
+ ota_start_ack: OtaStartAckEvent
826
+ ota_status: OtaStatusEvent
827
+ version_info: VersionInfoEvent
828
+ extraction_progress: ExtractionProgressEvent
613
829
  }
614
830
 
615
831
  export type BluetoothSdkEventName = keyof BluetoothSdkEventMap
@@ -651,32 +867,32 @@ export interface BluetoothSdkPublicModule {
651
867
  setHeadUpAngle(angleDegrees: number): Promise<void>
652
868
  setScreenDisabled(disabled: boolean): Promise<void>
653
869
 
654
- requestWifiScan(): Promise<void>
655
- sendWifiCredentials(ssid: string, password: string): Promise<void>
656
- forgetWifiNetwork(ssid: string): Promise<void>
657
- setHotspotState(enabled: boolean): Promise<void>
870
+ requestWifiScan(): Promise<WifiSearchResult[]>
871
+ sendWifiCredentials(ssid: string, password: string): Promise<WifiStatusChangeEvent>
872
+ forgetWifiNetwork(ssid: string): Promise<WifiStatusChangeEvent>
873
+ setHotspotState(enabled: boolean): Promise<HotspotStatusChangeEvent>
658
874
 
659
- setGalleryModeEnabled(enabled: boolean): Promise<void>
875
+ setGalleryModeEnabled(enabled: boolean): Promise<SettingsAckSuccessEvent>
660
876
  setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
661
- setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
662
- setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
663
- setButtonCameraLed(enabled: boolean): Promise<void>
664
- setButtonMaxRecordingTime(minutes: number): Promise<void>
665
- setCameraFov(fov: CameraFov): Promise<void>
666
- queryGalleryStatus(): Promise<void>
667
- requestPhoto(params: PhotoRequestParams): Promise<void>
668
- startVideoRecording(requestId: string, save: boolean, sound: boolean): Promise<void>
669
- stopVideoRecording(requestId: string): Promise<void>
670
-
671
- startStream(params: StreamStartRequest): Promise<void>
672
- stopStream(): Promise<void>
673
-
674
- setMicState(
675
- enabled: boolean,
676
- useGlassesMic?: boolean,
677
- sendTranscript?: boolean,
678
- sendLc3Data?: boolean,
679
- ): Promise<void>
877
+ setButtonPhotoSettings(size: ButtonPhotoSize): Promise<SettingsAckSuccessEvent>
878
+ setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<SettingsAckSuccessEvent>
879
+ setButtonCameraLed(enabled: boolean): Promise<SettingsAckSuccessEvent>
880
+ setButtonMaxRecordingTime(minutes: number): Promise<SettingsAckSuccessEvent>
881
+ setCameraFov(request: CameraFovRequest): Promise<CameraFovResult>
882
+ queryGalleryStatus(): Promise<GalleryStatusEvent>
883
+ requestPhoto(params: PhotoRequestParams): Promise<PhotoSuccessResponseEvent>
884
+ startVideoRecording(
885
+ requestId: string,
886
+ save: boolean,
887
+ sound: boolean,
888
+ settings?: VideoRecordingSettings,
889
+ ): Promise<VideoRecordingStartedStatusEvent>
890
+ stopVideoRecording(requestId: string): Promise<VideoRecordingStoppedStatusEvent>
891
+
892
+ startStream(params: StreamStartRequest): Promise<StreamStatusEvent>
893
+ stopStream(): Promise<StreamStatusEvent>
894
+
895
+ setMicState(enabled: boolean, useGlassesMic?: boolean, sendTranscript?: boolean, sendLc3Data?: boolean): Promise<void>
680
896
  setPreferredMic(preferredMic: MicPreference): Promise<void>
681
897
  setOwnAppAudioPlaying(playing: boolean): Promise<void>
682
898
  getGlassesMediaVolume(): Promise<GlassesMediaVolumeGetResult>
@@ -690,9 +906,52 @@ export interface BluetoothSdkPublicModule {
690
906
  onDurationMs: number,
691
907
  offDurationMs: number,
692
908
  count: number,
693
- ): Promise<void>
694
-
695
- requestVersionInfo(): Promise<void>
909
+ ): Promise<RgbLedControlSuccessResponseEvent>
910
+
911
+ requestVersionInfo(): Promise<VersionInfoResult>
912
+ /** Ask connected Mentra Live glasses to check/report OTA availability and status. */
913
+ checkForOtaUpdate(): Promise<OtaQueryResult>
914
+ /** Start the OTA flow after your app has presented the available update to the user. */
915
+ startOtaUpdate(): Promise<OtaStartAckEvent>
916
+ /** Re-run the glasses-side OTA version check, mainly after correcting clock skew/TLS failures. */
917
+ retryOtaVersionCheck(): Promise<OtaQueryResult>
918
+
919
+ // // stt commands (MOVE TO CRUST)
920
+ // setSttModelDetails(path: string, languageCode: string): Promise<void>
921
+ // getSttModelPath(): Promise<string>
922
+ // checkSttModelAvailable(): Promise<boolean>
923
+ // validateSttModel(path: string): Promise<boolean>
924
+ // extractTarBz2(sourcePath: string, destinationPath: string): Promise<boolean>
925
+
926
+ // // tts commands (MOVE TO CRUST)
927
+ // setTtsModelDetails(path: string, languageCode: string): Promise<void>
928
+ // getTtsModelPath(): Promise<string>
929
+ // getTtsModelLanguage(): Promise<string>
930
+ // checkTtsModelAvailable(): Promise<boolean>
931
+ // validateTtsModel(path: string): Promise<boolean>
932
+ // generateTtsAudio(text: string, path: string, outputPath: string, speakerId: number, speed: number): Promise<boolean>
933
+
934
+ // STT Commands (TODO: MOVE TO CRUST)
935
+ setSttModelDetails(path: string, languageCode: string): Promise<void>
936
+ getSttModelPath(): Promise<string>
937
+ checkSttModelAvailable(): Promise<boolean>
938
+ validateSttModel(path: string): Promise<boolean>
939
+ extractTarBz2(sourcePath: string, destinationPath: string): Promise<boolean>
940
+ restartTranscriber(): Promise<void>
941
+
942
+ // TTS Commands (TODO: MOVE TO CRUST)
943
+ setTtsModelDetails(path: string, languageCode: string): Promise<void>
944
+ getTtsModelPath(): Promise<string>
945
+ getTtsModelLanguage(): Promise<string>
946
+ checkTtsModelAvailable(): Promise<boolean>
947
+ validateTtsModel(path: string): Promise<boolean>
948
+ generateTtsAudio(
949
+ text: string,
950
+ modelPath: string,
951
+ outputPath: string,
952
+ speakerId: number,
953
+ speed: number,
954
+ ): Promise<boolean>
696
955
  }
697
956
 
698
957
  // OTA update status types
@@ -703,11 +962,11 @@ export interface OtaStatus {
703
962
  sessionId: string
704
963
  totalSteps: number
705
964
  currentStep: number
706
- stepType: 'apk' | 'mtk' | 'bes'
707
- phase: 'download' | 'install'
965
+ stepType: "apk" | "mtk" | "bes"
966
+ phase: "download" | "install"
708
967
  stepPercent: number
709
968
  overallPercent: number
710
- status: 'in_progress' | 'step_complete' | 'complete' | 'failed' | 'idle'
969
+ status: "in_progress" | "step_complete" | "complete" | "failed" | "idle"
711
970
  error?: string
712
971
  }
713
972
 
@@ -749,6 +1008,8 @@ export interface GlassesStatus {
749
1008
  leftMacAddress: string
750
1009
  rightMacAddress: string
751
1010
  buildNumber: string
1011
+ /** Glasses System.currentTimeMillis() from last version_info (clock skew detection). */
1012
+ systemTimeMs?: number
752
1013
  otaVersionUrl: string
753
1014
  appVersion: string
754
1015
  bluetoothName: string
@@ -784,8 +1045,16 @@ export interface CoreDashboardMenuItem {
784
1045
  running: boolean
785
1046
  }
786
1047
 
1048
+ export interface CalendarEvent {
1049
+ title: string
1050
+ location?: string
1051
+ time: string
1052
+ endDate: number
1053
+ }
1054
+
787
1055
  export interface CoreSettings {
788
1056
  menu_apps: CoreDashboardMenuItem[]
1057
+ calendar_events: CalendarEvent[]
789
1058
  }
790
1059
 
791
1060
  export interface Device {
@@ -876,6 +1145,9 @@ export type BluetoothSettingsUpdate = Partial<{
876
1145
  dashboard_height: number
877
1146
  dashboard_depth: number
878
1147
  menu_apps: DashboardMenuItem[] | CoreDashboardMenuItem[] | Array<Record<string, unknown>> | null
1148
+ calendar_events: CalendarEvent[]
1149
+ metric_system: boolean
1150
+ twelve_hour_time: boolean
879
1151
  gallery_mode: boolean
880
1152
  voice_activity_detection_enabled: boolean
881
1153
  button_photo_size: ButtonPhotoSize