@mentra/bluetooth-sdk 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +253 -44
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +37 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +43 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +25 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +177 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +68 -19
  98. package/ios/Source/DeviceManager.swift +167 -32
  99. package/ios/Source/DeviceStore.swift +24 -8
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +5 -5
  111. package/ios/Source/sgcs/G2.swift +1494 -870
  112. package/ios/Source/sgcs/Mach1.swift +4 -4
  113. package/ios/Source/sgcs/MentraLive.swift +814 -321
  114. package/ios/Source/sgcs/MentraNex.swift +604 -244
  115. package/ios/Source/sgcs/SGCManager.swift +68 -5
  116. package/ios/Source/sgcs/Simulated.swift +9 -5
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +344 -60
  131. package/src/_private/BluetoothSdkModule.ts +132 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +61 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -4,8 +4,11 @@ export type GlassesNotReadyEvent = {
4
4
  message: string
5
5
  }
6
6
 
7
+ // NOTE: unlike most events below, the native module does NOT include a `type`
8
+ // field on the button_press payload — it sends only {buttonId, pressType,
9
+ // timestamp} (see BluetoothSdkModule on both iOS and Android). Consumers must
10
+ // filter on `pressType` / the "button_press" listener name, never `event.type`.
7
11
  export type ButtonPressEvent = {
8
- type: "button_press"
9
12
  buttonId: string
10
13
  pressType: "long" | "short"
11
14
  timestamp: number
@@ -18,6 +21,14 @@ export type TouchEvent = {
18
21
  timestamp: number
19
22
  }
20
23
 
24
+ export type AccelEvent = {
25
+ type: "accel_event"
26
+ x: number
27
+ y: number
28
+ z: number
29
+ timestamp: number
30
+ }
31
+
21
32
  export type HeadUpEvent = {
22
33
  up: boolean
23
34
  }
@@ -27,6 +38,8 @@ export type VoiceActivityDetectionStatusEvent = {
27
38
  voiceActivityDetectionEnabled: boolean
28
39
  }
29
40
 
41
+ export const DEFAULT_VOICE_ACTIVITY_DETECTION_ENABLED = false
42
+
30
43
  export type SpeakingStatusEvent = {
31
44
  type: "speaking_status"
32
45
  speaking: boolean
@@ -41,34 +54,34 @@ export type BatteryStatusEvent = {
41
54
  }
42
55
 
43
56
  export type GlassesConnectionStatus =
44
- | {state: 'disconnected'}
45
- | {state: 'scanning'}
46
- | {state: 'connecting'}
47
- | {state: 'bonding'}
48
- | {state: 'connected'; fullyBooted: boolean}
57
+ | {state: "disconnected"}
58
+ | {state: "scanning"}
59
+ | {state: "connecting"}
60
+ | {state: "bonding"}
61
+ | {state: "connected"; fullyBooted: boolean}
49
62
 
50
- export type ConnectedGlassesConnectionStatus = Extract<GlassesConnectionStatus, {state: 'connected'}>
63
+ export type ConnectedGlassesConnectionStatus = Extract<GlassesConnectionStatus, {state: "connected"}>
51
64
 
52
65
  export function isConnectedGlassesConnectionStatus(
53
66
  status: GlassesConnectionStatus,
54
67
  ): status is ConnectedGlassesConnectionStatus {
55
- return status.state === 'connected'
68
+ return status.state === "connected"
56
69
  }
57
70
 
58
71
  export function isReadyGlassesConnectionStatus(status: GlassesConnectionStatus): boolean {
59
- return status.state === 'connected' && status.fullyBooted
72
+ return status.state === "connected" && status.fullyBooted
60
73
  }
61
74
 
62
75
  export function isBusyGlassesConnectionStatus(status: GlassesConnectionStatus): boolean {
63
- return status.state === 'scanning' || status.state === 'connecting' || status.state === 'bonding'
76
+ return status.state === "scanning" || status.state === "connecting" || status.state === "bonding"
64
77
  }
65
78
 
66
79
  export function createDisconnectedGlassesStatus(): Partial<GlassesStatus> {
67
80
  return {
68
- connection: {state: 'disconnected'},
69
- hotspot: {state: 'disabled'},
70
- voiceActivityDetectionEnabled: true,
71
- wifi: {state: 'disconnected'},
81
+ connection: {state: "disconnected"},
82
+ hotspot: {state: "disabled"},
83
+ voiceActivityDetectionEnabled: DEFAULT_VOICE_ACTIVITY_DETECTION_ENABLED,
84
+ wifi: {state: "disconnected"},
72
85
  }
73
86
  }
74
87
 
@@ -93,24 +106,24 @@ export type LogEvent = {
93
106
  message: string
94
107
  }
95
108
 
96
- export type WifiStatus = {state: 'disconnected'} | {state: 'connected'; ssid: string; localIp?: string}
109
+ export type WifiStatus = {state: "disconnected"} | {state: "connected"; ssid: string; localIp?: string}
97
110
 
98
- export type ConnectedWifiStatus = Extract<WifiStatus, {state: 'connected'}>
111
+ export type ConnectedWifiStatus = Extract<WifiStatus, {state: "connected"}>
99
112
 
100
113
  export function isConnectedWifiStatus(status: WifiStatus): status is ConnectedWifiStatus {
101
- return status.state === 'connected'
114
+ return status.state === "connected"
102
115
  }
103
116
 
104
117
  export type WifiStatusChangeEvent = WifiStatus & {
105
118
  type: "wifi_status_change"
106
119
  }
107
120
 
108
- export type HotspotStatus = {state: 'disabled'} | {state: 'enabled'; ssid: string; password: string; localIp: string}
121
+ export type HotspotStatus = {state: "disabled"} | {state: "enabled"; ssid: string; password: string; localIp: string}
109
122
 
110
- export type EnabledHotspotStatus = Extract<HotspotStatus, {state: 'enabled'}>
123
+ export type EnabledHotspotStatus = Extract<HotspotStatus, {state: "enabled"}>
111
124
 
112
125
  export function isEnabledHotspotStatus(status: HotspotStatus): status is EnabledHotspotStatus {
113
- return status.state === 'enabled'
126
+ return status.state === "enabled"
114
127
  }
115
128
 
116
129
  export type HotspotStatusChangeEvent = HotspotStatus & {
@@ -123,12 +136,37 @@ export type HotspotErrorEvent = {
123
136
  timestamp: number
124
137
  }
125
138
 
139
+ export type VersionInfoResult = {
140
+ androidVersion: string
141
+ firmwareVersion: string
142
+ besFirmwareVersion: string
143
+ mtkFirmwareVersion: string
144
+ buildNumber: string
145
+ systemTimeMs?: number
146
+ otaVersionUrl: string
147
+ appVersion: string
148
+ }
149
+
150
+ export type VersionInfoEvent = VersionInfoResult & {
151
+ type: "version_info"
152
+ }
153
+
154
+ export type WifiScanResultEvent = {
155
+ type: "wifi_scan_result"
156
+ networks: WifiSearchResult[]
157
+ scanComplete?: boolean
158
+ }
159
+
126
160
  export type PhotoResponseEvent =
127
161
  | {
128
162
  type: "photo_response"
129
163
  state: "success"
130
164
  requestId: string
131
165
  uploadUrl: string
166
+ photoUrl?: string
167
+ statusUrl?: string
168
+ contentType?: string
169
+ fileSizeBytes?: number
132
170
  timestamp: number
133
171
  }
134
172
  | {
@@ -140,6 +178,8 @@ export type PhotoResponseEvent =
140
178
  errorMessage: string
141
179
  }
142
180
 
181
+ export type PhotoSuccessResponseEvent = Extract<PhotoResponseEvent, {state: "success"}>
182
+
143
183
  export type PhotoStatusState =
144
184
  | "accepted"
145
185
  | "queued"
@@ -147,6 +187,7 @@ export type PhotoStatusState =
147
187
  | "capturing"
148
188
  | "captured"
149
189
  | "compressing"
190
+ | "ble_fallback_compression"
150
191
  | "uploading"
151
192
  | "uploaded"
152
193
  | "ready_for_transfer"
@@ -167,16 +208,105 @@ export type PhotoResolvedConfig = {
167
208
  iso?: number
168
209
  }
169
210
 
211
+ export type PhotoFpsRange = {
212
+ min?: number
213
+ max?: number
214
+ }
215
+
216
+ export type PhotoRequestedCaptureConfig = {
217
+ manual?: boolean
218
+ exposureTimeNs?: number
219
+ iso?: number
220
+ frameDurationNs?: number
221
+ aeMode?: number
222
+ aeLock?: boolean
223
+ aeExposureCompensation?: number
224
+ aeTargetFpsRange?: PhotoFpsRange
225
+ noiseReductionMode?: number
226
+ edgeMode?: number
227
+ afMode?: number
228
+ zsl?: boolean
229
+ }
230
+
231
+ export type PhotoMeteredPreview = {
232
+ exposureTimeNs?: number
233
+ iso?: number
234
+ totalLightProxy?: number
235
+ }
236
+
237
+ export type PhotoCaptureMetadata = {
238
+ manual?: boolean
239
+ exposureTimeNs?: number
240
+ iso?: number
241
+ frameDurationNs?: number
242
+ aeMode?: number
243
+ aeState?: number
244
+ aeStateName?: string
245
+ noiseReductionMode?: number
246
+ edgeMode?: number
247
+ zsl?: boolean
248
+ sensorTimestampNs?: number
249
+ totalLightProxy?: number
250
+ mfnrLikely?: boolean
251
+ }
252
+
170
253
  export type PhotoStatusEvent = {
171
254
  type: "photo_status"
172
255
  requestId: string
173
256
  status: PhotoStatusState | string
174
257
  timestamp: number
175
258
  resolvedConfig?: PhotoResolvedConfig
259
+ requestedCaptureConfig?: PhotoRequestedCaptureConfig
260
+ meteredPreview?: PhotoMeteredPreview
261
+ captureMetadata?: PhotoCaptureMetadata
176
262
  errorCode?: string
177
263
  errorMessage?: string
178
264
  }
179
265
 
266
+ export type VideoRecordingStatusEvent = {
267
+ type: "video_recording_status"
268
+ requestId?: string
269
+ success: boolean
270
+ status: VideoRecordingStatusState
271
+ details?: string | null
272
+ timestamp: number
273
+ data?: {
274
+ recording?: boolean
275
+ duration_ms?: number
276
+ duration_formatted?: string
277
+ [key: string]: unknown
278
+ }
279
+ }
280
+
281
+ export type VideoRecordingStatusState =
282
+ | "recording_started"
283
+ | "recording_status"
284
+ | "already_recording"
285
+ | "recording_stopped"
286
+ | "not_recording"
287
+ | "request_id_mismatch"
288
+ | "service_unavailable"
289
+ | "json_error"
290
+ | "battery_low"
291
+ | "camera_busy"
292
+ | "storage_unavailable"
293
+ | "integrity_failed"
294
+ | "error"
295
+
296
+ export type VideoRecordingStartedStatusEvent = Omit<VideoRecordingStatusEvent, "success" | "status"> & {
297
+ success: true
298
+ status: "recording_started"
299
+ }
300
+
301
+ export type VideoRecordingStoppedStatusEvent = Omit<VideoRecordingStatusEvent, "success" | "status"> & {
302
+ success: true
303
+ status: "recording_stopped"
304
+ }
305
+
306
+ export type VideoRecordingSuccessStatusEvent =
307
+ | VideoRecordingStartedStatusEvent
308
+ | VideoRecordingStoppedStatusEvent
309
+
180
310
  export type GalleryStatusEvent = {
181
311
  type: "gallery_status"
182
312
  photos: number
@@ -185,6 +315,7 @@ export type GalleryStatusEvent = {
185
315
  totalSize?: number
186
316
  hasContent: boolean
187
317
  cameraBusy: boolean
318
+ cameraBusyReason?: "video" | "stream" | (string & {})
188
319
  }
189
320
 
190
321
  export type CompatibleGlassesSearchStopEvent = {
@@ -232,11 +363,60 @@ export type RgbLedControlResponseEvent =
232
363
  errorCode: string
233
364
  }
234
365
 
366
+ export type RgbLedControlSuccessResponseEvent = Extract<RgbLedControlResponseEvent, {state: "success"}>
367
+
368
+ export type SettingsAckStatus = "applied" | "ready" | "error" | "failed" | "failure" | "rejected"
369
+
370
+ export type SettingsAckSetting =
371
+ | "gallery_mode"
372
+ | "button_photo"
373
+ | "button_video_recording"
374
+ | "button_camera_led"
375
+ | "button_max_recording_time"
376
+ | "camera_fov"
377
+
378
+ export type SettingsAckEvent = {
379
+ type: "settings_ack"
380
+ requestId: string
381
+ setting: SettingsAckSetting
382
+ status: SettingsAckStatus
383
+ timestamp: number
384
+ fov?: number
385
+ roiPosition?: CameraRoiPositionValue
386
+ hardwareApplied?: boolean
387
+ active?: boolean
388
+ size?: ButtonPhotoSize | string
389
+ width?: number
390
+ height?: number
391
+ fps?: number
392
+ enabled?: boolean
393
+ minutes?: number
394
+ errorCode?: string
395
+ errorMessage?: string
396
+ }
397
+
398
+ export type SettingsAckSuccessStatus = Exclude<SettingsAckStatus, "error" | "failed" | "failure" | "rejected">
399
+
400
+ export type SettingsAckSuccessEvent = Omit<SettingsAckEvent, "status"> & {
401
+ status: SettingsAckSuccessStatus
402
+ }
403
+
235
404
  export type RgbLedAction = "on" | "off"
236
405
  export type RgbLedColor = "red" | "green" | "blue" | "orange" | "white"
237
406
  export type PhotoSize = "small" | "medium" | "large" | "full"
238
- export type ButtonPhotoSize = "small" | "medium" | "large"
407
+ export type ButtonPhotoSize = "small" | "medium" | "large" | "max"
239
408
  export type PhotoCompression = "none" | "medium" | "heavy"
409
+
410
+ /**
411
+ * Optional per-recording video settings for {@link startVideoRecording}. When
412
+ * omitted, the glasses fall back to their saved button-video settings. Any
413
+ * field left undefined is omitted from the BLE command (glasses default applies).
414
+ */
415
+ export interface VideoRecordingSettings {
416
+ width?: number
417
+ height?: number
418
+ fps?: number
419
+ }
240
420
  export const DeviceModels = {
241
421
  Simulated: "Simulated Glasses",
242
422
  G1: "Even Realities G1",
@@ -258,25 +438,38 @@ export type DashboardMenuItem = {
258
438
  values?: Record<string, unknown>
259
439
  }
260
440
 
261
- export const CAMERA_FOV_MIN = 82
441
+ export const CAMERA_FOV_MIN = 62
262
442
  export const CAMERA_FOV_MAX = 118
263
443
  export const CAMERA_FOV_DEFAULT = 102
264
444
 
265
- export type CameraRoiPosition = 0 | 1 | 2
445
+ export type CameraRoiPosition = "center" | "bottom" | "top"
446
+ export type CameraRoiPositionValue = 0 | 1 | 2
447
+ export type CameraFovPreset = "narrow" | "standard" | "wide"
266
448
 
267
- export type CameraFov = {
449
+ export type CameraFovRequest =
450
+ | {
451
+ fov: number
452
+ roiPosition?: CameraRoiPosition
453
+ }
454
+ | {
455
+ preset: CameraFovPreset
456
+ }
457
+
458
+ export type CameraFovResult = {
459
+ requestId: string
268
460
  fov: number
269
- roiPosition?: CameraRoiPosition
461
+ roiPosition: CameraRoiPosition
462
+ timestamp: number
270
463
  }
271
464
 
272
465
  export type CameraFovSetting = {
273
466
  fov: number
274
- roiPosition: CameraRoiPosition
467
+ roiPosition: CameraRoiPositionValue
275
468
  }
276
469
 
277
470
  type NativeCameraFovSetting = {
278
471
  fov: number
279
- roi_position: CameraRoiPosition
472
+ roi_position: CameraRoiPositionValue
280
473
  }
281
474
 
282
475
  export type MicPreference = "auto" | "phone" | "glasses" | "bluetooth"
@@ -289,6 +482,7 @@ export type PhotoRequestParams = {
289
482
  webhookUrl: string | null
290
483
  authToken: string | null
291
484
  compress: PhotoCompression
485
+ save?: boolean
292
486
  sound: boolean
293
487
  exposureTimeNs?: number | null
294
488
  /** Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure. */
@@ -388,18 +582,27 @@ export type StreamStatusLifecycleState = "initializing" | "streaming" | "stoppin
388
582
  export type StreamStatusReconnectState = "reconnecting" | "reconnected" | "reconnect_failed"
389
583
  export type StreamStatusState = StreamStatusLifecycleState | StreamStatusReconnectState | "error"
390
584
 
585
+ /** Effective stream settings reported by the glasses after defaults and clamps. */
391
586
  export type StreamResolvedConfig = {
392
587
  transport?: "rtmp" | "srt" | "whip"
393
588
  video?: {
589
+ /** Encoded output width sent to the stream endpoint. */
394
590
  width: number
591
+ /** Encoded output height sent to the stream endpoint. */
395
592
  height: number
593
+ /** Native camera buffer width selected before crop/downscale. */
396
594
  captureWidth?: number
595
+ /** Native camera buffer height selected before crop/downscale. */
397
596
  captureHeight?: number
597
+ /** Encoded video bitrate in bits per second. */
398
598
  bitrate: number
599
+ /** Resolved capture/encode frame rate. */
399
600
  fps: number
400
601
  }
401
602
  audio?: {
603
+ /** Encoded audio bitrate in bits per second. */
402
604
  bitrate?: number
605
+ /** Audio sample rate in Hz. */
403
606
  sampleRate?: number
404
607
  echoCancellation?: boolean
405
608
  noiseSuppression?: boolean
@@ -492,14 +695,16 @@ export type OtaStatusEvent = {
492
695
  session_id: string
493
696
  total_steps: number
494
697
  current_step: number
495
- step_type: 'apk' | 'mtk' | 'bes'
496
- phase: 'download' | 'install'
698
+ step_type: "apk" | "mtk" | "bes"
699
+ phase: "download" | "install"
497
700
  step_percent: number
498
701
  overall_percent: number
499
- status: 'in_progress' | 'step_complete' | 'complete' | 'failed' | 'idle'
702
+ status: "in_progress" | "step_complete" | "complete" | "failed" | "idle"
500
703
  error_message?: string
501
704
  }
502
705
 
706
+ export type OtaQueryResult = OtaUpdateAvailableEvent | OtaStatusEvent
707
+
503
708
  /** Nex BLE protobuf trace (NexEventUtils); payload matches native Map keys. */
504
709
  export type BleCommandTraceEvent = {
505
710
  command: string
@@ -525,16 +730,19 @@ export type BluetoothSdkModuleEvents = {
525
730
  glasses_not_ready: (event: GlassesNotReadyEvent) => void
526
731
  button_press: (event: ButtonPressEvent) => void
527
732
  touch_event: (event: TouchEvent) => void
733
+ accel_event: (event: AccelEvent) => void
528
734
  head_up: (event: HeadUpEvent) => void
529
735
  voice_activity_detection_status: (event: VoiceActivityDetectionStatusEvent) => void
530
736
  speaking_status: (event: SpeakingStatusEvent) => void
531
737
  battery_status: (event: BatteryStatusEvent) => void
532
738
  local_transcription: (event: LocalTranscriptionEvent) => void
533
739
  wifi_status_change: (event: WifiStatusChangeEvent) => void
740
+ wifi_scan_result: (event: WifiScanResultEvent) => void
534
741
  hotspot_status_change: (event: HotspotStatusChangeEvent) => void
535
742
  hotspot_error: (event: HotspotErrorEvent) => void
536
743
  photo_response: (event: PhotoResponseEvent) => void
537
744
  photo_status: (event: PhotoStatusEvent) => void
745
+ video_recording_status: (event: VideoRecordingStatusEvent) => void
538
746
  gallery_status: (event: GalleryStatusEvent) => void
539
747
  compatible_glasses_search_stop: (event: CompatibleGlassesSearchStopEvent) => void
540
748
  heartbeat_sent: (event: HeartbeatSentEvent) => void
@@ -542,6 +750,7 @@ export type BluetoothSdkModuleEvents = {
542
750
  swipe_volume_status: (event: SwipeVolumeStatusEvent) => void
543
751
  switch_status: (event: SwitchStatusEvent) => void
544
752
  rgb_led_control_response: (event: RgbLedControlResponseEvent) => void
753
+ settings_ack: (event: SettingsAckEvent) => void
545
754
  pair_failure: (event: PairFailureEvent) => void
546
755
  audio_pairing_needed: (event: AudioPairingNeededEvent) => void
547
756
  audio_connected: (event: AudioConnectedEvent) => void
@@ -557,9 +766,17 @@ export type BluetoothSdkModuleEvents = {
557
766
  ota_update_available: (event: OtaUpdateAvailableEvent) => void
558
767
  ota_start_ack: (event: OtaStartAckEvent) => void
559
768
  ota_status: (event: OtaStatusEvent) => void
769
+ version_info: (event: VersionInfoEvent) => void
560
770
  send_command_to_ble: (event: BleCommandTraceEvent) => void
561
771
  receive_command_from_ble: (event: BleCommandTraceEvent) => void
562
772
  miniapp_selected: (event: MiniappSelectedEvent) => void
773
+ extraction_progress: (event: ExtractionProgressEvent) => void
774
+ }
775
+
776
+ export interface ExtractionProgressEvent {
777
+ percentage: number
778
+ bytesRead: number
779
+ totalBytes: number
563
780
  }
564
781
 
565
782
  export type PublicGlassesStatus = Omit<
@@ -588,21 +805,25 @@ export type BluetoothSdkEventMap = {
588
805
  glasses_not_ready: GlassesNotReadyEvent
589
806
  button_press: ButtonPressEvent
590
807
  touch_event: TouchEvent
808
+ accel_event: AccelEvent
591
809
  head_up: HeadUpEvent
592
810
  voice_activity_detection_status: VoiceActivityDetectionStatusEvent
593
811
  speaking_status: SpeakingStatusEvent
594
812
  battery_status: BatteryStatusEvent
595
813
  local_transcription: LocalTranscriptionEvent
596
814
  wifi_status_change: WifiStatusChangeEvent
815
+ wifi_scan_result: WifiScanResultEvent
597
816
  hotspot_status_change: HotspotStatusChangeEvent
598
817
  hotspot_error: HotspotErrorEvent
599
818
  photo_response: PhotoResponseEvent
600
819
  photo_status: PhotoStatusEvent
820
+ video_recording_status: VideoRecordingStatusEvent
601
821
  gallery_status: GalleryStatusEvent
602
822
  compatible_glasses_search_stop: CompatibleGlassesSearchStopEvent
603
823
  swipe_volume_status: SwipeVolumeStatusEvent
604
824
  switch_status: SwitchStatusEvent
605
825
  rgb_led_control_response: RgbLedControlResponseEvent
826
+ settings_ack: SettingsAckEvent
606
827
  pair_failure: PairFailureEvent
607
828
  audio_pairing_needed: AudioPairingNeededEvent
608
829
  audio_connected: AudioConnectedEvent
@@ -610,6 +831,11 @@ export type BluetoothSdkEventMap = {
610
831
  mic_pcm: MicPcmEvent
611
832
  mic_lc3: MicLc3Event
612
833
  stream_status: StreamStatusEvent
834
+ ota_update_available: OtaUpdateAvailableEvent
835
+ ota_start_ack: OtaStartAckEvent
836
+ ota_status: OtaStatusEvent
837
+ version_info: VersionInfoEvent
838
+ extraction_progress: ExtractionProgressEvent
613
839
  }
614
840
 
615
841
  export type BluetoothSdkEventName = keyof BluetoothSdkEventMap
@@ -649,34 +875,35 @@ export interface BluetoothSdkPublicModule {
649
875
  showDashboard(): Promise<void>
650
876
  setDashboardPosition(height: number, depth: number): Promise<void>
651
877
  setHeadUpAngle(angleDegrees: number): Promise<void>
878
+ setImuEnabled(enabled: boolean): Promise<void>
652
879
  setScreenDisabled(disabled: boolean): Promise<void>
653
880
 
654
- requestWifiScan(): Promise<void>
655
- sendWifiCredentials(ssid: string, password: string): Promise<void>
656
- forgetWifiNetwork(ssid: string): Promise<void>
657
- setHotspotState(enabled: boolean): Promise<void>
881
+ requestWifiScan(): Promise<WifiSearchResult[]>
882
+ sendWifiCredentials(ssid: string, password: string): Promise<WifiStatusChangeEvent>
883
+ forgetWifiNetwork(ssid: string): Promise<WifiStatusChangeEvent>
884
+ setHotspotState(enabled: boolean): Promise<HotspotStatusChangeEvent>
658
885
 
659
- setGalleryModeEnabled(enabled: boolean): Promise<void>
886
+ setGalleryModeEnabled(enabled: boolean): Promise<SettingsAckSuccessEvent>
660
887
  setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
661
- setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
662
- setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
663
- setButtonCameraLed(enabled: boolean): Promise<void>
664
- setButtonMaxRecordingTime(minutes: number): Promise<void>
665
- setCameraFov(fov: CameraFov): Promise<void>
666
- queryGalleryStatus(): Promise<void>
667
- requestPhoto(params: PhotoRequestParams): Promise<void>
668
- startVideoRecording(requestId: string, save: boolean, sound: boolean): Promise<void>
669
- stopVideoRecording(requestId: string): Promise<void>
670
-
671
- startStream(params: StreamStartRequest): Promise<void>
672
- stopStream(): Promise<void>
673
-
674
- setMicState(
675
- enabled: boolean,
676
- useGlassesMic?: boolean,
677
- sendTranscript?: boolean,
678
- sendLc3Data?: boolean,
679
- ): Promise<void>
888
+ setButtonPhotoSettings(size: ButtonPhotoSize): Promise<SettingsAckSuccessEvent>
889
+ setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<SettingsAckSuccessEvent>
890
+ setButtonCameraLed(enabled: boolean): Promise<SettingsAckSuccessEvent>
891
+ setButtonMaxRecordingTime(minutes: number): Promise<SettingsAckSuccessEvent>
892
+ setCameraFov(request: CameraFovRequest): Promise<CameraFovResult>
893
+ queryGalleryStatus(): Promise<GalleryStatusEvent>
894
+ requestPhoto(params: PhotoRequestParams): Promise<PhotoSuccessResponseEvent>
895
+ startVideoRecording(
896
+ requestId: string,
897
+ save: boolean,
898
+ sound: boolean,
899
+ settings?: VideoRecordingSettings,
900
+ ): Promise<VideoRecordingStartedStatusEvent>
901
+ stopVideoRecording(requestId: string): Promise<VideoRecordingStoppedStatusEvent>
902
+
903
+ startStream(params: StreamStartRequest): Promise<StreamStatusEvent>
904
+ stopStream(): Promise<StreamStatusEvent>
905
+
906
+ setMicState(enabled: boolean, useGlassesMic?: boolean, sendTranscript?: boolean, sendLc3Data?: boolean): Promise<void>
680
907
  setPreferredMic(preferredMic: MicPreference): Promise<void>
681
908
  setOwnAppAudioPlaying(playing: boolean): Promise<void>
682
909
  getGlassesMediaVolume(): Promise<GlassesMediaVolumeGetResult>
@@ -690,9 +917,52 @@ export interface BluetoothSdkPublicModule {
690
917
  onDurationMs: number,
691
918
  offDurationMs: number,
692
919
  count: number,
693
- ): Promise<void>
694
-
695
- requestVersionInfo(): Promise<void>
920
+ ): Promise<RgbLedControlSuccessResponseEvent>
921
+
922
+ requestVersionInfo(): Promise<VersionInfoResult>
923
+ /** Ask connected Mentra Live glasses to check/report OTA availability and status. */
924
+ checkForOtaUpdate(): Promise<OtaQueryResult>
925
+ /** Start the OTA flow after your app has presented the available update to the user. */
926
+ startOtaUpdate(): Promise<OtaStartAckEvent>
927
+ /** Re-run the glasses-side OTA version check, mainly after correcting clock skew/TLS failures. */
928
+ retryOtaVersionCheck(): Promise<OtaQueryResult>
929
+
930
+ // // stt commands (MOVE TO CRUST)
931
+ // setSttModelDetails(path: string, languageCode: string): Promise<void>
932
+ // getSttModelPath(): Promise<string>
933
+ // checkSttModelAvailable(): Promise<boolean>
934
+ // validateSttModel(path: string): Promise<boolean>
935
+ // extractTarBz2(sourcePath: string, destinationPath: string): Promise<boolean>
936
+
937
+ // // tts commands (MOVE TO CRUST)
938
+ // setTtsModelDetails(path: string, languageCode: string): Promise<void>
939
+ // getTtsModelPath(): Promise<string>
940
+ // getTtsModelLanguage(): Promise<string>
941
+ // checkTtsModelAvailable(): Promise<boolean>
942
+ // validateTtsModel(path: string): Promise<boolean>
943
+ // generateTtsAudio(text: string, path: string, outputPath: string, speakerId: number, speed: number): Promise<boolean>
944
+
945
+ // STT Commands (TODO: MOVE TO CRUST)
946
+ setSttModelDetails(path: string, languageCode: string): Promise<void>
947
+ getSttModelPath(): Promise<string>
948
+ checkSttModelAvailable(): Promise<boolean>
949
+ validateSttModel(path: string): Promise<boolean>
950
+ extractTarBz2(sourcePath: string, destinationPath: string): Promise<boolean>
951
+ restartTranscriber(): Promise<void>
952
+
953
+ // TTS Commands (TODO: MOVE TO CRUST)
954
+ setTtsModelDetails(path: string, languageCode: string): Promise<void>
955
+ getTtsModelPath(): Promise<string>
956
+ getTtsModelLanguage(): Promise<string>
957
+ checkTtsModelAvailable(): Promise<boolean>
958
+ validateTtsModel(path: string): Promise<boolean>
959
+ generateTtsAudio(
960
+ text: string,
961
+ modelPath: string,
962
+ outputPath: string,
963
+ speakerId: number,
964
+ speed: number,
965
+ ): Promise<boolean>
696
966
  }
697
967
 
698
968
  // OTA update status types
@@ -703,11 +973,11 @@ export interface OtaStatus {
703
973
  sessionId: string
704
974
  totalSteps: number
705
975
  currentStep: number
706
- stepType: 'apk' | 'mtk' | 'bes'
707
- phase: 'download' | 'install'
976
+ stepType: "apk" | "mtk" | "bes"
977
+ phase: "download" | "install"
708
978
  stepPercent: number
709
979
  overallPercent: number
710
- status: 'in_progress' | 'step_complete' | 'complete' | 'failed' | 'idle'
980
+ status: "in_progress" | "step_complete" | "complete" | "failed" | "idle"
711
981
  error?: string
712
982
  }
713
983
 
@@ -749,6 +1019,8 @@ export interface GlassesStatus {
749
1019
  leftMacAddress: string
750
1020
  rightMacAddress: string
751
1021
  buildNumber: string
1022
+ /** Glasses System.currentTimeMillis() from last version_info (clock skew detection). */
1023
+ systemTimeMs?: number
752
1024
  otaVersionUrl: string
753
1025
  appVersion: string
754
1026
  bluetoothName: string
@@ -784,8 +1056,16 @@ export interface CoreDashboardMenuItem {
784
1056
  running: boolean
785
1057
  }
786
1058
 
1059
+ export interface CalendarEvent {
1060
+ title: string
1061
+ location?: string
1062
+ time: string
1063
+ endDate: number
1064
+ }
1065
+
787
1066
  export interface CoreSettings {
788
1067
  menu_apps: CoreDashboardMenuItem[]
1068
+ calendar_events: CalendarEvent[]
789
1069
  }
790
1070
 
791
1071
  export interface Device {
@@ -871,11 +1151,15 @@ export type BluetoothSettingsUpdate = Partial<{
871
1151
  screen_disabled: boolean
872
1152
  contextual_dashboard: boolean
873
1153
  head_up_angle: number
1154
+ imu_enabled: boolean
874
1155
  brightness: number
875
1156
  auto_brightness: boolean
876
1157
  dashboard_height: number
877
1158
  dashboard_depth: number
878
1159
  menu_apps: DashboardMenuItem[] | CoreDashboardMenuItem[] | Array<Record<string, unknown>> | null
1160
+ calendar_events: CalendarEvent[]
1161
+ metric_system: boolean
1162
+ twelve_hour_time: boolean
879
1163
  gallery_mode: boolean
880
1164
  voice_activity_detection_enabled: boolean
881
1165
  button_photo_size: ButtonPhotoSize