@mentra/bluetooth-sdk 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -22
- package/android/.project +28 -0
- package/android/build.gradle +5 -9
- package/android/lc3Lib/.project +28 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
- package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
- package/build/BluetoothSdk.types.d.ts +107 -48
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +4 -0
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +8 -6
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +58 -9
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/index.d.ts +2 -2
- package/build/index.d.ts.map +1 -1
- package/build/index.js +6 -5
- package/build/index.js.map +1 -1
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
- package/build/photo-receiver/index.d.ts +3 -0
- package/build/photo-receiver/index.d.ts.map +1 -0
- package/build/photo-receiver/index.js +3 -0
- package/build/photo-receiver/index.js.map +1 -0
- package/build/react/useMentraBluetooth.d.ts +5 -3
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +15 -4
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/expo-module.config.json +5 -2
- package/ios/BluetoothSdkModule.swift +59 -26
- package/ios/LocalPhotoUploadServer.swift +411 -0
- package/ios/MentraBluetoothSDK.podspec +8 -5
- package/ios/MentraPhotoReceiverModule.swift +147 -0
- package/ios/Source/Audio/AudioModels.swift +10 -10
- package/ios/Source/Bridge.swift +42 -12
- package/ios/Source/Camera/CameraModels.swift +74 -19
- package/ios/Source/DeviceManager.swift +23 -98
- package/ios/Source/DeviceStore.swift +12 -4
- package/ios/Source/Events/BluetoothEvents.swift +37 -0
- package/ios/Source/MentraBluetoothSDK.swift +135 -14
- package/ios/Source/Status/DeviceStatus.swift +12 -4
- package/ios/Source/Status/RuntimeState.swift +22 -13
- package/ios/Source/Streaming/StreamModels.swift +239 -45
- package/ios/Source/controllers/ControllerManager.swift +3 -3
- package/ios/Source/controllers/R1.swift +9 -3
- package/ios/Source/sgcs/G1.swift +5 -4
- package/ios/Source/sgcs/G2.swift +5 -6
- package/ios/Source/sgcs/Mach1.swift +1 -1
- package/ios/Source/sgcs/MentraLive.swift +183 -22
- package/ios/Source/sgcs/MentraNex.swift +1 -1
- package/ios/Source/sgcs/SGCManager.swift +11 -3
- package/ios/Source/sgcs/Simulated.swift +2 -2
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
- package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
- package/package.json +6 -1
- package/src/BluetoothSdk.types.ts +133 -62
- package/src/_private/BluetoothSdkModule.ts +79 -26
- package/src/index.ts +15 -8
- package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
- package/src/photo-receiver/index.ts +2 -0
- package/src/react/useMentraBluetooth.ts +19 -7
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
- package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
- package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
- package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
- package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
- package/ios/Packages/VAD/Data/Configuration.swift +0 -68
- package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
- package/ios/Packages/VAD/Data/Record.swift +0 -13
- package/ios/Packages/VAD/Data/Result.swift +0 -22
- package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
- package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
- package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
- package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
|
@@ -22,9 +22,15 @@ export type HeadUpEvent = {
|
|
|
22
22
|
up: boolean
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
export type
|
|
26
|
-
type: "
|
|
27
|
-
|
|
25
|
+
export type VoiceActivityDetectionStatusEvent = {
|
|
26
|
+
type: "voice_activity_detection_status"
|
|
27
|
+
voiceActivityDetectionEnabled: boolean
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export type SpeakingStatusEvent = {
|
|
31
|
+
type: "speaking_status"
|
|
32
|
+
speaking: boolean
|
|
33
|
+
timestamp: number
|
|
28
34
|
}
|
|
29
35
|
|
|
30
36
|
export type BatteryStatusEvent = {
|
|
@@ -61,6 +67,7 @@ export function createDisconnectedGlassesStatus(): Partial<GlassesStatus> {
|
|
|
61
67
|
return {
|
|
62
68
|
connection: {state: 'disconnected'},
|
|
63
69
|
hotspot: {state: 'disabled'},
|
|
70
|
+
voiceActivityDetectionEnabled: true,
|
|
64
71
|
wifi: {state: 'disconnected'},
|
|
65
72
|
}
|
|
66
73
|
}
|
|
@@ -133,6 +140,43 @@ export type PhotoResponseEvent =
|
|
|
133
140
|
errorMessage: string
|
|
134
141
|
}
|
|
135
142
|
|
|
143
|
+
export type PhotoStatusState =
|
|
144
|
+
| "accepted"
|
|
145
|
+
| "queued"
|
|
146
|
+
| "configuring"
|
|
147
|
+
| "capturing"
|
|
148
|
+
| "captured"
|
|
149
|
+
| "compressing"
|
|
150
|
+
| "uploading"
|
|
151
|
+
| "uploaded"
|
|
152
|
+
| "ready_for_transfer"
|
|
153
|
+
| "transferring"
|
|
154
|
+
| "failed"
|
|
155
|
+
|
|
156
|
+
export type PhotoResolvedConfig = {
|
|
157
|
+
format?: "jpeg" | string
|
|
158
|
+
width?: number
|
|
159
|
+
height?: number
|
|
160
|
+
quality?: number
|
|
161
|
+
requestedSize?: PhotoSize | string
|
|
162
|
+
source?: "sdk" | "button" | string
|
|
163
|
+
transferMethod?: "webhook" | "ble" | "local" | string
|
|
164
|
+
compression?: PhotoCompression | string
|
|
165
|
+
saveToGallery?: boolean
|
|
166
|
+
exposureTimeNs?: number
|
|
167
|
+
iso?: number
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export type PhotoStatusEvent = {
|
|
171
|
+
type: "photo_status"
|
|
172
|
+
requestId: string
|
|
173
|
+
status: PhotoStatusState | string
|
|
174
|
+
timestamp: number
|
|
175
|
+
resolvedConfig?: PhotoResolvedConfig
|
|
176
|
+
errorCode?: string
|
|
177
|
+
errorMessage?: string
|
|
178
|
+
}
|
|
179
|
+
|
|
136
180
|
export type GalleryStatusEvent = {
|
|
137
181
|
type: "gallery_status"
|
|
138
182
|
photos: number
|
|
@@ -190,8 +234,6 @@ export type RgbLedControlResponseEvent =
|
|
|
190
234
|
|
|
191
235
|
export type RgbLedAction = "on" | "off"
|
|
192
236
|
export type RgbLedColor = "red" | "green" | "blue" | "orange" | "white"
|
|
193
|
-
/** `"auto"` enables local button photo/video capture; `"manual"` reports button events without local gallery capture. */
|
|
194
|
-
export type GalleryMode = "auto" | "manual"
|
|
195
237
|
export type PhotoSize = "small" | "medium" | "large" | "full"
|
|
196
238
|
export type ButtonPhotoSize = "small" | "medium" | "large"
|
|
197
239
|
export type PhotoCompression = "none" | "medium" | "heavy"
|
|
@@ -216,21 +258,48 @@ export type DashboardMenuItem = {
|
|
|
216
258
|
values?: Record<string, unknown>
|
|
217
259
|
}
|
|
218
260
|
|
|
219
|
-
export
|
|
261
|
+
export const CAMERA_FOV_MIN = 82
|
|
262
|
+
export const CAMERA_FOV_MAX = 118
|
|
263
|
+
export const CAMERA_FOV_DEFAULT = 102
|
|
264
|
+
|
|
265
|
+
export type CameraRoiPosition = 0 | 1 | 2
|
|
266
|
+
|
|
267
|
+
export type CameraFov = {
|
|
268
|
+
fov: number
|
|
269
|
+
roiPosition?: CameraRoiPosition
|
|
270
|
+
}
|
|
220
271
|
|
|
221
272
|
export type CameraFovSetting = {
|
|
222
273
|
fov: number
|
|
223
|
-
roiPosition:
|
|
274
|
+
roiPosition: CameraRoiPosition
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
type NativeCameraFovSetting = {
|
|
278
|
+
fov: number
|
|
279
|
+
roi_position: CameraRoiPosition
|
|
224
280
|
}
|
|
225
281
|
|
|
226
282
|
export type MicPreference = "auto" | "phone" | "glasses" | "bluetooth"
|
|
227
283
|
export type MicMode = "phone" | "glasses" | "bluetoothClassic" | "bluetooth"
|
|
228
284
|
|
|
285
|
+
export type PhotoRequestParams = {
|
|
286
|
+
requestId: string
|
|
287
|
+
appId: string
|
|
288
|
+
size: PhotoSize
|
|
289
|
+
webhookUrl: string | null
|
|
290
|
+
authToken: string | null
|
|
291
|
+
compress: PhotoCompression
|
|
292
|
+
sound: boolean
|
|
293
|
+
exposureTimeNs?: number | null
|
|
294
|
+
/** Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure. */
|
|
295
|
+
iso?: number | null
|
|
296
|
+
}
|
|
297
|
+
|
|
229
298
|
export type StreamVideoConfig = {
|
|
230
299
|
width?: number
|
|
231
300
|
height?: number
|
|
232
301
|
bitrate?: number
|
|
233
|
-
|
|
302
|
+
fps?: number
|
|
234
303
|
}
|
|
235
304
|
|
|
236
305
|
export type StreamAudioConfig = {
|
|
@@ -299,7 +368,7 @@ export type MicPcmEvent = {
|
|
|
299
368
|
bitsPerSample: 16
|
|
300
369
|
channels: 1
|
|
301
370
|
encoding: "pcm_s16le"
|
|
302
|
-
|
|
371
|
+
voiceActivityDetectionEnabled: boolean
|
|
303
372
|
}
|
|
304
373
|
|
|
305
374
|
export type MicLc3Event = {
|
|
@@ -312,65 +381,72 @@ export type MicLc3Event = {
|
|
|
312
381
|
frameSizeBytes: number
|
|
313
382
|
bitrate: number
|
|
314
383
|
packetizedFromGlasses: boolean
|
|
315
|
-
|
|
384
|
+
voiceActivityDetectionEnabled: boolean
|
|
316
385
|
}
|
|
317
386
|
|
|
318
387
|
export type StreamStatusLifecycleState = "initializing" | "streaming" | "stopping" | "stopped"
|
|
319
388
|
export type StreamStatusReconnectState = "reconnecting" | "reconnected" | "reconnect_failed"
|
|
320
389
|
export type StreamStatusState = StreamStatusLifecycleState | StreamStatusReconnectState | "error"
|
|
321
390
|
|
|
391
|
+
export type StreamResolvedConfig = {
|
|
392
|
+
transport?: "rtmp" | "srt" | "whip"
|
|
393
|
+
video?: {
|
|
394
|
+
width: number
|
|
395
|
+
height: number
|
|
396
|
+
captureWidth?: number
|
|
397
|
+
captureHeight?: number
|
|
398
|
+
bitrate: number
|
|
399
|
+
fps: number
|
|
400
|
+
}
|
|
401
|
+
audio?: {
|
|
402
|
+
bitrate?: number
|
|
403
|
+
sampleRate?: number
|
|
404
|
+
echoCancellation?: boolean
|
|
405
|
+
noiseSuppression?: boolean
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
type StreamStatusCommon = {
|
|
410
|
+
type: "stream_status"
|
|
411
|
+
streamId?: string
|
|
412
|
+
timestamp?: number
|
|
413
|
+
resolvedConfig?: StreamResolvedConfig
|
|
414
|
+
}
|
|
415
|
+
|
|
322
416
|
export type StreamStatusEvent =
|
|
323
|
-
| {
|
|
324
|
-
type: "stream_status"
|
|
417
|
+
| (StreamStatusCommon & {
|
|
325
418
|
kind: "lifecycle"
|
|
326
419
|
status: StreamStatusLifecycleState
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
}
|
|
330
|
-
| {
|
|
331
|
-
type: "stream_status"
|
|
420
|
+
})
|
|
421
|
+
| (StreamStatusCommon & {
|
|
332
422
|
kind: "reconnect"
|
|
333
423
|
status: "reconnecting"
|
|
334
|
-
streamId?: string
|
|
335
424
|
attempt: number
|
|
336
425
|
maxAttempts: number
|
|
337
426
|
reason: string
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
| {
|
|
341
|
-
type: "stream_status"
|
|
427
|
+
})
|
|
428
|
+
| (StreamStatusCommon & {
|
|
342
429
|
kind: "reconnect"
|
|
343
430
|
status: "reconnected"
|
|
344
|
-
streamId?: string
|
|
345
431
|
attempt: number
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
| {
|
|
349
|
-
type: "stream_status"
|
|
432
|
+
})
|
|
433
|
+
| (StreamStatusCommon & {
|
|
350
434
|
kind: "reconnect"
|
|
351
435
|
status: "reconnect_failed"
|
|
352
|
-
streamId?: string
|
|
353
436
|
maxAttempts: number
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
| {
|
|
357
|
-
type: "stream_status"
|
|
437
|
+
})
|
|
438
|
+
| (StreamStatusCommon & {
|
|
358
439
|
kind: "error"
|
|
359
440
|
status: "error"
|
|
360
|
-
streamId?: string
|
|
361
441
|
errorDetails: string
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
| {
|
|
365
|
-
type: "stream_status"
|
|
442
|
+
})
|
|
443
|
+
| (StreamStatusCommon & {
|
|
366
444
|
kind: "snapshot"
|
|
367
445
|
status: "streaming" | "reconnecting" | "stopped"
|
|
368
446
|
streaming: boolean
|
|
369
447
|
reconnecting: boolean
|
|
370
|
-
streamId?: string
|
|
371
448
|
attempt?: number
|
|
372
|
-
|
|
373
|
-
}
|
|
449
|
+
})
|
|
374
450
|
|
|
375
451
|
export type KeepAliveAckEvent = {
|
|
376
452
|
type: "keep_alive_ack"
|
|
@@ -450,13 +526,15 @@ export type BluetoothSdkModuleEvents = {
|
|
|
450
526
|
button_press: (event: ButtonPressEvent) => void
|
|
451
527
|
touch_event: (event: TouchEvent) => void
|
|
452
528
|
head_up: (event: HeadUpEvent) => void
|
|
453
|
-
|
|
529
|
+
voice_activity_detection_status: (event: VoiceActivityDetectionStatusEvent) => void
|
|
530
|
+
speaking_status: (event: SpeakingStatusEvent) => void
|
|
454
531
|
battery_status: (event: BatteryStatusEvent) => void
|
|
455
532
|
local_transcription: (event: LocalTranscriptionEvent) => void
|
|
456
533
|
wifi_status_change: (event: WifiStatusChangeEvent) => void
|
|
457
534
|
hotspot_status_change: (event: HotspotStatusChangeEvent) => void
|
|
458
535
|
hotspot_error: (event: HotspotErrorEvent) => void
|
|
459
536
|
photo_response: (event: PhotoResponseEvent) => void
|
|
537
|
+
photo_status: (event: PhotoStatusEvent) => void
|
|
460
538
|
gallery_status: (event: GalleryStatusEvent) => void
|
|
461
539
|
compatible_glasses_search_stop: (event: CompatibleGlassesSearchStopEvent) => void
|
|
462
540
|
heartbeat_sent: (event: HeartbeatSentEvent) => void
|
|
@@ -500,7 +578,7 @@ export type PublicBluetoothStatus = Pick<
|
|
|
500
578
|
| "wifiScanResults"
|
|
501
579
|
| "lastLog"
|
|
502
580
|
| "otherBtConnected"
|
|
503
|
-
| "
|
|
581
|
+
| "galleryModeEnabled"
|
|
504
582
|
>
|
|
505
583
|
|
|
506
584
|
export type BluetoothSdkEventMap = {
|
|
@@ -511,13 +589,15 @@ export type BluetoothSdkEventMap = {
|
|
|
511
589
|
button_press: ButtonPressEvent
|
|
512
590
|
touch_event: TouchEvent
|
|
513
591
|
head_up: HeadUpEvent
|
|
514
|
-
|
|
592
|
+
voice_activity_detection_status: VoiceActivityDetectionStatusEvent
|
|
593
|
+
speaking_status: SpeakingStatusEvent
|
|
515
594
|
battery_status: BatteryStatusEvent
|
|
516
595
|
local_transcription: LocalTranscriptionEvent
|
|
517
596
|
wifi_status_change: WifiStatusChangeEvent
|
|
518
597
|
hotspot_status_change: HotspotStatusChangeEvent
|
|
519
598
|
hotspot_error: HotspotErrorEvent
|
|
520
599
|
photo_response: PhotoResponseEvent
|
|
600
|
+
photo_status: PhotoStatusEvent
|
|
521
601
|
gallery_status: GalleryStatusEvent
|
|
522
602
|
compatible_glasses_search_stop: CompatibleGlassesSearchStopEvent
|
|
523
603
|
swipe_volume_status: SwipeVolumeStatusEvent
|
|
@@ -530,7 +610,6 @@ export type BluetoothSdkEventMap = {
|
|
|
530
610
|
mic_pcm: MicPcmEvent
|
|
531
611
|
mic_lc3: MicLc3Event
|
|
532
612
|
stream_status: StreamStatusEvent
|
|
533
|
-
keep_alive_ack: KeepAliveAckEvent
|
|
534
613
|
}
|
|
535
614
|
|
|
536
615
|
export type BluetoothSdkEventName = keyof BluetoothSdkEventMap
|
|
@@ -577,33 +656,24 @@ export interface BluetoothSdkPublicModule {
|
|
|
577
656
|
forgetWifiNetwork(ssid: string): Promise<void>
|
|
578
657
|
setHotspotState(enabled: boolean): Promise<void>
|
|
579
658
|
|
|
580
|
-
|
|
659
|
+
setGalleryModeEnabled(enabled: boolean): Promise<void>
|
|
660
|
+
setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
|
|
581
661
|
setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
|
|
582
|
-
setButtonVideoRecordingSettings(width: number, height: number,
|
|
662
|
+
setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
|
|
583
663
|
setButtonCameraLed(enabled: boolean): Promise<void>
|
|
584
664
|
setButtonMaxRecordingTime(minutes: number): Promise<void>
|
|
585
665
|
setCameraFov(fov: CameraFov): Promise<void>
|
|
586
666
|
queryGalleryStatus(): Promise<void>
|
|
587
|
-
requestPhoto(
|
|
588
|
-
requestId: string,
|
|
589
|
-
appId: string,
|
|
590
|
-
size: PhotoSize,
|
|
591
|
-
webhookUrl: string | null,
|
|
592
|
-
authToken: string | null,
|
|
593
|
-
compress: PhotoCompression,
|
|
594
|
-
sound: boolean,
|
|
595
|
-
): Promise<void>
|
|
667
|
+
requestPhoto(params: PhotoRequestParams): Promise<void>
|
|
596
668
|
startVideoRecording(requestId: string, save: boolean, sound: boolean): Promise<void>
|
|
597
669
|
stopVideoRecording(requestId: string): Promise<void>
|
|
598
670
|
|
|
599
671
|
startStream(params: StreamStartRequest): Promise<void>
|
|
600
672
|
stopStream(): Promise<void>
|
|
601
|
-
keepStreamAlive(params: StreamKeepAliveRequest): Promise<void>
|
|
602
673
|
|
|
603
674
|
setMicState(
|
|
604
675
|
enabled: boolean,
|
|
605
676
|
useGlassesMic?: boolean,
|
|
606
|
-
bypassVad?: boolean,
|
|
607
677
|
sendTranscript?: boolean,
|
|
608
678
|
sendLc3Data?: boolean,
|
|
609
679
|
): Promise<void>
|
|
@@ -664,6 +734,7 @@ export interface GlassesStatus {
|
|
|
664
734
|
// state:
|
|
665
735
|
connection: GlassesConnectionStatus
|
|
666
736
|
micEnabled: boolean
|
|
737
|
+
voiceActivityDetectionEnabled: boolean
|
|
667
738
|
bluetoothClassicConnected: boolean
|
|
668
739
|
signalStrength: number
|
|
669
740
|
/** Milliseconds since epoch when signalStrength was last refreshed by the phone BLE stack. */
|
|
@@ -787,7 +858,7 @@ export interface BluetoothStatus {
|
|
|
787
858
|
lastLog: string[]
|
|
788
859
|
otherBtConnected: boolean
|
|
789
860
|
// desired settings the SDK sends to compatible connected glasses:
|
|
790
|
-
|
|
861
|
+
galleryModeEnabled: boolean
|
|
791
862
|
}
|
|
792
863
|
|
|
793
864
|
export type BluetoothSettingsUpdate = Partial<{
|
|
@@ -805,19 +876,19 @@ export type BluetoothSettingsUpdate = Partial<{
|
|
|
805
876
|
dashboard_height: number
|
|
806
877
|
dashboard_depth: number
|
|
807
878
|
menu_apps: DashboardMenuItem[] | CoreDashboardMenuItem[] | Array<Record<string, unknown>> | null
|
|
808
|
-
|
|
879
|
+
gallery_mode: boolean
|
|
880
|
+
voice_activity_detection_enabled: boolean
|
|
809
881
|
button_photo_size: ButtonPhotoSize
|
|
810
|
-
button_video_settings: {width: number; height: number;
|
|
882
|
+
button_video_settings: {width: number; height: number; fps: number}
|
|
811
883
|
button_video_width: number
|
|
812
884
|
button_video_height: number
|
|
813
885
|
button_video_fps: number
|
|
814
886
|
button_camera_led: boolean
|
|
815
887
|
button_max_recording_time: number
|
|
816
|
-
camera_fov:
|
|
888
|
+
camera_fov: NativeCameraFovSetting
|
|
817
889
|
should_send_pcm: boolean
|
|
818
890
|
should_send_lc3: boolean
|
|
819
891
|
should_send_transcript: boolean
|
|
820
|
-
bypass_vad: boolean
|
|
821
892
|
offline_mode: boolean
|
|
822
893
|
offline_captions_running: boolean
|
|
823
894
|
local_stt_fallback_active: boolean
|
|
@@ -6,20 +6,20 @@ import {
|
|
|
6
6
|
BluetoothSdkModuleEvents,
|
|
7
7
|
BluetoothStatus,
|
|
8
8
|
ButtonPhotoSize,
|
|
9
|
+
CAMERA_FOV_DEFAULT,
|
|
10
|
+
CAMERA_FOV_MAX,
|
|
11
|
+
CAMERA_FOV_MIN,
|
|
9
12
|
CameraFov,
|
|
10
|
-
CameraFovSetting,
|
|
11
13
|
ConnectOptions,
|
|
12
14
|
DashboardMenuItem,
|
|
13
15
|
Device,
|
|
14
16
|
DeviceModel,
|
|
15
|
-
GalleryMode,
|
|
16
17
|
GlassesMediaVolumeGetResult,
|
|
17
18
|
GlassesMediaVolumeSetResult,
|
|
18
19
|
GlassesStatus,
|
|
19
20
|
MicPreference,
|
|
20
21
|
ObservableStoreCategory,
|
|
21
|
-
|
|
22
|
-
PhotoSize,
|
|
22
|
+
PhotoRequestParams,
|
|
23
23
|
PublicBluetoothStatus,
|
|
24
24
|
RgbLedAction,
|
|
25
25
|
RgbLedColor,
|
|
@@ -96,22 +96,15 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
|
|
|
96
96
|
logCurrentWifiFrequency(): Promise<void>
|
|
97
97
|
|
|
98
98
|
// Gallery Commands
|
|
99
|
-
|
|
99
|
+
setGalleryModeEnabled(enabled: boolean): Promise<void>
|
|
100
|
+
setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>
|
|
100
101
|
setButtonPhotoSettings(size: ButtonPhotoSize): Promise<void>
|
|
101
|
-
setButtonVideoRecordingSettings(width: number, height: number,
|
|
102
|
+
setButtonVideoRecordingSettings(width: number, height: number, fps: number): Promise<void>
|
|
102
103
|
setButtonCameraLed(enabled: boolean): Promise<void>
|
|
103
104
|
setButtonMaxRecordingTime(minutes: number): Promise<void>
|
|
104
105
|
setCameraFov(fov: CameraFov): Promise<void>
|
|
105
106
|
queryGalleryStatus(): Promise<void>
|
|
106
|
-
requestPhoto(
|
|
107
|
-
requestId: string,
|
|
108
|
-
appId: string,
|
|
109
|
-
size: PhotoSize,
|
|
110
|
-
webhookUrl: string | null,
|
|
111
|
-
authToken: string | null,
|
|
112
|
-
compress: PhotoCompression,
|
|
113
|
-
sound: boolean,
|
|
114
|
-
): Promise<void>
|
|
107
|
+
requestPhoto(params: PhotoRequestParams): Promise<void>
|
|
115
108
|
|
|
116
109
|
// OTA Commands
|
|
117
110
|
sendOtaStart(): Promise<void>
|
|
@@ -126,14 +119,14 @@ declare class BluetoothSdkNativeModule extends NativeModule<BluetoothSdkModuleEv
|
|
|
126
119
|
|
|
127
120
|
// Stream Commands
|
|
128
121
|
startStream(params: StreamStartRequest): Promise<void>
|
|
122
|
+
startCloudManagedStream(params: StreamStartRequest): Promise<void>
|
|
129
123
|
stopStream(): Promise<void>
|
|
130
|
-
|
|
124
|
+
sendCloudStreamKeepAlive(params: StreamKeepAliveRequest): Promise<void>
|
|
131
125
|
|
|
132
126
|
// Microphone Commands
|
|
133
127
|
setMicState(
|
|
134
128
|
enabled: boolean,
|
|
135
129
|
useGlassesMic?: boolean,
|
|
136
|
-
bypassVad?: boolean,
|
|
137
130
|
sendTranscript?: boolean,
|
|
138
131
|
sendLc3Data?: boolean,
|
|
139
132
|
): Promise<void>
|
|
@@ -191,9 +184,26 @@ const DEFAULT_CONNECT_OPTIONS: Required<ConnectOptions> = {
|
|
|
191
184
|
|
|
192
185
|
const DEFAULT_SCAN_TIMEOUT_MS = 15_000
|
|
193
186
|
|
|
194
|
-
const
|
|
195
|
-
|
|
196
|
-
|
|
187
|
+
const CAMERA_ROI_MIN = 0
|
|
188
|
+
const CAMERA_ROI_MAX = 2
|
|
189
|
+
|
|
190
|
+
function clampInteger(value: number, min: number, max: number): number {
|
|
191
|
+
return Math.max(min, Math.min(max, Math.round(value)))
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function normalizeCameraFov(setting: CameraFov): Required<CameraFov> {
|
|
195
|
+
return {
|
|
196
|
+
fov: clampInteger(
|
|
197
|
+
Number.isFinite(setting.fov) ? setting.fov : CAMERA_FOV_DEFAULT,
|
|
198
|
+
CAMERA_FOV_MIN,
|
|
199
|
+
CAMERA_FOV_MAX,
|
|
200
|
+
),
|
|
201
|
+
roiPosition: clampInteger(
|
|
202
|
+
Number.isFinite(setting.roiPosition ?? 0) ? (setting.roiPosition ?? 0) : 0,
|
|
203
|
+
CAMERA_ROI_MIN,
|
|
204
|
+
CAMERA_ROI_MAX,
|
|
205
|
+
) as Required<CameraFov>["roiPosition"],
|
|
206
|
+
}
|
|
197
207
|
}
|
|
198
208
|
|
|
199
209
|
function searchResultsForModel(status: Partial<PublicBluetoothStatus>, model: DeviceModel): Device[] {
|
|
@@ -301,7 +311,6 @@ NativeBluetoothSdkModule.getDefaultDevice = function () {
|
|
|
301
311
|
const nativeSetMicState = NativeBluetoothSdkModule.setMicState.bind(NativeBluetoothSdkModule) as (
|
|
302
312
|
enabled: boolean,
|
|
303
313
|
useGlassesMic: boolean,
|
|
304
|
-
bypassVad: boolean,
|
|
305
314
|
sendTranscript: boolean,
|
|
306
315
|
sendLc3Data: boolean,
|
|
307
316
|
) => MaybePromise<void>
|
|
@@ -355,15 +364,23 @@ NativeBluetoothSdkModule.setScreenDisabled = function (disabled: boolean) {
|
|
|
355
364
|
return this.updateBluetoothSettings({screen_disabled: disabled})
|
|
356
365
|
}
|
|
357
366
|
|
|
367
|
+
NativeBluetoothSdkModule.setGalleryModeEnabled = function (enabled: boolean) {
|
|
368
|
+
return this.updateBluetoothSettings({gallery_mode: enabled})
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
NativeBluetoothSdkModule.setVoiceActivityDetectionEnabled = function (enabled: boolean) {
|
|
372
|
+
return this.updateBluetoothSettings({voice_activity_detection_enabled: enabled})
|
|
373
|
+
}
|
|
374
|
+
|
|
358
375
|
NativeBluetoothSdkModule.setButtonPhotoSettings = function (size: ButtonPhotoSize) {
|
|
359
376
|
return this.updateBluetoothSettings({button_photo_size: size})
|
|
360
377
|
}
|
|
361
378
|
|
|
362
|
-
NativeBluetoothSdkModule.setButtonVideoRecordingSettings = function (width: number, height: number,
|
|
379
|
+
NativeBluetoothSdkModule.setButtonVideoRecordingSettings = function (width: number, height: number, fps: number) {
|
|
363
380
|
return this.updateBluetoothSettings({
|
|
364
381
|
button_video_width: width,
|
|
365
382
|
button_video_height: height,
|
|
366
|
-
button_video_fps:
|
|
383
|
+
button_video_fps: fps,
|
|
367
384
|
})
|
|
368
385
|
}
|
|
369
386
|
|
|
@@ -376,13 +393,15 @@ NativeBluetoothSdkModule.setButtonMaxRecordingTime = function (minutes: number)
|
|
|
376
393
|
}
|
|
377
394
|
|
|
378
395
|
NativeBluetoothSdkModule.setCameraFov = function (fov: CameraFov) {
|
|
379
|
-
|
|
396
|
+
const setting = normalizeCameraFov(fov)
|
|
397
|
+
return this.updateBluetoothSettings({
|
|
398
|
+
camera_fov: {fov: setting.fov, roi_position: setting.roiPosition},
|
|
399
|
+
})
|
|
380
400
|
}
|
|
381
401
|
|
|
382
402
|
NativeBluetoothSdkModule.setMicState = function (
|
|
383
403
|
enabled: boolean,
|
|
384
404
|
useGlassesMic?: boolean,
|
|
385
|
-
bypassVad?: boolean,
|
|
386
405
|
sendTranscript?: boolean,
|
|
387
406
|
sendLc3Data?: boolean,
|
|
388
407
|
) {
|
|
@@ -390,7 +409,6 @@ NativeBluetoothSdkModule.setMicState = function (
|
|
|
390
409
|
nativeSetMicState(
|
|
391
410
|
enabled,
|
|
392
411
|
useGlassesMic ?? true,
|
|
393
|
-
bypassVad ?? true,
|
|
394
412
|
sendTranscript ?? false,
|
|
395
413
|
sendLc3Data ?? false,
|
|
396
414
|
),
|
|
@@ -487,5 +505,40 @@ NativeBluetoothSdkModule.scan = async function (
|
|
|
487
505
|
})
|
|
488
506
|
}
|
|
489
507
|
|
|
508
|
+
/** Expo Android bridge rejects null values in Map<String, Any> — omit optional nullish fields. */
|
|
509
|
+
function photoRequestParamsForNative(params: PhotoRequestParams): Record<string, string | number | boolean> {
|
|
510
|
+
const payload: Record<string, string | number | boolean> = {
|
|
511
|
+
requestId: params.requestId,
|
|
512
|
+
appId: params.appId,
|
|
513
|
+
size: params.size,
|
|
514
|
+
webhookUrl: params.webhookUrl ?? "",
|
|
515
|
+
compress: params.compress,
|
|
516
|
+
flash: true,
|
|
517
|
+
sound: params.sound,
|
|
518
|
+
}
|
|
519
|
+
if (params.authToken != null && params.authToken.length > 0) {
|
|
520
|
+
payload.authToken = params.authToken
|
|
521
|
+
}
|
|
522
|
+
const exposureTimeNs = params.exposureTimeNs
|
|
523
|
+
const hasManualExposure = exposureTimeNs != null && Number.isFinite(exposureTimeNs) && exposureTimeNs > 0
|
|
524
|
+
if (hasManualExposure) {
|
|
525
|
+
payload.exposureTimeNs = exposureTimeNs
|
|
526
|
+
}
|
|
527
|
+
if (hasManualExposure && params.iso != null && Number.isFinite(params.iso) && params.iso > 0) {
|
|
528
|
+
payload.iso = Math.round(params.iso)
|
|
529
|
+
}
|
|
530
|
+
return payload
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
const nativeRequestPhoto = NativeBluetoothSdkModule.requestPhoto.bind(NativeBluetoothSdkModule)
|
|
534
|
+
NativeBluetoothSdkModule.requestPhoto = function (params: PhotoRequestParams) {
|
|
535
|
+
return nativeRequestPhoto(photoRequestParamsForNative(params) as unknown as PhotoRequestParams)
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const nativeStartStream = NativeBluetoothSdkModule.startStream.bind(NativeBluetoothSdkModule)
|
|
539
|
+
NativeBluetoothSdkModule.startCloudManagedStream = function (params: StreamStartRequest) {
|
|
540
|
+
return nativeStartStream({...params, keepAliveMode: "external"} as StreamStartRequest)
|
|
541
|
+
}
|
|
542
|
+
|
|
490
543
|
export default NativeBluetoothSdkModule
|
|
491
544
|
export const BluetoothSdk = NativeBluetoothSdkModule as BluetoothSdkPublicModule
|
package/src/index.ts
CHANGED
|
@@ -13,13 +13,15 @@ const PUBLIC_EVENT_NAMES = new Set<BluetoothSdkEventName>([
|
|
|
13
13
|
"button_press",
|
|
14
14
|
"touch_event",
|
|
15
15
|
"head_up",
|
|
16
|
-
"
|
|
16
|
+
"voice_activity_detection_status",
|
|
17
|
+
"speaking_status",
|
|
17
18
|
"battery_status",
|
|
18
19
|
"local_transcription",
|
|
19
20
|
"wifi_status_change",
|
|
20
21
|
"hotspot_status_change",
|
|
21
22
|
"hotspot_error",
|
|
22
23
|
"photo_response",
|
|
24
|
+
"photo_status",
|
|
23
25
|
"gallery_status",
|
|
24
26
|
"compatible_glasses_search_stop",
|
|
25
27
|
"swipe_volume_status",
|
|
@@ -32,7 +34,6 @@ const PUBLIC_EVENT_NAMES = new Set<BluetoothSdkEventName>([
|
|
|
32
34
|
"mic_pcm",
|
|
33
35
|
"mic_lc3",
|
|
34
36
|
"stream_status",
|
|
35
|
-
"keep_alive_ack",
|
|
36
37
|
])
|
|
37
38
|
|
|
38
39
|
const addListener: BluetoothSdkPublicModule["addListener"] = (eventName, listener) => {
|
|
@@ -70,7 +71,8 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
|
|
|
70
71
|
sendWifiCredentials: PrivateBluetoothSdkModule.sendWifiCredentials.bind(PrivateBluetoothSdkModule),
|
|
71
72
|
forgetWifiNetwork: PrivateBluetoothSdkModule.forgetWifiNetwork.bind(PrivateBluetoothSdkModule),
|
|
72
73
|
setHotspotState: PrivateBluetoothSdkModule.setHotspotState.bind(PrivateBluetoothSdkModule),
|
|
73
|
-
|
|
74
|
+
setGalleryModeEnabled: PrivateBluetoothSdkModule.setGalleryModeEnabled.bind(PrivateBluetoothSdkModule),
|
|
75
|
+
setVoiceActivityDetectionEnabled: PrivateBluetoothSdkModule.setVoiceActivityDetectionEnabled.bind(PrivateBluetoothSdkModule),
|
|
74
76
|
setButtonPhotoSettings: PrivateBluetoothSdkModule.setButtonPhotoSettings.bind(PrivateBluetoothSdkModule),
|
|
75
77
|
setButtonVideoRecordingSettings: PrivateBluetoothSdkModule.setButtonVideoRecordingSettings.bind(PrivateBluetoothSdkModule),
|
|
76
78
|
setButtonCameraLed: PrivateBluetoothSdkModule.setButtonCameraLed.bind(PrivateBluetoothSdkModule),
|
|
@@ -82,7 +84,6 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
|
|
|
82
84
|
stopVideoRecording: PrivateBluetoothSdkModule.stopVideoRecording.bind(PrivateBluetoothSdkModule),
|
|
83
85
|
startStream: PrivateBluetoothSdkModule.startStream.bind(PrivateBluetoothSdkModule),
|
|
84
86
|
stopStream: PrivateBluetoothSdkModule.stopStream.bind(PrivateBluetoothSdkModule),
|
|
85
|
-
keepStreamAlive: PrivateBluetoothSdkModule.keepStreamAlive.bind(PrivateBluetoothSdkModule),
|
|
86
87
|
setMicState: PrivateBluetoothSdkModule.setMicState.bind(PrivateBluetoothSdkModule),
|
|
87
88
|
setPreferredMic: PrivateBluetoothSdkModule.setPreferredMic.bind(PrivateBluetoothSdkModule),
|
|
88
89
|
setOwnAppAudioPlaying: PrivateBluetoothSdkModule.setOwnAppAudioPlaying.bind(PrivateBluetoothSdkModule),
|
|
@@ -95,6 +96,9 @@ export const BluetoothSdk: BluetoothSdkPublicModule = Object.freeze({
|
|
|
95
96
|
export default BluetoothSdk
|
|
96
97
|
|
|
97
98
|
export {
|
|
99
|
+
CAMERA_FOV_DEFAULT,
|
|
100
|
+
CAMERA_FOV_MAX,
|
|
101
|
+
CAMERA_FOV_MIN,
|
|
98
102
|
DeviceModels,
|
|
99
103
|
isBusyGlassesConnectionStatus,
|
|
100
104
|
isConnectedGlassesConnectionStatus,
|
|
@@ -117,6 +121,7 @@ export type {
|
|
|
117
121
|
ButtonPhotoSize,
|
|
118
122
|
ButtonPressEvent,
|
|
119
123
|
CameraFov,
|
|
124
|
+
CameraRoiPosition,
|
|
120
125
|
CompatibleGlassesSearchStopEvent,
|
|
121
126
|
ConnectOptions,
|
|
122
127
|
ConnectedGlassesConnectionStatus,
|
|
@@ -124,7 +129,6 @@ export type {
|
|
|
124
129
|
Device,
|
|
125
130
|
DeviceModel,
|
|
126
131
|
EnabledHotspotStatus,
|
|
127
|
-
GalleryMode,
|
|
128
132
|
GalleryStatusEvent,
|
|
129
133
|
GlassesConnectionStatus,
|
|
130
134
|
GlassesMediaVolumeGetResult,
|
|
@@ -134,7 +138,6 @@ export type {
|
|
|
134
138
|
HotspotErrorEvent,
|
|
135
139
|
HotspotStatus,
|
|
136
140
|
HotspotStatusChangeEvent,
|
|
137
|
-
KeepAliveAckEvent,
|
|
138
141
|
LocalTranscriptionEvent,
|
|
139
142
|
LogEvent,
|
|
140
143
|
MicLc3Event,
|
|
@@ -142,17 +145,21 @@ export type {
|
|
|
142
145
|
MicPcmEvent,
|
|
143
146
|
MicPreference,
|
|
144
147
|
PairFailureEvent,
|
|
148
|
+
PhotoResolvedConfig,
|
|
145
149
|
PhotoCompression,
|
|
146
150
|
PhotoResponseEvent,
|
|
147
151
|
PhotoSize,
|
|
152
|
+
PhotoStatusEvent,
|
|
153
|
+
PhotoStatusState,
|
|
148
154
|
RgbLedAction,
|
|
149
155
|
RgbLedColor,
|
|
150
156
|
RgbLedControlResponseEvent,
|
|
151
157
|
ScanModelOptions,
|
|
152
158
|
ScanOptions,
|
|
153
159
|
ScanResultsCallback,
|
|
160
|
+
SpeakingStatusEvent,
|
|
154
161
|
StreamAudioConfig,
|
|
155
|
-
|
|
162
|
+
StreamResolvedConfig,
|
|
156
163
|
StreamStartRequest,
|
|
157
164
|
StreamStatusEvent,
|
|
158
165
|
StreamStatusLifecycleState,
|
|
@@ -162,7 +169,7 @@ export type {
|
|
|
162
169
|
SwipeVolumeStatusEvent,
|
|
163
170
|
SwitchStatusEvent,
|
|
164
171
|
TouchEvent,
|
|
165
|
-
|
|
172
|
+
VoiceActivityDetectionStatusEvent,
|
|
166
173
|
WifiSearchResult,
|
|
167
174
|
WifiStatus,
|
|
168
175
|
WifiStatusChangeEvent,
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export type PhotoReceiverUploadEvent = {
|
|
2
|
+
byteCount: number
|
|
3
|
+
fileUri: string
|
|
4
|
+
requestId?: string | null
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export type PhotoReceiverStatusEvent = {
|
|
8
|
+
message: string
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type PhotoReceiverResult = {
|
|
12
|
+
host: string
|
|
13
|
+
port: number
|
|
14
|
+
uploadUrl: string
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export type MentraPhotoReceiverModuleEvents = {
|
|
18
|
+
photoUpload: (event: PhotoReceiverUploadEvent) => void
|
|
19
|
+
receiverStatus: (event: PhotoReceiverStatusEvent) => void
|
|
20
|
+
}
|