@mentra/engine 0.1.0-beta.0 → 0.1.0-dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -1
- package/build/engine.d.ts +11 -3
- package/build/engine.d.ts.map +1 -1
- package/build/engine.js +7 -2
- package/build/engine.js.map +1 -1
- package/build/facades/dev.d.ts.map +1 -1
- package/build/facades/dev.js +9 -5
- package/build/facades/dev.js.map +1 -1
- package/build/facades/gallery.js +1 -1
- package/build/facades/gallery.js.map +1 -1
- package/build/facades/glasses.d.ts +1 -0
- package/build/facades/glasses.d.ts.map +1 -1
- package/build/facades/glasses.js +19 -4
- package/build/facades/glasses.js.map +1 -1
- package/build/facades/glassesSettings.d.ts.map +1 -1
- package/build/facades/glassesSettings.js +1 -0
- package/build/facades/glassesSettings.js.map +1 -1
- package/build/facades/miniapps.d.ts +4 -0
- package/build/facades/miniapps.d.ts.map +1 -1
- package/build/facades/miniapps.js +5 -0
- package/build/facades/miniapps.js.map +1 -1
- package/build/facades/ota.d.ts +11 -2
- package/build/facades/ota.d.ts.map +1 -1
- package/build/facades/ota.js +19 -4
- package/build/facades/ota.js.map +1 -1
- package/build/facades/pairing.d.ts +4 -0
- package/build/facades/pairing.d.ts.map +1 -1
- package/build/facades/pairing.js +10 -8
- package/build/facades/pairing.js.map +1 -1
- package/build/facades/phoneNotifications.d.ts +0 -4
- package/build/facades/phoneNotifications.d.ts.map +1 -1
- package/build/facades/phoneNotifications.js +4 -14
- package/build/facades/phoneNotifications.js.map +1 -1
- package/build/index.d.ts +6 -3
- package/build/index.d.ts.map +1 -1
- package/build/index.js +4 -2
- package/build/index.js.map +1 -1
- package/build/internal.d.ts +2 -2
- package/build/internal.d.ts.map +1 -1
- package/build/internal.js +1 -1
- package/build/internal.js.map +1 -1
- package/build/react/MentraLiveOtaFlow.d.ts +38 -0
- package/build/react/MentraLiveOtaFlow.d.ts.map +1 -0
- package/build/react/MentraLiveOtaFlow.js +528 -0
- package/build/react/MentraLiveOtaFlow.js.map +1 -0
- package/build/react/index.d.ts +2 -0
- package/build/react/index.d.ts.map +1 -0
- package/build/react/index.js +2 -0
- package/build/react/index.js.map +1 -0
- package/build/react/useEngineSnapshot.d.ts +2 -0
- package/build/react/useEngineSnapshot.d.ts.map +1 -0
- package/build/react/useEngineSnapshot.js +14 -0
- package/build/react/useEngineSnapshot.js.map +1 -0
- package/build/runtime/bootstrap.d.ts +24 -1
- package/build/runtime/bootstrap.d.ts.map +1 -1
- package/build/runtime/bootstrap.js +17 -0
- package/build/runtime/bootstrap.js.map +1 -1
- package/build/runtime/scanQrSeam.d.ts +7 -0
- package/build/runtime/scanQrSeam.d.ts.map +1 -0
- package/build/runtime/scanQrSeam.js +16 -0
- package/build/runtime/scanQrSeam.js.map +1 -0
- package/build/services/AppRegistry.d.ts +18 -8
- package/build/services/AppRegistry.d.ts.map +1 -1
- package/build/services/AppRegistry.js +44 -45
- package/build/services/AppRegistry.js.map +1 -1
- package/build/services/AudioCloudUplink.d.ts +9 -3
- package/build/services/AudioCloudUplink.d.ts.map +1 -1
- package/build/services/AudioCloudUplink.js +9 -3
- package/build/services/AudioCloudUplink.js.map +1 -1
- package/build/services/AudioPlaybackService.d.ts +9 -1
- package/build/services/AudioPlaybackService.d.ts.map +1 -1
- package/build/services/AudioPlaybackService.js +77 -32
- package/build/services/AudioPlaybackService.js.map +1 -1
- package/build/services/BlobStore.d.ts +2 -1
- package/build/services/BlobStore.d.ts.map +1 -1
- package/build/services/BlobStore.js +17 -9
- package/build/services/BlobStore.js.map +1 -1
- package/build/services/CloudAudioSubscriptionSync.d.ts.map +1 -1
- package/build/services/CloudAudioSubscriptionSync.js +13 -1
- package/build/services/CloudAudioSubscriptionSync.js.map +1 -1
- package/build/services/CloudClientService.d.ts +15 -0
- package/build/services/CloudClientService.d.ts.map +1 -1
- package/build/services/CloudClientService.js +75 -0
- package/build/services/CloudClientService.js.map +1 -1
- package/build/services/ForegroundLocationPermission.d.ts +13 -0
- package/build/services/ForegroundLocationPermission.d.ts.map +1 -0
- package/build/services/ForegroundLocationPermission.js +12 -0
- package/build/services/ForegroundLocationPermission.js.map +1 -0
- package/build/services/GlassesSettingsSync.d.ts.map +1 -1
- package/build/services/GlassesSettingsSync.js +9 -3
- package/build/services/GlassesSettingsSync.js.map +1 -1
- package/build/services/GlassesStatusProjection.d.ts +2 -1
- package/build/services/GlassesStatusProjection.d.ts.map +1 -1
- package/build/services/GlassesStatusProjection.js +15 -8
- package/build/services/GlassesStatusProjection.js.map +1 -1
- package/build/services/HotspotOtaTransport.d.ts +29 -0
- package/build/services/HotspotOtaTransport.d.ts.map +1 -0
- package/build/services/HotspotOtaTransport.js +146 -0
- package/build/services/HotspotOtaTransport.js.map +1 -0
- package/build/services/HotspotShutdown.d.ts +17 -0
- package/build/services/HotspotShutdown.d.ts.map +1 -0
- package/build/services/HotspotShutdown.js +29 -0
- package/build/services/HotspotShutdown.js.map +1 -0
- package/build/services/LocalDisplayManager.d.ts +15 -0
- package/build/services/LocalDisplayManager.d.ts.map +1 -1
- package/build/services/LocalDisplayManager.js +106 -20
- package/build/services/LocalDisplayManager.js.map +1 -1
- package/build/services/LocalMiniappRuntime.d.ts +67 -1
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +486 -126
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/LocalMiniappStorage.d.ts +32 -0
- package/build/services/LocalMiniappStorage.d.ts.map +1 -0
- package/build/services/LocalMiniappStorage.js +66 -0
- package/build/services/LocalMiniappStorage.js.map +1 -0
- package/build/services/LocalMiniappUserIdentity.d.ts +20 -0
- package/build/services/LocalMiniappUserIdentity.d.ts.map +1 -0
- package/build/services/LocalMiniappUserIdentity.js +43 -0
- package/build/services/LocalMiniappUserIdentity.js.map +1 -0
- package/build/services/MicStateCoordinator.d.ts +42 -1
- package/build/services/MicStateCoordinator.d.ts.map +1 -1
- package/build/services/MicStateCoordinator.js +141 -3
- package/build/services/MicStateCoordinator.js.map +1 -1
- package/build/services/MiniappLauncher.d.ts.map +1 -1
- package/build/services/MiniappLauncher.js +23 -3
- package/build/services/MiniappLauncher.js.map +1 -1
- package/build/services/MiniappLiveness.d.ts +12 -0
- package/build/services/MiniappLiveness.d.ts.map +1 -0
- package/build/services/MiniappLiveness.js +16 -0
- package/build/services/MiniappLiveness.js.map +1 -0
- package/build/services/OtaArtifactDownloader.d.ts +65 -0
- package/build/services/OtaArtifactDownloader.d.ts.map +1 -0
- package/build/services/OtaArtifactDownloader.js +226 -0
- package/build/services/OtaArtifactDownloader.js.map +1 -0
- package/build/services/OtaAutoChain.d.ts +34 -0
- package/build/services/OtaAutoChain.d.ts.map +1 -0
- package/build/services/OtaAutoChain.js +83 -0
- package/build/services/OtaAutoChain.js.map +1 -0
- package/build/services/OtaErrorMapping.d.ts +18 -0
- package/build/services/OtaErrorMapping.d.ts.map +1 -0
- package/build/services/OtaErrorMapping.js +63 -0
- package/build/services/OtaErrorMapping.js.map +1 -0
- package/build/services/OtaInstallCoordinator.d.ts +94 -14
- package/build/services/OtaInstallCoordinator.d.ts.map +1 -1
- package/build/services/OtaInstallCoordinator.js +574 -130
- package/build/services/OtaInstallCoordinator.js.map +1 -1
- package/build/services/OtaService.d.ts.map +1 -1
- package/build/services/OtaService.js +8 -1
- package/build/services/OtaService.js.map +1 -1
- package/build/services/OtaUpdateCheckService.d.ts +38 -3
- package/build/services/OtaUpdateCheckService.d.ts.map +1 -1
- package/build/services/OtaUpdateCheckService.js +142 -22
- package/build/services/OtaUpdateCheckService.js.map +1 -1
- package/build/services/PhoneCalendarService.d.ts.map +1 -1
- package/build/services/PhoneCalendarService.js +22 -1
- package/build/services/PhoneCalendarService.js.map +1 -1
- package/build/services/PhoneCameraFovCoordinator.d.ts +2 -0
- package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -1
- package/build/services/PhoneCameraFovCoordinator.js +14 -0
- package/build/services/PhoneCameraFovCoordinator.js.map +1 -1
- package/build/services/PhoneNotificationsSync.d.ts.map +1 -1
- package/build/services/PhoneNotificationsSync.js +12 -10
- package/build/services/PhoneNotificationsSync.js.map +1 -1
- package/build/services/PhonePhotoCoordinator.d.ts +10 -3
- package/build/services/PhonePhotoCoordinator.d.ts.map +1 -1
- package/build/services/PhonePhotoCoordinator.js +21 -3
- package/build/services/PhonePhotoCoordinator.js.map +1 -1
- package/build/services/PhoneStreamCoordinator.d.ts +8 -1
- package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
- package/build/services/PhoneStreamCoordinator.js +125 -13
- package/build/services/PhoneStreamCoordinator.js.map +1 -1
- package/build/services/PhoneVideoCoordinator.d.ts +2 -0
- package/build/services/PhoneVideoCoordinator.d.ts.map +1 -1
- package/build/services/PhoneVideoCoordinator.js +10 -0
- package/build/services/PhoneVideoCoordinator.js.map +1 -1
- package/build/services/SceneRenderer.d.ts.map +1 -1
- package/build/services/SceneRenderer.js +15 -0
- package/build/services/SceneRenderer.js.map +1 -1
- package/build/services/SupportProfileSync.d.ts +22 -0
- package/build/services/SupportProfileSync.d.ts.map +1 -0
- package/build/services/SupportProfileSync.js +213 -0
- package/build/services/SupportProfileSync.js.map +1 -0
- package/build/services/TtsTextSanitizer.d.ts +5 -0
- package/build/services/TtsTextSanitizer.d.ts.map +1 -0
- package/build/services/TtsTextSanitizer.js +61 -0
- package/build/services/TtsTextSanitizer.js.map +1 -0
- package/build/services/WifiAdbRequest.d.ts +17 -0
- package/build/services/WifiAdbRequest.d.ts.map +1 -0
- package/build/services/WifiAdbRequest.js +35 -0
- package/build/services/WifiAdbRequest.js.map +1 -0
- package/build/services/asg/asgCameraApi.d.ts +3 -0
- package/build/services/asg/asgCameraApi.d.ts.map +1 -1
- package/build/services/asg/asgCameraApi.js +72 -2
- package/build/services/asg/asgCameraApi.js.map +1 -1
- package/build/services/asg/gallerySyncService.d.ts.map +1 -1
- package/build/services/asg/gallerySyncService.js +26 -14
- package/build/services/asg/gallerySyncService.js.map +1 -1
- package/build/services/asg/localNetworkTransport.d.ts +2 -2
- package/build/services/asg/localNetworkTransport.d.ts.map +1 -1
- package/build/services/asg/localNetworkTransport.js +6 -6
- package/build/services/asg/localNetworkTransport.js.map +1 -1
- package/build/services/audioPlaybackAssets.d.ts +8 -0
- package/build/services/audioPlaybackAssets.d.ts.map +1 -0
- package/build/services/audioPlaybackAssets.js +7 -0
- package/build/services/audioPlaybackAssets.js.map +1 -0
- package/build/services/glassesClockSync.js +2 -2
- package/build/services/glassesClockSync.js.map +1 -1
- package/build/services/manifestActions.d.ts +7 -0
- package/build/services/manifestActions.d.ts.map +1 -0
- package/build/services/manifestActions.js +25 -0
- package/build/services/manifestActions.js.map +1 -0
- package/build/services/otaDisplayState.d.ts +22 -11
- package/build/services/otaDisplayState.d.ts.map +1 -1
- package/build/services/otaDisplayState.js +36 -14
- package/build/services/otaDisplayState.js.map +1 -1
- package/build/services/otaInstallPolicy.d.ts +15 -3
- package/build/services/otaInstallPolicy.d.ts.map +1 -1
- package/build/services/otaInstallPolicy.js +24 -3
- package/build/services/otaInstallPolicy.js.map +1 -1
- package/build/services/otaManifestUrl.d.ts +2 -0
- package/build/services/otaManifestUrl.d.ts.map +1 -0
- package/build/services/{asgOtaVersionUrl.js → otaManifestUrl.js} +15 -5
- package/build/services/otaManifestUrl.js.map +1 -0
- package/build/services/slimStreamStatus.d.ts +11 -0
- package/build/services/slimStreamStatus.d.ts.map +1 -1
- package/build/services/slimStreamStatus.js +14 -4
- package/build/services/slimStreamStatus.js.map +1 -1
- package/build/stores/apps.d.ts.map +1 -1
- package/build/stores/apps.js +11 -0
- package/build/stores/apps.js.map +1 -1
- package/build/stores/bluetoothSettingKeys.d.ts.map +1 -1
- package/build/stores/bluetoothSettingKeys.js +2 -0
- package/build/stores/bluetoothSettingKeys.js.map +1 -1
- package/build/stores/gallerySync.d.ts +2 -0
- package/build/stores/gallerySync.d.ts.map +1 -1
- package/build/stores/gallerySync.js +2 -0
- package/build/stores/gallerySync.js.map +1 -1
- package/build/stores/glasses.d.ts.map +1 -1
- package/build/stores/glasses.js +3 -0
- package/build/stores/glasses.js.map +1 -1
- package/build/stores/settings.d.ts +17 -0
- package/build/stores/settings.d.ts.map +1 -1
- package/build/stores/settings.js +152 -19
- package/build/stores/settings.js.map +1 -1
- package/build/types/applet.d.ts +2 -0
- package/build/types/applet.d.ts.map +1 -1
- package/build/types/applet.js.map +1 -1
- package/build/types/enums.d.ts +1 -0
- package/build/types/enums.d.ts.map +1 -1
- package/build/types/enums.js +1 -0
- package/build/types/enums.js.map +1 -1
- package/build/types/hardware.d.ts +1 -0
- package/build/types/hardware.d.ts.map +1 -1
- package/build/types/hardware.js +14 -0
- package/build/types/hardware.js.map +1 -1
- package/build/utils/configuredDevHost.d.ts +6 -0
- package/build/utils/configuredDevHost.d.ts.map +1 -0
- package/build/utils/configuredDevHost.js +19 -0
- package/build/utils/configuredDevHost.js.map +1 -0
- package/build/utils/devMiniappLaunch.d.ts +19 -8
- package/build/utils/devMiniappLaunch.d.ts.map +1 -1
- package/build/utils/devMiniappLaunch.js +144 -21
- package/build/utils/devMiniappLaunch.js.map +1 -1
- package/build/utils/diagnosticContext.d.ts.map +1 -1
- package/build/utils/diagnosticContext.js +15 -17
- package/build/utils/diagnosticContext.js.map +1 -1
- package/build/utils/hardware/hardware.d.ts.map +1 -1
- package/build/utils/hardware/hardware.js +7 -3
- package/build/utils/hardware/hardware.js.map +1 -1
- package/build/utils/miniappDiagnostics.d.ts +64 -0
- package/build/utils/miniappDiagnostics.d.ts.map +1 -0
- package/build/utils/miniappDiagnostics.js +173 -0
- package/build/utils/miniappDiagnostics.js.map +1 -0
- package/build/utils/timers/timers.d.ts.map +1 -1
- package/build/utils/timers/timers.js +55 -18
- package/build/utils/timers/timers.js.map +1 -1
- package/package.json +27 -18
- package/src/assets/silence.wav +0 -0
- package/src/engine.ts +7 -2
- package/src/facades/dev.ts +9 -5
- package/src/facades/gallery.ts +1 -1
- package/src/facades/glasses.ts +17 -4
- package/src/facades/glassesSettings.ts +5 -0
- package/src/facades/miniapps.ts +7 -0
- package/src/facades/ota.ts +19 -4
- package/src/facades/pairing.ts +11 -7
- package/src/facades/phoneNotifications.ts +4 -13
- package/src/index.ts +26 -2
- package/src/internal.ts +2 -1
- package/src/react/MentraLiveOtaFlow.tsx +856 -0
- package/src/react/index.ts +7 -0
- package/src/react/useEngineSnapshot.ts +16 -0
- package/src/runtime/bootstrap.ts +33 -1
- package/src/runtime/scanQrSeam.ts +20 -0
- package/src/services/AppRegistry.ts +72 -48
- package/src/services/AudioCloudUplink.ts +9 -3
- package/src/services/AudioPlaybackService.ts +94 -34
- package/src/services/BlobStore.ts +22 -9
- package/src/services/CloudAudioSubscriptionSync.ts +13 -1
- package/src/services/CloudClientService.ts +74 -0
- package/src/services/ForegroundLocationPermission.ts +24 -0
- package/src/services/GlassesSettingsSync.ts +9 -3
- package/src/services/GlassesStatusProjection.ts +16 -8
- package/src/services/HotspotOtaTransport.ts +184 -0
- package/src/services/HotspotShutdown.ts +41 -0
- package/src/services/LocalDisplayManager.ts +112 -21
- package/src/services/LocalMiniappRuntime.ts +576 -130
- package/src/services/LocalMiniappStorage.ts +82 -0
- package/src/services/LocalMiniappUserIdentity.ts +46 -0
- package/src/services/MicStateCoordinator.ts +169 -4
- package/src/services/MiniappLauncher.ts +30 -3
- package/src/services/MiniappLiveness.ts +24 -0
- package/src/services/OtaArtifactDownloader.ts +312 -0
- package/src/services/OtaAutoChain.ts +106 -0
- package/src/services/OtaErrorMapping.ts +80 -0
- package/src/services/OtaInstallCoordinator.ts +645 -143
- package/src/services/OtaService.ts +11 -5
- package/src/services/OtaUpdateCheckService.ts +180 -20
- package/src/services/PhoneCalendarService.ts +27 -4
- package/src/services/PhoneCameraFovCoordinator.ts +15 -0
- package/src/services/PhoneNotificationsSync.ts +12 -10
- package/src/services/PhonePhotoCoordinator.ts +33 -7
- package/src/services/PhoneStreamCoordinator.ts +139 -25
- package/src/services/PhoneVideoCoordinator.ts +11 -0
- package/src/services/SceneRenderer.ts +16 -0
- package/src/services/SupportProfileSync.ts +218 -0
- package/src/services/TtsTextSanitizer.ts +66 -0
- package/src/services/WifiAdbRequest.ts +53 -0
- package/src/services/asg/asgCameraApi.ts +99 -8
- package/src/services/asg/gallerySyncService.ts +28 -19
- package/src/services/asg/localNetworkTransport.ts +7 -11
- package/src/services/audioPlaybackAssets.ts +8 -0
- package/src/services/glassesClockSync.ts +2 -2
- package/src/services/manifestActions.ts +30 -0
- package/src/services/otaDisplayState.ts +57 -16
- package/src/services/otaInstallPolicy.ts +29 -3
- package/src/services/{asgOtaVersionUrl.ts → otaManifestUrl.ts} +16 -4
- package/src/services/slimStreamStatus.ts +16 -5
- package/src/stores/apps.ts +12 -0
- package/src/stores/bluetoothSettingKeys.ts +2 -0
- package/src/stores/gallerySync.ts +4 -0
- package/src/stores/glasses.ts +3 -0
- package/src/stores/settings.ts +174 -21
- package/src/types/applet.ts +2 -0
- package/src/types/enums.ts +2 -1
- package/src/types/hardware.ts +17 -0
- package/src/utils/configuredDevHost.ts +17 -0
- package/src/utils/devMiniappLaunch.ts +151 -23
- package/src/utils/diagnosticContext.ts +17 -17
- package/src/utils/hardware/hardware.ts +7 -9
- package/src/utils/miniappDiagnostics.ts +226 -0
- package/src/utils/timers/timers.ts +57 -21
- package/build/services/asgOtaVersionUrl.d.ts +0 -2
- package/build/services/asgOtaVersionUrl.d.ts.map +0 -1
- package/build/services/asgOtaVersionUrl.js.map +0 -1
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* management (connect/disconnect/ping).
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import {Linking} from "react-native"
|
|
12
|
+
import {AppState, Linking} from "react-native"
|
|
13
13
|
import Share from "react-native-share"
|
|
14
14
|
import * as Battery from "expo-battery"
|
|
15
15
|
import * as Clipboard from "expo-clipboard"
|
|
@@ -51,6 +51,8 @@ import {phoneStreamCoordinator} from "./PhoneStreamCoordinator"
|
|
|
51
51
|
import {phoneVideoCoordinator} from "./PhoneVideoCoordinator"
|
|
52
52
|
import {runSentenceTtsPipeline} from "./SentenceTtsPipeline"
|
|
53
53
|
import {summarizeTranscriptionRoutes, transcriptionDeliveryRoute} from "./TranscriptionRouting"
|
|
54
|
+
import {prepareTtsSentences} from "./TtsTextSanitizer"
|
|
55
|
+
import {handleWifiAdbRequest} from "./WifiAdbRequest"
|
|
54
56
|
import {cloudClientService} from "./CloudClientService"
|
|
55
57
|
import {miniappLauncher} from "./MiniappLauncher"
|
|
56
58
|
import {
|
|
@@ -64,25 +66,38 @@ import {
|
|
|
64
66
|
type TtsSynthesisResult,
|
|
65
67
|
} from "../runtime/config"
|
|
66
68
|
import {getAnalytics, getUiSeams} from "../runtime/bootstrap"
|
|
69
|
+
import {invokeScanQrSeam} from "../runtime/scanQrSeam"
|
|
67
70
|
import {normalizeStreamAudioConfig, normalizeStreamVideoConfig} from "../runtime/streamConfig"
|
|
71
|
+
import {toLanguageHint} from "@mentra/cloud-protocol/languages"
|
|
68
72
|
import type {AudioSubscription, LanguageSource, TranscriptionData, TranslationData} from "@mentra/cloud-protocol"
|
|
73
|
+
import {buildMiniappManifestSnapshot, type MiniappRuntimeDiagnosticSnapshot} from "../utils/miniappDiagnostics"
|
|
69
74
|
import ttsModelManager from "./TTSModelManager"
|
|
70
75
|
import {NavigationHandlers} from "./NavigationHandlers"
|
|
71
76
|
import type {ClientApp} from "../types/applet"
|
|
72
77
|
import {useAppStatusStore} from "../stores/apps"
|
|
73
78
|
import {getDevAppAttestation, getDevAppSourcePackage} from "./AppRegistry"
|
|
79
|
+
import {resolveForegroundLocationPermission} from "./ForegroundLocationPermission"
|
|
80
|
+
import {advanceMiniappPingLiveness} from "./MiniappLiveness"
|
|
74
81
|
import {listPhoneCalendarEvents, PhoneCalendarError} from "./PhoneCalendarService"
|
|
82
|
+
import {LocalMiniappStorage} from "./LocalMiniappStorage"
|
|
75
83
|
|
|
76
84
|
// =============================================================================
|
|
77
85
|
// Types
|
|
78
86
|
// =============================================================================
|
|
79
87
|
|
|
80
88
|
export interface InstalledMiniappManifest {
|
|
89
|
+
packageName?: string
|
|
81
90
|
/** Human-facing app name (from miniapp.json `name`). Shown in the "Starting
|
|
82
91
|
* <name>…" boot message; falls back to the package name when absent. */
|
|
83
92
|
name?: string
|
|
93
|
+
version?: string
|
|
94
|
+
sdkVersion?: string
|
|
95
|
+
minHostVersion?: string
|
|
96
|
+
type?: string
|
|
97
|
+
entry?: {background?: string; ui?: string}
|
|
84
98
|
permissions?: Array<{type: string; required?: boolean; description?: string}>
|
|
85
99
|
hardwareRequirements?: Array<{type: string; level: string; description?: string}>
|
|
100
|
+
actions?: Array<{id?: unknown; description?: unknown; parameters?: unknown; outputSchema?: unknown}>
|
|
86
101
|
}
|
|
87
102
|
|
|
88
103
|
type SpeakerStateValue = "idle" | "loading" | "playing" | "stopped" | "error"
|
|
@@ -95,6 +110,7 @@ interface ConnectedMiniapp {
|
|
|
95
110
|
cloudTranscriptionStreams: Set<string>
|
|
96
111
|
sendMessage: (raw: string) => void
|
|
97
112
|
lastPongAt: number
|
|
113
|
+
unansweredPingRounds: number
|
|
98
114
|
installedManifest?: InstalledMiniappManifest
|
|
99
115
|
authRefreshTimerId: number | null
|
|
100
116
|
/**
|
|
@@ -168,6 +184,16 @@ function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | nul
|
|
|
168
184
|
}
|
|
169
185
|
|
|
170
186
|
const LOG_TAG = "LOCAL_MINIAPP"
|
|
187
|
+
const DIAGNOSTIC_MAX_LIST_ITEMS = 100
|
|
188
|
+
const DIAGNOSTIC_MAX_STRING_LENGTH = 512
|
|
189
|
+
|
|
190
|
+
function diagnosticStringList(values: Iterable<string>): string[] {
|
|
191
|
+
return [...values]
|
|
192
|
+
.map((value) => value.slice(0, DIAGNOSTIC_MAX_STRING_LENGTH))
|
|
193
|
+
.sort()
|
|
194
|
+
.slice(0, DIAGNOSTIC_MAX_LIST_ITEMS)
|
|
195
|
+
}
|
|
196
|
+
|
|
171
197
|
const SYSTEM_MINIAPP_PACKAGES = new Set([
|
|
172
198
|
"com.mentra.camera",
|
|
173
199
|
"com.mentra.gallery",
|
|
@@ -205,10 +231,29 @@ const RGB_LED_ACTIONS = new Set<RgbLedAction>(["on", "off"])
|
|
|
205
231
|
const RGB_LED_COLORS = new Set<RgbLedColor>(["red", "green", "blue", "orange", "white"])
|
|
206
232
|
const CAMERA_FOV_MIN = 62
|
|
207
233
|
const CAMERA_FOV_MAX = 118
|
|
208
|
-
const CAMERA_FOV_DEFAULT =
|
|
209
|
-
const CAMERA_FOV_PRESETS: Record<CameraFovPreset, number> = {narrow: 82, standard:
|
|
234
|
+
const CAMERA_FOV_DEFAULT = 118
|
|
235
|
+
const CAMERA_FOV_PRESETS: Record<CameraFovPreset, number> = {narrow: 82, standard: 102, wide: 118}
|
|
210
236
|
const CAMERA_ROI_POSITION_BY_NAME: Record<string, CameraRoiPosition> = {center: "center", bottom: "bottom", top: "top"}
|
|
211
237
|
|
|
238
|
+
const localMiniappStorageBackend = {
|
|
239
|
+
get(key: string): unknown | null {
|
|
240
|
+
const result = mmkvStorage.load<unknown>(key)
|
|
241
|
+
return result.is_ok() ? result.value : null
|
|
242
|
+
},
|
|
243
|
+
has(key: string): boolean {
|
|
244
|
+
return mmkvStorage.load<unknown>(key).is_ok()
|
|
245
|
+
},
|
|
246
|
+
set(key: string, value: unknown): void {
|
|
247
|
+
mmkvStorage.save(key, value)
|
|
248
|
+
},
|
|
249
|
+
remove(key: string): void {
|
|
250
|
+
mmkvStorage.remove(key)
|
|
251
|
+
},
|
|
252
|
+
keys(): string[] {
|
|
253
|
+
return mmkvStorage.getAllKeys()
|
|
254
|
+
},
|
|
255
|
+
}
|
|
256
|
+
|
|
212
257
|
// =============================================================================
|
|
213
258
|
// Declared-permission record helper (for CONNECT_ACK / PERMISSIONS_UPDATE)
|
|
214
259
|
// =============================================================================
|
|
@@ -350,11 +395,22 @@ class LocalMiniappRuntime {
|
|
|
350
395
|
/** Ref-counted stream subscriptions: stream → set of packageNames. */
|
|
351
396
|
private streamSubscribers: Map<string, Set<string>> = new Map()
|
|
352
397
|
|
|
398
|
+
/** Host observers for the Mentra Live button-capture policy. */
|
|
399
|
+
private buttonPressSubscriberListeners = new Set<(packageNames: string[]) => void>()
|
|
400
|
+
|
|
353
401
|
/** Guards one-time wiring of the cloud transcript/translation fan-out. */
|
|
354
402
|
private cloudResultsWired = false
|
|
355
403
|
private cloudStatusWired = false
|
|
356
404
|
private cloudAudioSubscriptionSync = new CloudAudioSubscriptionSync()
|
|
357
405
|
|
|
406
|
+
/**
|
|
407
|
+
* Per-miniapp language hints from TRANSCRIPTION_CONFIG (issue 021 WP3).
|
|
408
|
+
* Bare ISO 639-1 codes, already registry-validated by the SDK and
|
|
409
|
+
* re-normalized here (defense in depth). Merged (union) into the auto-mode
|
|
410
|
+
* cloud transcription subscription in buildCloudAudioSubscriptions.
|
|
411
|
+
*/
|
|
412
|
+
private transcriptionHintsByApp = new Map<string, string[]>()
|
|
413
|
+
|
|
358
414
|
/** Ping interval handle. */
|
|
359
415
|
private pingIntervalId: number | null = null
|
|
360
416
|
private foregroundProbeTimers: Map<string, number> = new Map()
|
|
@@ -381,6 +437,99 @@ class LocalMiniappRuntime {
|
|
|
381
437
|
|
|
382
438
|
private constructor() {}
|
|
383
439
|
|
|
440
|
+
/** Snapshot of miniapps actively subscribed to hardware button events. */
|
|
441
|
+
public getButtonPressSubscribers(): string[] {
|
|
442
|
+
return [...(this.streamSubscribers.get(MiniappStreamType.BUTTON_PRESS) ?? [])].sort()
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** Observe active hardware-button subscriptions. */
|
|
446
|
+
public onButtonPressSubscribersChanged(listener: (packageNames: string[]) => void): () => void {
|
|
447
|
+
this.buttonPressSubscriberListeners.add(listener)
|
|
448
|
+
return () => this.buttonPressSubscriberListeners.delete(listener)
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/** Report-safe live miniapp/subscription/resource snapshot. */
|
|
452
|
+
public getDiagnosticSnapshot(nowMs = Date.now()): MiniappRuntimeDiagnosticSnapshot {
|
|
453
|
+
const connectedApps = [...this.connectedApps.entries()]
|
|
454
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
455
|
+
.slice(0, DIAGNOSTIC_MAX_LIST_ITEMS)
|
|
456
|
+
.map(([packageName, app]) => ({
|
|
457
|
+
packageName: packageName.slice(0, DIAGNOSTIC_MAX_STRING_LENGTH),
|
|
458
|
+
handshakeComplete: this.handshookApps.has(packageName),
|
|
459
|
+
subscriptions: diagnosticStringList(app.subscriptions),
|
|
460
|
+
lastMessageAgeMs: Math.max(0, nowMs - app.lastPongAt),
|
|
461
|
+
requestedLocationRate: app.requestedLocationRate,
|
|
462
|
+
hasActiveSpeakerStream: app.activeSpeakerStreamId !== undefined,
|
|
463
|
+
...(app.installedManifest
|
|
464
|
+
? {
|
|
465
|
+
manifest: buildMiniappManifestSnapshot(app.installedManifest, {
|
|
466
|
+
packageName,
|
|
467
|
+
name: app.installedManifest.name,
|
|
468
|
+
version: app.installedManifest.version,
|
|
469
|
+
type: app.installedManifest.type,
|
|
470
|
+
}),
|
|
471
|
+
}
|
|
472
|
+
: {}),
|
|
473
|
+
}))
|
|
474
|
+
const subscriptionsByStream = Object.fromEntries(
|
|
475
|
+
[...this.streamSubscribers.entries()]
|
|
476
|
+
.filter(([, packageNames]) => packageNames.size > 0)
|
|
477
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
478
|
+
.slice(0, DIAGNOSTIC_MAX_LIST_ITEMS)
|
|
479
|
+
.map(([stream, packageNames]) => [
|
|
480
|
+
stream.slice(0, DIAGNOSTIC_MAX_STRING_LENGTH),
|
|
481
|
+
diagnosticStringList(packageNames),
|
|
482
|
+
]),
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
return {
|
|
486
|
+
connectedApps,
|
|
487
|
+
subscriptionsByStream,
|
|
488
|
+
resources: {
|
|
489
|
+
display: localDisplayManager.getDiagnosticSnapshot(),
|
|
490
|
+
camera: phonePhotoCoordinator.getDiagnosticSnapshot(),
|
|
491
|
+
video: phoneVideoCoordinator.getDiagnosticSnapshot(),
|
|
492
|
+
stream: phoneStreamCoordinator.getDiagnosticSnapshot(),
|
|
493
|
+
cameraFov: phoneCameraFovCoordinator.getDiagnosticSnapshot(),
|
|
494
|
+
},
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
private replaceStreamSubscribers(
|
|
499
|
+
packageName: string,
|
|
500
|
+
previousStreams: Iterable<string>,
|
|
501
|
+
nextStreams: Iterable<string>,
|
|
502
|
+
) {
|
|
503
|
+
const previousButtonSubscribers = this.getButtonPressSubscribers().join("\u0000")
|
|
504
|
+
|
|
505
|
+
for (const stream of previousStreams) {
|
|
506
|
+
const subscribers = this.streamSubscribers.get(stream)
|
|
507
|
+
if (!subscribers) continue
|
|
508
|
+
subscribers.delete(packageName)
|
|
509
|
+
if (subscribers.size === 0) this.streamSubscribers.delete(stream)
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
for (const stream of nextStreams) {
|
|
513
|
+
let subscribers = this.streamSubscribers.get(stream)
|
|
514
|
+
if (!subscribers) {
|
|
515
|
+
subscribers = new Set()
|
|
516
|
+
this.streamSubscribers.set(stream, subscribers)
|
|
517
|
+
}
|
|
518
|
+
subscribers.add(packageName)
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
const nextButtonSubscribers = this.getButtonPressSubscribers()
|
|
522
|
+
if (previousButtonSubscribers !== nextButtonSubscribers.join("\u0000")) {
|
|
523
|
+
for (const listener of this.buttonPressSubscriberListeners) {
|
|
524
|
+
try {
|
|
525
|
+
listener(nextButtonSubscribers)
|
|
526
|
+
} catch (error) {
|
|
527
|
+
console.warn(`${LOG_TAG}: button press subscriber listener threw:`, error)
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
384
533
|
/**
|
|
385
534
|
* Generate an HMAC-signed local session token for browser fallback auth.
|
|
386
535
|
* Token format: base64(JSON({userId, packageName, exp})).base64(HMAC-SHA256(payload, secret))
|
|
@@ -606,13 +755,7 @@ class LocalMiniappRuntime {
|
|
|
606
755
|
BgTimer.clearTimeout(existing.authRetryTimerId)
|
|
607
756
|
existing.authRetryTimerId = null
|
|
608
757
|
}
|
|
609
|
-
|
|
610
|
-
const subs = this.streamSubscribers.get(stream)
|
|
611
|
-
if (subs) {
|
|
612
|
-
subs.delete(packageName)
|
|
613
|
-
if (subs.size === 0) this.streamSubscribers.delete(stream)
|
|
614
|
-
}
|
|
615
|
-
}
|
|
758
|
+
this.replaceStreamSubscribers(packageName, existing.subscriptions, [])
|
|
616
759
|
this.recomputeMicRequirements()
|
|
617
760
|
this.updateCloudSubscriptions()
|
|
618
761
|
}
|
|
@@ -622,6 +765,7 @@ class LocalMiniappRuntime {
|
|
|
622
765
|
cloudTranscriptionStreams: new Set(),
|
|
623
766
|
sendMessage: sendFn,
|
|
624
767
|
lastPongAt: Date.now(),
|
|
768
|
+
unansweredPingRounds: 0,
|
|
625
769
|
installedManifest,
|
|
626
770
|
authRefreshTimerId: null,
|
|
627
771
|
authRetryTimerId: null,
|
|
@@ -742,6 +886,7 @@ class LocalMiniappRuntime {
|
|
|
742
886
|
|
|
743
887
|
public unregisterApp(packageName: string): void {
|
|
744
888
|
console.log(`${LOG_TAG}: unregisterApp(${packageName})`)
|
|
889
|
+
const releasedMicGateOverride = micStateCoordinator.clearMiniappGateOverrides(packageName)
|
|
745
890
|
this.clearForegroundProbe(packageName)
|
|
746
891
|
this.clearMiniappAuthRefresh(packageName)
|
|
747
892
|
this.clearMiniappAuthDeliveryRetry(packageName)
|
|
@@ -750,6 +895,9 @@ class LocalMiniappRuntime {
|
|
|
750
895
|
// the early-return so it runs even if the connectedApps entry is already gone.
|
|
751
896
|
this.handshookApps.delete(packageName)
|
|
752
897
|
this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`))
|
|
898
|
+
// Drop this app's transcription hints; the cloud subscription rebuild
|
|
899
|
+
// below (via replaceStreamSubscribers) re-derives the hint union without it.
|
|
900
|
+
this.transcriptionHintsByApp.delete(packageName)
|
|
753
901
|
|
|
754
902
|
// Warm-up is a request-owned camera lease. Cancel it even if the app record was already
|
|
755
903
|
// removed so a close racing the camera-open promise cannot leave the sensor running.
|
|
@@ -766,19 +914,18 @@ class LocalMiniappRuntime {
|
|
|
766
914
|
console.warn(`${LOG_TAG}: failed to release camera FOV override for ${packageName} on unregister`, error)
|
|
767
915
|
})
|
|
768
916
|
const app = this.connectedApps.get(packageName)
|
|
769
|
-
if (!app)
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
if (subs) {
|
|
775
|
-
subs.delete(packageName)
|
|
776
|
-
if (subs.size === 0) {
|
|
777
|
-
this.streamSubscribers.delete(stream)
|
|
778
|
-
}
|
|
917
|
+
if (!app) {
|
|
918
|
+
if (releasedMicGateOverride) {
|
|
919
|
+
void micStateCoordinator.syncEffectiveGatePolicy(this.getConfiguredMicGates()).catch((error) => {
|
|
920
|
+
console.warn(`${LOG_TAG}: failed to restore mic gate policy for ${packageName}`, error)
|
|
921
|
+
})
|
|
779
922
|
}
|
|
923
|
+
return
|
|
780
924
|
}
|
|
781
925
|
|
|
926
|
+
// Remove from all stream subscriber sets
|
|
927
|
+
this.replaceStreamSubscribers(packageName, app.subscriptions, [])
|
|
928
|
+
|
|
782
929
|
// Reset per-session warning dedup so a relaunch surfaces issues again.
|
|
783
930
|
resetPermissionWarnings(packageName)
|
|
784
931
|
|
|
@@ -844,6 +991,11 @@ class LocalMiniappRuntime {
|
|
|
844
991
|
|
|
845
992
|
this.connectedApps.delete(packageName)
|
|
846
993
|
this.recomputeMicRequirements()
|
|
994
|
+
if (releasedMicGateOverride) {
|
|
995
|
+
void micStateCoordinator.syncEffectiveGatePolicy(this.getConfiguredMicGates()).catch((error) => {
|
|
996
|
+
console.warn(`${LOG_TAG}: failed to restore mic gate policy for ${packageName}`, error)
|
|
997
|
+
})
|
|
998
|
+
}
|
|
847
999
|
this.updateCloudSubscriptions()
|
|
848
1000
|
|
|
849
1001
|
if (this.connectedApps.size === 0) {
|
|
@@ -1017,9 +1169,18 @@ class LocalMiniappRuntime {
|
|
|
1017
1169
|
case MiniappRequestType.CAMERA_FOV:
|
|
1018
1170
|
void this.handleCameraFov(packageName, payload, requestId)
|
|
1019
1171
|
break
|
|
1172
|
+
case MiniappRequestType.SET_WIFI_ADB_STATE:
|
|
1173
|
+
void this.handleSetWifiAdbState(packageName, payload, requestId)
|
|
1174
|
+
break
|
|
1020
1175
|
case MiniappRequestType.IMU_SET_ENABLED:
|
|
1021
1176
|
this.handleImuSetEnabled(packageName, payload, requestId)
|
|
1022
1177
|
break
|
|
1178
|
+
case MiniappRequestType.MIC_SET_VAD_ENABLED:
|
|
1179
|
+
void this.handleMicSetVadEnabled(packageName, payload, requestId)
|
|
1180
|
+
break
|
|
1181
|
+
case MiniappRequestType.MIC_SET_LOUDNESS_GATE_ENABLED:
|
|
1182
|
+
void this.handleMicSetLoudnessGateEnabled(packageName, payload, requestId)
|
|
1183
|
+
break
|
|
1023
1184
|
case MiniappRequestType.PING:
|
|
1024
1185
|
// SDK should handle this itself; reply PONG just in case
|
|
1025
1186
|
this.sendToMiniapp(packageName, {type: MiniappResponseType.PONG}, requestId)
|
|
@@ -1042,52 +1203,83 @@ class LocalMiniappRuntime {
|
|
|
1042
1203
|
case MiniappRequestType.DOWNLOAD:
|
|
1043
1204
|
this.handleDownload(packageName, payload, requestId)
|
|
1044
1205
|
break
|
|
1206
|
+
case MiniappRequestType.SCAN_QR:
|
|
1207
|
+
void this.handleScanQr(packageName, payload, requestId)
|
|
1208
|
+
break
|
|
1045
1209
|
|
|
1046
1210
|
// Persistent binary blob storage (session.blob)
|
|
1047
1211
|
case MiniappRequestType.BLOB_CREATE:
|
|
1048
|
-
this.
|
|
1212
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1213
|
+
this.blobStore.handleCreate(packageName, payload, requestId),
|
|
1214
|
+
)
|
|
1049
1215
|
break
|
|
1050
1216
|
case MiniappRequestType.BLOB_WRITE:
|
|
1051
|
-
this.
|
|
1217
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1218
|
+
this.blobStore.handleWrite(packageName, payload, requestId),
|
|
1219
|
+
)
|
|
1052
1220
|
break
|
|
1053
1221
|
case MiniappRequestType.BLOB_COMMIT:
|
|
1054
|
-
this.
|
|
1222
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1223
|
+
this.blobStore.handleCommit(packageName, payload, requestId),
|
|
1224
|
+
)
|
|
1055
1225
|
break
|
|
1056
1226
|
case MiniappRequestType.BLOB_ABORT:
|
|
1057
|
-
this.
|
|
1227
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1228
|
+
this.blobStore.handleAbort(packageName, payload, requestId),
|
|
1229
|
+
)
|
|
1058
1230
|
break
|
|
1059
1231
|
case MiniappRequestType.BLOB_SET_FROM_URL:
|
|
1060
|
-
|
|
1232
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1233
|
+
this.blobStore.handleSetFromUrl(packageName, payload, requestId),
|
|
1234
|
+
)
|
|
1061
1235
|
break
|
|
1062
1236
|
case MiniappRequestType.BLOB_IMPORT:
|
|
1063
|
-
|
|
1237
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1238
|
+
this.blobStore.handleImport(packageName, payload, requestId),
|
|
1239
|
+
)
|
|
1064
1240
|
break
|
|
1065
1241
|
case MiniappRequestType.BLOB_GET:
|
|
1066
|
-
this.blobStore.handleGet(packageName, payload, requestId)
|
|
1242
|
+
this.enqueueBlobRequest(packageName, requestId, () => this.blobStore.handleGet(packageName, payload, requestId))
|
|
1067
1243
|
break
|
|
1068
1244
|
case MiniappRequestType.BLOB_LIST:
|
|
1069
|
-
this.
|
|
1245
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1246
|
+
this.blobStore.handleList(packageName, payload, requestId),
|
|
1247
|
+
)
|
|
1070
1248
|
break
|
|
1071
1249
|
case MiniappRequestType.BLOB_USAGE:
|
|
1072
|
-
this.
|
|
1250
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1251
|
+
this.blobStore.handleUsage(packageName, payload, requestId),
|
|
1252
|
+
)
|
|
1073
1253
|
break
|
|
1074
1254
|
case MiniappRequestType.BLOB_DELETE:
|
|
1075
|
-
this.
|
|
1255
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1256
|
+
this.blobStore.handleDelete(packageName, payload, requestId),
|
|
1257
|
+
)
|
|
1076
1258
|
break
|
|
1077
1259
|
case MiniappRequestType.BLOB_CLEAR:
|
|
1078
|
-
this.
|
|
1260
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1261
|
+
this.blobStore.handleClear(packageName, payload, requestId),
|
|
1262
|
+
)
|
|
1079
1263
|
break
|
|
1080
1264
|
case MiniappRequestType.BLOB_OPEN_READ:
|
|
1081
|
-
this.
|
|
1265
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1266
|
+
this.blobStore.handleOpenRead(packageName, payload, requestId),
|
|
1267
|
+
)
|
|
1082
1268
|
break
|
|
1083
1269
|
case MiniappRequestType.BLOB_READ:
|
|
1084
|
-
this.
|
|
1270
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1271
|
+
this.blobStore.handleRead(packageName, payload, requestId),
|
|
1272
|
+
)
|
|
1085
1273
|
break
|
|
1086
1274
|
case MiniappRequestType.BLOB_CLOSE_READ:
|
|
1087
|
-
this.
|
|
1275
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1276
|
+
this.blobStore.handleCloseRead(packageName, payload, requestId),
|
|
1277
|
+
)
|
|
1088
1278
|
break
|
|
1089
1279
|
case MiniappRequestType.BLOB_SHARE:
|
|
1090
|
-
|
|
1280
|
+
this.enqueueBlobRequest(packageName, requestId, () =>
|
|
1281
|
+
this.blobStore.handleShare(packageName, payload, requestId),
|
|
1282
|
+
)
|
|
1091
1283
|
break
|
|
1092
1284
|
|
|
1093
1285
|
// Cloud-coordinated features
|
|
@@ -1144,8 +1336,20 @@ class LocalMiniappRuntime {
|
|
|
1144
1336
|
})
|
|
1145
1337
|
break
|
|
1146
1338
|
|
|
1339
|
+
case MiniappRequestType.TRANSCRIPTION_CONFIG:
|
|
1340
|
+
this.handleTranscriptionConfig(packageName, payload, requestId)
|
|
1341
|
+
break
|
|
1342
|
+
|
|
1147
1343
|
default:
|
|
1344
|
+
// NACK instead of silent drop (issue 021 S3): an SDK that sends a
|
|
1345
|
+
// request this runtime doesn't implement must get a rejected promise,
|
|
1346
|
+
// not a permanently-pending silent gap. One-shots (no requestId) can
|
|
1347
|
+
// only be logged.
|
|
1148
1348
|
console.warn(`${LOG_TAG}: Unknown request type from ${packageName}: ${requestType}`)
|
|
1349
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1350
|
+
code: MiniappErrorCode.NOT_IMPLEMENTED,
|
|
1351
|
+
message: `Unknown or unimplemented request type: ${requestType}`,
|
|
1352
|
+
})
|
|
1149
1353
|
break
|
|
1150
1354
|
}
|
|
1151
1355
|
}
|
|
@@ -1170,6 +1374,7 @@ class LocalMiniappRuntime {
|
|
|
1170
1374
|
|
|
1171
1375
|
// Update lastPongAt so it doesn't time out right away
|
|
1172
1376
|
existing.lastPongAt = Date.now()
|
|
1377
|
+
existing.unansweredPingRounds = 0
|
|
1173
1378
|
|
|
1174
1379
|
// Read current glasses capabilities from the settings store
|
|
1175
1380
|
const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable) as
|
|
@@ -1377,6 +1582,51 @@ class LocalMiniappRuntime {
|
|
|
1377
1582
|
}
|
|
1378
1583
|
}
|
|
1379
1584
|
|
|
1585
|
+
/**
|
|
1586
|
+
* TRANSCRIPTION_CONFIG (issue 021 WP3): store the miniapp's language hints
|
|
1587
|
+
* and rebuild the cloud subscription set so the auto-mode subscription
|
|
1588
|
+
* carries them. Replies with a REQUEST_RESULT so the SDK's configure()
|
|
1589
|
+
* promise settles: unknown hint values are an error, not a silent drop.
|
|
1590
|
+
*/
|
|
1591
|
+
private handleTranscriptionConfig(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
|
|
1592
|
+
const app = this.connectedApps.get(packageName)
|
|
1593
|
+
if (!app) return
|
|
1594
|
+
|
|
1595
|
+
const config = (payload.config ?? {}) as Record<string, unknown>
|
|
1596
|
+
const rawHints = config.languageHints
|
|
1597
|
+
if (rawHints !== undefined && !Array.isArray(rawHints)) {
|
|
1598
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1599
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1600
|
+
message: "languageHints must be an array of language codes",
|
|
1601
|
+
})
|
|
1602
|
+
return
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
if (rawHints === undefined || rawHints.length === 0) {
|
|
1606
|
+
this.transcriptionHintsByApp.delete(packageName)
|
|
1607
|
+
} else {
|
|
1608
|
+
// Defense in depth: the SDK already validated, but the wire is untyped.
|
|
1609
|
+
const hints: string[] = []
|
|
1610
|
+
const invalid: string[] = []
|
|
1611
|
+
for (const raw of rawHints as unknown[]) {
|
|
1612
|
+
const hint = typeof raw === "string" ? toLanguageHint(raw) : null
|
|
1613
|
+
if (hint) hints.push(hint)
|
|
1614
|
+
else invalid.push(String(raw))
|
|
1615
|
+
}
|
|
1616
|
+
if (invalid.length > 0) {
|
|
1617
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1618
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1619
|
+
message: `Unknown language hint(s): ${invalid.join(", ")}`,
|
|
1620
|
+
})
|
|
1621
|
+
return
|
|
1622
|
+
}
|
|
1623
|
+
this.transcriptionHintsByApp.set(packageName, [...new Set(hints)].sort())
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
this.updateCloudSubscriptions()
|
|
1627
|
+
this.sendResult(packageName, requestId, true, {applied: true})
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1380
1630
|
private handleSubscribe(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
|
|
1381
1631
|
const app = this.connectedApps.get(packageName)
|
|
1382
1632
|
if (!app) return
|
|
@@ -1450,18 +1700,8 @@ class LocalMiniappRuntime {
|
|
|
1450
1700
|
}
|
|
1451
1701
|
}
|
|
1452
1702
|
|
|
1453
|
-
// Remove old subscriptions for this app
|
|
1454
|
-
for (const oldStream of app.subscriptions) {
|
|
1455
|
-
const subs = this.streamSubscribers.get(oldStream)
|
|
1456
|
-
if (subs) {
|
|
1457
|
-
subs.delete(packageName)
|
|
1458
|
-
if (subs.size === 0) {
|
|
1459
|
-
this.streamSubscribers.delete(oldStream)
|
|
1460
|
-
}
|
|
1461
|
-
}
|
|
1462
|
-
}
|
|
1463
|
-
|
|
1464
1703
|
// Add new subscriptions
|
|
1704
|
+
const previousSubscriptions = app.subscriptions
|
|
1465
1705
|
app.subscriptions = new Set(streams)
|
|
1466
1706
|
app.forceLocalTranscriptionStreams = new Set(
|
|
1467
1707
|
[...forceLocalTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
|
|
@@ -1469,14 +1709,7 @@ class LocalMiniappRuntime {
|
|
|
1469
1709
|
app.cloudTranscriptionStreams = new Set(
|
|
1470
1710
|
[...cloudTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
|
|
1471
1711
|
)
|
|
1472
|
-
|
|
1473
|
-
let subs = this.streamSubscribers.get(stream)
|
|
1474
|
-
if (!subs) {
|
|
1475
|
-
subs = new Set()
|
|
1476
|
-
this.streamSubscribers.set(stream, subs)
|
|
1477
|
-
}
|
|
1478
|
-
subs.add(packageName)
|
|
1479
|
-
}
|
|
1712
|
+
this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
|
|
1480
1713
|
|
|
1481
1714
|
this.recomputeMicRequirements()
|
|
1482
1715
|
this.updateCloudSubscriptions()
|
|
@@ -1966,16 +2199,21 @@ class LocalMiniappRuntime {
|
|
|
1966
2199
|
|
|
1967
2200
|
private async handleSpeak(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
|
|
1968
2201
|
const rawText = payload.text
|
|
1969
|
-
const
|
|
2202
|
+
const rawSentences =
|
|
1970
2203
|
typeof rawText === "string"
|
|
1971
2204
|
? [rawText.trim()].filter(Boolean)
|
|
1972
2205
|
: Array.isArray(rawText)
|
|
1973
|
-
? rawText
|
|
2206
|
+
? rawText
|
|
2207
|
+
.filter((sentence): sentence is string => typeof sentence === "string")
|
|
2208
|
+
.map((sentence) => sentence.trim())
|
|
2209
|
+
.filter(Boolean)
|
|
1974
2210
|
: []
|
|
2211
|
+
const enableSanitization = payload.enableSanitization !== false
|
|
2212
|
+
const sentences = prepareTtsSentences(rawSentences, enableSanitization)
|
|
1975
2213
|
if (sentences.length === 0) {
|
|
1976
2214
|
this.sendResult(packageName, requestId, false, undefined, {
|
|
1977
2215
|
code: MiniappErrorCode.INTERNAL,
|
|
1978
|
-
message: "speak requires text or a non-empty sentence list",
|
|
2216
|
+
message: "speak requires text or a non-empty sentence list with speakable content",
|
|
1979
2217
|
})
|
|
1980
2218
|
return
|
|
1981
2219
|
}
|
|
@@ -2095,7 +2333,6 @@ class LocalMiniappRuntime {
|
|
|
2095
2333
|
appId: packageName,
|
|
2096
2334
|
volume,
|
|
2097
2335
|
stopOtherAudio,
|
|
2098
|
-
suppressCloudUplink: true,
|
|
2099
2336
|
},
|
|
2100
2337
|
(_responseId, success, error, duration, completionReason) => {
|
|
2101
2338
|
if (run.playbackRequestId === sentenceRequestId) run.playbackRequestId = undefined
|
|
@@ -2144,7 +2381,6 @@ class LocalMiniappRuntime {
|
|
|
2144
2381
|
appId: packageName,
|
|
2145
2382
|
volume,
|
|
2146
2383
|
stopOtherAudio,
|
|
2147
|
-
suppressCloudUplink: true,
|
|
2148
2384
|
},
|
|
2149
2385
|
(_respId, success, error, duration, completionReason) => {
|
|
2150
2386
|
if (run.playbackRequestId === audioRequestId) run.playbackRequestId = undefined
|
|
@@ -2197,7 +2433,6 @@ class LocalMiniappRuntime {
|
|
|
2197
2433
|
appId: packageName,
|
|
2198
2434
|
volume,
|
|
2199
2435
|
stopOtherAudio,
|
|
2200
|
-
suppressCloudUplink: true,
|
|
2201
2436
|
},
|
|
2202
2437
|
(_respId, success, error, duration, completionReason) => {
|
|
2203
2438
|
if (run.playbackRequestId === audioRequestId) run.playbackRequestId = undefined
|
|
@@ -2292,7 +2527,17 @@ class LocalMiniappRuntime {
|
|
|
2292
2527
|
|
|
2293
2528
|
private async handleLocationPoll(packageName: string, requestId?: string): Promise<void> {
|
|
2294
2529
|
try {
|
|
2295
|
-
|
|
2530
|
+
// Expo's Android requestForegroundPermissionsAsync always delegates to
|
|
2531
|
+
// Activity.requestPermissions(), even when permission is already granted.
|
|
2532
|
+
// Calling it after the user presses Home therefore waits until the Activity
|
|
2533
|
+
// resumes. Read the current grant first, and only enter the prompt flow
|
|
2534
|
+
// while the Activity is actually foregrounded.
|
|
2535
|
+
const permissionStartedAt = Date.now()
|
|
2536
|
+
const permission = await resolveForegroundLocationPermission(Location, () => AppState.currentState)
|
|
2537
|
+
const {status} = permission
|
|
2538
|
+
console.log(
|
|
2539
|
+
`${LOG_TAG}: location poll permission status=${status} appState=${AppState.currentState} elapsed=${Date.now() - permissionStartedAt}ms`,
|
|
2540
|
+
)
|
|
2296
2541
|
if (status !== "granted") {
|
|
2297
2542
|
this.sendResult(packageName, requestId, false, undefined, {
|
|
2298
2543
|
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
@@ -2305,8 +2550,11 @@ class LocalMiniappRuntime {
|
|
|
2305
2550
|
// a fresh fix if the cache is empty. Use Low accuracy on the fresh
|
|
2306
2551
|
// path so we get a cell/wifi-tower fix in ~1s instead of waiting
|
|
2307
2552
|
// for full GPS warm-up.
|
|
2553
|
+
const locationStartedAt = Date.now()
|
|
2308
2554
|
const cached = await Location.getLastKnownPositionAsync({maxAge: 60_000})
|
|
2555
|
+
console.log(`${LOG_TAG}: location poll cache hit=${cached !== null} elapsed=${Date.now() - locationStartedAt}ms`)
|
|
2309
2556
|
const location = cached ?? (await Location.getCurrentPositionAsync({accuracy: Location.Accuracy.Low}))
|
|
2557
|
+
console.log(`${LOG_TAG}: location poll resolved elapsed=${Date.now() - locationStartedAt}ms`)
|
|
2310
2558
|
|
|
2311
2559
|
this.sendResult(packageName, requestId, true, {
|
|
2312
2560
|
lat: location.coords.latitude,
|
|
@@ -2348,9 +2596,69 @@ class LocalMiniappRuntime {
|
|
|
2348
2596
|
private readonly blobStore = new BlobStore({
|
|
2349
2597
|
sendResult: (packageName, requestId, ok, result, error) =>
|
|
2350
2598
|
this.sendResult(packageName, requestId, ok, result, error),
|
|
2351
|
-
getUserId: () =>
|
|
2352
|
-
(useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous",
|
|
2599
|
+
getUserId: () => cloudClientService.getMentraUserId(),
|
|
2353
2600
|
})
|
|
2601
|
+
private readonly blobRequestQueues = new Map<string, Promise<void>>()
|
|
2602
|
+
|
|
2603
|
+
private readonly simpleStorage = new LocalMiniappStorage({
|
|
2604
|
+
backend: localMiniappStorageBackend,
|
|
2605
|
+
getUserId: () => cloudClientService.resolveMentraUserId(),
|
|
2606
|
+
})
|
|
2607
|
+
|
|
2608
|
+
/**
|
|
2609
|
+
* BlobStore's file helpers are synchronous once a user namespace is selected,
|
|
2610
|
+
* but first-boot Core auth is not. Queue requests per app behind stable
|
|
2611
|
+
* identity resolution so CREATE/WRITE/COMMIT ordering survives that wait.
|
|
2612
|
+
*/
|
|
2613
|
+
private enqueueBlobRequest(
|
|
2614
|
+
packageName: string,
|
|
2615
|
+
requestId: string | undefined,
|
|
2616
|
+
operation: () => Promise<void> | void,
|
|
2617
|
+
): void {
|
|
2618
|
+
const app = this.connectedApps.get(packageName)
|
|
2619
|
+
const previous = this.blobRequestQueues.get(packageName) ?? Promise.resolve()
|
|
2620
|
+
const queued = previous.then(async () => {
|
|
2621
|
+
try {
|
|
2622
|
+
await cloudClientService.resolveMentraUserId()
|
|
2623
|
+
} catch (err) {
|
|
2624
|
+
console.error(`${LOG_TAG}: blob identity error:`, err)
|
|
2625
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2626
|
+
code: MiniappErrorCode.NOT_CONNECTED,
|
|
2627
|
+
message: "Blob storage is unavailable before sign-in completes",
|
|
2628
|
+
})
|
|
2629
|
+
return
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
// unregisterApp() or a crash-respawn may have replaced this app while
|
|
2633
|
+
// identity resolution was pending. Never replay an old context's request
|
|
2634
|
+
// against its replacement.
|
|
2635
|
+
if (!app || this.connectedApps.get(packageName) !== app) return
|
|
2636
|
+
|
|
2637
|
+
// Invoke in request order, but do not keep the queue occupied for the
|
|
2638
|
+
// lifetime of an async picker, download, or share sheet. CREATE/WRITE/
|
|
2639
|
+
// COMMIT are synchronous and therefore still dispatch strictly in order.
|
|
2640
|
+
try {
|
|
2641
|
+
const pending = operation()
|
|
2642
|
+
if (pending) {
|
|
2643
|
+
void pending.catch((err) => this.handleBlobRequestError(packageName, requestId, err))
|
|
2644
|
+
}
|
|
2645
|
+
} catch (err) {
|
|
2646
|
+
this.handleBlobRequestError(packageName, requestId, err)
|
|
2647
|
+
}
|
|
2648
|
+
})
|
|
2649
|
+
this.blobRequestQueues.set(packageName, queued)
|
|
2650
|
+
void queued.finally(() => {
|
|
2651
|
+
if (this.blobRequestQueues.get(packageName) === queued) this.blobRequestQueues.delete(packageName)
|
|
2652
|
+
})
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2655
|
+
private handleBlobRequestError(packageName: string, requestId: string | undefined, err: unknown): void {
|
|
2656
|
+
console.error(`${LOG_TAG}: blob request error:`, err)
|
|
2657
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2658
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2659
|
+
message: err instanceof Error ? err.message : "Blob storage error",
|
|
2660
|
+
})
|
|
2661
|
+
}
|
|
2354
2662
|
|
|
2355
2663
|
/**
|
|
2356
2664
|
* Heading is a sensor stream — start the native compass when any mini
|
|
@@ -2427,12 +2735,6 @@ class LocalMiniappRuntime {
|
|
|
2427
2735
|
// Storage helpers
|
|
2428
2736
|
// ---------------------------------------------------------------------------
|
|
2429
2737
|
|
|
2430
|
-
private getStorageKeyPrefix(packageName: string): string {
|
|
2431
|
-
const userId =
|
|
2432
|
-
(useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous"
|
|
2433
|
-
return `mentraos_localstorage_${userId}_${packageName}_`
|
|
2434
|
-
}
|
|
2435
|
-
|
|
2436
2738
|
private async handleStorageGet(
|
|
2437
2739
|
packageName: string,
|
|
2438
2740
|
payload: Record<string, unknown>,
|
|
@@ -2447,9 +2749,8 @@ class LocalMiniappRuntime {
|
|
|
2447
2749
|
})
|
|
2448
2750
|
return
|
|
2449
2751
|
}
|
|
2450
|
-
const
|
|
2451
|
-
|
|
2452
|
-
this.sendResult(packageName, requestId, true, {key, value: result.is_ok() ? result.value : null})
|
|
2752
|
+
const value = await this.simpleStorage.get(packageName, key)
|
|
2753
|
+
this.sendResult(packageName, requestId, true, {key, value})
|
|
2453
2754
|
} catch (err) {
|
|
2454
2755
|
console.error(`${LOG_TAG}: storage_get error:`, err)
|
|
2455
2756
|
this.sendResult(packageName, requestId, false, undefined, {
|
|
@@ -2473,8 +2774,7 @@ class LocalMiniappRuntime {
|
|
|
2473
2774
|
})
|
|
2474
2775
|
return
|
|
2475
2776
|
}
|
|
2476
|
-
|
|
2477
|
-
mmkvStorage.save(fullKey, payload.value ?? null)
|
|
2777
|
+
await this.simpleStorage.set(packageName, key, payload.value ?? null)
|
|
2478
2778
|
this.sendResult(packageName, requestId, true)
|
|
2479
2779
|
} catch (err) {
|
|
2480
2780
|
console.error(`${LOG_TAG}: storage_set error:`, err)
|
|
@@ -2499,8 +2799,7 @@ class LocalMiniappRuntime {
|
|
|
2499
2799
|
})
|
|
2500
2800
|
return
|
|
2501
2801
|
}
|
|
2502
|
-
|
|
2503
|
-
mmkvStorage.remove(fullKey)
|
|
2802
|
+
await this.simpleStorage.delete(packageName, key)
|
|
2504
2803
|
this.sendResult(packageName, requestId, true)
|
|
2505
2804
|
} catch (err) {
|
|
2506
2805
|
console.error(`${LOG_TAG}: storage_delete error:`, err)
|
|
@@ -2517,9 +2816,7 @@ class LocalMiniappRuntime {
|
|
|
2517
2816
|
requestId?: string,
|
|
2518
2817
|
): Promise<void> {
|
|
2519
2818
|
try {
|
|
2520
|
-
const
|
|
2521
|
-
const allKeys = mmkvStorage.getAllKeys()
|
|
2522
|
-
const keys = allKeys.filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length))
|
|
2819
|
+
const keys = await this.simpleStorage.keys(packageName)
|
|
2523
2820
|
this.sendResult(packageName, requestId, true, {keys})
|
|
2524
2821
|
} catch (err) {
|
|
2525
2822
|
console.error(`${LOG_TAG}: storage_list error:`, err)
|
|
@@ -2532,11 +2829,7 @@ class LocalMiniappRuntime {
|
|
|
2532
2829
|
|
|
2533
2830
|
private async handleStorageClear(packageName: string, requestId?: string): Promise<void> {
|
|
2534
2831
|
try {
|
|
2535
|
-
|
|
2536
|
-
const allKeys = mmkvStorage.getAllKeys()
|
|
2537
|
-
for (const k of allKeys) {
|
|
2538
|
-
if (k.startsWith(prefix)) mmkvStorage.remove(k)
|
|
2539
|
-
}
|
|
2832
|
+
await this.simpleStorage.clear(packageName)
|
|
2540
2833
|
this.sendResult(packageName, requestId, true)
|
|
2541
2834
|
} catch (err) {
|
|
2542
2835
|
console.error(`${LOG_TAG}: storage_clear error:`, err)
|
|
@@ -2561,9 +2854,8 @@ class LocalMiniappRuntime {
|
|
|
2561
2854
|
})
|
|
2562
2855
|
return
|
|
2563
2856
|
}
|
|
2564
|
-
const
|
|
2565
|
-
|
|
2566
|
-
this.sendResult(packageName, requestId, true, {has: result.is_ok()})
|
|
2857
|
+
const has = await this.simpleStorage.has(packageName, key)
|
|
2858
|
+
this.sendResult(packageName, requestId, true, {has})
|
|
2567
2859
|
} catch (err) {
|
|
2568
2860
|
console.error(`${LOG_TAG}: storage_has error:`, err)
|
|
2569
2861
|
this.sendResult(packageName, requestId, false, undefined, {
|
|
@@ -2575,17 +2867,7 @@ class LocalMiniappRuntime {
|
|
|
2575
2867
|
|
|
2576
2868
|
private async handleStorageGetAll(packageName: string, requestId?: string): Promise<void> {
|
|
2577
2869
|
try {
|
|
2578
|
-
const
|
|
2579
|
-
const allKeys = mmkvStorage.getAllKeys()
|
|
2580
|
-
const values: Record<string, string> = {}
|
|
2581
|
-
for (const k of allKeys) {
|
|
2582
|
-
if (!k.startsWith(prefix)) continue
|
|
2583
|
-
const r = mmkvStorage.load<unknown>(k)
|
|
2584
|
-
if (r.is_ok()) {
|
|
2585
|
-
const v = r.value
|
|
2586
|
-
values[k.slice(prefix.length)] = typeof v === "string" ? v : String(v ?? "")
|
|
2587
|
-
}
|
|
2588
|
-
}
|
|
2870
|
+
const values = await this.simpleStorage.getAll(packageName)
|
|
2589
2871
|
this.sendResult(packageName, requestId, true, {values})
|
|
2590
2872
|
} catch (err) {
|
|
2591
2873
|
console.error(`${LOG_TAG}: storage_get_all error:`, err)
|
|
@@ -2610,10 +2892,7 @@ class LocalMiniappRuntime {
|
|
|
2610
2892
|
})
|
|
2611
2893
|
return
|
|
2612
2894
|
}
|
|
2613
|
-
|
|
2614
|
-
for (const [key, value] of Object.entries(values)) {
|
|
2615
|
-
mmkvStorage.save(prefix + key, value ?? null)
|
|
2616
|
-
}
|
|
2895
|
+
await this.simpleStorage.setMultiple(packageName, values)
|
|
2617
2896
|
this.sendResult(packageName, requestId, true)
|
|
2618
2897
|
} catch (err) {
|
|
2619
2898
|
console.error(`${LOG_TAG}: storage_set_multiple error:`, err)
|
|
@@ -2624,6 +2903,36 @@ class LocalMiniappRuntime {
|
|
|
2624
2903
|
}
|
|
2625
2904
|
}
|
|
2626
2905
|
|
|
2906
|
+
// ---------------------------------------------------------------------------
|
|
2907
|
+
// Wi-Fi ADB
|
|
2908
|
+
// ---------------------------------------------------------------------------
|
|
2909
|
+
|
|
2910
|
+
private async handleSetWifiAdbState(
|
|
2911
|
+
packageName: string,
|
|
2912
|
+
payload: Record<string, unknown>,
|
|
2913
|
+
requestId?: string,
|
|
2914
|
+
): Promise<void> {
|
|
2915
|
+
const result = await handleWifiAdbRequest(packageName, payload, {
|
|
2916
|
+
isSystemPackage: (candidate) => this.isSystemPackage(candidate),
|
|
2917
|
+
setWifiAdbState: async (enabled) => {
|
|
2918
|
+
console.log(`${LOG_TAG}: set_wifi_adb_state enabled=${enabled} from ${packageName}`)
|
|
2919
|
+
try {
|
|
2920
|
+
await BluetoothSdk.setWifiAdbState(enabled)
|
|
2921
|
+
} catch (error) {
|
|
2922
|
+
console.error(`${LOG_TAG}: set_wifi_adb_state error:`, error)
|
|
2923
|
+
throw error
|
|
2924
|
+
}
|
|
2925
|
+
},
|
|
2926
|
+
})
|
|
2927
|
+
|
|
2928
|
+
if (result.ok) {
|
|
2929
|
+
this.sendResult(packageName, requestId, true)
|
|
2930
|
+
return
|
|
2931
|
+
}
|
|
2932
|
+
|
|
2933
|
+
this.sendResult(packageName, requestId, false, undefined, result.error)
|
|
2934
|
+
}
|
|
2935
|
+
|
|
2627
2936
|
// ---------------------------------------------------------------------------
|
|
2628
2937
|
// Camera FOV
|
|
2629
2938
|
// ---------------------------------------------------------------------------
|
|
@@ -2689,6 +2998,124 @@ class LocalMiniappRuntime {
|
|
|
2689
2998
|
}
|
|
2690
2999
|
}
|
|
2691
3000
|
|
|
3001
|
+
/**
|
|
3002
|
+
* Lifecycle-scoped glasses-side VAD override from a miniapp
|
|
3003
|
+
* (session.mic.setVoiceActivityDetectionEnabled). Requires MICROPHONE in the
|
|
3004
|
+
* miniapp manifest. Mentra Live only today — other models no-op via DeviceStore.
|
|
3005
|
+
*/
|
|
3006
|
+
private async handleMicSetVadEnabled(
|
|
3007
|
+
packageName: string,
|
|
3008
|
+
payload: Record<string, unknown>,
|
|
3009
|
+
requestId?: string,
|
|
3010
|
+
): Promise<void> {
|
|
3011
|
+
const app = this.connectedApps.get(packageName)
|
|
3012
|
+
const hasMicPermission = app?.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE")
|
|
3013
|
+
if (!hasMicPermission) {
|
|
3014
|
+
logPermissionNotDeclared(packageName, "MICROPHONE", "to set VAD enabled", `{"type": "MICROPHONE"}`)
|
|
3015
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3016
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
3017
|
+
message: `MICROPHONE permission not declared in miniapp.json. Add {"type": "MICROPHONE"} to the "permissions" array.`,
|
|
3018
|
+
permission: "MICROPHONE",
|
|
3019
|
+
operation: MiniappRequestType.MIC_SET_VAD_ENABLED,
|
|
3020
|
+
})
|
|
3021
|
+
return
|
|
3022
|
+
}
|
|
3023
|
+
|
|
3024
|
+
if (typeof payload.enabled !== "boolean") {
|
|
3025
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3026
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
3027
|
+
message: "enabled must be a boolean",
|
|
3028
|
+
})
|
|
3029
|
+
return
|
|
3030
|
+
}
|
|
3031
|
+
|
|
3032
|
+
try {
|
|
3033
|
+
const enabled = payload.enabled
|
|
3034
|
+
console.log(`${LOG_TAG}: mic_set_vad_enabled ${enabled} (by ${packageName})`)
|
|
3035
|
+
await micStateCoordinator.setMiniappGateOverride(packageName, "vad", enabled, this.getConfiguredMicGates())
|
|
3036
|
+
this.sendResult(packageName, requestId, true)
|
|
3037
|
+
} catch (err) {
|
|
3038
|
+
console.error(`${LOG_TAG}: mic_set_vad_enabled error:`, err)
|
|
3039
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3040
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3041
|
+
message: err instanceof Error ? err.message : "VAD set-enabled error",
|
|
3042
|
+
})
|
|
3043
|
+
}
|
|
3044
|
+
}
|
|
3045
|
+
|
|
3046
|
+
/**
|
|
3047
|
+
* Lifecycle-scoped center-mic loudness gate ("Barrier") override from a miniapp
|
|
3048
|
+
* (session.mic.setLoudnessGateEnabled). Requires MICROPHONE in the miniapp
|
|
3049
|
+
* manifest. Mentra Live only today — other models no-op via DeviceStore.
|
|
3050
|
+
*/
|
|
3051
|
+
private async handleMicSetLoudnessGateEnabled(
|
|
3052
|
+
packageName: string,
|
|
3053
|
+
payload: Record<string, unknown>,
|
|
3054
|
+
requestId?: string,
|
|
3055
|
+
): Promise<void> {
|
|
3056
|
+
const app = this.connectedApps.get(packageName)
|
|
3057
|
+
const hasMicPermission = app?.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE")
|
|
3058
|
+
if (!hasMicPermission) {
|
|
3059
|
+
logPermissionNotDeclared(packageName, "MICROPHONE", "to set loudness gate enabled", `{"type": "MICROPHONE"}`)
|
|
3060
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3061
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
3062
|
+
message: `MICROPHONE permission not declared in miniapp.json. Add {"type": "MICROPHONE"} to the "permissions" array.`,
|
|
3063
|
+
permission: "MICROPHONE",
|
|
3064
|
+
operation: MiniappRequestType.MIC_SET_LOUDNESS_GATE_ENABLED,
|
|
3065
|
+
})
|
|
3066
|
+
return
|
|
3067
|
+
}
|
|
3068
|
+
|
|
3069
|
+
if (typeof payload.enabled !== "boolean") {
|
|
3070
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3071
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
3072
|
+
message: "enabled must be a boolean",
|
|
3073
|
+
})
|
|
3074
|
+
return
|
|
3075
|
+
}
|
|
3076
|
+
|
|
3077
|
+
try {
|
|
3078
|
+
const enabled = payload.enabled
|
|
3079
|
+
console.log(`${LOG_TAG}: mic_set_loudness_gate_enabled ${enabled} (by ${packageName})`)
|
|
3080
|
+
await micStateCoordinator.setMiniappGateOverride(
|
|
3081
|
+
packageName,
|
|
3082
|
+
"loudnessGate",
|
|
3083
|
+
enabled,
|
|
3084
|
+
this.getConfiguredMicGates(),
|
|
3085
|
+
)
|
|
3086
|
+
this.sendResult(packageName, requestId, true)
|
|
3087
|
+
} catch (err) {
|
|
3088
|
+
console.error(`${LOG_TAG}: mic_set_loudness_gate_enabled error:`, err)
|
|
3089
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3090
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3091
|
+
message: err instanceof Error ? err.message : "Loudness gate set-enabled error",
|
|
3092
|
+
})
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
|
|
3096
|
+
/**
|
|
3097
|
+
* session.system.scanQr — host camera overlay. Must not clear miniapp
|
|
3098
|
+
* foreground; the host seam is responsible for presenting a Modal on top.
|
|
3099
|
+
*/
|
|
3100
|
+
private async handleScanQr(
|
|
3101
|
+
packageName: string,
|
|
3102
|
+
payload: Record<string, unknown>,
|
|
3103
|
+
requestId?: string,
|
|
3104
|
+
): Promise<void> {
|
|
3105
|
+
try {
|
|
3106
|
+
const result = await invokeScanQrSeam(getUiSeams().scanQr, payload)
|
|
3107
|
+
this.sendResult(packageName, requestId, true, result)
|
|
3108
|
+
} catch (err) {
|
|
3109
|
+
if (!getUiSeams().scanQr) {
|
|
3110
|
+
console.warn(`${LOG_TAG}: scanQr seam missing — QR overlay is not registered`)
|
|
3111
|
+
}
|
|
3112
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3113
|
+
code: (err as {code?: string} | null | undefined)?.code || MiniappErrorCode.INTERNAL,
|
|
3114
|
+
message: err instanceof Error ? err.message : "QR scan failed",
|
|
3115
|
+
})
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
|
|
2692
3119
|
// ---------------------------------------------------------------------------
|
|
2693
3120
|
// System utilities (share, open URL, clipboard, download)
|
|
2694
3121
|
// ---------------------------------------------------------------------------
|
|
@@ -2833,7 +3260,7 @@ class LocalMiniappRuntime {
|
|
|
2833
3260
|
const result = await phonePhotoCoordinator.takePhoto(packageName, {
|
|
2834
3261
|
size: payload.size as "low" | "medium" | "high" | "max" | "small" | "large" | "full" | undefined,
|
|
2835
3262
|
mode: payload.mode as "photo" | "text" | undefined,
|
|
2836
|
-
transferMethod: payload.transferMethod as "auto" | "ble" | undefined,
|
|
3263
|
+
transferMethod: payload.transferMethod as "auto" | "direct" | "ble" | undefined,
|
|
2837
3264
|
compress: payload.compress as "none" | "low" | "medium" | "high" | undefined,
|
|
2838
3265
|
sound: payload.sound as boolean | undefined,
|
|
2839
3266
|
saveToGallery: payload.saveToGallery as boolean | undefined,
|
|
@@ -2843,8 +3270,8 @@ class LocalMiniappRuntime {
|
|
|
2843
3270
|
isoCap: payload.isoCap as number | undefined,
|
|
2844
3271
|
noiseReduction: payload.noiseReduction as boolean | undefined,
|
|
2845
3272
|
edgeEnhancement: payload.edgeEnhancement as boolean | undefined,
|
|
2846
|
-
mfnr: payload.mfnr as boolean | undefined,
|
|
2847
3273
|
zsl: payload.zsl as boolean | undefined,
|
|
3274
|
+
mfnr: payload.mfnr as boolean | undefined,
|
|
2848
3275
|
ispDigitalGain: payload.ispDigitalGain as number | undefined,
|
|
2849
3276
|
ispAnalogGain: payload.ispAnalogGain as string | undefined,
|
|
2850
3277
|
})
|
|
@@ -2884,6 +3311,8 @@ class LocalMiniappRuntime {
|
|
|
2884
3311
|
mode: payload.mode as "photo" | "text" | undefined,
|
|
2885
3312
|
exposureTimeNs: payload.exposureTimeNs as number | undefined,
|
|
2886
3313
|
durationMs: payload.durationMs as number | undefined,
|
|
3314
|
+
zsl: payload.zsl as boolean | undefined,
|
|
3315
|
+
mfnr: payload.mfnr as boolean | undefined,
|
|
2887
3316
|
})
|
|
2888
3317
|
this.sendResult(packageName, requestId, true)
|
|
2889
3318
|
} catch (err) {
|
|
@@ -2976,6 +3405,7 @@ class LocalMiniappRuntime {
|
|
|
2976
3405
|
video: normalizeStreamVideoConfig(payload.video),
|
|
2977
3406
|
audio: normalizeStreamAudioConfig(payload.audio),
|
|
2978
3407
|
sound: payload.sound as boolean | undefined,
|
|
3408
|
+
authToken: typeof payload.authToken === "string" ? payload.authToken : undefined,
|
|
2979
3409
|
})
|
|
2980
3410
|
this.sendResult(packageName, requestId, true, result)
|
|
2981
3411
|
} catch (err) {
|
|
@@ -3071,7 +3501,7 @@ class LocalMiniappRuntime {
|
|
|
3071
3501
|
return
|
|
3072
3502
|
}
|
|
3073
3503
|
try {
|
|
3074
|
-
await requestWifiSetup(payload.reason as string | undefined)
|
|
3504
|
+
await requestWifiSetup(payload.reason as string | undefined, packageName)
|
|
3075
3505
|
this.sendResult(packageName, requestId, true)
|
|
3076
3506
|
} catch (err) {
|
|
3077
3507
|
this.sendResult(packageName, requestId, false, undefined, {
|
|
@@ -3135,29 +3565,12 @@ class LocalMiniappRuntime {
|
|
|
3135
3565
|
}
|
|
3136
3566
|
}
|
|
3137
3567
|
|
|
3138
|
-
// Remove old subscriptions for this app
|
|
3139
|
-
for (const oldStream of app.subscriptions) {
|
|
3140
|
-
const subs = this.streamSubscribers.get(oldStream)
|
|
3141
|
-
if (subs) {
|
|
3142
|
-
subs.delete(packageName)
|
|
3143
|
-
if (subs.size === 0) {
|
|
3144
|
-
this.streamSubscribers.delete(oldStream)
|
|
3145
|
-
}
|
|
3146
|
-
}
|
|
3147
|
-
}
|
|
3148
|
-
|
|
3149
3568
|
// Add new subscriptions
|
|
3569
|
+
const previousSubscriptions = app.subscriptions
|
|
3150
3570
|
app.subscriptions = new Set(streams)
|
|
3151
3571
|
app.forceLocalTranscriptionStreams.clear()
|
|
3152
3572
|
app.cloudTranscriptionStreams = new Set(streams.filter((stream) => stream.startsWith("transcription:")))
|
|
3153
|
-
|
|
3154
|
-
let subs = this.streamSubscribers.get(stream)
|
|
3155
|
-
if (!subs) {
|
|
3156
|
-
subs = new Set()
|
|
3157
|
-
this.streamSubscribers.set(stream, subs)
|
|
3158
|
-
}
|
|
3159
|
-
subs.add(packageName)
|
|
3160
|
-
}
|
|
3573
|
+
this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
|
|
3161
3574
|
|
|
3162
3575
|
this.recomputeMicRequirements()
|
|
3163
3576
|
this.updateCloudSubscriptions()
|
|
@@ -3168,6 +3581,16 @@ class LocalMiniappRuntime {
|
|
|
3168
3581
|
// Mic requirements
|
|
3169
3582
|
// ===========================================================================
|
|
3170
3583
|
|
|
3584
|
+
private getConfiguredMicGates(): {vadEnabled: boolean | null; loudnessGateEnabled: boolean | null} {
|
|
3585
|
+
const bluetoothSettings = useSettingsStore.getState().getBluetoothSettings()
|
|
3586
|
+
const configuredVad = bluetoothSettings.voice_activity_detection_enabled
|
|
3587
|
+
const configuredLoudnessGate = bluetoothSettings.loudness_gate_enabled
|
|
3588
|
+
return {
|
|
3589
|
+
vadEnabled: typeof configuredVad === "boolean" ? configuredVad : null,
|
|
3590
|
+
loudnessGateEnabled: typeof configuredLoudnessGate === "boolean" ? configuredLoudnessGate : null,
|
|
3591
|
+
}
|
|
3592
|
+
}
|
|
3593
|
+
|
|
3171
3594
|
private recomputeMicRequirements(): void {
|
|
3172
3595
|
let anyPcm = false
|
|
3173
3596
|
let anyLc3 = false
|
|
@@ -3176,7 +3599,11 @@ class LocalMiniappRuntime {
|
|
|
3176
3599
|
if (stream === "audio_chunk") anyPcm = true
|
|
3177
3600
|
if (stream.startsWith("transcription:") || stream.startsWith("translation:") || stream === "vad") anyLc3 = true
|
|
3178
3601
|
}
|
|
3179
|
-
micStateCoordinator.setLocalRequirements({
|
|
3602
|
+
micStateCoordinator.setLocalRequirements({
|
|
3603
|
+
pcm: anyPcm,
|
|
3604
|
+
lc3: anyLc3,
|
|
3605
|
+
...this.getConfiguredMicGates(),
|
|
3606
|
+
})
|
|
3180
3607
|
}
|
|
3181
3608
|
|
|
3182
3609
|
/**
|
|
@@ -3254,12 +3681,21 @@ class LocalMiniappRuntime {
|
|
|
3254
3681
|
*/
|
|
3255
3682
|
private buildCloudAudioSubscriptions(cloudStreams: Set<string>): AudioSubscription[] {
|
|
3256
3683
|
const subs: AudioSubscription[] = []
|
|
3684
|
+
// Union of every connected app's TRANSCRIPTION_CONFIG hints. Hints only
|
|
3685
|
+
// apply to the auto-mode TRANSCRIPTION subscription (the wire schema has
|
|
3686
|
+
// no hints field on specific-language sources — the language IS the hint
|
|
3687
|
+
// there), and are deliberately NOT attached to translation sources.
|
|
3688
|
+
const hintUnion = [...new Set([...this.transcriptionHintsByApp.values()].flat())].sort()
|
|
3257
3689
|
const langSource = (code: string): LanguageSource =>
|
|
3258
3690
|
code === "auto" || code === "*" ? {mode: "auto"} : {mode: "specific", code}
|
|
3259
3691
|
for (const stream of cloudStreams) {
|
|
3260
3692
|
if (stream.startsWith("transcription:")) {
|
|
3261
3693
|
const lang = stream.substring("transcription:".length)
|
|
3262
|
-
|
|
3694
|
+
const source = langSource(lang)
|
|
3695
|
+
subs.push({
|
|
3696
|
+
kind: "transcription",
|
|
3697
|
+
language: source.mode === "auto" && hintUnion.length > 0 ? {mode: "auto", hints: hintUnion} : source,
|
|
3698
|
+
})
|
|
3263
3699
|
} else if (stream.startsWith("translation:")) {
|
|
3264
3700
|
const parts = stream.split(":")
|
|
3265
3701
|
// Only `translation:<source>:<target>` maps cleanly; a concrete target
|
|
@@ -4130,17 +4566,16 @@ class LocalMiniappRuntime {
|
|
|
4130
4566
|
}
|
|
4131
4567
|
|
|
4132
4568
|
private doPingRound(): void {
|
|
4133
|
-
const now = Date.now()
|
|
4134
|
-
const staleThreshold = PING_INTERVAL_MS * PING_TIMEOUT_THRESHOLD
|
|
4135
|
-
|
|
4136
4569
|
const toRemove: string[] = []
|
|
4137
4570
|
|
|
4138
4571
|
for (const [packageName, app] of this.connectedApps) {
|
|
4139
|
-
|
|
4572
|
+
const liveness = advanceMiniappPingLiveness(app.unansweredPingRounds, PING_TIMEOUT_THRESHOLD)
|
|
4573
|
+
if (liveness.shouldUnregister) {
|
|
4140
4574
|
console.warn(`${LOG_TAG}: ${packageName} missed ${PING_TIMEOUT_THRESHOLD} pings, unregistering`)
|
|
4141
4575
|
toRemove.push(packageName)
|
|
4142
4576
|
continue
|
|
4143
4577
|
}
|
|
4578
|
+
app.unansweredPingRounds = liveness.unansweredPingRounds
|
|
4144
4579
|
|
|
4145
4580
|
// Send PING — SDK auto-replies with PONG
|
|
4146
4581
|
this.sendToMiniapp(packageName, {
|
|
@@ -4164,6 +4599,7 @@ class LocalMiniappRuntime {
|
|
|
4164
4599
|
const app = this.connectedApps.get(packageName)
|
|
4165
4600
|
if (app) {
|
|
4166
4601
|
app.lastPongAt = Date.now()
|
|
4602
|
+
app.unansweredPingRounds = 0
|
|
4167
4603
|
this.clearForegroundProbe(packageName)
|
|
4168
4604
|
}
|
|
4169
4605
|
}
|
|
@@ -4191,7 +4627,17 @@ class LocalMiniappRuntime {
|
|
|
4191
4627
|
|
|
4192
4628
|
// Belt-and-suspenders: clear any remaining state
|
|
4193
4629
|
this.pendingCloudRequests.clear()
|
|
4630
|
+
const hadButtonPressSubscribers = this.getButtonPressSubscribers().length > 0
|
|
4194
4631
|
this.streamSubscribers.clear()
|
|
4632
|
+
if (hadButtonPressSubscribers) {
|
|
4633
|
+
for (const listener of this.buttonPressSubscriberListeners) {
|
|
4634
|
+
try {
|
|
4635
|
+
listener([])
|
|
4636
|
+
} catch (error) {
|
|
4637
|
+
console.warn(`${LOG_TAG}: button press subscriber listener threw during cleanup:`, error)
|
|
4638
|
+
}
|
|
4639
|
+
}
|
|
4640
|
+
}
|
|
4195
4641
|
this.connectedApps.clear()
|
|
4196
4642
|
for (const timerId of this.foregroundProbeTimers.values()) {
|
|
4197
4643
|
BgTimer.clearTimeout(timerId)
|