@mentra/engine 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/build/devtools.d.ts +11 -0
- package/build/devtools.d.ts.map +1 -0
- package/build/devtools.js +13 -0
- package/build/devtools.js.map +1 -0
- package/build/engine.d.ts +261 -0
- package/build/engine.d.ts.map +1 -0
- package/build/engine.js +170 -0
- package/build/engine.js.map +1 -0
- package/build/facades/dev.d.ts +58 -0
- package/build/facades/dev.d.ts.map +1 -0
- package/build/facades/dev.js +189 -0
- package/build/facades/dev.js.map +1 -0
- package/build/facades/display.d.ts +21 -0
- package/build/facades/display.d.ts.map +1 -0
- package/build/facades/display.js +17 -0
- package/build/facades/display.js.map +1 -0
- package/build/facades/displayMirror.d.ts +15 -0
- package/build/facades/displayMirror.d.ts.map +1 -0
- package/build/facades/displayMirror.js +29 -0
- package/build/facades/displayMirror.js.map +1 -0
- package/build/facades/gallery.d.ts +62 -0
- package/build/facades/gallery.d.ts.map +1 -0
- package/build/facades/gallery.js +61 -0
- package/build/facades/gallery.js.map +1 -0
- package/build/facades/glasses.d.ts +141 -0
- package/build/facades/glasses.d.ts.map +1 -0
- package/build/facades/glasses.js +199 -0
- package/build/facades/glasses.js.map +1 -0
- package/build/facades/glassesSettings.d.ts +13 -0
- package/build/facades/glassesSettings.d.ts.map +1 -0
- package/build/facades/glassesSettings.js +54 -0
- package/build/facades/glassesSettings.js.map +1 -0
- package/build/facades/glassesWifi.d.ts +25 -0
- package/build/facades/glassesWifi.d.ts.map +1 -0
- package/build/facades/glassesWifi.js +57 -0
- package/build/facades/glassesWifi.js.map +1 -0
- package/build/facades/miniapps.d.ts +43 -0
- package/build/facades/miniapps.d.ts.map +1 -0
- package/build/facades/miniapps.js +49 -0
- package/build/facades/miniapps.js.map +1 -0
- package/build/facades/notifications.d.ts +13 -0
- package/build/facades/notifications.d.ts.map +1 -0
- package/build/facades/notifications.js +12 -0
- package/build/facades/notifications.js.map +1 -0
- package/build/facades/ota.d.ts +103 -0
- package/build/facades/ota.d.ts.map +1 -0
- package/build/facades/ota.js +156 -0
- package/build/facades/ota.js.map +1 -0
- package/build/facades/pairing.d.ts +114 -0
- package/build/facades/pairing.d.ts.map +1 -0
- package/build/facades/pairing.js +275 -0
- package/build/facades/pairing.js.map +1 -0
- package/build/facades/permissions.d.ts +33 -0
- package/build/facades/permissions.d.ts.map +1 -0
- package/build/facades/permissions.js +155 -0
- package/build/facades/permissions.js.map +1 -0
- package/build/facades/phoneNotifications.d.ts +21 -0
- package/build/facades/phoneNotifications.d.ts.map +1 -0
- package/build/facades/phoneNotifications.js +51 -0
- package/build/facades/phoneNotifications.js.map +1 -0
- package/build/facades/reports.d.ts +36 -0
- package/build/facades/reports.d.ts.map +1 -0
- package/build/facades/reports.js +134 -0
- package/build/facades/reports.js.map +1 -0
- package/build/facades/session.d.ts +21 -0
- package/build/facades/session.d.ts.map +1 -0
- package/build/facades/session.js +52 -0
- package/build/facades/session.js.map +1 -0
- package/build/facades/settings.d.ts +37 -0
- package/build/facades/settings.d.ts.map +1 -0
- package/build/facades/settings.js +55 -0
- package/build/facades/settings.js.map +1 -0
- package/build/facades/speech.d.ts +30 -0
- package/build/facades/speech.d.ts.map +1 -0
- package/build/facades/speech.js +37 -0
- package/build/facades/speech.js.map +1 -0
- package/build/index.d.ts +43 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +45 -0
- package/build/index.js.map +1 -0
- package/build/internal.d.ts +67 -0
- package/build/internal.d.ts.map +1 -0
- package/build/internal.js +90 -0
- package/build/internal.js.map +1 -0
- package/build/runtime/bootstrap.d.ts +83 -0
- package/build/runtime/bootstrap.d.ts.map +1 -0
- package/build/runtime/bootstrap.js +56 -0
- package/build/runtime/bootstrap.js.map +1 -0
- package/build/runtime/config.d.ts +146 -0
- package/build/runtime/config.d.ts.map +1 -0
- package/build/runtime/config.js +18 -0
- package/build/runtime/config.js.map +1 -0
- package/build/runtime/streamConfig.d.ts +4 -0
- package/build/runtime/streamConfig.d.ts.map +1 -0
- package/build/runtime/streamConfig.js +38 -0
- package/build/runtime/streamConfig.js.map +1 -0
- package/build/services/AppRegistry.d.ts +269 -0
- package/build/services/AppRegistry.d.ts.map +1 -0
- package/build/services/AppRegistry.js +1079 -0
- package/build/services/AppRegistry.js.map +1 -0
- package/build/services/AudioCloudUplink.d.ts +9 -0
- package/build/services/AudioCloudUplink.d.ts.map +1 -0
- package/build/services/AudioCloudUplink.js +48 -0
- package/build/services/AudioCloudUplink.js.map +1 -0
- package/build/services/AudioPlaybackService.d.ts +175 -0
- package/build/services/AudioPlaybackService.d.ts.map +1 -0
- package/build/services/AudioPlaybackService.js +705 -0
- package/build/services/AudioPlaybackService.js.map +1 -0
- package/build/services/AutomaticReportResult.d.ts +20 -0
- package/build/services/AutomaticReportResult.d.ts.map +1 -0
- package/build/services/AutomaticReportResult.js +29 -0
- package/build/services/AutomaticReportResult.js.map +1 -0
- package/build/services/BlobStore.d.ts +91 -0
- package/build/services/BlobStore.d.ts.map +1 -0
- package/build/services/BlobStore.js +668 -0
- package/build/services/BlobStore.js.map +1 -0
- package/build/services/CaptionsTesterReportService.d.ts +4 -0
- package/build/services/CaptionsTesterReportService.d.ts.map +1 -0
- package/build/services/CaptionsTesterReportService.js +80 -0
- package/build/services/CaptionsTesterReportService.js.map +1 -0
- package/build/services/CloudAudioSubscriptionSync.d.ts +24 -0
- package/build/services/CloudAudioSubscriptionSync.d.ts.map +1 -0
- package/build/services/CloudAudioSubscriptionSync.js +39 -0
- package/build/services/CloudAudioSubscriptionSync.js.map +1 -0
- package/build/services/CloudClientService.d.ts +143 -0
- package/build/services/CloudClientService.d.ts.map +1 -0
- package/build/services/CloudClientService.js +644 -0
- package/build/services/CloudClientService.js.map +1 -0
- package/build/services/CloudTranscriptE2EMetrics.d.ts +4 -0
- package/build/services/CloudTranscriptE2EMetrics.d.ts.map +1 -0
- package/build/services/CloudTranscriptE2EMetrics.js +24 -0
- package/build/services/CloudTranscriptE2EMetrics.js.map +1 -0
- package/build/services/ConnectionCoordinator.d.ts +51 -0
- package/build/services/ConnectionCoordinator.d.ts.map +1 -0
- package/build/services/ConnectionCoordinator.js +34 -0
- package/build/services/ConnectionCoordinator.js.map +1 -0
- package/build/services/DevServerBridge.d.ts +63 -0
- package/build/services/DevServerBridge.d.ts.map +1 -0
- package/build/services/DevServerBridge.js +294 -0
- package/build/services/DevServerBridge.js.map +1 -0
- package/build/services/DeviceEventRouter.d.ts +3 -0
- package/build/services/DeviceEventRouter.d.ts.map +1 -0
- package/build/services/DeviceEventRouter.js +190 -0
- package/build/services/DeviceEventRouter.js.map +1 -0
- package/build/services/DeviceStoreHydration.d.ts +38 -0
- package/build/services/DeviceStoreHydration.d.ts.map +1 -0
- package/build/services/DeviceStoreHydration.js +124 -0
- package/build/services/DeviceStoreHydration.js.map +1 -0
- package/build/services/DisplayProcessor.d.ts +223 -0
- package/build/services/DisplayProcessor.d.ts.map +1 -0
- package/build/services/DisplayProcessor.js +552 -0
- package/build/services/DisplayProcessor.js.map +1 -0
- package/build/services/GlassesReadiness.d.ts +41 -0
- package/build/services/GlassesReadiness.d.ts.map +1 -0
- package/build/services/GlassesReadiness.js +82 -0
- package/build/services/GlassesReadiness.js.map +1 -0
- package/build/services/GlassesSettingsSync.d.ts +33 -0
- package/build/services/GlassesSettingsSync.d.ts.map +1 -0
- package/build/services/GlassesSettingsSync.js +121 -0
- package/build/services/GlassesSettingsSync.js.map +1 -0
- package/build/services/GlassesStatusProjection.d.ts +3 -0
- package/build/services/GlassesStatusProjection.d.ts.map +1 -0
- package/build/services/GlassesStatusProjection.js +65 -0
- package/build/services/GlassesStatusProjection.js.map +1 -0
- package/build/services/HeadingService.d.ts +27 -0
- package/build/services/HeadingService.d.ts.map +1 -0
- package/build/services/HeadingService.js +96 -0
- package/build/services/HeadingService.js.map +1 -0
- package/build/services/LocalDisplayManager.d.ts +139 -0
- package/build/services/LocalDisplayManager.d.ts.map +1 -0
- package/build/services/LocalDisplayManager.js +579 -0
- package/build/services/LocalDisplayManager.js.map +1 -0
- package/build/services/LocalMiniappRuntime.d.ts +439 -0
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -0
- package/build/services/LocalMiniappRuntime.js +3695 -0
- package/build/services/LocalMiniappRuntime.js.map +1 -0
- package/build/services/LocalSttFallbackCoordinator.d.ts +57 -0
- package/build/services/LocalSttFallbackCoordinator.d.ts.map +1 -0
- package/build/services/LocalSttFallbackCoordinator.js +137 -0
- package/build/services/LocalSttFallbackCoordinator.js.map +1 -0
- package/build/services/MentraJSCrashController.d.ts +97 -0
- package/build/services/MentraJSCrashController.d.ts.map +1 -0
- package/build/services/MentraJSCrashController.js +138 -0
- package/build/services/MentraJSCrashController.js.map +1 -0
- package/build/services/MentraJSCrashloopReportService.d.ts +9 -0
- package/build/services/MentraJSCrashloopReportService.d.ts.map +1 -0
- package/build/services/MentraJSCrashloopReportService.js +54 -0
- package/build/services/MentraJSCrashloopReportService.js.map +1 -0
- package/build/services/MentraJSLogPipeline.d.ts +87 -0
- package/build/services/MentraJSLogPipeline.d.ts.map +1 -0
- package/build/services/MentraJSLogPipeline.js +137 -0
- package/build/services/MentraJSLogPipeline.js.map +1 -0
- package/build/services/MentraJSRouter.d.ts +203 -0
- package/build/services/MentraJSRouter.d.ts.map +1 -0
- package/build/services/MentraJSRouter.js +480 -0
- package/build/services/MentraJSRouter.js.map +1 -0
- package/build/services/MentraUIRouter.d.ts +137 -0
- package/build/services/MentraUIRouter.d.ts.map +1 -0
- package/build/services/MentraUIRouter.js +210 -0
- package/build/services/MentraUIRouter.js.map +1 -0
- package/build/services/MicStateCoordinator.d.ts +37 -0
- package/build/services/MicStateCoordinator.d.ts.map +1 -0
- package/build/services/MicStateCoordinator.js +81 -0
- package/build/services/MicStateCoordinator.js.map +1 -0
- package/build/services/MiniappEngine.d.ts +31 -0
- package/build/services/MiniappEngine.d.ts.map +1 -0
- package/build/services/MiniappEngine.js +119 -0
- package/build/services/MiniappEngine.js.map +1 -0
- package/build/services/MiniappLauncher.d.ts +129 -0
- package/build/services/MiniappLauncher.d.ts.map +1 -0
- package/build/services/MiniappLauncher.js +291 -0
- package/build/services/MiniappLauncher.js.map +1 -0
- package/build/services/MiniappRunningRegistry.d.ts +24 -0
- package/build/services/MiniappRunningRegistry.d.ts.map +1 -0
- package/build/services/MiniappRunningRegistry.js +52 -0
- package/build/services/MiniappRunningRegistry.js.map +1 -0
- package/build/services/NativeHttpResponse.d.ts +3 -0
- package/build/services/NativeHttpResponse.d.ts.map +1 -0
- package/build/services/NativeHttpResponse.js +6 -0
- package/build/services/NativeHttpResponse.js.map +1 -0
- package/build/services/NavigationHandlers.d.ts +90 -0
- package/build/services/NavigationHandlers.d.ts.map +1 -0
- package/build/services/NavigationHandlers.js +379 -0
- package/build/services/NavigationHandlers.js.map +1 -0
- package/build/services/NavigationService.d.ts +254 -0
- package/build/services/NavigationService.d.ts.map +1 -0
- package/build/services/NavigationService.js +553 -0
- package/build/services/NavigationService.js.map +1 -0
- package/build/services/NotificationsEmitter.d.ts +28 -0
- package/build/services/NotificationsEmitter.d.ts.map +1 -0
- package/build/services/NotificationsEmitter.js +22 -0
- package/build/services/NotificationsEmitter.js.map +1 -0
- package/build/services/OfflineSpeechModelService.d.ts +33 -0
- package/build/services/OfflineSpeechModelService.d.ts.map +1 -0
- package/build/services/OfflineSpeechModelService.js +136 -0
- package/build/services/OfflineSpeechModelService.js.map +1 -0
- package/build/services/OtaInstallCoordinator.d.ts +195 -0
- package/build/services/OtaInstallCoordinator.d.ts.map +1 -0
- package/build/services/OtaInstallCoordinator.js +1047 -0
- package/build/services/OtaInstallCoordinator.js.map +1 -0
- package/build/services/OtaService.d.ts +3 -0
- package/build/services/OtaService.d.ts.map +1 -0
- package/build/services/OtaService.js +65 -0
- package/build/services/OtaService.js.map +1 -0
- package/build/services/OtaUpdateCheckService.d.ts +65 -0
- package/build/services/OtaUpdateCheckService.d.ts.map +1 -0
- package/build/services/OtaUpdateCheckService.js +260 -0
- package/build/services/OtaUpdateCheckService.js.map +1 -0
- package/build/services/PairingIdentity.d.ts +80 -0
- package/build/services/PairingIdentity.d.ts.map +1 -0
- package/build/services/PairingIdentity.js +145 -0
- package/build/services/PairingIdentity.js.map +1 -0
- package/build/services/PhoneCalendarHelpers.d.ts +32 -0
- package/build/services/PhoneCalendarHelpers.d.ts.map +1 -0
- package/build/services/PhoneCalendarHelpers.js +157 -0
- package/build/services/PhoneCalendarHelpers.js.map +1 -0
- package/build/services/PhoneCalendarService.d.ts +5 -0
- package/build/services/PhoneCalendarService.d.ts.map +1 -0
- package/build/services/PhoneCalendarService.js +24 -0
- package/build/services/PhoneCalendarService.js.map +1 -0
- package/build/services/PhoneCameraFovCoordinator.d.ts +28 -0
- package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -0
- package/build/services/PhoneCameraFovCoordinator.js +180 -0
- package/build/services/PhoneCameraFovCoordinator.js.map +1 -0
- package/build/services/PhoneLocationService.d.ts +36 -0
- package/build/services/PhoneLocationService.d.ts.map +1 -0
- package/build/services/PhoneLocationService.js +115 -0
- package/build/services/PhoneLocationService.js.map +1 -0
- package/build/services/PhoneNotificationsSync.d.ts +3 -0
- package/build/services/PhoneNotificationsSync.d.ts.map +1 -0
- package/build/services/PhoneNotificationsSync.js +35 -0
- package/build/services/PhoneNotificationsSync.js.map +1 -0
- package/build/services/PhonePhotoCoordinator.d.ts +103 -0
- package/build/services/PhonePhotoCoordinator.d.ts.map +1 -0
- package/build/services/PhonePhotoCoordinator.js +326 -0
- package/build/services/PhonePhotoCoordinator.js.map +1 -0
- package/build/services/PhoneStreamCoordinator.d.ts +168 -0
- package/build/services/PhoneStreamCoordinator.d.ts.map +1 -0
- package/build/services/PhoneStreamCoordinator.js +632 -0
- package/build/services/PhoneStreamCoordinator.js.map +1 -0
- package/build/services/PhoneVideoCoordinator.d.ts +50 -0
- package/build/services/PhoneVideoCoordinator.d.ts.map +1 -0
- package/build/services/PhoneVideoCoordinator.js +147 -0
- package/build/services/PhoneVideoCoordinator.js.map +1 -0
- package/build/services/STTModelManager.d.ts +65 -0
- package/build/services/STTModelManager.d.ts.map +1 -0
- package/build/services/STTModelManager.js +292 -0
- package/build/services/STTModelManager.js.map +1 -0
- package/build/services/SceneRenderer.d.ts +90 -0
- package/build/services/SceneRenderer.d.ts.map +1 -0
- package/build/services/SceneRenderer.js +248 -0
- package/build/services/SceneRenderer.js.map +1 -0
- package/build/services/SentenceTtsPipeline.d.ts +31 -0
- package/build/services/SentenceTtsPipeline.d.ts.map +1 -0
- package/build/services/SentenceTtsPipeline.js +67 -0
- package/build/services/SentenceTtsPipeline.js.map +1 -0
- package/build/services/StreamLifecycleController.d.ts +67 -0
- package/build/services/StreamLifecycleController.d.ts.map +1 -0
- package/build/services/StreamLifecycleController.js +152 -0
- package/build/services/StreamLifecycleController.js.map +1 -0
- package/build/services/TTSModelManager.d.ts +90 -0
- package/build/services/TTSModelManager.d.ts.map +1 -0
- package/build/services/TTSModelManager.js +366 -0
- package/build/services/TTSModelManager.js.map +1 -0
- package/build/services/TranscriptionRouting.d.ts +12 -0
- package/build/services/TranscriptionRouting.d.ts.map +1 -0
- package/build/services/TranscriptionRouting.js +24 -0
- package/build/services/TranscriptionRouting.js.map +1 -0
- package/build/services/WebviewBridge.d.ts +22 -0
- package/build/services/WebviewBridge.d.ts.map +1 -0
- package/build/services/WebviewBridge.js +47 -0
- package/build/services/WebviewBridge.js.map +1 -0
- package/build/services/asg/GalleryMediaIntegrityReportService.d.ts +17 -0
- package/build/services/asg/GalleryMediaIntegrityReportService.d.ts.map +1 -0
- package/build/services/asg/GalleryMediaIntegrityReportService.js +43 -0
- package/build/services/asg/GalleryMediaIntegrityReportService.js.map +1 -0
- package/build/services/asg/asgCameraApi.d.ts +205 -0
- package/build/services/asg/asgCameraApi.d.ts.map +1 -0
- package/build/services/asg/asgCameraApi.js +1410 -0
- package/build/services/asg/asgCameraApi.js.map +1 -0
- package/build/services/asg/cameraRollExportCoordinator.d.ts +60 -0
- package/build/services/asg/cameraRollExportCoordinator.d.ts.map +1 -0
- package/build/services/asg/cameraRollExportCoordinator.js +618 -0
- package/build/services/asg/cameraRollExportCoordinator.js.map +1 -0
- package/build/services/asg/cameraRollExportLedger.d.ts +44 -0
- package/build/services/asg/cameraRollExportLedger.d.ts.map +1 -0
- package/build/services/asg/cameraRollExportLedger.js +78 -0
- package/build/services/asg/cameraRollExportLedger.js.map +1 -0
- package/build/services/asg/galleryMediaValidation.d.ts +29 -0
- package/build/services/asg/galleryMediaValidation.d.ts.map +1 -0
- package/build/services/asg/galleryMediaValidation.js +149 -0
- package/build/services/asg/galleryMediaValidation.js.map +1 -0
- package/build/services/asg/galleryNotices.d.ts +33 -0
- package/build/services/asg/galleryNotices.d.ts.map +1 -0
- package/build/services/asg/galleryNotices.js +29 -0
- package/build/services/asg/galleryNotices.js.map +1 -0
- package/build/services/asg/galleryPathMigration.d.ts +3 -0
- package/build/services/asg/galleryPathMigration.d.ts.map +1 -0
- package/build/services/asg/galleryPathMigration.js +26 -0
- package/build/services/asg/galleryPathMigration.js.map +1 -0
- package/build/services/asg/gallerySettingsService.d.ts +20 -0
- package/build/services/asg/gallerySettingsService.d.ts.map +1 -0
- package/build/services/asg/gallerySettingsService.js +53 -0
- package/build/services/asg/gallerySettingsService.js.map +1 -0
- package/build/services/asg/gallerySyncNotifications.d.ts +56 -0
- package/build/services/asg/gallerySyncNotifications.d.ts.map +1 -0
- package/build/services/asg/gallerySyncNotifications.js +198 -0
- package/build/services/asg/gallerySyncNotifications.js.map +1 -0
- package/build/services/asg/gallerySyncService.d.ts +137 -0
- package/build/services/asg/gallerySyncService.d.ts.map +1 -0
- package/build/services/asg/gallerySyncService.js +1933 -0
- package/build/services/asg/gallerySyncService.js.map +1 -0
- package/build/services/asg/galleryTransferLedger.d.ts +58 -0
- package/build/services/asg/galleryTransferLedger.d.ts.map +1 -0
- package/build/services/asg/galleryTransferLedger.js +225 -0
- package/build/services/asg/galleryTransferLedger.js.map +1 -0
- package/build/services/asg/localNetworkTransport.d.ts +15 -0
- package/build/services/asg/localNetworkTransport.d.ts.map +1 -0
- package/build/services/asg/localNetworkTransport.js +144 -0
- package/build/services/asg/localNetworkTransport.js.map +1 -0
- package/build/services/asg/localStorageService.d.ts +146 -0
- package/build/services/asg/localStorageService.d.ts.map +1 -0
- package/build/services/asg/localStorageService.js +572 -0
- package/build/services/asg/localStorageService.js.map +1 -0
- package/build/services/asg/mediaProcessingQueue.d.ts +70 -0
- package/build/services/asg/mediaProcessingQueue.d.ts.map +1 -0
- package/build/services/asg/mediaProcessingQueue.js +451 -0
- package/build/services/asg/mediaProcessingQueue.js.map +1 -0
- package/build/services/asgOtaVersionUrl.d.ts +2 -0
- package/build/services/asgOtaVersionUrl.d.ts.map +1 -0
- package/build/services/asgOtaVersionUrl.js +36 -0
- package/build/services/asgOtaVersionUrl.js.map +1 -0
- package/build/services/blobPaths.d.ts +31 -0
- package/build/services/blobPaths.d.ts.map +1 -0
- package/build/services/blobPaths.js +86 -0
- package/build/services/blobPaths.js.map +1 -0
- package/build/services/cloudStreamApi.d.ts +59 -0
- package/build/services/cloudStreamApi.d.ts.map +1 -0
- package/build/services/cloudStreamApi.js +81 -0
- package/build/services/cloudStreamApi.js.map +1 -0
- package/build/services/gallerySyncClock.d.ts +17 -0
- package/build/services/gallerySyncClock.d.ts.map +1 -0
- package/build/services/gallerySyncClock.js +21 -0
- package/build/services/gallerySyncClock.js.map +1 -0
- package/build/services/glassesClockSync.d.ts +20 -0
- package/build/services/glassesClockSync.d.ts.map +1 -0
- package/build/services/glassesClockSync.js +97 -0
- package/build/services/glassesClockSync.js.map +1 -0
- package/build/services/manifestVersionGate.d.ts +28 -0
- package/build/services/manifestVersionGate.d.ts.map +1 -0
- package/build/services/manifestVersionGate.js +41 -0
- package/build/services/manifestVersionGate.js.map +1 -0
- package/build/services/mentraUiShim.d.ts +38 -0
- package/build/services/mentraUiShim.d.ts.map +1 -0
- package/build/services/mentraUiShim.js +360 -0
- package/build/services/mentraUiShim.js.map +1 -0
- package/build/services/navigation/roadNameResolver.d.ts +67 -0
- package/build/services/navigation/roadNameResolver.d.ts.map +1 -0
- package/build/services/navigation/roadNameResolver.js +96 -0
- package/build/services/navigation/roadNameResolver.js.map +1 -0
- package/build/services/navigation/routesApiCodec.d.ts +38 -0
- package/build/services/navigation/routesApiCodec.d.ts.map +1 -0
- package/build/services/navigation/routesApiCodec.js +68 -0
- package/build/services/navigation/routesApiCodec.js.map +1 -0
- package/build/services/otaDisplayState.d.ts +42 -0
- package/build/services/otaDisplayState.d.ts.map +1 -0
- package/build/services/otaDisplayState.js +92 -0
- package/build/services/otaDisplayState.js.map +1 -0
- package/build/services/otaInstallPolicy.d.ts +81 -0
- package/build/services/otaInstallPolicy.d.ts.map +1 -0
- package/build/services/otaInstallPolicy.js +96 -0
- package/build/services/otaInstallPolicy.js.map +1 -0
- package/build/services/otaLegacyMapping.d.ts +22 -0
- package/build/services/otaLegacyMapping.d.ts.map +1 -0
- package/build/services/otaLegacyMapping.js +56 -0
- package/build/services/otaLegacyMapping.js.map +1 -0
- package/build/services/slimStreamStatus.d.ts +7 -0
- package/build/services/slimStreamStatus.d.ts.map +1 -0
- package/build/services/slimStreamStatus.js +26 -0
- package/build/services/slimStreamStatus.js.map +1 -0
- package/build/stores/apps.d.ts +104 -0
- package/build/stores/apps.d.ts.map +1 -0
- package/build/stores/apps.js +492 -0
- package/build/stores/apps.js.map +1 -0
- package/build/stores/bluetoothSettingKeys.d.ts +4 -0
- package/build/stores/bluetoothSettingKeys.d.ts.map +1 -0
- package/build/stores/bluetoothSettingKeys.js +31 -0
- package/build/stores/bluetoothSettingKeys.js.map +1 -0
- package/build/stores/cloudClientStatus.d.ts +9 -0
- package/build/stores/cloudClientStatus.d.ts.map +1 -0
- package/build/stores/cloudClientStatus.js +24 -0
- package/build/stores/cloudClientStatus.js.map +1 -0
- package/build/stores/connection.d.ts +22 -0
- package/build/stores/connection.d.ts.map +1 -0
- package/build/stores/connection.js +43 -0
- package/build/stores/connection.js.map +1 -0
- package/build/stores/core.d.ts +15 -0
- package/build/stores/core.d.ts.map +1 -0
- package/build/stores/core.js +21 -0
- package/build/stores/core.js.map +1 -0
- package/build/stores/display.d.ts +24 -0
- package/build/stores/display.d.ts.map +1 -0
- package/build/stores/display.js +96 -0
- package/build/stores/display.js.map +1 -0
- package/build/stores/gallerySync.d.ts +101 -0
- package/build/stores/gallerySync.d.ts.map +1 -0
- package/build/stores/gallerySync.js +217 -0
- package/build/stores/gallerySync.js.map +1 -0
- package/build/stores/glasses.d.ts +62 -0
- package/build/stores/glasses.d.ts.map +1 -0
- package/build/stores/glasses.js +203 -0
- package/build/stores/glasses.js.map +1 -0
- package/build/stores/settings.d.ts +37 -0
- package/build/stores/settings.d.ts.map +1 -0
- package/build/stores/settings.js +922 -0
- package/build/stores/settings.js.map +1 -0
- package/build/types/applet.d.ts +96 -0
- package/build/types/applet.d.ts.map +1 -0
- package/build/types/applet.js +5 -0
- package/build/types/applet.js.map +1 -0
- package/build/types/asg.d.ts +53 -0
- package/build/types/asg.d.ts.map +1 -0
- package/build/types/asg.js +2 -0
- package/build/types/asg.js.map +1 -0
- package/build/types/capabilities/even-realities-g1.d.ts +12 -0
- package/build/types/capabilities/even-realities-g1.d.ts.map +1 -0
- package/build/types/capabilities/even-realities-g1.js +55 -0
- package/build/types/capabilities/even-realities-g1.js.map +1 -0
- package/build/types/capabilities/even-realities-g2.d.ts +12 -0
- package/build/types/capabilities/even-realities-g2.d.ts.map +1 -0
- package/build/types/capabilities/even-realities-g2.js +75 -0
- package/build/types/capabilities/even-realities-g2.js.map +1 -0
- package/build/types/capabilities/mentra-display.d.ts +12 -0
- package/build/types/capabilities/mentra-display.d.ts.map +1 -0
- package/build/types/capabilities/mentra-display.js +63 -0
- package/build/types/capabilities/mentra-display.js.map +1 -0
- package/build/types/capabilities/mentra-live.d.ts +12 -0
- package/build/types/capabilities/mentra-live.d.ts.map +1 -0
- package/build/types/capabilities/mentra-live.js +92 -0
- package/build/types/capabilities/mentra-live.js.map +1 -0
- package/build/types/capabilities/none.d.ts +13 -0
- package/build/types/capabilities/none.d.ts.map +1 -0
- package/build/types/capabilities/none.js +65 -0
- package/build/types/capabilities/none.js.map +1 -0
- package/build/types/capabilities/simulated-glasses.d.ts +13 -0
- package/build/types/capabilities/simulated-glasses.d.ts.map +1 -0
- package/build/types/capabilities/simulated-glasses.js +65 -0
- package/build/types/capabilities/simulated-glasses.js.map +1 -0
- package/build/types/capabilities/vuzix-z100.d.ts +12 -0
- package/build/types/capabilities/vuzix-z100.d.ts.map +1 -0
- package/build/types/capabilities/vuzix-z100.js +52 -0
- package/build/types/capabilities/vuzix-z100.js.map +1 -0
- package/build/types/enums.d.ts +40 -0
- package/build/types/enums.d.ts.map +1 -0
- package/build/types/enums.js +44 -0
- package/build/types/enums.js.map +1 -0
- package/build/types/hardware.d.ts +160 -0
- package/build/types/hardware.d.ts.map +1 -0
- package/build/types/hardware.js +36 -0
- package/build/types/hardware.js.map +1 -0
- package/build/types/index.d.ts +11 -0
- package/build/types/index.d.ts.map +1 -0
- package/build/types/index.js +10 -0
- package/build/types/index.js.map +1 -0
- package/build/utils/GlobalEventEmitter.d.ts +10 -0
- package/build/utils/GlobalEventEmitter.d.ts.map +1 -0
- package/build/utils/GlobalEventEmitter.js +13 -0
- package/build/utils/GlobalEventEmitter.js.map +1 -0
- package/build/utils/cloudClient/RnUdpAdapter.d.ts +3 -0
- package/build/utils/cloudClient/RnUdpAdapter.d.ts.map +1 -0
- package/build/utils/cloudClient/RnUdpAdapter.js +41 -0
- package/build/utils/cloudClient/RnUdpAdapter.js.map +1 -0
- package/build/utils/cloudClient/cloudSecureStore.d.ts +3 -0
- package/build/utils/cloudClient/cloudSecureStore.d.ts.map +1 -0
- package/build/utils/cloudClient/cloudSecureStore.js +26 -0
- package/build/utils/cloudClient/cloudSecureStore.js.map +1 -0
- package/build/utils/debouncedPatch.d.ts +6 -0
- package/build/utils/debouncedPatch.d.ts.map +1 -0
- package/build/utils/debouncedPatch.js +25 -0
- package/build/utils/debouncedPatch.js.map +1 -0
- package/build/utils/devLogging.d.ts +26 -0
- package/build/utils/devLogging.d.ts.map +1 -0
- package/build/utils/devLogging.js +116 -0
- package/build/utils/devLogging.js.map +1 -0
- package/build/utils/devMiniappLaunch.d.ts +44 -0
- package/build/utils/devMiniappLaunch.d.ts.map +1 -0
- package/build/utils/devMiniappLaunch.js +52 -0
- package/build/utils/devMiniappLaunch.js.map +1 -0
- package/build/utils/diagnosticContext.d.ts +3 -0
- package/build/utils/diagnosticContext.d.ts.map +1 -0
- package/build/utils/diagnosticContext.js +119 -0
- package/build/utils/diagnosticContext.js.map +1 -0
- package/build/utils/display/composer/ColumnComposer.d.ts +131 -0
- package/build/utils/display/composer/ColumnComposer.d.ts.map +1 -0
- package/build/utils/display/composer/ColumnComposer.js +204 -0
- package/build/utils/display/composer/ColumnComposer.js.map +1 -0
- package/build/utils/display/composer/index.d.ts +9 -0
- package/build/utils/display/composer/index.d.ts.map +1 -0
- package/build/utils/display/composer/index.js +9 -0
- package/build/utils/display/composer/index.js.map +1 -0
- package/build/utils/display/helpers/DisplayHelpers.d.ts +165 -0
- package/build/utils/display/helpers/DisplayHelpers.d.ts.map +1 -0
- package/build/utils/display/helpers/DisplayHelpers.js +312 -0
- package/build/utils/display/helpers/DisplayHelpers.js.map +1 -0
- package/build/utils/display/helpers/ScrollView.d.ts +183 -0
- package/build/utils/display/helpers/ScrollView.d.ts.map +1 -0
- package/build/utils/display/helpers/ScrollView.js +257 -0
- package/build/utils/display/helpers/ScrollView.js.map +1 -0
- package/build/utils/display/helpers/index.d.ts +11 -0
- package/build/utils/display/helpers/index.d.ts.map +1 -0
- package/build/utils/display/helpers/index.js +10 -0
- package/build/utils/display/helpers/index.js.map +1 -0
- package/build/utils/display/index.d.ts +174 -0
- package/build/utils/display/index.d.ts.map +1 -0
- package/build/utils/display/index.js +185 -0
- package/build/utils/display/index.js.map +1 -0
- package/build/utils/display/measurer/TextMeasurer.d.ts +160 -0
- package/build/utils/display/measurer/TextMeasurer.d.ts.map +1 -0
- package/build/utils/display/measurer/TextMeasurer.js +273 -0
- package/build/utils/display/measurer/TextMeasurer.js.map +1 -0
- package/build/utils/display/measurer/index.d.ts +10 -0
- package/build/utils/display/measurer/index.d.ts.map +1 -0
- package/build/utils/display/measurer/index.js +11 -0
- package/build/utils/display/measurer/index.js.map +1 -0
- package/build/utils/display/measurer/script-detection.d.ts +53 -0
- package/build/utils/display/measurer/script-detection.d.ts.map +1 -0
- package/build/utils/display/measurer/script-detection.js +184 -0
- package/build/utils/display/measurer/script-detection.js.map +1 -0
- package/build/utils/display/profiles/g1.d.ts +33 -0
- package/build/utils/display/profiles/g1.d.ts.map +1 -0
- package/build/utils/display/profiles/g1.js +213 -0
- package/build/utils/display/profiles/g1.js.map +1 -0
- package/build/utils/display/profiles/g2.d.ts +13 -0
- package/build/utils/display/profiles/g2.d.ts.map +1 -0
- package/build/utils/display/profiles/g2.js +34 -0
- package/build/utils/display/profiles/g2.js.map +1 -0
- package/build/utils/display/profiles/index.d.ts +12 -0
- package/build/utils/display/profiles/index.d.ts.map +1 -0
- package/build/utils/display/profiles/index.js +15 -0
- package/build/utils/display/profiles/index.js.map +1 -0
- package/build/utils/display/profiles/nex.d.ts +10 -0
- package/build/utils/display/profiles/nex.d.ts.map +1 -0
- package/build/utils/display/profiles/nex.js +150 -0
- package/build/utils/display/profiles/nex.js.map +1 -0
- package/build/utils/display/profiles/types.d.ts +103 -0
- package/build/utils/display/profiles/types.d.ts.map +1 -0
- package/build/utils/display/profiles/types.js +2 -0
- package/build/utils/display/profiles/types.js.map +1 -0
- package/build/utils/display/profiles/z100.d.ts +33 -0
- package/build/utils/display/profiles/z100.d.ts.map +1 -0
- package/build/utils/display/profiles/z100.js +182 -0
- package/build/utils/display/profiles/z100.js.map +1 -0
- package/build/utils/display/scene/degrade.d.ts +25 -0
- package/build/utils/display/scene/degrade.d.ts.map +1 -0
- package/build/utils/display/scene/degrade.js +64 -0
- package/build/utils/display/scene/degrade.js.map +1 -0
- package/build/utils/display/scene/differ.d.ts +38 -0
- package/build/utils/display/scene/differ.d.ts.map +1 -0
- package/build/utils/display/scene/differ.js +107 -0
- package/build/utils/display/scene/differ.js.map +1 -0
- package/build/utils/display/scene/index.d.ts +10 -0
- package/build/utils/display/scene/index.d.ts.map +1 -0
- package/build/utils/display/scene/index.js +6 -0
- package/build/utils/display/scene/index.js.map +1 -0
- package/build/utils/display/scene/process.d.ts +35 -0
- package/build/utils/display/scene/process.d.ts.map +1 -0
- package/build/utils/display/scene/process.js +187 -0
- package/build/utils/display/scene/process.js.map +1 -0
- package/build/utils/display/scene/store.d.ts +48 -0
- package/build/utils/display/scene/store.d.ts.map +1 -0
- package/build/utils/display/scene/store.js +0 -0
- package/build/utils/display/scene/store.js.map +1 -0
- package/build/utils/display/scene/types.d.ts +120 -0
- package/build/utils/display/scene/types.d.ts.map +1 -0
- package/build/utils/display/scene/types.js +0 -0
- package/build/utils/display/scene/types.js.map +1 -0
- package/build/utils/display/wrapper/TextWrapper.d.ts +102 -0
- package/build/utils/display/wrapper/TextWrapper.d.ts.map +1 -0
- package/build/utils/display/wrapper/TextWrapper.js +471 -0
- package/build/utils/display/wrapper/TextWrapper.js.map +1 -0
- package/build/utils/display/wrapper/index.d.ts +12 -0
- package/build/utils/display/wrapper/index.d.ts.map +1 -0
- package/build/utils/display/wrapper/index.js +12 -0
- package/build/utils/display/wrapper/index.js.map +1 -0
- package/build/utils/display/wrapper/types.d.ts +75 -0
- package/build/utils/display/wrapper/types.d.ts.map +1 -0
- package/build/utils/display/wrapper/types.js +11 -0
- package/build/utils/display/wrapper/types.js.map +1 -0
- package/build/utils/e2eMetrics.d.ts +6 -0
- package/build/utils/e2eMetrics.d.ts.map +1 -0
- package/build/utils/e2eMetrics.js +27 -0
- package/build/utils/e2eMetrics.js.map +1 -0
- package/build/utils/hardware/hardware.d.ts +42 -0
- package/build/utils/hardware/hardware.d.ts.map +1 -0
- package/build/utils/hardware/hardware.js +127 -0
- package/build/utils/hardware/hardware.js.map +1 -0
- package/build/utils/hardware/index.d.ts +2 -0
- package/build/utils/hardware/index.d.ts.map +1 -0
- package/build/utils/hardware/index.js +2 -0
- package/build/utils/hardware/index.js.map +1 -0
- package/build/utils/miniappGlobals.d.ts +60 -0
- package/build/utils/miniappGlobals.d.ts.map +1 -0
- package/build/utils/miniappGlobals.js +224 -0
- package/build/utils/miniappGlobals.js.map +1 -0
- package/build/utils/permissions/MediaLibraryPermissions.d.ts +45 -0
- package/build/utils/permissions/MediaLibraryPermissions.d.ts.map +1 -0
- package/build/utils/permissions/MediaLibraryPermissions.js +145 -0
- package/build/utils/permissions/MediaLibraryPermissions.js.map +1 -0
- package/build/utils/permissions/galleryDisplayName.d.ts +8 -0
- package/build/utils/permissions/galleryDisplayName.d.ts.map +1 -0
- package/build/utils/permissions/galleryDisplayName.js +25 -0
- package/build/utils/permissions/galleryDisplayName.js.map +1 -0
- package/build/utils/storage/index.d.ts +3 -0
- package/build/utils/storage/index.d.ts.map +1 -0
- package/build/utils/storage/index.js +3 -0
- package/build/utils/storage/index.js.map +1 -0
- package/build/utils/storage/storage.d.ts +24 -0
- package/build/utils/storage/storage.d.ts.map +1 -0
- package/build/utils/storage/storage.js +85 -0
- package/build/utils/storage/storage.js.map +1 -0
- package/build/utils/storage/zip.d.ts +3 -0
- package/build/utils/storage/zip.d.ts.map +1 -0
- package/build/utils/storage/zip.js +14 -0
- package/build/utils/storage/zip.js.map +1 -0
- package/build/utils/timers/index.d.ts +3 -0
- package/build/utils/timers/index.d.ts.map +1 -0
- package/build/utils/timers/index.js +3 -0
- package/build/utils/timers/index.js.map +1 -0
- package/build/utils/timers/timers.d.ts +9 -0
- package/build/utils/timers/timers.d.ts.map +1 -0
- package/build/utils/timers/timers.js +105 -0
- package/build/utils/timers/timers.js.map +1 -0
- package/package.json +120 -0
- package/src/devtools.ts +13 -0
- package/src/engine.ts +173 -0
- package/src/facades/dev.ts +198 -0
- package/src/facades/display.ts +17 -0
- package/src/facades/displayMirror.ts +33 -0
- package/src/facades/gallery.ts +69 -0
- package/src/facades/glasses.ts +206 -0
- package/src/facades/glassesSettings.ts +56 -0
- package/src/facades/glassesWifi.ts +68 -0
- package/src/facades/miniapps.ts +55 -0
- package/src/facades/notifications.ts +15 -0
- package/src/facades/ota.ts +174 -0
- package/src/facades/pairing.ts +329 -0
- package/src/facades/permissions.ts +165 -0
- package/src/facades/phoneNotifications.ts +50 -0
- package/src/facades/reports.ts +185 -0
- package/src/facades/session.ts +55 -0
- package/src/facades/settings.ts +57 -0
- package/src/facades/speech.ts +38 -0
- package/src/index.ts +191 -0
- package/src/internal.ts +159 -0
- package/src/runtime/bootstrap.ts +118 -0
- package/src/runtime/config.ts +147 -0
- package/src/runtime/streamConfig.ts +34 -0
- package/src/services/AppRegistry.ts +1190 -0
- package/src/services/AudioCloudUplink.ts +50 -0
- package/src/services/AudioPlaybackService.ts +813 -0
- package/src/services/AutomaticReportResult.ts +48 -0
- package/src/services/BlobStore.ts +746 -0
- package/src/services/CaptionsTesterReportService.ts +107 -0
- package/src/services/CloudAudioSubscriptionSync.ts +41 -0
- package/src/services/CloudClientService.ts +689 -0
- package/src/services/CloudTranscriptE2EMetrics.ts +27 -0
- package/src/services/ConnectionCoordinator.ts +65 -0
- package/src/services/DevServerBridge.ts +332 -0
- package/src/services/DeviceEventRouter.ts +238 -0
- package/src/services/DeviceStoreHydration.ts +130 -0
- package/src/services/DisplayProcessor.ts +775 -0
- package/src/services/GlassesReadiness.ts +103 -0
- package/src/services/GlassesSettingsSync.ts +134 -0
- package/src/services/GlassesStatusProjection.ts +73 -0
- package/src/services/HeadingService.ts +98 -0
- package/src/services/LocalDisplayManager.ts +703 -0
- package/src/services/LocalMiniappRuntime.ts +4213 -0
- package/src/services/LocalSttFallbackCoordinator.ts +147 -0
- package/src/services/MentraJSCrashController.ts +189 -0
- package/src/services/MentraJSCrashloopReportService.ts +66 -0
- package/src/services/MentraJSLogPipeline.ts +166 -0
- package/src/services/MentraJSRouter.ts +557 -0
- package/src/services/MentraUIRouter.ts +244 -0
- package/src/services/MicStateCoordinator.ts +94 -0
- package/src/services/MiniappEngine.ts +143 -0
- package/src/services/MiniappLauncher.ts +340 -0
- package/src/services/MiniappRunningRegistry.ts +57 -0
- package/src/services/NativeHttpResponse.ts +6 -0
- package/src/services/NavigationHandlers.ts +423 -0
- package/src/services/NavigationService.ts +731 -0
- package/src/services/NotificationsEmitter.ts +48 -0
- package/src/services/OfflineSpeechModelService.ts +168 -0
- package/src/services/OtaInstallCoordinator.ts +1205 -0
- package/src/services/OtaService.ts +83 -0
- package/src/services/OtaUpdateCheckService.ts +386 -0
- package/src/services/PairingIdentity.ts +154 -0
- package/src/services/PhoneCalendarHelpers.ts +212 -0
- package/src/services/PhoneCalendarService.ts +42 -0
- package/src/services/PhoneCameraFovCoordinator.ts +195 -0
- package/src/services/PhoneLocationService.ts +122 -0
- package/src/services/PhoneNotificationsSync.ts +44 -0
- package/src/services/PhonePhotoCoordinator.ts +409 -0
- package/src/services/PhoneStreamCoordinator.ts +777 -0
- package/src/services/PhoneVideoCoordinator.ts +189 -0
- package/src/services/STTModelManager.ts +360 -0
- package/src/services/SceneRenderer.ts +292 -0
- package/src/services/SentenceTtsPipeline.ts +106 -0
- package/src/services/StreamLifecycleController.ts +213 -0
- package/src/services/TTSModelManager.ts +466 -0
- package/src/services/TranscriptionRouting.ts +37 -0
- package/src/services/WebviewBridge.ts +50 -0
- package/src/services/asg/GalleryMediaIntegrityReportService.ts +70 -0
- package/src/services/asg/asgCameraApi.ts +1695 -0
- package/src/services/asg/cameraRollExportCoordinator.ts +627 -0
- package/src/services/asg/cameraRollExportLedger.ts +122 -0
- package/src/services/asg/galleryMediaValidation.ts +183 -0
- package/src/services/asg/galleryNotices.ts +61 -0
- package/src/services/asg/galleryPathMigration.ts +22 -0
- package/src/services/asg/gallerySettingsService.ts +68 -0
- package/src/services/asg/gallerySyncNotifications.ts +236 -0
- package/src/services/asg/gallerySyncService.ts +2269 -0
- package/src/services/asg/galleryTransferLedger.ts +312 -0
- package/src/services/asg/localNetworkTransport.ts +173 -0
- package/src/services/asg/localStorageService.ts +652 -0
- package/src/services/asg/mediaProcessingQueue.ts +518 -0
- package/src/services/asgOtaVersionUrl.ts +41 -0
- package/src/services/blobPaths.ts +86 -0
- package/src/services/cloudStreamApi.ts +154 -0
- package/src/services/gallerySyncClock.ts +39 -0
- package/src/services/glassesClockSync.ts +115 -0
- package/src/services/manifestVersionGate.ts +43 -0
- package/src/services/mentraUiShim.ts +366 -0
- package/src/services/navigation/roadNameResolver.ts +112 -0
- package/src/services/navigation/routesApiCodec.ts +69 -0
- package/src/services/otaDisplayState.ts +115 -0
- package/src/services/otaInstallPolicy.ts +106 -0
- package/src/services/otaLegacyMapping.ts +71 -0
- package/src/services/slimStreamStatus.ts +28 -0
- package/src/stores/apps.ts +614 -0
- package/src/stores/bluetoothSettingKeys.ts +36 -0
- package/src/stores/cloudClientStatus.ts +37 -0
- package/src/stores/connection.ts +69 -0
- package/src/stores/core.ts +32 -0
- package/src/stores/display.ts +121 -0
- package/src/stores/gallerySync.ts +380 -0
- package/src/stores/glasses.ts +309 -0
- package/src/stores/settings.ts +1001 -0
- package/src/types/applet.ts +112 -0
- package/src/types/asg.ts +58 -0
- package/src/types/capabilities/even-realities-g1.ts +67 -0
- package/src/types/capabilities/even-realities-g2.ts +88 -0
- package/src/types/capabilities/mentra-display.ts +75 -0
- package/src/types/capabilities/mentra-live.ts +103 -0
- package/src/types/capabilities/none.ts +76 -0
- package/src/types/capabilities/simulated-glasses.ts +76 -0
- package/src/types/capabilities/vuzix-z100.ts +64 -0
- package/src/types/enums.ts +43 -0
- package/src/types/hardware.ts +203 -0
- package/src/types/index.ts +37 -0
- package/src/utils/GlobalEventEmitter.ts +14 -0
- package/src/utils/cloudClient/RnUdpAdapter.ts +47 -0
- package/src/utils/cloudClient/cloudSecureStore.ts +28 -0
- package/src/utils/debouncedPatch.ts +28 -0
- package/src/utils/devLogging.ts +140 -0
- package/src/utils/devMiniappLaunch.ts +68 -0
- package/src/utils/diagnosticContext.ts +155 -0
- package/src/utils/display/composer/ColumnComposer.ts +285 -0
- package/src/utils/display/composer/index.ts +15 -0
- package/src/utils/display/helpers/DisplayHelpers.ts +413 -0
- package/src/utils/display/helpers/ScrollView.ts +337 -0
- package/src/utils/display/helpers/index.ts +14 -0
- package/src/utils/display/index.ts +273 -0
- package/src/utils/display/measurer/TextMeasurer.ts +346 -0
- package/src/utils/display/measurer/index.ts +21 -0
- package/src/utils/display/measurer/script-detection.ts +218 -0
- package/src/utils/display/profiles/g1.ts +236 -0
- package/src/utils/display/profiles/g2.ts +39 -0
- package/src/utils/display/profiles/index.ts +28 -0
- package/src/utils/display/profiles/nex.ts +166 -0
- package/src/utils/display/profiles/types.ts +132 -0
- package/src/utils/display/profiles/z100.ts +198 -0
- package/src/utils/display/scene/degrade.ts +80 -0
- package/src/utils/display/scene/differ.ts +128 -0
- package/src/utils/display/scene/index.ts +21 -0
- package/src/utils/display/scene/process.ts +214 -0
- package/src/utils/display/scene/store.ts +0 -0
- package/src/utils/display/scene/types.ts +0 -0
- package/src/utils/display/wrapper/TextWrapper.ts +546 -0
- package/src/utils/display/wrapper/index.ts +15 -0
- package/src/utils/display/wrapper/types.ts +102 -0
- package/src/utils/e2eMetrics.ts +30 -0
- package/src/utils/hardware/hardware.ts +166 -0
- package/src/utils/hardware/index.ts +1 -0
- package/src/utils/miniappGlobals.ts +260 -0
- package/src/utils/permissions/MediaLibraryPermissions.ts +161 -0
- package/src/utils/permissions/galleryDisplayName.ts +28 -0
- package/src/utils/storage/index.ts +2 -0
- package/src/utils/storage/storage.ts +101 -0
- package/src/utils/storage/zip.ts +13 -0
- package/src/utils/timers/index.ts +2 -0
- package/src/utils/timers/timers.ts +139 -0
|
@@ -0,0 +1,3695 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LocalMiniappRuntime
|
|
3
|
+
*
|
|
4
|
+
* Singleton service that bridges local miniapps (running in WebViews) with
|
|
5
|
+
* phone capabilities: display, audio, storage, sensors, LED, etc.
|
|
6
|
+
*
|
|
7
|
+
* Each miniapp communicates via the @mentra/miniapp envelope protocol.
|
|
8
|
+
* This runtime handles request dispatch, stream fan-out, and lifecycle
|
|
9
|
+
* management (connect/disconnect/ping).
|
|
10
|
+
*/
|
|
11
|
+
import { Linking } from "react-native";
|
|
12
|
+
import Share from "react-native-share";
|
|
13
|
+
import * as Battery from "expo-battery";
|
|
14
|
+
import * as Clipboard from "expo-clipboard";
|
|
15
|
+
import { File, Paths } from "expo-file-system";
|
|
16
|
+
import * as Location from "expo-location";
|
|
17
|
+
import BluetoothSdk from "@mentra/bluetooth-sdk";
|
|
18
|
+
import { MiniappErrorCode, MiniappRequestType, MiniappResponseType, MiniappStreamType, CLOUD_STATUS_STREAM, parseEnvelope, serializeEnvelope, } from "@mentra/miniapp";
|
|
19
|
+
import { DeviceTypes, getModelCapabilities } from "../types";
|
|
20
|
+
import { storage as mmkvStorage } from "../utils/storage/storage";
|
|
21
|
+
import { BgTimer } from "../utils/timers";
|
|
22
|
+
import devServerBridge from "./DevServerBridge";
|
|
23
|
+
import { islandNotifications } from "./NotificationsEmitter";
|
|
24
|
+
import { isGlassesConnected } from "./GlassesReadiness";
|
|
25
|
+
import audioPlaybackService from "./AudioPlaybackService";
|
|
26
|
+
import { phoneLocationService } from "./PhoneLocationService";
|
|
27
|
+
import { useGlassesStore } from "../stores/glasses";
|
|
28
|
+
import { useSettingsStore } from "../stores/settings";
|
|
29
|
+
import localDisplayManager from "./LocalDisplayManager";
|
|
30
|
+
import headingService from "./HeadingService";
|
|
31
|
+
import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator";
|
|
32
|
+
import micStateCoordinator from "./MicStateCoordinator";
|
|
33
|
+
import { BlobStore } from "./BlobStore";
|
|
34
|
+
import { CloudAudioSubscriptionSync } from "./CloudAudioSubscriptionSync";
|
|
35
|
+
import { phoneCameraFovCoordinator } from "./PhoneCameraFovCoordinator";
|
|
36
|
+
import { phonePhotoCoordinator } from "./PhonePhotoCoordinator";
|
|
37
|
+
import { phoneStreamCoordinator } from "./PhoneStreamCoordinator";
|
|
38
|
+
import { phoneVideoCoordinator } from "./PhoneVideoCoordinator";
|
|
39
|
+
import { runSentenceTtsPipeline } from "./SentenceTtsPipeline";
|
|
40
|
+
import { summarizeTranscriptionRoutes, transcriptionDeliveryRoute } from "./TranscriptionRouting";
|
|
41
|
+
import { cloudClientService } from "./CloudClientService";
|
|
42
|
+
import { miniappLauncher } from "./MiniappLauncher";
|
|
43
|
+
import { ISLAND_SETTINGS_KEYS, } from "../runtime/config";
|
|
44
|
+
import { getAnalytics, getUiSeams } from "../runtime/bootstrap";
|
|
45
|
+
import { normalizeStreamAudioConfig, normalizeStreamVideoConfig } from "../runtime/streamConfig";
|
|
46
|
+
import ttsModelManager from "./TTSModelManager";
|
|
47
|
+
import { NavigationHandlers } from "./NavigationHandlers";
|
|
48
|
+
import { useAppStatusStore } from "../stores/apps";
|
|
49
|
+
import { getDevAppAttestation, getDevAppSourcePackage } from "./AppRegistry";
|
|
50
|
+
import { listPhoneCalendarEvents, PhoneCalendarError } from "./PhoneCalendarService";
|
|
51
|
+
/**
|
|
52
|
+
* Strictness ordering for `location_stream` rate values. Higher index =
|
|
53
|
+
* higher accuracy and higher power cost. The aggregate tier applied to
|
|
54
|
+
* the OS is the max of every connected miniapp's requested rate;
|
|
55
|
+
* miniapps that didn't ask for `location_stream` contribute nothing.
|
|
56
|
+
*
|
|
57
|
+
* If a miniapp passes a value we don't recognize, we treat it like the
|
|
58
|
+
* weakest known rate ("passive") rather than blanket-overwriting an
|
|
59
|
+
* active stronger request from another app.
|
|
60
|
+
*/
|
|
61
|
+
const LOCATION_RATE_PRIORITY = ["off", "passive", "low", "high", "realtime"];
|
|
62
|
+
function locationRateRank(rate) {
|
|
63
|
+
if (!rate)
|
|
64
|
+
return -1;
|
|
65
|
+
const i = LOCATION_RATE_PRIORITY.indexOf(rate);
|
|
66
|
+
return i >= 0 ? i : LOCATION_RATE_PRIORITY.indexOf("passive");
|
|
67
|
+
}
|
|
68
|
+
function withTimeout(promise, timeoutMs) {
|
|
69
|
+
return new Promise((resolve) => {
|
|
70
|
+
let settled = false;
|
|
71
|
+
const done = (value) => {
|
|
72
|
+
if (settled)
|
|
73
|
+
return;
|
|
74
|
+
settled = true;
|
|
75
|
+
BgTimer.clearTimeout(timer);
|
|
76
|
+
resolve(value);
|
|
77
|
+
};
|
|
78
|
+
const timer = BgTimer.setTimeout(() => done(null), timeoutMs);
|
|
79
|
+
promise.then((value) => done(value), () => done(null));
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
const LOG_TAG = "LOCAL_MINIAPP";
|
|
83
|
+
const SYSTEM_MINIAPP_PACKAGES = new Set([
|
|
84
|
+
"com.mentra.camera",
|
|
85
|
+
"com.mentra.gallery",
|
|
86
|
+
"com.mentra.settings",
|
|
87
|
+
"com.mentra.simulated",
|
|
88
|
+
"com.mentra.mirror",
|
|
89
|
+
"com.mentra.ai",
|
|
90
|
+
"cloud.augmentos.notify",
|
|
91
|
+
"com.mentra.feedback",
|
|
92
|
+
"com.mentra.miniappdev",
|
|
93
|
+
]);
|
|
94
|
+
const PING_INTERVAL_MS = 5_000;
|
|
95
|
+
const MINIAPP_AUTH_REFRESH_HEADROOM_MS = 5 * 60 * 1000;
|
|
96
|
+
const MINIAPP_AUTH_REFRESH_MIN_DELAY_MS = 5_000;
|
|
97
|
+
// Backoff bounds for re-minting the INITIAL miniapp token after a failed mint
|
|
98
|
+
// (e.g. the cloud client had not finished its first-boot Core token exchange
|
|
99
|
+
// when the miniapp connected). Exponential from base, capped, so a transient
|
|
100
|
+
// not-yet-connected client self-heals within seconds instead of needing a full
|
|
101
|
+
// manual Cloud V2 reconnect.
|
|
102
|
+
const MINIAPP_AUTH_RETRY_BASE_MS = 1_000;
|
|
103
|
+
const MINIAPP_AUTH_RETRY_MAX_MS = 15_000;
|
|
104
|
+
const FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS = 2_500;
|
|
105
|
+
const REQUEST_WIFI_SETUP_TYPE = "miniapp_request_wifi_setup";
|
|
106
|
+
// Unregister after this many missed pongs. Generous on purpose: a busy
|
|
107
|
+
// context (heavy interim translation traffic) or OS scheduling while idle can
|
|
108
|
+
// delay pongs well past one interval, and killing a healthy-but-busy script
|
|
109
|
+
// drops its subscriptions (releasing the mic). With the liveness-timeout
|
|
110
|
+
// respawn path wired (MentraJSRouter.start), a genuinely dead context still
|
|
111
|
+
// comes back automatically — this threshold only bounds how long that takes.
|
|
112
|
+
const PING_TIMEOUT_THRESHOLD = 6; // ~30s
|
|
113
|
+
const TRANSCRIPT_TIMING_TELEMETRY = globalThis.__DEV__ === true;
|
|
114
|
+
const RGB_LED_ACTIONS = new Set(["on", "off"]);
|
|
115
|
+
const RGB_LED_COLORS = new Set(["red", "green", "blue", "orange", "white"]);
|
|
116
|
+
const CAMERA_FOV_MIN = 62;
|
|
117
|
+
const CAMERA_FOV_MAX = 118;
|
|
118
|
+
const CAMERA_FOV_DEFAULT = 102;
|
|
119
|
+
const CAMERA_FOV_PRESETS = { narrow: 82, standard: CAMERA_FOV_DEFAULT, wide: 118 };
|
|
120
|
+
const CAMERA_ROI_POSITION_BY_NAME = { center: "center", bottom: "bottom", top: "top" };
|
|
121
|
+
// =============================================================================
|
|
122
|
+
// Declared-permission record helper (for CONNECT_ACK / PERMISSIONS_UPDATE)
|
|
123
|
+
// =============================================================================
|
|
124
|
+
/**
|
|
125
|
+
* Map from manifest permission types (uppercase, snake-y) to the lowercase
|
|
126
|
+
* canonical permission keys the SDK exposes via session.permissions. Mirrors
|
|
127
|
+
* cloud SDK v3's PermissionType union.
|
|
128
|
+
*
|
|
129
|
+
* BACKGROUND_LOCATION + POST_NOTIFICATIONS aren't in v3's surface; they map
|
|
130
|
+
* onto the same canonical keys (location / notifications) since the SDK's
|
|
131
|
+
* `has()` is "do I have *any* form of this permission declared".
|
|
132
|
+
*/
|
|
133
|
+
const PERMISSION_TYPE_TO_CANONICAL = {
|
|
134
|
+
MICROPHONE: "microphone",
|
|
135
|
+
CAMERA: "camera",
|
|
136
|
+
LOCATION: "location",
|
|
137
|
+
BACKGROUND_LOCATION: "location",
|
|
138
|
+
READ_NOTIFICATIONS: "notifications",
|
|
139
|
+
POST_NOTIFICATIONS: "notifications",
|
|
140
|
+
CALENDAR: "calendar",
|
|
141
|
+
};
|
|
142
|
+
function normalizeRgbLedAction(value) {
|
|
143
|
+
return typeof value === "string" && RGB_LED_ACTIONS.has(value) ? value : "off";
|
|
144
|
+
}
|
|
145
|
+
function normalizeRgbLedColor(value) {
|
|
146
|
+
return typeof value === "string" && RGB_LED_COLORS.has(value) ? value : null;
|
|
147
|
+
}
|
|
148
|
+
function normalizeCameraFovPayload(payload) {
|
|
149
|
+
if (typeof payload.preset === "string") {
|
|
150
|
+
const preset = payload.preset in CAMERA_FOV_PRESETS ? payload.preset : "standard";
|
|
151
|
+
return { preset };
|
|
152
|
+
}
|
|
153
|
+
const rawFov = typeof payload.fov === "number" && Number.isFinite(payload.fov) ? payload.fov : CAMERA_FOV_DEFAULT;
|
|
154
|
+
return {
|
|
155
|
+
fov: Math.min(CAMERA_FOV_MAX, Math.max(CAMERA_FOV_MIN, rawFov)),
|
|
156
|
+
roiPosition: normalizeCameraRoiPosition(payload.roiPosition ?? payload.roi_position),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
function normalizeCameraRoiPosition(value) {
|
|
160
|
+
if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 2) {
|
|
161
|
+
return value === 1 ? "bottom" : value === 2 ? "top" : "center";
|
|
162
|
+
}
|
|
163
|
+
if (typeof value === "string") {
|
|
164
|
+
return CAMERA_ROI_POSITION_BY_NAME[value] ?? "center";
|
|
165
|
+
}
|
|
166
|
+
return "center";
|
|
167
|
+
}
|
|
168
|
+
const ALL_CANONICAL_PERMISSIONS = ["location", "microphone", "camera", "notifications", "calendar"];
|
|
169
|
+
/**
|
|
170
|
+
* Build the {location, microphone, camera, notifications, calendar} record the
|
|
171
|
+
* SDK's session.permissions module reads from. Missing types are false.
|
|
172
|
+
*/
|
|
173
|
+
function computeDeclaredPermissionRecord(manifest) {
|
|
174
|
+
const record = {};
|
|
175
|
+
for (const k of ALL_CANONICAL_PERMISSIONS)
|
|
176
|
+
record[k] = false;
|
|
177
|
+
for (const perm of manifest?.permissions ?? []) {
|
|
178
|
+
const canonical = PERMISSION_TYPE_TO_CANONICAL[perm.type?.toUpperCase()];
|
|
179
|
+
if (canonical)
|
|
180
|
+
record[canonical] = true;
|
|
181
|
+
}
|
|
182
|
+
return record;
|
|
183
|
+
}
|
|
184
|
+
// =============================================================================
|
|
185
|
+
// PERMISSION_NOT_DECLARED warnings — per-session dedup
|
|
186
|
+
// =============================================================================
|
|
187
|
+
//
|
|
188
|
+
// When a miniapp tries to subscribe / call something whose required permission
|
|
189
|
+
// isn't declared in miniapp.json, the runtime rejects with
|
|
190
|
+
// PERMISSION_NOT_DECLARED. The error reaches the SDK but most authors don't
|
|
191
|
+
// subscribe to session.on("error", ...), so it's silent in practice.
|
|
192
|
+
//
|
|
193
|
+
// To make the failure discoverable for developers running the MentraOS app
|
|
194
|
+
// from source, log a clear, copy-pasteable message in the phone console that
|
|
195
|
+
// names the offending permission, the offending stream/op, and the JSON
|
|
196
|
+
// snippet to add to miniapp.json. Once-per-session per (packageName, permission)
|
|
197
|
+
// to avoid spam from a tight retry loop.
|
|
198
|
+
//
|
|
199
|
+
// Production users running the App Store build of MentraOS won't see these
|
|
200
|
+
// (they don't watch Metro/adb logcat). For them, the WebView console bridge
|
|
201
|
+
// (#5 of the quick-fixes round) ships the structured error to the miniapp
|
|
202
|
+
// itself, and the miniapp's own console.warn flows to the dev terminal.
|
|
203
|
+
const warnedPermission = new Set(); // key: `${packageName}::${permission}`
|
|
204
|
+
function logPermissionNotDeclared(packageName, permission, context, manifestSnippet) {
|
|
205
|
+
const key = `${packageName}::${permission}`;
|
|
206
|
+
if (warnedPermission.has(key))
|
|
207
|
+
return;
|
|
208
|
+
warnedPermission.add(key);
|
|
209
|
+
console.warn(`${LOG_TAG}: ${packageName} attempted ${context}, but permission ${permission} is not declared in miniapp.json.\n` +
|
|
210
|
+
`Add this to the "permissions" array:\n ${manifestSnippet}`);
|
|
211
|
+
}
|
|
212
|
+
/** Reset the per-session dedup; called when a miniapp unregisters so a fresh launch warns again. */
|
|
213
|
+
function resetPermissionWarnings(packageName) {
|
|
214
|
+
for (const key of warnedPermission) {
|
|
215
|
+
if (key.startsWith(`${packageName}::`))
|
|
216
|
+
warnedPermission.delete(key);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
// =============================================================================
|
|
220
|
+
// LocalMiniappRuntime
|
|
221
|
+
// =============================================================================
|
|
222
|
+
class LocalMiniappRuntime {
|
|
223
|
+
static instance = null;
|
|
224
|
+
/** Connected miniapps keyed by packageName. */
|
|
225
|
+
connectedApps = new Map();
|
|
226
|
+
/**
|
|
227
|
+
* Packages that have completed their CONNECT handshake (sent
|
|
228
|
+
* `miniapp_connect`, got `CONNECT_ACK`). Distinct from `connectedApps`,
|
|
229
|
+
* which is populated at spawn/registration time — handshake is later.
|
|
230
|
+
* Drives {@link waitForConnect}.
|
|
231
|
+
*/
|
|
232
|
+
handshookApps = new Set();
|
|
233
|
+
/** Pending {@link waitForConnect} resolvers, keyed by packageName. */
|
|
234
|
+
connectWaiters = new Map();
|
|
235
|
+
/** Ref-counted stream subscriptions: stream → set of packageNames. */
|
|
236
|
+
streamSubscribers = new Map();
|
|
237
|
+
/** Guards one-time wiring of the cloud transcript/translation fan-out. */
|
|
238
|
+
cloudResultsWired = false;
|
|
239
|
+
cloudStatusWired = false;
|
|
240
|
+
cloudAudioSubscriptionSync = new CloudAudioSubscriptionSync();
|
|
241
|
+
/** Ping interval handle. */
|
|
242
|
+
pingIntervalId = null;
|
|
243
|
+
foregroundProbeTimers = new Map();
|
|
244
|
+
/**
|
|
245
|
+
* Notified when a miniapp is unregistered for missing liveness pings.
|
|
246
|
+
* MentraJSRouter wires this into its crash-respawn machinery so a
|
|
247
|
+
* silently-stalled background script is restarted (with crash-loop
|
|
248
|
+
* protection) instead of staying dead — a dead background drops its
|
|
249
|
+
* subscriptions, which releases the mic while the webview still looks
|
|
250
|
+
* alive.
|
|
251
|
+
*/
|
|
252
|
+
onLivenessTimeout = null;
|
|
253
|
+
/** Pending cloud requests: requestId → packageName that originated the request. */
|
|
254
|
+
pendingCloudRequests = new Map();
|
|
255
|
+
speechRuns = new Map();
|
|
256
|
+
// Browser fallback token auth — HMAC-signed blob with a phone-local
|
|
257
|
+
// secret. Both issuer and verifier are the same process, so the
|
|
258
|
+
// secret never leaves the device.
|
|
259
|
+
localSecret = `miniapp_${Date.now()}_${Math.random().toString(36).slice(2, 14)}`;
|
|
260
|
+
usedTokens = new Set();
|
|
261
|
+
constructor() { }
|
|
262
|
+
/**
|
|
263
|
+
* Generate an HMAC-signed local session token for browser fallback auth.
|
|
264
|
+
* Token format: base64(JSON({userId, packageName, exp})).base64(HMAC-SHA256(payload, secret))
|
|
265
|
+
* Single-use, 5-minute TTL.
|
|
266
|
+
*/
|
|
267
|
+
generateLocalToken(userId, packageName) {
|
|
268
|
+
const payload = JSON.stringify({
|
|
269
|
+
userId,
|
|
270
|
+
packageName,
|
|
271
|
+
exp: Date.now() + 5 * 60 * 1000, // 5 min TTL
|
|
272
|
+
nonce: Math.random().toString(36).slice(2, 10),
|
|
273
|
+
});
|
|
274
|
+
const payloadB64 = btoa(payload);
|
|
275
|
+
const sig = this.hmacSign(payload);
|
|
276
|
+
return `${payloadB64}.${sig}`;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Validate and consume a local session token. Single-use.
|
|
280
|
+
*/
|
|
281
|
+
validateLocalToken(token) {
|
|
282
|
+
const parts = token.split(".");
|
|
283
|
+
if (parts.length !== 2)
|
|
284
|
+
return null;
|
|
285
|
+
const [payloadB64, sig] = parts;
|
|
286
|
+
// Check single-use
|
|
287
|
+
if (this.usedTokens.has(token))
|
|
288
|
+
return null;
|
|
289
|
+
// Verify signature
|
|
290
|
+
let payload;
|
|
291
|
+
try {
|
|
292
|
+
payload = atob(payloadB64);
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return null;
|
|
296
|
+
}
|
|
297
|
+
if (this.hmacSign(payload) !== sig)
|
|
298
|
+
return null;
|
|
299
|
+
// Parse and check expiry
|
|
300
|
+
let parsed;
|
|
301
|
+
try {
|
|
302
|
+
parsed = JSON.parse(payload);
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
return null;
|
|
306
|
+
}
|
|
307
|
+
if (Date.now() > parsed.exp)
|
|
308
|
+
return null;
|
|
309
|
+
// Mark as used
|
|
310
|
+
this.usedTokens.add(token);
|
|
311
|
+
// Prune old used tokens periodically (keep set from growing unbounded)
|
|
312
|
+
if (this.usedTokens.size > 1000) {
|
|
313
|
+
this.usedTokens.clear();
|
|
314
|
+
}
|
|
315
|
+
return { userId: parsed.userId, packageName: parsed.packageName };
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Simple HMAC-SHA256 using Web Crypto API (available in React Native).
|
|
319
|
+
* Returns base64url-encoded signature.
|
|
320
|
+
* Falls back to a simpler hash if crypto.subtle is not available.
|
|
321
|
+
*/
|
|
322
|
+
hmacSign(payload) {
|
|
323
|
+
// Synchronous HMAC using a simple hash — Web Crypto's subtle.sign is async
|
|
324
|
+
// which doesn't fit cleanly here. For phone-local single-process auth,
|
|
325
|
+
// a keyed hash is sufficient.
|
|
326
|
+
let hash = 0;
|
|
327
|
+
const key = this.localSecret;
|
|
328
|
+
const input = key + payload + key;
|
|
329
|
+
for (let i = 0; i < input.length; i++) {
|
|
330
|
+
const ch = input.charCodeAt(i);
|
|
331
|
+
hash = ((hash << 5) - hash + ch) | 0;
|
|
332
|
+
}
|
|
333
|
+
// Mix in more bytes for collision resistance
|
|
334
|
+
let hash2 = 0x811c9dc5; // FNV offset basis
|
|
335
|
+
for (let i = 0; i < input.length; i++) {
|
|
336
|
+
hash2 ^= input.charCodeAt(i);
|
|
337
|
+
hash2 = Math.imul(hash2, 0x01000193); // FNV prime
|
|
338
|
+
}
|
|
339
|
+
return btoa(String(hash >>> 0) + "." + String(hash2 >>> 0));
|
|
340
|
+
}
|
|
341
|
+
static getInstance() {
|
|
342
|
+
if (!LocalMiniappRuntime.instance) {
|
|
343
|
+
LocalMiniappRuntime.instance = new LocalMiniappRuntime();
|
|
344
|
+
}
|
|
345
|
+
return LocalMiniappRuntime.instance;
|
|
346
|
+
}
|
|
347
|
+
initialized = false;
|
|
348
|
+
/**
|
|
349
|
+
* Initialize the runtime. Called from MantleManager.init().
|
|
350
|
+
* Idempotent — safe to call multiple times.
|
|
351
|
+
*/
|
|
352
|
+
initialize() {
|
|
353
|
+
if (this.initialized)
|
|
354
|
+
return;
|
|
355
|
+
this.initialized = true;
|
|
356
|
+
console.log(`${LOG_TAG}: initialize()`);
|
|
357
|
+
this.ensurePingLoop();
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Handle an incoming cloud message forwarded by SocketComms
|
|
361
|
+
* (phone_stream_status, phone_managed_stream_status).
|
|
362
|
+
*
|
|
363
|
+
* Routes the response back to the originating miniapp via the requestId
|
|
364
|
+
* that was stored when the miniapp first made the request.
|
|
365
|
+
*/
|
|
366
|
+
handleCloudMessage(msg) {
|
|
367
|
+
const requestId = msg.requestId;
|
|
368
|
+
const msgType = msg.type;
|
|
369
|
+
console.log(`${LOG_TAG}: Cloud message: ${msgType}, requestId=${requestId ?? "none"}`);
|
|
370
|
+
if (!requestId) {
|
|
371
|
+
console.warn(`${LOG_TAG}: Cloud message ${msgType} has no requestId, cannot route`);
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
const pending = this.pendingCloudRequests.get(requestId);
|
|
375
|
+
if (!pending) {
|
|
376
|
+
console.warn(`${LOG_TAG}: No pending request for requestId=${requestId}`);
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
this.pendingCloudRequests.delete(requestId);
|
|
380
|
+
switch (msgType) {
|
|
381
|
+
case "phone_stream_status": {
|
|
382
|
+
// Unreachable for phone-orchestrated streams (the coordinator owns
|
|
383
|
+
// their lifecycle and never registers a pending cloud request).
|
|
384
|
+
// Retained as a safety net for any legacy registration path.
|
|
385
|
+
this.sendToMiniapp(pending.packageName, {
|
|
386
|
+
type: MiniappResponseType.EVENT,
|
|
387
|
+
streamType: "stream_status",
|
|
388
|
+
data: {
|
|
389
|
+
streamId: msg.streamId,
|
|
390
|
+
status: msg.status,
|
|
391
|
+
errorDetails: msg.errorDetails,
|
|
392
|
+
},
|
|
393
|
+
});
|
|
394
|
+
// Re-register for ongoing status updates (streams send multiple status messages)
|
|
395
|
+
this.pendingCloudRequests.set(requestId, pending);
|
|
396
|
+
break;
|
|
397
|
+
}
|
|
398
|
+
case "phone_managed_stream_status": {
|
|
399
|
+
if (msg.status === "connected" || msg.status === "active") {
|
|
400
|
+
// Managed stream is ready — send back the playback URLs as the request result
|
|
401
|
+
this.sendResult(pending.packageName, pending.envelopeRequestId, true, {
|
|
402
|
+
streamId: msg.streamId,
|
|
403
|
+
hlsUrl: msg.hlsUrl,
|
|
404
|
+
dashUrl: msg.dashUrl,
|
|
405
|
+
webrtcUrl: msg.webrtcUrl,
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
// Forward all statuses as events too
|
|
409
|
+
this.sendToMiniapp(pending.packageName, {
|
|
410
|
+
type: MiniappResponseType.EVENT,
|
|
411
|
+
streamType: "stream_status",
|
|
412
|
+
data: {
|
|
413
|
+
streamId: msg.streamId,
|
|
414
|
+
status: msg.status,
|
|
415
|
+
hlsUrl: msg.hlsUrl,
|
|
416
|
+
dashUrl: msg.dashUrl,
|
|
417
|
+
webrtcUrl: msg.webrtcUrl,
|
|
418
|
+
},
|
|
419
|
+
});
|
|
420
|
+
// Re-register for ongoing updates
|
|
421
|
+
this.pendingCloudRequests.set(requestId, pending);
|
|
422
|
+
break;
|
|
423
|
+
}
|
|
424
|
+
default:
|
|
425
|
+
console.warn(`${LOG_TAG}: Unknown cloud message type: ${msgType}`);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Register a pending cloud request so we can route the response back.
|
|
430
|
+
*/
|
|
431
|
+
registerPendingCloudRequest(requestId, packageName, envelopeRequestId) {
|
|
432
|
+
this.pendingCloudRequests.set(requestId, { packageName, envelopeRequestId });
|
|
433
|
+
}
|
|
434
|
+
// ===========================================================================
|
|
435
|
+
// App registration
|
|
436
|
+
// ===========================================================================
|
|
437
|
+
registerApp(packageName, sendFn, installedManifest) {
|
|
438
|
+
console.log(`${LOG_TAG}: registerApp(${packageName})`);
|
|
439
|
+
this.clearForegroundProbe(packageName);
|
|
440
|
+
// Fresh spawn → fresh handshake. Clear any prior handshake flag so
|
|
441
|
+
// waitForConnect() blocks until this context's own CONNECT arrives.
|
|
442
|
+
this.handshookApps.delete(packageName);
|
|
443
|
+
// If the app is already registered (e.g. QR scanned again for same package),
|
|
444
|
+
// tear down its old subscriptions first so streamSubscribers doesn't keep
|
|
445
|
+
// dangling references. The WebView will re-subscribe after its CONNECT.
|
|
446
|
+
if (this.connectedApps.has(packageName)) {
|
|
447
|
+
const existing = this.connectedApps.get(packageName);
|
|
448
|
+
this.cancelSpeech(packageName);
|
|
449
|
+
// A crash respawn replaces the ConnectedMiniapp without going through
|
|
450
|
+
// unregisterApp(). Abort the old incarnation's native PCM player before
|
|
451
|
+
// its only stream id is lost. Target the exact stream rather than using
|
|
452
|
+
// stopForApp(), whose async playback cleanup could race with audio the
|
|
453
|
+
// replacement app starts after it connects.
|
|
454
|
+
const existingSpeakerStreamId = existing.activeSpeakerStreamId;
|
|
455
|
+
if (existingSpeakerStreamId) {
|
|
456
|
+
existing.activeSpeakerStreamId = undefined;
|
|
457
|
+
void audioPlaybackService.abortStream(existingSpeakerStreamId).catch((error) => {
|
|
458
|
+
console.warn(`${LOG_TAG}: failed to abort speaker stream while replacing ${packageName}`, error);
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
if (existing.authRefreshTimerId !== null) {
|
|
462
|
+
BgTimer.clearTimeout(existing.authRefreshTimerId);
|
|
463
|
+
existing.authRefreshTimerId = null;
|
|
464
|
+
}
|
|
465
|
+
if (existing.authRetryTimerId !== null) {
|
|
466
|
+
BgTimer.clearTimeout(existing.authRetryTimerId);
|
|
467
|
+
existing.authRetryTimerId = null;
|
|
468
|
+
}
|
|
469
|
+
for (const stream of existing.subscriptions) {
|
|
470
|
+
const subs = this.streamSubscribers.get(stream);
|
|
471
|
+
if (subs) {
|
|
472
|
+
subs.delete(packageName);
|
|
473
|
+
if (subs.size === 0)
|
|
474
|
+
this.streamSubscribers.delete(stream);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
this.recomputeMicRequirements();
|
|
478
|
+
this.updateCloudSubscriptions();
|
|
479
|
+
}
|
|
480
|
+
this.connectedApps.set(packageName, {
|
|
481
|
+
subscriptions: new Set(),
|
|
482
|
+
forceLocalTranscriptionStreams: new Set(),
|
|
483
|
+
cloudTranscriptionStreams: new Set(),
|
|
484
|
+
sendMessage: sendFn,
|
|
485
|
+
lastPongAt: Date.now(),
|
|
486
|
+
installedManifest,
|
|
487
|
+
authRefreshTimerId: null,
|
|
488
|
+
authRetryTimerId: null,
|
|
489
|
+
authDelivered: false,
|
|
490
|
+
speakerState: "idle",
|
|
491
|
+
requestedLocationRate: null,
|
|
492
|
+
});
|
|
493
|
+
// If we just clobbered an existing entry, its requestedLocationRate is
|
|
494
|
+
// now gone — recompute so the aggregate doesn't include a stale rate.
|
|
495
|
+
// The WebView will re-SUBSCRIBE shortly and the rate will reappear.
|
|
496
|
+
this.recomputeLocationTier();
|
|
497
|
+
this.ensureCloudStatusWired();
|
|
498
|
+
this.ensurePingLoop();
|
|
499
|
+
// Show the system boot message ("Starting <name>…") on the glasses for the
|
|
500
|
+
// bounded boot window, mirroring the cloud DisplayManager boot screen. The
|
|
501
|
+
// window ends early once this app pushes its first display (see
|
|
502
|
+
// LocalDisplayManager.request), or after ~1.5s. Fires once per spawn —
|
|
503
|
+
// including a crash-respawn, since each spawn is a fresh "app is starting".
|
|
504
|
+
localDisplayManager.onMount(packageName, installedManifest?.name ?? packageName);
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* Resolve once `packageName` has completed its CONNECT handshake. If it has
|
|
508
|
+
* already connected, resolves immediately. Rejects if the timeout elapses
|
|
509
|
+
* first, or if the app is unregistered while waiting.
|
|
510
|
+
*
|
|
511
|
+
* Used by the launcher's `ensureConnected` (and thus the action broker) so a
|
|
512
|
+
* just-spawned target is known to be live before anything is delivered to it.
|
|
513
|
+
*/
|
|
514
|
+
waitForConnect(packageName, timeoutMs) {
|
|
515
|
+
if (this.handshookApps.has(packageName))
|
|
516
|
+
return Promise.resolve();
|
|
517
|
+
return new Promise((resolve, reject) => {
|
|
518
|
+
let settled = false;
|
|
519
|
+
const waiter = (err) => {
|
|
520
|
+
if (settled)
|
|
521
|
+
return;
|
|
522
|
+
settled = true;
|
|
523
|
+
clearTimeout(timer);
|
|
524
|
+
this.connectWaiters.get(packageName)?.delete(waiter);
|
|
525
|
+
if (err)
|
|
526
|
+
reject(err);
|
|
527
|
+
else
|
|
528
|
+
resolve();
|
|
529
|
+
};
|
|
530
|
+
const timer = setTimeout(() => {
|
|
531
|
+
waiter(new Error(`waitForConnect: ${packageName} did not connect within ${timeoutMs}ms`));
|
|
532
|
+
}, timeoutMs);
|
|
533
|
+
let set = this.connectWaiters.get(packageName);
|
|
534
|
+
if (!set) {
|
|
535
|
+
set = new Set();
|
|
536
|
+
this.connectWaiters.set(packageName, set);
|
|
537
|
+
}
|
|
538
|
+
set.add(waiter);
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
/** Resolve/reject all pending {@link waitForConnect} waiters for a package. */
|
|
542
|
+
flushConnectWaiters(packageName, err) {
|
|
543
|
+
const waiters = this.connectWaiters.get(packageName);
|
|
544
|
+
if (!waiters)
|
|
545
|
+
return;
|
|
546
|
+
this.connectWaiters.delete(packageName);
|
|
547
|
+
for (const w of waiters)
|
|
548
|
+
w(err);
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Invalidate the CONNECT handshake for a package whose native context is
|
|
552
|
+
* being replaced WITHOUT going through register/unregister — i.e. a crash
|
|
553
|
+
* respawn. Without this, {@link waitForConnect} would treat the dead/
|
|
554
|
+
* not-yet-initialized context as connected and deliver to it. Pending waiters
|
|
555
|
+
* are failed so a wake mid-respawn retries rather than hanging.
|
|
556
|
+
*/
|
|
557
|
+
resetHandshake(packageName) {
|
|
558
|
+
this.handshookApps.delete(packageName);
|
|
559
|
+
this.flushConnectWaiters(packageName, new Error(`${packageName} respawning`));
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* Attach (or update) the installedManifest for an already-registered app.
|
|
563
|
+
* Used when the manifest is fetched asynchronously (dev miniapps) after the
|
|
564
|
+
* miniapp has already CONNECTed — preserves existing subscriptions.
|
|
565
|
+
*/
|
|
566
|
+
setInstalledManifest(packageName, installedManifest) {
|
|
567
|
+
const app = this.connectedApps.get(packageName);
|
|
568
|
+
if (!app)
|
|
569
|
+
return;
|
|
570
|
+
app.installedManifest = installedManifest;
|
|
571
|
+
// Push declared-permission record to the SDK so session.permissions stays
|
|
572
|
+
// in sync. Sent regardless of CONNECT_ACK timing — covers the dev-miniapp
|
|
573
|
+
// case where the manifest is fetched async after the miniapp CONNECTs.
|
|
574
|
+
this.sendToMiniapp(packageName, {
|
|
575
|
+
type: MiniappResponseType.PERMISSIONS_UPDATE,
|
|
576
|
+
permissions: computeDeclaredPermissionRecord(installedManifest),
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Graceful version of {@link unregisterApp}: notify the miniapp via
|
|
581
|
+
* `WILL_DISCONNECT`, wait ~50ms so its `beforeDisconnect` handlers can
|
|
582
|
+
* fire one last `sendOneShot` (e.g. `display.clear()`), then run the
|
|
583
|
+
* normal teardown. Use this for any disconnect path where the socket
|
|
584
|
+
* is still open. For ungraceful paths (transport already closed),
|
|
585
|
+
* call {@link unregisterApp} directly — the heads-up would be
|
|
586
|
+
* undeliverable anyway.
|
|
587
|
+
*/
|
|
588
|
+
async gracefullyUnregisterApp(packageName, reason = "unregistering") {
|
|
589
|
+
const app = this.connectedApps.get(packageName);
|
|
590
|
+
if (!app)
|
|
591
|
+
return;
|
|
592
|
+
console.log(`${LOG_TAG}: gracefullyUnregisterApp(${packageName}) — sending WILL_DISCONNECT`);
|
|
593
|
+
this.sendToMiniapp(packageName, {
|
|
594
|
+
type: MiniappResponseType.WILL_DISCONNECT,
|
|
595
|
+
reason,
|
|
596
|
+
});
|
|
597
|
+
// 50 ms: imperceptible to the user, plenty of time for the SDK's
|
|
598
|
+
// synchronous `beforeDisconnect` handlers to flush a final
|
|
599
|
+
// sendOneShot through the still-open transport.
|
|
600
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
601
|
+
this.unregisterApp(packageName);
|
|
602
|
+
}
|
|
603
|
+
unregisterApp(packageName) {
|
|
604
|
+
console.log(`${LOG_TAG}: unregisterApp(${packageName})`);
|
|
605
|
+
this.clearForegroundProbe(packageName);
|
|
606
|
+
this.clearMiniappAuthRefresh(packageName);
|
|
607
|
+
this.clearMiniappAuthDeliveryRetry(packageName);
|
|
608
|
+
// Drop the handshake flag and fail any in-flight waitForConnect() callers
|
|
609
|
+
// (e.g. a wake that's mid-handshake when the app is torn down). Done before
|
|
610
|
+
// the early-return so it runs even if the connectedApps entry is already gone.
|
|
611
|
+
this.handshookApps.delete(packageName);
|
|
612
|
+
this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`));
|
|
613
|
+
// Warm-up is a request-owned camera lease. Cancel it even if the app record was already
|
|
614
|
+
// removed so a close racing the camera-open promise cannot leave the sensor running.
|
|
615
|
+
// Keep camera teardown ordered: restoring FOV can restart the HAL, so the warm-up lease must
|
|
616
|
+
// finish closing the sensor before FOV reconciliation begins. Continue to FOV cleanup even if
|
|
617
|
+
// warm-up cancellation fails; phone ownership remains retryable until the ASG lease TTL.
|
|
618
|
+
void phonePhotoCoordinator
|
|
619
|
+
.stopWarmUpForApp(packageName)
|
|
620
|
+
.catch((error) => {
|
|
621
|
+
console.warn(`${LOG_TAG}: failed to stop camera warm-up for ${packageName} on unregister`, error);
|
|
622
|
+
})
|
|
623
|
+
.then(() => phoneCameraFovCoordinator.releaseForApp(packageName))
|
|
624
|
+
.catch((error) => {
|
|
625
|
+
console.warn(`${LOG_TAG}: failed to release camera FOV override for ${packageName} on unregister`, error);
|
|
626
|
+
});
|
|
627
|
+
const app = this.connectedApps.get(packageName);
|
|
628
|
+
if (!app)
|
|
629
|
+
return;
|
|
630
|
+
// Remove from all stream subscriber sets
|
|
631
|
+
for (const stream of app.subscriptions) {
|
|
632
|
+
const subs = this.streamSubscribers.get(stream);
|
|
633
|
+
if (subs) {
|
|
634
|
+
subs.delete(packageName);
|
|
635
|
+
if (subs.size === 0) {
|
|
636
|
+
this.streamSubscribers.delete(stream);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
// Reset per-session warning dedup so a relaunch surfaces issues again.
|
|
641
|
+
resetPermissionWarnings(packageName);
|
|
642
|
+
// Cancel lookahead/in-flight synthesis before stopping the current file so
|
|
643
|
+
// an interrupted sentence cannot advance and reclaim playback.
|
|
644
|
+
this.cancelSpeech(packageName);
|
|
645
|
+
// Stop audio for this app
|
|
646
|
+
audioPlaybackService.stopForApp(packageName);
|
|
647
|
+
// Tear down this app's blob state: abort in-flight uploads + close readers
|
|
648
|
+
// so a crashed/closed miniapp doesn't leak partial files or file handles.
|
|
649
|
+
this.blobStore.onAppGone(packageName);
|
|
650
|
+
// Release phone-owned camera streams. If a miniapp closes/crashes without
|
|
651
|
+
// sending STREAM_STOP, the host coordinator must drop its subscriber/owner
|
|
652
|
+
// so glasses publishing and managed Cloudflare inputs do not leak.
|
|
653
|
+
void phoneStreamCoordinator.stop(packageName).catch((error) => {
|
|
654
|
+
console.warn(`${LOG_TAG}: failed to stop stream for ${packageName} on unregister`, error);
|
|
655
|
+
});
|
|
656
|
+
// Stop any phone-owned video recordings for this app. A miniapp that
|
|
657
|
+
// closes/crashes mid-recording loses its recordingId, so without this the
|
|
658
|
+
// glasses keep recording until the max-recording timeout or thermal shutdown.
|
|
659
|
+
void phoneVideoCoordinator.stopForApp(packageName).catch((error) => {
|
|
660
|
+
console.warn(`${LOG_TAG}: failed to stop video recording for ${packageName} on unregister`, error);
|
|
661
|
+
});
|
|
662
|
+
// Detach the per-app nav event forwarder but leave the native nav session
|
|
663
|
+
// running. The user may have just closed the mini-app UI and will reopen
|
|
664
|
+
// it; stopping the session here would kill an active trip mid-route.
|
|
665
|
+
// Navigation is only stopped when the mini-app explicitly calls
|
|
666
|
+
// navigation.stop() or when the trip arrives/errors naturally.
|
|
667
|
+
// (See NavigationHandlers — activeNavApps stays populated so a reconnect
|
|
668
|
+
// can reattach listeners and resume.)
|
|
669
|
+
this.navigationHandlers.onDisconnect(packageName);
|
|
670
|
+
// Recompute heading subscription — if this app was the last subscriber,
|
|
671
|
+
// the sensor will stop.
|
|
672
|
+
this.recomputeHeadingSubscription();
|
|
673
|
+
// Same for the IMU/accelerometer stream.
|
|
674
|
+
this.recomputeImuSubscription();
|
|
675
|
+
// Drop this app's location-tier request before recomputing so the
|
|
676
|
+
// aggregate falls back down if it was the strictest. Done before
|
|
677
|
+
// `connectedApps.delete` below so it doesn't matter that the iter
|
|
678
|
+
// in `recomputeLocationTier` is over connectedApps — clearing the
|
|
679
|
+
// field is enough.
|
|
680
|
+
app.requestedLocationRate = null;
|
|
681
|
+
this.recomputeLocationTier();
|
|
682
|
+
// Clean up any pending cloud requests from this app
|
|
683
|
+
for (const [reqId, pending] of this.pendingCloudRequests) {
|
|
684
|
+
if (pending.packageName === packageName) {
|
|
685
|
+
this.pendingCloudRequests.delete(reqId);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
// Release any display real estate this app held — if it owned the
|
|
689
|
+
// current on-glasses frame, this clears the glasses (or restores the
|
|
690
|
+
// core app's saved frame).
|
|
691
|
+
localDisplayManager.onUnmount(packageName);
|
|
692
|
+
this.connectedApps.delete(packageName);
|
|
693
|
+
this.recomputeMicRequirements();
|
|
694
|
+
this.updateCloudSubscriptions();
|
|
695
|
+
if (this.connectedApps.size === 0) {
|
|
696
|
+
this.stopPingLoop();
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
// ===========================================================================
|
|
700
|
+
// Inbound message handling
|
|
701
|
+
// ===========================================================================
|
|
702
|
+
handleRawMessage(packageName, raw) {
|
|
703
|
+
const envelope = parseEnvelope(raw);
|
|
704
|
+
if (!envelope) {
|
|
705
|
+
// Not a miniapp envelope — ignore (could be legacy bridge message)
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
const payload = envelope.payload;
|
|
709
|
+
const requestType = payload.type;
|
|
710
|
+
const requestId = envelope.requestId;
|
|
711
|
+
if (!requestType) {
|
|
712
|
+
console.warn(`${LOG_TAG}: Envelope from ${packageName} missing payload.type`);
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
// ANY inbound message proves the context is alive — a busy background
|
|
716
|
+
// script streaming DISPLAY/storage traffic shouldn't be killed by the
|
|
717
|
+
// liveness watchdog just because its PONG replies queue behind real work.
|
|
718
|
+
this.handlePong(packageName);
|
|
719
|
+
// Console-tap forwarding. The miniapp's console.log/warn/etc is wrapped
|
|
720
|
+
// (via injected shim from miniappGlobals.ts) to post a `dev_log`
|
|
721
|
+
// envelope. We fan out to two destinations:
|
|
722
|
+
// 1. DevServerBridge — forwards to the laptop's `mentra-miniapp dev`
|
|
723
|
+
// terminal. No-op when there's no sidecar (installed miniapps).
|
|
724
|
+
// 2. React Native console — surfaces the log in Metro / Xcode console
|
|
725
|
+
// / adb logcat so installed-miniapp errors are still inspectable
|
|
726
|
+
// when there's no laptop to forward to.
|
|
727
|
+
if (requestType === "dev_log") {
|
|
728
|
+
const level = payload.level ?? "log";
|
|
729
|
+
const args = Array.isArray(payload.args) ? payload.args : [];
|
|
730
|
+
const timestamp = payload.timestamp ?? Date.now();
|
|
731
|
+
// Source is "ui" because the single-bundle WebView console-tap
|
|
732
|
+
// shim in miniappGlobals.ts is the only thing that posts
|
|
733
|
+
// `dev_log`; the two-layer path uses a separate envelope
|
|
734
|
+
// (`{type:"log", source}`) routed by MentraUIRouter.
|
|
735
|
+
devServerBridge.forwardLog(packageName, level, args, timestamp, "ui");
|
|
736
|
+
const tag = `[MINIAPP ${packageName}]`;
|
|
737
|
+
const fn = console[level] ?? console.log;
|
|
738
|
+
try {
|
|
739
|
+
fn(tag, ...args);
|
|
740
|
+
}
|
|
741
|
+
catch {
|
|
742
|
+
console.log(tag, ...args);
|
|
743
|
+
}
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
// Dispatch
|
|
747
|
+
switch (requestType) {
|
|
748
|
+
case MiniappRequestType.CONNECT:
|
|
749
|
+
void this.handleConnect(packageName, payload, requestId);
|
|
750
|
+
break;
|
|
751
|
+
case MiniappRequestType.AUTH_REFRESH:
|
|
752
|
+
void this.handleAuthRefresh(packageName, payload, requestId);
|
|
753
|
+
break;
|
|
754
|
+
case MiniappRequestType.SUBSCRIBE:
|
|
755
|
+
this.handleSubscribe(packageName, payload, requestId);
|
|
756
|
+
break;
|
|
757
|
+
case MiniappRequestType.DISPLAY:
|
|
758
|
+
this.handleDisplay(packageName, payload, requestId);
|
|
759
|
+
break;
|
|
760
|
+
case MiniappRequestType.RENDER:
|
|
761
|
+
this.handleRender(packageName, payload, requestId);
|
|
762
|
+
break;
|
|
763
|
+
case MiniappRequestType.PLAY_AUDIO:
|
|
764
|
+
this.handlePlayAudio(packageName, payload, requestId);
|
|
765
|
+
break;
|
|
766
|
+
case MiniappRequestType.STOP_AUDIO:
|
|
767
|
+
this.handleStopAudio(packageName, payload, requestId);
|
|
768
|
+
break;
|
|
769
|
+
case MiniappRequestType.SPEAK:
|
|
770
|
+
this.handleSpeak(packageName, payload, requestId);
|
|
771
|
+
break;
|
|
772
|
+
case MiniappRequestType.SPEAKER_STREAM_OPEN:
|
|
773
|
+
void this.handleSpeakerStreamOpen(packageName, payload, requestId);
|
|
774
|
+
break;
|
|
775
|
+
case MiniappRequestType.SPEAKER_STREAM_WRITE:
|
|
776
|
+
void this.handleSpeakerStreamWrite(packageName, payload, requestId);
|
|
777
|
+
break;
|
|
778
|
+
case MiniappRequestType.SPEAKER_STREAM_CLOSE:
|
|
779
|
+
void this.handleSpeakerStreamClose(packageName, payload, requestId);
|
|
780
|
+
break;
|
|
781
|
+
case MiniappRequestType.SPEAKER_STREAM_ABORT:
|
|
782
|
+
void this.handleSpeakerStreamAbort(packageName, payload, requestId);
|
|
783
|
+
break;
|
|
784
|
+
case MiniappRequestType.RGB_LED:
|
|
785
|
+
void this.handleRgbLed(packageName, payload, requestId);
|
|
786
|
+
break;
|
|
787
|
+
case MiniappRequestType.LOCATION_POLL:
|
|
788
|
+
this.handleLocationPoll(packageName, requestId);
|
|
789
|
+
break;
|
|
790
|
+
case MiniappRequestType.CALENDAR_LIST_EVENTS:
|
|
791
|
+
void this.handleCalendarListEvents(packageName, payload, requestId);
|
|
792
|
+
break;
|
|
793
|
+
case MiniappRequestType.NAVIGATION_START:
|
|
794
|
+
this.navigationHandlers.handleStart(packageName, payload, requestId);
|
|
795
|
+
break;
|
|
796
|
+
case MiniappRequestType.NAVIGATION_STOP:
|
|
797
|
+
this.navigationHandlers.handleStop(packageName, requestId);
|
|
798
|
+
break;
|
|
799
|
+
case MiniappRequestType.NAVIGATION_DEVIATE:
|
|
800
|
+
this.navigationHandlers.handleDeviate(packageName, payload, requestId);
|
|
801
|
+
break;
|
|
802
|
+
case MiniappRequestType.NAVIGATION_SET_WRONG_SIDEWALK:
|
|
803
|
+
this.navigationHandlers.handleSetWrongSidewalk(packageName, payload, requestId);
|
|
804
|
+
break;
|
|
805
|
+
case MiniappRequestType.NAVIGATION_SET_SKIP_CROSSINGS:
|
|
806
|
+
this.navigationHandlers.handleSetSkipCrossings(packageName, payload, requestId);
|
|
807
|
+
break;
|
|
808
|
+
case MiniappRequestType.NAVIGATION_GET_STATE:
|
|
809
|
+
this.navigationHandlers.handleGetState(packageName, requestId);
|
|
810
|
+
break;
|
|
811
|
+
case MiniappRequestType.NAVIGATION_COMPUTE_ROUTE:
|
|
812
|
+
this.navigationHandlers.handleComputeRoute(packageName, payload, requestId);
|
|
813
|
+
break;
|
|
814
|
+
case MiniappRequestType.NAVIGATION_REVERSE_GEOCODE:
|
|
815
|
+
this.navigationHandlers.handleReverseGeocode(packageName, payload, requestId);
|
|
816
|
+
break;
|
|
817
|
+
case MiniappRequestType.NAVIGATION_PLACE_AUTOCOMPLETE:
|
|
818
|
+
this.navigationHandlers.handlePlaceAutocomplete(packageName, payload, requestId);
|
|
819
|
+
break;
|
|
820
|
+
case MiniappRequestType.NAVIGATION_PLACE_DETAILS:
|
|
821
|
+
this.navigationHandlers.handlePlaceDetails(packageName, payload, requestId);
|
|
822
|
+
break;
|
|
823
|
+
case MiniappRequestType.NAVIGATION_REQUEST_PERMISSION:
|
|
824
|
+
this.navigationHandlers.handleRequestPermission(packageName, requestId);
|
|
825
|
+
break;
|
|
826
|
+
case MiniappRequestType.STORAGE_GET:
|
|
827
|
+
this.handleStorageGet(packageName, payload, requestId);
|
|
828
|
+
break;
|
|
829
|
+
case MiniappRequestType.STORAGE_SET:
|
|
830
|
+
this.handleStorageSet(packageName, payload, requestId);
|
|
831
|
+
break;
|
|
832
|
+
case MiniappRequestType.STORAGE_DELETE:
|
|
833
|
+
this.handleStorageDelete(packageName, payload, requestId);
|
|
834
|
+
break;
|
|
835
|
+
case MiniappRequestType.STORAGE_LIST:
|
|
836
|
+
this.handleStorageList(packageName, payload, requestId);
|
|
837
|
+
break;
|
|
838
|
+
case MiniappRequestType.STORAGE_CLEAR:
|
|
839
|
+
this.handleStorageClear(packageName, requestId);
|
|
840
|
+
break;
|
|
841
|
+
case MiniappRequestType.STORAGE_HAS:
|
|
842
|
+
this.handleStorageHas(packageName, payload, requestId);
|
|
843
|
+
break;
|
|
844
|
+
case MiniappRequestType.STORAGE_GET_ALL:
|
|
845
|
+
this.handleStorageGetAll(packageName, requestId);
|
|
846
|
+
break;
|
|
847
|
+
case MiniappRequestType.STORAGE_SET_MULTIPLE:
|
|
848
|
+
this.handleStorageSetMultiple(packageName, payload, requestId);
|
|
849
|
+
break;
|
|
850
|
+
case MiniappRequestType.STORAGE_FLUSH:
|
|
851
|
+
// No-op today — MMKV writes through synchronously. Reserved so
|
|
852
|
+
// a future debounced backend can honor "flush before I quit"
|
|
853
|
+
// calls without breaking the API.
|
|
854
|
+
this.sendResult(packageName, requestId, true);
|
|
855
|
+
break;
|
|
856
|
+
case MiniappRequestType.CAMERA_FOV:
|
|
857
|
+
void this.handleCameraFov(packageName, payload, requestId);
|
|
858
|
+
break;
|
|
859
|
+
case MiniappRequestType.IMU_SET_ENABLED:
|
|
860
|
+
this.handleImuSetEnabled(packageName, payload, requestId);
|
|
861
|
+
break;
|
|
862
|
+
case MiniappRequestType.PING:
|
|
863
|
+
// SDK should handle this itself; reply PONG just in case
|
|
864
|
+
this.sendToMiniapp(packageName, { type: MiniappResponseType.PONG }, requestId);
|
|
865
|
+
break;
|
|
866
|
+
case MiniappResponseType.PONG: {
|
|
867
|
+
// Liveness already touched above for every inbound message; the
|
|
868
|
+
// PONG carries no other action.
|
|
869
|
+
break;
|
|
870
|
+
}
|
|
871
|
+
case MiniappRequestType.SHARE:
|
|
872
|
+
this.handleShare(packageName, payload, requestId);
|
|
873
|
+
break;
|
|
874
|
+
case MiniappRequestType.OPEN_URL:
|
|
875
|
+
this.handleOpenUrl(packageName, payload);
|
|
876
|
+
break;
|
|
877
|
+
case MiniappRequestType.COPY_CLIPBOARD:
|
|
878
|
+
this.handleCopyClipboard(packageName, payload, requestId);
|
|
879
|
+
break;
|
|
880
|
+
case MiniappRequestType.DOWNLOAD:
|
|
881
|
+
this.handleDownload(packageName, payload, requestId);
|
|
882
|
+
break;
|
|
883
|
+
// Persistent binary blob storage (session.blob)
|
|
884
|
+
case MiniappRequestType.BLOB_CREATE:
|
|
885
|
+
this.blobStore.handleCreate(packageName, payload, requestId);
|
|
886
|
+
break;
|
|
887
|
+
case MiniappRequestType.BLOB_WRITE:
|
|
888
|
+
this.blobStore.handleWrite(packageName, payload, requestId);
|
|
889
|
+
break;
|
|
890
|
+
case MiniappRequestType.BLOB_COMMIT:
|
|
891
|
+
this.blobStore.handleCommit(packageName, payload, requestId);
|
|
892
|
+
break;
|
|
893
|
+
case MiniappRequestType.BLOB_ABORT:
|
|
894
|
+
this.blobStore.handleAbort(packageName, payload, requestId);
|
|
895
|
+
break;
|
|
896
|
+
case MiniappRequestType.BLOB_SET_FROM_URL:
|
|
897
|
+
void this.blobStore.handleSetFromUrl(packageName, payload, requestId);
|
|
898
|
+
break;
|
|
899
|
+
case MiniappRequestType.BLOB_IMPORT:
|
|
900
|
+
void this.blobStore.handleImport(packageName, payload, requestId);
|
|
901
|
+
break;
|
|
902
|
+
case MiniappRequestType.BLOB_GET:
|
|
903
|
+
this.blobStore.handleGet(packageName, payload, requestId);
|
|
904
|
+
break;
|
|
905
|
+
case MiniappRequestType.BLOB_LIST:
|
|
906
|
+
this.blobStore.handleList(packageName, payload, requestId);
|
|
907
|
+
break;
|
|
908
|
+
case MiniappRequestType.BLOB_USAGE:
|
|
909
|
+
this.blobStore.handleUsage(packageName, payload, requestId);
|
|
910
|
+
break;
|
|
911
|
+
case MiniappRequestType.BLOB_DELETE:
|
|
912
|
+
this.blobStore.handleDelete(packageName, payload, requestId);
|
|
913
|
+
break;
|
|
914
|
+
case MiniappRequestType.BLOB_CLEAR:
|
|
915
|
+
this.blobStore.handleClear(packageName, payload, requestId);
|
|
916
|
+
break;
|
|
917
|
+
case MiniappRequestType.BLOB_OPEN_READ:
|
|
918
|
+
this.blobStore.handleOpenRead(packageName, payload, requestId);
|
|
919
|
+
break;
|
|
920
|
+
case MiniappRequestType.BLOB_READ:
|
|
921
|
+
this.blobStore.handleRead(packageName, payload, requestId);
|
|
922
|
+
break;
|
|
923
|
+
case MiniappRequestType.BLOB_CLOSE_READ:
|
|
924
|
+
this.blobStore.handleCloseRead(packageName, payload, requestId);
|
|
925
|
+
break;
|
|
926
|
+
case MiniappRequestType.BLOB_SHARE:
|
|
927
|
+
void this.blobStore.handleShare(packageName, payload, requestId);
|
|
928
|
+
break;
|
|
929
|
+
// Cloud-coordinated features
|
|
930
|
+
case MiniappRequestType.PHOTO:
|
|
931
|
+
void this.handlePhoto(packageName, payload, requestId);
|
|
932
|
+
break;
|
|
933
|
+
case MiniappRequestType.CAMERA_WARM_UP:
|
|
934
|
+
void this.handleCameraWarmUp(packageName, payload, requestId);
|
|
935
|
+
break;
|
|
936
|
+
case MiniappRequestType.VIDEO_RECORDING_START:
|
|
937
|
+
void this.handleVideoRecordingStart(packageName, payload, requestId);
|
|
938
|
+
break;
|
|
939
|
+
case MiniappRequestType.VIDEO_RECORDING_STOP:
|
|
940
|
+
void this.handleVideoRecordingStop(packageName, payload, requestId);
|
|
941
|
+
break;
|
|
942
|
+
case MiniappRequestType.STREAM_START:
|
|
943
|
+
void this.handleStreamStart(packageName, payload, requestId);
|
|
944
|
+
break;
|
|
945
|
+
case MiniappRequestType.STREAM_STOP:
|
|
946
|
+
void this.handleStreamStop(packageName, payload, requestId);
|
|
947
|
+
break;
|
|
948
|
+
case MiniappRequestType.MANAGED_STREAM_START:
|
|
949
|
+
void this.handleManagedStreamStart(packageName, payload, requestId);
|
|
950
|
+
break;
|
|
951
|
+
case MiniappRequestType.MANAGED_STREAM_STOP:
|
|
952
|
+
void this.handleManagedStreamStop(packageName, payload, requestId);
|
|
953
|
+
break;
|
|
954
|
+
case REQUEST_WIFI_SETUP_TYPE:
|
|
955
|
+
void this.handleRequestWifiSetup(packageName, payload, requestId);
|
|
956
|
+
break;
|
|
957
|
+
// Inter-miniapp interop (SYSTEM apps only)
|
|
958
|
+
case MiniappRequestType.MINIAPPS_LIST:
|
|
959
|
+
this.handleMiniappsList(packageName, payload, requestId);
|
|
960
|
+
break;
|
|
961
|
+
case MiniappRequestType.MINIAPPS_START:
|
|
962
|
+
void this.handleMiniappsStart(packageName, payload, requestId);
|
|
963
|
+
break;
|
|
964
|
+
case MiniappRequestType.MINIAPPS_STOP:
|
|
965
|
+
void this.handleMiniappsStop(packageName, payload, requestId);
|
|
966
|
+
break;
|
|
967
|
+
case MiniappRequestType.ACTION_INVOKE:
|
|
968
|
+
void this.handleActionInvoke(packageName, payload, requestId);
|
|
969
|
+
break;
|
|
970
|
+
case MiniappRequestType.ACTION_RESULT:
|
|
971
|
+
this.handleActionResult(packageName, payload);
|
|
972
|
+
break;
|
|
973
|
+
// Deferred in v1
|
|
974
|
+
case MiniappRequestType.DASHBOARD_CONTENT_UPDATE:
|
|
975
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
976
|
+
code: MiniappErrorCode.NOT_IMPLEMENTED,
|
|
977
|
+
message: "Dashboard API is deferred in v1",
|
|
978
|
+
});
|
|
979
|
+
break;
|
|
980
|
+
default:
|
|
981
|
+
console.warn(`${LOG_TAG}: Unknown request type from ${packageName}: ${requestType}`);
|
|
982
|
+
break;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
// ===========================================================================
|
|
986
|
+
// Request handlers
|
|
987
|
+
// ===========================================================================
|
|
988
|
+
async handleConnect(packageName, _payload, requestId) {
|
|
989
|
+
console.log(`${LOG_TAG}: CONNECT from ${packageName}`);
|
|
990
|
+
// Register if not already
|
|
991
|
+
const existing = this.connectedApps.get(packageName);
|
|
992
|
+
if (!existing) {
|
|
993
|
+
console.warn(`${LOG_TAG}: CONNECT from unregistered app ${packageName}, ignoring`);
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
996
|
+
// Update lastPongAt so it doesn't time out right away
|
|
997
|
+
existing.lastPongAt = Date.now();
|
|
998
|
+
// Read current glasses capabilities from the settings store
|
|
999
|
+
const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable);
|
|
1000
|
+
const capabilities = getModelCapabilities(defaultWearable || DeviceTypes.NONE);
|
|
1001
|
+
// Build the declared-permission record for the SDK's session.permissions
|
|
1002
|
+
// module. Lower-cased to match v3's PermissionType union (microphone,
|
|
1003
|
+
// camera, location, notifications, calendar). Missing permissions default
|
|
1004
|
+
// to false; this is manifest-declaration tracking only — OS-grant state
|
|
1005
|
+
// is intentionally not modeled here.
|
|
1006
|
+
const declaredPermissions = computeDeclaredPermissionRecord(existing.installedManifest);
|
|
1007
|
+
// Fresh handshake → clear any auth state from a prior context so a stale
|
|
1008
|
+
// retry timer can't fire against this new connection and so the retry loop
|
|
1009
|
+
// below re-arms from scratch.
|
|
1010
|
+
existing.authDelivered = false;
|
|
1011
|
+
this.clearMiniappAuthDeliveryRetry(packageName);
|
|
1012
|
+
const authPromise = this.requestMiniappAuth(packageName);
|
|
1013
|
+
const initialAuth = await withTimeout(authPromise, 1_500);
|
|
1014
|
+
const userId = initialAuth?.mentraUserId ?? "";
|
|
1015
|
+
if (initialAuth) {
|
|
1016
|
+
existing.authDelivered = true;
|
|
1017
|
+
this.scheduleMiniappAuthRefresh(packageName, initialAuth);
|
|
1018
|
+
}
|
|
1019
|
+
this.sendToMiniapp(packageName, {
|
|
1020
|
+
type: MiniappResponseType.CONNECT_ACK,
|
|
1021
|
+
userId,
|
|
1022
|
+
packageName,
|
|
1023
|
+
capabilities,
|
|
1024
|
+
permissions: declaredPermissions,
|
|
1025
|
+
...(initialAuth ? { auth: initialAuth } : {}),
|
|
1026
|
+
}, requestId);
|
|
1027
|
+
if (!initialAuth) {
|
|
1028
|
+
// The mint timed out or (more importantly) REJECTED — the latter happens
|
|
1029
|
+
// when the cloud client hasn't finished its first-boot Core token exchange
|
|
1030
|
+
// yet, which is exactly the case when a dev miniapp is scanned/launched
|
|
1031
|
+
// right after app start. CONNECT_ACK already went out without auth; keep
|
|
1032
|
+
// trying to mint (and re-drive on cloud-connect) so the app authenticates
|
|
1033
|
+
// to its backend on its own, instead of staying dead until a full manual
|
|
1034
|
+
// Cloud V2 reconnect re-runs the whole handshake.
|
|
1035
|
+
this.deliverInitialMiniappAuth(packageName, authPromise, 0);
|
|
1036
|
+
}
|
|
1037
|
+
this.sendCloudStatusToMiniapp(packageName);
|
|
1038
|
+
// Handshake complete — unblock any launcher.waitForConnect() callers.
|
|
1039
|
+
this.handshookApps.add(packageName);
|
|
1040
|
+
this.flushConnectWaiters(packageName);
|
|
1041
|
+
}
|
|
1042
|
+
async requestMiniappAuth(packageName, opts) {
|
|
1043
|
+
const authPackageName = getDevAppSourcePackage(packageName) ?? packageName;
|
|
1044
|
+
const devAttestation = getDevAppAttestation(packageName) ?? undefined;
|
|
1045
|
+
return cloudClientService.getMiniappAuthToken(authPackageName, {
|
|
1046
|
+
...opts,
|
|
1047
|
+
...(devAttestation ? { devAttestation } : {}),
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
async handleAuthRefresh(packageName, payload, requestId) {
|
|
1051
|
+
try {
|
|
1052
|
+
const auth = await this.refreshMiniappAuth(packageName, this.authRefreshOptions(payload));
|
|
1053
|
+
if (!auth) {
|
|
1054
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1055
|
+
code: MiniappErrorCode.NOT_CONNECTED,
|
|
1056
|
+
message: "Miniapp auth is not configured",
|
|
1057
|
+
});
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
this.sendResult(packageName, requestId, true, { auth });
|
|
1061
|
+
}
|
|
1062
|
+
catch (err) {
|
|
1063
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1064
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1065
|
+
message: err?.message ?? "Miniapp auth refresh failed",
|
|
1066
|
+
});
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
async refreshMiniappAuth(packageName, opts) {
|
|
1070
|
+
const auth = await this.requestMiniappAuth(packageName, opts);
|
|
1071
|
+
if (!auth)
|
|
1072
|
+
return null;
|
|
1073
|
+
this.scheduleMiniappAuthRefresh(packageName, auth);
|
|
1074
|
+
this.sendToMiniapp(packageName, {
|
|
1075
|
+
type: MiniappResponseType.AUTH_UPDATE,
|
|
1076
|
+
auth,
|
|
1077
|
+
});
|
|
1078
|
+
return auth;
|
|
1079
|
+
}
|
|
1080
|
+
authRefreshOptions(payload) {
|
|
1081
|
+
const minTtlMs = Number(payload.minTtlMs);
|
|
1082
|
+
if (!Number.isFinite(minTtlMs) || minTtlMs <= 0)
|
|
1083
|
+
return undefined;
|
|
1084
|
+
return { minTtlMs };
|
|
1085
|
+
}
|
|
1086
|
+
scheduleMiniappAuthRefresh(packageName, auth) {
|
|
1087
|
+
const app = this.connectedApps.get(packageName);
|
|
1088
|
+
if (!app)
|
|
1089
|
+
return;
|
|
1090
|
+
this.clearMiniappAuthRefresh(packageName);
|
|
1091
|
+
const refreshAt = auth.expiresAt - MINIAPP_AUTH_REFRESH_HEADROOM_MS;
|
|
1092
|
+
const delay = Math.max(MINIAPP_AUTH_REFRESH_MIN_DELAY_MS, refreshAt - Date.now());
|
|
1093
|
+
app.authRefreshTimerId = BgTimer.setTimeout(() => {
|
|
1094
|
+
const current = this.connectedApps.get(packageName);
|
|
1095
|
+
if (!current)
|
|
1096
|
+
return;
|
|
1097
|
+
current.authRefreshTimerId = null;
|
|
1098
|
+
void this.refreshMiniappAuth(packageName).catch((err) => {
|
|
1099
|
+
console.warn(`${LOG_TAG}: miniapp auth refresh failed for ${packageName}: ${err?.message ?? err}`);
|
|
1100
|
+
});
|
|
1101
|
+
}, delay);
|
|
1102
|
+
}
|
|
1103
|
+
clearMiniappAuthRefresh(packageName) {
|
|
1104
|
+
const app = this.connectedApps.get(packageName);
|
|
1105
|
+
if (!app || app.authRefreshTimerId === null)
|
|
1106
|
+
return;
|
|
1107
|
+
const timerId = app.authRefreshTimerId;
|
|
1108
|
+
BgTimer.clearTimeout(timerId);
|
|
1109
|
+
app.authRefreshTimerId = null;
|
|
1110
|
+
}
|
|
1111
|
+
/**
|
|
1112
|
+
* Deliver the miniapp's FIRST auth token, retrying until it lands. Called from
|
|
1113
|
+
* {@link handleConnect} when the initial mint didn't resolve in the CONNECT_ACK
|
|
1114
|
+
* window. `inFlight` lets attempt 0 reuse the mint the handshake already
|
|
1115
|
+
* started (avoids a redundant duplicate request); later attempts mint fresh.
|
|
1116
|
+
*
|
|
1117
|
+
* A `null` result means auth is fundamentally unavailable (no miniappAuth hook)
|
|
1118
|
+
* — permanent, so we do NOT retry. A REJECTION means a transient failure
|
|
1119
|
+
* (typically the cloud client isn't connection-ready yet) — retry with backoff.
|
|
1120
|
+
* Either way we bail if the app has since disconnected or already got a token
|
|
1121
|
+
* (e.g. via {@link redriveMiniappAuthDelivery} on cloud-connect).
|
|
1122
|
+
*/
|
|
1123
|
+
deliverInitialMiniappAuth(packageName, inFlight, attempt) {
|
|
1124
|
+
const app = this.connectedApps.get(packageName);
|
|
1125
|
+
if (!app || app.authDelivered)
|
|
1126
|
+
return;
|
|
1127
|
+
const mint = inFlight ?? this.requestMiniappAuth(packageName);
|
|
1128
|
+
void mint
|
|
1129
|
+
.then((auth) => {
|
|
1130
|
+
const current = this.connectedApps.get(packageName);
|
|
1131
|
+
if (!current || current.authDelivered)
|
|
1132
|
+
return;
|
|
1133
|
+
if (!auth)
|
|
1134
|
+
return; // no auth hook — permanent, don't spin
|
|
1135
|
+
current.authDelivered = true;
|
|
1136
|
+
this.clearMiniappAuthDeliveryRetry(packageName);
|
|
1137
|
+
this.scheduleMiniappAuthRefresh(packageName, auth);
|
|
1138
|
+
this.sendToMiniapp(packageName, { type: MiniappResponseType.AUTH_UPDATE, auth });
|
|
1139
|
+
})
|
|
1140
|
+
.catch((err) => {
|
|
1141
|
+
console.warn(`${LOG_TAG}: miniapp auth mint failed for ${packageName} (attempt ${attempt}): ${err?.message ?? err}`);
|
|
1142
|
+
this.scheduleInitialMiniappAuthRetry(packageName, attempt);
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
1145
|
+
scheduleInitialMiniappAuthRetry(packageName, attempt) {
|
|
1146
|
+
const app = this.connectedApps.get(packageName);
|
|
1147
|
+
if (!app || app.authDelivered)
|
|
1148
|
+
return;
|
|
1149
|
+
this.clearMiniappAuthDeliveryRetry(packageName);
|
|
1150
|
+
const delay = Math.min(MINIAPP_AUTH_RETRY_MAX_MS, MINIAPP_AUTH_RETRY_BASE_MS * 2 ** attempt);
|
|
1151
|
+
app.authRetryTimerId = BgTimer.setTimeout(() => {
|
|
1152
|
+
const current = this.connectedApps.get(packageName);
|
|
1153
|
+
if (!current)
|
|
1154
|
+
return;
|
|
1155
|
+
current.authRetryTimerId = null;
|
|
1156
|
+
this.deliverInitialMiniappAuth(packageName, undefined, attempt + 1);
|
|
1157
|
+
}, delay);
|
|
1158
|
+
}
|
|
1159
|
+
clearMiniappAuthDeliveryRetry(packageName) {
|
|
1160
|
+
const app = this.connectedApps.get(packageName);
|
|
1161
|
+
if (!app || app.authRetryTimerId === null)
|
|
1162
|
+
return;
|
|
1163
|
+
BgTimer.clearTimeout(app.authRetryTimerId);
|
|
1164
|
+
app.authRetryTimerId = null;
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* When the cloud client (re)connects, immediately re-attempt the initial mint
|
|
1168
|
+
* for any handshook app still missing its token, instead of waiting out the
|
|
1169
|
+
* backoff. This is what turns "scan a dev app before the cloud is ready" from
|
|
1170
|
+
* "dead until you manually reconnect Cloud V2" into "authenticates on its own
|
|
1171
|
+
* the instant the connection comes up".
|
|
1172
|
+
*/
|
|
1173
|
+
redriveMiniappAuthDelivery() {
|
|
1174
|
+
for (const [packageName, app] of this.connectedApps) {
|
|
1175
|
+
if (app.authDelivered)
|
|
1176
|
+
continue;
|
|
1177
|
+
// Only apps that finished their CONNECT handshake are in the mint-retry
|
|
1178
|
+
// state; skip anything still mid-handshake (handleConnect owns that).
|
|
1179
|
+
if (!this.handshookApps.has(packageName))
|
|
1180
|
+
continue;
|
|
1181
|
+
this.clearMiniappAuthDeliveryRetry(packageName);
|
|
1182
|
+
this.deliverInitialMiniappAuth(packageName, undefined, 0);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
handleSubscribe(packageName, payload, requestId) {
|
|
1186
|
+
const app = this.connectedApps.get(packageName);
|
|
1187
|
+
if (!app)
|
|
1188
|
+
return;
|
|
1189
|
+
const rawStreams = (payload.subscriptions ?? payload.streams);
|
|
1190
|
+
// Normalize: objects like {stream: "location_stream", rate: "realtime"} → extract stream name
|
|
1191
|
+
// "location_stream" is the rate-bearing alias for "location_update".
|
|
1192
|
+
// The strictest rate wins when multiple `location_stream` entries appear
|
|
1193
|
+
// in one SUBSCRIBE — the same rule we apply across apps.
|
|
1194
|
+
let requestedLocationRate = null;
|
|
1195
|
+
const forceLocalTranscriptionStreams = new Set();
|
|
1196
|
+
const cloudTranscriptionStreams = new Set();
|
|
1197
|
+
const streams = rawStreams?.map((s) => {
|
|
1198
|
+
if (typeof s === "object" && s !== null) {
|
|
1199
|
+
if (s.stream === "location_stream") {
|
|
1200
|
+
if (s.rate && locationRateRank(s.rate) > locationRateRank(requestedLocationRate)) {
|
|
1201
|
+
requestedLocationRate = s.rate;
|
|
1202
|
+
}
|
|
1203
|
+
return "location_update";
|
|
1204
|
+
}
|
|
1205
|
+
if (s.forceLocal === true && s.stream.startsWith("transcription:")) {
|
|
1206
|
+
forceLocalTranscriptionStreams.add(s.stream);
|
|
1207
|
+
}
|
|
1208
|
+
if (s.stream.startsWith("transcription:") && (s.forceLocal !== true || s.includeCloud === true)) {
|
|
1209
|
+
cloudTranscriptionStreams.add(s.stream);
|
|
1210
|
+
}
|
|
1211
|
+
return s.stream;
|
|
1212
|
+
}
|
|
1213
|
+
if (s.startsWith("transcription:"))
|
|
1214
|
+
cloudTranscriptionStreams.add(s);
|
|
1215
|
+
return s;
|
|
1216
|
+
});
|
|
1217
|
+
if (!Array.isArray(streams)) {
|
|
1218
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1219
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1220
|
+
message: "subscribe requires a subscriptions array",
|
|
1221
|
+
});
|
|
1222
|
+
return;
|
|
1223
|
+
}
|
|
1224
|
+
console.log(`${LOG_TAG}: SUBSCRIBE from ${packageName}: [${streams.join(", ")}]`);
|
|
1225
|
+
// Gate each stream on the permission type its data requires. The manifest
|
|
1226
|
+
// must declare the permission (miniapp.json -> permissions) or we reject
|
|
1227
|
+
// the whole subscribe with PERMISSION_NOT_DECLARED.
|
|
1228
|
+
const declaredTypes = new Set((app.installedManifest?.permissions ?? []).map((p) => p.type?.toUpperCase()));
|
|
1229
|
+
const permissionForStream = (s) => {
|
|
1230
|
+
if (s === "audio_chunk" || s === "vad")
|
|
1231
|
+
return "MICROPHONE";
|
|
1232
|
+
if (s.startsWith("transcription") || s.startsWith("translation"))
|
|
1233
|
+
return "MICROPHONE";
|
|
1234
|
+
if (s === "location_update")
|
|
1235
|
+
return "LOCATION";
|
|
1236
|
+
if (s === "phone_notification")
|
|
1237
|
+
return "READ_NOTIFICATIONS";
|
|
1238
|
+
if (s === "phone_notification_dismissed")
|
|
1239
|
+
return "READ_NOTIFICATIONS";
|
|
1240
|
+
return null;
|
|
1241
|
+
};
|
|
1242
|
+
for (const stream of streams) {
|
|
1243
|
+
const required = permissionForStream(stream);
|
|
1244
|
+
if (required && !declaredTypes.has(required)) {
|
|
1245
|
+
logPermissionNotDeclared(packageName, required, `to subscribe to "${stream}"`, `{"type": "${required}"}`);
|
|
1246
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1247
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
1248
|
+
message: `${required} permission not declared in miniapp.json (required for "${stream}"). Add {"type": "${required}"} to the "permissions" array.`,
|
|
1249
|
+
// Extra context fields read by the SDK so authors that subscribe
|
|
1250
|
+
// to session.on("error") can format their own messages.
|
|
1251
|
+
permission: required,
|
|
1252
|
+
subscription: stream,
|
|
1253
|
+
});
|
|
1254
|
+
return;
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
// Remove old subscriptions for this app
|
|
1258
|
+
for (const oldStream of app.subscriptions) {
|
|
1259
|
+
const subs = this.streamSubscribers.get(oldStream);
|
|
1260
|
+
if (subs) {
|
|
1261
|
+
subs.delete(packageName);
|
|
1262
|
+
if (subs.size === 0) {
|
|
1263
|
+
this.streamSubscribers.delete(oldStream);
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
// Add new subscriptions
|
|
1268
|
+
app.subscriptions = new Set(streams);
|
|
1269
|
+
app.forceLocalTranscriptionStreams = new Set([...forceLocalTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)));
|
|
1270
|
+
app.cloudTranscriptionStreams = new Set([...cloudTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)));
|
|
1271
|
+
for (const stream of streams) {
|
|
1272
|
+
let subs = this.streamSubscribers.get(stream);
|
|
1273
|
+
if (!subs) {
|
|
1274
|
+
subs = new Set();
|
|
1275
|
+
this.streamSubscribers.set(stream, subs);
|
|
1276
|
+
}
|
|
1277
|
+
subs.add(packageName);
|
|
1278
|
+
}
|
|
1279
|
+
this.recomputeMicRequirements();
|
|
1280
|
+
this.updateCloudSubscriptions();
|
|
1281
|
+
this.recomputeHeadingSubscription();
|
|
1282
|
+
this.recomputeImuSubscription();
|
|
1283
|
+
// Persist this app's requested rate (or clear it if SUBSCRIBE didn't
|
|
1284
|
+
// include `location_stream` this time), then ask the host for the
|
|
1285
|
+
// strictest rate across all connected apps.
|
|
1286
|
+
app.requestedLocationRate = requestedLocationRate;
|
|
1287
|
+
this.recomputeLocationTier();
|
|
1288
|
+
this.sendResult(packageName, requestId, true);
|
|
1289
|
+
// Fire initial snapshot values for stateful streams so miniapps don't have
|
|
1290
|
+
// to wait for the first change event.
|
|
1291
|
+
this.emitInitialSnapshots(packageName, streams);
|
|
1292
|
+
}
|
|
1293
|
+
async handleCalendarListEvents(packageName, payload, requestId) {
|
|
1294
|
+
const app = this.connectedApps.get(packageName);
|
|
1295
|
+
const declared = app?.installedManifest?.permissions?.some((permission) => permission.type?.toUpperCase() === "CALENDAR");
|
|
1296
|
+
if (!declared) {
|
|
1297
|
+
logPermissionNotDeclared(packageName, "CALENDAR", "to list calendar events", '{"type": "CALENDAR"}');
|
|
1298
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1299
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
1300
|
+
message: 'CALENDAR permission not declared in miniapp.json. Add {"type": "CALENDAR"} to the permissions array.',
|
|
1301
|
+
permission: "CALENDAR",
|
|
1302
|
+
});
|
|
1303
|
+
return;
|
|
1304
|
+
}
|
|
1305
|
+
try {
|
|
1306
|
+
const result = await listPhoneCalendarEvents(payload);
|
|
1307
|
+
this.sendResult(packageName, requestId, true, result);
|
|
1308
|
+
}
|
|
1309
|
+
catch (error) {
|
|
1310
|
+
const known = error instanceof PhoneCalendarError;
|
|
1311
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1312
|
+
code: known ? error.code : MiniappErrorCode.INTERNAL,
|
|
1313
|
+
message: error instanceof Error ? error.message : "Failed to read phone calendar events",
|
|
1314
|
+
});
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
/**
|
|
1318
|
+
* For state-bearing streams (battery, connection), deliver the current value
|
|
1319
|
+
* immediately on subscribe. Other streams (button press, transcription, etc.)
|
|
1320
|
+
* are pure event streams with no "current value" to snapshot.
|
|
1321
|
+
*/
|
|
1322
|
+
emitInitialSnapshots(packageName, streams) {
|
|
1323
|
+
// Read the engine glasses store directly (was a host glassesStatus hook).
|
|
1324
|
+
const gs = useGlassesStore.getState();
|
|
1325
|
+
const glassesState = {
|
|
1326
|
+
connected: isGlassesConnected(gs.connection),
|
|
1327
|
+
deviceModel: gs.deviceModel,
|
|
1328
|
+
batteryLevel: gs.batteryLevel,
|
|
1329
|
+
charging: gs.charging,
|
|
1330
|
+
};
|
|
1331
|
+
const isSimulated = (glassesState.deviceModel || "").toLowerCase().includes("simulated");
|
|
1332
|
+
for (const stream of streams) {
|
|
1333
|
+
if (stream === "glasses_battery") {
|
|
1334
|
+
// Simulated glasses have no real battery; mirror the phone's battery
|
|
1335
|
+
// so miniapps see a sensible value during development.
|
|
1336
|
+
if (isSimulated) {
|
|
1337
|
+
void this.emitPhoneBatteryAs(packageName, "glasses_battery");
|
|
1338
|
+
}
|
|
1339
|
+
else if (typeof glassesState.batteryLevel === "number" && glassesState.batteryLevel >= 0) {
|
|
1340
|
+
this.sendToMiniapp(packageName, {
|
|
1341
|
+
type: MiniappResponseType.EVENT,
|
|
1342
|
+
streamType: "glasses_battery",
|
|
1343
|
+
data: {
|
|
1344
|
+
level: glassesState.batteryLevel,
|
|
1345
|
+
charging: !!glassesState.charging,
|
|
1346
|
+
timestamp: Date.now(),
|
|
1347
|
+
},
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
else if (stream === "phone_battery") {
|
|
1352
|
+
void this.emitPhoneBatteryAs(packageName, "phone_battery");
|
|
1353
|
+
}
|
|
1354
|
+
else if (stream === "glasses_connection") {
|
|
1355
|
+
this.sendToMiniapp(packageName, {
|
|
1356
|
+
type: MiniappResponseType.EVENT,
|
|
1357
|
+
streamType: "glasses_connection",
|
|
1358
|
+
data: glassesState,
|
|
1359
|
+
});
|
|
1360
|
+
}
|
|
1361
|
+
else if (stream === "glasses_wifi") {
|
|
1362
|
+
// Snapshot the current glasses Wi-Fi state on subscribe (like battery).
|
|
1363
|
+
// Read the canonical nested `wifi: {state, ssid}` from the STORE state
|
|
1364
|
+
// (`glassesState` is the narrowed 4-field projection and has no `wifi`;
|
|
1365
|
+
// the pre-engine host hook spread the full store, so read gs directly —
|
|
1366
|
+
// NOT the legacy flat wifiConnected). Effective connectivity requires the
|
|
1367
|
+
// glasses connected AND on Wi-Fi — mirrors the host's store-derived
|
|
1368
|
+
// forward, so an already-disconnected device reports `connected: false`
|
|
1369
|
+
// rather than silently emitting nothing. Always emits so onWifi gets an
|
|
1370
|
+
// initial value.
|
|
1371
|
+
const wifi = gs.wifi.state === "connected" ? gs.wifi : null;
|
|
1372
|
+
const connected = glassesState.connected === true && wifi !== null;
|
|
1373
|
+
this.sendToMiniapp(packageName, {
|
|
1374
|
+
type: MiniappResponseType.EVENT,
|
|
1375
|
+
streamType: "glasses_wifi",
|
|
1376
|
+
data: {
|
|
1377
|
+
connected,
|
|
1378
|
+
ssid: connected ? wifi?.ssid : undefined,
|
|
1379
|
+
localIp: connected ? wifi?.localIp : undefined,
|
|
1380
|
+
timestamp: Date.now(),
|
|
1381
|
+
},
|
|
1382
|
+
});
|
|
1383
|
+
}
|
|
1384
|
+
else if (stream === "head_position") {
|
|
1385
|
+
// The glasses store has no headUp field (same on the pre-engine host
|
|
1386
|
+
// hook), so this stays a no-emit until head state lands in the store.
|
|
1387
|
+
const headUp = gs.headUp;
|
|
1388
|
+
if (typeof headUp === "boolean") {
|
|
1389
|
+
this.sendToMiniapp(packageName, {
|
|
1390
|
+
type: MiniappResponseType.EVENT,
|
|
1391
|
+
streamType: "head_position",
|
|
1392
|
+
data: {
|
|
1393
|
+
position: headUp ? "up" : "down",
|
|
1394
|
+
timestamp: Date.now(),
|
|
1395
|
+
},
|
|
1396
|
+
});
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
/**
|
|
1402
|
+
* Read the phone's battery state right now and emit it as the given stream.
|
|
1403
|
+
* Used for both phone_battery snapshot on subscribe, and as a stand-in for
|
|
1404
|
+
* glasses_battery when connected to Simulated Glasses.
|
|
1405
|
+
*/
|
|
1406
|
+
async emitPhoneBatteryAs(packageName, streamType) {
|
|
1407
|
+
try {
|
|
1408
|
+
const level = await Battery.getBatteryLevelAsync();
|
|
1409
|
+
const state = await Battery.getBatteryStateAsync();
|
|
1410
|
+
const charging = state === Battery.BatteryState.CHARGING || state === Battery.BatteryState.FULL;
|
|
1411
|
+
this.sendToMiniapp(packageName, {
|
|
1412
|
+
type: MiniappResponseType.EVENT,
|
|
1413
|
+
streamType,
|
|
1414
|
+
data: {
|
|
1415
|
+
level: Math.round(level * 100),
|
|
1416
|
+
charging,
|
|
1417
|
+
timestamp: Date.now(),
|
|
1418
|
+
},
|
|
1419
|
+
});
|
|
1420
|
+
}
|
|
1421
|
+
catch (err) {
|
|
1422
|
+
console.log(`${LOG_TAG}: phone battery snapshot failed`, err);
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
handleDisplay(packageName, payload, requestId) {
|
|
1426
|
+
try {
|
|
1427
|
+
// Zombie-frame gate: a dying miniapp's last in-flight frame can arrive
|
|
1428
|
+
// AFTER onUnmount cleared its display — repainting a dead app's UI (and
|
|
1429
|
+
// poisoning the next boot's pre-boot restore snapshot). Killed mid-route
|
|
1430
|
+
// nav reproduced this reliably; its 3s re-push raced the teardown.
|
|
1431
|
+
if (!this.connectedApps.has(packageName)) {
|
|
1432
|
+
console.log(`${LOG_TAG}: dropping display from unmounted app ${packageName}`);
|
|
1433
|
+
return;
|
|
1434
|
+
}
|
|
1435
|
+
if (!payload.layout || typeof payload.layout !== "object") {
|
|
1436
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1437
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1438
|
+
message: "display request missing layout object",
|
|
1439
|
+
});
|
|
1440
|
+
return;
|
|
1441
|
+
}
|
|
1442
|
+
// Hand off to LocalDisplayManager — it owns boot/throttle/arbitration/
|
|
1443
|
+
// expiry + the native BluetoothSdk.displayEvent call + useDisplayStore.
|
|
1444
|
+
localDisplayManager.request(packageName, {
|
|
1445
|
+
view: payload.view ?? "main",
|
|
1446
|
+
layout: payload.layout,
|
|
1447
|
+
durationMs: payload.durationMs,
|
|
1448
|
+
});
|
|
1449
|
+
this.sendResult(packageName, requestId, true);
|
|
1450
|
+
}
|
|
1451
|
+
catch (err) {
|
|
1452
|
+
console.error(`${LOG_TAG}: display error:`, err);
|
|
1453
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1454
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1455
|
+
message: err instanceof Error ? err.message : "Display error",
|
|
1456
|
+
});
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
/**
|
|
1460
|
+
* Scene render (session.display.render()). Same arbitration as DISPLAY, but
|
|
1461
|
+
* the payload is a whole frame of positioned elements and the caller may be
|
|
1462
|
+
* awaiting the outcome: LocalDisplayManager guarantees the resolver fires
|
|
1463
|
+
* exactly once (displayed/blocked, with degraded/dropped reporting), and that
|
|
1464
|
+
* becomes the REQUEST_RESULT data.
|
|
1465
|
+
*/
|
|
1466
|
+
handleRender(packageName, payload, requestId) {
|
|
1467
|
+
try {
|
|
1468
|
+
// Zombie-frame gate — see handleDisplay.
|
|
1469
|
+
if (!this.connectedApps.has(packageName)) {
|
|
1470
|
+
this.sendResult(packageName, requestId, true, { status: "blocked", reason: "app stopped" });
|
|
1471
|
+
return;
|
|
1472
|
+
}
|
|
1473
|
+
if (!Array.isArray(payload.elements)) {
|
|
1474
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1475
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1476
|
+
message: "render request missing elements array",
|
|
1477
|
+
});
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
localDisplayManager.request(packageName, {
|
|
1481
|
+
view: payload.view ?? "main",
|
|
1482
|
+
scene: payload.elements,
|
|
1483
|
+
durationMs: payload.durationMs,
|
|
1484
|
+
}, (result) => this.sendResult(packageName, requestId, true, result));
|
|
1485
|
+
}
|
|
1486
|
+
catch (err) {
|
|
1487
|
+
console.error(`${LOG_TAG}: render error:`, err);
|
|
1488
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1489
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1490
|
+
message: err instanceof Error ? err.message : "Render error",
|
|
1491
|
+
});
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
handlePlayAudio(packageName, payload, requestId) {
|
|
1495
|
+
const audioUrl = (payload.audioUrl ?? payload.url);
|
|
1496
|
+
if (!audioUrl) {
|
|
1497
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1498
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1499
|
+
message: "play_audio requires an audioUrl",
|
|
1500
|
+
});
|
|
1501
|
+
return;
|
|
1502
|
+
}
|
|
1503
|
+
const audioRequestId = requestId || `local_${Date.now()}`;
|
|
1504
|
+
const volume = typeof payload.volume === "number" ? payload.volume : 1.0;
|
|
1505
|
+
const stopOtherAudio = payload.stopOtherAudio !== false;
|
|
1506
|
+
this.cancelSpeech(packageName);
|
|
1507
|
+
this.setSpeakerState(packageName, "loading");
|
|
1508
|
+
audioPlaybackService.play({ requestId: audioRequestId, audioUrl, appId: packageName, volume, stopOtherAudio }, (_respId, success, error, duration) => {
|
|
1509
|
+
if (success) {
|
|
1510
|
+
this.setSpeakerState(packageName, "stopped", { durationMs: duration ?? undefined });
|
|
1511
|
+
}
|
|
1512
|
+
else {
|
|
1513
|
+
this.setSpeakerState(packageName, "error", {
|
|
1514
|
+
errorCode: MiniappErrorCode.INTERNAL,
|
|
1515
|
+
errorMessage: error ?? "play failed",
|
|
1516
|
+
durationMs: duration ?? undefined,
|
|
1517
|
+
});
|
|
1518
|
+
}
|
|
1519
|
+
this.sendResult(packageName, requestId, success, { duration }, error
|
|
1520
|
+
? {
|
|
1521
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1522
|
+
message: error,
|
|
1523
|
+
}
|
|
1524
|
+
: undefined);
|
|
1525
|
+
});
|
|
1526
|
+
// Optimistic "started playing" transition. The audio service doesn't
|
|
1527
|
+
// have a "playback actually started" callback today; cloud SDK v3 has
|
|
1528
|
+
// the same constraint and uses optimistic timing. Microtask so the
|
|
1529
|
+
// initial "loading" envelope flushes first.
|
|
1530
|
+
queueMicrotask(() => this.setSpeakerState(packageName, "playing"));
|
|
1531
|
+
}
|
|
1532
|
+
handleStopAudio(packageName, _payload, requestId) {
|
|
1533
|
+
this.cancelSpeech(packageName);
|
|
1534
|
+
audioPlaybackService.stopForApp(packageName);
|
|
1535
|
+
this.setSpeakerState(packageName, "stopped");
|
|
1536
|
+
this.sendResult(packageName, requestId, true);
|
|
1537
|
+
}
|
|
1538
|
+
// ── speaker.createStream (live PCM output) ────────────────────────────────
|
|
1539
|
+
/** Valid PCM sample rates for speaker streams. Mirrors the SDK contract. */
|
|
1540
|
+
static SPEAKER_STREAM_SAMPLE_RATES = new Set([16000, 24000, 48000]);
|
|
1541
|
+
static SPEAKER_STREAM_MAX_BASE64_CHARS = Math.ceil((256 * 1024) / 3) * 4;
|
|
1542
|
+
speakerStreamCounter = 0;
|
|
1543
|
+
async handleSpeakerStreamOpen(packageName, payload, requestId) {
|
|
1544
|
+
const app = this.connectedApps.get(packageName);
|
|
1545
|
+
if (!app)
|
|
1546
|
+
return;
|
|
1547
|
+
// One stream per miniapp: opening a second closes the first.
|
|
1548
|
+
const previous = app.activeSpeakerStreamId;
|
|
1549
|
+
if (previous) {
|
|
1550
|
+
app.activeSpeakerStreamId = undefined;
|
|
1551
|
+
await audioPlaybackService.abortStream(previous).catch((error) => {
|
|
1552
|
+
console.warn(`${LOG_TAG}: failed to abort previous speaker stream for ${packageName}`, error);
|
|
1553
|
+
});
|
|
1554
|
+
}
|
|
1555
|
+
const sampleRate = typeof payload.sampleRate === "number" ? payload.sampleRate : 16000;
|
|
1556
|
+
if (!LocalMiniappRuntime.SPEAKER_STREAM_SAMPLE_RATES.has(sampleRate)) {
|
|
1557
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1558
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
1559
|
+
message: `unsupported sampleRate ${sampleRate} (use 16000, 24000, or 48000)`,
|
|
1560
|
+
});
|
|
1561
|
+
return;
|
|
1562
|
+
}
|
|
1563
|
+
const channels = typeof payload.channels === "number" ? payload.channels : 1;
|
|
1564
|
+
if (channels !== 1) {
|
|
1565
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1566
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
1567
|
+
message: "speaker streams currently support mono PCM only",
|
|
1568
|
+
});
|
|
1569
|
+
return;
|
|
1570
|
+
}
|
|
1571
|
+
const volume = typeof payload.volume === "number" ? payload.volume : 1;
|
|
1572
|
+
if (!Number.isFinite(volume) || volume < 0 || volume > 1) {
|
|
1573
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1574
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
1575
|
+
message: "speaker stream volume must be between 0 and 1",
|
|
1576
|
+
});
|
|
1577
|
+
return;
|
|
1578
|
+
}
|
|
1579
|
+
const streamId = `spkstream_${Date.now()}_${++this.speakerStreamCounter}`;
|
|
1580
|
+
this.setSpeakerState(packageName, "loading");
|
|
1581
|
+
try {
|
|
1582
|
+
await audioPlaybackService.openStream({
|
|
1583
|
+
streamId,
|
|
1584
|
+
appId: packageName,
|
|
1585
|
+
sampleRate,
|
|
1586
|
+
channels,
|
|
1587
|
+
volume,
|
|
1588
|
+
stopOtherAudio: payload.stopOtherAudio !== false,
|
|
1589
|
+
onEnded: (endedId, success, error, durationMs) => {
|
|
1590
|
+
const current = this.connectedApps.get(packageName);
|
|
1591
|
+
// Ignore completion from an app incarnation replaced by a crash
|
|
1592
|
+
// respawn. Its abort may finish after the new app has registered and
|
|
1593
|
+
// must not overwrite that app's speaker state.
|
|
1594
|
+
if (current !== app)
|
|
1595
|
+
return;
|
|
1596
|
+
if (current.activeSpeakerStreamId === endedId)
|
|
1597
|
+
current.activeSpeakerStreamId = undefined;
|
|
1598
|
+
if (success) {
|
|
1599
|
+
this.setSpeakerState(packageName, "stopped", { durationMs: durationMs ?? undefined });
|
|
1600
|
+
}
|
|
1601
|
+
else {
|
|
1602
|
+
this.setSpeakerState(packageName, "error", {
|
|
1603
|
+
errorCode: MiniappErrorCode.INTERNAL,
|
|
1604
|
+
errorMessage: error ?? "speaker stream failed",
|
|
1605
|
+
durationMs: durationMs ?? undefined,
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1608
|
+
},
|
|
1609
|
+
});
|
|
1610
|
+
}
|
|
1611
|
+
catch (error) {
|
|
1612
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1613
|
+
this.setSpeakerState(packageName, "error", {
|
|
1614
|
+
errorCode: MiniappErrorCode.INTERNAL,
|
|
1615
|
+
errorMessage: message,
|
|
1616
|
+
});
|
|
1617
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1618
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1619
|
+
message,
|
|
1620
|
+
});
|
|
1621
|
+
return;
|
|
1622
|
+
}
|
|
1623
|
+
// The native open can outlive a miniapp disconnect. Its ordinary teardown
|
|
1624
|
+
// ran while no StreamState existed yet, so close this late result here
|
|
1625
|
+
// instead of leaving an orphan AudioTrack/AVAudioEngine session.
|
|
1626
|
+
if (this.connectedApps.get(packageName) !== app) {
|
|
1627
|
+
await audioPlaybackService.abortStream(streamId);
|
|
1628
|
+
return;
|
|
1629
|
+
}
|
|
1630
|
+
app.activeSpeakerStreamId = streamId;
|
|
1631
|
+
// Optimistic "playing", same as play(): the chunk player starts as soon
|
|
1632
|
+
// as the first write lands. Microtask so "loading" flushes first.
|
|
1633
|
+
queueMicrotask(() => this.setSpeakerState(packageName, "playing"));
|
|
1634
|
+
this.sendResult(packageName, requestId, true, { streamId });
|
|
1635
|
+
}
|
|
1636
|
+
/** Resolve + validate a stream id from a payload against the app's active stream. */
|
|
1637
|
+
activeSpeakerStream(packageName, payload) {
|
|
1638
|
+
const streamId = typeof payload.streamId === "string" ? payload.streamId : undefined;
|
|
1639
|
+
const app = this.connectedApps.get(packageName);
|
|
1640
|
+
if (!streamId || !app || app.activeSpeakerStreamId !== streamId)
|
|
1641
|
+
return null;
|
|
1642
|
+
return streamId;
|
|
1643
|
+
}
|
|
1644
|
+
async handleSpeakerStreamWrite(packageName, payload, requestId) {
|
|
1645
|
+
const streamId = this.activeSpeakerStream(packageName, payload);
|
|
1646
|
+
if (!streamId) {
|
|
1647
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1648
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1649
|
+
message: "speaker stream is not open",
|
|
1650
|
+
});
|
|
1651
|
+
return;
|
|
1652
|
+
}
|
|
1653
|
+
const base64 = typeof payload.base64 === "string" ? payload.base64 : "";
|
|
1654
|
+
if (!base64 || base64.length % 4 !== 0) {
|
|
1655
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1656
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
1657
|
+
message: "speaker stream write requires padded base64 PCM data",
|
|
1658
|
+
});
|
|
1659
|
+
return;
|
|
1660
|
+
}
|
|
1661
|
+
if (base64.length > LocalMiniappRuntime.SPEAKER_STREAM_MAX_BASE64_CHARS) {
|
|
1662
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1663
|
+
code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
|
|
1664
|
+
message: "speaker stream chunk exceeds the 256 KB raw PCM limit",
|
|
1665
|
+
});
|
|
1666
|
+
return;
|
|
1667
|
+
}
|
|
1668
|
+
try {
|
|
1669
|
+
const result = await audioPlaybackService.writeStreamChunk(streamId, base64);
|
|
1670
|
+
this.sendResult(packageName, requestId, true, result);
|
|
1671
|
+
}
|
|
1672
|
+
catch (error) {
|
|
1673
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1674
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1675
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1676
|
+
});
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
async handleSpeakerStreamClose(packageName, payload, requestId) {
|
|
1680
|
+
const streamId = this.activeSpeakerStream(packageName, payload);
|
|
1681
|
+
if (!streamId) {
|
|
1682
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1683
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1684
|
+
message: "speaker stream is not open",
|
|
1685
|
+
});
|
|
1686
|
+
return;
|
|
1687
|
+
}
|
|
1688
|
+
try {
|
|
1689
|
+
// Resolves once the buffered audio has drained; onEnded delivers the
|
|
1690
|
+
// terminal SPEAKER_STATE ("stopped").
|
|
1691
|
+
const result = await audioPlaybackService.closeStream(streamId);
|
|
1692
|
+
this.sendResult(packageName, requestId, true, result);
|
|
1693
|
+
}
|
|
1694
|
+
catch (error) {
|
|
1695
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1696
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1697
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1698
|
+
});
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
async handleSpeakerStreamAbort(packageName, payload, requestId) {
|
|
1702
|
+
const streamId = this.activeSpeakerStream(packageName, payload);
|
|
1703
|
+
if (!streamId) {
|
|
1704
|
+
// Aborting an already-gone stream is fine — abort() is idempotent.
|
|
1705
|
+
this.sendResult(packageName, requestId, true);
|
|
1706
|
+
return;
|
|
1707
|
+
}
|
|
1708
|
+
try {
|
|
1709
|
+
await audioPlaybackService.abortStream(streamId);
|
|
1710
|
+
this.sendResult(packageName, requestId, true);
|
|
1711
|
+
}
|
|
1712
|
+
catch (error) {
|
|
1713
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1714
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1715
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
async handleSpeak(packageName, payload, requestId) {
|
|
1720
|
+
const rawText = payload.text;
|
|
1721
|
+
const sentences = typeof rawText === "string"
|
|
1722
|
+
? [rawText.trim()].filter(Boolean)
|
|
1723
|
+
: Array.isArray(rawText)
|
|
1724
|
+
? rawText.filter((sentence) => typeof sentence === "string").map((sentence) => sentence.trim()).filter(Boolean)
|
|
1725
|
+
: [];
|
|
1726
|
+
if (sentences.length === 0) {
|
|
1727
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1728
|
+
code: MiniappErrorCode.INTERNAL,
|
|
1729
|
+
message: "speak requires text or a non-empty sentence list",
|
|
1730
|
+
});
|
|
1731
|
+
return;
|
|
1732
|
+
}
|
|
1733
|
+
const cloudText = sentences.join(" ");
|
|
1734
|
+
const voice = (payload.voice_id ?? payload.voice) || "default";
|
|
1735
|
+
const audioRequestId = requestId || `tts_${Date.now()}`;
|
|
1736
|
+
const volume = typeof payload.volume === "number" ? payload.volume : 1.0;
|
|
1737
|
+
const stopOtherAudio = payload.stopOtherAudio !== false;
|
|
1738
|
+
const voiceSettings = payload.voice_settings && typeof payload.voice_settings === "object"
|
|
1739
|
+
? payload.voice_settings
|
|
1740
|
+
: undefined;
|
|
1741
|
+
const speed = typeof voiceSettings?.speed === "number" ? voiceSettings.speed : undefined;
|
|
1742
|
+
const run = { id: audioRequestId, cancelled: false };
|
|
1743
|
+
this.cancelSpeech(packageName);
|
|
1744
|
+
this.speechRuns.set(packageName, run);
|
|
1745
|
+
this.setSpeakerState(packageName, "loading");
|
|
1746
|
+
let terminalSent = false;
|
|
1747
|
+
const sendPlaybackResult = (success, error, duration, fallbackErrorMessage, completed = success, errorCode = MiniappErrorCode.TTS_UPSTREAM_ERROR) => {
|
|
1748
|
+
if (terminalSent)
|
|
1749
|
+
return;
|
|
1750
|
+
terminalSent = true;
|
|
1751
|
+
// A superseded run still owns its bridge request, but it no longer owns
|
|
1752
|
+
// the package's shared speaker state.
|
|
1753
|
+
const ownsSpeakerState = this.speechRuns.get(packageName) === run;
|
|
1754
|
+
if (ownsSpeakerState)
|
|
1755
|
+
this.speechRuns.delete(packageName);
|
|
1756
|
+
if (ownsSpeakerState) {
|
|
1757
|
+
if (success) {
|
|
1758
|
+
this.setSpeakerState(packageName, "stopped", { durationMs: duration ?? undefined });
|
|
1759
|
+
}
|
|
1760
|
+
else {
|
|
1761
|
+
this.setSpeakerState(packageName, "error", {
|
|
1762
|
+
errorCode,
|
|
1763
|
+
errorMessage: error ?? fallbackErrorMessage,
|
|
1764
|
+
durationMs: duration ?? undefined,
|
|
1765
|
+
});
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
this.sendResult(packageName, requestId, success, { completed, duration }, error
|
|
1769
|
+
? {
|
|
1770
|
+
code: errorCode,
|
|
1771
|
+
message: error,
|
|
1772
|
+
}
|
|
1773
|
+
: undefined);
|
|
1774
|
+
};
|
|
1775
|
+
run.onCancelled = () => sendPlaybackResult(true, null, null, "tts playback stopped", false);
|
|
1776
|
+
try {
|
|
1777
|
+
const voiceExplicit = payload.voice_id !== undefined || payload.voice !== undefined;
|
|
1778
|
+
const offlineSupportsVoice = !voiceExplicit || voice === "default" || ttsModelManager.getAvailableLanguages().some((l) => l.code === voice);
|
|
1779
|
+
const modelId = typeof payload.model_id === "string" ? payload.model_id : undefined;
|
|
1780
|
+
const forceLocal = payload.forceLocal === true;
|
|
1781
|
+
const cloudConnected = cloudClientService.isConnected();
|
|
1782
|
+
console.log(`${LOG_TAG}: TTS decision for ${packageName}: cloudConnected=${cloudConnected}, runtimeTts=yes, offlineVoice=${offlineSupportsVoice}, forceLocal=${forceLocal}`);
|
|
1783
|
+
const playOfflineTts = async (reason) => {
|
|
1784
|
+
if (run.cancelled)
|
|
1785
|
+
return true;
|
|
1786
|
+
if (!offlineSupportsVoice || !(await ttsModelManager.isModelAvailable())) {
|
|
1787
|
+
return false;
|
|
1788
|
+
}
|
|
1789
|
+
if (run.cancelled)
|
|
1790
|
+
return true;
|
|
1791
|
+
const languageCode = ttsModelManager.getAvailableLanguages().some((l) => l.code === voice) ? voice : undefined;
|
|
1792
|
+
const offlineSentences = sentences;
|
|
1793
|
+
if (offlineSentences.length > 1) {
|
|
1794
|
+
let firstSentence;
|
|
1795
|
+
try {
|
|
1796
|
+
firstSentence = await ttsModelManager.synthesizeToFile(offlineSentences[0], { languageCode, speed });
|
|
1797
|
+
}
|
|
1798
|
+
catch (offlineErr) {
|
|
1799
|
+
if (run.cancelled)
|
|
1800
|
+
return true;
|
|
1801
|
+
console.warn(`${LOG_TAG}: offline sentence TTS synthesize failed${reason ? ` after ${reason}` : ""}:`, offlineErr);
|
|
1802
|
+
return false;
|
|
1803
|
+
}
|
|
1804
|
+
if (run.cancelled) {
|
|
1805
|
+
await Promise.resolve(firstSentence.cleanup?.());
|
|
1806
|
+
return true;
|
|
1807
|
+
}
|
|
1808
|
+
void runSentenceTtsPipeline({
|
|
1809
|
+
sentences: offlineSentences,
|
|
1810
|
+
firstAudio: firstSentence,
|
|
1811
|
+
run,
|
|
1812
|
+
synthesize: (sentence) => ttsModelManager.synthesizeToFile(sentence, { languageCode, speed }),
|
|
1813
|
+
play: (audio, index) => new Promise((resolve) => {
|
|
1814
|
+
const sentenceRequestId = `${audioRequestId}_sentence_${index}`;
|
|
1815
|
+
run.playbackRequestId = sentenceRequestId;
|
|
1816
|
+
audioPlaybackService.play({
|
|
1817
|
+
requestId: sentenceRequestId,
|
|
1818
|
+
audioUrl: audio.audioUrl,
|
|
1819
|
+
appId: packageName,
|
|
1820
|
+
volume,
|
|
1821
|
+
stopOtherAudio,
|
|
1822
|
+
suppressCloudUplink: true,
|
|
1823
|
+
}, (_responseId, success, error, duration, completionReason) => {
|
|
1824
|
+
if (run.playbackRequestId === sentenceRequestId)
|
|
1825
|
+
run.playbackRequestId = undefined;
|
|
1826
|
+
resolve({ success, completed: completionReason === "completed", error, duration });
|
|
1827
|
+
});
|
|
1828
|
+
}),
|
|
1829
|
+
onCleanupError: (cleanupError) => {
|
|
1830
|
+
console.warn(`${LOG_TAG}: sentence TTS cleanup failed`, cleanupError);
|
|
1831
|
+
},
|
|
1832
|
+
}).then(({ success, completed, error, duration }) => {
|
|
1833
|
+
sendPlaybackResult(success, error, duration, "offline sentence tts playback failed", completed);
|
|
1834
|
+
}, (pipelineError) => {
|
|
1835
|
+
const message = pipelineError instanceof Error ? pipelineError.message : String(pipelineError);
|
|
1836
|
+
sendPlaybackResult(false, message, null, "offline sentence tts playback failed", false);
|
|
1837
|
+
});
|
|
1838
|
+
queueMicrotask(() => {
|
|
1839
|
+
if (this.speechRuns.get(packageName) === run)
|
|
1840
|
+
this.setSpeakerState(packageName, "playing");
|
|
1841
|
+
});
|
|
1842
|
+
return true;
|
|
1843
|
+
}
|
|
1844
|
+
let offlineGenerated;
|
|
1845
|
+
try {
|
|
1846
|
+
offlineGenerated = await ttsModelManager.synthesizeToFile(offlineSentences[0], { languageCode, speed });
|
|
1847
|
+
}
|
|
1848
|
+
catch (offlineErr) {
|
|
1849
|
+
if (run.cancelled)
|
|
1850
|
+
return true;
|
|
1851
|
+
console.warn(`${LOG_TAG}: offline TTS synthesize failed${reason ? ` after ${reason}` : ""}:`, offlineErr);
|
|
1852
|
+
return false;
|
|
1853
|
+
}
|
|
1854
|
+
const generated = offlineGenerated;
|
|
1855
|
+
if (run.cancelled) {
|
|
1856
|
+
await Promise.resolve(generated.cleanup?.());
|
|
1857
|
+
return true;
|
|
1858
|
+
}
|
|
1859
|
+
run.playbackRequestId = audioRequestId;
|
|
1860
|
+
audioPlaybackService.play({
|
|
1861
|
+
requestId: audioRequestId,
|
|
1862
|
+
audioUrl: generated.audioUrl,
|
|
1863
|
+
appId: packageName,
|
|
1864
|
+
volume,
|
|
1865
|
+
stopOtherAudio,
|
|
1866
|
+
suppressCloudUplink: true,
|
|
1867
|
+
}, (_respId, success, error, duration, completionReason) => {
|
|
1868
|
+
if (run.playbackRequestId === audioRequestId)
|
|
1869
|
+
run.playbackRequestId = undefined;
|
|
1870
|
+
void Promise.resolve(generated.cleanup?.()).catch((cleanupError) => {
|
|
1871
|
+
console.warn(`${LOG_TAG}: offline TTS cleanup failed`, cleanupError);
|
|
1872
|
+
});
|
|
1873
|
+
sendPlaybackResult(success, error, duration, "offline tts playback failed", completionReason === "completed");
|
|
1874
|
+
});
|
|
1875
|
+
queueMicrotask(() => {
|
|
1876
|
+
if (this.speechRuns.get(packageName) === run)
|
|
1877
|
+
this.setSpeakerState(packageName, "playing");
|
|
1878
|
+
});
|
|
1879
|
+
return true;
|
|
1880
|
+
};
|
|
1881
|
+
const playCloudTts = async (fallbackToOffline) => {
|
|
1882
|
+
if (run.cancelled)
|
|
1883
|
+
return true;
|
|
1884
|
+
let source;
|
|
1885
|
+
try {
|
|
1886
|
+
source = await cloudClientService.tts.speak(cloudText, {
|
|
1887
|
+
...(voiceExplicit && voice !== "default" ? { voice_id: voice } : {}),
|
|
1888
|
+
...(modelId ? { model_id: modelId } : {}),
|
|
1889
|
+
...(voiceSettings ? { voice_settings: voiceSettings } : {}),
|
|
1890
|
+
});
|
|
1891
|
+
}
|
|
1892
|
+
catch (cloudErr) {
|
|
1893
|
+
if (run.cancelled)
|
|
1894
|
+
return true;
|
|
1895
|
+
const error = cloudErr instanceof Error ? cloudErr.message : String(cloudErr);
|
|
1896
|
+
console.warn(`${LOG_TAG}: cloud TTS source failed: ${error}`);
|
|
1897
|
+
if (fallbackToOffline && (await playOfflineTts("cloud tts source failed"))) {
|
|
1898
|
+
return true;
|
|
1899
|
+
}
|
|
1900
|
+
sendPlaybackResult(false, error, null, "tts failed");
|
|
1901
|
+
return true;
|
|
1902
|
+
}
|
|
1903
|
+
if (run.cancelled)
|
|
1904
|
+
return true;
|
|
1905
|
+
run.playbackRequestId = audioRequestId;
|
|
1906
|
+
await Promise.resolve(audioPlaybackService.play({
|
|
1907
|
+
requestId: audioRequestId,
|
|
1908
|
+
audioUrl: source.audioUrl,
|
|
1909
|
+
appId: packageName,
|
|
1910
|
+
volume,
|
|
1911
|
+
stopOtherAudio,
|
|
1912
|
+
suppressCloudUplink: true,
|
|
1913
|
+
}, (_respId, success, error, duration, completionReason) => {
|
|
1914
|
+
if (run.playbackRequestId === audioRequestId)
|
|
1915
|
+
run.playbackRequestId = undefined;
|
|
1916
|
+
if (!success && fallbackToOffline) {
|
|
1917
|
+
void playOfflineTts("cloud tts playback failed").then((started) => {
|
|
1918
|
+
if (!started) {
|
|
1919
|
+
sendPlaybackResult(false, error, duration, "tts failed");
|
|
1920
|
+
}
|
|
1921
|
+
});
|
|
1922
|
+
return;
|
|
1923
|
+
}
|
|
1924
|
+
sendPlaybackResult(success, error, duration, "tts failed", completionReason === "completed");
|
|
1925
|
+
}));
|
|
1926
|
+
queueMicrotask(() => {
|
|
1927
|
+
if (this.speechRuns.get(packageName) === run)
|
|
1928
|
+
this.setSpeakerState(packageName, "playing");
|
|
1929
|
+
});
|
|
1930
|
+
return true;
|
|
1931
|
+
};
|
|
1932
|
+
if (forceLocal) {
|
|
1933
|
+
if (await playOfflineTts()) {
|
|
1934
|
+
return;
|
|
1935
|
+
}
|
|
1936
|
+
if (run.cancelled)
|
|
1937
|
+
return;
|
|
1938
|
+
const message = "tts unavailable: forceLocal requested but offline TTS model is not available";
|
|
1939
|
+
sendPlaybackResult(false, message, null, message, false, MiniappErrorCode.TTS_LOCAL_UNAVAILABLE);
|
|
1940
|
+
return;
|
|
1941
|
+
}
|
|
1942
|
+
if (cloudConnected && (await playCloudTts(true))) {
|
|
1943
|
+
return;
|
|
1944
|
+
}
|
|
1945
|
+
// Offline is the disconnected-cloud fallback. If the model is not
|
|
1946
|
+
// available, preserve the old behavior by trying cloud as a last resort.
|
|
1947
|
+
if (await playOfflineTts(cloudConnected ? undefined : "cloud disconnected")) {
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1950
|
+
if (await playCloudTts(false)) {
|
|
1951
|
+
return;
|
|
1952
|
+
}
|
|
1953
|
+
const message = "tts unavailable: no cloud TTS URL or offline TTS model";
|
|
1954
|
+
sendPlaybackResult(false, message, null, message, false);
|
|
1955
|
+
}
|
|
1956
|
+
catch (err) {
|
|
1957
|
+
console.error(`${LOG_TAG}: speak error:`, err);
|
|
1958
|
+
const message = err instanceof Error ? err.message : "TTS error";
|
|
1959
|
+
sendPlaybackResult(false, message, null, "TTS error", false);
|
|
1960
|
+
}
|
|
1961
|
+
}
|
|
1962
|
+
cancelSpeech(packageName) {
|
|
1963
|
+
const run = this.speechRuns.get(packageName);
|
|
1964
|
+
if (!run)
|
|
1965
|
+
return;
|
|
1966
|
+
run.cancelled = true;
|
|
1967
|
+
this.speechRuns.delete(packageName);
|
|
1968
|
+
if (run.playbackRequestId)
|
|
1969
|
+
audioPlaybackService.cancelPlayback(run.playbackRequestId);
|
|
1970
|
+
run.onCancelled?.();
|
|
1971
|
+
}
|
|
1972
|
+
async handleRgbLed(packageName, payload, requestId) {
|
|
1973
|
+
const coerceNumber = (value, fallback) => {
|
|
1974
|
+
const coerced = Number(value);
|
|
1975
|
+
return Number.isFinite(coerced) ? coerced : fallback;
|
|
1976
|
+
};
|
|
1977
|
+
const ledRequestId = requestId || `led_${Date.now()}`;
|
|
1978
|
+
const action = normalizeRgbLedAction(payload.action);
|
|
1979
|
+
const color = normalizeRgbLedColor(payload.color);
|
|
1980
|
+
try {
|
|
1981
|
+
const result = await BluetoothSdk.rgbLedControl(ledRequestId, packageName, action, color, coerceNumber(payload.ontime, 1000), coerceNumber(payload.offtime, 0), coerceNumber(payload.count, 1));
|
|
1982
|
+
this.sendResult(packageName, requestId, true, result);
|
|
1983
|
+
}
|
|
1984
|
+
catch (err) {
|
|
1985
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1986
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
1987
|
+
message: err instanceof Error ? err.message : "RGB LED command failed",
|
|
1988
|
+
});
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
async handleLocationPoll(packageName, requestId) {
|
|
1992
|
+
try {
|
|
1993
|
+
const { status } = await Location.requestForegroundPermissionsAsync();
|
|
1994
|
+
if (status !== "granted") {
|
|
1995
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
1996
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
1997
|
+
message: "Location permission not granted",
|
|
1998
|
+
});
|
|
1999
|
+
return;
|
|
2000
|
+
}
|
|
2001
|
+
// Try the OS cache first — usually instant — and only fall back to
|
|
2002
|
+
// a fresh fix if the cache is empty. Use Low accuracy on the fresh
|
|
2003
|
+
// path so we get a cell/wifi-tower fix in ~1s instead of waiting
|
|
2004
|
+
// for full GPS warm-up.
|
|
2005
|
+
const cached = await Location.getLastKnownPositionAsync({ maxAge: 60_000 });
|
|
2006
|
+
const location = cached ?? (await Location.getCurrentPositionAsync({ accuracy: Location.Accuracy.Low }));
|
|
2007
|
+
this.sendResult(packageName, requestId, true, {
|
|
2008
|
+
lat: location.coords.latitude,
|
|
2009
|
+
lng: location.coords.longitude,
|
|
2010
|
+
accuracy: location.coords.accuracy ?? undefined,
|
|
2011
|
+
timestamp: location.timestamp,
|
|
2012
|
+
});
|
|
2013
|
+
}
|
|
2014
|
+
catch (err) {
|
|
2015
|
+
console.error(`${LOG_TAG}: location poll error:`, err);
|
|
2016
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2017
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2018
|
+
message: err instanceof Error ? err.message : "Location error",
|
|
2019
|
+
});
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
// ---------------------------------------------------------------------------
|
|
2023
|
+
// Navigation
|
|
2024
|
+
// ---------------------------------------------------------------------------
|
|
2025
|
+
/**
|
|
2026
|
+
* All navigation handlers + their state live in NavigationHandlers. The
|
|
2027
|
+
* dispatcher's NAVIGATION_* cases delegate here; the rest of the runtime
|
|
2028
|
+
* cross-cuts navigation through {@link NavigationHandlers.onDisconnect}
|
|
2029
|
+
* (mini-app disconnect) and {@link NavigationHandlers.isTripActive}
|
|
2030
|
+
* (location-stream gating during a trip).
|
|
2031
|
+
*/
|
|
2032
|
+
navigationHandlers = new NavigationHandlers((packageName, envelope) => this.sendToMiniapp(packageName, envelope), (packageName, requestId, ok, result, error) => this.sendResult(packageName, requestId, ok, result, error));
|
|
2033
|
+
/**
|
|
2034
|
+
* `session.blob` — persistent, per-app binary storage. Like NavigationHandlers,
|
|
2035
|
+
* the BLOB_* dispatcher cases delegate here. It's a generic byte store; audio
|
|
2036
|
+
* recording lives entirely in the miniapp (mic.onAudioChunk → chunked
|
|
2037
|
+
* BLOB_WRITE), so nothing audio-specific runs in the host.
|
|
2038
|
+
*/
|
|
2039
|
+
blobStore = new BlobStore({
|
|
2040
|
+
sendResult: (packageName, requestId, ok, result, error) => this.sendResult(packageName, requestId, ok, result, error),
|
|
2041
|
+
getUserId: () => useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) || "anonymous",
|
|
2042
|
+
});
|
|
2043
|
+
/**
|
|
2044
|
+
* Heading is a sensor stream — start the native compass when any mini
|
|
2045
|
+
* app is subscribed to "heading_update", stop it when none are.
|
|
2046
|
+
*/
|
|
2047
|
+
headingUnsub = null;
|
|
2048
|
+
recomputeHeadingSubscription() {
|
|
2049
|
+
const wantsHeading = this.streamSubscribers.has(MiniappStreamType.HEADING_UPDATE);
|
|
2050
|
+
if (wantsHeading && !this.headingUnsub) {
|
|
2051
|
+
// Heading sensor lives in engine (HeadingService) — subscribe directly.
|
|
2052
|
+
this.headingUnsub = headingService.addListener((degrees) => {
|
|
2053
|
+
this.forwardEvent(MiniappStreamType.HEADING_UPDATE, { degrees });
|
|
2054
|
+
});
|
|
2055
|
+
}
|
|
2056
|
+
else if (!wantsHeading && this.headingUnsub) {
|
|
2057
|
+
this.headingUnsub();
|
|
2058
|
+
this.headingUnsub = null;
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
/**
|
|
2062
|
+
* Accelerometer is a sensor stream — enable the glasses IMU when any mini
|
|
2063
|
+
* app is subscribed to "accel_data", disable it when none are. Mirrors the
|
|
2064
|
+
* heading pattern. Only G2 streams IMU today; on other devices the native
|
|
2065
|
+
* call is a no-op so this is harmless.
|
|
2066
|
+
*/
|
|
2067
|
+
imuEnabled = false;
|
|
2068
|
+
recomputeImuSubscription() {
|
|
2069
|
+
const wantsImu = this.streamSubscribers.has(MiniappStreamType.ACCEL_DATA);
|
|
2070
|
+
if (wantsImu === this.imuEnabled)
|
|
2071
|
+
return;
|
|
2072
|
+
this.imuEnabled = wantsImu;
|
|
2073
|
+
void BluetoothSdk.setImuEnabled(wantsImu);
|
|
2074
|
+
}
|
|
2075
|
+
/**
|
|
2076
|
+
* Last rate we asked the host for. Lets us skip the cross-process call
|
|
2077
|
+
* when nothing actually changed (most SUBSCRIBE / unregister churn
|
|
2078
|
+
* leaves the aggregate unchanged) and tracks state for the no-active-
|
|
2079
|
+
* subscribers → tell host to back off case.
|
|
2080
|
+
*/
|
|
2081
|
+
lastAppliedLocationRate = null;
|
|
2082
|
+
/**
|
|
2083
|
+
* Recompute the aggregate location tier across every connected
|
|
2084
|
+
* miniapp and push the result to the host. The aggregate is the
|
|
2085
|
+
* STRICTEST rate any single app has requested — see
|
|
2086
|
+
* {@link LOCATION_RATE_PRIORITY} — because the GPS sample rate is a
|
|
2087
|
+
* shared OS-level setting that we can't bias per-app.
|
|
2088
|
+
*
|
|
2089
|
+
* Called after every SUBSCRIBE and every unregister so that when a
|
|
2090
|
+
* realtime-requesting app goes away the tier falls back down. If no
|
|
2091
|
+
* connected app is asking for a location rate, we tell the host
|
|
2092
|
+
* "off" so it can drop GPS power.
|
|
2093
|
+
*/
|
|
2094
|
+
recomputeLocationTier() {
|
|
2095
|
+
let bestRank = -1;
|
|
2096
|
+
for (const app of this.connectedApps.values()) {
|
|
2097
|
+
const r = app.requestedLocationRate;
|
|
2098
|
+
if (!r)
|
|
2099
|
+
continue;
|
|
2100
|
+
const rank = locationRateRank(r);
|
|
2101
|
+
if (rank > bestRank) {
|
|
2102
|
+
bestRank = rank;
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
// When no app is asking, downgrade to "off". Without this we'd
|
|
2106
|
+
// leak the last-known-strictest rate forever (the original bug).
|
|
2107
|
+
const next = bestRank >= 0 ? LOCATION_RATE_PRIORITY[bestRank] : "off";
|
|
2108
|
+
if (next === this.lastAppliedLocationRate)
|
|
2109
|
+
return;
|
|
2110
|
+
this.lastAppliedLocationRate = next;
|
|
2111
|
+
console.log(`[LOCATION] aggregate tier → ${next}`);
|
|
2112
|
+
void phoneLocationService.setLocationTier(next);
|
|
2113
|
+
}
|
|
2114
|
+
// ---------------------------------------------------------------------------
|
|
2115
|
+
// Storage helpers
|
|
2116
|
+
// ---------------------------------------------------------------------------
|
|
2117
|
+
getStorageKeyPrefix(packageName) {
|
|
2118
|
+
const userId = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) || "anonymous";
|
|
2119
|
+
return `mentraos_localstorage_${userId}_${packageName}_`;
|
|
2120
|
+
}
|
|
2121
|
+
async handleStorageGet(packageName, payload, requestId) {
|
|
2122
|
+
try {
|
|
2123
|
+
const key = payload.key;
|
|
2124
|
+
if (!key) {
|
|
2125
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2126
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2127
|
+
message: "storage_get requires a key",
|
|
2128
|
+
});
|
|
2129
|
+
return;
|
|
2130
|
+
}
|
|
2131
|
+
const fullKey = this.getStorageKeyPrefix(packageName) + key;
|
|
2132
|
+
const result = mmkvStorage.load(fullKey);
|
|
2133
|
+
this.sendResult(packageName, requestId, true, { key, value: result.is_ok() ? result.value : null });
|
|
2134
|
+
}
|
|
2135
|
+
catch (err) {
|
|
2136
|
+
console.error(`${LOG_TAG}: storage_get error:`, err);
|
|
2137
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2138
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2139
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2140
|
+
});
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
async handleStorageSet(packageName, payload, requestId) {
|
|
2144
|
+
try {
|
|
2145
|
+
const key = payload.key;
|
|
2146
|
+
if (!key) {
|
|
2147
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2148
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2149
|
+
message: "storage_set requires a key",
|
|
2150
|
+
});
|
|
2151
|
+
return;
|
|
2152
|
+
}
|
|
2153
|
+
const fullKey = this.getStorageKeyPrefix(packageName) + key;
|
|
2154
|
+
mmkvStorage.save(fullKey, payload.value ?? null);
|
|
2155
|
+
this.sendResult(packageName, requestId, true);
|
|
2156
|
+
}
|
|
2157
|
+
catch (err) {
|
|
2158
|
+
console.error(`${LOG_TAG}: storage_set error:`, err);
|
|
2159
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2160
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2161
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2162
|
+
});
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
async handleStorageDelete(packageName, payload, requestId) {
|
|
2166
|
+
try {
|
|
2167
|
+
const key = payload.key;
|
|
2168
|
+
if (!key) {
|
|
2169
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2170
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2171
|
+
message: "storage_delete requires a key",
|
|
2172
|
+
});
|
|
2173
|
+
return;
|
|
2174
|
+
}
|
|
2175
|
+
const fullKey = this.getStorageKeyPrefix(packageName) + key;
|
|
2176
|
+
mmkvStorage.remove(fullKey);
|
|
2177
|
+
this.sendResult(packageName, requestId, true);
|
|
2178
|
+
}
|
|
2179
|
+
catch (err) {
|
|
2180
|
+
console.error(`${LOG_TAG}: storage_delete error:`, err);
|
|
2181
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2182
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2183
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2184
|
+
});
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
async handleStorageList(packageName, _payload, requestId) {
|
|
2188
|
+
try {
|
|
2189
|
+
const prefix = this.getStorageKeyPrefix(packageName);
|
|
2190
|
+
const allKeys = mmkvStorage.getAllKeys();
|
|
2191
|
+
const keys = allKeys.filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length));
|
|
2192
|
+
this.sendResult(packageName, requestId, true, { keys });
|
|
2193
|
+
}
|
|
2194
|
+
catch (err) {
|
|
2195
|
+
console.error(`${LOG_TAG}: storage_list error:`, err);
|
|
2196
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2197
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2198
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2199
|
+
});
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
async handleStorageClear(packageName, requestId) {
|
|
2203
|
+
try {
|
|
2204
|
+
const prefix = this.getStorageKeyPrefix(packageName);
|
|
2205
|
+
const allKeys = mmkvStorage.getAllKeys();
|
|
2206
|
+
for (const k of allKeys) {
|
|
2207
|
+
if (k.startsWith(prefix))
|
|
2208
|
+
mmkvStorage.remove(k);
|
|
2209
|
+
}
|
|
2210
|
+
this.sendResult(packageName, requestId, true);
|
|
2211
|
+
}
|
|
2212
|
+
catch (err) {
|
|
2213
|
+
console.error(`${LOG_TAG}: storage_clear error:`, err);
|
|
2214
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2215
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2216
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2217
|
+
});
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
async handleStorageHas(packageName, payload, requestId) {
|
|
2221
|
+
try {
|
|
2222
|
+
const key = payload.key;
|
|
2223
|
+
if (!key) {
|
|
2224
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2225
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2226
|
+
message: "storage_has requires a key",
|
|
2227
|
+
});
|
|
2228
|
+
return;
|
|
2229
|
+
}
|
|
2230
|
+
const fullKey = this.getStorageKeyPrefix(packageName) + key;
|
|
2231
|
+
const result = mmkvStorage.load(fullKey);
|
|
2232
|
+
this.sendResult(packageName, requestId, true, { has: result.is_ok() });
|
|
2233
|
+
}
|
|
2234
|
+
catch (err) {
|
|
2235
|
+
console.error(`${LOG_TAG}: storage_has error:`, err);
|
|
2236
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2237
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2238
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2239
|
+
});
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
async handleStorageGetAll(packageName, requestId) {
|
|
2243
|
+
try {
|
|
2244
|
+
const prefix = this.getStorageKeyPrefix(packageName);
|
|
2245
|
+
const allKeys = mmkvStorage.getAllKeys();
|
|
2246
|
+
const values = {};
|
|
2247
|
+
for (const k of allKeys) {
|
|
2248
|
+
if (!k.startsWith(prefix))
|
|
2249
|
+
continue;
|
|
2250
|
+
const r = mmkvStorage.load(k);
|
|
2251
|
+
if (r.is_ok()) {
|
|
2252
|
+
const v = r.value;
|
|
2253
|
+
values[k.slice(prefix.length)] = typeof v === "string" ? v : String(v ?? "");
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
this.sendResult(packageName, requestId, true, { values });
|
|
2257
|
+
}
|
|
2258
|
+
catch (err) {
|
|
2259
|
+
console.error(`${LOG_TAG}: storage_get_all error:`, err);
|
|
2260
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2261
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2262
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2263
|
+
});
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
async handleStorageSetMultiple(packageName, payload, requestId) {
|
|
2267
|
+
try {
|
|
2268
|
+
const values = payload.values;
|
|
2269
|
+
if (!values || typeof values !== "object") {
|
|
2270
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2271
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2272
|
+
message: "storage_set_multiple requires a values object",
|
|
2273
|
+
});
|
|
2274
|
+
return;
|
|
2275
|
+
}
|
|
2276
|
+
const prefix = this.getStorageKeyPrefix(packageName);
|
|
2277
|
+
for (const [key, value] of Object.entries(values)) {
|
|
2278
|
+
mmkvStorage.save(prefix + key, value ?? null);
|
|
2279
|
+
}
|
|
2280
|
+
this.sendResult(packageName, requestId, true);
|
|
2281
|
+
}
|
|
2282
|
+
catch (err) {
|
|
2283
|
+
console.error(`${LOG_TAG}: storage_set_multiple error:`, err);
|
|
2284
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2285
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2286
|
+
message: err instanceof Error ? err.message : "Storage error",
|
|
2287
|
+
});
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
// ---------------------------------------------------------------------------
|
|
2291
|
+
// Camera FOV
|
|
2292
|
+
// ---------------------------------------------------------------------------
|
|
2293
|
+
async handleCameraFov(packageName, payload, requestId) {
|
|
2294
|
+
const app = this.connectedApps.get(packageName);
|
|
2295
|
+
const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
|
|
2296
|
+
if (!hasCameraPermission) {
|
|
2297
|
+
logPermissionNotDeclared(packageName, "CAMERA", "to set camera FOV", `{"type": "CAMERA"}`);
|
|
2298
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2299
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
2300
|
+
message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
|
|
2301
|
+
permission: "CAMERA",
|
|
2302
|
+
operation: MiniappRequestType.CAMERA_FOV,
|
|
2303
|
+
});
|
|
2304
|
+
return;
|
|
2305
|
+
}
|
|
2306
|
+
try {
|
|
2307
|
+
const request = normalizeCameraFovPayload(payload);
|
|
2308
|
+
const description = "preset" in request ? `preset=${request.preset}` : `fov=${request.fov} roi=${request.roiPosition ?? "center"}`;
|
|
2309
|
+
console.log(`${LOG_TAG}: camera_fov_set ${description}`);
|
|
2310
|
+
const result = await phoneCameraFovCoordinator.setOverride(packageName, request);
|
|
2311
|
+
this.sendResult(packageName, requestId, true, result);
|
|
2312
|
+
}
|
|
2313
|
+
catch (err) {
|
|
2314
|
+
console.error(`${LOG_TAG}: camera_fov error:`, err);
|
|
2315
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2316
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2317
|
+
message: err instanceof Error ? err.message : "Camera FOV error",
|
|
2318
|
+
});
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
/**
|
|
2322
|
+
* Explicit IMU enable/disable from a miniapp (session.imu.setEnabled).
|
|
2323
|
+
*
|
|
2324
|
+
* The accel stream already auto-toggles the native IMU on subscribe (see
|
|
2325
|
+
* {@link recomputeImuSubscription}); this is a direct override. We sync
|
|
2326
|
+
* `this.imuEnabled` to the requested state so a subsequent subscription
|
|
2327
|
+
* recompute with the same aggregate doesn't issue a redundant native call.
|
|
2328
|
+
* Note that a later subscribe/unsubscribe that changes the aggregate will
|
|
2329
|
+
* still re-derive the sensor state from subscriptions.
|
|
2330
|
+
*/
|
|
2331
|
+
handleImuSetEnabled(packageName, payload, requestId) {
|
|
2332
|
+
try {
|
|
2333
|
+
const enabled = !!payload.enabled;
|
|
2334
|
+
console.log(`${LOG_TAG}: imu_set_enabled ${enabled} (by ${packageName})`);
|
|
2335
|
+
this.imuEnabled = enabled;
|
|
2336
|
+
void BluetoothSdk.setImuEnabled(enabled);
|
|
2337
|
+
this.sendResult(packageName, requestId, true);
|
|
2338
|
+
}
|
|
2339
|
+
catch (err) {
|
|
2340
|
+
console.error(`${LOG_TAG}: imu_set_enabled error:`, err);
|
|
2341
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2342
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2343
|
+
message: err instanceof Error ? err.message : "IMU set-enabled error",
|
|
2344
|
+
});
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
// ---------------------------------------------------------------------------
|
|
2348
|
+
// System utilities (share, open URL, clipboard, download)
|
|
2349
|
+
// ---------------------------------------------------------------------------
|
|
2350
|
+
async handleShare(packageName, payload, requestId) {
|
|
2351
|
+
const { text, title, base64, mimeType, filename, url } = payload;
|
|
2352
|
+
try {
|
|
2353
|
+
if (base64) {
|
|
2354
|
+
// File share via base64 — write to temp file then share
|
|
2355
|
+
const tempFile = new File(Paths.cache, filename || "shared_file");
|
|
2356
|
+
tempFile.write(base64, { encoding: "base64" });
|
|
2357
|
+
await Share.open({
|
|
2358
|
+
url: tempFile.uri,
|
|
2359
|
+
type: mimeType || "application/octet-stream",
|
|
2360
|
+
filename: filename,
|
|
2361
|
+
title: title,
|
|
2362
|
+
});
|
|
2363
|
+
}
|
|
2364
|
+
else if (url) {
|
|
2365
|
+
await Share.open({ url, title, message: text });
|
|
2366
|
+
}
|
|
2367
|
+
else {
|
|
2368
|
+
await Share.open({ message: text || "", title });
|
|
2369
|
+
}
|
|
2370
|
+
this.sendResult(packageName, requestId, true, { success: true });
|
|
2371
|
+
}
|
|
2372
|
+
catch (error) {
|
|
2373
|
+
// react-native-share throws when user dismisses the share sheet
|
|
2374
|
+
if (error?.message?.includes("User did not share")) {
|
|
2375
|
+
this.sendResult(packageName, requestId, true, { success: false, cancelled: true });
|
|
2376
|
+
}
|
|
2377
|
+
else {
|
|
2378
|
+
console.error(`${LOG_TAG}: share error:`, error);
|
|
2379
|
+
this.sendResult(packageName, requestId, true, { success: false });
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2382
|
+
}
|
|
2383
|
+
async handleOpenUrl(packageName, payload) {
|
|
2384
|
+
const url = payload.url;
|
|
2385
|
+
if (!url || typeof url !== "string") {
|
|
2386
|
+
console.warn(`${LOG_TAG}: open_url missing url`);
|
|
2387
|
+
return;
|
|
2388
|
+
}
|
|
2389
|
+
// Block dangerous schemes
|
|
2390
|
+
if (url.startsWith("javascript:") || url.startsWith("file:")) {
|
|
2391
|
+
console.warn(`${LOG_TAG}: open_url blocked dangerous scheme: ${url}`);
|
|
2392
|
+
return;
|
|
2393
|
+
}
|
|
2394
|
+
try {
|
|
2395
|
+
await Linking.openURL(url);
|
|
2396
|
+
}
|
|
2397
|
+
catch (error) {
|
|
2398
|
+
console.error(`${LOG_TAG}: open_url error:`, error);
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
async handleCopyClipboard(packageName, payload, requestId) {
|
|
2402
|
+
const text = payload.text;
|
|
2403
|
+
if (typeof text !== "string") {
|
|
2404
|
+
console.warn(`${LOG_TAG}: copy_clipboard missing text`);
|
|
2405
|
+
return;
|
|
2406
|
+
}
|
|
2407
|
+
try {
|
|
2408
|
+
await Clipboard.setStringAsync(text);
|
|
2409
|
+
this.sendResult(packageName, requestId, true);
|
|
2410
|
+
}
|
|
2411
|
+
catch (error) {
|
|
2412
|
+
console.error(`${LOG_TAG}: clipboard error:`, error);
|
|
2413
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2414
|
+
code: MiniappErrorCode.INTERNAL,
|
|
2415
|
+
message: error?.message || "Clipboard error",
|
|
2416
|
+
});
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
async handleDownload(packageName, payload, requestId) {
|
|
2420
|
+
const { base64, url, mimeType, filename } = payload;
|
|
2421
|
+
const name = filename || "download";
|
|
2422
|
+
try {
|
|
2423
|
+
let file;
|
|
2424
|
+
if (base64) {
|
|
2425
|
+
file = new File(Paths.cache, name);
|
|
2426
|
+
file.write(base64, { encoding: "base64" });
|
|
2427
|
+
}
|
|
2428
|
+
else if (url) {
|
|
2429
|
+
file = await File.downloadFileAsync(url, new File(Paths.cache, name), { idempotent: true });
|
|
2430
|
+
}
|
|
2431
|
+
else {
|
|
2432
|
+
console.warn(`${LOG_TAG}: download missing base64 or url`);
|
|
2433
|
+
return;
|
|
2434
|
+
}
|
|
2435
|
+
// Open share sheet so user can choose where to save
|
|
2436
|
+
await Share.open({
|
|
2437
|
+
url: file.uri,
|
|
2438
|
+
type: mimeType || "application/octet-stream",
|
|
2439
|
+
filename: name,
|
|
2440
|
+
});
|
|
2441
|
+
this.sendResult(packageName, requestId, true, { success: true });
|
|
2442
|
+
}
|
|
2443
|
+
catch (error) {
|
|
2444
|
+
if (error?.message?.includes("User did not share")) {
|
|
2445
|
+
this.sendResult(packageName, requestId, true, { success: true, cancelled: true });
|
|
2446
|
+
}
|
|
2447
|
+
else {
|
|
2448
|
+
console.error(`${LOG_TAG}: download error:`, error);
|
|
2449
|
+
this.sendResult(packageName, requestId, true, { success: false });
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
}
|
|
2453
|
+
// ===========================================================================
|
|
2454
|
+
// Photo + streaming handlers (cloud-coordinated)
|
|
2455
|
+
// ===========================================================================
|
|
2456
|
+
async handlePhoto(packageName, payload, requestId) {
|
|
2457
|
+
// Manifest CAMERA permission gate.
|
|
2458
|
+
const app = this.connectedApps.get(packageName);
|
|
2459
|
+
const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
|
|
2460
|
+
if (!hasCameraPermission) {
|
|
2461
|
+
logPermissionNotDeclared(packageName, "CAMERA", "to take a photo", `{"type": "CAMERA"}`);
|
|
2462
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2463
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
2464
|
+
message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
|
|
2465
|
+
permission: "CAMERA",
|
|
2466
|
+
operation: MiniappRequestType.PHOTO,
|
|
2467
|
+
});
|
|
2468
|
+
return;
|
|
2469
|
+
}
|
|
2470
|
+
try {
|
|
2471
|
+
const result = await phonePhotoCoordinator.takePhoto(packageName, {
|
|
2472
|
+
size: payload.size,
|
|
2473
|
+
mode: payload.mode,
|
|
2474
|
+
transferMethod: payload.transferMethod,
|
|
2475
|
+
compress: payload.compress,
|
|
2476
|
+
sound: payload.sound,
|
|
2477
|
+
saveToGallery: payload.saveToGallery,
|
|
2478
|
+
exposureTimeNs: payload.exposureTimeNs,
|
|
2479
|
+
iso: payload.iso,
|
|
2480
|
+
aeExposureDivisor: payload.aeExposureDivisor,
|
|
2481
|
+
isoCap: payload.isoCap,
|
|
2482
|
+
noiseReduction: payload.noiseReduction,
|
|
2483
|
+
edgeEnhancement: payload.edgeEnhancement,
|
|
2484
|
+
mfnr: payload.mfnr,
|
|
2485
|
+
zsl: payload.zsl,
|
|
2486
|
+
ispDigitalGain: payload.ispDigitalGain,
|
|
2487
|
+
ispAnalogGain: payload.ispAnalogGain,
|
|
2488
|
+
});
|
|
2489
|
+
this.sendResult(packageName, requestId, true, result);
|
|
2490
|
+
}
|
|
2491
|
+
catch (err) {
|
|
2492
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2493
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2494
|
+
message: err instanceof Error ? err.message : "Photo request failed",
|
|
2495
|
+
stage: err.stage,
|
|
2496
|
+
transport: err.transport,
|
|
2497
|
+
});
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
async handleCameraWarmUp(packageName, payload, requestId) {
|
|
2501
|
+
// Manifest CAMERA permission gate (same as photo).
|
|
2502
|
+
const app = this.connectedApps.get(packageName);
|
|
2503
|
+
const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
|
|
2504
|
+
if (!hasCameraPermission) {
|
|
2505
|
+
logPermissionNotDeclared(packageName, "CAMERA", "to warm up the camera", `{"type": "CAMERA"}`);
|
|
2506
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2507
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
2508
|
+
message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
|
|
2509
|
+
permission: "CAMERA",
|
|
2510
|
+
operation: MiniappRequestType.CAMERA_WARM_UP,
|
|
2511
|
+
});
|
|
2512
|
+
return;
|
|
2513
|
+
}
|
|
2514
|
+
try {
|
|
2515
|
+
await phonePhotoCoordinator.warmUpCamera(packageName, {
|
|
2516
|
+
size: payload.size,
|
|
2517
|
+
mode: payload.mode,
|
|
2518
|
+
exposureTimeNs: payload.exposureTimeNs,
|
|
2519
|
+
durationMs: payload.durationMs,
|
|
2520
|
+
});
|
|
2521
|
+
this.sendResult(packageName, requestId, true);
|
|
2522
|
+
}
|
|
2523
|
+
catch (err) {
|
|
2524
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2525
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2526
|
+
message: err instanceof Error ? err.message : "Camera warm-up failed",
|
|
2527
|
+
stage: err.stage,
|
|
2528
|
+
transport: err.transport,
|
|
2529
|
+
});
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
async handleVideoRecordingStart(packageName, payload, requestId) {
|
|
2533
|
+
// Manifest CAMERA permission gate (same as photo).
|
|
2534
|
+
const app = this.connectedApps.get(packageName);
|
|
2535
|
+
const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
|
|
2536
|
+
if (!hasCameraPermission) {
|
|
2537
|
+
logPermissionNotDeclared(packageName, "CAMERA", "to record video", `{"type": "CAMERA"}`);
|
|
2538
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2539
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
2540
|
+
message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
|
|
2541
|
+
permission: "CAMERA",
|
|
2542
|
+
operation: MiniappRequestType.VIDEO_RECORDING_START,
|
|
2543
|
+
});
|
|
2544
|
+
return;
|
|
2545
|
+
}
|
|
2546
|
+
try {
|
|
2547
|
+
const result = await phoneVideoCoordinator.startRecording(packageName, {
|
|
2548
|
+
width: payload.width,
|
|
2549
|
+
height: payload.height,
|
|
2550
|
+
fps: payload.fps,
|
|
2551
|
+
sound: payload.sound,
|
|
2552
|
+
save: payload.save,
|
|
2553
|
+
});
|
|
2554
|
+
this.sendResult(packageName, requestId, true, result);
|
|
2555
|
+
}
|
|
2556
|
+
catch (err) {
|
|
2557
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2558
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2559
|
+
message: err instanceof Error ? err.message : "Video recording start failed",
|
|
2560
|
+
});
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
async handleVideoRecordingStop(packageName, payload, requestId) {
|
|
2564
|
+
try {
|
|
2565
|
+
await phoneVideoCoordinator.stopRecording(packageName, payload.recordingId, {
|
|
2566
|
+
uploadUrl: payload.uploadUrl,
|
|
2567
|
+
uploadAuthToken: payload.uploadAuthToken,
|
|
2568
|
+
});
|
|
2569
|
+
this.sendResult(packageName, requestId, true);
|
|
2570
|
+
}
|
|
2571
|
+
catch (err) {
|
|
2572
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2573
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2574
|
+
message: err instanceof Error ? err.message : "Video recording stop failed",
|
|
2575
|
+
});
|
|
2576
|
+
}
|
|
2577
|
+
}
|
|
2578
|
+
/**
|
|
2579
|
+
* Stream handlers — dispatched to the engine PhoneStreamCoordinator. For managed
|
|
2580
|
+
* streams the coordinator additionally calls the v2 client REST route to
|
|
2581
|
+
* provision Cloudflare. Cloud-SDK apps (third-party developers) use a
|
|
2582
|
+
* separate cloud-side path that does not pass through here.
|
|
2583
|
+
*/
|
|
2584
|
+
async handleStreamStart(packageName, payload, requestId) {
|
|
2585
|
+
const streaming = phoneStreamCoordinator;
|
|
2586
|
+
if (!streaming) {
|
|
2587
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2588
|
+
code: MiniappErrorCode.NOT_IMPLEMENTED,
|
|
2589
|
+
message: "Streaming is not configured on this host",
|
|
2590
|
+
});
|
|
2591
|
+
return;
|
|
2592
|
+
}
|
|
2593
|
+
try {
|
|
2594
|
+
const result = await streaming.startUnmanaged(packageName, {
|
|
2595
|
+
streamUrl: payload.streamUrl,
|
|
2596
|
+
video: normalizeStreamVideoConfig(payload.video),
|
|
2597
|
+
audio: normalizeStreamAudioConfig(payload.audio),
|
|
2598
|
+
sound: payload.sound,
|
|
2599
|
+
});
|
|
2600
|
+
this.sendResult(packageName, requestId, true, result);
|
|
2601
|
+
}
|
|
2602
|
+
catch (err) {
|
|
2603
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2604
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2605
|
+
message: err instanceof Error ? err.message : "Stream start failed",
|
|
2606
|
+
stage: err.stage,
|
|
2607
|
+
transport: err.transport,
|
|
2608
|
+
});
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
async handleStreamStop(packageName, payload, requestId) {
|
|
2612
|
+
const streaming = phoneStreamCoordinator;
|
|
2613
|
+
if (!streaming) {
|
|
2614
|
+
// No adapter wired — treat as already-stopped.
|
|
2615
|
+
this.sendResult(packageName, requestId, true);
|
|
2616
|
+
return;
|
|
2617
|
+
}
|
|
2618
|
+
try {
|
|
2619
|
+
await streaming.stop(packageName, payload.streamId);
|
|
2620
|
+
this.sendResult(packageName, requestId, true);
|
|
2621
|
+
}
|
|
2622
|
+
catch (err) {
|
|
2623
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2624
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2625
|
+
message: err instanceof Error ? err.message : "Stream stop failed",
|
|
2626
|
+
stage: err.stage,
|
|
2627
|
+
transport: err.transport,
|
|
2628
|
+
});
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
async handleManagedStreamStart(packageName, payload, requestId) {
|
|
2632
|
+
const streaming = phoneStreamCoordinator;
|
|
2633
|
+
if (!streaming) {
|
|
2634
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2635
|
+
code: MiniappErrorCode.NOT_IMPLEMENTED,
|
|
2636
|
+
message: "Streaming is not configured on this host",
|
|
2637
|
+
});
|
|
2638
|
+
return;
|
|
2639
|
+
}
|
|
2640
|
+
try {
|
|
2641
|
+
const result = await streaming.startManaged(packageName, {
|
|
2642
|
+
restreamDestinations: payload.restreamDestinations,
|
|
2643
|
+
video: normalizeStreamVideoConfig(payload.video),
|
|
2644
|
+
audio: normalizeStreamAudioConfig(payload.audio),
|
|
2645
|
+
sound: payload.sound,
|
|
2646
|
+
ingest: payload.ingest,
|
|
2647
|
+
});
|
|
2648
|
+
this.sendResult(packageName, requestId, true, result);
|
|
2649
|
+
}
|
|
2650
|
+
catch (err) {
|
|
2651
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2652
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2653
|
+
message: err instanceof Error ? err.message : "Managed stream start failed",
|
|
2654
|
+
stage: err.stage,
|
|
2655
|
+
transport: err.transport,
|
|
2656
|
+
});
|
|
2657
|
+
}
|
|
2658
|
+
}
|
|
2659
|
+
async handleManagedStreamStop(packageName, payload, requestId) {
|
|
2660
|
+
return this.handleStreamStop(packageName, payload, requestId);
|
|
2661
|
+
}
|
|
2662
|
+
/**
|
|
2663
|
+
* session.glasses.requestWifiSetup — open the phone's glasses Wi-Fi setup
|
|
2664
|
+
* flow. The host owns the actual UI via the `engine.configure({ui})` seam;
|
|
2665
|
+
* this just forwards the request and reports success/failure.
|
|
2666
|
+
*/
|
|
2667
|
+
async handleRequestWifiSetup(packageName, payload, requestId) {
|
|
2668
|
+
const requestWifiSetup = getUiSeams().requestWifiSetup;
|
|
2669
|
+
if (!requestWifiSetup) {
|
|
2670
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2671
|
+
code: MiniappErrorCode.NOT_IMPLEMENTED,
|
|
2672
|
+
message: "Wi-Fi setup is not configured on this host",
|
|
2673
|
+
});
|
|
2674
|
+
return;
|
|
2675
|
+
}
|
|
2676
|
+
try {
|
|
2677
|
+
await requestWifiSetup(payload.reason);
|
|
2678
|
+
this.sendResult(packageName, requestId, true);
|
|
2679
|
+
}
|
|
2680
|
+
catch (err) {
|
|
2681
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
2682
|
+
code: err.code || MiniappErrorCode.INTERNAL,
|
|
2683
|
+
message: err instanceof Error ? err.message : "Wi-Fi setup failed",
|
|
2684
|
+
});
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
2687
|
+
/**
|
|
2688
|
+
* Wire the PhoneStreamCoordinator's status callback so coordinator-emitted
|
|
2689
|
+
* updates become `EVENT { streamType: "stream_status" }` envelopes on the
|
|
2690
|
+
* subscribing miniapp(s).
|
|
2691
|
+
*/
|
|
2692
|
+
wireStreamingStatusFanout() {
|
|
2693
|
+
const streaming = phoneStreamCoordinator;
|
|
2694
|
+
if (!streaming)
|
|
2695
|
+
return;
|
|
2696
|
+
streaming.setStatusSubscriber((packageName, update) => {
|
|
2697
|
+
this.sendToMiniapp(packageName, {
|
|
2698
|
+
type: MiniappResponseType.EVENT,
|
|
2699
|
+
streamType: "stream_status",
|
|
2700
|
+
data: {
|
|
2701
|
+
streamId: update.streamId,
|
|
2702
|
+
status: update.status,
|
|
2703
|
+
source: update.source,
|
|
2704
|
+
...(update.data || {}),
|
|
2705
|
+
},
|
|
2706
|
+
});
|
|
2707
|
+
});
|
|
2708
|
+
}
|
|
2709
|
+
// ===========================================================================
|
|
2710
|
+
// Public subscribe helper
|
|
2711
|
+
// ===========================================================================
|
|
2712
|
+
/**
|
|
2713
|
+
* Standalone subscribe — lets external code subscribe a registered miniapp to
|
|
2714
|
+
* streams without going through the envelope protocol.
|
|
2715
|
+
*
|
|
2716
|
+
* Returns `{ok: true}` on success, or `{ok: false, error}` on failure.
|
|
2717
|
+
*/
|
|
2718
|
+
subscribe(packageName, streams) {
|
|
2719
|
+
const app = this.connectedApps.get(packageName);
|
|
2720
|
+
if (!app) {
|
|
2721
|
+
return { ok: false, error: `App ${packageName} is not registered` };
|
|
2722
|
+
}
|
|
2723
|
+
if (!Array.isArray(streams)) {
|
|
2724
|
+
return { ok: false, error: "streams must be an array" };
|
|
2725
|
+
}
|
|
2726
|
+
console.log(`${LOG_TAG}: subscribe(${packageName}, [${streams.join(", ")}])`);
|
|
2727
|
+
// Check microphone permission for mic-requiring streams
|
|
2728
|
+
const micStreams = ["transcription", "translation", "audio_chunk", "vad"];
|
|
2729
|
+
const needsMic = streams.some((s) => micStreams.some((m) => s.startsWith(m) || s === m));
|
|
2730
|
+
if (needsMic) {
|
|
2731
|
+
const hasMicPermission = app.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE");
|
|
2732
|
+
if (!hasMicPermission) {
|
|
2733
|
+
return { ok: false, error: "MICROPHONE permission not declared in miniapp.json" };
|
|
2734
|
+
}
|
|
2735
|
+
}
|
|
2736
|
+
// Remove old subscriptions for this app
|
|
2737
|
+
for (const oldStream of app.subscriptions) {
|
|
2738
|
+
const subs = this.streamSubscribers.get(oldStream);
|
|
2739
|
+
if (subs) {
|
|
2740
|
+
subs.delete(packageName);
|
|
2741
|
+
if (subs.size === 0) {
|
|
2742
|
+
this.streamSubscribers.delete(oldStream);
|
|
2743
|
+
}
|
|
2744
|
+
}
|
|
2745
|
+
}
|
|
2746
|
+
// Add new subscriptions
|
|
2747
|
+
app.subscriptions = new Set(streams);
|
|
2748
|
+
app.forceLocalTranscriptionStreams.clear();
|
|
2749
|
+
app.cloudTranscriptionStreams = new Set(streams.filter((stream) => stream.startsWith("transcription:")));
|
|
2750
|
+
for (const stream of streams) {
|
|
2751
|
+
let subs = this.streamSubscribers.get(stream);
|
|
2752
|
+
if (!subs) {
|
|
2753
|
+
subs = new Set();
|
|
2754
|
+
this.streamSubscribers.set(stream, subs);
|
|
2755
|
+
}
|
|
2756
|
+
subs.add(packageName);
|
|
2757
|
+
}
|
|
2758
|
+
this.recomputeMicRequirements();
|
|
2759
|
+
this.updateCloudSubscriptions();
|
|
2760
|
+
return { ok: true };
|
|
2761
|
+
}
|
|
2762
|
+
// ===========================================================================
|
|
2763
|
+
// Mic requirements
|
|
2764
|
+
// ===========================================================================
|
|
2765
|
+
recomputeMicRequirements() {
|
|
2766
|
+
let anyPcm = false;
|
|
2767
|
+
let anyLc3 = false;
|
|
2768
|
+
for (const [stream, subscribers] of this.streamSubscribers) {
|
|
2769
|
+
if (subscribers.size === 0)
|
|
2770
|
+
continue;
|
|
2771
|
+
if (stream === "audio_chunk")
|
|
2772
|
+
anyPcm = true;
|
|
2773
|
+
if (stream.startsWith("transcription:") || stream.startsWith("translation:") || stream === "vad")
|
|
2774
|
+
anyLc3 = true;
|
|
2775
|
+
}
|
|
2776
|
+
micStateCoordinator.setLocalRequirements({ pcm: anyPcm, lc3: anyLc3 });
|
|
2777
|
+
}
|
|
2778
|
+
/**
|
|
2779
|
+
* Recompute the aggregated v2 cloud subscription list across all local miniapps.
|
|
2780
|
+
*
|
|
2781
|
+
* Cloud-dependent streams (transcription:*, translation:*) only flow if the
|
|
2782
|
+
* v2 runtime knows the phone wants them. Local-only streams (button_press, etc.)
|
|
2783
|
+
* are NOT sent — they come from the Bluetooth SDK, not from cloud.
|
|
2784
|
+
*/
|
|
2785
|
+
updateCloudSubscriptions() {
|
|
2786
|
+
const cloudStreams = new Set();
|
|
2787
|
+
let transcriptionLang = null;
|
|
2788
|
+
let forceLocalTranscriptionLang = null;
|
|
2789
|
+
let hasForceLocalTranscription = false;
|
|
2790
|
+
for (const [stream, subscribers] of this.streamSubscribers) {
|
|
2791
|
+
if (subscribers.size === 0)
|
|
2792
|
+
continue;
|
|
2793
|
+
// Only transcription / translation need cloud delivery. Location and
|
|
2794
|
+
// notifications are sourced natively on the phone — no cloud hop.
|
|
2795
|
+
if (stream.startsWith("translation:")) {
|
|
2796
|
+
cloudStreams.add(stream);
|
|
2797
|
+
}
|
|
2798
|
+
if (stream.startsWith("transcription:")) {
|
|
2799
|
+
const language = stream.substring("transcription:".length);
|
|
2800
|
+
if (transcriptionLang === null)
|
|
2801
|
+
transcriptionLang = language;
|
|
2802
|
+
const routes = summarizeTranscriptionRoutes(subscribers, (packageName) => this.appTranscriptionRoutesForSubscription(packageName, stream));
|
|
2803
|
+
if (routes.hasForceLocalSubscriber) {
|
|
2804
|
+
hasForceLocalTranscription = true;
|
|
2805
|
+
if (forceLocalTranscriptionLang === null)
|
|
2806
|
+
forceLocalTranscriptionLang = language;
|
|
2807
|
+
}
|
|
2808
|
+
if (routes.hasCloudSubscriber)
|
|
2809
|
+
cloudStreams.add(stream);
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
localSttFallbackCoordinator.onSubscriptionChange(transcriptionLang !== null, forceLocalTranscriptionLang ?? transcriptionLang, hasForceLocalTranscription);
|
|
2813
|
+
// Mirror the same set as typed AudioSubscription[] and push it to the cloud
|
|
2814
|
+
// runtime.
|
|
2815
|
+
this.ensureCloudResultsWired();
|
|
2816
|
+
const subs = this.buildCloudAudioSubscriptions(cloudStreams);
|
|
2817
|
+
const nextKey = this.audioSubscriptionKey(subs);
|
|
2818
|
+
if (!this.cloudAudioSubscriptionSync.begin(nextKey))
|
|
2819
|
+
return;
|
|
2820
|
+
console.log(`${LOG_TAG}: updateCloudSubscriptions streams=[${Array.from(cloudStreams).join(", ")}] cloudSubs=${subs.length}`);
|
|
2821
|
+
cloudClientService
|
|
2822
|
+
.setSubscriptions(subs)
|
|
2823
|
+
.then(() => {
|
|
2824
|
+
this.cloudAudioSubscriptionSync.succeeded(nextKey);
|
|
2825
|
+
})
|
|
2826
|
+
.catch((err) => {
|
|
2827
|
+
// Best-effort: the cloud may not be connected yet. Keep failed writes
|
|
2828
|
+
// retryable; otherwise a reconnect that recomputes the same desired
|
|
2829
|
+
// transcription set gets deduped and local captions never recover.
|
|
2830
|
+
this.cloudAudioSubscriptionSync.failed(nextKey);
|
|
2831
|
+
console.warn(`${LOG_TAG}: cloud setSubscriptions failed: ${err?.message ?? err}`);
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2834
|
+
/**
|
|
2835
|
+
* Build the cloud `AudioSubscription[]` from the miniapp cloud-stream key set.
|
|
2836
|
+
*
|
|
2837
|
+
* Stream keys: `transcription:<lang>` (lang may be `auto`, `en-US`, …) and
|
|
2838
|
+
* `translation:<source>:<target>` (3 colon-parts; source/target may be `*`
|
|
2839
|
+
* or `auto` wildcards, target may also be `*`). The cloud protocol requires a
|
|
2840
|
+
* concrete `target: string` for translation, so wildcard-target translation
|
|
2841
|
+
* keys (`translation:<source>:*`, `translation:*:*`, `translation:auto`) have
|
|
2842
|
+
* no cloud equivalent and are skipped. Wildcard/`auto` SOURCE maps to
|
|
2843
|
+
* `{mode: "auto"}`.
|
|
2844
|
+
*/
|
|
2845
|
+
buildCloudAudioSubscriptions(cloudStreams) {
|
|
2846
|
+
const subs = [];
|
|
2847
|
+
const langSource = (code) => code === "auto" || code === "*" ? { mode: "auto" } : { mode: "specific", code };
|
|
2848
|
+
for (const stream of cloudStreams) {
|
|
2849
|
+
if (stream.startsWith("transcription:")) {
|
|
2850
|
+
const lang = stream.substring("transcription:".length);
|
|
2851
|
+
subs.push({ kind: "transcription", language: langSource(lang) });
|
|
2852
|
+
}
|
|
2853
|
+
else if (stream.startsWith("translation:")) {
|
|
2854
|
+
const parts = stream.split(":");
|
|
2855
|
+
// Only `translation:<source>:<target>` maps cleanly; a concrete target
|
|
2856
|
+
// is required by the cloud schema.
|
|
2857
|
+
if (parts.length === 3) {
|
|
2858
|
+
const [, source, target] = parts;
|
|
2859
|
+
if (target === "*")
|
|
2860
|
+
continue;
|
|
2861
|
+
subs.push({ kind: "translation", source: langSource(source), target });
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
return subs;
|
|
2866
|
+
}
|
|
2867
|
+
audioSubscriptionKey(subs) {
|
|
2868
|
+
return JSON.stringify(subs
|
|
2869
|
+
.map((sub) => {
|
|
2870
|
+
if (sub.kind === "transcription") {
|
|
2871
|
+
return {
|
|
2872
|
+
kind: sub.kind,
|
|
2873
|
+
language: sub.language,
|
|
2874
|
+
};
|
|
2875
|
+
}
|
|
2876
|
+
return {
|
|
2877
|
+
kind: sub.kind,
|
|
2878
|
+
source: sub.source,
|
|
2879
|
+
target: sub.target,
|
|
2880
|
+
};
|
|
2881
|
+
})
|
|
2882
|
+
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))));
|
|
2883
|
+
}
|
|
2884
|
+
/**
|
|
2885
|
+
* Wire the cloud's transcription/translation results into the existing
|
|
2886
|
+
* miniapp fan-out exactly once. Maps each cloud result back to the
|
|
2887
|
+
* cloud-to-app data shape `forwardEvent` already forwards, keyed on the
|
|
2888
|
+
* `transcription:<lang>` / `translation:<source>:<target>` stream strings, so
|
|
2889
|
+
* subscribed miniapps receive identical envelopes.
|
|
2890
|
+
*/
|
|
2891
|
+
ensureCloudResultsWired() {
|
|
2892
|
+
if (this.cloudResultsWired)
|
|
2893
|
+
return;
|
|
2894
|
+
this.cloudResultsWired = true;
|
|
2895
|
+
cloudClientService.onTranscript((d) => {
|
|
2896
|
+
const receivedAt = Date.now();
|
|
2897
|
+
if (TRANSCRIPT_TIMING_TELEMETRY) {
|
|
2898
|
+
console.log(`${LOG_TAG}: transcript cloud_recv t=${receivedAt} final=${d.isFinal} lang=${d.resolvedLanguage} text="${d.text.slice(0, 48)}"`);
|
|
2899
|
+
}
|
|
2900
|
+
this.forwardEvent(`transcription:${d.resolvedLanguage}`, {
|
|
2901
|
+
type: "transcription",
|
|
2902
|
+
text: d.text,
|
|
2903
|
+
isFinal: d.isFinal,
|
|
2904
|
+
utteranceId: d.utteranceId,
|
|
2905
|
+
transcribeLanguage: d.resolvedLanguage,
|
|
2906
|
+
detectedLanguage: d.languageDetected ? d.resolvedLanguage : undefined,
|
|
2907
|
+
startTime: d.startMs,
|
|
2908
|
+
endTime: d.endMs,
|
|
2909
|
+
speakerId: d.speakerId,
|
|
2910
|
+
duration: d.durationMs,
|
|
2911
|
+
provider: d.provider,
|
|
2912
|
+
confidence: d.confidence,
|
|
2913
|
+
__hostReceivedAt: receivedAt,
|
|
2914
|
+
}, "cloud");
|
|
2915
|
+
});
|
|
2916
|
+
cloudClientService.onTranslation((d) => {
|
|
2917
|
+
this.forwardEvent(`translation:${d.source.language}:${d.target.language}`, {
|
|
2918
|
+
type: "translation",
|
|
2919
|
+
text: d.text,
|
|
2920
|
+
originalText: d.originalText,
|
|
2921
|
+
isFinal: d.isFinal,
|
|
2922
|
+
utteranceId: d.utteranceId,
|
|
2923
|
+
startTime: d.startMs,
|
|
2924
|
+
endTime: d.endMs,
|
|
2925
|
+
speakerId: d.speakerId,
|
|
2926
|
+
duration: d.durationMs,
|
|
2927
|
+
transcribeLanguage: d.source.language,
|
|
2928
|
+
translateLanguage: d.target.language,
|
|
2929
|
+
didTranslate: true,
|
|
2930
|
+
provider: d.provider,
|
|
2931
|
+
confidence: d.confidence,
|
|
2932
|
+
});
|
|
2933
|
+
});
|
|
2934
|
+
}
|
|
2935
|
+
ensureCloudStatusWired() {
|
|
2936
|
+
if (this.cloudStatusWired)
|
|
2937
|
+
return;
|
|
2938
|
+
this.cloudStatusWired = true;
|
|
2939
|
+
cloudClientService.onStatusChanged((status) => {
|
|
2940
|
+
if (status.status === "connected") {
|
|
2941
|
+
this.updateCloudSubscriptions();
|
|
2942
|
+
// A miniapp that connected before the cloud client was ready never got
|
|
2943
|
+
// its auth token; now that we're connected, mint it without waiting for
|
|
2944
|
+
// the retry backoff (or a full manual reconnect).
|
|
2945
|
+
this.redriveMiniappAuthDelivery();
|
|
2946
|
+
}
|
|
2947
|
+
this.broadcastCloudStatus();
|
|
2948
|
+
});
|
|
2949
|
+
useSettingsStore.subscribe((s) => s.getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive), () => {
|
|
2950
|
+
this.broadcastCloudStatus();
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
currentCloudStatus() {
|
|
2954
|
+
const base = cloudClientService.getStatus();
|
|
2955
|
+
const fallbackActive = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive) ===
|
|
2956
|
+
true;
|
|
2957
|
+
return {
|
|
2958
|
+
status: base.status,
|
|
2959
|
+
audioTransport: fallbackActive ? "offline" : base.audioTransport,
|
|
2960
|
+
};
|
|
2961
|
+
}
|
|
2962
|
+
broadcastCloudStatus() {
|
|
2963
|
+
const status = this.currentCloudStatus();
|
|
2964
|
+
for (const packageName of this.connectedApps.keys()) {
|
|
2965
|
+
this.sendCloudStatusToMiniapp(packageName, status);
|
|
2966
|
+
}
|
|
2967
|
+
}
|
|
2968
|
+
sendCloudStatusToMiniapp(packageName, status = this.currentCloudStatus()) {
|
|
2969
|
+
this.sendToMiniapp(packageName, {
|
|
2970
|
+
type: MiniappResponseType.EVENT,
|
|
2971
|
+
streamType: CLOUD_STATUS_STREAM,
|
|
2972
|
+
data: status,
|
|
2973
|
+
});
|
|
2974
|
+
}
|
|
2975
|
+
// ===========================================================================
|
|
2976
|
+
// Stream fan-out
|
|
2977
|
+
// ===========================================================================
|
|
2978
|
+
/**
|
|
2979
|
+
* Forward a streamed event to all miniapps subscribed to the given stream.
|
|
2980
|
+
*
|
|
2981
|
+
* Event name translation:
|
|
2982
|
+
* - Cloud sends "head_up" → miniapp protocol uses "head_position" (HEAD_POSITION)
|
|
2983
|
+
* - Cloud sends "VAD" (uppercase) → miniapp protocol uses "vad" (lowercase)
|
|
2984
|
+
*/
|
|
2985
|
+
forwardEvent(streamType, data, transcriptionSource) {
|
|
2986
|
+
// Translate cloud event names to miniapp protocol stream types
|
|
2987
|
+
const normalizedStream = this.normalizeStreamType(streamType);
|
|
2988
|
+
// Collect all subscribers: exact match, plus wildcard matches for streams
|
|
2989
|
+
// that carry a language tag. A miniapp subscribed to "transcription:auto"
|
|
2990
|
+
// should receive any "transcription:<lang>" event (the detected language
|
|
2991
|
+
// is conveyed in the event data, not the stream key).
|
|
2992
|
+
//
|
|
2993
|
+
// Translation streams support cloud v3's wildcard patterns. Incoming
|
|
2994
|
+
// events are keyed `translation:<source>:<target>` and we match against
|
|
2995
|
+
// any of:
|
|
2996
|
+
// translation:*:* — all-pairs (TranslationModule.on)
|
|
2997
|
+
// translation:*:<target> — any-source → target (TranslationModule.to)
|
|
2998
|
+
// translation:<source>:* — source → any-target (rare)
|
|
2999
|
+
// translation:<source>:<target> — exact (TranslationModule.fromTo)
|
|
3000
|
+
// translation:auto — back-compat alias for *:*
|
|
3001
|
+
const matchedSubs = new Set();
|
|
3002
|
+
const exact = this.streamSubscribers.get(normalizedStream);
|
|
3003
|
+
if (exact)
|
|
3004
|
+
for (const p of exact)
|
|
3005
|
+
matchedSubs.add(p);
|
|
3006
|
+
if (normalizedStream.startsWith("transcription:")) {
|
|
3007
|
+
const autoSubs = this.streamSubscribers.get("transcription:auto");
|
|
3008
|
+
if (autoSubs)
|
|
3009
|
+
for (const p of autoSubs)
|
|
3010
|
+
matchedSubs.add(p);
|
|
3011
|
+
}
|
|
3012
|
+
else if (normalizedStream.startsWith("translation:")) {
|
|
3013
|
+
// translation:<source>:<target> — match each wildcard variant.
|
|
3014
|
+
const parts = normalizedStream.split(":");
|
|
3015
|
+
// parts[0] = "translation", parts[1] = source, parts[2] = target
|
|
3016
|
+
if (parts.length === 3) {
|
|
3017
|
+
const [, source, target] = parts;
|
|
3018
|
+
const patterns = [
|
|
3019
|
+
"translation:auto", // legacy alias for *:*
|
|
3020
|
+
"translation:*:*", // cloud v3 "all pairs"
|
|
3021
|
+
`translation:*:${target}`, // cloud v3 "to <target>"
|
|
3022
|
+
`translation:${source}:*`, // cloud v3 "from <source>"
|
|
3023
|
+
];
|
|
3024
|
+
for (const pat of patterns) {
|
|
3025
|
+
const subs = this.streamSubscribers.get(pat);
|
|
3026
|
+
if (subs)
|
|
3027
|
+
for (const p of subs)
|
|
3028
|
+
matchedSubs.add(p);
|
|
3029
|
+
}
|
|
3030
|
+
}
|
|
3031
|
+
else {
|
|
3032
|
+
// Malformed stream key — still honor the legacy auto alias.
|
|
3033
|
+
const autoSubs = this.streamSubscribers.get("translation:auto");
|
|
3034
|
+
if (autoSubs)
|
|
3035
|
+
for (const p of autoSubs)
|
|
3036
|
+
matchedSubs.add(p);
|
|
3037
|
+
}
|
|
3038
|
+
}
|
|
3039
|
+
// Touch event per-gesture fan-out. SDK-side dispatch is keyed on the
|
|
3040
|
+
// exact streamType, so per-gesture subscribers (`touch_event:single_tap`,
|
|
3041
|
+
// etc.) need their own EVENT envelope with the gesture-tagged streamType.
|
|
3042
|
+
// The bare `touch_event` stream above still catches `onTouch(handler)`.
|
|
3043
|
+
let perGestureStream = null;
|
|
3044
|
+
let outboundData = data;
|
|
3045
|
+
if (normalizedStream === MiniappStreamType.TOUCH_EVENT) {
|
|
3046
|
+
// The Bluetooth SDK delivers the gesture under `gestureName` (single_tap /
|
|
3047
|
+
// double_tap / triple_tap / long_press / swipe_up / swipe_down). Surface it
|
|
3048
|
+
// to miniapps as `TouchData.kind` and tag a per-gesture stream so
|
|
3049
|
+
// onTouch("single_tap", ...) routes correctly.
|
|
3050
|
+
const touch = data;
|
|
3051
|
+
const gesture = touch?.gestureName ?? touch?.kind;
|
|
3052
|
+
if (typeof gesture === "string" && gesture.length > 0) {
|
|
3053
|
+
outboundData = { ...touch, kind: gesture };
|
|
3054
|
+
perGestureStream = `${MiniappStreamType.TOUCH_EVENT}:${gesture}`;
|
|
3055
|
+
}
|
|
3056
|
+
}
|
|
3057
|
+
if (normalizedStream.startsWith("transcription:")) {
|
|
3058
|
+
// const known = Array.from(this.streamSubscribers.keys())
|
|
3059
|
+
// console.log(
|
|
3060
|
+
// `${LOG_TAG}: forwardEvent(${streamType} → ${normalizedStream}) matched=${matchedSubs.size} known=[${known.join(", ")}]`,
|
|
3061
|
+
// )
|
|
3062
|
+
}
|
|
3063
|
+
if (matchedSubs.size === 0 && !perGestureStream)
|
|
3064
|
+
return;
|
|
3065
|
+
for (const packageName of matchedSubs) {
|
|
3066
|
+
let transcriptionRoute;
|
|
3067
|
+
if (normalizedStream.startsWith("transcription:") && transcriptionSource) {
|
|
3068
|
+
const routes = this.appTranscriptionRoutesForEvent(packageName, normalizedStream);
|
|
3069
|
+
transcriptionRoute =
|
|
3070
|
+
transcriptionDeliveryRoute(transcriptionSource, routes, cloudClientService.isConnected()) ?? undefined;
|
|
3071
|
+
if (!transcriptionRoute)
|
|
3072
|
+
continue;
|
|
3073
|
+
}
|
|
3074
|
+
if (TRANSCRIPT_TIMING_TELEMETRY && normalizedStream.startsWith("transcription:")) {
|
|
3075
|
+
const transcript = data;
|
|
3076
|
+
const now = Date.now();
|
|
3077
|
+
console.log(`${LOG_TAG}: transcript fanout t=${now} app=${packageName} final=${!!transcript?.isFinal} hostDelta=${transcript?.__hostReceivedAt ? now - transcript.__hostReceivedAt : -1}ms text="${(transcript?.text ?? "").slice(0, 48)}"`);
|
|
3078
|
+
}
|
|
3079
|
+
// While a nav trip is active the Nav SDK's road-snapped fixes are
|
|
3080
|
+
// already being forwarded via addLocationListener (see NAV_START handler).
|
|
3081
|
+
// Suppress the raw background-GPS forward for that miniapp so the two
|
|
3082
|
+
// streams don't interleave and cause the position to jump back to the
|
|
3083
|
+
// real-phone location during simulation.
|
|
3084
|
+
if (normalizedStream === MiniappStreamType.LOCATION_UPDATE && this.navigationHandlers.isTripActive(packageName)) {
|
|
3085
|
+
continue;
|
|
3086
|
+
}
|
|
3087
|
+
this.sendToMiniapp(packageName, {
|
|
3088
|
+
type: MiniappResponseType.EVENT,
|
|
3089
|
+
streamType: normalizedStream,
|
|
3090
|
+
data: outboundData,
|
|
3091
|
+
...(transcriptionRoute ? { transcriptionRoute } : {}),
|
|
3092
|
+
});
|
|
3093
|
+
}
|
|
3094
|
+
// Send a separate envelope to per-gesture subscribers tagged with the
|
|
3095
|
+
// gesture-specific streamType so SDK-side dispatch routes correctly.
|
|
3096
|
+
if (perGestureStream) {
|
|
3097
|
+
const gestureSubs = this.streamSubscribers.get(perGestureStream);
|
|
3098
|
+
if (gestureSubs) {
|
|
3099
|
+
for (const packageName of gestureSubs) {
|
|
3100
|
+
this.sendToMiniapp(packageName, {
|
|
3101
|
+
type: MiniappResponseType.EVENT,
|
|
3102
|
+
streamType: perGestureStream,
|
|
3103
|
+
data: outboundData,
|
|
3104
|
+
});
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
}
|
|
3108
|
+
}
|
|
3109
|
+
appTranscriptionRoutesForSubscription(packageName, stream) {
|
|
3110
|
+
const app = this.connectedApps.get(packageName);
|
|
3111
|
+
return {
|
|
3112
|
+
cloud: app?.cloudTranscriptionStreams.has(stream) === true,
|
|
3113
|
+
forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true,
|
|
3114
|
+
};
|
|
3115
|
+
}
|
|
3116
|
+
appTranscriptionRoutesForEvent(packageName, stream) {
|
|
3117
|
+
const app = this.connectedApps.get(packageName);
|
|
3118
|
+
const autoStream = `${MiniappStreamType.TRANSCRIPTION}:auto`;
|
|
3119
|
+
return {
|
|
3120
|
+
cloud: app?.cloudTranscriptionStreams.has(stream) === true || app?.cloudTranscriptionStreams.has(autoStream) === true,
|
|
3121
|
+
forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true ||
|
|
3122
|
+
app?.forceLocalTranscriptionStreams.has(autoStream) === true,
|
|
3123
|
+
};
|
|
3124
|
+
}
|
|
3125
|
+
/**
|
|
3126
|
+
* Translate cloud event names to miniapp stream type values.
|
|
3127
|
+
*/
|
|
3128
|
+
normalizeStreamType(cloudEventName) {
|
|
3129
|
+
// Cloud / Bluetooth SDK → miniapp protocol translations.
|
|
3130
|
+
// Bluetooth SDK event names don't always match the miniapp wire values.
|
|
3131
|
+
switch (cloudEventName) {
|
|
3132
|
+
case "head_up":
|
|
3133
|
+
return MiniappStreamType.HEAD_POSITION; // head_up → head_position
|
|
3134
|
+
case "accel_event":
|
|
3135
|
+
return MiniappStreamType.ACCEL_DATA; // accel_event (native) → accel_data
|
|
3136
|
+
case "VAD":
|
|
3137
|
+
return MiniappStreamType.VAD; // VAD (uppercase) → vad (lowercase)
|
|
3138
|
+
case "glasses_battery_update":
|
|
3139
|
+
return MiniappStreamType.GLASSES_BATTERY; // glasses_battery_update → glasses_battery
|
|
3140
|
+
case "glasses_connection_state":
|
|
3141
|
+
return MiniappStreamType.GLASSES_CONNECTION; // glasses_connection_state → glasses_connection
|
|
3142
|
+
default:
|
|
3143
|
+
// Preserve case for typed streams like "transcription:en-US" / "translation:en-US:fr-FR"
|
|
3144
|
+
// whose language tags are case-sensitive (BCP-47). Lowercase only plain names.
|
|
3145
|
+
if (cloudEventName.includes(":"))
|
|
3146
|
+
return cloudEventName;
|
|
3147
|
+
return cloudEventName.toLowerCase();
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
// ===========================================================================
|
|
3151
|
+
// Outbound helpers
|
|
3152
|
+
// ===========================================================================
|
|
3153
|
+
/**
|
|
3154
|
+
* Send a payload to a connected miniapp, wrapped in an envelope.
|
|
3155
|
+
*/
|
|
3156
|
+
/**
|
|
3157
|
+
* Push a speaker-state transition to the owning miniapp. Idempotent: if
|
|
3158
|
+
* the new state matches the cached one (and isn't an error), no envelope
|
|
3159
|
+
* is sent. Error events are always sent — they're transient and the SDK
|
|
3160
|
+
* settles back to a non-error state immediately after.
|
|
3161
|
+
*/
|
|
3162
|
+
setSpeakerState(packageName, next, extra) {
|
|
3163
|
+
const app = this.connectedApps.get(packageName);
|
|
3164
|
+
if (!app)
|
|
3165
|
+
return;
|
|
3166
|
+
if (app.speakerState === next && next !== "error")
|
|
3167
|
+
return;
|
|
3168
|
+
app.speakerState = next === "error" ? "error" : next;
|
|
3169
|
+
this.sendToMiniapp(packageName, {
|
|
3170
|
+
type: MiniappResponseType.SPEAKER_STATE,
|
|
3171
|
+
state: next,
|
|
3172
|
+
...(extra?.errorCode !== undefined ? { errorCode: extra.errorCode } : {}),
|
|
3173
|
+
...(extra?.errorMessage !== undefined ? { errorMessage: extra.errorMessage } : {}),
|
|
3174
|
+
...(extra?.durationMs !== undefined ? { durationMs: extra.durationMs } : {}),
|
|
3175
|
+
});
|
|
3176
|
+
// After an error event, immediately transition to "stopped" so the
|
|
3177
|
+
// SDK's isPlaying getter reads false and a new play()/speak() call
|
|
3178
|
+
// starts cleanly from idle/stopped.
|
|
3179
|
+
if (next === "error") {
|
|
3180
|
+
app.speakerState = "stopped";
|
|
3181
|
+
this.sendToMiniapp(packageName, {
|
|
3182
|
+
type: MiniappResponseType.SPEAKER_STATE,
|
|
3183
|
+
state: "stopped",
|
|
3184
|
+
});
|
|
3185
|
+
}
|
|
3186
|
+
}
|
|
3187
|
+
sendToMiniapp(packageName, payload, requestId) {
|
|
3188
|
+
const app = this.connectedApps.get(packageName);
|
|
3189
|
+
if (!app) {
|
|
3190
|
+
console.warn(`${LOG_TAG}: sendToMiniapp — ${packageName} not connected`);
|
|
3191
|
+
return;
|
|
3192
|
+
}
|
|
3193
|
+
const envelope = {
|
|
3194
|
+
payload,
|
|
3195
|
+
requestId,
|
|
3196
|
+
};
|
|
3197
|
+
const serialized = serializeEnvelope(envelope);
|
|
3198
|
+
if (payload?.streamType?.toString().startsWith("transcription")) {
|
|
3199
|
+
// console.log(
|
|
3200
|
+
// `${LOG_TAG}: sendToMiniapp → ${packageName} streamType=${(payload as Record<string, unknown>).streamType}`,
|
|
3201
|
+
// )
|
|
3202
|
+
}
|
|
3203
|
+
try {
|
|
3204
|
+
app.sendMessage(serialized);
|
|
3205
|
+
}
|
|
3206
|
+
catch (err) {
|
|
3207
|
+
console.error(`${LOG_TAG}: sendToMiniapp error for ${packageName}:`, err);
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
/**
|
|
3211
|
+
* Send a REQUEST_RESULT response.
|
|
3212
|
+
*/
|
|
3213
|
+
// ===========================================================================
|
|
3214
|
+
// Inter-miniapp interop (session.miniapps + session.actions.invoke)
|
|
3215
|
+
// ===========================================================================
|
|
3216
|
+
/** Max serialized size of an action call's params / result (256 KB). */
|
|
3217
|
+
static ACTION_PAYLOAD_CAP = 256 * 1024;
|
|
3218
|
+
/** Outstanding action invocations, keyed by host-generated callId. */
|
|
3219
|
+
actionCalls = new Map();
|
|
3220
|
+
actionCallSeq = 0;
|
|
3221
|
+
interopApps() {
|
|
3222
|
+
return useAppStatusStore.getState().apps;
|
|
3223
|
+
}
|
|
3224
|
+
isSystemPackage(packageName) {
|
|
3225
|
+
if (SYSTEM_MINIAPP_PACKAGES.has(packageName))
|
|
3226
|
+
return true;
|
|
3227
|
+
return this.interopApps().find((app) => app.packageName === packageName)?.isMiniappDev === true;
|
|
3228
|
+
}
|
|
3229
|
+
async startInteropApp(packageName) {
|
|
3230
|
+
const app = this.interopApps().find((a) => a.packageName === packageName);
|
|
3231
|
+
if (!app)
|
|
3232
|
+
return false;
|
|
3233
|
+
if (app.local) {
|
|
3234
|
+
await miniappLauncher.ensureConnected(packageName);
|
|
3235
|
+
return true;
|
|
3236
|
+
}
|
|
3237
|
+
return useAppStatusStore.getState().start(app, { skipNavigation: true });
|
|
3238
|
+
}
|
|
3239
|
+
stopInteropApp(packageName) {
|
|
3240
|
+
return useAppStatusStore.getState().stop(packageName);
|
|
3241
|
+
}
|
|
3242
|
+
wakeInteropApp(packageName) {
|
|
3243
|
+
return miniappLauncher.ensureConnected(packageName);
|
|
3244
|
+
}
|
|
3245
|
+
/** Emit an interop audit event (best-effort — never let telemetry break a call). */
|
|
3246
|
+
auditInterop(event) {
|
|
3247
|
+
try {
|
|
3248
|
+
getAnalytics()?.("miniapp_interop", {
|
|
3249
|
+
caller: event.caller,
|
|
3250
|
+
op: event.op,
|
|
3251
|
+
target: event.target ?? "",
|
|
3252
|
+
actionId: event.actionId ?? "",
|
|
3253
|
+
ok: event.ok,
|
|
3254
|
+
errorCode: event.errorCode ?? "",
|
|
3255
|
+
});
|
|
3256
|
+
}
|
|
3257
|
+
catch {
|
|
3258
|
+
/* telemetry must never break an interop call */
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
3261
|
+
/** Reject the request with NOT_PERMITTED unless the caller is a system app. */
|
|
3262
|
+
requireSystemCaller(packageName, requestId, op, target) {
|
|
3263
|
+
if (this.isSystemPackage(packageName))
|
|
3264
|
+
return true;
|
|
3265
|
+
this.auditInterop({ caller: packageName, op, target, ok: false, errorCode: MiniappErrorCode.NOT_PERMITTED });
|
|
3266
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3267
|
+
code: MiniappErrorCode.NOT_PERMITTED,
|
|
3268
|
+
message: "Inter-miniapp APIs are restricted to system apps",
|
|
3269
|
+
});
|
|
3270
|
+
return false;
|
|
3271
|
+
}
|
|
3272
|
+
/** Project a host ClientApp into the SDK's MiniappInfo shape (JSON over the wire). */
|
|
3273
|
+
buildMiniappInfo(app) {
|
|
3274
|
+
const compatibility = app.compatibility ?? {
|
|
3275
|
+
isCompatible: true,
|
|
3276
|
+
missingRequired: [],
|
|
3277
|
+
missingOptional: [],
|
|
3278
|
+
warnings: [],
|
|
3279
|
+
};
|
|
3280
|
+
return {
|
|
3281
|
+
packageName: app.packageName,
|
|
3282
|
+
name: app.name,
|
|
3283
|
+
version: app.version ?? "",
|
|
3284
|
+
running: app.running,
|
|
3285
|
+
compatibility,
|
|
3286
|
+
actions: app.actions ?? [],
|
|
3287
|
+
};
|
|
3288
|
+
}
|
|
3289
|
+
handleMiniappsList(packageName, payload, requestId) {
|
|
3290
|
+
if (!this.requireSystemCaller(packageName, requestId, "list"))
|
|
3291
|
+
return;
|
|
3292
|
+
const includeIncompatible = payload.includeIncompatible === true;
|
|
3293
|
+
const infos = this.interopApps()
|
|
3294
|
+
.filter((a) => a.packageName &&
|
|
3295
|
+
a.packageName !== packageName &&
|
|
3296
|
+
!a.packageName.includes("@empty") &&
|
|
3297
|
+
(includeIncompatible || a.compatibility?.isCompatible !== false))
|
|
3298
|
+
.map((a) => this.buildMiniappInfo(a));
|
|
3299
|
+
this.sendResult(packageName, requestId, true, infos);
|
|
3300
|
+
this.auditInterop({ caller: packageName, op: "list", ok: true });
|
|
3301
|
+
}
|
|
3302
|
+
async handleMiniappsStart(packageName, payload, requestId) {
|
|
3303
|
+
const target = payload.packageName;
|
|
3304
|
+
if (!this.requireSystemCaller(packageName, requestId, "start", target))
|
|
3305
|
+
return;
|
|
3306
|
+
const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined;
|
|
3307
|
+
if (!target || !app) {
|
|
3308
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3309
|
+
code: MiniappErrorCode.APP_NOT_FOUND,
|
|
3310
|
+
message: `Miniapp not found: ${target ?? "(missing packageName)"}`,
|
|
3311
|
+
});
|
|
3312
|
+
this.auditInterop({
|
|
3313
|
+
caller: packageName,
|
|
3314
|
+
op: "start",
|
|
3315
|
+
target,
|
|
3316
|
+
ok: false,
|
|
3317
|
+
errorCode: MiniappErrorCode.APP_NOT_FOUND,
|
|
3318
|
+
});
|
|
3319
|
+
return;
|
|
3320
|
+
}
|
|
3321
|
+
// Pre-flight the hardware gate so the caller gets a precise reason rather
|
|
3322
|
+
// than a generic rejection (start()'s own gate would otherwise alert + abort).
|
|
3323
|
+
if (app.compatibility && app.compatibility.isCompatible === false) {
|
|
3324
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3325
|
+
code: MiniappErrorCode.APP_NOT_COMPATIBLE,
|
|
3326
|
+
message: `${target} is not compatible with the connected glasses`,
|
|
3327
|
+
});
|
|
3328
|
+
islandNotifications.emit({
|
|
3329
|
+
kind: "version_incompatible",
|
|
3330
|
+
packageName: target,
|
|
3331
|
+
reason: `${target} is not compatible with the connected glasses`,
|
|
3332
|
+
metadata: { missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.start" },
|
|
3333
|
+
timestamp: Date.now(),
|
|
3334
|
+
});
|
|
3335
|
+
this.auditInterop({
|
|
3336
|
+
caller: packageName,
|
|
3337
|
+
op: "start",
|
|
3338
|
+
target,
|
|
3339
|
+
ok: false,
|
|
3340
|
+
errorCode: MiniappErrorCode.APP_NOT_COMPATIBLE,
|
|
3341
|
+
});
|
|
3342
|
+
return;
|
|
3343
|
+
}
|
|
3344
|
+
try {
|
|
3345
|
+
// startInteropApp resolves to false when the app-store gate rejected the
|
|
3346
|
+
// launch or the background context failed to spawn — don't report success.
|
|
3347
|
+
const started = await this.startInteropApp(target);
|
|
3348
|
+
if (started) {
|
|
3349
|
+
this.sendResult(packageName, requestId, true);
|
|
3350
|
+
this.auditInterop({ caller: packageName, op: "start", target, ok: true });
|
|
3351
|
+
}
|
|
3352
|
+
else {
|
|
3353
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3354
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3355
|
+
message: `start of ${target} was rejected by the host (gated or failed to spawn)`,
|
|
3356
|
+
});
|
|
3357
|
+
this.auditInterop({
|
|
3358
|
+
caller: packageName,
|
|
3359
|
+
op: "start",
|
|
3360
|
+
target,
|
|
3361
|
+
ok: false,
|
|
3362
|
+
errorCode: MiniappErrorCode.INTERNAL,
|
|
3363
|
+
});
|
|
3364
|
+
}
|
|
3365
|
+
}
|
|
3366
|
+
catch (e) {
|
|
3367
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3368
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3369
|
+
message: e?.message ?? "start failed",
|
|
3370
|
+
});
|
|
3371
|
+
this.auditInterop({
|
|
3372
|
+
caller: packageName,
|
|
3373
|
+
op: "start",
|
|
3374
|
+
target,
|
|
3375
|
+
ok: false,
|
|
3376
|
+
errorCode: MiniappErrorCode.INTERNAL,
|
|
3377
|
+
});
|
|
3378
|
+
}
|
|
3379
|
+
}
|
|
3380
|
+
async handleMiniappsStop(packageName, payload, requestId) {
|
|
3381
|
+
const target = payload.packageName;
|
|
3382
|
+
if (!this.requireSystemCaller(packageName, requestId, "stop", target))
|
|
3383
|
+
return;
|
|
3384
|
+
if (!target) {
|
|
3385
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3386
|
+
code: MiniappErrorCode.APP_NOT_FOUND,
|
|
3387
|
+
message: "stop requires a packageName",
|
|
3388
|
+
});
|
|
3389
|
+
return;
|
|
3390
|
+
}
|
|
3391
|
+
try {
|
|
3392
|
+
await this.stopInteropApp(target);
|
|
3393
|
+
this.sendResult(packageName, requestId, true);
|
|
3394
|
+
this.auditInterop({ caller: packageName, op: "stop", target, ok: true });
|
|
3395
|
+
}
|
|
3396
|
+
catch (e) {
|
|
3397
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3398
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3399
|
+
message: e?.message ?? "stop failed",
|
|
3400
|
+
});
|
|
3401
|
+
}
|
|
3402
|
+
}
|
|
3403
|
+
/** True iff `value` serializes larger than the action payload cap. */
|
|
3404
|
+
actionPayloadTooLarge(value) {
|
|
3405
|
+
try {
|
|
3406
|
+
return JSON.stringify(value ?? null).length > LocalMiniappRuntime.ACTION_PAYLOAD_CAP;
|
|
3407
|
+
}
|
|
3408
|
+
catch {
|
|
3409
|
+
// Unserializable — let it through; the transport's own serialize handles it.
|
|
3410
|
+
return false;
|
|
3411
|
+
}
|
|
3412
|
+
}
|
|
3413
|
+
/**
|
|
3414
|
+
* session.actions.invoke → wake the target headlessly, deliver an ACTION_CALL,
|
|
3415
|
+
* and correlate its ACTION_RESULT back to the caller's pending request.
|
|
3416
|
+
*/
|
|
3417
|
+
async handleActionInvoke(callerPackageName, payload, requestId) {
|
|
3418
|
+
const targetForGate = payload.targetPackageName;
|
|
3419
|
+
if (!this.requireSystemCaller(callerPackageName, requestId, "invoke", targetForGate))
|
|
3420
|
+
return;
|
|
3421
|
+
const target = payload.targetPackageName;
|
|
3422
|
+
const actionId = payload.actionId;
|
|
3423
|
+
const params = payload.params ?? {};
|
|
3424
|
+
// Floor the timeout at 6s so the host never rejects with ACTION_TIMEOUT before
|
|
3425
|
+
// the target SDK's handler-registration window closes. A freshly-woken miniapp
|
|
3426
|
+
// buffers an undelivered ACTION_CALL for HANDLER_WAIT_MS (5s in @mentra/miniapp's
|
|
3427
|
+
// actions module) waiting for session.actions.handle; a shorter host timeout
|
|
3428
|
+
// could fire while the target is still registering, then the action would run
|
|
3429
|
+
// with no caller left to receive its result. 6s = that 5s buffer + 1s for the
|
|
3430
|
+
// ACTION_RESULT/NO_ACTION_HANDLER reply to travel back. Keep these in sync.
|
|
3431
|
+
const timeoutMs = Math.min(Math.max(Number(payload.timeoutMs) || 30_000, 6_000), 120_000);
|
|
3432
|
+
if (this.actionPayloadTooLarge(params)) {
|
|
3433
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3434
|
+
code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
|
|
3435
|
+
message: "action params exceeded the 256 KB cap",
|
|
3436
|
+
});
|
|
3437
|
+
return;
|
|
3438
|
+
}
|
|
3439
|
+
const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined;
|
|
3440
|
+
if (!target || !app) {
|
|
3441
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3442
|
+
code: MiniappErrorCode.APP_NOT_FOUND,
|
|
3443
|
+
message: `Miniapp not found: ${target ?? "(missing targetPackageName)"}`,
|
|
3444
|
+
});
|
|
3445
|
+
return;
|
|
3446
|
+
}
|
|
3447
|
+
if (!actionId) {
|
|
3448
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3449
|
+
code: MiniappErrorCode.ACTION_NOT_FOUND,
|
|
3450
|
+
message: "invoke requires an actionId",
|
|
3451
|
+
});
|
|
3452
|
+
return;
|
|
3453
|
+
}
|
|
3454
|
+
if (app.compatibility && app.compatibility.isCompatible === false) {
|
|
3455
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3456
|
+
code: MiniappErrorCode.APP_NOT_COMPATIBLE,
|
|
3457
|
+
message: `${target} is not compatible with the connected glasses`,
|
|
3458
|
+
});
|
|
3459
|
+
islandNotifications.emit({
|
|
3460
|
+
kind: "version_incompatible",
|
|
3461
|
+
packageName: target,
|
|
3462
|
+
reason: `${target} is not compatible with the connected glasses`,
|
|
3463
|
+
metadata: { missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.invoke" },
|
|
3464
|
+
timestamp: Date.now(),
|
|
3465
|
+
});
|
|
3466
|
+
return;
|
|
3467
|
+
}
|
|
3468
|
+
// Declared-action gate — only once the host populates app.actions (Phase 2);
|
|
3469
|
+
// until then app.actions is undefined and we fall through to NO_ACTION_HANDLER.
|
|
3470
|
+
if (!(app.actions ?? []).some((a) => a.id === actionId)) {
|
|
3471
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3472
|
+
code: MiniappErrorCode.ACTION_NOT_FOUND,
|
|
3473
|
+
message: `${target} does not declare action "${actionId}"`,
|
|
3474
|
+
});
|
|
3475
|
+
return;
|
|
3476
|
+
}
|
|
3477
|
+
// Headless wake + wait for CONNECT (idempotent / fast if already connected).
|
|
3478
|
+
try {
|
|
3479
|
+
await this.wakeInteropApp(target);
|
|
3480
|
+
}
|
|
3481
|
+
catch (e) {
|
|
3482
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3483
|
+
code: MiniappErrorCode.WAKE_FAILED,
|
|
3484
|
+
message: e?.message ?? `failed to wake ${target}`,
|
|
3485
|
+
});
|
|
3486
|
+
return;
|
|
3487
|
+
}
|
|
3488
|
+
// The wake waited for CONNECT, but the target could have dropped in the gap
|
|
3489
|
+
// before delivery — fail fast rather than arming a timer for a call that
|
|
3490
|
+
// sendToMiniapp would silently drop (the caller would otherwise wait the
|
|
3491
|
+
// full invoke timeout for an ACTION_CALL that was never delivered).
|
|
3492
|
+
if (!this.connectedApps.has(target)) {
|
|
3493
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3494
|
+
code: MiniappErrorCode.WAKE_FAILED,
|
|
3495
|
+
message: `${target} disconnected before the action could be delivered`,
|
|
3496
|
+
});
|
|
3497
|
+
this.auditInterop({
|
|
3498
|
+
caller: callerPackageName,
|
|
3499
|
+
op: "invoke",
|
|
3500
|
+
target,
|
|
3501
|
+
actionId,
|
|
3502
|
+
ok: false,
|
|
3503
|
+
errorCode: MiniappErrorCode.WAKE_FAILED,
|
|
3504
|
+
});
|
|
3505
|
+
return;
|
|
3506
|
+
}
|
|
3507
|
+
// Deliver the call and arm the handler timeout; ACTION_RESULT resolves it.
|
|
3508
|
+
const callId = `act-${Date.now().toString(36)}-${this.actionCallSeq++}`;
|
|
3509
|
+
const timer = BgTimer.setTimeout(() => {
|
|
3510
|
+
this.actionCalls.delete(callId);
|
|
3511
|
+
this.sendResult(callerPackageName, requestId, false, undefined, {
|
|
3512
|
+
code: MiniappErrorCode.ACTION_TIMEOUT,
|
|
3513
|
+
message: `action "${actionId}" timed out after ${timeoutMs}ms`,
|
|
3514
|
+
});
|
|
3515
|
+
}, timeoutMs);
|
|
3516
|
+
this.actionCalls.set(callId, { callerPackageName, targetPackageName: target, callerRequestId: requestId, timer });
|
|
3517
|
+
this.sendToMiniapp(target, {
|
|
3518
|
+
type: MiniappResponseType.ACTION_CALL,
|
|
3519
|
+
callId,
|
|
3520
|
+
actionId,
|
|
3521
|
+
params,
|
|
3522
|
+
callerPackageName,
|
|
3523
|
+
});
|
|
3524
|
+
this.auditInterop({ caller: callerPackageName, op: "invoke", target, actionId, ok: true });
|
|
3525
|
+
}
|
|
3526
|
+
/** Target → host: forward an ACTION_RESULT back to the caller's pending invoke. */
|
|
3527
|
+
handleActionResult(targetPackageName, payload) {
|
|
3528
|
+
const callId = payload.callId;
|
|
3529
|
+
if (!callId)
|
|
3530
|
+
return;
|
|
3531
|
+
const pending = this.actionCalls.get(callId);
|
|
3532
|
+
if (!pending)
|
|
3533
|
+
return;
|
|
3534
|
+
// Only the invoked target may resolve this call. callIds are guessable
|
|
3535
|
+
// (timestamp + seq), so a different connected miniapp must not be able to
|
|
3536
|
+
// spoof a result/error for someone else's invoke.
|
|
3537
|
+
if (pending.targetPackageName !== targetPackageName)
|
|
3538
|
+
return;
|
|
3539
|
+
this.actionCalls.delete(callId);
|
|
3540
|
+
BgTimer.clearTimeout(pending.timer);
|
|
3541
|
+
if (payload.ok === true) {
|
|
3542
|
+
const result = payload.result;
|
|
3543
|
+
if (this.actionPayloadTooLarge(result)) {
|
|
3544
|
+
this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, {
|
|
3545
|
+
code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
|
|
3546
|
+
message: "action result exceeded the 256 KB cap",
|
|
3547
|
+
});
|
|
3548
|
+
return;
|
|
3549
|
+
}
|
|
3550
|
+
this.sendResult(pending.callerPackageName, pending.callerRequestId, true, result ?? null);
|
|
3551
|
+
}
|
|
3552
|
+
else {
|
|
3553
|
+
const error = payload.error ?? {
|
|
3554
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3555
|
+
message: "action handler error",
|
|
3556
|
+
};
|
|
3557
|
+
this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, error);
|
|
3558
|
+
}
|
|
3559
|
+
}
|
|
3560
|
+
sendResult(packageName, requestId, ok, data, error) {
|
|
3561
|
+
if (!requestId)
|
|
3562
|
+
return;
|
|
3563
|
+
this.sendToMiniapp(packageName, {
|
|
3564
|
+
type: MiniappResponseType.REQUEST_RESULT,
|
|
3565
|
+
requestId,
|
|
3566
|
+
ok,
|
|
3567
|
+
...(data !== undefined ? { data } : {}),
|
|
3568
|
+
...(error ? { error } : {}),
|
|
3569
|
+
}, requestId);
|
|
3570
|
+
}
|
|
3571
|
+
/**
|
|
3572
|
+
* Send a VISIBILITY_CHANGE push to a miniapp.
|
|
3573
|
+
*/
|
|
3574
|
+
sendVisibilityChange(packageName, visibility) {
|
|
3575
|
+
this.sendToMiniapp(packageName, {
|
|
3576
|
+
type: MiniappResponseType.VISIBILITY_CHANGE,
|
|
3577
|
+
visibility,
|
|
3578
|
+
});
|
|
3579
|
+
}
|
|
3580
|
+
/**
|
|
3581
|
+
* Fast foreground liveness check used when a local miniapp UI is opened or
|
|
3582
|
+
* resumed. The normal watchdog intentionally waits ~30s to avoid killing a
|
|
3583
|
+
* healthy-but-busy background script. When the user is actively looking at a
|
|
3584
|
+
* WebView, a stale JSContext reads as "cloud offline" / no captions, so probe
|
|
3585
|
+
* immediately and reuse the crash-respawn path if no message comes back.
|
|
3586
|
+
*/
|
|
3587
|
+
probeForegroundLiveness(packageName, reason = "foreground-open", timeoutMs = FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS) {
|
|
3588
|
+
const app = this.connectedApps.get(packageName);
|
|
3589
|
+
if (!app)
|
|
3590
|
+
return;
|
|
3591
|
+
this.clearForegroundProbe(packageName);
|
|
3592
|
+
const probeStartedAt = Date.now();
|
|
3593
|
+
this.sendToMiniapp(packageName, {
|
|
3594
|
+
type: MiniappRequestType.PING,
|
|
3595
|
+
});
|
|
3596
|
+
const timerId = BgTimer.setTimeout(() => {
|
|
3597
|
+
this.foregroundProbeTimers.delete(packageName);
|
|
3598
|
+
const current = this.connectedApps.get(packageName);
|
|
3599
|
+
if (!current)
|
|
3600
|
+
return;
|
|
3601
|
+
if (current.lastPongAt >= probeStartedAt)
|
|
3602
|
+
return;
|
|
3603
|
+
console.warn(`${LOG_TAG}: ${packageName} failed foreground liveness probe (${reason}), respawning`);
|
|
3604
|
+
this.unregisterApp(packageName);
|
|
3605
|
+
this.onLivenessTimeout?.(packageName);
|
|
3606
|
+
}, timeoutMs);
|
|
3607
|
+
this.foregroundProbeTimers.set(packageName, timerId);
|
|
3608
|
+
}
|
|
3609
|
+
// ===========================================================================
|
|
3610
|
+
// Ping / pong liveness
|
|
3611
|
+
// ===========================================================================
|
|
3612
|
+
ensurePingLoop() {
|
|
3613
|
+
if (this.pingIntervalId !== null)
|
|
3614
|
+
return;
|
|
3615
|
+
this.pingIntervalId = BgTimer.setInterval(() => {
|
|
3616
|
+
this.doPingRound();
|
|
3617
|
+
}, PING_INTERVAL_MS);
|
|
3618
|
+
}
|
|
3619
|
+
stopPingLoop() {
|
|
3620
|
+
if (this.pingIntervalId !== null) {
|
|
3621
|
+
BgTimer.clearInterval(this.pingIntervalId);
|
|
3622
|
+
this.pingIntervalId = null;
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
doPingRound() {
|
|
3626
|
+
const now = Date.now();
|
|
3627
|
+
const staleThreshold = PING_INTERVAL_MS * PING_TIMEOUT_THRESHOLD;
|
|
3628
|
+
const toRemove = [];
|
|
3629
|
+
for (const [packageName, app] of this.connectedApps) {
|
|
3630
|
+
if (now - app.lastPongAt > staleThreshold) {
|
|
3631
|
+
console.warn(`${LOG_TAG}: ${packageName} missed ${PING_TIMEOUT_THRESHOLD} pings, unregistering`);
|
|
3632
|
+
toRemove.push(packageName);
|
|
3633
|
+
continue;
|
|
3634
|
+
}
|
|
3635
|
+
// Send PING — SDK auto-replies with PONG
|
|
3636
|
+
this.sendToMiniapp(packageName, {
|
|
3637
|
+
type: MiniappRequestType.PING,
|
|
3638
|
+
});
|
|
3639
|
+
}
|
|
3640
|
+
for (const pkg of toRemove) {
|
|
3641
|
+
this.unregisterApp(pkg);
|
|
3642
|
+
// Hand the package to the router's respawn machinery — a missed-pings
|
|
3643
|
+
// death is treated like a crash so the background script comes back.
|
|
3644
|
+
this.onLivenessTimeout?.(pkg);
|
|
3645
|
+
}
|
|
3646
|
+
}
|
|
3647
|
+
/**
|
|
3648
|
+
* Called when a PONG is received from a miniapp (or any message, really).
|
|
3649
|
+
* Updates lastPongAt to keep the app alive.
|
|
3650
|
+
*/
|
|
3651
|
+
handlePong(packageName) {
|
|
3652
|
+
const app = this.connectedApps.get(packageName);
|
|
3653
|
+
if (app) {
|
|
3654
|
+
app.lastPongAt = Date.now();
|
|
3655
|
+
this.clearForegroundProbe(packageName);
|
|
3656
|
+
}
|
|
3657
|
+
}
|
|
3658
|
+
clearForegroundProbe(packageName) {
|
|
3659
|
+
const timerId = this.foregroundProbeTimers.get(packageName);
|
|
3660
|
+
if (timerId == null)
|
|
3661
|
+
return;
|
|
3662
|
+
BgTimer.clearTimeout(timerId);
|
|
3663
|
+
this.foregroundProbeTimers.delete(packageName);
|
|
3664
|
+
}
|
|
3665
|
+
// ===========================================================================
|
|
3666
|
+
// Lifecycle
|
|
3667
|
+
// ===========================================================================
|
|
3668
|
+
cleanup() {
|
|
3669
|
+
console.log(`${LOG_TAG}: cleanup()`);
|
|
3670
|
+
this.stopPingLoop();
|
|
3671
|
+
// Copy keys since unregisterApp mutates the map
|
|
3672
|
+
const packageNames = [...this.connectedApps.keys()];
|
|
3673
|
+
for (const pkg of packageNames) {
|
|
3674
|
+
this.unregisterApp(pkg);
|
|
3675
|
+
}
|
|
3676
|
+
// Belt-and-suspenders: clear any remaining state
|
|
3677
|
+
this.pendingCloudRequests.clear();
|
|
3678
|
+
this.streamSubscribers.clear();
|
|
3679
|
+
this.connectedApps.clear();
|
|
3680
|
+
for (const timerId of this.foregroundProbeTimers.values()) {
|
|
3681
|
+
BgTimer.clearTimeout(timerId);
|
|
3682
|
+
}
|
|
3683
|
+
this.foregroundProbeTimers.clear();
|
|
3684
|
+
LocalMiniappRuntime.instance = null;
|
|
3685
|
+
}
|
|
3686
|
+
/**
|
|
3687
|
+
* Number of currently connected miniapps (for diagnostics).
|
|
3688
|
+
*/
|
|
3689
|
+
get connectedAppCount() {
|
|
3690
|
+
return this.connectedApps.size;
|
|
3691
|
+
}
|
|
3692
|
+
}
|
|
3693
|
+
const localMiniappRuntime = LocalMiniappRuntime.getInstance();
|
|
3694
|
+
export default localMiniappRuntime;
|
|
3695
|
+
//# sourceMappingURL=LocalMiniappRuntime.js.map
|