@mentra/engine 0.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (842) hide show
  1. package/README.md +79 -0
  2. package/build/devtools.d.ts +11 -0
  3. package/build/devtools.d.ts.map +1 -0
  4. package/build/devtools.js +13 -0
  5. package/build/devtools.js.map +1 -0
  6. package/build/engine.d.ts +261 -0
  7. package/build/engine.d.ts.map +1 -0
  8. package/build/engine.js +170 -0
  9. package/build/engine.js.map +1 -0
  10. package/build/facades/dev.d.ts +58 -0
  11. package/build/facades/dev.d.ts.map +1 -0
  12. package/build/facades/dev.js +189 -0
  13. package/build/facades/dev.js.map +1 -0
  14. package/build/facades/display.d.ts +21 -0
  15. package/build/facades/display.d.ts.map +1 -0
  16. package/build/facades/display.js +17 -0
  17. package/build/facades/display.js.map +1 -0
  18. package/build/facades/displayMirror.d.ts +15 -0
  19. package/build/facades/displayMirror.d.ts.map +1 -0
  20. package/build/facades/displayMirror.js +29 -0
  21. package/build/facades/displayMirror.js.map +1 -0
  22. package/build/facades/gallery.d.ts +62 -0
  23. package/build/facades/gallery.d.ts.map +1 -0
  24. package/build/facades/gallery.js +61 -0
  25. package/build/facades/gallery.js.map +1 -0
  26. package/build/facades/glasses.d.ts +141 -0
  27. package/build/facades/glasses.d.ts.map +1 -0
  28. package/build/facades/glasses.js +199 -0
  29. package/build/facades/glasses.js.map +1 -0
  30. package/build/facades/glassesSettings.d.ts +13 -0
  31. package/build/facades/glassesSettings.d.ts.map +1 -0
  32. package/build/facades/glassesSettings.js +54 -0
  33. package/build/facades/glassesSettings.js.map +1 -0
  34. package/build/facades/glassesWifi.d.ts +25 -0
  35. package/build/facades/glassesWifi.d.ts.map +1 -0
  36. package/build/facades/glassesWifi.js +57 -0
  37. package/build/facades/glassesWifi.js.map +1 -0
  38. package/build/facades/miniapps.d.ts +43 -0
  39. package/build/facades/miniapps.d.ts.map +1 -0
  40. package/build/facades/miniapps.js +49 -0
  41. package/build/facades/miniapps.js.map +1 -0
  42. package/build/facades/notifications.d.ts +13 -0
  43. package/build/facades/notifications.d.ts.map +1 -0
  44. package/build/facades/notifications.js +12 -0
  45. package/build/facades/notifications.js.map +1 -0
  46. package/build/facades/ota.d.ts +103 -0
  47. package/build/facades/ota.d.ts.map +1 -0
  48. package/build/facades/ota.js +156 -0
  49. package/build/facades/ota.js.map +1 -0
  50. package/build/facades/pairing.d.ts +114 -0
  51. package/build/facades/pairing.d.ts.map +1 -0
  52. package/build/facades/pairing.js +275 -0
  53. package/build/facades/pairing.js.map +1 -0
  54. package/build/facades/permissions.d.ts +33 -0
  55. package/build/facades/permissions.d.ts.map +1 -0
  56. package/build/facades/permissions.js +155 -0
  57. package/build/facades/permissions.js.map +1 -0
  58. package/build/facades/phoneNotifications.d.ts +21 -0
  59. package/build/facades/phoneNotifications.d.ts.map +1 -0
  60. package/build/facades/phoneNotifications.js +51 -0
  61. package/build/facades/phoneNotifications.js.map +1 -0
  62. package/build/facades/reports.d.ts +36 -0
  63. package/build/facades/reports.d.ts.map +1 -0
  64. package/build/facades/reports.js +134 -0
  65. package/build/facades/reports.js.map +1 -0
  66. package/build/facades/session.d.ts +21 -0
  67. package/build/facades/session.d.ts.map +1 -0
  68. package/build/facades/session.js +52 -0
  69. package/build/facades/session.js.map +1 -0
  70. package/build/facades/settings.d.ts +37 -0
  71. package/build/facades/settings.d.ts.map +1 -0
  72. package/build/facades/settings.js +55 -0
  73. package/build/facades/settings.js.map +1 -0
  74. package/build/facades/speech.d.ts +30 -0
  75. package/build/facades/speech.d.ts.map +1 -0
  76. package/build/facades/speech.js +37 -0
  77. package/build/facades/speech.js.map +1 -0
  78. package/build/index.d.ts +43 -0
  79. package/build/index.d.ts.map +1 -0
  80. package/build/index.js +45 -0
  81. package/build/index.js.map +1 -0
  82. package/build/internal.d.ts +67 -0
  83. package/build/internal.d.ts.map +1 -0
  84. package/build/internal.js +90 -0
  85. package/build/internal.js.map +1 -0
  86. package/build/runtime/bootstrap.d.ts +83 -0
  87. package/build/runtime/bootstrap.d.ts.map +1 -0
  88. package/build/runtime/bootstrap.js +56 -0
  89. package/build/runtime/bootstrap.js.map +1 -0
  90. package/build/runtime/config.d.ts +146 -0
  91. package/build/runtime/config.d.ts.map +1 -0
  92. package/build/runtime/config.js +18 -0
  93. package/build/runtime/config.js.map +1 -0
  94. package/build/runtime/streamConfig.d.ts +4 -0
  95. package/build/runtime/streamConfig.d.ts.map +1 -0
  96. package/build/runtime/streamConfig.js +38 -0
  97. package/build/runtime/streamConfig.js.map +1 -0
  98. package/build/services/AppRegistry.d.ts +269 -0
  99. package/build/services/AppRegistry.d.ts.map +1 -0
  100. package/build/services/AppRegistry.js +1079 -0
  101. package/build/services/AppRegistry.js.map +1 -0
  102. package/build/services/AudioCloudUplink.d.ts +9 -0
  103. package/build/services/AudioCloudUplink.d.ts.map +1 -0
  104. package/build/services/AudioCloudUplink.js +48 -0
  105. package/build/services/AudioCloudUplink.js.map +1 -0
  106. package/build/services/AudioPlaybackService.d.ts +175 -0
  107. package/build/services/AudioPlaybackService.d.ts.map +1 -0
  108. package/build/services/AudioPlaybackService.js +705 -0
  109. package/build/services/AudioPlaybackService.js.map +1 -0
  110. package/build/services/AutomaticReportResult.d.ts +20 -0
  111. package/build/services/AutomaticReportResult.d.ts.map +1 -0
  112. package/build/services/AutomaticReportResult.js +29 -0
  113. package/build/services/AutomaticReportResult.js.map +1 -0
  114. package/build/services/BlobStore.d.ts +91 -0
  115. package/build/services/BlobStore.d.ts.map +1 -0
  116. package/build/services/BlobStore.js +668 -0
  117. package/build/services/BlobStore.js.map +1 -0
  118. package/build/services/CaptionsTesterReportService.d.ts +4 -0
  119. package/build/services/CaptionsTesterReportService.d.ts.map +1 -0
  120. package/build/services/CaptionsTesterReportService.js +80 -0
  121. package/build/services/CaptionsTesterReportService.js.map +1 -0
  122. package/build/services/CloudAudioSubscriptionSync.d.ts +24 -0
  123. package/build/services/CloudAudioSubscriptionSync.d.ts.map +1 -0
  124. package/build/services/CloudAudioSubscriptionSync.js +39 -0
  125. package/build/services/CloudAudioSubscriptionSync.js.map +1 -0
  126. package/build/services/CloudClientService.d.ts +143 -0
  127. package/build/services/CloudClientService.d.ts.map +1 -0
  128. package/build/services/CloudClientService.js +644 -0
  129. package/build/services/CloudClientService.js.map +1 -0
  130. package/build/services/CloudTranscriptE2EMetrics.d.ts +4 -0
  131. package/build/services/CloudTranscriptE2EMetrics.d.ts.map +1 -0
  132. package/build/services/CloudTranscriptE2EMetrics.js +24 -0
  133. package/build/services/CloudTranscriptE2EMetrics.js.map +1 -0
  134. package/build/services/ConnectionCoordinator.d.ts +51 -0
  135. package/build/services/ConnectionCoordinator.d.ts.map +1 -0
  136. package/build/services/ConnectionCoordinator.js +34 -0
  137. package/build/services/ConnectionCoordinator.js.map +1 -0
  138. package/build/services/DevServerBridge.d.ts +63 -0
  139. package/build/services/DevServerBridge.d.ts.map +1 -0
  140. package/build/services/DevServerBridge.js +294 -0
  141. package/build/services/DevServerBridge.js.map +1 -0
  142. package/build/services/DeviceEventRouter.d.ts +3 -0
  143. package/build/services/DeviceEventRouter.d.ts.map +1 -0
  144. package/build/services/DeviceEventRouter.js +190 -0
  145. package/build/services/DeviceEventRouter.js.map +1 -0
  146. package/build/services/DeviceStoreHydration.d.ts +38 -0
  147. package/build/services/DeviceStoreHydration.d.ts.map +1 -0
  148. package/build/services/DeviceStoreHydration.js +124 -0
  149. package/build/services/DeviceStoreHydration.js.map +1 -0
  150. package/build/services/DisplayProcessor.d.ts +223 -0
  151. package/build/services/DisplayProcessor.d.ts.map +1 -0
  152. package/build/services/DisplayProcessor.js +552 -0
  153. package/build/services/DisplayProcessor.js.map +1 -0
  154. package/build/services/GlassesReadiness.d.ts +41 -0
  155. package/build/services/GlassesReadiness.d.ts.map +1 -0
  156. package/build/services/GlassesReadiness.js +82 -0
  157. package/build/services/GlassesReadiness.js.map +1 -0
  158. package/build/services/GlassesSettingsSync.d.ts +33 -0
  159. package/build/services/GlassesSettingsSync.d.ts.map +1 -0
  160. package/build/services/GlassesSettingsSync.js +121 -0
  161. package/build/services/GlassesSettingsSync.js.map +1 -0
  162. package/build/services/GlassesStatusProjection.d.ts +3 -0
  163. package/build/services/GlassesStatusProjection.d.ts.map +1 -0
  164. package/build/services/GlassesStatusProjection.js +65 -0
  165. package/build/services/GlassesStatusProjection.js.map +1 -0
  166. package/build/services/HeadingService.d.ts +27 -0
  167. package/build/services/HeadingService.d.ts.map +1 -0
  168. package/build/services/HeadingService.js +96 -0
  169. package/build/services/HeadingService.js.map +1 -0
  170. package/build/services/LocalDisplayManager.d.ts +139 -0
  171. package/build/services/LocalDisplayManager.d.ts.map +1 -0
  172. package/build/services/LocalDisplayManager.js +579 -0
  173. package/build/services/LocalDisplayManager.js.map +1 -0
  174. package/build/services/LocalMiniappRuntime.d.ts +439 -0
  175. package/build/services/LocalMiniappRuntime.d.ts.map +1 -0
  176. package/build/services/LocalMiniappRuntime.js +3695 -0
  177. package/build/services/LocalMiniappRuntime.js.map +1 -0
  178. package/build/services/LocalSttFallbackCoordinator.d.ts +57 -0
  179. package/build/services/LocalSttFallbackCoordinator.d.ts.map +1 -0
  180. package/build/services/LocalSttFallbackCoordinator.js +137 -0
  181. package/build/services/LocalSttFallbackCoordinator.js.map +1 -0
  182. package/build/services/MentraJSCrashController.d.ts +97 -0
  183. package/build/services/MentraJSCrashController.d.ts.map +1 -0
  184. package/build/services/MentraJSCrashController.js +138 -0
  185. package/build/services/MentraJSCrashController.js.map +1 -0
  186. package/build/services/MentraJSCrashloopReportService.d.ts +9 -0
  187. package/build/services/MentraJSCrashloopReportService.d.ts.map +1 -0
  188. package/build/services/MentraJSCrashloopReportService.js +54 -0
  189. package/build/services/MentraJSCrashloopReportService.js.map +1 -0
  190. package/build/services/MentraJSLogPipeline.d.ts +87 -0
  191. package/build/services/MentraJSLogPipeline.d.ts.map +1 -0
  192. package/build/services/MentraJSLogPipeline.js +137 -0
  193. package/build/services/MentraJSLogPipeline.js.map +1 -0
  194. package/build/services/MentraJSRouter.d.ts +203 -0
  195. package/build/services/MentraJSRouter.d.ts.map +1 -0
  196. package/build/services/MentraJSRouter.js +480 -0
  197. package/build/services/MentraJSRouter.js.map +1 -0
  198. package/build/services/MentraUIRouter.d.ts +137 -0
  199. package/build/services/MentraUIRouter.d.ts.map +1 -0
  200. package/build/services/MentraUIRouter.js +210 -0
  201. package/build/services/MentraUIRouter.js.map +1 -0
  202. package/build/services/MicStateCoordinator.d.ts +37 -0
  203. package/build/services/MicStateCoordinator.d.ts.map +1 -0
  204. package/build/services/MicStateCoordinator.js +81 -0
  205. package/build/services/MicStateCoordinator.js.map +1 -0
  206. package/build/services/MiniappEngine.d.ts +31 -0
  207. package/build/services/MiniappEngine.d.ts.map +1 -0
  208. package/build/services/MiniappEngine.js +119 -0
  209. package/build/services/MiniappEngine.js.map +1 -0
  210. package/build/services/MiniappLauncher.d.ts +129 -0
  211. package/build/services/MiniappLauncher.d.ts.map +1 -0
  212. package/build/services/MiniappLauncher.js +291 -0
  213. package/build/services/MiniappLauncher.js.map +1 -0
  214. package/build/services/MiniappRunningRegistry.d.ts +24 -0
  215. package/build/services/MiniappRunningRegistry.d.ts.map +1 -0
  216. package/build/services/MiniappRunningRegistry.js +52 -0
  217. package/build/services/MiniappRunningRegistry.js.map +1 -0
  218. package/build/services/NativeHttpResponse.d.ts +3 -0
  219. package/build/services/NativeHttpResponse.d.ts.map +1 -0
  220. package/build/services/NativeHttpResponse.js +6 -0
  221. package/build/services/NativeHttpResponse.js.map +1 -0
  222. package/build/services/NavigationHandlers.d.ts +90 -0
  223. package/build/services/NavigationHandlers.d.ts.map +1 -0
  224. package/build/services/NavigationHandlers.js +379 -0
  225. package/build/services/NavigationHandlers.js.map +1 -0
  226. package/build/services/NavigationService.d.ts +254 -0
  227. package/build/services/NavigationService.d.ts.map +1 -0
  228. package/build/services/NavigationService.js +553 -0
  229. package/build/services/NavigationService.js.map +1 -0
  230. package/build/services/NotificationsEmitter.d.ts +28 -0
  231. package/build/services/NotificationsEmitter.d.ts.map +1 -0
  232. package/build/services/NotificationsEmitter.js +22 -0
  233. package/build/services/NotificationsEmitter.js.map +1 -0
  234. package/build/services/OfflineSpeechModelService.d.ts +33 -0
  235. package/build/services/OfflineSpeechModelService.d.ts.map +1 -0
  236. package/build/services/OfflineSpeechModelService.js +136 -0
  237. package/build/services/OfflineSpeechModelService.js.map +1 -0
  238. package/build/services/OtaInstallCoordinator.d.ts +195 -0
  239. package/build/services/OtaInstallCoordinator.d.ts.map +1 -0
  240. package/build/services/OtaInstallCoordinator.js +1047 -0
  241. package/build/services/OtaInstallCoordinator.js.map +1 -0
  242. package/build/services/OtaService.d.ts +3 -0
  243. package/build/services/OtaService.d.ts.map +1 -0
  244. package/build/services/OtaService.js +65 -0
  245. package/build/services/OtaService.js.map +1 -0
  246. package/build/services/OtaUpdateCheckService.d.ts +65 -0
  247. package/build/services/OtaUpdateCheckService.d.ts.map +1 -0
  248. package/build/services/OtaUpdateCheckService.js +260 -0
  249. package/build/services/OtaUpdateCheckService.js.map +1 -0
  250. package/build/services/PairingIdentity.d.ts +80 -0
  251. package/build/services/PairingIdentity.d.ts.map +1 -0
  252. package/build/services/PairingIdentity.js +145 -0
  253. package/build/services/PairingIdentity.js.map +1 -0
  254. package/build/services/PhoneCalendarHelpers.d.ts +32 -0
  255. package/build/services/PhoneCalendarHelpers.d.ts.map +1 -0
  256. package/build/services/PhoneCalendarHelpers.js +157 -0
  257. package/build/services/PhoneCalendarHelpers.js.map +1 -0
  258. package/build/services/PhoneCalendarService.d.ts +5 -0
  259. package/build/services/PhoneCalendarService.d.ts.map +1 -0
  260. package/build/services/PhoneCalendarService.js +24 -0
  261. package/build/services/PhoneCalendarService.js.map +1 -0
  262. package/build/services/PhoneCameraFovCoordinator.d.ts +28 -0
  263. package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -0
  264. package/build/services/PhoneCameraFovCoordinator.js +180 -0
  265. package/build/services/PhoneCameraFovCoordinator.js.map +1 -0
  266. package/build/services/PhoneLocationService.d.ts +36 -0
  267. package/build/services/PhoneLocationService.d.ts.map +1 -0
  268. package/build/services/PhoneLocationService.js +115 -0
  269. package/build/services/PhoneLocationService.js.map +1 -0
  270. package/build/services/PhoneNotificationsSync.d.ts +3 -0
  271. package/build/services/PhoneNotificationsSync.d.ts.map +1 -0
  272. package/build/services/PhoneNotificationsSync.js +35 -0
  273. package/build/services/PhoneNotificationsSync.js.map +1 -0
  274. package/build/services/PhonePhotoCoordinator.d.ts +103 -0
  275. package/build/services/PhonePhotoCoordinator.d.ts.map +1 -0
  276. package/build/services/PhonePhotoCoordinator.js +326 -0
  277. package/build/services/PhonePhotoCoordinator.js.map +1 -0
  278. package/build/services/PhoneStreamCoordinator.d.ts +168 -0
  279. package/build/services/PhoneStreamCoordinator.d.ts.map +1 -0
  280. package/build/services/PhoneStreamCoordinator.js +632 -0
  281. package/build/services/PhoneStreamCoordinator.js.map +1 -0
  282. package/build/services/PhoneVideoCoordinator.d.ts +50 -0
  283. package/build/services/PhoneVideoCoordinator.d.ts.map +1 -0
  284. package/build/services/PhoneVideoCoordinator.js +147 -0
  285. package/build/services/PhoneVideoCoordinator.js.map +1 -0
  286. package/build/services/STTModelManager.d.ts +65 -0
  287. package/build/services/STTModelManager.d.ts.map +1 -0
  288. package/build/services/STTModelManager.js +292 -0
  289. package/build/services/STTModelManager.js.map +1 -0
  290. package/build/services/SceneRenderer.d.ts +90 -0
  291. package/build/services/SceneRenderer.d.ts.map +1 -0
  292. package/build/services/SceneRenderer.js +248 -0
  293. package/build/services/SceneRenderer.js.map +1 -0
  294. package/build/services/SentenceTtsPipeline.d.ts +31 -0
  295. package/build/services/SentenceTtsPipeline.d.ts.map +1 -0
  296. package/build/services/SentenceTtsPipeline.js +67 -0
  297. package/build/services/SentenceTtsPipeline.js.map +1 -0
  298. package/build/services/StreamLifecycleController.d.ts +67 -0
  299. package/build/services/StreamLifecycleController.d.ts.map +1 -0
  300. package/build/services/StreamLifecycleController.js +152 -0
  301. package/build/services/StreamLifecycleController.js.map +1 -0
  302. package/build/services/TTSModelManager.d.ts +90 -0
  303. package/build/services/TTSModelManager.d.ts.map +1 -0
  304. package/build/services/TTSModelManager.js +366 -0
  305. package/build/services/TTSModelManager.js.map +1 -0
  306. package/build/services/TranscriptionRouting.d.ts +12 -0
  307. package/build/services/TranscriptionRouting.d.ts.map +1 -0
  308. package/build/services/TranscriptionRouting.js +24 -0
  309. package/build/services/TranscriptionRouting.js.map +1 -0
  310. package/build/services/WebviewBridge.d.ts +22 -0
  311. package/build/services/WebviewBridge.d.ts.map +1 -0
  312. package/build/services/WebviewBridge.js +47 -0
  313. package/build/services/WebviewBridge.js.map +1 -0
  314. package/build/services/asg/GalleryMediaIntegrityReportService.d.ts +17 -0
  315. package/build/services/asg/GalleryMediaIntegrityReportService.d.ts.map +1 -0
  316. package/build/services/asg/GalleryMediaIntegrityReportService.js +43 -0
  317. package/build/services/asg/GalleryMediaIntegrityReportService.js.map +1 -0
  318. package/build/services/asg/asgCameraApi.d.ts +205 -0
  319. package/build/services/asg/asgCameraApi.d.ts.map +1 -0
  320. package/build/services/asg/asgCameraApi.js +1410 -0
  321. package/build/services/asg/asgCameraApi.js.map +1 -0
  322. package/build/services/asg/cameraRollExportCoordinator.d.ts +60 -0
  323. package/build/services/asg/cameraRollExportCoordinator.d.ts.map +1 -0
  324. package/build/services/asg/cameraRollExportCoordinator.js +618 -0
  325. package/build/services/asg/cameraRollExportCoordinator.js.map +1 -0
  326. package/build/services/asg/cameraRollExportLedger.d.ts +44 -0
  327. package/build/services/asg/cameraRollExportLedger.d.ts.map +1 -0
  328. package/build/services/asg/cameraRollExportLedger.js +78 -0
  329. package/build/services/asg/cameraRollExportLedger.js.map +1 -0
  330. package/build/services/asg/galleryMediaValidation.d.ts +29 -0
  331. package/build/services/asg/galleryMediaValidation.d.ts.map +1 -0
  332. package/build/services/asg/galleryMediaValidation.js +149 -0
  333. package/build/services/asg/galleryMediaValidation.js.map +1 -0
  334. package/build/services/asg/galleryNotices.d.ts +33 -0
  335. package/build/services/asg/galleryNotices.d.ts.map +1 -0
  336. package/build/services/asg/galleryNotices.js +29 -0
  337. package/build/services/asg/galleryNotices.js.map +1 -0
  338. package/build/services/asg/galleryPathMigration.d.ts +3 -0
  339. package/build/services/asg/galleryPathMigration.d.ts.map +1 -0
  340. package/build/services/asg/galleryPathMigration.js +26 -0
  341. package/build/services/asg/galleryPathMigration.js.map +1 -0
  342. package/build/services/asg/gallerySettingsService.d.ts +20 -0
  343. package/build/services/asg/gallerySettingsService.d.ts.map +1 -0
  344. package/build/services/asg/gallerySettingsService.js +53 -0
  345. package/build/services/asg/gallerySettingsService.js.map +1 -0
  346. package/build/services/asg/gallerySyncNotifications.d.ts +56 -0
  347. package/build/services/asg/gallerySyncNotifications.d.ts.map +1 -0
  348. package/build/services/asg/gallerySyncNotifications.js +198 -0
  349. package/build/services/asg/gallerySyncNotifications.js.map +1 -0
  350. package/build/services/asg/gallerySyncService.d.ts +137 -0
  351. package/build/services/asg/gallerySyncService.d.ts.map +1 -0
  352. package/build/services/asg/gallerySyncService.js +1933 -0
  353. package/build/services/asg/gallerySyncService.js.map +1 -0
  354. package/build/services/asg/galleryTransferLedger.d.ts +58 -0
  355. package/build/services/asg/galleryTransferLedger.d.ts.map +1 -0
  356. package/build/services/asg/galleryTransferLedger.js +225 -0
  357. package/build/services/asg/galleryTransferLedger.js.map +1 -0
  358. package/build/services/asg/localNetworkTransport.d.ts +15 -0
  359. package/build/services/asg/localNetworkTransport.d.ts.map +1 -0
  360. package/build/services/asg/localNetworkTransport.js +144 -0
  361. package/build/services/asg/localNetworkTransport.js.map +1 -0
  362. package/build/services/asg/localStorageService.d.ts +146 -0
  363. package/build/services/asg/localStorageService.d.ts.map +1 -0
  364. package/build/services/asg/localStorageService.js +572 -0
  365. package/build/services/asg/localStorageService.js.map +1 -0
  366. package/build/services/asg/mediaProcessingQueue.d.ts +70 -0
  367. package/build/services/asg/mediaProcessingQueue.d.ts.map +1 -0
  368. package/build/services/asg/mediaProcessingQueue.js +451 -0
  369. package/build/services/asg/mediaProcessingQueue.js.map +1 -0
  370. package/build/services/asgOtaVersionUrl.d.ts +2 -0
  371. package/build/services/asgOtaVersionUrl.d.ts.map +1 -0
  372. package/build/services/asgOtaVersionUrl.js +36 -0
  373. package/build/services/asgOtaVersionUrl.js.map +1 -0
  374. package/build/services/blobPaths.d.ts +31 -0
  375. package/build/services/blobPaths.d.ts.map +1 -0
  376. package/build/services/blobPaths.js +86 -0
  377. package/build/services/blobPaths.js.map +1 -0
  378. package/build/services/cloudStreamApi.d.ts +59 -0
  379. package/build/services/cloudStreamApi.d.ts.map +1 -0
  380. package/build/services/cloudStreamApi.js +81 -0
  381. package/build/services/cloudStreamApi.js.map +1 -0
  382. package/build/services/gallerySyncClock.d.ts +17 -0
  383. package/build/services/gallerySyncClock.d.ts.map +1 -0
  384. package/build/services/gallerySyncClock.js +21 -0
  385. package/build/services/gallerySyncClock.js.map +1 -0
  386. package/build/services/glassesClockSync.d.ts +20 -0
  387. package/build/services/glassesClockSync.d.ts.map +1 -0
  388. package/build/services/glassesClockSync.js +97 -0
  389. package/build/services/glassesClockSync.js.map +1 -0
  390. package/build/services/manifestVersionGate.d.ts +28 -0
  391. package/build/services/manifestVersionGate.d.ts.map +1 -0
  392. package/build/services/manifestVersionGate.js +41 -0
  393. package/build/services/manifestVersionGate.js.map +1 -0
  394. package/build/services/mentraUiShim.d.ts +38 -0
  395. package/build/services/mentraUiShim.d.ts.map +1 -0
  396. package/build/services/mentraUiShim.js +360 -0
  397. package/build/services/mentraUiShim.js.map +1 -0
  398. package/build/services/navigation/roadNameResolver.d.ts +67 -0
  399. package/build/services/navigation/roadNameResolver.d.ts.map +1 -0
  400. package/build/services/navigation/roadNameResolver.js +96 -0
  401. package/build/services/navigation/roadNameResolver.js.map +1 -0
  402. package/build/services/navigation/routesApiCodec.d.ts +38 -0
  403. package/build/services/navigation/routesApiCodec.d.ts.map +1 -0
  404. package/build/services/navigation/routesApiCodec.js +68 -0
  405. package/build/services/navigation/routesApiCodec.js.map +1 -0
  406. package/build/services/otaDisplayState.d.ts +42 -0
  407. package/build/services/otaDisplayState.d.ts.map +1 -0
  408. package/build/services/otaDisplayState.js +92 -0
  409. package/build/services/otaDisplayState.js.map +1 -0
  410. package/build/services/otaInstallPolicy.d.ts +81 -0
  411. package/build/services/otaInstallPolicy.d.ts.map +1 -0
  412. package/build/services/otaInstallPolicy.js +96 -0
  413. package/build/services/otaInstallPolicy.js.map +1 -0
  414. package/build/services/otaLegacyMapping.d.ts +22 -0
  415. package/build/services/otaLegacyMapping.d.ts.map +1 -0
  416. package/build/services/otaLegacyMapping.js +56 -0
  417. package/build/services/otaLegacyMapping.js.map +1 -0
  418. package/build/services/slimStreamStatus.d.ts +7 -0
  419. package/build/services/slimStreamStatus.d.ts.map +1 -0
  420. package/build/services/slimStreamStatus.js +26 -0
  421. package/build/services/slimStreamStatus.js.map +1 -0
  422. package/build/stores/apps.d.ts +104 -0
  423. package/build/stores/apps.d.ts.map +1 -0
  424. package/build/stores/apps.js +492 -0
  425. package/build/stores/apps.js.map +1 -0
  426. package/build/stores/bluetoothSettingKeys.d.ts +4 -0
  427. package/build/stores/bluetoothSettingKeys.d.ts.map +1 -0
  428. package/build/stores/bluetoothSettingKeys.js +31 -0
  429. package/build/stores/bluetoothSettingKeys.js.map +1 -0
  430. package/build/stores/cloudClientStatus.d.ts +9 -0
  431. package/build/stores/cloudClientStatus.d.ts.map +1 -0
  432. package/build/stores/cloudClientStatus.js +24 -0
  433. package/build/stores/cloudClientStatus.js.map +1 -0
  434. package/build/stores/connection.d.ts +22 -0
  435. package/build/stores/connection.d.ts.map +1 -0
  436. package/build/stores/connection.js +43 -0
  437. package/build/stores/connection.js.map +1 -0
  438. package/build/stores/core.d.ts +15 -0
  439. package/build/stores/core.d.ts.map +1 -0
  440. package/build/stores/core.js +21 -0
  441. package/build/stores/core.js.map +1 -0
  442. package/build/stores/display.d.ts +24 -0
  443. package/build/stores/display.d.ts.map +1 -0
  444. package/build/stores/display.js +96 -0
  445. package/build/stores/display.js.map +1 -0
  446. package/build/stores/gallerySync.d.ts +101 -0
  447. package/build/stores/gallerySync.d.ts.map +1 -0
  448. package/build/stores/gallerySync.js +217 -0
  449. package/build/stores/gallerySync.js.map +1 -0
  450. package/build/stores/glasses.d.ts +62 -0
  451. package/build/stores/glasses.d.ts.map +1 -0
  452. package/build/stores/glasses.js +203 -0
  453. package/build/stores/glasses.js.map +1 -0
  454. package/build/stores/settings.d.ts +37 -0
  455. package/build/stores/settings.d.ts.map +1 -0
  456. package/build/stores/settings.js +922 -0
  457. package/build/stores/settings.js.map +1 -0
  458. package/build/types/applet.d.ts +96 -0
  459. package/build/types/applet.d.ts.map +1 -0
  460. package/build/types/applet.js +5 -0
  461. package/build/types/applet.js.map +1 -0
  462. package/build/types/asg.d.ts +53 -0
  463. package/build/types/asg.d.ts.map +1 -0
  464. package/build/types/asg.js +2 -0
  465. package/build/types/asg.js.map +1 -0
  466. package/build/types/capabilities/even-realities-g1.d.ts +12 -0
  467. package/build/types/capabilities/even-realities-g1.d.ts.map +1 -0
  468. package/build/types/capabilities/even-realities-g1.js +55 -0
  469. package/build/types/capabilities/even-realities-g1.js.map +1 -0
  470. package/build/types/capabilities/even-realities-g2.d.ts +12 -0
  471. package/build/types/capabilities/even-realities-g2.d.ts.map +1 -0
  472. package/build/types/capabilities/even-realities-g2.js +75 -0
  473. package/build/types/capabilities/even-realities-g2.js.map +1 -0
  474. package/build/types/capabilities/mentra-display.d.ts +12 -0
  475. package/build/types/capabilities/mentra-display.d.ts.map +1 -0
  476. package/build/types/capabilities/mentra-display.js +63 -0
  477. package/build/types/capabilities/mentra-display.js.map +1 -0
  478. package/build/types/capabilities/mentra-live.d.ts +12 -0
  479. package/build/types/capabilities/mentra-live.d.ts.map +1 -0
  480. package/build/types/capabilities/mentra-live.js +92 -0
  481. package/build/types/capabilities/mentra-live.js.map +1 -0
  482. package/build/types/capabilities/none.d.ts +13 -0
  483. package/build/types/capabilities/none.d.ts.map +1 -0
  484. package/build/types/capabilities/none.js +65 -0
  485. package/build/types/capabilities/none.js.map +1 -0
  486. package/build/types/capabilities/simulated-glasses.d.ts +13 -0
  487. package/build/types/capabilities/simulated-glasses.d.ts.map +1 -0
  488. package/build/types/capabilities/simulated-glasses.js +65 -0
  489. package/build/types/capabilities/simulated-glasses.js.map +1 -0
  490. package/build/types/capabilities/vuzix-z100.d.ts +12 -0
  491. package/build/types/capabilities/vuzix-z100.d.ts.map +1 -0
  492. package/build/types/capabilities/vuzix-z100.js +52 -0
  493. package/build/types/capabilities/vuzix-z100.js.map +1 -0
  494. package/build/types/enums.d.ts +40 -0
  495. package/build/types/enums.d.ts.map +1 -0
  496. package/build/types/enums.js +44 -0
  497. package/build/types/enums.js.map +1 -0
  498. package/build/types/hardware.d.ts +160 -0
  499. package/build/types/hardware.d.ts.map +1 -0
  500. package/build/types/hardware.js +36 -0
  501. package/build/types/hardware.js.map +1 -0
  502. package/build/types/index.d.ts +11 -0
  503. package/build/types/index.d.ts.map +1 -0
  504. package/build/types/index.js +10 -0
  505. package/build/types/index.js.map +1 -0
  506. package/build/utils/GlobalEventEmitter.d.ts +10 -0
  507. package/build/utils/GlobalEventEmitter.d.ts.map +1 -0
  508. package/build/utils/GlobalEventEmitter.js +13 -0
  509. package/build/utils/GlobalEventEmitter.js.map +1 -0
  510. package/build/utils/cloudClient/RnUdpAdapter.d.ts +3 -0
  511. package/build/utils/cloudClient/RnUdpAdapter.d.ts.map +1 -0
  512. package/build/utils/cloudClient/RnUdpAdapter.js +41 -0
  513. package/build/utils/cloudClient/RnUdpAdapter.js.map +1 -0
  514. package/build/utils/cloudClient/cloudSecureStore.d.ts +3 -0
  515. package/build/utils/cloudClient/cloudSecureStore.d.ts.map +1 -0
  516. package/build/utils/cloudClient/cloudSecureStore.js +26 -0
  517. package/build/utils/cloudClient/cloudSecureStore.js.map +1 -0
  518. package/build/utils/debouncedPatch.d.ts +6 -0
  519. package/build/utils/debouncedPatch.d.ts.map +1 -0
  520. package/build/utils/debouncedPatch.js +25 -0
  521. package/build/utils/debouncedPatch.js.map +1 -0
  522. package/build/utils/devLogging.d.ts +26 -0
  523. package/build/utils/devLogging.d.ts.map +1 -0
  524. package/build/utils/devLogging.js +116 -0
  525. package/build/utils/devLogging.js.map +1 -0
  526. package/build/utils/devMiniappLaunch.d.ts +44 -0
  527. package/build/utils/devMiniappLaunch.d.ts.map +1 -0
  528. package/build/utils/devMiniappLaunch.js +52 -0
  529. package/build/utils/devMiniappLaunch.js.map +1 -0
  530. package/build/utils/diagnosticContext.d.ts +3 -0
  531. package/build/utils/diagnosticContext.d.ts.map +1 -0
  532. package/build/utils/diagnosticContext.js +119 -0
  533. package/build/utils/diagnosticContext.js.map +1 -0
  534. package/build/utils/display/composer/ColumnComposer.d.ts +131 -0
  535. package/build/utils/display/composer/ColumnComposer.d.ts.map +1 -0
  536. package/build/utils/display/composer/ColumnComposer.js +204 -0
  537. package/build/utils/display/composer/ColumnComposer.js.map +1 -0
  538. package/build/utils/display/composer/index.d.ts +9 -0
  539. package/build/utils/display/composer/index.d.ts.map +1 -0
  540. package/build/utils/display/composer/index.js +9 -0
  541. package/build/utils/display/composer/index.js.map +1 -0
  542. package/build/utils/display/helpers/DisplayHelpers.d.ts +165 -0
  543. package/build/utils/display/helpers/DisplayHelpers.d.ts.map +1 -0
  544. package/build/utils/display/helpers/DisplayHelpers.js +312 -0
  545. package/build/utils/display/helpers/DisplayHelpers.js.map +1 -0
  546. package/build/utils/display/helpers/ScrollView.d.ts +183 -0
  547. package/build/utils/display/helpers/ScrollView.d.ts.map +1 -0
  548. package/build/utils/display/helpers/ScrollView.js +257 -0
  549. package/build/utils/display/helpers/ScrollView.js.map +1 -0
  550. package/build/utils/display/helpers/index.d.ts +11 -0
  551. package/build/utils/display/helpers/index.d.ts.map +1 -0
  552. package/build/utils/display/helpers/index.js +10 -0
  553. package/build/utils/display/helpers/index.js.map +1 -0
  554. package/build/utils/display/index.d.ts +174 -0
  555. package/build/utils/display/index.d.ts.map +1 -0
  556. package/build/utils/display/index.js +185 -0
  557. package/build/utils/display/index.js.map +1 -0
  558. package/build/utils/display/measurer/TextMeasurer.d.ts +160 -0
  559. package/build/utils/display/measurer/TextMeasurer.d.ts.map +1 -0
  560. package/build/utils/display/measurer/TextMeasurer.js +273 -0
  561. package/build/utils/display/measurer/TextMeasurer.js.map +1 -0
  562. package/build/utils/display/measurer/index.d.ts +10 -0
  563. package/build/utils/display/measurer/index.d.ts.map +1 -0
  564. package/build/utils/display/measurer/index.js +11 -0
  565. package/build/utils/display/measurer/index.js.map +1 -0
  566. package/build/utils/display/measurer/script-detection.d.ts +53 -0
  567. package/build/utils/display/measurer/script-detection.d.ts.map +1 -0
  568. package/build/utils/display/measurer/script-detection.js +184 -0
  569. package/build/utils/display/measurer/script-detection.js.map +1 -0
  570. package/build/utils/display/profiles/g1.d.ts +33 -0
  571. package/build/utils/display/profiles/g1.d.ts.map +1 -0
  572. package/build/utils/display/profiles/g1.js +213 -0
  573. package/build/utils/display/profiles/g1.js.map +1 -0
  574. package/build/utils/display/profiles/g2.d.ts +13 -0
  575. package/build/utils/display/profiles/g2.d.ts.map +1 -0
  576. package/build/utils/display/profiles/g2.js +34 -0
  577. package/build/utils/display/profiles/g2.js.map +1 -0
  578. package/build/utils/display/profiles/index.d.ts +12 -0
  579. package/build/utils/display/profiles/index.d.ts.map +1 -0
  580. package/build/utils/display/profiles/index.js +15 -0
  581. package/build/utils/display/profiles/index.js.map +1 -0
  582. package/build/utils/display/profiles/nex.d.ts +10 -0
  583. package/build/utils/display/profiles/nex.d.ts.map +1 -0
  584. package/build/utils/display/profiles/nex.js +150 -0
  585. package/build/utils/display/profiles/nex.js.map +1 -0
  586. package/build/utils/display/profiles/types.d.ts +103 -0
  587. package/build/utils/display/profiles/types.d.ts.map +1 -0
  588. package/build/utils/display/profiles/types.js +2 -0
  589. package/build/utils/display/profiles/types.js.map +1 -0
  590. package/build/utils/display/profiles/z100.d.ts +33 -0
  591. package/build/utils/display/profiles/z100.d.ts.map +1 -0
  592. package/build/utils/display/profiles/z100.js +182 -0
  593. package/build/utils/display/profiles/z100.js.map +1 -0
  594. package/build/utils/display/scene/degrade.d.ts +25 -0
  595. package/build/utils/display/scene/degrade.d.ts.map +1 -0
  596. package/build/utils/display/scene/degrade.js +64 -0
  597. package/build/utils/display/scene/degrade.js.map +1 -0
  598. package/build/utils/display/scene/differ.d.ts +38 -0
  599. package/build/utils/display/scene/differ.d.ts.map +1 -0
  600. package/build/utils/display/scene/differ.js +107 -0
  601. package/build/utils/display/scene/differ.js.map +1 -0
  602. package/build/utils/display/scene/index.d.ts +10 -0
  603. package/build/utils/display/scene/index.d.ts.map +1 -0
  604. package/build/utils/display/scene/index.js +6 -0
  605. package/build/utils/display/scene/index.js.map +1 -0
  606. package/build/utils/display/scene/process.d.ts +35 -0
  607. package/build/utils/display/scene/process.d.ts.map +1 -0
  608. package/build/utils/display/scene/process.js +187 -0
  609. package/build/utils/display/scene/process.js.map +1 -0
  610. package/build/utils/display/scene/store.d.ts +48 -0
  611. package/build/utils/display/scene/store.d.ts.map +1 -0
  612. package/build/utils/display/scene/store.js +0 -0
  613. package/build/utils/display/scene/store.js.map +1 -0
  614. package/build/utils/display/scene/types.d.ts +120 -0
  615. package/build/utils/display/scene/types.d.ts.map +1 -0
  616. package/build/utils/display/scene/types.js +0 -0
  617. package/build/utils/display/scene/types.js.map +1 -0
  618. package/build/utils/display/wrapper/TextWrapper.d.ts +102 -0
  619. package/build/utils/display/wrapper/TextWrapper.d.ts.map +1 -0
  620. package/build/utils/display/wrapper/TextWrapper.js +471 -0
  621. package/build/utils/display/wrapper/TextWrapper.js.map +1 -0
  622. package/build/utils/display/wrapper/index.d.ts +12 -0
  623. package/build/utils/display/wrapper/index.d.ts.map +1 -0
  624. package/build/utils/display/wrapper/index.js +12 -0
  625. package/build/utils/display/wrapper/index.js.map +1 -0
  626. package/build/utils/display/wrapper/types.d.ts +75 -0
  627. package/build/utils/display/wrapper/types.d.ts.map +1 -0
  628. package/build/utils/display/wrapper/types.js +11 -0
  629. package/build/utils/display/wrapper/types.js.map +1 -0
  630. package/build/utils/e2eMetrics.d.ts +6 -0
  631. package/build/utils/e2eMetrics.d.ts.map +1 -0
  632. package/build/utils/e2eMetrics.js +27 -0
  633. package/build/utils/e2eMetrics.js.map +1 -0
  634. package/build/utils/hardware/hardware.d.ts +42 -0
  635. package/build/utils/hardware/hardware.d.ts.map +1 -0
  636. package/build/utils/hardware/hardware.js +127 -0
  637. package/build/utils/hardware/hardware.js.map +1 -0
  638. package/build/utils/hardware/index.d.ts +2 -0
  639. package/build/utils/hardware/index.d.ts.map +1 -0
  640. package/build/utils/hardware/index.js +2 -0
  641. package/build/utils/hardware/index.js.map +1 -0
  642. package/build/utils/miniappGlobals.d.ts +60 -0
  643. package/build/utils/miniappGlobals.d.ts.map +1 -0
  644. package/build/utils/miniappGlobals.js +224 -0
  645. package/build/utils/miniappGlobals.js.map +1 -0
  646. package/build/utils/permissions/MediaLibraryPermissions.d.ts +45 -0
  647. package/build/utils/permissions/MediaLibraryPermissions.d.ts.map +1 -0
  648. package/build/utils/permissions/MediaLibraryPermissions.js +145 -0
  649. package/build/utils/permissions/MediaLibraryPermissions.js.map +1 -0
  650. package/build/utils/permissions/galleryDisplayName.d.ts +8 -0
  651. package/build/utils/permissions/galleryDisplayName.d.ts.map +1 -0
  652. package/build/utils/permissions/galleryDisplayName.js +25 -0
  653. package/build/utils/permissions/galleryDisplayName.js.map +1 -0
  654. package/build/utils/storage/index.d.ts +3 -0
  655. package/build/utils/storage/index.d.ts.map +1 -0
  656. package/build/utils/storage/index.js +3 -0
  657. package/build/utils/storage/index.js.map +1 -0
  658. package/build/utils/storage/storage.d.ts +24 -0
  659. package/build/utils/storage/storage.d.ts.map +1 -0
  660. package/build/utils/storage/storage.js +85 -0
  661. package/build/utils/storage/storage.js.map +1 -0
  662. package/build/utils/storage/zip.d.ts +3 -0
  663. package/build/utils/storage/zip.d.ts.map +1 -0
  664. package/build/utils/storage/zip.js +14 -0
  665. package/build/utils/storage/zip.js.map +1 -0
  666. package/build/utils/timers/index.d.ts +3 -0
  667. package/build/utils/timers/index.d.ts.map +1 -0
  668. package/build/utils/timers/index.js +3 -0
  669. package/build/utils/timers/index.js.map +1 -0
  670. package/build/utils/timers/timers.d.ts +9 -0
  671. package/build/utils/timers/timers.d.ts.map +1 -0
  672. package/build/utils/timers/timers.js +105 -0
  673. package/build/utils/timers/timers.js.map +1 -0
  674. package/package.json +120 -0
  675. package/src/devtools.ts +13 -0
  676. package/src/engine.ts +173 -0
  677. package/src/facades/dev.ts +198 -0
  678. package/src/facades/display.ts +17 -0
  679. package/src/facades/displayMirror.ts +33 -0
  680. package/src/facades/gallery.ts +69 -0
  681. package/src/facades/glasses.ts +206 -0
  682. package/src/facades/glassesSettings.ts +56 -0
  683. package/src/facades/glassesWifi.ts +68 -0
  684. package/src/facades/miniapps.ts +55 -0
  685. package/src/facades/notifications.ts +15 -0
  686. package/src/facades/ota.ts +174 -0
  687. package/src/facades/pairing.ts +329 -0
  688. package/src/facades/permissions.ts +165 -0
  689. package/src/facades/phoneNotifications.ts +50 -0
  690. package/src/facades/reports.ts +185 -0
  691. package/src/facades/session.ts +55 -0
  692. package/src/facades/settings.ts +57 -0
  693. package/src/facades/speech.ts +38 -0
  694. package/src/index.ts +191 -0
  695. package/src/internal.ts +159 -0
  696. package/src/runtime/bootstrap.ts +118 -0
  697. package/src/runtime/config.ts +147 -0
  698. package/src/runtime/streamConfig.ts +34 -0
  699. package/src/services/AppRegistry.ts +1190 -0
  700. package/src/services/AudioCloudUplink.ts +50 -0
  701. package/src/services/AudioPlaybackService.ts +813 -0
  702. package/src/services/AutomaticReportResult.ts +48 -0
  703. package/src/services/BlobStore.ts +746 -0
  704. package/src/services/CaptionsTesterReportService.ts +107 -0
  705. package/src/services/CloudAudioSubscriptionSync.ts +41 -0
  706. package/src/services/CloudClientService.ts +689 -0
  707. package/src/services/CloudTranscriptE2EMetrics.ts +27 -0
  708. package/src/services/ConnectionCoordinator.ts +65 -0
  709. package/src/services/DevServerBridge.ts +332 -0
  710. package/src/services/DeviceEventRouter.ts +238 -0
  711. package/src/services/DeviceStoreHydration.ts +130 -0
  712. package/src/services/DisplayProcessor.ts +775 -0
  713. package/src/services/GlassesReadiness.ts +103 -0
  714. package/src/services/GlassesSettingsSync.ts +134 -0
  715. package/src/services/GlassesStatusProjection.ts +73 -0
  716. package/src/services/HeadingService.ts +98 -0
  717. package/src/services/LocalDisplayManager.ts +703 -0
  718. package/src/services/LocalMiniappRuntime.ts +4213 -0
  719. package/src/services/LocalSttFallbackCoordinator.ts +147 -0
  720. package/src/services/MentraJSCrashController.ts +189 -0
  721. package/src/services/MentraJSCrashloopReportService.ts +66 -0
  722. package/src/services/MentraJSLogPipeline.ts +166 -0
  723. package/src/services/MentraJSRouter.ts +557 -0
  724. package/src/services/MentraUIRouter.ts +244 -0
  725. package/src/services/MicStateCoordinator.ts +94 -0
  726. package/src/services/MiniappEngine.ts +143 -0
  727. package/src/services/MiniappLauncher.ts +340 -0
  728. package/src/services/MiniappRunningRegistry.ts +57 -0
  729. package/src/services/NativeHttpResponse.ts +6 -0
  730. package/src/services/NavigationHandlers.ts +423 -0
  731. package/src/services/NavigationService.ts +731 -0
  732. package/src/services/NotificationsEmitter.ts +48 -0
  733. package/src/services/OfflineSpeechModelService.ts +168 -0
  734. package/src/services/OtaInstallCoordinator.ts +1205 -0
  735. package/src/services/OtaService.ts +83 -0
  736. package/src/services/OtaUpdateCheckService.ts +386 -0
  737. package/src/services/PairingIdentity.ts +154 -0
  738. package/src/services/PhoneCalendarHelpers.ts +212 -0
  739. package/src/services/PhoneCalendarService.ts +42 -0
  740. package/src/services/PhoneCameraFovCoordinator.ts +195 -0
  741. package/src/services/PhoneLocationService.ts +122 -0
  742. package/src/services/PhoneNotificationsSync.ts +44 -0
  743. package/src/services/PhonePhotoCoordinator.ts +409 -0
  744. package/src/services/PhoneStreamCoordinator.ts +777 -0
  745. package/src/services/PhoneVideoCoordinator.ts +189 -0
  746. package/src/services/STTModelManager.ts +360 -0
  747. package/src/services/SceneRenderer.ts +292 -0
  748. package/src/services/SentenceTtsPipeline.ts +106 -0
  749. package/src/services/StreamLifecycleController.ts +213 -0
  750. package/src/services/TTSModelManager.ts +466 -0
  751. package/src/services/TranscriptionRouting.ts +37 -0
  752. package/src/services/WebviewBridge.ts +50 -0
  753. package/src/services/asg/GalleryMediaIntegrityReportService.ts +70 -0
  754. package/src/services/asg/asgCameraApi.ts +1695 -0
  755. package/src/services/asg/cameraRollExportCoordinator.ts +627 -0
  756. package/src/services/asg/cameraRollExportLedger.ts +122 -0
  757. package/src/services/asg/galleryMediaValidation.ts +183 -0
  758. package/src/services/asg/galleryNotices.ts +61 -0
  759. package/src/services/asg/galleryPathMigration.ts +22 -0
  760. package/src/services/asg/gallerySettingsService.ts +68 -0
  761. package/src/services/asg/gallerySyncNotifications.ts +236 -0
  762. package/src/services/asg/gallerySyncService.ts +2269 -0
  763. package/src/services/asg/galleryTransferLedger.ts +312 -0
  764. package/src/services/asg/localNetworkTransport.ts +173 -0
  765. package/src/services/asg/localStorageService.ts +652 -0
  766. package/src/services/asg/mediaProcessingQueue.ts +518 -0
  767. package/src/services/asgOtaVersionUrl.ts +41 -0
  768. package/src/services/blobPaths.ts +86 -0
  769. package/src/services/cloudStreamApi.ts +154 -0
  770. package/src/services/gallerySyncClock.ts +39 -0
  771. package/src/services/glassesClockSync.ts +115 -0
  772. package/src/services/manifestVersionGate.ts +43 -0
  773. package/src/services/mentraUiShim.ts +366 -0
  774. package/src/services/navigation/roadNameResolver.ts +112 -0
  775. package/src/services/navigation/routesApiCodec.ts +69 -0
  776. package/src/services/otaDisplayState.ts +115 -0
  777. package/src/services/otaInstallPolicy.ts +106 -0
  778. package/src/services/otaLegacyMapping.ts +71 -0
  779. package/src/services/slimStreamStatus.ts +28 -0
  780. package/src/stores/apps.ts +614 -0
  781. package/src/stores/bluetoothSettingKeys.ts +36 -0
  782. package/src/stores/cloudClientStatus.ts +37 -0
  783. package/src/stores/connection.ts +69 -0
  784. package/src/stores/core.ts +32 -0
  785. package/src/stores/display.ts +121 -0
  786. package/src/stores/gallerySync.ts +380 -0
  787. package/src/stores/glasses.ts +309 -0
  788. package/src/stores/settings.ts +1001 -0
  789. package/src/types/applet.ts +112 -0
  790. package/src/types/asg.ts +58 -0
  791. package/src/types/capabilities/even-realities-g1.ts +67 -0
  792. package/src/types/capabilities/even-realities-g2.ts +88 -0
  793. package/src/types/capabilities/mentra-display.ts +75 -0
  794. package/src/types/capabilities/mentra-live.ts +103 -0
  795. package/src/types/capabilities/none.ts +76 -0
  796. package/src/types/capabilities/simulated-glasses.ts +76 -0
  797. package/src/types/capabilities/vuzix-z100.ts +64 -0
  798. package/src/types/enums.ts +43 -0
  799. package/src/types/hardware.ts +203 -0
  800. package/src/types/index.ts +37 -0
  801. package/src/utils/GlobalEventEmitter.ts +14 -0
  802. package/src/utils/cloudClient/RnUdpAdapter.ts +47 -0
  803. package/src/utils/cloudClient/cloudSecureStore.ts +28 -0
  804. package/src/utils/debouncedPatch.ts +28 -0
  805. package/src/utils/devLogging.ts +140 -0
  806. package/src/utils/devMiniappLaunch.ts +68 -0
  807. package/src/utils/diagnosticContext.ts +155 -0
  808. package/src/utils/display/composer/ColumnComposer.ts +285 -0
  809. package/src/utils/display/composer/index.ts +15 -0
  810. package/src/utils/display/helpers/DisplayHelpers.ts +413 -0
  811. package/src/utils/display/helpers/ScrollView.ts +337 -0
  812. package/src/utils/display/helpers/index.ts +14 -0
  813. package/src/utils/display/index.ts +273 -0
  814. package/src/utils/display/measurer/TextMeasurer.ts +346 -0
  815. package/src/utils/display/measurer/index.ts +21 -0
  816. package/src/utils/display/measurer/script-detection.ts +218 -0
  817. package/src/utils/display/profiles/g1.ts +236 -0
  818. package/src/utils/display/profiles/g2.ts +39 -0
  819. package/src/utils/display/profiles/index.ts +28 -0
  820. package/src/utils/display/profiles/nex.ts +166 -0
  821. package/src/utils/display/profiles/types.ts +132 -0
  822. package/src/utils/display/profiles/z100.ts +198 -0
  823. package/src/utils/display/scene/degrade.ts +80 -0
  824. package/src/utils/display/scene/differ.ts +128 -0
  825. package/src/utils/display/scene/index.ts +21 -0
  826. package/src/utils/display/scene/process.ts +214 -0
  827. package/src/utils/display/scene/store.ts +0 -0
  828. package/src/utils/display/scene/types.ts +0 -0
  829. package/src/utils/display/wrapper/TextWrapper.ts +546 -0
  830. package/src/utils/display/wrapper/index.ts +15 -0
  831. package/src/utils/display/wrapper/types.ts +102 -0
  832. package/src/utils/e2eMetrics.ts +30 -0
  833. package/src/utils/hardware/hardware.ts +166 -0
  834. package/src/utils/hardware/index.ts +1 -0
  835. package/src/utils/miniappGlobals.ts +260 -0
  836. package/src/utils/permissions/MediaLibraryPermissions.ts +161 -0
  837. package/src/utils/permissions/galleryDisplayName.ts +28 -0
  838. package/src/utils/storage/index.ts +2 -0
  839. package/src/utils/storage/storage.ts +101 -0
  840. package/src/utils/storage/zip.ts +13 -0
  841. package/src/utils/timers/index.ts +2 -0
  842. package/src/utils/timers/timers.ts +139 -0
@@ -0,0 +1,3695 @@
1
+ /**
2
+ * LocalMiniappRuntime
3
+ *
4
+ * Singleton service that bridges local miniapps (running in WebViews) with
5
+ * phone capabilities: display, audio, storage, sensors, LED, etc.
6
+ *
7
+ * Each miniapp communicates via the @mentra/miniapp envelope protocol.
8
+ * This runtime handles request dispatch, stream fan-out, and lifecycle
9
+ * management (connect/disconnect/ping).
10
+ */
11
+ import { Linking } from "react-native";
12
+ import Share from "react-native-share";
13
+ import * as Battery from "expo-battery";
14
+ import * as Clipboard from "expo-clipboard";
15
+ import { File, Paths } from "expo-file-system";
16
+ import * as Location from "expo-location";
17
+ import BluetoothSdk from "@mentra/bluetooth-sdk";
18
+ import { MiniappErrorCode, MiniappRequestType, MiniappResponseType, MiniappStreamType, CLOUD_STATUS_STREAM, parseEnvelope, serializeEnvelope, } from "@mentra/miniapp";
19
+ import { DeviceTypes, getModelCapabilities } from "../types";
20
+ import { storage as mmkvStorage } from "../utils/storage/storage";
21
+ import { BgTimer } from "../utils/timers";
22
+ import devServerBridge from "./DevServerBridge";
23
+ import { islandNotifications } from "./NotificationsEmitter";
24
+ import { isGlassesConnected } from "./GlassesReadiness";
25
+ import audioPlaybackService from "./AudioPlaybackService";
26
+ import { phoneLocationService } from "./PhoneLocationService";
27
+ import { useGlassesStore } from "../stores/glasses";
28
+ import { useSettingsStore } from "../stores/settings";
29
+ import localDisplayManager from "./LocalDisplayManager";
30
+ import headingService from "./HeadingService";
31
+ import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator";
32
+ import micStateCoordinator from "./MicStateCoordinator";
33
+ import { BlobStore } from "./BlobStore";
34
+ import { CloudAudioSubscriptionSync } from "./CloudAudioSubscriptionSync";
35
+ import { phoneCameraFovCoordinator } from "./PhoneCameraFovCoordinator";
36
+ import { phonePhotoCoordinator } from "./PhonePhotoCoordinator";
37
+ import { phoneStreamCoordinator } from "./PhoneStreamCoordinator";
38
+ import { phoneVideoCoordinator } from "./PhoneVideoCoordinator";
39
+ import { runSentenceTtsPipeline } from "./SentenceTtsPipeline";
40
+ import { summarizeTranscriptionRoutes, transcriptionDeliveryRoute } from "./TranscriptionRouting";
41
+ import { cloudClientService } from "./CloudClientService";
42
+ import { miniappLauncher } from "./MiniappLauncher";
43
+ import { ISLAND_SETTINGS_KEYS, } from "../runtime/config";
44
+ import { getAnalytics, getUiSeams } from "../runtime/bootstrap";
45
+ import { normalizeStreamAudioConfig, normalizeStreamVideoConfig } from "../runtime/streamConfig";
46
+ import ttsModelManager from "./TTSModelManager";
47
+ import { NavigationHandlers } from "./NavigationHandlers";
48
+ import { useAppStatusStore } from "../stores/apps";
49
+ import { getDevAppAttestation, getDevAppSourcePackage } from "./AppRegistry";
50
+ import { listPhoneCalendarEvents, PhoneCalendarError } from "./PhoneCalendarService";
51
+ /**
52
+ * Strictness ordering for `location_stream` rate values. Higher index =
53
+ * higher accuracy and higher power cost. The aggregate tier applied to
54
+ * the OS is the max of every connected miniapp's requested rate;
55
+ * miniapps that didn't ask for `location_stream` contribute nothing.
56
+ *
57
+ * If a miniapp passes a value we don't recognize, we treat it like the
58
+ * weakest known rate ("passive") rather than blanket-overwriting an
59
+ * active stronger request from another app.
60
+ */
61
+ const LOCATION_RATE_PRIORITY = ["off", "passive", "low", "high", "realtime"];
62
+ function locationRateRank(rate) {
63
+ if (!rate)
64
+ return -1;
65
+ const i = LOCATION_RATE_PRIORITY.indexOf(rate);
66
+ return i >= 0 ? i : LOCATION_RATE_PRIORITY.indexOf("passive");
67
+ }
68
+ function withTimeout(promise, timeoutMs) {
69
+ return new Promise((resolve) => {
70
+ let settled = false;
71
+ const done = (value) => {
72
+ if (settled)
73
+ return;
74
+ settled = true;
75
+ BgTimer.clearTimeout(timer);
76
+ resolve(value);
77
+ };
78
+ const timer = BgTimer.setTimeout(() => done(null), timeoutMs);
79
+ promise.then((value) => done(value), () => done(null));
80
+ });
81
+ }
82
+ const LOG_TAG = "LOCAL_MINIAPP";
83
+ const SYSTEM_MINIAPP_PACKAGES = new Set([
84
+ "com.mentra.camera",
85
+ "com.mentra.gallery",
86
+ "com.mentra.settings",
87
+ "com.mentra.simulated",
88
+ "com.mentra.mirror",
89
+ "com.mentra.ai",
90
+ "cloud.augmentos.notify",
91
+ "com.mentra.feedback",
92
+ "com.mentra.miniappdev",
93
+ ]);
94
+ const PING_INTERVAL_MS = 5_000;
95
+ const MINIAPP_AUTH_REFRESH_HEADROOM_MS = 5 * 60 * 1000;
96
+ const MINIAPP_AUTH_REFRESH_MIN_DELAY_MS = 5_000;
97
+ // Backoff bounds for re-minting the INITIAL miniapp token after a failed mint
98
+ // (e.g. the cloud client had not finished its first-boot Core token exchange
99
+ // when the miniapp connected). Exponential from base, capped, so a transient
100
+ // not-yet-connected client self-heals within seconds instead of needing a full
101
+ // manual Cloud V2 reconnect.
102
+ const MINIAPP_AUTH_RETRY_BASE_MS = 1_000;
103
+ const MINIAPP_AUTH_RETRY_MAX_MS = 15_000;
104
+ const FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS = 2_500;
105
+ const REQUEST_WIFI_SETUP_TYPE = "miniapp_request_wifi_setup";
106
+ // Unregister after this many missed pongs. Generous on purpose: a busy
107
+ // context (heavy interim translation traffic) or OS scheduling while idle can
108
+ // delay pongs well past one interval, and killing a healthy-but-busy script
109
+ // drops its subscriptions (releasing the mic). With the liveness-timeout
110
+ // respawn path wired (MentraJSRouter.start), a genuinely dead context still
111
+ // comes back automatically — this threshold only bounds how long that takes.
112
+ const PING_TIMEOUT_THRESHOLD = 6; // ~30s
113
+ const TRANSCRIPT_TIMING_TELEMETRY = globalThis.__DEV__ === true;
114
+ const RGB_LED_ACTIONS = new Set(["on", "off"]);
115
+ const RGB_LED_COLORS = new Set(["red", "green", "blue", "orange", "white"]);
116
+ const CAMERA_FOV_MIN = 62;
117
+ const CAMERA_FOV_MAX = 118;
118
+ const CAMERA_FOV_DEFAULT = 102;
119
+ const CAMERA_FOV_PRESETS = { narrow: 82, standard: CAMERA_FOV_DEFAULT, wide: 118 };
120
+ const CAMERA_ROI_POSITION_BY_NAME = { center: "center", bottom: "bottom", top: "top" };
121
+ // =============================================================================
122
+ // Declared-permission record helper (for CONNECT_ACK / PERMISSIONS_UPDATE)
123
+ // =============================================================================
124
+ /**
125
+ * Map from manifest permission types (uppercase, snake-y) to the lowercase
126
+ * canonical permission keys the SDK exposes via session.permissions. Mirrors
127
+ * cloud SDK v3's PermissionType union.
128
+ *
129
+ * BACKGROUND_LOCATION + POST_NOTIFICATIONS aren't in v3's surface; they map
130
+ * onto the same canonical keys (location / notifications) since the SDK's
131
+ * `has()` is "do I have *any* form of this permission declared".
132
+ */
133
+ const PERMISSION_TYPE_TO_CANONICAL = {
134
+ MICROPHONE: "microphone",
135
+ CAMERA: "camera",
136
+ LOCATION: "location",
137
+ BACKGROUND_LOCATION: "location",
138
+ READ_NOTIFICATIONS: "notifications",
139
+ POST_NOTIFICATIONS: "notifications",
140
+ CALENDAR: "calendar",
141
+ };
142
+ function normalizeRgbLedAction(value) {
143
+ return typeof value === "string" && RGB_LED_ACTIONS.has(value) ? value : "off";
144
+ }
145
+ function normalizeRgbLedColor(value) {
146
+ return typeof value === "string" && RGB_LED_COLORS.has(value) ? value : null;
147
+ }
148
+ function normalizeCameraFovPayload(payload) {
149
+ if (typeof payload.preset === "string") {
150
+ const preset = payload.preset in CAMERA_FOV_PRESETS ? payload.preset : "standard";
151
+ return { preset };
152
+ }
153
+ const rawFov = typeof payload.fov === "number" && Number.isFinite(payload.fov) ? payload.fov : CAMERA_FOV_DEFAULT;
154
+ return {
155
+ fov: Math.min(CAMERA_FOV_MAX, Math.max(CAMERA_FOV_MIN, rawFov)),
156
+ roiPosition: normalizeCameraRoiPosition(payload.roiPosition ?? payload.roi_position),
157
+ };
158
+ }
159
+ function normalizeCameraRoiPosition(value) {
160
+ if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 2) {
161
+ return value === 1 ? "bottom" : value === 2 ? "top" : "center";
162
+ }
163
+ if (typeof value === "string") {
164
+ return CAMERA_ROI_POSITION_BY_NAME[value] ?? "center";
165
+ }
166
+ return "center";
167
+ }
168
+ const ALL_CANONICAL_PERMISSIONS = ["location", "microphone", "camera", "notifications", "calendar"];
169
+ /**
170
+ * Build the {location, microphone, camera, notifications, calendar} record the
171
+ * SDK's session.permissions module reads from. Missing types are false.
172
+ */
173
+ function computeDeclaredPermissionRecord(manifest) {
174
+ const record = {};
175
+ for (const k of ALL_CANONICAL_PERMISSIONS)
176
+ record[k] = false;
177
+ for (const perm of manifest?.permissions ?? []) {
178
+ const canonical = PERMISSION_TYPE_TO_CANONICAL[perm.type?.toUpperCase()];
179
+ if (canonical)
180
+ record[canonical] = true;
181
+ }
182
+ return record;
183
+ }
184
+ // =============================================================================
185
+ // PERMISSION_NOT_DECLARED warnings — per-session dedup
186
+ // =============================================================================
187
+ //
188
+ // When a miniapp tries to subscribe / call something whose required permission
189
+ // isn't declared in miniapp.json, the runtime rejects with
190
+ // PERMISSION_NOT_DECLARED. The error reaches the SDK but most authors don't
191
+ // subscribe to session.on("error", ...), so it's silent in practice.
192
+ //
193
+ // To make the failure discoverable for developers running the MentraOS app
194
+ // from source, log a clear, copy-pasteable message in the phone console that
195
+ // names the offending permission, the offending stream/op, and the JSON
196
+ // snippet to add to miniapp.json. Once-per-session per (packageName, permission)
197
+ // to avoid spam from a tight retry loop.
198
+ //
199
+ // Production users running the App Store build of MentraOS won't see these
200
+ // (they don't watch Metro/adb logcat). For them, the WebView console bridge
201
+ // (#5 of the quick-fixes round) ships the structured error to the miniapp
202
+ // itself, and the miniapp's own console.warn flows to the dev terminal.
203
+ const warnedPermission = new Set(); // key: `${packageName}::${permission}`
204
+ function logPermissionNotDeclared(packageName, permission, context, manifestSnippet) {
205
+ const key = `${packageName}::${permission}`;
206
+ if (warnedPermission.has(key))
207
+ return;
208
+ warnedPermission.add(key);
209
+ console.warn(`${LOG_TAG}: ${packageName} attempted ${context}, but permission ${permission} is not declared in miniapp.json.\n` +
210
+ `Add this to the "permissions" array:\n ${manifestSnippet}`);
211
+ }
212
+ /** Reset the per-session dedup; called when a miniapp unregisters so a fresh launch warns again. */
213
+ function resetPermissionWarnings(packageName) {
214
+ for (const key of warnedPermission) {
215
+ if (key.startsWith(`${packageName}::`))
216
+ warnedPermission.delete(key);
217
+ }
218
+ }
219
+ // =============================================================================
220
+ // LocalMiniappRuntime
221
+ // =============================================================================
222
+ class LocalMiniappRuntime {
223
+ static instance = null;
224
+ /** Connected miniapps keyed by packageName. */
225
+ connectedApps = new Map();
226
+ /**
227
+ * Packages that have completed their CONNECT handshake (sent
228
+ * `miniapp_connect`, got `CONNECT_ACK`). Distinct from `connectedApps`,
229
+ * which is populated at spawn/registration time — handshake is later.
230
+ * Drives {@link waitForConnect}.
231
+ */
232
+ handshookApps = new Set();
233
+ /** Pending {@link waitForConnect} resolvers, keyed by packageName. */
234
+ connectWaiters = new Map();
235
+ /** Ref-counted stream subscriptions: stream → set of packageNames. */
236
+ streamSubscribers = new Map();
237
+ /** Guards one-time wiring of the cloud transcript/translation fan-out. */
238
+ cloudResultsWired = false;
239
+ cloudStatusWired = false;
240
+ cloudAudioSubscriptionSync = new CloudAudioSubscriptionSync();
241
+ /** Ping interval handle. */
242
+ pingIntervalId = null;
243
+ foregroundProbeTimers = new Map();
244
+ /**
245
+ * Notified when a miniapp is unregistered for missing liveness pings.
246
+ * MentraJSRouter wires this into its crash-respawn machinery so a
247
+ * silently-stalled background script is restarted (with crash-loop
248
+ * protection) instead of staying dead — a dead background drops its
249
+ * subscriptions, which releases the mic while the webview still looks
250
+ * alive.
251
+ */
252
+ onLivenessTimeout = null;
253
+ /** Pending cloud requests: requestId → packageName that originated the request. */
254
+ pendingCloudRequests = new Map();
255
+ speechRuns = new Map();
256
+ // Browser fallback token auth — HMAC-signed blob with a phone-local
257
+ // secret. Both issuer and verifier are the same process, so the
258
+ // secret never leaves the device.
259
+ localSecret = `miniapp_${Date.now()}_${Math.random().toString(36).slice(2, 14)}`;
260
+ usedTokens = new Set();
261
+ constructor() { }
262
+ /**
263
+ * Generate an HMAC-signed local session token for browser fallback auth.
264
+ * Token format: base64(JSON({userId, packageName, exp})).base64(HMAC-SHA256(payload, secret))
265
+ * Single-use, 5-minute TTL.
266
+ */
267
+ generateLocalToken(userId, packageName) {
268
+ const payload = JSON.stringify({
269
+ userId,
270
+ packageName,
271
+ exp: Date.now() + 5 * 60 * 1000, // 5 min TTL
272
+ nonce: Math.random().toString(36).slice(2, 10),
273
+ });
274
+ const payloadB64 = btoa(payload);
275
+ const sig = this.hmacSign(payload);
276
+ return `${payloadB64}.${sig}`;
277
+ }
278
+ /**
279
+ * Validate and consume a local session token. Single-use.
280
+ */
281
+ validateLocalToken(token) {
282
+ const parts = token.split(".");
283
+ if (parts.length !== 2)
284
+ return null;
285
+ const [payloadB64, sig] = parts;
286
+ // Check single-use
287
+ if (this.usedTokens.has(token))
288
+ return null;
289
+ // Verify signature
290
+ let payload;
291
+ try {
292
+ payload = atob(payloadB64);
293
+ }
294
+ catch {
295
+ return null;
296
+ }
297
+ if (this.hmacSign(payload) !== sig)
298
+ return null;
299
+ // Parse and check expiry
300
+ let parsed;
301
+ try {
302
+ parsed = JSON.parse(payload);
303
+ }
304
+ catch {
305
+ return null;
306
+ }
307
+ if (Date.now() > parsed.exp)
308
+ return null;
309
+ // Mark as used
310
+ this.usedTokens.add(token);
311
+ // Prune old used tokens periodically (keep set from growing unbounded)
312
+ if (this.usedTokens.size > 1000) {
313
+ this.usedTokens.clear();
314
+ }
315
+ return { userId: parsed.userId, packageName: parsed.packageName };
316
+ }
317
+ /**
318
+ * Simple HMAC-SHA256 using Web Crypto API (available in React Native).
319
+ * Returns base64url-encoded signature.
320
+ * Falls back to a simpler hash if crypto.subtle is not available.
321
+ */
322
+ hmacSign(payload) {
323
+ // Synchronous HMAC using a simple hash — Web Crypto's subtle.sign is async
324
+ // which doesn't fit cleanly here. For phone-local single-process auth,
325
+ // a keyed hash is sufficient.
326
+ let hash = 0;
327
+ const key = this.localSecret;
328
+ const input = key + payload + key;
329
+ for (let i = 0; i < input.length; i++) {
330
+ const ch = input.charCodeAt(i);
331
+ hash = ((hash << 5) - hash + ch) | 0;
332
+ }
333
+ // Mix in more bytes for collision resistance
334
+ let hash2 = 0x811c9dc5; // FNV offset basis
335
+ for (let i = 0; i < input.length; i++) {
336
+ hash2 ^= input.charCodeAt(i);
337
+ hash2 = Math.imul(hash2, 0x01000193); // FNV prime
338
+ }
339
+ return btoa(String(hash >>> 0) + "." + String(hash2 >>> 0));
340
+ }
341
+ static getInstance() {
342
+ if (!LocalMiniappRuntime.instance) {
343
+ LocalMiniappRuntime.instance = new LocalMiniappRuntime();
344
+ }
345
+ return LocalMiniappRuntime.instance;
346
+ }
347
+ initialized = false;
348
+ /**
349
+ * Initialize the runtime. Called from MantleManager.init().
350
+ * Idempotent — safe to call multiple times.
351
+ */
352
+ initialize() {
353
+ if (this.initialized)
354
+ return;
355
+ this.initialized = true;
356
+ console.log(`${LOG_TAG}: initialize()`);
357
+ this.ensurePingLoop();
358
+ }
359
+ /**
360
+ * Handle an incoming cloud message forwarded by SocketComms
361
+ * (phone_stream_status, phone_managed_stream_status).
362
+ *
363
+ * Routes the response back to the originating miniapp via the requestId
364
+ * that was stored when the miniapp first made the request.
365
+ */
366
+ handleCloudMessage(msg) {
367
+ const requestId = msg.requestId;
368
+ const msgType = msg.type;
369
+ console.log(`${LOG_TAG}: Cloud message: ${msgType}, requestId=${requestId ?? "none"}`);
370
+ if (!requestId) {
371
+ console.warn(`${LOG_TAG}: Cloud message ${msgType} has no requestId, cannot route`);
372
+ return;
373
+ }
374
+ const pending = this.pendingCloudRequests.get(requestId);
375
+ if (!pending) {
376
+ console.warn(`${LOG_TAG}: No pending request for requestId=${requestId}`);
377
+ return;
378
+ }
379
+ this.pendingCloudRequests.delete(requestId);
380
+ switch (msgType) {
381
+ case "phone_stream_status": {
382
+ // Unreachable for phone-orchestrated streams (the coordinator owns
383
+ // their lifecycle and never registers a pending cloud request).
384
+ // Retained as a safety net for any legacy registration path.
385
+ this.sendToMiniapp(pending.packageName, {
386
+ type: MiniappResponseType.EVENT,
387
+ streamType: "stream_status",
388
+ data: {
389
+ streamId: msg.streamId,
390
+ status: msg.status,
391
+ errorDetails: msg.errorDetails,
392
+ },
393
+ });
394
+ // Re-register for ongoing status updates (streams send multiple status messages)
395
+ this.pendingCloudRequests.set(requestId, pending);
396
+ break;
397
+ }
398
+ case "phone_managed_stream_status": {
399
+ if (msg.status === "connected" || msg.status === "active") {
400
+ // Managed stream is ready — send back the playback URLs as the request result
401
+ this.sendResult(pending.packageName, pending.envelopeRequestId, true, {
402
+ streamId: msg.streamId,
403
+ hlsUrl: msg.hlsUrl,
404
+ dashUrl: msg.dashUrl,
405
+ webrtcUrl: msg.webrtcUrl,
406
+ });
407
+ }
408
+ // Forward all statuses as events too
409
+ this.sendToMiniapp(pending.packageName, {
410
+ type: MiniappResponseType.EVENT,
411
+ streamType: "stream_status",
412
+ data: {
413
+ streamId: msg.streamId,
414
+ status: msg.status,
415
+ hlsUrl: msg.hlsUrl,
416
+ dashUrl: msg.dashUrl,
417
+ webrtcUrl: msg.webrtcUrl,
418
+ },
419
+ });
420
+ // Re-register for ongoing updates
421
+ this.pendingCloudRequests.set(requestId, pending);
422
+ break;
423
+ }
424
+ default:
425
+ console.warn(`${LOG_TAG}: Unknown cloud message type: ${msgType}`);
426
+ }
427
+ }
428
+ /**
429
+ * Register a pending cloud request so we can route the response back.
430
+ */
431
+ registerPendingCloudRequest(requestId, packageName, envelopeRequestId) {
432
+ this.pendingCloudRequests.set(requestId, { packageName, envelopeRequestId });
433
+ }
434
+ // ===========================================================================
435
+ // App registration
436
+ // ===========================================================================
437
+ registerApp(packageName, sendFn, installedManifest) {
438
+ console.log(`${LOG_TAG}: registerApp(${packageName})`);
439
+ this.clearForegroundProbe(packageName);
440
+ // Fresh spawn → fresh handshake. Clear any prior handshake flag so
441
+ // waitForConnect() blocks until this context's own CONNECT arrives.
442
+ this.handshookApps.delete(packageName);
443
+ // If the app is already registered (e.g. QR scanned again for same package),
444
+ // tear down its old subscriptions first so streamSubscribers doesn't keep
445
+ // dangling references. The WebView will re-subscribe after its CONNECT.
446
+ if (this.connectedApps.has(packageName)) {
447
+ const existing = this.connectedApps.get(packageName);
448
+ this.cancelSpeech(packageName);
449
+ // A crash respawn replaces the ConnectedMiniapp without going through
450
+ // unregisterApp(). Abort the old incarnation's native PCM player before
451
+ // its only stream id is lost. Target the exact stream rather than using
452
+ // stopForApp(), whose async playback cleanup could race with audio the
453
+ // replacement app starts after it connects.
454
+ const existingSpeakerStreamId = existing.activeSpeakerStreamId;
455
+ if (existingSpeakerStreamId) {
456
+ existing.activeSpeakerStreamId = undefined;
457
+ void audioPlaybackService.abortStream(existingSpeakerStreamId).catch((error) => {
458
+ console.warn(`${LOG_TAG}: failed to abort speaker stream while replacing ${packageName}`, error);
459
+ });
460
+ }
461
+ if (existing.authRefreshTimerId !== null) {
462
+ BgTimer.clearTimeout(existing.authRefreshTimerId);
463
+ existing.authRefreshTimerId = null;
464
+ }
465
+ if (existing.authRetryTimerId !== null) {
466
+ BgTimer.clearTimeout(existing.authRetryTimerId);
467
+ existing.authRetryTimerId = null;
468
+ }
469
+ for (const stream of existing.subscriptions) {
470
+ const subs = this.streamSubscribers.get(stream);
471
+ if (subs) {
472
+ subs.delete(packageName);
473
+ if (subs.size === 0)
474
+ this.streamSubscribers.delete(stream);
475
+ }
476
+ }
477
+ this.recomputeMicRequirements();
478
+ this.updateCloudSubscriptions();
479
+ }
480
+ this.connectedApps.set(packageName, {
481
+ subscriptions: new Set(),
482
+ forceLocalTranscriptionStreams: new Set(),
483
+ cloudTranscriptionStreams: new Set(),
484
+ sendMessage: sendFn,
485
+ lastPongAt: Date.now(),
486
+ installedManifest,
487
+ authRefreshTimerId: null,
488
+ authRetryTimerId: null,
489
+ authDelivered: false,
490
+ speakerState: "idle",
491
+ requestedLocationRate: null,
492
+ });
493
+ // If we just clobbered an existing entry, its requestedLocationRate is
494
+ // now gone — recompute so the aggregate doesn't include a stale rate.
495
+ // The WebView will re-SUBSCRIBE shortly and the rate will reappear.
496
+ this.recomputeLocationTier();
497
+ this.ensureCloudStatusWired();
498
+ this.ensurePingLoop();
499
+ // Show the system boot message ("Starting <name>…") on the glasses for the
500
+ // bounded boot window, mirroring the cloud DisplayManager boot screen. The
501
+ // window ends early once this app pushes its first display (see
502
+ // LocalDisplayManager.request), or after ~1.5s. Fires once per spawn —
503
+ // including a crash-respawn, since each spawn is a fresh "app is starting".
504
+ localDisplayManager.onMount(packageName, installedManifest?.name ?? packageName);
505
+ }
506
+ /**
507
+ * Resolve once `packageName` has completed its CONNECT handshake. If it has
508
+ * already connected, resolves immediately. Rejects if the timeout elapses
509
+ * first, or if the app is unregistered while waiting.
510
+ *
511
+ * Used by the launcher's `ensureConnected` (and thus the action broker) so a
512
+ * just-spawned target is known to be live before anything is delivered to it.
513
+ */
514
+ waitForConnect(packageName, timeoutMs) {
515
+ if (this.handshookApps.has(packageName))
516
+ return Promise.resolve();
517
+ return new Promise((resolve, reject) => {
518
+ let settled = false;
519
+ const waiter = (err) => {
520
+ if (settled)
521
+ return;
522
+ settled = true;
523
+ clearTimeout(timer);
524
+ this.connectWaiters.get(packageName)?.delete(waiter);
525
+ if (err)
526
+ reject(err);
527
+ else
528
+ resolve();
529
+ };
530
+ const timer = setTimeout(() => {
531
+ waiter(new Error(`waitForConnect: ${packageName} did not connect within ${timeoutMs}ms`));
532
+ }, timeoutMs);
533
+ let set = this.connectWaiters.get(packageName);
534
+ if (!set) {
535
+ set = new Set();
536
+ this.connectWaiters.set(packageName, set);
537
+ }
538
+ set.add(waiter);
539
+ });
540
+ }
541
+ /** Resolve/reject all pending {@link waitForConnect} waiters for a package. */
542
+ flushConnectWaiters(packageName, err) {
543
+ const waiters = this.connectWaiters.get(packageName);
544
+ if (!waiters)
545
+ return;
546
+ this.connectWaiters.delete(packageName);
547
+ for (const w of waiters)
548
+ w(err);
549
+ }
550
+ /**
551
+ * Invalidate the CONNECT handshake for a package whose native context is
552
+ * being replaced WITHOUT going through register/unregister — i.e. a crash
553
+ * respawn. Without this, {@link waitForConnect} would treat the dead/
554
+ * not-yet-initialized context as connected and deliver to it. Pending waiters
555
+ * are failed so a wake mid-respawn retries rather than hanging.
556
+ */
557
+ resetHandshake(packageName) {
558
+ this.handshookApps.delete(packageName);
559
+ this.flushConnectWaiters(packageName, new Error(`${packageName} respawning`));
560
+ }
561
+ /**
562
+ * Attach (or update) the installedManifest for an already-registered app.
563
+ * Used when the manifest is fetched asynchronously (dev miniapps) after the
564
+ * miniapp has already CONNECTed — preserves existing subscriptions.
565
+ */
566
+ setInstalledManifest(packageName, installedManifest) {
567
+ const app = this.connectedApps.get(packageName);
568
+ if (!app)
569
+ return;
570
+ app.installedManifest = installedManifest;
571
+ // Push declared-permission record to the SDK so session.permissions stays
572
+ // in sync. Sent regardless of CONNECT_ACK timing — covers the dev-miniapp
573
+ // case where the manifest is fetched async after the miniapp CONNECTs.
574
+ this.sendToMiniapp(packageName, {
575
+ type: MiniappResponseType.PERMISSIONS_UPDATE,
576
+ permissions: computeDeclaredPermissionRecord(installedManifest),
577
+ });
578
+ }
579
+ /**
580
+ * Graceful version of {@link unregisterApp}: notify the miniapp via
581
+ * `WILL_DISCONNECT`, wait ~50ms so its `beforeDisconnect` handlers can
582
+ * fire one last `sendOneShot` (e.g. `display.clear()`), then run the
583
+ * normal teardown. Use this for any disconnect path where the socket
584
+ * is still open. For ungraceful paths (transport already closed),
585
+ * call {@link unregisterApp} directly — the heads-up would be
586
+ * undeliverable anyway.
587
+ */
588
+ async gracefullyUnregisterApp(packageName, reason = "unregistering") {
589
+ const app = this.connectedApps.get(packageName);
590
+ if (!app)
591
+ return;
592
+ console.log(`${LOG_TAG}: gracefullyUnregisterApp(${packageName}) — sending WILL_DISCONNECT`);
593
+ this.sendToMiniapp(packageName, {
594
+ type: MiniappResponseType.WILL_DISCONNECT,
595
+ reason,
596
+ });
597
+ // 50 ms: imperceptible to the user, plenty of time for the SDK's
598
+ // synchronous `beforeDisconnect` handlers to flush a final
599
+ // sendOneShot through the still-open transport.
600
+ await new Promise((resolve) => setTimeout(resolve, 50));
601
+ this.unregisterApp(packageName);
602
+ }
603
+ unregisterApp(packageName) {
604
+ console.log(`${LOG_TAG}: unregisterApp(${packageName})`);
605
+ this.clearForegroundProbe(packageName);
606
+ this.clearMiniappAuthRefresh(packageName);
607
+ this.clearMiniappAuthDeliveryRetry(packageName);
608
+ // Drop the handshake flag and fail any in-flight waitForConnect() callers
609
+ // (e.g. a wake that's mid-handshake when the app is torn down). Done before
610
+ // the early-return so it runs even if the connectedApps entry is already gone.
611
+ this.handshookApps.delete(packageName);
612
+ this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`));
613
+ // Warm-up is a request-owned camera lease. Cancel it even if the app record was already
614
+ // removed so a close racing the camera-open promise cannot leave the sensor running.
615
+ // Keep camera teardown ordered: restoring FOV can restart the HAL, so the warm-up lease must
616
+ // finish closing the sensor before FOV reconciliation begins. Continue to FOV cleanup even if
617
+ // warm-up cancellation fails; phone ownership remains retryable until the ASG lease TTL.
618
+ void phonePhotoCoordinator
619
+ .stopWarmUpForApp(packageName)
620
+ .catch((error) => {
621
+ console.warn(`${LOG_TAG}: failed to stop camera warm-up for ${packageName} on unregister`, error);
622
+ })
623
+ .then(() => phoneCameraFovCoordinator.releaseForApp(packageName))
624
+ .catch((error) => {
625
+ console.warn(`${LOG_TAG}: failed to release camera FOV override for ${packageName} on unregister`, error);
626
+ });
627
+ const app = this.connectedApps.get(packageName);
628
+ if (!app)
629
+ return;
630
+ // Remove from all stream subscriber sets
631
+ for (const stream of app.subscriptions) {
632
+ const subs = this.streamSubscribers.get(stream);
633
+ if (subs) {
634
+ subs.delete(packageName);
635
+ if (subs.size === 0) {
636
+ this.streamSubscribers.delete(stream);
637
+ }
638
+ }
639
+ }
640
+ // Reset per-session warning dedup so a relaunch surfaces issues again.
641
+ resetPermissionWarnings(packageName);
642
+ // Cancel lookahead/in-flight synthesis before stopping the current file so
643
+ // an interrupted sentence cannot advance and reclaim playback.
644
+ this.cancelSpeech(packageName);
645
+ // Stop audio for this app
646
+ audioPlaybackService.stopForApp(packageName);
647
+ // Tear down this app's blob state: abort in-flight uploads + close readers
648
+ // so a crashed/closed miniapp doesn't leak partial files or file handles.
649
+ this.blobStore.onAppGone(packageName);
650
+ // Release phone-owned camera streams. If a miniapp closes/crashes without
651
+ // sending STREAM_STOP, the host coordinator must drop its subscriber/owner
652
+ // so glasses publishing and managed Cloudflare inputs do not leak.
653
+ void phoneStreamCoordinator.stop(packageName).catch((error) => {
654
+ console.warn(`${LOG_TAG}: failed to stop stream for ${packageName} on unregister`, error);
655
+ });
656
+ // Stop any phone-owned video recordings for this app. A miniapp that
657
+ // closes/crashes mid-recording loses its recordingId, so without this the
658
+ // glasses keep recording until the max-recording timeout or thermal shutdown.
659
+ void phoneVideoCoordinator.stopForApp(packageName).catch((error) => {
660
+ console.warn(`${LOG_TAG}: failed to stop video recording for ${packageName} on unregister`, error);
661
+ });
662
+ // Detach the per-app nav event forwarder but leave the native nav session
663
+ // running. The user may have just closed the mini-app UI and will reopen
664
+ // it; stopping the session here would kill an active trip mid-route.
665
+ // Navigation is only stopped when the mini-app explicitly calls
666
+ // navigation.stop() or when the trip arrives/errors naturally.
667
+ // (See NavigationHandlers — activeNavApps stays populated so a reconnect
668
+ // can reattach listeners and resume.)
669
+ this.navigationHandlers.onDisconnect(packageName);
670
+ // Recompute heading subscription — if this app was the last subscriber,
671
+ // the sensor will stop.
672
+ this.recomputeHeadingSubscription();
673
+ // Same for the IMU/accelerometer stream.
674
+ this.recomputeImuSubscription();
675
+ // Drop this app's location-tier request before recomputing so the
676
+ // aggregate falls back down if it was the strictest. Done before
677
+ // `connectedApps.delete` below so it doesn't matter that the iter
678
+ // in `recomputeLocationTier` is over connectedApps — clearing the
679
+ // field is enough.
680
+ app.requestedLocationRate = null;
681
+ this.recomputeLocationTier();
682
+ // Clean up any pending cloud requests from this app
683
+ for (const [reqId, pending] of this.pendingCloudRequests) {
684
+ if (pending.packageName === packageName) {
685
+ this.pendingCloudRequests.delete(reqId);
686
+ }
687
+ }
688
+ // Release any display real estate this app held — if it owned the
689
+ // current on-glasses frame, this clears the glasses (or restores the
690
+ // core app's saved frame).
691
+ localDisplayManager.onUnmount(packageName);
692
+ this.connectedApps.delete(packageName);
693
+ this.recomputeMicRequirements();
694
+ this.updateCloudSubscriptions();
695
+ if (this.connectedApps.size === 0) {
696
+ this.stopPingLoop();
697
+ }
698
+ }
699
+ // ===========================================================================
700
+ // Inbound message handling
701
+ // ===========================================================================
702
+ handleRawMessage(packageName, raw) {
703
+ const envelope = parseEnvelope(raw);
704
+ if (!envelope) {
705
+ // Not a miniapp envelope — ignore (could be legacy bridge message)
706
+ return;
707
+ }
708
+ const payload = envelope.payload;
709
+ const requestType = payload.type;
710
+ const requestId = envelope.requestId;
711
+ if (!requestType) {
712
+ console.warn(`${LOG_TAG}: Envelope from ${packageName} missing payload.type`);
713
+ return;
714
+ }
715
+ // ANY inbound message proves the context is alive — a busy background
716
+ // script streaming DISPLAY/storage traffic shouldn't be killed by the
717
+ // liveness watchdog just because its PONG replies queue behind real work.
718
+ this.handlePong(packageName);
719
+ // Console-tap forwarding. The miniapp's console.log/warn/etc is wrapped
720
+ // (via injected shim from miniappGlobals.ts) to post a `dev_log`
721
+ // envelope. We fan out to two destinations:
722
+ // 1. DevServerBridge — forwards to the laptop's `mentra-miniapp dev`
723
+ // terminal. No-op when there's no sidecar (installed miniapps).
724
+ // 2. React Native console — surfaces the log in Metro / Xcode console
725
+ // / adb logcat so installed-miniapp errors are still inspectable
726
+ // when there's no laptop to forward to.
727
+ if (requestType === "dev_log") {
728
+ const level = payload.level ?? "log";
729
+ const args = Array.isArray(payload.args) ? payload.args : [];
730
+ const timestamp = payload.timestamp ?? Date.now();
731
+ // Source is "ui" because the single-bundle WebView console-tap
732
+ // shim in miniappGlobals.ts is the only thing that posts
733
+ // `dev_log`; the two-layer path uses a separate envelope
734
+ // (`{type:"log", source}`) routed by MentraUIRouter.
735
+ devServerBridge.forwardLog(packageName, level, args, timestamp, "ui");
736
+ const tag = `[MINIAPP ${packageName}]`;
737
+ const fn = console[level] ?? console.log;
738
+ try {
739
+ fn(tag, ...args);
740
+ }
741
+ catch {
742
+ console.log(tag, ...args);
743
+ }
744
+ return;
745
+ }
746
+ // Dispatch
747
+ switch (requestType) {
748
+ case MiniappRequestType.CONNECT:
749
+ void this.handleConnect(packageName, payload, requestId);
750
+ break;
751
+ case MiniappRequestType.AUTH_REFRESH:
752
+ void this.handleAuthRefresh(packageName, payload, requestId);
753
+ break;
754
+ case MiniappRequestType.SUBSCRIBE:
755
+ this.handleSubscribe(packageName, payload, requestId);
756
+ break;
757
+ case MiniappRequestType.DISPLAY:
758
+ this.handleDisplay(packageName, payload, requestId);
759
+ break;
760
+ case MiniappRequestType.RENDER:
761
+ this.handleRender(packageName, payload, requestId);
762
+ break;
763
+ case MiniappRequestType.PLAY_AUDIO:
764
+ this.handlePlayAudio(packageName, payload, requestId);
765
+ break;
766
+ case MiniappRequestType.STOP_AUDIO:
767
+ this.handleStopAudio(packageName, payload, requestId);
768
+ break;
769
+ case MiniappRequestType.SPEAK:
770
+ this.handleSpeak(packageName, payload, requestId);
771
+ break;
772
+ case MiniappRequestType.SPEAKER_STREAM_OPEN:
773
+ void this.handleSpeakerStreamOpen(packageName, payload, requestId);
774
+ break;
775
+ case MiniappRequestType.SPEAKER_STREAM_WRITE:
776
+ void this.handleSpeakerStreamWrite(packageName, payload, requestId);
777
+ break;
778
+ case MiniappRequestType.SPEAKER_STREAM_CLOSE:
779
+ void this.handleSpeakerStreamClose(packageName, payload, requestId);
780
+ break;
781
+ case MiniappRequestType.SPEAKER_STREAM_ABORT:
782
+ void this.handleSpeakerStreamAbort(packageName, payload, requestId);
783
+ break;
784
+ case MiniappRequestType.RGB_LED:
785
+ void this.handleRgbLed(packageName, payload, requestId);
786
+ break;
787
+ case MiniappRequestType.LOCATION_POLL:
788
+ this.handleLocationPoll(packageName, requestId);
789
+ break;
790
+ case MiniappRequestType.CALENDAR_LIST_EVENTS:
791
+ void this.handleCalendarListEvents(packageName, payload, requestId);
792
+ break;
793
+ case MiniappRequestType.NAVIGATION_START:
794
+ this.navigationHandlers.handleStart(packageName, payload, requestId);
795
+ break;
796
+ case MiniappRequestType.NAVIGATION_STOP:
797
+ this.navigationHandlers.handleStop(packageName, requestId);
798
+ break;
799
+ case MiniappRequestType.NAVIGATION_DEVIATE:
800
+ this.navigationHandlers.handleDeviate(packageName, payload, requestId);
801
+ break;
802
+ case MiniappRequestType.NAVIGATION_SET_WRONG_SIDEWALK:
803
+ this.navigationHandlers.handleSetWrongSidewalk(packageName, payload, requestId);
804
+ break;
805
+ case MiniappRequestType.NAVIGATION_SET_SKIP_CROSSINGS:
806
+ this.navigationHandlers.handleSetSkipCrossings(packageName, payload, requestId);
807
+ break;
808
+ case MiniappRequestType.NAVIGATION_GET_STATE:
809
+ this.navigationHandlers.handleGetState(packageName, requestId);
810
+ break;
811
+ case MiniappRequestType.NAVIGATION_COMPUTE_ROUTE:
812
+ this.navigationHandlers.handleComputeRoute(packageName, payload, requestId);
813
+ break;
814
+ case MiniappRequestType.NAVIGATION_REVERSE_GEOCODE:
815
+ this.navigationHandlers.handleReverseGeocode(packageName, payload, requestId);
816
+ break;
817
+ case MiniappRequestType.NAVIGATION_PLACE_AUTOCOMPLETE:
818
+ this.navigationHandlers.handlePlaceAutocomplete(packageName, payload, requestId);
819
+ break;
820
+ case MiniappRequestType.NAVIGATION_PLACE_DETAILS:
821
+ this.navigationHandlers.handlePlaceDetails(packageName, payload, requestId);
822
+ break;
823
+ case MiniappRequestType.NAVIGATION_REQUEST_PERMISSION:
824
+ this.navigationHandlers.handleRequestPermission(packageName, requestId);
825
+ break;
826
+ case MiniappRequestType.STORAGE_GET:
827
+ this.handleStorageGet(packageName, payload, requestId);
828
+ break;
829
+ case MiniappRequestType.STORAGE_SET:
830
+ this.handleStorageSet(packageName, payload, requestId);
831
+ break;
832
+ case MiniappRequestType.STORAGE_DELETE:
833
+ this.handleStorageDelete(packageName, payload, requestId);
834
+ break;
835
+ case MiniappRequestType.STORAGE_LIST:
836
+ this.handleStorageList(packageName, payload, requestId);
837
+ break;
838
+ case MiniappRequestType.STORAGE_CLEAR:
839
+ this.handleStorageClear(packageName, requestId);
840
+ break;
841
+ case MiniappRequestType.STORAGE_HAS:
842
+ this.handleStorageHas(packageName, payload, requestId);
843
+ break;
844
+ case MiniappRequestType.STORAGE_GET_ALL:
845
+ this.handleStorageGetAll(packageName, requestId);
846
+ break;
847
+ case MiniappRequestType.STORAGE_SET_MULTIPLE:
848
+ this.handleStorageSetMultiple(packageName, payload, requestId);
849
+ break;
850
+ case MiniappRequestType.STORAGE_FLUSH:
851
+ // No-op today — MMKV writes through synchronously. Reserved so
852
+ // a future debounced backend can honor "flush before I quit"
853
+ // calls without breaking the API.
854
+ this.sendResult(packageName, requestId, true);
855
+ break;
856
+ case MiniappRequestType.CAMERA_FOV:
857
+ void this.handleCameraFov(packageName, payload, requestId);
858
+ break;
859
+ case MiniappRequestType.IMU_SET_ENABLED:
860
+ this.handleImuSetEnabled(packageName, payload, requestId);
861
+ break;
862
+ case MiniappRequestType.PING:
863
+ // SDK should handle this itself; reply PONG just in case
864
+ this.sendToMiniapp(packageName, { type: MiniappResponseType.PONG }, requestId);
865
+ break;
866
+ case MiniappResponseType.PONG: {
867
+ // Liveness already touched above for every inbound message; the
868
+ // PONG carries no other action.
869
+ break;
870
+ }
871
+ case MiniappRequestType.SHARE:
872
+ this.handleShare(packageName, payload, requestId);
873
+ break;
874
+ case MiniappRequestType.OPEN_URL:
875
+ this.handleOpenUrl(packageName, payload);
876
+ break;
877
+ case MiniappRequestType.COPY_CLIPBOARD:
878
+ this.handleCopyClipboard(packageName, payload, requestId);
879
+ break;
880
+ case MiniappRequestType.DOWNLOAD:
881
+ this.handleDownload(packageName, payload, requestId);
882
+ break;
883
+ // Persistent binary blob storage (session.blob)
884
+ case MiniappRequestType.BLOB_CREATE:
885
+ this.blobStore.handleCreate(packageName, payload, requestId);
886
+ break;
887
+ case MiniappRequestType.BLOB_WRITE:
888
+ this.blobStore.handleWrite(packageName, payload, requestId);
889
+ break;
890
+ case MiniappRequestType.BLOB_COMMIT:
891
+ this.blobStore.handleCommit(packageName, payload, requestId);
892
+ break;
893
+ case MiniappRequestType.BLOB_ABORT:
894
+ this.blobStore.handleAbort(packageName, payload, requestId);
895
+ break;
896
+ case MiniappRequestType.BLOB_SET_FROM_URL:
897
+ void this.blobStore.handleSetFromUrl(packageName, payload, requestId);
898
+ break;
899
+ case MiniappRequestType.BLOB_IMPORT:
900
+ void this.blobStore.handleImport(packageName, payload, requestId);
901
+ break;
902
+ case MiniappRequestType.BLOB_GET:
903
+ this.blobStore.handleGet(packageName, payload, requestId);
904
+ break;
905
+ case MiniappRequestType.BLOB_LIST:
906
+ this.blobStore.handleList(packageName, payload, requestId);
907
+ break;
908
+ case MiniappRequestType.BLOB_USAGE:
909
+ this.blobStore.handleUsage(packageName, payload, requestId);
910
+ break;
911
+ case MiniappRequestType.BLOB_DELETE:
912
+ this.blobStore.handleDelete(packageName, payload, requestId);
913
+ break;
914
+ case MiniappRequestType.BLOB_CLEAR:
915
+ this.blobStore.handleClear(packageName, payload, requestId);
916
+ break;
917
+ case MiniappRequestType.BLOB_OPEN_READ:
918
+ this.blobStore.handleOpenRead(packageName, payload, requestId);
919
+ break;
920
+ case MiniappRequestType.BLOB_READ:
921
+ this.blobStore.handleRead(packageName, payload, requestId);
922
+ break;
923
+ case MiniappRequestType.BLOB_CLOSE_READ:
924
+ this.blobStore.handleCloseRead(packageName, payload, requestId);
925
+ break;
926
+ case MiniappRequestType.BLOB_SHARE:
927
+ void this.blobStore.handleShare(packageName, payload, requestId);
928
+ break;
929
+ // Cloud-coordinated features
930
+ case MiniappRequestType.PHOTO:
931
+ void this.handlePhoto(packageName, payload, requestId);
932
+ break;
933
+ case MiniappRequestType.CAMERA_WARM_UP:
934
+ void this.handleCameraWarmUp(packageName, payload, requestId);
935
+ break;
936
+ case MiniappRequestType.VIDEO_RECORDING_START:
937
+ void this.handleVideoRecordingStart(packageName, payload, requestId);
938
+ break;
939
+ case MiniappRequestType.VIDEO_RECORDING_STOP:
940
+ void this.handleVideoRecordingStop(packageName, payload, requestId);
941
+ break;
942
+ case MiniappRequestType.STREAM_START:
943
+ void this.handleStreamStart(packageName, payload, requestId);
944
+ break;
945
+ case MiniappRequestType.STREAM_STOP:
946
+ void this.handleStreamStop(packageName, payload, requestId);
947
+ break;
948
+ case MiniappRequestType.MANAGED_STREAM_START:
949
+ void this.handleManagedStreamStart(packageName, payload, requestId);
950
+ break;
951
+ case MiniappRequestType.MANAGED_STREAM_STOP:
952
+ void this.handleManagedStreamStop(packageName, payload, requestId);
953
+ break;
954
+ case REQUEST_WIFI_SETUP_TYPE:
955
+ void this.handleRequestWifiSetup(packageName, payload, requestId);
956
+ break;
957
+ // Inter-miniapp interop (SYSTEM apps only)
958
+ case MiniappRequestType.MINIAPPS_LIST:
959
+ this.handleMiniappsList(packageName, payload, requestId);
960
+ break;
961
+ case MiniappRequestType.MINIAPPS_START:
962
+ void this.handleMiniappsStart(packageName, payload, requestId);
963
+ break;
964
+ case MiniappRequestType.MINIAPPS_STOP:
965
+ void this.handleMiniappsStop(packageName, payload, requestId);
966
+ break;
967
+ case MiniappRequestType.ACTION_INVOKE:
968
+ void this.handleActionInvoke(packageName, payload, requestId);
969
+ break;
970
+ case MiniappRequestType.ACTION_RESULT:
971
+ this.handleActionResult(packageName, payload);
972
+ break;
973
+ // Deferred in v1
974
+ case MiniappRequestType.DASHBOARD_CONTENT_UPDATE:
975
+ this.sendResult(packageName, requestId, false, undefined, {
976
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
977
+ message: "Dashboard API is deferred in v1",
978
+ });
979
+ break;
980
+ default:
981
+ console.warn(`${LOG_TAG}: Unknown request type from ${packageName}: ${requestType}`);
982
+ break;
983
+ }
984
+ }
985
+ // ===========================================================================
986
+ // Request handlers
987
+ // ===========================================================================
988
+ async handleConnect(packageName, _payload, requestId) {
989
+ console.log(`${LOG_TAG}: CONNECT from ${packageName}`);
990
+ // Register if not already
991
+ const existing = this.connectedApps.get(packageName);
992
+ if (!existing) {
993
+ console.warn(`${LOG_TAG}: CONNECT from unregistered app ${packageName}, ignoring`);
994
+ return;
995
+ }
996
+ // Update lastPongAt so it doesn't time out right away
997
+ existing.lastPongAt = Date.now();
998
+ // Read current glasses capabilities from the settings store
999
+ const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable);
1000
+ const capabilities = getModelCapabilities(defaultWearable || DeviceTypes.NONE);
1001
+ // Build the declared-permission record for the SDK's session.permissions
1002
+ // module. Lower-cased to match v3's PermissionType union (microphone,
1003
+ // camera, location, notifications, calendar). Missing permissions default
1004
+ // to false; this is manifest-declaration tracking only — OS-grant state
1005
+ // is intentionally not modeled here.
1006
+ const declaredPermissions = computeDeclaredPermissionRecord(existing.installedManifest);
1007
+ // Fresh handshake → clear any auth state from a prior context so a stale
1008
+ // retry timer can't fire against this new connection and so the retry loop
1009
+ // below re-arms from scratch.
1010
+ existing.authDelivered = false;
1011
+ this.clearMiniappAuthDeliveryRetry(packageName);
1012
+ const authPromise = this.requestMiniappAuth(packageName);
1013
+ const initialAuth = await withTimeout(authPromise, 1_500);
1014
+ const userId = initialAuth?.mentraUserId ?? "";
1015
+ if (initialAuth) {
1016
+ existing.authDelivered = true;
1017
+ this.scheduleMiniappAuthRefresh(packageName, initialAuth);
1018
+ }
1019
+ this.sendToMiniapp(packageName, {
1020
+ type: MiniappResponseType.CONNECT_ACK,
1021
+ userId,
1022
+ packageName,
1023
+ capabilities,
1024
+ permissions: declaredPermissions,
1025
+ ...(initialAuth ? { auth: initialAuth } : {}),
1026
+ }, requestId);
1027
+ if (!initialAuth) {
1028
+ // The mint timed out or (more importantly) REJECTED — the latter happens
1029
+ // when the cloud client hasn't finished its first-boot Core token exchange
1030
+ // yet, which is exactly the case when a dev miniapp is scanned/launched
1031
+ // right after app start. CONNECT_ACK already went out without auth; keep
1032
+ // trying to mint (and re-drive on cloud-connect) so the app authenticates
1033
+ // to its backend on its own, instead of staying dead until a full manual
1034
+ // Cloud V2 reconnect re-runs the whole handshake.
1035
+ this.deliverInitialMiniappAuth(packageName, authPromise, 0);
1036
+ }
1037
+ this.sendCloudStatusToMiniapp(packageName);
1038
+ // Handshake complete — unblock any launcher.waitForConnect() callers.
1039
+ this.handshookApps.add(packageName);
1040
+ this.flushConnectWaiters(packageName);
1041
+ }
1042
+ async requestMiniappAuth(packageName, opts) {
1043
+ const authPackageName = getDevAppSourcePackage(packageName) ?? packageName;
1044
+ const devAttestation = getDevAppAttestation(packageName) ?? undefined;
1045
+ return cloudClientService.getMiniappAuthToken(authPackageName, {
1046
+ ...opts,
1047
+ ...(devAttestation ? { devAttestation } : {}),
1048
+ });
1049
+ }
1050
+ async handleAuthRefresh(packageName, payload, requestId) {
1051
+ try {
1052
+ const auth = await this.refreshMiniappAuth(packageName, this.authRefreshOptions(payload));
1053
+ if (!auth) {
1054
+ this.sendResult(packageName, requestId, false, undefined, {
1055
+ code: MiniappErrorCode.NOT_CONNECTED,
1056
+ message: "Miniapp auth is not configured",
1057
+ });
1058
+ return;
1059
+ }
1060
+ this.sendResult(packageName, requestId, true, { auth });
1061
+ }
1062
+ catch (err) {
1063
+ this.sendResult(packageName, requestId, false, undefined, {
1064
+ code: MiniappErrorCode.INTERNAL,
1065
+ message: err?.message ?? "Miniapp auth refresh failed",
1066
+ });
1067
+ }
1068
+ }
1069
+ async refreshMiniappAuth(packageName, opts) {
1070
+ const auth = await this.requestMiniappAuth(packageName, opts);
1071
+ if (!auth)
1072
+ return null;
1073
+ this.scheduleMiniappAuthRefresh(packageName, auth);
1074
+ this.sendToMiniapp(packageName, {
1075
+ type: MiniappResponseType.AUTH_UPDATE,
1076
+ auth,
1077
+ });
1078
+ return auth;
1079
+ }
1080
+ authRefreshOptions(payload) {
1081
+ const minTtlMs = Number(payload.minTtlMs);
1082
+ if (!Number.isFinite(minTtlMs) || minTtlMs <= 0)
1083
+ return undefined;
1084
+ return { minTtlMs };
1085
+ }
1086
+ scheduleMiniappAuthRefresh(packageName, auth) {
1087
+ const app = this.connectedApps.get(packageName);
1088
+ if (!app)
1089
+ return;
1090
+ this.clearMiniappAuthRefresh(packageName);
1091
+ const refreshAt = auth.expiresAt - MINIAPP_AUTH_REFRESH_HEADROOM_MS;
1092
+ const delay = Math.max(MINIAPP_AUTH_REFRESH_MIN_DELAY_MS, refreshAt - Date.now());
1093
+ app.authRefreshTimerId = BgTimer.setTimeout(() => {
1094
+ const current = this.connectedApps.get(packageName);
1095
+ if (!current)
1096
+ return;
1097
+ current.authRefreshTimerId = null;
1098
+ void this.refreshMiniappAuth(packageName).catch((err) => {
1099
+ console.warn(`${LOG_TAG}: miniapp auth refresh failed for ${packageName}: ${err?.message ?? err}`);
1100
+ });
1101
+ }, delay);
1102
+ }
1103
+ clearMiniappAuthRefresh(packageName) {
1104
+ const app = this.connectedApps.get(packageName);
1105
+ if (!app || app.authRefreshTimerId === null)
1106
+ return;
1107
+ const timerId = app.authRefreshTimerId;
1108
+ BgTimer.clearTimeout(timerId);
1109
+ app.authRefreshTimerId = null;
1110
+ }
1111
+ /**
1112
+ * Deliver the miniapp's FIRST auth token, retrying until it lands. Called from
1113
+ * {@link handleConnect} when the initial mint didn't resolve in the CONNECT_ACK
1114
+ * window. `inFlight` lets attempt 0 reuse the mint the handshake already
1115
+ * started (avoids a redundant duplicate request); later attempts mint fresh.
1116
+ *
1117
+ * A `null` result means auth is fundamentally unavailable (no miniappAuth hook)
1118
+ * — permanent, so we do NOT retry. A REJECTION means a transient failure
1119
+ * (typically the cloud client isn't connection-ready yet) — retry with backoff.
1120
+ * Either way we bail if the app has since disconnected or already got a token
1121
+ * (e.g. via {@link redriveMiniappAuthDelivery} on cloud-connect).
1122
+ */
1123
+ deliverInitialMiniappAuth(packageName, inFlight, attempt) {
1124
+ const app = this.connectedApps.get(packageName);
1125
+ if (!app || app.authDelivered)
1126
+ return;
1127
+ const mint = inFlight ?? this.requestMiniappAuth(packageName);
1128
+ void mint
1129
+ .then((auth) => {
1130
+ const current = this.connectedApps.get(packageName);
1131
+ if (!current || current.authDelivered)
1132
+ return;
1133
+ if (!auth)
1134
+ return; // no auth hook — permanent, don't spin
1135
+ current.authDelivered = true;
1136
+ this.clearMiniappAuthDeliveryRetry(packageName);
1137
+ this.scheduleMiniappAuthRefresh(packageName, auth);
1138
+ this.sendToMiniapp(packageName, { type: MiniappResponseType.AUTH_UPDATE, auth });
1139
+ })
1140
+ .catch((err) => {
1141
+ console.warn(`${LOG_TAG}: miniapp auth mint failed for ${packageName} (attempt ${attempt}): ${err?.message ?? err}`);
1142
+ this.scheduleInitialMiniappAuthRetry(packageName, attempt);
1143
+ });
1144
+ }
1145
+ scheduleInitialMiniappAuthRetry(packageName, attempt) {
1146
+ const app = this.connectedApps.get(packageName);
1147
+ if (!app || app.authDelivered)
1148
+ return;
1149
+ this.clearMiniappAuthDeliveryRetry(packageName);
1150
+ const delay = Math.min(MINIAPP_AUTH_RETRY_MAX_MS, MINIAPP_AUTH_RETRY_BASE_MS * 2 ** attempt);
1151
+ app.authRetryTimerId = BgTimer.setTimeout(() => {
1152
+ const current = this.connectedApps.get(packageName);
1153
+ if (!current)
1154
+ return;
1155
+ current.authRetryTimerId = null;
1156
+ this.deliverInitialMiniappAuth(packageName, undefined, attempt + 1);
1157
+ }, delay);
1158
+ }
1159
+ clearMiniappAuthDeliveryRetry(packageName) {
1160
+ const app = this.connectedApps.get(packageName);
1161
+ if (!app || app.authRetryTimerId === null)
1162
+ return;
1163
+ BgTimer.clearTimeout(app.authRetryTimerId);
1164
+ app.authRetryTimerId = null;
1165
+ }
1166
+ /**
1167
+ * When the cloud client (re)connects, immediately re-attempt the initial mint
1168
+ * for any handshook app still missing its token, instead of waiting out the
1169
+ * backoff. This is what turns "scan a dev app before the cloud is ready" from
1170
+ * "dead until you manually reconnect Cloud V2" into "authenticates on its own
1171
+ * the instant the connection comes up".
1172
+ */
1173
+ redriveMiniappAuthDelivery() {
1174
+ for (const [packageName, app] of this.connectedApps) {
1175
+ if (app.authDelivered)
1176
+ continue;
1177
+ // Only apps that finished their CONNECT handshake are in the mint-retry
1178
+ // state; skip anything still mid-handshake (handleConnect owns that).
1179
+ if (!this.handshookApps.has(packageName))
1180
+ continue;
1181
+ this.clearMiniappAuthDeliveryRetry(packageName);
1182
+ this.deliverInitialMiniappAuth(packageName, undefined, 0);
1183
+ }
1184
+ }
1185
+ handleSubscribe(packageName, payload, requestId) {
1186
+ const app = this.connectedApps.get(packageName);
1187
+ if (!app)
1188
+ return;
1189
+ const rawStreams = (payload.subscriptions ?? payload.streams);
1190
+ // Normalize: objects like {stream: "location_stream", rate: "realtime"} → extract stream name
1191
+ // "location_stream" is the rate-bearing alias for "location_update".
1192
+ // The strictest rate wins when multiple `location_stream` entries appear
1193
+ // in one SUBSCRIBE — the same rule we apply across apps.
1194
+ let requestedLocationRate = null;
1195
+ const forceLocalTranscriptionStreams = new Set();
1196
+ const cloudTranscriptionStreams = new Set();
1197
+ const streams = rawStreams?.map((s) => {
1198
+ if (typeof s === "object" && s !== null) {
1199
+ if (s.stream === "location_stream") {
1200
+ if (s.rate && locationRateRank(s.rate) > locationRateRank(requestedLocationRate)) {
1201
+ requestedLocationRate = s.rate;
1202
+ }
1203
+ return "location_update";
1204
+ }
1205
+ if (s.forceLocal === true && s.stream.startsWith("transcription:")) {
1206
+ forceLocalTranscriptionStreams.add(s.stream);
1207
+ }
1208
+ if (s.stream.startsWith("transcription:") && (s.forceLocal !== true || s.includeCloud === true)) {
1209
+ cloudTranscriptionStreams.add(s.stream);
1210
+ }
1211
+ return s.stream;
1212
+ }
1213
+ if (s.startsWith("transcription:"))
1214
+ cloudTranscriptionStreams.add(s);
1215
+ return s;
1216
+ });
1217
+ if (!Array.isArray(streams)) {
1218
+ this.sendResult(packageName, requestId, false, undefined, {
1219
+ code: MiniappErrorCode.INTERNAL,
1220
+ message: "subscribe requires a subscriptions array",
1221
+ });
1222
+ return;
1223
+ }
1224
+ console.log(`${LOG_TAG}: SUBSCRIBE from ${packageName}: [${streams.join(", ")}]`);
1225
+ // Gate each stream on the permission type its data requires. The manifest
1226
+ // must declare the permission (miniapp.json -> permissions) or we reject
1227
+ // the whole subscribe with PERMISSION_NOT_DECLARED.
1228
+ const declaredTypes = new Set((app.installedManifest?.permissions ?? []).map((p) => p.type?.toUpperCase()));
1229
+ const permissionForStream = (s) => {
1230
+ if (s === "audio_chunk" || s === "vad")
1231
+ return "MICROPHONE";
1232
+ if (s.startsWith("transcription") || s.startsWith("translation"))
1233
+ return "MICROPHONE";
1234
+ if (s === "location_update")
1235
+ return "LOCATION";
1236
+ if (s === "phone_notification")
1237
+ return "READ_NOTIFICATIONS";
1238
+ if (s === "phone_notification_dismissed")
1239
+ return "READ_NOTIFICATIONS";
1240
+ return null;
1241
+ };
1242
+ for (const stream of streams) {
1243
+ const required = permissionForStream(stream);
1244
+ if (required && !declaredTypes.has(required)) {
1245
+ logPermissionNotDeclared(packageName, required, `to subscribe to "${stream}"`, `{"type": "${required}"}`);
1246
+ this.sendResult(packageName, requestId, false, undefined, {
1247
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
1248
+ message: `${required} permission not declared in miniapp.json (required for "${stream}"). Add {"type": "${required}"} to the "permissions" array.`,
1249
+ // Extra context fields read by the SDK so authors that subscribe
1250
+ // to session.on("error") can format their own messages.
1251
+ permission: required,
1252
+ subscription: stream,
1253
+ });
1254
+ return;
1255
+ }
1256
+ }
1257
+ // Remove old subscriptions for this app
1258
+ for (const oldStream of app.subscriptions) {
1259
+ const subs = this.streamSubscribers.get(oldStream);
1260
+ if (subs) {
1261
+ subs.delete(packageName);
1262
+ if (subs.size === 0) {
1263
+ this.streamSubscribers.delete(oldStream);
1264
+ }
1265
+ }
1266
+ }
1267
+ // Add new subscriptions
1268
+ app.subscriptions = new Set(streams);
1269
+ app.forceLocalTranscriptionStreams = new Set([...forceLocalTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)));
1270
+ app.cloudTranscriptionStreams = new Set([...cloudTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)));
1271
+ for (const stream of streams) {
1272
+ let subs = this.streamSubscribers.get(stream);
1273
+ if (!subs) {
1274
+ subs = new Set();
1275
+ this.streamSubscribers.set(stream, subs);
1276
+ }
1277
+ subs.add(packageName);
1278
+ }
1279
+ this.recomputeMicRequirements();
1280
+ this.updateCloudSubscriptions();
1281
+ this.recomputeHeadingSubscription();
1282
+ this.recomputeImuSubscription();
1283
+ // Persist this app's requested rate (or clear it if SUBSCRIBE didn't
1284
+ // include `location_stream` this time), then ask the host for the
1285
+ // strictest rate across all connected apps.
1286
+ app.requestedLocationRate = requestedLocationRate;
1287
+ this.recomputeLocationTier();
1288
+ this.sendResult(packageName, requestId, true);
1289
+ // Fire initial snapshot values for stateful streams so miniapps don't have
1290
+ // to wait for the first change event.
1291
+ this.emitInitialSnapshots(packageName, streams);
1292
+ }
1293
+ async handleCalendarListEvents(packageName, payload, requestId) {
1294
+ const app = this.connectedApps.get(packageName);
1295
+ const declared = app?.installedManifest?.permissions?.some((permission) => permission.type?.toUpperCase() === "CALENDAR");
1296
+ if (!declared) {
1297
+ logPermissionNotDeclared(packageName, "CALENDAR", "to list calendar events", '{"type": "CALENDAR"}');
1298
+ this.sendResult(packageName, requestId, false, undefined, {
1299
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
1300
+ message: 'CALENDAR permission not declared in miniapp.json. Add {"type": "CALENDAR"} to the permissions array.',
1301
+ permission: "CALENDAR",
1302
+ });
1303
+ return;
1304
+ }
1305
+ try {
1306
+ const result = await listPhoneCalendarEvents(payload);
1307
+ this.sendResult(packageName, requestId, true, result);
1308
+ }
1309
+ catch (error) {
1310
+ const known = error instanceof PhoneCalendarError;
1311
+ this.sendResult(packageName, requestId, false, undefined, {
1312
+ code: known ? error.code : MiniappErrorCode.INTERNAL,
1313
+ message: error instanceof Error ? error.message : "Failed to read phone calendar events",
1314
+ });
1315
+ }
1316
+ }
1317
+ /**
1318
+ * For state-bearing streams (battery, connection), deliver the current value
1319
+ * immediately on subscribe. Other streams (button press, transcription, etc.)
1320
+ * are pure event streams with no "current value" to snapshot.
1321
+ */
1322
+ emitInitialSnapshots(packageName, streams) {
1323
+ // Read the engine glasses store directly (was a host glassesStatus hook).
1324
+ const gs = useGlassesStore.getState();
1325
+ const glassesState = {
1326
+ connected: isGlassesConnected(gs.connection),
1327
+ deviceModel: gs.deviceModel,
1328
+ batteryLevel: gs.batteryLevel,
1329
+ charging: gs.charging,
1330
+ };
1331
+ const isSimulated = (glassesState.deviceModel || "").toLowerCase().includes("simulated");
1332
+ for (const stream of streams) {
1333
+ if (stream === "glasses_battery") {
1334
+ // Simulated glasses have no real battery; mirror the phone's battery
1335
+ // so miniapps see a sensible value during development.
1336
+ if (isSimulated) {
1337
+ void this.emitPhoneBatteryAs(packageName, "glasses_battery");
1338
+ }
1339
+ else if (typeof glassesState.batteryLevel === "number" && glassesState.batteryLevel >= 0) {
1340
+ this.sendToMiniapp(packageName, {
1341
+ type: MiniappResponseType.EVENT,
1342
+ streamType: "glasses_battery",
1343
+ data: {
1344
+ level: glassesState.batteryLevel,
1345
+ charging: !!glassesState.charging,
1346
+ timestamp: Date.now(),
1347
+ },
1348
+ });
1349
+ }
1350
+ }
1351
+ else if (stream === "phone_battery") {
1352
+ void this.emitPhoneBatteryAs(packageName, "phone_battery");
1353
+ }
1354
+ else if (stream === "glasses_connection") {
1355
+ this.sendToMiniapp(packageName, {
1356
+ type: MiniappResponseType.EVENT,
1357
+ streamType: "glasses_connection",
1358
+ data: glassesState,
1359
+ });
1360
+ }
1361
+ else if (stream === "glasses_wifi") {
1362
+ // Snapshot the current glasses Wi-Fi state on subscribe (like battery).
1363
+ // Read the canonical nested `wifi: {state, ssid}` from the STORE state
1364
+ // (`glassesState` is the narrowed 4-field projection and has no `wifi`;
1365
+ // the pre-engine host hook spread the full store, so read gs directly —
1366
+ // NOT the legacy flat wifiConnected). Effective connectivity requires the
1367
+ // glasses connected AND on Wi-Fi — mirrors the host's store-derived
1368
+ // forward, so an already-disconnected device reports `connected: false`
1369
+ // rather than silently emitting nothing. Always emits so onWifi gets an
1370
+ // initial value.
1371
+ const wifi = gs.wifi.state === "connected" ? gs.wifi : null;
1372
+ const connected = glassesState.connected === true && wifi !== null;
1373
+ this.sendToMiniapp(packageName, {
1374
+ type: MiniappResponseType.EVENT,
1375
+ streamType: "glasses_wifi",
1376
+ data: {
1377
+ connected,
1378
+ ssid: connected ? wifi?.ssid : undefined,
1379
+ localIp: connected ? wifi?.localIp : undefined,
1380
+ timestamp: Date.now(),
1381
+ },
1382
+ });
1383
+ }
1384
+ else if (stream === "head_position") {
1385
+ // The glasses store has no headUp field (same on the pre-engine host
1386
+ // hook), so this stays a no-emit until head state lands in the store.
1387
+ const headUp = gs.headUp;
1388
+ if (typeof headUp === "boolean") {
1389
+ this.sendToMiniapp(packageName, {
1390
+ type: MiniappResponseType.EVENT,
1391
+ streamType: "head_position",
1392
+ data: {
1393
+ position: headUp ? "up" : "down",
1394
+ timestamp: Date.now(),
1395
+ },
1396
+ });
1397
+ }
1398
+ }
1399
+ }
1400
+ }
1401
+ /**
1402
+ * Read the phone's battery state right now and emit it as the given stream.
1403
+ * Used for both phone_battery snapshot on subscribe, and as a stand-in for
1404
+ * glasses_battery when connected to Simulated Glasses.
1405
+ */
1406
+ async emitPhoneBatteryAs(packageName, streamType) {
1407
+ try {
1408
+ const level = await Battery.getBatteryLevelAsync();
1409
+ const state = await Battery.getBatteryStateAsync();
1410
+ const charging = state === Battery.BatteryState.CHARGING || state === Battery.BatteryState.FULL;
1411
+ this.sendToMiniapp(packageName, {
1412
+ type: MiniappResponseType.EVENT,
1413
+ streamType,
1414
+ data: {
1415
+ level: Math.round(level * 100),
1416
+ charging,
1417
+ timestamp: Date.now(),
1418
+ },
1419
+ });
1420
+ }
1421
+ catch (err) {
1422
+ console.log(`${LOG_TAG}: phone battery snapshot failed`, err);
1423
+ }
1424
+ }
1425
+ handleDisplay(packageName, payload, requestId) {
1426
+ try {
1427
+ // Zombie-frame gate: a dying miniapp's last in-flight frame can arrive
1428
+ // AFTER onUnmount cleared its display — repainting a dead app's UI (and
1429
+ // poisoning the next boot's pre-boot restore snapshot). Killed mid-route
1430
+ // nav reproduced this reliably; its 3s re-push raced the teardown.
1431
+ if (!this.connectedApps.has(packageName)) {
1432
+ console.log(`${LOG_TAG}: dropping display from unmounted app ${packageName}`);
1433
+ return;
1434
+ }
1435
+ if (!payload.layout || typeof payload.layout !== "object") {
1436
+ this.sendResult(packageName, requestId, false, undefined, {
1437
+ code: MiniappErrorCode.INTERNAL,
1438
+ message: "display request missing layout object",
1439
+ });
1440
+ return;
1441
+ }
1442
+ // Hand off to LocalDisplayManager — it owns boot/throttle/arbitration/
1443
+ // expiry + the native BluetoothSdk.displayEvent call + useDisplayStore.
1444
+ localDisplayManager.request(packageName, {
1445
+ view: payload.view ?? "main",
1446
+ layout: payload.layout,
1447
+ durationMs: payload.durationMs,
1448
+ });
1449
+ this.sendResult(packageName, requestId, true);
1450
+ }
1451
+ catch (err) {
1452
+ console.error(`${LOG_TAG}: display error:`, err);
1453
+ this.sendResult(packageName, requestId, false, undefined, {
1454
+ code: MiniappErrorCode.INTERNAL,
1455
+ message: err instanceof Error ? err.message : "Display error",
1456
+ });
1457
+ }
1458
+ }
1459
+ /**
1460
+ * Scene render (session.display.render()). Same arbitration as DISPLAY, but
1461
+ * the payload is a whole frame of positioned elements and the caller may be
1462
+ * awaiting the outcome: LocalDisplayManager guarantees the resolver fires
1463
+ * exactly once (displayed/blocked, with degraded/dropped reporting), and that
1464
+ * becomes the REQUEST_RESULT data.
1465
+ */
1466
+ handleRender(packageName, payload, requestId) {
1467
+ try {
1468
+ // Zombie-frame gate — see handleDisplay.
1469
+ if (!this.connectedApps.has(packageName)) {
1470
+ this.sendResult(packageName, requestId, true, { status: "blocked", reason: "app stopped" });
1471
+ return;
1472
+ }
1473
+ if (!Array.isArray(payload.elements)) {
1474
+ this.sendResult(packageName, requestId, false, undefined, {
1475
+ code: MiniappErrorCode.INTERNAL,
1476
+ message: "render request missing elements array",
1477
+ });
1478
+ return;
1479
+ }
1480
+ localDisplayManager.request(packageName, {
1481
+ view: payload.view ?? "main",
1482
+ scene: payload.elements,
1483
+ durationMs: payload.durationMs,
1484
+ }, (result) => this.sendResult(packageName, requestId, true, result));
1485
+ }
1486
+ catch (err) {
1487
+ console.error(`${LOG_TAG}: render error:`, err);
1488
+ this.sendResult(packageName, requestId, false, undefined, {
1489
+ code: MiniappErrorCode.INTERNAL,
1490
+ message: err instanceof Error ? err.message : "Render error",
1491
+ });
1492
+ }
1493
+ }
1494
+ handlePlayAudio(packageName, payload, requestId) {
1495
+ const audioUrl = (payload.audioUrl ?? payload.url);
1496
+ if (!audioUrl) {
1497
+ this.sendResult(packageName, requestId, false, undefined, {
1498
+ code: MiniappErrorCode.INTERNAL,
1499
+ message: "play_audio requires an audioUrl",
1500
+ });
1501
+ return;
1502
+ }
1503
+ const audioRequestId = requestId || `local_${Date.now()}`;
1504
+ const volume = typeof payload.volume === "number" ? payload.volume : 1.0;
1505
+ const stopOtherAudio = payload.stopOtherAudio !== false;
1506
+ this.cancelSpeech(packageName);
1507
+ this.setSpeakerState(packageName, "loading");
1508
+ audioPlaybackService.play({ requestId: audioRequestId, audioUrl, appId: packageName, volume, stopOtherAudio }, (_respId, success, error, duration) => {
1509
+ if (success) {
1510
+ this.setSpeakerState(packageName, "stopped", { durationMs: duration ?? undefined });
1511
+ }
1512
+ else {
1513
+ this.setSpeakerState(packageName, "error", {
1514
+ errorCode: MiniappErrorCode.INTERNAL,
1515
+ errorMessage: error ?? "play failed",
1516
+ durationMs: duration ?? undefined,
1517
+ });
1518
+ }
1519
+ this.sendResult(packageName, requestId, success, { duration }, error
1520
+ ? {
1521
+ code: MiniappErrorCode.INTERNAL,
1522
+ message: error,
1523
+ }
1524
+ : undefined);
1525
+ });
1526
+ // Optimistic "started playing" transition. The audio service doesn't
1527
+ // have a "playback actually started" callback today; cloud SDK v3 has
1528
+ // the same constraint and uses optimistic timing. Microtask so the
1529
+ // initial "loading" envelope flushes first.
1530
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"));
1531
+ }
1532
+ handleStopAudio(packageName, _payload, requestId) {
1533
+ this.cancelSpeech(packageName);
1534
+ audioPlaybackService.stopForApp(packageName);
1535
+ this.setSpeakerState(packageName, "stopped");
1536
+ this.sendResult(packageName, requestId, true);
1537
+ }
1538
+ // ── speaker.createStream (live PCM output) ────────────────────────────────
1539
+ /** Valid PCM sample rates for speaker streams. Mirrors the SDK contract. */
1540
+ static SPEAKER_STREAM_SAMPLE_RATES = new Set([16000, 24000, 48000]);
1541
+ static SPEAKER_STREAM_MAX_BASE64_CHARS = Math.ceil((256 * 1024) / 3) * 4;
1542
+ speakerStreamCounter = 0;
1543
+ async handleSpeakerStreamOpen(packageName, payload, requestId) {
1544
+ const app = this.connectedApps.get(packageName);
1545
+ if (!app)
1546
+ return;
1547
+ // One stream per miniapp: opening a second closes the first.
1548
+ const previous = app.activeSpeakerStreamId;
1549
+ if (previous) {
1550
+ app.activeSpeakerStreamId = undefined;
1551
+ await audioPlaybackService.abortStream(previous).catch((error) => {
1552
+ console.warn(`${LOG_TAG}: failed to abort previous speaker stream for ${packageName}`, error);
1553
+ });
1554
+ }
1555
+ const sampleRate = typeof payload.sampleRate === "number" ? payload.sampleRate : 16000;
1556
+ if (!LocalMiniappRuntime.SPEAKER_STREAM_SAMPLE_RATES.has(sampleRate)) {
1557
+ this.sendResult(packageName, requestId, false, undefined, {
1558
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1559
+ message: `unsupported sampleRate ${sampleRate} (use 16000, 24000, or 48000)`,
1560
+ });
1561
+ return;
1562
+ }
1563
+ const channels = typeof payload.channels === "number" ? payload.channels : 1;
1564
+ if (channels !== 1) {
1565
+ this.sendResult(packageName, requestId, false, undefined, {
1566
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1567
+ message: "speaker streams currently support mono PCM only",
1568
+ });
1569
+ return;
1570
+ }
1571
+ const volume = typeof payload.volume === "number" ? payload.volume : 1;
1572
+ if (!Number.isFinite(volume) || volume < 0 || volume > 1) {
1573
+ this.sendResult(packageName, requestId, false, undefined, {
1574
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1575
+ message: "speaker stream volume must be between 0 and 1",
1576
+ });
1577
+ return;
1578
+ }
1579
+ const streamId = `spkstream_${Date.now()}_${++this.speakerStreamCounter}`;
1580
+ this.setSpeakerState(packageName, "loading");
1581
+ try {
1582
+ await audioPlaybackService.openStream({
1583
+ streamId,
1584
+ appId: packageName,
1585
+ sampleRate,
1586
+ channels,
1587
+ volume,
1588
+ stopOtherAudio: payload.stopOtherAudio !== false,
1589
+ onEnded: (endedId, success, error, durationMs) => {
1590
+ const current = this.connectedApps.get(packageName);
1591
+ // Ignore completion from an app incarnation replaced by a crash
1592
+ // respawn. Its abort may finish after the new app has registered and
1593
+ // must not overwrite that app's speaker state.
1594
+ if (current !== app)
1595
+ return;
1596
+ if (current.activeSpeakerStreamId === endedId)
1597
+ current.activeSpeakerStreamId = undefined;
1598
+ if (success) {
1599
+ this.setSpeakerState(packageName, "stopped", { durationMs: durationMs ?? undefined });
1600
+ }
1601
+ else {
1602
+ this.setSpeakerState(packageName, "error", {
1603
+ errorCode: MiniappErrorCode.INTERNAL,
1604
+ errorMessage: error ?? "speaker stream failed",
1605
+ durationMs: durationMs ?? undefined,
1606
+ });
1607
+ }
1608
+ },
1609
+ });
1610
+ }
1611
+ catch (error) {
1612
+ const message = error instanceof Error ? error.message : String(error);
1613
+ this.setSpeakerState(packageName, "error", {
1614
+ errorCode: MiniappErrorCode.INTERNAL,
1615
+ errorMessage: message,
1616
+ });
1617
+ this.sendResult(packageName, requestId, false, undefined, {
1618
+ code: MiniappErrorCode.INTERNAL,
1619
+ message,
1620
+ });
1621
+ return;
1622
+ }
1623
+ // The native open can outlive a miniapp disconnect. Its ordinary teardown
1624
+ // ran while no StreamState existed yet, so close this late result here
1625
+ // instead of leaving an orphan AudioTrack/AVAudioEngine session.
1626
+ if (this.connectedApps.get(packageName) !== app) {
1627
+ await audioPlaybackService.abortStream(streamId);
1628
+ return;
1629
+ }
1630
+ app.activeSpeakerStreamId = streamId;
1631
+ // Optimistic "playing", same as play(): the chunk player starts as soon
1632
+ // as the first write lands. Microtask so "loading" flushes first.
1633
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"));
1634
+ this.sendResult(packageName, requestId, true, { streamId });
1635
+ }
1636
+ /** Resolve + validate a stream id from a payload against the app's active stream. */
1637
+ activeSpeakerStream(packageName, payload) {
1638
+ const streamId = typeof payload.streamId === "string" ? payload.streamId : undefined;
1639
+ const app = this.connectedApps.get(packageName);
1640
+ if (!streamId || !app || app.activeSpeakerStreamId !== streamId)
1641
+ return null;
1642
+ return streamId;
1643
+ }
1644
+ async handleSpeakerStreamWrite(packageName, payload, requestId) {
1645
+ const streamId = this.activeSpeakerStream(packageName, payload);
1646
+ if (!streamId) {
1647
+ this.sendResult(packageName, requestId, false, undefined, {
1648
+ code: MiniappErrorCode.INTERNAL,
1649
+ message: "speaker stream is not open",
1650
+ });
1651
+ return;
1652
+ }
1653
+ const base64 = typeof payload.base64 === "string" ? payload.base64 : "";
1654
+ if (!base64 || base64.length % 4 !== 0) {
1655
+ this.sendResult(packageName, requestId, false, undefined, {
1656
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1657
+ message: "speaker stream write requires padded base64 PCM data",
1658
+ });
1659
+ return;
1660
+ }
1661
+ if (base64.length > LocalMiniappRuntime.SPEAKER_STREAM_MAX_BASE64_CHARS) {
1662
+ this.sendResult(packageName, requestId, false, undefined, {
1663
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
1664
+ message: "speaker stream chunk exceeds the 256 KB raw PCM limit",
1665
+ });
1666
+ return;
1667
+ }
1668
+ try {
1669
+ const result = await audioPlaybackService.writeStreamChunk(streamId, base64);
1670
+ this.sendResult(packageName, requestId, true, result);
1671
+ }
1672
+ catch (error) {
1673
+ this.sendResult(packageName, requestId, false, undefined, {
1674
+ code: MiniappErrorCode.INTERNAL,
1675
+ message: error instanceof Error ? error.message : String(error),
1676
+ });
1677
+ }
1678
+ }
1679
+ async handleSpeakerStreamClose(packageName, payload, requestId) {
1680
+ const streamId = this.activeSpeakerStream(packageName, payload);
1681
+ if (!streamId) {
1682
+ this.sendResult(packageName, requestId, false, undefined, {
1683
+ code: MiniappErrorCode.INTERNAL,
1684
+ message: "speaker stream is not open",
1685
+ });
1686
+ return;
1687
+ }
1688
+ try {
1689
+ // Resolves once the buffered audio has drained; onEnded delivers the
1690
+ // terminal SPEAKER_STATE ("stopped").
1691
+ const result = await audioPlaybackService.closeStream(streamId);
1692
+ this.sendResult(packageName, requestId, true, result);
1693
+ }
1694
+ catch (error) {
1695
+ this.sendResult(packageName, requestId, false, undefined, {
1696
+ code: MiniappErrorCode.INTERNAL,
1697
+ message: error instanceof Error ? error.message : String(error),
1698
+ });
1699
+ }
1700
+ }
1701
+ async handleSpeakerStreamAbort(packageName, payload, requestId) {
1702
+ const streamId = this.activeSpeakerStream(packageName, payload);
1703
+ if (!streamId) {
1704
+ // Aborting an already-gone stream is fine — abort() is idempotent.
1705
+ this.sendResult(packageName, requestId, true);
1706
+ return;
1707
+ }
1708
+ try {
1709
+ await audioPlaybackService.abortStream(streamId);
1710
+ this.sendResult(packageName, requestId, true);
1711
+ }
1712
+ catch (error) {
1713
+ this.sendResult(packageName, requestId, false, undefined, {
1714
+ code: MiniappErrorCode.INTERNAL,
1715
+ message: error instanceof Error ? error.message : String(error),
1716
+ });
1717
+ }
1718
+ }
1719
+ async handleSpeak(packageName, payload, requestId) {
1720
+ const rawText = payload.text;
1721
+ const sentences = typeof rawText === "string"
1722
+ ? [rawText.trim()].filter(Boolean)
1723
+ : Array.isArray(rawText)
1724
+ ? rawText.filter((sentence) => typeof sentence === "string").map((sentence) => sentence.trim()).filter(Boolean)
1725
+ : [];
1726
+ if (sentences.length === 0) {
1727
+ this.sendResult(packageName, requestId, false, undefined, {
1728
+ code: MiniappErrorCode.INTERNAL,
1729
+ message: "speak requires text or a non-empty sentence list",
1730
+ });
1731
+ return;
1732
+ }
1733
+ const cloudText = sentences.join(" ");
1734
+ const voice = (payload.voice_id ?? payload.voice) || "default";
1735
+ const audioRequestId = requestId || `tts_${Date.now()}`;
1736
+ const volume = typeof payload.volume === "number" ? payload.volume : 1.0;
1737
+ const stopOtherAudio = payload.stopOtherAudio !== false;
1738
+ const voiceSettings = payload.voice_settings && typeof payload.voice_settings === "object"
1739
+ ? payload.voice_settings
1740
+ : undefined;
1741
+ const speed = typeof voiceSettings?.speed === "number" ? voiceSettings.speed : undefined;
1742
+ const run = { id: audioRequestId, cancelled: false };
1743
+ this.cancelSpeech(packageName);
1744
+ this.speechRuns.set(packageName, run);
1745
+ this.setSpeakerState(packageName, "loading");
1746
+ let terminalSent = false;
1747
+ const sendPlaybackResult = (success, error, duration, fallbackErrorMessage, completed = success, errorCode = MiniappErrorCode.TTS_UPSTREAM_ERROR) => {
1748
+ if (terminalSent)
1749
+ return;
1750
+ terminalSent = true;
1751
+ // A superseded run still owns its bridge request, but it no longer owns
1752
+ // the package's shared speaker state.
1753
+ const ownsSpeakerState = this.speechRuns.get(packageName) === run;
1754
+ if (ownsSpeakerState)
1755
+ this.speechRuns.delete(packageName);
1756
+ if (ownsSpeakerState) {
1757
+ if (success) {
1758
+ this.setSpeakerState(packageName, "stopped", { durationMs: duration ?? undefined });
1759
+ }
1760
+ else {
1761
+ this.setSpeakerState(packageName, "error", {
1762
+ errorCode,
1763
+ errorMessage: error ?? fallbackErrorMessage,
1764
+ durationMs: duration ?? undefined,
1765
+ });
1766
+ }
1767
+ }
1768
+ this.sendResult(packageName, requestId, success, { completed, duration }, error
1769
+ ? {
1770
+ code: errorCode,
1771
+ message: error,
1772
+ }
1773
+ : undefined);
1774
+ };
1775
+ run.onCancelled = () => sendPlaybackResult(true, null, null, "tts playback stopped", false);
1776
+ try {
1777
+ const voiceExplicit = payload.voice_id !== undefined || payload.voice !== undefined;
1778
+ const offlineSupportsVoice = !voiceExplicit || voice === "default" || ttsModelManager.getAvailableLanguages().some((l) => l.code === voice);
1779
+ const modelId = typeof payload.model_id === "string" ? payload.model_id : undefined;
1780
+ const forceLocal = payload.forceLocal === true;
1781
+ const cloudConnected = cloudClientService.isConnected();
1782
+ console.log(`${LOG_TAG}: TTS decision for ${packageName}: cloudConnected=${cloudConnected}, runtimeTts=yes, offlineVoice=${offlineSupportsVoice}, forceLocal=${forceLocal}`);
1783
+ const playOfflineTts = async (reason) => {
1784
+ if (run.cancelled)
1785
+ return true;
1786
+ if (!offlineSupportsVoice || !(await ttsModelManager.isModelAvailable())) {
1787
+ return false;
1788
+ }
1789
+ if (run.cancelled)
1790
+ return true;
1791
+ const languageCode = ttsModelManager.getAvailableLanguages().some((l) => l.code === voice) ? voice : undefined;
1792
+ const offlineSentences = sentences;
1793
+ if (offlineSentences.length > 1) {
1794
+ let firstSentence;
1795
+ try {
1796
+ firstSentence = await ttsModelManager.synthesizeToFile(offlineSentences[0], { languageCode, speed });
1797
+ }
1798
+ catch (offlineErr) {
1799
+ if (run.cancelled)
1800
+ return true;
1801
+ console.warn(`${LOG_TAG}: offline sentence TTS synthesize failed${reason ? ` after ${reason}` : ""}:`, offlineErr);
1802
+ return false;
1803
+ }
1804
+ if (run.cancelled) {
1805
+ await Promise.resolve(firstSentence.cleanup?.());
1806
+ return true;
1807
+ }
1808
+ void runSentenceTtsPipeline({
1809
+ sentences: offlineSentences,
1810
+ firstAudio: firstSentence,
1811
+ run,
1812
+ synthesize: (sentence) => ttsModelManager.synthesizeToFile(sentence, { languageCode, speed }),
1813
+ play: (audio, index) => new Promise((resolve) => {
1814
+ const sentenceRequestId = `${audioRequestId}_sentence_${index}`;
1815
+ run.playbackRequestId = sentenceRequestId;
1816
+ audioPlaybackService.play({
1817
+ requestId: sentenceRequestId,
1818
+ audioUrl: audio.audioUrl,
1819
+ appId: packageName,
1820
+ volume,
1821
+ stopOtherAudio,
1822
+ suppressCloudUplink: true,
1823
+ }, (_responseId, success, error, duration, completionReason) => {
1824
+ if (run.playbackRequestId === sentenceRequestId)
1825
+ run.playbackRequestId = undefined;
1826
+ resolve({ success, completed: completionReason === "completed", error, duration });
1827
+ });
1828
+ }),
1829
+ onCleanupError: (cleanupError) => {
1830
+ console.warn(`${LOG_TAG}: sentence TTS cleanup failed`, cleanupError);
1831
+ },
1832
+ }).then(({ success, completed, error, duration }) => {
1833
+ sendPlaybackResult(success, error, duration, "offline sentence tts playback failed", completed);
1834
+ }, (pipelineError) => {
1835
+ const message = pipelineError instanceof Error ? pipelineError.message : String(pipelineError);
1836
+ sendPlaybackResult(false, message, null, "offline sentence tts playback failed", false);
1837
+ });
1838
+ queueMicrotask(() => {
1839
+ if (this.speechRuns.get(packageName) === run)
1840
+ this.setSpeakerState(packageName, "playing");
1841
+ });
1842
+ return true;
1843
+ }
1844
+ let offlineGenerated;
1845
+ try {
1846
+ offlineGenerated = await ttsModelManager.synthesizeToFile(offlineSentences[0], { languageCode, speed });
1847
+ }
1848
+ catch (offlineErr) {
1849
+ if (run.cancelled)
1850
+ return true;
1851
+ console.warn(`${LOG_TAG}: offline TTS synthesize failed${reason ? ` after ${reason}` : ""}:`, offlineErr);
1852
+ return false;
1853
+ }
1854
+ const generated = offlineGenerated;
1855
+ if (run.cancelled) {
1856
+ await Promise.resolve(generated.cleanup?.());
1857
+ return true;
1858
+ }
1859
+ run.playbackRequestId = audioRequestId;
1860
+ audioPlaybackService.play({
1861
+ requestId: audioRequestId,
1862
+ audioUrl: generated.audioUrl,
1863
+ appId: packageName,
1864
+ volume,
1865
+ stopOtherAudio,
1866
+ suppressCloudUplink: true,
1867
+ }, (_respId, success, error, duration, completionReason) => {
1868
+ if (run.playbackRequestId === audioRequestId)
1869
+ run.playbackRequestId = undefined;
1870
+ void Promise.resolve(generated.cleanup?.()).catch((cleanupError) => {
1871
+ console.warn(`${LOG_TAG}: offline TTS cleanup failed`, cleanupError);
1872
+ });
1873
+ sendPlaybackResult(success, error, duration, "offline tts playback failed", completionReason === "completed");
1874
+ });
1875
+ queueMicrotask(() => {
1876
+ if (this.speechRuns.get(packageName) === run)
1877
+ this.setSpeakerState(packageName, "playing");
1878
+ });
1879
+ return true;
1880
+ };
1881
+ const playCloudTts = async (fallbackToOffline) => {
1882
+ if (run.cancelled)
1883
+ return true;
1884
+ let source;
1885
+ try {
1886
+ source = await cloudClientService.tts.speak(cloudText, {
1887
+ ...(voiceExplicit && voice !== "default" ? { voice_id: voice } : {}),
1888
+ ...(modelId ? { model_id: modelId } : {}),
1889
+ ...(voiceSettings ? { voice_settings: voiceSettings } : {}),
1890
+ });
1891
+ }
1892
+ catch (cloudErr) {
1893
+ if (run.cancelled)
1894
+ return true;
1895
+ const error = cloudErr instanceof Error ? cloudErr.message : String(cloudErr);
1896
+ console.warn(`${LOG_TAG}: cloud TTS source failed: ${error}`);
1897
+ if (fallbackToOffline && (await playOfflineTts("cloud tts source failed"))) {
1898
+ return true;
1899
+ }
1900
+ sendPlaybackResult(false, error, null, "tts failed");
1901
+ return true;
1902
+ }
1903
+ if (run.cancelled)
1904
+ return true;
1905
+ run.playbackRequestId = audioRequestId;
1906
+ await Promise.resolve(audioPlaybackService.play({
1907
+ requestId: audioRequestId,
1908
+ audioUrl: source.audioUrl,
1909
+ appId: packageName,
1910
+ volume,
1911
+ stopOtherAudio,
1912
+ suppressCloudUplink: true,
1913
+ }, (_respId, success, error, duration, completionReason) => {
1914
+ if (run.playbackRequestId === audioRequestId)
1915
+ run.playbackRequestId = undefined;
1916
+ if (!success && fallbackToOffline) {
1917
+ void playOfflineTts("cloud tts playback failed").then((started) => {
1918
+ if (!started) {
1919
+ sendPlaybackResult(false, error, duration, "tts failed");
1920
+ }
1921
+ });
1922
+ return;
1923
+ }
1924
+ sendPlaybackResult(success, error, duration, "tts failed", completionReason === "completed");
1925
+ }));
1926
+ queueMicrotask(() => {
1927
+ if (this.speechRuns.get(packageName) === run)
1928
+ this.setSpeakerState(packageName, "playing");
1929
+ });
1930
+ return true;
1931
+ };
1932
+ if (forceLocal) {
1933
+ if (await playOfflineTts()) {
1934
+ return;
1935
+ }
1936
+ if (run.cancelled)
1937
+ return;
1938
+ const message = "tts unavailable: forceLocal requested but offline TTS model is not available";
1939
+ sendPlaybackResult(false, message, null, message, false, MiniappErrorCode.TTS_LOCAL_UNAVAILABLE);
1940
+ return;
1941
+ }
1942
+ if (cloudConnected && (await playCloudTts(true))) {
1943
+ return;
1944
+ }
1945
+ // Offline is the disconnected-cloud fallback. If the model is not
1946
+ // available, preserve the old behavior by trying cloud as a last resort.
1947
+ if (await playOfflineTts(cloudConnected ? undefined : "cloud disconnected")) {
1948
+ return;
1949
+ }
1950
+ if (await playCloudTts(false)) {
1951
+ return;
1952
+ }
1953
+ const message = "tts unavailable: no cloud TTS URL or offline TTS model";
1954
+ sendPlaybackResult(false, message, null, message, false);
1955
+ }
1956
+ catch (err) {
1957
+ console.error(`${LOG_TAG}: speak error:`, err);
1958
+ const message = err instanceof Error ? err.message : "TTS error";
1959
+ sendPlaybackResult(false, message, null, "TTS error", false);
1960
+ }
1961
+ }
1962
+ cancelSpeech(packageName) {
1963
+ const run = this.speechRuns.get(packageName);
1964
+ if (!run)
1965
+ return;
1966
+ run.cancelled = true;
1967
+ this.speechRuns.delete(packageName);
1968
+ if (run.playbackRequestId)
1969
+ audioPlaybackService.cancelPlayback(run.playbackRequestId);
1970
+ run.onCancelled?.();
1971
+ }
1972
+ async handleRgbLed(packageName, payload, requestId) {
1973
+ const coerceNumber = (value, fallback) => {
1974
+ const coerced = Number(value);
1975
+ return Number.isFinite(coerced) ? coerced : fallback;
1976
+ };
1977
+ const ledRequestId = requestId || `led_${Date.now()}`;
1978
+ const action = normalizeRgbLedAction(payload.action);
1979
+ const color = normalizeRgbLedColor(payload.color);
1980
+ try {
1981
+ const result = await BluetoothSdk.rgbLedControl(ledRequestId, packageName, action, color, coerceNumber(payload.ontime, 1000), coerceNumber(payload.offtime, 0), coerceNumber(payload.count, 1));
1982
+ this.sendResult(packageName, requestId, true, result);
1983
+ }
1984
+ catch (err) {
1985
+ this.sendResult(packageName, requestId, false, undefined, {
1986
+ code: err.code || MiniappErrorCode.INTERNAL,
1987
+ message: err instanceof Error ? err.message : "RGB LED command failed",
1988
+ });
1989
+ }
1990
+ }
1991
+ async handleLocationPoll(packageName, requestId) {
1992
+ try {
1993
+ const { status } = await Location.requestForegroundPermissionsAsync();
1994
+ if (status !== "granted") {
1995
+ this.sendResult(packageName, requestId, false, undefined, {
1996
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
1997
+ message: "Location permission not granted",
1998
+ });
1999
+ return;
2000
+ }
2001
+ // Try the OS cache first — usually instant — and only fall back to
2002
+ // a fresh fix if the cache is empty. Use Low accuracy on the fresh
2003
+ // path so we get a cell/wifi-tower fix in ~1s instead of waiting
2004
+ // for full GPS warm-up.
2005
+ const cached = await Location.getLastKnownPositionAsync({ maxAge: 60_000 });
2006
+ const location = cached ?? (await Location.getCurrentPositionAsync({ accuracy: Location.Accuracy.Low }));
2007
+ this.sendResult(packageName, requestId, true, {
2008
+ lat: location.coords.latitude,
2009
+ lng: location.coords.longitude,
2010
+ accuracy: location.coords.accuracy ?? undefined,
2011
+ timestamp: location.timestamp,
2012
+ });
2013
+ }
2014
+ catch (err) {
2015
+ console.error(`${LOG_TAG}: location poll error:`, err);
2016
+ this.sendResult(packageName, requestId, false, undefined, {
2017
+ code: MiniappErrorCode.INTERNAL,
2018
+ message: err instanceof Error ? err.message : "Location error",
2019
+ });
2020
+ }
2021
+ }
2022
+ // ---------------------------------------------------------------------------
2023
+ // Navigation
2024
+ // ---------------------------------------------------------------------------
2025
+ /**
2026
+ * All navigation handlers + their state live in NavigationHandlers. The
2027
+ * dispatcher's NAVIGATION_* cases delegate here; the rest of the runtime
2028
+ * cross-cuts navigation through {@link NavigationHandlers.onDisconnect}
2029
+ * (mini-app disconnect) and {@link NavigationHandlers.isTripActive}
2030
+ * (location-stream gating during a trip).
2031
+ */
2032
+ navigationHandlers = new NavigationHandlers((packageName, envelope) => this.sendToMiniapp(packageName, envelope), (packageName, requestId, ok, result, error) => this.sendResult(packageName, requestId, ok, result, error));
2033
+ /**
2034
+ * `session.blob` — persistent, per-app binary storage. Like NavigationHandlers,
2035
+ * the BLOB_* dispatcher cases delegate here. It's a generic byte store; audio
2036
+ * recording lives entirely in the miniapp (mic.onAudioChunk → chunked
2037
+ * BLOB_WRITE), so nothing audio-specific runs in the host.
2038
+ */
2039
+ blobStore = new BlobStore({
2040
+ sendResult: (packageName, requestId, ok, result, error) => this.sendResult(packageName, requestId, ok, result, error),
2041
+ getUserId: () => useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) || "anonymous",
2042
+ });
2043
+ /**
2044
+ * Heading is a sensor stream — start the native compass when any mini
2045
+ * app is subscribed to "heading_update", stop it when none are.
2046
+ */
2047
+ headingUnsub = null;
2048
+ recomputeHeadingSubscription() {
2049
+ const wantsHeading = this.streamSubscribers.has(MiniappStreamType.HEADING_UPDATE);
2050
+ if (wantsHeading && !this.headingUnsub) {
2051
+ // Heading sensor lives in engine (HeadingService) — subscribe directly.
2052
+ this.headingUnsub = headingService.addListener((degrees) => {
2053
+ this.forwardEvent(MiniappStreamType.HEADING_UPDATE, { degrees });
2054
+ });
2055
+ }
2056
+ else if (!wantsHeading && this.headingUnsub) {
2057
+ this.headingUnsub();
2058
+ this.headingUnsub = null;
2059
+ }
2060
+ }
2061
+ /**
2062
+ * Accelerometer is a sensor stream — enable the glasses IMU when any mini
2063
+ * app is subscribed to "accel_data", disable it when none are. Mirrors the
2064
+ * heading pattern. Only G2 streams IMU today; on other devices the native
2065
+ * call is a no-op so this is harmless.
2066
+ */
2067
+ imuEnabled = false;
2068
+ recomputeImuSubscription() {
2069
+ const wantsImu = this.streamSubscribers.has(MiniappStreamType.ACCEL_DATA);
2070
+ if (wantsImu === this.imuEnabled)
2071
+ return;
2072
+ this.imuEnabled = wantsImu;
2073
+ void BluetoothSdk.setImuEnabled(wantsImu);
2074
+ }
2075
+ /**
2076
+ * Last rate we asked the host for. Lets us skip the cross-process call
2077
+ * when nothing actually changed (most SUBSCRIBE / unregister churn
2078
+ * leaves the aggregate unchanged) and tracks state for the no-active-
2079
+ * subscribers → tell host to back off case.
2080
+ */
2081
+ lastAppliedLocationRate = null;
2082
+ /**
2083
+ * Recompute the aggregate location tier across every connected
2084
+ * miniapp and push the result to the host. The aggregate is the
2085
+ * STRICTEST rate any single app has requested — see
2086
+ * {@link LOCATION_RATE_PRIORITY} — because the GPS sample rate is a
2087
+ * shared OS-level setting that we can't bias per-app.
2088
+ *
2089
+ * Called after every SUBSCRIBE and every unregister so that when a
2090
+ * realtime-requesting app goes away the tier falls back down. If no
2091
+ * connected app is asking for a location rate, we tell the host
2092
+ * "off" so it can drop GPS power.
2093
+ */
2094
+ recomputeLocationTier() {
2095
+ let bestRank = -1;
2096
+ for (const app of this.connectedApps.values()) {
2097
+ const r = app.requestedLocationRate;
2098
+ if (!r)
2099
+ continue;
2100
+ const rank = locationRateRank(r);
2101
+ if (rank > bestRank) {
2102
+ bestRank = rank;
2103
+ }
2104
+ }
2105
+ // When no app is asking, downgrade to "off". Without this we'd
2106
+ // leak the last-known-strictest rate forever (the original bug).
2107
+ const next = bestRank >= 0 ? LOCATION_RATE_PRIORITY[bestRank] : "off";
2108
+ if (next === this.lastAppliedLocationRate)
2109
+ return;
2110
+ this.lastAppliedLocationRate = next;
2111
+ console.log(`[LOCATION] aggregate tier → ${next}`);
2112
+ void phoneLocationService.setLocationTier(next);
2113
+ }
2114
+ // ---------------------------------------------------------------------------
2115
+ // Storage helpers
2116
+ // ---------------------------------------------------------------------------
2117
+ getStorageKeyPrefix(packageName) {
2118
+ const userId = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) || "anonymous";
2119
+ return `mentraos_localstorage_${userId}_${packageName}_`;
2120
+ }
2121
+ async handleStorageGet(packageName, payload, requestId) {
2122
+ try {
2123
+ const key = payload.key;
2124
+ if (!key) {
2125
+ this.sendResult(packageName, requestId, false, undefined, {
2126
+ code: MiniappErrorCode.INTERNAL,
2127
+ message: "storage_get requires a key",
2128
+ });
2129
+ return;
2130
+ }
2131
+ const fullKey = this.getStorageKeyPrefix(packageName) + key;
2132
+ const result = mmkvStorage.load(fullKey);
2133
+ this.sendResult(packageName, requestId, true, { key, value: result.is_ok() ? result.value : null });
2134
+ }
2135
+ catch (err) {
2136
+ console.error(`${LOG_TAG}: storage_get error:`, err);
2137
+ this.sendResult(packageName, requestId, false, undefined, {
2138
+ code: MiniappErrorCode.INTERNAL,
2139
+ message: err instanceof Error ? err.message : "Storage error",
2140
+ });
2141
+ }
2142
+ }
2143
+ async handleStorageSet(packageName, payload, requestId) {
2144
+ try {
2145
+ const key = payload.key;
2146
+ if (!key) {
2147
+ this.sendResult(packageName, requestId, false, undefined, {
2148
+ code: MiniappErrorCode.INTERNAL,
2149
+ message: "storage_set requires a key",
2150
+ });
2151
+ return;
2152
+ }
2153
+ const fullKey = this.getStorageKeyPrefix(packageName) + key;
2154
+ mmkvStorage.save(fullKey, payload.value ?? null);
2155
+ this.sendResult(packageName, requestId, true);
2156
+ }
2157
+ catch (err) {
2158
+ console.error(`${LOG_TAG}: storage_set error:`, err);
2159
+ this.sendResult(packageName, requestId, false, undefined, {
2160
+ code: MiniappErrorCode.INTERNAL,
2161
+ message: err instanceof Error ? err.message : "Storage error",
2162
+ });
2163
+ }
2164
+ }
2165
+ async handleStorageDelete(packageName, payload, requestId) {
2166
+ try {
2167
+ const key = payload.key;
2168
+ if (!key) {
2169
+ this.sendResult(packageName, requestId, false, undefined, {
2170
+ code: MiniappErrorCode.INTERNAL,
2171
+ message: "storage_delete requires a key",
2172
+ });
2173
+ return;
2174
+ }
2175
+ const fullKey = this.getStorageKeyPrefix(packageName) + key;
2176
+ mmkvStorage.remove(fullKey);
2177
+ this.sendResult(packageName, requestId, true);
2178
+ }
2179
+ catch (err) {
2180
+ console.error(`${LOG_TAG}: storage_delete error:`, err);
2181
+ this.sendResult(packageName, requestId, false, undefined, {
2182
+ code: MiniappErrorCode.INTERNAL,
2183
+ message: err instanceof Error ? err.message : "Storage error",
2184
+ });
2185
+ }
2186
+ }
2187
+ async handleStorageList(packageName, _payload, requestId) {
2188
+ try {
2189
+ const prefix = this.getStorageKeyPrefix(packageName);
2190
+ const allKeys = mmkvStorage.getAllKeys();
2191
+ const keys = allKeys.filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length));
2192
+ this.sendResult(packageName, requestId, true, { keys });
2193
+ }
2194
+ catch (err) {
2195
+ console.error(`${LOG_TAG}: storage_list error:`, err);
2196
+ this.sendResult(packageName, requestId, false, undefined, {
2197
+ code: MiniappErrorCode.INTERNAL,
2198
+ message: err instanceof Error ? err.message : "Storage error",
2199
+ });
2200
+ }
2201
+ }
2202
+ async handleStorageClear(packageName, requestId) {
2203
+ try {
2204
+ const prefix = this.getStorageKeyPrefix(packageName);
2205
+ const allKeys = mmkvStorage.getAllKeys();
2206
+ for (const k of allKeys) {
2207
+ if (k.startsWith(prefix))
2208
+ mmkvStorage.remove(k);
2209
+ }
2210
+ this.sendResult(packageName, requestId, true);
2211
+ }
2212
+ catch (err) {
2213
+ console.error(`${LOG_TAG}: storage_clear error:`, err);
2214
+ this.sendResult(packageName, requestId, false, undefined, {
2215
+ code: MiniappErrorCode.INTERNAL,
2216
+ message: err instanceof Error ? err.message : "Storage error",
2217
+ });
2218
+ }
2219
+ }
2220
+ async handleStorageHas(packageName, payload, requestId) {
2221
+ try {
2222
+ const key = payload.key;
2223
+ if (!key) {
2224
+ this.sendResult(packageName, requestId, false, undefined, {
2225
+ code: MiniappErrorCode.INTERNAL,
2226
+ message: "storage_has requires a key",
2227
+ });
2228
+ return;
2229
+ }
2230
+ const fullKey = this.getStorageKeyPrefix(packageName) + key;
2231
+ const result = mmkvStorage.load(fullKey);
2232
+ this.sendResult(packageName, requestId, true, { has: result.is_ok() });
2233
+ }
2234
+ catch (err) {
2235
+ console.error(`${LOG_TAG}: storage_has error:`, err);
2236
+ this.sendResult(packageName, requestId, false, undefined, {
2237
+ code: MiniappErrorCode.INTERNAL,
2238
+ message: err instanceof Error ? err.message : "Storage error",
2239
+ });
2240
+ }
2241
+ }
2242
+ async handleStorageGetAll(packageName, requestId) {
2243
+ try {
2244
+ const prefix = this.getStorageKeyPrefix(packageName);
2245
+ const allKeys = mmkvStorage.getAllKeys();
2246
+ const values = {};
2247
+ for (const k of allKeys) {
2248
+ if (!k.startsWith(prefix))
2249
+ continue;
2250
+ const r = mmkvStorage.load(k);
2251
+ if (r.is_ok()) {
2252
+ const v = r.value;
2253
+ values[k.slice(prefix.length)] = typeof v === "string" ? v : String(v ?? "");
2254
+ }
2255
+ }
2256
+ this.sendResult(packageName, requestId, true, { values });
2257
+ }
2258
+ catch (err) {
2259
+ console.error(`${LOG_TAG}: storage_get_all error:`, err);
2260
+ this.sendResult(packageName, requestId, false, undefined, {
2261
+ code: MiniappErrorCode.INTERNAL,
2262
+ message: err instanceof Error ? err.message : "Storage error",
2263
+ });
2264
+ }
2265
+ }
2266
+ async handleStorageSetMultiple(packageName, payload, requestId) {
2267
+ try {
2268
+ const values = payload.values;
2269
+ if (!values || typeof values !== "object") {
2270
+ this.sendResult(packageName, requestId, false, undefined, {
2271
+ code: MiniappErrorCode.INTERNAL,
2272
+ message: "storage_set_multiple requires a values object",
2273
+ });
2274
+ return;
2275
+ }
2276
+ const prefix = this.getStorageKeyPrefix(packageName);
2277
+ for (const [key, value] of Object.entries(values)) {
2278
+ mmkvStorage.save(prefix + key, value ?? null);
2279
+ }
2280
+ this.sendResult(packageName, requestId, true);
2281
+ }
2282
+ catch (err) {
2283
+ console.error(`${LOG_TAG}: storage_set_multiple error:`, err);
2284
+ this.sendResult(packageName, requestId, false, undefined, {
2285
+ code: MiniappErrorCode.INTERNAL,
2286
+ message: err instanceof Error ? err.message : "Storage error",
2287
+ });
2288
+ }
2289
+ }
2290
+ // ---------------------------------------------------------------------------
2291
+ // Camera FOV
2292
+ // ---------------------------------------------------------------------------
2293
+ async handleCameraFov(packageName, payload, requestId) {
2294
+ const app = this.connectedApps.get(packageName);
2295
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
2296
+ if (!hasCameraPermission) {
2297
+ logPermissionNotDeclared(packageName, "CAMERA", "to set camera FOV", `{"type": "CAMERA"}`);
2298
+ this.sendResult(packageName, requestId, false, undefined, {
2299
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2300
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2301
+ permission: "CAMERA",
2302
+ operation: MiniappRequestType.CAMERA_FOV,
2303
+ });
2304
+ return;
2305
+ }
2306
+ try {
2307
+ const request = normalizeCameraFovPayload(payload);
2308
+ const description = "preset" in request ? `preset=${request.preset}` : `fov=${request.fov} roi=${request.roiPosition ?? "center"}`;
2309
+ console.log(`${LOG_TAG}: camera_fov_set ${description}`);
2310
+ const result = await phoneCameraFovCoordinator.setOverride(packageName, request);
2311
+ this.sendResult(packageName, requestId, true, result);
2312
+ }
2313
+ catch (err) {
2314
+ console.error(`${LOG_TAG}: camera_fov error:`, err);
2315
+ this.sendResult(packageName, requestId, false, undefined, {
2316
+ code: err.code || MiniappErrorCode.INTERNAL,
2317
+ message: err instanceof Error ? err.message : "Camera FOV error",
2318
+ });
2319
+ }
2320
+ }
2321
+ /**
2322
+ * Explicit IMU enable/disable from a miniapp (session.imu.setEnabled).
2323
+ *
2324
+ * The accel stream already auto-toggles the native IMU on subscribe (see
2325
+ * {@link recomputeImuSubscription}); this is a direct override. We sync
2326
+ * `this.imuEnabled` to the requested state so a subsequent subscription
2327
+ * recompute with the same aggregate doesn't issue a redundant native call.
2328
+ * Note that a later subscribe/unsubscribe that changes the aggregate will
2329
+ * still re-derive the sensor state from subscriptions.
2330
+ */
2331
+ handleImuSetEnabled(packageName, payload, requestId) {
2332
+ try {
2333
+ const enabled = !!payload.enabled;
2334
+ console.log(`${LOG_TAG}: imu_set_enabled ${enabled} (by ${packageName})`);
2335
+ this.imuEnabled = enabled;
2336
+ void BluetoothSdk.setImuEnabled(enabled);
2337
+ this.sendResult(packageName, requestId, true);
2338
+ }
2339
+ catch (err) {
2340
+ console.error(`${LOG_TAG}: imu_set_enabled error:`, err);
2341
+ this.sendResult(packageName, requestId, false, undefined, {
2342
+ code: MiniappErrorCode.INTERNAL,
2343
+ message: err instanceof Error ? err.message : "IMU set-enabled error",
2344
+ });
2345
+ }
2346
+ }
2347
+ // ---------------------------------------------------------------------------
2348
+ // System utilities (share, open URL, clipboard, download)
2349
+ // ---------------------------------------------------------------------------
2350
+ async handleShare(packageName, payload, requestId) {
2351
+ const { text, title, base64, mimeType, filename, url } = payload;
2352
+ try {
2353
+ if (base64) {
2354
+ // File share via base64 — write to temp file then share
2355
+ const tempFile = new File(Paths.cache, filename || "shared_file");
2356
+ tempFile.write(base64, { encoding: "base64" });
2357
+ await Share.open({
2358
+ url: tempFile.uri,
2359
+ type: mimeType || "application/octet-stream",
2360
+ filename: filename,
2361
+ title: title,
2362
+ });
2363
+ }
2364
+ else if (url) {
2365
+ await Share.open({ url, title, message: text });
2366
+ }
2367
+ else {
2368
+ await Share.open({ message: text || "", title });
2369
+ }
2370
+ this.sendResult(packageName, requestId, true, { success: true });
2371
+ }
2372
+ catch (error) {
2373
+ // react-native-share throws when user dismisses the share sheet
2374
+ if (error?.message?.includes("User did not share")) {
2375
+ this.sendResult(packageName, requestId, true, { success: false, cancelled: true });
2376
+ }
2377
+ else {
2378
+ console.error(`${LOG_TAG}: share error:`, error);
2379
+ this.sendResult(packageName, requestId, true, { success: false });
2380
+ }
2381
+ }
2382
+ }
2383
+ async handleOpenUrl(packageName, payload) {
2384
+ const url = payload.url;
2385
+ if (!url || typeof url !== "string") {
2386
+ console.warn(`${LOG_TAG}: open_url missing url`);
2387
+ return;
2388
+ }
2389
+ // Block dangerous schemes
2390
+ if (url.startsWith("javascript:") || url.startsWith("file:")) {
2391
+ console.warn(`${LOG_TAG}: open_url blocked dangerous scheme: ${url}`);
2392
+ return;
2393
+ }
2394
+ try {
2395
+ await Linking.openURL(url);
2396
+ }
2397
+ catch (error) {
2398
+ console.error(`${LOG_TAG}: open_url error:`, error);
2399
+ }
2400
+ }
2401
+ async handleCopyClipboard(packageName, payload, requestId) {
2402
+ const text = payload.text;
2403
+ if (typeof text !== "string") {
2404
+ console.warn(`${LOG_TAG}: copy_clipboard missing text`);
2405
+ return;
2406
+ }
2407
+ try {
2408
+ await Clipboard.setStringAsync(text);
2409
+ this.sendResult(packageName, requestId, true);
2410
+ }
2411
+ catch (error) {
2412
+ console.error(`${LOG_TAG}: clipboard error:`, error);
2413
+ this.sendResult(packageName, requestId, false, undefined, {
2414
+ code: MiniappErrorCode.INTERNAL,
2415
+ message: error?.message || "Clipboard error",
2416
+ });
2417
+ }
2418
+ }
2419
+ async handleDownload(packageName, payload, requestId) {
2420
+ const { base64, url, mimeType, filename } = payload;
2421
+ const name = filename || "download";
2422
+ try {
2423
+ let file;
2424
+ if (base64) {
2425
+ file = new File(Paths.cache, name);
2426
+ file.write(base64, { encoding: "base64" });
2427
+ }
2428
+ else if (url) {
2429
+ file = await File.downloadFileAsync(url, new File(Paths.cache, name), { idempotent: true });
2430
+ }
2431
+ else {
2432
+ console.warn(`${LOG_TAG}: download missing base64 or url`);
2433
+ return;
2434
+ }
2435
+ // Open share sheet so user can choose where to save
2436
+ await Share.open({
2437
+ url: file.uri,
2438
+ type: mimeType || "application/octet-stream",
2439
+ filename: name,
2440
+ });
2441
+ this.sendResult(packageName, requestId, true, { success: true });
2442
+ }
2443
+ catch (error) {
2444
+ if (error?.message?.includes("User did not share")) {
2445
+ this.sendResult(packageName, requestId, true, { success: true, cancelled: true });
2446
+ }
2447
+ else {
2448
+ console.error(`${LOG_TAG}: download error:`, error);
2449
+ this.sendResult(packageName, requestId, true, { success: false });
2450
+ }
2451
+ }
2452
+ }
2453
+ // ===========================================================================
2454
+ // Photo + streaming handlers (cloud-coordinated)
2455
+ // ===========================================================================
2456
+ async handlePhoto(packageName, payload, requestId) {
2457
+ // Manifest CAMERA permission gate.
2458
+ const app = this.connectedApps.get(packageName);
2459
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
2460
+ if (!hasCameraPermission) {
2461
+ logPermissionNotDeclared(packageName, "CAMERA", "to take a photo", `{"type": "CAMERA"}`);
2462
+ this.sendResult(packageName, requestId, false, undefined, {
2463
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2464
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2465
+ permission: "CAMERA",
2466
+ operation: MiniappRequestType.PHOTO,
2467
+ });
2468
+ return;
2469
+ }
2470
+ try {
2471
+ const result = await phonePhotoCoordinator.takePhoto(packageName, {
2472
+ size: payload.size,
2473
+ mode: payload.mode,
2474
+ transferMethod: payload.transferMethod,
2475
+ compress: payload.compress,
2476
+ sound: payload.sound,
2477
+ saveToGallery: payload.saveToGallery,
2478
+ exposureTimeNs: payload.exposureTimeNs,
2479
+ iso: payload.iso,
2480
+ aeExposureDivisor: payload.aeExposureDivisor,
2481
+ isoCap: payload.isoCap,
2482
+ noiseReduction: payload.noiseReduction,
2483
+ edgeEnhancement: payload.edgeEnhancement,
2484
+ mfnr: payload.mfnr,
2485
+ zsl: payload.zsl,
2486
+ ispDigitalGain: payload.ispDigitalGain,
2487
+ ispAnalogGain: payload.ispAnalogGain,
2488
+ });
2489
+ this.sendResult(packageName, requestId, true, result);
2490
+ }
2491
+ catch (err) {
2492
+ this.sendResult(packageName, requestId, false, undefined, {
2493
+ code: err.code || MiniappErrorCode.INTERNAL,
2494
+ message: err instanceof Error ? err.message : "Photo request failed",
2495
+ stage: err.stage,
2496
+ transport: err.transport,
2497
+ });
2498
+ }
2499
+ }
2500
+ async handleCameraWarmUp(packageName, payload, requestId) {
2501
+ // Manifest CAMERA permission gate (same as photo).
2502
+ const app = this.connectedApps.get(packageName);
2503
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
2504
+ if (!hasCameraPermission) {
2505
+ logPermissionNotDeclared(packageName, "CAMERA", "to warm up the camera", `{"type": "CAMERA"}`);
2506
+ this.sendResult(packageName, requestId, false, undefined, {
2507
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2508
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2509
+ permission: "CAMERA",
2510
+ operation: MiniappRequestType.CAMERA_WARM_UP,
2511
+ });
2512
+ return;
2513
+ }
2514
+ try {
2515
+ await phonePhotoCoordinator.warmUpCamera(packageName, {
2516
+ size: payload.size,
2517
+ mode: payload.mode,
2518
+ exposureTimeNs: payload.exposureTimeNs,
2519
+ durationMs: payload.durationMs,
2520
+ });
2521
+ this.sendResult(packageName, requestId, true);
2522
+ }
2523
+ catch (err) {
2524
+ this.sendResult(packageName, requestId, false, undefined, {
2525
+ code: err.code || MiniappErrorCode.INTERNAL,
2526
+ message: err instanceof Error ? err.message : "Camera warm-up failed",
2527
+ stage: err.stage,
2528
+ transport: err.transport,
2529
+ });
2530
+ }
2531
+ }
2532
+ async handleVideoRecordingStart(packageName, payload, requestId) {
2533
+ // Manifest CAMERA permission gate (same as photo).
2534
+ const app = this.connectedApps.get(packageName);
2535
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA");
2536
+ if (!hasCameraPermission) {
2537
+ logPermissionNotDeclared(packageName, "CAMERA", "to record video", `{"type": "CAMERA"}`);
2538
+ this.sendResult(packageName, requestId, false, undefined, {
2539
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2540
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2541
+ permission: "CAMERA",
2542
+ operation: MiniappRequestType.VIDEO_RECORDING_START,
2543
+ });
2544
+ return;
2545
+ }
2546
+ try {
2547
+ const result = await phoneVideoCoordinator.startRecording(packageName, {
2548
+ width: payload.width,
2549
+ height: payload.height,
2550
+ fps: payload.fps,
2551
+ sound: payload.sound,
2552
+ save: payload.save,
2553
+ });
2554
+ this.sendResult(packageName, requestId, true, result);
2555
+ }
2556
+ catch (err) {
2557
+ this.sendResult(packageName, requestId, false, undefined, {
2558
+ code: err.code || MiniappErrorCode.INTERNAL,
2559
+ message: err instanceof Error ? err.message : "Video recording start failed",
2560
+ });
2561
+ }
2562
+ }
2563
+ async handleVideoRecordingStop(packageName, payload, requestId) {
2564
+ try {
2565
+ await phoneVideoCoordinator.stopRecording(packageName, payload.recordingId, {
2566
+ uploadUrl: payload.uploadUrl,
2567
+ uploadAuthToken: payload.uploadAuthToken,
2568
+ });
2569
+ this.sendResult(packageName, requestId, true);
2570
+ }
2571
+ catch (err) {
2572
+ this.sendResult(packageName, requestId, false, undefined, {
2573
+ code: err.code || MiniappErrorCode.INTERNAL,
2574
+ message: err instanceof Error ? err.message : "Video recording stop failed",
2575
+ });
2576
+ }
2577
+ }
2578
+ /**
2579
+ * Stream handlers — dispatched to the engine PhoneStreamCoordinator. For managed
2580
+ * streams the coordinator additionally calls the v2 client REST route to
2581
+ * provision Cloudflare. Cloud-SDK apps (third-party developers) use a
2582
+ * separate cloud-side path that does not pass through here.
2583
+ */
2584
+ async handleStreamStart(packageName, payload, requestId) {
2585
+ const streaming = phoneStreamCoordinator;
2586
+ if (!streaming) {
2587
+ this.sendResult(packageName, requestId, false, undefined, {
2588
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
2589
+ message: "Streaming is not configured on this host",
2590
+ });
2591
+ return;
2592
+ }
2593
+ try {
2594
+ const result = await streaming.startUnmanaged(packageName, {
2595
+ streamUrl: payload.streamUrl,
2596
+ video: normalizeStreamVideoConfig(payload.video),
2597
+ audio: normalizeStreamAudioConfig(payload.audio),
2598
+ sound: payload.sound,
2599
+ });
2600
+ this.sendResult(packageName, requestId, true, result);
2601
+ }
2602
+ catch (err) {
2603
+ this.sendResult(packageName, requestId, false, undefined, {
2604
+ code: err.code || MiniappErrorCode.INTERNAL,
2605
+ message: err instanceof Error ? err.message : "Stream start failed",
2606
+ stage: err.stage,
2607
+ transport: err.transport,
2608
+ });
2609
+ }
2610
+ }
2611
+ async handleStreamStop(packageName, payload, requestId) {
2612
+ const streaming = phoneStreamCoordinator;
2613
+ if (!streaming) {
2614
+ // No adapter wired — treat as already-stopped.
2615
+ this.sendResult(packageName, requestId, true);
2616
+ return;
2617
+ }
2618
+ try {
2619
+ await streaming.stop(packageName, payload.streamId);
2620
+ this.sendResult(packageName, requestId, true);
2621
+ }
2622
+ catch (err) {
2623
+ this.sendResult(packageName, requestId, false, undefined, {
2624
+ code: err.code || MiniappErrorCode.INTERNAL,
2625
+ message: err instanceof Error ? err.message : "Stream stop failed",
2626
+ stage: err.stage,
2627
+ transport: err.transport,
2628
+ });
2629
+ }
2630
+ }
2631
+ async handleManagedStreamStart(packageName, payload, requestId) {
2632
+ const streaming = phoneStreamCoordinator;
2633
+ if (!streaming) {
2634
+ this.sendResult(packageName, requestId, false, undefined, {
2635
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
2636
+ message: "Streaming is not configured on this host",
2637
+ });
2638
+ return;
2639
+ }
2640
+ try {
2641
+ const result = await streaming.startManaged(packageName, {
2642
+ restreamDestinations: payload.restreamDestinations,
2643
+ video: normalizeStreamVideoConfig(payload.video),
2644
+ audio: normalizeStreamAudioConfig(payload.audio),
2645
+ sound: payload.sound,
2646
+ ingest: payload.ingest,
2647
+ });
2648
+ this.sendResult(packageName, requestId, true, result);
2649
+ }
2650
+ catch (err) {
2651
+ this.sendResult(packageName, requestId, false, undefined, {
2652
+ code: err.code || MiniappErrorCode.INTERNAL,
2653
+ message: err instanceof Error ? err.message : "Managed stream start failed",
2654
+ stage: err.stage,
2655
+ transport: err.transport,
2656
+ });
2657
+ }
2658
+ }
2659
+ async handleManagedStreamStop(packageName, payload, requestId) {
2660
+ return this.handleStreamStop(packageName, payload, requestId);
2661
+ }
2662
+ /**
2663
+ * session.glasses.requestWifiSetup — open the phone's glasses Wi-Fi setup
2664
+ * flow. The host owns the actual UI via the `engine.configure({ui})` seam;
2665
+ * this just forwards the request and reports success/failure.
2666
+ */
2667
+ async handleRequestWifiSetup(packageName, payload, requestId) {
2668
+ const requestWifiSetup = getUiSeams().requestWifiSetup;
2669
+ if (!requestWifiSetup) {
2670
+ this.sendResult(packageName, requestId, false, undefined, {
2671
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
2672
+ message: "Wi-Fi setup is not configured on this host",
2673
+ });
2674
+ return;
2675
+ }
2676
+ try {
2677
+ await requestWifiSetup(payload.reason);
2678
+ this.sendResult(packageName, requestId, true);
2679
+ }
2680
+ catch (err) {
2681
+ this.sendResult(packageName, requestId, false, undefined, {
2682
+ code: err.code || MiniappErrorCode.INTERNAL,
2683
+ message: err instanceof Error ? err.message : "Wi-Fi setup failed",
2684
+ });
2685
+ }
2686
+ }
2687
+ /**
2688
+ * Wire the PhoneStreamCoordinator's status callback so coordinator-emitted
2689
+ * updates become `EVENT { streamType: "stream_status" }` envelopes on the
2690
+ * subscribing miniapp(s).
2691
+ */
2692
+ wireStreamingStatusFanout() {
2693
+ const streaming = phoneStreamCoordinator;
2694
+ if (!streaming)
2695
+ return;
2696
+ streaming.setStatusSubscriber((packageName, update) => {
2697
+ this.sendToMiniapp(packageName, {
2698
+ type: MiniappResponseType.EVENT,
2699
+ streamType: "stream_status",
2700
+ data: {
2701
+ streamId: update.streamId,
2702
+ status: update.status,
2703
+ source: update.source,
2704
+ ...(update.data || {}),
2705
+ },
2706
+ });
2707
+ });
2708
+ }
2709
+ // ===========================================================================
2710
+ // Public subscribe helper
2711
+ // ===========================================================================
2712
+ /**
2713
+ * Standalone subscribe — lets external code subscribe a registered miniapp to
2714
+ * streams without going through the envelope protocol.
2715
+ *
2716
+ * Returns `{ok: true}` on success, or `{ok: false, error}` on failure.
2717
+ */
2718
+ subscribe(packageName, streams) {
2719
+ const app = this.connectedApps.get(packageName);
2720
+ if (!app) {
2721
+ return { ok: false, error: `App ${packageName} is not registered` };
2722
+ }
2723
+ if (!Array.isArray(streams)) {
2724
+ return { ok: false, error: "streams must be an array" };
2725
+ }
2726
+ console.log(`${LOG_TAG}: subscribe(${packageName}, [${streams.join(", ")}])`);
2727
+ // Check microphone permission for mic-requiring streams
2728
+ const micStreams = ["transcription", "translation", "audio_chunk", "vad"];
2729
+ const needsMic = streams.some((s) => micStreams.some((m) => s.startsWith(m) || s === m));
2730
+ if (needsMic) {
2731
+ const hasMicPermission = app.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE");
2732
+ if (!hasMicPermission) {
2733
+ return { ok: false, error: "MICROPHONE permission not declared in miniapp.json" };
2734
+ }
2735
+ }
2736
+ // Remove old subscriptions for this app
2737
+ for (const oldStream of app.subscriptions) {
2738
+ const subs = this.streamSubscribers.get(oldStream);
2739
+ if (subs) {
2740
+ subs.delete(packageName);
2741
+ if (subs.size === 0) {
2742
+ this.streamSubscribers.delete(oldStream);
2743
+ }
2744
+ }
2745
+ }
2746
+ // Add new subscriptions
2747
+ app.subscriptions = new Set(streams);
2748
+ app.forceLocalTranscriptionStreams.clear();
2749
+ app.cloudTranscriptionStreams = new Set(streams.filter((stream) => stream.startsWith("transcription:")));
2750
+ for (const stream of streams) {
2751
+ let subs = this.streamSubscribers.get(stream);
2752
+ if (!subs) {
2753
+ subs = new Set();
2754
+ this.streamSubscribers.set(stream, subs);
2755
+ }
2756
+ subs.add(packageName);
2757
+ }
2758
+ this.recomputeMicRequirements();
2759
+ this.updateCloudSubscriptions();
2760
+ return { ok: true };
2761
+ }
2762
+ // ===========================================================================
2763
+ // Mic requirements
2764
+ // ===========================================================================
2765
+ recomputeMicRequirements() {
2766
+ let anyPcm = false;
2767
+ let anyLc3 = false;
2768
+ for (const [stream, subscribers] of this.streamSubscribers) {
2769
+ if (subscribers.size === 0)
2770
+ continue;
2771
+ if (stream === "audio_chunk")
2772
+ anyPcm = true;
2773
+ if (stream.startsWith("transcription:") || stream.startsWith("translation:") || stream === "vad")
2774
+ anyLc3 = true;
2775
+ }
2776
+ micStateCoordinator.setLocalRequirements({ pcm: anyPcm, lc3: anyLc3 });
2777
+ }
2778
+ /**
2779
+ * Recompute the aggregated v2 cloud subscription list across all local miniapps.
2780
+ *
2781
+ * Cloud-dependent streams (transcription:*, translation:*) only flow if the
2782
+ * v2 runtime knows the phone wants them. Local-only streams (button_press, etc.)
2783
+ * are NOT sent — they come from the Bluetooth SDK, not from cloud.
2784
+ */
2785
+ updateCloudSubscriptions() {
2786
+ const cloudStreams = new Set();
2787
+ let transcriptionLang = null;
2788
+ let forceLocalTranscriptionLang = null;
2789
+ let hasForceLocalTranscription = false;
2790
+ for (const [stream, subscribers] of this.streamSubscribers) {
2791
+ if (subscribers.size === 0)
2792
+ continue;
2793
+ // Only transcription / translation need cloud delivery. Location and
2794
+ // notifications are sourced natively on the phone — no cloud hop.
2795
+ if (stream.startsWith("translation:")) {
2796
+ cloudStreams.add(stream);
2797
+ }
2798
+ if (stream.startsWith("transcription:")) {
2799
+ const language = stream.substring("transcription:".length);
2800
+ if (transcriptionLang === null)
2801
+ transcriptionLang = language;
2802
+ const routes = summarizeTranscriptionRoutes(subscribers, (packageName) => this.appTranscriptionRoutesForSubscription(packageName, stream));
2803
+ if (routes.hasForceLocalSubscriber) {
2804
+ hasForceLocalTranscription = true;
2805
+ if (forceLocalTranscriptionLang === null)
2806
+ forceLocalTranscriptionLang = language;
2807
+ }
2808
+ if (routes.hasCloudSubscriber)
2809
+ cloudStreams.add(stream);
2810
+ }
2811
+ }
2812
+ localSttFallbackCoordinator.onSubscriptionChange(transcriptionLang !== null, forceLocalTranscriptionLang ?? transcriptionLang, hasForceLocalTranscription);
2813
+ // Mirror the same set as typed AudioSubscription[] and push it to the cloud
2814
+ // runtime.
2815
+ this.ensureCloudResultsWired();
2816
+ const subs = this.buildCloudAudioSubscriptions(cloudStreams);
2817
+ const nextKey = this.audioSubscriptionKey(subs);
2818
+ if (!this.cloudAudioSubscriptionSync.begin(nextKey))
2819
+ return;
2820
+ console.log(`${LOG_TAG}: updateCloudSubscriptions streams=[${Array.from(cloudStreams).join(", ")}] cloudSubs=${subs.length}`);
2821
+ cloudClientService
2822
+ .setSubscriptions(subs)
2823
+ .then(() => {
2824
+ this.cloudAudioSubscriptionSync.succeeded(nextKey);
2825
+ })
2826
+ .catch((err) => {
2827
+ // Best-effort: the cloud may not be connected yet. Keep failed writes
2828
+ // retryable; otherwise a reconnect that recomputes the same desired
2829
+ // transcription set gets deduped and local captions never recover.
2830
+ this.cloudAudioSubscriptionSync.failed(nextKey);
2831
+ console.warn(`${LOG_TAG}: cloud setSubscriptions failed: ${err?.message ?? err}`);
2832
+ });
2833
+ }
2834
+ /**
2835
+ * Build the cloud `AudioSubscription[]` from the miniapp cloud-stream key set.
2836
+ *
2837
+ * Stream keys: `transcription:<lang>` (lang may be `auto`, `en-US`, …) and
2838
+ * `translation:<source>:<target>` (3 colon-parts; source/target may be `*`
2839
+ * or `auto` wildcards, target may also be `*`). The cloud protocol requires a
2840
+ * concrete `target: string` for translation, so wildcard-target translation
2841
+ * keys (`translation:<source>:*`, `translation:*:*`, `translation:auto`) have
2842
+ * no cloud equivalent and are skipped. Wildcard/`auto` SOURCE maps to
2843
+ * `{mode: "auto"}`.
2844
+ */
2845
+ buildCloudAudioSubscriptions(cloudStreams) {
2846
+ const subs = [];
2847
+ const langSource = (code) => code === "auto" || code === "*" ? { mode: "auto" } : { mode: "specific", code };
2848
+ for (const stream of cloudStreams) {
2849
+ if (stream.startsWith("transcription:")) {
2850
+ const lang = stream.substring("transcription:".length);
2851
+ subs.push({ kind: "transcription", language: langSource(lang) });
2852
+ }
2853
+ else if (stream.startsWith("translation:")) {
2854
+ const parts = stream.split(":");
2855
+ // Only `translation:<source>:<target>` maps cleanly; a concrete target
2856
+ // is required by the cloud schema.
2857
+ if (parts.length === 3) {
2858
+ const [, source, target] = parts;
2859
+ if (target === "*")
2860
+ continue;
2861
+ subs.push({ kind: "translation", source: langSource(source), target });
2862
+ }
2863
+ }
2864
+ }
2865
+ return subs;
2866
+ }
2867
+ audioSubscriptionKey(subs) {
2868
+ return JSON.stringify(subs
2869
+ .map((sub) => {
2870
+ if (sub.kind === "transcription") {
2871
+ return {
2872
+ kind: sub.kind,
2873
+ language: sub.language,
2874
+ };
2875
+ }
2876
+ return {
2877
+ kind: sub.kind,
2878
+ source: sub.source,
2879
+ target: sub.target,
2880
+ };
2881
+ })
2882
+ .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))));
2883
+ }
2884
+ /**
2885
+ * Wire the cloud's transcription/translation results into the existing
2886
+ * miniapp fan-out exactly once. Maps each cloud result back to the
2887
+ * cloud-to-app data shape `forwardEvent` already forwards, keyed on the
2888
+ * `transcription:<lang>` / `translation:<source>:<target>` stream strings, so
2889
+ * subscribed miniapps receive identical envelopes.
2890
+ */
2891
+ ensureCloudResultsWired() {
2892
+ if (this.cloudResultsWired)
2893
+ return;
2894
+ this.cloudResultsWired = true;
2895
+ cloudClientService.onTranscript((d) => {
2896
+ const receivedAt = Date.now();
2897
+ if (TRANSCRIPT_TIMING_TELEMETRY) {
2898
+ console.log(`${LOG_TAG}: transcript cloud_recv t=${receivedAt} final=${d.isFinal} lang=${d.resolvedLanguage} text="${d.text.slice(0, 48)}"`);
2899
+ }
2900
+ this.forwardEvent(`transcription:${d.resolvedLanguage}`, {
2901
+ type: "transcription",
2902
+ text: d.text,
2903
+ isFinal: d.isFinal,
2904
+ utteranceId: d.utteranceId,
2905
+ transcribeLanguage: d.resolvedLanguage,
2906
+ detectedLanguage: d.languageDetected ? d.resolvedLanguage : undefined,
2907
+ startTime: d.startMs,
2908
+ endTime: d.endMs,
2909
+ speakerId: d.speakerId,
2910
+ duration: d.durationMs,
2911
+ provider: d.provider,
2912
+ confidence: d.confidence,
2913
+ __hostReceivedAt: receivedAt,
2914
+ }, "cloud");
2915
+ });
2916
+ cloudClientService.onTranslation((d) => {
2917
+ this.forwardEvent(`translation:${d.source.language}:${d.target.language}`, {
2918
+ type: "translation",
2919
+ text: d.text,
2920
+ originalText: d.originalText,
2921
+ isFinal: d.isFinal,
2922
+ utteranceId: d.utteranceId,
2923
+ startTime: d.startMs,
2924
+ endTime: d.endMs,
2925
+ speakerId: d.speakerId,
2926
+ duration: d.durationMs,
2927
+ transcribeLanguage: d.source.language,
2928
+ translateLanguage: d.target.language,
2929
+ didTranslate: true,
2930
+ provider: d.provider,
2931
+ confidence: d.confidence,
2932
+ });
2933
+ });
2934
+ }
2935
+ ensureCloudStatusWired() {
2936
+ if (this.cloudStatusWired)
2937
+ return;
2938
+ this.cloudStatusWired = true;
2939
+ cloudClientService.onStatusChanged((status) => {
2940
+ if (status.status === "connected") {
2941
+ this.updateCloudSubscriptions();
2942
+ // A miniapp that connected before the cloud client was ready never got
2943
+ // its auth token; now that we're connected, mint it without waiting for
2944
+ // the retry backoff (or a full manual reconnect).
2945
+ this.redriveMiniappAuthDelivery();
2946
+ }
2947
+ this.broadcastCloudStatus();
2948
+ });
2949
+ useSettingsStore.subscribe((s) => s.getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive), () => {
2950
+ this.broadcastCloudStatus();
2951
+ });
2952
+ }
2953
+ currentCloudStatus() {
2954
+ const base = cloudClientService.getStatus();
2955
+ const fallbackActive = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive) ===
2956
+ true;
2957
+ return {
2958
+ status: base.status,
2959
+ audioTransport: fallbackActive ? "offline" : base.audioTransport,
2960
+ };
2961
+ }
2962
+ broadcastCloudStatus() {
2963
+ const status = this.currentCloudStatus();
2964
+ for (const packageName of this.connectedApps.keys()) {
2965
+ this.sendCloudStatusToMiniapp(packageName, status);
2966
+ }
2967
+ }
2968
+ sendCloudStatusToMiniapp(packageName, status = this.currentCloudStatus()) {
2969
+ this.sendToMiniapp(packageName, {
2970
+ type: MiniappResponseType.EVENT,
2971
+ streamType: CLOUD_STATUS_STREAM,
2972
+ data: status,
2973
+ });
2974
+ }
2975
+ // ===========================================================================
2976
+ // Stream fan-out
2977
+ // ===========================================================================
2978
+ /**
2979
+ * Forward a streamed event to all miniapps subscribed to the given stream.
2980
+ *
2981
+ * Event name translation:
2982
+ * - Cloud sends "head_up" → miniapp protocol uses "head_position" (HEAD_POSITION)
2983
+ * - Cloud sends "VAD" (uppercase) → miniapp protocol uses "vad" (lowercase)
2984
+ */
2985
+ forwardEvent(streamType, data, transcriptionSource) {
2986
+ // Translate cloud event names to miniapp protocol stream types
2987
+ const normalizedStream = this.normalizeStreamType(streamType);
2988
+ // Collect all subscribers: exact match, plus wildcard matches for streams
2989
+ // that carry a language tag. A miniapp subscribed to "transcription:auto"
2990
+ // should receive any "transcription:<lang>" event (the detected language
2991
+ // is conveyed in the event data, not the stream key).
2992
+ //
2993
+ // Translation streams support cloud v3's wildcard patterns. Incoming
2994
+ // events are keyed `translation:<source>:<target>` and we match against
2995
+ // any of:
2996
+ // translation:*:* — all-pairs (TranslationModule.on)
2997
+ // translation:*:<target> — any-source → target (TranslationModule.to)
2998
+ // translation:<source>:* — source → any-target (rare)
2999
+ // translation:<source>:<target> — exact (TranslationModule.fromTo)
3000
+ // translation:auto — back-compat alias for *:*
3001
+ const matchedSubs = new Set();
3002
+ const exact = this.streamSubscribers.get(normalizedStream);
3003
+ if (exact)
3004
+ for (const p of exact)
3005
+ matchedSubs.add(p);
3006
+ if (normalizedStream.startsWith("transcription:")) {
3007
+ const autoSubs = this.streamSubscribers.get("transcription:auto");
3008
+ if (autoSubs)
3009
+ for (const p of autoSubs)
3010
+ matchedSubs.add(p);
3011
+ }
3012
+ else if (normalizedStream.startsWith("translation:")) {
3013
+ // translation:<source>:<target> — match each wildcard variant.
3014
+ const parts = normalizedStream.split(":");
3015
+ // parts[0] = "translation", parts[1] = source, parts[2] = target
3016
+ if (parts.length === 3) {
3017
+ const [, source, target] = parts;
3018
+ const patterns = [
3019
+ "translation:auto", // legacy alias for *:*
3020
+ "translation:*:*", // cloud v3 "all pairs"
3021
+ `translation:*:${target}`, // cloud v3 "to <target>"
3022
+ `translation:${source}:*`, // cloud v3 "from <source>"
3023
+ ];
3024
+ for (const pat of patterns) {
3025
+ const subs = this.streamSubscribers.get(pat);
3026
+ if (subs)
3027
+ for (const p of subs)
3028
+ matchedSubs.add(p);
3029
+ }
3030
+ }
3031
+ else {
3032
+ // Malformed stream key — still honor the legacy auto alias.
3033
+ const autoSubs = this.streamSubscribers.get("translation:auto");
3034
+ if (autoSubs)
3035
+ for (const p of autoSubs)
3036
+ matchedSubs.add(p);
3037
+ }
3038
+ }
3039
+ // Touch event per-gesture fan-out. SDK-side dispatch is keyed on the
3040
+ // exact streamType, so per-gesture subscribers (`touch_event:single_tap`,
3041
+ // etc.) need their own EVENT envelope with the gesture-tagged streamType.
3042
+ // The bare `touch_event` stream above still catches `onTouch(handler)`.
3043
+ let perGestureStream = null;
3044
+ let outboundData = data;
3045
+ if (normalizedStream === MiniappStreamType.TOUCH_EVENT) {
3046
+ // The Bluetooth SDK delivers the gesture under `gestureName` (single_tap /
3047
+ // double_tap / triple_tap / long_press / swipe_up / swipe_down). Surface it
3048
+ // to miniapps as `TouchData.kind` and tag a per-gesture stream so
3049
+ // onTouch("single_tap", ...) routes correctly.
3050
+ const touch = data;
3051
+ const gesture = touch?.gestureName ?? touch?.kind;
3052
+ if (typeof gesture === "string" && gesture.length > 0) {
3053
+ outboundData = { ...touch, kind: gesture };
3054
+ perGestureStream = `${MiniappStreamType.TOUCH_EVENT}:${gesture}`;
3055
+ }
3056
+ }
3057
+ if (normalizedStream.startsWith("transcription:")) {
3058
+ // const known = Array.from(this.streamSubscribers.keys())
3059
+ // console.log(
3060
+ // `${LOG_TAG}: forwardEvent(${streamType} → ${normalizedStream}) matched=${matchedSubs.size} known=[${known.join(", ")}]`,
3061
+ // )
3062
+ }
3063
+ if (matchedSubs.size === 0 && !perGestureStream)
3064
+ return;
3065
+ for (const packageName of matchedSubs) {
3066
+ let transcriptionRoute;
3067
+ if (normalizedStream.startsWith("transcription:") && transcriptionSource) {
3068
+ const routes = this.appTranscriptionRoutesForEvent(packageName, normalizedStream);
3069
+ transcriptionRoute =
3070
+ transcriptionDeliveryRoute(transcriptionSource, routes, cloudClientService.isConnected()) ?? undefined;
3071
+ if (!transcriptionRoute)
3072
+ continue;
3073
+ }
3074
+ if (TRANSCRIPT_TIMING_TELEMETRY && normalizedStream.startsWith("transcription:")) {
3075
+ const transcript = data;
3076
+ const now = Date.now();
3077
+ console.log(`${LOG_TAG}: transcript fanout t=${now} app=${packageName} final=${!!transcript?.isFinal} hostDelta=${transcript?.__hostReceivedAt ? now - transcript.__hostReceivedAt : -1}ms text="${(transcript?.text ?? "").slice(0, 48)}"`);
3078
+ }
3079
+ // While a nav trip is active the Nav SDK's road-snapped fixes are
3080
+ // already being forwarded via addLocationListener (see NAV_START handler).
3081
+ // Suppress the raw background-GPS forward for that miniapp so the two
3082
+ // streams don't interleave and cause the position to jump back to the
3083
+ // real-phone location during simulation.
3084
+ if (normalizedStream === MiniappStreamType.LOCATION_UPDATE && this.navigationHandlers.isTripActive(packageName)) {
3085
+ continue;
3086
+ }
3087
+ this.sendToMiniapp(packageName, {
3088
+ type: MiniappResponseType.EVENT,
3089
+ streamType: normalizedStream,
3090
+ data: outboundData,
3091
+ ...(transcriptionRoute ? { transcriptionRoute } : {}),
3092
+ });
3093
+ }
3094
+ // Send a separate envelope to per-gesture subscribers tagged with the
3095
+ // gesture-specific streamType so SDK-side dispatch routes correctly.
3096
+ if (perGestureStream) {
3097
+ const gestureSubs = this.streamSubscribers.get(perGestureStream);
3098
+ if (gestureSubs) {
3099
+ for (const packageName of gestureSubs) {
3100
+ this.sendToMiniapp(packageName, {
3101
+ type: MiniappResponseType.EVENT,
3102
+ streamType: perGestureStream,
3103
+ data: outboundData,
3104
+ });
3105
+ }
3106
+ }
3107
+ }
3108
+ }
3109
+ appTranscriptionRoutesForSubscription(packageName, stream) {
3110
+ const app = this.connectedApps.get(packageName);
3111
+ return {
3112
+ cloud: app?.cloudTranscriptionStreams.has(stream) === true,
3113
+ forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true,
3114
+ };
3115
+ }
3116
+ appTranscriptionRoutesForEvent(packageName, stream) {
3117
+ const app = this.connectedApps.get(packageName);
3118
+ const autoStream = `${MiniappStreamType.TRANSCRIPTION}:auto`;
3119
+ return {
3120
+ cloud: app?.cloudTranscriptionStreams.has(stream) === true || app?.cloudTranscriptionStreams.has(autoStream) === true,
3121
+ forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true ||
3122
+ app?.forceLocalTranscriptionStreams.has(autoStream) === true,
3123
+ };
3124
+ }
3125
+ /**
3126
+ * Translate cloud event names to miniapp stream type values.
3127
+ */
3128
+ normalizeStreamType(cloudEventName) {
3129
+ // Cloud / Bluetooth SDK → miniapp protocol translations.
3130
+ // Bluetooth SDK event names don't always match the miniapp wire values.
3131
+ switch (cloudEventName) {
3132
+ case "head_up":
3133
+ return MiniappStreamType.HEAD_POSITION; // head_up → head_position
3134
+ case "accel_event":
3135
+ return MiniappStreamType.ACCEL_DATA; // accel_event (native) → accel_data
3136
+ case "VAD":
3137
+ return MiniappStreamType.VAD; // VAD (uppercase) → vad (lowercase)
3138
+ case "glasses_battery_update":
3139
+ return MiniappStreamType.GLASSES_BATTERY; // glasses_battery_update → glasses_battery
3140
+ case "glasses_connection_state":
3141
+ return MiniappStreamType.GLASSES_CONNECTION; // glasses_connection_state → glasses_connection
3142
+ default:
3143
+ // Preserve case for typed streams like "transcription:en-US" / "translation:en-US:fr-FR"
3144
+ // whose language tags are case-sensitive (BCP-47). Lowercase only plain names.
3145
+ if (cloudEventName.includes(":"))
3146
+ return cloudEventName;
3147
+ return cloudEventName.toLowerCase();
3148
+ }
3149
+ }
3150
+ // ===========================================================================
3151
+ // Outbound helpers
3152
+ // ===========================================================================
3153
+ /**
3154
+ * Send a payload to a connected miniapp, wrapped in an envelope.
3155
+ */
3156
+ /**
3157
+ * Push a speaker-state transition to the owning miniapp. Idempotent: if
3158
+ * the new state matches the cached one (and isn't an error), no envelope
3159
+ * is sent. Error events are always sent — they're transient and the SDK
3160
+ * settles back to a non-error state immediately after.
3161
+ */
3162
+ setSpeakerState(packageName, next, extra) {
3163
+ const app = this.connectedApps.get(packageName);
3164
+ if (!app)
3165
+ return;
3166
+ if (app.speakerState === next && next !== "error")
3167
+ return;
3168
+ app.speakerState = next === "error" ? "error" : next;
3169
+ this.sendToMiniapp(packageName, {
3170
+ type: MiniappResponseType.SPEAKER_STATE,
3171
+ state: next,
3172
+ ...(extra?.errorCode !== undefined ? { errorCode: extra.errorCode } : {}),
3173
+ ...(extra?.errorMessage !== undefined ? { errorMessage: extra.errorMessage } : {}),
3174
+ ...(extra?.durationMs !== undefined ? { durationMs: extra.durationMs } : {}),
3175
+ });
3176
+ // After an error event, immediately transition to "stopped" so the
3177
+ // SDK's isPlaying getter reads false and a new play()/speak() call
3178
+ // starts cleanly from idle/stopped.
3179
+ if (next === "error") {
3180
+ app.speakerState = "stopped";
3181
+ this.sendToMiniapp(packageName, {
3182
+ type: MiniappResponseType.SPEAKER_STATE,
3183
+ state: "stopped",
3184
+ });
3185
+ }
3186
+ }
3187
+ sendToMiniapp(packageName, payload, requestId) {
3188
+ const app = this.connectedApps.get(packageName);
3189
+ if (!app) {
3190
+ console.warn(`${LOG_TAG}: sendToMiniapp — ${packageName} not connected`);
3191
+ return;
3192
+ }
3193
+ const envelope = {
3194
+ payload,
3195
+ requestId,
3196
+ };
3197
+ const serialized = serializeEnvelope(envelope);
3198
+ if (payload?.streamType?.toString().startsWith("transcription")) {
3199
+ // console.log(
3200
+ // `${LOG_TAG}: sendToMiniapp → ${packageName} streamType=${(payload as Record<string, unknown>).streamType}`,
3201
+ // )
3202
+ }
3203
+ try {
3204
+ app.sendMessage(serialized);
3205
+ }
3206
+ catch (err) {
3207
+ console.error(`${LOG_TAG}: sendToMiniapp error for ${packageName}:`, err);
3208
+ }
3209
+ }
3210
+ /**
3211
+ * Send a REQUEST_RESULT response.
3212
+ */
3213
+ // ===========================================================================
3214
+ // Inter-miniapp interop (session.miniapps + session.actions.invoke)
3215
+ // ===========================================================================
3216
+ /** Max serialized size of an action call's params / result (256 KB). */
3217
+ static ACTION_PAYLOAD_CAP = 256 * 1024;
3218
+ /** Outstanding action invocations, keyed by host-generated callId. */
3219
+ actionCalls = new Map();
3220
+ actionCallSeq = 0;
3221
+ interopApps() {
3222
+ return useAppStatusStore.getState().apps;
3223
+ }
3224
+ isSystemPackage(packageName) {
3225
+ if (SYSTEM_MINIAPP_PACKAGES.has(packageName))
3226
+ return true;
3227
+ return this.interopApps().find((app) => app.packageName === packageName)?.isMiniappDev === true;
3228
+ }
3229
+ async startInteropApp(packageName) {
3230
+ const app = this.interopApps().find((a) => a.packageName === packageName);
3231
+ if (!app)
3232
+ return false;
3233
+ if (app.local) {
3234
+ await miniappLauncher.ensureConnected(packageName);
3235
+ return true;
3236
+ }
3237
+ return useAppStatusStore.getState().start(app, { skipNavigation: true });
3238
+ }
3239
+ stopInteropApp(packageName) {
3240
+ return useAppStatusStore.getState().stop(packageName);
3241
+ }
3242
+ wakeInteropApp(packageName) {
3243
+ return miniappLauncher.ensureConnected(packageName);
3244
+ }
3245
+ /** Emit an interop audit event (best-effort — never let telemetry break a call). */
3246
+ auditInterop(event) {
3247
+ try {
3248
+ getAnalytics()?.("miniapp_interop", {
3249
+ caller: event.caller,
3250
+ op: event.op,
3251
+ target: event.target ?? "",
3252
+ actionId: event.actionId ?? "",
3253
+ ok: event.ok,
3254
+ errorCode: event.errorCode ?? "",
3255
+ });
3256
+ }
3257
+ catch {
3258
+ /* telemetry must never break an interop call */
3259
+ }
3260
+ }
3261
+ /** Reject the request with NOT_PERMITTED unless the caller is a system app. */
3262
+ requireSystemCaller(packageName, requestId, op, target) {
3263
+ if (this.isSystemPackage(packageName))
3264
+ return true;
3265
+ this.auditInterop({ caller: packageName, op, target, ok: false, errorCode: MiniappErrorCode.NOT_PERMITTED });
3266
+ this.sendResult(packageName, requestId, false, undefined, {
3267
+ code: MiniappErrorCode.NOT_PERMITTED,
3268
+ message: "Inter-miniapp APIs are restricted to system apps",
3269
+ });
3270
+ return false;
3271
+ }
3272
+ /** Project a host ClientApp into the SDK's MiniappInfo shape (JSON over the wire). */
3273
+ buildMiniappInfo(app) {
3274
+ const compatibility = app.compatibility ?? {
3275
+ isCompatible: true,
3276
+ missingRequired: [],
3277
+ missingOptional: [],
3278
+ warnings: [],
3279
+ };
3280
+ return {
3281
+ packageName: app.packageName,
3282
+ name: app.name,
3283
+ version: app.version ?? "",
3284
+ running: app.running,
3285
+ compatibility,
3286
+ actions: app.actions ?? [],
3287
+ };
3288
+ }
3289
+ handleMiniappsList(packageName, payload, requestId) {
3290
+ if (!this.requireSystemCaller(packageName, requestId, "list"))
3291
+ return;
3292
+ const includeIncompatible = payload.includeIncompatible === true;
3293
+ const infos = this.interopApps()
3294
+ .filter((a) => a.packageName &&
3295
+ a.packageName !== packageName &&
3296
+ !a.packageName.includes("@empty") &&
3297
+ (includeIncompatible || a.compatibility?.isCompatible !== false))
3298
+ .map((a) => this.buildMiniappInfo(a));
3299
+ this.sendResult(packageName, requestId, true, infos);
3300
+ this.auditInterop({ caller: packageName, op: "list", ok: true });
3301
+ }
3302
+ async handleMiniappsStart(packageName, payload, requestId) {
3303
+ const target = payload.packageName;
3304
+ if (!this.requireSystemCaller(packageName, requestId, "start", target))
3305
+ return;
3306
+ const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined;
3307
+ if (!target || !app) {
3308
+ this.sendResult(packageName, requestId, false, undefined, {
3309
+ code: MiniappErrorCode.APP_NOT_FOUND,
3310
+ message: `Miniapp not found: ${target ?? "(missing packageName)"}`,
3311
+ });
3312
+ this.auditInterop({
3313
+ caller: packageName,
3314
+ op: "start",
3315
+ target,
3316
+ ok: false,
3317
+ errorCode: MiniappErrorCode.APP_NOT_FOUND,
3318
+ });
3319
+ return;
3320
+ }
3321
+ // Pre-flight the hardware gate so the caller gets a precise reason rather
3322
+ // than a generic rejection (start()'s own gate would otherwise alert + abort).
3323
+ if (app.compatibility && app.compatibility.isCompatible === false) {
3324
+ this.sendResult(packageName, requestId, false, undefined, {
3325
+ code: MiniappErrorCode.APP_NOT_COMPATIBLE,
3326
+ message: `${target} is not compatible with the connected glasses`,
3327
+ });
3328
+ islandNotifications.emit({
3329
+ kind: "version_incompatible",
3330
+ packageName: target,
3331
+ reason: `${target} is not compatible with the connected glasses`,
3332
+ metadata: { missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.start" },
3333
+ timestamp: Date.now(),
3334
+ });
3335
+ this.auditInterop({
3336
+ caller: packageName,
3337
+ op: "start",
3338
+ target,
3339
+ ok: false,
3340
+ errorCode: MiniappErrorCode.APP_NOT_COMPATIBLE,
3341
+ });
3342
+ return;
3343
+ }
3344
+ try {
3345
+ // startInteropApp resolves to false when the app-store gate rejected the
3346
+ // launch or the background context failed to spawn — don't report success.
3347
+ const started = await this.startInteropApp(target);
3348
+ if (started) {
3349
+ this.sendResult(packageName, requestId, true);
3350
+ this.auditInterop({ caller: packageName, op: "start", target, ok: true });
3351
+ }
3352
+ else {
3353
+ this.sendResult(packageName, requestId, false, undefined, {
3354
+ code: MiniappErrorCode.INTERNAL,
3355
+ message: `start of ${target} was rejected by the host (gated or failed to spawn)`,
3356
+ });
3357
+ this.auditInterop({
3358
+ caller: packageName,
3359
+ op: "start",
3360
+ target,
3361
+ ok: false,
3362
+ errorCode: MiniappErrorCode.INTERNAL,
3363
+ });
3364
+ }
3365
+ }
3366
+ catch (e) {
3367
+ this.sendResult(packageName, requestId, false, undefined, {
3368
+ code: MiniappErrorCode.INTERNAL,
3369
+ message: e?.message ?? "start failed",
3370
+ });
3371
+ this.auditInterop({
3372
+ caller: packageName,
3373
+ op: "start",
3374
+ target,
3375
+ ok: false,
3376
+ errorCode: MiniappErrorCode.INTERNAL,
3377
+ });
3378
+ }
3379
+ }
3380
+ async handleMiniappsStop(packageName, payload, requestId) {
3381
+ const target = payload.packageName;
3382
+ if (!this.requireSystemCaller(packageName, requestId, "stop", target))
3383
+ return;
3384
+ if (!target) {
3385
+ this.sendResult(packageName, requestId, false, undefined, {
3386
+ code: MiniappErrorCode.APP_NOT_FOUND,
3387
+ message: "stop requires a packageName",
3388
+ });
3389
+ return;
3390
+ }
3391
+ try {
3392
+ await this.stopInteropApp(target);
3393
+ this.sendResult(packageName, requestId, true);
3394
+ this.auditInterop({ caller: packageName, op: "stop", target, ok: true });
3395
+ }
3396
+ catch (e) {
3397
+ this.sendResult(packageName, requestId, false, undefined, {
3398
+ code: MiniappErrorCode.INTERNAL,
3399
+ message: e?.message ?? "stop failed",
3400
+ });
3401
+ }
3402
+ }
3403
+ /** True iff `value` serializes larger than the action payload cap. */
3404
+ actionPayloadTooLarge(value) {
3405
+ try {
3406
+ return JSON.stringify(value ?? null).length > LocalMiniappRuntime.ACTION_PAYLOAD_CAP;
3407
+ }
3408
+ catch {
3409
+ // Unserializable — let it through; the transport's own serialize handles it.
3410
+ return false;
3411
+ }
3412
+ }
3413
+ /**
3414
+ * session.actions.invoke → wake the target headlessly, deliver an ACTION_CALL,
3415
+ * and correlate its ACTION_RESULT back to the caller's pending request.
3416
+ */
3417
+ async handleActionInvoke(callerPackageName, payload, requestId) {
3418
+ const targetForGate = payload.targetPackageName;
3419
+ if (!this.requireSystemCaller(callerPackageName, requestId, "invoke", targetForGate))
3420
+ return;
3421
+ const target = payload.targetPackageName;
3422
+ const actionId = payload.actionId;
3423
+ const params = payload.params ?? {};
3424
+ // Floor the timeout at 6s so the host never rejects with ACTION_TIMEOUT before
3425
+ // the target SDK's handler-registration window closes. A freshly-woken miniapp
3426
+ // buffers an undelivered ACTION_CALL for HANDLER_WAIT_MS (5s in @mentra/miniapp's
3427
+ // actions module) waiting for session.actions.handle; a shorter host timeout
3428
+ // could fire while the target is still registering, then the action would run
3429
+ // with no caller left to receive its result. 6s = that 5s buffer + 1s for the
3430
+ // ACTION_RESULT/NO_ACTION_HANDLER reply to travel back. Keep these in sync.
3431
+ const timeoutMs = Math.min(Math.max(Number(payload.timeoutMs) || 30_000, 6_000), 120_000);
3432
+ if (this.actionPayloadTooLarge(params)) {
3433
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3434
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
3435
+ message: "action params exceeded the 256 KB cap",
3436
+ });
3437
+ return;
3438
+ }
3439
+ const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined;
3440
+ if (!target || !app) {
3441
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3442
+ code: MiniappErrorCode.APP_NOT_FOUND,
3443
+ message: `Miniapp not found: ${target ?? "(missing targetPackageName)"}`,
3444
+ });
3445
+ return;
3446
+ }
3447
+ if (!actionId) {
3448
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3449
+ code: MiniappErrorCode.ACTION_NOT_FOUND,
3450
+ message: "invoke requires an actionId",
3451
+ });
3452
+ return;
3453
+ }
3454
+ if (app.compatibility && app.compatibility.isCompatible === false) {
3455
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3456
+ code: MiniappErrorCode.APP_NOT_COMPATIBLE,
3457
+ message: `${target} is not compatible with the connected glasses`,
3458
+ });
3459
+ islandNotifications.emit({
3460
+ kind: "version_incompatible",
3461
+ packageName: target,
3462
+ reason: `${target} is not compatible with the connected glasses`,
3463
+ metadata: { missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.invoke" },
3464
+ timestamp: Date.now(),
3465
+ });
3466
+ return;
3467
+ }
3468
+ // Declared-action gate — only once the host populates app.actions (Phase 2);
3469
+ // until then app.actions is undefined and we fall through to NO_ACTION_HANDLER.
3470
+ if (!(app.actions ?? []).some((a) => a.id === actionId)) {
3471
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3472
+ code: MiniappErrorCode.ACTION_NOT_FOUND,
3473
+ message: `${target} does not declare action "${actionId}"`,
3474
+ });
3475
+ return;
3476
+ }
3477
+ // Headless wake + wait for CONNECT (idempotent / fast if already connected).
3478
+ try {
3479
+ await this.wakeInteropApp(target);
3480
+ }
3481
+ catch (e) {
3482
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3483
+ code: MiniappErrorCode.WAKE_FAILED,
3484
+ message: e?.message ?? `failed to wake ${target}`,
3485
+ });
3486
+ return;
3487
+ }
3488
+ // The wake waited for CONNECT, but the target could have dropped in the gap
3489
+ // before delivery — fail fast rather than arming a timer for a call that
3490
+ // sendToMiniapp would silently drop (the caller would otherwise wait the
3491
+ // full invoke timeout for an ACTION_CALL that was never delivered).
3492
+ if (!this.connectedApps.has(target)) {
3493
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3494
+ code: MiniappErrorCode.WAKE_FAILED,
3495
+ message: `${target} disconnected before the action could be delivered`,
3496
+ });
3497
+ this.auditInterop({
3498
+ caller: callerPackageName,
3499
+ op: "invoke",
3500
+ target,
3501
+ actionId,
3502
+ ok: false,
3503
+ errorCode: MiniappErrorCode.WAKE_FAILED,
3504
+ });
3505
+ return;
3506
+ }
3507
+ // Deliver the call and arm the handler timeout; ACTION_RESULT resolves it.
3508
+ const callId = `act-${Date.now().toString(36)}-${this.actionCallSeq++}`;
3509
+ const timer = BgTimer.setTimeout(() => {
3510
+ this.actionCalls.delete(callId);
3511
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3512
+ code: MiniappErrorCode.ACTION_TIMEOUT,
3513
+ message: `action "${actionId}" timed out after ${timeoutMs}ms`,
3514
+ });
3515
+ }, timeoutMs);
3516
+ this.actionCalls.set(callId, { callerPackageName, targetPackageName: target, callerRequestId: requestId, timer });
3517
+ this.sendToMiniapp(target, {
3518
+ type: MiniappResponseType.ACTION_CALL,
3519
+ callId,
3520
+ actionId,
3521
+ params,
3522
+ callerPackageName,
3523
+ });
3524
+ this.auditInterop({ caller: callerPackageName, op: "invoke", target, actionId, ok: true });
3525
+ }
3526
+ /** Target → host: forward an ACTION_RESULT back to the caller's pending invoke. */
3527
+ handleActionResult(targetPackageName, payload) {
3528
+ const callId = payload.callId;
3529
+ if (!callId)
3530
+ return;
3531
+ const pending = this.actionCalls.get(callId);
3532
+ if (!pending)
3533
+ return;
3534
+ // Only the invoked target may resolve this call. callIds are guessable
3535
+ // (timestamp + seq), so a different connected miniapp must not be able to
3536
+ // spoof a result/error for someone else's invoke.
3537
+ if (pending.targetPackageName !== targetPackageName)
3538
+ return;
3539
+ this.actionCalls.delete(callId);
3540
+ BgTimer.clearTimeout(pending.timer);
3541
+ if (payload.ok === true) {
3542
+ const result = payload.result;
3543
+ if (this.actionPayloadTooLarge(result)) {
3544
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, {
3545
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
3546
+ message: "action result exceeded the 256 KB cap",
3547
+ });
3548
+ return;
3549
+ }
3550
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, true, result ?? null);
3551
+ }
3552
+ else {
3553
+ const error = payload.error ?? {
3554
+ code: MiniappErrorCode.INTERNAL,
3555
+ message: "action handler error",
3556
+ };
3557
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, error);
3558
+ }
3559
+ }
3560
+ sendResult(packageName, requestId, ok, data, error) {
3561
+ if (!requestId)
3562
+ return;
3563
+ this.sendToMiniapp(packageName, {
3564
+ type: MiniappResponseType.REQUEST_RESULT,
3565
+ requestId,
3566
+ ok,
3567
+ ...(data !== undefined ? { data } : {}),
3568
+ ...(error ? { error } : {}),
3569
+ }, requestId);
3570
+ }
3571
+ /**
3572
+ * Send a VISIBILITY_CHANGE push to a miniapp.
3573
+ */
3574
+ sendVisibilityChange(packageName, visibility) {
3575
+ this.sendToMiniapp(packageName, {
3576
+ type: MiniappResponseType.VISIBILITY_CHANGE,
3577
+ visibility,
3578
+ });
3579
+ }
3580
+ /**
3581
+ * Fast foreground liveness check used when a local miniapp UI is opened or
3582
+ * resumed. The normal watchdog intentionally waits ~30s to avoid killing a
3583
+ * healthy-but-busy background script. When the user is actively looking at a
3584
+ * WebView, a stale JSContext reads as "cloud offline" / no captions, so probe
3585
+ * immediately and reuse the crash-respawn path if no message comes back.
3586
+ */
3587
+ probeForegroundLiveness(packageName, reason = "foreground-open", timeoutMs = FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS) {
3588
+ const app = this.connectedApps.get(packageName);
3589
+ if (!app)
3590
+ return;
3591
+ this.clearForegroundProbe(packageName);
3592
+ const probeStartedAt = Date.now();
3593
+ this.sendToMiniapp(packageName, {
3594
+ type: MiniappRequestType.PING,
3595
+ });
3596
+ const timerId = BgTimer.setTimeout(() => {
3597
+ this.foregroundProbeTimers.delete(packageName);
3598
+ const current = this.connectedApps.get(packageName);
3599
+ if (!current)
3600
+ return;
3601
+ if (current.lastPongAt >= probeStartedAt)
3602
+ return;
3603
+ console.warn(`${LOG_TAG}: ${packageName} failed foreground liveness probe (${reason}), respawning`);
3604
+ this.unregisterApp(packageName);
3605
+ this.onLivenessTimeout?.(packageName);
3606
+ }, timeoutMs);
3607
+ this.foregroundProbeTimers.set(packageName, timerId);
3608
+ }
3609
+ // ===========================================================================
3610
+ // Ping / pong liveness
3611
+ // ===========================================================================
3612
+ ensurePingLoop() {
3613
+ if (this.pingIntervalId !== null)
3614
+ return;
3615
+ this.pingIntervalId = BgTimer.setInterval(() => {
3616
+ this.doPingRound();
3617
+ }, PING_INTERVAL_MS);
3618
+ }
3619
+ stopPingLoop() {
3620
+ if (this.pingIntervalId !== null) {
3621
+ BgTimer.clearInterval(this.pingIntervalId);
3622
+ this.pingIntervalId = null;
3623
+ }
3624
+ }
3625
+ doPingRound() {
3626
+ const now = Date.now();
3627
+ const staleThreshold = PING_INTERVAL_MS * PING_TIMEOUT_THRESHOLD;
3628
+ const toRemove = [];
3629
+ for (const [packageName, app] of this.connectedApps) {
3630
+ if (now - app.lastPongAt > staleThreshold) {
3631
+ console.warn(`${LOG_TAG}: ${packageName} missed ${PING_TIMEOUT_THRESHOLD} pings, unregistering`);
3632
+ toRemove.push(packageName);
3633
+ continue;
3634
+ }
3635
+ // Send PING — SDK auto-replies with PONG
3636
+ this.sendToMiniapp(packageName, {
3637
+ type: MiniappRequestType.PING,
3638
+ });
3639
+ }
3640
+ for (const pkg of toRemove) {
3641
+ this.unregisterApp(pkg);
3642
+ // Hand the package to the router's respawn machinery — a missed-pings
3643
+ // death is treated like a crash so the background script comes back.
3644
+ this.onLivenessTimeout?.(pkg);
3645
+ }
3646
+ }
3647
+ /**
3648
+ * Called when a PONG is received from a miniapp (or any message, really).
3649
+ * Updates lastPongAt to keep the app alive.
3650
+ */
3651
+ handlePong(packageName) {
3652
+ const app = this.connectedApps.get(packageName);
3653
+ if (app) {
3654
+ app.lastPongAt = Date.now();
3655
+ this.clearForegroundProbe(packageName);
3656
+ }
3657
+ }
3658
+ clearForegroundProbe(packageName) {
3659
+ const timerId = this.foregroundProbeTimers.get(packageName);
3660
+ if (timerId == null)
3661
+ return;
3662
+ BgTimer.clearTimeout(timerId);
3663
+ this.foregroundProbeTimers.delete(packageName);
3664
+ }
3665
+ // ===========================================================================
3666
+ // Lifecycle
3667
+ // ===========================================================================
3668
+ cleanup() {
3669
+ console.log(`${LOG_TAG}: cleanup()`);
3670
+ this.stopPingLoop();
3671
+ // Copy keys since unregisterApp mutates the map
3672
+ const packageNames = [...this.connectedApps.keys()];
3673
+ for (const pkg of packageNames) {
3674
+ this.unregisterApp(pkg);
3675
+ }
3676
+ // Belt-and-suspenders: clear any remaining state
3677
+ this.pendingCloudRequests.clear();
3678
+ this.streamSubscribers.clear();
3679
+ this.connectedApps.clear();
3680
+ for (const timerId of this.foregroundProbeTimers.values()) {
3681
+ BgTimer.clearTimeout(timerId);
3682
+ }
3683
+ this.foregroundProbeTimers.clear();
3684
+ LocalMiniappRuntime.instance = null;
3685
+ }
3686
+ /**
3687
+ * Number of currently connected miniapps (for diagnostics).
3688
+ */
3689
+ get connectedAppCount() {
3690
+ return this.connectedApps.size;
3691
+ }
3692
+ }
3693
+ const localMiniappRuntime = LocalMiniappRuntime.getInstance();
3694
+ export default localMiniappRuntime;
3695
+ //# sourceMappingURL=LocalMiniappRuntime.js.map