@mentra/engine 0.1.0-dev.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (787) hide show
  1. package/README.md +79 -0
  2. package/build/devtools.d.ts +11 -0
  3. package/build/devtools.d.ts.map +1 -0
  4. package/build/devtools.js +13 -0
  5. package/build/devtools.js.map +1 -0
  6. package/build/engine.d.ts +261 -0
  7. package/build/engine.d.ts.map +1 -0
  8. package/build/engine.js +170 -0
  9. package/build/engine.js.map +1 -0
  10. package/build/facades/dev.d.ts +57 -0
  11. package/build/facades/dev.d.ts.map +1 -0
  12. package/build/facades/dev.js +187 -0
  13. package/build/facades/dev.js.map +1 -0
  14. package/build/facades/display.d.ts +21 -0
  15. package/build/facades/display.d.ts.map +1 -0
  16. package/build/facades/display.js +17 -0
  17. package/build/facades/display.js.map +1 -0
  18. package/build/facades/displayMirror.d.ts +15 -0
  19. package/build/facades/displayMirror.d.ts.map +1 -0
  20. package/build/facades/displayMirror.js +29 -0
  21. package/build/facades/displayMirror.js.map +1 -0
  22. package/build/facades/gallery.d.ts +62 -0
  23. package/build/facades/gallery.d.ts.map +1 -0
  24. package/build/facades/gallery.js +61 -0
  25. package/build/facades/gallery.js.map +1 -0
  26. package/build/facades/glasses.d.ts +141 -0
  27. package/build/facades/glasses.d.ts.map +1 -0
  28. package/build/facades/glasses.js +199 -0
  29. package/build/facades/glasses.js.map +1 -0
  30. package/build/facades/glassesSettings.d.ts +13 -0
  31. package/build/facades/glassesSettings.d.ts.map +1 -0
  32. package/build/facades/glassesSettings.js +54 -0
  33. package/build/facades/glassesSettings.js.map +1 -0
  34. package/build/facades/glassesWifi.d.ts +25 -0
  35. package/build/facades/glassesWifi.d.ts.map +1 -0
  36. package/build/facades/glassesWifi.js +57 -0
  37. package/build/facades/glassesWifi.js.map +1 -0
  38. package/build/facades/miniapps.d.ts +43 -0
  39. package/build/facades/miniapps.d.ts.map +1 -0
  40. package/build/facades/miniapps.js +49 -0
  41. package/build/facades/miniapps.js.map +1 -0
  42. package/build/facades/notifications.d.ts +13 -0
  43. package/build/facades/notifications.d.ts.map +1 -0
  44. package/build/facades/notifications.js +12 -0
  45. package/build/facades/notifications.js.map +1 -0
  46. package/build/facades/ota.d.ts +103 -0
  47. package/build/facades/ota.d.ts.map +1 -0
  48. package/build/facades/ota.js +156 -0
  49. package/build/facades/ota.js.map +1 -0
  50. package/build/facades/pairing.d.ts +114 -0
  51. package/build/facades/pairing.d.ts.map +1 -0
  52. package/build/facades/pairing.js +275 -0
  53. package/build/facades/pairing.js.map +1 -0
  54. package/build/facades/permissions.d.ts +23 -0
  55. package/build/facades/permissions.d.ts.map +1 -0
  56. package/build/facades/permissions.js +145 -0
  57. package/build/facades/permissions.js.map +1 -0
  58. package/build/facades/phoneNotifications.d.ts +21 -0
  59. package/build/facades/phoneNotifications.d.ts.map +1 -0
  60. package/build/facades/phoneNotifications.js +51 -0
  61. package/build/facades/phoneNotifications.js.map +1 -0
  62. package/build/facades/reports.d.ts +36 -0
  63. package/build/facades/reports.d.ts.map +1 -0
  64. package/build/facades/reports.js +134 -0
  65. package/build/facades/reports.js.map +1 -0
  66. package/build/facades/session.d.ts +21 -0
  67. package/build/facades/session.d.ts.map +1 -0
  68. package/build/facades/session.js +52 -0
  69. package/build/facades/session.js.map +1 -0
  70. package/build/facades/settings.d.ts +37 -0
  71. package/build/facades/settings.d.ts.map +1 -0
  72. package/build/facades/settings.js +55 -0
  73. package/build/facades/settings.js.map +1 -0
  74. package/build/facades/speech.d.ts +30 -0
  75. package/build/facades/speech.d.ts.map +1 -0
  76. package/build/facades/speech.js +37 -0
  77. package/build/facades/speech.js.map +1 -0
  78. package/build/index.d.ts +43 -0
  79. package/build/index.d.ts.map +1 -0
  80. package/build/index.js +45 -0
  81. package/build/index.js.map +1 -0
  82. package/build/internal.d.ts +66 -0
  83. package/build/internal.d.ts.map +1 -0
  84. package/build/internal.js +89 -0
  85. package/build/internal.js.map +1 -0
  86. package/build/runtime/bootstrap.d.ts +83 -0
  87. package/build/runtime/bootstrap.d.ts.map +1 -0
  88. package/build/runtime/bootstrap.js +56 -0
  89. package/build/runtime/bootstrap.js.map +1 -0
  90. package/build/runtime/config.d.ts +146 -0
  91. package/build/runtime/config.d.ts.map +1 -0
  92. package/build/runtime/config.js +18 -0
  93. package/build/runtime/config.js.map +1 -0
  94. package/build/runtime/streamConfig.d.ts +4 -0
  95. package/build/runtime/streamConfig.d.ts.map +1 -0
  96. package/build/runtime/streamConfig.js +38 -0
  97. package/build/runtime/streamConfig.js.map +1 -0
  98. package/build/services/AppRegistry.d.ts +303 -0
  99. package/build/services/AppRegistry.d.ts.map +1 -0
  100. package/build/services/AppRegistry.js +1029 -0
  101. package/build/services/AppRegistry.js.map +1 -0
  102. package/build/services/AudioCloudUplink.d.ts +3 -0
  103. package/build/services/AudioCloudUplink.d.ts.map +1 -0
  104. package/build/services/AudioCloudUplink.js +33 -0
  105. package/build/services/AudioCloudUplink.js.map +1 -0
  106. package/build/services/AudioPlaybackService.d.ts +144 -0
  107. package/build/services/AudioPlaybackService.d.ts.map +1 -0
  108. package/build/services/AudioPlaybackService.js +533 -0
  109. package/build/services/AudioPlaybackService.js.map +1 -0
  110. package/build/services/AutomaticReportResult.d.ts +20 -0
  111. package/build/services/AutomaticReportResult.d.ts.map +1 -0
  112. package/build/services/AutomaticReportResult.js +29 -0
  113. package/build/services/AutomaticReportResult.js.map +1 -0
  114. package/build/services/BlobStore.d.ts +91 -0
  115. package/build/services/BlobStore.d.ts.map +1 -0
  116. package/build/services/BlobStore.js +668 -0
  117. package/build/services/BlobStore.js.map +1 -0
  118. package/build/services/CaptionsTesterReportService.d.ts +4 -0
  119. package/build/services/CaptionsTesterReportService.d.ts.map +1 -0
  120. package/build/services/CaptionsTesterReportService.js +80 -0
  121. package/build/services/CaptionsTesterReportService.js.map +1 -0
  122. package/build/services/CloudAudioSubscriptionSync.d.ts +24 -0
  123. package/build/services/CloudAudioSubscriptionSync.d.ts.map +1 -0
  124. package/build/services/CloudAudioSubscriptionSync.js +39 -0
  125. package/build/services/CloudAudioSubscriptionSync.js.map +1 -0
  126. package/build/services/CloudClientService.d.ts +143 -0
  127. package/build/services/CloudClientService.d.ts.map +1 -0
  128. package/build/services/CloudClientService.js +627 -0
  129. package/build/services/CloudClientService.js.map +1 -0
  130. package/build/services/CloudTranscriptE2EMetrics.d.ts +4 -0
  131. package/build/services/CloudTranscriptE2EMetrics.d.ts.map +1 -0
  132. package/build/services/CloudTranscriptE2EMetrics.js +24 -0
  133. package/build/services/CloudTranscriptE2EMetrics.js.map +1 -0
  134. package/build/services/ConnectionCoordinator.d.ts +51 -0
  135. package/build/services/ConnectionCoordinator.d.ts.map +1 -0
  136. package/build/services/ConnectionCoordinator.js +34 -0
  137. package/build/services/ConnectionCoordinator.js.map +1 -0
  138. package/build/services/DevServerBridge.d.ts +63 -0
  139. package/build/services/DevServerBridge.d.ts.map +1 -0
  140. package/build/services/DevServerBridge.js +294 -0
  141. package/build/services/DevServerBridge.js.map +1 -0
  142. package/build/services/DeviceEventRouter.d.ts +3 -0
  143. package/build/services/DeviceEventRouter.d.ts.map +1 -0
  144. package/build/services/DeviceEventRouter.js +190 -0
  145. package/build/services/DeviceEventRouter.js.map +1 -0
  146. package/build/services/DeviceStoreHydration.d.ts +38 -0
  147. package/build/services/DeviceStoreHydration.d.ts.map +1 -0
  148. package/build/services/DeviceStoreHydration.js +124 -0
  149. package/build/services/DeviceStoreHydration.js.map +1 -0
  150. package/build/services/DisplayProcessor.d.ts +223 -0
  151. package/build/services/DisplayProcessor.d.ts.map +1 -0
  152. package/build/services/DisplayProcessor.js +552 -0
  153. package/build/services/DisplayProcessor.js.map +1 -0
  154. package/build/services/GlassesReadiness.d.ts +41 -0
  155. package/build/services/GlassesReadiness.d.ts.map +1 -0
  156. package/build/services/GlassesReadiness.js +82 -0
  157. package/build/services/GlassesReadiness.js.map +1 -0
  158. package/build/services/GlassesSettingsSync.d.ts +33 -0
  159. package/build/services/GlassesSettingsSync.d.ts.map +1 -0
  160. package/build/services/GlassesSettingsSync.js +121 -0
  161. package/build/services/GlassesSettingsSync.js.map +1 -0
  162. package/build/services/GlassesStatusProjection.d.ts +3 -0
  163. package/build/services/GlassesStatusProjection.d.ts.map +1 -0
  164. package/build/services/GlassesStatusProjection.js +65 -0
  165. package/build/services/GlassesStatusProjection.js.map +1 -0
  166. package/build/services/HeadingService.d.ts +27 -0
  167. package/build/services/HeadingService.d.ts.map +1 -0
  168. package/build/services/HeadingService.js +96 -0
  169. package/build/services/HeadingService.js.map +1 -0
  170. package/build/services/LocalDisplayManager.d.ts +139 -0
  171. package/build/services/LocalDisplayManager.d.ts.map +1 -0
  172. package/build/services/LocalDisplayManager.js +579 -0
  173. package/build/services/LocalDisplayManager.js.map +1 -0
  174. package/build/services/LocalMiniappRuntime.d.ts +433 -0
  175. package/build/services/LocalMiniappRuntime.d.ts.map +1 -0
  176. package/build/services/LocalMiniappRuntime.js +3561 -0
  177. package/build/services/LocalMiniappRuntime.js.map +1 -0
  178. package/build/services/LocalSttFallbackCoordinator.d.ts +54 -0
  179. package/build/services/LocalSttFallbackCoordinator.d.ts.map +1 -0
  180. package/build/services/LocalSttFallbackCoordinator.js +131 -0
  181. package/build/services/LocalSttFallbackCoordinator.js.map +1 -0
  182. package/build/services/MentraJSCrashController.d.ts +97 -0
  183. package/build/services/MentraJSCrashController.d.ts.map +1 -0
  184. package/build/services/MentraJSCrashController.js +138 -0
  185. package/build/services/MentraJSCrashController.js.map +1 -0
  186. package/build/services/MentraJSCrashloopReportService.d.ts +9 -0
  187. package/build/services/MentraJSCrashloopReportService.d.ts.map +1 -0
  188. package/build/services/MentraJSCrashloopReportService.js +54 -0
  189. package/build/services/MentraJSCrashloopReportService.js.map +1 -0
  190. package/build/services/MentraJSLogPipeline.d.ts +87 -0
  191. package/build/services/MentraJSLogPipeline.d.ts.map +1 -0
  192. package/build/services/MentraJSLogPipeline.js +137 -0
  193. package/build/services/MentraJSLogPipeline.js.map +1 -0
  194. package/build/services/MentraJSRouter.d.ts +203 -0
  195. package/build/services/MentraJSRouter.d.ts.map +1 -0
  196. package/build/services/MentraJSRouter.js +480 -0
  197. package/build/services/MentraJSRouter.js.map +1 -0
  198. package/build/services/MentraUIRouter.d.ts +137 -0
  199. package/build/services/MentraUIRouter.d.ts.map +1 -0
  200. package/build/services/MentraUIRouter.js +210 -0
  201. package/build/services/MentraUIRouter.js.map +1 -0
  202. package/build/services/MicStateCoordinator.d.ts +37 -0
  203. package/build/services/MicStateCoordinator.d.ts.map +1 -0
  204. package/build/services/MicStateCoordinator.js +81 -0
  205. package/build/services/MicStateCoordinator.js.map +1 -0
  206. package/build/services/MiniappEngine.d.ts +31 -0
  207. package/build/services/MiniappEngine.d.ts.map +1 -0
  208. package/build/services/MiniappEngine.js +119 -0
  209. package/build/services/MiniappEngine.js.map +1 -0
  210. package/build/services/MiniappLauncher.d.ts +129 -0
  211. package/build/services/MiniappLauncher.d.ts.map +1 -0
  212. package/build/services/MiniappLauncher.js +291 -0
  213. package/build/services/MiniappLauncher.js.map +1 -0
  214. package/build/services/MiniappRunningRegistry.d.ts +24 -0
  215. package/build/services/MiniappRunningRegistry.d.ts.map +1 -0
  216. package/build/services/MiniappRunningRegistry.js +52 -0
  217. package/build/services/MiniappRunningRegistry.js.map +1 -0
  218. package/build/services/NavigationHandlers.d.ts +90 -0
  219. package/build/services/NavigationHandlers.d.ts.map +1 -0
  220. package/build/services/NavigationHandlers.js +379 -0
  221. package/build/services/NavigationHandlers.js.map +1 -0
  222. package/build/services/NavigationService.d.ts +254 -0
  223. package/build/services/NavigationService.d.ts.map +1 -0
  224. package/build/services/NavigationService.js +553 -0
  225. package/build/services/NavigationService.js.map +1 -0
  226. package/build/services/NotificationsEmitter.d.ts +28 -0
  227. package/build/services/NotificationsEmitter.d.ts.map +1 -0
  228. package/build/services/NotificationsEmitter.js +22 -0
  229. package/build/services/NotificationsEmitter.js.map +1 -0
  230. package/build/services/OfflineSpeechModelService.d.ts +33 -0
  231. package/build/services/OfflineSpeechModelService.d.ts.map +1 -0
  232. package/build/services/OfflineSpeechModelService.js +136 -0
  233. package/build/services/OfflineSpeechModelService.js.map +1 -0
  234. package/build/services/OtaInstallCoordinator.d.ts +195 -0
  235. package/build/services/OtaInstallCoordinator.d.ts.map +1 -0
  236. package/build/services/OtaInstallCoordinator.js +1047 -0
  237. package/build/services/OtaInstallCoordinator.js.map +1 -0
  238. package/build/services/OtaService.d.ts +3 -0
  239. package/build/services/OtaService.d.ts.map +1 -0
  240. package/build/services/OtaService.js +65 -0
  241. package/build/services/OtaService.js.map +1 -0
  242. package/build/services/OtaUpdateCheckService.d.ts +65 -0
  243. package/build/services/OtaUpdateCheckService.d.ts.map +1 -0
  244. package/build/services/OtaUpdateCheckService.js +260 -0
  245. package/build/services/OtaUpdateCheckService.js.map +1 -0
  246. package/build/services/PairingIdentity.d.ts +80 -0
  247. package/build/services/PairingIdentity.d.ts.map +1 -0
  248. package/build/services/PairingIdentity.js +145 -0
  249. package/build/services/PairingIdentity.js.map +1 -0
  250. package/build/services/PhoneLocationService.d.ts +36 -0
  251. package/build/services/PhoneLocationService.d.ts.map +1 -0
  252. package/build/services/PhoneLocationService.js +115 -0
  253. package/build/services/PhoneLocationService.js.map +1 -0
  254. package/build/services/PhoneNotificationsSync.d.ts +3 -0
  255. package/build/services/PhoneNotificationsSync.d.ts.map +1 -0
  256. package/build/services/PhoneNotificationsSync.js +35 -0
  257. package/build/services/PhoneNotificationsSync.js.map +1 -0
  258. package/build/services/PhonePhotoCoordinator.d.ts +86 -0
  259. package/build/services/PhonePhotoCoordinator.d.ts.map +1 -0
  260. package/build/services/PhonePhotoCoordinator.js +274 -0
  261. package/build/services/PhonePhotoCoordinator.js.map +1 -0
  262. package/build/services/PhoneStreamCoordinator.d.ts +168 -0
  263. package/build/services/PhoneStreamCoordinator.d.ts.map +1 -0
  264. package/build/services/PhoneStreamCoordinator.js +632 -0
  265. package/build/services/PhoneStreamCoordinator.js.map +1 -0
  266. package/build/services/PhoneVideoCoordinator.d.ts +50 -0
  267. package/build/services/PhoneVideoCoordinator.d.ts.map +1 -0
  268. package/build/services/PhoneVideoCoordinator.js +147 -0
  269. package/build/services/PhoneVideoCoordinator.js.map +1 -0
  270. package/build/services/STTModelManager.d.ts +65 -0
  271. package/build/services/STTModelManager.d.ts.map +1 -0
  272. package/build/services/STTModelManager.js +292 -0
  273. package/build/services/STTModelManager.js.map +1 -0
  274. package/build/services/SceneRenderer.d.ts +90 -0
  275. package/build/services/SceneRenderer.d.ts.map +1 -0
  276. package/build/services/SceneRenderer.js +248 -0
  277. package/build/services/SceneRenderer.js.map +1 -0
  278. package/build/services/StreamLifecycleController.d.ts +67 -0
  279. package/build/services/StreamLifecycleController.d.ts.map +1 -0
  280. package/build/services/StreamLifecycleController.js +152 -0
  281. package/build/services/StreamLifecycleController.js.map +1 -0
  282. package/build/services/TTSModelManager.d.ts +90 -0
  283. package/build/services/TTSModelManager.d.ts.map +1 -0
  284. package/build/services/TTSModelManager.js +366 -0
  285. package/build/services/TTSModelManager.js.map +1 -0
  286. package/build/services/WebviewBridge.d.ts +22 -0
  287. package/build/services/WebviewBridge.d.ts.map +1 -0
  288. package/build/services/WebviewBridge.js +47 -0
  289. package/build/services/WebviewBridge.js.map +1 -0
  290. package/build/services/asg/GalleryMediaIntegrityReportService.d.ts +17 -0
  291. package/build/services/asg/GalleryMediaIntegrityReportService.d.ts.map +1 -0
  292. package/build/services/asg/GalleryMediaIntegrityReportService.js +43 -0
  293. package/build/services/asg/GalleryMediaIntegrityReportService.js.map +1 -0
  294. package/build/services/asg/asgCameraApi.d.ts +165 -0
  295. package/build/services/asg/asgCameraApi.d.ts.map +1 -0
  296. package/build/services/asg/asgCameraApi.js +997 -0
  297. package/build/services/asg/asgCameraApi.js.map +1 -0
  298. package/build/services/asg/galleryMediaValidation.d.ts +29 -0
  299. package/build/services/asg/galleryMediaValidation.d.ts.map +1 -0
  300. package/build/services/asg/galleryMediaValidation.js +149 -0
  301. package/build/services/asg/galleryMediaValidation.js.map +1 -0
  302. package/build/services/asg/galleryNotices.d.ts +33 -0
  303. package/build/services/asg/galleryNotices.d.ts.map +1 -0
  304. package/build/services/asg/galleryNotices.js +29 -0
  305. package/build/services/asg/galleryNotices.js.map +1 -0
  306. package/build/services/asg/gallerySettingsService.d.ts +19 -0
  307. package/build/services/asg/gallerySettingsService.d.ts.map +1 -0
  308. package/build/services/asg/gallerySettingsService.js +51 -0
  309. package/build/services/asg/gallerySettingsService.js.map +1 -0
  310. package/build/services/asg/gallerySyncNotifications.d.ts +56 -0
  311. package/build/services/asg/gallerySyncNotifications.d.ts.map +1 -0
  312. package/build/services/asg/gallerySyncNotifications.js +198 -0
  313. package/build/services/asg/gallerySyncNotifications.js.map +1 -0
  314. package/build/services/asg/gallerySyncService.d.ts +137 -0
  315. package/build/services/asg/gallerySyncService.d.ts.map +1 -0
  316. package/build/services/asg/gallerySyncService.js +1839 -0
  317. package/build/services/asg/gallerySyncService.js.map +1 -0
  318. package/build/services/asg/localStorageService.d.ts +128 -0
  319. package/build/services/asg/localStorageService.d.ts.map +1 -0
  320. package/build/services/asg/localStorageService.js +423 -0
  321. package/build/services/asg/localStorageService.js.map +1 -0
  322. package/build/services/asg/mediaProcessingQueue.d.ts +60 -0
  323. package/build/services/asg/mediaProcessingQueue.d.ts.map +1 -0
  324. package/build/services/asg/mediaProcessingQueue.js +320 -0
  325. package/build/services/asg/mediaProcessingQueue.js.map +1 -0
  326. package/build/services/asgOtaVersionUrl.d.ts +2 -0
  327. package/build/services/asgOtaVersionUrl.d.ts.map +1 -0
  328. package/build/services/asgOtaVersionUrl.js +36 -0
  329. package/build/services/asgOtaVersionUrl.js.map +1 -0
  330. package/build/services/blobPaths.d.ts +31 -0
  331. package/build/services/blobPaths.d.ts.map +1 -0
  332. package/build/services/blobPaths.js +86 -0
  333. package/build/services/blobPaths.js.map +1 -0
  334. package/build/services/cloudStreamApi.d.ts +59 -0
  335. package/build/services/cloudStreamApi.d.ts.map +1 -0
  336. package/build/services/cloudStreamApi.js +81 -0
  337. package/build/services/cloudStreamApi.js.map +1 -0
  338. package/build/services/gallerySyncClock.d.ts +17 -0
  339. package/build/services/gallerySyncClock.d.ts.map +1 -0
  340. package/build/services/gallerySyncClock.js +21 -0
  341. package/build/services/gallerySyncClock.js.map +1 -0
  342. package/build/services/glassesClockSync.d.ts +20 -0
  343. package/build/services/glassesClockSync.d.ts.map +1 -0
  344. package/build/services/glassesClockSync.js +97 -0
  345. package/build/services/glassesClockSync.js.map +1 -0
  346. package/build/services/manifestVersionGate.d.ts +28 -0
  347. package/build/services/manifestVersionGate.d.ts.map +1 -0
  348. package/build/services/manifestVersionGate.js +41 -0
  349. package/build/services/manifestVersionGate.js.map +1 -0
  350. package/build/services/mentraUiShim.d.ts +38 -0
  351. package/build/services/mentraUiShim.d.ts.map +1 -0
  352. package/build/services/mentraUiShim.js +360 -0
  353. package/build/services/mentraUiShim.js.map +1 -0
  354. package/build/services/navigation/roadNameResolver.d.ts +67 -0
  355. package/build/services/navigation/roadNameResolver.d.ts.map +1 -0
  356. package/build/services/navigation/roadNameResolver.js +96 -0
  357. package/build/services/navigation/roadNameResolver.js.map +1 -0
  358. package/build/services/navigation/routesApiCodec.d.ts +38 -0
  359. package/build/services/navigation/routesApiCodec.d.ts.map +1 -0
  360. package/build/services/navigation/routesApiCodec.js +68 -0
  361. package/build/services/navigation/routesApiCodec.js.map +1 -0
  362. package/build/services/otaDisplayState.d.ts +42 -0
  363. package/build/services/otaDisplayState.d.ts.map +1 -0
  364. package/build/services/otaDisplayState.js +92 -0
  365. package/build/services/otaDisplayState.js.map +1 -0
  366. package/build/services/otaInstallPolicy.d.ts +81 -0
  367. package/build/services/otaInstallPolicy.d.ts.map +1 -0
  368. package/build/services/otaInstallPolicy.js +96 -0
  369. package/build/services/otaInstallPolicy.js.map +1 -0
  370. package/build/services/otaLegacyMapping.d.ts +22 -0
  371. package/build/services/otaLegacyMapping.d.ts.map +1 -0
  372. package/build/services/otaLegacyMapping.js +56 -0
  373. package/build/services/otaLegacyMapping.js.map +1 -0
  374. package/build/services/slimStreamStatus.d.ts +7 -0
  375. package/build/services/slimStreamStatus.d.ts.map +1 -0
  376. package/build/services/slimStreamStatus.js +26 -0
  377. package/build/services/slimStreamStatus.js.map +1 -0
  378. package/build/stores/apps.d.ts +104 -0
  379. package/build/stores/apps.d.ts.map +1 -0
  380. package/build/stores/apps.js +492 -0
  381. package/build/stores/apps.js.map +1 -0
  382. package/build/stores/bluetoothSettingKeys.d.ts +4 -0
  383. package/build/stores/bluetoothSettingKeys.d.ts.map +1 -0
  384. package/build/stores/bluetoothSettingKeys.js +31 -0
  385. package/build/stores/bluetoothSettingKeys.js.map +1 -0
  386. package/build/stores/cloudClientStatus.d.ts +9 -0
  387. package/build/stores/cloudClientStatus.d.ts.map +1 -0
  388. package/build/stores/cloudClientStatus.js +24 -0
  389. package/build/stores/cloudClientStatus.js.map +1 -0
  390. package/build/stores/connection.d.ts +22 -0
  391. package/build/stores/connection.d.ts.map +1 -0
  392. package/build/stores/connection.js +43 -0
  393. package/build/stores/connection.js.map +1 -0
  394. package/build/stores/core.d.ts +15 -0
  395. package/build/stores/core.d.ts.map +1 -0
  396. package/build/stores/core.js +21 -0
  397. package/build/stores/core.js.map +1 -0
  398. package/build/stores/display.d.ts +24 -0
  399. package/build/stores/display.d.ts.map +1 -0
  400. package/build/stores/display.js +96 -0
  401. package/build/stores/display.js.map +1 -0
  402. package/build/stores/gallerySync.d.ts +101 -0
  403. package/build/stores/gallerySync.d.ts.map +1 -0
  404. package/build/stores/gallerySync.js +217 -0
  405. package/build/stores/gallerySync.js.map +1 -0
  406. package/build/stores/glasses.d.ts +62 -0
  407. package/build/stores/glasses.d.ts.map +1 -0
  408. package/build/stores/glasses.js +203 -0
  409. package/build/stores/glasses.js.map +1 -0
  410. package/build/stores/settings.d.ts +37 -0
  411. package/build/stores/settings.d.ts.map +1 -0
  412. package/build/stores/settings.js +915 -0
  413. package/build/stores/settings.js.map +1 -0
  414. package/build/types/applet.d.ts +96 -0
  415. package/build/types/applet.d.ts.map +1 -0
  416. package/build/types/applet.js +5 -0
  417. package/build/types/applet.js.map +1 -0
  418. package/build/types/asg.d.ts +46 -0
  419. package/build/types/asg.d.ts.map +1 -0
  420. package/build/types/asg.js +2 -0
  421. package/build/types/asg.js.map +1 -0
  422. package/build/types/capabilities/even-realities-g1.d.ts +12 -0
  423. package/build/types/capabilities/even-realities-g1.d.ts.map +1 -0
  424. package/build/types/capabilities/even-realities-g1.js +55 -0
  425. package/build/types/capabilities/even-realities-g1.js.map +1 -0
  426. package/build/types/capabilities/even-realities-g2.d.ts +12 -0
  427. package/build/types/capabilities/even-realities-g2.d.ts.map +1 -0
  428. package/build/types/capabilities/even-realities-g2.js +75 -0
  429. package/build/types/capabilities/even-realities-g2.js.map +1 -0
  430. package/build/types/capabilities/mentra-display.d.ts +12 -0
  431. package/build/types/capabilities/mentra-display.d.ts.map +1 -0
  432. package/build/types/capabilities/mentra-display.js +63 -0
  433. package/build/types/capabilities/mentra-display.js.map +1 -0
  434. package/build/types/capabilities/mentra-live.d.ts +12 -0
  435. package/build/types/capabilities/mentra-live.d.ts.map +1 -0
  436. package/build/types/capabilities/mentra-live.js +92 -0
  437. package/build/types/capabilities/mentra-live.js.map +1 -0
  438. package/build/types/capabilities/none.d.ts +13 -0
  439. package/build/types/capabilities/none.d.ts.map +1 -0
  440. package/build/types/capabilities/none.js +65 -0
  441. package/build/types/capabilities/none.js.map +1 -0
  442. package/build/types/capabilities/simulated-glasses.d.ts +13 -0
  443. package/build/types/capabilities/simulated-glasses.d.ts.map +1 -0
  444. package/build/types/capabilities/simulated-glasses.js +65 -0
  445. package/build/types/capabilities/simulated-glasses.js.map +1 -0
  446. package/build/types/capabilities/vuzix-z100.d.ts +12 -0
  447. package/build/types/capabilities/vuzix-z100.d.ts.map +1 -0
  448. package/build/types/capabilities/vuzix-z100.js +52 -0
  449. package/build/types/capabilities/vuzix-z100.js.map +1 -0
  450. package/build/types/enums.d.ts +40 -0
  451. package/build/types/enums.d.ts.map +1 -0
  452. package/build/types/enums.js +44 -0
  453. package/build/types/enums.js.map +1 -0
  454. package/build/types/hardware.d.ts +160 -0
  455. package/build/types/hardware.d.ts.map +1 -0
  456. package/build/types/hardware.js +36 -0
  457. package/build/types/hardware.js.map +1 -0
  458. package/build/types/index.d.ts +11 -0
  459. package/build/types/index.d.ts.map +1 -0
  460. package/build/types/index.js +10 -0
  461. package/build/types/index.js.map +1 -0
  462. package/build/utils/GlobalEventEmitter.d.ts +10 -0
  463. package/build/utils/GlobalEventEmitter.d.ts.map +1 -0
  464. package/build/utils/GlobalEventEmitter.js +13 -0
  465. package/build/utils/GlobalEventEmitter.js.map +1 -0
  466. package/build/utils/cloudClient/RnUdpAdapter.d.ts +3 -0
  467. package/build/utils/cloudClient/RnUdpAdapter.d.ts.map +1 -0
  468. package/build/utils/cloudClient/RnUdpAdapter.js +41 -0
  469. package/build/utils/cloudClient/RnUdpAdapter.js.map +1 -0
  470. package/build/utils/cloudClient/cloudSecureStore.d.ts +3 -0
  471. package/build/utils/cloudClient/cloudSecureStore.d.ts.map +1 -0
  472. package/build/utils/cloudClient/cloudSecureStore.js +26 -0
  473. package/build/utils/cloudClient/cloudSecureStore.js.map +1 -0
  474. package/build/utils/debouncedPatch.d.ts +6 -0
  475. package/build/utils/debouncedPatch.d.ts.map +1 -0
  476. package/build/utils/debouncedPatch.js +25 -0
  477. package/build/utils/debouncedPatch.js.map +1 -0
  478. package/build/utils/devLogging.d.ts +26 -0
  479. package/build/utils/devLogging.d.ts.map +1 -0
  480. package/build/utils/devLogging.js +116 -0
  481. package/build/utils/devLogging.js.map +1 -0
  482. package/build/utils/devMiniappLaunch.d.ts +44 -0
  483. package/build/utils/devMiniappLaunch.d.ts.map +1 -0
  484. package/build/utils/devMiniappLaunch.js +52 -0
  485. package/build/utils/devMiniappLaunch.js.map +1 -0
  486. package/build/utils/diagnosticContext.d.ts +3 -0
  487. package/build/utils/diagnosticContext.d.ts.map +1 -0
  488. package/build/utils/diagnosticContext.js +119 -0
  489. package/build/utils/diagnosticContext.js.map +1 -0
  490. package/build/utils/display/composer/ColumnComposer.d.ts +131 -0
  491. package/build/utils/display/composer/ColumnComposer.d.ts.map +1 -0
  492. package/build/utils/display/composer/ColumnComposer.js +204 -0
  493. package/build/utils/display/composer/ColumnComposer.js.map +1 -0
  494. package/build/utils/display/composer/index.d.ts +9 -0
  495. package/build/utils/display/composer/index.d.ts.map +1 -0
  496. package/build/utils/display/composer/index.js +9 -0
  497. package/build/utils/display/composer/index.js.map +1 -0
  498. package/build/utils/display/helpers/DisplayHelpers.d.ts +165 -0
  499. package/build/utils/display/helpers/DisplayHelpers.d.ts.map +1 -0
  500. package/build/utils/display/helpers/DisplayHelpers.js +312 -0
  501. package/build/utils/display/helpers/DisplayHelpers.js.map +1 -0
  502. package/build/utils/display/helpers/ScrollView.d.ts +183 -0
  503. package/build/utils/display/helpers/ScrollView.d.ts.map +1 -0
  504. package/build/utils/display/helpers/ScrollView.js +257 -0
  505. package/build/utils/display/helpers/ScrollView.js.map +1 -0
  506. package/build/utils/display/helpers/index.d.ts +11 -0
  507. package/build/utils/display/helpers/index.d.ts.map +1 -0
  508. package/build/utils/display/helpers/index.js +10 -0
  509. package/build/utils/display/helpers/index.js.map +1 -0
  510. package/build/utils/display/index.d.ts +174 -0
  511. package/build/utils/display/index.d.ts.map +1 -0
  512. package/build/utils/display/index.js +185 -0
  513. package/build/utils/display/index.js.map +1 -0
  514. package/build/utils/display/measurer/TextMeasurer.d.ts +160 -0
  515. package/build/utils/display/measurer/TextMeasurer.d.ts.map +1 -0
  516. package/build/utils/display/measurer/TextMeasurer.js +273 -0
  517. package/build/utils/display/measurer/TextMeasurer.js.map +1 -0
  518. package/build/utils/display/measurer/index.d.ts +10 -0
  519. package/build/utils/display/measurer/index.d.ts.map +1 -0
  520. package/build/utils/display/measurer/index.js +11 -0
  521. package/build/utils/display/measurer/index.js.map +1 -0
  522. package/build/utils/display/measurer/script-detection.d.ts +53 -0
  523. package/build/utils/display/measurer/script-detection.d.ts.map +1 -0
  524. package/build/utils/display/measurer/script-detection.js +184 -0
  525. package/build/utils/display/measurer/script-detection.js.map +1 -0
  526. package/build/utils/display/profiles/g1.d.ts +33 -0
  527. package/build/utils/display/profiles/g1.d.ts.map +1 -0
  528. package/build/utils/display/profiles/g1.js +213 -0
  529. package/build/utils/display/profiles/g1.js.map +1 -0
  530. package/build/utils/display/profiles/g2.d.ts +13 -0
  531. package/build/utils/display/profiles/g2.d.ts.map +1 -0
  532. package/build/utils/display/profiles/g2.js +34 -0
  533. package/build/utils/display/profiles/g2.js.map +1 -0
  534. package/build/utils/display/profiles/index.d.ts +12 -0
  535. package/build/utils/display/profiles/index.d.ts.map +1 -0
  536. package/build/utils/display/profiles/index.js +15 -0
  537. package/build/utils/display/profiles/index.js.map +1 -0
  538. package/build/utils/display/profiles/nex.d.ts +10 -0
  539. package/build/utils/display/profiles/nex.d.ts.map +1 -0
  540. package/build/utils/display/profiles/nex.js +150 -0
  541. package/build/utils/display/profiles/nex.js.map +1 -0
  542. package/build/utils/display/profiles/types.d.ts +103 -0
  543. package/build/utils/display/profiles/types.d.ts.map +1 -0
  544. package/build/utils/display/profiles/types.js +2 -0
  545. package/build/utils/display/profiles/types.js.map +1 -0
  546. package/build/utils/display/profiles/z100.d.ts +33 -0
  547. package/build/utils/display/profiles/z100.d.ts.map +1 -0
  548. package/build/utils/display/profiles/z100.js +182 -0
  549. package/build/utils/display/profiles/z100.js.map +1 -0
  550. package/build/utils/display/scene/degrade.d.ts +25 -0
  551. package/build/utils/display/scene/degrade.d.ts.map +1 -0
  552. package/build/utils/display/scene/degrade.js +64 -0
  553. package/build/utils/display/scene/degrade.js.map +1 -0
  554. package/build/utils/display/scene/differ.d.ts +38 -0
  555. package/build/utils/display/scene/differ.d.ts.map +1 -0
  556. package/build/utils/display/scene/differ.js +107 -0
  557. package/build/utils/display/scene/differ.js.map +1 -0
  558. package/build/utils/display/scene/index.d.ts +10 -0
  559. package/build/utils/display/scene/index.d.ts.map +1 -0
  560. package/build/utils/display/scene/index.js +6 -0
  561. package/build/utils/display/scene/index.js.map +1 -0
  562. package/build/utils/display/scene/process.d.ts +35 -0
  563. package/build/utils/display/scene/process.d.ts.map +1 -0
  564. package/build/utils/display/scene/process.js +187 -0
  565. package/build/utils/display/scene/process.js.map +1 -0
  566. package/build/utils/display/scene/store.d.ts +48 -0
  567. package/build/utils/display/scene/store.d.ts.map +1 -0
  568. package/build/utils/display/scene/store.js +0 -0
  569. package/build/utils/display/scene/store.js.map +1 -0
  570. package/build/utils/display/scene/types.d.ts +120 -0
  571. package/build/utils/display/scene/types.d.ts.map +1 -0
  572. package/build/utils/display/scene/types.js +0 -0
  573. package/build/utils/display/scene/types.js.map +1 -0
  574. package/build/utils/display/wrapper/TextWrapper.d.ts +102 -0
  575. package/build/utils/display/wrapper/TextWrapper.d.ts.map +1 -0
  576. package/build/utils/display/wrapper/TextWrapper.js +471 -0
  577. package/build/utils/display/wrapper/TextWrapper.js.map +1 -0
  578. package/build/utils/display/wrapper/index.d.ts +12 -0
  579. package/build/utils/display/wrapper/index.d.ts.map +1 -0
  580. package/build/utils/display/wrapper/index.js +12 -0
  581. package/build/utils/display/wrapper/index.js.map +1 -0
  582. package/build/utils/display/wrapper/types.d.ts +75 -0
  583. package/build/utils/display/wrapper/types.d.ts.map +1 -0
  584. package/build/utils/display/wrapper/types.js +11 -0
  585. package/build/utils/display/wrapper/types.js.map +1 -0
  586. package/build/utils/e2eMetrics.d.ts +6 -0
  587. package/build/utils/e2eMetrics.d.ts.map +1 -0
  588. package/build/utils/e2eMetrics.js +27 -0
  589. package/build/utils/e2eMetrics.js.map +1 -0
  590. package/build/utils/hardware/hardware.d.ts +42 -0
  591. package/build/utils/hardware/hardware.d.ts.map +1 -0
  592. package/build/utils/hardware/hardware.js +127 -0
  593. package/build/utils/hardware/hardware.js.map +1 -0
  594. package/build/utils/hardware/index.d.ts +2 -0
  595. package/build/utils/hardware/index.d.ts.map +1 -0
  596. package/build/utils/hardware/index.js +2 -0
  597. package/build/utils/hardware/index.js.map +1 -0
  598. package/build/utils/miniappGlobals.d.ts +60 -0
  599. package/build/utils/miniappGlobals.d.ts.map +1 -0
  600. package/build/utils/miniappGlobals.js +224 -0
  601. package/build/utils/miniappGlobals.js.map +1 -0
  602. package/build/utils/permissions/MediaLibraryPermissions.d.ts +34 -0
  603. package/build/utils/permissions/MediaLibraryPermissions.d.ts.map +1 -0
  604. package/build/utils/permissions/MediaLibraryPermissions.js +122 -0
  605. package/build/utils/permissions/MediaLibraryPermissions.js.map +1 -0
  606. package/build/utils/permissions/galleryDisplayName.d.ts +8 -0
  607. package/build/utils/permissions/galleryDisplayName.d.ts.map +1 -0
  608. package/build/utils/permissions/galleryDisplayName.js +25 -0
  609. package/build/utils/permissions/galleryDisplayName.js.map +1 -0
  610. package/build/utils/storage/index.d.ts +3 -0
  611. package/build/utils/storage/index.d.ts.map +1 -0
  612. package/build/utils/storage/index.js +3 -0
  613. package/build/utils/storage/index.js.map +1 -0
  614. package/build/utils/storage/storage.d.ts +24 -0
  615. package/build/utils/storage/storage.d.ts.map +1 -0
  616. package/build/utils/storage/storage.js +85 -0
  617. package/build/utils/storage/storage.js.map +1 -0
  618. package/build/utils/storage/zip.d.ts +3 -0
  619. package/build/utils/storage/zip.d.ts.map +1 -0
  620. package/build/utils/storage/zip.js +14 -0
  621. package/build/utils/storage/zip.js.map +1 -0
  622. package/build/utils/timers/index.d.ts +3 -0
  623. package/build/utils/timers/index.d.ts.map +1 -0
  624. package/build/utils/timers/index.js +3 -0
  625. package/build/utils/timers/index.js.map +1 -0
  626. package/build/utils/timers/timers.d.ts +9 -0
  627. package/build/utils/timers/timers.d.ts.map +1 -0
  628. package/build/utils/timers/timers.js +105 -0
  629. package/build/utils/timers/timers.js.map +1 -0
  630. package/package.json +119 -0
  631. package/src/devtools.ts +13 -0
  632. package/src/engine.ts +173 -0
  633. package/src/facades/dev.ts +197 -0
  634. package/src/facades/display.ts +17 -0
  635. package/src/facades/displayMirror.ts +33 -0
  636. package/src/facades/gallery.ts +69 -0
  637. package/src/facades/glasses.ts +206 -0
  638. package/src/facades/glassesSettings.ts +56 -0
  639. package/src/facades/glassesWifi.ts +68 -0
  640. package/src/facades/miniapps.ts +55 -0
  641. package/src/facades/notifications.ts +15 -0
  642. package/src/facades/ota.ts +174 -0
  643. package/src/facades/pairing.ts +329 -0
  644. package/src/facades/permissions.ts +151 -0
  645. package/src/facades/phoneNotifications.ts +50 -0
  646. package/src/facades/reports.ts +185 -0
  647. package/src/facades/session.ts +55 -0
  648. package/src/facades/settings.ts +57 -0
  649. package/src/facades/speech.ts +38 -0
  650. package/src/index.ts +191 -0
  651. package/src/internal.ts +158 -0
  652. package/src/runtime/bootstrap.ts +118 -0
  653. package/src/runtime/config.ts +147 -0
  654. package/src/runtime/streamConfig.ts +34 -0
  655. package/src/services/AppRegistry.ts +1139 -0
  656. package/src/services/AudioCloudUplink.ts +34 -0
  657. package/src/services/AudioPlaybackService.ts +624 -0
  658. package/src/services/AutomaticReportResult.ts +48 -0
  659. package/src/services/BlobStore.ts +746 -0
  660. package/src/services/CaptionsTesterReportService.ts +107 -0
  661. package/src/services/CloudAudioSubscriptionSync.ts +41 -0
  662. package/src/services/CloudClientService.ts +668 -0
  663. package/src/services/CloudTranscriptE2EMetrics.ts +27 -0
  664. package/src/services/ConnectionCoordinator.ts +65 -0
  665. package/src/services/DevServerBridge.ts +332 -0
  666. package/src/services/DeviceEventRouter.ts +238 -0
  667. package/src/services/DeviceStoreHydration.ts +130 -0
  668. package/src/services/DisplayProcessor.ts +775 -0
  669. package/src/services/GlassesReadiness.ts +103 -0
  670. package/src/services/GlassesSettingsSync.ts +134 -0
  671. package/src/services/GlassesStatusProjection.ts +73 -0
  672. package/src/services/HeadingService.ts +98 -0
  673. package/src/services/LocalDisplayManager.ts +703 -0
  674. package/src/services/LocalMiniappRuntime.ts +4038 -0
  675. package/src/services/LocalSttFallbackCoordinator.ts +140 -0
  676. package/src/services/MentraJSCrashController.ts +189 -0
  677. package/src/services/MentraJSCrashloopReportService.ts +66 -0
  678. package/src/services/MentraJSLogPipeline.ts +166 -0
  679. package/src/services/MentraJSRouter.ts +557 -0
  680. package/src/services/MentraUIRouter.ts +244 -0
  681. package/src/services/MicStateCoordinator.ts +94 -0
  682. package/src/services/MiniappEngine.ts +143 -0
  683. package/src/services/MiniappLauncher.ts +340 -0
  684. package/src/services/MiniappRunningRegistry.ts +57 -0
  685. package/src/services/NavigationHandlers.ts +423 -0
  686. package/src/services/NavigationService.ts +731 -0
  687. package/src/services/NotificationsEmitter.ts +48 -0
  688. package/src/services/OfflineSpeechModelService.ts +168 -0
  689. package/src/services/OtaInstallCoordinator.ts +1205 -0
  690. package/src/services/OtaService.ts +83 -0
  691. package/src/services/OtaUpdateCheckService.ts +386 -0
  692. package/src/services/PairingIdentity.ts +154 -0
  693. package/src/services/PhoneLocationService.ts +122 -0
  694. package/src/services/PhoneNotificationsSync.ts +44 -0
  695. package/src/services/PhonePhotoCoordinator.ts +337 -0
  696. package/src/services/PhoneStreamCoordinator.ts +777 -0
  697. package/src/services/PhoneVideoCoordinator.ts +189 -0
  698. package/src/services/STTModelManager.ts +360 -0
  699. package/src/services/SceneRenderer.ts +292 -0
  700. package/src/services/StreamLifecycleController.ts +213 -0
  701. package/src/services/TTSModelManager.ts +466 -0
  702. package/src/services/WebviewBridge.ts +50 -0
  703. package/src/services/asg/GalleryMediaIntegrityReportService.ts +70 -0
  704. package/src/services/asg/asgCameraApi.ts +1204 -0
  705. package/src/services/asg/galleryMediaValidation.ts +183 -0
  706. package/src/services/asg/galleryNotices.ts +60 -0
  707. package/src/services/asg/gallerySettingsService.ts +65 -0
  708. package/src/services/asg/gallerySyncNotifications.ts +236 -0
  709. package/src/services/asg/gallerySyncService.ts +2167 -0
  710. package/src/services/asg/localStorageService.ts +497 -0
  711. package/src/services/asg/mediaProcessingQueue.ts +371 -0
  712. package/src/services/asgOtaVersionUrl.ts +41 -0
  713. package/src/services/blobPaths.ts +86 -0
  714. package/src/services/cloudStreamApi.ts +154 -0
  715. package/src/services/gallerySyncClock.ts +39 -0
  716. package/src/services/glassesClockSync.ts +115 -0
  717. package/src/services/manifestVersionGate.ts +43 -0
  718. package/src/services/mentraUiShim.ts +366 -0
  719. package/src/services/navigation/roadNameResolver.ts +112 -0
  720. package/src/services/navigation/routesApiCodec.ts +69 -0
  721. package/src/services/otaDisplayState.ts +115 -0
  722. package/src/services/otaInstallPolicy.ts +106 -0
  723. package/src/services/otaLegacyMapping.ts +71 -0
  724. package/src/services/slimStreamStatus.ts +28 -0
  725. package/src/stores/apps.ts +614 -0
  726. package/src/stores/bluetoothSettingKeys.ts +36 -0
  727. package/src/stores/cloudClientStatus.ts +37 -0
  728. package/src/stores/connection.ts +69 -0
  729. package/src/stores/core.ts +32 -0
  730. package/src/stores/display.ts +121 -0
  731. package/src/stores/gallerySync.ts +380 -0
  732. package/src/stores/glasses.ts +309 -0
  733. package/src/stores/settings.ts +993 -0
  734. package/src/types/applet.ts +112 -0
  735. package/src/types/asg.ts +51 -0
  736. package/src/types/capabilities/even-realities-g1.ts +67 -0
  737. package/src/types/capabilities/even-realities-g2.ts +88 -0
  738. package/src/types/capabilities/mentra-display.ts +75 -0
  739. package/src/types/capabilities/mentra-live.ts +103 -0
  740. package/src/types/capabilities/none.ts +76 -0
  741. package/src/types/capabilities/simulated-glasses.ts +76 -0
  742. package/src/types/capabilities/vuzix-z100.ts +64 -0
  743. package/src/types/enums.ts +43 -0
  744. package/src/types/hardware.ts +203 -0
  745. package/src/types/index.ts +37 -0
  746. package/src/utils/GlobalEventEmitter.ts +14 -0
  747. package/src/utils/cloudClient/RnUdpAdapter.ts +47 -0
  748. package/src/utils/cloudClient/cloudSecureStore.ts +28 -0
  749. package/src/utils/debouncedPatch.ts +28 -0
  750. package/src/utils/devLogging.ts +140 -0
  751. package/src/utils/devMiniappLaunch.ts +68 -0
  752. package/src/utils/diagnosticContext.ts +155 -0
  753. package/src/utils/display/composer/ColumnComposer.ts +285 -0
  754. package/src/utils/display/composer/index.ts +15 -0
  755. package/src/utils/display/helpers/DisplayHelpers.ts +413 -0
  756. package/src/utils/display/helpers/ScrollView.ts +337 -0
  757. package/src/utils/display/helpers/index.ts +14 -0
  758. package/src/utils/display/index.ts +273 -0
  759. package/src/utils/display/measurer/TextMeasurer.ts +346 -0
  760. package/src/utils/display/measurer/index.ts +21 -0
  761. package/src/utils/display/measurer/script-detection.ts +218 -0
  762. package/src/utils/display/profiles/g1.ts +236 -0
  763. package/src/utils/display/profiles/g2.ts +39 -0
  764. package/src/utils/display/profiles/index.ts +28 -0
  765. package/src/utils/display/profiles/nex.ts +166 -0
  766. package/src/utils/display/profiles/types.ts +132 -0
  767. package/src/utils/display/profiles/z100.ts +198 -0
  768. package/src/utils/display/scene/degrade.ts +80 -0
  769. package/src/utils/display/scene/differ.ts +128 -0
  770. package/src/utils/display/scene/index.ts +21 -0
  771. package/src/utils/display/scene/process.ts +214 -0
  772. package/src/utils/display/scene/store.ts +0 -0
  773. package/src/utils/display/scene/types.ts +0 -0
  774. package/src/utils/display/wrapper/TextWrapper.ts +546 -0
  775. package/src/utils/display/wrapper/index.ts +15 -0
  776. package/src/utils/display/wrapper/types.ts +102 -0
  777. package/src/utils/e2eMetrics.ts +30 -0
  778. package/src/utils/hardware/hardware.ts +166 -0
  779. package/src/utils/hardware/index.ts +1 -0
  780. package/src/utils/miniappGlobals.ts +260 -0
  781. package/src/utils/permissions/MediaLibraryPermissions.ts +130 -0
  782. package/src/utils/permissions/galleryDisplayName.ts +28 -0
  783. package/src/utils/storage/index.ts +2 -0
  784. package/src/utils/storage/storage.ts +101 -0
  785. package/src/utils/storage/zip.ts +13 -0
  786. package/src/utils/timers/index.ts +2 -0
  787. package/src/utils/timers/timers.ts +139 -0
@@ -0,0 +1,4038 @@
1
+ /**
2
+ * LocalMiniappRuntime
3
+ *
4
+ * Singleton service that bridges local miniapps (running in WebViews) with
5
+ * phone capabilities: display, audio, storage, sensors, LED, etc.
6
+ *
7
+ * Each miniapp communicates via the @mentra/miniapp envelope protocol.
8
+ * This runtime handles request dispatch, stream fan-out, and lifecycle
9
+ * management (connect/disconnect/ping).
10
+ */
11
+
12
+ import {Linking} from "react-native"
13
+ import Share from "react-native-share"
14
+ import * as Battery from "expo-battery"
15
+ import * as Clipboard from "expo-clipboard"
16
+ import {File, Paths} from "expo-file-system"
17
+ import * as Location from "expo-location"
18
+ import BluetoothSdk, {type RgbLedAction, type RgbLedColor} from "@mentra/bluetooth-sdk"
19
+
20
+ import {
21
+ MiniappErrorCode,
22
+ MiniappRequestType,
23
+ MiniappResponseType,
24
+ MiniappStreamType,
25
+ CLOUD_STATUS_STREAM,
26
+ parseEnvelope,
27
+ serializeEnvelope,
28
+ } from "@mentra/miniapp"
29
+ import type {MiniappEnvelope} from "@mentra/miniapp"
30
+
31
+ import {DeviceTypes, getModelCapabilities} from "../types"
32
+ import {storage as mmkvStorage} from "../utils/storage/storage"
33
+ import {BgTimer} from "../utils/timers"
34
+ import devServerBridge from "./DevServerBridge"
35
+ import {islandNotifications} from "./NotificationsEmitter"
36
+ import {isGlassesConnected} from "./GlassesReadiness"
37
+ import audioPlaybackService from "./AudioPlaybackService"
38
+ import {phoneLocationService} from "./PhoneLocationService"
39
+ import {useGlassesStore} from "../stores/glasses"
40
+ import {useSettingsStore} from "../stores/settings"
41
+ import localDisplayManager from "./LocalDisplayManager"
42
+ import type {DisplayPayload} from "./LocalDisplayManager"
43
+ import headingService from "./HeadingService"
44
+ import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator"
45
+ import micStateCoordinator from "./MicStateCoordinator"
46
+ import {BlobStore} from "./BlobStore"
47
+ import {CloudAudioSubscriptionSync} from "./CloudAudioSubscriptionSync"
48
+ import {phonePhotoCoordinator} from "./PhonePhotoCoordinator"
49
+ import {phoneStreamCoordinator} from "./PhoneStreamCoordinator"
50
+ import {phoneVideoCoordinator} from "./PhoneVideoCoordinator"
51
+ import {cloudClientService} from "./CloudClientService"
52
+ import {miniappLauncher} from "./MiniappLauncher"
53
+ import {
54
+ ISLAND_SETTINGS_KEYS,
55
+ type CameraFovPreset,
56
+ type CameraFovRequest,
57
+ type CameraRoiPosition,
58
+ type CloudClientStatusSnapshot,
59
+ type InteropAuditEvent,
60
+ type MiniappAuthToken,
61
+ type TtsSynthesisResult,
62
+ } from "../runtime/config"
63
+ import {getAnalytics, getUiSeams} from "../runtime/bootstrap"
64
+ import {normalizeStreamAudioConfig, normalizeStreamVideoConfig} from "../runtime/streamConfig"
65
+ import type {
66
+ AudioSubscription,
67
+ LanguageSource,
68
+ TranscriptionData,
69
+ TranslationData,
70
+ } from "@mentra/cloud-protocol"
71
+ import ttsModelManager from "./TTSModelManager"
72
+ import {NavigationHandlers} from "./NavigationHandlers"
73
+ import type {ClientApp} from "../types/applet"
74
+ import {useAppStatusStore} from "../stores/apps"
75
+ import {DEV_APP_PACKAGE_NAME, getDevAppSourcePackage} from "./AppRegistry"
76
+
77
+ // =============================================================================
78
+ // Types
79
+ // =============================================================================
80
+
81
+ export interface InstalledMiniappManifest {
82
+ /** Human-facing app name (from miniapp.json `name`). Shown in the "Starting
83
+ * <name>…" boot message; falls back to the package name when absent. */
84
+ name?: string
85
+ permissions?: Array<{type: string; required?: boolean; description?: string}>
86
+ hardwareRequirements?: Array<{type: string; level: string; description?: string}>
87
+ }
88
+
89
+ type SpeakerStateValue = "idle" | "loading" | "playing" | "stopped" | "error"
90
+
91
+ interface ConnectedMiniapp {
92
+ subscriptions: Set<string>
93
+ sendMessage: (raw: string) => void
94
+ lastPongAt: number
95
+ /** BGCAP diagnostic: wall-clock of the most recent PING send, for PONG RTT. */
96
+ bgcapLastPingSentAt?: number
97
+ installedManifest?: InstalledMiniappManifest
98
+ authRefreshTimerId: number | null
99
+ /**
100
+ * Backoff timer for re-attempting the INITIAL miniapp-token mint after it
101
+ * failed (distinct from `authRefreshTimerId`, which renews an already-issued
102
+ * token near expiry). Null when no retry is queued.
103
+ */
104
+ authRetryTimerId: number | null
105
+ /**
106
+ * True once this app has received a miniapp auth token (in CONNECT_ACK or a
107
+ * follow-up AUTH_UPDATE). Gates the initial-mint retry loop and the
108
+ * on-reconnect re-drive so we stop hammering once auth has landed.
109
+ */
110
+ authDelivered: boolean
111
+ /** Last speaker state pushed to this app. Used to dedup SPEAKER_STATE pushes. */
112
+ speakerState: SpeakerStateValue
113
+ /** Live PCM output stream this app has open (speaker.createStream), if any. */
114
+ activeSpeakerStreamId?: string
115
+ /**
116
+ * Location-tier rate this app is currently asking for via its
117
+ * `location_stream` subscription, or null if it hasn't asked for
118
+ * one. The aggregate tier handed to the host is the strictest
119
+ * (highest-accuracy) rate across all currently connected apps —
120
+ * see {@link LOCATION_RATE_PRIORITY}.
121
+ */
122
+ requestedLocationRate: string | null
123
+ }
124
+
125
+ /**
126
+ * Strictness ordering for `location_stream` rate values. Higher index =
127
+ * higher accuracy and higher power cost. The aggregate tier applied to
128
+ * the OS is the max of every connected miniapp's requested rate;
129
+ * miniapps that didn't ask for `location_stream` contribute nothing.
130
+ *
131
+ * If a miniapp passes a value we don't recognize, we treat it like the
132
+ * weakest known rate ("passive") rather than blanket-overwriting an
133
+ * active stronger request from another app.
134
+ */
135
+ const LOCATION_RATE_PRIORITY = ["off", "passive", "low", "high", "realtime"] as const
136
+ type LocationRate = (typeof LOCATION_RATE_PRIORITY)[number]
137
+ function locationRateRank(rate: string | null | undefined): number {
138
+ if (!rate) return -1
139
+ const i = LOCATION_RATE_PRIORITY.indexOf(rate as LocationRate)
140
+ return i >= 0 ? i : LOCATION_RATE_PRIORITY.indexOf("passive")
141
+ }
142
+
143
+ function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {
144
+ return new Promise((resolve) => {
145
+ let settled = false
146
+ const done = (value: T | null) => {
147
+ if (settled) return
148
+ settled = true
149
+ BgTimer.clearTimeout(timer)
150
+ resolve(value)
151
+ }
152
+ const timer = BgTimer.setTimeout(() => done(null), timeoutMs)
153
+ promise.then(
154
+ (value) => done(value),
155
+ () => done(null),
156
+ )
157
+ })
158
+ }
159
+
160
+ const LOG_TAG = "LOCAL_MINIAPP"
161
+ const SYSTEM_MINIAPP_PACKAGES = new Set([
162
+ "com.mentra.camera",
163
+ "com.mentra.gallery",
164
+ "com.mentra.settings",
165
+ "com.mentra.simulated",
166
+ "com.mentra.mirror",
167
+ "com.mentra.ai",
168
+ "cloud.augmentos.notify",
169
+ "com.mentra.feedback",
170
+ "com.mentra.miniappdev",
171
+ ])
172
+ const PING_INTERVAL_MS = 5_000
173
+ const MINIAPP_AUTH_REFRESH_HEADROOM_MS = 5 * 60 * 1000
174
+ const MINIAPP_AUTH_REFRESH_MIN_DELAY_MS = 5_000
175
+ // Backoff bounds for re-minting the INITIAL miniapp token after a failed mint
176
+ // (e.g. the cloud client had not finished its first-boot Core token exchange
177
+ // when the miniapp connected). Exponential from base, capped, so a transient
178
+ // not-yet-connected client self-heals within seconds instead of needing a full
179
+ // manual Cloud V2 reconnect.
180
+ const MINIAPP_AUTH_RETRY_BASE_MS = 1_000
181
+ const MINIAPP_AUTH_RETRY_MAX_MS = 15_000
182
+ const FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS = 2_500
183
+ const REQUEST_WIFI_SETUP_TYPE = "miniapp_request_wifi_setup"
184
+ // Unregister after this many missed pongs. Generous on purpose: a busy
185
+ // context (heavy interim translation traffic) or OS scheduling while idle can
186
+ // delay pongs well past one interval, and killing a healthy-but-busy script
187
+ // drops its subscriptions (releasing the mic). With the liveness-timeout
188
+ // respawn path wired (MentraJSRouter.start), a genuinely dead context still
189
+ // comes back automatically — this threshold only bounds how long that takes.
190
+ const PING_TIMEOUT_THRESHOLD = 6 // ~30s
191
+
192
+ // ===========================================================================
193
+ // BGCAP: temporary background-caption telemetry (DIAGNOSTIC — remove after the
194
+ // "captions fall behind then flood in waves" investigation lands a fix).
195
+ //
196
+ // Captions now run as a local miniapp inside the per-context JS engine
197
+ // (Crust: QuickJS on Android / JSC on iOS), driven off a single serial
198
+ // queue/executor with NO background awareness. The hypothesis is that the OS
199
+ // starves/freezes that queue when the screen is off (iOS app suspension /
200
+ // Android Doze), so inbound transcript deliveries pile up and drain in a burst
201
+ // on resume. These logs measure exactly that. All lines are prefixed "BGCAP:"
202
+ // so they're trivial to grep in incident logs and to delete. Cross-platform —
203
+ // host-side only, no native changes.
204
+ // ===========================================================================
205
+ const BGCAP_TELEMETRY = false
206
+ const TRANSCRIPT_TIMING_TELEMETRY = (globalThis as {__DEV__?: boolean}).__DEV__ === true
207
+ let bgcapHbTimer: ReturnType<typeof setInterval> | null = null
208
+ let bgcapHbLast = 0
209
+ let bgcapTxCount = 0
210
+ let bgcapTxLastFinalAt = 0
211
+ let bgcapTxLastLogAt = 0
212
+
213
+ function bgcapStartHeartbeat(): void {
214
+ if (!BGCAP_TELEMETRY || bgcapHbTimer !== null) return
215
+ bgcapHbLast = Date.now()
216
+ // PLAIN setInterval (NOT BgTimer): a plain JS timer only fires when the RN
217
+ // JS thread is actually executing. A gap >> 1s between ticks therefore proves
218
+ // the JS thread was suspended/throttled (iOS suspend / Android Doze) for that
219
+ // long — the smoking gun for the freeze. Silent when healthy; logs the stall
220
+ // duration when it happens. Pair with the "App state changed to:" lines.
221
+ bgcapHbTimer = setInterval(() => {
222
+ const now = Date.now()
223
+ const gap = now - bgcapHbLast
224
+ bgcapHbLast = now
225
+ if (gap > 2000) {
226
+ console.log(`BGCAP: host-js stall — no tick for ${gap}ms (RN JS thread frozen/throttled)`)
227
+ }
228
+ }, 1000)
229
+ }
230
+
231
+ function bgcapStopHeartbeat(): void {
232
+ if (bgcapHbTimer !== null) {
233
+ clearInterval(bgcapHbTimer)
234
+ bgcapHbTimer = null
235
+ }
236
+ }
237
+
238
+ // Rate-limited (1/s) summary of transcripts being FORWARDED to miniapps. Shows
239
+ // whether transcripts keep arriving + being handed to the captions miniapp
240
+ // during background. Compare its rate against the native "MAN: displayEvent"
241
+ // (render) rate: fwd keeps flowing but render lags => miniapp executor is
242
+ // starved; fwd itself stops => upstream (cloud delivery / RN thread) is the gap.
243
+ function bgcapNoteTranscriptForward(isFinal: boolean): void {
244
+ if (!BGCAP_TELEMETRY) return
245
+ bgcapTxCount++
246
+ if (isFinal) bgcapTxLastFinalAt = Date.now()
247
+ const now = Date.now()
248
+ if (now - bgcapTxLastLogAt >= 1000) {
249
+ console.log(
250
+ `BGCAP: tx forwarded=${bgcapTxCount} in last ${now - bgcapTxLastLogAt}ms (lastFinal ${
251
+ bgcapTxLastFinalAt ? now - bgcapTxLastFinalAt : -1
252
+ }ms ago)`,
253
+ )
254
+ bgcapTxCount = 0
255
+ bgcapTxLastLogAt = now
256
+ }
257
+ }
258
+
259
+ const RGB_LED_ACTIONS = new Set<RgbLedAction>(["on", "off"])
260
+ const RGB_LED_COLORS = new Set<RgbLedColor>(["red", "green", "blue", "orange", "white"])
261
+ const CAMERA_FOV_MIN = 62
262
+ const CAMERA_FOV_MAX = 118
263
+ const CAMERA_FOV_DEFAULT = 102
264
+ const CAMERA_FOV_PRESETS: Record<CameraFovPreset, number> = {narrow: 82, standard: CAMERA_FOV_DEFAULT, wide: 118}
265
+ const CAMERA_ROI_POSITION_BY_NAME: Record<string, CameraRoiPosition> = {center: "center", bottom: "bottom", top: "top"}
266
+ const CAMERA_ROI_POSITION_VALUES: Record<CameraRoiPosition, 0 | 1 | 2> = {center: 0, bottom: 1, top: 2}
267
+
268
+ // =============================================================================
269
+ // Declared-permission record helper (for CONNECT_ACK / PERMISSIONS_UPDATE)
270
+ // =============================================================================
271
+
272
+ /**
273
+ * Map from manifest permission types (uppercase, snake-y) to the lowercase
274
+ * canonical permission keys the SDK exposes via session.permissions. Mirrors
275
+ * cloud SDK v3's PermissionType union.
276
+ *
277
+ * BACKGROUND_LOCATION + POST_NOTIFICATIONS aren't in v3's surface; they map
278
+ * onto the same canonical keys (location / notifications) since the SDK's
279
+ * `has()` is "do I have *any* form of this permission declared".
280
+ */
281
+ const PERMISSION_TYPE_TO_CANONICAL: Record<string, string> = {
282
+ MICROPHONE: "microphone",
283
+ CAMERA: "camera",
284
+ LOCATION: "location",
285
+ BACKGROUND_LOCATION: "location",
286
+ READ_NOTIFICATIONS: "notifications",
287
+ POST_NOTIFICATIONS: "notifications",
288
+ CALENDAR: "calendar",
289
+ }
290
+
291
+ function normalizeRgbLedAction(value: unknown): RgbLedAction {
292
+ return typeof value === "string" && RGB_LED_ACTIONS.has(value as RgbLedAction) ? (value as RgbLedAction) : "off"
293
+ }
294
+
295
+ function normalizeRgbLedColor(value: unknown): RgbLedColor | null {
296
+ return typeof value === "string" && RGB_LED_COLORS.has(value as RgbLedColor) ? (value as RgbLedColor) : null
297
+ }
298
+
299
+ function normalizeCameraFovPayload(payload: Record<string, unknown>): CameraFovRequest {
300
+ if (typeof payload.preset === "string") {
301
+ const preset = payload.preset in CAMERA_FOV_PRESETS ? (payload.preset as CameraFovPreset) : "standard"
302
+ return {preset}
303
+ }
304
+
305
+ const rawFov = typeof payload.fov === "number" && Number.isFinite(payload.fov) ? payload.fov : CAMERA_FOV_DEFAULT
306
+
307
+ return {
308
+ fov: Math.min(CAMERA_FOV_MAX, Math.max(CAMERA_FOV_MIN, rawFov)),
309
+ roiPosition: normalizeCameraRoiPosition(payload.roiPosition ?? payload.roi_position),
310
+ }
311
+ }
312
+
313
+ function normalizeCameraRoiPosition(value: unknown): CameraRoiPosition {
314
+ if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 2) {
315
+ return value === 1 ? "bottom" : value === 2 ? "top" : "center"
316
+ }
317
+
318
+ if (typeof value === "string") {
319
+ return CAMERA_ROI_POSITION_BY_NAME[value] ?? "center"
320
+ }
321
+
322
+ return "center"
323
+ }
324
+
325
+ const ALL_CANONICAL_PERMISSIONS = ["location", "microphone", "camera", "notifications", "calendar"] as const
326
+
327
+ /**
328
+ * Build the {location, microphone, camera, notifications, calendar} record the
329
+ * SDK's session.permissions module reads from. Missing types are false.
330
+ */
331
+ function computeDeclaredPermissionRecord(manifest: InstalledMiniappManifest | undefined): Record<string, boolean> {
332
+ const record: Record<string, boolean> = {}
333
+ for (const k of ALL_CANONICAL_PERMISSIONS) record[k] = false
334
+ for (const perm of manifest?.permissions ?? []) {
335
+ const canonical = PERMISSION_TYPE_TO_CANONICAL[perm.type?.toUpperCase()]
336
+ if (canonical) record[canonical] = true
337
+ }
338
+ return record
339
+ }
340
+
341
+ // =============================================================================
342
+ // PERMISSION_NOT_DECLARED warnings — per-session dedup
343
+ // =============================================================================
344
+ //
345
+ // When a miniapp tries to subscribe / call something whose required permission
346
+ // isn't declared in miniapp.json, the runtime rejects with
347
+ // PERMISSION_NOT_DECLARED. The error reaches the SDK but most authors don't
348
+ // subscribe to session.on("error", ...), so it's silent in practice.
349
+ //
350
+ // To make the failure discoverable for developers running the MentraOS app
351
+ // from source, log a clear, copy-pasteable message in the phone console that
352
+ // names the offending permission, the offending stream/op, and the JSON
353
+ // snippet to add to miniapp.json. Once-per-session per (packageName, permission)
354
+ // to avoid spam from a tight retry loop.
355
+ //
356
+ // Production users running the App Store build of MentraOS won't see these
357
+ // (they don't watch Metro/adb logcat). For them, the WebView console bridge
358
+ // (#5 of the quick-fixes round) ships the structured error to the miniapp
359
+ // itself, and the miniapp's own console.warn flows to the dev terminal.
360
+
361
+ const warnedPermission = new Set<string>() // key: `${packageName}::${permission}`
362
+
363
+ function logPermissionNotDeclared(
364
+ packageName: string,
365
+ permission: string,
366
+ context: string,
367
+ manifestSnippet: string,
368
+ ): void {
369
+ const key = `${packageName}::${permission}`
370
+ if (warnedPermission.has(key)) return
371
+ warnedPermission.add(key)
372
+ console.warn(
373
+ `${LOG_TAG}: ${packageName} attempted ${context}, but permission ${permission} is not declared in miniapp.json.\n` +
374
+ `Add this to the "permissions" array:\n ${manifestSnippet}`,
375
+ )
376
+ }
377
+
378
+ /** Reset the per-session dedup; called when a miniapp unregisters so a fresh launch warns again. */
379
+ function resetPermissionWarnings(packageName: string): void {
380
+ for (const key of warnedPermission) {
381
+ if (key.startsWith(`${packageName}::`)) warnedPermission.delete(key)
382
+ }
383
+ }
384
+
385
+ // =============================================================================
386
+ // LocalMiniappRuntime
387
+ // =============================================================================
388
+
389
+ class LocalMiniappRuntime {
390
+ private static instance: LocalMiniappRuntime | null = null
391
+
392
+ /** Connected miniapps keyed by packageName. */
393
+ private connectedApps: Map<string, ConnectedMiniapp> = new Map()
394
+
395
+ /**
396
+ * Packages that have completed their CONNECT handshake (sent
397
+ * `miniapp_connect`, got `CONNECT_ACK`). Distinct from `connectedApps`,
398
+ * which is populated at spawn/registration time — handshake is later.
399
+ * Drives {@link waitForConnect}.
400
+ */
401
+ private handshookApps: Set<string> = new Set()
402
+
403
+ /** Pending {@link waitForConnect} resolvers, keyed by packageName. */
404
+ private connectWaiters: Map<string, Set<(err?: Error) => void>> = new Map()
405
+
406
+ /** Ref-counted stream subscriptions: stream → set of packageNames. */
407
+ private streamSubscribers: Map<string, Set<string>> = new Map()
408
+
409
+ /** Guards one-time wiring of the cloud transcript/translation fan-out. */
410
+ private cloudResultsWired = false
411
+ private cloudStatusWired = false
412
+ private cloudAudioSubscriptionSync = new CloudAudioSubscriptionSync()
413
+
414
+ /** Ping interval handle. */
415
+ private pingIntervalId: number | null = null
416
+ private foregroundProbeTimers: Map<string, number> = new Map()
417
+
418
+ /**
419
+ * Notified when a miniapp is unregistered for missing liveness pings.
420
+ * MentraJSRouter wires this into its crash-respawn machinery so a
421
+ * silently-stalled background script is restarted (with crash-loop
422
+ * protection) instead of staying dead — a dead background drops its
423
+ * subscriptions, which releases the mic while the webview still looks
424
+ * alive.
425
+ */
426
+ public onLivenessTimeout: ((packageName: string) => void) | null = null
427
+
428
+ /** Pending cloud requests: requestId → packageName that originated the request. */
429
+ private pendingCloudRequests: Map<string, {packageName: string; envelopeRequestId?: string}> = new Map()
430
+
431
+ // Browser fallback token auth — HMAC-signed blob with a phone-local
432
+ // secret. Both issuer and verifier are the same process, so the
433
+ // secret never leaves the device.
434
+ private localSecret = `miniapp_${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
435
+ private usedTokens = new Set<string>()
436
+
437
+ private constructor() {}
438
+
439
+ /**
440
+ * Generate an HMAC-signed local session token for browser fallback auth.
441
+ * Token format: base64(JSON({userId, packageName, exp})).base64(HMAC-SHA256(payload, secret))
442
+ * Single-use, 5-minute TTL.
443
+ */
444
+ public generateLocalToken(userId: string, packageName: string): string {
445
+ const payload = JSON.stringify({
446
+ userId,
447
+ packageName,
448
+ exp: Date.now() + 5 * 60 * 1000, // 5 min TTL
449
+ nonce: Math.random().toString(36).slice(2, 10),
450
+ })
451
+ const payloadB64 = btoa(payload)
452
+ const sig = this.hmacSign(payload)
453
+ return `${payloadB64}.${sig}`
454
+ }
455
+
456
+ /**
457
+ * Validate and consume a local session token. Single-use.
458
+ */
459
+ public validateLocalToken(token: string): {userId: string; packageName: string} | null {
460
+ const parts = token.split(".")
461
+ if (parts.length !== 2) return null
462
+ const [payloadB64, sig] = parts
463
+
464
+ // Check single-use
465
+ if (this.usedTokens.has(token)) return null
466
+
467
+ // Verify signature
468
+ let payload: string
469
+ try {
470
+ payload = atob(payloadB64!)
471
+ } catch {
472
+ return null
473
+ }
474
+ if (this.hmacSign(payload) !== sig) return null
475
+
476
+ // Parse and check expiry
477
+ let parsed: {userId: string; packageName: string; exp: number}
478
+ try {
479
+ parsed = JSON.parse(payload)
480
+ } catch {
481
+ return null
482
+ }
483
+ if (Date.now() > parsed.exp) return null
484
+
485
+ // Mark as used
486
+ this.usedTokens.add(token)
487
+ // Prune old used tokens periodically (keep set from growing unbounded)
488
+ if (this.usedTokens.size > 1000) {
489
+ this.usedTokens.clear()
490
+ }
491
+
492
+ return {userId: parsed.userId, packageName: parsed.packageName}
493
+ }
494
+
495
+ /**
496
+ * Simple HMAC-SHA256 using Web Crypto API (available in React Native).
497
+ * Returns base64url-encoded signature.
498
+ * Falls back to a simpler hash if crypto.subtle is not available.
499
+ */
500
+ private hmacSign(payload: string): string {
501
+ // Synchronous HMAC using a simple hash — Web Crypto's subtle.sign is async
502
+ // which doesn't fit cleanly here. For phone-local single-process auth,
503
+ // a keyed hash is sufficient.
504
+ let hash = 0
505
+ const key = this.localSecret
506
+ const input = key + payload + key
507
+ for (let i = 0; i < input.length; i++) {
508
+ const ch = input.charCodeAt(i)
509
+ hash = ((hash << 5) - hash + ch) | 0
510
+ }
511
+ // Mix in more bytes for collision resistance
512
+ let hash2 = 0x811c9dc5 // FNV offset basis
513
+ for (let i = 0; i < input.length; i++) {
514
+ hash2 ^= input.charCodeAt(i)
515
+ hash2 = Math.imul(hash2, 0x01000193) // FNV prime
516
+ }
517
+ return btoa(String(hash >>> 0) + "." + String(hash2 >>> 0))
518
+ }
519
+
520
+ public static getInstance(): LocalMiniappRuntime {
521
+ if (!LocalMiniappRuntime.instance) {
522
+ LocalMiniappRuntime.instance = new LocalMiniappRuntime()
523
+ }
524
+ return LocalMiniappRuntime.instance
525
+ }
526
+
527
+ private initialized = false
528
+
529
+ /**
530
+ * Initialize the runtime. Called from MantleManager.init().
531
+ * Idempotent — safe to call multiple times.
532
+ */
533
+ public initialize(): void {
534
+ if (this.initialized) return
535
+ this.initialized = true
536
+ console.log(`${LOG_TAG}: initialize()`)
537
+ this.ensurePingLoop()
538
+ }
539
+
540
+ /**
541
+ * Handle an incoming cloud message forwarded by SocketComms
542
+ * (phone_stream_status, phone_managed_stream_status).
543
+ *
544
+ * Routes the response back to the originating miniapp via the requestId
545
+ * that was stored when the miniapp first made the request.
546
+ */
547
+ public handleCloudMessage(msg: any): void {
548
+ const requestId = msg.requestId as string | undefined
549
+ const msgType = msg.type as string
550
+
551
+ console.log(`${LOG_TAG}: Cloud message: ${msgType}, requestId=${requestId ?? "none"}`)
552
+
553
+ if (!requestId) {
554
+ console.warn(`${LOG_TAG}: Cloud message ${msgType} has no requestId, cannot route`)
555
+ return
556
+ }
557
+
558
+ const pending = this.pendingCloudRequests.get(requestId)
559
+ if (!pending) {
560
+ console.warn(`${LOG_TAG}: No pending request for requestId=${requestId}`)
561
+ return
562
+ }
563
+
564
+ this.pendingCloudRequests.delete(requestId)
565
+
566
+ switch (msgType) {
567
+ case "phone_stream_status": {
568
+ // Unreachable for phone-orchestrated streams (the coordinator owns
569
+ // their lifecycle and never registers a pending cloud request).
570
+ // Retained as a safety net for any legacy registration path.
571
+ this.sendToMiniapp(pending.packageName, {
572
+ type: MiniappResponseType.EVENT,
573
+ streamType: "stream_status",
574
+ data: {
575
+ streamId: msg.streamId,
576
+ status: msg.status,
577
+ errorDetails: msg.errorDetails,
578
+ },
579
+ })
580
+ // Re-register for ongoing status updates (streams send multiple status messages)
581
+ this.pendingCloudRequests.set(requestId, pending)
582
+ break
583
+ }
584
+
585
+ case "phone_managed_stream_status": {
586
+ if (msg.status === "connected" || msg.status === "active") {
587
+ // Managed stream is ready — send back the playback URLs as the request result
588
+ this.sendResult(pending.packageName, pending.envelopeRequestId, true, {
589
+ streamId: msg.streamId,
590
+ hlsUrl: msg.hlsUrl,
591
+ dashUrl: msg.dashUrl,
592
+ webrtcUrl: msg.webrtcUrl,
593
+ })
594
+ }
595
+ // Forward all statuses as events too
596
+ this.sendToMiniapp(pending.packageName, {
597
+ type: MiniappResponseType.EVENT,
598
+ streamType: "stream_status",
599
+ data: {
600
+ streamId: msg.streamId,
601
+ status: msg.status,
602
+ hlsUrl: msg.hlsUrl,
603
+ dashUrl: msg.dashUrl,
604
+ webrtcUrl: msg.webrtcUrl,
605
+ },
606
+ })
607
+ // Re-register for ongoing updates
608
+ this.pendingCloudRequests.set(requestId, pending)
609
+ break
610
+ }
611
+
612
+ default:
613
+ console.warn(`${LOG_TAG}: Unknown cloud message type: ${msgType}`)
614
+ }
615
+ }
616
+
617
+ /**
618
+ * Register a pending cloud request so we can route the response back.
619
+ */
620
+ public registerPendingCloudRequest(requestId: string, packageName: string, envelopeRequestId?: string): void {
621
+ this.pendingCloudRequests.set(requestId, {packageName, envelopeRequestId})
622
+ }
623
+
624
+ // ===========================================================================
625
+ // App registration
626
+ // ===========================================================================
627
+
628
+ public registerApp(
629
+ packageName: string,
630
+ sendFn: (raw: string) => void,
631
+ installedManifest?: InstalledMiniappManifest,
632
+ ): void {
633
+ console.log(`${LOG_TAG}: registerApp(${packageName})`)
634
+ this.clearForegroundProbe(packageName)
635
+ // Fresh spawn → fresh handshake. Clear any prior handshake flag so
636
+ // waitForConnect() blocks until this context's own CONNECT arrives.
637
+ this.handshookApps.delete(packageName)
638
+ // If the app is already registered (e.g. QR scanned again for same package),
639
+ // tear down its old subscriptions first so streamSubscribers doesn't keep
640
+ // dangling references. The WebView will re-subscribe after its CONNECT.
641
+ if (this.connectedApps.has(packageName)) {
642
+ const existing = this.connectedApps.get(packageName)!
643
+ // A crash respawn replaces the ConnectedMiniapp without going through
644
+ // unregisterApp(). Abort the old incarnation's native PCM player before
645
+ // its only stream id is lost. Target the exact stream rather than using
646
+ // stopForApp(), whose async playback cleanup could race with audio the
647
+ // replacement app starts after it connects.
648
+ const existingSpeakerStreamId = existing.activeSpeakerStreamId
649
+ if (existingSpeakerStreamId) {
650
+ existing.activeSpeakerStreamId = undefined
651
+ void audioPlaybackService.abortStream(existingSpeakerStreamId).catch((error) => {
652
+ console.warn(
653
+ `${LOG_TAG}: failed to abort speaker stream while replacing ${packageName}`,
654
+ error,
655
+ )
656
+ })
657
+ }
658
+ if (existing.authRefreshTimerId !== null) {
659
+ BgTimer.clearTimeout(existing.authRefreshTimerId)
660
+ existing.authRefreshTimerId = null
661
+ }
662
+ if (existing.authRetryTimerId !== null) {
663
+ BgTimer.clearTimeout(existing.authRetryTimerId)
664
+ existing.authRetryTimerId = null
665
+ }
666
+ for (const stream of existing.subscriptions) {
667
+ const subs = this.streamSubscribers.get(stream)
668
+ if (subs) {
669
+ subs.delete(packageName)
670
+ if (subs.size === 0) this.streamSubscribers.delete(stream)
671
+ }
672
+ }
673
+ this.recomputeMicRequirements()
674
+ this.updateCloudSubscriptions()
675
+ }
676
+ this.connectedApps.set(packageName, {
677
+ subscriptions: new Set(),
678
+ sendMessage: sendFn,
679
+ lastPongAt: Date.now(),
680
+ installedManifest,
681
+ authRefreshTimerId: null,
682
+ authRetryTimerId: null,
683
+ authDelivered: false,
684
+ speakerState: "idle",
685
+ requestedLocationRate: null,
686
+ })
687
+ // If we just clobbered an existing entry, its requestedLocationRate is
688
+ // now gone — recompute so the aggregate doesn't include a stale rate.
689
+ // The WebView will re-SUBSCRIBE shortly and the rate will reappear.
690
+ this.recomputeLocationTier()
691
+ this.ensureCloudStatusWired()
692
+ this.ensurePingLoop()
693
+
694
+ // Show the system boot message ("Starting <name>…") on the glasses for the
695
+ // bounded boot window, mirroring the cloud DisplayManager boot screen. The
696
+ // window ends early once this app pushes its first display (see
697
+ // LocalDisplayManager.request), or after ~1.5s. Fires once per spawn —
698
+ // including a crash-respawn, since each spawn is a fresh "app is starting".
699
+ localDisplayManager.onMount(packageName, installedManifest?.name ?? packageName)
700
+ }
701
+
702
+ /**
703
+ * Resolve once `packageName` has completed its CONNECT handshake. If it has
704
+ * already connected, resolves immediately. Rejects if the timeout elapses
705
+ * first, or if the app is unregistered while waiting.
706
+ *
707
+ * Used by the launcher's `ensureConnected` (and thus the action broker) so a
708
+ * just-spawned target is known to be live before anything is delivered to it.
709
+ */
710
+ public waitForConnect(packageName: string, timeoutMs: number): Promise<void> {
711
+ if (this.handshookApps.has(packageName)) return Promise.resolve()
712
+ return new Promise<void>((resolve, reject) => {
713
+ let settled = false
714
+ const waiter = (err?: Error) => {
715
+ if (settled) return
716
+ settled = true
717
+ clearTimeout(timer)
718
+ this.connectWaiters.get(packageName)?.delete(waiter)
719
+ if (err) reject(err)
720
+ else resolve()
721
+ }
722
+ const timer = setTimeout(() => {
723
+ waiter(new Error(`waitForConnect: ${packageName} did not connect within ${timeoutMs}ms`))
724
+ }, timeoutMs)
725
+ let set = this.connectWaiters.get(packageName)
726
+ if (!set) {
727
+ set = new Set()
728
+ this.connectWaiters.set(packageName, set)
729
+ }
730
+ set.add(waiter)
731
+ })
732
+ }
733
+
734
+ /** Resolve/reject all pending {@link waitForConnect} waiters for a package. */
735
+ private flushConnectWaiters(packageName: string, err?: Error): void {
736
+ const waiters = this.connectWaiters.get(packageName)
737
+ if (!waiters) return
738
+ this.connectWaiters.delete(packageName)
739
+ for (const w of waiters) w(err)
740
+ }
741
+
742
+ /**
743
+ * Invalidate the CONNECT handshake for a package whose native context is
744
+ * being replaced WITHOUT going through register/unregister — i.e. a crash
745
+ * respawn. Without this, {@link waitForConnect} would treat the dead/
746
+ * not-yet-initialized context as connected and deliver to it. Pending waiters
747
+ * are failed so a wake mid-respawn retries rather than hanging.
748
+ */
749
+ public resetHandshake(packageName: string): void {
750
+ this.handshookApps.delete(packageName)
751
+ this.flushConnectWaiters(packageName, new Error(`${packageName} respawning`))
752
+ }
753
+
754
+ /**
755
+ * Attach (or update) the installedManifest for an already-registered app.
756
+ * Used when the manifest is fetched asynchronously (dev miniapps) after the
757
+ * miniapp has already CONNECTed — preserves existing subscriptions.
758
+ */
759
+ public setInstalledManifest(packageName: string, installedManifest: InstalledMiniappManifest): void {
760
+ const app = this.connectedApps.get(packageName)
761
+ if (!app) return
762
+ app.installedManifest = installedManifest
763
+
764
+ // Push declared-permission record to the SDK so session.permissions stays
765
+ // in sync. Sent regardless of CONNECT_ACK timing — covers the dev-miniapp
766
+ // case where the manifest is fetched async after the miniapp CONNECTs.
767
+ this.sendToMiniapp(packageName, {
768
+ type: MiniappResponseType.PERMISSIONS_UPDATE,
769
+ permissions: computeDeclaredPermissionRecord(installedManifest),
770
+ })
771
+ }
772
+
773
+ /**
774
+ * Graceful version of {@link unregisterApp}: notify the miniapp via
775
+ * `WILL_DISCONNECT`, wait ~50ms so its `beforeDisconnect` handlers can
776
+ * fire one last `sendOneShot` (e.g. `display.clear()`), then run the
777
+ * normal teardown. Use this for any disconnect path where the socket
778
+ * is still open. For ungraceful paths (transport already closed),
779
+ * call {@link unregisterApp} directly — the heads-up would be
780
+ * undeliverable anyway.
781
+ */
782
+ public async gracefullyUnregisterApp(packageName: string, reason = "unregistering"): Promise<void> {
783
+ const app = this.connectedApps.get(packageName)
784
+ if (!app) return
785
+
786
+ console.log(`${LOG_TAG}: gracefullyUnregisterApp(${packageName}) — sending WILL_DISCONNECT`)
787
+ this.sendToMiniapp(packageName, {
788
+ type: MiniappResponseType.WILL_DISCONNECT,
789
+ reason,
790
+ })
791
+ // 50 ms: imperceptible to the user, plenty of time for the SDK's
792
+ // synchronous `beforeDisconnect` handlers to flush a final
793
+ // sendOneShot through the still-open transport.
794
+ await new Promise<void>((resolve) => setTimeout(resolve, 50))
795
+ this.unregisterApp(packageName)
796
+ }
797
+
798
+ public unregisterApp(packageName: string): void {
799
+ console.log(`${LOG_TAG}: unregisterApp(${packageName})`)
800
+ this.clearForegroundProbe(packageName)
801
+ this.clearMiniappAuthRefresh(packageName)
802
+ this.clearMiniappAuthDeliveryRetry(packageName)
803
+ // Drop the handshake flag and fail any in-flight waitForConnect() callers
804
+ // (e.g. a wake that's mid-handshake when the app is torn down). Done before
805
+ // the early-return so it runs even if the connectedApps entry is already gone.
806
+ this.handshookApps.delete(packageName)
807
+ this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`))
808
+ const app = this.connectedApps.get(packageName)
809
+ if (!app) return
810
+
811
+ // Remove from all stream subscriber sets
812
+ for (const stream of app.subscriptions) {
813
+ const subs = this.streamSubscribers.get(stream)
814
+ if (subs) {
815
+ subs.delete(packageName)
816
+ if (subs.size === 0) {
817
+ this.streamSubscribers.delete(stream)
818
+ }
819
+ }
820
+ }
821
+
822
+ // Reset per-session warning dedup so a relaunch surfaces issues again.
823
+ resetPermissionWarnings(packageName)
824
+
825
+ // Stop audio for this app
826
+ audioPlaybackService.stopForApp(packageName)
827
+
828
+ // Tear down this app's blob state: abort in-flight uploads + close readers
829
+ // so a crashed/closed miniapp doesn't leak partial files or file handles.
830
+ this.blobStore.onAppGone(packageName)
831
+
832
+ // Release phone-owned camera streams. If a miniapp closes/crashes without
833
+ // sending STREAM_STOP, the host coordinator must drop its subscriber/owner
834
+ // so glasses publishing and managed Cloudflare inputs do not leak.
835
+ void phoneStreamCoordinator.stop(packageName).catch((error) => {
836
+ console.warn(`${LOG_TAG}: failed to stop stream for ${packageName} on unregister`, error)
837
+ })
838
+
839
+ // Stop any phone-owned video recordings for this app. A miniapp that
840
+ // closes/crashes mid-recording loses its recordingId, so without this the
841
+ // glasses keep recording until the max-recording timeout or thermal shutdown.
842
+ void phoneVideoCoordinator.stopForApp(packageName).catch((error) => {
843
+ console.warn(`${LOG_TAG}: failed to stop video recording for ${packageName} on unregister`, error)
844
+ })
845
+
846
+ // Detach the per-app nav event forwarder but leave the native nav session
847
+ // running. The user may have just closed the mini-app UI and will reopen
848
+ // it; stopping the session here would kill an active trip mid-route.
849
+ // Navigation is only stopped when the mini-app explicitly calls
850
+ // navigation.stop() or when the trip arrives/errors naturally.
851
+ // (See NavigationHandlers — activeNavApps stays populated so a reconnect
852
+ // can reattach listeners and resume.)
853
+ this.navigationHandlers.onDisconnect(packageName)
854
+
855
+ // Recompute heading subscription — if this app was the last subscriber,
856
+ // the sensor will stop.
857
+ this.recomputeHeadingSubscription()
858
+ // Same for the IMU/accelerometer stream.
859
+ this.recomputeImuSubscription()
860
+
861
+ // Drop this app's location-tier request before recomputing so the
862
+ // aggregate falls back down if it was the strictest. Done before
863
+ // `connectedApps.delete` below so it doesn't matter that the iter
864
+ // in `recomputeLocationTier` is over connectedApps — clearing the
865
+ // field is enough.
866
+ app.requestedLocationRate = null
867
+ this.recomputeLocationTier()
868
+
869
+ // Clean up any pending cloud requests from this app
870
+ for (const [reqId, pending] of this.pendingCloudRequests) {
871
+ if (pending.packageName === packageName) {
872
+ this.pendingCloudRequests.delete(reqId)
873
+ }
874
+ }
875
+
876
+ // Release any display real estate this app held — if it owned the
877
+ // current on-glasses frame, this clears the glasses (or restores the
878
+ // core app's saved frame).
879
+ localDisplayManager.onUnmount(packageName)
880
+
881
+ this.connectedApps.delete(packageName)
882
+ this.recomputeMicRequirements()
883
+ this.updateCloudSubscriptions()
884
+
885
+ if (this.connectedApps.size === 0) {
886
+ this.stopPingLoop()
887
+ }
888
+ }
889
+
890
+ // ===========================================================================
891
+ // Inbound message handling
892
+ // ===========================================================================
893
+
894
+ public handleRawMessage(packageName: string, raw: string): void {
895
+ const envelope = parseEnvelope(raw)
896
+ if (!envelope) {
897
+ // Not a miniapp envelope — ignore (could be legacy bridge message)
898
+ return
899
+ }
900
+
901
+ const payload = envelope.payload as Record<string, unknown>
902
+ const requestType = payload.type as string | undefined
903
+ const requestId = envelope.requestId
904
+
905
+ if (!requestType) {
906
+ console.warn(`${LOG_TAG}: Envelope from ${packageName} missing payload.type`)
907
+ return
908
+ }
909
+
910
+ // ANY inbound message proves the context is alive — a busy background
911
+ // script streaming DISPLAY/storage traffic shouldn't be killed by the
912
+ // liveness watchdog just because its PONG replies queue behind real work.
913
+ this.handlePong(packageName)
914
+
915
+ // Console-tap forwarding. The miniapp's console.log/warn/etc is wrapped
916
+ // (via injected shim from miniappGlobals.ts) to post a `dev_log`
917
+ // envelope. We fan out to two destinations:
918
+ // 1. DevServerBridge — forwards to the laptop's `mentra-miniapp dev`
919
+ // terminal. No-op when there's no sidecar (installed miniapps).
920
+ // 2. React Native console — surfaces the log in Metro / Xcode console
921
+ // / adb logcat so installed-miniapp errors are still inspectable
922
+ // when there's no laptop to forward to.
923
+ if (requestType === "dev_log") {
924
+ const level = (payload.level as string | undefined) ?? "log"
925
+ const args = Array.isArray(payload.args) ? (payload.args as unknown[]) : []
926
+ const timestamp = (payload.timestamp as number | undefined) ?? Date.now()
927
+ // Source is "ui" because the single-bundle WebView console-tap
928
+ // shim in miniappGlobals.ts is the only thing that posts
929
+ // `dev_log`; the two-layer path uses a separate envelope
930
+ // (`{type:"log", source}`) routed by MentraUIRouter.
931
+ devServerBridge.forwardLog(packageName, level, args, timestamp, "ui")
932
+
933
+ const tag = `[MINIAPP ${packageName}]`
934
+ const fn = (console as unknown as Record<string, (...a: unknown[]) => void>)[level] ?? console.log
935
+ try {
936
+ fn(tag, ...args)
937
+ } catch {
938
+ console.log(tag, ...args)
939
+ }
940
+ return
941
+ }
942
+
943
+ // Dispatch
944
+ switch (requestType) {
945
+ case MiniappRequestType.CONNECT:
946
+ void this.handleConnect(packageName, payload, requestId)
947
+ break
948
+ case MiniappRequestType.AUTH_REFRESH:
949
+ void this.handleAuthRefresh(packageName, payload, requestId)
950
+ break
951
+ case MiniappRequestType.SUBSCRIBE:
952
+ this.handleSubscribe(packageName, payload, requestId)
953
+ break
954
+ case MiniappRequestType.DISPLAY:
955
+ this.handleDisplay(packageName, payload, requestId)
956
+ break
957
+ case MiniappRequestType.RENDER:
958
+ this.handleRender(packageName, payload, requestId)
959
+ break
960
+ case MiniappRequestType.PLAY_AUDIO:
961
+ this.handlePlayAudio(packageName, payload, requestId)
962
+ break
963
+ case MiniappRequestType.STOP_AUDIO:
964
+ this.handleStopAudio(packageName, payload, requestId)
965
+ break
966
+ case MiniappRequestType.SPEAK:
967
+ this.handleSpeak(packageName, payload, requestId)
968
+ break
969
+ case MiniappRequestType.SPEAKER_STREAM_OPEN:
970
+ void this.handleSpeakerStreamOpen(packageName, payload, requestId)
971
+ break
972
+ case MiniappRequestType.SPEAKER_STREAM_WRITE:
973
+ void this.handleSpeakerStreamWrite(packageName, payload, requestId)
974
+ break
975
+ case MiniappRequestType.SPEAKER_STREAM_CLOSE:
976
+ void this.handleSpeakerStreamClose(packageName, payload, requestId)
977
+ break
978
+ case MiniappRequestType.SPEAKER_STREAM_ABORT:
979
+ void this.handleSpeakerStreamAbort(packageName, payload, requestId)
980
+ break
981
+ case MiniappRequestType.RGB_LED:
982
+ void this.handleRgbLed(packageName, payload, requestId)
983
+ break
984
+ case MiniappRequestType.LOCATION_POLL:
985
+ this.handleLocationPoll(packageName, requestId)
986
+ break
987
+ case MiniappRequestType.NAVIGATION_START:
988
+ this.navigationHandlers.handleStart(packageName, payload, requestId)
989
+ break
990
+ case MiniappRequestType.NAVIGATION_STOP:
991
+ this.navigationHandlers.handleStop(packageName, requestId)
992
+ break
993
+ case MiniappRequestType.NAVIGATION_DEVIATE:
994
+ this.navigationHandlers.handleDeviate(packageName, payload, requestId)
995
+ break
996
+ case MiniappRequestType.NAVIGATION_SET_WRONG_SIDEWALK:
997
+ this.navigationHandlers.handleSetWrongSidewalk(packageName, payload, requestId)
998
+ break
999
+ case MiniappRequestType.NAVIGATION_SET_SKIP_CROSSINGS:
1000
+ this.navigationHandlers.handleSetSkipCrossings(packageName, payload, requestId)
1001
+ break
1002
+ case MiniappRequestType.NAVIGATION_GET_STATE:
1003
+ this.navigationHandlers.handleGetState(packageName, requestId)
1004
+ break
1005
+ case MiniappRequestType.NAVIGATION_COMPUTE_ROUTE:
1006
+ this.navigationHandlers.handleComputeRoute(packageName, payload, requestId)
1007
+ break
1008
+ case MiniappRequestType.NAVIGATION_REVERSE_GEOCODE:
1009
+ this.navigationHandlers.handleReverseGeocode(packageName, payload, requestId)
1010
+ break
1011
+ case MiniappRequestType.NAVIGATION_PLACE_AUTOCOMPLETE:
1012
+ this.navigationHandlers.handlePlaceAutocomplete(packageName, payload, requestId)
1013
+ break
1014
+ case MiniappRequestType.NAVIGATION_PLACE_DETAILS:
1015
+ this.navigationHandlers.handlePlaceDetails(packageName, payload, requestId)
1016
+ break
1017
+ case MiniappRequestType.NAVIGATION_REQUEST_PERMISSION:
1018
+ this.navigationHandlers.handleRequestPermission(packageName, requestId)
1019
+ break
1020
+ case MiniappRequestType.STORAGE_GET:
1021
+ this.handleStorageGet(packageName, payload, requestId)
1022
+ break
1023
+ case MiniappRequestType.STORAGE_SET:
1024
+ this.handleStorageSet(packageName, payload, requestId)
1025
+ break
1026
+ case MiniappRequestType.STORAGE_DELETE:
1027
+ this.handleStorageDelete(packageName, payload, requestId)
1028
+ break
1029
+ case MiniappRequestType.STORAGE_LIST:
1030
+ this.handleStorageList(packageName, payload, requestId)
1031
+ break
1032
+ case MiniappRequestType.STORAGE_CLEAR:
1033
+ this.handleStorageClear(packageName, requestId)
1034
+ break
1035
+ case MiniappRequestType.STORAGE_HAS:
1036
+ this.handleStorageHas(packageName, payload, requestId)
1037
+ break
1038
+ case MiniappRequestType.STORAGE_GET_ALL:
1039
+ this.handleStorageGetAll(packageName, requestId)
1040
+ break
1041
+ case MiniappRequestType.STORAGE_SET_MULTIPLE:
1042
+ this.handleStorageSetMultiple(packageName, payload, requestId)
1043
+ break
1044
+ case MiniappRequestType.STORAGE_FLUSH:
1045
+ // No-op today — MMKV writes through synchronously. Reserved so
1046
+ // a future debounced backend can honor "flush before I quit"
1047
+ // calls without breaking the API.
1048
+ this.sendResult(packageName, requestId, true)
1049
+ break
1050
+ case MiniappRequestType.CAMERA_FOV:
1051
+ void this.handleCameraFov(packageName, payload, requestId)
1052
+ break
1053
+ case MiniappRequestType.IMU_SET_ENABLED:
1054
+ this.handleImuSetEnabled(packageName, payload, requestId)
1055
+ break
1056
+ case MiniappRequestType.PING:
1057
+ // SDK should handle this itself; reply PONG just in case
1058
+ this.sendToMiniapp(packageName, {type: MiniappResponseType.PONG}, requestId)
1059
+ break
1060
+ case MiniappResponseType.PONG: {
1061
+ // Liveness already touched above for every inbound message; the
1062
+ // PONG carries no other action.
1063
+ // BGCAP: measure PING→PONG round-trip ONLY here (a real pong), not in
1064
+ // handlePong (which fires for every inbound envelope). RTT = how long
1065
+ // the miniapp's own JS engine (Crust serial queue) took to answer — a
1066
+ // large value means that queue was starved (the transcript→display
1067
+ // layer). bgcapLastPingSentAt holds the OLDEST outstanding ping (see
1068
+ // doPingRound), so a multi-round background stall reports its full
1069
+ // duration on the first pong after resume.
1070
+ if (BGCAP_TELEMETRY) {
1071
+ const app = this.connectedApps.get(packageName)
1072
+ if (app?.bgcapLastPingSentAt != null) {
1073
+ const rtt = Date.now() - app.bgcapLastPingSentAt
1074
+ if (rtt > 1500) {
1075
+ console.log(`BGCAP: ${packageName} pong rtt=${rtt}ms (miniapp JS engine was starved)`)
1076
+ }
1077
+ app.bgcapLastPingSentAt = undefined
1078
+ }
1079
+ }
1080
+ break
1081
+ }
1082
+
1083
+ case MiniappRequestType.SHARE:
1084
+ this.handleShare(packageName, payload, requestId)
1085
+ break
1086
+ case MiniappRequestType.OPEN_URL:
1087
+ this.handleOpenUrl(packageName, payload)
1088
+ break
1089
+ case MiniappRequestType.COPY_CLIPBOARD:
1090
+ this.handleCopyClipboard(packageName, payload, requestId)
1091
+ break
1092
+ case MiniappRequestType.DOWNLOAD:
1093
+ this.handleDownload(packageName, payload, requestId)
1094
+ break
1095
+
1096
+ // Persistent binary blob storage (session.blob)
1097
+ case MiniappRequestType.BLOB_CREATE:
1098
+ this.blobStore.handleCreate(packageName, payload, requestId)
1099
+ break
1100
+ case MiniappRequestType.BLOB_WRITE:
1101
+ this.blobStore.handleWrite(packageName, payload, requestId)
1102
+ break
1103
+ case MiniappRequestType.BLOB_COMMIT:
1104
+ this.blobStore.handleCommit(packageName, payload, requestId)
1105
+ break
1106
+ case MiniappRequestType.BLOB_ABORT:
1107
+ this.blobStore.handleAbort(packageName, payload, requestId)
1108
+ break
1109
+ case MiniappRequestType.BLOB_SET_FROM_URL:
1110
+ void this.blobStore.handleSetFromUrl(packageName, payload, requestId)
1111
+ break
1112
+ case MiniappRequestType.BLOB_IMPORT:
1113
+ void this.blobStore.handleImport(packageName, payload, requestId)
1114
+ break
1115
+ case MiniappRequestType.BLOB_GET:
1116
+ this.blobStore.handleGet(packageName, payload, requestId)
1117
+ break
1118
+ case MiniappRequestType.BLOB_LIST:
1119
+ this.blobStore.handleList(packageName, payload, requestId)
1120
+ break
1121
+ case MiniappRequestType.BLOB_USAGE:
1122
+ this.blobStore.handleUsage(packageName, payload, requestId)
1123
+ break
1124
+ case MiniappRequestType.BLOB_DELETE:
1125
+ this.blobStore.handleDelete(packageName, payload, requestId)
1126
+ break
1127
+ case MiniappRequestType.BLOB_CLEAR:
1128
+ this.blobStore.handleClear(packageName, payload, requestId)
1129
+ break
1130
+ case MiniappRequestType.BLOB_OPEN_READ:
1131
+ this.blobStore.handleOpenRead(packageName, payload, requestId)
1132
+ break
1133
+ case MiniappRequestType.BLOB_READ:
1134
+ this.blobStore.handleRead(packageName, payload, requestId)
1135
+ break
1136
+ case MiniappRequestType.BLOB_CLOSE_READ:
1137
+ this.blobStore.handleCloseRead(packageName, payload, requestId)
1138
+ break
1139
+ case MiniappRequestType.BLOB_SHARE:
1140
+ void this.blobStore.handleShare(packageName, payload, requestId)
1141
+ break
1142
+
1143
+ // Cloud-coordinated features
1144
+ case MiniappRequestType.PHOTO:
1145
+ void this.handlePhoto(packageName, payload, requestId)
1146
+ break
1147
+ case MiniappRequestType.CAMERA_WARM_UP:
1148
+ void this.handleCameraWarmUp(packageName, payload, requestId)
1149
+ break
1150
+ case MiniappRequestType.VIDEO_RECORDING_START:
1151
+ void this.handleVideoRecordingStart(packageName, payload, requestId)
1152
+ break
1153
+ case MiniappRequestType.VIDEO_RECORDING_STOP:
1154
+ void this.handleVideoRecordingStop(packageName, payload, requestId)
1155
+ break
1156
+ case MiniappRequestType.STREAM_START:
1157
+ void this.handleStreamStart(packageName, payload, requestId)
1158
+ break
1159
+ case MiniappRequestType.STREAM_STOP:
1160
+ void this.handleStreamStop(packageName, payload, requestId)
1161
+ break
1162
+ case MiniappRequestType.MANAGED_STREAM_START:
1163
+ void this.handleManagedStreamStart(packageName, payload, requestId)
1164
+ break
1165
+ case MiniappRequestType.MANAGED_STREAM_STOP:
1166
+ void this.handleManagedStreamStop(packageName, payload, requestId)
1167
+ break
1168
+ case REQUEST_WIFI_SETUP_TYPE:
1169
+ void this.handleRequestWifiSetup(packageName, payload, requestId)
1170
+ break
1171
+
1172
+ // Inter-miniapp interop (SYSTEM apps only)
1173
+ case MiniappRequestType.MINIAPPS_LIST:
1174
+ this.handleMiniappsList(packageName, payload, requestId)
1175
+ break
1176
+ case MiniappRequestType.MINIAPPS_START:
1177
+ void this.handleMiniappsStart(packageName, payload, requestId)
1178
+ break
1179
+ case MiniappRequestType.MINIAPPS_STOP:
1180
+ void this.handleMiniappsStop(packageName, payload, requestId)
1181
+ break
1182
+ case MiniappRequestType.ACTION_INVOKE:
1183
+ void this.handleActionInvoke(packageName, payload, requestId)
1184
+ break
1185
+ case MiniappRequestType.ACTION_RESULT:
1186
+ this.handleActionResult(packageName, payload)
1187
+ break
1188
+
1189
+ // Deferred in v1
1190
+ case MiniappRequestType.DASHBOARD_CONTENT_UPDATE:
1191
+ this.sendResult(packageName, requestId, false, undefined, {
1192
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
1193
+ message: "Dashboard API is deferred in v1",
1194
+ })
1195
+ break
1196
+
1197
+ default:
1198
+ console.warn(`${LOG_TAG}: Unknown request type from ${packageName}: ${requestType}`)
1199
+ break
1200
+ }
1201
+ }
1202
+
1203
+ // ===========================================================================
1204
+ // Request handlers
1205
+ // ===========================================================================
1206
+
1207
+ private async handleConnect(
1208
+ packageName: string,
1209
+ _payload: Record<string, unknown>,
1210
+ requestId?: string,
1211
+ ): Promise<void> {
1212
+ console.log(`${LOG_TAG}: CONNECT from ${packageName}`)
1213
+
1214
+ // Register if not already
1215
+ const existing = this.connectedApps.get(packageName)
1216
+ if (!existing) {
1217
+ console.warn(`${LOG_TAG}: CONNECT from unregistered app ${packageName}, ignoring`)
1218
+ return
1219
+ }
1220
+
1221
+ // Update lastPongAt so it doesn't time out right away
1222
+ existing.lastPongAt = Date.now()
1223
+
1224
+ // Read current glasses capabilities from the settings store
1225
+ const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable) as
1226
+ | DeviceTypes
1227
+ | undefined
1228
+ const capabilities = getModelCapabilities(defaultWearable || DeviceTypes.NONE)
1229
+
1230
+ // Build the declared-permission record for the SDK's session.permissions
1231
+ // module. Lower-cased to match v3's PermissionType union (microphone,
1232
+ // camera, location, notifications, calendar). Missing permissions default
1233
+ // to false; this is manifest-declaration tracking only — OS-grant state
1234
+ // is intentionally not modeled here.
1235
+ const declaredPermissions = computeDeclaredPermissionRecord(existing.installedManifest)
1236
+ // Fresh handshake → clear any auth state from a prior context so a stale
1237
+ // retry timer can't fire against this new connection and so the retry loop
1238
+ // below re-arms from scratch.
1239
+ existing.authDelivered = false
1240
+ this.clearMiniappAuthDeliveryRetry(packageName)
1241
+ const authPromise = this.requestMiniappAuth(packageName)
1242
+ const initialAuth = await withTimeout(authPromise, 1_500)
1243
+ const userId = initialAuth?.mentraUserId ?? ""
1244
+ if (initialAuth) {
1245
+ existing.authDelivered = true
1246
+ this.scheduleMiniappAuthRefresh(packageName, initialAuth)
1247
+ }
1248
+
1249
+ this.sendToMiniapp(
1250
+ packageName,
1251
+ {
1252
+ type: MiniappResponseType.CONNECT_ACK,
1253
+ userId,
1254
+ packageName,
1255
+ capabilities,
1256
+ permissions: declaredPermissions,
1257
+ ...(initialAuth ? {auth: initialAuth} : {}),
1258
+ },
1259
+ requestId,
1260
+ )
1261
+ if (!initialAuth) {
1262
+ // The mint timed out or (more importantly) REJECTED — the latter happens
1263
+ // when the cloud client hasn't finished its first-boot Core token exchange
1264
+ // yet, which is exactly the case when a dev miniapp is scanned/launched
1265
+ // right after app start. CONNECT_ACK already went out without auth; keep
1266
+ // trying to mint (and re-drive on cloud-connect) so the app authenticates
1267
+ // to its backend on its own, instead of staying dead until a full manual
1268
+ // Cloud V2 reconnect re-runs the whole handshake.
1269
+ this.deliverInitialMiniappAuth(packageName, authPromise, 0)
1270
+ }
1271
+ this.sendCloudStatusToMiniapp(packageName)
1272
+
1273
+ // Handshake complete — unblock any launcher.waitForConnect() callers.
1274
+ this.handshookApps.add(packageName)
1275
+ this.flushConnectWaiters(packageName)
1276
+ }
1277
+
1278
+ private async requestMiniappAuth(packageName: string, opts?: {minTtlMs?: number}): Promise<MiniappAuthToken | null> {
1279
+ const authPackageName = packageName === DEV_APP_PACKAGE_NAME ? getDevAppSourcePackage() : packageName
1280
+ if (!authPackageName) return null
1281
+ return cloudClientService.getMiniappAuthToken(authPackageName, opts)
1282
+ }
1283
+
1284
+ private async handleAuthRefresh(
1285
+ packageName: string,
1286
+ payload: Record<string, unknown>,
1287
+ requestId?: string,
1288
+ ): Promise<void> {
1289
+ try {
1290
+ const auth = await this.refreshMiniappAuth(packageName, this.authRefreshOptions(payload))
1291
+ if (!auth) {
1292
+ this.sendResult(packageName, requestId, false, undefined, {
1293
+ code: MiniappErrorCode.NOT_CONNECTED,
1294
+ message: "Miniapp auth is not configured",
1295
+ })
1296
+ return
1297
+ }
1298
+ this.sendResult(packageName, requestId, true, {auth})
1299
+ } catch (err) {
1300
+ this.sendResult(packageName, requestId, false, undefined, {
1301
+ code: MiniappErrorCode.INTERNAL,
1302
+ message: (err as Error)?.message ?? "Miniapp auth refresh failed",
1303
+ })
1304
+ }
1305
+ }
1306
+
1307
+ private async refreshMiniappAuth(packageName: string, opts?: {minTtlMs?: number}): Promise<MiniappAuthToken | null> {
1308
+ const auth = await this.requestMiniappAuth(packageName, opts)
1309
+ if (!auth) return null
1310
+ this.scheduleMiniappAuthRefresh(packageName, auth)
1311
+ this.sendToMiniapp(packageName, {
1312
+ type: MiniappResponseType.AUTH_UPDATE,
1313
+ auth,
1314
+ })
1315
+ return auth
1316
+ }
1317
+
1318
+ private authRefreshOptions(payload: Record<string, unknown>): {minTtlMs?: number} | undefined {
1319
+ const minTtlMs = Number(payload.minTtlMs)
1320
+ if (!Number.isFinite(minTtlMs) || minTtlMs <= 0) return undefined
1321
+ return {minTtlMs}
1322
+ }
1323
+
1324
+ private scheduleMiniappAuthRefresh(packageName: string, auth: MiniappAuthToken): void {
1325
+ const app = this.connectedApps.get(packageName)
1326
+ if (!app) return
1327
+ this.clearMiniappAuthRefresh(packageName)
1328
+
1329
+ const refreshAt = auth.expiresAt - MINIAPP_AUTH_REFRESH_HEADROOM_MS
1330
+ const delay = Math.max(MINIAPP_AUTH_REFRESH_MIN_DELAY_MS, refreshAt - Date.now())
1331
+ app.authRefreshTimerId = BgTimer.setTimeout(() => {
1332
+ const current = this.connectedApps.get(packageName)
1333
+ if (!current) return
1334
+ current.authRefreshTimerId = null
1335
+ void this.refreshMiniappAuth(packageName).catch((err) => {
1336
+ console.warn(`${LOG_TAG}: miniapp auth refresh failed for ${packageName}: ${(err as Error)?.message ?? err}`)
1337
+ })
1338
+ }, delay)
1339
+ }
1340
+
1341
+ private clearMiniappAuthRefresh(packageName: string): void {
1342
+ const app = this.connectedApps.get(packageName)
1343
+ if (!app || app.authRefreshTimerId === null) return
1344
+ const timerId = app.authRefreshTimerId
1345
+ BgTimer.clearTimeout(timerId)
1346
+ app.authRefreshTimerId = null
1347
+ }
1348
+
1349
+ /**
1350
+ * Deliver the miniapp's FIRST auth token, retrying until it lands. Called from
1351
+ * {@link handleConnect} when the initial mint didn't resolve in the CONNECT_ACK
1352
+ * window. `inFlight` lets attempt 0 reuse the mint the handshake already
1353
+ * started (avoids a redundant duplicate request); later attempts mint fresh.
1354
+ *
1355
+ * A `null` result means auth is fundamentally unavailable (no miniappAuth hook)
1356
+ * — permanent, so we do NOT retry. A REJECTION means a transient failure
1357
+ * (typically the cloud client isn't connection-ready yet) — retry with backoff.
1358
+ * Either way we bail if the app has since disconnected or already got a token
1359
+ * (e.g. via {@link redriveMiniappAuthDelivery} on cloud-connect).
1360
+ */
1361
+ private deliverInitialMiniappAuth(
1362
+ packageName: string,
1363
+ inFlight: Promise<MiniappAuthToken | null> | undefined,
1364
+ attempt: number,
1365
+ ): void {
1366
+ const app = this.connectedApps.get(packageName)
1367
+ if (!app || app.authDelivered) return
1368
+ const mint = inFlight ?? this.requestMiniappAuth(packageName)
1369
+ void mint
1370
+ .then((auth) => {
1371
+ const current = this.connectedApps.get(packageName)
1372
+ if (!current || current.authDelivered) return
1373
+ if (!auth) return // no auth hook — permanent, don't spin
1374
+ current.authDelivered = true
1375
+ this.clearMiniappAuthDeliveryRetry(packageName)
1376
+ this.scheduleMiniappAuthRefresh(packageName, auth)
1377
+ this.sendToMiniapp(packageName, {type: MiniappResponseType.AUTH_UPDATE, auth})
1378
+ })
1379
+ .catch((err) => {
1380
+ console.warn(
1381
+ `${LOG_TAG}: miniapp auth mint failed for ${packageName} (attempt ${attempt}): ${
1382
+ (err as Error)?.message ?? err
1383
+ }`,
1384
+ )
1385
+ this.scheduleInitialMiniappAuthRetry(packageName, attempt)
1386
+ })
1387
+ }
1388
+
1389
+ private scheduleInitialMiniappAuthRetry(packageName: string, attempt: number): void {
1390
+ const app = this.connectedApps.get(packageName)
1391
+ if (!app || app.authDelivered) return
1392
+ this.clearMiniappAuthDeliveryRetry(packageName)
1393
+ const delay = Math.min(MINIAPP_AUTH_RETRY_MAX_MS, MINIAPP_AUTH_RETRY_BASE_MS * 2 ** attempt)
1394
+ app.authRetryTimerId = BgTimer.setTimeout(() => {
1395
+ const current = this.connectedApps.get(packageName)
1396
+ if (!current) return
1397
+ current.authRetryTimerId = null
1398
+ this.deliverInitialMiniappAuth(packageName, undefined, attempt + 1)
1399
+ }, delay)
1400
+ }
1401
+
1402
+ private clearMiniappAuthDeliveryRetry(packageName: string): void {
1403
+ const app = this.connectedApps.get(packageName)
1404
+ if (!app || app.authRetryTimerId === null) return
1405
+ BgTimer.clearTimeout(app.authRetryTimerId)
1406
+ app.authRetryTimerId = null
1407
+ }
1408
+
1409
+ /**
1410
+ * When the cloud client (re)connects, immediately re-attempt the initial mint
1411
+ * for any handshook app still missing its token, instead of waiting out the
1412
+ * backoff. This is what turns "scan a dev app before the cloud is ready" from
1413
+ * "dead until you manually reconnect Cloud V2" into "authenticates on its own
1414
+ * the instant the connection comes up".
1415
+ */
1416
+ private redriveMiniappAuthDelivery(): void {
1417
+ for (const [packageName, app] of this.connectedApps) {
1418
+ if (app.authDelivered) continue
1419
+ // Only apps that finished their CONNECT handshake are in the mint-retry
1420
+ // state; skip anything still mid-handshake (handleConnect owns that).
1421
+ if (!this.handshookApps.has(packageName)) continue
1422
+ this.clearMiniappAuthDeliveryRetry(packageName)
1423
+ this.deliverInitialMiniappAuth(packageName, undefined, 0)
1424
+ }
1425
+ }
1426
+
1427
+ private handleSubscribe(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1428
+ const app = this.connectedApps.get(packageName)
1429
+ if (!app) return
1430
+
1431
+ const rawStreams = (payload.subscriptions ?? payload.streams) as
1432
+ | (string | {stream: string; rate?: string})[]
1433
+ | undefined
1434
+ // Normalize: objects like {stream: "location_stream", rate: "realtime"} → extract stream name
1435
+ // "location_stream" is the rate-bearing alias for "location_update".
1436
+ // The strictest rate wins when multiple `location_stream` entries appear
1437
+ // in one SUBSCRIBE — the same rule we apply across apps.
1438
+ let requestedLocationRate: string | null = null
1439
+ const streams = rawStreams?.map((s) => {
1440
+ if (typeof s === "object" && s !== null) {
1441
+ if (s.stream === "location_stream") {
1442
+ if (s.rate && locationRateRank(s.rate) > locationRateRank(requestedLocationRate)) {
1443
+ requestedLocationRate = s.rate
1444
+ }
1445
+ return "location_update"
1446
+ }
1447
+ return s.stream
1448
+ }
1449
+ return s
1450
+ }) as string[] | undefined
1451
+ if (!Array.isArray(streams)) {
1452
+ this.sendResult(packageName, requestId, false, undefined, {
1453
+ code: MiniappErrorCode.INTERNAL,
1454
+ message: "subscribe requires a subscriptions array",
1455
+ })
1456
+ return
1457
+ }
1458
+
1459
+ console.log(`${LOG_TAG}: SUBSCRIBE from ${packageName}: [${streams.join(", ")}]`)
1460
+
1461
+ // Gate each stream on the permission type its data requires. The manifest
1462
+ // must declare the permission (miniapp.json -> permissions) or we reject
1463
+ // the whole subscribe with PERMISSION_NOT_DECLARED.
1464
+ const declaredTypes = new Set((app.installedManifest?.permissions ?? []).map((p) => p.type?.toUpperCase()))
1465
+
1466
+ const permissionForStream = (s: string): string | null => {
1467
+ if (s === "audio_chunk" || s === "vad") return "MICROPHONE"
1468
+ if (s.startsWith("transcription") || s.startsWith("translation")) return "MICROPHONE"
1469
+ if (s === "location_update") return "LOCATION"
1470
+ if (s === "phone_notification") return "READ_NOTIFICATIONS"
1471
+ if (s === "phone_notification_dismissed") return "READ_NOTIFICATIONS"
1472
+ if (s === "calendar_event") return "CALENDAR"
1473
+ return null
1474
+ }
1475
+
1476
+ for (const stream of streams) {
1477
+ const required = permissionForStream(stream)
1478
+ if (required && !declaredTypes.has(required)) {
1479
+ logPermissionNotDeclared(packageName, required, `to subscribe to "${stream}"`, `{"type": "${required}"}`)
1480
+ this.sendResult(packageName, requestId, false, undefined, {
1481
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
1482
+ message: `${required} permission not declared in miniapp.json (required for "${stream}"). Add {"type": "${required}"} to the "permissions" array.`,
1483
+ // Extra context fields read by the SDK so authors that subscribe
1484
+ // to session.on("error") can format their own messages.
1485
+ permission: required,
1486
+ subscription: stream,
1487
+ })
1488
+ return
1489
+ }
1490
+ }
1491
+
1492
+ // Remove old subscriptions for this app
1493
+ for (const oldStream of app.subscriptions) {
1494
+ const subs = this.streamSubscribers.get(oldStream)
1495
+ if (subs) {
1496
+ subs.delete(packageName)
1497
+ if (subs.size === 0) {
1498
+ this.streamSubscribers.delete(oldStream)
1499
+ }
1500
+ }
1501
+ }
1502
+
1503
+ // Add new subscriptions
1504
+ app.subscriptions = new Set(streams)
1505
+ for (const stream of streams) {
1506
+ let subs = this.streamSubscribers.get(stream)
1507
+ if (!subs) {
1508
+ subs = new Set()
1509
+ this.streamSubscribers.set(stream, subs)
1510
+ }
1511
+ subs.add(packageName)
1512
+ }
1513
+
1514
+ this.recomputeMicRequirements()
1515
+ this.updateCloudSubscriptions()
1516
+ this.recomputeHeadingSubscription()
1517
+ this.recomputeImuSubscription()
1518
+ // Persist this app's requested rate (or clear it if SUBSCRIBE didn't
1519
+ // include `location_stream` this time), then ask the host for the
1520
+ // strictest rate across all connected apps.
1521
+ app.requestedLocationRate = requestedLocationRate
1522
+ this.recomputeLocationTier()
1523
+ this.sendResult(packageName, requestId, true)
1524
+
1525
+ // Fire initial snapshot values for stateful streams so miniapps don't have
1526
+ // to wait for the first change event.
1527
+ this.emitInitialSnapshots(packageName, streams)
1528
+ }
1529
+
1530
+ /**
1531
+ * For state-bearing streams (battery, connection), deliver the current value
1532
+ * immediately on subscribe. Other streams (button press, transcription, etc.)
1533
+ * are pure event streams with no "current value" to snapshot.
1534
+ */
1535
+ private emitInitialSnapshots(packageName: string, streams: string[]): void {
1536
+ // Read the engine glasses store directly (was a host glassesStatus hook).
1537
+ const gs = useGlassesStore.getState()
1538
+ const glassesState = {
1539
+ connected: isGlassesConnected(gs.connection),
1540
+ deviceModel: gs.deviceModel,
1541
+ batteryLevel: gs.batteryLevel,
1542
+ charging: gs.charging,
1543
+ }
1544
+ const isSimulated = (glassesState.deviceModel || "").toLowerCase().includes("simulated")
1545
+
1546
+ for (const stream of streams) {
1547
+ if (stream === "glasses_battery") {
1548
+ // Simulated glasses have no real battery; mirror the phone's battery
1549
+ // so miniapps see a sensible value during development.
1550
+ if (isSimulated) {
1551
+ void this.emitPhoneBatteryAs(packageName, "glasses_battery")
1552
+ } else if (typeof glassesState.batteryLevel === "number" && glassesState.batteryLevel >= 0) {
1553
+ this.sendToMiniapp(packageName, {
1554
+ type: MiniappResponseType.EVENT,
1555
+ streamType: "glasses_battery",
1556
+ data: {
1557
+ level: glassesState.batteryLevel,
1558
+ charging: !!glassesState.charging,
1559
+ timestamp: Date.now(),
1560
+ },
1561
+ })
1562
+ }
1563
+ } else if (stream === "phone_battery") {
1564
+ void this.emitPhoneBatteryAs(packageName, "phone_battery")
1565
+ } else if (stream === "glasses_connection") {
1566
+ this.sendToMiniapp(packageName, {
1567
+ type: MiniappResponseType.EVENT,
1568
+ streamType: "glasses_connection",
1569
+ data: glassesState,
1570
+ })
1571
+ } else if (stream === "glasses_wifi") {
1572
+ // Snapshot the current glasses Wi-Fi state on subscribe (like battery).
1573
+ // Read the canonical nested `wifi: {state, ssid}` from the STORE state
1574
+ // (`glassesState` is the narrowed 4-field projection and has no `wifi`;
1575
+ // the pre-engine host hook spread the full store, so read gs directly —
1576
+ // NOT the legacy flat wifiConnected). Effective connectivity requires the
1577
+ // glasses connected AND on Wi-Fi — mirrors the host's store-derived
1578
+ // forward, so an already-disconnected device reports `connected: false`
1579
+ // rather than silently emitting nothing. Always emits so onWifi gets an
1580
+ // initial value.
1581
+ const wifi = gs.wifi.state === "connected" ? gs.wifi : null
1582
+ const connected = glassesState.connected === true && wifi !== null
1583
+ this.sendToMiniapp(packageName, {
1584
+ type: MiniappResponseType.EVENT,
1585
+ streamType: "glasses_wifi",
1586
+ data: {
1587
+ connected,
1588
+ ssid: connected ? wifi?.ssid : undefined,
1589
+ localIp: connected ? wifi?.localIp : undefined,
1590
+ timestamp: Date.now(),
1591
+ },
1592
+ })
1593
+ } else if (stream === "head_position") {
1594
+ // The glasses store has no headUp field (same on the pre-engine host
1595
+ // hook), so this stays a no-emit until head state lands in the store.
1596
+ const headUp = (gs as {headUp?: boolean}).headUp
1597
+ if (typeof headUp === "boolean") {
1598
+ this.sendToMiniapp(packageName, {
1599
+ type: MiniappResponseType.EVENT,
1600
+ streamType: "head_position",
1601
+ data: {
1602
+ position: headUp ? "up" : "down",
1603
+ timestamp: Date.now(),
1604
+ },
1605
+ })
1606
+ }
1607
+ }
1608
+ }
1609
+ }
1610
+
1611
+ /**
1612
+ * Read the phone's battery state right now and emit it as the given stream.
1613
+ * Used for both phone_battery snapshot on subscribe, and as a stand-in for
1614
+ * glasses_battery when connected to Simulated Glasses.
1615
+ */
1616
+ private async emitPhoneBatteryAs(
1617
+ packageName: string,
1618
+ streamType: "phone_battery" | "glasses_battery",
1619
+ ): Promise<void> {
1620
+ try {
1621
+ const level = await Battery.getBatteryLevelAsync()
1622
+ const state = await Battery.getBatteryStateAsync()
1623
+ const charging = state === Battery.BatteryState.CHARGING || state === Battery.BatteryState.FULL
1624
+ this.sendToMiniapp(packageName, {
1625
+ type: MiniappResponseType.EVENT,
1626
+ streamType,
1627
+ data: {
1628
+ level: Math.round(level * 100),
1629
+ charging,
1630
+ timestamp: Date.now(),
1631
+ },
1632
+ })
1633
+ } catch (err) {
1634
+ console.log(`${LOG_TAG}: phone battery snapshot failed`, err)
1635
+ }
1636
+ }
1637
+
1638
+ private handleDisplay(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1639
+ try {
1640
+ // Zombie-frame gate: a dying miniapp's last in-flight frame can arrive
1641
+ // AFTER onUnmount cleared its display — repainting a dead app's UI (and
1642
+ // poisoning the next boot's pre-boot restore snapshot). Killed mid-route
1643
+ // nav reproduced this reliably; its 3s re-push raced the teardown.
1644
+ if (!this.connectedApps.has(packageName)) {
1645
+ console.log(`${LOG_TAG}: dropping display from unmounted app ${packageName}`)
1646
+ return
1647
+ }
1648
+ if (!payload.layout || typeof payload.layout !== "object") {
1649
+ this.sendResult(packageName, requestId, false, undefined, {
1650
+ code: MiniappErrorCode.INTERNAL,
1651
+ message: "display request missing layout object",
1652
+ })
1653
+ return
1654
+ }
1655
+
1656
+ // Hand off to LocalDisplayManager — it owns boot/throttle/arbitration/
1657
+ // expiry + the native BluetoothSdk.displayEvent call + useDisplayStore.
1658
+ localDisplayManager.request(packageName, {
1659
+ view: (payload.view as DisplayPayload["view"]) ?? "main",
1660
+ layout: payload.layout as DisplayPayload["layout"],
1661
+ durationMs: payload.durationMs as number | undefined,
1662
+ })
1663
+
1664
+ this.sendResult(packageName, requestId, true)
1665
+ } catch (err) {
1666
+ console.error(`${LOG_TAG}: display error:`, err)
1667
+ this.sendResult(packageName, requestId, false, undefined, {
1668
+ code: MiniappErrorCode.INTERNAL,
1669
+ message: err instanceof Error ? err.message : "Display error",
1670
+ })
1671
+ }
1672
+ }
1673
+
1674
+ /**
1675
+ * Scene render (session.display.render()). Same arbitration as DISPLAY, but
1676
+ * the payload is a whole frame of positioned elements and the caller may be
1677
+ * awaiting the outcome: LocalDisplayManager guarantees the resolver fires
1678
+ * exactly once (displayed/blocked, with degraded/dropped reporting), and that
1679
+ * becomes the REQUEST_RESULT data.
1680
+ */
1681
+ private handleRender(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1682
+ try {
1683
+ // Zombie-frame gate — see handleDisplay.
1684
+ if (!this.connectedApps.has(packageName)) {
1685
+ this.sendResult(packageName, requestId, true, {status: "blocked", reason: "app stopped"})
1686
+ return
1687
+ }
1688
+ if (!Array.isArray(payload.elements)) {
1689
+ this.sendResult(packageName, requestId, false, undefined, {
1690
+ code: MiniappErrorCode.INTERNAL,
1691
+ message: "render request missing elements array",
1692
+ })
1693
+ return
1694
+ }
1695
+
1696
+ localDisplayManager.request(
1697
+ packageName,
1698
+ {
1699
+ view: (payload.view as DisplayPayload["view"]) ?? "main",
1700
+ scene: payload.elements as DisplayPayload["scene"],
1701
+ durationMs: payload.durationMs as number | undefined,
1702
+ },
1703
+ (result) => this.sendResult(packageName, requestId, true, result),
1704
+ )
1705
+ } catch (err) {
1706
+ console.error(`${LOG_TAG}: render error:`, err)
1707
+ this.sendResult(packageName, requestId, false, undefined, {
1708
+ code: MiniappErrorCode.INTERNAL,
1709
+ message: err instanceof Error ? err.message : "Render error",
1710
+ })
1711
+ }
1712
+ }
1713
+
1714
+ private handlePlayAudio(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1715
+ const audioUrl = (payload.audioUrl ?? payload.url) as string | undefined
1716
+ if (!audioUrl) {
1717
+ this.sendResult(packageName, requestId, false, undefined, {
1718
+ code: MiniappErrorCode.INTERNAL,
1719
+ message: "play_audio requires an audioUrl",
1720
+ })
1721
+ return
1722
+ }
1723
+
1724
+ const audioRequestId = requestId || `local_${Date.now()}`
1725
+ const volume = typeof payload.volume === "number" ? payload.volume : 1.0
1726
+ const stopOtherAudio = payload.stopOtherAudio !== false
1727
+
1728
+ this.setSpeakerState(packageName, "loading")
1729
+ audioPlaybackService.play(
1730
+ {requestId: audioRequestId, audioUrl, appId: packageName, volume, stopOtherAudio},
1731
+ (_respId, success, error, duration) => {
1732
+ if (success) {
1733
+ this.setSpeakerState(packageName, "stopped", {durationMs: duration ?? undefined})
1734
+ } else {
1735
+ this.setSpeakerState(packageName, "error", {
1736
+ errorCode: MiniappErrorCode.INTERNAL,
1737
+ errorMessage: error ?? "play failed",
1738
+ durationMs: duration ?? undefined,
1739
+ })
1740
+ }
1741
+ this.sendResult(
1742
+ packageName,
1743
+ requestId,
1744
+ success,
1745
+ {duration},
1746
+ error
1747
+ ? {
1748
+ code: MiniappErrorCode.INTERNAL,
1749
+ message: error,
1750
+ }
1751
+ : undefined,
1752
+ )
1753
+ },
1754
+ )
1755
+ // Optimistic "started playing" transition. The audio service doesn't
1756
+ // have a "playback actually started" callback today; cloud SDK v3 has
1757
+ // the same constraint and uses optimistic timing. Microtask so the
1758
+ // initial "loading" envelope flushes first.
1759
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"))
1760
+ }
1761
+
1762
+ private handleStopAudio(packageName: string, _payload: Record<string, unknown>, requestId?: string): void {
1763
+ audioPlaybackService.stopForApp(packageName)
1764
+ this.setSpeakerState(packageName, "stopped")
1765
+ this.sendResult(packageName, requestId, true)
1766
+ }
1767
+
1768
+ // ── speaker.createStream (live PCM output) ────────────────────────────────
1769
+
1770
+ /** Valid PCM sample rates for speaker streams. Mirrors the SDK contract. */
1771
+ private static readonly SPEAKER_STREAM_SAMPLE_RATES = new Set([16000, 24000, 48000])
1772
+ private static readonly SPEAKER_STREAM_MAX_BASE64_CHARS = Math.ceil((256 * 1024) / 3) * 4
1773
+ private speakerStreamCounter = 0
1774
+
1775
+ private async handleSpeakerStreamOpen(
1776
+ packageName: string,
1777
+ payload: Record<string, unknown>,
1778
+ requestId?: string,
1779
+ ): Promise<void> {
1780
+ const app = this.connectedApps.get(packageName)
1781
+ if (!app) return
1782
+
1783
+ // One stream per miniapp: opening a second closes the first.
1784
+ const previous = app.activeSpeakerStreamId
1785
+ if (previous) {
1786
+ app.activeSpeakerStreamId = undefined
1787
+ await audioPlaybackService.abortStream(previous).catch((error) => {
1788
+ console.warn(`${LOG_TAG}: failed to abort previous speaker stream for ${packageName}`, error)
1789
+ })
1790
+ }
1791
+
1792
+ const sampleRate = typeof payload.sampleRate === "number" ? payload.sampleRate : 16000
1793
+ if (!LocalMiniappRuntime.SPEAKER_STREAM_SAMPLE_RATES.has(sampleRate)) {
1794
+ this.sendResult(packageName, requestId, false, undefined, {
1795
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1796
+ message: `unsupported sampleRate ${sampleRate} (use 16000, 24000, or 48000)`,
1797
+ })
1798
+ return
1799
+ }
1800
+ const channels = typeof payload.channels === "number" ? payload.channels : 1
1801
+ if (channels !== 1) {
1802
+ this.sendResult(packageName, requestId, false, undefined, {
1803
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1804
+ message: "speaker streams currently support mono PCM only",
1805
+ })
1806
+ return
1807
+ }
1808
+ const volume = typeof payload.volume === "number" ? payload.volume : 1
1809
+ if (!Number.isFinite(volume) || volume < 0 || volume > 1) {
1810
+ this.sendResult(packageName, requestId, false, undefined, {
1811
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1812
+ message: "speaker stream volume must be between 0 and 1",
1813
+ })
1814
+ return
1815
+ }
1816
+
1817
+ const streamId = `spkstream_${Date.now()}_${++this.speakerStreamCounter}`
1818
+ this.setSpeakerState(packageName, "loading")
1819
+ try {
1820
+ await audioPlaybackService.openStream({
1821
+ streamId,
1822
+ appId: packageName,
1823
+ sampleRate,
1824
+ channels,
1825
+ volume,
1826
+ stopOtherAudio: payload.stopOtherAudio !== false,
1827
+ onEnded: (endedId, success, error, durationMs) => {
1828
+ const current = this.connectedApps.get(packageName)
1829
+ // Ignore completion from an app incarnation replaced by a crash
1830
+ // respawn. Its abort may finish after the new app has registered and
1831
+ // must not overwrite that app's speaker state.
1832
+ if (current !== app) return
1833
+ if (current.activeSpeakerStreamId === endedId) current.activeSpeakerStreamId = undefined
1834
+ if (success) {
1835
+ this.setSpeakerState(packageName, "stopped", {durationMs: durationMs ?? undefined})
1836
+ } else {
1837
+ this.setSpeakerState(packageName, "error", {
1838
+ errorCode: MiniappErrorCode.INTERNAL,
1839
+ errorMessage: error ?? "speaker stream failed",
1840
+ durationMs: durationMs ?? undefined,
1841
+ })
1842
+ }
1843
+ },
1844
+ })
1845
+ } catch (error) {
1846
+ const message = error instanceof Error ? error.message : String(error)
1847
+ this.setSpeakerState(packageName, "error", {
1848
+ errorCode: MiniappErrorCode.INTERNAL,
1849
+ errorMessage: message,
1850
+ })
1851
+ this.sendResult(packageName, requestId, false, undefined, {
1852
+ code: MiniappErrorCode.INTERNAL,
1853
+ message,
1854
+ })
1855
+ return
1856
+ }
1857
+
1858
+ // The native open can outlive a miniapp disconnect. Its ordinary teardown
1859
+ // ran while no StreamState existed yet, so close this late result here
1860
+ // instead of leaving an orphan AudioTrack/AVAudioEngine session.
1861
+ if (this.connectedApps.get(packageName) !== app) {
1862
+ await audioPlaybackService.abortStream(streamId)
1863
+ return
1864
+ }
1865
+ app.activeSpeakerStreamId = streamId
1866
+ // Optimistic "playing", same as play(): the chunk player starts as soon
1867
+ // as the first write lands. Microtask so "loading" flushes first.
1868
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"))
1869
+ this.sendResult(packageName, requestId, true, {streamId})
1870
+ }
1871
+
1872
+ /** Resolve + validate a stream id from a payload against the app's active stream. */
1873
+ private activeSpeakerStream(packageName: string, payload: Record<string, unknown>): string | null {
1874
+ const streamId = typeof payload.streamId === "string" ? payload.streamId : undefined
1875
+ const app = this.connectedApps.get(packageName)
1876
+ if (!streamId || !app || app.activeSpeakerStreamId !== streamId) return null
1877
+ return streamId
1878
+ }
1879
+
1880
+ private async handleSpeakerStreamWrite(
1881
+ packageName: string,
1882
+ payload: Record<string, unknown>,
1883
+ requestId?: string,
1884
+ ): Promise<void> {
1885
+ const streamId = this.activeSpeakerStream(packageName, payload)
1886
+ if (!streamId) {
1887
+ this.sendResult(packageName, requestId, false, undefined, {
1888
+ code: MiniappErrorCode.INTERNAL,
1889
+ message: "speaker stream is not open",
1890
+ })
1891
+ return
1892
+ }
1893
+ const base64 = typeof payload.base64 === "string" ? payload.base64 : ""
1894
+ if (!base64 || base64.length % 4 !== 0) {
1895
+ this.sendResult(packageName, requestId, false, undefined, {
1896
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1897
+ message: "speaker stream write requires padded base64 PCM data",
1898
+ })
1899
+ return
1900
+ }
1901
+ if (base64.length > LocalMiniappRuntime.SPEAKER_STREAM_MAX_BASE64_CHARS) {
1902
+ this.sendResult(packageName, requestId, false, undefined, {
1903
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
1904
+ message: "speaker stream chunk exceeds the 256 KB raw PCM limit",
1905
+ })
1906
+ return
1907
+ }
1908
+ try {
1909
+ const result = await audioPlaybackService.writeStreamChunk(streamId, base64)
1910
+ this.sendResult(packageName, requestId, true, result)
1911
+ } catch (error) {
1912
+ this.sendResult(packageName, requestId, false, undefined, {
1913
+ code: MiniappErrorCode.INTERNAL,
1914
+ message: error instanceof Error ? error.message : String(error),
1915
+ })
1916
+ }
1917
+ }
1918
+
1919
+ private async handleSpeakerStreamClose(
1920
+ packageName: string,
1921
+ payload: Record<string, unknown>,
1922
+ requestId?: string,
1923
+ ): Promise<void> {
1924
+ const streamId = this.activeSpeakerStream(packageName, payload)
1925
+ if (!streamId) {
1926
+ this.sendResult(packageName, requestId, false, undefined, {
1927
+ code: MiniappErrorCode.INTERNAL,
1928
+ message: "speaker stream is not open",
1929
+ })
1930
+ return
1931
+ }
1932
+ try {
1933
+ // Resolves once the buffered audio has drained; onEnded delivers the
1934
+ // terminal SPEAKER_STATE ("stopped").
1935
+ const result = await audioPlaybackService.closeStream(streamId)
1936
+ this.sendResult(packageName, requestId, true, result)
1937
+ } catch (error) {
1938
+ this.sendResult(packageName, requestId, false, undefined, {
1939
+ code: MiniappErrorCode.INTERNAL,
1940
+ message: error instanceof Error ? error.message : String(error),
1941
+ })
1942
+ }
1943
+ }
1944
+
1945
+ private async handleSpeakerStreamAbort(
1946
+ packageName: string,
1947
+ payload: Record<string, unknown>,
1948
+ requestId?: string,
1949
+ ): Promise<void> {
1950
+ const streamId = this.activeSpeakerStream(packageName, payload)
1951
+ if (!streamId) {
1952
+ // Aborting an already-gone stream is fine — abort() is idempotent.
1953
+ this.sendResult(packageName, requestId, true)
1954
+ return
1955
+ }
1956
+ try {
1957
+ await audioPlaybackService.abortStream(streamId)
1958
+ this.sendResult(packageName, requestId, true)
1959
+ } catch (error) {
1960
+ this.sendResult(packageName, requestId, false, undefined, {
1961
+ code: MiniappErrorCode.INTERNAL,
1962
+ message: error instanceof Error ? error.message : String(error),
1963
+ })
1964
+ }
1965
+ }
1966
+
1967
+ private async handleSpeak(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
1968
+ const text = payload.text as string | undefined
1969
+ if (!text) {
1970
+ this.sendResult(packageName, requestId, false, undefined, {
1971
+ code: MiniappErrorCode.INTERNAL,
1972
+ message: "speak requires text",
1973
+ })
1974
+ return
1975
+ }
1976
+
1977
+ try {
1978
+ const voice = ((payload.voice_id ?? payload.voice) as string) || "default"
1979
+ const audioRequestId = requestId || `tts_${Date.now()}`
1980
+ const volume = typeof payload.volume === "number" ? payload.volume : 1.0
1981
+ const stopOtherAudio = payload.stopOtherAudio !== false
1982
+ const voiceSettings =
1983
+ payload.voice_settings && typeof payload.voice_settings === "object"
1984
+ ? (payload.voice_settings as Record<string, unknown>)
1985
+ : undefined
1986
+ const speed = typeof voiceSettings?.speed === "number" ? voiceSettings.speed : undefined
1987
+
1988
+ this.setSpeakerState(packageName, "loading")
1989
+
1990
+ const voiceExplicit = payload.voice_id !== undefined || payload.voice !== undefined
1991
+ const offlineSupportsVoice =
1992
+ !voiceExplicit || voice === "default" || ttsModelManager.getAvailableLanguages().some((l) => l.code === voice)
1993
+
1994
+ const modelId = typeof payload.model_id === "string" ? payload.model_id : undefined
1995
+ const cloudConnected = cloudClientService.isConnected()
1996
+ console.log(
1997
+ `${LOG_TAG}: TTS decision for ${packageName}: cloudConnected=${cloudConnected}, runtimeTts=yes, offlineVoice=${offlineSupportsVoice}`,
1998
+ )
1999
+
2000
+ let terminalSent = false
2001
+ const sendPlaybackResult = (
2002
+ success: boolean,
2003
+ error: string | null,
2004
+ duration: number | null,
2005
+ fallbackErrorMessage: string,
2006
+ ) => {
2007
+ if (terminalSent) return
2008
+ terminalSent = true
2009
+ if (success) {
2010
+ this.setSpeakerState(packageName, "stopped", {durationMs: duration ?? undefined})
2011
+ } else {
2012
+ this.setSpeakerState(packageName, "error", {
2013
+ errorCode: MiniappErrorCode.TTS_UPSTREAM_ERROR,
2014
+ errorMessage: error ?? fallbackErrorMessage,
2015
+ durationMs: duration ?? undefined,
2016
+ })
2017
+ }
2018
+ this.sendResult(
2019
+ packageName,
2020
+ requestId,
2021
+ success,
2022
+ {completed: success, duration},
2023
+ error
2024
+ ? {
2025
+ code: MiniappErrorCode.TTS_UPSTREAM_ERROR,
2026
+ message: error,
2027
+ }
2028
+ : undefined,
2029
+ )
2030
+ }
2031
+
2032
+ const playOfflineTts = async (reason?: string): Promise<boolean> => {
2033
+ if (!offlineSupportsVoice || !(await ttsModelManager.isModelAvailable())) {
2034
+ return false
2035
+ }
2036
+
2037
+ let offlineGenerated: TtsSynthesisResult | undefined
2038
+
2039
+ try {
2040
+ const languageCode = ttsModelManager.getAvailableLanguages().some((l) => l.code === voice) ? voice : undefined
2041
+ offlineGenerated = await ttsModelManager.synthesizeToFile(text, {languageCode, speed})
2042
+ } catch (offlineErr) {
2043
+ console.warn(`${LOG_TAG}: offline TTS synthesize failed${reason ? ` after ${reason}` : ""}:`, offlineErr)
2044
+ return false
2045
+ }
2046
+
2047
+ const generated = offlineGenerated
2048
+ audioPlaybackService.play(
2049
+ {requestId: audioRequestId, audioUrl: generated.audioUrl, appId: packageName, volume, stopOtherAudio},
2050
+ (_respId, success, error, duration) => {
2051
+ void Promise.resolve(generated.cleanup?.()).catch((cleanupError) => {
2052
+ console.warn(`${LOG_TAG}: offline TTS cleanup failed`, cleanupError)
2053
+ })
2054
+ sendPlaybackResult(success, error, duration, "offline tts playback failed")
2055
+ },
2056
+ )
2057
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"))
2058
+ return true
2059
+ }
2060
+
2061
+ const playCloudTts = async (fallbackToOffline: boolean): Promise<boolean> => {
2062
+ let source: Awaited<ReturnType<typeof cloudClientService.tts.speak>>
2063
+ try {
2064
+ source = await cloudClientService.tts.speak(text, {
2065
+ ...(voiceExplicit && voice !== "default" ? {voice_id: voice} : {}),
2066
+ ...(modelId ? {model_id: modelId} : {}),
2067
+ ...(voiceSettings ? {voice_settings: voiceSettings} : {}),
2068
+ })
2069
+ } catch (cloudErr) {
2070
+ const error = cloudErr instanceof Error ? cloudErr.message : String(cloudErr)
2071
+ console.warn(`${LOG_TAG}: cloud TTS source failed: ${error}`)
2072
+ if (fallbackToOffline && (await playOfflineTts("cloud tts source failed"))) {
2073
+ return true
2074
+ }
2075
+ sendPlaybackResult(false, error, null, "tts failed")
2076
+ return true
2077
+ }
2078
+
2079
+ await Promise.resolve(
2080
+ audioPlaybackService.play(
2081
+ {requestId: audioRequestId, audioUrl: source.audioUrl, appId: packageName, volume, stopOtherAudio},
2082
+ (_respId, success, error, duration) => {
2083
+ if (!success && fallbackToOffline) {
2084
+ void playOfflineTts("cloud tts playback failed").then((started) => {
2085
+ if (!started) {
2086
+ sendPlaybackResult(false, error, duration, "tts failed")
2087
+ }
2088
+ })
2089
+ return
2090
+ }
2091
+ sendPlaybackResult(success, error, duration, "tts failed")
2092
+ },
2093
+ ),
2094
+ )
2095
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"))
2096
+ return true
2097
+ }
2098
+
2099
+ if (cloudConnected && (await playCloudTts(true))) {
2100
+ return
2101
+ }
2102
+
2103
+ // Offline is the disconnected-cloud fallback. If the model is not
2104
+ // available, preserve the old behavior by trying cloud as a last resort.
2105
+ if (await playOfflineTts(cloudConnected ? undefined : "cloud disconnected")) {
2106
+ return
2107
+ }
2108
+
2109
+ if (await playCloudTts(false)) {
2110
+ return
2111
+ }
2112
+
2113
+ const message = "tts unavailable: no cloud TTS URL or offline TTS model"
2114
+ this.setSpeakerState(packageName, "error", {
2115
+ errorCode: MiniappErrorCode.TTS_UPSTREAM_ERROR,
2116
+ errorMessage: message,
2117
+ })
2118
+ this.sendResult(packageName, requestId, false, undefined, {
2119
+ code: MiniappErrorCode.TTS_UPSTREAM_ERROR,
2120
+ message,
2121
+ })
2122
+ } catch (err) {
2123
+ console.error(`${LOG_TAG}: speak error:`, err)
2124
+ const message = err instanceof Error ? err.message : "TTS error"
2125
+ this.setSpeakerState(packageName, "error", {
2126
+ errorCode: MiniappErrorCode.TTS_UPSTREAM_ERROR,
2127
+ errorMessage: message,
2128
+ })
2129
+ this.sendResult(packageName, requestId, false, undefined, {
2130
+ code: MiniappErrorCode.TTS_UPSTREAM_ERROR,
2131
+ message,
2132
+ })
2133
+ }
2134
+ }
2135
+
2136
+ private async handleRgbLed(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
2137
+ const coerceNumber = (value: unknown, fallback: number): number => {
2138
+ const coerced = Number(value)
2139
+ return Number.isFinite(coerced) ? coerced : fallback
2140
+ }
2141
+
2142
+ const ledRequestId = requestId || `led_${Date.now()}`
2143
+ const action = normalizeRgbLedAction(payload.action)
2144
+ const color = normalizeRgbLedColor(payload.color)
2145
+
2146
+ try {
2147
+ const result = await BluetoothSdk.rgbLedControl(
2148
+ ledRequestId,
2149
+ packageName,
2150
+ action,
2151
+ color,
2152
+ coerceNumber(payload.ontime, 1000),
2153
+ coerceNumber(payload.offtime, 0),
2154
+ coerceNumber(payload.count, 1),
2155
+ )
2156
+ this.sendResult(packageName, requestId, true, result)
2157
+ } catch (err) {
2158
+ this.sendResult(packageName, requestId, false, undefined, {
2159
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2160
+ message: err instanceof Error ? err.message : "RGB LED command failed",
2161
+ })
2162
+ }
2163
+ }
2164
+
2165
+ private async handleLocationPoll(packageName: string, requestId?: string): Promise<void> {
2166
+ try {
2167
+ const {status} = await Location.requestForegroundPermissionsAsync()
2168
+ if (status !== "granted") {
2169
+ this.sendResult(packageName, requestId, false, undefined, {
2170
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2171
+ message: "Location permission not granted",
2172
+ })
2173
+ return
2174
+ }
2175
+
2176
+ // Try the OS cache first — usually instant — and only fall back to
2177
+ // a fresh fix if the cache is empty. Use Low accuracy on the fresh
2178
+ // path so we get a cell/wifi-tower fix in ~1s instead of waiting
2179
+ // for full GPS warm-up.
2180
+ const cached = await Location.getLastKnownPositionAsync({maxAge: 60_000})
2181
+ const location = cached ?? (await Location.getCurrentPositionAsync({accuracy: Location.Accuracy.Low}))
2182
+
2183
+ this.sendResult(packageName, requestId, true, {
2184
+ lat: location.coords.latitude,
2185
+ lng: location.coords.longitude,
2186
+ accuracy: location.coords.accuracy ?? undefined,
2187
+ timestamp: location.timestamp,
2188
+ })
2189
+ } catch (err) {
2190
+ console.error(`${LOG_TAG}: location poll error:`, err)
2191
+ this.sendResult(packageName, requestId, false, undefined, {
2192
+ code: MiniappErrorCode.INTERNAL,
2193
+ message: err instanceof Error ? err.message : "Location error",
2194
+ })
2195
+ }
2196
+ }
2197
+
2198
+ // ---------------------------------------------------------------------------
2199
+ // Navigation
2200
+ // ---------------------------------------------------------------------------
2201
+
2202
+ /**
2203
+ * All navigation handlers + their state live in NavigationHandlers. The
2204
+ * dispatcher's NAVIGATION_* cases delegate here; the rest of the runtime
2205
+ * cross-cuts navigation through {@link NavigationHandlers.onDisconnect}
2206
+ * (mini-app disconnect) and {@link NavigationHandlers.isTripActive}
2207
+ * (location-stream gating during a trip).
2208
+ */
2209
+ private readonly navigationHandlers = new NavigationHandlers(
2210
+ (packageName, envelope) => this.sendToMiniapp(packageName, envelope),
2211
+ (packageName, requestId, ok, result, error) => this.sendResult(packageName, requestId, ok, result, error),
2212
+ )
2213
+
2214
+ /**
2215
+ * `session.blob` — persistent, per-app binary storage. Like NavigationHandlers,
2216
+ * the BLOB_* dispatcher cases delegate here. It's a generic byte store; audio
2217
+ * recording lives entirely in the miniapp (mic.onAudioChunk → chunked
2218
+ * BLOB_WRITE), so nothing audio-specific runs in the host.
2219
+ */
2220
+ private readonly blobStore = new BlobStore({
2221
+ sendResult: (packageName, requestId, ok, result, error) =>
2222
+ this.sendResult(packageName, requestId, ok, result, error),
2223
+ getUserId: () =>
2224
+ (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous",
2225
+ })
2226
+
2227
+ /**
2228
+ * Heading is a sensor stream — start the native compass when any mini
2229
+ * app is subscribed to "heading_update", stop it when none are.
2230
+ */
2231
+ private headingUnsub: (() => void) | null = null
2232
+ private recomputeHeadingSubscription(): void {
2233
+ const wantsHeading = this.streamSubscribers.has(MiniappStreamType.HEADING_UPDATE)
2234
+ if (wantsHeading && !this.headingUnsub) {
2235
+ // Heading sensor lives in engine (HeadingService) — subscribe directly.
2236
+ this.headingUnsub = headingService.addListener((degrees: number) => {
2237
+ this.forwardEvent(MiniappStreamType.HEADING_UPDATE, {degrees})
2238
+ })
2239
+ } else if (!wantsHeading && this.headingUnsub) {
2240
+ this.headingUnsub()
2241
+ this.headingUnsub = null
2242
+ }
2243
+ }
2244
+
2245
+ /**
2246
+ * Accelerometer is a sensor stream — enable the glasses IMU when any mini
2247
+ * app is subscribed to "accel_data", disable it when none are. Mirrors the
2248
+ * heading pattern. Only G2 streams IMU today; on other devices the native
2249
+ * call is a no-op so this is harmless.
2250
+ */
2251
+ private imuEnabled = false
2252
+ private recomputeImuSubscription(): void {
2253
+ const wantsImu = this.streamSubscribers.has(MiniappStreamType.ACCEL_DATA)
2254
+ if (wantsImu === this.imuEnabled) return
2255
+ this.imuEnabled = wantsImu
2256
+ void BluetoothSdk.setImuEnabled(wantsImu)
2257
+ }
2258
+
2259
+ /**
2260
+ * Last rate we asked the host for. Lets us skip the cross-process call
2261
+ * when nothing actually changed (most SUBSCRIBE / unregister churn
2262
+ * leaves the aggregate unchanged) and tracks state for the no-active-
2263
+ * subscribers → tell host to back off case.
2264
+ */
2265
+ private lastAppliedLocationRate: LocationRate | null = null
2266
+
2267
+ /**
2268
+ * Recompute the aggregate location tier across every connected
2269
+ * miniapp and push the result to the host. The aggregate is the
2270
+ * STRICTEST rate any single app has requested — see
2271
+ * {@link LOCATION_RATE_PRIORITY} — because the GPS sample rate is a
2272
+ * shared OS-level setting that we can't bias per-app.
2273
+ *
2274
+ * Called after every SUBSCRIBE and every unregister so that when a
2275
+ * realtime-requesting app goes away the tier falls back down. If no
2276
+ * connected app is asking for a location rate, we tell the host
2277
+ * "off" so it can drop GPS power.
2278
+ */
2279
+ private recomputeLocationTier(): void {
2280
+ let bestRank = -1
2281
+ for (const app of this.connectedApps.values()) {
2282
+ const r = app.requestedLocationRate
2283
+ if (!r) continue
2284
+ const rank = locationRateRank(r)
2285
+ if (rank > bestRank) {
2286
+ bestRank = rank
2287
+ }
2288
+ }
2289
+ // When no app is asking, downgrade to "off". Without this we'd
2290
+ // leak the last-known-strictest rate forever (the original bug).
2291
+ const next: LocationRate = bestRank >= 0 ? LOCATION_RATE_PRIORITY[bestRank] : "off"
2292
+ if (next === this.lastAppliedLocationRate) return
2293
+ this.lastAppliedLocationRate = next
2294
+ console.log(`[LOCATION] aggregate tier → ${next}`)
2295
+ void phoneLocationService.setLocationTier(next)
2296
+ }
2297
+
2298
+ // ---------------------------------------------------------------------------
2299
+ // Storage helpers
2300
+ // ---------------------------------------------------------------------------
2301
+
2302
+ private getStorageKeyPrefix(packageName: string): string {
2303
+ const userId =
2304
+ (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous"
2305
+ return `mentraos_localstorage_${userId}_${packageName}_`
2306
+ }
2307
+
2308
+ private async handleStorageGet(
2309
+ packageName: string,
2310
+ payload: Record<string, unknown>,
2311
+ requestId?: string,
2312
+ ): Promise<void> {
2313
+ try {
2314
+ const key = payload.key as string
2315
+ if (!key) {
2316
+ this.sendResult(packageName, requestId, false, undefined, {
2317
+ code: MiniappErrorCode.INTERNAL,
2318
+ message: "storage_get requires a key",
2319
+ })
2320
+ return
2321
+ }
2322
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2323
+ const result = mmkvStorage.load<unknown>(fullKey)
2324
+ this.sendResult(packageName, requestId, true, {key, value: result.is_ok() ? result.value : null})
2325
+ } catch (err) {
2326
+ console.error(`${LOG_TAG}: storage_get error:`, err)
2327
+ this.sendResult(packageName, requestId, false, undefined, {
2328
+ code: MiniappErrorCode.INTERNAL,
2329
+ message: err instanceof Error ? err.message : "Storage error",
2330
+ })
2331
+ }
2332
+ }
2333
+
2334
+ private async handleStorageSet(
2335
+ packageName: string,
2336
+ payload: Record<string, unknown>,
2337
+ requestId?: string,
2338
+ ): Promise<void> {
2339
+ try {
2340
+ const key = payload.key as string
2341
+ if (!key) {
2342
+ this.sendResult(packageName, requestId, false, undefined, {
2343
+ code: MiniappErrorCode.INTERNAL,
2344
+ message: "storage_set requires a key",
2345
+ })
2346
+ return
2347
+ }
2348
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2349
+ mmkvStorage.save(fullKey, payload.value ?? null)
2350
+ this.sendResult(packageName, requestId, true)
2351
+ } catch (err) {
2352
+ console.error(`${LOG_TAG}: storage_set error:`, err)
2353
+ this.sendResult(packageName, requestId, false, undefined, {
2354
+ code: MiniappErrorCode.INTERNAL,
2355
+ message: err instanceof Error ? err.message : "Storage error",
2356
+ })
2357
+ }
2358
+ }
2359
+
2360
+ private async handleStorageDelete(
2361
+ packageName: string,
2362
+ payload: Record<string, unknown>,
2363
+ requestId?: string,
2364
+ ): Promise<void> {
2365
+ try {
2366
+ const key = payload.key as string
2367
+ if (!key) {
2368
+ this.sendResult(packageName, requestId, false, undefined, {
2369
+ code: MiniappErrorCode.INTERNAL,
2370
+ message: "storage_delete requires a key",
2371
+ })
2372
+ return
2373
+ }
2374
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2375
+ mmkvStorage.remove(fullKey)
2376
+ this.sendResult(packageName, requestId, true)
2377
+ } catch (err) {
2378
+ console.error(`${LOG_TAG}: storage_delete error:`, err)
2379
+ this.sendResult(packageName, requestId, false, undefined, {
2380
+ code: MiniappErrorCode.INTERNAL,
2381
+ message: err instanceof Error ? err.message : "Storage error",
2382
+ })
2383
+ }
2384
+ }
2385
+
2386
+ private async handleStorageList(
2387
+ packageName: string,
2388
+ _payload: Record<string, unknown>,
2389
+ requestId?: string,
2390
+ ): Promise<void> {
2391
+ try {
2392
+ const prefix = this.getStorageKeyPrefix(packageName)
2393
+ const allKeys = mmkvStorage.getAllKeys()
2394
+ const keys = allKeys.filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length))
2395
+ this.sendResult(packageName, requestId, true, {keys})
2396
+ } catch (err) {
2397
+ console.error(`${LOG_TAG}: storage_list error:`, err)
2398
+ this.sendResult(packageName, requestId, false, undefined, {
2399
+ code: MiniappErrorCode.INTERNAL,
2400
+ message: err instanceof Error ? err.message : "Storage error",
2401
+ })
2402
+ }
2403
+ }
2404
+
2405
+ private async handleStorageClear(packageName: string, requestId?: string): Promise<void> {
2406
+ try {
2407
+ const prefix = this.getStorageKeyPrefix(packageName)
2408
+ const allKeys = mmkvStorage.getAllKeys()
2409
+ for (const k of allKeys) {
2410
+ if (k.startsWith(prefix)) mmkvStorage.remove(k)
2411
+ }
2412
+ this.sendResult(packageName, requestId, true)
2413
+ } catch (err) {
2414
+ console.error(`${LOG_TAG}: storage_clear error:`, err)
2415
+ this.sendResult(packageName, requestId, false, undefined, {
2416
+ code: MiniappErrorCode.INTERNAL,
2417
+ message: err instanceof Error ? err.message : "Storage error",
2418
+ })
2419
+ }
2420
+ }
2421
+
2422
+ private async handleStorageHas(
2423
+ packageName: string,
2424
+ payload: Record<string, unknown>,
2425
+ requestId?: string,
2426
+ ): Promise<void> {
2427
+ try {
2428
+ const key = payload.key as string
2429
+ if (!key) {
2430
+ this.sendResult(packageName, requestId, false, undefined, {
2431
+ code: MiniappErrorCode.INTERNAL,
2432
+ message: "storage_has requires a key",
2433
+ })
2434
+ return
2435
+ }
2436
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2437
+ const result = mmkvStorage.load<unknown>(fullKey)
2438
+ this.sendResult(packageName, requestId, true, {has: result.is_ok()})
2439
+ } catch (err) {
2440
+ console.error(`${LOG_TAG}: storage_has error:`, err)
2441
+ this.sendResult(packageName, requestId, false, undefined, {
2442
+ code: MiniappErrorCode.INTERNAL,
2443
+ message: err instanceof Error ? err.message : "Storage error",
2444
+ })
2445
+ }
2446
+ }
2447
+
2448
+ private async handleStorageGetAll(packageName: string, requestId?: string): Promise<void> {
2449
+ try {
2450
+ const prefix = this.getStorageKeyPrefix(packageName)
2451
+ const allKeys = mmkvStorage.getAllKeys()
2452
+ const values: Record<string, string> = {}
2453
+ for (const k of allKeys) {
2454
+ if (!k.startsWith(prefix)) continue
2455
+ const r = mmkvStorage.load<unknown>(k)
2456
+ if (r.is_ok()) {
2457
+ const v = r.value
2458
+ values[k.slice(prefix.length)] = typeof v === "string" ? v : String(v ?? "")
2459
+ }
2460
+ }
2461
+ this.sendResult(packageName, requestId, true, {values})
2462
+ } catch (err) {
2463
+ console.error(`${LOG_TAG}: storage_get_all error:`, err)
2464
+ this.sendResult(packageName, requestId, false, undefined, {
2465
+ code: MiniappErrorCode.INTERNAL,
2466
+ message: err instanceof Error ? err.message : "Storage error",
2467
+ })
2468
+ }
2469
+ }
2470
+
2471
+ private async handleStorageSetMultiple(
2472
+ packageName: string,
2473
+ payload: Record<string, unknown>,
2474
+ requestId?: string,
2475
+ ): Promise<void> {
2476
+ try {
2477
+ const values = payload.values as Record<string, unknown> | undefined
2478
+ if (!values || typeof values !== "object") {
2479
+ this.sendResult(packageName, requestId, false, undefined, {
2480
+ code: MiniappErrorCode.INTERNAL,
2481
+ message: "storage_set_multiple requires a values object",
2482
+ })
2483
+ return
2484
+ }
2485
+ const prefix = this.getStorageKeyPrefix(packageName)
2486
+ for (const [key, value] of Object.entries(values)) {
2487
+ mmkvStorage.save(prefix + key, value ?? null)
2488
+ }
2489
+ this.sendResult(packageName, requestId, true)
2490
+ } catch (err) {
2491
+ console.error(`${LOG_TAG}: storage_set_multiple error:`, err)
2492
+ this.sendResult(packageName, requestId, false, undefined, {
2493
+ code: MiniappErrorCode.INTERNAL,
2494
+ message: err instanceof Error ? err.message : "Storage error",
2495
+ })
2496
+ }
2497
+ }
2498
+
2499
+ // ---------------------------------------------------------------------------
2500
+ // Camera FOV
2501
+ // ---------------------------------------------------------------------------
2502
+
2503
+ private async handleCameraFov(
2504
+ packageName: string,
2505
+ payload: Record<string, unknown>,
2506
+ requestId?: string,
2507
+ ): Promise<void> {
2508
+ const app = this.connectedApps.get(packageName)
2509
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2510
+ if (!hasCameraPermission) {
2511
+ logPermissionNotDeclared(packageName, "CAMERA", "to set camera FOV", `{"type": "CAMERA"}`)
2512
+ this.sendResult(packageName, requestId, false, undefined, {
2513
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2514
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2515
+ permission: "CAMERA",
2516
+ operation: MiniappRequestType.CAMERA_FOV,
2517
+ })
2518
+ return
2519
+ }
2520
+
2521
+ try {
2522
+ const request = normalizeCameraFovPayload(payload)
2523
+ const description =
2524
+ "preset" in request ? `preset=${request.preset}` : `fov=${request.fov} roi=${request.roiPosition ?? "center"}`
2525
+ console.log(`${LOG_TAG}: camera_fov_set ${description}`)
2526
+
2527
+ // Camera FOV is a direct btsdk call now (was the host-injected `cameraSettings`
2528
+ // runtime hook — a pure BluetoothSdk.setCameraFov passthrough with no host coupling).
2529
+ const result = await BluetoothSdk.setCameraFov(request)
2530
+
2531
+ useSettingsStore
2532
+ .getState()
2533
+ .setSetting(
2534
+ ISLAND_SETTINGS_KEYS.cameraFov,
2535
+ {fov: result.fov, roi_position: CAMERA_ROI_POSITION_VALUES[result.roiPosition]},
2536
+ false,
2537
+ )
2538
+ this.sendResult(packageName, requestId, true, result)
2539
+ } catch (err) {
2540
+ console.error(`${LOG_TAG}: camera_fov error:`, err)
2541
+ this.sendResult(packageName, requestId, false, undefined, {
2542
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2543
+ message: err instanceof Error ? err.message : "Camera FOV error",
2544
+ })
2545
+ }
2546
+ }
2547
+
2548
+ /**
2549
+ * Explicit IMU enable/disable from a miniapp (session.imu.setEnabled).
2550
+ *
2551
+ * The accel stream already auto-toggles the native IMU on subscribe (see
2552
+ * {@link recomputeImuSubscription}); this is a direct override. We sync
2553
+ * `this.imuEnabled` to the requested state so a subsequent subscription
2554
+ * recompute with the same aggregate doesn't issue a redundant native call.
2555
+ * Note that a later subscribe/unsubscribe that changes the aggregate will
2556
+ * still re-derive the sensor state from subscriptions.
2557
+ */
2558
+ private handleImuSetEnabled(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
2559
+ try {
2560
+ const enabled = !!payload.enabled
2561
+ console.log(`${LOG_TAG}: imu_set_enabled ${enabled} (by ${packageName})`)
2562
+ this.imuEnabled = enabled
2563
+ void BluetoothSdk.setImuEnabled(enabled)
2564
+ this.sendResult(packageName, requestId, true)
2565
+ } catch (err) {
2566
+ console.error(`${LOG_TAG}: imu_set_enabled error:`, err)
2567
+ this.sendResult(packageName, requestId, false, undefined, {
2568
+ code: MiniappErrorCode.INTERNAL,
2569
+ message: err instanceof Error ? err.message : "IMU set-enabled error",
2570
+ })
2571
+ }
2572
+ }
2573
+
2574
+ // ---------------------------------------------------------------------------
2575
+ // System utilities (share, open URL, clipboard, download)
2576
+ // ---------------------------------------------------------------------------
2577
+
2578
+ private async handleShare(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
2579
+ const {text, title, base64, mimeType, filename, url} = payload as {
2580
+ text?: string
2581
+ title?: string
2582
+ base64?: string
2583
+ mimeType?: string
2584
+ filename?: string
2585
+ url?: string
2586
+ }
2587
+ try {
2588
+ if (base64) {
2589
+ // File share via base64 — write to temp file then share
2590
+ const tempFile = new File(Paths.cache, filename || "shared_file")
2591
+ tempFile.write(base64, {encoding: "base64"})
2592
+ await Share.open({
2593
+ url: tempFile.uri,
2594
+ type: mimeType || "application/octet-stream",
2595
+ filename: filename,
2596
+ title: title,
2597
+ })
2598
+ } else if (url) {
2599
+ await Share.open({url, title, message: text})
2600
+ } else {
2601
+ await Share.open({message: text || "", title})
2602
+ }
2603
+ this.sendResult(packageName, requestId, true, {success: true})
2604
+ } catch (error: any) {
2605
+ // react-native-share throws when user dismisses the share sheet
2606
+ if (error?.message?.includes("User did not share")) {
2607
+ this.sendResult(packageName, requestId, true, {success: false, cancelled: true})
2608
+ } else {
2609
+ console.error(`${LOG_TAG}: share error:`, error)
2610
+ this.sendResult(packageName, requestId, true, {success: false})
2611
+ }
2612
+ }
2613
+ }
2614
+
2615
+ private async handleOpenUrl(packageName: string, payload: Record<string, unknown>): Promise<void> {
2616
+ const url = payload.url as string | undefined
2617
+ if (!url || typeof url !== "string") {
2618
+ console.warn(`${LOG_TAG}: open_url missing url`)
2619
+ return
2620
+ }
2621
+ // Block dangerous schemes
2622
+ if (url.startsWith("javascript:") || url.startsWith("file:")) {
2623
+ console.warn(`${LOG_TAG}: open_url blocked dangerous scheme: ${url}`)
2624
+ return
2625
+ }
2626
+ try {
2627
+ await Linking.openURL(url)
2628
+ } catch (error) {
2629
+ console.error(`${LOG_TAG}: open_url error:`, error)
2630
+ }
2631
+ }
2632
+
2633
+ private async handleCopyClipboard(
2634
+ packageName: string,
2635
+ payload: Record<string, unknown>,
2636
+ requestId?: string,
2637
+ ): Promise<void> {
2638
+ const text = payload.text as string | undefined
2639
+ if (typeof text !== "string") {
2640
+ console.warn(`${LOG_TAG}: copy_clipboard missing text`)
2641
+ return
2642
+ }
2643
+ try {
2644
+ await Clipboard.setStringAsync(text)
2645
+ this.sendResult(packageName, requestId, true)
2646
+ } catch (error: any) {
2647
+ console.error(`${LOG_TAG}: clipboard error:`, error)
2648
+ this.sendResult(packageName, requestId, false, undefined, {
2649
+ code: MiniappErrorCode.INTERNAL,
2650
+ message: error?.message || "Clipboard error",
2651
+ })
2652
+ }
2653
+ }
2654
+
2655
+ private async handleDownload(
2656
+ packageName: string,
2657
+ payload: Record<string, unknown>,
2658
+ requestId?: string,
2659
+ ): Promise<void> {
2660
+ const {base64, url, mimeType, filename} = payload as {
2661
+ base64?: string
2662
+ url?: string
2663
+ mimeType?: string
2664
+ filename?: string
2665
+ }
2666
+ const name = filename || "download"
2667
+ try {
2668
+ let file: File
2669
+ if (base64) {
2670
+ file = new File(Paths.cache, name)
2671
+ file.write(base64, {encoding: "base64"})
2672
+ } else if (url) {
2673
+ file = await File.downloadFileAsync(url, new File(Paths.cache, name), {idempotent: true})
2674
+ } else {
2675
+ console.warn(`${LOG_TAG}: download missing base64 or url`)
2676
+ return
2677
+ }
2678
+ // Open share sheet so user can choose where to save
2679
+ await Share.open({
2680
+ url: file.uri,
2681
+ type: mimeType || "application/octet-stream",
2682
+ filename: name,
2683
+ })
2684
+ this.sendResult(packageName, requestId, true, {success: true})
2685
+ } catch (error: any) {
2686
+ if (error?.message?.includes("User did not share")) {
2687
+ this.sendResult(packageName, requestId, true, {success: true, cancelled: true})
2688
+ } else {
2689
+ console.error(`${LOG_TAG}: download error:`, error)
2690
+ this.sendResult(packageName, requestId, true, {success: false})
2691
+ }
2692
+ }
2693
+ }
2694
+
2695
+ // ===========================================================================
2696
+ // Photo + streaming handlers (cloud-coordinated)
2697
+ // ===========================================================================
2698
+
2699
+ private async handlePhoto(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
2700
+ // Manifest CAMERA permission gate.
2701
+ const app = this.connectedApps.get(packageName)
2702
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2703
+ if (!hasCameraPermission) {
2704
+ logPermissionNotDeclared(packageName, "CAMERA", "to take a photo", `{"type": "CAMERA"}`)
2705
+ this.sendResult(packageName, requestId, false, undefined, {
2706
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2707
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2708
+ permission: "CAMERA",
2709
+ operation: MiniappRequestType.PHOTO,
2710
+ })
2711
+ return
2712
+ }
2713
+
2714
+ try {
2715
+ const result = await phonePhotoCoordinator.takePhoto(packageName, {
2716
+ size: payload.size as "low" | "medium" | "high" | "max" | "small" | "large" | "full" | undefined,
2717
+ mode: payload.mode as "photo" | "text" | undefined,
2718
+ compress: payload.compress as "none" | "low" | "medium" | "high" | undefined,
2719
+ sound: payload.sound as boolean | undefined,
2720
+ saveToGallery: payload.saveToGallery as boolean | undefined,
2721
+ exposureTimeNs: payload.exposureTimeNs as number | undefined,
2722
+ })
2723
+ this.sendResult(packageName, requestId, true, result)
2724
+ } catch (err) {
2725
+ this.sendResult(packageName, requestId, false, undefined, {
2726
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2727
+ message: err instanceof Error ? err.message : "Photo request failed",
2728
+ stage: (err as {stage?: string}).stage,
2729
+ transport: (err as {transport?: string}).transport,
2730
+ })
2731
+ }
2732
+ }
2733
+
2734
+ private async handleCameraWarmUp(
2735
+ packageName: string,
2736
+ payload: Record<string, unknown>,
2737
+ requestId?: string,
2738
+ ): Promise<void> {
2739
+ // Manifest CAMERA permission gate (same as photo).
2740
+ const app = this.connectedApps.get(packageName)
2741
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2742
+ if (!hasCameraPermission) {
2743
+ logPermissionNotDeclared(packageName, "CAMERA", "to warm up the camera", `{"type": "CAMERA"}`)
2744
+ this.sendResult(packageName, requestId, false, undefined, {
2745
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2746
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2747
+ permission: "CAMERA",
2748
+ operation: MiniappRequestType.CAMERA_WARM_UP,
2749
+ })
2750
+ return
2751
+ }
2752
+
2753
+ try {
2754
+ await phonePhotoCoordinator.warmUpCamera(packageName, {
2755
+ size: payload.size as "low" | "medium" | "high" | "max" | undefined,
2756
+ exposureTimeNs: payload.exposureTimeNs as number | undefined,
2757
+ durationMs: payload.durationMs as number | undefined,
2758
+ })
2759
+ this.sendResult(packageName, requestId, true)
2760
+ } catch (err) {
2761
+ this.sendResult(packageName, requestId, false, undefined, {
2762
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2763
+ message: err instanceof Error ? err.message : "Camera warm-up failed",
2764
+ stage: (err as {stage?: string}).stage,
2765
+ transport: (err as {transport?: string}).transport,
2766
+ })
2767
+ }
2768
+ }
2769
+
2770
+ private async handleVideoRecordingStart(
2771
+ packageName: string,
2772
+ payload: Record<string, unknown>,
2773
+ requestId?: string,
2774
+ ): Promise<void> {
2775
+ // Manifest CAMERA permission gate (same as photo).
2776
+ const app = this.connectedApps.get(packageName)
2777
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2778
+ if (!hasCameraPermission) {
2779
+ logPermissionNotDeclared(packageName, "CAMERA", "to record video", `{"type": "CAMERA"}`)
2780
+ this.sendResult(packageName, requestId, false, undefined, {
2781
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2782
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2783
+ permission: "CAMERA",
2784
+ operation: MiniappRequestType.VIDEO_RECORDING_START,
2785
+ })
2786
+ return
2787
+ }
2788
+
2789
+ try {
2790
+ const result = await phoneVideoCoordinator.startRecording(packageName, {
2791
+ width: payload.width as number | undefined,
2792
+ height: payload.height as number | undefined,
2793
+ fps: payload.fps as number | undefined,
2794
+ sound: payload.sound as boolean | undefined,
2795
+ save: payload.save as boolean | undefined,
2796
+ })
2797
+ this.sendResult(packageName, requestId, true, result)
2798
+ } catch (err) {
2799
+ this.sendResult(packageName, requestId, false, undefined, {
2800
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2801
+ message: err instanceof Error ? err.message : "Video recording start failed",
2802
+ })
2803
+ }
2804
+ }
2805
+
2806
+ private async handleVideoRecordingStop(
2807
+ packageName: string,
2808
+ payload: Record<string, unknown>,
2809
+ requestId?: string,
2810
+ ): Promise<void> {
2811
+ try {
2812
+ await phoneVideoCoordinator.stopRecording(packageName, payload.recordingId as string | undefined, {
2813
+ uploadUrl: payload.uploadUrl as string | undefined,
2814
+ uploadAuthToken: payload.uploadAuthToken as string | undefined,
2815
+ })
2816
+ this.sendResult(packageName, requestId, true)
2817
+ } catch (err) {
2818
+ this.sendResult(packageName, requestId, false, undefined, {
2819
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2820
+ message: err instanceof Error ? err.message : "Video recording stop failed",
2821
+ })
2822
+ }
2823
+ }
2824
+
2825
+ /**
2826
+ * Stream handlers — dispatched to the engine PhoneStreamCoordinator. For managed
2827
+ * streams the coordinator additionally calls the v2 client REST route to
2828
+ * provision Cloudflare. Cloud-SDK apps (third-party developers) use a
2829
+ * separate cloud-side path that does not pass through here.
2830
+ */
2831
+ private async handleStreamStart(
2832
+ packageName: string,
2833
+ payload: Record<string, unknown>,
2834
+ requestId?: string,
2835
+ ): Promise<void> {
2836
+ const streaming = phoneStreamCoordinator
2837
+ if (!streaming) {
2838
+ this.sendResult(packageName, requestId, false, undefined, {
2839
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
2840
+ message: "Streaming is not configured on this host",
2841
+ })
2842
+ return
2843
+ }
2844
+ try {
2845
+ const result = await streaming.startUnmanaged(packageName, {
2846
+ streamUrl: payload.streamUrl as string,
2847
+ video: normalizeStreamVideoConfig(payload.video),
2848
+ audio: normalizeStreamAudioConfig(payload.audio),
2849
+ sound: payload.sound as boolean | undefined,
2850
+ })
2851
+ this.sendResult(packageName, requestId, true, result)
2852
+ } catch (err) {
2853
+ this.sendResult(packageName, requestId, false, undefined, {
2854
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2855
+ message: err instanceof Error ? err.message : "Stream start failed",
2856
+ stage: (err as {stage?: string}).stage,
2857
+ transport: (err as {transport?: string}).transport,
2858
+ })
2859
+ }
2860
+ }
2861
+
2862
+ private async handleStreamStop(
2863
+ packageName: string,
2864
+ payload: Record<string, unknown>,
2865
+ requestId?: string,
2866
+ ): Promise<void> {
2867
+ const streaming = phoneStreamCoordinator
2868
+ if (!streaming) {
2869
+ // No adapter wired — treat as already-stopped.
2870
+ this.sendResult(packageName, requestId, true)
2871
+ return
2872
+ }
2873
+ try {
2874
+ await streaming.stop(packageName, payload.streamId as string | undefined)
2875
+ this.sendResult(packageName, requestId, true)
2876
+ } catch (err) {
2877
+ this.sendResult(packageName, requestId, false, undefined, {
2878
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2879
+ message: err instanceof Error ? err.message : "Stream stop failed",
2880
+ stage: (err as {stage?: string}).stage,
2881
+ transport: (err as {transport?: string}).transport,
2882
+ })
2883
+ }
2884
+ }
2885
+
2886
+ private async handleManagedStreamStart(
2887
+ packageName: string,
2888
+ payload: Record<string, unknown>,
2889
+ requestId?: string,
2890
+ ): Promise<void> {
2891
+ const streaming = phoneStreamCoordinator
2892
+ if (!streaming) {
2893
+ this.sendResult(packageName, requestId, false, undefined, {
2894
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
2895
+ message: "Streaming is not configured on this host",
2896
+ })
2897
+ return
2898
+ }
2899
+ try {
2900
+ const result = await streaming.startManaged(packageName, {
2901
+ restreamDestinations: payload.restreamDestinations as Array<string | {url: string; name?: string}> | undefined,
2902
+ video: normalizeStreamVideoConfig(payload.video),
2903
+ audio: normalizeStreamAudioConfig(payload.audio),
2904
+ sound: payload.sound as boolean | undefined,
2905
+ ingest: payload.ingest as "srt" | "whip" | undefined,
2906
+ })
2907
+ this.sendResult(packageName, requestId, true, result)
2908
+ } catch (err) {
2909
+ this.sendResult(packageName, requestId, false, undefined, {
2910
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2911
+ message: err instanceof Error ? err.message : "Managed stream start failed",
2912
+ stage: (err as {stage?: string}).stage,
2913
+ transport: (err as {transport?: string}).transport,
2914
+ })
2915
+ }
2916
+ }
2917
+
2918
+ private async handleManagedStreamStop(
2919
+ packageName: string,
2920
+ payload: Record<string, unknown>,
2921
+ requestId?: string,
2922
+ ): Promise<void> {
2923
+ return this.handleStreamStop(packageName, payload, requestId)
2924
+ }
2925
+
2926
+ /**
2927
+ * session.glasses.requestWifiSetup — open the phone's glasses Wi-Fi setup
2928
+ * flow. The host owns the actual UI via the `engine.configure({ui})` seam;
2929
+ * this just forwards the request and reports success/failure.
2930
+ */
2931
+ private async handleRequestWifiSetup(
2932
+ packageName: string,
2933
+ payload: Record<string, unknown>,
2934
+ requestId?: string,
2935
+ ): Promise<void> {
2936
+ const requestWifiSetup = getUiSeams().requestWifiSetup
2937
+ if (!requestWifiSetup) {
2938
+ this.sendResult(packageName, requestId, false, undefined, {
2939
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
2940
+ message: "Wi-Fi setup is not configured on this host",
2941
+ })
2942
+ return
2943
+ }
2944
+ try {
2945
+ await requestWifiSetup(payload.reason as string | undefined)
2946
+ this.sendResult(packageName, requestId, true)
2947
+ } catch (err) {
2948
+ this.sendResult(packageName, requestId, false, undefined, {
2949
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2950
+ message: err instanceof Error ? err.message : "Wi-Fi setup failed",
2951
+ })
2952
+ }
2953
+ }
2954
+
2955
+ /**
2956
+ * Wire the PhoneStreamCoordinator's status callback so coordinator-emitted
2957
+ * updates become `EVENT { streamType: "stream_status" }` envelopes on the
2958
+ * subscribing miniapp(s).
2959
+ */
2960
+ public wireStreamingStatusFanout(): void {
2961
+ const streaming = phoneStreamCoordinator
2962
+ if (!streaming) return
2963
+ streaming.setStatusSubscriber((packageName, update) => {
2964
+ this.sendToMiniapp(packageName, {
2965
+ type: MiniappResponseType.EVENT,
2966
+ streamType: "stream_status",
2967
+ data: {
2968
+ streamId: update.streamId,
2969
+ status: update.status,
2970
+ source: update.source,
2971
+ ...(update.data || {}),
2972
+ },
2973
+ })
2974
+ })
2975
+ }
2976
+
2977
+ // ===========================================================================
2978
+ // Public subscribe helper
2979
+ // ===========================================================================
2980
+
2981
+ /**
2982
+ * Standalone subscribe — lets external code subscribe a registered miniapp to
2983
+ * streams without going through the envelope protocol.
2984
+ *
2985
+ * Returns `{ok: true}` on success, or `{ok: false, error}` on failure.
2986
+ */
2987
+ public subscribe(packageName: string, streams: string[]): {ok: boolean; error?: string} {
2988
+ const app = this.connectedApps.get(packageName)
2989
+ if (!app) {
2990
+ return {ok: false, error: `App ${packageName} is not registered`}
2991
+ }
2992
+
2993
+ if (!Array.isArray(streams)) {
2994
+ return {ok: false, error: "streams must be an array"}
2995
+ }
2996
+
2997
+ console.log(`${LOG_TAG}: subscribe(${packageName}, [${streams.join(", ")}])`)
2998
+
2999
+ // Check microphone permission for mic-requiring streams
3000
+ const micStreams = ["transcription", "translation", "audio_chunk", "vad"]
3001
+ const needsMic = streams.some((s: string) => micStreams.some((m) => s.startsWith(m) || s === m))
3002
+ if (needsMic) {
3003
+ const hasMicPermission = app.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE")
3004
+ if (!hasMicPermission) {
3005
+ return {ok: false, error: "MICROPHONE permission not declared in miniapp.json"}
3006
+ }
3007
+ }
3008
+
3009
+ // Remove old subscriptions for this app
3010
+ for (const oldStream of app.subscriptions) {
3011
+ const subs = this.streamSubscribers.get(oldStream)
3012
+ if (subs) {
3013
+ subs.delete(packageName)
3014
+ if (subs.size === 0) {
3015
+ this.streamSubscribers.delete(oldStream)
3016
+ }
3017
+ }
3018
+ }
3019
+
3020
+ // Add new subscriptions
3021
+ app.subscriptions = new Set(streams)
3022
+ for (const stream of streams) {
3023
+ let subs = this.streamSubscribers.get(stream)
3024
+ if (!subs) {
3025
+ subs = new Set()
3026
+ this.streamSubscribers.set(stream, subs)
3027
+ }
3028
+ subs.add(packageName)
3029
+ }
3030
+
3031
+ this.recomputeMicRequirements()
3032
+ this.updateCloudSubscriptions()
3033
+ return {ok: true}
3034
+ }
3035
+
3036
+ // ===========================================================================
3037
+ // Mic requirements
3038
+ // ===========================================================================
3039
+
3040
+ private recomputeMicRequirements(): void {
3041
+ let anyPcm = false
3042
+ let anyLc3 = false
3043
+ for (const [stream, subscribers] of this.streamSubscribers) {
3044
+ if (subscribers.size === 0) continue
3045
+ if (stream === "audio_chunk") anyPcm = true
3046
+ if (stream.startsWith("transcription:") || stream.startsWith("translation:") || stream === "vad") anyLc3 = true
3047
+ }
3048
+ micStateCoordinator.setLocalRequirements({pcm: anyPcm, lc3: anyLc3})
3049
+ }
3050
+
3051
+ /**
3052
+ * Recompute the aggregated v2 cloud subscription list across all local miniapps.
3053
+ *
3054
+ * Cloud-dependent streams (transcription:*, translation:*) only flow if the
3055
+ * v2 runtime knows the phone wants them. Local-only streams (button_press, etc.)
3056
+ * are NOT sent — they come from the Bluetooth SDK, not from cloud.
3057
+ */
3058
+ private updateCloudSubscriptions(): void {
3059
+ const cloudStreams = new Set<string>()
3060
+ let transcriptionLang: string | null = null
3061
+ for (const [stream, subscribers] of this.streamSubscribers) {
3062
+ if (subscribers.size === 0) continue
3063
+ // Only transcription / translation need cloud delivery. Location,
3064
+ // notifications, and calendar events are sourced natively on the phone
3065
+ // and forwarded to miniapps directly via MantleManager — no cloud hop.
3066
+ if (stream.startsWith("transcription:") || stream.startsWith("translation:")) {
3067
+ cloudStreams.add(stream)
3068
+ }
3069
+ if (stream.startsWith("transcription:") && transcriptionLang === null) {
3070
+ transcriptionLang = stream.substring("transcription:".length)
3071
+ }
3072
+ }
3073
+ localSttFallbackCoordinator.onSubscriptionChange(transcriptionLang !== null, transcriptionLang)
3074
+
3075
+ // Mirror the same set as typed AudioSubscription[] and push it to the cloud
3076
+ // runtime.
3077
+ this.ensureCloudResultsWired()
3078
+ const subs = this.buildCloudAudioSubscriptions(cloudStreams)
3079
+ const nextKey = this.audioSubscriptionKey(subs)
3080
+ if (!this.cloudAudioSubscriptionSync.begin(nextKey)) return
3081
+ console.log(
3082
+ `${LOG_TAG}: updateCloudSubscriptions streams=[${Array.from(cloudStreams).join(", ")}] cloudSubs=${subs.length}`,
3083
+ )
3084
+ cloudClientService
3085
+ .setSubscriptions(subs)
3086
+ .then(() => {
3087
+ this.cloudAudioSubscriptionSync.succeeded(nextKey)
3088
+ })
3089
+ .catch((err) => {
3090
+ // Best-effort: the cloud may not be connected yet. Keep failed writes
3091
+ // retryable; otherwise a reconnect that recomputes the same desired
3092
+ // transcription set gets deduped and local captions never recover.
3093
+ this.cloudAudioSubscriptionSync.failed(nextKey)
3094
+ console.warn(`${LOG_TAG}: cloud setSubscriptions failed: ${(err as Error)?.message ?? err}`)
3095
+ })
3096
+ }
3097
+
3098
+ /**
3099
+ * Build the cloud `AudioSubscription[]` from the miniapp cloud-stream key set.
3100
+ *
3101
+ * Stream keys: `transcription:<lang>` (lang may be `auto`, `en-US`, …) and
3102
+ * `translation:<source>:<target>` (3 colon-parts; source/target may be `*`
3103
+ * or `auto` wildcards, target may also be `*`). The cloud protocol requires a
3104
+ * concrete `target: string` for translation, so wildcard-target translation
3105
+ * keys (`translation:<source>:*`, `translation:*:*`, `translation:auto`) have
3106
+ * no cloud equivalent and are skipped. Wildcard/`auto` SOURCE maps to
3107
+ * `{mode: "auto"}`.
3108
+ */
3109
+ private buildCloudAudioSubscriptions(cloudStreams: Set<string>): AudioSubscription[] {
3110
+ const subs: AudioSubscription[] = []
3111
+ const langSource = (code: string): LanguageSource =>
3112
+ code === "auto" || code === "*" ? {mode: "auto"} : {mode: "specific", code}
3113
+ for (const stream of cloudStreams) {
3114
+ if (stream.startsWith("transcription:")) {
3115
+ const lang = stream.substring("transcription:".length)
3116
+ subs.push({kind: "transcription", language: langSource(lang)})
3117
+ } else if (stream.startsWith("translation:")) {
3118
+ const parts = stream.split(":")
3119
+ // Only `translation:<source>:<target>` maps cleanly; a concrete target
3120
+ // is required by the cloud schema.
3121
+ if (parts.length === 3) {
3122
+ const [, source, target] = parts
3123
+ if (target === "*") continue
3124
+ subs.push({kind: "translation", source: langSource(source), target})
3125
+ }
3126
+ }
3127
+ }
3128
+ return subs
3129
+ }
3130
+
3131
+ private audioSubscriptionKey(subs: AudioSubscription[]): string {
3132
+ return JSON.stringify(
3133
+ subs
3134
+ .map((sub) => {
3135
+ if (sub.kind === "transcription") {
3136
+ return {
3137
+ kind: sub.kind,
3138
+ language: sub.language,
3139
+ }
3140
+ }
3141
+ return {
3142
+ kind: sub.kind,
3143
+ source: sub.source,
3144
+ target: sub.target,
3145
+ }
3146
+ })
3147
+ .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
3148
+ )
3149
+ }
3150
+
3151
+ /**
3152
+ * Wire the cloud's transcription/translation results into the existing
3153
+ * miniapp fan-out exactly once. Maps each cloud result back to the
3154
+ * cloud-to-app data shape `forwardEvent` already forwards, keyed on the
3155
+ * `transcription:<lang>` / `translation:<source>:<target>` stream strings, so
3156
+ * subscribed miniapps receive identical envelopes.
3157
+ */
3158
+ private ensureCloudResultsWired(): void {
3159
+ if (this.cloudResultsWired) return
3160
+ this.cloudResultsWired = true
3161
+
3162
+ cloudClientService.onTranscript((d: TranscriptionData) => {
3163
+ const receivedAt = Date.now()
3164
+ if (TRANSCRIPT_TIMING_TELEMETRY) {
3165
+ console.log(
3166
+ `${LOG_TAG}: transcript cloud_recv t=${receivedAt} final=${d.isFinal} lang=${
3167
+ d.resolvedLanguage
3168
+ } text="${d.text.slice(0, 48)}"`,
3169
+ )
3170
+ }
3171
+ this.forwardEvent(`transcription:${d.resolvedLanguage}`, {
3172
+ type: "transcription",
3173
+ text: d.text,
3174
+ isFinal: d.isFinal,
3175
+ utteranceId: d.utteranceId,
3176
+ transcribeLanguage: d.resolvedLanguage,
3177
+ detectedLanguage: d.languageDetected ? d.resolvedLanguage : undefined,
3178
+ startTime: d.startMs,
3179
+ endTime: d.endMs,
3180
+ speakerId: d.speakerId,
3181
+ duration: d.durationMs,
3182
+ provider: d.provider,
3183
+ confidence: d.confidence,
3184
+ __hostReceivedAt: receivedAt,
3185
+ })
3186
+ })
3187
+
3188
+ cloudClientService.onTranslation((d: TranslationData) => {
3189
+ this.forwardEvent(`translation:${d.source.language}:${d.target.language}`, {
3190
+ type: "translation",
3191
+ text: d.text,
3192
+ originalText: d.originalText,
3193
+ isFinal: d.isFinal,
3194
+ utteranceId: d.utteranceId,
3195
+ startTime: d.startMs,
3196
+ endTime: d.endMs,
3197
+ speakerId: d.speakerId,
3198
+ duration: d.durationMs,
3199
+ transcribeLanguage: d.source.language,
3200
+ translateLanguage: d.target.language,
3201
+ didTranslate: true,
3202
+ provider: d.provider,
3203
+ confidence: d.confidence,
3204
+ })
3205
+ })
3206
+ }
3207
+
3208
+ private ensureCloudStatusWired(): void {
3209
+ if (this.cloudStatusWired) return
3210
+ this.cloudStatusWired = true
3211
+
3212
+ cloudClientService.onStatusChanged((status) => {
3213
+ if (status.status === "connected") {
3214
+ this.updateCloudSubscriptions()
3215
+ // A miniapp that connected before the cloud client was ready never got
3216
+ // its auth token; now that we're connected, mint it without waiting for
3217
+ // the retry backoff (or a full manual reconnect).
3218
+ this.redriveMiniappAuthDelivery()
3219
+ }
3220
+ this.broadcastCloudStatus()
3221
+ })
3222
+
3223
+ useSettingsStore.subscribe(
3224
+ (s) => s.getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive),
3225
+ () => {
3226
+ this.broadcastCloudStatus()
3227
+ },
3228
+ )
3229
+ }
3230
+
3231
+ private currentCloudStatus(): CloudClientStatusSnapshot {
3232
+ const base = cloudClientService.getStatus()
3233
+ const fallbackActive =
3234
+ (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive) as boolean | undefined) ===
3235
+ true
3236
+ return {
3237
+ status: base.status,
3238
+ audioTransport: fallbackActive ? "offline" : base.audioTransport,
3239
+ }
3240
+ }
3241
+
3242
+ private broadcastCloudStatus(): void {
3243
+ const status = this.currentCloudStatus()
3244
+ for (const packageName of this.connectedApps.keys()) {
3245
+ this.sendCloudStatusToMiniapp(packageName, status)
3246
+ }
3247
+ }
3248
+
3249
+ private sendCloudStatusToMiniapp(packageName: string, status = this.currentCloudStatus()): void {
3250
+ this.sendToMiniapp(packageName, {
3251
+ type: MiniappResponseType.EVENT,
3252
+ streamType: CLOUD_STATUS_STREAM,
3253
+ data: status,
3254
+ })
3255
+ }
3256
+
3257
+ // ===========================================================================
3258
+ // Stream fan-out
3259
+ // ===========================================================================
3260
+
3261
+ /**
3262
+ * Forward a streamed event to all miniapps subscribed to the given stream.
3263
+ *
3264
+ * Event name translation:
3265
+ * - Cloud sends "head_up" → miniapp protocol uses "head_position" (HEAD_POSITION)
3266
+ * - Cloud sends "VAD" (uppercase) → miniapp protocol uses "vad" (lowercase)
3267
+ */
3268
+ public forwardEvent(streamType: string, data: unknown): void {
3269
+ // Translate cloud event names to miniapp protocol stream types
3270
+ const normalizedStream = this.normalizeStreamType(streamType)
3271
+
3272
+ // Collect all subscribers: exact match, plus wildcard matches for streams
3273
+ // that carry a language tag. A miniapp subscribed to "transcription:auto"
3274
+ // should receive any "transcription:<lang>" event (the detected language
3275
+ // is conveyed in the event data, not the stream key).
3276
+ //
3277
+ // Translation streams support cloud v3's wildcard patterns. Incoming
3278
+ // events are keyed `translation:<source>:<target>` and we match against
3279
+ // any of:
3280
+ // translation:*:* — all-pairs (TranslationModule.on)
3281
+ // translation:*:<target> — any-source → target (TranslationModule.to)
3282
+ // translation:<source>:* — source → any-target (rare)
3283
+ // translation:<source>:<target> — exact (TranslationModule.fromTo)
3284
+ // translation:auto — back-compat alias for *:*
3285
+ const matchedSubs = new Set<string>()
3286
+ const exact = this.streamSubscribers.get(normalizedStream)
3287
+ if (exact) for (const p of exact) matchedSubs.add(p)
3288
+
3289
+ if (normalizedStream.startsWith("transcription:")) {
3290
+ const autoSubs = this.streamSubscribers.get("transcription:auto")
3291
+ if (autoSubs) for (const p of autoSubs) matchedSubs.add(p)
3292
+ } else if (normalizedStream.startsWith("translation:")) {
3293
+ // translation:<source>:<target> — match each wildcard variant.
3294
+ const parts = normalizedStream.split(":")
3295
+ // parts[0] = "translation", parts[1] = source, parts[2] = target
3296
+ if (parts.length === 3) {
3297
+ const [, source, target] = parts
3298
+ const patterns = [
3299
+ "translation:auto", // legacy alias for *:*
3300
+ "translation:*:*", // cloud v3 "all pairs"
3301
+ `translation:*:${target}`, // cloud v3 "to <target>"
3302
+ `translation:${source}:*`, // cloud v3 "from <source>"
3303
+ ]
3304
+ for (const pat of patterns) {
3305
+ const subs = this.streamSubscribers.get(pat)
3306
+ if (subs) for (const p of subs) matchedSubs.add(p)
3307
+ }
3308
+ } else {
3309
+ // Malformed stream key — still honor the legacy auto alias.
3310
+ const autoSubs = this.streamSubscribers.get("translation:auto")
3311
+ if (autoSubs) for (const p of autoSubs) matchedSubs.add(p)
3312
+ }
3313
+ }
3314
+
3315
+ // Touch event per-gesture fan-out. SDK-side dispatch is keyed on the
3316
+ // exact streamType, so per-gesture subscribers (`touch_event:single_tap`,
3317
+ // etc.) need their own EVENT envelope with the gesture-tagged streamType.
3318
+ // The bare `touch_event` stream above still catches `onTouch(handler)`.
3319
+ let perGestureStream: string | null = null
3320
+ let outboundData = data
3321
+ if (normalizedStream === MiniappStreamType.TOUCH_EVENT) {
3322
+ // The Bluetooth SDK delivers the gesture under `gestureName` (single_tap /
3323
+ // double_tap / triple_tap / long_press / swipe_up / swipe_down). Surface it
3324
+ // to miniapps as `TouchData.kind` and tag a per-gesture stream so
3325
+ // onTouch("single_tap", ...) routes correctly.
3326
+ const touch = data as {gestureName?: string; kind?: string} | null
3327
+ const gesture = touch?.gestureName ?? touch?.kind
3328
+ if (typeof gesture === "string" && gesture.length > 0) {
3329
+ outboundData = {...(touch as object), kind: gesture}
3330
+ perGestureStream = `${MiniappStreamType.TOUCH_EVENT}:${gesture}`
3331
+ }
3332
+ }
3333
+
3334
+ if (normalizedStream.startsWith("transcription:")) {
3335
+ // const known = Array.from(this.streamSubscribers.keys())
3336
+ // console.log(
3337
+ // `${LOG_TAG}: forwardEvent(${streamType} → ${normalizedStream}) matched=${matchedSubs.size} known=[${known.join(", ")}]`,
3338
+ // )
3339
+ }
3340
+
3341
+ if (matchedSubs.size === 0 && !perGestureStream) return
3342
+
3343
+ // BGCAP: count transcripts only once we know they're actually being handed
3344
+ // to a subscriber. If the captions miniapp is unregistered/respawning (or
3345
+ // hasn't subscribed yet) the cloud may still push transcripts that match no
3346
+ // one — counting those would falsely show "forwarded" during the exact gap
3347
+ // the diagnostic is meant to expose.
3348
+ if (BGCAP_TELEMETRY && normalizedStream.startsWith("transcription:")) {
3349
+ bgcapNoteTranscriptForward(!!(data as {isFinal?: boolean} | null)?.isFinal)
3350
+ }
3351
+
3352
+ for (const packageName of matchedSubs) {
3353
+ if (TRANSCRIPT_TIMING_TELEMETRY && normalizedStream.startsWith("transcription:")) {
3354
+ const transcript = data as {text?: string; isFinal?: boolean; __hostReceivedAt?: number} | null
3355
+ const now = Date.now()
3356
+ console.log(
3357
+ `${LOG_TAG}: transcript fanout t=${now} app=${packageName} final=${!!transcript?.isFinal} hostDelta=${
3358
+ transcript?.__hostReceivedAt ? now - transcript.__hostReceivedAt : -1
3359
+ }ms text="${(transcript?.text ?? "").slice(0, 48)}"`,
3360
+ )
3361
+ }
3362
+ // While a nav trip is active the Nav SDK's road-snapped fixes are
3363
+ // already being forwarded via addLocationListener (see NAV_START handler).
3364
+ // Suppress the raw background-GPS forward for that miniapp so the two
3365
+ // streams don't interleave and cause the position to jump back to the
3366
+ // real-phone location during simulation.
3367
+ if (normalizedStream === MiniappStreamType.LOCATION_UPDATE && this.navigationHandlers.isTripActive(packageName)) {
3368
+ continue
3369
+ }
3370
+ this.sendToMiniapp(packageName, {
3371
+ type: MiniappResponseType.EVENT,
3372
+ streamType: normalizedStream,
3373
+ data: outboundData,
3374
+ })
3375
+ }
3376
+
3377
+ // Send a separate envelope to per-gesture subscribers tagged with the
3378
+ // gesture-specific streamType so SDK-side dispatch routes correctly.
3379
+ if (perGestureStream) {
3380
+ const gestureSubs = this.streamSubscribers.get(perGestureStream)
3381
+ if (gestureSubs) {
3382
+ for (const packageName of gestureSubs) {
3383
+ this.sendToMiniapp(packageName, {
3384
+ type: MiniappResponseType.EVENT,
3385
+ streamType: perGestureStream,
3386
+ data: outboundData,
3387
+ })
3388
+ }
3389
+ }
3390
+ }
3391
+ }
3392
+
3393
+ /**
3394
+ * Translate cloud event names to miniapp stream type values.
3395
+ */
3396
+ private normalizeStreamType(cloudEventName: string): string {
3397
+ // Cloud / Bluetooth SDK → miniapp protocol translations.
3398
+ // Bluetooth SDK event names don't always match the miniapp wire values.
3399
+ switch (cloudEventName) {
3400
+ case "head_up":
3401
+ return MiniappStreamType.HEAD_POSITION // head_up → head_position
3402
+ case "accel_event":
3403
+ return MiniappStreamType.ACCEL_DATA // accel_event (native) → accel_data
3404
+ case "VAD":
3405
+ return MiniappStreamType.VAD // VAD (uppercase) → vad (lowercase)
3406
+ case "glasses_battery_update":
3407
+ return MiniappStreamType.GLASSES_BATTERY // glasses_battery_update → glasses_battery
3408
+ case "glasses_connection_state":
3409
+ return MiniappStreamType.GLASSES_CONNECTION // glasses_connection_state → glasses_connection
3410
+ default:
3411
+ // Preserve case for typed streams like "transcription:en-US" / "translation:en-US:fr-FR"
3412
+ // whose language tags are case-sensitive (BCP-47). Lowercase only plain names.
3413
+ if (cloudEventName.includes(":")) return cloudEventName
3414
+ return cloudEventName.toLowerCase()
3415
+ }
3416
+ }
3417
+
3418
+ // ===========================================================================
3419
+ // Outbound helpers
3420
+ // ===========================================================================
3421
+
3422
+ /**
3423
+ * Send a payload to a connected miniapp, wrapped in an envelope.
3424
+ */
3425
+ /**
3426
+ * Push a speaker-state transition to the owning miniapp. Idempotent: if
3427
+ * the new state matches the cached one (and isn't an error), no envelope
3428
+ * is sent. Error events are always sent — they're transient and the SDK
3429
+ * settles back to a non-error state immediately after.
3430
+ */
3431
+ private setSpeakerState(
3432
+ packageName: string,
3433
+ next: SpeakerStateValue,
3434
+ extra?: {errorCode?: string; errorMessage?: string; durationMs?: number},
3435
+ ): void {
3436
+ const app = this.connectedApps.get(packageName)
3437
+ if (!app) return
3438
+ if (app.speakerState === next && next !== "error") return
3439
+ app.speakerState = next === "error" ? "error" : next
3440
+ this.sendToMiniapp(packageName, {
3441
+ type: MiniappResponseType.SPEAKER_STATE,
3442
+ state: next,
3443
+ ...(extra?.errorCode !== undefined ? {errorCode: extra.errorCode} : {}),
3444
+ ...(extra?.errorMessage !== undefined ? {errorMessage: extra.errorMessage} : {}),
3445
+ ...(extra?.durationMs !== undefined ? {durationMs: extra.durationMs} : {}),
3446
+ })
3447
+ // After an error event, immediately transition to "stopped" so the
3448
+ // SDK's isPlaying getter reads false and a new play()/speak() call
3449
+ // starts cleanly from idle/stopped.
3450
+ if (next === "error") {
3451
+ app.speakerState = "stopped"
3452
+ this.sendToMiniapp(packageName, {
3453
+ type: MiniappResponseType.SPEAKER_STATE,
3454
+ state: "stopped",
3455
+ })
3456
+ }
3457
+ }
3458
+
3459
+ private sendToMiniapp(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
3460
+ const app = this.connectedApps.get(packageName)
3461
+ if (!app) {
3462
+ console.warn(`${LOG_TAG}: sendToMiniapp — ${packageName} not connected`)
3463
+ return
3464
+ }
3465
+
3466
+ const envelope: MiniappEnvelope = {
3467
+ payload,
3468
+ requestId,
3469
+ }
3470
+
3471
+ const serialized = serializeEnvelope(envelope)
3472
+ if ((payload as Record<string, unknown>)?.streamType?.toString().startsWith("transcription")) {
3473
+ // console.log(
3474
+ // `${LOG_TAG}: sendToMiniapp → ${packageName} streamType=${(payload as Record<string, unknown>).streamType}`,
3475
+ // )
3476
+ }
3477
+
3478
+ try {
3479
+ app.sendMessage(serialized)
3480
+ } catch (err) {
3481
+ console.error(`${LOG_TAG}: sendToMiniapp error for ${packageName}:`, err)
3482
+ }
3483
+ }
3484
+
3485
+ /**
3486
+ * Send a REQUEST_RESULT response.
3487
+ */
3488
+ // ===========================================================================
3489
+ // Inter-miniapp interop (session.miniapps + session.actions.invoke)
3490
+ // ===========================================================================
3491
+
3492
+ /** Max serialized size of an action call's params / result (256 KB). */
3493
+ private static readonly ACTION_PAYLOAD_CAP = 256 * 1024
3494
+
3495
+ /** Outstanding action invocations, keyed by host-generated callId. */
3496
+ private actionCalls = new Map<
3497
+ string,
3498
+ {callerPackageName: string; targetPackageName: string; callerRequestId?: string; timer: number}
3499
+ >()
3500
+ private actionCallSeq = 0
3501
+
3502
+ private interopApps(): ClientApp[] {
3503
+ return useAppStatusStore.getState().apps
3504
+ }
3505
+
3506
+ private isSystemPackage(packageName: string): boolean {
3507
+ if (SYSTEM_MINIAPP_PACKAGES.has(packageName)) return true
3508
+ return this.interopApps().find((app) => app.packageName === packageName)?.isMiniappDev === true
3509
+ }
3510
+
3511
+ private async startInteropApp(packageName: string): Promise<boolean> {
3512
+ const app = this.interopApps().find((a) => a.packageName === packageName)
3513
+ if (!app) return false
3514
+ if (app.local) {
3515
+ await miniappLauncher.ensureConnected(packageName)
3516
+ return true
3517
+ }
3518
+ return useAppStatusStore.getState().start(app, {skipNavigation: true})
3519
+ }
3520
+
3521
+ private stopInteropApp(packageName: string): Promise<void> {
3522
+ return useAppStatusStore.getState().stop(packageName)
3523
+ }
3524
+
3525
+ private wakeInteropApp(packageName: string): Promise<void> {
3526
+ return miniappLauncher.ensureConnected(packageName)
3527
+ }
3528
+
3529
+ /** Emit an interop audit event (best-effort — never let telemetry break a call). */
3530
+ private auditInterop(event: InteropAuditEvent): void {
3531
+ try {
3532
+ getAnalytics()?.("miniapp_interop", {
3533
+ caller: event.caller,
3534
+ op: event.op,
3535
+ target: event.target ?? "",
3536
+ actionId: event.actionId ?? "",
3537
+ ok: event.ok,
3538
+ errorCode: event.errorCode ?? "",
3539
+ })
3540
+ } catch {
3541
+ /* telemetry must never break an interop call */
3542
+ }
3543
+ }
3544
+
3545
+ /** Reject the request with NOT_PERMITTED unless the caller is a system app. */
3546
+ private requireSystemCaller(
3547
+ packageName: string,
3548
+ requestId: string | undefined,
3549
+ op: InteropAuditEvent["op"],
3550
+ target?: string,
3551
+ ): boolean {
3552
+ if (this.isSystemPackage(packageName)) return true
3553
+ this.auditInterop({caller: packageName, op, target, ok: false, errorCode: MiniappErrorCode.NOT_PERMITTED})
3554
+ this.sendResult(packageName, requestId, false, undefined, {
3555
+ code: MiniappErrorCode.NOT_PERMITTED,
3556
+ message: "Inter-miniapp APIs are restricted to system apps",
3557
+ })
3558
+ return false
3559
+ }
3560
+
3561
+ /** Project a host ClientApp into the SDK's MiniappInfo shape (JSON over the wire). */
3562
+ private buildMiniappInfo(app: ClientApp): Record<string, unknown> {
3563
+ const compatibility = app.compatibility ?? {
3564
+ isCompatible: true,
3565
+ missingRequired: [],
3566
+ missingOptional: [],
3567
+ warnings: [],
3568
+ }
3569
+ return {
3570
+ packageName: app.packageName,
3571
+ name: app.name,
3572
+ version: app.version ?? "",
3573
+ running: app.running,
3574
+ compatibility,
3575
+ actions: app.actions ?? [],
3576
+ }
3577
+ }
3578
+
3579
+ private handleMiniappsList(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
3580
+ if (!this.requireSystemCaller(packageName, requestId, "list")) return
3581
+ const includeIncompatible = payload.includeIncompatible === true
3582
+ const infos = this.interopApps()
3583
+ .filter(
3584
+ (a) =>
3585
+ a.packageName &&
3586
+ a.packageName !== packageName &&
3587
+ !a.packageName.includes("@empty") &&
3588
+ (includeIncompatible || a.compatibility?.isCompatible !== false),
3589
+ )
3590
+ .map((a) => this.buildMiniappInfo(a))
3591
+ this.sendResult(packageName, requestId, true, infos)
3592
+ this.auditInterop({caller: packageName, op: "list", ok: true})
3593
+ }
3594
+
3595
+ private async handleMiniappsStart(
3596
+ packageName: string,
3597
+ payload: Record<string, unknown>,
3598
+ requestId?: string,
3599
+ ): Promise<void> {
3600
+ const target = payload.packageName as string | undefined
3601
+ if (!this.requireSystemCaller(packageName, requestId, "start", target)) return
3602
+ const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined
3603
+ if (!target || !app) {
3604
+ this.sendResult(packageName, requestId, false, undefined, {
3605
+ code: MiniappErrorCode.APP_NOT_FOUND,
3606
+ message: `Miniapp not found: ${target ?? "(missing packageName)"}`,
3607
+ })
3608
+ this.auditInterop({
3609
+ caller: packageName,
3610
+ op: "start",
3611
+ target,
3612
+ ok: false,
3613
+ errorCode: MiniappErrorCode.APP_NOT_FOUND,
3614
+ })
3615
+ return
3616
+ }
3617
+ // Pre-flight the hardware gate so the caller gets a precise reason rather
3618
+ // than a generic rejection (start()'s own gate would otherwise alert + abort).
3619
+ if (app.compatibility && app.compatibility.isCompatible === false) {
3620
+ this.sendResult(packageName, requestId, false, undefined, {
3621
+ code: MiniappErrorCode.APP_NOT_COMPATIBLE,
3622
+ message: `${target} is not compatible with the connected glasses`,
3623
+ })
3624
+ islandNotifications.emit({
3625
+ kind: "version_incompatible",
3626
+ packageName: target,
3627
+ reason: `${target} is not compatible with the connected glasses`,
3628
+ metadata: {missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.start"},
3629
+ timestamp: Date.now(),
3630
+ })
3631
+ this.auditInterop({
3632
+ caller: packageName,
3633
+ op: "start",
3634
+ target,
3635
+ ok: false,
3636
+ errorCode: MiniappErrorCode.APP_NOT_COMPATIBLE,
3637
+ })
3638
+ return
3639
+ }
3640
+ try {
3641
+ // startInteropApp resolves to false when the app-store gate rejected the
3642
+ // launch or the background context failed to spawn — don't report success.
3643
+ const started = await this.startInteropApp(target)
3644
+ if (started) {
3645
+ this.sendResult(packageName, requestId, true)
3646
+ this.auditInterop({caller: packageName, op: "start", target, ok: true})
3647
+ } else {
3648
+ this.sendResult(packageName, requestId, false, undefined, {
3649
+ code: MiniappErrorCode.INTERNAL,
3650
+ message: `start of ${target} was rejected by the host (gated or failed to spawn)`,
3651
+ })
3652
+ this.auditInterop({
3653
+ caller: packageName,
3654
+ op: "start",
3655
+ target,
3656
+ ok: false,
3657
+ errorCode: MiniappErrorCode.INTERNAL,
3658
+ })
3659
+ }
3660
+ } catch (e) {
3661
+ this.sendResult(packageName, requestId, false, undefined, {
3662
+ code: MiniappErrorCode.INTERNAL,
3663
+ message: (e as Error)?.message ?? "start failed",
3664
+ })
3665
+ this.auditInterop({
3666
+ caller: packageName,
3667
+ op: "start",
3668
+ target,
3669
+ ok: false,
3670
+ errorCode: MiniappErrorCode.INTERNAL,
3671
+ })
3672
+ }
3673
+ }
3674
+
3675
+ private async handleMiniappsStop(
3676
+ packageName: string,
3677
+ payload: Record<string, unknown>,
3678
+ requestId?: string,
3679
+ ): Promise<void> {
3680
+ const target = payload.packageName as string | undefined
3681
+ if (!this.requireSystemCaller(packageName, requestId, "stop", target)) return
3682
+ if (!target) {
3683
+ this.sendResult(packageName, requestId, false, undefined, {
3684
+ code: MiniappErrorCode.APP_NOT_FOUND,
3685
+ message: "stop requires a packageName",
3686
+ })
3687
+ return
3688
+ }
3689
+ try {
3690
+ await this.stopInteropApp(target)
3691
+ this.sendResult(packageName, requestId, true)
3692
+ this.auditInterop({caller: packageName, op: "stop", target, ok: true})
3693
+ } catch (e) {
3694
+ this.sendResult(packageName, requestId, false, undefined, {
3695
+ code: MiniappErrorCode.INTERNAL,
3696
+ message: (e as Error)?.message ?? "stop failed",
3697
+ })
3698
+ }
3699
+ }
3700
+
3701
+ /** True iff `value` serializes larger than the action payload cap. */
3702
+ private actionPayloadTooLarge(value: unknown): boolean {
3703
+ try {
3704
+ return JSON.stringify(value ?? null).length > LocalMiniappRuntime.ACTION_PAYLOAD_CAP
3705
+ } catch {
3706
+ // Unserializable — let it through; the transport's own serialize handles it.
3707
+ return false
3708
+ }
3709
+ }
3710
+
3711
+ /**
3712
+ * session.actions.invoke → wake the target headlessly, deliver an ACTION_CALL,
3713
+ * and correlate its ACTION_RESULT back to the caller's pending request.
3714
+ */
3715
+ private async handleActionInvoke(
3716
+ callerPackageName: string,
3717
+ payload: Record<string, unknown>,
3718
+ requestId?: string,
3719
+ ): Promise<void> {
3720
+ const targetForGate = payload.targetPackageName as string | undefined
3721
+ if (!this.requireSystemCaller(callerPackageName, requestId, "invoke", targetForGate)) return
3722
+
3723
+ const target = payload.targetPackageName as string | undefined
3724
+ const actionId = payload.actionId as string | undefined
3725
+ const params = (payload.params as Record<string, unknown> | undefined) ?? {}
3726
+ // Floor the timeout at 6s so the host never rejects with ACTION_TIMEOUT before
3727
+ // the target SDK's handler-registration window closes. A freshly-woken miniapp
3728
+ // buffers an undelivered ACTION_CALL for HANDLER_WAIT_MS (5s in @mentra/miniapp's
3729
+ // actions module) waiting for session.actions.handle; a shorter host timeout
3730
+ // could fire while the target is still registering, then the action would run
3731
+ // with no caller left to receive its result. 6s = that 5s buffer + 1s for the
3732
+ // ACTION_RESULT/NO_ACTION_HANDLER reply to travel back. Keep these in sync.
3733
+ const timeoutMs = Math.min(Math.max(Number(payload.timeoutMs) || 30_000, 6_000), 120_000)
3734
+
3735
+ if (this.actionPayloadTooLarge(params)) {
3736
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3737
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
3738
+ message: "action params exceeded the 256 KB cap",
3739
+ })
3740
+ return
3741
+ }
3742
+
3743
+ const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined
3744
+ if (!target || !app) {
3745
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3746
+ code: MiniappErrorCode.APP_NOT_FOUND,
3747
+ message: `Miniapp not found: ${target ?? "(missing targetPackageName)"}`,
3748
+ })
3749
+ return
3750
+ }
3751
+ if (!actionId) {
3752
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3753
+ code: MiniappErrorCode.ACTION_NOT_FOUND,
3754
+ message: "invoke requires an actionId",
3755
+ })
3756
+ return
3757
+ }
3758
+ if (app.compatibility && app.compatibility.isCompatible === false) {
3759
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3760
+ code: MiniappErrorCode.APP_NOT_COMPATIBLE,
3761
+ message: `${target} is not compatible with the connected glasses`,
3762
+ })
3763
+ islandNotifications.emit({
3764
+ kind: "version_incompatible",
3765
+ packageName: target,
3766
+ reason: `${target} is not compatible with the connected glasses`,
3767
+ metadata: {missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.invoke"},
3768
+ timestamp: Date.now(),
3769
+ })
3770
+ return
3771
+ }
3772
+ // Declared-action gate — only once the host populates app.actions (Phase 2);
3773
+ // until then app.actions is undefined and we fall through to NO_ACTION_HANDLER.
3774
+ if (!(app.actions ?? []).some((a) => a.id === actionId)) {
3775
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3776
+ code: MiniappErrorCode.ACTION_NOT_FOUND,
3777
+ message: `${target} does not declare action "${actionId}"`,
3778
+ })
3779
+ return
3780
+ }
3781
+
3782
+ // Headless wake + wait for CONNECT (idempotent / fast if already connected).
3783
+ try {
3784
+ await this.wakeInteropApp(target)
3785
+ } catch (e) {
3786
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3787
+ code: MiniappErrorCode.WAKE_FAILED,
3788
+ message: (e as Error)?.message ?? `failed to wake ${target}`,
3789
+ })
3790
+ return
3791
+ }
3792
+
3793
+ // The wake waited for CONNECT, but the target could have dropped in the gap
3794
+ // before delivery — fail fast rather than arming a timer for a call that
3795
+ // sendToMiniapp would silently drop (the caller would otherwise wait the
3796
+ // full invoke timeout for an ACTION_CALL that was never delivered).
3797
+ if (!this.connectedApps.has(target)) {
3798
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3799
+ code: MiniappErrorCode.WAKE_FAILED,
3800
+ message: `${target} disconnected before the action could be delivered`,
3801
+ })
3802
+ this.auditInterop({
3803
+ caller: callerPackageName,
3804
+ op: "invoke",
3805
+ target,
3806
+ actionId,
3807
+ ok: false,
3808
+ errorCode: MiniappErrorCode.WAKE_FAILED,
3809
+ })
3810
+ return
3811
+ }
3812
+
3813
+ // Deliver the call and arm the handler timeout; ACTION_RESULT resolves it.
3814
+ const callId = `act-${Date.now().toString(36)}-${this.actionCallSeq++}`
3815
+ const timer = BgTimer.setTimeout(() => {
3816
+ this.actionCalls.delete(callId)
3817
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3818
+ code: MiniappErrorCode.ACTION_TIMEOUT,
3819
+ message: `action "${actionId}" timed out after ${timeoutMs}ms`,
3820
+ })
3821
+ }, timeoutMs)
3822
+ this.actionCalls.set(callId, {callerPackageName, targetPackageName: target, callerRequestId: requestId, timer})
3823
+ this.sendToMiniapp(target, {
3824
+ type: MiniappResponseType.ACTION_CALL,
3825
+ callId,
3826
+ actionId,
3827
+ params,
3828
+ callerPackageName,
3829
+ })
3830
+ this.auditInterop({caller: callerPackageName, op: "invoke", target, actionId, ok: true})
3831
+ }
3832
+
3833
+ /** Target → host: forward an ACTION_RESULT back to the caller's pending invoke. */
3834
+ private handleActionResult(targetPackageName: string, payload: Record<string, unknown>): void {
3835
+ const callId = payload.callId as string | undefined
3836
+ if (!callId) return
3837
+ const pending = this.actionCalls.get(callId)
3838
+ if (!pending) return
3839
+ // Only the invoked target may resolve this call. callIds are guessable
3840
+ // (timestamp + seq), so a different connected miniapp must not be able to
3841
+ // spoof a result/error for someone else's invoke.
3842
+ if (pending.targetPackageName !== targetPackageName) return
3843
+ this.actionCalls.delete(callId)
3844
+ BgTimer.clearTimeout(pending.timer)
3845
+
3846
+ if (payload.ok === true) {
3847
+ const result = payload.result
3848
+ if (this.actionPayloadTooLarge(result)) {
3849
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, {
3850
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
3851
+ message: "action result exceeded the 256 KB cap",
3852
+ })
3853
+ return
3854
+ }
3855
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, true, result ?? null)
3856
+ } else {
3857
+ const error = (payload.error as ({code: string; message: string} & Record<string, unknown>) | undefined) ?? {
3858
+ code: MiniappErrorCode.INTERNAL,
3859
+ message: "action handler error",
3860
+ }
3861
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, error)
3862
+ }
3863
+ }
3864
+
3865
+ private sendResult(
3866
+ packageName: string,
3867
+ requestId: string | undefined,
3868
+ ok: boolean,
3869
+ data?: unknown,
3870
+ error?: {code: string; message: string} & Record<string, unknown>,
3871
+ ): void {
3872
+ if (!requestId) return
3873
+
3874
+ this.sendToMiniapp(
3875
+ packageName,
3876
+ {
3877
+ type: MiniappResponseType.REQUEST_RESULT,
3878
+ requestId,
3879
+ ok,
3880
+ ...(data !== undefined ? {data} : {}),
3881
+ ...(error ? {error} : {}),
3882
+ },
3883
+ requestId,
3884
+ )
3885
+ }
3886
+
3887
+ /**
3888
+ * Send a VISIBILITY_CHANGE push to a miniapp.
3889
+ */
3890
+ public sendVisibilityChange(packageName: string, visibility: "foreground" | "background"): void {
3891
+ this.sendToMiniapp(packageName, {
3892
+ type: MiniappResponseType.VISIBILITY_CHANGE,
3893
+ visibility,
3894
+ })
3895
+ }
3896
+
3897
+ /**
3898
+ * Fast foreground liveness check used when a local miniapp UI is opened or
3899
+ * resumed. The normal watchdog intentionally waits ~30s to avoid killing a
3900
+ * healthy-but-busy background script. When the user is actively looking at a
3901
+ * WebView, a stale JSContext reads as "cloud offline" / no captions, so probe
3902
+ * immediately and reuse the crash-respawn path if no message comes back.
3903
+ */
3904
+ public probeForegroundLiveness(
3905
+ packageName: string,
3906
+ reason = "foreground-open",
3907
+ timeoutMs = FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS,
3908
+ ): void {
3909
+ const app = this.connectedApps.get(packageName)
3910
+ if (!app) return
3911
+
3912
+ this.clearForegroundProbe(packageName)
3913
+ const probeStartedAt = Date.now()
3914
+
3915
+ this.sendToMiniapp(packageName, {
3916
+ type: MiniappRequestType.PING,
3917
+ })
3918
+
3919
+ const timerId = BgTimer.setTimeout(() => {
3920
+ this.foregroundProbeTimers.delete(packageName)
3921
+ const current = this.connectedApps.get(packageName)
3922
+ if (!current) return
3923
+ if (current.lastPongAt >= probeStartedAt) return
3924
+
3925
+ console.warn(`${LOG_TAG}: ${packageName} failed foreground liveness probe (${reason}), respawning`)
3926
+ this.unregisterApp(packageName)
3927
+ this.onLivenessTimeout?.(packageName)
3928
+ }, timeoutMs)
3929
+ this.foregroundProbeTimers.set(packageName, timerId)
3930
+ }
3931
+
3932
+ // ===========================================================================
3933
+ // Ping / pong liveness
3934
+ // ===========================================================================
3935
+
3936
+ private ensurePingLoop(): void {
3937
+ bgcapStartHeartbeat() // BGCAP diagnostic
3938
+ if (this.pingIntervalId !== null) return
3939
+
3940
+ this.pingIntervalId = BgTimer.setInterval(() => {
3941
+ this.doPingRound()
3942
+ }, PING_INTERVAL_MS)
3943
+ }
3944
+
3945
+ private stopPingLoop(): void {
3946
+ bgcapStopHeartbeat() // BGCAP diagnostic
3947
+ if (this.pingIntervalId !== null) {
3948
+ BgTimer.clearInterval(this.pingIntervalId)
3949
+ this.pingIntervalId = null
3950
+ }
3951
+ }
3952
+
3953
+ private doPingRound(): void {
3954
+ const now = Date.now()
3955
+ const staleThreshold = PING_INTERVAL_MS * PING_TIMEOUT_THRESHOLD
3956
+
3957
+ const toRemove: string[] = []
3958
+
3959
+ for (const [packageName, app] of this.connectedApps) {
3960
+ if (now - app.lastPongAt > staleThreshold) {
3961
+ console.warn(`${LOG_TAG}: ${packageName} missed ${PING_TIMEOUT_THRESHOLD} pings, unregistering`)
3962
+ toRemove.push(packageName)
3963
+ continue
3964
+ }
3965
+
3966
+ // Send PING — SDK auto-replies with PONG
3967
+ // BGCAP: stamp the OLDEST outstanding ping only — don't overwrite, or a
3968
+ // background stall (pings keep firing on the native BgTimer while the
3969
+ // miniapp engine is frozen) would reset the clock and hide its duration.
3970
+ if (BGCAP_TELEMETRY && app.bgcapLastPingSentAt == null) app.bgcapLastPingSentAt = now
3971
+ this.sendToMiniapp(packageName, {
3972
+ type: MiniappRequestType.PING,
3973
+ })
3974
+ }
3975
+
3976
+ for (const pkg of toRemove) {
3977
+ this.unregisterApp(pkg)
3978
+ // Hand the package to the router's respawn machinery — a missed-pings
3979
+ // death is treated like a crash so the background script comes back.
3980
+ this.onLivenessTimeout?.(pkg)
3981
+ }
3982
+ }
3983
+
3984
+ /**
3985
+ * Called when a PONG is received from a miniapp (or any message, really).
3986
+ * Updates lastPongAt to keep the app alive.
3987
+ */
3988
+ public handlePong(packageName: string): void {
3989
+ const app = this.connectedApps.get(packageName)
3990
+ if (app) {
3991
+ app.lastPongAt = Date.now()
3992
+ this.clearForegroundProbe(packageName)
3993
+ }
3994
+ }
3995
+
3996
+ private clearForegroundProbe(packageName: string): void {
3997
+ const timerId = this.foregroundProbeTimers.get(packageName)
3998
+ if (timerId == null) return
3999
+ BgTimer.clearTimeout(timerId)
4000
+ this.foregroundProbeTimers.delete(packageName)
4001
+ }
4002
+
4003
+ // ===========================================================================
4004
+ // Lifecycle
4005
+ // ===========================================================================
4006
+
4007
+ public cleanup(): void {
4008
+ console.log(`${LOG_TAG}: cleanup()`)
4009
+ this.stopPingLoop()
4010
+
4011
+ // Copy keys since unregisterApp mutates the map
4012
+ const packageNames = [...this.connectedApps.keys()]
4013
+ for (const pkg of packageNames) {
4014
+ this.unregisterApp(pkg)
4015
+ }
4016
+
4017
+ // Belt-and-suspenders: clear any remaining state
4018
+ this.pendingCloudRequests.clear()
4019
+ this.streamSubscribers.clear()
4020
+ this.connectedApps.clear()
4021
+ for (const timerId of this.foregroundProbeTimers.values()) {
4022
+ BgTimer.clearTimeout(timerId)
4023
+ }
4024
+ this.foregroundProbeTimers.clear()
4025
+
4026
+ LocalMiniappRuntime.instance = null
4027
+ }
4028
+
4029
+ /**
4030
+ * Number of currently connected miniapps (for diagnostics).
4031
+ */
4032
+ public get connectedAppCount(): number {
4033
+ return this.connectedApps.size
4034
+ }
4035
+ }
4036
+
4037
+ const localMiniappRuntime = LocalMiniappRuntime.getInstance()
4038
+ export default localMiniappRuntime