@mentra/engine 0.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (842) hide show
  1. package/README.md +79 -0
  2. package/build/devtools.d.ts +11 -0
  3. package/build/devtools.d.ts.map +1 -0
  4. package/build/devtools.js +13 -0
  5. package/build/devtools.js.map +1 -0
  6. package/build/engine.d.ts +261 -0
  7. package/build/engine.d.ts.map +1 -0
  8. package/build/engine.js +170 -0
  9. package/build/engine.js.map +1 -0
  10. package/build/facades/dev.d.ts +58 -0
  11. package/build/facades/dev.d.ts.map +1 -0
  12. package/build/facades/dev.js +189 -0
  13. package/build/facades/dev.js.map +1 -0
  14. package/build/facades/display.d.ts +21 -0
  15. package/build/facades/display.d.ts.map +1 -0
  16. package/build/facades/display.js +17 -0
  17. package/build/facades/display.js.map +1 -0
  18. package/build/facades/displayMirror.d.ts +15 -0
  19. package/build/facades/displayMirror.d.ts.map +1 -0
  20. package/build/facades/displayMirror.js +29 -0
  21. package/build/facades/displayMirror.js.map +1 -0
  22. package/build/facades/gallery.d.ts +62 -0
  23. package/build/facades/gallery.d.ts.map +1 -0
  24. package/build/facades/gallery.js +61 -0
  25. package/build/facades/gallery.js.map +1 -0
  26. package/build/facades/glasses.d.ts +141 -0
  27. package/build/facades/glasses.d.ts.map +1 -0
  28. package/build/facades/glasses.js +199 -0
  29. package/build/facades/glasses.js.map +1 -0
  30. package/build/facades/glassesSettings.d.ts +13 -0
  31. package/build/facades/glassesSettings.d.ts.map +1 -0
  32. package/build/facades/glassesSettings.js +54 -0
  33. package/build/facades/glassesSettings.js.map +1 -0
  34. package/build/facades/glassesWifi.d.ts +25 -0
  35. package/build/facades/glassesWifi.d.ts.map +1 -0
  36. package/build/facades/glassesWifi.js +57 -0
  37. package/build/facades/glassesWifi.js.map +1 -0
  38. package/build/facades/miniapps.d.ts +43 -0
  39. package/build/facades/miniapps.d.ts.map +1 -0
  40. package/build/facades/miniapps.js +49 -0
  41. package/build/facades/miniapps.js.map +1 -0
  42. package/build/facades/notifications.d.ts +13 -0
  43. package/build/facades/notifications.d.ts.map +1 -0
  44. package/build/facades/notifications.js +12 -0
  45. package/build/facades/notifications.js.map +1 -0
  46. package/build/facades/ota.d.ts +103 -0
  47. package/build/facades/ota.d.ts.map +1 -0
  48. package/build/facades/ota.js +156 -0
  49. package/build/facades/ota.js.map +1 -0
  50. package/build/facades/pairing.d.ts +114 -0
  51. package/build/facades/pairing.d.ts.map +1 -0
  52. package/build/facades/pairing.js +275 -0
  53. package/build/facades/pairing.js.map +1 -0
  54. package/build/facades/permissions.d.ts +33 -0
  55. package/build/facades/permissions.d.ts.map +1 -0
  56. package/build/facades/permissions.js +155 -0
  57. package/build/facades/permissions.js.map +1 -0
  58. package/build/facades/phoneNotifications.d.ts +21 -0
  59. package/build/facades/phoneNotifications.d.ts.map +1 -0
  60. package/build/facades/phoneNotifications.js +51 -0
  61. package/build/facades/phoneNotifications.js.map +1 -0
  62. package/build/facades/reports.d.ts +36 -0
  63. package/build/facades/reports.d.ts.map +1 -0
  64. package/build/facades/reports.js +134 -0
  65. package/build/facades/reports.js.map +1 -0
  66. package/build/facades/session.d.ts +21 -0
  67. package/build/facades/session.d.ts.map +1 -0
  68. package/build/facades/session.js +52 -0
  69. package/build/facades/session.js.map +1 -0
  70. package/build/facades/settings.d.ts +37 -0
  71. package/build/facades/settings.d.ts.map +1 -0
  72. package/build/facades/settings.js +55 -0
  73. package/build/facades/settings.js.map +1 -0
  74. package/build/facades/speech.d.ts +30 -0
  75. package/build/facades/speech.d.ts.map +1 -0
  76. package/build/facades/speech.js +37 -0
  77. package/build/facades/speech.js.map +1 -0
  78. package/build/index.d.ts +43 -0
  79. package/build/index.d.ts.map +1 -0
  80. package/build/index.js +45 -0
  81. package/build/index.js.map +1 -0
  82. package/build/internal.d.ts +67 -0
  83. package/build/internal.d.ts.map +1 -0
  84. package/build/internal.js +90 -0
  85. package/build/internal.js.map +1 -0
  86. package/build/runtime/bootstrap.d.ts +83 -0
  87. package/build/runtime/bootstrap.d.ts.map +1 -0
  88. package/build/runtime/bootstrap.js +56 -0
  89. package/build/runtime/bootstrap.js.map +1 -0
  90. package/build/runtime/config.d.ts +146 -0
  91. package/build/runtime/config.d.ts.map +1 -0
  92. package/build/runtime/config.js +18 -0
  93. package/build/runtime/config.js.map +1 -0
  94. package/build/runtime/streamConfig.d.ts +4 -0
  95. package/build/runtime/streamConfig.d.ts.map +1 -0
  96. package/build/runtime/streamConfig.js +38 -0
  97. package/build/runtime/streamConfig.js.map +1 -0
  98. package/build/services/AppRegistry.d.ts +269 -0
  99. package/build/services/AppRegistry.d.ts.map +1 -0
  100. package/build/services/AppRegistry.js +1079 -0
  101. package/build/services/AppRegistry.js.map +1 -0
  102. package/build/services/AudioCloudUplink.d.ts +9 -0
  103. package/build/services/AudioCloudUplink.d.ts.map +1 -0
  104. package/build/services/AudioCloudUplink.js +48 -0
  105. package/build/services/AudioCloudUplink.js.map +1 -0
  106. package/build/services/AudioPlaybackService.d.ts +175 -0
  107. package/build/services/AudioPlaybackService.d.ts.map +1 -0
  108. package/build/services/AudioPlaybackService.js +705 -0
  109. package/build/services/AudioPlaybackService.js.map +1 -0
  110. package/build/services/AutomaticReportResult.d.ts +20 -0
  111. package/build/services/AutomaticReportResult.d.ts.map +1 -0
  112. package/build/services/AutomaticReportResult.js +29 -0
  113. package/build/services/AutomaticReportResult.js.map +1 -0
  114. package/build/services/BlobStore.d.ts +91 -0
  115. package/build/services/BlobStore.d.ts.map +1 -0
  116. package/build/services/BlobStore.js +668 -0
  117. package/build/services/BlobStore.js.map +1 -0
  118. package/build/services/CaptionsTesterReportService.d.ts +4 -0
  119. package/build/services/CaptionsTesterReportService.d.ts.map +1 -0
  120. package/build/services/CaptionsTesterReportService.js +80 -0
  121. package/build/services/CaptionsTesterReportService.js.map +1 -0
  122. package/build/services/CloudAudioSubscriptionSync.d.ts +24 -0
  123. package/build/services/CloudAudioSubscriptionSync.d.ts.map +1 -0
  124. package/build/services/CloudAudioSubscriptionSync.js +39 -0
  125. package/build/services/CloudAudioSubscriptionSync.js.map +1 -0
  126. package/build/services/CloudClientService.d.ts +143 -0
  127. package/build/services/CloudClientService.d.ts.map +1 -0
  128. package/build/services/CloudClientService.js +644 -0
  129. package/build/services/CloudClientService.js.map +1 -0
  130. package/build/services/CloudTranscriptE2EMetrics.d.ts +4 -0
  131. package/build/services/CloudTranscriptE2EMetrics.d.ts.map +1 -0
  132. package/build/services/CloudTranscriptE2EMetrics.js +24 -0
  133. package/build/services/CloudTranscriptE2EMetrics.js.map +1 -0
  134. package/build/services/ConnectionCoordinator.d.ts +51 -0
  135. package/build/services/ConnectionCoordinator.d.ts.map +1 -0
  136. package/build/services/ConnectionCoordinator.js +34 -0
  137. package/build/services/ConnectionCoordinator.js.map +1 -0
  138. package/build/services/DevServerBridge.d.ts +63 -0
  139. package/build/services/DevServerBridge.d.ts.map +1 -0
  140. package/build/services/DevServerBridge.js +294 -0
  141. package/build/services/DevServerBridge.js.map +1 -0
  142. package/build/services/DeviceEventRouter.d.ts +3 -0
  143. package/build/services/DeviceEventRouter.d.ts.map +1 -0
  144. package/build/services/DeviceEventRouter.js +190 -0
  145. package/build/services/DeviceEventRouter.js.map +1 -0
  146. package/build/services/DeviceStoreHydration.d.ts +38 -0
  147. package/build/services/DeviceStoreHydration.d.ts.map +1 -0
  148. package/build/services/DeviceStoreHydration.js +124 -0
  149. package/build/services/DeviceStoreHydration.js.map +1 -0
  150. package/build/services/DisplayProcessor.d.ts +223 -0
  151. package/build/services/DisplayProcessor.d.ts.map +1 -0
  152. package/build/services/DisplayProcessor.js +552 -0
  153. package/build/services/DisplayProcessor.js.map +1 -0
  154. package/build/services/GlassesReadiness.d.ts +41 -0
  155. package/build/services/GlassesReadiness.d.ts.map +1 -0
  156. package/build/services/GlassesReadiness.js +82 -0
  157. package/build/services/GlassesReadiness.js.map +1 -0
  158. package/build/services/GlassesSettingsSync.d.ts +33 -0
  159. package/build/services/GlassesSettingsSync.d.ts.map +1 -0
  160. package/build/services/GlassesSettingsSync.js +121 -0
  161. package/build/services/GlassesSettingsSync.js.map +1 -0
  162. package/build/services/GlassesStatusProjection.d.ts +3 -0
  163. package/build/services/GlassesStatusProjection.d.ts.map +1 -0
  164. package/build/services/GlassesStatusProjection.js +65 -0
  165. package/build/services/GlassesStatusProjection.js.map +1 -0
  166. package/build/services/HeadingService.d.ts +27 -0
  167. package/build/services/HeadingService.d.ts.map +1 -0
  168. package/build/services/HeadingService.js +96 -0
  169. package/build/services/HeadingService.js.map +1 -0
  170. package/build/services/LocalDisplayManager.d.ts +139 -0
  171. package/build/services/LocalDisplayManager.d.ts.map +1 -0
  172. package/build/services/LocalDisplayManager.js +579 -0
  173. package/build/services/LocalDisplayManager.js.map +1 -0
  174. package/build/services/LocalMiniappRuntime.d.ts +439 -0
  175. package/build/services/LocalMiniappRuntime.d.ts.map +1 -0
  176. package/build/services/LocalMiniappRuntime.js +3695 -0
  177. package/build/services/LocalMiniappRuntime.js.map +1 -0
  178. package/build/services/LocalSttFallbackCoordinator.d.ts +57 -0
  179. package/build/services/LocalSttFallbackCoordinator.d.ts.map +1 -0
  180. package/build/services/LocalSttFallbackCoordinator.js +137 -0
  181. package/build/services/LocalSttFallbackCoordinator.js.map +1 -0
  182. package/build/services/MentraJSCrashController.d.ts +97 -0
  183. package/build/services/MentraJSCrashController.d.ts.map +1 -0
  184. package/build/services/MentraJSCrashController.js +138 -0
  185. package/build/services/MentraJSCrashController.js.map +1 -0
  186. package/build/services/MentraJSCrashloopReportService.d.ts +9 -0
  187. package/build/services/MentraJSCrashloopReportService.d.ts.map +1 -0
  188. package/build/services/MentraJSCrashloopReportService.js +54 -0
  189. package/build/services/MentraJSCrashloopReportService.js.map +1 -0
  190. package/build/services/MentraJSLogPipeline.d.ts +87 -0
  191. package/build/services/MentraJSLogPipeline.d.ts.map +1 -0
  192. package/build/services/MentraJSLogPipeline.js +137 -0
  193. package/build/services/MentraJSLogPipeline.js.map +1 -0
  194. package/build/services/MentraJSRouter.d.ts +203 -0
  195. package/build/services/MentraJSRouter.d.ts.map +1 -0
  196. package/build/services/MentraJSRouter.js +480 -0
  197. package/build/services/MentraJSRouter.js.map +1 -0
  198. package/build/services/MentraUIRouter.d.ts +137 -0
  199. package/build/services/MentraUIRouter.d.ts.map +1 -0
  200. package/build/services/MentraUIRouter.js +210 -0
  201. package/build/services/MentraUIRouter.js.map +1 -0
  202. package/build/services/MicStateCoordinator.d.ts +37 -0
  203. package/build/services/MicStateCoordinator.d.ts.map +1 -0
  204. package/build/services/MicStateCoordinator.js +81 -0
  205. package/build/services/MicStateCoordinator.js.map +1 -0
  206. package/build/services/MiniappEngine.d.ts +31 -0
  207. package/build/services/MiniappEngine.d.ts.map +1 -0
  208. package/build/services/MiniappEngine.js +119 -0
  209. package/build/services/MiniappEngine.js.map +1 -0
  210. package/build/services/MiniappLauncher.d.ts +129 -0
  211. package/build/services/MiniappLauncher.d.ts.map +1 -0
  212. package/build/services/MiniappLauncher.js +291 -0
  213. package/build/services/MiniappLauncher.js.map +1 -0
  214. package/build/services/MiniappRunningRegistry.d.ts +24 -0
  215. package/build/services/MiniappRunningRegistry.d.ts.map +1 -0
  216. package/build/services/MiniappRunningRegistry.js +52 -0
  217. package/build/services/MiniappRunningRegistry.js.map +1 -0
  218. package/build/services/NativeHttpResponse.d.ts +3 -0
  219. package/build/services/NativeHttpResponse.d.ts.map +1 -0
  220. package/build/services/NativeHttpResponse.js +6 -0
  221. package/build/services/NativeHttpResponse.js.map +1 -0
  222. package/build/services/NavigationHandlers.d.ts +90 -0
  223. package/build/services/NavigationHandlers.d.ts.map +1 -0
  224. package/build/services/NavigationHandlers.js +379 -0
  225. package/build/services/NavigationHandlers.js.map +1 -0
  226. package/build/services/NavigationService.d.ts +254 -0
  227. package/build/services/NavigationService.d.ts.map +1 -0
  228. package/build/services/NavigationService.js +553 -0
  229. package/build/services/NavigationService.js.map +1 -0
  230. package/build/services/NotificationsEmitter.d.ts +28 -0
  231. package/build/services/NotificationsEmitter.d.ts.map +1 -0
  232. package/build/services/NotificationsEmitter.js +22 -0
  233. package/build/services/NotificationsEmitter.js.map +1 -0
  234. package/build/services/OfflineSpeechModelService.d.ts +33 -0
  235. package/build/services/OfflineSpeechModelService.d.ts.map +1 -0
  236. package/build/services/OfflineSpeechModelService.js +136 -0
  237. package/build/services/OfflineSpeechModelService.js.map +1 -0
  238. package/build/services/OtaInstallCoordinator.d.ts +195 -0
  239. package/build/services/OtaInstallCoordinator.d.ts.map +1 -0
  240. package/build/services/OtaInstallCoordinator.js +1047 -0
  241. package/build/services/OtaInstallCoordinator.js.map +1 -0
  242. package/build/services/OtaService.d.ts +3 -0
  243. package/build/services/OtaService.d.ts.map +1 -0
  244. package/build/services/OtaService.js +65 -0
  245. package/build/services/OtaService.js.map +1 -0
  246. package/build/services/OtaUpdateCheckService.d.ts +65 -0
  247. package/build/services/OtaUpdateCheckService.d.ts.map +1 -0
  248. package/build/services/OtaUpdateCheckService.js +260 -0
  249. package/build/services/OtaUpdateCheckService.js.map +1 -0
  250. package/build/services/PairingIdentity.d.ts +80 -0
  251. package/build/services/PairingIdentity.d.ts.map +1 -0
  252. package/build/services/PairingIdentity.js +145 -0
  253. package/build/services/PairingIdentity.js.map +1 -0
  254. package/build/services/PhoneCalendarHelpers.d.ts +32 -0
  255. package/build/services/PhoneCalendarHelpers.d.ts.map +1 -0
  256. package/build/services/PhoneCalendarHelpers.js +157 -0
  257. package/build/services/PhoneCalendarHelpers.js.map +1 -0
  258. package/build/services/PhoneCalendarService.d.ts +5 -0
  259. package/build/services/PhoneCalendarService.d.ts.map +1 -0
  260. package/build/services/PhoneCalendarService.js +24 -0
  261. package/build/services/PhoneCalendarService.js.map +1 -0
  262. package/build/services/PhoneCameraFovCoordinator.d.ts +28 -0
  263. package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -0
  264. package/build/services/PhoneCameraFovCoordinator.js +180 -0
  265. package/build/services/PhoneCameraFovCoordinator.js.map +1 -0
  266. package/build/services/PhoneLocationService.d.ts +36 -0
  267. package/build/services/PhoneLocationService.d.ts.map +1 -0
  268. package/build/services/PhoneLocationService.js +115 -0
  269. package/build/services/PhoneLocationService.js.map +1 -0
  270. package/build/services/PhoneNotificationsSync.d.ts +3 -0
  271. package/build/services/PhoneNotificationsSync.d.ts.map +1 -0
  272. package/build/services/PhoneNotificationsSync.js +35 -0
  273. package/build/services/PhoneNotificationsSync.js.map +1 -0
  274. package/build/services/PhonePhotoCoordinator.d.ts +103 -0
  275. package/build/services/PhonePhotoCoordinator.d.ts.map +1 -0
  276. package/build/services/PhonePhotoCoordinator.js +326 -0
  277. package/build/services/PhonePhotoCoordinator.js.map +1 -0
  278. package/build/services/PhoneStreamCoordinator.d.ts +168 -0
  279. package/build/services/PhoneStreamCoordinator.d.ts.map +1 -0
  280. package/build/services/PhoneStreamCoordinator.js +632 -0
  281. package/build/services/PhoneStreamCoordinator.js.map +1 -0
  282. package/build/services/PhoneVideoCoordinator.d.ts +50 -0
  283. package/build/services/PhoneVideoCoordinator.d.ts.map +1 -0
  284. package/build/services/PhoneVideoCoordinator.js +147 -0
  285. package/build/services/PhoneVideoCoordinator.js.map +1 -0
  286. package/build/services/STTModelManager.d.ts +65 -0
  287. package/build/services/STTModelManager.d.ts.map +1 -0
  288. package/build/services/STTModelManager.js +292 -0
  289. package/build/services/STTModelManager.js.map +1 -0
  290. package/build/services/SceneRenderer.d.ts +90 -0
  291. package/build/services/SceneRenderer.d.ts.map +1 -0
  292. package/build/services/SceneRenderer.js +248 -0
  293. package/build/services/SceneRenderer.js.map +1 -0
  294. package/build/services/SentenceTtsPipeline.d.ts +31 -0
  295. package/build/services/SentenceTtsPipeline.d.ts.map +1 -0
  296. package/build/services/SentenceTtsPipeline.js +67 -0
  297. package/build/services/SentenceTtsPipeline.js.map +1 -0
  298. package/build/services/StreamLifecycleController.d.ts +67 -0
  299. package/build/services/StreamLifecycleController.d.ts.map +1 -0
  300. package/build/services/StreamLifecycleController.js +152 -0
  301. package/build/services/StreamLifecycleController.js.map +1 -0
  302. package/build/services/TTSModelManager.d.ts +90 -0
  303. package/build/services/TTSModelManager.d.ts.map +1 -0
  304. package/build/services/TTSModelManager.js +366 -0
  305. package/build/services/TTSModelManager.js.map +1 -0
  306. package/build/services/TranscriptionRouting.d.ts +12 -0
  307. package/build/services/TranscriptionRouting.d.ts.map +1 -0
  308. package/build/services/TranscriptionRouting.js +24 -0
  309. package/build/services/TranscriptionRouting.js.map +1 -0
  310. package/build/services/WebviewBridge.d.ts +22 -0
  311. package/build/services/WebviewBridge.d.ts.map +1 -0
  312. package/build/services/WebviewBridge.js +47 -0
  313. package/build/services/WebviewBridge.js.map +1 -0
  314. package/build/services/asg/GalleryMediaIntegrityReportService.d.ts +17 -0
  315. package/build/services/asg/GalleryMediaIntegrityReportService.d.ts.map +1 -0
  316. package/build/services/asg/GalleryMediaIntegrityReportService.js +43 -0
  317. package/build/services/asg/GalleryMediaIntegrityReportService.js.map +1 -0
  318. package/build/services/asg/asgCameraApi.d.ts +205 -0
  319. package/build/services/asg/asgCameraApi.d.ts.map +1 -0
  320. package/build/services/asg/asgCameraApi.js +1410 -0
  321. package/build/services/asg/asgCameraApi.js.map +1 -0
  322. package/build/services/asg/cameraRollExportCoordinator.d.ts +60 -0
  323. package/build/services/asg/cameraRollExportCoordinator.d.ts.map +1 -0
  324. package/build/services/asg/cameraRollExportCoordinator.js +618 -0
  325. package/build/services/asg/cameraRollExportCoordinator.js.map +1 -0
  326. package/build/services/asg/cameraRollExportLedger.d.ts +44 -0
  327. package/build/services/asg/cameraRollExportLedger.d.ts.map +1 -0
  328. package/build/services/asg/cameraRollExportLedger.js +78 -0
  329. package/build/services/asg/cameraRollExportLedger.js.map +1 -0
  330. package/build/services/asg/galleryMediaValidation.d.ts +29 -0
  331. package/build/services/asg/galleryMediaValidation.d.ts.map +1 -0
  332. package/build/services/asg/galleryMediaValidation.js +149 -0
  333. package/build/services/asg/galleryMediaValidation.js.map +1 -0
  334. package/build/services/asg/galleryNotices.d.ts +33 -0
  335. package/build/services/asg/galleryNotices.d.ts.map +1 -0
  336. package/build/services/asg/galleryNotices.js +29 -0
  337. package/build/services/asg/galleryNotices.js.map +1 -0
  338. package/build/services/asg/galleryPathMigration.d.ts +3 -0
  339. package/build/services/asg/galleryPathMigration.d.ts.map +1 -0
  340. package/build/services/asg/galleryPathMigration.js +26 -0
  341. package/build/services/asg/galleryPathMigration.js.map +1 -0
  342. package/build/services/asg/gallerySettingsService.d.ts +20 -0
  343. package/build/services/asg/gallerySettingsService.d.ts.map +1 -0
  344. package/build/services/asg/gallerySettingsService.js +53 -0
  345. package/build/services/asg/gallerySettingsService.js.map +1 -0
  346. package/build/services/asg/gallerySyncNotifications.d.ts +56 -0
  347. package/build/services/asg/gallerySyncNotifications.d.ts.map +1 -0
  348. package/build/services/asg/gallerySyncNotifications.js +198 -0
  349. package/build/services/asg/gallerySyncNotifications.js.map +1 -0
  350. package/build/services/asg/gallerySyncService.d.ts +137 -0
  351. package/build/services/asg/gallerySyncService.d.ts.map +1 -0
  352. package/build/services/asg/gallerySyncService.js +1933 -0
  353. package/build/services/asg/gallerySyncService.js.map +1 -0
  354. package/build/services/asg/galleryTransferLedger.d.ts +58 -0
  355. package/build/services/asg/galleryTransferLedger.d.ts.map +1 -0
  356. package/build/services/asg/galleryTransferLedger.js +225 -0
  357. package/build/services/asg/galleryTransferLedger.js.map +1 -0
  358. package/build/services/asg/localNetworkTransport.d.ts +15 -0
  359. package/build/services/asg/localNetworkTransport.d.ts.map +1 -0
  360. package/build/services/asg/localNetworkTransport.js +144 -0
  361. package/build/services/asg/localNetworkTransport.js.map +1 -0
  362. package/build/services/asg/localStorageService.d.ts +146 -0
  363. package/build/services/asg/localStorageService.d.ts.map +1 -0
  364. package/build/services/asg/localStorageService.js +572 -0
  365. package/build/services/asg/localStorageService.js.map +1 -0
  366. package/build/services/asg/mediaProcessingQueue.d.ts +70 -0
  367. package/build/services/asg/mediaProcessingQueue.d.ts.map +1 -0
  368. package/build/services/asg/mediaProcessingQueue.js +451 -0
  369. package/build/services/asg/mediaProcessingQueue.js.map +1 -0
  370. package/build/services/asgOtaVersionUrl.d.ts +2 -0
  371. package/build/services/asgOtaVersionUrl.d.ts.map +1 -0
  372. package/build/services/asgOtaVersionUrl.js +36 -0
  373. package/build/services/asgOtaVersionUrl.js.map +1 -0
  374. package/build/services/blobPaths.d.ts +31 -0
  375. package/build/services/blobPaths.d.ts.map +1 -0
  376. package/build/services/blobPaths.js +86 -0
  377. package/build/services/blobPaths.js.map +1 -0
  378. package/build/services/cloudStreamApi.d.ts +59 -0
  379. package/build/services/cloudStreamApi.d.ts.map +1 -0
  380. package/build/services/cloudStreamApi.js +81 -0
  381. package/build/services/cloudStreamApi.js.map +1 -0
  382. package/build/services/gallerySyncClock.d.ts +17 -0
  383. package/build/services/gallerySyncClock.d.ts.map +1 -0
  384. package/build/services/gallerySyncClock.js +21 -0
  385. package/build/services/gallerySyncClock.js.map +1 -0
  386. package/build/services/glassesClockSync.d.ts +20 -0
  387. package/build/services/glassesClockSync.d.ts.map +1 -0
  388. package/build/services/glassesClockSync.js +97 -0
  389. package/build/services/glassesClockSync.js.map +1 -0
  390. package/build/services/manifestVersionGate.d.ts +28 -0
  391. package/build/services/manifestVersionGate.d.ts.map +1 -0
  392. package/build/services/manifestVersionGate.js +41 -0
  393. package/build/services/manifestVersionGate.js.map +1 -0
  394. package/build/services/mentraUiShim.d.ts +38 -0
  395. package/build/services/mentraUiShim.d.ts.map +1 -0
  396. package/build/services/mentraUiShim.js +360 -0
  397. package/build/services/mentraUiShim.js.map +1 -0
  398. package/build/services/navigation/roadNameResolver.d.ts +67 -0
  399. package/build/services/navigation/roadNameResolver.d.ts.map +1 -0
  400. package/build/services/navigation/roadNameResolver.js +96 -0
  401. package/build/services/navigation/roadNameResolver.js.map +1 -0
  402. package/build/services/navigation/routesApiCodec.d.ts +38 -0
  403. package/build/services/navigation/routesApiCodec.d.ts.map +1 -0
  404. package/build/services/navigation/routesApiCodec.js +68 -0
  405. package/build/services/navigation/routesApiCodec.js.map +1 -0
  406. package/build/services/otaDisplayState.d.ts +42 -0
  407. package/build/services/otaDisplayState.d.ts.map +1 -0
  408. package/build/services/otaDisplayState.js +92 -0
  409. package/build/services/otaDisplayState.js.map +1 -0
  410. package/build/services/otaInstallPolicy.d.ts +81 -0
  411. package/build/services/otaInstallPolicy.d.ts.map +1 -0
  412. package/build/services/otaInstallPolicy.js +96 -0
  413. package/build/services/otaInstallPolicy.js.map +1 -0
  414. package/build/services/otaLegacyMapping.d.ts +22 -0
  415. package/build/services/otaLegacyMapping.d.ts.map +1 -0
  416. package/build/services/otaLegacyMapping.js +56 -0
  417. package/build/services/otaLegacyMapping.js.map +1 -0
  418. package/build/services/slimStreamStatus.d.ts +7 -0
  419. package/build/services/slimStreamStatus.d.ts.map +1 -0
  420. package/build/services/slimStreamStatus.js +26 -0
  421. package/build/services/slimStreamStatus.js.map +1 -0
  422. package/build/stores/apps.d.ts +104 -0
  423. package/build/stores/apps.d.ts.map +1 -0
  424. package/build/stores/apps.js +492 -0
  425. package/build/stores/apps.js.map +1 -0
  426. package/build/stores/bluetoothSettingKeys.d.ts +4 -0
  427. package/build/stores/bluetoothSettingKeys.d.ts.map +1 -0
  428. package/build/stores/bluetoothSettingKeys.js +31 -0
  429. package/build/stores/bluetoothSettingKeys.js.map +1 -0
  430. package/build/stores/cloudClientStatus.d.ts +9 -0
  431. package/build/stores/cloudClientStatus.d.ts.map +1 -0
  432. package/build/stores/cloudClientStatus.js +24 -0
  433. package/build/stores/cloudClientStatus.js.map +1 -0
  434. package/build/stores/connection.d.ts +22 -0
  435. package/build/stores/connection.d.ts.map +1 -0
  436. package/build/stores/connection.js +43 -0
  437. package/build/stores/connection.js.map +1 -0
  438. package/build/stores/core.d.ts +15 -0
  439. package/build/stores/core.d.ts.map +1 -0
  440. package/build/stores/core.js +21 -0
  441. package/build/stores/core.js.map +1 -0
  442. package/build/stores/display.d.ts +24 -0
  443. package/build/stores/display.d.ts.map +1 -0
  444. package/build/stores/display.js +96 -0
  445. package/build/stores/display.js.map +1 -0
  446. package/build/stores/gallerySync.d.ts +101 -0
  447. package/build/stores/gallerySync.d.ts.map +1 -0
  448. package/build/stores/gallerySync.js +217 -0
  449. package/build/stores/gallerySync.js.map +1 -0
  450. package/build/stores/glasses.d.ts +62 -0
  451. package/build/stores/glasses.d.ts.map +1 -0
  452. package/build/stores/glasses.js +203 -0
  453. package/build/stores/glasses.js.map +1 -0
  454. package/build/stores/settings.d.ts +37 -0
  455. package/build/stores/settings.d.ts.map +1 -0
  456. package/build/stores/settings.js +922 -0
  457. package/build/stores/settings.js.map +1 -0
  458. package/build/types/applet.d.ts +96 -0
  459. package/build/types/applet.d.ts.map +1 -0
  460. package/build/types/applet.js +5 -0
  461. package/build/types/applet.js.map +1 -0
  462. package/build/types/asg.d.ts +53 -0
  463. package/build/types/asg.d.ts.map +1 -0
  464. package/build/types/asg.js +2 -0
  465. package/build/types/asg.js.map +1 -0
  466. package/build/types/capabilities/even-realities-g1.d.ts +12 -0
  467. package/build/types/capabilities/even-realities-g1.d.ts.map +1 -0
  468. package/build/types/capabilities/even-realities-g1.js +55 -0
  469. package/build/types/capabilities/even-realities-g1.js.map +1 -0
  470. package/build/types/capabilities/even-realities-g2.d.ts +12 -0
  471. package/build/types/capabilities/even-realities-g2.d.ts.map +1 -0
  472. package/build/types/capabilities/even-realities-g2.js +75 -0
  473. package/build/types/capabilities/even-realities-g2.js.map +1 -0
  474. package/build/types/capabilities/mentra-display.d.ts +12 -0
  475. package/build/types/capabilities/mentra-display.d.ts.map +1 -0
  476. package/build/types/capabilities/mentra-display.js +63 -0
  477. package/build/types/capabilities/mentra-display.js.map +1 -0
  478. package/build/types/capabilities/mentra-live.d.ts +12 -0
  479. package/build/types/capabilities/mentra-live.d.ts.map +1 -0
  480. package/build/types/capabilities/mentra-live.js +92 -0
  481. package/build/types/capabilities/mentra-live.js.map +1 -0
  482. package/build/types/capabilities/none.d.ts +13 -0
  483. package/build/types/capabilities/none.d.ts.map +1 -0
  484. package/build/types/capabilities/none.js +65 -0
  485. package/build/types/capabilities/none.js.map +1 -0
  486. package/build/types/capabilities/simulated-glasses.d.ts +13 -0
  487. package/build/types/capabilities/simulated-glasses.d.ts.map +1 -0
  488. package/build/types/capabilities/simulated-glasses.js +65 -0
  489. package/build/types/capabilities/simulated-glasses.js.map +1 -0
  490. package/build/types/capabilities/vuzix-z100.d.ts +12 -0
  491. package/build/types/capabilities/vuzix-z100.d.ts.map +1 -0
  492. package/build/types/capabilities/vuzix-z100.js +52 -0
  493. package/build/types/capabilities/vuzix-z100.js.map +1 -0
  494. package/build/types/enums.d.ts +40 -0
  495. package/build/types/enums.d.ts.map +1 -0
  496. package/build/types/enums.js +44 -0
  497. package/build/types/enums.js.map +1 -0
  498. package/build/types/hardware.d.ts +160 -0
  499. package/build/types/hardware.d.ts.map +1 -0
  500. package/build/types/hardware.js +36 -0
  501. package/build/types/hardware.js.map +1 -0
  502. package/build/types/index.d.ts +11 -0
  503. package/build/types/index.d.ts.map +1 -0
  504. package/build/types/index.js +10 -0
  505. package/build/types/index.js.map +1 -0
  506. package/build/utils/GlobalEventEmitter.d.ts +10 -0
  507. package/build/utils/GlobalEventEmitter.d.ts.map +1 -0
  508. package/build/utils/GlobalEventEmitter.js +13 -0
  509. package/build/utils/GlobalEventEmitter.js.map +1 -0
  510. package/build/utils/cloudClient/RnUdpAdapter.d.ts +3 -0
  511. package/build/utils/cloudClient/RnUdpAdapter.d.ts.map +1 -0
  512. package/build/utils/cloudClient/RnUdpAdapter.js +41 -0
  513. package/build/utils/cloudClient/RnUdpAdapter.js.map +1 -0
  514. package/build/utils/cloudClient/cloudSecureStore.d.ts +3 -0
  515. package/build/utils/cloudClient/cloudSecureStore.d.ts.map +1 -0
  516. package/build/utils/cloudClient/cloudSecureStore.js +26 -0
  517. package/build/utils/cloudClient/cloudSecureStore.js.map +1 -0
  518. package/build/utils/debouncedPatch.d.ts +6 -0
  519. package/build/utils/debouncedPatch.d.ts.map +1 -0
  520. package/build/utils/debouncedPatch.js +25 -0
  521. package/build/utils/debouncedPatch.js.map +1 -0
  522. package/build/utils/devLogging.d.ts +26 -0
  523. package/build/utils/devLogging.d.ts.map +1 -0
  524. package/build/utils/devLogging.js +116 -0
  525. package/build/utils/devLogging.js.map +1 -0
  526. package/build/utils/devMiniappLaunch.d.ts +44 -0
  527. package/build/utils/devMiniappLaunch.d.ts.map +1 -0
  528. package/build/utils/devMiniappLaunch.js +52 -0
  529. package/build/utils/devMiniappLaunch.js.map +1 -0
  530. package/build/utils/diagnosticContext.d.ts +3 -0
  531. package/build/utils/diagnosticContext.d.ts.map +1 -0
  532. package/build/utils/diagnosticContext.js +119 -0
  533. package/build/utils/diagnosticContext.js.map +1 -0
  534. package/build/utils/display/composer/ColumnComposer.d.ts +131 -0
  535. package/build/utils/display/composer/ColumnComposer.d.ts.map +1 -0
  536. package/build/utils/display/composer/ColumnComposer.js +204 -0
  537. package/build/utils/display/composer/ColumnComposer.js.map +1 -0
  538. package/build/utils/display/composer/index.d.ts +9 -0
  539. package/build/utils/display/composer/index.d.ts.map +1 -0
  540. package/build/utils/display/composer/index.js +9 -0
  541. package/build/utils/display/composer/index.js.map +1 -0
  542. package/build/utils/display/helpers/DisplayHelpers.d.ts +165 -0
  543. package/build/utils/display/helpers/DisplayHelpers.d.ts.map +1 -0
  544. package/build/utils/display/helpers/DisplayHelpers.js +312 -0
  545. package/build/utils/display/helpers/DisplayHelpers.js.map +1 -0
  546. package/build/utils/display/helpers/ScrollView.d.ts +183 -0
  547. package/build/utils/display/helpers/ScrollView.d.ts.map +1 -0
  548. package/build/utils/display/helpers/ScrollView.js +257 -0
  549. package/build/utils/display/helpers/ScrollView.js.map +1 -0
  550. package/build/utils/display/helpers/index.d.ts +11 -0
  551. package/build/utils/display/helpers/index.d.ts.map +1 -0
  552. package/build/utils/display/helpers/index.js +10 -0
  553. package/build/utils/display/helpers/index.js.map +1 -0
  554. package/build/utils/display/index.d.ts +174 -0
  555. package/build/utils/display/index.d.ts.map +1 -0
  556. package/build/utils/display/index.js +185 -0
  557. package/build/utils/display/index.js.map +1 -0
  558. package/build/utils/display/measurer/TextMeasurer.d.ts +160 -0
  559. package/build/utils/display/measurer/TextMeasurer.d.ts.map +1 -0
  560. package/build/utils/display/measurer/TextMeasurer.js +273 -0
  561. package/build/utils/display/measurer/TextMeasurer.js.map +1 -0
  562. package/build/utils/display/measurer/index.d.ts +10 -0
  563. package/build/utils/display/measurer/index.d.ts.map +1 -0
  564. package/build/utils/display/measurer/index.js +11 -0
  565. package/build/utils/display/measurer/index.js.map +1 -0
  566. package/build/utils/display/measurer/script-detection.d.ts +53 -0
  567. package/build/utils/display/measurer/script-detection.d.ts.map +1 -0
  568. package/build/utils/display/measurer/script-detection.js +184 -0
  569. package/build/utils/display/measurer/script-detection.js.map +1 -0
  570. package/build/utils/display/profiles/g1.d.ts +33 -0
  571. package/build/utils/display/profiles/g1.d.ts.map +1 -0
  572. package/build/utils/display/profiles/g1.js +213 -0
  573. package/build/utils/display/profiles/g1.js.map +1 -0
  574. package/build/utils/display/profiles/g2.d.ts +13 -0
  575. package/build/utils/display/profiles/g2.d.ts.map +1 -0
  576. package/build/utils/display/profiles/g2.js +34 -0
  577. package/build/utils/display/profiles/g2.js.map +1 -0
  578. package/build/utils/display/profiles/index.d.ts +12 -0
  579. package/build/utils/display/profiles/index.d.ts.map +1 -0
  580. package/build/utils/display/profiles/index.js +15 -0
  581. package/build/utils/display/profiles/index.js.map +1 -0
  582. package/build/utils/display/profiles/nex.d.ts +10 -0
  583. package/build/utils/display/profiles/nex.d.ts.map +1 -0
  584. package/build/utils/display/profiles/nex.js +150 -0
  585. package/build/utils/display/profiles/nex.js.map +1 -0
  586. package/build/utils/display/profiles/types.d.ts +103 -0
  587. package/build/utils/display/profiles/types.d.ts.map +1 -0
  588. package/build/utils/display/profiles/types.js +2 -0
  589. package/build/utils/display/profiles/types.js.map +1 -0
  590. package/build/utils/display/profiles/z100.d.ts +33 -0
  591. package/build/utils/display/profiles/z100.d.ts.map +1 -0
  592. package/build/utils/display/profiles/z100.js +182 -0
  593. package/build/utils/display/profiles/z100.js.map +1 -0
  594. package/build/utils/display/scene/degrade.d.ts +25 -0
  595. package/build/utils/display/scene/degrade.d.ts.map +1 -0
  596. package/build/utils/display/scene/degrade.js +64 -0
  597. package/build/utils/display/scene/degrade.js.map +1 -0
  598. package/build/utils/display/scene/differ.d.ts +38 -0
  599. package/build/utils/display/scene/differ.d.ts.map +1 -0
  600. package/build/utils/display/scene/differ.js +107 -0
  601. package/build/utils/display/scene/differ.js.map +1 -0
  602. package/build/utils/display/scene/index.d.ts +10 -0
  603. package/build/utils/display/scene/index.d.ts.map +1 -0
  604. package/build/utils/display/scene/index.js +6 -0
  605. package/build/utils/display/scene/index.js.map +1 -0
  606. package/build/utils/display/scene/process.d.ts +35 -0
  607. package/build/utils/display/scene/process.d.ts.map +1 -0
  608. package/build/utils/display/scene/process.js +187 -0
  609. package/build/utils/display/scene/process.js.map +1 -0
  610. package/build/utils/display/scene/store.d.ts +48 -0
  611. package/build/utils/display/scene/store.d.ts.map +1 -0
  612. package/build/utils/display/scene/store.js +0 -0
  613. package/build/utils/display/scene/store.js.map +1 -0
  614. package/build/utils/display/scene/types.d.ts +120 -0
  615. package/build/utils/display/scene/types.d.ts.map +1 -0
  616. package/build/utils/display/scene/types.js +0 -0
  617. package/build/utils/display/scene/types.js.map +1 -0
  618. package/build/utils/display/wrapper/TextWrapper.d.ts +102 -0
  619. package/build/utils/display/wrapper/TextWrapper.d.ts.map +1 -0
  620. package/build/utils/display/wrapper/TextWrapper.js +471 -0
  621. package/build/utils/display/wrapper/TextWrapper.js.map +1 -0
  622. package/build/utils/display/wrapper/index.d.ts +12 -0
  623. package/build/utils/display/wrapper/index.d.ts.map +1 -0
  624. package/build/utils/display/wrapper/index.js +12 -0
  625. package/build/utils/display/wrapper/index.js.map +1 -0
  626. package/build/utils/display/wrapper/types.d.ts +75 -0
  627. package/build/utils/display/wrapper/types.d.ts.map +1 -0
  628. package/build/utils/display/wrapper/types.js +11 -0
  629. package/build/utils/display/wrapper/types.js.map +1 -0
  630. package/build/utils/e2eMetrics.d.ts +6 -0
  631. package/build/utils/e2eMetrics.d.ts.map +1 -0
  632. package/build/utils/e2eMetrics.js +27 -0
  633. package/build/utils/e2eMetrics.js.map +1 -0
  634. package/build/utils/hardware/hardware.d.ts +42 -0
  635. package/build/utils/hardware/hardware.d.ts.map +1 -0
  636. package/build/utils/hardware/hardware.js +127 -0
  637. package/build/utils/hardware/hardware.js.map +1 -0
  638. package/build/utils/hardware/index.d.ts +2 -0
  639. package/build/utils/hardware/index.d.ts.map +1 -0
  640. package/build/utils/hardware/index.js +2 -0
  641. package/build/utils/hardware/index.js.map +1 -0
  642. package/build/utils/miniappGlobals.d.ts +60 -0
  643. package/build/utils/miniappGlobals.d.ts.map +1 -0
  644. package/build/utils/miniappGlobals.js +224 -0
  645. package/build/utils/miniappGlobals.js.map +1 -0
  646. package/build/utils/permissions/MediaLibraryPermissions.d.ts +45 -0
  647. package/build/utils/permissions/MediaLibraryPermissions.d.ts.map +1 -0
  648. package/build/utils/permissions/MediaLibraryPermissions.js +145 -0
  649. package/build/utils/permissions/MediaLibraryPermissions.js.map +1 -0
  650. package/build/utils/permissions/galleryDisplayName.d.ts +8 -0
  651. package/build/utils/permissions/galleryDisplayName.d.ts.map +1 -0
  652. package/build/utils/permissions/galleryDisplayName.js +25 -0
  653. package/build/utils/permissions/galleryDisplayName.js.map +1 -0
  654. package/build/utils/storage/index.d.ts +3 -0
  655. package/build/utils/storage/index.d.ts.map +1 -0
  656. package/build/utils/storage/index.js +3 -0
  657. package/build/utils/storage/index.js.map +1 -0
  658. package/build/utils/storage/storage.d.ts +24 -0
  659. package/build/utils/storage/storage.d.ts.map +1 -0
  660. package/build/utils/storage/storage.js +85 -0
  661. package/build/utils/storage/storage.js.map +1 -0
  662. package/build/utils/storage/zip.d.ts +3 -0
  663. package/build/utils/storage/zip.d.ts.map +1 -0
  664. package/build/utils/storage/zip.js +14 -0
  665. package/build/utils/storage/zip.js.map +1 -0
  666. package/build/utils/timers/index.d.ts +3 -0
  667. package/build/utils/timers/index.d.ts.map +1 -0
  668. package/build/utils/timers/index.js +3 -0
  669. package/build/utils/timers/index.js.map +1 -0
  670. package/build/utils/timers/timers.d.ts +9 -0
  671. package/build/utils/timers/timers.d.ts.map +1 -0
  672. package/build/utils/timers/timers.js +105 -0
  673. package/build/utils/timers/timers.js.map +1 -0
  674. package/package.json +120 -0
  675. package/src/devtools.ts +13 -0
  676. package/src/engine.ts +173 -0
  677. package/src/facades/dev.ts +198 -0
  678. package/src/facades/display.ts +17 -0
  679. package/src/facades/displayMirror.ts +33 -0
  680. package/src/facades/gallery.ts +69 -0
  681. package/src/facades/glasses.ts +206 -0
  682. package/src/facades/glassesSettings.ts +56 -0
  683. package/src/facades/glassesWifi.ts +68 -0
  684. package/src/facades/miniapps.ts +55 -0
  685. package/src/facades/notifications.ts +15 -0
  686. package/src/facades/ota.ts +174 -0
  687. package/src/facades/pairing.ts +329 -0
  688. package/src/facades/permissions.ts +165 -0
  689. package/src/facades/phoneNotifications.ts +50 -0
  690. package/src/facades/reports.ts +185 -0
  691. package/src/facades/session.ts +55 -0
  692. package/src/facades/settings.ts +57 -0
  693. package/src/facades/speech.ts +38 -0
  694. package/src/index.ts +191 -0
  695. package/src/internal.ts +159 -0
  696. package/src/runtime/bootstrap.ts +118 -0
  697. package/src/runtime/config.ts +147 -0
  698. package/src/runtime/streamConfig.ts +34 -0
  699. package/src/services/AppRegistry.ts +1190 -0
  700. package/src/services/AudioCloudUplink.ts +50 -0
  701. package/src/services/AudioPlaybackService.ts +813 -0
  702. package/src/services/AutomaticReportResult.ts +48 -0
  703. package/src/services/BlobStore.ts +746 -0
  704. package/src/services/CaptionsTesterReportService.ts +107 -0
  705. package/src/services/CloudAudioSubscriptionSync.ts +41 -0
  706. package/src/services/CloudClientService.ts +689 -0
  707. package/src/services/CloudTranscriptE2EMetrics.ts +27 -0
  708. package/src/services/ConnectionCoordinator.ts +65 -0
  709. package/src/services/DevServerBridge.ts +332 -0
  710. package/src/services/DeviceEventRouter.ts +238 -0
  711. package/src/services/DeviceStoreHydration.ts +130 -0
  712. package/src/services/DisplayProcessor.ts +775 -0
  713. package/src/services/GlassesReadiness.ts +103 -0
  714. package/src/services/GlassesSettingsSync.ts +134 -0
  715. package/src/services/GlassesStatusProjection.ts +73 -0
  716. package/src/services/HeadingService.ts +98 -0
  717. package/src/services/LocalDisplayManager.ts +703 -0
  718. package/src/services/LocalMiniappRuntime.ts +4213 -0
  719. package/src/services/LocalSttFallbackCoordinator.ts +147 -0
  720. package/src/services/MentraJSCrashController.ts +189 -0
  721. package/src/services/MentraJSCrashloopReportService.ts +66 -0
  722. package/src/services/MentraJSLogPipeline.ts +166 -0
  723. package/src/services/MentraJSRouter.ts +557 -0
  724. package/src/services/MentraUIRouter.ts +244 -0
  725. package/src/services/MicStateCoordinator.ts +94 -0
  726. package/src/services/MiniappEngine.ts +143 -0
  727. package/src/services/MiniappLauncher.ts +340 -0
  728. package/src/services/MiniappRunningRegistry.ts +57 -0
  729. package/src/services/NativeHttpResponse.ts +6 -0
  730. package/src/services/NavigationHandlers.ts +423 -0
  731. package/src/services/NavigationService.ts +731 -0
  732. package/src/services/NotificationsEmitter.ts +48 -0
  733. package/src/services/OfflineSpeechModelService.ts +168 -0
  734. package/src/services/OtaInstallCoordinator.ts +1205 -0
  735. package/src/services/OtaService.ts +83 -0
  736. package/src/services/OtaUpdateCheckService.ts +386 -0
  737. package/src/services/PairingIdentity.ts +154 -0
  738. package/src/services/PhoneCalendarHelpers.ts +212 -0
  739. package/src/services/PhoneCalendarService.ts +42 -0
  740. package/src/services/PhoneCameraFovCoordinator.ts +195 -0
  741. package/src/services/PhoneLocationService.ts +122 -0
  742. package/src/services/PhoneNotificationsSync.ts +44 -0
  743. package/src/services/PhonePhotoCoordinator.ts +409 -0
  744. package/src/services/PhoneStreamCoordinator.ts +777 -0
  745. package/src/services/PhoneVideoCoordinator.ts +189 -0
  746. package/src/services/STTModelManager.ts +360 -0
  747. package/src/services/SceneRenderer.ts +292 -0
  748. package/src/services/SentenceTtsPipeline.ts +106 -0
  749. package/src/services/StreamLifecycleController.ts +213 -0
  750. package/src/services/TTSModelManager.ts +466 -0
  751. package/src/services/TranscriptionRouting.ts +37 -0
  752. package/src/services/WebviewBridge.ts +50 -0
  753. package/src/services/asg/GalleryMediaIntegrityReportService.ts +70 -0
  754. package/src/services/asg/asgCameraApi.ts +1695 -0
  755. package/src/services/asg/cameraRollExportCoordinator.ts +627 -0
  756. package/src/services/asg/cameraRollExportLedger.ts +122 -0
  757. package/src/services/asg/galleryMediaValidation.ts +183 -0
  758. package/src/services/asg/galleryNotices.ts +61 -0
  759. package/src/services/asg/galleryPathMigration.ts +22 -0
  760. package/src/services/asg/gallerySettingsService.ts +68 -0
  761. package/src/services/asg/gallerySyncNotifications.ts +236 -0
  762. package/src/services/asg/gallerySyncService.ts +2269 -0
  763. package/src/services/asg/galleryTransferLedger.ts +312 -0
  764. package/src/services/asg/localNetworkTransport.ts +173 -0
  765. package/src/services/asg/localStorageService.ts +652 -0
  766. package/src/services/asg/mediaProcessingQueue.ts +518 -0
  767. package/src/services/asgOtaVersionUrl.ts +41 -0
  768. package/src/services/blobPaths.ts +86 -0
  769. package/src/services/cloudStreamApi.ts +154 -0
  770. package/src/services/gallerySyncClock.ts +39 -0
  771. package/src/services/glassesClockSync.ts +115 -0
  772. package/src/services/manifestVersionGate.ts +43 -0
  773. package/src/services/mentraUiShim.ts +366 -0
  774. package/src/services/navigation/roadNameResolver.ts +112 -0
  775. package/src/services/navigation/routesApiCodec.ts +69 -0
  776. package/src/services/otaDisplayState.ts +115 -0
  777. package/src/services/otaInstallPolicy.ts +106 -0
  778. package/src/services/otaLegacyMapping.ts +71 -0
  779. package/src/services/slimStreamStatus.ts +28 -0
  780. package/src/stores/apps.ts +614 -0
  781. package/src/stores/bluetoothSettingKeys.ts +36 -0
  782. package/src/stores/cloudClientStatus.ts +37 -0
  783. package/src/stores/connection.ts +69 -0
  784. package/src/stores/core.ts +32 -0
  785. package/src/stores/display.ts +121 -0
  786. package/src/stores/gallerySync.ts +380 -0
  787. package/src/stores/glasses.ts +309 -0
  788. package/src/stores/settings.ts +1001 -0
  789. package/src/types/applet.ts +112 -0
  790. package/src/types/asg.ts +58 -0
  791. package/src/types/capabilities/even-realities-g1.ts +67 -0
  792. package/src/types/capabilities/even-realities-g2.ts +88 -0
  793. package/src/types/capabilities/mentra-display.ts +75 -0
  794. package/src/types/capabilities/mentra-live.ts +103 -0
  795. package/src/types/capabilities/none.ts +76 -0
  796. package/src/types/capabilities/simulated-glasses.ts +76 -0
  797. package/src/types/capabilities/vuzix-z100.ts +64 -0
  798. package/src/types/enums.ts +43 -0
  799. package/src/types/hardware.ts +203 -0
  800. package/src/types/index.ts +37 -0
  801. package/src/utils/GlobalEventEmitter.ts +14 -0
  802. package/src/utils/cloudClient/RnUdpAdapter.ts +47 -0
  803. package/src/utils/cloudClient/cloudSecureStore.ts +28 -0
  804. package/src/utils/debouncedPatch.ts +28 -0
  805. package/src/utils/devLogging.ts +140 -0
  806. package/src/utils/devMiniappLaunch.ts +68 -0
  807. package/src/utils/diagnosticContext.ts +155 -0
  808. package/src/utils/display/composer/ColumnComposer.ts +285 -0
  809. package/src/utils/display/composer/index.ts +15 -0
  810. package/src/utils/display/helpers/DisplayHelpers.ts +413 -0
  811. package/src/utils/display/helpers/ScrollView.ts +337 -0
  812. package/src/utils/display/helpers/index.ts +14 -0
  813. package/src/utils/display/index.ts +273 -0
  814. package/src/utils/display/measurer/TextMeasurer.ts +346 -0
  815. package/src/utils/display/measurer/index.ts +21 -0
  816. package/src/utils/display/measurer/script-detection.ts +218 -0
  817. package/src/utils/display/profiles/g1.ts +236 -0
  818. package/src/utils/display/profiles/g2.ts +39 -0
  819. package/src/utils/display/profiles/index.ts +28 -0
  820. package/src/utils/display/profiles/nex.ts +166 -0
  821. package/src/utils/display/profiles/types.ts +132 -0
  822. package/src/utils/display/profiles/z100.ts +198 -0
  823. package/src/utils/display/scene/degrade.ts +80 -0
  824. package/src/utils/display/scene/differ.ts +128 -0
  825. package/src/utils/display/scene/index.ts +21 -0
  826. package/src/utils/display/scene/process.ts +214 -0
  827. package/src/utils/display/scene/store.ts +0 -0
  828. package/src/utils/display/scene/types.ts +0 -0
  829. package/src/utils/display/wrapper/TextWrapper.ts +546 -0
  830. package/src/utils/display/wrapper/index.ts +15 -0
  831. package/src/utils/display/wrapper/types.ts +102 -0
  832. package/src/utils/e2eMetrics.ts +30 -0
  833. package/src/utils/hardware/hardware.ts +166 -0
  834. package/src/utils/hardware/index.ts +1 -0
  835. package/src/utils/miniappGlobals.ts +260 -0
  836. package/src/utils/permissions/MediaLibraryPermissions.ts +161 -0
  837. package/src/utils/permissions/galleryDisplayName.ts +28 -0
  838. package/src/utils/storage/index.ts +2 -0
  839. package/src/utils/storage/storage.ts +101 -0
  840. package/src/utils/storage/zip.ts +13 -0
  841. package/src/utils/timers/index.ts +2 -0
  842. package/src/utils/timers/timers.ts +139 -0
@@ -0,0 +1,4213 @@
1
+ /**
2
+ * LocalMiniappRuntime
3
+ *
4
+ * Singleton service that bridges local miniapps (running in WebViews) with
5
+ * phone capabilities: display, audio, storage, sensors, LED, etc.
6
+ *
7
+ * Each miniapp communicates via the @mentra/miniapp envelope protocol.
8
+ * This runtime handles request dispatch, stream fan-out, and lifecycle
9
+ * management (connect/disconnect/ping).
10
+ */
11
+
12
+ import {Linking} from "react-native"
13
+ import Share from "react-native-share"
14
+ import * as Battery from "expo-battery"
15
+ import * as Clipboard from "expo-clipboard"
16
+ import {File, Paths} from "expo-file-system"
17
+ import * as Location from "expo-location"
18
+ import BluetoothSdk, {type RgbLedAction, type RgbLedColor} from "@mentra/bluetooth-sdk"
19
+
20
+ import {
21
+ MiniappErrorCode,
22
+ MiniappRequestType,
23
+ MiniappResponseType,
24
+ MiniappStreamType,
25
+ CLOUD_STATUS_STREAM,
26
+ parseEnvelope,
27
+ serializeEnvelope,
28
+ } from "@mentra/miniapp"
29
+ import type {MiniappEnvelope} from "@mentra/miniapp"
30
+
31
+ import {DeviceTypes, getModelCapabilities} from "../types"
32
+ import {storage as mmkvStorage} from "../utils/storage/storage"
33
+ import {BgTimer} from "../utils/timers"
34
+ import devServerBridge from "./DevServerBridge"
35
+ import {islandNotifications} from "./NotificationsEmitter"
36
+ import {isGlassesConnected} from "./GlassesReadiness"
37
+ import audioPlaybackService from "./AudioPlaybackService"
38
+ import {phoneLocationService} from "./PhoneLocationService"
39
+ import {useGlassesStore} from "../stores/glasses"
40
+ import {useSettingsStore} from "../stores/settings"
41
+ import localDisplayManager from "./LocalDisplayManager"
42
+ import type {DisplayPayload} from "./LocalDisplayManager"
43
+ import headingService from "./HeadingService"
44
+ import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator"
45
+ import micStateCoordinator from "./MicStateCoordinator"
46
+ import {BlobStore} from "./BlobStore"
47
+ import {CloudAudioSubscriptionSync} from "./CloudAudioSubscriptionSync"
48
+ import {phoneCameraFovCoordinator} from "./PhoneCameraFovCoordinator"
49
+ import {phonePhotoCoordinator} from "./PhonePhotoCoordinator"
50
+ import {phoneStreamCoordinator} from "./PhoneStreamCoordinator"
51
+ import {phoneVideoCoordinator} from "./PhoneVideoCoordinator"
52
+ import {runSentenceTtsPipeline} from "./SentenceTtsPipeline"
53
+ import {summarizeTranscriptionRoutes, transcriptionDeliveryRoute} from "./TranscriptionRouting"
54
+ import {cloudClientService} from "./CloudClientService"
55
+ import {miniappLauncher} from "./MiniappLauncher"
56
+ import {
57
+ ISLAND_SETTINGS_KEYS,
58
+ type CameraFovPreset,
59
+ type CameraFovRequest,
60
+ type CameraRoiPosition,
61
+ type CloudClientStatusSnapshot,
62
+ type InteropAuditEvent,
63
+ type MiniappAuthToken,
64
+ type TtsSynthesisResult,
65
+ } from "../runtime/config"
66
+ import {getAnalytics, getUiSeams} from "../runtime/bootstrap"
67
+ import {normalizeStreamAudioConfig, normalizeStreamVideoConfig} from "../runtime/streamConfig"
68
+ import type {AudioSubscription, LanguageSource, TranscriptionData, TranslationData} from "@mentra/cloud-protocol"
69
+ import ttsModelManager from "./TTSModelManager"
70
+ import {NavigationHandlers} from "./NavigationHandlers"
71
+ import type {ClientApp} from "../types/applet"
72
+ import {useAppStatusStore} from "../stores/apps"
73
+ import {getDevAppAttestation, getDevAppSourcePackage} from "./AppRegistry"
74
+ import {listPhoneCalendarEvents, PhoneCalendarError} from "./PhoneCalendarService"
75
+
76
+ // =============================================================================
77
+ // Types
78
+ // =============================================================================
79
+
80
+ export interface InstalledMiniappManifest {
81
+ /** Human-facing app name (from miniapp.json `name`). Shown in the "Starting
82
+ * <name>…" boot message; falls back to the package name when absent. */
83
+ name?: string
84
+ permissions?: Array<{type: string; required?: boolean; description?: string}>
85
+ hardwareRequirements?: Array<{type: string; level: string; description?: string}>
86
+ }
87
+
88
+ type SpeakerStateValue = "idle" | "loading" | "playing" | "stopped" | "error"
89
+
90
+ interface ConnectedMiniapp {
91
+ subscriptions: Set<string>
92
+ /** Transcription streams this app explicitly requires to stay on-device. */
93
+ forceLocalTranscriptionStreams: Set<string>
94
+ /** Transcription streams with at least one listener using the default/cloud route. */
95
+ cloudTranscriptionStreams: Set<string>
96
+ sendMessage: (raw: string) => void
97
+ lastPongAt: number
98
+ installedManifest?: InstalledMiniappManifest
99
+ authRefreshTimerId: number | null
100
+ /**
101
+ * Backoff timer for re-attempting the INITIAL miniapp-token mint after it
102
+ * failed (distinct from `authRefreshTimerId`, which renews an already-issued
103
+ * token near expiry). Null when no retry is queued.
104
+ */
105
+ authRetryTimerId: number | null
106
+ /**
107
+ * True once this app has received a miniapp auth token (in CONNECT_ACK or a
108
+ * follow-up AUTH_UPDATE). Gates the initial-mint retry loop and the
109
+ * on-reconnect re-drive so we stop hammering once auth has landed.
110
+ */
111
+ authDelivered: boolean
112
+ /** Last speaker state pushed to this app. Used to dedup SPEAKER_STATE pushes. */
113
+ speakerState: SpeakerStateValue
114
+ /** Live PCM output stream this app has open (speaker.createStream), if any. */
115
+ activeSpeakerStreamId?: string
116
+ /**
117
+ * Location-tier rate this app is currently asking for via its
118
+ * `location_stream` subscription, or null if it hasn't asked for
119
+ * one. The aggregate tier handed to the host is the strictest
120
+ * (highest-accuracy) rate across all currently connected apps —
121
+ * see {@link LOCATION_RATE_PRIORITY}.
122
+ */
123
+ requestedLocationRate: string | null
124
+ }
125
+
126
+ interface SpeechRun {
127
+ id: string
128
+ cancelled: boolean
129
+ playbackRequestId?: string
130
+ onCancelled?: () => void
131
+ }
132
+
133
+ type TranscriptionEventSource = "cloud" | "local"
134
+
135
+ /**
136
+ * Strictness ordering for `location_stream` rate values. Higher index =
137
+ * higher accuracy and higher power cost. The aggregate tier applied to
138
+ * the OS is the max of every connected miniapp's requested rate;
139
+ * miniapps that didn't ask for `location_stream` contribute nothing.
140
+ *
141
+ * If a miniapp passes a value we don't recognize, we treat it like the
142
+ * weakest known rate ("passive") rather than blanket-overwriting an
143
+ * active stronger request from another app.
144
+ */
145
+ const LOCATION_RATE_PRIORITY = ["off", "passive", "low", "high", "realtime"] as const
146
+ type LocationRate = (typeof LOCATION_RATE_PRIORITY)[number]
147
+ function locationRateRank(rate: string | null | undefined): number {
148
+ if (!rate) return -1
149
+ const i = LOCATION_RATE_PRIORITY.indexOf(rate as LocationRate)
150
+ return i >= 0 ? i : LOCATION_RATE_PRIORITY.indexOf("passive")
151
+ }
152
+
153
+ function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {
154
+ return new Promise((resolve) => {
155
+ let settled = false
156
+ const done = (value: T | null) => {
157
+ if (settled) return
158
+ settled = true
159
+ BgTimer.clearTimeout(timer)
160
+ resolve(value)
161
+ }
162
+ const timer = BgTimer.setTimeout(() => done(null), timeoutMs)
163
+ promise.then(
164
+ (value) => done(value),
165
+ () => done(null),
166
+ )
167
+ })
168
+ }
169
+
170
+ const LOG_TAG = "LOCAL_MINIAPP"
171
+ const SYSTEM_MINIAPP_PACKAGES = new Set([
172
+ "com.mentra.camera",
173
+ "com.mentra.gallery",
174
+ "com.mentra.settings",
175
+ "com.mentra.simulated",
176
+ "com.mentra.mirror",
177
+ "com.mentra.ai",
178
+ "cloud.augmentos.notify",
179
+ "com.mentra.feedback",
180
+ "com.mentra.miniappdev",
181
+ ])
182
+ const PING_INTERVAL_MS = 5_000
183
+ const MINIAPP_AUTH_REFRESH_HEADROOM_MS = 5 * 60 * 1000
184
+ const MINIAPP_AUTH_REFRESH_MIN_DELAY_MS = 5_000
185
+ // Backoff bounds for re-minting the INITIAL miniapp token after a failed mint
186
+ // (e.g. the cloud client had not finished its first-boot Core token exchange
187
+ // when the miniapp connected). Exponential from base, capped, so a transient
188
+ // not-yet-connected client self-heals within seconds instead of needing a full
189
+ // manual Cloud V2 reconnect.
190
+ const MINIAPP_AUTH_RETRY_BASE_MS = 1_000
191
+ const MINIAPP_AUTH_RETRY_MAX_MS = 15_000
192
+ const FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS = 2_500
193
+ const REQUEST_WIFI_SETUP_TYPE = "miniapp_request_wifi_setup"
194
+ // Unregister after this many missed pongs. Generous on purpose: a busy
195
+ // context (heavy interim translation traffic) or OS scheduling while idle can
196
+ // delay pongs well past one interval, and killing a healthy-but-busy script
197
+ // drops its subscriptions (releasing the mic). With the liveness-timeout
198
+ // respawn path wired (MentraJSRouter.start), a genuinely dead context still
199
+ // comes back automatically — this threshold only bounds how long that takes.
200
+ const PING_TIMEOUT_THRESHOLD = 6 // ~30s
201
+
202
+ const TRANSCRIPT_TIMING_TELEMETRY = (globalThis as {__DEV__?: boolean}).__DEV__ === true
203
+
204
+ const RGB_LED_ACTIONS = new Set<RgbLedAction>(["on", "off"])
205
+ const RGB_LED_COLORS = new Set<RgbLedColor>(["red", "green", "blue", "orange", "white"])
206
+ const CAMERA_FOV_MIN = 62
207
+ const CAMERA_FOV_MAX = 118
208
+ const CAMERA_FOV_DEFAULT = 102
209
+ const CAMERA_FOV_PRESETS: Record<CameraFovPreset, number> = {narrow: 82, standard: CAMERA_FOV_DEFAULT, wide: 118}
210
+ const CAMERA_ROI_POSITION_BY_NAME: Record<string, CameraRoiPosition> = {center: "center", bottom: "bottom", top: "top"}
211
+
212
+ // =============================================================================
213
+ // Declared-permission record helper (for CONNECT_ACK / PERMISSIONS_UPDATE)
214
+ // =============================================================================
215
+
216
+ /**
217
+ * Map from manifest permission types (uppercase, snake-y) to the lowercase
218
+ * canonical permission keys the SDK exposes via session.permissions. Mirrors
219
+ * cloud SDK v3's PermissionType union.
220
+ *
221
+ * BACKGROUND_LOCATION + POST_NOTIFICATIONS aren't in v3's surface; they map
222
+ * onto the same canonical keys (location / notifications) since the SDK's
223
+ * `has()` is "do I have *any* form of this permission declared".
224
+ */
225
+ const PERMISSION_TYPE_TO_CANONICAL: Record<string, string> = {
226
+ MICROPHONE: "microphone",
227
+ CAMERA: "camera",
228
+ LOCATION: "location",
229
+ BACKGROUND_LOCATION: "location",
230
+ READ_NOTIFICATIONS: "notifications",
231
+ POST_NOTIFICATIONS: "notifications",
232
+ CALENDAR: "calendar",
233
+ }
234
+
235
+ function normalizeRgbLedAction(value: unknown): RgbLedAction {
236
+ return typeof value === "string" && RGB_LED_ACTIONS.has(value as RgbLedAction) ? (value as RgbLedAction) : "off"
237
+ }
238
+
239
+ function normalizeRgbLedColor(value: unknown): RgbLedColor | null {
240
+ return typeof value === "string" && RGB_LED_COLORS.has(value as RgbLedColor) ? (value as RgbLedColor) : null
241
+ }
242
+
243
+ function normalizeCameraFovPayload(payload: Record<string, unknown>): CameraFovRequest {
244
+ if (typeof payload.preset === "string") {
245
+ const preset = payload.preset in CAMERA_FOV_PRESETS ? (payload.preset as CameraFovPreset) : "standard"
246
+ return {preset}
247
+ }
248
+
249
+ const rawFov = typeof payload.fov === "number" && Number.isFinite(payload.fov) ? payload.fov : CAMERA_FOV_DEFAULT
250
+
251
+ return {
252
+ fov: Math.min(CAMERA_FOV_MAX, Math.max(CAMERA_FOV_MIN, rawFov)),
253
+ roiPosition: normalizeCameraRoiPosition(payload.roiPosition ?? payload.roi_position),
254
+ }
255
+ }
256
+
257
+ function normalizeCameraRoiPosition(value: unknown): CameraRoiPosition {
258
+ if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 2) {
259
+ return value === 1 ? "bottom" : value === 2 ? "top" : "center"
260
+ }
261
+
262
+ if (typeof value === "string") {
263
+ return CAMERA_ROI_POSITION_BY_NAME[value] ?? "center"
264
+ }
265
+
266
+ return "center"
267
+ }
268
+
269
+ const ALL_CANONICAL_PERMISSIONS = ["location", "microphone", "camera", "notifications", "calendar"] as const
270
+
271
+ /**
272
+ * Build the {location, microphone, camera, notifications, calendar} record the
273
+ * SDK's session.permissions module reads from. Missing types are false.
274
+ */
275
+ function computeDeclaredPermissionRecord(manifest: InstalledMiniappManifest | undefined): Record<string, boolean> {
276
+ const record: Record<string, boolean> = {}
277
+ for (const k of ALL_CANONICAL_PERMISSIONS) record[k] = false
278
+ for (const perm of manifest?.permissions ?? []) {
279
+ const canonical = PERMISSION_TYPE_TO_CANONICAL[perm.type?.toUpperCase()]
280
+ if (canonical) record[canonical] = true
281
+ }
282
+ return record
283
+ }
284
+
285
+ // =============================================================================
286
+ // PERMISSION_NOT_DECLARED warnings — per-session dedup
287
+ // =============================================================================
288
+ //
289
+ // When a miniapp tries to subscribe / call something whose required permission
290
+ // isn't declared in miniapp.json, the runtime rejects with
291
+ // PERMISSION_NOT_DECLARED. The error reaches the SDK but most authors don't
292
+ // subscribe to session.on("error", ...), so it's silent in practice.
293
+ //
294
+ // To make the failure discoverable for developers running the MentraOS app
295
+ // from source, log a clear, copy-pasteable message in the phone console that
296
+ // names the offending permission, the offending stream/op, and the JSON
297
+ // snippet to add to miniapp.json. Once-per-session per (packageName, permission)
298
+ // to avoid spam from a tight retry loop.
299
+ //
300
+ // Production users running the App Store build of MentraOS won't see these
301
+ // (they don't watch Metro/adb logcat). For them, the WebView console bridge
302
+ // (#5 of the quick-fixes round) ships the structured error to the miniapp
303
+ // itself, and the miniapp's own console.warn flows to the dev terminal.
304
+
305
+ const warnedPermission = new Set<string>() // key: `${packageName}::${permission}`
306
+
307
+ function logPermissionNotDeclared(
308
+ packageName: string,
309
+ permission: string,
310
+ context: string,
311
+ manifestSnippet: string,
312
+ ): void {
313
+ const key = `${packageName}::${permission}`
314
+ if (warnedPermission.has(key)) return
315
+ warnedPermission.add(key)
316
+ console.warn(
317
+ `${LOG_TAG}: ${packageName} attempted ${context}, but permission ${permission} is not declared in miniapp.json.\n` +
318
+ `Add this to the "permissions" array:\n ${manifestSnippet}`,
319
+ )
320
+ }
321
+
322
+ /** Reset the per-session dedup; called when a miniapp unregisters so a fresh launch warns again. */
323
+ function resetPermissionWarnings(packageName: string): void {
324
+ for (const key of warnedPermission) {
325
+ if (key.startsWith(`${packageName}::`)) warnedPermission.delete(key)
326
+ }
327
+ }
328
+
329
+ // =============================================================================
330
+ // LocalMiniappRuntime
331
+ // =============================================================================
332
+
333
+ class LocalMiniappRuntime {
334
+ private static instance: LocalMiniappRuntime | null = null
335
+
336
+ /** Connected miniapps keyed by packageName. */
337
+ private connectedApps: Map<string, ConnectedMiniapp> = new Map()
338
+
339
+ /**
340
+ * Packages that have completed their CONNECT handshake (sent
341
+ * `miniapp_connect`, got `CONNECT_ACK`). Distinct from `connectedApps`,
342
+ * which is populated at spawn/registration time — handshake is later.
343
+ * Drives {@link waitForConnect}.
344
+ */
345
+ private handshookApps: Set<string> = new Set()
346
+
347
+ /** Pending {@link waitForConnect} resolvers, keyed by packageName. */
348
+ private connectWaiters: Map<string, Set<(err?: Error) => void>> = new Map()
349
+
350
+ /** Ref-counted stream subscriptions: stream → set of packageNames. */
351
+ private streamSubscribers: Map<string, Set<string>> = new Map()
352
+
353
+ /** Guards one-time wiring of the cloud transcript/translation fan-out. */
354
+ private cloudResultsWired = false
355
+ private cloudStatusWired = false
356
+ private cloudAudioSubscriptionSync = new CloudAudioSubscriptionSync()
357
+
358
+ /** Ping interval handle. */
359
+ private pingIntervalId: number | null = null
360
+ private foregroundProbeTimers: Map<string, number> = new Map()
361
+
362
+ /**
363
+ * Notified when a miniapp is unregistered for missing liveness pings.
364
+ * MentraJSRouter wires this into its crash-respawn machinery so a
365
+ * silently-stalled background script is restarted (with crash-loop
366
+ * protection) instead of staying dead — a dead background drops its
367
+ * subscriptions, which releases the mic while the webview still looks
368
+ * alive.
369
+ */
370
+ public onLivenessTimeout: ((packageName: string) => void) | null = null
371
+
372
+ /** Pending cloud requests: requestId → packageName that originated the request. */
373
+ private pendingCloudRequests: Map<string, {packageName: string; envelopeRequestId?: string}> = new Map()
374
+ private speechRuns = new Map<string, SpeechRun>()
375
+
376
+ // Browser fallback token auth — HMAC-signed blob with a phone-local
377
+ // secret. Both issuer and verifier are the same process, so the
378
+ // secret never leaves the device.
379
+ private localSecret = `miniapp_${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
380
+ private usedTokens = new Set<string>()
381
+
382
+ private constructor() {}
383
+
384
+ /**
385
+ * Generate an HMAC-signed local session token for browser fallback auth.
386
+ * Token format: base64(JSON({userId, packageName, exp})).base64(HMAC-SHA256(payload, secret))
387
+ * Single-use, 5-minute TTL.
388
+ */
389
+ public generateLocalToken(userId: string, packageName: string): string {
390
+ const payload = JSON.stringify({
391
+ userId,
392
+ packageName,
393
+ exp: Date.now() + 5 * 60 * 1000, // 5 min TTL
394
+ nonce: Math.random().toString(36).slice(2, 10),
395
+ })
396
+ const payloadB64 = btoa(payload)
397
+ const sig = this.hmacSign(payload)
398
+ return `${payloadB64}.${sig}`
399
+ }
400
+
401
+ /**
402
+ * Validate and consume a local session token. Single-use.
403
+ */
404
+ public validateLocalToken(token: string): {userId: string; packageName: string} | null {
405
+ const parts = token.split(".")
406
+ if (parts.length !== 2) return null
407
+ const [payloadB64, sig] = parts
408
+
409
+ // Check single-use
410
+ if (this.usedTokens.has(token)) return null
411
+
412
+ // Verify signature
413
+ let payload: string
414
+ try {
415
+ payload = atob(payloadB64!)
416
+ } catch {
417
+ return null
418
+ }
419
+ if (this.hmacSign(payload) !== sig) return null
420
+
421
+ // Parse and check expiry
422
+ let parsed: {userId: string; packageName: string; exp: number}
423
+ try {
424
+ parsed = JSON.parse(payload)
425
+ } catch {
426
+ return null
427
+ }
428
+ if (Date.now() > parsed.exp) return null
429
+
430
+ // Mark as used
431
+ this.usedTokens.add(token)
432
+ // Prune old used tokens periodically (keep set from growing unbounded)
433
+ if (this.usedTokens.size > 1000) {
434
+ this.usedTokens.clear()
435
+ }
436
+
437
+ return {userId: parsed.userId, packageName: parsed.packageName}
438
+ }
439
+
440
+ /**
441
+ * Simple HMAC-SHA256 using Web Crypto API (available in React Native).
442
+ * Returns base64url-encoded signature.
443
+ * Falls back to a simpler hash if crypto.subtle is not available.
444
+ */
445
+ private hmacSign(payload: string): string {
446
+ // Synchronous HMAC using a simple hash — Web Crypto's subtle.sign is async
447
+ // which doesn't fit cleanly here. For phone-local single-process auth,
448
+ // a keyed hash is sufficient.
449
+ let hash = 0
450
+ const key = this.localSecret
451
+ const input = key + payload + key
452
+ for (let i = 0; i < input.length; i++) {
453
+ const ch = input.charCodeAt(i)
454
+ hash = ((hash << 5) - hash + ch) | 0
455
+ }
456
+ // Mix in more bytes for collision resistance
457
+ let hash2 = 0x811c9dc5 // FNV offset basis
458
+ for (let i = 0; i < input.length; i++) {
459
+ hash2 ^= input.charCodeAt(i)
460
+ hash2 = Math.imul(hash2, 0x01000193) // FNV prime
461
+ }
462
+ return btoa(String(hash >>> 0) + "." + String(hash2 >>> 0))
463
+ }
464
+
465
+ public static getInstance(): LocalMiniappRuntime {
466
+ if (!LocalMiniappRuntime.instance) {
467
+ LocalMiniappRuntime.instance = new LocalMiniappRuntime()
468
+ }
469
+ return LocalMiniappRuntime.instance
470
+ }
471
+
472
+ private initialized = false
473
+
474
+ /**
475
+ * Initialize the runtime. Called from MantleManager.init().
476
+ * Idempotent — safe to call multiple times.
477
+ */
478
+ public initialize(): void {
479
+ if (this.initialized) return
480
+ this.initialized = true
481
+ console.log(`${LOG_TAG}: initialize()`)
482
+ this.ensurePingLoop()
483
+ }
484
+
485
+ /**
486
+ * Handle an incoming cloud message forwarded by SocketComms
487
+ * (phone_stream_status, phone_managed_stream_status).
488
+ *
489
+ * Routes the response back to the originating miniapp via the requestId
490
+ * that was stored when the miniapp first made the request.
491
+ */
492
+ public handleCloudMessage(msg: any): void {
493
+ const requestId = msg.requestId as string | undefined
494
+ const msgType = msg.type as string
495
+
496
+ console.log(`${LOG_TAG}: Cloud message: ${msgType}, requestId=${requestId ?? "none"}`)
497
+
498
+ if (!requestId) {
499
+ console.warn(`${LOG_TAG}: Cloud message ${msgType} has no requestId, cannot route`)
500
+ return
501
+ }
502
+
503
+ const pending = this.pendingCloudRequests.get(requestId)
504
+ if (!pending) {
505
+ console.warn(`${LOG_TAG}: No pending request for requestId=${requestId}`)
506
+ return
507
+ }
508
+
509
+ this.pendingCloudRequests.delete(requestId)
510
+
511
+ switch (msgType) {
512
+ case "phone_stream_status": {
513
+ // Unreachable for phone-orchestrated streams (the coordinator owns
514
+ // their lifecycle and never registers a pending cloud request).
515
+ // Retained as a safety net for any legacy registration path.
516
+ this.sendToMiniapp(pending.packageName, {
517
+ type: MiniappResponseType.EVENT,
518
+ streamType: "stream_status",
519
+ data: {
520
+ streamId: msg.streamId,
521
+ status: msg.status,
522
+ errorDetails: msg.errorDetails,
523
+ },
524
+ })
525
+ // Re-register for ongoing status updates (streams send multiple status messages)
526
+ this.pendingCloudRequests.set(requestId, pending)
527
+ break
528
+ }
529
+
530
+ case "phone_managed_stream_status": {
531
+ if (msg.status === "connected" || msg.status === "active") {
532
+ // Managed stream is ready — send back the playback URLs as the request result
533
+ this.sendResult(pending.packageName, pending.envelopeRequestId, true, {
534
+ streamId: msg.streamId,
535
+ hlsUrl: msg.hlsUrl,
536
+ dashUrl: msg.dashUrl,
537
+ webrtcUrl: msg.webrtcUrl,
538
+ })
539
+ }
540
+ // Forward all statuses as events too
541
+ this.sendToMiniapp(pending.packageName, {
542
+ type: MiniappResponseType.EVENT,
543
+ streamType: "stream_status",
544
+ data: {
545
+ streamId: msg.streamId,
546
+ status: msg.status,
547
+ hlsUrl: msg.hlsUrl,
548
+ dashUrl: msg.dashUrl,
549
+ webrtcUrl: msg.webrtcUrl,
550
+ },
551
+ })
552
+ // Re-register for ongoing updates
553
+ this.pendingCloudRequests.set(requestId, pending)
554
+ break
555
+ }
556
+
557
+ default:
558
+ console.warn(`${LOG_TAG}: Unknown cloud message type: ${msgType}`)
559
+ }
560
+ }
561
+
562
+ /**
563
+ * Register a pending cloud request so we can route the response back.
564
+ */
565
+ public registerPendingCloudRequest(requestId: string, packageName: string, envelopeRequestId?: string): void {
566
+ this.pendingCloudRequests.set(requestId, {packageName, envelopeRequestId})
567
+ }
568
+
569
+ // ===========================================================================
570
+ // App registration
571
+ // ===========================================================================
572
+
573
+ public registerApp(
574
+ packageName: string,
575
+ sendFn: (raw: string) => void,
576
+ installedManifest?: InstalledMiniappManifest,
577
+ ): void {
578
+ console.log(`${LOG_TAG}: registerApp(${packageName})`)
579
+ this.clearForegroundProbe(packageName)
580
+ // Fresh spawn → fresh handshake. Clear any prior handshake flag so
581
+ // waitForConnect() blocks until this context's own CONNECT arrives.
582
+ this.handshookApps.delete(packageName)
583
+ // If the app is already registered (e.g. QR scanned again for same package),
584
+ // tear down its old subscriptions first so streamSubscribers doesn't keep
585
+ // dangling references. The WebView will re-subscribe after its CONNECT.
586
+ if (this.connectedApps.has(packageName)) {
587
+ const existing = this.connectedApps.get(packageName)!
588
+ this.cancelSpeech(packageName)
589
+ // A crash respawn replaces the ConnectedMiniapp without going through
590
+ // unregisterApp(). Abort the old incarnation's native PCM player before
591
+ // its only stream id is lost. Target the exact stream rather than using
592
+ // stopForApp(), whose async playback cleanup could race with audio the
593
+ // replacement app starts after it connects.
594
+ const existingSpeakerStreamId = existing.activeSpeakerStreamId
595
+ if (existingSpeakerStreamId) {
596
+ existing.activeSpeakerStreamId = undefined
597
+ void audioPlaybackService.abortStream(existingSpeakerStreamId).catch((error) => {
598
+ console.warn(`${LOG_TAG}: failed to abort speaker stream while replacing ${packageName}`, error)
599
+ })
600
+ }
601
+ if (existing.authRefreshTimerId !== null) {
602
+ BgTimer.clearTimeout(existing.authRefreshTimerId)
603
+ existing.authRefreshTimerId = null
604
+ }
605
+ if (existing.authRetryTimerId !== null) {
606
+ BgTimer.clearTimeout(existing.authRetryTimerId)
607
+ existing.authRetryTimerId = null
608
+ }
609
+ for (const stream of existing.subscriptions) {
610
+ const subs = this.streamSubscribers.get(stream)
611
+ if (subs) {
612
+ subs.delete(packageName)
613
+ if (subs.size === 0) this.streamSubscribers.delete(stream)
614
+ }
615
+ }
616
+ this.recomputeMicRequirements()
617
+ this.updateCloudSubscriptions()
618
+ }
619
+ this.connectedApps.set(packageName, {
620
+ subscriptions: new Set(),
621
+ forceLocalTranscriptionStreams: new Set(),
622
+ cloudTranscriptionStreams: new Set(),
623
+ sendMessage: sendFn,
624
+ lastPongAt: Date.now(),
625
+ installedManifest,
626
+ authRefreshTimerId: null,
627
+ authRetryTimerId: null,
628
+ authDelivered: false,
629
+ speakerState: "idle",
630
+ requestedLocationRate: null,
631
+ })
632
+ // If we just clobbered an existing entry, its requestedLocationRate is
633
+ // now gone — recompute so the aggregate doesn't include a stale rate.
634
+ // The WebView will re-SUBSCRIBE shortly and the rate will reappear.
635
+ this.recomputeLocationTier()
636
+ this.ensureCloudStatusWired()
637
+ this.ensurePingLoop()
638
+
639
+ // Show the system boot message ("Starting <name>…") on the glasses for the
640
+ // bounded boot window, mirroring the cloud DisplayManager boot screen. The
641
+ // window ends early once this app pushes its first display (see
642
+ // LocalDisplayManager.request), or after ~1.5s. Fires once per spawn —
643
+ // including a crash-respawn, since each spawn is a fresh "app is starting".
644
+ localDisplayManager.onMount(packageName, installedManifest?.name ?? packageName)
645
+ }
646
+
647
+ /**
648
+ * Resolve once `packageName` has completed its CONNECT handshake. If it has
649
+ * already connected, resolves immediately. Rejects if the timeout elapses
650
+ * first, or if the app is unregistered while waiting.
651
+ *
652
+ * Used by the launcher's `ensureConnected` (and thus the action broker) so a
653
+ * just-spawned target is known to be live before anything is delivered to it.
654
+ */
655
+ public waitForConnect(packageName: string, timeoutMs: number): Promise<void> {
656
+ if (this.handshookApps.has(packageName)) return Promise.resolve()
657
+ return new Promise<void>((resolve, reject) => {
658
+ let settled = false
659
+ const waiter = (err?: Error) => {
660
+ if (settled) return
661
+ settled = true
662
+ clearTimeout(timer)
663
+ this.connectWaiters.get(packageName)?.delete(waiter)
664
+ if (err) reject(err)
665
+ else resolve()
666
+ }
667
+ const timer = setTimeout(() => {
668
+ waiter(new Error(`waitForConnect: ${packageName} did not connect within ${timeoutMs}ms`))
669
+ }, timeoutMs)
670
+ let set = this.connectWaiters.get(packageName)
671
+ if (!set) {
672
+ set = new Set()
673
+ this.connectWaiters.set(packageName, set)
674
+ }
675
+ set.add(waiter)
676
+ })
677
+ }
678
+
679
+ /** Resolve/reject all pending {@link waitForConnect} waiters for a package. */
680
+ private flushConnectWaiters(packageName: string, err?: Error): void {
681
+ const waiters = this.connectWaiters.get(packageName)
682
+ if (!waiters) return
683
+ this.connectWaiters.delete(packageName)
684
+ for (const w of waiters) w(err)
685
+ }
686
+
687
+ /**
688
+ * Invalidate the CONNECT handshake for a package whose native context is
689
+ * being replaced WITHOUT going through register/unregister — i.e. a crash
690
+ * respawn. Without this, {@link waitForConnect} would treat the dead/
691
+ * not-yet-initialized context as connected and deliver to it. Pending waiters
692
+ * are failed so a wake mid-respawn retries rather than hanging.
693
+ */
694
+ public resetHandshake(packageName: string): void {
695
+ this.handshookApps.delete(packageName)
696
+ this.flushConnectWaiters(packageName, new Error(`${packageName} respawning`))
697
+ }
698
+
699
+ /**
700
+ * Attach (or update) the installedManifest for an already-registered app.
701
+ * Used when the manifest is fetched asynchronously (dev miniapps) after the
702
+ * miniapp has already CONNECTed — preserves existing subscriptions.
703
+ */
704
+ public setInstalledManifest(packageName: string, installedManifest: InstalledMiniappManifest): void {
705
+ const app = this.connectedApps.get(packageName)
706
+ if (!app) return
707
+ app.installedManifest = installedManifest
708
+
709
+ // Push declared-permission record to the SDK so session.permissions stays
710
+ // in sync. Sent regardless of CONNECT_ACK timing — covers the dev-miniapp
711
+ // case where the manifest is fetched async after the miniapp CONNECTs.
712
+ this.sendToMiniapp(packageName, {
713
+ type: MiniappResponseType.PERMISSIONS_UPDATE,
714
+ permissions: computeDeclaredPermissionRecord(installedManifest),
715
+ })
716
+ }
717
+
718
+ /**
719
+ * Graceful version of {@link unregisterApp}: notify the miniapp via
720
+ * `WILL_DISCONNECT`, wait ~50ms so its `beforeDisconnect` handlers can
721
+ * fire one last `sendOneShot` (e.g. `display.clear()`), then run the
722
+ * normal teardown. Use this for any disconnect path where the socket
723
+ * is still open. For ungraceful paths (transport already closed),
724
+ * call {@link unregisterApp} directly — the heads-up would be
725
+ * undeliverable anyway.
726
+ */
727
+ public async gracefullyUnregisterApp(packageName: string, reason = "unregistering"): Promise<void> {
728
+ const app = this.connectedApps.get(packageName)
729
+ if (!app) return
730
+
731
+ console.log(`${LOG_TAG}: gracefullyUnregisterApp(${packageName}) — sending WILL_DISCONNECT`)
732
+ this.sendToMiniapp(packageName, {
733
+ type: MiniappResponseType.WILL_DISCONNECT,
734
+ reason,
735
+ })
736
+ // 50 ms: imperceptible to the user, plenty of time for the SDK's
737
+ // synchronous `beforeDisconnect` handlers to flush a final
738
+ // sendOneShot through the still-open transport.
739
+ await new Promise<void>((resolve) => setTimeout(resolve, 50))
740
+ this.unregisterApp(packageName)
741
+ }
742
+
743
+ public unregisterApp(packageName: string): void {
744
+ console.log(`${LOG_TAG}: unregisterApp(${packageName})`)
745
+ this.clearForegroundProbe(packageName)
746
+ this.clearMiniappAuthRefresh(packageName)
747
+ this.clearMiniappAuthDeliveryRetry(packageName)
748
+ // Drop the handshake flag and fail any in-flight waitForConnect() callers
749
+ // (e.g. a wake that's mid-handshake when the app is torn down). Done before
750
+ // the early-return so it runs even if the connectedApps entry is already gone.
751
+ this.handshookApps.delete(packageName)
752
+ this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`))
753
+
754
+ // Warm-up is a request-owned camera lease. Cancel it even if the app record was already
755
+ // removed so a close racing the camera-open promise cannot leave the sensor running.
756
+ // Keep camera teardown ordered: restoring FOV can restart the HAL, so the warm-up lease must
757
+ // finish closing the sensor before FOV reconciliation begins. Continue to FOV cleanup even if
758
+ // warm-up cancellation fails; phone ownership remains retryable until the ASG lease TTL.
759
+ void phonePhotoCoordinator
760
+ .stopWarmUpForApp(packageName)
761
+ .catch((error) => {
762
+ console.warn(`${LOG_TAG}: failed to stop camera warm-up for ${packageName} on unregister`, error)
763
+ })
764
+ .then(() => phoneCameraFovCoordinator.releaseForApp(packageName))
765
+ .catch((error) => {
766
+ console.warn(`${LOG_TAG}: failed to release camera FOV override for ${packageName} on unregister`, error)
767
+ })
768
+ const app = this.connectedApps.get(packageName)
769
+ if (!app) return
770
+
771
+ // Remove from all stream subscriber sets
772
+ for (const stream of app.subscriptions) {
773
+ const subs = this.streamSubscribers.get(stream)
774
+ if (subs) {
775
+ subs.delete(packageName)
776
+ if (subs.size === 0) {
777
+ this.streamSubscribers.delete(stream)
778
+ }
779
+ }
780
+ }
781
+
782
+ // Reset per-session warning dedup so a relaunch surfaces issues again.
783
+ resetPermissionWarnings(packageName)
784
+
785
+ // Cancel lookahead/in-flight synthesis before stopping the current file so
786
+ // an interrupted sentence cannot advance and reclaim playback.
787
+ this.cancelSpeech(packageName)
788
+
789
+ // Stop audio for this app
790
+ audioPlaybackService.stopForApp(packageName)
791
+
792
+ // Tear down this app's blob state: abort in-flight uploads + close readers
793
+ // so a crashed/closed miniapp doesn't leak partial files or file handles.
794
+ this.blobStore.onAppGone(packageName)
795
+
796
+ // Release phone-owned camera streams. If a miniapp closes/crashes without
797
+ // sending STREAM_STOP, the host coordinator must drop its subscriber/owner
798
+ // so glasses publishing and managed Cloudflare inputs do not leak.
799
+ void phoneStreamCoordinator.stop(packageName).catch((error) => {
800
+ console.warn(`${LOG_TAG}: failed to stop stream for ${packageName} on unregister`, error)
801
+ })
802
+
803
+ // Stop any phone-owned video recordings for this app. A miniapp that
804
+ // closes/crashes mid-recording loses its recordingId, so without this the
805
+ // glasses keep recording until the max-recording timeout or thermal shutdown.
806
+ void phoneVideoCoordinator.stopForApp(packageName).catch((error) => {
807
+ console.warn(`${LOG_TAG}: failed to stop video recording for ${packageName} on unregister`, error)
808
+ })
809
+
810
+ // Detach the per-app nav event forwarder but leave the native nav session
811
+ // running. The user may have just closed the mini-app UI and will reopen
812
+ // it; stopping the session here would kill an active trip mid-route.
813
+ // Navigation is only stopped when the mini-app explicitly calls
814
+ // navigation.stop() or when the trip arrives/errors naturally.
815
+ // (See NavigationHandlers — activeNavApps stays populated so a reconnect
816
+ // can reattach listeners and resume.)
817
+ this.navigationHandlers.onDisconnect(packageName)
818
+
819
+ // Recompute heading subscription — if this app was the last subscriber,
820
+ // the sensor will stop.
821
+ this.recomputeHeadingSubscription()
822
+ // Same for the IMU/accelerometer stream.
823
+ this.recomputeImuSubscription()
824
+
825
+ // Drop this app's location-tier request before recomputing so the
826
+ // aggregate falls back down if it was the strictest. Done before
827
+ // `connectedApps.delete` below so it doesn't matter that the iter
828
+ // in `recomputeLocationTier` is over connectedApps — clearing the
829
+ // field is enough.
830
+ app.requestedLocationRate = null
831
+ this.recomputeLocationTier()
832
+
833
+ // Clean up any pending cloud requests from this app
834
+ for (const [reqId, pending] of this.pendingCloudRequests) {
835
+ if (pending.packageName === packageName) {
836
+ this.pendingCloudRequests.delete(reqId)
837
+ }
838
+ }
839
+
840
+ // Release any display real estate this app held — if it owned the
841
+ // current on-glasses frame, this clears the glasses (or restores the
842
+ // core app's saved frame).
843
+ localDisplayManager.onUnmount(packageName)
844
+
845
+ this.connectedApps.delete(packageName)
846
+ this.recomputeMicRequirements()
847
+ this.updateCloudSubscriptions()
848
+
849
+ if (this.connectedApps.size === 0) {
850
+ this.stopPingLoop()
851
+ }
852
+ }
853
+
854
+ // ===========================================================================
855
+ // Inbound message handling
856
+ // ===========================================================================
857
+
858
+ public handleRawMessage(packageName: string, raw: string): void {
859
+ const envelope = parseEnvelope(raw)
860
+ if (!envelope) {
861
+ // Not a miniapp envelope — ignore (could be legacy bridge message)
862
+ return
863
+ }
864
+
865
+ const payload = envelope.payload as Record<string, unknown>
866
+ const requestType = payload.type as string | undefined
867
+ const requestId = envelope.requestId
868
+
869
+ if (!requestType) {
870
+ console.warn(`${LOG_TAG}: Envelope from ${packageName} missing payload.type`)
871
+ return
872
+ }
873
+
874
+ // ANY inbound message proves the context is alive — a busy background
875
+ // script streaming DISPLAY/storage traffic shouldn't be killed by the
876
+ // liveness watchdog just because its PONG replies queue behind real work.
877
+ this.handlePong(packageName)
878
+
879
+ // Console-tap forwarding. The miniapp's console.log/warn/etc is wrapped
880
+ // (via injected shim from miniappGlobals.ts) to post a `dev_log`
881
+ // envelope. We fan out to two destinations:
882
+ // 1. DevServerBridge — forwards to the laptop's `mentra-miniapp dev`
883
+ // terminal. No-op when there's no sidecar (installed miniapps).
884
+ // 2. React Native console — surfaces the log in Metro / Xcode console
885
+ // / adb logcat so installed-miniapp errors are still inspectable
886
+ // when there's no laptop to forward to.
887
+ if (requestType === "dev_log") {
888
+ const level = (payload.level as string | undefined) ?? "log"
889
+ const args = Array.isArray(payload.args) ? (payload.args as unknown[]) : []
890
+ const timestamp = (payload.timestamp as number | undefined) ?? Date.now()
891
+ // Source is "ui" because the single-bundle WebView console-tap
892
+ // shim in miniappGlobals.ts is the only thing that posts
893
+ // `dev_log`; the two-layer path uses a separate envelope
894
+ // (`{type:"log", source}`) routed by MentraUIRouter.
895
+ devServerBridge.forwardLog(packageName, level, args, timestamp, "ui")
896
+
897
+ const tag = `[MINIAPP ${packageName}]`
898
+ const fn = (console as unknown as Record<string, (...a: unknown[]) => void>)[level] ?? console.log
899
+ try {
900
+ fn(tag, ...args)
901
+ } catch {
902
+ console.log(tag, ...args)
903
+ }
904
+ return
905
+ }
906
+
907
+ // Dispatch
908
+ switch (requestType) {
909
+ case MiniappRequestType.CONNECT:
910
+ void this.handleConnect(packageName, payload, requestId)
911
+ break
912
+ case MiniappRequestType.AUTH_REFRESH:
913
+ void this.handleAuthRefresh(packageName, payload, requestId)
914
+ break
915
+ case MiniappRequestType.SUBSCRIBE:
916
+ this.handleSubscribe(packageName, payload, requestId)
917
+ break
918
+ case MiniappRequestType.DISPLAY:
919
+ this.handleDisplay(packageName, payload, requestId)
920
+ break
921
+ case MiniappRequestType.RENDER:
922
+ this.handleRender(packageName, payload, requestId)
923
+ break
924
+ case MiniappRequestType.PLAY_AUDIO:
925
+ this.handlePlayAudio(packageName, payload, requestId)
926
+ break
927
+ case MiniappRequestType.STOP_AUDIO:
928
+ this.handleStopAudio(packageName, payload, requestId)
929
+ break
930
+ case MiniappRequestType.SPEAK:
931
+ this.handleSpeak(packageName, payload, requestId)
932
+ break
933
+ case MiniappRequestType.SPEAKER_STREAM_OPEN:
934
+ void this.handleSpeakerStreamOpen(packageName, payload, requestId)
935
+ break
936
+ case MiniappRequestType.SPEAKER_STREAM_WRITE:
937
+ void this.handleSpeakerStreamWrite(packageName, payload, requestId)
938
+ break
939
+ case MiniappRequestType.SPEAKER_STREAM_CLOSE:
940
+ void this.handleSpeakerStreamClose(packageName, payload, requestId)
941
+ break
942
+ case MiniappRequestType.SPEAKER_STREAM_ABORT:
943
+ void this.handleSpeakerStreamAbort(packageName, payload, requestId)
944
+ break
945
+ case MiniappRequestType.RGB_LED:
946
+ void this.handleRgbLed(packageName, payload, requestId)
947
+ break
948
+ case MiniappRequestType.LOCATION_POLL:
949
+ this.handleLocationPoll(packageName, requestId)
950
+ break
951
+ case MiniappRequestType.CALENDAR_LIST_EVENTS:
952
+ void this.handleCalendarListEvents(packageName, payload, requestId)
953
+ break
954
+ case MiniappRequestType.NAVIGATION_START:
955
+ this.navigationHandlers.handleStart(packageName, payload, requestId)
956
+ break
957
+ case MiniappRequestType.NAVIGATION_STOP:
958
+ this.navigationHandlers.handleStop(packageName, requestId)
959
+ break
960
+ case MiniappRequestType.NAVIGATION_DEVIATE:
961
+ this.navigationHandlers.handleDeviate(packageName, payload, requestId)
962
+ break
963
+ case MiniappRequestType.NAVIGATION_SET_WRONG_SIDEWALK:
964
+ this.navigationHandlers.handleSetWrongSidewalk(packageName, payload, requestId)
965
+ break
966
+ case MiniappRequestType.NAVIGATION_SET_SKIP_CROSSINGS:
967
+ this.navigationHandlers.handleSetSkipCrossings(packageName, payload, requestId)
968
+ break
969
+ case MiniappRequestType.NAVIGATION_GET_STATE:
970
+ this.navigationHandlers.handleGetState(packageName, requestId)
971
+ break
972
+ case MiniappRequestType.NAVIGATION_COMPUTE_ROUTE:
973
+ this.navigationHandlers.handleComputeRoute(packageName, payload, requestId)
974
+ break
975
+ case MiniappRequestType.NAVIGATION_REVERSE_GEOCODE:
976
+ this.navigationHandlers.handleReverseGeocode(packageName, payload, requestId)
977
+ break
978
+ case MiniappRequestType.NAVIGATION_PLACE_AUTOCOMPLETE:
979
+ this.navigationHandlers.handlePlaceAutocomplete(packageName, payload, requestId)
980
+ break
981
+ case MiniappRequestType.NAVIGATION_PLACE_DETAILS:
982
+ this.navigationHandlers.handlePlaceDetails(packageName, payload, requestId)
983
+ break
984
+ case MiniappRequestType.NAVIGATION_REQUEST_PERMISSION:
985
+ this.navigationHandlers.handleRequestPermission(packageName, requestId)
986
+ break
987
+ case MiniappRequestType.STORAGE_GET:
988
+ this.handleStorageGet(packageName, payload, requestId)
989
+ break
990
+ case MiniappRequestType.STORAGE_SET:
991
+ this.handleStorageSet(packageName, payload, requestId)
992
+ break
993
+ case MiniappRequestType.STORAGE_DELETE:
994
+ this.handleStorageDelete(packageName, payload, requestId)
995
+ break
996
+ case MiniappRequestType.STORAGE_LIST:
997
+ this.handleStorageList(packageName, payload, requestId)
998
+ break
999
+ case MiniappRequestType.STORAGE_CLEAR:
1000
+ this.handleStorageClear(packageName, requestId)
1001
+ break
1002
+ case MiniappRequestType.STORAGE_HAS:
1003
+ this.handleStorageHas(packageName, payload, requestId)
1004
+ break
1005
+ case MiniappRequestType.STORAGE_GET_ALL:
1006
+ this.handleStorageGetAll(packageName, requestId)
1007
+ break
1008
+ case MiniappRequestType.STORAGE_SET_MULTIPLE:
1009
+ this.handleStorageSetMultiple(packageName, payload, requestId)
1010
+ break
1011
+ case MiniappRequestType.STORAGE_FLUSH:
1012
+ // No-op today — MMKV writes through synchronously. Reserved so
1013
+ // a future debounced backend can honor "flush before I quit"
1014
+ // calls without breaking the API.
1015
+ this.sendResult(packageName, requestId, true)
1016
+ break
1017
+ case MiniappRequestType.CAMERA_FOV:
1018
+ void this.handleCameraFov(packageName, payload, requestId)
1019
+ break
1020
+ case MiniappRequestType.IMU_SET_ENABLED:
1021
+ this.handleImuSetEnabled(packageName, payload, requestId)
1022
+ break
1023
+ case MiniappRequestType.PING:
1024
+ // SDK should handle this itself; reply PONG just in case
1025
+ this.sendToMiniapp(packageName, {type: MiniappResponseType.PONG}, requestId)
1026
+ break
1027
+ case MiniappResponseType.PONG: {
1028
+ // Liveness already touched above for every inbound message; the
1029
+ // PONG carries no other action.
1030
+ break
1031
+ }
1032
+
1033
+ case MiniappRequestType.SHARE:
1034
+ this.handleShare(packageName, payload, requestId)
1035
+ break
1036
+ case MiniappRequestType.OPEN_URL:
1037
+ this.handleOpenUrl(packageName, payload)
1038
+ break
1039
+ case MiniappRequestType.COPY_CLIPBOARD:
1040
+ this.handleCopyClipboard(packageName, payload, requestId)
1041
+ break
1042
+ case MiniappRequestType.DOWNLOAD:
1043
+ this.handleDownload(packageName, payload, requestId)
1044
+ break
1045
+
1046
+ // Persistent binary blob storage (session.blob)
1047
+ case MiniappRequestType.BLOB_CREATE:
1048
+ this.blobStore.handleCreate(packageName, payload, requestId)
1049
+ break
1050
+ case MiniappRequestType.BLOB_WRITE:
1051
+ this.blobStore.handleWrite(packageName, payload, requestId)
1052
+ break
1053
+ case MiniappRequestType.BLOB_COMMIT:
1054
+ this.blobStore.handleCommit(packageName, payload, requestId)
1055
+ break
1056
+ case MiniappRequestType.BLOB_ABORT:
1057
+ this.blobStore.handleAbort(packageName, payload, requestId)
1058
+ break
1059
+ case MiniappRequestType.BLOB_SET_FROM_URL:
1060
+ void this.blobStore.handleSetFromUrl(packageName, payload, requestId)
1061
+ break
1062
+ case MiniappRequestType.BLOB_IMPORT:
1063
+ void this.blobStore.handleImport(packageName, payload, requestId)
1064
+ break
1065
+ case MiniappRequestType.BLOB_GET:
1066
+ this.blobStore.handleGet(packageName, payload, requestId)
1067
+ break
1068
+ case MiniappRequestType.BLOB_LIST:
1069
+ this.blobStore.handleList(packageName, payload, requestId)
1070
+ break
1071
+ case MiniappRequestType.BLOB_USAGE:
1072
+ this.blobStore.handleUsage(packageName, payload, requestId)
1073
+ break
1074
+ case MiniappRequestType.BLOB_DELETE:
1075
+ this.blobStore.handleDelete(packageName, payload, requestId)
1076
+ break
1077
+ case MiniappRequestType.BLOB_CLEAR:
1078
+ this.blobStore.handleClear(packageName, payload, requestId)
1079
+ break
1080
+ case MiniappRequestType.BLOB_OPEN_READ:
1081
+ this.blobStore.handleOpenRead(packageName, payload, requestId)
1082
+ break
1083
+ case MiniappRequestType.BLOB_READ:
1084
+ this.blobStore.handleRead(packageName, payload, requestId)
1085
+ break
1086
+ case MiniappRequestType.BLOB_CLOSE_READ:
1087
+ this.blobStore.handleCloseRead(packageName, payload, requestId)
1088
+ break
1089
+ case MiniappRequestType.BLOB_SHARE:
1090
+ void this.blobStore.handleShare(packageName, payload, requestId)
1091
+ break
1092
+
1093
+ // Cloud-coordinated features
1094
+ case MiniappRequestType.PHOTO:
1095
+ void this.handlePhoto(packageName, payload, requestId)
1096
+ break
1097
+ case MiniappRequestType.CAMERA_WARM_UP:
1098
+ void this.handleCameraWarmUp(packageName, payload, requestId)
1099
+ break
1100
+ case MiniappRequestType.VIDEO_RECORDING_START:
1101
+ void this.handleVideoRecordingStart(packageName, payload, requestId)
1102
+ break
1103
+ case MiniappRequestType.VIDEO_RECORDING_STOP:
1104
+ void this.handleVideoRecordingStop(packageName, payload, requestId)
1105
+ break
1106
+ case MiniappRequestType.STREAM_START:
1107
+ void this.handleStreamStart(packageName, payload, requestId)
1108
+ break
1109
+ case MiniappRequestType.STREAM_STOP:
1110
+ void this.handleStreamStop(packageName, payload, requestId)
1111
+ break
1112
+ case MiniappRequestType.MANAGED_STREAM_START:
1113
+ void this.handleManagedStreamStart(packageName, payload, requestId)
1114
+ break
1115
+ case MiniappRequestType.MANAGED_STREAM_STOP:
1116
+ void this.handleManagedStreamStop(packageName, payload, requestId)
1117
+ break
1118
+ case REQUEST_WIFI_SETUP_TYPE:
1119
+ void this.handleRequestWifiSetup(packageName, payload, requestId)
1120
+ break
1121
+
1122
+ // Inter-miniapp interop (SYSTEM apps only)
1123
+ case MiniappRequestType.MINIAPPS_LIST:
1124
+ this.handleMiniappsList(packageName, payload, requestId)
1125
+ break
1126
+ case MiniappRequestType.MINIAPPS_START:
1127
+ void this.handleMiniappsStart(packageName, payload, requestId)
1128
+ break
1129
+ case MiniappRequestType.MINIAPPS_STOP:
1130
+ void this.handleMiniappsStop(packageName, payload, requestId)
1131
+ break
1132
+ case MiniappRequestType.ACTION_INVOKE:
1133
+ void this.handleActionInvoke(packageName, payload, requestId)
1134
+ break
1135
+ case MiniappRequestType.ACTION_RESULT:
1136
+ this.handleActionResult(packageName, payload)
1137
+ break
1138
+
1139
+ // Deferred in v1
1140
+ case MiniappRequestType.DASHBOARD_CONTENT_UPDATE:
1141
+ this.sendResult(packageName, requestId, false, undefined, {
1142
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
1143
+ message: "Dashboard API is deferred in v1",
1144
+ })
1145
+ break
1146
+
1147
+ default:
1148
+ console.warn(`${LOG_TAG}: Unknown request type from ${packageName}: ${requestType}`)
1149
+ break
1150
+ }
1151
+ }
1152
+
1153
+ // ===========================================================================
1154
+ // Request handlers
1155
+ // ===========================================================================
1156
+
1157
+ private async handleConnect(
1158
+ packageName: string,
1159
+ _payload: Record<string, unknown>,
1160
+ requestId?: string,
1161
+ ): Promise<void> {
1162
+ console.log(`${LOG_TAG}: CONNECT from ${packageName}`)
1163
+
1164
+ // Register if not already
1165
+ const existing = this.connectedApps.get(packageName)
1166
+ if (!existing) {
1167
+ console.warn(`${LOG_TAG}: CONNECT from unregistered app ${packageName}, ignoring`)
1168
+ return
1169
+ }
1170
+
1171
+ // Update lastPongAt so it doesn't time out right away
1172
+ existing.lastPongAt = Date.now()
1173
+
1174
+ // Read current glasses capabilities from the settings store
1175
+ const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable) as
1176
+ | DeviceTypes
1177
+ | undefined
1178
+ const capabilities = getModelCapabilities(defaultWearable || DeviceTypes.NONE)
1179
+
1180
+ // Build the declared-permission record for the SDK's session.permissions
1181
+ // module. Lower-cased to match v3's PermissionType union (microphone,
1182
+ // camera, location, notifications, calendar). Missing permissions default
1183
+ // to false; this is manifest-declaration tracking only — OS-grant state
1184
+ // is intentionally not modeled here.
1185
+ const declaredPermissions = computeDeclaredPermissionRecord(existing.installedManifest)
1186
+ // Fresh handshake → clear any auth state from a prior context so a stale
1187
+ // retry timer can't fire against this new connection and so the retry loop
1188
+ // below re-arms from scratch.
1189
+ existing.authDelivered = false
1190
+ this.clearMiniappAuthDeliveryRetry(packageName)
1191
+ const authPromise = this.requestMiniappAuth(packageName)
1192
+ const initialAuth = await withTimeout(authPromise, 1_500)
1193
+ const userId = initialAuth?.mentraUserId ?? ""
1194
+ if (initialAuth) {
1195
+ existing.authDelivered = true
1196
+ this.scheduleMiniappAuthRefresh(packageName, initialAuth)
1197
+ }
1198
+
1199
+ this.sendToMiniapp(
1200
+ packageName,
1201
+ {
1202
+ type: MiniappResponseType.CONNECT_ACK,
1203
+ userId,
1204
+ packageName,
1205
+ capabilities,
1206
+ permissions: declaredPermissions,
1207
+ ...(initialAuth ? {auth: initialAuth} : {}),
1208
+ },
1209
+ requestId,
1210
+ )
1211
+ if (!initialAuth) {
1212
+ // The mint timed out or (more importantly) REJECTED — the latter happens
1213
+ // when the cloud client hasn't finished its first-boot Core token exchange
1214
+ // yet, which is exactly the case when a dev miniapp is scanned/launched
1215
+ // right after app start. CONNECT_ACK already went out without auth; keep
1216
+ // trying to mint (and re-drive on cloud-connect) so the app authenticates
1217
+ // to its backend on its own, instead of staying dead until a full manual
1218
+ // Cloud V2 reconnect re-runs the whole handshake.
1219
+ this.deliverInitialMiniappAuth(packageName, authPromise, 0)
1220
+ }
1221
+ this.sendCloudStatusToMiniapp(packageName)
1222
+
1223
+ // Handshake complete — unblock any launcher.waitForConnect() callers.
1224
+ this.handshookApps.add(packageName)
1225
+ this.flushConnectWaiters(packageName)
1226
+ }
1227
+
1228
+ private async requestMiniappAuth(packageName: string, opts?: {minTtlMs?: number}): Promise<MiniappAuthToken | null> {
1229
+ const authPackageName = getDevAppSourcePackage(packageName) ?? packageName
1230
+ const devAttestation = getDevAppAttestation(packageName) ?? undefined
1231
+ return cloudClientService.getMiniappAuthToken(authPackageName, {
1232
+ ...opts,
1233
+ ...(devAttestation ? {devAttestation} : {}),
1234
+ })
1235
+ }
1236
+
1237
+ private async handleAuthRefresh(
1238
+ packageName: string,
1239
+ payload: Record<string, unknown>,
1240
+ requestId?: string,
1241
+ ): Promise<void> {
1242
+ try {
1243
+ const auth = await this.refreshMiniappAuth(packageName, this.authRefreshOptions(payload))
1244
+ if (!auth) {
1245
+ this.sendResult(packageName, requestId, false, undefined, {
1246
+ code: MiniappErrorCode.NOT_CONNECTED,
1247
+ message: "Miniapp auth is not configured",
1248
+ })
1249
+ return
1250
+ }
1251
+ this.sendResult(packageName, requestId, true, {auth})
1252
+ } catch (err) {
1253
+ this.sendResult(packageName, requestId, false, undefined, {
1254
+ code: MiniappErrorCode.INTERNAL,
1255
+ message: (err as Error)?.message ?? "Miniapp auth refresh failed",
1256
+ })
1257
+ }
1258
+ }
1259
+
1260
+ private async refreshMiniappAuth(packageName: string, opts?: {minTtlMs?: number}): Promise<MiniappAuthToken | null> {
1261
+ const auth = await this.requestMiniappAuth(packageName, opts)
1262
+ if (!auth) return null
1263
+ this.scheduleMiniappAuthRefresh(packageName, auth)
1264
+ this.sendToMiniapp(packageName, {
1265
+ type: MiniappResponseType.AUTH_UPDATE,
1266
+ auth,
1267
+ })
1268
+ return auth
1269
+ }
1270
+
1271
+ private authRefreshOptions(payload: Record<string, unknown>): {minTtlMs?: number} | undefined {
1272
+ const minTtlMs = Number(payload.minTtlMs)
1273
+ if (!Number.isFinite(minTtlMs) || minTtlMs <= 0) return undefined
1274
+ return {minTtlMs}
1275
+ }
1276
+
1277
+ private scheduleMiniappAuthRefresh(packageName: string, auth: MiniappAuthToken): void {
1278
+ const app = this.connectedApps.get(packageName)
1279
+ if (!app) return
1280
+ this.clearMiniappAuthRefresh(packageName)
1281
+
1282
+ const refreshAt = auth.expiresAt - MINIAPP_AUTH_REFRESH_HEADROOM_MS
1283
+ const delay = Math.max(MINIAPP_AUTH_REFRESH_MIN_DELAY_MS, refreshAt - Date.now())
1284
+ app.authRefreshTimerId = BgTimer.setTimeout(() => {
1285
+ const current = this.connectedApps.get(packageName)
1286
+ if (!current) return
1287
+ current.authRefreshTimerId = null
1288
+ void this.refreshMiniappAuth(packageName).catch((err) => {
1289
+ console.warn(`${LOG_TAG}: miniapp auth refresh failed for ${packageName}: ${(err as Error)?.message ?? err}`)
1290
+ })
1291
+ }, delay)
1292
+ }
1293
+
1294
+ private clearMiniappAuthRefresh(packageName: string): void {
1295
+ const app = this.connectedApps.get(packageName)
1296
+ if (!app || app.authRefreshTimerId === null) return
1297
+ const timerId = app.authRefreshTimerId
1298
+ BgTimer.clearTimeout(timerId)
1299
+ app.authRefreshTimerId = null
1300
+ }
1301
+
1302
+ /**
1303
+ * Deliver the miniapp's FIRST auth token, retrying until it lands. Called from
1304
+ * {@link handleConnect} when the initial mint didn't resolve in the CONNECT_ACK
1305
+ * window. `inFlight` lets attempt 0 reuse the mint the handshake already
1306
+ * started (avoids a redundant duplicate request); later attempts mint fresh.
1307
+ *
1308
+ * A `null` result means auth is fundamentally unavailable (no miniappAuth hook)
1309
+ * — permanent, so we do NOT retry. A REJECTION means a transient failure
1310
+ * (typically the cloud client isn't connection-ready yet) — retry with backoff.
1311
+ * Either way we bail if the app has since disconnected or already got a token
1312
+ * (e.g. via {@link redriveMiniappAuthDelivery} on cloud-connect).
1313
+ */
1314
+ private deliverInitialMiniappAuth(
1315
+ packageName: string,
1316
+ inFlight: Promise<MiniappAuthToken | null> | undefined,
1317
+ attempt: number,
1318
+ ): void {
1319
+ const app = this.connectedApps.get(packageName)
1320
+ if (!app || app.authDelivered) return
1321
+ const mint = inFlight ?? this.requestMiniappAuth(packageName)
1322
+ void mint
1323
+ .then((auth) => {
1324
+ const current = this.connectedApps.get(packageName)
1325
+ if (!current || current.authDelivered) return
1326
+ if (!auth) return // no auth hook — permanent, don't spin
1327
+ current.authDelivered = true
1328
+ this.clearMiniappAuthDeliveryRetry(packageName)
1329
+ this.scheduleMiniappAuthRefresh(packageName, auth)
1330
+ this.sendToMiniapp(packageName, {type: MiniappResponseType.AUTH_UPDATE, auth})
1331
+ })
1332
+ .catch((err) => {
1333
+ console.warn(
1334
+ `${LOG_TAG}: miniapp auth mint failed for ${packageName} (attempt ${attempt}): ${
1335
+ (err as Error)?.message ?? err
1336
+ }`,
1337
+ )
1338
+ this.scheduleInitialMiniappAuthRetry(packageName, attempt)
1339
+ })
1340
+ }
1341
+
1342
+ private scheduleInitialMiniappAuthRetry(packageName: string, attempt: number): void {
1343
+ const app = this.connectedApps.get(packageName)
1344
+ if (!app || app.authDelivered) return
1345
+ this.clearMiniappAuthDeliveryRetry(packageName)
1346
+ const delay = Math.min(MINIAPP_AUTH_RETRY_MAX_MS, MINIAPP_AUTH_RETRY_BASE_MS * 2 ** attempt)
1347
+ app.authRetryTimerId = BgTimer.setTimeout(() => {
1348
+ const current = this.connectedApps.get(packageName)
1349
+ if (!current) return
1350
+ current.authRetryTimerId = null
1351
+ this.deliverInitialMiniappAuth(packageName, undefined, attempt + 1)
1352
+ }, delay)
1353
+ }
1354
+
1355
+ private clearMiniappAuthDeliveryRetry(packageName: string): void {
1356
+ const app = this.connectedApps.get(packageName)
1357
+ if (!app || app.authRetryTimerId === null) return
1358
+ BgTimer.clearTimeout(app.authRetryTimerId)
1359
+ app.authRetryTimerId = null
1360
+ }
1361
+
1362
+ /**
1363
+ * When the cloud client (re)connects, immediately re-attempt the initial mint
1364
+ * for any handshook app still missing its token, instead of waiting out the
1365
+ * backoff. This is what turns "scan a dev app before the cloud is ready" from
1366
+ * "dead until you manually reconnect Cloud V2" into "authenticates on its own
1367
+ * the instant the connection comes up".
1368
+ */
1369
+ private redriveMiniappAuthDelivery(): void {
1370
+ for (const [packageName, app] of this.connectedApps) {
1371
+ if (app.authDelivered) continue
1372
+ // Only apps that finished their CONNECT handshake are in the mint-retry
1373
+ // state; skip anything still mid-handshake (handleConnect owns that).
1374
+ if (!this.handshookApps.has(packageName)) continue
1375
+ this.clearMiniappAuthDeliveryRetry(packageName)
1376
+ this.deliverInitialMiniappAuth(packageName, undefined, 0)
1377
+ }
1378
+ }
1379
+
1380
+ private handleSubscribe(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1381
+ const app = this.connectedApps.get(packageName)
1382
+ if (!app) return
1383
+
1384
+ const rawStreams = (payload.subscriptions ?? payload.streams) as
1385
+ | (string | {stream: string; rate?: string; forceLocal?: boolean; includeCloud?: boolean})[]
1386
+ | undefined
1387
+ // Normalize: objects like {stream: "location_stream", rate: "realtime"} → extract stream name
1388
+ // "location_stream" is the rate-bearing alias for "location_update".
1389
+ // The strictest rate wins when multiple `location_stream` entries appear
1390
+ // in one SUBSCRIBE — the same rule we apply across apps.
1391
+ let requestedLocationRate: string | null = null
1392
+ const forceLocalTranscriptionStreams = new Set<string>()
1393
+ const cloudTranscriptionStreams = new Set<string>()
1394
+ const streams = rawStreams?.map((s) => {
1395
+ if (typeof s === "object" && s !== null) {
1396
+ if (s.stream === "location_stream") {
1397
+ if (s.rate && locationRateRank(s.rate) > locationRateRank(requestedLocationRate)) {
1398
+ requestedLocationRate = s.rate
1399
+ }
1400
+ return "location_update"
1401
+ }
1402
+ if (s.forceLocal === true && s.stream.startsWith("transcription:")) {
1403
+ forceLocalTranscriptionStreams.add(s.stream)
1404
+ }
1405
+ if (s.stream.startsWith("transcription:") && (s.forceLocal !== true || s.includeCloud === true)) {
1406
+ cloudTranscriptionStreams.add(s.stream)
1407
+ }
1408
+ return s.stream
1409
+ }
1410
+ if (s.startsWith("transcription:")) cloudTranscriptionStreams.add(s)
1411
+ return s
1412
+ }) as string[] | undefined
1413
+ if (!Array.isArray(streams)) {
1414
+ this.sendResult(packageName, requestId, false, undefined, {
1415
+ code: MiniappErrorCode.INTERNAL,
1416
+ message: "subscribe requires a subscriptions array",
1417
+ })
1418
+ return
1419
+ }
1420
+
1421
+ console.log(`${LOG_TAG}: SUBSCRIBE from ${packageName}: [${streams.join(", ")}]`)
1422
+
1423
+ // Gate each stream on the permission type its data requires. The manifest
1424
+ // must declare the permission (miniapp.json -> permissions) or we reject
1425
+ // the whole subscribe with PERMISSION_NOT_DECLARED.
1426
+ const declaredTypes = new Set((app.installedManifest?.permissions ?? []).map((p) => p.type?.toUpperCase()))
1427
+
1428
+ const permissionForStream = (s: string): string | null => {
1429
+ if (s === "audio_chunk" || s === "vad") return "MICROPHONE"
1430
+ if (s.startsWith("transcription") || s.startsWith("translation")) return "MICROPHONE"
1431
+ if (s === "location_update") return "LOCATION"
1432
+ if (s === "phone_notification") return "READ_NOTIFICATIONS"
1433
+ if (s === "phone_notification_dismissed") return "READ_NOTIFICATIONS"
1434
+ return null
1435
+ }
1436
+
1437
+ for (const stream of streams) {
1438
+ const required = permissionForStream(stream)
1439
+ if (required && !declaredTypes.has(required)) {
1440
+ logPermissionNotDeclared(packageName, required, `to subscribe to "${stream}"`, `{"type": "${required}"}`)
1441
+ this.sendResult(packageName, requestId, false, undefined, {
1442
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
1443
+ message: `${required} permission not declared in miniapp.json (required for "${stream}"). Add {"type": "${required}"} to the "permissions" array.`,
1444
+ // Extra context fields read by the SDK so authors that subscribe
1445
+ // to session.on("error") can format their own messages.
1446
+ permission: required,
1447
+ subscription: stream,
1448
+ })
1449
+ return
1450
+ }
1451
+ }
1452
+
1453
+ // Remove old subscriptions for this app
1454
+ for (const oldStream of app.subscriptions) {
1455
+ const subs = this.streamSubscribers.get(oldStream)
1456
+ if (subs) {
1457
+ subs.delete(packageName)
1458
+ if (subs.size === 0) {
1459
+ this.streamSubscribers.delete(oldStream)
1460
+ }
1461
+ }
1462
+ }
1463
+
1464
+ // Add new subscriptions
1465
+ app.subscriptions = new Set(streams)
1466
+ app.forceLocalTranscriptionStreams = new Set(
1467
+ [...forceLocalTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
1468
+ )
1469
+ app.cloudTranscriptionStreams = new Set(
1470
+ [...cloudTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
1471
+ )
1472
+ for (const stream of streams) {
1473
+ let subs = this.streamSubscribers.get(stream)
1474
+ if (!subs) {
1475
+ subs = new Set()
1476
+ this.streamSubscribers.set(stream, subs)
1477
+ }
1478
+ subs.add(packageName)
1479
+ }
1480
+
1481
+ this.recomputeMicRequirements()
1482
+ this.updateCloudSubscriptions()
1483
+ this.recomputeHeadingSubscription()
1484
+ this.recomputeImuSubscription()
1485
+ // Persist this app's requested rate (or clear it if SUBSCRIBE didn't
1486
+ // include `location_stream` this time), then ask the host for the
1487
+ // strictest rate across all connected apps.
1488
+ app.requestedLocationRate = requestedLocationRate
1489
+ this.recomputeLocationTier()
1490
+ this.sendResult(packageName, requestId, true)
1491
+
1492
+ // Fire initial snapshot values for stateful streams so miniapps don't have
1493
+ // to wait for the first change event.
1494
+ this.emitInitialSnapshots(packageName, streams)
1495
+ }
1496
+
1497
+ private async handleCalendarListEvents(
1498
+ packageName: string,
1499
+ payload: Record<string, unknown>,
1500
+ requestId?: string,
1501
+ ): Promise<void> {
1502
+ const app = this.connectedApps.get(packageName)
1503
+ const declared = app?.installedManifest?.permissions?.some(
1504
+ (permission) => permission.type?.toUpperCase() === "CALENDAR",
1505
+ )
1506
+ if (!declared) {
1507
+ logPermissionNotDeclared(packageName, "CALENDAR", "to list calendar events", '{"type": "CALENDAR"}')
1508
+ this.sendResult(packageName, requestId, false, undefined, {
1509
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
1510
+ message: 'CALENDAR permission not declared in miniapp.json. Add {"type": "CALENDAR"} to the permissions array.',
1511
+ permission: "CALENDAR",
1512
+ })
1513
+ return
1514
+ }
1515
+
1516
+ try {
1517
+ const result = await listPhoneCalendarEvents(payload)
1518
+ this.sendResult(packageName, requestId, true, result)
1519
+ } catch (error) {
1520
+ const known = error instanceof PhoneCalendarError
1521
+ this.sendResult(packageName, requestId, false, undefined, {
1522
+ code: known ? error.code : MiniappErrorCode.INTERNAL,
1523
+ message: error instanceof Error ? error.message : "Failed to read phone calendar events",
1524
+ })
1525
+ }
1526
+ }
1527
+
1528
+ /**
1529
+ * For state-bearing streams (battery, connection), deliver the current value
1530
+ * immediately on subscribe. Other streams (button press, transcription, etc.)
1531
+ * are pure event streams with no "current value" to snapshot.
1532
+ */
1533
+ private emitInitialSnapshots(packageName: string, streams: string[]): void {
1534
+ // Read the engine glasses store directly (was a host glassesStatus hook).
1535
+ const gs = useGlassesStore.getState()
1536
+ const glassesState = {
1537
+ connected: isGlassesConnected(gs.connection),
1538
+ deviceModel: gs.deviceModel,
1539
+ batteryLevel: gs.batteryLevel,
1540
+ charging: gs.charging,
1541
+ }
1542
+ const isSimulated = (glassesState.deviceModel || "").toLowerCase().includes("simulated")
1543
+
1544
+ for (const stream of streams) {
1545
+ if (stream === "glasses_battery") {
1546
+ // Simulated glasses have no real battery; mirror the phone's battery
1547
+ // so miniapps see a sensible value during development.
1548
+ if (isSimulated) {
1549
+ void this.emitPhoneBatteryAs(packageName, "glasses_battery")
1550
+ } else if (typeof glassesState.batteryLevel === "number" && glassesState.batteryLevel >= 0) {
1551
+ this.sendToMiniapp(packageName, {
1552
+ type: MiniappResponseType.EVENT,
1553
+ streamType: "glasses_battery",
1554
+ data: {
1555
+ level: glassesState.batteryLevel,
1556
+ charging: !!glassesState.charging,
1557
+ timestamp: Date.now(),
1558
+ },
1559
+ })
1560
+ }
1561
+ } else if (stream === "phone_battery") {
1562
+ void this.emitPhoneBatteryAs(packageName, "phone_battery")
1563
+ } else if (stream === "glasses_connection") {
1564
+ this.sendToMiniapp(packageName, {
1565
+ type: MiniappResponseType.EVENT,
1566
+ streamType: "glasses_connection",
1567
+ data: glassesState,
1568
+ })
1569
+ } else if (stream === "glasses_wifi") {
1570
+ // Snapshot the current glasses Wi-Fi state on subscribe (like battery).
1571
+ // Read the canonical nested `wifi: {state, ssid}` from the STORE state
1572
+ // (`glassesState` is the narrowed 4-field projection and has no `wifi`;
1573
+ // the pre-engine host hook spread the full store, so read gs directly —
1574
+ // NOT the legacy flat wifiConnected). Effective connectivity requires the
1575
+ // glasses connected AND on Wi-Fi — mirrors the host's store-derived
1576
+ // forward, so an already-disconnected device reports `connected: false`
1577
+ // rather than silently emitting nothing. Always emits so onWifi gets an
1578
+ // initial value.
1579
+ const wifi = gs.wifi.state === "connected" ? gs.wifi : null
1580
+ const connected = glassesState.connected === true && wifi !== null
1581
+ this.sendToMiniapp(packageName, {
1582
+ type: MiniappResponseType.EVENT,
1583
+ streamType: "glasses_wifi",
1584
+ data: {
1585
+ connected,
1586
+ ssid: connected ? wifi?.ssid : undefined,
1587
+ localIp: connected ? wifi?.localIp : undefined,
1588
+ timestamp: Date.now(),
1589
+ },
1590
+ })
1591
+ } else if (stream === "head_position") {
1592
+ // The glasses store has no headUp field (same on the pre-engine host
1593
+ // hook), so this stays a no-emit until head state lands in the store.
1594
+ const headUp = (gs as {headUp?: boolean}).headUp
1595
+ if (typeof headUp === "boolean") {
1596
+ this.sendToMiniapp(packageName, {
1597
+ type: MiniappResponseType.EVENT,
1598
+ streamType: "head_position",
1599
+ data: {
1600
+ position: headUp ? "up" : "down",
1601
+ timestamp: Date.now(),
1602
+ },
1603
+ })
1604
+ }
1605
+ }
1606
+ }
1607
+ }
1608
+
1609
+ /**
1610
+ * Read the phone's battery state right now and emit it as the given stream.
1611
+ * Used for both phone_battery snapshot on subscribe, and as a stand-in for
1612
+ * glasses_battery when connected to Simulated Glasses.
1613
+ */
1614
+ private async emitPhoneBatteryAs(
1615
+ packageName: string,
1616
+ streamType: "phone_battery" | "glasses_battery",
1617
+ ): Promise<void> {
1618
+ try {
1619
+ const level = await Battery.getBatteryLevelAsync()
1620
+ const state = await Battery.getBatteryStateAsync()
1621
+ const charging = state === Battery.BatteryState.CHARGING || state === Battery.BatteryState.FULL
1622
+ this.sendToMiniapp(packageName, {
1623
+ type: MiniappResponseType.EVENT,
1624
+ streamType,
1625
+ data: {
1626
+ level: Math.round(level * 100),
1627
+ charging,
1628
+ timestamp: Date.now(),
1629
+ },
1630
+ })
1631
+ } catch (err) {
1632
+ console.log(`${LOG_TAG}: phone battery snapshot failed`, err)
1633
+ }
1634
+ }
1635
+
1636
+ private handleDisplay(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1637
+ try {
1638
+ // Zombie-frame gate: a dying miniapp's last in-flight frame can arrive
1639
+ // AFTER onUnmount cleared its display — repainting a dead app's UI (and
1640
+ // poisoning the next boot's pre-boot restore snapshot). Killed mid-route
1641
+ // nav reproduced this reliably; its 3s re-push raced the teardown.
1642
+ if (!this.connectedApps.has(packageName)) {
1643
+ console.log(`${LOG_TAG}: dropping display from unmounted app ${packageName}`)
1644
+ return
1645
+ }
1646
+ if (!payload.layout || typeof payload.layout !== "object") {
1647
+ this.sendResult(packageName, requestId, false, undefined, {
1648
+ code: MiniappErrorCode.INTERNAL,
1649
+ message: "display request missing layout object",
1650
+ })
1651
+ return
1652
+ }
1653
+
1654
+ // Hand off to LocalDisplayManager — it owns boot/throttle/arbitration/
1655
+ // expiry + the native BluetoothSdk.displayEvent call + useDisplayStore.
1656
+ localDisplayManager.request(packageName, {
1657
+ view: (payload.view as DisplayPayload["view"]) ?? "main",
1658
+ layout: payload.layout as DisplayPayload["layout"],
1659
+ durationMs: payload.durationMs as number | undefined,
1660
+ })
1661
+
1662
+ this.sendResult(packageName, requestId, true)
1663
+ } catch (err) {
1664
+ console.error(`${LOG_TAG}: display error:`, err)
1665
+ this.sendResult(packageName, requestId, false, undefined, {
1666
+ code: MiniappErrorCode.INTERNAL,
1667
+ message: err instanceof Error ? err.message : "Display error",
1668
+ })
1669
+ }
1670
+ }
1671
+
1672
+ /**
1673
+ * Scene render (session.display.render()). Same arbitration as DISPLAY, but
1674
+ * the payload is a whole frame of positioned elements and the caller may be
1675
+ * awaiting the outcome: LocalDisplayManager guarantees the resolver fires
1676
+ * exactly once (displayed/blocked, with degraded/dropped reporting), and that
1677
+ * becomes the REQUEST_RESULT data.
1678
+ */
1679
+ private handleRender(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1680
+ try {
1681
+ // Zombie-frame gate — see handleDisplay.
1682
+ if (!this.connectedApps.has(packageName)) {
1683
+ this.sendResult(packageName, requestId, true, {status: "blocked", reason: "app stopped"})
1684
+ return
1685
+ }
1686
+ if (!Array.isArray(payload.elements)) {
1687
+ this.sendResult(packageName, requestId, false, undefined, {
1688
+ code: MiniappErrorCode.INTERNAL,
1689
+ message: "render request missing elements array",
1690
+ })
1691
+ return
1692
+ }
1693
+
1694
+ localDisplayManager.request(
1695
+ packageName,
1696
+ {
1697
+ view: (payload.view as DisplayPayload["view"]) ?? "main",
1698
+ scene: payload.elements as DisplayPayload["scene"],
1699
+ durationMs: payload.durationMs as number | undefined,
1700
+ },
1701
+ (result) => this.sendResult(packageName, requestId, true, result),
1702
+ )
1703
+ } catch (err) {
1704
+ console.error(`${LOG_TAG}: render error:`, err)
1705
+ this.sendResult(packageName, requestId, false, undefined, {
1706
+ code: MiniappErrorCode.INTERNAL,
1707
+ message: err instanceof Error ? err.message : "Render error",
1708
+ })
1709
+ }
1710
+ }
1711
+
1712
+ private handlePlayAudio(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1713
+ const audioUrl = (payload.audioUrl ?? payload.url) as string | undefined
1714
+ if (!audioUrl) {
1715
+ this.sendResult(packageName, requestId, false, undefined, {
1716
+ code: MiniappErrorCode.INTERNAL,
1717
+ message: "play_audio requires an audioUrl",
1718
+ })
1719
+ return
1720
+ }
1721
+
1722
+ const audioRequestId = requestId || `local_${Date.now()}`
1723
+ const volume = typeof payload.volume === "number" ? payload.volume : 1.0
1724
+ const stopOtherAudio = payload.stopOtherAudio !== false
1725
+
1726
+ this.cancelSpeech(packageName)
1727
+ this.setSpeakerState(packageName, "loading")
1728
+ audioPlaybackService.play(
1729
+ {requestId: audioRequestId, audioUrl, appId: packageName, volume, stopOtherAudio},
1730
+ (_respId, success, error, duration) => {
1731
+ if (success) {
1732
+ this.setSpeakerState(packageName, "stopped", {durationMs: duration ?? undefined})
1733
+ } else {
1734
+ this.setSpeakerState(packageName, "error", {
1735
+ errorCode: MiniappErrorCode.INTERNAL,
1736
+ errorMessage: error ?? "play failed",
1737
+ durationMs: duration ?? undefined,
1738
+ })
1739
+ }
1740
+ this.sendResult(
1741
+ packageName,
1742
+ requestId,
1743
+ success,
1744
+ {duration},
1745
+ error
1746
+ ? {
1747
+ code: MiniappErrorCode.INTERNAL,
1748
+ message: error,
1749
+ }
1750
+ : undefined,
1751
+ )
1752
+ },
1753
+ )
1754
+ // Optimistic "started playing" transition. The audio service doesn't
1755
+ // have a "playback actually started" callback today; cloud SDK v3 has
1756
+ // the same constraint and uses optimistic timing. Microtask so the
1757
+ // initial "loading" envelope flushes first.
1758
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"))
1759
+ }
1760
+
1761
+ private handleStopAudio(packageName: string, _payload: Record<string, unknown>, requestId?: string): void {
1762
+ this.cancelSpeech(packageName)
1763
+ audioPlaybackService.stopForApp(packageName)
1764
+ this.setSpeakerState(packageName, "stopped")
1765
+ this.sendResult(packageName, requestId, true)
1766
+ }
1767
+
1768
+ // ── speaker.createStream (live PCM output) ────────────────────────────────
1769
+
1770
+ /** Valid PCM sample rates for speaker streams. Mirrors the SDK contract. */
1771
+ private static readonly SPEAKER_STREAM_SAMPLE_RATES = new Set([16000, 24000, 48000])
1772
+ private static readonly SPEAKER_STREAM_MAX_BASE64_CHARS = Math.ceil((256 * 1024) / 3) * 4
1773
+ private speakerStreamCounter = 0
1774
+
1775
+ private async handleSpeakerStreamOpen(
1776
+ packageName: string,
1777
+ payload: Record<string, unknown>,
1778
+ requestId?: string,
1779
+ ): Promise<void> {
1780
+ const app = this.connectedApps.get(packageName)
1781
+ if (!app) return
1782
+
1783
+ // One stream per miniapp: opening a second closes the first.
1784
+ const previous = app.activeSpeakerStreamId
1785
+ if (previous) {
1786
+ app.activeSpeakerStreamId = undefined
1787
+ await audioPlaybackService.abortStream(previous).catch((error) => {
1788
+ console.warn(`${LOG_TAG}: failed to abort previous speaker stream for ${packageName}`, error)
1789
+ })
1790
+ }
1791
+
1792
+ const sampleRate = typeof payload.sampleRate === "number" ? payload.sampleRate : 16000
1793
+ if (!LocalMiniappRuntime.SPEAKER_STREAM_SAMPLE_RATES.has(sampleRate)) {
1794
+ this.sendResult(packageName, requestId, false, undefined, {
1795
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1796
+ message: `unsupported sampleRate ${sampleRate} (use 16000, 24000, or 48000)`,
1797
+ })
1798
+ return
1799
+ }
1800
+ const channels = typeof payload.channels === "number" ? payload.channels : 1
1801
+ if (channels !== 1) {
1802
+ this.sendResult(packageName, requestId, false, undefined, {
1803
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1804
+ message: "speaker streams currently support mono PCM only",
1805
+ })
1806
+ return
1807
+ }
1808
+ const volume = typeof payload.volume === "number" ? payload.volume : 1
1809
+ if (!Number.isFinite(volume) || volume < 0 || volume > 1) {
1810
+ this.sendResult(packageName, requestId, false, undefined, {
1811
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1812
+ message: "speaker stream volume must be between 0 and 1",
1813
+ })
1814
+ return
1815
+ }
1816
+
1817
+ const streamId = `spkstream_${Date.now()}_${++this.speakerStreamCounter}`
1818
+ this.setSpeakerState(packageName, "loading")
1819
+ try {
1820
+ await audioPlaybackService.openStream({
1821
+ streamId,
1822
+ appId: packageName,
1823
+ sampleRate,
1824
+ channels,
1825
+ volume,
1826
+ stopOtherAudio: payload.stopOtherAudio !== false,
1827
+ onEnded: (endedId, success, error, durationMs) => {
1828
+ const current = this.connectedApps.get(packageName)
1829
+ // Ignore completion from an app incarnation replaced by a crash
1830
+ // respawn. Its abort may finish after the new app has registered and
1831
+ // must not overwrite that app's speaker state.
1832
+ if (current !== app) return
1833
+ if (current.activeSpeakerStreamId === endedId) current.activeSpeakerStreamId = undefined
1834
+ if (success) {
1835
+ this.setSpeakerState(packageName, "stopped", {durationMs: durationMs ?? undefined})
1836
+ } else {
1837
+ this.setSpeakerState(packageName, "error", {
1838
+ errorCode: MiniappErrorCode.INTERNAL,
1839
+ errorMessage: error ?? "speaker stream failed",
1840
+ durationMs: durationMs ?? undefined,
1841
+ })
1842
+ }
1843
+ },
1844
+ })
1845
+ } catch (error) {
1846
+ const message = error instanceof Error ? error.message : String(error)
1847
+ this.setSpeakerState(packageName, "error", {
1848
+ errorCode: MiniappErrorCode.INTERNAL,
1849
+ errorMessage: message,
1850
+ })
1851
+ this.sendResult(packageName, requestId, false, undefined, {
1852
+ code: MiniappErrorCode.INTERNAL,
1853
+ message,
1854
+ })
1855
+ return
1856
+ }
1857
+
1858
+ // The native open can outlive a miniapp disconnect. Its ordinary teardown
1859
+ // ran while no StreamState existed yet, so close this late result here
1860
+ // instead of leaving an orphan AudioTrack/AVAudioEngine session.
1861
+ if (this.connectedApps.get(packageName) !== app) {
1862
+ await audioPlaybackService.abortStream(streamId)
1863
+ return
1864
+ }
1865
+ app.activeSpeakerStreamId = streamId
1866
+ // Optimistic "playing", same as play(): the chunk player starts as soon
1867
+ // as the first write lands. Microtask so "loading" flushes first.
1868
+ queueMicrotask(() => this.setSpeakerState(packageName, "playing"))
1869
+ this.sendResult(packageName, requestId, true, {streamId})
1870
+ }
1871
+
1872
+ /** Resolve + validate a stream id from a payload against the app's active stream. */
1873
+ private activeSpeakerStream(packageName: string, payload: Record<string, unknown>): string | null {
1874
+ const streamId = typeof payload.streamId === "string" ? payload.streamId : undefined
1875
+ const app = this.connectedApps.get(packageName)
1876
+ if (!streamId || !app || app.activeSpeakerStreamId !== streamId) return null
1877
+ return streamId
1878
+ }
1879
+
1880
+ private async handleSpeakerStreamWrite(
1881
+ packageName: string,
1882
+ payload: Record<string, unknown>,
1883
+ requestId?: string,
1884
+ ): Promise<void> {
1885
+ const streamId = this.activeSpeakerStream(packageName, payload)
1886
+ if (!streamId) {
1887
+ this.sendResult(packageName, requestId, false, undefined, {
1888
+ code: MiniappErrorCode.INTERNAL,
1889
+ message: "speaker stream is not open",
1890
+ })
1891
+ return
1892
+ }
1893
+ const base64 = typeof payload.base64 === "string" ? payload.base64 : ""
1894
+ if (!base64 || base64.length % 4 !== 0) {
1895
+ this.sendResult(packageName, requestId, false, undefined, {
1896
+ code: MiniappErrorCode.INVALID_ARGUMENT,
1897
+ message: "speaker stream write requires padded base64 PCM data",
1898
+ })
1899
+ return
1900
+ }
1901
+ if (base64.length > LocalMiniappRuntime.SPEAKER_STREAM_MAX_BASE64_CHARS) {
1902
+ this.sendResult(packageName, requestId, false, undefined, {
1903
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
1904
+ message: "speaker stream chunk exceeds the 256 KB raw PCM limit",
1905
+ })
1906
+ return
1907
+ }
1908
+ try {
1909
+ const result = await audioPlaybackService.writeStreamChunk(streamId, base64)
1910
+ this.sendResult(packageName, requestId, true, result)
1911
+ } catch (error) {
1912
+ this.sendResult(packageName, requestId, false, undefined, {
1913
+ code: MiniappErrorCode.INTERNAL,
1914
+ message: error instanceof Error ? error.message : String(error),
1915
+ })
1916
+ }
1917
+ }
1918
+
1919
+ private async handleSpeakerStreamClose(
1920
+ packageName: string,
1921
+ payload: Record<string, unknown>,
1922
+ requestId?: string,
1923
+ ): Promise<void> {
1924
+ const streamId = this.activeSpeakerStream(packageName, payload)
1925
+ if (!streamId) {
1926
+ this.sendResult(packageName, requestId, false, undefined, {
1927
+ code: MiniappErrorCode.INTERNAL,
1928
+ message: "speaker stream is not open",
1929
+ })
1930
+ return
1931
+ }
1932
+ try {
1933
+ // Resolves once the buffered audio has drained; onEnded delivers the
1934
+ // terminal SPEAKER_STATE ("stopped").
1935
+ const result = await audioPlaybackService.closeStream(streamId)
1936
+ this.sendResult(packageName, requestId, true, result)
1937
+ } catch (error) {
1938
+ this.sendResult(packageName, requestId, false, undefined, {
1939
+ code: MiniappErrorCode.INTERNAL,
1940
+ message: error instanceof Error ? error.message : String(error),
1941
+ })
1942
+ }
1943
+ }
1944
+
1945
+ private async handleSpeakerStreamAbort(
1946
+ packageName: string,
1947
+ payload: Record<string, unknown>,
1948
+ requestId?: string,
1949
+ ): Promise<void> {
1950
+ const streamId = this.activeSpeakerStream(packageName, payload)
1951
+ if (!streamId) {
1952
+ // Aborting an already-gone stream is fine — abort() is idempotent.
1953
+ this.sendResult(packageName, requestId, true)
1954
+ return
1955
+ }
1956
+ try {
1957
+ await audioPlaybackService.abortStream(streamId)
1958
+ this.sendResult(packageName, requestId, true)
1959
+ } catch (error) {
1960
+ this.sendResult(packageName, requestId, false, undefined, {
1961
+ code: MiniappErrorCode.INTERNAL,
1962
+ message: error instanceof Error ? error.message : String(error),
1963
+ })
1964
+ }
1965
+ }
1966
+
1967
+ private async handleSpeak(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
1968
+ const rawText = payload.text
1969
+ const sentences =
1970
+ typeof rawText === "string"
1971
+ ? [rawText.trim()].filter(Boolean)
1972
+ : Array.isArray(rawText)
1973
+ ? rawText.filter((sentence): sentence is string => typeof sentence === "string").map((sentence) => sentence.trim()).filter(Boolean)
1974
+ : []
1975
+ if (sentences.length === 0) {
1976
+ this.sendResult(packageName, requestId, false, undefined, {
1977
+ code: MiniappErrorCode.INTERNAL,
1978
+ message: "speak requires text or a non-empty sentence list",
1979
+ })
1980
+ return
1981
+ }
1982
+ const cloudText = sentences.join(" ")
1983
+
1984
+ const voice = ((payload.voice_id ?? payload.voice) as string) || "default"
1985
+ const audioRequestId = requestId || `tts_${Date.now()}`
1986
+ const volume = typeof payload.volume === "number" ? payload.volume : 1.0
1987
+ const stopOtherAudio = payload.stopOtherAudio !== false
1988
+ const voiceSettings =
1989
+ payload.voice_settings && typeof payload.voice_settings === "object"
1990
+ ? (payload.voice_settings as Record<string, unknown>)
1991
+ : undefined
1992
+ const speed = typeof voiceSettings?.speed === "number" ? voiceSettings.speed : undefined
1993
+ const run: SpeechRun = {id: audioRequestId, cancelled: false}
1994
+
1995
+ this.cancelSpeech(packageName)
1996
+ this.speechRuns.set(packageName, run)
1997
+ this.setSpeakerState(packageName, "loading")
1998
+
1999
+ let terminalSent = false
2000
+ const sendPlaybackResult = (
2001
+ success: boolean,
2002
+ error: string | null,
2003
+ duration: number | null,
2004
+ fallbackErrorMessage: string,
2005
+ completed = success,
2006
+ errorCode: MiniappErrorCode = MiniappErrorCode.TTS_UPSTREAM_ERROR,
2007
+ ) => {
2008
+ if (terminalSent) return
2009
+ terminalSent = true
2010
+
2011
+ // A superseded run still owns its bridge request, but it no longer owns
2012
+ // the package's shared speaker state.
2013
+ const ownsSpeakerState = this.speechRuns.get(packageName) === run
2014
+ if (ownsSpeakerState) this.speechRuns.delete(packageName)
2015
+
2016
+ if (ownsSpeakerState) {
2017
+ if (success) {
2018
+ this.setSpeakerState(packageName, "stopped", {durationMs: duration ?? undefined})
2019
+ } else {
2020
+ this.setSpeakerState(packageName, "error", {
2021
+ errorCode,
2022
+ errorMessage: error ?? fallbackErrorMessage,
2023
+ durationMs: duration ?? undefined,
2024
+ })
2025
+ }
2026
+ }
2027
+
2028
+ this.sendResult(
2029
+ packageName,
2030
+ requestId,
2031
+ success,
2032
+ {completed, duration},
2033
+ error
2034
+ ? {
2035
+ code: errorCode,
2036
+ message: error,
2037
+ }
2038
+ : undefined,
2039
+ )
2040
+ }
2041
+ run.onCancelled = () => sendPlaybackResult(true, null, null, "tts playback stopped", false)
2042
+
2043
+ try {
2044
+ const voiceExplicit = payload.voice_id !== undefined || payload.voice !== undefined
2045
+ const offlineSupportsVoice =
2046
+ !voiceExplicit || voice === "default" || ttsModelManager.getAvailableLanguages().some((l) => l.code === voice)
2047
+
2048
+ const modelId = typeof payload.model_id === "string" ? payload.model_id : undefined
2049
+ const forceLocal = payload.forceLocal === true
2050
+ const cloudConnected = cloudClientService.isConnected()
2051
+ console.log(
2052
+ `${LOG_TAG}: TTS decision for ${packageName}: cloudConnected=${cloudConnected}, runtimeTts=yes, offlineVoice=${offlineSupportsVoice}, forceLocal=${forceLocal}`,
2053
+ )
2054
+
2055
+ const playOfflineTts = async (reason?: string): Promise<boolean> => {
2056
+ if (run.cancelled) return true
2057
+ if (!offlineSupportsVoice || !(await ttsModelManager.isModelAvailable())) {
2058
+ return false
2059
+ }
2060
+ if (run.cancelled) return true
2061
+
2062
+ const languageCode = ttsModelManager.getAvailableLanguages().some((l) => l.code === voice) ? voice : undefined
2063
+ const offlineSentences = sentences
2064
+
2065
+ if (offlineSentences.length > 1) {
2066
+ let firstSentence: TtsSynthesisResult
2067
+ try {
2068
+ firstSentence = await ttsModelManager.synthesizeToFile(offlineSentences[0], {languageCode, speed})
2069
+ } catch (offlineErr) {
2070
+ if (run.cancelled) return true
2071
+ console.warn(
2072
+ `${LOG_TAG}: offline sentence TTS synthesize failed${reason ? ` after ${reason}` : ""}:`,
2073
+ offlineErr,
2074
+ )
2075
+ return false
2076
+ }
2077
+
2078
+ if (run.cancelled) {
2079
+ await Promise.resolve(firstSentence.cleanup?.())
2080
+ return true
2081
+ }
2082
+ void runSentenceTtsPipeline({
2083
+ sentences: offlineSentences,
2084
+ firstAudio: firstSentence,
2085
+ run,
2086
+ synthesize: (sentence) => ttsModelManager.synthesizeToFile(sentence, {languageCode, speed}),
2087
+ play: (audio, index) =>
2088
+ new Promise((resolve) => {
2089
+ const sentenceRequestId = `${audioRequestId}_sentence_${index}`
2090
+ run.playbackRequestId = sentenceRequestId
2091
+ audioPlaybackService.play(
2092
+ {
2093
+ requestId: sentenceRequestId,
2094
+ audioUrl: audio.audioUrl,
2095
+ appId: packageName,
2096
+ volume,
2097
+ stopOtherAudio,
2098
+ suppressCloudUplink: true,
2099
+ },
2100
+ (_responseId, success, error, duration, completionReason) => {
2101
+ if (run.playbackRequestId === sentenceRequestId) run.playbackRequestId = undefined
2102
+ resolve({success, completed: completionReason === "completed", error, duration})
2103
+ },
2104
+ )
2105
+ }),
2106
+ onCleanupError: (cleanupError) => {
2107
+ console.warn(`${LOG_TAG}: sentence TTS cleanup failed`, cleanupError)
2108
+ },
2109
+ }).then(
2110
+ ({success, completed, error, duration}) => {
2111
+ sendPlaybackResult(success, error, duration, "offline sentence tts playback failed", completed)
2112
+ },
2113
+ (pipelineError: unknown) => {
2114
+ const message = pipelineError instanceof Error ? pipelineError.message : String(pipelineError)
2115
+ sendPlaybackResult(false, message, null, "offline sentence tts playback failed", false)
2116
+ },
2117
+ )
2118
+ queueMicrotask(() => {
2119
+ if (this.speechRuns.get(packageName) === run) this.setSpeakerState(packageName, "playing")
2120
+ })
2121
+ return true
2122
+ }
2123
+
2124
+ let offlineGenerated: TtsSynthesisResult | undefined
2125
+
2126
+ try {
2127
+ offlineGenerated = await ttsModelManager.synthesizeToFile(offlineSentences[0], {languageCode, speed})
2128
+ } catch (offlineErr) {
2129
+ if (run.cancelled) return true
2130
+ console.warn(`${LOG_TAG}: offline TTS synthesize failed${reason ? ` after ${reason}` : ""}:`, offlineErr)
2131
+ return false
2132
+ }
2133
+
2134
+ const generated = offlineGenerated
2135
+ if (run.cancelled) {
2136
+ await Promise.resolve(generated.cleanup?.())
2137
+ return true
2138
+ }
2139
+ run.playbackRequestId = audioRequestId
2140
+ audioPlaybackService.play(
2141
+ {
2142
+ requestId: audioRequestId,
2143
+ audioUrl: generated.audioUrl,
2144
+ appId: packageName,
2145
+ volume,
2146
+ stopOtherAudio,
2147
+ suppressCloudUplink: true,
2148
+ },
2149
+ (_respId, success, error, duration, completionReason) => {
2150
+ if (run.playbackRequestId === audioRequestId) run.playbackRequestId = undefined
2151
+ void Promise.resolve(generated.cleanup?.()).catch((cleanupError) => {
2152
+ console.warn(`${LOG_TAG}: offline TTS cleanup failed`, cleanupError)
2153
+ })
2154
+ sendPlaybackResult(
2155
+ success,
2156
+ error,
2157
+ duration,
2158
+ "offline tts playback failed",
2159
+ completionReason === "completed",
2160
+ )
2161
+ },
2162
+ )
2163
+ queueMicrotask(() => {
2164
+ if (this.speechRuns.get(packageName) === run) this.setSpeakerState(packageName, "playing")
2165
+ })
2166
+ return true
2167
+ }
2168
+
2169
+ const playCloudTts = async (fallbackToOffline: boolean): Promise<boolean> => {
2170
+ if (run.cancelled) return true
2171
+ let source: Awaited<ReturnType<typeof cloudClientService.tts.speak>>
2172
+ try {
2173
+ source = await cloudClientService.tts.speak(cloudText, {
2174
+ ...(voiceExplicit && voice !== "default" ? {voice_id: voice} : {}),
2175
+ ...(modelId ? {model_id: modelId} : {}),
2176
+ ...(voiceSettings ? {voice_settings: voiceSettings} : {}),
2177
+ })
2178
+ } catch (cloudErr) {
2179
+ if (run.cancelled) return true
2180
+ const error = cloudErr instanceof Error ? cloudErr.message : String(cloudErr)
2181
+ console.warn(`${LOG_TAG}: cloud TTS source failed: ${error}`)
2182
+ if (fallbackToOffline && (await playOfflineTts("cloud tts source failed"))) {
2183
+ return true
2184
+ }
2185
+ sendPlaybackResult(false, error, null, "tts failed")
2186
+ return true
2187
+ }
2188
+
2189
+ if (run.cancelled) return true
2190
+
2191
+ run.playbackRequestId = audioRequestId
2192
+ await Promise.resolve(
2193
+ audioPlaybackService.play(
2194
+ {
2195
+ requestId: audioRequestId,
2196
+ audioUrl: source.audioUrl,
2197
+ appId: packageName,
2198
+ volume,
2199
+ stopOtherAudio,
2200
+ suppressCloudUplink: true,
2201
+ },
2202
+ (_respId, success, error, duration, completionReason) => {
2203
+ if (run.playbackRequestId === audioRequestId) run.playbackRequestId = undefined
2204
+ if (!success && fallbackToOffline) {
2205
+ void playOfflineTts("cloud tts playback failed").then((started) => {
2206
+ if (!started) {
2207
+ sendPlaybackResult(false, error, duration, "tts failed")
2208
+ }
2209
+ })
2210
+ return
2211
+ }
2212
+ sendPlaybackResult(success, error, duration, "tts failed", completionReason === "completed")
2213
+ },
2214
+ ),
2215
+ )
2216
+ queueMicrotask(() => {
2217
+ if (this.speechRuns.get(packageName) === run) this.setSpeakerState(packageName, "playing")
2218
+ })
2219
+ return true
2220
+ }
2221
+
2222
+ if (forceLocal) {
2223
+ if (await playOfflineTts()) {
2224
+ return
2225
+ }
2226
+ if (run.cancelled) return
2227
+ const message = "tts unavailable: forceLocal requested but offline TTS model is not available"
2228
+ sendPlaybackResult(false, message, null, message, false, MiniappErrorCode.TTS_LOCAL_UNAVAILABLE)
2229
+ return
2230
+ }
2231
+
2232
+ if (cloudConnected && (await playCloudTts(true))) {
2233
+ return
2234
+ }
2235
+
2236
+ // Offline is the disconnected-cloud fallback. If the model is not
2237
+ // available, preserve the old behavior by trying cloud as a last resort.
2238
+ if (await playOfflineTts(cloudConnected ? undefined : "cloud disconnected")) {
2239
+ return
2240
+ }
2241
+
2242
+ if (await playCloudTts(false)) {
2243
+ return
2244
+ }
2245
+
2246
+ const message = "tts unavailable: no cloud TTS URL or offline TTS model"
2247
+ sendPlaybackResult(false, message, null, message, false)
2248
+ } catch (err) {
2249
+ console.error(`${LOG_TAG}: speak error:`, err)
2250
+ const message = err instanceof Error ? err.message : "TTS error"
2251
+ sendPlaybackResult(false, message, null, "TTS error", false)
2252
+ }
2253
+ }
2254
+
2255
+ private cancelSpeech(packageName: string): void {
2256
+ const run = this.speechRuns.get(packageName)
2257
+ if (!run) return
2258
+ run.cancelled = true
2259
+ this.speechRuns.delete(packageName)
2260
+ if (run.playbackRequestId) audioPlaybackService.cancelPlayback(run.playbackRequestId)
2261
+ run.onCancelled?.()
2262
+ }
2263
+
2264
+ private async handleRgbLed(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
2265
+ const coerceNumber = (value: unknown, fallback: number): number => {
2266
+ const coerced = Number(value)
2267
+ return Number.isFinite(coerced) ? coerced : fallback
2268
+ }
2269
+
2270
+ const ledRequestId = requestId || `led_${Date.now()}`
2271
+ const action = normalizeRgbLedAction(payload.action)
2272
+ const color = normalizeRgbLedColor(payload.color)
2273
+
2274
+ try {
2275
+ const result = await BluetoothSdk.rgbLedControl(
2276
+ ledRequestId,
2277
+ packageName,
2278
+ action,
2279
+ color,
2280
+ coerceNumber(payload.ontime, 1000),
2281
+ coerceNumber(payload.offtime, 0),
2282
+ coerceNumber(payload.count, 1),
2283
+ )
2284
+ this.sendResult(packageName, requestId, true, result)
2285
+ } catch (err) {
2286
+ this.sendResult(packageName, requestId, false, undefined, {
2287
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2288
+ message: err instanceof Error ? err.message : "RGB LED command failed",
2289
+ })
2290
+ }
2291
+ }
2292
+
2293
+ private async handleLocationPoll(packageName: string, requestId?: string): Promise<void> {
2294
+ try {
2295
+ const {status} = await Location.requestForegroundPermissionsAsync()
2296
+ if (status !== "granted") {
2297
+ this.sendResult(packageName, requestId, false, undefined, {
2298
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2299
+ message: "Location permission not granted",
2300
+ })
2301
+ return
2302
+ }
2303
+
2304
+ // Try the OS cache first — usually instant — and only fall back to
2305
+ // a fresh fix if the cache is empty. Use Low accuracy on the fresh
2306
+ // path so we get a cell/wifi-tower fix in ~1s instead of waiting
2307
+ // for full GPS warm-up.
2308
+ const cached = await Location.getLastKnownPositionAsync({maxAge: 60_000})
2309
+ const location = cached ?? (await Location.getCurrentPositionAsync({accuracy: Location.Accuracy.Low}))
2310
+
2311
+ this.sendResult(packageName, requestId, true, {
2312
+ lat: location.coords.latitude,
2313
+ lng: location.coords.longitude,
2314
+ accuracy: location.coords.accuracy ?? undefined,
2315
+ timestamp: location.timestamp,
2316
+ })
2317
+ } catch (err) {
2318
+ console.error(`${LOG_TAG}: location poll error:`, err)
2319
+ this.sendResult(packageName, requestId, false, undefined, {
2320
+ code: MiniappErrorCode.INTERNAL,
2321
+ message: err instanceof Error ? err.message : "Location error",
2322
+ })
2323
+ }
2324
+ }
2325
+
2326
+ // ---------------------------------------------------------------------------
2327
+ // Navigation
2328
+ // ---------------------------------------------------------------------------
2329
+
2330
+ /**
2331
+ * All navigation handlers + their state live in NavigationHandlers. The
2332
+ * dispatcher's NAVIGATION_* cases delegate here; the rest of the runtime
2333
+ * cross-cuts navigation through {@link NavigationHandlers.onDisconnect}
2334
+ * (mini-app disconnect) and {@link NavigationHandlers.isTripActive}
2335
+ * (location-stream gating during a trip).
2336
+ */
2337
+ private readonly navigationHandlers = new NavigationHandlers(
2338
+ (packageName, envelope) => this.sendToMiniapp(packageName, envelope),
2339
+ (packageName, requestId, ok, result, error) => this.sendResult(packageName, requestId, ok, result, error),
2340
+ )
2341
+
2342
+ /**
2343
+ * `session.blob` — persistent, per-app binary storage. Like NavigationHandlers,
2344
+ * the BLOB_* dispatcher cases delegate here. It's a generic byte store; audio
2345
+ * recording lives entirely in the miniapp (mic.onAudioChunk → chunked
2346
+ * BLOB_WRITE), so nothing audio-specific runs in the host.
2347
+ */
2348
+ private readonly blobStore = new BlobStore({
2349
+ sendResult: (packageName, requestId, ok, result, error) =>
2350
+ this.sendResult(packageName, requestId, ok, result, error),
2351
+ getUserId: () =>
2352
+ (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous",
2353
+ })
2354
+
2355
+ /**
2356
+ * Heading is a sensor stream — start the native compass when any mini
2357
+ * app is subscribed to "heading_update", stop it when none are.
2358
+ */
2359
+ private headingUnsub: (() => void) | null = null
2360
+ private recomputeHeadingSubscription(): void {
2361
+ const wantsHeading = this.streamSubscribers.has(MiniappStreamType.HEADING_UPDATE)
2362
+ if (wantsHeading && !this.headingUnsub) {
2363
+ // Heading sensor lives in engine (HeadingService) — subscribe directly.
2364
+ this.headingUnsub = headingService.addListener((degrees: number) => {
2365
+ this.forwardEvent(MiniappStreamType.HEADING_UPDATE, {degrees})
2366
+ })
2367
+ } else if (!wantsHeading && this.headingUnsub) {
2368
+ this.headingUnsub()
2369
+ this.headingUnsub = null
2370
+ }
2371
+ }
2372
+
2373
+ /**
2374
+ * Accelerometer is a sensor stream — enable the glasses IMU when any mini
2375
+ * app is subscribed to "accel_data", disable it when none are. Mirrors the
2376
+ * heading pattern. Only G2 streams IMU today; on other devices the native
2377
+ * call is a no-op so this is harmless.
2378
+ */
2379
+ private imuEnabled = false
2380
+ private recomputeImuSubscription(): void {
2381
+ const wantsImu = this.streamSubscribers.has(MiniappStreamType.ACCEL_DATA)
2382
+ if (wantsImu === this.imuEnabled) return
2383
+ this.imuEnabled = wantsImu
2384
+ void BluetoothSdk.setImuEnabled(wantsImu)
2385
+ }
2386
+
2387
+ /**
2388
+ * Last rate we asked the host for. Lets us skip the cross-process call
2389
+ * when nothing actually changed (most SUBSCRIBE / unregister churn
2390
+ * leaves the aggregate unchanged) and tracks state for the no-active-
2391
+ * subscribers → tell host to back off case.
2392
+ */
2393
+ private lastAppliedLocationRate: LocationRate | null = null
2394
+
2395
+ /**
2396
+ * Recompute the aggregate location tier across every connected
2397
+ * miniapp and push the result to the host. The aggregate is the
2398
+ * STRICTEST rate any single app has requested — see
2399
+ * {@link LOCATION_RATE_PRIORITY} — because the GPS sample rate is a
2400
+ * shared OS-level setting that we can't bias per-app.
2401
+ *
2402
+ * Called after every SUBSCRIBE and every unregister so that when a
2403
+ * realtime-requesting app goes away the tier falls back down. If no
2404
+ * connected app is asking for a location rate, we tell the host
2405
+ * "off" so it can drop GPS power.
2406
+ */
2407
+ private recomputeLocationTier(): void {
2408
+ let bestRank = -1
2409
+ for (const app of this.connectedApps.values()) {
2410
+ const r = app.requestedLocationRate
2411
+ if (!r) continue
2412
+ const rank = locationRateRank(r)
2413
+ if (rank > bestRank) {
2414
+ bestRank = rank
2415
+ }
2416
+ }
2417
+ // When no app is asking, downgrade to "off". Without this we'd
2418
+ // leak the last-known-strictest rate forever (the original bug).
2419
+ const next: LocationRate = bestRank >= 0 ? LOCATION_RATE_PRIORITY[bestRank] : "off"
2420
+ if (next === this.lastAppliedLocationRate) return
2421
+ this.lastAppliedLocationRate = next
2422
+ console.log(`[LOCATION] aggregate tier → ${next}`)
2423
+ void phoneLocationService.setLocationTier(next)
2424
+ }
2425
+
2426
+ // ---------------------------------------------------------------------------
2427
+ // Storage helpers
2428
+ // ---------------------------------------------------------------------------
2429
+
2430
+ private getStorageKeyPrefix(packageName: string): string {
2431
+ const userId =
2432
+ (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous"
2433
+ return `mentraos_localstorage_${userId}_${packageName}_`
2434
+ }
2435
+
2436
+ private async handleStorageGet(
2437
+ packageName: string,
2438
+ payload: Record<string, unknown>,
2439
+ requestId?: string,
2440
+ ): Promise<void> {
2441
+ try {
2442
+ const key = payload.key as string
2443
+ if (!key) {
2444
+ this.sendResult(packageName, requestId, false, undefined, {
2445
+ code: MiniappErrorCode.INTERNAL,
2446
+ message: "storage_get requires a key",
2447
+ })
2448
+ return
2449
+ }
2450
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2451
+ const result = mmkvStorage.load<unknown>(fullKey)
2452
+ this.sendResult(packageName, requestId, true, {key, value: result.is_ok() ? result.value : null})
2453
+ } catch (err) {
2454
+ console.error(`${LOG_TAG}: storage_get error:`, err)
2455
+ this.sendResult(packageName, requestId, false, undefined, {
2456
+ code: MiniappErrorCode.INTERNAL,
2457
+ message: err instanceof Error ? err.message : "Storage error",
2458
+ })
2459
+ }
2460
+ }
2461
+
2462
+ private async handleStorageSet(
2463
+ packageName: string,
2464
+ payload: Record<string, unknown>,
2465
+ requestId?: string,
2466
+ ): Promise<void> {
2467
+ try {
2468
+ const key = payload.key as string
2469
+ if (!key) {
2470
+ this.sendResult(packageName, requestId, false, undefined, {
2471
+ code: MiniappErrorCode.INTERNAL,
2472
+ message: "storage_set requires a key",
2473
+ })
2474
+ return
2475
+ }
2476
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2477
+ mmkvStorage.save(fullKey, payload.value ?? null)
2478
+ this.sendResult(packageName, requestId, true)
2479
+ } catch (err) {
2480
+ console.error(`${LOG_TAG}: storage_set error:`, err)
2481
+ this.sendResult(packageName, requestId, false, undefined, {
2482
+ code: MiniappErrorCode.INTERNAL,
2483
+ message: err instanceof Error ? err.message : "Storage error",
2484
+ })
2485
+ }
2486
+ }
2487
+
2488
+ private async handleStorageDelete(
2489
+ packageName: string,
2490
+ payload: Record<string, unknown>,
2491
+ requestId?: string,
2492
+ ): Promise<void> {
2493
+ try {
2494
+ const key = payload.key as string
2495
+ if (!key) {
2496
+ this.sendResult(packageName, requestId, false, undefined, {
2497
+ code: MiniappErrorCode.INTERNAL,
2498
+ message: "storage_delete requires a key",
2499
+ })
2500
+ return
2501
+ }
2502
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2503
+ mmkvStorage.remove(fullKey)
2504
+ this.sendResult(packageName, requestId, true)
2505
+ } catch (err) {
2506
+ console.error(`${LOG_TAG}: storage_delete error:`, err)
2507
+ this.sendResult(packageName, requestId, false, undefined, {
2508
+ code: MiniappErrorCode.INTERNAL,
2509
+ message: err instanceof Error ? err.message : "Storage error",
2510
+ })
2511
+ }
2512
+ }
2513
+
2514
+ private async handleStorageList(
2515
+ packageName: string,
2516
+ _payload: Record<string, unknown>,
2517
+ requestId?: string,
2518
+ ): Promise<void> {
2519
+ try {
2520
+ const prefix = this.getStorageKeyPrefix(packageName)
2521
+ const allKeys = mmkvStorage.getAllKeys()
2522
+ const keys = allKeys.filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length))
2523
+ this.sendResult(packageName, requestId, true, {keys})
2524
+ } catch (err) {
2525
+ console.error(`${LOG_TAG}: storage_list error:`, err)
2526
+ this.sendResult(packageName, requestId, false, undefined, {
2527
+ code: MiniappErrorCode.INTERNAL,
2528
+ message: err instanceof Error ? err.message : "Storage error",
2529
+ })
2530
+ }
2531
+ }
2532
+
2533
+ private async handleStorageClear(packageName: string, requestId?: string): Promise<void> {
2534
+ try {
2535
+ const prefix = this.getStorageKeyPrefix(packageName)
2536
+ const allKeys = mmkvStorage.getAllKeys()
2537
+ for (const k of allKeys) {
2538
+ if (k.startsWith(prefix)) mmkvStorage.remove(k)
2539
+ }
2540
+ this.sendResult(packageName, requestId, true)
2541
+ } catch (err) {
2542
+ console.error(`${LOG_TAG}: storage_clear error:`, err)
2543
+ this.sendResult(packageName, requestId, false, undefined, {
2544
+ code: MiniappErrorCode.INTERNAL,
2545
+ message: err instanceof Error ? err.message : "Storage error",
2546
+ })
2547
+ }
2548
+ }
2549
+
2550
+ private async handleStorageHas(
2551
+ packageName: string,
2552
+ payload: Record<string, unknown>,
2553
+ requestId?: string,
2554
+ ): Promise<void> {
2555
+ try {
2556
+ const key = payload.key as string
2557
+ if (!key) {
2558
+ this.sendResult(packageName, requestId, false, undefined, {
2559
+ code: MiniappErrorCode.INTERNAL,
2560
+ message: "storage_has requires a key",
2561
+ })
2562
+ return
2563
+ }
2564
+ const fullKey = this.getStorageKeyPrefix(packageName) + key
2565
+ const result = mmkvStorage.load<unknown>(fullKey)
2566
+ this.sendResult(packageName, requestId, true, {has: result.is_ok()})
2567
+ } catch (err) {
2568
+ console.error(`${LOG_TAG}: storage_has error:`, err)
2569
+ this.sendResult(packageName, requestId, false, undefined, {
2570
+ code: MiniappErrorCode.INTERNAL,
2571
+ message: err instanceof Error ? err.message : "Storage error",
2572
+ })
2573
+ }
2574
+ }
2575
+
2576
+ private async handleStorageGetAll(packageName: string, requestId?: string): Promise<void> {
2577
+ try {
2578
+ const prefix = this.getStorageKeyPrefix(packageName)
2579
+ const allKeys = mmkvStorage.getAllKeys()
2580
+ const values: Record<string, string> = {}
2581
+ for (const k of allKeys) {
2582
+ if (!k.startsWith(prefix)) continue
2583
+ const r = mmkvStorage.load<unknown>(k)
2584
+ if (r.is_ok()) {
2585
+ const v = r.value
2586
+ values[k.slice(prefix.length)] = typeof v === "string" ? v : String(v ?? "")
2587
+ }
2588
+ }
2589
+ this.sendResult(packageName, requestId, true, {values})
2590
+ } catch (err) {
2591
+ console.error(`${LOG_TAG}: storage_get_all error:`, err)
2592
+ this.sendResult(packageName, requestId, false, undefined, {
2593
+ code: MiniappErrorCode.INTERNAL,
2594
+ message: err instanceof Error ? err.message : "Storage error",
2595
+ })
2596
+ }
2597
+ }
2598
+
2599
+ private async handleStorageSetMultiple(
2600
+ packageName: string,
2601
+ payload: Record<string, unknown>,
2602
+ requestId?: string,
2603
+ ): Promise<void> {
2604
+ try {
2605
+ const values = payload.values as Record<string, unknown> | undefined
2606
+ if (!values || typeof values !== "object") {
2607
+ this.sendResult(packageName, requestId, false, undefined, {
2608
+ code: MiniappErrorCode.INTERNAL,
2609
+ message: "storage_set_multiple requires a values object",
2610
+ })
2611
+ return
2612
+ }
2613
+ const prefix = this.getStorageKeyPrefix(packageName)
2614
+ for (const [key, value] of Object.entries(values)) {
2615
+ mmkvStorage.save(prefix + key, value ?? null)
2616
+ }
2617
+ this.sendResult(packageName, requestId, true)
2618
+ } catch (err) {
2619
+ console.error(`${LOG_TAG}: storage_set_multiple error:`, err)
2620
+ this.sendResult(packageName, requestId, false, undefined, {
2621
+ code: MiniappErrorCode.INTERNAL,
2622
+ message: err instanceof Error ? err.message : "Storage error",
2623
+ })
2624
+ }
2625
+ }
2626
+
2627
+ // ---------------------------------------------------------------------------
2628
+ // Camera FOV
2629
+ // ---------------------------------------------------------------------------
2630
+
2631
+ private async handleCameraFov(
2632
+ packageName: string,
2633
+ payload: Record<string, unknown>,
2634
+ requestId?: string,
2635
+ ): Promise<void> {
2636
+ const app = this.connectedApps.get(packageName)
2637
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2638
+ if (!hasCameraPermission) {
2639
+ logPermissionNotDeclared(packageName, "CAMERA", "to set camera FOV", `{"type": "CAMERA"}`)
2640
+ this.sendResult(packageName, requestId, false, undefined, {
2641
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2642
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2643
+ permission: "CAMERA",
2644
+ operation: MiniappRequestType.CAMERA_FOV,
2645
+ })
2646
+ return
2647
+ }
2648
+
2649
+ try {
2650
+ const request = normalizeCameraFovPayload(payload)
2651
+ const description =
2652
+ "preset" in request ? `preset=${request.preset}` : `fov=${request.fov} roi=${request.roiPosition ?? "center"}`
2653
+ console.log(`${LOG_TAG}: camera_fov_set ${description}`)
2654
+
2655
+ const result = await phoneCameraFovCoordinator.setOverride(packageName, request)
2656
+ this.sendResult(packageName, requestId, true, result)
2657
+ } catch (err) {
2658
+ console.error(`${LOG_TAG}: camera_fov error:`, err)
2659
+ this.sendResult(packageName, requestId, false, undefined, {
2660
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2661
+ message: err instanceof Error ? err.message : "Camera FOV error",
2662
+ })
2663
+ }
2664
+ }
2665
+
2666
+ /**
2667
+ * Explicit IMU enable/disable from a miniapp (session.imu.setEnabled).
2668
+ *
2669
+ * The accel stream already auto-toggles the native IMU on subscribe (see
2670
+ * {@link recomputeImuSubscription}); this is a direct override. We sync
2671
+ * `this.imuEnabled` to the requested state so a subsequent subscription
2672
+ * recompute with the same aggregate doesn't issue a redundant native call.
2673
+ * Note that a later subscribe/unsubscribe that changes the aggregate will
2674
+ * still re-derive the sensor state from subscriptions.
2675
+ */
2676
+ private handleImuSetEnabled(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
2677
+ try {
2678
+ const enabled = !!payload.enabled
2679
+ console.log(`${LOG_TAG}: imu_set_enabled ${enabled} (by ${packageName})`)
2680
+ this.imuEnabled = enabled
2681
+ void BluetoothSdk.setImuEnabled(enabled)
2682
+ this.sendResult(packageName, requestId, true)
2683
+ } catch (err) {
2684
+ console.error(`${LOG_TAG}: imu_set_enabled error:`, err)
2685
+ this.sendResult(packageName, requestId, false, undefined, {
2686
+ code: MiniappErrorCode.INTERNAL,
2687
+ message: err instanceof Error ? err.message : "IMU set-enabled error",
2688
+ })
2689
+ }
2690
+ }
2691
+
2692
+ // ---------------------------------------------------------------------------
2693
+ // System utilities (share, open URL, clipboard, download)
2694
+ // ---------------------------------------------------------------------------
2695
+
2696
+ private async handleShare(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
2697
+ const {text, title, base64, mimeType, filename, url} = payload as {
2698
+ text?: string
2699
+ title?: string
2700
+ base64?: string
2701
+ mimeType?: string
2702
+ filename?: string
2703
+ url?: string
2704
+ }
2705
+ try {
2706
+ if (base64) {
2707
+ // File share via base64 — write to temp file then share
2708
+ const tempFile = new File(Paths.cache, filename || "shared_file")
2709
+ tempFile.write(base64, {encoding: "base64"})
2710
+ await Share.open({
2711
+ url: tempFile.uri,
2712
+ type: mimeType || "application/octet-stream",
2713
+ filename: filename,
2714
+ title: title,
2715
+ })
2716
+ } else if (url) {
2717
+ await Share.open({url, title, message: text})
2718
+ } else {
2719
+ await Share.open({message: text || "", title})
2720
+ }
2721
+ this.sendResult(packageName, requestId, true, {success: true})
2722
+ } catch (error: any) {
2723
+ // react-native-share throws when user dismisses the share sheet
2724
+ if (error?.message?.includes("User did not share")) {
2725
+ this.sendResult(packageName, requestId, true, {success: false, cancelled: true})
2726
+ } else {
2727
+ console.error(`${LOG_TAG}: share error:`, error)
2728
+ this.sendResult(packageName, requestId, true, {success: false})
2729
+ }
2730
+ }
2731
+ }
2732
+
2733
+ private async handleOpenUrl(packageName: string, payload: Record<string, unknown>): Promise<void> {
2734
+ const url = payload.url as string | undefined
2735
+ if (!url || typeof url !== "string") {
2736
+ console.warn(`${LOG_TAG}: open_url missing url`)
2737
+ return
2738
+ }
2739
+ // Block dangerous schemes
2740
+ if (url.startsWith("javascript:") || url.startsWith("file:")) {
2741
+ console.warn(`${LOG_TAG}: open_url blocked dangerous scheme: ${url}`)
2742
+ return
2743
+ }
2744
+ try {
2745
+ await Linking.openURL(url)
2746
+ } catch (error) {
2747
+ console.error(`${LOG_TAG}: open_url error:`, error)
2748
+ }
2749
+ }
2750
+
2751
+ private async handleCopyClipboard(
2752
+ packageName: string,
2753
+ payload: Record<string, unknown>,
2754
+ requestId?: string,
2755
+ ): Promise<void> {
2756
+ const text = payload.text as string | undefined
2757
+ if (typeof text !== "string") {
2758
+ console.warn(`${LOG_TAG}: copy_clipboard missing text`)
2759
+ return
2760
+ }
2761
+ try {
2762
+ await Clipboard.setStringAsync(text)
2763
+ this.sendResult(packageName, requestId, true)
2764
+ } catch (error: any) {
2765
+ console.error(`${LOG_TAG}: clipboard error:`, error)
2766
+ this.sendResult(packageName, requestId, false, undefined, {
2767
+ code: MiniappErrorCode.INTERNAL,
2768
+ message: error?.message || "Clipboard error",
2769
+ })
2770
+ }
2771
+ }
2772
+
2773
+ private async handleDownload(
2774
+ packageName: string,
2775
+ payload: Record<string, unknown>,
2776
+ requestId?: string,
2777
+ ): Promise<void> {
2778
+ const {base64, url, mimeType, filename} = payload as {
2779
+ base64?: string
2780
+ url?: string
2781
+ mimeType?: string
2782
+ filename?: string
2783
+ }
2784
+ const name = filename || "download"
2785
+ try {
2786
+ let file: File
2787
+ if (base64) {
2788
+ file = new File(Paths.cache, name)
2789
+ file.write(base64, {encoding: "base64"})
2790
+ } else if (url) {
2791
+ file = await File.downloadFileAsync(url, new File(Paths.cache, name), {idempotent: true})
2792
+ } else {
2793
+ console.warn(`${LOG_TAG}: download missing base64 or url`)
2794
+ return
2795
+ }
2796
+ // Open share sheet so user can choose where to save
2797
+ await Share.open({
2798
+ url: file.uri,
2799
+ type: mimeType || "application/octet-stream",
2800
+ filename: name,
2801
+ })
2802
+ this.sendResult(packageName, requestId, true, {success: true})
2803
+ } catch (error: any) {
2804
+ if (error?.message?.includes("User did not share")) {
2805
+ this.sendResult(packageName, requestId, true, {success: true, cancelled: true})
2806
+ } else {
2807
+ console.error(`${LOG_TAG}: download error:`, error)
2808
+ this.sendResult(packageName, requestId, true, {success: false})
2809
+ }
2810
+ }
2811
+ }
2812
+
2813
+ // ===========================================================================
2814
+ // Photo + streaming handlers (cloud-coordinated)
2815
+ // ===========================================================================
2816
+
2817
+ private async handlePhoto(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
2818
+ // Manifest CAMERA permission gate.
2819
+ const app = this.connectedApps.get(packageName)
2820
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2821
+ if (!hasCameraPermission) {
2822
+ logPermissionNotDeclared(packageName, "CAMERA", "to take a photo", `{"type": "CAMERA"}`)
2823
+ this.sendResult(packageName, requestId, false, undefined, {
2824
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2825
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2826
+ permission: "CAMERA",
2827
+ operation: MiniappRequestType.PHOTO,
2828
+ })
2829
+ return
2830
+ }
2831
+
2832
+ try {
2833
+ const result = await phonePhotoCoordinator.takePhoto(packageName, {
2834
+ size: payload.size as "low" | "medium" | "high" | "max" | "small" | "large" | "full" | undefined,
2835
+ mode: payload.mode as "photo" | "text" | undefined,
2836
+ transferMethod: payload.transferMethod as "auto" | "ble" | undefined,
2837
+ compress: payload.compress as "none" | "low" | "medium" | "high" | undefined,
2838
+ sound: payload.sound as boolean | undefined,
2839
+ saveToGallery: payload.saveToGallery as boolean | undefined,
2840
+ exposureTimeNs: payload.exposureTimeNs as number | undefined,
2841
+ iso: payload.iso as number | null | undefined,
2842
+ aeExposureDivisor: payload.aeExposureDivisor as number | undefined,
2843
+ isoCap: payload.isoCap as number | undefined,
2844
+ noiseReduction: payload.noiseReduction as boolean | undefined,
2845
+ edgeEnhancement: payload.edgeEnhancement as boolean | undefined,
2846
+ mfnr: payload.mfnr as boolean | undefined,
2847
+ zsl: payload.zsl as boolean | undefined,
2848
+ ispDigitalGain: payload.ispDigitalGain as number | undefined,
2849
+ ispAnalogGain: payload.ispAnalogGain as string | undefined,
2850
+ })
2851
+ this.sendResult(packageName, requestId, true, result)
2852
+ } catch (err) {
2853
+ this.sendResult(packageName, requestId, false, undefined, {
2854
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2855
+ message: err instanceof Error ? err.message : "Photo request failed",
2856
+ stage: (err as {stage?: string}).stage,
2857
+ transport: (err as {transport?: string}).transport,
2858
+ })
2859
+ }
2860
+ }
2861
+
2862
+ private async handleCameraWarmUp(
2863
+ packageName: string,
2864
+ payload: Record<string, unknown>,
2865
+ requestId?: string,
2866
+ ): Promise<void> {
2867
+ // Manifest CAMERA permission gate (same as photo).
2868
+ const app = this.connectedApps.get(packageName)
2869
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2870
+ if (!hasCameraPermission) {
2871
+ logPermissionNotDeclared(packageName, "CAMERA", "to warm up the camera", `{"type": "CAMERA"}`)
2872
+ this.sendResult(packageName, requestId, false, undefined, {
2873
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2874
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2875
+ permission: "CAMERA",
2876
+ operation: MiniappRequestType.CAMERA_WARM_UP,
2877
+ })
2878
+ return
2879
+ }
2880
+
2881
+ try {
2882
+ await phonePhotoCoordinator.warmUpCamera(packageName, {
2883
+ size: payload.size as "low" | "medium" | "high" | "max" | undefined,
2884
+ mode: payload.mode as "photo" | "text" | undefined,
2885
+ exposureTimeNs: payload.exposureTimeNs as number | undefined,
2886
+ durationMs: payload.durationMs as number | undefined,
2887
+ })
2888
+ this.sendResult(packageName, requestId, true)
2889
+ } catch (err) {
2890
+ this.sendResult(packageName, requestId, false, undefined, {
2891
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2892
+ message: err instanceof Error ? err.message : "Camera warm-up failed",
2893
+ stage: (err as {stage?: string}).stage,
2894
+ transport: (err as {transport?: string}).transport,
2895
+ })
2896
+ }
2897
+ }
2898
+
2899
+ private async handleVideoRecordingStart(
2900
+ packageName: string,
2901
+ payload: Record<string, unknown>,
2902
+ requestId?: string,
2903
+ ): Promise<void> {
2904
+ // Manifest CAMERA permission gate (same as photo).
2905
+ const app = this.connectedApps.get(packageName)
2906
+ const hasCameraPermission = app?.installedManifest?.permissions?.some((p) => p.type === "CAMERA")
2907
+ if (!hasCameraPermission) {
2908
+ logPermissionNotDeclared(packageName, "CAMERA", "to record video", `{"type": "CAMERA"}`)
2909
+ this.sendResult(packageName, requestId, false, undefined, {
2910
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
2911
+ message: `CAMERA permission not declared in miniapp.json. Add {"type": "CAMERA"} to the "permissions" array.`,
2912
+ permission: "CAMERA",
2913
+ operation: MiniappRequestType.VIDEO_RECORDING_START,
2914
+ })
2915
+ return
2916
+ }
2917
+
2918
+ try {
2919
+ const result = await phoneVideoCoordinator.startRecording(packageName, {
2920
+ width: payload.width as number | undefined,
2921
+ height: payload.height as number | undefined,
2922
+ fps: payload.fps as number | undefined,
2923
+ sound: payload.sound as boolean | undefined,
2924
+ save: payload.save as boolean | undefined,
2925
+ })
2926
+ this.sendResult(packageName, requestId, true, result)
2927
+ } catch (err) {
2928
+ this.sendResult(packageName, requestId, false, undefined, {
2929
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2930
+ message: err instanceof Error ? err.message : "Video recording start failed",
2931
+ })
2932
+ }
2933
+ }
2934
+
2935
+ private async handleVideoRecordingStop(
2936
+ packageName: string,
2937
+ payload: Record<string, unknown>,
2938
+ requestId?: string,
2939
+ ): Promise<void> {
2940
+ try {
2941
+ await phoneVideoCoordinator.stopRecording(packageName, payload.recordingId as string | undefined, {
2942
+ uploadUrl: payload.uploadUrl as string | undefined,
2943
+ uploadAuthToken: payload.uploadAuthToken as string | undefined,
2944
+ })
2945
+ this.sendResult(packageName, requestId, true)
2946
+ } catch (err) {
2947
+ this.sendResult(packageName, requestId, false, undefined, {
2948
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2949
+ message: err instanceof Error ? err.message : "Video recording stop failed",
2950
+ })
2951
+ }
2952
+ }
2953
+
2954
+ /**
2955
+ * Stream handlers — dispatched to the engine PhoneStreamCoordinator. For managed
2956
+ * streams the coordinator additionally calls the v2 client REST route to
2957
+ * provision Cloudflare. Cloud-SDK apps (third-party developers) use a
2958
+ * separate cloud-side path that does not pass through here.
2959
+ */
2960
+ private async handleStreamStart(
2961
+ packageName: string,
2962
+ payload: Record<string, unknown>,
2963
+ requestId?: string,
2964
+ ): Promise<void> {
2965
+ const streaming = phoneStreamCoordinator
2966
+ if (!streaming) {
2967
+ this.sendResult(packageName, requestId, false, undefined, {
2968
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
2969
+ message: "Streaming is not configured on this host",
2970
+ })
2971
+ return
2972
+ }
2973
+ try {
2974
+ const result = await streaming.startUnmanaged(packageName, {
2975
+ streamUrl: payload.streamUrl as string,
2976
+ video: normalizeStreamVideoConfig(payload.video),
2977
+ audio: normalizeStreamAudioConfig(payload.audio),
2978
+ sound: payload.sound as boolean | undefined,
2979
+ })
2980
+ this.sendResult(packageName, requestId, true, result)
2981
+ } catch (err) {
2982
+ this.sendResult(packageName, requestId, false, undefined, {
2983
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
2984
+ message: err instanceof Error ? err.message : "Stream start failed",
2985
+ stage: (err as {stage?: string}).stage,
2986
+ transport: (err as {transport?: string}).transport,
2987
+ })
2988
+ }
2989
+ }
2990
+
2991
+ private async handleStreamStop(
2992
+ packageName: string,
2993
+ payload: Record<string, unknown>,
2994
+ requestId?: string,
2995
+ ): Promise<void> {
2996
+ const streaming = phoneStreamCoordinator
2997
+ if (!streaming) {
2998
+ // No adapter wired — treat as already-stopped.
2999
+ this.sendResult(packageName, requestId, true)
3000
+ return
3001
+ }
3002
+ try {
3003
+ await streaming.stop(packageName, payload.streamId as string | undefined)
3004
+ this.sendResult(packageName, requestId, true)
3005
+ } catch (err) {
3006
+ this.sendResult(packageName, requestId, false, undefined, {
3007
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
3008
+ message: err instanceof Error ? err.message : "Stream stop failed",
3009
+ stage: (err as {stage?: string}).stage,
3010
+ transport: (err as {transport?: string}).transport,
3011
+ })
3012
+ }
3013
+ }
3014
+
3015
+ private async handleManagedStreamStart(
3016
+ packageName: string,
3017
+ payload: Record<string, unknown>,
3018
+ requestId?: string,
3019
+ ): Promise<void> {
3020
+ const streaming = phoneStreamCoordinator
3021
+ if (!streaming) {
3022
+ this.sendResult(packageName, requestId, false, undefined, {
3023
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
3024
+ message: "Streaming is not configured on this host",
3025
+ })
3026
+ return
3027
+ }
3028
+ try {
3029
+ const result = await streaming.startManaged(packageName, {
3030
+ restreamDestinations: payload.restreamDestinations as Array<string | {url: string; name?: string}> | undefined,
3031
+ video: normalizeStreamVideoConfig(payload.video),
3032
+ audio: normalizeStreamAudioConfig(payload.audio),
3033
+ sound: payload.sound as boolean | undefined,
3034
+ ingest: payload.ingest as "srt" | "whip" | undefined,
3035
+ })
3036
+ this.sendResult(packageName, requestId, true, result)
3037
+ } catch (err) {
3038
+ this.sendResult(packageName, requestId, false, undefined, {
3039
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
3040
+ message: err instanceof Error ? err.message : "Managed stream start failed",
3041
+ stage: (err as {stage?: string}).stage,
3042
+ transport: (err as {transport?: string}).transport,
3043
+ })
3044
+ }
3045
+ }
3046
+
3047
+ private async handleManagedStreamStop(
3048
+ packageName: string,
3049
+ payload: Record<string, unknown>,
3050
+ requestId?: string,
3051
+ ): Promise<void> {
3052
+ return this.handleStreamStop(packageName, payload, requestId)
3053
+ }
3054
+
3055
+ /**
3056
+ * session.glasses.requestWifiSetup — open the phone's glasses Wi-Fi setup
3057
+ * flow. The host owns the actual UI via the `engine.configure({ui})` seam;
3058
+ * this just forwards the request and reports success/failure.
3059
+ */
3060
+ private async handleRequestWifiSetup(
3061
+ packageName: string,
3062
+ payload: Record<string, unknown>,
3063
+ requestId?: string,
3064
+ ): Promise<void> {
3065
+ const requestWifiSetup = getUiSeams().requestWifiSetup
3066
+ if (!requestWifiSetup) {
3067
+ this.sendResult(packageName, requestId, false, undefined, {
3068
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
3069
+ message: "Wi-Fi setup is not configured on this host",
3070
+ })
3071
+ return
3072
+ }
3073
+ try {
3074
+ await requestWifiSetup(payload.reason as string | undefined)
3075
+ this.sendResult(packageName, requestId, true)
3076
+ } catch (err) {
3077
+ this.sendResult(packageName, requestId, false, undefined, {
3078
+ code: (err as {code?: string}).code || MiniappErrorCode.INTERNAL,
3079
+ message: err instanceof Error ? err.message : "Wi-Fi setup failed",
3080
+ })
3081
+ }
3082
+ }
3083
+
3084
+ /**
3085
+ * Wire the PhoneStreamCoordinator's status callback so coordinator-emitted
3086
+ * updates become `EVENT { streamType: "stream_status" }` envelopes on the
3087
+ * subscribing miniapp(s).
3088
+ */
3089
+ public wireStreamingStatusFanout(): void {
3090
+ const streaming = phoneStreamCoordinator
3091
+ if (!streaming) return
3092
+ streaming.setStatusSubscriber((packageName, update) => {
3093
+ this.sendToMiniapp(packageName, {
3094
+ type: MiniappResponseType.EVENT,
3095
+ streamType: "stream_status",
3096
+ data: {
3097
+ streamId: update.streamId,
3098
+ status: update.status,
3099
+ source: update.source,
3100
+ ...(update.data || {}),
3101
+ },
3102
+ })
3103
+ })
3104
+ }
3105
+
3106
+ // ===========================================================================
3107
+ // Public subscribe helper
3108
+ // ===========================================================================
3109
+
3110
+ /**
3111
+ * Standalone subscribe — lets external code subscribe a registered miniapp to
3112
+ * streams without going through the envelope protocol.
3113
+ *
3114
+ * Returns `{ok: true}` on success, or `{ok: false, error}` on failure.
3115
+ */
3116
+ public subscribe(packageName: string, streams: string[]): {ok: boolean; error?: string} {
3117
+ const app = this.connectedApps.get(packageName)
3118
+ if (!app) {
3119
+ return {ok: false, error: `App ${packageName} is not registered`}
3120
+ }
3121
+
3122
+ if (!Array.isArray(streams)) {
3123
+ return {ok: false, error: "streams must be an array"}
3124
+ }
3125
+
3126
+ console.log(`${LOG_TAG}: subscribe(${packageName}, [${streams.join(", ")}])`)
3127
+
3128
+ // Check microphone permission for mic-requiring streams
3129
+ const micStreams = ["transcription", "translation", "audio_chunk", "vad"]
3130
+ const needsMic = streams.some((s: string) => micStreams.some((m) => s.startsWith(m) || s === m))
3131
+ if (needsMic) {
3132
+ const hasMicPermission = app.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE")
3133
+ if (!hasMicPermission) {
3134
+ return {ok: false, error: "MICROPHONE permission not declared in miniapp.json"}
3135
+ }
3136
+ }
3137
+
3138
+ // Remove old subscriptions for this app
3139
+ for (const oldStream of app.subscriptions) {
3140
+ const subs = this.streamSubscribers.get(oldStream)
3141
+ if (subs) {
3142
+ subs.delete(packageName)
3143
+ if (subs.size === 0) {
3144
+ this.streamSubscribers.delete(oldStream)
3145
+ }
3146
+ }
3147
+ }
3148
+
3149
+ // Add new subscriptions
3150
+ app.subscriptions = new Set(streams)
3151
+ app.forceLocalTranscriptionStreams.clear()
3152
+ app.cloudTranscriptionStreams = new Set(streams.filter((stream) => stream.startsWith("transcription:")))
3153
+ for (const stream of streams) {
3154
+ let subs = this.streamSubscribers.get(stream)
3155
+ if (!subs) {
3156
+ subs = new Set()
3157
+ this.streamSubscribers.set(stream, subs)
3158
+ }
3159
+ subs.add(packageName)
3160
+ }
3161
+
3162
+ this.recomputeMicRequirements()
3163
+ this.updateCloudSubscriptions()
3164
+ return {ok: true}
3165
+ }
3166
+
3167
+ // ===========================================================================
3168
+ // Mic requirements
3169
+ // ===========================================================================
3170
+
3171
+ private recomputeMicRequirements(): void {
3172
+ let anyPcm = false
3173
+ let anyLc3 = false
3174
+ for (const [stream, subscribers] of this.streamSubscribers) {
3175
+ if (subscribers.size === 0) continue
3176
+ if (stream === "audio_chunk") anyPcm = true
3177
+ if (stream.startsWith("transcription:") || stream.startsWith("translation:") || stream === "vad") anyLc3 = true
3178
+ }
3179
+ micStateCoordinator.setLocalRequirements({pcm: anyPcm, lc3: anyLc3})
3180
+ }
3181
+
3182
+ /**
3183
+ * Recompute the aggregated v2 cloud subscription list across all local miniapps.
3184
+ *
3185
+ * Cloud-dependent streams (transcription:*, translation:*) only flow if the
3186
+ * v2 runtime knows the phone wants them. Local-only streams (button_press, etc.)
3187
+ * are NOT sent — they come from the Bluetooth SDK, not from cloud.
3188
+ */
3189
+ private updateCloudSubscriptions(): void {
3190
+ const cloudStreams = new Set<string>()
3191
+ let transcriptionLang: string | null = null
3192
+ let forceLocalTranscriptionLang: string | null = null
3193
+ let hasForceLocalTranscription = false
3194
+ for (const [stream, subscribers] of this.streamSubscribers) {
3195
+ if (subscribers.size === 0) continue
3196
+ // Only transcription / translation need cloud delivery. Location and
3197
+ // notifications are sourced natively on the phone — no cloud hop.
3198
+ if (stream.startsWith("translation:")) {
3199
+ cloudStreams.add(stream)
3200
+ }
3201
+ if (stream.startsWith("transcription:")) {
3202
+ const language = stream.substring("transcription:".length)
3203
+ if (transcriptionLang === null) transcriptionLang = language
3204
+
3205
+ const routes = summarizeTranscriptionRoutes(subscribers, (packageName) =>
3206
+ this.appTranscriptionRoutesForSubscription(packageName, stream),
3207
+ )
3208
+ if (routes.hasForceLocalSubscriber) {
3209
+ hasForceLocalTranscription = true
3210
+ if (forceLocalTranscriptionLang === null) forceLocalTranscriptionLang = language
3211
+ }
3212
+ if (routes.hasCloudSubscriber) cloudStreams.add(stream)
3213
+ }
3214
+ }
3215
+ localSttFallbackCoordinator.onSubscriptionChange(
3216
+ transcriptionLang !== null,
3217
+ forceLocalTranscriptionLang ?? transcriptionLang,
3218
+ hasForceLocalTranscription,
3219
+ )
3220
+
3221
+ // Mirror the same set as typed AudioSubscription[] and push it to the cloud
3222
+ // runtime.
3223
+ this.ensureCloudResultsWired()
3224
+ const subs = this.buildCloudAudioSubscriptions(cloudStreams)
3225
+ const nextKey = this.audioSubscriptionKey(subs)
3226
+ if (!this.cloudAudioSubscriptionSync.begin(nextKey)) return
3227
+ console.log(
3228
+ `${LOG_TAG}: updateCloudSubscriptions streams=[${Array.from(cloudStreams).join(", ")}] cloudSubs=${subs.length}`,
3229
+ )
3230
+ cloudClientService
3231
+ .setSubscriptions(subs)
3232
+ .then(() => {
3233
+ this.cloudAudioSubscriptionSync.succeeded(nextKey)
3234
+ })
3235
+ .catch((err) => {
3236
+ // Best-effort: the cloud may not be connected yet. Keep failed writes
3237
+ // retryable; otherwise a reconnect that recomputes the same desired
3238
+ // transcription set gets deduped and local captions never recover.
3239
+ this.cloudAudioSubscriptionSync.failed(nextKey)
3240
+ console.warn(`${LOG_TAG}: cloud setSubscriptions failed: ${(err as Error)?.message ?? err}`)
3241
+ })
3242
+ }
3243
+
3244
+ /**
3245
+ * Build the cloud `AudioSubscription[]` from the miniapp cloud-stream key set.
3246
+ *
3247
+ * Stream keys: `transcription:<lang>` (lang may be `auto`, `en-US`, …) and
3248
+ * `translation:<source>:<target>` (3 colon-parts; source/target may be `*`
3249
+ * or `auto` wildcards, target may also be `*`). The cloud protocol requires a
3250
+ * concrete `target: string` for translation, so wildcard-target translation
3251
+ * keys (`translation:<source>:*`, `translation:*:*`, `translation:auto`) have
3252
+ * no cloud equivalent and are skipped. Wildcard/`auto` SOURCE maps to
3253
+ * `{mode: "auto"}`.
3254
+ */
3255
+ private buildCloudAudioSubscriptions(cloudStreams: Set<string>): AudioSubscription[] {
3256
+ const subs: AudioSubscription[] = []
3257
+ const langSource = (code: string): LanguageSource =>
3258
+ code === "auto" || code === "*" ? {mode: "auto"} : {mode: "specific", code}
3259
+ for (const stream of cloudStreams) {
3260
+ if (stream.startsWith("transcription:")) {
3261
+ const lang = stream.substring("transcription:".length)
3262
+ subs.push({kind: "transcription", language: langSource(lang)})
3263
+ } else if (stream.startsWith("translation:")) {
3264
+ const parts = stream.split(":")
3265
+ // Only `translation:<source>:<target>` maps cleanly; a concrete target
3266
+ // is required by the cloud schema.
3267
+ if (parts.length === 3) {
3268
+ const [, source, target] = parts
3269
+ if (target === "*") continue
3270
+ subs.push({kind: "translation", source: langSource(source), target})
3271
+ }
3272
+ }
3273
+ }
3274
+ return subs
3275
+ }
3276
+
3277
+ private audioSubscriptionKey(subs: AudioSubscription[]): string {
3278
+ return JSON.stringify(
3279
+ subs
3280
+ .map((sub) => {
3281
+ if (sub.kind === "transcription") {
3282
+ return {
3283
+ kind: sub.kind,
3284
+ language: sub.language,
3285
+ }
3286
+ }
3287
+ return {
3288
+ kind: sub.kind,
3289
+ source: sub.source,
3290
+ target: sub.target,
3291
+ }
3292
+ })
3293
+ .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
3294
+ )
3295
+ }
3296
+
3297
+ /**
3298
+ * Wire the cloud's transcription/translation results into the existing
3299
+ * miniapp fan-out exactly once. Maps each cloud result back to the
3300
+ * cloud-to-app data shape `forwardEvent` already forwards, keyed on the
3301
+ * `transcription:<lang>` / `translation:<source>:<target>` stream strings, so
3302
+ * subscribed miniapps receive identical envelopes.
3303
+ */
3304
+ private ensureCloudResultsWired(): void {
3305
+ if (this.cloudResultsWired) return
3306
+ this.cloudResultsWired = true
3307
+
3308
+ cloudClientService.onTranscript((d: TranscriptionData) => {
3309
+ const receivedAt = Date.now()
3310
+ if (TRANSCRIPT_TIMING_TELEMETRY) {
3311
+ console.log(
3312
+ `${LOG_TAG}: transcript cloud_recv t=${receivedAt} final=${d.isFinal} lang=${
3313
+ d.resolvedLanguage
3314
+ } text="${d.text.slice(0, 48)}"`,
3315
+ )
3316
+ }
3317
+ this.forwardEvent(
3318
+ `transcription:${d.resolvedLanguage}`,
3319
+ {
3320
+ type: "transcription",
3321
+ text: d.text,
3322
+ isFinal: d.isFinal,
3323
+ utteranceId: d.utteranceId,
3324
+ transcribeLanguage: d.resolvedLanguage,
3325
+ detectedLanguage: d.languageDetected ? d.resolvedLanguage : undefined,
3326
+ startTime: d.startMs,
3327
+ endTime: d.endMs,
3328
+ speakerId: d.speakerId,
3329
+ duration: d.durationMs,
3330
+ provider: d.provider,
3331
+ confidence: d.confidence,
3332
+ __hostReceivedAt: receivedAt,
3333
+ },
3334
+ "cloud",
3335
+ )
3336
+ })
3337
+
3338
+ cloudClientService.onTranslation((d: TranslationData) => {
3339
+ this.forwardEvent(`translation:${d.source.language}:${d.target.language}`, {
3340
+ type: "translation",
3341
+ text: d.text,
3342
+ originalText: d.originalText,
3343
+ isFinal: d.isFinal,
3344
+ utteranceId: d.utteranceId,
3345
+ startTime: d.startMs,
3346
+ endTime: d.endMs,
3347
+ speakerId: d.speakerId,
3348
+ duration: d.durationMs,
3349
+ transcribeLanguage: d.source.language,
3350
+ translateLanguage: d.target.language,
3351
+ didTranslate: true,
3352
+ provider: d.provider,
3353
+ confidence: d.confidence,
3354
+ })
3355
+ })
3356
+ }
3357
+
3358
+ private ensureCloudStatusWired(): void {
3359
+ if (this.cloudStatusWired) return
3360
+ this.cloudStatusWired = true
3361
+
3362
+ cloudClientService.onStatusChanged((status) => {
3363
+ if (status.status === "connected") {
3364
+ this.updateCloudSubscriptions()
3365
+ // A miniapp that connected before the cloud client was ready never got
3366
+ // its auth token; now that we're connected, mint it without waiting for
3367
+ // the retry backoff (or a full manual reconnect).
3368
+ this.redriveMiniappAuthDelivery()
3369
+ }
3370
+ this.broadcastCloudStatus()
3371
+ })
3372
+
3373
+ useSettingsStore.subscribe(
3374
+ (s) => s.getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive),
3375
+ () => {
3376
+ this.broadcastCloudStatus()
3377
+ },
3378
+ )
3379
+ }
3380
+
3381
+ private currentCloudStatus(): CloudClientStatusSnapshot {
3382
+ const base = cloudClientService.getStatus()
3383
+ const fallbackActive =
3384
+ (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive) as boolean | undefined) ===
3385
+ true
3386
+ return {
3387
+ status: base.status,
3388
+ audioTransport: fallbackActive ? "offline" : base.audioTransport,
3389
+ }
3390
+ }
3391
+
3392
+ private broadcastCloudStatus(): void {
3393
+ const status = this.currentCloudStatus()
3394
+ for (const packageName of this.connectedApps.keys()) {
3395
+ this.sendCloudStatusToMiniapp(packageName, status)
3396
+ }
3397
+ }
3398
+
3399
+ private sendCloudStatusToMiniapp(packageName: string, status = this.currentCloudStatus()): void {
3400
+ this.sendToMiniapp(packageName, {
3401
+ type: MiniappResponseType.EVENT,
3402
+ streamType: CLOUD_STATUS_STREAM,
3403
+ data: status,
3404
+ })
3405
+ }
3406
+
3407
+ // ===========================================================================
3408
+ // Stream fan-out
3409
+ // ===========================================================================
3410
+
3411
+ /**
3412
+ * Forward a streamed event to all miniapps subscribed to the given stream.
3413
+ *
3414
+ * Event name translation:
3415
+ * - Cloud sends "head_up" → miniapp protocol uses "head_position" (HEAD_POSITION)
3416
+ * - Cloud sends "VAD" (uppercase) → miniapp protocol uses "vad" (lowercase)
3417
+ */
3418
+ public forwardEvent(streamType: string, data: unknown, transcriptionSource?: TranscriptionEventSource): void {
3419
+ // Translate cloud event names to miniapp protocol stream types
3420
+ const normalizedStream = this.normalizeStreamType(streamType)
3421
+
3422
+ // Collect all subscribers: exact match, plus wildcard matches for streams
3423
+ // that carry a language tag. A miniapp subscribed to "transcription:auto"
3424
+ // should receive any "transcription:<lang>" event (the detected language
3425
+ // is conveyed in the event data, not the stream key).
3426
+ //
3427
+ // Translation streams support cloud v3's wildcard patterns. Incoming
3428
+ // events are keyed `translation:<source>:<target>` and we match against
3429
+ // any of:
3430
+ // translation:*:* — all-pairs (TranslationModule.on)
3431
+ // translation:*:<target> — any-source → target (TranslationModule.to)
3432
+ // translation:<source>:* — source → any-target (rare)
3433
+ // translation:<source>:<target> — exact (TranslationModule.fromTo)
3434
+ // translation:auto — back-compat alias for *:*
3435
+ const matchedSubs = new Set<string>()
3436
+ const exact = this.streamSubscribers.get(normalizedStream)
3437
+ if (exact) for (const p of exact) matchedSubs.add(p)
3438
+
3439
+ if (normalizedStream.startsWith("transcription:")) {
3440
+ const autoSubs = this.streamSubscribers.get("transcription:auto")
3441
+ if (autoSubs) for (const p of autoSubs) matchedSubs.add(p)
3442
+ } else if (normalizedStream.startsWith("translation:")) {
3443
+ // translation:<source>:<target> — match each wildcard variant.
3444
+ const parts = normalizedStream.split(":")
3445
+ // parts[0] = "translation", parts[1] = source, parts[2] = target
3446
+ if (parts.length === 3) {
3447
+ const [, source, target] = parts
3448
+ const patterns = [
3449
+ "translation:auto", // legacy alias for *:*
3450
+ "translation:*:*", // cloud v3 "all pairs"
3451
+ `translation:*:${target}`, // cloud v3 "to <target>"
3452
+ `translation:${source}:*`, // cloud v3 "from <source>"
3453
+ ]
3454
+ for (const pat of patterns) {
3455
+ const subs = this.streamSubscribers.get(pat)
3456
+ if (subs) for (const p of subs) matchedSubs.add(p)
3457
+ }
3458
+ } else {
3459
+ // Malformed stream key — still honor the legacy auto alias.
3460
+ const autoSubs = this.streamSubscribers.get("translation:auto")
3461
+ if (autoSubs) for (const p of autoSubs) matchedSubs.add(p)
3462
+ }
3463
+ }
3464
+
3465
+ // Touch event per-gesture fan-out. SDK-side dispatch is keyed on the
3466
+ // exact streamType, so per-gesture subscribers (`touch_event:single_tap`,
3467
+ // etc.) need their own EVENT envelope with the gesture-tagged streamType.
3468
+ // The bare `touch_event` stream above still catches `onTouch(handler)`.
3469
+ let perGestureStream: string | null = null
3470
+ let outboundData = data
3471
+ if (normalizedStream === MiniappStreamType.TOUCH_EVENT) {
3472
+ // The Bluetooth SDK delivers the gesture under `gestureName` (single_tap /
3473
+ // double_tap / triple_tap / long_press / swipe_up / swipe_down). Surface it
3474
+ // to miniapps as `TouchData.kind` and tag a per-gesture stream so
3475
+ // onTouch("single_tap", ...) routes correctly.
3476
+ const touch = data as {gestureName?: string; kind?: string} | null
3477
+ const gesture = touch?.gestureName ?? touch?.kind
3478
+ if (typeof gesture === "string" && gesture.length > 0) {
3479
+ outboundData = {...(touch as object), kind: gesture}
3480
+ perGestureStream = `${MiniappStreamType.TOUCH_EVENT}:${gesture}`
3481
+ }
3482
+ }
3483
+
3484
+ if (normalizedStream.startsWith("transcription:")) {
3485
+ // const known = Array.from(this.streamSubscribers.keys())
3486
+ // console.log(
3487
+ // `${LOG_TAG}: forwardEvent(${streamType} → ${normalizedStream}) matched=${matchedSubs.size} known=[${known.join(", ")}]`,
3488
+ // )
3489
+ }
3490
+
3491
+ if (matchedSubs.size === 0 && !perGestureStream) return
3492
+
3493
+ for (const packageName of matchedSubs) {
3494
+ let transcriptionRoute: "default" | "forceLocal" | "all" | undefined
3495
+ if (normalizedStream.startsWith("transcription:") && transcriptionSource) {
3496
+ const routes = this.appTranscriptionRoutesForEvent(packageName, normalizedStream)
3497
+ transcriptionRoute =
3498
+ transcriptionDeliveryRoute(transcriptionSource, routes, cloudClientService.isConnected()) ?? undefined
3499
+ if (!transcriptionRoute) continue
3500
+ }
3501
+ if (TRANSCRIPT_TIMING_TELEMETRY && normalizedStream.startsWith("transcription:")) {
3502
+ const transcript = data as {text?: string; isFinal?: boolean; __hostReceivedAt?: number} | null
3503
+ const now = Date.now()
3504
+ console.log(
3505
+ `${LOG_TAG}: transcript fanout t=${now} app=${packageName} final=${!!transcript?.isFinal} hostDelta=${
3506
+ transcript?.__hostReceivedAt ? now - transcript.__hostReceivedAt : -1
3507
+ }ms text="${(transcript?.text ?? "").slice(0, 48)}"`,
3508
+ )
3509
+ }
3510
+ // While a nav trip is active the Nav SDK's road-snapped fixes are
3511
+ // already being forwarded via addLocationListener (see NAV_START handler).
3512
+ // Suppress the raw background-GPS forward for that miniapp so the two
3513
+ // streams don't interleave and cause the position to jump back to the
3514
+ // real-phone location during simulation.
3515
+ if (normalizedStream === MiniappStreamType.LOCATION_UPDATE && this.navigationHandlers.isTripActive(packageName)) {
3516
+ continue
3517
+ }
3518
+ this.sendToMiniapp(packageName, {
3519
+ type: MiniappResponseType.EVENT,
3520
+ streamType: normalizedStream,
3521
+ data: outboundData,
3522
+ ...(transcriptionRoute ? {transcriptionRoute} : {}),
3523
+ })
3524
+ }
3525
+
3526
+ // Send a separate envelope to per-gesture subscribers tagged with the
3527
+ // gesture-specific streamType so SDK-side dispatch routes correctly.
3528
+ if (perGestureStream) {
3529
+ const gestureSubs = this.streamSubscribers.get(perGestureStream)
3530
+ if (gestureSubs) {
3531
+ for (const packageName of gestureSubs) {
3532
+ this.sendToMiniapp(packageName, {
3533
+ type: MiniappResponseType.EVENT,
3534
+ streamType: perGestureStream,
3535
+ data: outboundData,
3536
+ })
3537
+ }
3538
+ }
3539
+ }
3540
+ }
3541
+
3542
+ private appTranscriptionRoutesForSubscription(
3543
+ packageName: string,
3544
+ stream: string,
3545
+ ): {
3546
+ cloud: boolean
3547
+ forceLocal: boolean
3548
+ } {
3549
+ const app = this.connectedApps.get(packageName)
3550
+ return {
3551
+ cloud: app?.cloudTranscriptionStreams.has(stream) === true,
3552
+ forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true,
3553
+ }
3554
+ }
3555
+
3556
+ private appTranscriptionRoutesForEvent(
3557
+ packageName: string,
3558
+ stream: string,
3559
+ ): {
3560
+ cloud: boolean
3561
+ forceLocal: boolean
3562
+ } {
3563
+ const app = this.connectedApps.get(packageName)
3564
+ const autoStream = `${MiniappStreamType.TRANSCRIPTION}:auto`
3565
+ return {
3566
+ cloud:
3567
+ app?.cloudTranscriptionStreams.has(stream) === true || app?.cloudTranscriptionStreams.has(autoStream) === true,
3568
+ forceLocal:
3569
+ app?.forceLocalTranscriptionStreams.has(stream) === true ||
3570
+ app?.forceLocalTranscriptionStreams.has(autoStream) === true,
3571
+ }
3572
+ }
3573
+
3574
+ /**
3575
+ * Translate cloud event names to miniapp stream type values.
3576
+ */
3577
+ private normalizeStreamType(cloudEventName: string): string {
3578
+ // Cloud / Bluetooth SDK → miniapp protocol translations.
3579
+ // Bluetooth SDK event names don't always match the miniapp wire values.
3580
+ switch (cloudEventName) {
3581
+ case "head_up":
3582
+ return MiniappStreamType.HEAD_POSITION // head_up → head_position
3583
+ case "accel_event":
3584
+ return MiniappStreamType.ACCEL_DATA // accel_event (native) → accel_data
3585
+ case "VAD":
3586
+ return MiniappStreamType.VAD // VAD (uppercase) → vad (lowercase)
3587
+ case "glasses_battery_update":
3588
+ return MiniappStreamType.GLASSES_BATTERY // glasses_battery_update → glasses_battery
3589
+ case "glasses_connection_state":
3590
+ return MiniappStreamType.GLASSES_CONNECTION // glasses_connection_state → glasses_connection
3591
+ default:
3592
+ // Preserve case for typed streams like "transcription:en-US" / "translation:en-US:fr-FR"
3593
+ // whose language tags are case-sensitive (BCP-47). Lowercase only plain names.
3594
+ if (cloudEventName.includes(":")) return cloudEventName
3595
+ return cloudEventName.toLowerCase()
3596
+ }
3597
+ }
3598
+
3599
+ // ===========================================================================
3600
+ // Outbound helpers
3601
+ // ===========================================================================
3602
+
3603
+ /**
3604
+ * Send a payload to a connected miniapp, wrapped in an envelope.
3605
+ */
3606
+ /**
3607
+ * Push a speaker-state transition to the owning miniapp. Idempotent: if
3608
+ * the new state matches the cached one (and isn't an error), no envelope
3609
+ * is sent. Error events are always sent — they're transient and the SDK
3610
+ * settles back to a non-error state immediately after.
3611
+ */
3612
+ private setSpeakerState(
3613
+ packageName: string,
3614
+ next: SpeakerStateValue,
3615
+ extra?: {errorCode?: string; errorMessage?: string; durationMs?: number},
3616
+ ): void {
3617
+ const app = this.connectedApps.get(packageName)
3618
+ if (!app) return
3619
+ if (app.speakerState === next && next !== "error") return
3620
+ app.speakerState = next === "error" ? "error" : next
3621
+ this.sendToMiniapp(packageName, {
3622
+ type: MiniappResponseType.SPEAKER_STATE,
3623
+ state: next,
3624
+ ...(extra?.errorCode !== undefined ? {errorCode: extra.errorCode} : {}),
3625
+ ...(extra?.errorMessage !== undefined ? {errorMessage: extra.errorMessage} : {}),
3626
+ ...(extra?.durationMs !== undefined ? {durationMs: extra.durationMs} : {}),
3627
+ })
3628
+ // After an error event, immediately transition to "stopped" so the
3629
+ // SDK's isPlaying getter reads false and a new play()/speak() call
3630
+ // starts cleanly from idle/stopped.
3631
+ if (next === "error") {
3632
+ app.speakerState = "stopped"
3633
+ this.sendToMiniapp(packageName, {
3634
+ type: MiniappResponseType.SPEAKER_STATE,
3635
+ state: "stopped",
3636
+ })
3637
+ }
3638
+ }
3639
+
3640
+ private sendToMiniapp(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
3641
+ const app = this.connectedApps.get(packageName)
3642
+ if (!app) {
3643
+ console.warn(`${LOG_TAG}: sendToMiniapp — ${packageName} not connected`)
3644
+ return
3645
+ }
3646
+
3647
+ const envelope: MiniappEnvelope = {
3648
+ payload,
3649
+ requestId,
3650
+ }
3651
+
3652
+ const serialized = serializeEnvelope(envelope)
3653
+ if ((payload as Record<string, unknown>)?.streamType?.toString().startsWith("transcription")) {
3654
+ // console.log(
3655
+ // `${LOG_TAG}: sendToMiniapp → ${packageName} streamType=${(payload as Record<string, unknown>).streamType}`,
3656
+ // )
3657
+ }
3658
+
3659
+ try {
3660
+ app.sendMessage(serialized)
3661
+ } catch (err) {
3662
+ console.error(`${LOG_TAG}: sendToMiniapp error for ${packageName}:`, err)
3663
+ }
3664
+ }
3665
+
3666
+ /**
3667
+ * Send a REQUEST_RESULT response.
3668
+ */
3669
+ // ===========================================================================
3670
+ // Inter-miniapp interop (session.miniapps + session.actions.invoke)
3671
+ // ===========================================================================
3672
+
3673
+ /** Max serialized size of an action call's params / result (256 KB). */
3674
+ private static readonly ACTION_PAYLOAD_CAP = 256 * 1024
3675
+
3676
+ /** Outstanding action invocations, keyed by host-generated callId. */
3677
+ private actionCalls = new Map<
3678
+ string,
3679
+ {callerPackageName: string; targetPackageName: string; callerRequestId?: string; timer: number}
3680
+ >()
3681
+ private actionCallSeq = 0
3682
+
3683
+ private interopApps(): ClientApp[] {
3684
+ return useAppStatusStore.getState().apps
3685
+ }
3686
+
3687
+ private isSystemPackage(packageName: string): boolean {
3688
+ if (SYSTEM_MINIAPP_PACKAGES.has(packageName)) return true
3689
+ return this.interopApps().find((app) => app.packageName === packageName)?.isMiniappDev === true
3690
+ }
3691
+
3692
+ private async startInteropApp(packageName: string): Promise<boolean> {
3693
+ const app = this.interopApps().find((a) => a.packageName === packageName)
3694
+ if (!app) return false
3695
+ if (app.local) {
3696
+ await miniappLauncher.ensureConnected(packageName)
3697
+ return true
3698
+ }
3699
+ return useAppStatusStore.getState().start(app, {skipNavigation: true})
3700
+ }
3701
+
3702
+ private stopInteropApp(packageName: string): Promise<void> {
3703
+ return useAppStatusStore.getState().stop(packageName)
3704
+ }
3705
+
3706
+ private wakeInteropApp(packageName: string): Promise<void> {
3707
+ return miniappLauncher.ensureConnected(packageName)
3708
+ }
3709
+
3710
+ /** Emit an interop audit event (best-effort — never let telemetry break a call). */
3711
+ private auditInterop(event: InteropAuditEvent): void {
3712
+ try {
3713
+ getAnalytics()?.("miniapp_interop", {
3714
+ caller: event.caller,
3715
+ op: event.op,
3716
+ target: event.target ?? "",
3717
+ actionId: event.actionId ?? "",
3718
+ ok: event.ok,
3719
+ errorCode: event.errorCode ?? "",
3720
+ })
3721
+ } catch {
3722
+ /* telemetry must never break an interop call */
3723
+ }
3724
+ }
3725
+
3726
+ /** Reject the request with NOT_PERMITTED unless the caller is a system app. */
3727
+ private requireSystemCaller(
3728
+ packageName: string,
3729
+ requestId: string | undefined,
3730
+ op: InteropAuditEvent["op"],
3731
+ target?: string,
3732
+ ): boolean {
3733
+ if (this.isSystemPackage(packageName)) return true
3734
+ this.auditInterop({caller: packageName, op, target, ok: false, errorCode: MiniappErrorCode.NOT_PERMITTED})
3735
+ this.sendResult(packageName, requestId, false, undefined, {
3736
+ code: MiniappErrorCode.NOT_PERMITTED,
3737
+ message: "Inter-miniapp APIs are restricted to system apps",
3738
+ })
3739
+ return false
3740
+ }
3741
+
3742
+ /** Project a host ClientApp into the SDK's MiniappInfo shape (JSON over the wire). */
3743
+ private buildMiniappInfo(app: ClientApp): Record<string, unknown> {
3744
+ const compatibility = app.compatibility ?? {
3745
+ isCompatible: true,
3746
+ missingRequired: [],
3747
+ missingOptional: [],
3748
+ warnings: [],
3749
+ }
3750
+ return {
3751
+ packageName: app.packageName,
3752
+ name: app.name,
3753
+ version: app.version ?? "",
3754
+ running: app.running,
3755
+ compatibility,
3756
+ actions: app.actions ?? [],
3757
+ }
3758
+ }
3759
+
3760
+ private handleMiniappsList(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
3761
+ if (!this.requireSystemCaller(packageName, requestId, "list")) return
3762
+ const includeIncompatible = payload.includeIncompatible === true
3763
+ const infos = this.interopApps()
3764
+ .filter(
3765
+ (a) =>
3766
+ a.packageName &&
3767
+ a.packageName !== packageName &&
3768
+ !a.packageName.includes("@empty") &&
3769
+ (includeIncompatible || a.compatibility?.isCompatible !== false),
3770
+ )
3771
+ .map((a) => this.buildMiniappInfo(a))
3772
+ this.sendResult(packageName, requestId, true, infos)
3773
+ this.auditInterop({caller: packageName, op: "list", ok: true})
3774
+ }
3775
+
3776
+ private async handleMiniappsStart(
3777
+ packageName: string,
3778
+ payload: Record<string, unknown>,
3779
+ requestId?: string,
3780
+ ): Promise<void> {
3781
+ const target = payload.packageName as string | undefined
3782
+ if (!this.requireSystemCaller(packageName, requestId, "start", target)) return
3783
+ const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined
3784
+ if (!target || !app) {
3785
+ this.sendResult(packageName, requestId, false, undefined, {
3786
+ code: MiniappErrorCode.APP_NOT_FOUND,
3787
+ message: `Miniapp not found: ${target ?? "(missing packageName)"}`,
3788
+ })
3789
+ this.auditInterop({
3790
+ caller: packageName,
3791
+ op: "start",
3792
+ target,
3793
+ ok: false,
3794
+ errorCode: MiniappErrorCode.APP_NOT_FOUND,
3795
+ })
3796
+ return
3797
+ }
3798
+ // Pre-flight the hardware gate so the caller gets a precise reason rather
3799
+ // than a generic rejection (start()'s own gate would otherwise alert + abort).
3800
+ if (app.compatibility && app.compatibility.isCompatible === false) {
3801
+ this.sendResult(packageName, requestId, false, undefined, {
3802
+ code: MiniappErrorCode.APP_NOT_COMPATIBLE,
3803
+ message: `${target} is not compatible with the connected glasses`,
3804
+ })
3805
+ islandNotifications.emit({
3806
+ kind: "version_incompatible",
3807
+ packageName: target,
3808
+ reason: `${target} is not compatible with the connected glasses`,
3809
+ metadata: {missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.start"},
3810
+ timestamp: Date.now(),
3811
+ })
3812
+ this.auditInterop({
3813
+ caller: packageName,
3814
+ op: "start",
3815
+ target,
3816
+ ok: false,
3817
+ errorCode: MiniappErrorCode.APP_NOT_COMPATIBLE,
3818
+ })
3819
+ return
3820
+ }
3821
+ try {
3822
+ // startInteropApp resolves to false when the app-store gate rejected the
3823
+ // launch or the background context failed to spawn — don't report success.
3824
+ const started = await this.startInteropApp(target)
3825
+ if (started) {
3826
+ this.sendResult(packageName, requestId, true)
3827
+ this.auditInterop({caller: packageName, op: "start", target, ok: true})
3828
+ } else {
3829
+ this.sendResult(packageName, requestId, false, undefined, {
3830
+ code: MiniappErrorCode.INTERNAL,
3831
+ message: `start of ${target} was rejected by the host (gated or failed to spawn)`,
3832
+ })
3833
+ this.auditInterop({
3834
+ caller: packageName,
3835
+ op: "start",
3836
+ target,
3837
+ ok: false,
3838
+ errorCode: MiniappErrorCode.INTERNAL,
3839
+ })
3840
+ }
3841
+ } catch (e) {
3842
+ this.sendResult(packageName, requestId, false, undefined, {
3843
+ code: MiniappErrorCode.INTERNAL,
3844
+ message: (e as Error)?.message ?? "start failed",
3845
+ })
3846
+ this.auditInterop({
3847
+ caller: packageName,
3848
+ op: "start",
3849
+ target,
3850
+ ok: false,
3851
+ errorCode: MiniappErrorCode.INTERNAL,
3852
+ })
3853
+ }
3854
+ }
3855
+
3856
+ private async handleMiniappsStop(
3857
+ packageName: string,
3858
+ payload: Record<string, unknown>,
3859
+ requestId?: string,
3860
+ ): Promise<void> {
3861
+ const target = payload.packageName as string | undefined
3862
+ if (!this.requireSystemCaller(packageName, requestId, "stop", target)) return
3863
+ if (!target) {
3864
+ this.sendResult(packageName, requestId, false, undefined, {
3865
+ code: MiniappErrorCode.APP_NOT_FOUND,
3866
+ message: "stop requires a packageName",
3867
+ })
3868
+ return
3869
+ }
3870
+ try {
3871
+ await this.stopInteropApp(target)
3872
+ this.sendResult(packageName, requestId, true)
3873
+ this.auditInterop({caller: packageName, op: "stop", target, ok: true})
3874
+ } catch (e) {
3875
+ this.sendResult(packageName, requestId, false, undefined, {
3876
+ code: MiniappErrorCode.INTERNAL,
3877
+ message: (e as Error)?.message ?? "stop failed",
3878
+ })
3879
+ }
3880
+ }
3881
+
3882
+ /** True iff `value` serializes larger than the action payload cap. */
3883
+ private actionPayloadTooLarge(value: unknown): boolean {
3884
+ try {
3885
+ return JSON.stringify(value ?? null).length > LocalMiniappRuntime.ACTION_PAYLOAD_CAP
3886
+ } catch {
3887
+ // Unserializable — let it through; the transport's own serialize handles it.
3888
+ return false
3889
+ }
3890
+ }
3891
+
3892
+ /**
3893
+ * session.actions.invoke → wake the target headlessly, deliver an ACTION_CALL,
3894
+ * and correlate its ACTION_RESULT back to the caller's pending request.
3895
+ */
3896
+ private async handleActionInvoke(
3897
+ callerPackageName: string,
3898
+ payload: Record<string, unknown>,
3899
+ requestId?: string,
3900
+ ): Promise<void> {
3901
+ const targetForGate = payload.targetPackageName as string | undefined
3902
+ if (!this.requireSystemCaller(callerPackageName, requestId, "invoke", targetForGate)) return
3903
+
3904
+ const target = payload.targetPackageName as string | undefined
3905
+ const actionId = payload.actionId as string | undefined
3906
+ const params = (payload.params as Record<string, unknown> | undefined) ?? {}
3907
+ // Floor the timeout at 6s so the host never rejects with ACTION_TIMEOUT before
3908
+ // the target SDK's handler-registration window closes. A freshly-woken miniapp
3909
+ // buffers an undelivered ACTION_CALL for HANDLER_WAIT_MS (5s in @mentra/miniapp's
3910
+ // actions module) waiting for session.actions.handle; a shorter host timeout
3911
+ // could fire while the target is still registering, then the action would run
3912
+ // with no caller left to receive its result. 6s = that 5s buffer + 1s for the
3913
+ // ACTION_RESULT/NO_ACTION_HANDLER reply to travel back. Keep these in sync.
3914
+ const timeoutMs = Math.min(Math.max(Number(payload.timeoutMs) || 30_000, 6_000), 120_000)
3915
+
3916
+ if (this.actionPayloadTooLarge(params)) {
3917
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3918
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
3919
+ message: "action params exceeded the 256 KB cap",
3920
+ })
3921
+ return
3922
+ }
3923
+
3924
+ const app = target ? this.interopApps().find((a) => a.packageName === target) : undefined
3925
+ if (!target || !app) {
3926
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3927
+ code: MiniappErrorCode.APP_NOT_FOUND,
3928
+ message: `Miniapp not found: ${target ?? "(missing targetPackageName)"}`,
3929
+ })
3930
+ return
3931
+ }
3932
+ if (!actionId) {
3933
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3934
+ code: MiniappErrorCode.ACTION_NOT_FOUND,
3935
+ message: "invoke requires an actionId",
3936
+ })
3937
+ return
3938
+ }
3939
+ if (app.compatibility && app.compatibility.isCompatible === false) {
3940
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3941
+ code: MiniappErrorCode.APP_NOT_COMPATIBLE,
3942
+ message: `${target} is not compatible with the connected glasses`,
3943
+ })
3944
+ islandNotifications.emit({
3945
+ kind: "version_incompatible",
3946
+ packageName: target,
3947
+ reason: `${target} is not compatible with the connected glasses`,
3948
+ metadata: {missingRequired: app.compatibility.missingRequired ?? [], via: "miniapps.invoke"},
3949
+ timestamp: Date.now(),
3950
+ })
3951
+ return
3952
+ }
3953
+ // Declared-action gate — only once the host populates app.actions (Phase 2);
3954
+ // until then app.actions is undefined and we fall through to NO_ACTION_HANDLER.
3955
+ if (!(app.actions ?? []).some((a) => a.id === actionId)) {
3956
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3957
+ code: MiniappErrorCode.ACTION_NOT_FOUND,
3958
+ message: `${target} does not declare action "${actionId}"`,
3959
+ })
3960
+ return
3961
+ }
3962
+
3963
+ // Headless wake + wait for CONNECT (idempotent / fast if already connected).
3964
+ try {
3965
+ await this.wakeInteropApp(target)
3966
+ } catch (e) {
3967
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3968
+ code: MiniappErrorCode.WAKE_FAILED,
3969
+ message: (e as Error)?.message ?? `failed to wake ${target}`,
3970
+ })
3971
+ return
3972
+ }
3973
+
3974
+ // The wake waited for CONNECT, but the target could have dropped in the gap
3975
+ // before delivery — fail fast rather than arming a timer for a call that
3976
+ // sendToMiniapp would silently drop (the caller would otherwise wait the
3977
+ // full invoke timeout for an ACTION_CALL that was never delivered).
3978
+ if (!this.connectedApps.has(target)) {
3979
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3980
+ code: MiniappErrorCode.WAKE_FAILED,
3981
+ message: `${target} disconnected before the action could be delivered`,
3982
+ })
3983
+ this.auditInterop({
3984
+ caller: callerPackageName,
3985
+ op: "invoke",
3986
+ target,
3987
+ actionId,
3988
+ ok: false,
3989
+ errorCode: MiniappErrorCode.WAKE_FAILED,
3990
+ })
3991
+ return
3992
+ }
3993
+
3994
+ // Deliver the call and arm the handler timeout; ACTION_RESULT resolves it.
3995
+ const callId = `act-${Date.now().toString(36)}-${this.actionCallSeq++}`
3996
+ const timer = BgTimer.setTimeout(() => {
3997
+ this.actionCalls.delete(callId)
3998
+ this.sendResult(callerPackageName, requestId, false, undefined, {
3999
+ code: MiniappErrorCode.ACTION_TIMEOUT,
4000
+ message: `action "${actionId}" timed out after ${timeoutMs}ms`,
4001
+ })
4002
+ }, timeoutMs)
4003
+ this.actionCalls.set(callId, {callerPackageName, targetPackageName: target, callerRequestId: requestId, timer})
4004
+ this.sendToMiniapp(target, {
4005
+ type: MiniappResponseType.ACTION_CALL,
4006
+ callId,
4007
+ actionId,
4008
+ params,
4009
+ callerPackageName,
4010
+ })
4011
+ this.auditInterop({caller: callerPackageName, op: "invoke", target, actionId, ok: true})
4012
+ }
4013
+
4014
+ /** Target → host: forward an ACTION_RESULT back to the caller's pending invoke. */
4015
+ private handleActionResult(targetPackageName: string, payload: Record<string, unknown>): void {
4016
+ const callId = payload.callId as string | undefined
4017
+ if (!callId) return
4018
+ const pending = this.actionCalls.get(callId)
4019
+ if (!pending) return
4020
+ // Only the invoked target may resolve this call. callIds are guessable
4021
+ // (timestamp + seq), so a different connected miniapp must not be able to
4022
+ // spoof a result/error for someone else's invoke.
4023
+ if (pending.targetPackageName !== targetPackageName) return
4024
+ this.actionCalls.delete(callId)
4025
+ BgTimer.clearTimeout(pending.timer)
4026
+
4027
+ if (payload.ok === true) {
4028
+ const result = payload.result
4029
+ if (this.actionPayloadTooLarge(result)) {
4030
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, {
4031
+ code: MiniappErrorCode.PAYLOAD_TOO_LARGE,
4032
+ message: "action result exceeded the 256 KB cap",
4033
+ })
4034
+ return
4035
+ }
4036
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, true, result ?? null)
4037
+ } else {
4038
+ const error = (payload.error as ({code: string; message: string} & Record<string, unknown>) | undefined) ?? {
4039
+ code: MiniappErrorCode.INTERNAL,
4040
+ message: "action handler error",
4041
+ }
4042
+ this.sendResult(pending.callerPackageName, pending.callerRequestId, false, undefined, error)
4043
+ }
4044
+ }
4045
+
4046
+ private sendResult(
4047
+ packageName: string,
4048
+ requestId: string | undefined,
4049
+ ok: boolean,
4050
+ data?: unknown,
4051
+ error?: {code: string; message: string} & Record<string, unknown>,
4052
+ ): void {
4053
+ if (!requestId) return
4054
+
4055
+ this.sendToMiniapp(
4056
+ packageName,
4057
+ {
4058
+ type: MiniappResponseType.REQUEST_RESULT,
4059
+ requestId,
4060
+ ok,
4061
+ ...(data !== undefined ? {data} : {}),
4062
+ ...(error ? {error} : {}),
4063
+ },
4064
+ requestId,
4065
+ )
4066
+ }
4067
+
4068
+ /**
4069
+ * Send a VISIBILITY_CHANGE push to a miniapp.
4070
+ */
4071
+ public sendVisibilityChange(packageName: string, visibility: "foreground" | "background"): void {
4072
+ this.sendToMiniapp(packageName, {
4073
+ type: MiniappResponseType.VISIBILITY_CHANGE,
4074
+ visibility,
4075
+ })
4076
+ }
4077
+
4078
+ /**
4079
+ * Fast foreground liveness check used when a local miniapp UI is opened or
4080
+ * resumed. The normal watchdog intentionally waits ~30s to avoid killing a
4081
+ * healthy-but-busy background script. When the user is actively looking at a
4082
+ * WebView, a stale JSContext reads as "cloud offline" / no captions, so probe
4083
+ * immediately and reuse the crash-respawn path if no message comes back.
4084
+ */
4085
+ public probeForegroundLiveness(
4086
+ packageName: string,
4087
+ reason = "foreground-open",
4088
+ timeoutMs = FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS,
4089
+ ): void {
4090
+ const app = this.connectedApps.get(packageName)
4091
+ if (!app) return
4092
+
4093
+ this.clearForegroundProbe(packageName)
4094
+ const probeStartedAt = Date.now()
4095
+
4096
+ this.sendToMiniapp(packageName, {
4097
+ type: MiniappRequestType.PING,
4098
+ })
4099
+
4100
+ const timerId = BgTimer.setTimeout(() => {
4101
+ this.foregroundProbeTimers.delete(packageName)
4102
+ const current = this.connectedApps.get(packageName)
4103
+ if (!current) return
4104
+ if (current.lastPongAt >= probeStartedAt) return
4105
+
4106
+ console.warn(`${LOG_TAG}: ${packageName} failed foreground liveness probe (${reason}), respawning`)
4107
+ this.unregisterApp(packageName)
4108
+ this.onLivenessTimeout?.(packageName)
4109
+ }, timeoutMs)
4110
+ this.foregroundProbeTimers.set(packageName, timerId)
4111
+ }
4112
+
4113
+ // ===========================================================================
4114
+ // Ping / pong liveness
4115
+ // ===========================================================================
4116
+
4117
+ private ensurePingLoop(): void {
4118
+ if (this.pingIntervalId !== null) return
4119
+
4120
+ this.pingIntervalId = BgTimer.setInterval(() => {
4121
+ this.doPingRound()
4122
+ }, PING_INTERVAL_MS)
4123
+ }
4124
+
4125
+ private stopPingLoop(): void {
4126
+ if (this.pingIntervalId !== null) {
4127
+ BgTimer.clearInterval(this.pingIntervalId)
4128
+ this.pingIntervalId = null
4129
+ }
4130
+ }
4131
+
4132
+ private doPingRound(): void {
4133
+ const now = Date.now()
4134
+ const staleThreshold = PING_INTERVAL_MS * PING_TIMEOUT_THRESHOLD
4135
+
4136
+ const toRemove: string[] = []
4137
+
4138
+ for (const [packageName, app] of this.connectedApps) {
4139
+ if (now - app.lastPongAt > staleThreshold) {
4140
+ console.warn(`${LOG_TAG}: ${packageName} missed ${PING_TIMEOUT_THRESHOLD} pings, unregistering`)
4141
+ toRemove.push(packageName)
4142
+ continue
4143
+ }
4144
+
4145
+ // Send PING — SDK auto-replies with PONG
4146
+ this.sendToMiniapp(packageName, {
4147
+ type: MiniappRequestType.PING,
4148
+ })
4149
+ }
4150
+
4151
+ for (const pkg of toRemove) {
4152
+ this.unregisterApp(pkg)
4153
+ // Hand the package to the router's respawn machinery — a missed-pings
4154
+ // death is treated like a crash so the background script comes back.
4155
+ this.onLivenessTimeout?.(pkg)
4156
+ }
4157
+ }
4158
+
4159
+ /**
4160
+ * Called when a PONG is received from a miniapp (or any message, really).
4161
+ * Updates lastPongAt to keep the app alive.
4162
+ */
4163
+ public handlePong(packageName: string): void {
4164
+ const app = this.connectedApps.get(packageName)
4165
+ if (app) {
4166
+ app.lastPongAt = Date.now()
4167
+ this.clearForegroundProbe(packageName)
4168
+ }
4169
+ }
4170
+
4171
+ private clearForegroundProbe(packageName: string): void {
4172
+ const timerId = this.foregroundProbeTimers.get(packageName)
4173
+ if (timerId == null) return
4174
+ BgTimer.clearTimeout(timerId)
4175
+ this.foregroundProbeTimers.delete(packageName)
4176
+ }
4177
+
4178
+ // ===========================================================================
4179
+ // Lifecycle
4180
+ // ===========================================================================
4181
+
4182
+ public cleanup(): void {
4183
+ console.log(`${LOG_TAG}: cleanup()`)
4184
+ this.stopPingLoop()
4185
+
4186
+ // Copy keys since unregisterApp mutates the map
4187
+ const packageNames = [...this.connectedApps.keys()]
4188
+ for (const pkg of packageNames) {
4189
+ this.unregisterApp(pkg)
4190
+ }
4191
+
4192
+ // Belt-and-suspenders: clear any remaining state
4193
+ this.pendingCloudRequests.clear()
4194
+ this.streamSubscribers.clear()
4195
+ this.connectedApps.clear()
4196
+ for (const timerId of this.foregroundProbeTimers.values()) {
4197
+ BgTimer.clearTimeout(timerId)
4198
+ }
4199
+ this.foregroundProbeTimers.clear()
4200
+
4201
+ LocalMiniappRuntime.instance = null
4202
+ }
4203
+
4204
+ /**
4205
+ * Number of currently connected miniapps (for diagnostics).
4206
+ */
4207
+ public get connectedAppCount(): number {
4208
+ return this.connectedApps.size
4209
+ }
4210
+ }
4211
+
4212
+ const localMiniappRuntime = LocalMiniappRuntime.getInstance()
4213
+ export default localMiniappRuntime