@mentra/engine 0.1.0-beta.0 → 0.1.0-dev.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (355) hide show
  1. package/README.md +42 -1
  2. package/build/engine.d.ts +11 -3
  3. package/build/engine.d.ts.map +1 -1
  4. package/build/engine.js +7 -2
  5. package/build/engine.js.map +1 -1
  6. package/build/facades/dev.d.ts.map +1 -1
  7. package/build/facades/dev.js +9 -5
  8. package/build/facades/dev.js.map +1 -1
  9. package/build/facades/gallery.js +1 -1
  10. package/build/facades/gallery.js.map +1 -1
  11. package/build/facades/glasses.d.ts +1 -0
  12. package/build/facades/glasses.d.ts.map +1 -1
  13. package/build/facades/glasses.js +19 -4
  14. package/build/facades/glasses.js.map +1 -1
  15. package/build/facades/glassesSettings.d.ts.map +1 -1
  16. package/build/facades/glassesSettings.js +1 -0
  17. package/build/facades/glassesSettings.js.map +1 -1
  18. package/build/facades/miniapps.d.ts +4 -0
  19. package/build/facades/miniapps.d.ts.map +1 -1
  20. package/build/facades/miniapps.js +5 -0
  21. package/build/facades/miniapps.js.map +1 -1
  22. package/build/facades/ota.d.ts +11 -2
  23. package/build/facades/ota.d.ts.map +1 -1
  24. package/build/facades/ota.js +19 -4
  25. package/build/facades/ota.js.map +1 -1
  26. package/build/facades/pairing.d.ts +4 -0
  27. package/build/facades/pairing.d.ts.map +1 -1
  28. package/build/facades/pairing.js +10 -8
  29. package/build/facades/pairing.js.map +1 -1
  30. package/build/facades/phoneNotifications.d.ts +0 -4
  31. package/build/facades/phoneNotifications.d.ts.map +1 -1
  32. package/build/facades/phoneNotifications.js +4 -14
  33. package/build/facades/phoneNotifications.js.map +1 -1
  34. package/build/index.d.ts +6 -3
  35. package/build/index.d.ts.map +1 -1
  36. package/build/index.js +4 -2
  37. package/build/index.js.map +1 -1
  38. package/build/internal.d.ts +2 -2
  39. package/build/internal.d.ts.map +1 -1
  40. package/build/internal.js +1 -1
  41. package/build/internal.js.map +1 -1
  42. package/build/react/MentraLiveOtaFlow.d.ts +38 -0
  43. package/build/react/MentraLiveOtaFlow.d.ts.map +1 -0
  44. package/build/react/MentraLiveOtaFlow.js +528 -0
  45. package/build/react/MentraLiveOtaFlow.js.map +1 -0
  46. package/build/react/index.d.ts +2 -0
  47. package/build/react/index.d.ts.map +1 -0
  48. package/build/react/index.js +2 -0
  49. package/build/react/index.js.map +1 -0
  50. package/build/react/useEngineSnapshot.d.ts +2 -0
  51. package/build/react/useEngineSnapshot.d.ts.map +1 -0
  52. package/build/react/useEngineSnapshot.js +14 -0
  53. package/build/react/useEngineSnapshot.js.map +1 -0
  54. package/build/runtime/bootstrap.d.ts +24 -1
  55. package/build/runtime/bootstrap.d.ts.map +1 -1
  56. package/build/runtime/bootstrap.js +17 -0
  57. package/build/runtime/bootstrap.js.map +1 -1
  58. package/build/runtime/scanQrSeam.d.ts +7 -0
  59. package/build/runtime/scanQrSeam.d.ts.map +1 -0
  60. package/build/runtime/scanQrSeam.js +16 -0
  61. package/build/runtime/scanQrSeam.js.map +1 -0
  62. package/build/services/AppRegistry.d.ts +18 -8
  63. package/build/services/AppRegistry.d.ts.map +1 -1
  64. package/build/services/AppRegistry.js +44 -45
  65. package/build/services/AppRegistry.js.map +1 -1
  66. package/build/services/AudioCloudUplink.d.ts +9 -3
  67. package/build/services/AudioCloudUplink.d.ts.map +1 -1
  68. package/build/services/AudioCloudUplink.js +9 -3
  69. package/build/services/AudioCloudUplink.js.map +1 -1
  70. package/build/services/AudioPlaybackService.d.ts +9 -1
  71. package/build/services/AudioPlaybackService.d.ts.map +1 -1
  72. package/build/services/AudioPlaybackService.js +77 -32
  73. package/build/services/AudioPlaybackService.js.map +1 -1
  74. package/build/services/BlobStore.d.ts +2 -1
  75. package/build/services/BlobStore.d.ts.map +1 -1
  76. package/build/services/BlobStore.js +17 -9
  77. package/build/services/BlobStore.js.map +1 -1
  78. package/build/services/CloudAudioSubscriptionSync.d.ts.map +1 -1
  79. package/build/services/CloudAudioSubscriptionSync.js +13 -1
  80. package/build/services/CloudAudioSubscriptionSync.js.map +1 -1
  81. package/build/services/CloudClientService.d.ts +15 -0
  82. package/build/services/CloudClientService.d.ts.map +1 -1
  83. package/build/services/CloudClientService.js +75 -0
  84. package/build/services/CloudClientService.js.map +1 -1
  85. package/build/services/ForegroundLocationPermission.d.ts +13 -0
  86. package/build/services/ForegroundLocationPermission.d.ts.map +1 -0
  87. package/build/services/ForegroundLocationPermission.js +12 -0
  88. package/build/services/ForegroundLocationPermission.js.map +1 -0
  89. package/build/services/GlassesSettingsSync.d.ts.map +1 -1
  90. package/build/services/GlassesSettingsSync.js +9 -3
  91. package/build/services/GlassesSettingsSync.js.map +1 -1
  92. package/build/services/GlassesStatusProjection.d.ts +2 -1
  93. package/build/services/GlassesStatusProjection.d.ts.map +1 -1
  94. package/build/services/GlassesStatusProjection.js +15 -8
  95. package/build/services/GlassesStatusProjection.js.map +1 -1
  96. package/build/services/HotspotOtaTransport.d.ts +29 -0
  97. package/build/services/HotspotOtaTransport.d.ts.map +1 -0
  98. package/build/services/HotspotOtaTransport.js +146 -0
  99. package/build/services/HotspotOtaTransport.js.map +1 -0
  100. package/build/services/HotspotShutdown.d.ts +17 -0
  101. package/build/services/HotspotShutdown.d.ts.map +1 -0
  102. package/build/services/HotspotShutdown.js +29 -0
  103. package/build/services/HotspotShutdown.js.map +1 -0
  104. package/build/services/LocalDisplayManager.d.ts +15 -0
  105. package/build/services/LocalDisplayManager.d.ts.map +1 -1
  106. package/build/services/LocalDisplayManager.js +106 -20
  107. package/build/services/LocalDisplayManager.js.map +1 -1
  108. package/build/services/LocalMiniappRuntime.d.ts +67 -1
  109. package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
  110. package/build/services/LocalMiniappRuntime.js +486 -126
  111. package/build/services/LocalMiniappRuntime.js.map +1 -1
  112. package/build/services/LocalMiniappStorage.d.ts +32 -0
  113. package/build/services/LocalMiniappStorage.d.ts.map +1 -0
  114. package/build/services/LocalMiniappStorage.js +66 -0
  115. package/build/services/LocalMiniappStorage.js.map +1 -0
  116. package/build/services/LocalMiniappUserIdentity.d.ts +20 -0
  117. package/build/services/LocalMiniappUserIdentity.d.ts.map +1 -0
  118. package/build/services/LocalMiniappUserIdentity.js +43 -0
  119. package/build/services/LocalMiniappUserIdentity.js.map +1 -0
  120. package/build/services/MicStateCoordinator.d.ts +42 -1
  121. package/build/services/MicStateCoordinator.d.ts.map +1 -1
  122. package/build/services/MicStateCoordinator.js +141 -3
  123. package/build/services/MicStateCoordinator.js.map +1 -1
  124. package/build/services/MiniappLauncher.d.ts.map +1 -1
  125. package/build/services/MiniappLauncher.js +23 -3
  126. package/build/services/MiniappLauncher.js.map +1 -1
  127. package/build/services/MiniappLiveness.d.ts +12 -0
  128. package/build/services/MiniappLiveness.d.ts.map +1 -0
  129. package/build/services/MiniappLiveness.js +16 -0
  130. package/build/services/MiniappLiveness.js.map +1 -0
  131. package/build/services/OtaArtifactDownloader.d.ts +65 -0
  132. package/build/services/OtaArtifactDownloader.d.ts.map +1 -0
  133. package/build/services/OtaArtifactDownloader.js +226 -0
  134. package/build/services/OtaArtifactDownloader.js.map +1 -0
  135. package/build/services/OtaAutoChain.d.ts +34 -0
  136. package/build/services/OtaAutoChain.d.ts.map +1 -0
  137. package/build/services/OtaAutoChain.js +83 -0
  138. package/build/services/OtaAutoChain.js.map +1 -0
  139. package/build/services/OtaErrorMapping.d.ts +18 -0
  140. package/build/services/OtaErrorMapping.d.ts.map +1 -0
  141. package/build/services/OtaErrorMapping.js +63 -0
  142. package/build/services/OtaErrorMapping.js.map +1 -0
  143. package/build/services/OtaInstallCoordinator.d.ts +94 -14
  144. package/build/services/OtaInstallCoordinator.d.ts.map +1 -1
  145. package/build/services/OtaInstallCoordinator.js +574 -130
  146. package/build/services/OtaInstallCoordinator.js.map +1 -1
  147. package/build/services/OtaService.d.ts.map +1 -1
  148. package/build/services/OtaService.js +8 -1
  149. package/build/services/OtaService.js.map +1 -1
  150. package/build/services/OtaUpdateCheckService.d.ts +38 -3
  151. package/build/services/OtaUpdateCheckService.d.ts.map +1 -1
  152. package/build/services/OtaUpdateCheckService.js +142 -22
  153. package/build/services/OtaUpdateCheckService.js.map +1 -1
  154. package/build/services/PhoneCalendarService.d.ts.map +1 -1
  155. package/build/services/PhoneCalendarService.js +22 -1
  156. package/build/services/PhoneCalendarService.js.map +1 -1
  157. package/build/services/PhoneCameraFovCoordinator.d.ts +2 -0
  158. package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -1
  159. package/build/services/PhoneCameraFovCoordinator.js +14 -0
  160. package/build/services/PhoneCameraFovCoordinator.js.map +1 -1
  161. package/build/services/PhoneNotificationsSync.d.ts.map +1 -1
  162. package/build/services/PhoneNotificationsSync.js +12 -10
  163. package/build/services/PhoneNotificationsSync.js.map +1 -1
  164. package/build/services/PhonePhotoCoordinator.d.ts +10 -3
  165. package/build/services/PhonePhotoCoordinator.d.ts.map +1 -1
  166. package/build/services/PhonePhotoCoordinator.js +21 -3
  167. package/build/services/PhonePhotoCoordinator.js.map +1 -1
  168. package/build/services/PhoneStreamCoordinator.d.ts +8 -1
  169. package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
  170. package/build/services/PhoneStreamCoordinator.js +125 -13
  171. package/build/services/PhoneStreamCoordinator.js.map +1 -1
  172. package/build/services/PhoneVideoCoordinator.d.ts +2 -0
  173. package/build/services/PhoneVideoCoordinator.d.ts.map +1 -1
  174. package/build/services/PhoneVideoCoordinator.js +10 -0
  175. package/build/services/PhoneVideoCoordinator.js.map +1 -1
  176. package/build/services/SceneRenderer.d.ts.map +1 -1
  177. package/build/services/SceneRenderer.js +15 -0
  178. package/build/services/SceneRenderer.js.map +1 -1
  179. package/build/services/SupportProfileSync.d.ts +22 -0
  180. package/build/services/SupportProfileSync.d.ts.map +1 -0
  181. package/build/services/SupportProfileSync.js +213 -0
  182. package/build/services/SupportProfileSync.js.map +1 -0
  183. package/build/services/TtsTextSanitizer.d.ts +5 -0
  184. package/build/services/TtsTextSanitizer.d.ts.map +1 -0
  185. package/build/services/TtsTextSanitizer.js +61 -0
  186. package/build/services/TtsTextSanitizer.js.map +1 -0
  187. package/build/services/WifiAdbRequest.d.ts +17 -0
  188. package/build/services/WifiAdbRequest.d.ts.map +1 -0
  189. package/build/services/WifiAdbRequest.js +35 -0
  190. package/build/services/WifiAdbRequest.js.map +1 -0
  191. package/build/services/asg/asgCameraApi.d.ts +3 -0
  192. package/build/services/asg/asgCameraApi.d.ts.map +1 -1
  193. package/build/services/asg/asgCameraApi.js +72 -2
  194. package/build/services/asg/asgCameraApi.js.map +1 -1
  195. package/build/services/asg/gallerySyncService.d.ts.map +1 -1
  196. package/build/services/asg/gallerySyncService.js +26 -14
  197. package/build/services/asg/gallerySyncService.js.map +1 -1
  198. package/build/services/asg/localNetworkTransport.d.ts +2 -2
  199. package/build/services/asg/localNetworkTransport.d.ts.map +1 -1
  200. package/build/services/asg/localNetworkTransport.js +6 -6
  201. package/build/services/asg/localNetworkTransport.js.map +1 -1
  202. package/build/services/audioPlaybackAssets.d.ts +8 -0
  203. package/build/services/audioPlaybackAssets.d.ts.map +1 -0
  204. package/build/services/audioPlaybackAssets.js +7 -0
  205. package/build/services/audioPlaybackAssets.js.map +1 -0
  206. package/build/services/glassesClockSync.js +2 -2
  207. package/build/services/glassesClockSync.js.map +1 -1
  208. package/build/services/manifestActions.d.ts +7 -0
  209. package/build/services/manifestActions.d.ts.map +1 -0
  210. package/build/services/manifestActions.js +25 -0
  211. package/build/services/manifestActions.js.map +1 -0
  212. package/build/services/otaDisplayState.d.ts +22 -11
  213. package/build/services/otaDisplayState.d.ts.map +1 -1
  214. package/build/services/otaDisplayState.js +36 -14
  215. package/build/services/otaDisplayState.js.map +1 -1
  216. package/build/services/otaInstallPolicy.d.ts +15 -3
  217. package/build/services/otaInstallPolicy.d.ts.map +1 -1
  218. package/build/services/otaInstallPolicy.js +24 -3
  219. package/build/services/otaInstallPolicy.js.map +1 -1
  220. package/build/services/otaManifestUrl.d.ts +2 -0
  221. package/build/services/otaManifestUrl.d.ts.map +1 -0
  222. package/build/services/{asgOtaVersionUrl.js → otaManifestUrl.js} +15 -5
  223. package/build/services/otaManifestUrl.js.map +1 -0
  224. package/build/services/slimStreamStatus.d.ts +11 -0
  225. package/build/services/slimStreamStatus.d.ts.map +1 -1
  226. package/build/services/slimStreamStatus.js +14 -4
  227. package/build/services/slimStreamStatus.js.map +1 -1
  228. package/build/stores/apps.d.ts.map +1 -1
  229. package/build/stores/apps.js +11 -0
  230. package/build/stores/apps.js.map +1 -1
  231. package/build/stores/bluetoothSettingKeys.d.ts.map +1 -1
  232. package/build/stores/bluetoothSettingKeys.js +2 -0
  233. package/build/stores/bluetoothSettingKeys.js.map +1 -1
  234. package/build/stores/gallerySync.d.ts +2 -0
  235. package/build/stores/gallerySync.d.ts.map +1 -1
  236. package/build/stores/gallerySync.js +2 -0
  237. package/build/stores/gallerySync.js.map +1 -1
  238. package/build/stores/glasses.d.ts.map +1 -1
  239. package/build/stores/glasses.js +3 -0
  240. package/build/stores/glasses.js.map +1 -1
  241. package/build/stores/settings.d.ts +17 -0
  242. package/build/stores/settings.d.ts.map +1 -1
  243. package/build/stores/settings.js +152 -19
  244. package/build/stores/settings.js.map +1 -1
  245. package/build/types/applet.d.ts +2 -0
  246. package/build/types/applet.d.ts.map +1 -1
  247. package/build/types/applet.js.map +1 -1
  248. package/build/types/enums.d.ts +1 -0
  249. package/build/types/enums.d.ts.map +1 -1
  250. package/build/types/enums.js +1 -0
  251. package/build/types/enums.js.map +1 -1
  252. package/build/types/hardware.d.ts +1 -0
  253. package/build/types/hardware.d.ts.map +1 -1
  254. package/build/types/hardware.js +14 -0
  255. package/build/types/hardware.js.map +1 -1
  256. package/build/utils/configuredDevHost.d.ts +6 -0
  257. package/build/utils/configuredDevHost.d.ts.map +1 -0
  258. package/build/utils/configuredDevHost.js +19 -0
  259. package/build/utils/configuredDevHost.js.map +1 -0
  260. package/build/utils/devMiniappLaunch.d.ts +19 -8
  261. package/build/utils/devMiniappLaunch.d.ts.map +1 -1
  262. package/build/utils/devMiniappLaunch.js +144 -21
  263. package/build/utils/devMiniappLaunch.js.map +1 -1
  264. package/build/utils/diagnosticContext.d.ts.map +1 -1
  265. package/build/utils/diagnosticContext.js +15 -17
  266. package/build/utils/diagnosticContext.js.map +1 -1
  267. package/build/utils/hardware/hardware.d.ts.map +1 -1
  268. package/build/utils/hardware/hardware.js +7 -3
  269. package/build/utils/hardware/hardware.js.map +1 -1
  270. package/build/utils/miniappDiagnostics.d.ts +64 -0
  271. package/build/utils/miniappDiagnostics.d.ts.map +1 -0
  272. package/build/utils/miniappDiagnostics.js +173 -0
  273. package/build/utils/miniappDiagnostics.js.map +1 -0
  274. package/build/utils/timers/timers.d.ts.map +1 -1
  275. package/build/utils/timers/timers.js +55 -18
  276. package/build/utils/timers/timers.js.map +1 -1
  277. package/package.json +27 -18
  278. package/src/assets/silence.wav +0 -0
  279. package/src/engine.ts +7 -2
  280. package/src/facades/dev.ts +9 -5
  281. package/src/facades/gallery.ts +1 -1
  282. package/src/facades/glasses.ts +17 -4
  283. package/src/facades/glassesSettings.ts +5 -0
  284. package/src/facades/miniapps.ts +7 -0
  285. package/src/facades/ota.ts +19 -4
  286. package/src/facades/pairing.ts +11 -7
  287. package/src/facades/phoneNotifications.ts +4 -13
  288. package/src/index.ts +26 -2
  289. package/src/internal.ts +2 -1
  290. package/src/react/MentraLiveOtaFlow.tsx +856 -0
  291. package/src/react/index.ts +7 -0
  292. package/src/react/useEngineSnapshot.ts +16 -0
  293. package/src/runtime/bootstrap.ts +33 -1
  294. package/src/runtime/scanQrSeam.ts +20 -0
  295. package/src/services/AppRegistry.ts +72 -48
  296. package/src/services/AudioCloudUplink.ts +9 -3
  297. package/src/services/AudioPlaybackService.ts +94 -34
  298. package/src/services/BlobStore.ts +22 -9
  299. package/src/services/CloudAudioSubscriptionSync.ts +13 -1
  300. package/src/services/CloudClientService.ts +74 -0
  301. package/src/services/ForegroundLocationPermission.ts +24 -0
  302. package/src/services/GlassesSettingsSync.ts +9 -3
  303. package/src/services/GlassesStatusProjection.ts +16 -8
  304. package/src/services/HotspotOtaTransport.ts +184 -0
  305. package/src/services/HotspotShutdown.ts +41 -0
  306. package/src/services/LocalDisplayManager.ts +112 -21
  307. package/src/services/LocalMiniappRuntime.ts +576 -130
  308. package/src/services/LocalMiniappStorage.ts +82 -0
  309. package/src/services/LocalMiniappUserIdentity.ts +46 -0
  310. package/src/services/MicStateCoordinator.ts +169 -4
  311. package/src/services/MiniappLauncher.ts +30 -3
  312. package/src/services/MiniappLiveness.ts +24 -0
  313. package/src/services/OtaArtifactDownloader.ts +312 -0
  314. package/src/services/OtaAutoChain.ts +106 -0
  315. package/src/services/OtaErrorMapping.ts +80 -0
  316. package/src/services/OtaInstallCoordinator.ts +645 -143
  317. package/src/services/OtaService.ts +11 -5
  318. package/src/services/OtaUpdateCheckService.ts +180 -20
  319. package/src/services/PhoneCalendarService.ts +27 -4
  320. package/src/services/PhoneCameraFovCoordinator.ts +15 -0
  321. package/src/services/PhoneNotificationsSync.ts +12 -10
  322. package/src/services/PhonePhotoCoordinator.ts +33 -7
  323. package/src/services/PhoneStreamCoordinator.ts +139 -25
  324. package/src/services/PhoneVideoCoordinator.ts +11 -0
  325. package/src/services/SceneRenderer.ts +16 -0
  326. package/src/services/SupportProfileSync.ts +218 -0
  327. package/src/services/TtsTextSanitizer.ts +66 -0
  328. package/src/services/WifiAdbRequest.ts +53 -0
  329. package/src/services/asg/asgCameraApi.ts +99 -8
  330. package/src/services/asg/gallerySyncService.ts +28 -19
  331. package/src/services/asg/localNetworkTransport.ts +7 -11
  332. package/src/services/audioPlaybackAssets.ts +8 -0
  333. package/src/services/glassesClockSync.ts +2 -2
  334. package/src/services/manifestActions.ts +30 -0
  335. package/src/services/otaDisplayState.ts +57 -16
  336. package/src/services/otaInstallPolicy.ts +29 -3
  337. package/src/services/{asgOtaVersionUrl.ts → otaManifestUrl.ts} +16 -4
  338. package/src/services/slimStreamStatus.ts +16 -5
  339. package/src/stores/apps.ts +12 -0
  340. package/src/stores/bluetoothSettingKeys.ts +2 -0
  341. package/src/stores/gallerySync.ts +4 -0
  342. package/src/stores/glasses.ts +3 -0
  343. package/src/stores/settings.ts +174 -21
  344. package/src/types/applet.ts +2 -0
  345. package/src/types/enums.ts +2 -1
  346. package/src/types/hardware.ts +17 -0
  347. package/src/utils/configuredDevHost.ts +17 -0
  348. package/src/utils/devMiniappLaunch.ts +151 -23
  349. package/src/utils/diagnosticContext.ts +17 -17
  350. package/src/utils/hardware/hardware.ts +7 -9
  351. package/src/utils/miniappDiagnostics.ts +226 -0
  352. package/src/utils/timers/timers.ts +57 -21
  353. package/build/services/asgOtaVersionUrl.d.ts +0 -2
  354. package/build/services/asgOtaVersionUrl.d.ts.map +0 -1
  355. package/build/services/asgOtaVersionUrl.js.map +0 -1
@@ -9,7 +9,7 @@
9
9
  * management (connect/disconnect/ping).
10
10
  */
11
11
 
12
- import {Linking} from "react-native"
12
+ import {AppState, Linking} from "react-native"
13
13
  import Share from "react-native-share"
14
14
  import * as Battery from "expo-battery"
15
15
  import * as Clipboard from "expo-clipboard"
@@ -51,6 +51,8 @@ import {phoneStreamCoordinator} from "./PhoneStreamCoordinator"
51
51
  import {phoneVideoCoordinator} from "./PhoneVideoCoordinator"
52
52
  import {runSentenceTtsPipeline} from "./SentenceTtsPipeline"
53
53
  import {summarizeTranscriptionRoutes, transcriptionDeliveryRoute} from "./TranscriptionRouting"
54
+ import {prepareTtsSentences} from "./TtsTextSanitizer"
55
+ import {handleWifiAdbRequest} from "./WifiAdbRequest"
54
56
  import {cloudClientService} from "./CloudClientService"
55
57
  import {miniappLauncher} from "./MiniappLauncher"
56
58
  import {
@@ -64,25 +66,38 @@ import {
64
66
  type TtsSynthesisResult,
65
67
  } from "../runtime/config"
66
68
  import {getAnalytics, getUiSeams} from "../runtime/bootstrap"
69
+ import {invokeScanQrSeam} from "../runtime/scanQrSeam"
67
70
  import {normalizeStreamAudioConfig, normalizeStreamVideoConfig} from "../runtime/streamConfig"
71
+ import {toLanguageHint} from "@mentra/cloud-protocol/languages"
68
72
  import type {AudioSubscription, LanguageSource, TranscriptionData, TranslationData} from "@mentra/cloud-protocol"
73
+ import {buildMiniappManifestSnapshot, type MiniappRuntimeDiagnosticSnapshot} from "../utils/miniappDiagnostics"
69
74
  import ttsModelManager from "./TTSModelManager"
70
75
  import {NavigationHandlers} from "./NavigationHandlers"
71
76
  import type {ClientApp} from "../types/applet"
72
77
  import {useAppStatusStore} from "../stores/apps"
73
78
  import {getDevAppAttestation, getDevAppSourcePackage} from "./AppRegistry"
79
+ import {resolveForegroundLocationPermission} from "./ForegroundLocationPermission"
80
+ import {advanceMiniappPingLiveness} from "./MiniappLiveness"
74
81
  import {listPhoneCalendarEvents, PhoneCalendarError} from "./PhoneCalendarService"
82
+ import {LocalMiniappStorage} from "./LocalMiniappStorage"
75
83
 
76
84
  // =============================================================================
77
85
  // Types
78
86
  // =============================================================================
79
87
 
80
88
  export interface InstalledMiniappManifest {
89
+ packageName?: string
81
90
  /** Human-facing app name (from miniapp.json `name`). Shown in the "Starting
82
91
  * <name>…" boot message; falls back to the package name when absent. */
83
92
  name?: string
93
+ version?: string
94
+ sdkVersion?: string
95
+ minHostVersion?: string
96
+ type?: string
97
+ entry?: {background?: string; ui?: string}
84
98
  permissions?: Array<{type: string; required?: boolean; description?: string}>
85
99
  hardwareRequirements?: Array<{type: string; level: string; description?: string}>
100
+ actions?: Array<{id?: unknown; description?: unknown; parameters?: unknown; outputSchema?: unknown}>
86
101
  }
87
102
 
88
103
  type SpeakerStateValue = "idle" | "loading" | "playing" | "stopped" | "error"
@@ -95,6 +110,7 @@ interface ConnectedMiniapp {
95
110
  cloudTranscriptionStreams: Set<string>
96
111
  sendMessage: (raw: string) => void
97
112
  lastPongAt: number
113
+ unansweredPingRounds: number
98
114
  installedManifest?: InstalledMiniappManifest
99
115
  authRefreshTimerId: number | null
100
116
  /**
@@ -168,6 +184,16 @@ function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | nul
168
184
  }
169
185
 
170
186
  const LOG_TAG = "LOCAL_MINIAPP"
187
+ const DIAGNOSTIC_MAX_LIST_ITEMS = 100
188
+ const DIAGNOSTIC_MAX_STRING_LENGTH = 512
189
+
190
+ function diagnosticStringList(values: Iterable<string>): string[] {
191
+ return [...values]
192
+ .map((value) => value.slice(0, DIAGNOSTIC_MAX_STRING_LENGTH))
193
+ .sort()
194
+ .slice(0, DIAGNOSTIC_MAX_LIST_ITEMS)
195
+ }
196
+
171
197
  const SYSTEM_MINIAPP_PACKAGES = new Set([
172
198
  "com.mentra.camera",
173
199
  "com.mentra.gallery",
@@ -205,10 +231,29 @@ const RGB_LED_ACTIONS = new Set<RgbLedAction>(["on", "off"])
205
231
  const RGB_LED_COLORS = new Set<RgbLedColor>(["red", "green", "blue", "orange", "white"])
206
232
  const CAMERA_FOV_MIN = 62
207
233
  const CAMERA_FOV_MAX = 118
208
- const CAMERA_FOV_DEFAULT = 102
209
- const CAMERA_FOV_PRESETS: Record<CameraFovPreset, number> = {narrow: 82, standard: CAMERA_FOV_DEFAULT, wide: 118}
234
+ const CAMERA_FOV_DEFAULT = 118
235
+ const CAMERA_FOV_PRESETS: Record<CameraFovPreset, number> = {narrow: 82, standard: 102, wide: 118}
210
236
  const CAMERA_ROI_POSITION_BY_NAME: Record<string, CameraRoiPosition> = {center: "center", bottom: "bottom", top: "top"}
211
237
 
238
+ const localMiniappStorageBackend = {
239
+ get(key: string): unknown | null {
240
+ const result = mmkvStorage.load<unknown>(key)
241
+ return result.is_ok() ? result.value : null
242
+ },
243
+ has(key: string): boolean {
244
+ return mmkvStorage.load<unknown>(key).is_ok()
245
+ },
246
+ set(key: string, value: unknown): void {
247
+ mmkvStorage.save(key, value)
248
+ },
249
+ remove(key: string): void {
250
+ mmkvStorage.remove(key)
251
+ },
252
+ keys(): string[] {
253
+ return mmkvStorage.getAllKeys()
254
+ },
255
+ }
256
+
212
257
  // =============================================================================
213
258
  // Declared-permission record helper (for CONNECT_ACK / PERMISSIONS_UPDATE)
214
259
  // =============================================================================
@@ -350,11 +395,22 @@ class LocalMiniappRuntime {
350
395
  /** Ref-counted stream subscriptions: stream → set of packageNames. */
351
396
  private streamSubscribers: Map<string, Set<string>> = new Map()
352
397
 
398
+ /** Host observers for the Mentra Live button-capture policy. */
399
+ private buttonPressSubscriberListeners = new Set<(packageNames: string[]) => void>()
400
+
353
401
  /** Guards one-time wiring of the cloud transcript/translation fan-out. */
354
402
  private cloudResultsWired = false
355
403
  private cloudStatusWired = false
356
404
  private cloudAudioSubscriptionSync = new CloudAudioSubscriptionSync()
357
405
 
406
+ /**
407
+ * Per-miniapp language hints from TRANSCRIPTION_CONFIG (issue 021 WP3).
408
+ * Bare ISO 639-1 codes, already registry-validated by the SDK and
409
+ * re-normalized here (defense in depth). Merged (union) into the auto-mode
410
+ * cloud transcription subscription in buildCloudAudioSubscriptions.
411
+ */
412
+ private transcriptionHintsByApp = new Map<string, string[]>()
413
+
358
414
  /** Ping interval handle. */
359
415
  private pingIntervalId: number | null = null
360
416
  private foregroundProbeTimers: Map<string, number> = new Map()
@@ -381,6 +437,99 @@ class LocalMiniappRuntime {
381
437
 
382
438
  private constructor() {}
383
439
 
440
+ /** Snapshot of miniapps actively subscribed to hardware button events. */
441
+ public getButtonPressSubscribers(): string[] {
442
+ return [...(this.streamSubscribers.get(MiniappStreamType.BUTTON_PRESS) ?? [])].sort()
443
+ }
444
+
445
+ /** Observe active hardware-button subscriptions. */
446
+ public onButtonPressSubscribersChanged(listener: (packageNames: string[]) => void): () => void {
447
+ this.buttonPressSubscriberListeners.add(listener)
448
+ return () => this.buttonPressSubscriberListeners.delete(listener)
449
+ }
450
+
451
+ /** Report-safe live miniapp/subscription/resource snapshot. */
452
+ public getDiagnosticSnapshot(nowMs = Date.now()): MiniappRuntimeDiagnosticSnapshot {
453
+ const connectedApps = [...this.connectedApps.entries()]
454
+ .sort(([a], [b]) => a.localeCompare(b))
455
+ .slice(0, DIAGNOSTIC_MAX_LIST_ITEMS)
456
+ .map(([packageName, app]) => ({
457
+ packageName: packageName.slice(0, DIAGNOSTIC_MAX_STRING_LENGTH),
458
+ handshakeComplete: this.handshookApps.has(packageName),
459
+ subscriptions: diagnosticStringList(app.subscriptions),
460
+ lastMessageAgeMs: Math.max(0, nowMs - app.lastPongAt),
461
+ requestedLocationRate: app.requestedLocationRate,
462
+ hasActiveSpeakerStream: app.activeSpeakerStreamId !== undefined,
463
+ ...(app.installedManifest
464
+ ? {
465
+ manifest: buildMiniappManifestSnapshot(app.installedManifest, {
466
+ packageName,
467
+ name: app.installedManifest.name,
468
+ version: app.installedManifest.version,
469
+ type: app.installedManifest.type,
470
+ }),
471
+ }
472
+ : {}),
473
+ }))
474
+ const subscriptionsByStream = Object.fromEntries(
475
+ [...this.streamSubscribers.entries()]
476
+ .filter(([, packageNames]) => packageNames.size > 0)
477
+ .sort(([a], [b]) => a.localeCompare(b))
478
+ .slice(0, DIAGNOSTIC_MAX_LIST_ITEMS)
479
+ .map(([stream, packageNames]) => [
480
+ stream.slice(0, DIAGNOSTIC_MAX_STRING_LENGTH),
481
+ diagnosticStringList(packageNames),
482
+ ]),
483
+ )
484
+
485
+ return {
486
+ connectedApps,
487
+ subscriptionsByStream,
488
+ resources: {
489
+ display: localDisplayManager.getDiagnosticSnapshot(),
490
+ camera: phonePhotoCoordinator.getDiagnosticSnapshot(),
491
+ video: phoneVideoCoordinator.getDiagnosticSnapshot(),
492
+ stream: phoneStreamCoordinator.getDiagnosticSnapshot(),
493
+ cameraFov: phoneCameraFovCoordinator.getDiagnosticSnapshot(),
494
+ },
495
+ }
496
+ }
497
+
498
+ private replaceStreamSubscribers(
499
+ packageName: string,
500
+ previousStreams: Iterable<string>,
501
+ nextStreams: Iterable<string>,
502
+ ) {
503
+ const previousButtonSubscribers = this.getButtonPressSubscribers().join("\u0000")
504
+
505
+ for (const stream of previousStreams) {
506
+ const subscribers = this.streamSubscribers.get(stream)
507
+ if (!subscribers) continue
508
+ subscribers.delete(packageName)
509
+ if (subscribers.size === 0) this.streamSubscribers.delete(stream)
510
+ }
511
+
512
+ for (const stream of nextStreams) {
513
+ let subscribers = this.streamSubscribers.get(stream)
514
+ if (!subscribers) {
515
+ subscribers = new Set()
516
+ this.streamSubscribers.set(stream, subscribers)
517
+ }
518
+ subscribers.add(packageName)
519
+ }
520
+
521
+ const nextButtonSubscribers = this.getButtonPressSubscribers()
522
+ if (previousButtonSubscribers !== nextButtonSubscribers.join("\u0000")) {
523
+ for (const listener of this.buttonPressSubscriberListeners) {
524
+ try {
525
+ listener(nextButtonSubscribers)
526
+ } catch (error) {
527
+ console.warn(`${LOG_TAG}: button press subscriber listener threw:`, error)
528
+ }
529
+ }
530
+ }
531
+ }
532
+
384
533
  /**
385
534
  * Generate an HMAC-signed local session token for browser fallback auth.
386
535
  * Token format: base64(JSON({userId, packageName, exp})).base64(HMAC-SHA256(payload, secret))
@@ -606,13 +755,7 @@ class LocalMiniappRuntime {
606
755
  BgTimer.clearTimeout(existing.authRetryTimerId)
607
756
  existing.authRetryTimerId = null
608
757
  }
609
- for (const stream of existing.subscriptions) {
610
- const subs = this.streamSubscribers.get(stream)
611
- if (subs) {
612
- subs.delete(packageName)
613
- if (subs.size === 0) this.streamSubscribers.delete(stream)
614
- }
615
- }
758
+ this.replaceStreamSubscribers(packageName, existing.subscriptions, [])
616
759
  this.recomputeMicRequirements()
617
760
  this.updateCloudSubscriptions()
618
761
  }
@@ -622,6 +765,7 @@ class LocalMiniappRuntime {
622
765
  cloudTranscriptionStreams: new Set(),
623
766
  sendMessage: sendFn,
624
767
  lastPongAt: Date.now(),
768
+ unansweredPingRounds: 0,
625
769
  installedManifest,
626
770
  authRefreshTimerId: null,
627
771
  authRetryTimerId: null,
@@ -742,6 +886,7 @@ class LocalMiniappRuntime {
742
886
 
743
887
  public unregisterApp(packageName: string): void {
744
888
  console.log(`${LOG_TAG}: unregisterApp(${packageName})`)
889
+ const releasedMicGateOverride = micStateCoordinator.clearMiniappGateOverrides(packageName)
745
890
  this.clearForegroundProbe(packageName)
746
891
  this.clearMiniappAuthRefresh(packageName)
747
892
  this.clearMiniappAuthDeliveryRetry(packageName)
@@ -750,6 +895,9 @@ class LocalMiniappRuntime {
750
895
  // the early-return so it runs even if the connectedApps entry is already gone.
751
896
  this.handshookApps.delete(packageName)
752
897
  this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`))
898
+ // Drop this app's transcription hints; the cloud subscription rebuild
899
+ // below (via replaceStreamSubscribers) re-derives the hint union without it.
900
+ this.transcriptionHintsByApp.delete(packageName)
753
901
 
754
902
  // Warm-up is a request-owned camera lease. Cancel it even if the app record was already
755
903
  // removed so a close racing the camera-open promise cannot leave the sensor running.
@@ -766,19 +914,18 @@ class LocalMiniappRuntime {
766
914
  console.warn(`${LOG_TAG}: failed to release camera FOV override for ${packageName} on unregister`, error)
767
915
  })
768
916
  const app = this.connectedApps.get(packageName)
769
- if (!app) return
770
-
771
- // Remove from all stream subscriber sets
772
- for (const stream of app.subscriptions) {
773
- const subs = this.streamSubscribers.get(stream)
774
- if (subs) {
775
- subs.delete(packageName)
776
- if (subs.size === 0) {
777
- this.streamSubscribers.delete(stream)
778
- }
917
+ if (!app) {
918
+ if (releasedMicGateOverride) {
919
+ void micStateCoordinator.syncEffectiveGatePolicy(this.getConfiguredMicGates()).catch((error) => {
920
+ console.warn(`${LOG_TAG}: failed to restore mic gate policy for ${packageName}`, error)
921
+ })
779
922
  }
923
+ return
780
924
  }
781
925
 
926
+ // Remove from all stream subscriber sets
927
+ this.replaceStreamSubscribers(packageName, app.subscriptions, [])
928
+
782
929
  // Reset per-session warning dedup so a relaunch surfaces issues again.
783
930
  resetPermissionWarnings(packageName)
784
931
 
@@ -844,6 +991,11 @@ class LocalMiniappRuntime {
844
991
 
845
992
  this.connectedApps.delete(packageName)
846
993
  this.recomputeMicRequirements()
994
+ if (releasedMicGateOverride) {
995
+ void micStateCoordinator.syncEffectiveGatePolicy(this.getConfiguredMicGates()).catch((error) => {
996
+ console.warn(`${LOG_TAG}: failed to restore mic gate policy for ${packageName}`, error)
997
+ })
998
+ }
847
999
  this.updateCloudSubscriptions()
848
1000
 
849
1001
  if (this.connectedApps.size === 0) {
@@ -1017,9 +1169,18 @@ class LocalMiniappRuntime {
1017
1169
  case MiniappRequestType.CAMERA_FOV:
1018
1170
  void this.handleCameraFov(packageName, payload, requestId)
1019
1171
  break
1172
+ case MiniappRequestType.SET_WIFI_ADB_STATE:
1173
+ void this.handleSetWifiAdbState(packageName, payload, requestId)
1174
+ break
1020
1175
  case MiniappRequestType.IMU_SET_ENABLED:
1021
1176
  this.handleImuSetEnabled(packageName, payload, requestId)
1022
1177
  break
1178
+ case MiniappRequestType.MIC_SET_VAD_ENABLED:
1179
+ void this.handleMicSetVadEnabled(packageName, payload, requestId)
1180
+ break
1181
+ case MiniappRequestType.MIC_SET_LOUDNESS_GATE_ENABLED:
1182
+ void this.handleMicSetLoudnessGateEnabled(packageName, payload, requestId)
1183
+ break
1023
1184
  case MiniappRequestType.PING:
1024
1185
  // SDK should handle this itself; reply PONG just in case
1025
1186
  this.sendToMiniapp(packageName, {type: MiniappResponseType.PONG}, requestId)
@@ -1042,52 +1203,83 @@ class LocalMiniappRuntime {
1042
1203
  case MiniappRequestType.DOWNLOAD:
1043
1204
  this.handleDownload(packageName, payload, requestId)
1044
1205
  break
1206
+ case MiniappRequestType.SCAN_QR:
1207
+ void this.handleScanQr(packageName, payload, requestId)
1208
+ break
1045
1209
 
1046
1210
  // Persistent binary blob storage (session.blob)
1047
1211
  case MiniappRequestType.BLOB_CREATE:
1048
- this.blobStore.handleCreate(packageName, payload, requestId)
1212
+ this.enqueueBlobRequest(packageName, requestId, () =>
1213
+ this.blobStore.handleCreate(packageName, payload, requestId),
1214
+ )
1049
1215
  break
1050
1216
  case MiniappRequestType.BLOB_WRITE:
1051
- this.blobStore.handleWrite(packageName, payload, requestId)
1217
+ this.enqueueBlobRequest(packageName, requestId, () =>
1218
+ this.blobStore.handleWrite(packageName, payload, requestId),
1219
+ )
1052
1220
  break
1053
1221
  case MiniappRequestType.BLOB_COMMIT:
1054
- this.blobStore.handleCommit(packageName, payload, requestId)
1222
+ this.enqueueBlobRequest(packageName, requestId, () =>
1223
+ this.blobStore.handleCommit(packageName, payload, requestId),
1224
+ )
1055
1225
  break
1056
1226
  case MiniappRequestType.BLOB_ABORT:
1057
- this.blobStore.handleAbort(packageName, payload, requestId)
1227
+ this.enqueueBlobRequest(packageName, requestId, () =>
1228
+ this.blobStore.handleAbort(packageName, payload, requestId),
1229
+ )
1058
1230
  break
1059
1231
  case MiniappRequestType.BLOB_SET_FROM_URL:
1060
- void this.blobStore.handleSetFromUrl(packageName, payload, requestId)
1232
+ this.enqueueBlobRequest(packageName, requestId, () =>
1233
+ this.blobStore.handleSetFromUrl(packageName, payload, requestId),
1234
+ )
1061
1235
  break
1062
1236
  case MiniappRequestType.BLOB_IMPORT:
1063
- void this.blobStore.handleImport(packageName, payload, requestId)
1237
+ this.enqueueBlobRequest(packageName, requestId, () =>
1238
+ this.blobStore.handleImport(packageName, payload, requestId),
1239
+ )
1064
1240
  break
1065
1241
  case MiniappRequestType.BLOB_GET:
1066
- this.blobStore.handleGet(packageName, payload, requestId)
1242
+ this.enqueueBlobRequest(packageName, requestId, () => this.blobStore.handleGet(packageName, payload, requestId))
1067
1243
  break
1068
1244
  case MiniappRequestType.BLOB_LIST:
1069
- this.blobStore.handleList(packageName, payload, requestId)
1245
+ this.enqueueBlobRequest(packageName, requestId, () =>
1246
+ this.blobStore.handleList(packageName, payload, requestId),
1247
+ )
1070
1248
  break
1071
1249
  case MiniappRequestType.BLOB_USAGE:
1072
- this.blobStore.handleUsage(packageName, payload, requestId)
1250
+ this.enqueueBlobRequest(packageName, requestId, () =>
1251
+ this.blobStore.handleUsage(packageName, payload, requestId),
1252
+ )
1073
1253
  break
1074
1254
  case MiniappRequestType.BLOB_DELETE:
1075
- this.blobStore.handleDelete(packageName, payload, requestId)
1255
+ this.enqueueBlobRequest(packageName, requestId, () =>
1256
+ this.blobStore.handleDelete(packageName, payload, requestId),
1257
+ )
1076
1258
  break
1077
1259
  case MiniappRequestType.BLOB_CLEAR:
1078
- this.blobStore.handleClear(packageName, payload, requestId)
1260
+ this.enqueueBlobRequest(packageName, requestId, () =>
1261
+ this.blobStore.handleClear(packageName, payload, requestId),
1262
+ )
1079
1263
  break
1080
1264
  case MiniappRequestType.BLOB_OPEN_READ:
1081
- this.blobStore.handleOpenRead(packageName, payload, requestId)
1265
+ this.enqueueBlobRequest(packageName, requestId, () =>
1266
+ this.blobStore.handleOpenRead(packageName, payload, requestId),
1267
+ )
1082
1268
  break
1083
1269
  case MiniappRequestType.BLOB_READ:
1084
- this.blobStore.handleRead(packageName, payload, requestId)
1270
+ this.enqueueBlobRequest(packageName, requestId, () =>
1271
+ this.blobStore.handleRead(packageName, payload, requestId),
1272
+ )
1085
1273
  break
1086
1274
  case MiniappRequestType.BLOB_CLOSE_READ:
1087
- this.blobStore.handleCloseRead(packageName, payload, requestId)
1275
+ this.enqueueBlobRequest(packageName, requestId, () =>
1276
+ this.blobStore.handleCloseRead(packageName, payload, requestId),
1277
+ )
1088
1278
  break
1089
1279
  case MiniappRequestType.BLOB_SHARE:
1090
- void this.blobStore.handleShare(packageName, payload, requestId)
1280
+ this.enqueueBlobRequest(packageName, requestId, () =>
1281
+ this.blobStore.handleShare(packageName, payload, requestId),
1282
+ )
1091
1283
  break
1092
1284
 
1093
1285
  // Cloud-coordinated features
@@ -1144,8 +1336,20 @@ class LocalMiniappRuntime {
1144
1336
  })
1145
1337
  break
1146
1338
 
1339
+ case MiniappRequestType.TRANSCRIPTION_CONFIG:
1340
+ this.handleTranscriptionConfig(packageName, payload, requestId)
1341
+ break
1342
+
1147
1343
  default:
1344
+ // NACK instead of silent drop (issue 021 S3): an SDK that sends a
1345
+ // request this runtime doesn't implement must get a rejected promise,
1346
+ // not a permanently-pending silent gap. One-shots (no requestId) can
1347
+ // only be logged.
1148
1348
  console.warn(`${LOG_TAG}: Unknown request type from ${packageName}: ${requestType}`)
1349
+ this.sendResult(packageName, requestId, false, undefined, {
1350
+ code: MiniappErrorCode.NOT_IMPLEMENTED,
1351
+ message: `Unknown or unimplemented request type: ${requestType}`,
1352
+ })
1149
1353
  break
1150
1354
  }
1151
1355
  }
@@ -1170,6 +1374,7 @@ class LocalMiniappRuntime {
1170
1374
 
1171
1375
  // Update lastPongAt so it doesn't time out right away
1172
1376
  existing.lastPongAt = Date.now()
1377
+ existing.unansweredPingRounds = 0
1173
1378
 
1174
1379
  // Read current glasses capabilities from the settings store
1175
1380
  const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable) as
@@ -1377,6 +1582,51 @@ class LocalMiniappRuntime {
1377
1582
  }
1378
1583
  }
1379
1584
 
1585
+ /**
1586
+ * TRANSCRIPTION_CONFIG (issue 021 WP3): store the miniapp's language hints
1587
+ * and rebuild the cloud subscription set so the auto-mode subscription
1588
+ * carries them. Replies with a REQUEST_RESULT so the SDK's configure()
1589
+ * promise settles: unknown hint values are an error, not a silent drop.
1590
+ */
1591
+ private handleTranscriptionConfig(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1592
+ const app = this.connectedApps.get(packageName)
1593
+ if (!app) return
1594
+
1595
+ const config = (payload.config ?? {}) as Record<string, unknown>
1596
+ const rawHints = config.languageHints
1597
+ if (rawHints !== undefined && !Array.isArray(rawHints)) {
1598
+ this.sendResult(packageName, requestId, false, undefined, {
1599
+ code: MiniappErrorCode.INTERNAL,
1600
+ message: "languageHints must be an array of language codes",
1601
+ })
1602
+ return
1603
+ }
1604
+
1605
+ if (rawHints === undefined || rawHints.length === 0) {
1606
+ this.transcriptionHintsByApp.delete(packageName)
1607
+ } else {
1608
+ // Defense in depth: the SDK already validated, but the wire is untyped.
1609
+ const hints: string[] = []
1610
+ const invalid: string[] = []
1611
+ for (const raw of rawHints as unknown[]) {
1612
+ const hint = typeof raw === "string" ? toLanguageHint(raw) : null
1613
+ if (hint) hints.push(hint)
1614
+ else invalid.push(String(raw))
1615
+ }
1616
+ if (invalid.length > 0) {
1617
+ this.sendResult(packageName, requestId, false, undefined, {
1618
+ code: MiniappErrorCode.INTERNAL,
1619
+ message: `Unknown language hint(s): ${invalid.join(", ")}`,
1620
+ })
1621
+ return
1622
+ }
1623
+ this.transcriptionHintsByApp.set(packageName, [...new Set(hints)].sort())
1624
+ }
1625
+
1626
+ this.updateCloudSubscriptions()
1627
+ this.sendResult(packageName, requestId, true, {applied: true})
1628
+ }
1629
+
1380
1630
  private handleSubscribe(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
1381
1631
  const app = this.connectedApps.get(packageName)
1382
1632
  if (!app) return
@@ -1450,18 +1700,8 @@ class LocalMiniappRuntime {
1450
1700
  }
1451
1701
  }
1452
1702
 
1453
- // Remove old subscriptions for this app
1454
- for (const oldStream of app.subscriptions) {
1455
- const subs = this.streamSubscribers.get(oldStream)
1456
- if (subs) {
1457
- subs.delete(packageName)
1458
- if (subs.size === 0) {
1459
- this.streamSubscribers.delete(oldStream)
1460
- }
1461
- }
1462
- }
1463
-
1464
1703
  // Add new subscriptions
1704
+ const previousSubscriptions = app.subscriptions
1465
1705
  app.subscriptions = new Set(streams)
1466
1706
  app.forceLocalTranscriptionStreams = new Set(
1467
1707
  [...forceLocalTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
@@ -1469,14 +1709,7 @@ class LocalMiniappRuntime {
1469
1709
  app.cloudTranscriptionStreams = new Set(
1470
1710
  [...cloudTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
1471
1711
  )
1472
- for (const stream of streams) {
1473
- let subs = this.streamSubscribers.get(stream)
1474
- if (!subs) {
1475
- subs = new Set()
1476
- this.streamSubscribers.set(stream, subs)
1477
- }
1478
- subs.add(packageName)
1479
- }
1712
+ this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
1480
1713
 
1481
1714
  this.recomputeMicRequirements()
1482
1715
  this.updateCloudSubscriptions()
@@ -1966,16 +2199,21 @@ class LocalMiniappRuntime {
1966
2199
 
1967
2200
  private async handleSpeak(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
1968
2201
  const rawText = payload.text
1969
- const sentences =
2202
+ const rawSentences =
1970
2203
  typeof rawText === "string"
1971
2204
  ? [rawText.trim()].filter(Boolean)
1972
2205
  : Array.isArray(rawText)
1973
- ? rawText.filter((sentence): sentence is string => typeof sentence === "string").map((sentence) => sentence.trim()).filter(Boolean)
2206
+ ? rawText
2207
+ .filter((sentence): sentence is string => typeof sentence === "string")
2208
+ .map((sentence) => sentence.trim())
2209
+ .filter(Boolean)
1974
2210
  : []
2211
+ const enableSanitization = payload.enableSanitization !== false
2212
+ const sentences = prepareTtsSentences(rawSentences, enableSanitization)
1975
2213
  if (sentences.length === 0) {
1976
2214
  this.sendResult(packageName, requestId, false, undefined, {
1977
2215
  code: MiniappErrorCode.INTERNAL,
1978
- message: "speak requires text or a non-empty sentence list",
2216
+ message: "speak requires text or a non-empty sentence list with speakable content",
1979
2217
  })
1980
2218
  return
1981
2219
  }
@@ -2095,7 +2333,6 @@ class LocalMiniappRuntime {
2095
2333
  appId: packageName,
2096
2334
  volume,
2097
2335
  stopOtherAudio,
2098
- suppressCloudUplink: true,
2099
2336
  },
2100
2337
  (_responseId, success, error, duration, completionReason) => {
2101
2338
  if (run.playbackRequestId === sentenceRequestId) run.playbackRequestId = undefined
@@ -2144,7 +2381,6 @@ class LocalMiniappRuntime {
2144
2381
  appId: packageName,
2145
2382
  volume,
2146
2383
  stopOtherAudio,
2147
- suppressCloudUplink: true,
2148
2384
  },
2149
2385
  (_respId, success, error, duration, completionReason) => {
2150
2386
  if (run.playbackRequestId === audioRequestId) run.playbackRequestId = undefined
@@ -2197,7 +2433,6 @@ class LocalMiniappRuntime {
2197
2433
  appId: packageName,
2198
2434
  volume,
2199
2435
  stopOtherAudio,
2200
- suppressCloudUplink: true,
2201
2436
  },
2202
2437
  (_respId, success, error, duration, completionReason) => {
2203
2438
  if (run.playbackRequestId === audioRequestId) run.playbackRequestId = undefined
@@ -2292,7 +2527,17 @@ class LocalMiniappRuntime {
2292
2527
 
2293
2528
  private async handleLocationPoll(packageName: string, requestId?: string): Promise<void> {
2294
2529
  try {
2295
- const {status} = await Location.requestForegroundPermissionsAsync()
2530
+ // Expo's Android requestForegroundPermissionsAsync always delegates to
2531
+ // Activity.requestPermissions(), even when permission is already granted.
2532
+ // Calling it after the user presses Home therefore waits until the Activity
2533
+ // resumes. Read the current grant first, and only enter the prompt flow
2534
+ // while the Activity is actually foregrounded.
2535
+ const permissionStartedAt = Date.now()
2536
+ const permission = await resolveForegroundLocationPermission(Location, () => AppState.currentState)
2537
+ const {status} = permission
2538
+ console.log(
2539
+ `${LOG_TAG}: location poll permission status=${status} appState=${AppState.currentState} elapsed=${Date.now() - permissionStartedAt}ms`,
2540
+ )
2296
2541
  if (status !== "granted") {
2297
2542
  this.sendResult(packageName, requestId, false, undefined, {
2298
2543
  code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
@@ -2305,8 +2550,11 @@ class LocalMiniappRuntime {
2305
2550
  // a fresh fix if the cache is empty. Use Low accuracy on the fresh
2306
2551
  // path so we get a cell/wifi-tower fix in ~1s instead of waiting
2307
2552
  // for full GPS warm-up.
2553
+ const locationStartedAt = Date.now()
2308
2554
  const cached = await Location.getLastKnownPositionAsync({maxAge: 60_000})
2555
+ console.log(`${LOG_TAG}: location poll cache hit=${cached !== null} elapsed=${Date.now() - locationStartedAt}ms`)
2309
2556
  const location = cached ?? (await Location.getCurrentPositionAsync({accuracy: Location.Accuracy.Low}))
2557
+ console.log(`${LOG_TAG}: location poll resolved elapsed=${Date.now() - locationStartedAt}ms`)
2310
2558
 
2311
2559
  this.sendResult(packageName, requestId, true, {
2312
2560
  lat: location.coords.latitude,
@@ -2348,9 +2596,69 @@ class LocalMiniappRuntime {
2348
2596
  private readonly blobStore = new BlobStore({
2349
2597
  sendResult: (packageName, requestId, ok, result, error) =>
2350
2598
  this.sendResult(packageName, requestId, ok, result, error),
2351
- getUserId: () =>
2352
- (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous",
2599
+ getUserId: () => cloudClientService.getMentraUserId(),
2353
2600
  })
2601
+ private readonly blobRequestQueues = new Map<string, Promise<void>>()
2602
+
2603
+ private readonly simpleStorage = new LocalMiniappStorage({
2604
+ backend: localMiniappStorageBackend,
2605
+ getUserId: () => cloudClientService.resolveMentraUserId(),
2606
+ })
2607
+
2608
+ /**
2609
+ * BlobStore's file helpers are synchronous once a user namespace is selected,
2610
+ * but first-boot Core auth is not. Queue requests per app behind stable
2611
+ * identity resolution so CREATE/WRITE/COMMIT ordering survives that wait.
2612
+ */
2613
+ private enqueueBlobRequest(
2614
+ packageName: string,
2615
+ requestId: string | undefined,
2616
+ operation: () => Promise<void> | void,
2617
+ ): void {
2618
+ const app = this.connectedApps.get(packageName)
2619
+ const previous = this.blobRequestQueues.get(packageName) ?? Promise.resolve()
2620
+ const queued = previous.then(async () => {
2621
+ try {
2622
+ await cloudClientService.resolveMentraUserId()
2623
+ } catch (err) {
2624
+ console.error(`${LOG_TAG}: blob identity error:`, err)
2625
+ this.sendResult(packageName, requestId, false, undefined, {
2626
+ code: MiniappErrorCode.NOT_CONNECTED,
2627
+ message: "Blob storage is unavailable before sign-in completes",
2628
+ })
2629
+ return
2630
+ }
2631
+
2632
+ // unregisterApp() or a crash-respawn may have replaced this app while
2633
+ // identity resolution was pending. Never replay an old context's request
2634
+ // against its replacement.
2635
+ if (!app || this.connectedApps.get(packageName) !== app) return
2636
+
2637
+ // Invoke in request order, but do not keep the queue occupied for the
2638
+ // lifetime of an async picker, download, or share sheet. CREATE/WRITE/
2639
+ // COMMIT are synchronous and therefore still dispatch strictly in order.
2640
+ try {
2641
+ const pending = operation()
2642
+ if (pending) {
2643
+ void pending.catch((err) => this.handleBlobRequestError(packageName, requestId, err))
2644
+ }
2645
+ } catch (err) {
2646
+ this.handleBlobRequestError(packageName, requestId, err)
2647
+ }
2648
+ })
2649
+ this.blobRequestQueues.set(packageName, queued)
2650
+ void queued.finally(() => {
2651
+ if (this.blobRequestQueues.get(packageName) === queued) this.blobRequestQueues.delete(packageName)
2652
+ })
2653
+ }
2654
+
2655
+ private handleBlobRequestError(packageName: string, requestId: string | undefined, err: unknown): void {
2656
+ console.error(`${LOG_TAG}: blob request error:`, err)
2657
+ this.sendResult(packageName, requestId, false, undefined, {
2658
+ code: MiniappErrorCode.INTERNAL,
2659
+ message: err instanceof Error ? err.message : "Blob storage error",
2660
+ })
2661
+ }
2354
2662
 
2355
2663
  /**
2356
2664
  * Heading is a sensor stream — start the native compass when any mini
@@ -2427,12 +2735,6 @@ class LocalMiniappRuntime {
2427
2735
  // Storage helpers
2428
2736
  // ---------------------------------------------------------------------------
2429
2737
 
2430
- private getStorageKeyPrefix(packageName: string): string {
2431
- const userId =
2432
- (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.coreToken) as string | undefined) || "anonymous"
2433
- return `mentraos_localstorage_${userId}_${packageName}_`
2434
- }
2435
-
2436
2738
  private async handleStorageGet(
2437
2739
  packageName: string,
2438
2740
  payload: Record<string, unknown>,
@@ -2447,9 +2749,8 @@ class LocalMiniappRuntime {
2447
2749
  })
2448
2750
  return
2449
2751
  }
2450
- const fullKey = this.getStorageKeyPrefix(packageName) + key
2451
- const result = mmkvStorage.load<unknown>(fullKey)
2452
- this.sendResult(packageName, requestId, true, {key, value: result.is_ok() ? result.value : null})
2752
+ const value = await this.simpleStorage.get(packageName, key)
2753
+ this.sendResult(packageName, requestId, true, {key, value})
2453
2754
  } catch (err) {
2454
2755
  console.error(`${LOG_TAG}: storage_get error:`, err)
2455
2756
  this.sendResult(packageName, requestId, false, undefined, {
@@ -2473,8 +2774,7 @@ class LocalMiniappRuntime {
2473
2774
  })
2474
2775
  return
2475
2776
  }
2476
- const fullKey = this.getStorageKeyPrefix(packageName) + key
2477
- mmkvStorage.save(fullKey, payload.value ?? null)
2777
+ await this.simpleStorage.set(packageName, key, payload.value ?? null)
2478
2778
  this.sendResult(packageName, requestId, true)
2479
2779
  } catch (err) {
2480
2780
  console.error(`${LOG_TAG}: storage_set error:`, err)
@@ -2499,8 +2799,7 @@ class LocalMiniappRuntime {
2499
2799
  })
2500
2800
  return
2501
2801
  }
2502
- const fullKey = this.getStorageKeyPrefix(packageName) + key
2503
- mmkvStorage.remove(fullKey)
2802
+ await this.simpleStorage.delete(packageName, key)
2504
2803
  this.sendResult(packageName, requestId, true)
2505
2804
  } catch (err) {
2506
2805
  console.error(`${LOG_TAG}: storage_delete error:`, err)
@@ -2517,9 +2816,7 @@ class LocalMiniappRuntime {
2517
2816
  requestId?: string,
2518
2817
  ): Promise<void> {
2519
2818
  try {
2520
- const prefix = this.getStorageKeyPrefix(packageName)
2521
- const allKeys = mmkvStorage.getAllKeys()
2522
- const keys = allKeys.filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length))
2819
+ const keys = await this.simpleStorage.keys(packageName)
2523
2820
  this.sendResult(packageName, requestId, true, {keys})
2524
2821
  } catch (err) {
2525
2822
  console.error(`${LOG_TAG}: storage_list error:`, err)
@@ -2532,11 +2829,7 @@ class LocalMiniappRuntime {
2532
2829
 
2533
2830
  private async handleStorageClear(packageName: string, requestId?: string): Promise<void> {
2534
2831
  try {
2535
- const prefix = this.getStorageKeyPrefix(packageName)
2536
- const allKeys = mmkvStorage.getAllKeys()
2537
- for (const k of allKeys) {
2538
- if (k.startsWith(prefix)) mmkvStorage.remove(k)
2539
- }
2832
+ await this.simpleStorage.clear(packageName)
2540
2833
  this.sendResult(packageName, requestId, true)
2541
2834
  } catch (err) {
2542
2835
  console.error(`${LOG_TAG}: storage_clear error:`, err)
@@ -2561,9 +2854,8 @@ class LocalMiniappRuntime {
2561
2854
  })
2562
2855
  return
2563
2856
  }
2564
- const fullKey = this.getStorageKeyPrefix(packageName) + key
2565
- const result = mmkvStorage.load<unknown>(fullKey)
2566
- this.sendResult(packageName, requestId, true, {has: result.is_ok()})
2857
+ const has = await this.simpleStorage.has(packageName, key)
2858
+ this.sendResult(packageName, requestId, true, {has})
2567
2859
  } catch (err) {
2568
2860
  console.error(`${LOG_TAG}: storage_has error:`, err)
2569
2861
  this.sendResult(packageName, requestId, false, undefined, {
@@ -2575,17 +2867,7 @@ class LocalMiniappRuntime {
2575
2867
 
2576
2868
  private async handleStorageGetAll(packageName: string, requestId?: string): Promise<void> {
2577
2869
  try {
2578
- const prefix = this.getStorageKeyPrefix(packageName)
2579
- const allKeys = mmkvStorage.getAllKeys()
2580
- const values: Record<string, string> = {}
2581
- for (const k of allKeys) {
2582
- if (!k.startsWith(prefix)) continue
2583
- const r = mmkvStorage.load<unknown>(k)
2584
- if (r.is_ok()) {
2585
- const v = r.value
2586
- values[k.slice(prefix.length)] = typeof v === "string" ? v : String(v ?? "")
2587
- }
2588
- }
2870
+ const values = await this.simpleStorage.getAll(packageName)
2589
2871
  this.sendResult(packageName, requestId, true, {values})
2590
2872
  } catch (err) {
2591
2873
  console.error(`${LOG_TAG}: storage_get_all error:`, err)
@@ -2610,10 +2892,7 @@ class LocalMiniappRuntime {
2610
2892
  })
2611
2893
  return
2612
2894
  }
2613
- const prefix = this.getStorageKeyPrefix(packageName)
2614
- for (const [key, value] of Object.entries(values)) {
2615
- mmkvStorage.save(prefix + key, value ?? null)
2616
- }
2895
+ await this.simpleStorage.setMultiple(packageName, values)
2617
2896
  this.sendResult(packageName, requestId, true)
2618
2897
  } catch (err) {
2619
2898
  console.error(`${LOG_TAG}: storage_set_multiple error:`, err)
@@ -2624,6 +2903,36 @@ class LocalMiniappRuntime {
2624
2903
  }
2625
2904
  }
2626
2905
 
2906
+ // ---------------------------------------------------------------------------
2907
+ // Wi-Fi ADB
2908
+ // ---------------------------------------------------------------------------
2909
+
2910
+ private async handleSetWifiAdbState(
2911
+ packageName: string,
2912
+ payload: Record<string, unknown>,
2913
+ requestId?: string,
2914
+ ): Promise<void> {
2915
+ const result = await handleWifiAdbRequest(packageName, payload, {
2916
+ isSystemPackage: (candidate) => this.isSystemPackage(candidate),
2917
+ setWifiAdbState: async (enabled) => {
2918
+ console.log(`${LOG_TAG}: set_wifi_adb_state enabled=${enabled} from ${packageName}`)
2919
+ try {
2920
+ await BluetoothSdk.setWifiAdbState(enabled)
2921
+ } catch (error) {
2922
+ console.error(`${LOG_TAG}: set_wifi_adb_state error:`, error)
2923
+ throw error
2924
+ }
2925
+ },
2926
+ })
2927
+
2928
+ if (result.ok) {
2929
+ this.sendResult(packageName, requestId, true)
2930
+ return
2931
+ }
2932
+
2933
+ this.sendResult(packageName, requestId, false, undefined, result.error)
2934
+ }
2935
+
2627
2936
  // ---------------------------------------------------------------------------
2628
2937
  // Camera FOV
2629
2938
  // ---------------------------------------------------------------------------
@@ -2689,6 +2998,124 @@ class LocalMiniappRuntime {
2689
2998
  }
2690
2999
  }
2691
3000
 
3001
+ /**
3002
+ * Lifecycle-scoped glasses-side VAD override from a miniapp
3003
+ * (session.mic.setVoiceActivityDetectionEnabled). Requires MICROPHONE in the
3004
+ * miniapp manifest. Mentra Live only today — other models no-op via DeviceStore.
3005
+ */
3006
+ private async handleMicSetVadEnabled(
3007
+ packageName: string,
3008
+ payload: Record<string, unknown>,
3009
+ requestId?: string,
3010
+ ): Promise<void> {
3011
+ const app = this.connectedApps.get(packageName)
3012
+ const hasMicPermission = app?.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE")
3013
+ if (!hasMicPermission) {
3014
+ logPermissionNotDeclared(packageName, "MICROPHONE", "to set VAD enabled", `{"type": "MICROPHONE"}`)
3015
+ this.sendResult(packageName, requestId, false, undefined, {
3016
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
3017
+ message: `MICROPHONE permission not declared in miniapp.json. Add {"type": "MICROPHONE"} to the "permissions" array.`,
3018
+ permission: "MICROPHONE",
3019
+ operation: MiniappRequestType.MIC_SET_VAD_ENABLED,
3020
+ })
3021
+ return
3022
+ }
3023
+
3024
+ if (typeof payload.enabled !== "boolean") {
3025
+ this.sendResult(packageName, requestId, false, undefined, {
3026
+ code: MiniappErrorCode.INVALID_ARGUMENT,
3027
+ message: "enabled must be a boolean",
3028
+ })
3029
+ return
3030
+ }
3031
+
3032
+ try {
3033
+ const enabled = payload.enabled
3034
+ console.log(`${LOG_TAG}: mic_set_vad_enabled ${enabled} (by ${packageName})`)
3035
+ await micStateCoordinator.setMiniappGateOverride(packageName, "vad", enabled, this.getConfiguredMicGates())
3036
+ this.sendResult(packageName, requestId, true)
3037
+ } catch (err) {
3038
+ console.error(`${LOG_TAG}: mic_set_vad_enabled error:`, err)
3039
+ this.sendResult(packageName, requestId, false, undefined, {
3040
+ code: MiniappErrorCode.INTERNAL,
3041
+ message: err instanceof Error ? err.message : "VAD set-enabled error",
3042
+ })
3043
+ }
3044
+ }
3045
+
3046
+ /**
3047
+ * Lifecycle-scoped center-mic loudness gate ("Barrier") override from a miniapp
3048
+ * (session.mic.setLoudnessGateEnabled). Requires MICROPHONE in the miniapp
3049
+ * manifest. Mentra Live only today — other models no-op via DeviceStore.
3050
+ */
3051
+ private async handleMicSetLoudnessGateEnabled(
3052
+ packageName: string,
3053
+ payload: Record<string, unknown>,
3054
+ requestId?: string,
3055
+ ): Promise<void> {
3056
+ const app = this.connectedApps.get(packageName)
3057
+ const hasMicPermission = app?.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE")
3058
+ if (!hasMicPermission) {
3059
+ logPermissionNotDeclared(packageName, "MICROPHONE", "to set loudness gate enabled", `{"type": "MICROPHONE"}`)
3060
+ this.sendResult(packageName, requestId, false, undefined, {
3061
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
3062
+ message: `MICROPHONE permission not declared in miniapp.json. Add {"type": "MICROPHONE"} to the "permissions" array.`,
3063
+ permission: "MICROPHONE",
3064
+ operation: MiniappRequestType.MIC_SET_LOUDNESS_GATE_ENABLED,
3065
+ })
3066
+ return
3067
+ }
3068
+
3069
+ if (typeof payload.enabled !== "boolean") {
3070
+ this.sendResult(packageName, requestId, false, undefined, {
3071
+ code: MiniappErrorCode.INVALID_ARGUMENT,
3072
+ message: "enabled must be a boolean",
3073
+ })
3074
+ return
3075
+ }
3076
+
3077
+ try {
3078
+ const enabled = payload.enabled
3079
+ console.log(`${LOG_TAG}: mic_set_loudness_gate_enabled ${enabled} (by ${packageName})`)
3080
+ await micStateCoordinator.setMiniappGateOverride(
3081
+ packageName,
3082
+ "loudnessGate",
3083
+ enabled,
3084
+ this.getConfiguredMicGates(),
3085
+ )
3086
+ this.sendResult(packageName, requestId, true)
3087
+ } catch (err) {
3088
+ console.error(`${LOG_TAG}: mic_set_loudness_gate_enabled error:`, err)
3089
+ this.sendResult(packageName, requestId, false, undefined, {
3090
+ code: MiniappErrorCode.INTERNAL,
3091
+ message: err instanceof Error ? err.message : "Loudness gate set-enabled error",
3092
+ })
3093
+ }
3094
+ }
3095
+
3096
+ /**
3097
+ * session.system.scanQr — host camera overlay. Must not clear miniapp
3098
+ * foreground; the host seam is responsible for presenting a Modal on top.
3099
+ */
3100
+ private async handleScanQr(
3101
+ packageName: string,
3102
+ payload: Record<string, unknown>,
3103
+ requestId?: string,
3104
+ ): Promise<void> {
3105
+ try {
3106
+ const result = await invokeScanQrSeam(getUiSeams().scanQr, payload)
3107
+ this.sendResult(packageName, requestId, true, result)
3108
+ } catch (err) {
3109
+ if (!getUiSeams().scanQr) {
3110
+ console.warn(`${LOG_TAG}: scanQr seam missing — QR overlay is not registered`)
3111
+ }
3112
+ this.sendResult(packageName, requestId, false, undefined, {
3113
+ code: (err as {code?: string} | null | undefined)?.code || MiniappErrorCode.INTERNAL,
3114
+ message: err instanceof Error ? err.message : "QR scan failed",
3115
+ })
3116
+ }
3117
+ }
3118
+
2692
3119
  // ---------------------------------------------------------------------------
2693
3120
  // System utilities (share, open URL, clipboard, download)
2694
3121
  // ---------------------------------------------------------------------------
@@ -2833,7 +3260,7 @@ class LocalMiniappRuntime {
2833
3260
  const result = await phonePhotoCoordinator.takePhoto(packageName, {
2834
3261
  size: payload.size as "low" | "medium" | "high" | "max" | "small" | "large" | "full" | undefined,
2835
3262
  mode: payload.mode as "photo" | "text" | undefined,
2836
- transferMethod: payload.transferMethod as "auto" | "ble" | undefined,
3263
+ transferMethod: payload.transferMethod as "auto" | "direct" | "ble" | undefined,
2837
3264
  compress: payload.compress as "none" | "low" | "medium" | "high" | undefined,
2838
3265
  sound: payload.sound as boolean | undefined,
2839
3266
  saveToGallery: payload.saveToGallery as boolean | undefined,
@@ -2843,8 +3270,8 @@ class LocalMiniappRuntime {
2843
3270
  isoCap: payload.isoCap as number | undefined,
2844
3271
  noiseReduction: payload.noiseReduction as boolean | undefined,
2845
3272
  edgeEnhancement: payload.edgeEnhancement as boolean | undefined,
2846
- mfnr: payload.mfnr as boolean | undefined,
2847
3273
  zsl: payload.zsl as boolean | undefined,
3274
+ mfnr: payload.mfnr as boolean | undefined,
2848
3275
  ispDigitalGain: payload.ispDigitalGain as number | undefined,
2849
3276
  ispAnalogGain: payload.ispAnalogGain as string | undefined,
2850
3277
  })
@@ -2884,6 +3311,8 @@ class LocalMiniappRuntime {
2884
3311
  mode: payload.mode as "photo" | "text" | undefined,
2885
3312
  exposureTimeNs: payload.exposureTimeNs as number | undefined,
2886
3313
  durationMs: payload.durationMs as number | undefined,
3314
+ zsl: payload.zsl as boolean | undefined,
3315
+ mfnr: payload.mfnr as boolean | undefined,
2887
3316
  })
2888
3317
  this.sendResult(packageName, requestId, true)
2889
3318
  } catch (err) {
@@ -2976,6 +3405,7 @@ class LocalMiniappRuntime {
2976
3405
  video: normalizeStreamVideoConfig(payload.video),
2977
3406
  audio: normalizeStreamAudioConfig(payload.audio),
2978
3407
  sound: payload.sound as boolean | undefined,
3408
+ authToken: typeof payload.authToken === "string" ? payload.authToken : undefined,
2979
3409
  })
2980
3410
  this.sendResult(packageName, requestId, true, result)
2981
3411
  } catch (err) {
@@ -3071,7 +3501,7 @@ class LocalMiniappRuntime {
3071
3501
  return
3072
3502
  }
3073
3503
  try {
3074
- await requestWifiSetup(payload.reason as string | undefined)
3504
+ await requestWifiSetup(payload.reason as string | undefined, packageName)
3075
3505
  this.sendResult(packageName, requestId, true)
3076
3506
  } catch (err) {
3077
3507
  this.sendResult(packageName, requestId, false, undefined, {
@@ -3135,29 +3565,12 @@ class LocalMiniappRuntime {
3135
3565
  }
3136
3566
  }
3137
3567
 
3138
- // Remove old subscriptions for this app
3139
- for (const oldStream of app.subscriptions) {
3140
- const subs = this.streamSubscribers.get(oldStream)
3141
- if (subs) {
3142
- subs.delete(packageName)
3143
- if (subs.size === 0) {
3144
- this.streamSubscribers.delete(oldStream)
3145
- }
3146
- }
3147
- }
3148
-
3149
3568
  // Add new subscriptions
3569
+ const previousSubscriptions = app.subscriptions
3150
3570
  app.subscriptions = new Set(streams)
3151
3571
  app.forceLocalTranscriptionStreams.clear()
3152
3572
  app.cloudTranscriptionStreams = new Set(streams.filter((stream) => stream.startsWith("transcription:")))
3153
- for (const stream of streams) {
3154
- let subs = this.streamSubscribers.get(stream)
3155
- if (!subs) {
3156
- subs = new Set()
3157
- this.streamSubscribers.set(stream, subs)
3158
- }
3159
- subs.add(packageName)
3160
- }
3573
+ this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
3161
3574
 
3162
3575
  this.recomputeMicRequirements()
3163
3576
  this.updateCloudSubscriptions()
@@ -3168,6 +3581,16 @@ class LocalMiniappRuntime {
3168
3581
  // Mic requirements
3169
3582
  // ===========================================================================
3170
3583
 
3584
+ private getConfiguredMicGates(): {vadEnabled: boolean | null; loudnessGateEnabled: boolean | null} {
3585
+ const bluetoothSettings = useSettingsStore.getState().getBluetoothSettings()
3586
+ const configuredVad = bluetoothSettings.voice_activity_detection_enabled
3587
+ const configuredLoudnessGate = bluetoothSettings.loudness_gate_enabled
3588
+ return {
3589
+ vadEnabled: typeof configuredVad === "boolean" ? configuredVad : null,
3590
+ loudnessGateEnabled: typeof configuredLoudnessGate === "boolean" ? configuredLoudnessGate : null,
3591
+ }
3592
+ }
3593
+
3171
3594
  private recomputeMicRequirements(): void {
3172
3595
  let anyPcm = false
3173
3596
  let anyLc3 = false
@@ -3176,7 +3599,11 @@ class LocalMiniappRuntime {
3176
3599
  if (stream === "audio_chunk") anyPcm = true
3177
3600
  if (stream.startsWith("transcription:") || stream.startsWith("translation:") || stream === "vad") anyLc3 = true
3178
3601
  }
3179
- micStateCoordinator.setLocalRequirements({pcm: anyPcm, lc3: anyLc3})
3602
+ micStateCoordinator.setLocalRequirements({
3603
+ pcm: anyPcm,
3604
+ lc3: anyLc3,
3605
+ ...this.getConfiguredMicGates(),
3606
+ })
3180
3607
  }
3181
3608
 
3182
3609
  /**
@@ -3254,12 +3681,21 @@ class LocalMiniappRuntime {
3254
3681
  */
3255
3682
  private buildCloudAudioSubscriptions(cloudStreams: Set<string>): AudioSubscription[] {
3256
3683
  const subs: AudioSubscription[] = []
3684
+ // Union of every connected app's TRANSCRIPTION_CONFIG hints. Hints only
3685
+ // apply to the auto-mode TRANSCRIPTION subscription (the wire schema has
3686
+ // no hints field on specific-language sources — the language IS the hint
3687
+ // there), and are deliberately NOT attached to translation sources.
3688
+ const hintUnion = [...new Set([...this.transcriptionHintsByApp.values()].flat())].sort()
3257
3689
  const langSource = (code: string): LanguageSource =>
3258
3690
  code === "auto" || code === "*" ? {mode: "auto"} : {mode: "specific", code}
3259
3691
  for (const stream of cloudStreams) {
3260
3692
  if (stream.startsWith("transcription:")) {
3261
3693
  const lang = stream.substring("transcription:".length)
3262
- subs.push({kind: "transcription", language: langSource(lang)})
3694
+ const source = langSource(lang)
3695
+ subs.push({
3696
+ kind: "transcription",
3697
+ language: source.mode === "auto" && hintUnion.length > 0 ? {mode: "auto", hints: hintUnion} : source,
3698
+ })
3263
3699
  } else if (stream.startsWith("translation:")) {
3264
3700
  const parts = stream.split(":")
3265
3701
  // Only `translation:<source>:<target>` maps cleanly; a concrete target
@@ -4130,17 +4566,16 @@ class LocalMiniappRuntime {
4130
4566
  }
4131
4567
 
4132
4568
  private doPingRound(): void {
4133
- const now = Date.now()
4134
- const staleThreshold = PING_INTERVAL_MS * PING_TIMEOUT_THRESHOLD
4135
-
4136
4569
  const toRemove: string[] = []
4137
4570
 
4138
4571
  for (const [packageName, app] of this.connectedApps) {
4139
- if (now - app.lastPongAt > staleThreshold) {
4572
+ const liveness = advanceMiniappPingLiveness(app.unansweredPingRounds, PING_TIMEOUT_THRESHOLD)
4573
+ if (liveness.shouldUnregister) {
4140
4574
  console.warn(`${LOG_TAG}: ${packageName} missed ${PING_TIMEOUT_THRESHOLD} pings, unregistering`)
4141
4575
  toRemove.push(packageName)
4142
4576
  continue
4143
4577
  }
4578
+ app.unansweredPingRounds = liveness.unansweredPingRounds
4144
4579
 
4145
4580
  // Send PING — SDK auto-replies with PONG
4146
4581
  this.sendToMiniapp(packageName, {
@@ -4164,6 +4599,7 @@ class LocalMiniappRuntime {
4164
4599
  const app = this.connectedApps.get(packageName)
4165
4600
  if (app) {
4166
4601
  app.lastPongAt = Date.now()
4602
+ app.unansweredPingRounds = 0
4167
4603
  this.clearForegroundProbe(packageName)
4168
4604
  }
4169
4605
  }
@@ -4191,7 +4627,17 @@ class LocalMiniappRuntime {
4191
4627
 
4192
4628
  // Belt-and-suspenders: clear any remaining state
4193
4629
  this.pendingCloudRequests.clear()
4630
+ const hadButtonPressSubscribers = this.getButtonPressSubscribers().length > 0
4194
4631
  this.streamSubscribers.clear()
4632
+ if (hadButtonPressSubscribers) {
4633
+ for (const listener of this.buttonPressSubscriberListeners) {
4634
+ try {
4635
+ listener([])
4636
+ } catch (error) {
4637
+ console.warn(`${LOG_TAG}: button press subscriber listener threw during cleanup:`, error)
4638
+ }
4639
+ }
4640
+ }
4195
4641
  this.connectedApps.clear()
4196
4642
  for (const timerId of this.foregroundProbeTimers.values()) {
4197
4643
  BgTimer.clearTimeout(timerId)