@glassly/engine 0.4.0-beta.1023 → 0.4.0-beta.1045

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/build/engine.d.ts +2 -1
  2. package/build/engine.d.ts.map +1 -1
  3. package/build/engine.js +6 -4
  4. package/build/engine.js.map +1 -1
  5. package/build/facades/glasses.js +1 -1
  6. package/build/facades/glasses.js.map +1 -1
  7. package/build/facades/providerKeys.d.ts +1 -1
  8. package/build/facades/providerKeys.js +1 -1
  9. package/build/facades/providerKeys.js.map +1 -1
  10. package/build/facades/reports.d.ts +15 -0
  11. package/build/facades/reports.d.ts.map +1 -1
  12. package/build/facades/reports.js +37 -0
  13. package/build/facades/reports.js.map +1 -1
  14. package/build/generated/releaseMetadata.js +3 -3
  15. package/build/generated/releaseMetadata.js.map +1 -1
  16. package/build/react/useMentraLiveOta.d.ts.map +1 -1
  17. package/build/react/useMentraLiveOta.js.map +1 -1
  18. package/build/runtime/config.d.ts.map +1 -1
  19. package/build/runtime/config.js.map +1 -1
  20. package/build/services/AiTierResolution.d.ts +1 -1
  21. package/build/services/AiTierResolution.d.ts.map +1 -1
  22. package/build/services/AiTierResolution.js.map +1 -1
  23. package/build/services/AutomaticReportResult.d.ts.map +1 -1
  24. package/build/services/AutomaticReportResult.js.map +1 -1
  25. package/build/services/CloudClientService.d.ts +7 -5
  26. package/build/services/CloudClientService.d.ts.map +1 -1
  27. package/build/services/CloudClientService.js +17 -14
  28. package/build/services/CloudClientService.js.map +1 -1
  29. package/build/services/DisplayProcessor.d.ts.map +1 -1
  30. package/build/services/DisplayProcessor.js +2 -1
  31. package/build/services/DisplayProcessor.js.map +1 -1
  32. package/build/services/GestureDotsService.js +1 -1
  33. package/build/services/GestureDotsService.js.map +1 -1
  34. package/build/services/HotspotOtaTransport.d.ts.map +1 -1
  35. package/build/services/HotspotOtaTransport.js +0 -1
  36. package/build/services/HotspotOtaTransport.js.map +1 -1
  37. package/build/services/LocalMiniappRuntime.d.ts +5 -2
  38. package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
  39. package/build/services/LocalMiniappRuntime.js +54 -23
  40. package/build/services/LocalMiniappRuntime.js.map +1 -1
  41. package/build/services/LocalSttFallbackCoordinator.d.ts.map +1 -1
  42. package/build/services/LocalSttFallbackCoordinator.js.map +1 -1
  43. package/build/services/NativeNavigationService.d.ts.map +1 -1
  44. package/build/services/NativeNavigationService.js.map +1 -1
  45. package/build/services/NavigationHandlers.d.ts +1 -1
  46. package/build/services/NavigationHandlers.js +2 -2
  47. package/build/services/NavigationHandlers.js.map +1 -1
  48. package/build/services/NavigationService.d.ts +2 -2
  49. package/build/services/NavigationService.d.ts.map +1 -1
  50. package/build/services/NavigationService.js +11 -11
  51. package/build/services/NavigationService.js.map +1 -1
  52. package/build/services/PairingIdentity.d.ts.map +1 -1
  53. package/build/services/PairingIdentity.js.map +1 -1
  54. package/build/services/PhoneCalendarService.d.ts.map +1 -1
  55. package/build/services/PhoneCalendarService.js.map +1 -1
  56. package/build/services/PhoneNotificationsSync.d.ts.map +1 -1
  57. package/build/services/PhoneNotificationsSync.js +2 -1
  58. package/build/services/PhoneNotificationsSync.js.map +1 -1
  59. package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
  60. package/build/services/PhoneStreamCoordinator.js.map +1 -1
  61. package/build/services/ProviderKeyValidation.d.ts +1 -1
  62. package/build/services/ProviderKeyValidation.js +1 -1
  63. package/build/services/ProviderKeyValidation.js.map +1 -1
  64. package/build/services/STTModelManager.d.ts.map +1 -1
  65. package/build/services/STTModelManager.js +1 -6
  66. package/build/services/STTModelManager.js.map +1 -1
  67. package/build/services/SupportProfileSync.d.ts.map +1 -1
  68. package/build/services/SupportProfileSync.js +3 -0
  69. package/build/services/SupportProfileSync.js.map +1 -1
  70. package/build/services/TranscriptionRouting.d.ts +3 -0
  71. package/build/services/TranscriptionRouting.d.ts.map +1 -1
  72. package/build/services/TranscriptionRouting.js +14 -0
  73. package/build/services/TranscriptionRouting.js.map +1 -1
  74. package/build/services/UIRouter.d.ts.map +1 -1
  75. package/build/services/UIRouter.js.map +1 -1
  76. package/build/services/controllers.d.ts.map +1 -1
  77. package/build/services/controllers.js +1 -0
  78. package/build/services/controllers.js.map +1 -1
  79. package/build/services/effectiveCapabilities.js +3 -2
  80. package/build/services/effectiveCapabilities.js.map +1 -1
  81. package/build/services/gestureDotGlyphs.d.ts.map +1 -1
  82. package/build/services/gestureDotGlyphs.js.map +1 -1
  83. package/build/services/manifestActions.d.ts.map +1 -1
  84. package/build/services/manifestActions.js.map +1 -1
  85. package/build/services/notificationAppPayload.d.ts +14 -0
  86. package/build/services/notificationAppPayload.d.ts.map +1 -0
  87. package/build/services/notificationAppPayload.js +15 -0
  88. package/build/services/notificationAppPayload.js.map +1 -0
  89. package/build/services/settingsBridge.d.ts.map +1 -1
  90. package/build/services/settingsBridge.js.map +1 -1
  91. package/build/stores/appVisibility.d.ts.map +1 -1
  92. package/build/stores/appVisibility.js.map +1 -1
  93. package/build/stores/gallerySync.d.ts.map +1 -1
  94. package/build/stores/gallerySync.js.map +1 -1
  95. package/build/stores/glasses.d.ts.map +1 -1
  96. package/build/stores/glasses.js +1 -0
  97. package/build/stores/glasses.js.map +1 -1
  98. package/build/stores/settings.d.ts.map +1 -1
  99. package/build/stores/settings.js +5 -3
  100. package/build/stores/settings.js.map +1 -1
  101. package/build/types/capabilities/nimo.d.ts.map +1 -1
  102. package/build/types/capabilities/nimo.js +19 -7
  103. package/build/types/capabilities/nimo.js.map +1 -1
  104. package/build/types/enums.d.ts +2 -1
  105. package/build/types/enums.d.ts.map +1 -1
  106. package/build/types/enums.js +1 -0
  107. package/build/types/enums.js.map +1 -1
  108. package/build/utils/display/index.d.ts +1 -0
  109. package/build/utils/display/index.d.ts.map +1 -1
  110. package/build/utils/display/index.js +2 -2
  111. package/build/utils/display/index.js.map +1 -1
  112. package/build/utils/display/profiles/index.d.ts +1 -0
  113. package/build/utils/display/profiles/index.d.ts.map +1 -1
  114. package/build/utils/display/profiles/index.js +1 -0
  115. package/build/utils/display/profiles/index.js.map +1 -1
  116. package/build/utils/display/profiles/nimo.d.ts +2 -0
  117. package/build/utils/display/profiles/nimo.d.ts.map +1 -0
  118. package/build/utils/display/profiles/nimo.js +3 -0
  119. package/build/utils/display/profiles/nimo.js.map +1 -0
  120. package/build/utils/display/scene/svgPath.js.map +1 -1
  121. package/build/utils/memory/search.d.ts.map +1 -1
  122. package/build/utils/memory/search.js +17 -7
  123. package/build/utils/memory/search.js.map +1 -1
  124. package/package.json +7 -7
  125. package/src/engine.ts +9 -7
  126. package/src/facades/glasses.ts +1 -1
  127. package/src/facades/providerKeys.ts +1 -1
  128. package/src/facades/reports.ts +41 -0
  129. package/src/generated/releaseMetadata.ts +3 -3
  130. package/src/react/useMentraLiveOta.ts +1 -4
  131. package/src/runtime/config.ts +1 -6
  132. package/src/services/AiTierResolution.ts +1 -1
  133. package/src/services/AutomaticReportResult.ts +1 -3
  134. package/src/services/CloudClientService.ts +17 -11
  135. package/src/services/DisplayProcessor.ts +2 -0
  136. package/src/services/GestureDotsService.ts +1 -1
  137. package/src/services/HotspotOtaTransport.ts +1 -5
  138. package/src/services/LocalMiniappRuntime.ts +66 -24
  139. package/src/services/LocalSttFallbackCoordinator.ts +1 -5
  140. package/src/services/NativeNavigationService.ts +1 -2
  141. package/src/services/NavigationHandlers.ts +2 -2
  142. package/src/services/NavigationService.ts +12 -11
  143. package/src/services/PairingIdentity.ts +1 -3
  144. package/src/services/PhoneCalendarService.ts +1 -3
  145. package/src/services/PhoneNotificationsSync.ts +2 -1
  146. package/src/services/PhoneStreamCoordinator.ts +1 -2
  147. package/src/services/ProviderKeyValidation.ts +1 -1
  148. package/src/services/STTModelManager.ts +1 -6
  149. package/src/services/SupportProfileSync.ts +2 -0
  150. package/src/services/TranscriptionRouting.ts +18 -0
  151. package/src/services/UIRouter.ts +3 -9
  152. package/src/services/controllers.ts +1 -0
  153. package/src/services/effectiveCapabilities.ts +3 -2
  154. package/src/services/gestureDotGlyphs.ts +10 -5
  155. package/src/services/manifestActions.ts +2 -1
  156. package/src/services/notificationAppPayload.ts +15 -0
  157. package/src/services/settingsBridge.ts +1 -2
  158. package/src/stores/appVisibility.ts +2 -5
  159. package/src/stores/gallerySync.ts +1 -7
  160. package/src/stores/glasses.ts +1 -0
  161. package/src/stores/settings.ts +5 -3
  162. package/src/types/capabilities/nimo.ts +20 -7
  163. package/src/types/enums.ts +1 -0
  164. package/src/utils/display/index.ts +2 -2
  165. package/src/utils/display/profiles/index.ts +2 -0
  166. package/src/utils/display/profiles/nimo.ts +2 -0
  167. package/src/utils/display/scene/svgPath.ts +1 -1
  168. package/src/utils/memory/search.ts +17 -7
@@ -20,6 +20,7 @@ import type {
20
20
  StoreCatalogResponse,
21
21
  StoreListing,
22
22
  StoreReleasesResponse,
23
+ StoreHostOptions,
23
24
  } from "@glassly/cloud-client/react-native"
24
25
  import type {SubjectTokenType} from "@glassly/cloud-client"
25
26
  import {AppState, Platform} from "react-native"
@@ -190,9 +191,16 @@ function resolveEndpoints(): {core: string; runtime: string} {
190
191
  }
191
192
  }
192
193
 
194
+ // Via init(), never construct(): init() holds the disable-cloud gate, and a
195
+ // client built past it opens the runtime socket and retries it forever.
196
+ function requireClient(): CloudClient {
197
+ if (!client) cloudClientService.init()
198
+ if (!client) throw new Error("Glassly cloud is disabled")
199
+ return client
200
+ }
201
+
193
202
  function getCoreClient(): CloudCore {
194
- if (!client) construct()
195
- const core = client?.core
203
+ const core = requireClient().core
196
204
  if (!core) throw new Error("cloud client core not configured")
197
205
  return core
198
206
  }
@@ -207,9 +215,7 @@ function syncCoreAccessTokenToBluetooth(): Promise<string> {
207
215
  }
208
216
 
209
217
  async function syncCoreAccessTokenToBluetoothInternal(): Promise<string> {
210
- if (!client) construct()
211
- const c = client
212
- if (!c) throw new Error("cloud client not initialized")
218
+ const c = requireClient()
213
219
 
214
220
  const token = await c.auth.getCoreToken()
215
221
  if (c !== client) {
@@ -765,25 +771,25 @@ export const cloudClientService = {
765
771
  return c.core.miniapps.getRegistry()
766
772
  },
767
773
 
768
- async getStoreCatalog(): Promise<StoreCatalogResponse> {
774
+ async getStoreCatalog(opts?: StoreHostOptions): Promise<StoreCatalogResponse> {
769
775
  if (!client) this.init()
770
776
  const c = client
771
777
  if (!c?.core) throw new Error("cloud client core is unavailable")
772
- return c.core.store.list()
778
+ return c.core.store.list(opts)
773
779
  },
774
780
 
775
- async getStoreListing(packageName: string): Promise<StoreListing> {
781
+ async getStoreListing(packageName: string, opts?: StoreHostOptions): Promise<StoreListing> {
776
782
  if (!client) this.init()
777
783
  const c = client
778
784
  if (!c?.core) throw new Error("cloud client core is unavailable")
779
- return c.core.store.get(packageName)
785
+ return c.core.store.get(packageName, opts)
780
786
  },
781
787
 
782
- async getStoreReleases(packageName: string): Promise<StoreReleasesResponse> {
788
+ async getStoreReleases(packageName: string, opts?: StoreHostOptions): Promise<StoreReleasesResponse> {
783
789
  if (!client) this.init()
784
790
  const c = client
785
791
  if (!c?.core) throw new Error("cloud client core is unavailable")
786
- return c.core.store.releases(packageName)
792
+ return c.core.store.releases(packageName, opts)
787
793
  },
788
794
 
789
795
  /** Bytes of a store icon or screenshot; these routes need the user Bearer. */
@@ -21,6 +21,7 @@ import {
21
21
  NEX_PROFILE,
22
22
  RAYNEO_IO_PROFILE,
23
23
  MEMOMIND_ONE_PROFILE,
24
+ NIMO_PROFILE,
24
25
  RAYNEO_IO_CFW_PROFILE,
25
26
  TextMeasurer,
26
27
  TextWrapper,
@@ -206,6 +207,7 @@ const DEVICE_PROFILES: Partial<Record<DeviceTypes, DisplayProfile>> = {
206
207
  [DeviceTypes.MACH1]: Z100_PROFILE, // Mach1 uses same hardware as Vuzix Z100
207
208
  [DeviceTypes.RAYNEO_IO]: RAYNEO_IO_PROFILE,
208
209
  [DeviceTypes.MEMOMIND_ONE]: MEMOMIND_ONE_PROFILE,
210
+ [DeviceTypes.NIMO]: NIMO_PROFILE,
209
211
  }
210
212
 
211
213
  /**
@@ -50,7 +50,7 @@ const MIDDLE_FINGER_GLYPHS: Partial<Record<GestureDotGlyph, GestureDotGlyph>> =
50
50
 
51
51
  /** `glyph` as `finger` draws it: the middle finger's shapes are hollow. */
52
52
  function fingerGlyph(glyph: GestureDotGlyph, finger: number): GestureDotGlyph {
53
- return finger === 1 ? MIDDLE_FINGER_GLYPHS[glyph] ?? glyph : glyph
53
+ return finger === 1 ? (MIDDLE_FINGER_GLYPHS[glyph] ?? glyph) : glyph
54
54
  }
55
55
 
56
56
  function fingerKey(controllerId: string | undefined, finger: number): string {
@@ -1,7 +1,6 @@
1
1
  import BluetoothSdk, {isEnabledHotspotStatus} from "@glassly/bluetooth-sdk"
2
2
  import {otaServer} from "@glassly/bluetooth-sdk/ota-transport"
3
3
  // One coordinator selects the native Android or iOS staging implementation at runtime.
4
- // eslint-disable-next-line react-native/split-platform-components
5
4
  import {PermissionsAndroid, Platform} from "react-native"
6
5
  import type {OtaCheckCurrentGlassesResult} from "./OtaUpdateCheckService"
7
6
  import {
@@ -24,10 +23,7 @@ export type HotspotOtaProgress = {
24
23
  }
25
24
 
26
25
  export type HotspotOtaErrorCode =
27
- | "hotspot_wifi_permission_denied"
28
- | "hotspot_start_failed"
29
- | "hotspot_join_failed"
30
- | "hotspot_server_failed"
26
+ "hotspot_wifi_permission_denied" | "hotspot_start_failed" | "hotspot_join_failed" | "hotspot_server_failed"
31
27
 
32
28
  export class HotspotOtaTransportError extends Error {
33
29
  constructor(
@@ -141,6 +141,7 @@ import {phoneStreamCoordinator} from "./PhoneStreamCoordinator"
141
141
  import {phoneVideoCoordinator} from "./PhoneVideoCoordinator"
142
142
  import {runSentenceTtsPipeline} from "./SentenceTtsPipeline"
143
143
  import {
144
+ matchesTranscriptionSubscription,
144
145
  summarizeTranscriptionRoutes,
145
146
  transcriptionDeliveryRoute,
146
147
  type TranscriptionDeliveryRoute,
@@ -433,7 +434,7 @@ const PERMISSION_TYPE_TO_CANONICAL: Record<string, string> = {
433
434
  }
434
435
 
435
436
  /**
436
- * User API keys (Settings > API Keys) a miniapp may read via
437
+ * User API keys (Settings > Privacy > API Keys) a miniapp may read via
437
438
  * session.keys.get(), mapped provider → settings-store key. Explicit
438
439
  * allowlist: the Soniox key is deliberately absent — it gates the
439
440
  * phone→Soniox direct transcription path and is not exposed to miniapps.
@@ -477,9 +478,9 @@ function directLlmProvider(name: AiTierProvider): LlmProviderImpl {
477
478
  }
478
479
 
479
480
  const QUOTA_EXCEEDED_MESSAGE =
480
- "AI credits are used up. Get more in Settings > Membership, or add your own API key in Settings > API Keys."
481
+ "AI credits are used up. Get more in Settings > Membership, or add your own API key in Settings > Privacy > API Keys."
481
482
 
482
- /** The user's saved key (Settings > API Keys) for an LLM provider, "" if none. */
483
+ /** The user's saved key (Settings > Privacy > API Keys) for an LLM provider, "" if none. */
483
484
  function userLlmKey(provider: AiTierProvider): string {
484
485
  const raw = useSettingsStore.getState().getSetting(MINIAPP_READABLE_API_KEYS[provider])
485
486
  return typeof raw === "string" ? raw.trim() : ""
@@ -4149,7 +4150,7 @@ class LocalMiniappRuntime {
4149
4150
  }
4150
4151
 
4151
4152
  // ---------------------------------------------------------------------------
4152
- // User API keys (Settings > API Keys)
4153
+ // User API keys (Settings > Privacy > API Keys)
4153
4154
  // ---------------------------------------------------------------------------
4154
4155
 
4155
4156
  /**
@@ -4217,7 +4218,7 @@ class LocalMiniappRuntime {
4217
4218
  * The request names a TIER; this handler resolves it to a provider+model
4218
4219
  * (Settings > AI Models dropdowns, or auto from saved keys) and then picks
4219
4220
  * one of two transport paths, decided by whether the user saved a key for
4220
- * the resolved provider (Settings > API Keys) — the phone is where both
4221
+ * the resolved provider (Settings > Privacy > API Keys) — the phone is where both
4221
4222
  * settings live, so this is the only place the choice can be made:
4222
4223
  *
4223
4224
  * - User key saved → the phone calls the vendor DIRECTLY via
@@ -4490,7 +4491,7 @@ class LocalMiniappRuntime {
4490
4491
  this.sendResult(packageName, requestId, false, undefined, {
4491
4492
  code: MiniappErrorCode.QUOTA_EXCEEDED,
4492
4493
  message:
4493
- "AI credits are used up. Get more in Settings > Membership, or add your own API key in Settings > API Keys.",
4494
+ "AI credits are used up. Get more in Settings > Membership, or add your own API key in Settings > Privacy > API Keys.",
4494
4495
  })
4495
4496
  return
4496
4497
  }
@@ -4744,7 +4745,11 @@ class LocalMiniappRuntime {
4744
4745
  * session.system.webLogin — host in-app browser sheet. Same contract as
4745
4746
  * scanQr: the seam presents a Modal and never clears miniapp foreground.
4746
4747
  */
4747
- private async handleWebLogin(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
4748
+ private async handleWebLogin(
4749
+ packageName: string,
4750
+ payload: Record<string, unknown>,
4751
+ requestId?: string,
4752
+ ): Promise<void> {
4748
4753
  try {
4749
4754
  const result = await invokeWebLoginSeam(getUiSeams().webLogin, payload, packageName)
4750
4755
  this.sendResult(packageName, requestId, true, result)
@@ -5385,6 +5390,9 @@ class LocalMiniappRuntime {
5385
5390
  */
5386
5391
  private buildCloudAudioSubscriptions(cloudStreams: Set<string>): AudioSubscription[] {
5387
5392
  const subs: AudioSubscription[] = []
5393
+ // One auto recognizer serves every transcription listener. Explicit-language
5394
+ // listeners are filtered at delivery; translations still need their own jobs.
5395
+ const hasAutoTranscription = cloudStreams.has("transcription:auto")
5388
5396
  // Union of every connected app's TRANSCRIPTION_CONFIG hints. Hints only
5389
5397
  // apply to the auto-mode TRANSCRIPTION subscription (the wire schema has
5390
5398
  // no hints field on specific-language sources — the language IS the hint
@@ -5395,6 +5403,7 @@ class LocalMiniappRuntime {
5395
5403
  for (const stream of cloudStreams) {
5396
5404
  if (stream.startsWith("transcription:")) {
5397
5405
  const lang = stream.substring("transcription:".length)
5406
+ if (hasAutoTranscription && lang !== "auto") continue
5398
5407
  const source = langSource(lang)
5399
5408
  subs.push({
5400
5409
  kind: "transcription",
@@ -5527,6 +5536,12 @@ class LocalMiniappRuntime {
5527
5536
  }
5528
5537
 
5529
5538
  private handleTranscriptionData(d: TranscriptionData): void {
5539
+ const expectsAuto = [...(this.streamSubscribers.get("transcription:auto") ?? [])].some(
5540
+ (packageName) => this.appTranscriptionRoutesForSubscription(packageName, "transcription:auto").cloud,
5541
+ )
5542
+ // Subscription replacement is asynchronous. Ignore late results from the
5543
+ // old pipeline while switching between auto and language-specific jobs.
5544
+ if ((d.subscription.language.mode === "auto") !== expectsAuto) return
5530
5545
  const receivedAt = Date.now()
5531
5546
  if (TRANSCRIPT_TIMING_TELEMETRY) {
5532
5547
  console.log(
@@ -5565,6 +5580,8 @@ class LocalMiniappRuntime {
5565
5580
  __hostReceivedAt: receivedAt,
5566
5581
  },
5567
5582
  "cloud",
5583
+ undefined,
5584
+ d.subscription,
5568
5585
  )
5569
5586
  }
5570
5587
 
@@ -5642,8 +5659,7 @@ class LocalMiniappRuntime {
5642
5659
  /** Model profile + runtime overrides (the G2 CFW scene canvas) for the default wearable. */
5643
5660
  private currentCapabilities(): Capabilities {
5644
5661
  const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable) as
5645
- | DeviceTypes
5646
- | undefined
5662
+ DeviceTypes | undefined
5647
5663
  return getEffectiveCapabilities(defaultWearable || DeviceTypes.NONE)
5648
5664
  }
5649
5665
 
@@ -5661,7 +5677,7 @@ class LocalMiniappRuntime {
5661
5677
  // connected controllers add their input surfaces to `button`.
5662
5678
  useGlassesStore.subscribe(
5663
5679
  (s) =>
5664
- `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}:${readyControllersKey(s.controllers)}`,
5680
+ `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.sceneStereo === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}:${readyControllersKey(s.controllers)}`,
5665
5681
  broadcast,
5666
5682
  )
5667
5683
  // The debug knob flips hasNativeNavigation on custom firmware.
@@ -5790,6 +5806,8 @@ class LocalMiniappRuntime {
5790
5806
  transcriptionSource?: TranscriptionEventSource,
5791
5807
  /** Deliver to this subscriber alone (the designated dashboard app's gestures while it shows). */
5792
5808
  only?: string,
5809
+ /** Producing cloud/direct subscription; auto results also serve matching language listeners. */
5810
+ transcriptionSubscription?: TranscriptionData["subscription"],
5793
5811
  ): void {
5794
5812
  // Paused miniapps (glasses disconnected) hear no streams. The resume push
5795
5813
  // lands before the reconnect's own glasses_connection event, so nothing
@@ -5801,6 +5819,12 @@ class LocalMiniappRuntime {
5801
5819
  if (transcriptionSource === "local") markTranscriptSourceRecent("local")
5802
5820
  // Translate cloud event names to miniapp protocol stream types
5803
5821
  const normalizedStream = this.normalizeStreamType(streamType)
5822
+ const transcriptionStreams =
5823
+ transcriptionSource === "cloud" && transcriptionSubscription
5824
+ ? [...this.streamSubscribers.keys()].filter((stream) =>
5825
+ matchesTranscriptionSubscription(stream, normalizedStream, transcriptionSubscription),
5826
+ )
5827
+ : undefined
5804
5828
 
5805
5829
  // Passive memory capture happens HERE, before subscriber fan-out, so the
5806
5830
  // timeline reflects what was said around the user regardless of which
@@ -5809,10 +5833,10 @@ class LocalMiniappRuntime {
5809
5833
  // recorded while it was open.)
5810
5834
  this.captureTranscriptToMemory(normalizedStream, data)
5811
5835
 
5812
- // Collect all subscribers: exact match, plus wildcard matches for streams
5813
- // that carry a language tag. A miniapp subscribed to "transcription:auto"
5814
- // should receive any "transcription:<lang>" event (the detected language
5815
- // is conveyed in the event data, not the stream key).
5836
+ // Auto cloud/direct results serve auto listeners and listeners requesting
5837
+ // the detected language. Specific jobs serve only their own subscription.
5838
+ // Local/legacy events have no producing subscription and retain wildcard
5839
+ // matching so one on-device recognizer can serve auto and explicit listeners.
5816
5840
  //
5817
5841
  // Translation streams support cloud v3's wildcard patterns. Incoming
5818
5842
  // events are keyed `translation:<source>:<target>` and we match against
@@ -5824,11 +5848,13 @@ class LocalMiniappRuntime {
5824
5848
  // translation:auto — back-compat alias for *:*
5825
5849
  const matchedSubs = new Set<string>()
5826
5850
  let translationPatterns: string[] = []
5827
- const exact = this.streamSubscribers.get(normalizedStream)
5828
- if (exact) for (const p of exact) matchedSubs.add(p)
5851
+ for (const stream of transcriptionStreams ?? [normalizedStream]) {
5852
+ const exact = this.streamSubscribers.get(stream)
5853
+ if (exact) for (const p of exact) matchedSubs.add(p)
5854
+ }
5829
5855
 
5830
5856
  if (normalizedStream.startsWith("transcription:")) {
5831
- const autoSubs = this.streamSubscribers.get("transcription:auto")
5857
+ const autoSubs = transcriptionStreams ? undefined : this.streamSubscribers.get("transcription:auto")
5832
5858
  if (autoSubs) for (const p of autoSubs) matchedSubs.add(p)
5833
5859
  } else if (normalizedStream.startsWith("translation:")) {
5834
5860
  // translation:<source>:<target> — match each wildcard variant.
@@ -5907,8 +5933,18 @@ class LocalMiniappRuntime {
5907
5933
  // The wearer's touches reach a widget-only app through its host's widget events alone.
5908
5934
  if (normalizedStream === MiniappStreamType.TOUCH_EVENT && miniappWidgetMode.isWidgetOnly(packageName)) continue
5909
5935
  let transcriptionRoute: TranscriptionDeliveryRoute | undefined
5936
+ let outboundStream = normalizedStream
5910
5937
  if (normalizedStream.startsWith("transcription:") && transcriptionSource) {
5911
- const routes = this.appTranscriptionRoutesForEvent(packageName, normalizedStream)
5938
+ const cloudStreams = transcriptionStreams?.filter(
5939
+ (stream) => this.appTranscriptionRoutesForSubscription(packageName, stream).cloud,
5940
+ )
5941
+ const routes = cloudStreams
5942
+ ? {cloud: cloudStreams.length > 0, forceLocal: false}
5943
+ : this.appTranscriptionRoutesForEvent(packageName, normalizedStream)
5944
+ // Existing miniapp bundles dispatch explicit listeners by their BCP-47
5945
+ // subscription key, while providers detect bare codes (es vs es-ES).
5946
+ // Send one envelope per app, even when it also subscribes to auto.
5947
+ outboundStream = cloudStreams?.find((stream) => stream !== "transcription:auto") ?? normalizedStream
5912
5948
  transcriptionRoute =
5913
5949
  transcriptionDeliveryRoute(transcriptionSource, routes, this.transcriptionPrimaryUp()) ?? undefined
5914
5950
  if (!transcriptionRoute) continue
@@ -5937,7 +5973,7 @@ class LocalMiniappRuntime {
5937
5973
  }
5938
5974
  this.sendToMiniapp(packageName, {
5939
5975
  type: MiniappResponseType.EVENT,
5940
- streamType: normalizedStream,
5976
+ streamType: outboundStream,
5941
5977
  data: outboundData,
5942
5978
  ...(transcriptionRoute ? {transcriptionRoute} : {}),
5943
5979
  })
@@ -6538,7 +6574,12 @@ class LocalMiniappRuntime {
6538
6574
  // and an unplace racing it cannot stop the provider early. A provider the
6539
6575
  // store already runs (a wearer's launch still spawning, say) stays a
6540
6576
  // normal run; the registry alone lags that launch.
6541
- this.widgetPlacements = placeWidget(this.widgetPlacements, {hostPackageName, providerPackageName: provider, widgetId, box})
6577
+ this.widgetPlacements = placeWidget(this.widgetPlacements, {
6578
+ hostPackageName,
6579
+ providerPackageName: provider,
6580
+ widgetId,
6581
+ box,
6582
+ })
6542
6583
  miniappWidgetMode.setPlaced(provider, true)
6543
6584
  if (!app.running && !miniappRunningRegistry.has(provider)) miniappWidgetMode.setWidgetOnly(provider, true)
6544
6585
  try {
@@ -6575,7 +6616,10 @@ class LocalMiniappRuntime {
6575
6616
  this.widgetPlacements = unplaceWidget(this.widgetPlacements, hostPackageName, provider, target.widgetId)
6576
6617
  this.sendResult(hostPackageName, requestId, true)
6577
6618
  if (!had) return
6578
- if (this.connectedApps.has(provider) && widgetHostsOf(this.widgetPlacements, provider, target.widgetId).length === 0) {
6619
+ if (
6620
+ this.connectedApps.has(provider) &&
6621
+ widgetHostsOf(this.widgetPlacements, provider, target.widgetId).length === 0
6622
+ ) {
6579
6623
  this.sendWidgetCall(provider, target.widgetId, {kind: "box", box: null})
6580
6624
  }
6581
6625
  await this.settleWidgetProvider(provider)
@@ -6706,9 +6750,7 @@ class LocalMiniappRuntime {
6706
6750
  if (!widget) return
6707
6751
  const view: WidgetView = payload.view === "expanded" ? "expanded" : "card"
6708
6752
  if (view === "expanded" && !widget.expandable) {
6709
- console.warn(
6710
- `${LOG_TAG}: dropping expanded frame from ${providerPackageName}/${widgetId}: it does not list EX`,
6711
- )
6753
+ console.warn(`${LOG_TAG}: dropping expanded frame from ${providerPackageName}/${widgetId}: it does not list EX`)
6712
6754
  return
6713
6755
  }
6714
6756
  const cap =
@@ -28,11 +28,7 @@ import sttModelManager from "./STTModelManager"
28
28
  * `force_local` (a forced subscription would keep local STT on either way).
29
29
  */
30
30
  export type LocalSttFallbackReason =
31
- | "force_local"
32
- | "cloud_disconnected"
33
- | "cloud_disabled"
34
- | "stt_quota"
35
- | "provider_unreachable"
31
+ "force_local" | "cloud_disconnected" | "cloud_disabled" | "stt_quota" | "provider_unreachable"
36
32
 
37
33
  class LocalSttFallbackCoordinator {
38
34
  private static instance: LocalSttFallbackCoordinator
@@ -61,8 +61,7 @@ class NativeNavigationService {
61
61
  */
62
62
  public isSupported(): boolean {
63
63
  const defaultWearable = useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.defaultWearable) as
64
- | DeviceTypes
65
- | undefined
64
+ DeviceTypes | undefined
66
65
  // Effective, not model: custom firmware drops hasNativeNavigation so the
67
66
  // miniapp's scene HUD owns the display (see effectiveCapabilities).
68
67
  const capabilities = getEffectiveCapabilities(defaultWearable || DeviceTypes.NONE)
@@ -364,7 +364,7 @@ export class NavigationHandlers {
364
364
 
365
365
  async handleComputeRoute(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
366
366
  try {
367
- // With the user's own Mapbox key (Settings > API Keys), compute
367
+ // With the user's own Mapbox key (Settings > Privacy > API Keys), compute
368
368
  // on-device so routing works without the cloud and bills their account.
369
369
  // Without one, go through the v2 cloud maps proxy — org-token Mapbox
370
370
  // usage only ever happens cloud-side.
@@ -437,7 +437,7 @@ export class NavigationHandlers {
437
437
 
438
438
  /**
439
439
  * Type-ahead place search. Runs on-device against Mapbox Search Box when the
440
- * user saved their own key (Settings > API Keys) — that key exists precisely
440
+ * user saved their own key (Settings > Privacy > API Keys) — that key exists precisely
441
441
  * so search works without the cloud — and falls back to the v2 cloud maps
442
442
  * service otherwise. Resolves `{ok: true, suggestions}` — an empty query or no
443
443
  * matches is `{ok: true, suggestions: []}`, not an error.
@@ -211,7 +211,7 @@ class NavigationService {
211
211
  const mapboxRenderToken = await resolveMapboxRenderToken()
212
212
  if (!mapboxRenderToken) {
213
213
  const error =
214
- "No Mapbox token available — add your key in Settings > API Keys" +
214
+ "No Mapbox token available — add your key in Settings > Privacy > API Keys" +
215
215
  " (required while Glassly Cloud is disabled or unreachable)"
216
216
  console.warn(`${LOG_TAG}: start blocked — ${error}`)
217
217
  return {ok: false, error}
@@ -329,7 +329,7 @@ class NavigationService {
329
329
  * Compute one or more routes without starting a trip.
330
330
  *
331
331
  * Reached from the miniapp bridge (NavigationHandlers.handleComputeRoute)
332
- * when the user has saved their own Mapbox key in Settings > API Keys —
332
+ * when the user has saved their own Mapbox key in Settings > Privacy > API Keys —
333
333
  * that key exists so navigation works without the cloud. Without a user
334
334
  * key the bridge uses the v2 cloud maps service, which holds a provider
335
335
  * token server-side and caches results. Implemented by
@@ -379,7 +379,7 @@ class NavigationService {
379
379
  return !!resolveMapboxAccessToken()
380
380
  }
381
381
 
382
- /** True when the user saved their own Mapbox key in Settings > API Keys. */
382
+ /** True when the user saved their own Mapbox key in Settings > Privacy > API Keys. */
383
383
  public hasUserMapboxKey(): boolean {
384
384
  return !!userMapboxApiKey()
385
385
  }
@@ -501,7 +501,7 @@ export default navigationService
501
501
  // ---------------------------------------------------------------------------
502
502
 
503
503
  // Routes + Geocoding go DIRECT to Mapbox from the phone process ONLY when the
504
- // user saved their own key (Settings > API Keys) — "use my account". Without
504
+ // user saved their own key (Settings > Privacy > API Keys) — "use my account". Without
505
505
  // one, callers (NavigationHandlers) route these operations through the v2
506
506
  // cloud maps proxy instead; the binary ships no baked Mapbox key.
507
507
  const MAPBOX_DIRECTIONS_BASE = "https://api.mapbox.com/directions/v5/mapbox"
@@ -532,7 +532,7 @@ function mapboxToken(): string {
532
532
 
533
533
  /**
534
534
  * Phone-side helper: hit the Mapbox Directions API directly with the user's own
535
- * key (Settings > API Keys). Preferred over the v2 cloud maps service whenever
535
+ * key (Settings > Privacy > API Keys). Preferred over the v2 cloud maps service whenever
536
536
  * that key is set, so routing works without the cloud. Lives here
537
537
  * (not in Kotlin) so it's platform-independent across the JS bridge.
538
538
  * Returns the SDK-shaped result with primary + alternates.
@@ -583,7 +583,8 @@ async function computeRouteViaRoutesApi(payload: Record<string, unknown>): Promi
583
583
  }
584
584
 
585
585
  const token = mapboxToken()
586
- if (!token) return {ok: false, error: "computeRoute: no Mapbox key saved (add yours in Settings > API Keys)"}
586
+ if (!token)
587
+ return {ok: false, error: "computeRoute: no Mapbox key saved (add yours in Settings > Privacy > API Keys)"}
587
588
 
588
589
  // Mapbox path is a semicolon-joined `lng,lat` list: origin then each stop.
589
590
  const coords = [`${oLng},${oLat}`, ...stops.map((s) => `${s.lng},${s.lat}`)].join(";")
@@ -739,7 +740,7 @@ function mapboxManeuverToKind(type?: string, modifier?: string): string | undefi
739
740
 
740
741
  /**
741
742
  * Reverse-geocode a coordinate via the Mapbox Geocoding v6 API directly, using
742
- * the user's own key (Settings > API Keys). Preferred over the v2 cloud maps
743
+ * the user's own key (Settings > Privacy > API Keys). Preferred over the v2 cloud maps
743
744
  * service whenever that key is set, so pin labels work without the cloud.
744
745
  *
745
746
  * Returns the road/street name (e.g. "Octavia Blvd") plus the full formatted
@@ -767,7 +768,7 @@ async function reverseGeocodeRoadViaGeocodingApi(coord: {lat: number; lng: numbe
767
768
  if (!token)
768
769
  return {
769
770
  ok: false,
770
- error: "reverseGeocode: no Mapbox key saved (add yours in Settings > API Keys)",
771
+ error: "reverseGeocode: no Mapbox key saved (add yours in Settings > Privacy > API Keys)",
771
772
  }
772
773
  const params = new URLSearchParams({
773
774
  longitude: String(coord.lng),
@@ -813,7 +814,7 @@ async function reverseGeocodeRoadViaGeocodingApi(coord: {lat: number; lng: numbe
813
814
 
814
815
  /**
815
816
  * Type-ahead place search straight from the device via Mapbox Search Box,
816
- * using the user's own key (Settings > API Keys). This is the offline-from-
817
+ * using the user's own key (Settings > Privacy > API Keys). This is the offline-from-
817
818
  * the-cloud path: with a key saved, place search works with no runtime
818
819
  * involvement at all.
819
820
  *
@@ -840,7 +841,7 @@ async function placeAutocompleteViaSearchBox(
840
841
  if (!token) {
841
842
  return {
842
843
  ok: false,
843
- error: "placeAutocomplete: no Mapbox key saved (add yours in Settings > API Keys)",
844
+ error: "placeAutocomplete: no Mapbox key saved (add yours in Settings > Privacy > API Keys)",
844
845
  }
845
846
  }
846
847
  const params = new URLSearchParams({
@@ -904,7 +905,7 @@ async function placeDetailsViaSearchBox(
904
905
  if (!token) {
905
906
  return {
906
907
  ok: false,
907
- error: "placeDetails: no Mapbox key saved (add yours in Settings > API Keys)",
908
+ error: "placeDetails: no Mapbox key saved (add yours in Settings > Privacy > API Keys)",
908
909
  }
909
910
  }
910
911
  const params = new URLSearchParams({access_token: token, session_token: sessionToken})
@@ -40,9 +40,7 @@ import {useSettingsStore, SETTINGS} from "../stores/settings"
40
40
  import {DeviceTypes} from "../types"
41
41
 
42
42
  export type IdentitySnapshot =
43
- | {kind: "none"}
44
- | {kind: "pending"; model: string}
45
- | {kind: "paired"; model: string; name: string; address?: string}
43
+ {kind: "none"} | {kind: "pending"; model: string} | {kind: "paired"; model: string; name: string; address?: string}
46
44
 
47
45
  // Identity keys hold model/name strings; anything else (legacy null, a stray
48
46
  // non-string) reads as absent, matching native currentDefaultDevice()'s
@@ -14,9 +14,7 @@ import {
14
14
 
15
15
  export {PhoneCalendarError} from "./PhoneCalendarHelpers"
16
16
 
17
- type CalendarPermissionStatus = Awaited<
18
- ReturnType<typeof Calendar.getCalendarPermissionsAsync>
19
- >["status"]
17
+ type CalendarPermissionStatus = Awaited<ReturnType<typeof Calendar.getCalendarPermissionsAsync>>["status"]
20
18
 
21
19
  /**
22
20
  * expo-calendar reports anything below EventKit full access as not granted,
@@ -1,3 +1,4 @@
1
+ import {notificationAppPayload} from "./notificationAppPayload"
1
2
  /**
2
3
  * Phone-notifications sync — engine-owned, started by `engine.start()`. Keeps
3
4
  * three things in step with settings and the connected glasses, for any host:
@@ -78,7 +79,7 @@ export function getKnownNotificationApps(): NotificationApp[] {
78
79
  export function installedNotificationApps(): Promise<
79
80
  Array<{packageName: string; appName: string; icon: string | null}>
80
81
  > {
81
- if (Platform.OS === "android") return CrustModule.getInstalledApps()
82
+ if (Platform.OS === "android") return CrustModule.getInstalledApps().then(notificationAppPayload)
82
83
  if (Platform.OS !== "ios") return Promise.resolve([])
83
84
  return Promise.resolve(
84
85
  getKnownNotificationApps().map((app) => ({packageName: app.appId, appName: app.displayName, icon: null})),
@@ -339,8 +339,7 @@ export class PhoneStreamCoordinator {
339
339
  // block subsequent start/stop transitions on this coordinator.
340
340
  assertGlassesConnected()
341
341
  type JoinDecision =
342
- | {kind: "join"; entry: ManagedEntry; immediate: ManagedStartResult | null}
343
- | {kind: "fresh"; entry: ManagedEntry}
342
+ {kind: "join"; entry: ManagedEntry; immediate: ManagedStartResult | null} | {kind: "fresh"; entry: ManagedEntry}
344
343
 
345
344
  const decision = await this.runExclusive(async (): Promise<JoinDecision> => {
346
345
  if (this.current && this.current.kind === "unmanaged") {
@@ -2,7 +2,7 @@
2
2
  * ProviderKeyValidation
3
3
  *
4
4
  * Cheap validate-before-save probes for the AI/voice provider keys on the
5
- * Keys screen (Settings > API Keys), mirroring validateMapboxApiKey's shape:
5
+ * Keys screen (Settings > Privacy > API Keys), mirroring validateMapboxApiKey's shape:
6
6
  * each probe hits the provider's cheapest authenticated read endpoint (no
7
7
  * metered usage), maps auth failures to `invalid_key`, and maps everything
8
8
  * else to `network` so the caller can offer "save anyway".
@@ -151,12 +151,7 @@ class STTModelManager {
151
151
  fileName: "sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-480ms-int8",
152
152
  size: 101 * 1024 * 1024,
153
153
  type: "transducer",
154
- requiredFiles: [
155
- "encoder.int8.onnx",
156
- "decoder.int8.onnx",
157
- "joiner.int8.onnx",
158
- "tokens.txt",
159
- ],
154
+ requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
160
155
  languageCode: "en-US",
161
156
  },
162
157
  // NVIDIA Nemotron streaming ASR. These are model choices rather than plain
@@ -83,6 +83,8 @@ function armDebounce(generation: number): void {
83
83
 
84
84
  async function sendCurrentSnapshot(force: boolean, generation = syncGeneration): Promise<void> {
85
85
  if (!unsubscribe || generation !== syncGeneration) return
86
+ // Every send would fail into the retry backoff with no client to send through.
87
+ if (cloudClientService.isCloudDisabled()) return
86
88
  if (sendingGeneration === generation) {
87
89
  const fingerprint = snapshotFingerprint(buildSnapshot())
88
90
  if (fingerprint !== lastSentFingerprint) queuedFingerprint = fingerprint
@@ -1,3 +1,21 @@
1
+ import type {TranscriptionSubscription} from "@glassly/cloud-protocol"
2
+
3
+ /** Match a provider result to a logical miniapp subscription. */
4
+ export function matchesTranscriptionSubscription(
5
+ stream: string,
6
+ resolvedStream: string,
7
+ subscription: TranscriptionSubscription,
8
+ ): boolean {
9
+ const prefix = "transcription:"
10
+ if (!stream.startsWith(prefix) || !resolvedStream.startsWith(prefix)) return false
11
+ if (subscription.language.mode === "specific") return stream === `${prefix}${subscription.language.code}`
12
+ if (stream === `${prefix}auto`) return true
13
+ // Providers detect ISO language codes; miniapps request regional BCP-47 tags.
14
+ // Unknown language stays auto-only until the recognizer identifies it.
15
+ const detected = resolvedStream.slice(prefix.length).split("-")[0].toLowerCase()
16
+ return detected !== "auto" && detected !== "" && stream.slice(prefix.length).split("-")[0].toLowerCase() === detected
17
+ }
18
+
1
19
  export interface TranscriptionRouteSummary {
2
20
  hasCloudSubscriber: boolean
3
21
  hasForceLocalSubscriber: boolean
@@ -193,9 +193,7 @@ export class UIRouter {
193
193
  const cancel = {type: "cancel", requestId: uiSendPayload.requestId}
194
194
  const literal = JSON.stringify(cancel)
195
195
  const escaped = JSON.stringify(literal)
196
- binding.inject(
197
- `if (window.__host && window.__host.recv) window.__host.recv(JSON.parse(${escaped})); true;`,
198
- )
196
+ binding.inject(`if (window.__host && window.__host.recv) window.__host.recv(JSON.parse(${escaped})); true;`)
199
197
  return
200
198
  }
201
199
  const outbound: Record<string, unknown> = {
@@ -207,9 +205,7 @@ export class UIRouter {
207
205
  if (typeof uiSendPayload.requestId === "string") outbound.requestId = uiSendPayload.requestId
208
206
  const literal = JSON.stringify(outbound)
209
207
  const escaped = JSON.stringify(literal)
210
- binding.inject(
211
- `if (window.__host && window.__host.recv) window.__host.recv(JSON.parse(${escaped})); true;`,
212
- )
208
+ binding.inject(`if (window.__host && window.__host.recv) window.__host.recv(JSON.parse(${escaped})); true;`)
213
209
  }
214
210
 
215
211
  /**
@@ -222,9 +218,7 @@ export class UIRouter {
222
218
  if (!binding) return
223
219
  const literal = JSON.stringify(frame)
224
220
  const escaped = JSON.stringify(literal)
225
- binding.inject(
226
- `if (window.__host && window.__host.recv) window.__host.recv(JSON.parse(${escaped})); true;`,
227
- )
221
+ binding.inject(`if (window.__host && window.__host.recv) window.__host.recv(JSON.parse(${escaped})); true;`)
228
222
  }
229
223
 
230
224
  /**
@@ -57,6 +57,7 @@ const INPUT_SURFACES: Readonly<Record<string, readonly InputSurface[]>> = {
57
57
  [ControllerTypes.IINE_ANANKE]: [GAMEBRICK_SURFACE],
58
58
  [ControllerTypes.ZERO2]: [GAMEBRICK_SURFACE],
59
59
  [ControllerTypes.MICRO]: [GAMEBRICK_SURFACE],
60
+ [ControllerTypes.CRKD_ATOM_PLUS]: [GAMEBRICK_SURFACE],
60
61
  [ControllerTypes.R1]: [],
61
62
  }
62
63