@glassly/engine 0.3.1-dev.494 → 0.3.1-dev.523

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/build/engine.d.ts +9 -2
  2. package/build/engine.d.ts.map +1 -1
  3. package/build/engine.js +5 -0
  4. package/build/engine.js.map +1 -1
  5. package/build/facades/dev.js +1 -1
  6. package/build/facades/dev.js.map +1 -1
  7. package/build/facades/miniapps.d.ts +5 -5
  8. package/build/facades/miniapps.d.ts.map +1 -1
  9. package/build/facades/miniapps.js +5 -6
  10. package/build/facades/miniapps.js.map +1 -1
  11. package/build/facades/speech.d.ts +10 -0
  12. package/build/facades/speech.d.ts.map +1 -1
  13. package/build/facades/speech.js +10 -0
  14. package/build/facades/speech.js.map +1 -1
  15. package/build/generated/releaseMetadata.js +3 -3
  16. package/build/generated/releaseMetadata.js.map +1 -1
  17. package/build/index.d.ts +1 -1
  18. package/build/index.d.ts.map +1 -1
  19. package/build/index.js +1 -1
  20. package/build/index.js.map +1 -1
  21. package/build/services/AppRegistry.d.ts +1 -7
  22. package/build/services/AppRegistry.d.ts.map +1 -1
  23. package/build/services/AppRegistry.js +15 -26
  24. package/build/services/AppRegistry.js.map +1 -1
  25. package/build/services/DevServerBridge.d.ts +1 -1
  26. package/build/services/DevServerBridge.js +2 -2
  27. package/build/services/DevServerBridge.js.map +1 -1
  28. package/build/services/DeviceEventRouter.d.ts.map +1 -1
  29. package/build/services/DeviceEventRouter.js +2 -0
  30. package/build/services/DeviceEventRouter.js.map +1 -1
  31. package/build/services/JSRouter.js +1 -1
  32. package/build/services/JSRouter.js.map +1 -1
  33. package/build/services/LocalMiniappRuntime.d.ts +21 -6
  34. package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
  35. package/build/services/LocalMiniappRuntime.js +130 -48
  36. package/build/services/LocalMiniappRuntime.js.map +1 -1
  37. package/build/services/LocalTranslationService.d.ts +51 -0
  38. package/build/services/LocalTranslationService.d.ts.map +1 -0
  39. package/build/services/LocalTranslationService.js +141 -0
  40. package/build/services/LocalTranslationService.js.map +1 -0
  41. package/build/services/MiniappLauncher.d.ts +17 -23
  42. package/build/services/MiniappLauncher.d.ts.map +1 -1
  43. package/build/services/MiniappLauncher.js +30 -34
  44. package/build/services/MiniappLauncher.js.map +1 -1
  45. package/build/services/MiniappPauseCoordinator.d.ts +15 -0
  46. package/build/services/MiniappPauseCoordinator.d.ts.map +1 -0
  47. package/build/services/MiniappPauseCoordinator.js +93 -0
  48. package/build/services/MiniappPauseCoordinator.js.map +1 -0
  49. package/build/services/OfflineTranslationManager.d.ts +19 -0
  50. package/build/services/OfflineTranslationManager.d.ts.map +1 -0
  51. package/build/services/OfflineTranslationManager.js +35 -0
  52. package/build/services/OfflineTranslationManager.js.map +1 -0
  53. package/build/services/pages/OverlayPages.d.ts +5 -5
  54. package/build/services/pages/OverlayPages.d.ts.map +1 -1
  55. package/build/services/pages/OverlayPages.js +5 -5
  56. package/build/services/pages/OverlayPages.js.map +1 -1
  57. package/build/services/pages/overlayApps.d.ts +14 -11
  58. package/build/services/pages/overlayApps.d.ts.map +1 -1
  59. package/build/services/pages/overlayApps.js +21 -19
  60. package/build/services/pages/overlayApps.js.map +1 -1
  61. package/build/services/settingsBridge.d.ts +8 -8
  62. package/build/services/settingsBridge.d.ts.map +1 -1
  63. package/build/services/settingsBridge.js +12 -13
  64. package/build/services/settingsBridge.js.map +1 -1
  65. package/build/stores/apps.d.ts +0 -5
  66. package/build/stores/apps.d.ts.map +1 -1
  67. package/build/stores/apps.js +32 -52
  68. package/build/stores/apps.js.map +1 -1
  69. package/build/stores/settings.d.ts.map +1 -1
  70. package/build/stores/settings.js +0 -10
  71. package/build/stores/settings.js.map +1 -1
  72. package/build/types/applet.d.ts +3 -1
  73. package/build/types/applet.d.ts.map +1 -1
  74. package/build/types/applet.js.map +1 -1
  75. package/build/types/capabilities/even-realities-g1.d.ts.map +1 -1
  76. package/build/types/capabilities/even-realities-g1.js +1 -0
  77. package/build/types/capabilities/even-realities-g1.js.map +1 -1
  78. package/build/types/capabilities/even-realities-g2.d.ts.map +1 -1
  79. package/build/types/capabilities/even-realities-g2.js +1 -0
  80. package/build/types/capabilities/even-realities-g2.js.map +1 -1
  81. package/build/types/capabilities/mentra-display.d.ts.map +1 -1
  82. package/build/types/capabilities/mentra-display.js +1 -0
  83. package/build/types/capabilities/mentra-display.js.map +1 -1
  84. package/build/types/capabilities/nimo.d.ts.map +1 -1
  85. package/build/types/capabilities/nimo.js +1 -0
  86. package/build/types/capabilities/nimo.js.map +1 -1
  87. package/build/types/hardware.d.ts +9 -0
  88. package/build/types/hardware.d.ts.map +1 -1
  89. package/build/types/hardware.js +2 -0
  90. package/build/types/hardware.js.map +1 -1
  91. package/build/utils/miniappGlobals.js +1 -1
  92. package/build/utils/miniappGlobals.js.map +1 -1
  93. package/package.json +7 -7
  94. package/src/engine.ts +5 -0
  95. package/src/facades/dev.ts +1 -1
  96. package/src/facades/miniapps.ts +7 -7
  97. package/src/facades/speech.ts +12 -0
  98. package/src/generated/releaseMetadata.ts +3 -3
  99. package/src/index.ts +0 -2
  100. package/src/services/AppRegistry.ts +15 -27
  101. package/src/services/DevServerBridge.ts +2 -2
  102. package/src/services/DeviceEventRouter.ts +2 -0
  103. package/src/services/JSRouter.ts +1 -1
  104. package/src/services/LocalMiniappRuntime.ts +165 -66
  105. package/src/services/LocalTranslationService.ts +184 -0
  106. package/src/services/MiniappLauncher.ts +31 -34
  107. package/src/services/MiniappPauseCoordinator.ts +100 -0
  108. package/src/services/OfflineTranslationManager.ts +40 -0
  109. package/src/services/pages/OverlayPages.ts +5 -5
  110. package/src/services/pages/overlayApps.ts +19 -19
  111. package/src/services/settingsBridge.ts +12 -13
  112. package/src/stores/apps.ts +30 -58
  113. package/src/stores/settings.ts +0 -10
  114. package/src/types/applet.ts +3 -1
  115. package/src/types/capabilities/even-realities-g1.ts +1 -0
  116. package/src/types/capabilities/even-realities-g2.ts +1 -0
  117. package/src/types/capabilities/mentra-display.ts +1 -0
  118. package/src/types/capabilities/nimo.ts +1 -0
  119. package/src/types/hardware.ts +9 -0
  120. package/src/utils/miniappGlobals.ts +1 -1
  121. package/build/services/pages/activeDashboard.d.ts +0 -7
  122. package/build/services/pages/activeDashboard.d.ts.map +0 -1
  123. package/build/services/pages/activeDashboard.js +0 -21
  124. package/build/services/pages/activeDashboard.js.map +0 -1
  125. package/src/services/pages/activeDashboard.ts +0 -24
@@ -55,6 +55,9 @@ import {normalizeAiTier, resolveAiTier} from "./AiTierResolution"
55
55
  import type {AiTier, AiTierProvider, SavedLlmKeys} from "./AiTierResolution"
56
56
  import {HttpError} from "@glassly/cloud-client"
57
57
  import localDisplayManager from "./LocalDisplayManager"
58
+ import sttModelManager from "./STTModelManager"
59
+ import localTranslationService, {type LocalTranslationSubscription} from "./LocalTranslationService"
60
+ import {offlineTranslationSupported} from "./OfflineTranslationManager"
58
61
  import {
59
62
  dropGestureClaims,
60
63
  emptyGestureClaims,
@@ -78,7 +81,6 @@ import {
78
81
  type DerivedSettingsKey,
79
82
  } from "./settingsBridge"
80
83
  import {mayUsePermission} from "./miniappPermissions"
81
- import {activeDashboardAmong} from "./pages/activeDashboard"
82
84
  import {
83
85
  activeDashboard,
84
86
  addDashboardApp,
@@ -87,7 +89,6 @@ import {
87
89
  isOverlayApp,
88
90
  onOverlayAppsChange,
89
91
  removeDashboardApp,
90
- setActiveDashboard,
91
92
  setOverlayViewer,
92
93
  } from "./pages/overlayApps"
93
94
  import {pagesEligible} from "./pages/pageEligibility"
@@ -98,6 +99,7 @@ import glassesSensorService from "./GlassesSensorService"
98
99
  import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator"
99
100
  import speakerLabelResolver from "./SpeakerLabelResolver"
100
101
  import micStateCoordinator from "./MicStateCoordinator"
102
+ import {isMiniappsPaused, onMiniappsPauseChange} from "./MiniappPauseCoordinator"
101
103
  import sonioxDirectService from "./soniox/SonioxDirectService"
102
104
  import {BlobStore} from "./BlobStore"
103
105
  import {CloudAudioSubscriptionSync} from "./CloudAudioSubscriptionSync"
@@ -165,9 +167,9 @@ type SpeakerStateValue = "idle" | "loading" | "playing" | "stopped" | "error"
165
167
 
166
168
  interface ConnectedMiniapp {
167
169
  subscriptions: Set<string>
168
- /** Transcription streams this app explicitly requires to stay on-device. */
170
+ /** Transcription/translation streams this app explicitly requires to stay on-device. */
169
171
  forceLocalTranscriptionStreams: Set<string>
170
- /** Transcription streams with at least one listener using the default/cloud route. */
172
+ /** Transcription/translation streams with at least one listener using the default/cloud route. */
171
173
  cloudTranscriptionStreams: Set<string>
172
174
  sendMessage: (raw: string) => void
173
175
  lastPongAt: number
@@ -211,6 +213,10 @@ interface SpeechRun {
211
213
 
212
214
  type TranscriptionEventSource = "cloud" | "local"
213
215
 
216
+ /** Streams that carry speech text and so have a cloud route and an on-device route. */
217
+ const isSpeechTextStream = (stream: string): boolean =>
218
+ stream.startsWith("transcription:") || stream.startsWith("translation:")
219
+
214
220
  /**
215
221
  * Strictness ordering for `location_stream` rate values. Higher index =
216
222
  * higher accuracy and higher power cost. The aggregate tier applied to
@@ -598,6 +604,9 @@ class LocalMiniappRuntime {
598
604
  */
599
605
  private handshookApps: Set<string> = new Set()
600
606
 
607
+ /** Links that arrived before the target finished its handshake; sent right after CONNECT_ACK. */
608
+ private pendingLinks: Map<string, Record<string, string>> = new Map()
609
+
601
610
  /** Pending {@link waitForConnect} resolvers, keyed by packageName. */
602
611
  private connectWaiters: Map<string, Set<(err?: Error) => void>> = new Map()
603
612
 
@@ -612,6 +621,7 @@ class LocalMiniappRuntime {
612
621
  private cloudStatusWired = false
613
622
  private capabilitiesWired = false
614
623
  private preferencesWired = false
624
+ private pauseWired = false
615
625
  private settingsBridgeWired = false
616
626
  /** Last derived read-only values pushed on the settings stream, for change detection. */
617
627
  private derivedSettings = new Map<DerivedSettingsKey, string>()
@@ -1056,6 +1066,7 @@ class LocalMiniappRuntime {
1056
1066
  this.ensureCloudStatusWired()
1057
1067
  this.ensureCapabilitiesWired()
1058
1068
  this.ensurePreferencesWired()
1069
+ this.ensurePauseWired()
1059
1070
  this.ensureSettingsBridgeWired()
1060
1071
  this.ensurePingLoop()
1061
1072
 
@@ -1173,6 +1184,7 @@ class LocalMiniappRuntime {
1173
1184
  // (e.g. a wake that's mid-handshake when the app is torn down). Done before
1174
1185
  // the early-return so it runs even if the connectedApps entry is already gone.
1175
1186
  this.handshookApps.delete(packageName)
1187
+ this.pendingLinks.delete(packageName)
1176
1188
  this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`))
1177
1189
  // Drop this app's transcription hints; the cloud subscription rebuild
1178
1190
  // below (via replaceStreamSubscribers) re-derives the hint union without it.
@@ -1325,7 +1337,7 @@ class LocalMiniappRuntime {
1325
1337
  // Console-tap forwarding. The miniapp's console.log/warn/etc is wrapped
1326
1338
  // (via injected shim from miniappGlobals.ts) to post a `dev_log`
1327
1339
  // envelope. We fan out to two destinations:
1328
- // 1. DevServerBridge — forwards to the laptop's `glassly-miniapp dev`
1340
+ // 1. DevServerBridge — forwards to the laptop's `glassly dev`
1329
1341
  // terminal. No-op when there's no sidecar (installed miniapps).
1330
1342
  // 2. React Native console — surfaces the log in Metro / Xcode console
1331
1343
  // / adb logcat so installed-miniapp errors are still inspectable
@@ -1766,12 +1778,9 @@ class LocalMiniappRuntime {
1766
1778
  existing.lastPongAt = Date.now()
1767
1779
  existing.unansweredPingRounds = 0
1768
1780
 
1769
- // The dashboard that holds the role may take the overlay slot: its
1770
- // capabilities carry the dashboard view from the start.
1771
- if (existing.installedManifest?.type === "dashboard") {
1772
- addDashboardApp(packageName)
1773
- this.syncActiveDashboard()
1774
- }
1781
+ // The connected dashboard holds the role and may take the overlay slot:
1782
+ // its capabilities carry the dashboard view from the start.
1783
+ if (existing.installedManifest?.type === "dashboard") addDashboardApp(packageName)
1775
1784
  const capabilities = this.capabilitiesFor(packageName)
1776
1785
 
1777
1786
  // Build the declared-permission record for the SDK's session.permissions
@@ -1802,6 +1811,8 @@ class LocalMiniappRuntime {
1802
1811
  capabilities,
1803
1812
  preferences: this.currentPreferences(),
1804
1813
  visibility: this.currentVisibility(),
1814
+ paused: isMiniappsPaused(),
1815
+ ...(isMiniappsPaused() ? {pauseReason: "glasses_disconnected"} : {}),
1805
1816
  permissions: declaredPermissions,
1806
1817
  ...(existing.launchSource ? {launchSource: existing.launchSource} : {}),
1807
1818
  ...(initialAuth ? {auth: initialAuth} : {}),
@@ -1823,6 +1834,11 @@ class LocalMiniappRuntime {
1823
1834
  // Handshake complete — unblock any launcher.waitForConnect() callers.
1824
1835
  this.handshookApps.add(packageName)
1825
1836
  this.flushConnectWaiters(packageName)
1837
+ const pendingLink = this.pendingLinks.get(packageName)
1838
+ if (pendingLink) {
1839
+ this.pendingLinks.delete(packageName)
1840
+ this.sendToMiniapp(packageName, {type: MiniappResponseType.LINK, params: pendingLink})
1841
+ }
1826
1842
  }
1827
1843
 
1828
1844
  private async requestMiniappAuth(packageName: string, opts?: {minTtlMs?: number}): Promise<MiniappAuthToken | null> {
@@ -2102,15 +2118,15 @@ class LocalMiniappRuntime {
2102
2118
  }
2103
2119
  return "location_update"
2104
2120
  }
2105
- if (s.forceLocal === true && s.stream.startsWith("transcription:")) {
2121
+ if (s.forceLocal === true && isSpeechTextStream(s.stream)) {
2106
2122
  forceLocalTranscriptionStreams.add(s.stream)
2107
2123
  }
2108
- if (s.stream.startsWith("transcription:") && (s.forceLocal !== true || s.includeCloud === true)) {
2124
+ if (isSpeechTextStream(s.stream) && (s.forceLocal !== true || s.includeCloud === true)) {
2109
2125
  cloudTranscriptionStreams.add(s.stream)
2110
2126
  }
2111
2127
  return s.stream
2112
2128
  }
2113
- if (s.startsWith("transcription:")) cloudTranscriptionStreams.add(s)
2129
+ if (isSpeechTextStream(s)) cloudTranscriptionStreams.add(s)
2114
2130
  return s
2115
2131
  }) as string[] | undefined
2116
2132
  if (!Array.isArray(streams)) {
@@ -2538,8 +2554,6 @@ class LocalMiniappRuntime {
2538
2554
  return localDisplayManager.foregroundAppOnScreen()
2539
2555
  case "ring_battery":
2540
2556
  return ringBatteryFrom(useGlassesStore.getState())
2541
- case "active_dashboard":
2542
- return activeDashboard()
2543
2557
  }
2544
2558
  }
2545
2559
 
@@ -4849,7 +4863,7 @@ class LocalMiniappRuntime {
4849
4863
  const previousSubscriptions = app.subscriptions
4850
4864
  app.subscriptions = new Set(streams)
4851
4865
  app.forceLocalTranscriptionStreams.clear()
4852
- app.cloudTranscriptionStreams = new Set(streams.filter((stream) => stream.startsWith("transcription:")))
4866
+ app.cloudTranscriptionStreams = new Set(streams.filter(isSpeechTextStream))
4853
4867
  this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
4854
4868
 
4855
4869
  this.recomputeMicRequirements()
@@ -4871,6 +4885,15 @@ class LocalMiniappRuntime {
4871
4885
  }
4872
4886
  }
4873
4887
 
4888
+ /**
4889
+ * Subscriptions that hold mic and transcription demand right now. A paused
4890
+ * app keeps its subscriptions registered (they come back on resume) but
4891
+ * demands nothing while the glasses are away.
4892
+ */
4893
+ private activeStreamSubscribers(): Iterable<[string, Set<string>]> {
4894
+ return isMiniappsPaused() ? [] : this.streamSubscribers
4895
+ }
4896
+
4874
4897
  private recomputeMicRequirements(): void {
4875
4898
  // Soniox-direct (user API key) consumes decoded PCM on the phone, so
4876
4899
  // transcription/translation subscriptions request PCM instead of LC3 —
@@ -4878,7 +4901,7 @@ class LocalMiniappRuntime {
4878
4901
  const sonioxDirect = sonioxDirectService.isEnabled()
4879
4902
  let anyPcm = false
4880
4903
  let anyLc3 = false
4881
- for (const [stream, subscribers] of this.streamSubscribers) {
4904
+ for (const [stream, subscribers] of this.activeStreamSubscribers()) {
4882
4905
  if (subscribers.size === 0) continue
4883
4906
  if (stream === "audio_chunk") anyPcm = true
4884
4907
  if (stream.startsWith("transcription:") || stream.startsWith("translation:")) {
@@ -4906,12 +4929,25 @@ class LocalMiniappRuntime {
4906
4929
  let transcriptionLang: string | null = null
4907
4930
  let forceLocalTranscriptionLang: string | null = null
4908
4931
  let hasForceLocalTranscription = false
4909
- for (const [stream, subscribers] of this.streamSubscribers) {
4932
+ const localTranslations: LocalTranslationSubscription[] = []
4933
+ for (const [stream, subscribers] of this.activeStreamSubscribers()) {
4910
4934
  if (subscribers.size === 0) continue
4911
4935
  // Only transcription / translation need cloud delivery. Location and
4912
4936
  // notifications are sourced natively on the phone — no cloud hop.
4913
4937
  if (stream.startsWith("translation:")) {
4914
- cloudStreams.add(stream)
4938
+ const routes = summarizeTranscriptionRoutes(subscribers, (packageName) =>
4939
+ this.appTranscriptionRoutesForSubscription(packageName, stream),
4940
+ )
4941
+ if (routes.hasCloudSubscriber) cloudStreams.add(stream)
4942
+ const [, source, target, ...rest] = stream.split(":")
4943
+ if (target && target !== "*" && rest.length === 0) {
4944
+ localTranslations.push({
4945
+ source,
4946
+ target,
4947
+ cloud: routes.hasCloudSubscriber,
4948
+ forceLocal: routes.hasForceLocalSubscriber,
4949
+ })
4950
+ }
4915
4951
  }
4916
4952
  if (stream.startsWith("transcription:")) {
4917
4953
  const language = stream.substring("transcription:".length)
@@ -4927,10 +4963,17 @@ class LocalMiniappRuntime {
4927
4963
  if (routes.hasCloudSubscriber) cloudStreams.add(stream)
4928
4964
  }
4929
4965
  }
4966
+ // On-device translation is fed by on-device transcripts, so a translation
4967
+ // subscription it can serve keeps local STT eligible exactly like a
4968
+ // transcription subscription does.
4969
+ const translatesLocally = offlineTranslationSupported() ? localTranslations : []
4970
+ localTranslationService.setSubscriptions(translatesLocally)
4971
+ const hasForceLocalTranslation = translatesLocally.some((sub) => sub.forceLocal)
4972
+ const localSourceLang = translatesLocally.length > 0 ? sttModelManager.getBcp47ForLanguage() : null
4930
4973
  localSttFallbackCoordinator.onSubscriptionChange(
4931
- transcriptionLang !== null,
4932
- forceLocalTranscriptionLang ?? transcriptionLang,
4933
- hasForceLocalTranscription,
4974
+ transcriptionLang !== null || localSourceLang !== null,
4975
+ forceLocalTranscriptionLang ?? transcriptionLang ?? localSourceLang,
4976
+ hasForceLocalTranscription || hasForceLocalTranslation,
4934
4977
  )
4935
4978
 
4936
4979
  // Mirror the same set as typed AudioSubscription[]. When Soniox-direct is
@@ -5065,6 +5108,14 @@ class LocalMiniappRuntime {
5065
5108
  markTranscriptSourceRecent("direct")
5066
5109
  this.handleTranslationData(d)
5067
5110
  })
5111
+ localTranslationService.attach({
5112
+ deliver: (stream, data) => this.forwardEvent(stream, data, "local"),
5113
+ primaryUp: () => this.transcriptionPrimaryUp(),
5114
+ onPairUnavailable: () =>
5115
+ localDisplayManager.showSystemNotice(
5116
+ "Offline translation language not downloaded.\nGet it in Settings > Offline models.",
5117
+ ),
5118
+ })
5068
5119
  // When the user sets/clears their Soniox key mid-session, re-partition
5069
5120
  // subscriptions and mic requirements without waiting for a subscription
5070
5121
  // change.
@@ -5158,22 +5209,26 @@ class LocalMiniappRuntime {
5158
5209
  }
5159
5210
 
5160
5211
  private handleTranslationData(d: TranslationData): void {
5161
- this.forwardEvent(`translation:${d.source.language}:${d.target.language}`, {
5162
- type: "translation",
5163
- text: d.text,
5164
- originalText: d.originalText,
5165
- isFinal: d.isFinal,
5166
- utteranceId: d.utteranceId,
5167
- startTime: d.startMs,
5168
- endTime: d.endMs,
5169
- speakerId: d.speakerId,
5170
- duration: d.durationMs,
5171
- transcribeLanguage: d.source.language,
5172
- translateLanguage: d.target.language,
5173
- didTranslate: true,
5174
- provider: d.provider,
5175
- confidence: d.confidence,
5176
- })
5212
+ this.forwardEvent(
5213
+ `translation:${d.source.language}:${d.target.language}`,
5214
+ {
5215
+ type: "translation",
5216
+ text: d.text,
5217
+ originalText: d.originalText,
5218
+ isFinal: d.isFinal,
5219
+ utteranceId: d.utteranceId,
5220
+ startTime: d.startMs,
5221
+ endTime: d.endMs,
5222
+ speakerId: d.speakerId,
5223
+ duration: d.durationMs,
5224
+ transcribeLanguage: d.source.language,
5225
+ translateLanguage: d.target.language,
5226
+ didTranslate: true,
5227
+ provider: d.provider,
5228
+ confidence: d.confidence,
5229
+ },
5230
+ "cloud",
5231
+ )
5177
5232
  }
5178
5233
 
5179
5234
  /** Re-partition audio routing after a Soniox-direct enable/disable flip. */
@@ -5252,31 +5307,15 @@ class LocalMiniappRuntime {
5252
5307
  useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.cfw_native_navigation.key), broadcast)
5253
5308
  // The dashboard master switch flips the dashboard view.
5254
5309
  useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.custom_dashboard_enabled.key), broadcast)
5255
- // The wearer's choice and the installed dashboards decide who holds the role.
5256
- const syncRole = () => this.syncActiveDashboard()
5257
- useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.active_dashboard.key), syncRole)
5258
- useAppStatusStore.subscribe(syncRole)
5259
- syncRole()
5260
- }
5261
-
5262
- /**
5263
- * Recompute who holds the dashboard role (ADR-0009). The previous holder
5264
- * gives up the slot first, so it hears the view change while it is still
5265
- * the overlay app; then every app sees its views again.
5266
- */
5267
- private syncActiveDashboard(): void {
5268
- const candidates: Array<{packageName: string; type?: string}> = [...useAppStatusStore.getState().apps]
5269
- // A dashboard can connect at launch before the apps store has listed it.
5270
- for (const [packageName, app] of this.connectedApps) {
5271
- if (candidates.some((candidate) => candidate.packageName === packageName)) continue
5272
- candidates.push({packageName, type: app.installedManifest?.type})
5273
- }
5274
- const next = activeDashboardAmong(candidates)
5275
- const previous = activeDashboard()
5276
- if (next === previous) return
5277
- if (previous !== null) overlayPages.hide(previous, "replaced")
5278
- setActiveDashboard(next)
5279
- this.broadcastCapabilities()
5310
+ // The role follows the connected dashboard: when it passes, every app
5311
+ // sees its views again.
5312
+ let role = activeDashboard()
5313
+ onOverlayAppsChange(() => {
5314
+ const next = activeDashboard()
5315
+ if (next === role) return
5316
+ role = next
5317
+ broadcast()
5318
+ })
5280
5319
  }
5281
5320
 
5282
5321
  /** Capabilities as `packageName` sees them: the model profile plus the views this app may draw. */
@@ -5296,6 +5335,30 @@ class LocalMiniappRuntime {
5296
5335
  }
5297
5336
  }
5298
5337
 
5338
+ /**
5339
+ * Pause/resume (MiniappPauseCoordinator) lands as one push per handshook
5340
+ * app, and the mic and cloud transcription demand follow it.
5341
+ */
5342
+ private ensurePauseWired(): void {
5343
+ if (this.pauseWired) return
5344
+ this.pauseWired = true
5345
+ onMiniappsPauseChange((paused) => {
5346
+ this.broadcastPauseState(paused)
5347
+ this.recomputeMicRequirements()
5348
+ this.updateCloudSubscriptions()
5349
+ })
5350
+ }
5351
+
5352
+ private broadcastPauseState(paused: boolean): void {
5353
+ for (const packageName of this.handshookApps) {
5354
+ this.sendToMiniapp(packageName, {
5355
+ type: MiniappResponseType.PAUSE_CHANGE,
5356
+ paused,
5357
+ ...(paused ? {reason: "glasses_disconnected"} : {}),
5358
+ })
5359
+ }
5360
+ }
5361
+
5299
5362
  /** Host user preferences miniapps follow by default (Settings > Appearance). */
5300
5363
  private currentPreferences(): MiniappPreferences {
5301
5364
  const settings = useSettingsStore.getState()
@@ -5367,6 +5430,10 @@ class LocalMiniappRuntime {
5367
5430
  /** Deliver to this subscriber alone (the designated dashboard app's gestures while it shows). */
5368
5431
  only?: string,
5369
5432
  ): void {
5433
+ // Paused miniapps (glasses disconnected) hear no streams. The resume push
5434
+ // lands before the reconnect's own glasses_connection event, so nothing
5435
+ // a resumed app needs is lost here.
5436
+ if (isMiniappsPaused()) return
5370
5437
  // Local sherpa results enter the runtime only through here (DeviceEventRouter's
5371
5438
  // local_transcription listener); cloud/direct are marked at their onTranscript
5372
5439
  // wiring, which never passes "local".
@@ -5395,6 +5462,7 @@ class LocalMiniappRuntime {
5395
5462
  // translation:<source>:<target> — exact (TranslationModule.fromTo)
5396
5463
  // translation:auto — back-compat alias for *:*
5397
5464
  const matchedSubs = new Set<string>()
5465
+ let translationPatterns: string[] = []
5398
5466
  const exact = this.streamSubscribers.get(normalizedStream)
5399
5467
  if (exact) for (const p of exact) matchedSubs.add(p)
5400
5468
 
@@ -5407,13 +5475,13 @@ class LocalMiniappRuntime {
5407
5475
  // parts[0] = "translation", parts[1] = source, parts[2] = target
5408
5476
  if (parts.length === 3) {
5409
5477
  const [, source, target] = parts
5410
- const patterns = [
5478
+ translationPatterns = [
5411
5479
  "translation:auto", // legacy alias for *:*
5412
5480
  "translation:*:*", // cloud v3 "all pairs"
5413
5481
  `translation:*:${target}`, // cloud v3 "to <target>"
5414
5482
  `translation:${source}:*`, // cloud v3 "from <source>"
5415
5483
  ]
5416
- for (const pat of patterns) {
5484
+ for (const pat of translationPatterns) {
5417
5485
  const subs = this.streamSubscribers.get(pat)
5418
5486
  if (subs) for (const p of subs) matchedSubs.add(p)
5419
5487
  }
@@ -5481,6 +5549,11 @@ class LocalMiniappRuntime {
5481
5549
  transcriptionRoute =
5482
5550
  transcriptionDeliveryRoute(transcriptionSource, routes, this.transcriptionPrimaryUp()) ?? undefined
5483
5551
  if (!transcriptionRoute) continue
5552
+ } else if (normalizedStream.startsWith("translation:") && transcriptionSource) {
5553
+ const routes = this.appTranslationRoutesForEvent(packageName, [normalizedStream, ...translationPatterns])
5554
+ transcriptionRoute =
5555
+ transcriptionDeliveryRoute(transcriptionSource, routes, this.transcriptionPrimaryUp()) ?? undefined
5556
+ if (!transcriptionRoute) continue
5484
5557
  }
5485
5558
  if (TRANSCRIPT_TIMING_TELEMETRY && normalizedStream.startsWith("transcription:")) {
5486
5559
  const transcript = data as {text?: string; isFinal?: boolean; __hostReceivedAt?: number} | null
@@ -5579,6 +5652,20 @@ class LocalMiniappRuntime {
5579
5652
  }
5580
5653
  }
5581
5654
 
5655
+ private appTranslationRoutesForEvent(
5656
+ packageName: string,
5657
+ patterns: string[],
5658
+ ): {
5659
+ cloud: boolean
5660
+ forceLocal: boolean
5661
+ } {
5662
+ const app = this.connectedApps.get(packageName)
5663
+ return {
5664
+ cloud: patterns.some((pattern) => app?.cloudTranscriptionStreams.has(pattern) === true),
5665
+ forceLocal: patterns.some((pattern) => app?.forceLocalTranscriptionStreams.has(pattern) === true),
5666
+ }
5667
+ }
5668
+
5582
5669
  private appTranscriptionRoutesForEvent(
5583
5670
  packageName: string,
5584
5671
  stream: string,
@@ -6148,6 +6235,18 @@ class LocalMiniappRuntime {
6148
6235
  )
6149
6236
  }
6150
6237
 
6238
+ /**
6239
+ * Hand an external link's query parameters to a miniapp. A miniapp that is
6240
+ * still spawning gets them once its handshake completes.
6241
+ */
6242
+ public deliverLink(packageName: string, params: Record<string, string>): void {
6243
+ if (!this.handshookApps.has(packageName)) {
6244
+ this.pendingLinks.set(packageName, params)
6245
+ return
6246
+ }
6247
+ this.sendToMiniapp(packageName, {type: MiniappResponseType.LINK, params})
6248
+ }
6249
+
6151
6250
  /**
6152
6251
  * Send a VISIBILITY_CHANGE push to a miniapp.
6153
6252
  */
@@ -0,0 +1,184 @@
1
+ import {offlineTranslationManager} from "./OfflineTranslationManager"
2
+
3
+ /** One concrete-target `translation:<source>:<target>` stream and who listens to it. */
4
+ export interface LocalTranslationSubscription {
5
+ /** Subscribed source: a language code, or `*` / `auto` for any. */
6
+ source: string
7
+ target: string
8
+ /** At least one listener on the default (cloud) route. */
9
+ cloud: boolean
10
+ /** At least one listener that requires on-device translation. */
11
+ forceLocal: boolean
12
+ }
13
+
14
+ export interface LocalTranscript {
15
+ text: string
16
+ isFinal?: boolean
17
+ speakerId?: number | string
18
+ startTime?: number
19
+ endTime?: number
20
+ }
21
+
22
+ interface Host {
23
+ /** Fan a `translation:<source>:<target>` event out to miniapps. */
24
+ deliver: (stream: string, data: Record<string, unknown>) => void
25
+ /** Whether the primary (cloud / Soniox-direct) path is up. */
26
+ primaryUp: () => boolean
27
+ /** A wanted pair has no language pack on the device. */
28
+ onPairUnavailable?: (source: string, target: string) => void
29
+ }
30
+
31
+ interface Job {
32
+ text: string
33
+ isFinal: boolean
34
+ utteranceId: string
35
+ transcript: LocalTranscript
36
+ }
37
+
38
+ interface Lane {
39
+ busy: boolean
40
+ finals: Job[]
41
+ pendingInterim: Job | null
42
+ lastInterimAt: number
43
+ retryAt: number
44
+ reportedUnavailable: boolean
45
+ }
46
+
47
+ /** Interims are translated at most this often; finals always are. */
48
+ const INTERIM_MIN_INTERVAL_MS = 300
49
+ const FAILURE_BACKOFF_MS = 5000
50
+
51
+ const baseLanguage = (code: string): string => code.split("-")[0].toLowerCase()
52
+
53
+ /**
54
+ * Turns on-device transcripts into on-device translations. Apple's Translation
55
+ * framework has no streaming API, so each update is a whole `translate` call:
56
+ * one runs at a time per language pair, a newer interim replaces a waiting
57
+ * one, and finals are never dropped.
58
+ *
59
+ * A pair is translated here when a subscriber forces local, or when the
60
+ * primary path is down and would otherwise leave default subscribers silent.
61
+ */
62
+ export class LocalTranslationService {
63
+ private subscriptions: LocalTranslationSubscription[] = []
64
+ private lanes = new Map<string, Lane>()
65
+ private host: Host | null = null
66
+ private utteranceCounter = 0
67
+
68
+ attach(host: Host): void {
69
+ this.host = host
70
+ }
71
+
72
+ setSubscriptions(subscriptions: LocalTranslationSubscription[]): void {
73
+ this.subscriptions = subscriptions
74
+ const live = new Set(subscriptions.map((s) => s.target))
75
+ for (const key of this.lanes.keys()) {
76
+ if (!live.has(key.substring(key.indexOf(">") + 1))) this.lanes.delete(key)
77
+ }
78
+ }
79
+
80
+ onLocalTranscript(transcript: LocalTranscript, sourceLanguage: string): void {
81
+ const text = transcript.text?.trim()
82
+ if (!text || !this.host || !offlineTranslationManager.isSupported()) return
83
+ const isFinal = transcript.isFinal === true
84
+ const utteranceId = `local-${this.utteranceCounter}`
85
+ if (isFinal) this.utteranceCounter++
86
+
87
+ for (const target of this.wantedTargets(sourceLanguage)) {
88
+ const job: Job = {text, isFinal, utteranceId, transcript}
89
+ if (baseLanguage(target) === baseLanguage(sourceLanguage)) {
90
+ // Same-language passthrough, as the cloud emits it: no originalText.
91
+ this.emit(sourceLanguage, target, job, text, undefined)
92
+ continue
93
+ }
94
+ this.enqueue(sourceLanguage, target, job)
95
+ }
96
+ }
97
+
98
+ private wantedTargets(sourceLanguage: string): Set<string> {
99
+ const targets = new Set<string>()
100
+ let primaryUp: boolean | null = null
101
+ for (const sub of this.subscriptions) {
102
+ const anySource = sub.source === "*" || sub.source === "auto"
103
+ if (!anySource && baseLanguage(sub.source) !== baseLanguage(sourceLanguage)) continue
104
+ if (!sub.forceLocal) {
105
+ primaryUp ??= this.host!.primaryUp()
106
+ if (primaryUp || !sub.cloud) continue
107
+ }
108
+ targets.add(sub.target)
109
+ }
110
+ return targets
111
+ }
112
+
113
+ private enqueue(source: string, target: string, job: Job): void {
114
+ const key = `${source}>${target}`
115
+ let lane = this.lanes.get(key)
116
+ if (!lane) {
117
+ lane = {busy: false, finals: [], pendingInterim: null, lastInterimAt: 0, retryAt: 0, reportedUnavailable: false}
118
+ this.lanes.set(key, lane)
119
+ }
120
+ const now = Date.now()
121
+ if (now < lane.retryAt) return
122
+ if (job.isFinal) {
123
+ lane.pendingInterim = null
124
+ lane.finals.push(job)
125
+ } else {
126
+ if (now - lane.lastInterimAt < INTERIM_MIN_INTERVAL_MS) return
127
+ lane.lastInterimAt = now
128
+ lane.pendingInterim = job
129
+ }
130
+ if (!lane.busy) void this.drain(source, target, lane)
131
+ }
132
+
133
+ private async drain(source: string, target: string, lane: Lane): Promise<void> {
134
+ lane.busy = true
135
+ try {
136
+ for (;;) {
137
+ const job = lane.finals.shift() ?? lane.pendingInterim
138
+ if (!job) break
139
+ if (!job.isFinal) lane.pendingInterim = null
140
+ try {
141
+ const translated = await offlineTranslationManager.translate(job.text, source, target)
142
+ this.emit(source, target, job, translated, job.text)
143
+ } catch (err) {
144
+ this.handleFailure(source, target, lane, err)
145
+ break
146
+ }
147
+ }
148
+ } finally {
149
+ lane.busy = false
150
+ }
151
+ }
152
+
153
+ private handleFailure(source: string, target: string, lane: Lane, err: unknown): void {
154
+ const message = (err as Error)?.message ?? String(err)
155
+ console.warn(`[LocalTranslation] ${source}>${target} failed: ${message}`)
156
+ lane.finals = []
157
+ lane.pendingInterim = null
158
+ lane.retryAt = Date.now() + FAILURE_BACKOFF_MS
159
+ if (message.includes("not_installed") && !lane.reportedUnavailable) {
160
+ lane.reportedUnavailable = true
161
+ this.host?.onPairUnavailable?.(source, target)
162
+ }
163
+ }
164
+
165
+ private emit(source: string, target: string, job: Job, text: string, originalText: string | undefined): void {
166
+ this.host?.deliver(`translation:${source}:${target}`, {
167
+ type: "translation",
168
+ text,
169
+ originalText,
170
+ isFinal: job.isFinal,
171
+ utteranceId: job.utteranceId,
172
+ startTime: job.transcript.startTime,
173
+ endTime: job.transcript.endTime,
174
+ speakerId: job.transcript.speakerId,
175
+ transcribeLanguage: source,
176
+ translateLanguage: target,
177
+ didTranslate: originalText !== undefined,
178
+ provider: "apple-translation",
179
+ })
180
+ }
181
+ }
182
+
183
+ const localTranslationService = new LocalTranslationService()
184
+ export default localTranslationService