@glassly/engine 0.3.1-dev.494 → 0.3.1-dev.523
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/engine.d.ts +9 -2
- package/build/engine.d.ts.map +1 -1
- package/build/engine.js +5 -0
- package/build/engine.js.map +1 -1
- package/build/facades/dev.js +1 -1
- package/build/facades/dev.js.map +1 -1
- package/build/facades/miniapps.d.ts +5 -5
- package/build/facades/miniapps.d.ts.map +1 -1
- package/build/facades/miniapps.js +5 -6
- package/build/facades/miniapps.js.map +1 -1
- package/build/facades/speech.d.ts +10 -0
- package/build/facades/speech.d.ts.map +1 -1
- package/build/facades/speech.js +10 -0
- package/build/facades/speech.js.map +1 -1
- package/build/generated/releaseMetadata.js +3 -3
- package/build/generated/releaseMetadata.js.map +1 -1
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +1 -1
- package/build/index.js.map +1 -1
- package/build/services/AppRegistry.d.ts +1 -7
- package/build/services/AppRegistry.d.ts.map +1 -1
- package/build/services/AppRegistry.js +15 -26
- package/build/services/AppRegistry.js.map +1 -1
- package/build/services/DevServerBridge.d.ts +1 -1
- package/build/services/DevServerBridge.js +2 -2
- package/build/services/DevServerBridge.js.map +1 -1
- package/build/services/DeviceEventRouter.d.ts.map +1 -1
- package/build/services/DeviceEventRouter.js +2 -0
- package/build/services/DeviceEventRouter.js.map +1 -1
- package/build/services/JSRouter.js +1 -1
- package/build/services/JSRouter.js.map +1 -1
- package/build/services/LocalMiniappRuntime.d.ts +21 -6
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +130 -48
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/LocalTranslationService.d.ts +51 -0
- package/build/services/LocalTranslationService.d.ts.map +1 -0
- package/build/services/LocalTranslationService.js +141 -0
- package/build/services/LocalTranslationService.js.map +1 -0
- package/build/services/MiniappLauncher.d.ts +17 -23
- package/build/services/MiniappLauncher.d.ts.map +1 -1
- package/build/services/MiniappLauncher.js +30 -34
- package/build/services/MiniappLauncher.js.map +1 -1
- package/build/services/MiniappPauseCoordinator.d.ts +15 -0
- package/build/services/MiniappPauseCoordinator.d.ts.map +1 -0
- package/build/services/MiniappPauseCoordinator.js +93 -0
- package/build/services/MiniappPauseCoordinator.js.map +1 -0
- package/build/services/OfflineTranslationManager.d.ts +19 -0
- package/build/services/OfflineTranslationManager.d.ts.map +1 -0
- package/build/services/OfflineTranslationManager.js +35 -0
- package/build/services/OfflineTranslationManager.js.map +1 -0
- package/build/services/pages/OverlayPages.d.ts +5 -5
- package/build/services/pages/OverlayPages.d.ts.map +1 -1
- package/build/services/pages/OverlayPages.js +5 -5
- package/build/services/pages/OverlayPages.js.map +1 -1
- package/build/services/pages/overlayApps.d.ts +14 -11
- package/build/services/pages/overlayApps.d.ts.map +1 -1
- package/build/services/pages/overlayApps.js +21 -19
- package/build/services/pages/overlayApps.js.map +1 -1
- package/build/services/settingsBridge.d.ts +8 -8
- package/build/services/settingsBridge.d.ts.map +1 -1
- package/build/services/settingsBridge.js +12 -13
- package/build/services/settingsBridge.js.map +1 -1
- package/build/stores/apps.d.ts +0 -5
- package/build/stores/apps.d.ts.map +1 -1
- package/build/stores/apps.js +32 -52
- package/build/stores/apps.js.map +1 -1
- package/build/stores/settings.d.ts.map +1 -1
- package/build/stores/settings.js +0 -10
- package/build/stores/settings.js.map +1 -1
- package/build/types/applet.d.ts +3 -1
- package/build/types/applet.d.ts.map +1 -1
- package/build/types/applet.js.map +1 -1
- package/build/types/capabilities/even-realities-g1.d.ts.map +1 -1
- package/build/types/capabilities/even-realities-g1.js +1 -0
- package/build/types/capabilities/even-realities-g1.js.map +1 -1
- package/build/types/capabilities/even-realities-g2.d.ts.map +1 -1
- package/build/types/capabilities/even-realities-g2.js +1 -0
- package/build/types/capabilities/even-realities-g2.js.map +1 -1
- package/build/types/capabilities/mentra-display.d.ts.map +1 -1
- package/build/types/capabilities/mentra-display.js +1 -0
- package/build/types/capabilities/mentra-display.js.map +1 -1
- package/build/types/capabilities/nimo.d.ts.map +1 -1
- package/build/types/capabilities/nimo.js +1 -0
- package/build/types/capabilities/nimo.js.map +1 -1
- package/build/types/hardware.d.ts +9 -0
- package/build/types/hardware.d.ts.map +1 -1
- package/build/types/hardware.js +2 -0
- package/build/types/hardware.js.map +1 -1
- package/build/utils/miniappGlobals.js +1 -1
- package/build/utils/miniappGlobals.js.map +1 -1
- package/package.json +7 -7
- package/src/engine.ts +5 -0
- package/src/facades/dev.ts +1 -1
- package/src/facades/miniapps.ts +7 -7
- package/src/facades/speech.ts +12 -0
- package/src/generated/releaseMetadata.ts +3 -3
- package/src/index.ts +0 -2
- package/src/services/AppRegistry.ts +15 -27
- package/src/services/DevServerBridge.ts +2 -2
- package/src/services/DeviceEventRouter.ts +2 -0
- package/src/services/JSRouter.ts +1 -1
- package/src/services/LocalMiniappRuntime.ts +165 -66
- package/src/services/LocalTranslationService.ts +184 -0
- package/src/services/MiniappLauncher.ts +31 -34
- package/src/services/MiniappPauseCoordinator.ts +100 -0
- package/src/services/OfflineTranslationManager.ts +40 -0
- package/src/services/pages/OverlayPages.ts +5 -5
- package/src/services/pages/overlayApps.ts +19 -19
- package/src/services/settingsBridge.ts +12 -13
- package/src/stores/apps.ts +30 -58
- package/src/stores/settings.ts +0 -10
- package/src/types/applet.ts +3 -1
- package/src/types/capabilities/even-realities-g1.ts +1 -0
- package/src/types/capabilities/even-realities-g2.ts +1 -0
- package/src/types/capabilities/mentra-display.ts +1 -0
- package/src/types/capabilities/nimo.ts +1 -0
- package/src/types/hardware.ts +9 -0
- package/src/utils/miniappGlobals.ts +1 -1
- package/build/services/pages/activeDashboard.d.ts +0 -7
- package/build/services/pages/activeDashboard.d.ts.map +0 -1
- package/build/services/pages/activeDashboard.js +0 -21
- package/build/services/pages/activeDashboard.js.map +0 -1
- package/src/services/pages/activeDashboard.ts +0 -24
|
@@ -55,6 +55,9 @@ import {normalizeAiTier, resolveAiTier} from "./AiTierResolution"
|
|
|
55
55
|
import type {AiTier, AiTierProvider, SavedLlmKeys} from "./AiTierResolution"
|
|
56
56
|
import {HttpError} from "@glassly/cloud-client"
|
|
57
57
|
import localDisplayManager from "./LocalDisplayManager"
|
|
58
|
+
import sttModelManager from "./STTModelManager"
|
|
59
|
+
import localTranslationService, {type LocalTranslationSubscription} from "./LocalTranslationService"
|
|
60
|
+
import {offlineTranslationSupported} from "./OfflineTranslationManager"
|
|
58
61
|
import {
|
|
59
62
|
dropGestureClaims,
|
|
60
63
|
emptyGestureClaims,
|
|
@@ -78,7 +81,6 @@ import {
|
|
|
78
81
|
type DerivedSettingsKey,
|
|
79
82
|
} from "./settingsBridge"
|
|
80
83
|
import {mayUsePermission} from "./miniappPermissions"
|
|
81
|
-
import {activeDashboardAmong} from "./pages/activeDashboard"
|
|
82
84
|
import {
|
|
83
85
|
activeDashboard,
|
|
84
86
|
addDashboardApp,
|
|
@@ -87,7 +89,6 @@ import {
|
|
|
87
89
|
isOverlayApp,
|
|
88
90
|
onOverlayAppsChange,
|
|
89
91
|
removeDashboardApp,
|
|
90
|
-
setActiveDashboard,
|
|
91
92
|
setOverlayViewer,
|
|
92
93
|
} from "./pages/overlayApps"
|
|
93
94
|
import {pagesEligible} from "./pages/pageEligibility"
|
|
@@ -98,6 +99,7 @@ import glassesSensorService from "./GlassesSensorService"
|
|
|
98
99
|
import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator"
|
|
99
100
|
import speakerLabelResolver from "./SpeakerLabelResolver"
|
|
100
101
|
import micStateCoordinator from "./MicStateCoordinator"
|
|
102
|
+
import {isMiniappsPaused, onMiniappsPauseChange} from "./MiniappPauseCoordinator"
|
|
101
103
|
import sonioxDirectService from "./soniox/SonioxDirectService"
|
|
102
104
|
import {BlobStore} from "./BlobStore"
|
|
103
105
|
import {CloudAudioSubscriptionSync} from "./CloudAudioSubscriptionSync"
|
|
@@ -165,9 +167,9 @@ type SpeakerStateValue = "idle" | "loading" | "playing" | "stopped" | "error"
|
|
|
165
167
|
|
|
166
168
|
interface ConnectedMiniapp {
|
|
167
169
|
subscriptions: Set<string>
|
|
168
|
-
/** Transcription streams this app explicitly requires to stay on-device. */
|
|
170
|
+
/** Transcription/translation streams this app explicitly requires to stay on-device. */
|
|
169
171
|
forceLocalTranscriptionStreams: Set<string>
|
|
170
|
-
/** Transcription streams with at least one listener using the default/cloud route. */
|
|
172
|
+
/** Transcription/translation streams with at least one listener using the default/cloud route. */
|
|
171
173
|
cloudTranscriptionStreams: Set<string>
|
|
172
174
|
sendMessage: (raw: string) => void
|
|
173
175
|
lastPongAt: number
|
|
@@ -211,6 +213,10 @@ interface SpeechRun {
|
|
|
211
213
|
|
|
212
214
|
type TranscriptionEventSource = "cloud" | "local"
|
|
213
215
|
|
|
216
|
+
/** Streams that carry speech text and so have a cloud route and an on-device route. */
|
|
217
|
+
const isSpeechTextStream = (stream: string): boolean =>
|
|
218
|
+
stream.startsWith("transcription:") || stream.startsWith("translation:")
|
|
219
|
+
|
|
214
220
|
/**
|
|
215
221
|
* Strictness ordering for `location_stream` rate values. Higher index =
|
|
216
222
|
* higher accuracy and higher power cost. The aggregate tier applied to
|
|
@@ -598,6 +604,9 @@ class LocalMiniappRuntime {
|
|
|
598
604
|
*/
|
|
599
605
|
private handshookApps: Set<string> = new Set()
|
|
600
606
|
|
|
607
|
+
/** Links that arrived before the target finished its handshake; sent right after CONNECT_ACK. */
|
|
608
|
+
private pendingLinks: Map<string, Record<string, string>> = new Map()
|
|
609
|
+
|
|
601
610
|
/** Pending {@link waitForConnect} resolvers, keyed by packageName. */
|
|
602
611
|
private connectWaiters: Map<string, Set<(err?: Error) => void>> = new Map()
|
|
603
612
|
|
|
@@ -612,6 +621,7 @@ class LocalMiniappRuntime {
|
|
|
612
621
|
private cloudStatusWired = false
|
|
613
622
|
private capabilitiesWired = false
|
|
614
623
|
private preferencesWired = false
|
|
624
|
+
private pauseWired = false
|
|
615
625
|
private settingsBridgeWired = false
|
|
616
626
|
/** Last derived read-only values pushed on the settings stream, for change detection. */
|
|
617
627
|
private derivedSettings = new Map<DerivedSettingsKey, string>()
|
|
@@ -1056,6 +1066,7 @@ class LocalMiniappRuntime {
|
|
|
1056
1066
|
this.ensureCloudStatusWired()
|
|
1057
1067
|
this.ensureCapabilitiesWired()
|
|
1058
1068
|
this.ensurePreferencesWired()
|
|
1069
|
+
this.ensurePauseWired()
|
|
1059
1070
|
this.ensureSettingsBridgeWired()
|
|
1060
1071
|
this.ensurePingLoop()
|
|
1061
1072
|
|
|
@@ -1173,6 +1184,7 @@ class LocalMiniappRuntime {
|
|
|
1173
1184
|
// (e.g. a wake that's mid-handshake when the app is torn down). Done before
|
|
1174
1185
|
// the early-return so it runs even if the connectedApps entry is already gone.
|
|
1175
1186
|
this.handshookApps.delete(packageName)
|
|
1187
|
+
this.pendingLinks.delete(packageName)
|
|
1176
1188
|
this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`))
|
|
1177
1189
|
// Drop this app's transcription hints; the cloud subscription rebuild
|
|
1178
1190
|
// below (via replaceStreamSubscribers) re-derives the hint union without it.
|
|
@@ -1325,7 +1337,7 @@ class LocalMiniappRuntime {
|
|
|
1325
1337
|
// Console-tap forwarding. The miniapp's console.log/warn/etc is wrapped
|
|
1326
1338
|
// (via injected shim from miniappGlobals.ts) to post a `dev_log`
|
|
1327
1339
|
// envelope. We fan out to two destinations:
|
|
1328
|
-
// 1. DevServerBridge — forwards to the laptop's `glassly
|
|
1340
|
+
// 1. DevServerBridge — forwards to the laptop's `glassly dev`
|
|
1329
1341
|
// terminal. No-op when there's no sidecar (installed miniapps).
|
|
1330
1342
|
// 2. React Native console — surfaces the log in Metro / Xcode console
|
|
1331
1343
|
// / adb logcat so installed-miniapp errors are still inspectable
|
|
@@ -1766,12 +1778,9 @@ class LocalMiniappRuntime {
|
|
|
1766
1778
|
existing.lastPongAt = Date.now()
|
|
1767
1779
|
existing.unansweredPingRounds = 0
|
|
1768
1780
|
|
|
1769
|
-
// The dashboard
|
|
1770
|
-
// capabilities carry the dashboard view from the start.
|
|
1771
|
-
if (existing.installedManifest?.type === "dashboard")
|
|
1772
|
-
addDashboardApp(packageName)
|
|
1773
|
-
this.syncActiveDashboard()
|
|
1774
|
-
}
|
|
1781
|
+
// The connected dashboard holds the role and may take the overlay slot:
|
|
1782
|
+
// its capabilities carry the dashboard view from the start.
|
|
1783
|
+
if (existing.installedManifest?.type === "dashboard") addDashboardApp(packageName)
|
|
1775
1784
|
const capabilities = this.capabilitiesFor(packageName)
|
|
1776
1785
|
|
|
1777
1786
|
// Build the declared-permission record for the SDK's session.permissions
|
|
@@ -1802,6 +1811,8 @@ class LocalMiniappRuntime {
|
|
|
1802
1811
|
capabilities,
|
|
1803
1812
|
preferences: this.currentPreferences(),
|
|
1804
1813
|
visibility: this.currentVisibility(),
|
|
1814
|
+
paused: isMiniappsPaused(),
|
|
1815
|
+
...(isMiniappsPaused() ? {pauseReason: "glasses_disconnected"} : {}),
|
|
1805
1816
|
permissions: declaredPermissions,
|
|
1806
1817
|
...(existing.launchSource ? {launchSource: existing.launchSource} : {}),
|
|
1807
1818
|
...(initialAuth ? {auth: initialAuth} : {}),
|
|
@@ -1823,6 +1834,11 @@ class LocalMiniappRuntime {
|
|
|
1823
1834
|
// Handshake complete — unblock any launcher.waitForConnect() callers.
|
|
1824
1835
|
this.handshookApps.add(packageName)
|
|
1825
1836
|
this.flushConnectWaiters(packageName)
|
|
1837
|
+
const pendingLink = this.pendingLinks.get(packageName)
|
|
1838
|
+
if (pendingLink) {
|
|
1839
|
+
this.pendingLinks.delete(packageName)
|
|
1840
|
+
this.sendToMiniapp(packageName, {type: MiniappResponseType.LINK, params: pendingLink})
|
|
1841
|
+
}
|
|
1826
1842
|
}
|
|
1827
1843
|
|
|
1828
1844
|
private async requestMiniappAuth(packageName: string, opts?: {minTtlMs?: number}): Promise<MiniappAuthToken | null> {
|
|
@@ -2102,15 +2118,15 @@ class LocalMiniappRuntime {
|
|
|
2102
2118
|
}
|
|
2103
2119
|
return "location_update"
|
|
2104
2120
|
}
|
|
2105
|
-
if (s.forceLocal === true && s.stream
|
|
2121
|
+
if (s.forceLocal === true && isSpeechTextStream(s.stream)) {
|
|
2106
2122
|
forceLocalTranscriptionStreams.add(s.stream)
|
|
2107
2123
|
}
|
|
2108
|
-
if (s.stream
|
|
2124
|
+
if (isSpeechTextStream(s.stream) && (s.forceLocal !== true || s.includeCloud === true)) {
|
|
2109
2125
|
cloudTranscriptionStreams.add(s.stream)
|
|
2110
2126
|
}
|
|
2111
2127
|
return s.stream
|
|
2112
2128
|
}
|
|
2113
|
-
if (s
|
|
2129
|
+
if (isSpeechTextStream(s)) cloudTranscriptionStreams.add(s)
|
|
2114
2130
|
return s
|
|
2115
2131
|
}) as string[] | undefined
|
|
2116
2132
|
if (!Array.isArray(streams)) {
|
|
@@ -2538,8 +2554,6 @@ class LocalMiniappRuntime {
|
|
|
2538
2554
|
return localDisplayManager.foregroundAppOnScreen()
|
|
2539
2555
|
case "ring_battery":
|
|
2540
2556
|
return ringBatteryFrom(useGlassesStore.getState())
|
|
2541
|
-
case "active_dashboard":
|
|
2542
|
-
return activeDashboard()
|
|
2543
2557
|
}
|
|
2544
2558
|
}
|
|
2545
2559
|
|
|
@@ -4849,7 +4863,7 @@ class LocalMiniappRuntime {
|
|
|
4849
4863
|
const previousSubscriptions = app.subscriptions
|
|
4850
4864
|
app.subscriptions = new Set(streams)
|
|
4851
4865
|
app.forceLocalTranscriptionStreams.clear()
|
|
4852
|
-
app.cloudTranscriptionStreams = new Set(streams.filter(
|
|
4866
|
+
app.cloudTranscriptionStreams = new Set(streams.filter(isSpeechTextStream))
|
|
4853
4867
|
this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
|
|
4854
4868
|
|
|
4855
4869
|
this.recomputeMicRequirements()
|
|
@@ -4871,6 +4885,15 @@ class LocalMiniappRuntime {
|
|
|
4871
4885
|
}
|
|
4872
4886
|
}
|
|
4873
4887
|
|
|
4888
|
+
/**
|
|
4889
|
+
* Subscriptions that hold mic and transcription demand right now. A paused
|
|
4890
|
+
* app keeps its subscriptions registered (they come back on resume) but
|
|
4891
|
+
* demands nothing while the glasses are away.
|
|
4892
|
+
*/
|
|
4893
|
+
private activeStreamSubscribers(): Iterable<[string, Set<string>]> {
|
|
4894
|
+
return isMiniappsPaused() ? [] : this.streamSubscribers
|
|
4895
|
+
}
|
|
4896
|
+
|
|
4874
4897
|
private recomputeMicRequirements(): void {
|
|
4875
4898
|
// Soniox-direct (user API key) consumes decoded PCM on the phone, so
|
|
4876
4899
|
// transcription/translation subscriptions request PCM instead of LC3 —
|
|
@@ -4878,7 +4901,7 @@ class LocalMiniappRuntime {
|
|
|
4878
4901
|
const sonioxDirect = sonioxDirectService.isEnabled()
|
|
4879
4902
|
let anyPcm = false
|
|
4880
4903
|
let anyLc3 = false
|
|
4881
|
-
for (const [stream, subscribers] of this.
|
|
4904
|
+
for (const [stream, subscribers] of this.activeStreamSubscribers()) {
|
|
4882
4905
|
if (subscribers.size === 0) continue
|
|
4883
4906
|
if (stream === "audio_chunk") anyPcm = true
|
|
4884
4907
|
if (stream.startsWith("transcription:") || stream.startsWith("translation:")) {
|
|
@@ -4906,12 +4929,25 @@ class LocalMiniappRuntime {
|
|
|
4906
4929
|
let transcriptionLang: string | null = null
|
|
4907
4930
|
let forceLocalTranscriptionLang: string | null = null
|
|
4908
4931
|
let hasForceLocalTranscription = false
|
|
4909
|
-
|
|
4932
|
+
const localTranslations: LocalTranslationSubscription[] = []
|
|
4933
|
+
for (const [stream, subscribers] of this.activeStreamSubscribers()) {
|
|
4910
4934
|
if (subscribers.size === 0) continue
|
|
4911
4935
|
// Only transcription / translation need cloud delivery. Location and
|
|
4912
4936
|
// notifications are sourced natively on the phone — no cloud hop.
|
|
4913
4937
|
if (stream.startsWith("translation:")) {
|
|
4914
|
-
|
|
4938
|
+
const routes = summarizeTranscriptionRoutes(subscribers, (packageName) =>
|
|
4939
|
+
this.appTranscriptionRoutesForSubscription(packageName, stream),
|
|
4940
|
+
)
|
|
4941
|
+
if (routes.hasCloudSubscriber) cloudStreams.add(stream)
|
|
4942
|
+
const [, source, target, ...rest] = stream.split(":")
|
|
4943
|
+
if (target && target !== "*" && rest.length === 0) {
|
|
4944
|
+
localTranslations.push({
|
|
4945
|
+
source,
|
|
4946
|
+
target,
|
|
4947
|
+
cloud: routes.hasCloudSubscriber,
|
|
4948
|
+
forceLocal: routes.hasForceLocalSubscriber,
|
|
4949
|
+
})
|
|
4950
|
+
}
|
|
4915
4951
|
}
|
|
4916
4952
|
if (stream.startsWith("transcription:")) {
|
|
4917
4953
|
const language = stream.substring("transcription:".length)
|
|
@@ -4927,10 +4963,17 @@ class LocalMiniappRuntime {
|
|
|
4927
4963
|
if (routes.hasCloudSubscriber) cloudStreams.add(stream)
|
|
4928
4964
|
}
|
|
4929
4965
|
}
|
|
4966
|
+
// On-device translation is fed by on-device transcripts, so a translation
|
|
4967
|
+
// subscription it can serve keeps local STT eligible exactly like a
|
|
4968
|
+
// transcription subscription does.
|
|
4969
|
+
const translatesLocally = offlineTranslationSupported() ? localTranslations : []
|
|
4970
|
+
localTranslationService.setSubscriptions(translatesLocally)
|
|
4971
|
+
const hasForceLocalTranslation = translatesLocally.some((sub) => sub.forceLocal)
|
|
4972
|
+
const localSourceLang = translatesLocally.length > 0 ? sttModelManager.getBcp47ForLanguage() : null
|
|
4930
4973
|
localSttFallbackCoordinator.onSubscriptionChange(
|
|
4931
|
-
transcriptionLang !== null,
|
|
4932
|
-
forceLocalTranscriptionLang ?? transcriptionLang,
|
|
4933
|
-
hasForceLocalTranscription,
|
|
4974
|
+
transcriptionLang !== null || localSourceLang !== null,
|
|
4975
|
+
forceLocalTranscriptionLang ?? transcriptionLang ?? localSourceLang,
|
|
4976
|
+
hasForceLocalTranscription || hasForceLocalTranslation,
|
|
4934
4977
|
)
|
|
4935
4978
|
|
|
4936
4979
|
// Mirror the same set as typed AudioSubscription[]. When Soniox-direct is
|
|
@@ -5065,6 +5108,14 @@ class LocalMiniappRuntime {
|
|
|
5065
5108
|
markTranscriptSourceRecent("direct")
|
|
5066
5109
|
this.handleTranslationData(d)
|
|
5067
5110
|
})
|
|
5111
|
+
localTranslationService.attach({
|
|
5112
|
+
deliver: (stream, data) => this.forwardEvent(stream, data, "local"),
|
|
5113
|
+
primaryUp: () => this.transcriptionPrimaryUp(),
|
|
5114
|
+
onPairUnavailable: () =>
|
|
5115
|
+
localDisplayManager.showSystemNotice(
|
|
5116
|
+
"Offline translation language not downloaded.\nGet it in Settings > Offline models.",
|
|
5117
|
+
),
|
|
5118
|
+
})
|
|
5068
5119
|
// When the user sets/clears their Soniox key mid-session, re-partition
|
|
5069
5120
|
// subscriptions and mic requirements without waiting for a subscription
|
|
5070
5121
|
// change.
|
|
@@ -5158,22 +5209,26 @@ class LocalMiniappRuntime {
|
|
|
5158
5209
|
}
|
|
5159
5210
|
|
|
5160
5211
|
private handleTranslationData(d: TranslationData): void {
|
|
5161
|
-
this.forwardEvent(
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5212
|
+
this.forwardEvent(
|
|
5213
|
+
`translation:${d.source.language}:${d.target.language}`,
|
|
5214
|
+
{
|
|
5215
|
+
type: "translation",
|
|
5216
|
+
text: d.text,
|
|
5217
|
+
originalText: d.originalText,
|
|
5218
|
+
isFinal: d.isFinal,
|
|
5219
|
+
utteranceId: d.utteranceId,
|
|
5220
|
+
startTime: d.startMs,
|
|
5221
|
+
endTime: d.endMs,
|
|
5222
|
+
speakerId: d.speakerId,
|
|
5223
|
+
duration: d.durationMs,
|
|
5224
|
+
transcribeLanguage: d.source.language,
|
|
5225
|
+
translateLanguage: d.target.language,
|
|
5226
|
+
didTranslate: true,
|
|
5227
|
+
provider: d.provider,
|
|
5228
|
+
confidence: d.confidence,
|
|
5229
|
+
},
|
|
5230
|
+
"cloud",
|
|
5231
|
+
)
|
|
5177
5232
|
}
|
|
5178
5233
|
|
|
5179
5234
|
/** Re-partition audio routing after a Soniox-direct enable/disable flip. */
|
|
@@ -5252,31 +5307,15 @@ class LocalMiniappRuntime {
|
|
|
5252
5307
|
useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.cfw_native_navigation.key), broadcast)
|
|
5253
5308
|
// The dashboard master switch flips the dashboard view.
|
|
5254
5309
|
useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.custom_dashboard_enabled.key), broadcast)
|
|
5255
|
-
// The
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
* gives up the slot first, so it hears the view change while it is still
|
|
5265
|
-
* the overlay app; then every app sees its views again.
|
|
5266
|
-
*/
|
|
5267
|
-
private syncActiveDashboard(): void {
|
|
5268
|
-
const candidates: Array<{packageName: string; type?: string}> = [...useAppStatusStore.getState().apps]
|
|
5269
|
-
// A dashboard can connect at launch before the apps store has listed it.
|
|
5270
|
-
for (const [packageName, app] of this.connectedApps) {
|
|
5271
|
-
if (candidates.some((candidate) => candidate.packageName === packageName)) continue
|
|
5272
|
-
candidates.push({packageName, type: app.installedManifest?.type})
|
|
5273
|
-
}
|
|
5274
|
-
const next = activeDashboardAmong(candidates)
|
|
5275
|
-
const previous = activeDashboard()
|
|
5276
|
-
if (next === previous) return
|
|
5277
|
-
if (previous !== null) overlayPages.hide(previous, "replaced")
|
|
5278
|
-
setActiveDashboard(next)
|
|
5279
|
-
this.broadcastCapabilities()
|
|
5310
|
+
// The role follows the connected dashboard: when it passes, every app
|
|
5311
|
+
// sees its views again.
|
|
5312
|
+
let role = activeDashboard()
|
|
5313
|
+
onOverlayAppsChange(() => {
|
|
5314
|
+
const next = activeDashboard()
|
|
5315
|
+
if (next === role) return
|
|
5316
|
+
role = next
|
|
5317
|
+
broadcast()
|
|
5318
|
+
})
|
|
5280
5319
|
}
|
|
5281
5320
|
|
|
5282
5321
|
/** Capabilities as `packageName` sees them: the model profile plus the views this app may draw. */
|
|
@@ -5296,6 +5335,30 @@ class LocalMiniappRuntime {
|
|
|
5296
5335
|
}
|
|
5297
5336
|
}
|
|
5298
5337
|
|
|
5338
|
+
/**
|
|
5339
|
+
* Pause/resume (MiniappPauseCoordinator) lands as one push per handshook
|
|
5340
|
+
* app, and the mic and cloud transcription demand follow it.
|
|
5341
|
+
*/
|
|
5342
|
+
private ensurePauseWired(): void {
|
|
5343
|
+
if (this.pauseWired) return
|
|
5344
|
+
this.pauseWired = true
|
|
5345
|
+
onMiniappsPauseChange((paused) => {
|
|
5346
|
+
this.broadcastPauseState(paused)
|
|
5347
|
+
this.recomputeMicRequirements()
|
|
5348
|
+
this.updateCloudSubscriptions()
|
|
5349
|
+
})
|
|
5350
|
+
}
|
|
5351
|
+
|
|
5352
|
+
private broadcastPauseState(paused: boolean): void {
|
|
5353
|
+
for (const packageName of this.handshookApps) {
|
|
5354
|
+
this.sendToMiniapp(packageName, {
|
|
5355
|
+
type: MiniappResponseType.PAUSE_CHANGE,
|
|
5356
|
+
paused,
|
|
5357
|
+
...(paused ? {reason: "glasses_disconnected"} : {}),
|
|
5358
|
+
})
|
|
5359
|
+
}
|
|
5360
|
+
}
|
|
5361
|
+
|
|
5299
5362
|
/** Host user preferences miniapps follow by default (Settings > Appearance). */
|
|
5300
5363
|
private currentPreferences(): MiniappPreferences {
|
|
5301
5364
|
const settings = useSettingsStore.getState()
|
|
@@ -5367,6 +5430,10 @@ class LocalMiniappRuntime {
|
|
|
5367
5430
|
/** Deliver to this subscriber alone (the designated dashboard app's gestures while it shows). */
|
|
5368
5431
|
only?: string,
|
|
5369
5432
|
): void {
|
|
5433
|
+
// Paused miniapps (glasses disconnected) hear no streams. The resume push
|
|
5434
|
+
// lands before the reconnect's own glasses_connection event, so nothing
|
|
5435
|
+
// a resumed app needs is lost here.
|
|
5436
|
+
if (isMiniappsPaused()) return
|
|
5370
5437
|
// Local sherpa results enter the runtime only through here (DeviceEventRouter's
|
|
5371
5438
|
// local_transcription listener); cloud/direct are marked at their onTranscript
|
|
5372
5439
|
// wiring, which never passes "local".
|
|
@@ -5395,6 +5462,7 @@ class LocalMiniappRuntime {
|
|
|
5395
5462
|
// translation:<source>:<target> — exact (TranslationModule.fromTo)
|
|
5396
5463
|
// translation:auto — back-compat alias for *:*
|
|
5397
5464
|
const matchedSubs = new Set<string>()
|
|
5465
|
+
let translationPatterns: string[] = []
|
|
5398
5466
|
const exact = this.streamSubscribers.get(normalizedStream)
|
|
5399
5467
|
if (exact) for (const p of exact) matchedSubs.add(p)
|
|
5400
5468
|
|
|
@@ -5407,13 +5475,13 @@ class LocalMiniappRuntime {
|
|
|
5407
5475
|
// parts[0] = "translation", parts[1] = source, parts[2] = target
|
|
5408
5476
|
if (parts.length === 3) {
|
|
5409
5477
|
const [, source, target] = parts
|
|
5410
|
-
|
|
5478
|
+
translationPatterns = [
|
|
5411
5479
|
"translation:auto", // legacy alias for *:*
|
|
5412
5480
|
"translation:*:*", // cloud v3 "all pairs"
|
|
5413
5481
|
`translation:*:${target}`, // cloud v3 "to <target>"
|
|
5414
5482
|
`translation:${source}:*`, // cloud v3 "from <source>"
|
|
5415
5483
|
]
|
|
5416
|
-
for (const pat of
|
|
5484
|
+
for (const pat of translationPatterns) {
|
|
5417
5485
|
const subs = this.streamSubscribers.get(pat)
|
|
5418
5486
|
if (subs) for (const p of subs) matchedSubs.add(p)
|
|
5419
5487
|
}
|
|
@@ -5481,6 +5549,11 @@ class LocalMiniappRuntime {
|
|
|
5481
5549
|
transcriptionRoute =
|
|
5482
5550
|
transcriptionDeliveryRoute(transcriptionSource, routes, this.transcriptionPrimaryUp()) ?? undefined
|
|
5483
5551
|
if (!transcriptionRoute) continue
|
|
5552
|
+
} else if (normalizedStream.startsWith("translation:") && transcriptionSource) {
|
|
5553
|
+
const routes = this.appTranslationRoutesForEvent(packageName, [normalizedStream, ...translationPatterns])
|
|
5554
|
+
transcriptionRoute =
|
|
5555
|
+
transcriptionDeliveryRoute(transcriptionSource, routes, this.transcriptionPrimaryUp()) ?? undefined
|
|
5556
|
+
if (!transcriptionRoute) continue
|
|
5484
5557
|
}
|
|
5485
5558
|
if (TRANSCRIPT_TIMING_TELEMETRY && normalizedStream.startsWith("transcription:")) {
|
|
5486
5559
|
const transcript = data as {text?: string; isFinal?: boolean; __hostReceivedAt?: number} | null
|
|
@@ -5579,6 +5652,20 @@ class LocalMiniappRuntime {
|
|
|
5579
5652
|
}
|
|
5580
5653
|
}
|
|
5581
5654
|
|
|
5655
|
+
private appTranslationRoutesForEvent(
|
|
5656
|
+
packageName: string,
|
|
5657
|
+
patterns: string[],
|
|
5658
|
+
): {
|
|
5659
|
+
cloud: boolean
|
|
5660
|
+
forceLocal: boolean
|
|
5661
|
+
} {
|
|
5662
|
+
const app = this.connectedApps.get(packageName)
|
|
5663
|
+
return {
|
|
5664
|
+
cloud: patterns.some((pattern) => app?.cloudTranscriptionStreams.has(pattern) === true),
|
|
5665
|
+
forceLocal: patterns.some((pattern) => app?.forceLocalTranscriptionStreams.has(pattern) === true),
|
|
5666
|
+
}
|
|
5667
|
+
}
|
|
5668
|
+
|
|
5582
5669
|
private appTranscriptionRoutesForEvent(
|
|
5583
5670
|
packageName: string,
|
|
5584
5671
|
stream: string,
|
|
@@ -6148,6 +6235,18 @@ class LocalMiniappRuntime {
|
|
|
6148
6235
|
)
|
|
6149
6236
|
}
|
|
6150
6237
|
|
|
6238
|
+
/**
|
|
6239
|
+
* Hand an external link's query parameters to a miniapp. A miniapp that is
|
|
6240
|
+
* still spawning gets them once its handshake completes.
|
|
6241
|
+
*/
|
|
6242
|
+
public deliverLink(packageName: string, params: Record<string, string>): void {
|
|
6243
|
+
if (!this.handshookApps.has(packageName)) {
|
|
6244
|
+
this.pendingLinks.set(packageName, params)
|
|
6245
|
+
return
|
|
6246
|
+
}
|
|
6247
|
+
this.sendToMiniapp(packageName, {type: MiniappResponseType.LINK, params})
|
|
6248
|
+
}
|
|
6249
|
+
|
|
6151
6250
|
/**
|
|
6152
6251
|
* Send a VISIBILITY_CHANGE push to a miniapp.
|
|
6153
6252
|
*/
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import {offlineTranslationManager} from "./OfflineTranslationManager"
|
|
2
|
+
|
|
3
|
+
/** One concrete-target `translation:<source>:<target>` stream and who listens to it. */
|
|
4
|
+
export interface LocalTranslationSubscription {
|
|
5
|
+
/** Subscribed source: a language code, or `*` / `auto` for any. */
|
|
6
|
+
source: string
|
|
7
|
+
target: string
|
|
8
|
+
/** At least one listener on the default (cloud) route. */
|
|
9
|
+
cloud: boolean
|
|
10
|
+
/** At least one listener that requires on-device translation. */
|
|
11
|
+
forceLocal: boolean
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface LocalTranscript {
|
|
15
|
+
text: string
|
|
16
|
+
isFinal?: boolean
|
|
17
|
+
speakerId?: number | string
|
|
18
|
+
startTime?: number
|
|
19
|
+
endTime?: number
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface Host {
|
|
23
|
+
/** Fan a `translation:<source>:<target>` event out to miniapps. */
|
|
24
|
+
deliver: (stream: string, data: Record<string, unknown>) => void
|
|
25
|
+
/** Whether the primary (cloud / Soniox-direct) path is up. */
|
|
26
|
+
primaryUp: () => boolean
|
|
27
|
+
/** A wanted pair has no language pack on the device. */
|
|
28
|
+
onPairUnavailable?: (source: string, target: string) => void
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
interface Job {
|
|
32
|
+
text: string
|
|
33
|
+
isFinal: boolean
|
|
34
|
+
utteranceId: string
|
|
35
|
+
transcript: LocalTranscript
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface Lane {
|
|
39
|
+
busy: boolean
|
|
40
|
+
finals: Job[]
|
|
41
|
+
pendingInterim: Job | null
|
|
42
|
+
lastInterimAt: number
|
|
43
|
+
retryAt: number
|
|
44
|
+
reportedUnavailable: boolean
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Interims are translated at most this often; finals always are. */
|
|
48
|
+
const INTERIM_MIN_INTERVAL_MS = 300
|
|
49
|
+
const FAILURE_BACKOFF_MS = 5000
|
|
50
|
+
|
|
51
|
+
const baseLanguage = (code: string): string => code.split("-")[0].toLowerCase()
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Turns on-device transcripts into on-device translations. Apple's Translation
|
|
55
|
+
* framework has no streaming API, so each update is a whole `translate` call:
|
|
56
|
+
* one runs at a time per language pair, a newer interim replaces a waiting
|
|
57
|
+
* one, and finals are never dropped.
|
|
58
|
+
*
|
|
59
|
+
* A pair is translated here when a subscriber forces local, or when the
|
|
60
|
+
* primary path is down and would otherwise leave default subscribers silent.
|
|
61
|
+
*/
|
|
62
|
+
export class LocalTranslationService {
|
|
63
|
+
private subscriptions: LocalTranslationSubscription[] = []
|
|
64
|
+
private lanes = new Map<string, Lane>()
|
|
65
|
+
private host: Host | null = null
|
|
66
|
+
private utteranceCounter = 0
|
|
67
|
+
|
|
68
|
+
attach(host: Host): void {
|
|
69
|
+
this.host = host
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
setSubscriptions(subscriptions: LocalTranslationSubscription[]): void {
|
|
73
|
+
this.subscriptions = subscriptions
|
|
74
|
+
const live = new Set(subscriptions.map((s) => s.target))
|
|
75
|
+
for (const key of this.lanes.keys()) {
|
|
76
|
+
if (!live.has(key.substring(key.indexOf(">") + 1))) this.lanes.delete(key)
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
onLocalTranscript(transcript: LocalTranscript, sourceLanguage: string): void {
|
|
81
|
+
const text = transcript.text?.trim()
|
|
82
|
+
if (!text || !this.host || !offlineTranslationManager.isSupported()) return
|
|
83
|
+
const isFinal = transcript.isFinal === true
|
|
84
|
+
const utteranceId = `local-${this.utteranceCounter}`
|
|
85
|
+
if (isFinal) this.utteranceCounter++
|
|
86
|
+
|
|
87
|
+
for (const target of this.wantedTargets(sourceLanguage)) {
|
|
88
|
+
const job: Job = {text, isFinal, utteranceId, transcript}
|
|
89
|
+
if (baseLanguage(target) === baseLanguage(sourceLanguage)) {
|
|
90
|
+
// Same-language passthrough, as the cloud emits it: no originalText.
|
|
91
|
+
this.emit(sourceLanguage, target, job, text, undefined)
|
|
92
|
+
continue
|
|
93
|
+
}
|
|
94
|
+
this.enqueue(sourceLanguage, target, job)
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
private wantedTargets(sourceLanguage: string): Set<string> {
|
|
99
|
+
const targets = new Set<string>()
|
|
100
|
+
let primaryUp: boolean | null = null
|
|
101
|
+
for (const sub of this.subscriptions) {
|
|
102
|
+
const anySource = sub.source === "*" || sub.source === "auto"
|
|
103
|
+
if (!anySource && baseLanguage(sub.source) !== baseLanguage(sourceLanguage)) continue
|
|
104
|
+
if (!sub.forceLocal) {
|
|
105
|
+
primaryUp ??= this.host!.primaryUp()
|
|
106
|
+
if (primaryUp || !sub.cloud) continue
|
|
107
|
+
}
|
|
108
|
+
targets.add(sub.target)
|
|
109
|
+
}
|
|
110
|
+
return targets
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
private enqueue(source: string, target: string, job: Job): void {
|
|
114
|
+
const key = `${source}>${target}`
|
|
115
|
+
let lane = this.lanes.get(key)
|
|
116
|
+
if (!lane) {
|
|
117
|
+
lane = {busy: false, finals: [], pendingInterim: null, lastInterimAt: 0, retryAt: 0, reportedUnavailable: false}
|
|
118
|
+
this.lanes.set(key, lane)
|
|
119
|
+
}
|
|
120
|
+
const now = Date.now()
|
|
121
|
+
if (now < lane.retryAt) return
|
|
122
|
+
if (job.isFinal) {
|
|
123
|
+
lane.pendingInterim = null
|
|
124
|
+
lane.finals.push(job)
|
|
125
|
+
} else {
|
|
126
|
+
if (now - lane.lastInterimAt < INTERIM_MIN_INTERVAL_MS) return
|
|
127
|
+
lane.lastInterimAt = now
|
|
128
|
+
lane.pendingInterim = job
|
|
129
|
+
}
|
|
130
|
+
if (!lane.busy) void this.drain(source, target, lane)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
private async drain(source: string, target: string, lane: Lane): Promise<void> {
|
|
134
|
+
lane.busy = true
|
|
135
|
+
try {
|
|
136
|
+
for (;;) {
|
|
137
|
+
const job = lane.finals.shift() ?? lane.pendingInterim
|
|
138
|
+
if (!job) break
|
|
139
|
+
if (!job.isFinal) lane.pendingInterim = null
|
|
140
|
+
try {
|
|
141
|
+
const translated = await offlineTranslationManager.translate(job.text, source, target)
|
|
142
|
+
this.emit(source, target, job, translated, job.text)
|
|
143
|
+
} catch (err) {
|
|
144
|
+
this.handleFailure(source, target, lane, err)
|
|
145
|
+
break
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
} finally {
|
|
149
|
+
lane.busy = false
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private handleFailure(source: string, target: string, lane: Lane, err: unknown): void {
|
|
154
|
+
const message = (err as Error)?.message ?? String(err)
|
|
155
|
+
console.warn(`[LocalTranslation] ${source}>${target} failed: ${message}`)
|
|
156
|
+
lane.finals = []
|
|
157
|
+
lane.pendingInterim = null
|
|
158
|
+
lane.retryAt = Date.now() + FAILURE_BACKOFF_MS
|
|
159
|
+
if (message.includes("not_installed") && !lane.reportedUnavailable) {
|
|
160
|
+
lane.reportedUnavailable = true
|
|
161
|
+
this.host?.onPairUnavailable?.(source, target)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private emit(source: string, target: string, job: Job, text: string, originalText: string | undefined): void {
|
|
166
|
+
this.host?.deliver(`translation:${source}:${target}`, {
|
|
167
|
+
type: "translation",
|
|
168
|
+
text,
|
|
169
|
+
originalText,
|
|
170
|
+
isFinal: job.isFinal,
|
|
171
|
+
utteranceId: job.utteranceId,
|
|
172
|
+
startTime: job.transcript.startTime,
|
|
173
|
+
endTime: job.transcript.endTime,
|
|
174
|
+
speakerId: job.transcript.speakerId,
|
|
175
|
+
transcribeLanguage: source,
|
|
176
|
+
translateLanguage: target,
|
|
177
|
+
didTranslate: originalText !== undefined,
|
|
178
|
+
provider: "apple-translation",
|
|
179
|
+
})
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const localTranslationService = new LocalTranslationService()
|
|
184
|
+
export default localTranslationService
|