@glassly/engine 0.3.1-dev.494 → 0.3.1-dev.523

package/build/services/LocalMiniappRuntime.js

@@ -36,6 +36,9 @@ import { createAnthropicProvider, createGeminiProvider, createOpenAiProvider, }
 import { normalizeAiTier, resolveAiTier } from "./AiTierResolution";
 import { HttpError } from "@glassly/cloud-client";
 import localDisplayManager from "./LocalDisplayManager";
+import sttModelManager from "./STTModelManager";
+import localTranslationService from "./LocalTranslationService";
+import { offlineTranslationSupported } from "./OfflineTranslationManager";
 import { dropGestureClaims, emptyGestureClaims, gestureClaimant, hasGestureClaim, setGestureClaims, } from "./gestureClaims";
 import overlayPages from "./pages/OverlayPages";
 import dashboardOpenTrace from "./debug/DashboardOpenTrace";
@@ -43,8 +46,7 @@ import { isAppForegrounded, isPhoneActive, onAppForegroundChange, onPhoneActivit
 import { installedNotificationApps } from "./PhoneNotificationsSync";
 import { DERIVED_SETTINGS_KEYS, SETTINGS_BRIDGE_KEYS, isDerivedSettingsKey, ringBatteryFrom, selectSettingsBridgeKeys, validateSettingsSet, } from "./settingsBridge";
 import { mayUsePermission } from "./miniappPermissions";
-import { activeDashboardAmong } from "./pages/activeDashboard";
-import { activeDashboard, addDashboardApp, displayViewsFor, isDashboardApp, isOverlayApp, onOverlayAppsChange, removeDashboardApp, setActiveDashboard, setOverlayViewer, } from "./pages/overlayApps";
+import { activeDashboard, addDashboardApp, displayViewsFor, isDashboardApp, isOverlayApp, onOverlayAppsChange, removeDashboardApp, setOverlayViewer, } from "./pages/overlayApps";
 import { pagesEligible } from "./pages/pageEligibility";
 import nativeNavigationService from "./NativeNavigationService";
 import headingService from "./HeadingService";
@@ -52,6 +54,7 @@ import glassesSensorService from "./GlassesSensorService";
 import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator";
 import speakerLabelResolver from "./SpeakerLabelResolver";
 import micStateCoordinator from "./MicStateCoordinator";
+import { isMiniappsPaused, onMiniappsPauseChange } from "./MiniappPauseCoordinator";
 import sonioxDirectService from "./soniox/SonioxDirectService";
 import { BlobStore } from "./BlobStore";
 import { CloudAudioSubscriptionSync } from "./CloudAudioSubscriptionSync";
@@ -84,6 +87,8 @@ import { LocalMiniappStorage } from "./LocalMiniappStorage";
 import { captureDecisionFor } from "./MemoryCapture";
 import { memoryStore } from "./memoryStoreInstance";
 import { speakerProfileStore } from "./speakerProfileStoreInstance";
+/** Streams that carry speech text and so have a cloud route and an on-device route. */
+const isSpeechTextStream = (stream) => stream.startsWith("transcription:") || stream.startsWith("translation:");
 /**
  * Strictness ordering for `location_stream` rate values. Higher index =
  * higher accuracy and higher power cost. The aggregate tier applied to
@@ -428,6 +433,8 @@ class LocalMiniappRuntime {
      * Drives {@link waitForConnect}.
      */
     handshookApps = new Set();
+    /** Links that arrived before the target finished its handshake; sent right after CONNECT_ACK. */
+    pendingLinks = new Map();
     /** Pending {@link waitForConnect} resolvers, keyed by packageName. */
     connectWaiters = new Map();
     /** Ref-counted stream subscriptions: stream → set of packageNames. */
@@ -439,6 +446,7 @@ class LocalMiniappRuntime {
     cloudStatusWired = false;
     capabilitiesWired = false;
     preferencesWired = false;
+    pauseWired = false;
     settingsBridgeWired = false;
     /** Last derived read-only values pushed on the settings stream, for change detection. */
     derivedSettings = new Map();
@@ -845,6 +853,7 @@ class LocalMiniappRuntime {
         this.ensureCloudStatusWired();
         this.ensureCapabilitiesWired();
         this.ensurePreferencesWired();
+        this.ensurePauseWired();
         this.ensureSettingsBridgeWired();
         this.ensurePingLoop();
         // Show the system boot message ("Starting <name>…") on the glasses for the
@@ -961,6 +970,7 @@ class LocalMiniappRuntime {
         // (e.g. a wake that's mid-handshake when the app is torn down). Done before
         // the early-return so it runs even if the connectedApps entry is already gone.
         this.handshookApps.delete(packageName);
+        this.pendingLinks.delete(packageName);
         this.flushConnectWaiters(packageName, new Error(`${packageName} unregistered before connect`));
         // Drop this app's transcription hints; the cloud subscription rebuild
         // below (via replaceStreamSubscribers) re-derives the hint union without it.
@@ -1092,7 +1102,7 @@ class LocalMiniappRuntime {
         // Console-tap forwarding. The miniapp's console.log/warn/etc is wrapped
         // (via injected shim from miniappGlobals.ts) to post a `dev_log`
         // envelope. We fan out to two destinations:
-        //   1. DevServerBridge — forwards to the laptop's `glassly-miniapp dev`
+        //   1. DevServerBridge — forwards to the laptop's `glassly dev`
         //      terminal. No-op when there's no sidecar (installed miniapps).
         //   2. React Native console — surfaces the log in Metro / Xcode console
         //      / adb logcat so installed-miniapp errors are still inspectable
@@ -1485,12 +1495,10 @@ class LocalMiniappRuntime {
         // Update lastPongAt so it doesn't time out right away
         existing.lastPongAt = Date.now();
         existing.unansweredPingRounds = 0;
-        // The dashboard that holds the role may take the overlay slot: its
-        // capabilities carry the dashboard view from the start.
-        if (existing.installedManifest?.type === "dashboard") {
+        // The connected dashboard holds the role and may take the overlay slot:
+        // its capabilities carry the dashboard view from the start.
+        if (existing.installedManifest?.type === "dashboard")
             addDashboardApp(packageName);
-            this.syncActiveDashboard();
-        }
         const capabilities = this.capabilitiesFor(packageName);
         // Build the declared-permission record for the SDK's session.permissions
         // module. Lower-cased to match v3's PermissionType union (microphone,
@@ -1517,6 +1525,8 @@ class LocalMiniappRuntime {
             capabilities,
             preferences: this.currentPreferences(),
             visibility: this.currentVisibility(),
+            paused: isMiniappsPaused(),
+            ...(isMiniappsPaused() ? { pauseReason: "glasses_disconnected" } : {}),
             permissions: declaredPermissions,
             ...(existing.launchSource ? { launchSource: existing.launchSource } : {}),
             ...(initialAuth ? { auth: initialAuth } : {}),
@@ -1535,6 +1545,11 @@ class LocalMiniappRuntime {
         // Handshake complete — unblock any launcher.waitForConnect() callers.
         this.handshookApps.add(packageName);
         this.flushConnectWaiters(packageName);
+        const pendingLink = this.pendingLinks.get(packageName);
+        if (pendingLink) {
+            this.pendingLinks.delete(packageName);
+            this.sendToMiniapp(packageName, { type: MiniappResponseType.LINK, params: pendingLink });
+        }
     }
     async requestMiniappAuth(packageName, opts) {
         const authPackageName = getDevAppSourcePackage(packageName) ?? packageName;
@@ -1791,15 +1806,15 @@ class LocalMiniappRuntime {
                     }
                     return "location_update";
                 }
-                if (s.forceLocal === true && s.stream.startsWith("transcription:")) {
+                if (s.forceLocal === true && isSpeechTextStream(s.stream)) {
                     forceLocalTranscriptionStreams.add(s.stream);
                 }
-                if (s.stream.startsWith("transcription:") && (s.forceLocal !== true || s.includeCloud === true)) {
+                if (isSpeechTextStream(s.stream) && (s.forceLocal !== true || s.includeCloud === true)) {
                     cloudTranscriptionStreams.add(s.stream);
                 }
                 return s.stream;
             }
-            if (s.startsWith("transcription:"))
+            if (isSpeechTextStream(s))
                 cloudTranscriptionStreams.add(s);
             return s;
         });
@@ -2197,8 +2212,6 @@ class LocalMiniappRuntime {
                 return localDisplayManager.foregroundAppOnScreen();
             case "ring_battery":
                 return ringBatteryFrom(useGlassesStore.getState());
-            case "active_dashboard":
-                return activeDashboard();
         }
     }
     handleSettingsGet(packageName, payload, requestId) {
@@ -4232,7 +4245,7 @@ class LocalMiniappRuntime {
         const previousSubscriptions = app.subscriptions;
         app.subscriptions = new Set(streams);
         app.forceLocalTranscriptionStreams.clear();
-        app.cloudTranscriptionStreams = new Set(streams.filter((stream) => stream.startsWith("transcription:")));
+        app.cloudTranscriptionStreams = new Set(streams.filter(isSpeechTextStream));
         this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions);
         this.recomputeMicRequirements();
         this.updateCloudSubscriptions();
@@ -4250,6 +4263,14 @@ class LocalMiniappRuntime {
             loudnessGateEnabled: typeof configuredLoudnessGate === "boolean" ? configuredLoudnessGate : null,
         };
     }
+    /**
+     * Subscriptions that hold mic and transcription demand right now. A paused
+     * app keeps its subscriptions registered (they come back on resume) but
+     * demands nothing while the glasses are away.
+     */
+    activeStreamSubscribers() {
+        return isMiniappsPaused() ? [] : this.streamSubscribers;
+    }
     recomputeMicRequirements() {
         // Soniox-direct (user API key) consumes decoded PCM on the phone, so
         // transcription/translation subscriptions request PCM instead of LC3 —
@@ -4257,7 +4278,7 @@ class LocalMiniappRuntime {
         const sonioxDirect = sonioxDirectService.isEnabled();
         let anyPcm = false;
         let anyLc3 = false;
-        for (const [stream, subscribers] of this.streamSubscribers) {
+        for (const [stream, subscribers] of this.activeStreamSubscribers()) {
             if (subscribers.size === 0)
                 continue;
             if (stream === "audio_chunk")
@@ -4289,13 +4310,25 @@ class LocalMiniappRuntime {
         let transcriptionLang = null;
         let forceLocalTranscriptionLang = null;
         let hasForceLocalTranscription = false;
-        for (const [stream, subscribers] of this.streamSubscribers) {
+        const localTranslations = [];
+        for (const [stream, subscribers] of this.activeStreamSubscribers()) {
             if (subscribers.size === 0)
                 continue;
             // Only transcription / translation need cloud delivery. Location and
             // notifications are sourced natively on the phone — no cloud hop.
             if (stream.startsWith("translation:")) {
-                cloudStreams.add(stream);
+                const routes = summarizeTranscriptionRoutes(subscribers, (packageName) => this.appTranscriptionRoutesForSubscription(packageName, stream));
+                if (routes.hasCloudSubscriber)
+                    cloudStreams.add(stream);
+                const [, source, target, ...rest] = stream.split(":");
+                if (target && target !== "*" && rest.length === 0) {
+                    localTranslations.push({
+                        source,
+                        target,
+                        cloud: routes.hasCloudSubscriber,
+                        forceLocal: routes.hasForceLocalSubscriber,
+                    });
+                }
             }
             if (stream.startsWith("transcription:")) {
                 const language = stream.substring("transcription:".length);
@@ -4311,7 +4344,14 @@ class LocalMiniappRuntime {
                     cloudStreams.add(stream);
             }
         }
-        localSttFallbackCoordinator.onSubscriptionChange(transcriptionLang !== null, forceLocalTranscriptionLang ?? transcriptionLang, hasForceLocalTranscription);
+        // On-device translation is fed by on-device transcripts, so a translation
+        // subscription it can serve keeps local STT eligible exactly like a
+        // transcription subscription does.
+        const translatesLocally = offlineTranslationSupported() ? localTranslations : [];
+        localTranslationService.setSubscriptions(translatesLocally);
+        const hasForceLocalTranslation = translatesLocally.some((sub) => sub.forceLocal);
+        const localSourceLang = translatesLocally.length > 0 ? sttModelManager.getBcp47ForLanguage() : null;
+        localSttFallbackCoordinator.onSubscriptionChange(transcriptionLang !== null || localSourceLang !== null, forceLocalTranscriptionLang ?? transcriptionLang ?? localSourceLang, hasForceLocalTranscription || hasForceLocalTranslation);
         // Mirror the same set as typed AudioSubscription[]. When Soniox-direct is
         // enabled (user API key set), the DIRECT service serves the whole set and
         // the cloud gets an EMPTY one — pushed as a real PUT, not skipped, so
@@ -4437,6 +4477,11 @@ class LocalMiniappRuntime {
             markTranscriptSourceRecent("direct");
             this.handleTranslationData(d);
         });
+        localTranslationService.attach({
+            deliver: (stream, data) => this.forwardEvent(stream, data, "local"),
+            primaryUp: () => this.transcriptionPrimaryUp(),
+            onPairUnavailable: () => localDisplayManager.showSystemNotice("Offline translation language not downloaded.\nGet it in Settings > Offline models."),
+        });
         // When the user sets/clears their Soniox key mid-session, re-partition
         // subscriptions and mic requirements without waiting for a subscription
         // change.
@@ -4533,7 +4578,7 @@ class LocalMiniappRuntime {
             didTranslate: true,
             provider: d.provider,
             confidence: d.confidence,
-        });
+        }, "cloud");
     }
     /** Re-partition audio routing after a Soniox-direct enable/disable flip. */
     recomputeAudioRouting() {
@@ -4597,33 +4642,16 @@ class LocalMiniappRuntime {
         useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.cfw_native_navigation.key), broadcast);
         // The dashboard master switch flips the dashboard view.
         useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.custom_dashboard_enabled.key), broadcast);
-        // The wearer's choice and the installed dashboards decide who holds the role.
-        const syncRole = () => this.syncActiveDashboard();
-        useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.active_dashboard.key), syncRole);
-        useAppStatusStore.subscribe(syncRole);
-        syncRole();
-    }
-    /**
-     * Recompute who holds the dashboard role (ADR-0009). The previous holder
-     * gives up the slot first, so it hears the view change while it is still
-     * the overlay app; then every app sees its views again.
-     */
-    syncActiveDashboard() {
-        const candidates = [...useAppStatusStore.getState().apps];
-        // A dashboard can connect at launch before the apps store has listed it.
-        for (const [packageName, app] of this.connectedApps) {
-            if (candidates.some((candidate) => candidate.packageName === packageName))
-                continue;
-            candidates.push({ packageName, type: app.installedManifest?.type });
-        }
-        const next = activeDashboardAmong(candidates);
-        const previous = activeDashboard();
-        if (next === previous)
-            return;
-        if (previous !== null)
-            overlayPages.hide(previous, "replaced");
-        setActiveDashboard(next);
-        this.broadcastCapabilities();
+        // The role follows the connected dashboard: when it passes, every app
+        // sees its views again.
+        let role = activeDashboard();
+        onOverlayAppsChange(() => {
+            const next = activeDashboard();
+            if (next === role)
+                return;
+            role = next;
+            broadcast();
+        });
     }
     /** Capabilities as `packageName` sees them: the model profile plus the views this app may draw. */
     capabilitiesFor(packageName) {
@@ -4641,6 +4669,29 @@ class LocalMiniappRuntime {
             });
         }
     }
+    /**
+     * Pause/resume (MiniappPauseCoordinator) lands as one push per handshook
+     * app, and the mic and cloud transcription demand follow it.
+     */
+    ensurePauseWired() {
+        if (this.pauseWired)
+            return;
+        this.pauseWired = true;
+        onMiniappsPauseChange((paused) => {
+            this.broadcastPauseState(paused);
+            this.recomputeMicRequirements();
+            this.updateCloudSubscriptions();
+        });
+    }
+    broadcastPauseState(paused) {
+        for (const packageName of this.handshookApps) {
+            this.sendToMiniapp(packageName, {
+                type: MiniappResponseType.PAUSE_CHANGE,
+                paused,
+                ...(paused ? { reason: "glasses_disconnected" } : {}),
+            });
+        }
+    }
     /** Host user preferences miniapps follow by default (Settings > Appearance). */
     currentPreferences() {
         const settings = useSettingsStore.getState();
@@ -4698,6 +4749,11 @@ class LocalMiniappRuntime {
     forwardEvent(streamType, data, transcriptionSource, 
     /** Deliver to this subscriber alone (the designated dashboard app's gestures while it shows). */
     only) {
+        // Paused miniapps (glasses disconnected) hear no streams. The resume push
+        // lands before the reconnect's own glasses_connection event, so nothing
+        // a resumed app needs is lost here.
+        if (isMiniappsPaused())
+            return;
         // Local sherpa results enter the runtime only through here (DeviceEventRouter's
         // local_transcription listener); cloud/direct are marked at their onTranscript
         // wiring, which never passes "local".
@@ -4725,6 +4781,7 @@ class LocalMiniappRuntime {
         //   translation:<source>:<target> — exact (TranslationModule.fromTo)
         //   translation:auto            — back-compat alias for *:*
         const matchedSubs = new Set();
+        let translationPatterns = [];
         const exact = this.streamSubscribers.get(normalizedStream);
         if (exact)
             for (const p of exact)
@@ -4741,13 +4798,13 @@ class LocalMiniappRuntime {
             // parts[0] = "translation", parts[1] = source, parts[2] = target
             if (parts.length === 3) {
                 const [, source, target] = parts;
-                const patterns = [
+                translationPatterns = [
                     "translation:auto", // legacy alias for *:*
                     "translation:*:*", // cloud v3 "all pairs"
                     `translation:*:${target}`, // cloud v3 "to <target>"
                     `translation:${source}:*`, // cloud v3 "from <source>"
                 ];
-                for (const pat of patterns) {
+                for (const pat of translationPatterns) {
                     const subs = this.streamSubscribers.get(pat);
                     if (subs)
                         for (const p of subs)
@@ -4820,6 +4877,13 @@ class LocalMiniappRuntime {
                 if (!transcriptionRoute)
                     continue;
             }
+            else if (normalizedStream.startsWith("translation:") && transcriptionSource) {
+                const routes = this.appTranslationRoutesForEvent(packageName, [normalizedStream, ...translationPatterns]);
+                transcriptionRoute =
+                    transcriptionDeliveryRoute(transcriptionSource, routes, this.transcriptionPrimaryUp()) ?? undefined;
+                if (!transcriptionRoute)
+                    continue;
+            }
             if (TRANSCRIPT_TIMING_TELEMETRY && normalizedStream.startsWith("transcription:")) {
                 const transcript = data;
                 const now = Date.now();
@@ -4905,6 +4969,13 @@ class LocalMiniappRuntime {
             forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true,
         };
     }
+    appTranslationRoutesForEvent(packageName, patterns) {
+        const app = this.connectedApps.get(packageName);
+        return {
+            cloud: patterns.some((pattern) => app?.cloudTranscriptionStreams.has(pattern) === true),
+            forceLocal: patterns.some((pattern) => app?.forceLocalTranscriptionStreams.has(pattern) === true),
+        };
+    }
     appTranscriptionRoutesForEvent(packageName, stream) {
         const app = this.connectedApps.get(packageName);
         const autoStream = `${MiniappStreamType.TRANSCRIPTION}:auto`;
@@ -5415,6 +5486,17 @@ class LocalMiniappRuntime {
             ...(error ? { error } : {}),
         }, requestId);
     }
+    /**
+     * Hand an external link's query parameters to a miniapp. A miniapp that is
+     * still spawning gets them once its handshake completes.
+     */
+    deliverLink(packageName, params) {
+        if (!this.handshookApps.has(packageName)) {
+            this.pendingLinks.set(packageName, params);
+            return;
+        }
+        this.sendToMiniapp(packageName, { type: MiniappResponseType.LINK, params });
+    }
     /**
      * Send a VISIBILITY_CHANGE push to a miniapp.
      */