@glassly/engine 0.3.2-beta.572 → 0.3.2-beta.602

package/build/services/LocalMiniappRuntime.js

@@ -44,7 +44,7 @@ import overlayPages from "./pages/OverlayPages";
 import dashboardOpenTrace from "./debug/DashboardOpenTrace";
 import { isAppForegrounded, isPhoneActive, onAppForegroundChange, onPhoneActivityChange } from "./PhoneActivity";
 import { installedNotificationApps } from "./PhoneNotificationsSync";
-import { DERIVED_SETTINGS_KEYS, SETTINGS_BRIDGE_KEYS, isDerivedSettingsKey, ringBatteryFrom, selectSettingsBridgeKeys, validateSettingsSet, } from "./settingsBridge";
+import { DERIVED_SETTINGS_KEYS, SETTINGS_BRIDGE_KEYS, input2dFrom, isDerivedSettingsKey, ringBatteryFrom, selectSettingsBridgeKeys, validateSettingsSet, } from "./settingsBridge";
 import { mayUsePermission } from "./miniappPermissions";
 import { activeDashboard, addDashboardApp, displayViewsFor, isDashboardApp, isOverlayApp, onOverlayAppsChange, removeDashboardApp, setOverlayViewer, } from "./pages/overlayApps";
 import { pagesEligible } from "./pages/pageEligibility";
@@ -63,7 +63,7 @@ import { phonePhotoCoordinator } from "./PhonePhotoCoordinator";
 import { phoneStreamCoordinator } from "./PhoneStreamCoordinator";
 import { phoneVideoCoordinator } from "./PhoneVideoCoordinator";
 import { runSentenceTtsPipeline } from "./SentenceTtsPipeline";
-import { summarizeTranscriptionRoutes, transcriptionDeliveryRoute } from "./TranscriptionRouting";
+import { summarizeTranscriptionRoutes, transcriptionDeliveryRoute, } from "./TranscriptionRouting";
 import { prepareTtsSentences } from "./TtsTextSanitizer";
 import { handleWifiAdbRequest } from "./WifiAdbRequest";
 import { cloudClientService, setBlobNamespaceMigrator } from "./CloudClientService";
@@ -152,7 +152,7 @@ const MINIAPP_AUTH_REFRESH_MIN_DELAY_MS = 5_000;
 // (e.g. the cloud client had not finished its first-boot Core token exchange
 // when the miniapp connected). Exponential from base, capped, so a transient
 // not-yet-connected client self-heals within seconds instead of needing a full
-// manual Cloud V2 reconnect.
+// manual Cloud reconnect.
 const MINIAPP_AUTH_RETRY_BASE_MS = 1_000;
 const MINIAPP_AUTH_RETRY_MAX_MS = 15_000;
 const FOREGROUND_LIVENESS_PROBE_TIMEOUT_MS = 2_500;
@@ -835,6 +835,7 @@ class LocalMiniappRuntime {
             subscriptions: new Set(),
             forceLocalTranscriptionStreams: new Set(),
             cloudTranscriptionStreams: new Set(),
+            noFallbackTranscriptionStreams: new Set(),
             sendMessage: sendFn,
             lastPongAt: Date.now(),
             unansweredPingRounds: 0,
@@ -1538,7 +1539,7 @@ class LocalMiniappRuntime {
             // right after app start. CONNECT_ACK already went out without auth; keep
             // trying to mint (and re-drive on cloud-connect) so the app authenticates
             // to its backend on its own, instead of staying dead until a full manual
-            // Cloud V2 reconnect re-runs the whole handshake.
+            // Cloud reconnect re-runs the whole handshake.
             this.deliverInitialMiniappAuth(packageName, authPromise, 0);
         }
         this.sendCloudStatusToMiniapp(packageName);
@@ -1688,7 +1689,7 @@ class LocalMiniappRuntime {
      * When the cloud client (re)connects, immediately re-attempt the initial mint
      * for any handshook app still missing its token, instead of waiting out the
      * backoff. This is what turns "scan a dev app before the cloud is ready" from
-     * "dead until you manually reconnect Cloud V2" into "authenticates on its own
+     * "dead until you manually reconnect Cloud" into "authenticates on its own
      * the instant the connection comes up".
      */
     redriveMiniappAuthDelivery() {
@@ -1798,6 +1799,7 @@ class LocalMiniappRuntime {
         let requestedLocationRate = null;
         const forceLocalTranscriptionStreams = new Set();
         const cloudTranscriptionStreams = new Set();
+        const noFallbackTranscriptionStreams = new Set();
         const streams = rawStreams?.map((s) => {
             if (typeof s === "object" && s !== null) {
                 if (s.stream === "location_stream") {
@@ -1811,6 +1813,8 @@ class LocalMiniappRuntime {
                 }
                 if (isSpeechTextStream(s.stream) && (s.forceLocal !== true || s.includeCloud === true)) {
                     cloudTranscriptionStreams.add(s.stream);
+                    if (s.localFallback === false)
+                        noFallbackTranscriptionStreams.add(s.stream);
                 }
                 return s.stream;
             }
@@ -1875,6 +1879,7 @@ class LocalMiniappRuntime {
         app.subscriptions = new Set(streams);
         app.forceLocalTranscriptionStreams = new Set([...forceLocalTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)));
         app.cloudTranscriptionStreams = new Set([...cloudTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)));
+        app.noFallbackTranscriptionStreams = new Set([...noFallbackTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)));
         this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions);
         // The overlay app presents phone notifications on the glasses while it
         // listens to them; recorded for every dashboard so the role can move.
@@ -2212,6 +2217,10 @@ class LocalMiniappRuntime {
                 return localDisplayManager.foregroundAppOnScreen();
             case "ring_battery":
                 return ringBatteryFrom(useGlassesStore.getState());
+            case "input_2d": {
+                const controller = useSettingsStore.getState().getSetting(SETTINGS.default_controller.key);
+                return input2dFrom(useGlassesStore.getState(), controller);
+            }
         }
     }
     handleSettingsGet(packageName, payload, requestId) {
@@ -2298,6 +2307,7 @@ class LocalMiniappRuntime {
         useAppStatusStore.subscribe(pushDerived);
         onOverlayAppsChange(pushDerived);
         useGlassesStore.subscribe((s) => `${s.ringConnected}:${s.ringBatteryLevel}:${s.controllerConnected}:${s.controllerBatteryLevel}`, pushDerived);
+        useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.default_controller.key), pushDerived);
     }
     /**
      * Native nav UI feed (display.sendNativeNav*). Gated on the device's nav
@@ -4245,6 +4255,7 @@ class LocalMiniappRuntime {
         const previousSubscriptions = app.subscriptions;
         app.subscriptions = new Set(streams);
         app.forceLocalTranscriptionStreams.clear();
+        app.noFallbackTranscriptionStreams.clear();
         app.cloudTranscriptionStreams = new Set(streams.filter(isSpeechTextStream));
         this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions);
         this.recomputeMicRequirements();
@@ -4310,6 +4321,7 @@ class LocalMiniappRuntime {
         let transcriptionLang = null;
         let forceLocalTranscriptionLang = null;
         let hasForceLocalTranscription = false;
+        let hasFallbackTranscription = false;
         const localTranslations = [];
         for (const [stream, subscribers] of this.activeStreamSubscribers()) {
             if (subscribers.size === 0)
@@ -4340,6 +4352,8 @@ class LocalMiniappRuntime {
                     if (forceLocalTranscriptionLang === null)
                         forceLocalTranscriptionLang = language;
                 }
+                if (routes.hasFallbackSubscriber)
+                    hasFallbackTranscription = true;
                 if (routes.hasCloudSubscriber)
                     cloudStreams.add(stream);
             }
@@ -4351,7 +4365,7 @@ class LocalMiniappRuntime {
         localTranslationService.setSubscriptions(translatesLocally);
         const hasForceLocalTranslation = translatesLocally.some((sub) => sub.forceLocal);
         const localSourceLang = translatesLocally.length > 0 ? sttModelManager.getBcp47ForLanguage() : null;
-        localSttFallbackCoordinator.onSubscriptionChange(transcriptionLang !== null || localSourceLang !== null, forceLocalTranscriptionLang ?? transcriptionLang ?? localSourceLang, hasForceLocalTranscription || hasForceLocalTranslation);
+        localSttFallbackCoordinator.onSubscriptionChange(transcriptionLang !== null || localSourceLang !== null, forceLocalTranscriptionLang ?? transcriptionLang ?? localSourceLang, hasForceLocalTranscription || hasForceLocalTranslation, hasFallbackTranscription || localSourceLang !== null);
         // Mirror the same set as typed AudioSubscription[]. When Soniox-direct is
         // enabled (user API key set), the DIRECT service serves the whole set and
         // the cloud gets an EMPTY one — pushed as a real PUT, not skipped, so
@@ -4500,12 +4514,15 @@ class LocalMiniappRuntime {
                 return;
             if (!q.blocked) {
                 this.lastSttQuotaNoticeAt = 0;
-                void localSttFallbackCoordinator.onSttQuotaBlockedChange(false);
+                void localSttFallbackCoordinator.onSttQuotaBlockedChange(false).then(() => this.broadcastCloudStatus());
                 this.onQuotaRestored?.("transcription");
                 return;
             }
             console.warn(`${LOG_TAG}: cloud transcription blocked by plan quota`);
             void localSttFallbackCoordinator.onSttQuotaBlockedChange(true).then(() => {
+                // Miniapps learn of the block through the status snapshot; the lens
+                // notice below is for the wearer.
+                this.broadcastCloudStatus();
                 const localSttActive = localSttFallbackCoordinator.isActive();
                 const now = Date.now();
                 if (now - this.lastSttQuotaNoticeAt >= STT_QUOTA_NOTICE_DEBOUNCE_MS) {
@@ -4721,6 +4738,7 @@ class LocalMiniappRuntime {
         return {
             status: base.status,
             audioTransport: fallbackActive || sonioxDegraded ? "offline" : base.audioTransport,
+            quotaBlocked: localSttFallbackCoordinator.isSttQuotaBlocked() ? ["transcription"] : [],
         };
     }
     broadcastCloudStatus() {
@@ -4967,6 +4985,7 @@ class LocalMiniappRuntime {
         return {
             cloud: app?.cloudTranscriptionStreams.has(stream) === true,
             forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true,
+            localFallback: app?.noFallbackTranscriptionStreams.has(stream) !== true,
         };
     }
     appTranslationRoutesForEvent(packageName, patterns) {
@@ -4979,10 +4998,15 @@ class LocalMiniappRuntime {
     appTranscriptionRoutesForEvent(packageName, stream) {
         const app = this.connectedApps.get(packageName);
         const autoStream = `${MiniappStreamType.TRANSCRIPTION}:auto`;
+        const cloudExact = app?.cloudTranscriptionStreams.has(stream) === true;
+        const cloudAuto = app?.cloudTranscriptionStreams.has(autoStream) === true;
         return {
-            cloud: app?.cloudTranscriptionStreams.has(stream) === true || app?.cloudTranscriptionStreams.has(autoStream) === true,
+            cloud: cloudExact || cloudAuto,
             forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true ||
                 app?.forceLocalTranscriptionStreams.has(autoStream) === true,
+            // Fallback stays accepted while any matching default subscription takes it.
+            localFallback: (cloudExact && app?.noFallbackTranscriptionStreams.has(stream) !== true) ||
+                (cloudAuto && app?.noFallbackTranscriptionStreams.has(autoStream) !== true),
         };
     }
     /**