@glassly/engine 0.3.2-dev.574 → 0.3.2-dev.580

package/src/services/LocalMiniappRuntime.ts

@@ -54,6 +54,7 @@ import {
 import {normalizeAiTier, resolveAiTier} from "./AiTierResolution"
 import type {AiTier, AiTierProvider, SavedLlmKeys} from "./AiTierResolution"
 import {HttpError} from "@glassly/cloud-client"
+import type {QuotaStatusPayload} from "@glassly/cloud-protocol"
 import localDisplayManager from "./LocalDisplayManager"
 import sttModelManager from "./STTModelManager"
 import localTranslationService, {type LocalTranslationSubscription} from "./LocalTranslationService"
@@ -108,7 +109,12 @@ import {phonePhotoCoordinator} from "./PhonePhotoCoordinator"
 import {phoneStreamCoordinator} from "./PhoneStreamCoordinator"
 import {phoneVideoCoordinator} from "./PhoneVideoCoordinator"
 import {runSentenceTtsPipeline} from "./SentenceTtsPipeline"
-import {summarizeTranscriptionRoutes, transcriptionDeliveryRoute} from "./TranscriptionRouting"
+import {
+  summarizeTranscriptionRoutes,
+  transcriptionDeliveryRoute,
+  type TranscriptionDeliveryRoute,
+  type TranscriptionSubscriberRoutes,
+} from "./TranscriptionRouting"
 import {prepareTtsSentences} from "./TtsTextSanitizer"
 import {handleWifiAdbRequest} from "./WifiAdbRequest"
 import {cloudClientService, setBlobNamespaceMigrator} from "./CloudClientService"
@@ -171,6 +177,8 @@ interface ConnectedMiniapp {
   forceLocalTranscriptionStreams: Set<string>
   /** Transcription/translation streams with at least one listener using the default/cloud route. */
   cloudTranscriptionStreams: Set<string>
+  /** Transcription streams whose default-route listeners all refuse on-device fallback text. */
+  noFallbackTranscriptionStreams: Set<string>
   sendMessage: (raw: string) => void
   lastPongAt: number
   unansweredPingRounds: number
@@ -213,6 +221,9 @@ interface SpeechRun {
 
 type TranscriptionEventSource = "cloud" | "local"
 
+/** The cloud snapshot miniapps see: the phone's connection state plus plan-quota blocks. */
+type MiniappCloudStatus = CloudClientStatusSnapshot & {quotaBlocked: QuotaStatusPayload["feature"][]}
+
 /** Streams that carry speech text and so have a cloud route and an on-device route. */
 const isSpeechTextStream = (stream: string): boolean =>
   stream.startsWith("transcription:") || stream.startsWith("translation:")
@@ -1048,6 +1059,7 @@ class LocalMiniappRuntime {
       subscriptions: new Set(),
       forceLocalTranscriptionStreams: new Set(),
       cloudTranscriptionStreams: new Set(),
+      noFallbackTranscriptionStreams: new Set(),
       sendMessage: sendFn,
       lastPongAt: Date.now(),
       unansweredPingRounds: 0,
@@ -2101,7 +2113,10 @@ class LocalMiniappRuntime {
     if (!app) return
 
     const rawStreams = (payload.subscriptions ?? payload.streams) as
-      | (string | {stream: string; rate?: string; forceLocal?: boolean; includeCloud?: boolean})[]
+      | (
+          | string
+          | {stream: string; rate?: string; forceLocal?: boolean; includeCloud?: boolean; localFallback?: boolean}
+        )[]
       | undefined
     // Normalize: objects like {stream: "location_stream", rate: "realtime"} → extract stream name
     // "location_stream" is the rate-bearing alias for "location_update".
@@ -2110,6 +2125,7 @@ class LocalMiniappRuntime {
     let requestedLocationRate: string | null = null
     const forceLocalTranscriptionStreams = new Set<string>()
     const cloudTranscriptionStreams = new Set<string>()
+    const noFallbackTranscriptionStreams = new Set<string>()
     const streams = rawStreams?.map((s) => {
       if (typeof s === "object" && s !== null) {
         if (s.stream === "location_stream") {
@@ -2123,6 +2139,7 @@ class LocalMiniappRuntime {
         }
         if (isSpeechTextStream(s.stream) && (s.forceLocal !== true || s.includeCloud === true)) {
           cloudTranscriptionStreams.add(s.stream)
+          if (s.localFallback === false) noFallbackTranscriptionStreams.add(s.stream)
         }
         return s.stream
       }
@@ -2190,6 +2207,9 @@ class LocalMiniappRuntime {
     app.cloudTranscriptionStreams = new Set(
       [...cloudTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
     )
+    app.noFallbackTranscriptionStreams = new Set(
+      [...noFallbackTranscriptionStreams].filter((stream) => app.subscriptions.has(stream)),
+    )
     this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
     // The overlay app presents phone notifications on the glasses while it
     // listens to them; recorded for every dashboard so the role can move.
@@ -4863,6 +4883,7 @@ class LocalMiniappRuntime {
     const previousSubscriptions = app.subscriptions
     app.subscriptions = new Set(streams)
     app.forceLocalTranscriptionStreams.clear()
+    app.noFallbackTranscriptionStreams.clear()
     app.cloudTranscriptionStreams = new Set(streams.filter(isSpeechTextStream))
     this.replaceStreamSubscribers(packageName, previousSubscriptions, app.subscriptions)
 
@@ -4929,6 +4950,7 @@ class LocalMiniappRuntime {
     let transcriptionLang: string | null = null
     let forceLocalTranscriptionLang: string | null = null
     let hasForceLocalTranscription = false
+    let hasFallbackTranscription = false
     const localTranslations: LocalTranslationSubscription[] = []
     for (const [stream, subscribers] of this.activeStreamSubscribers()) {
       if (subscribers.size === 0) continue
@@ -4960,6 +4982,7 @@ class LocalMiniappRuntime {
           hasForceLocalTranscription = true
           if (forceLocalTranscriptionLang === null) forceLocalTranscriptionLang = language
         }
+        if (routes.hasFallbackSubscriber) hasFallbackTranscription = true
         if (routes.hasCloudSubscriber) cloudStreams.add(stream)
       }
     }
@@ -4974,6 +4997,7 @@ class LocalMiniappRuntime {
       transcriptionLang !== null || localSourceLang !== null,
       forceLocalTranscriptionLang ?? transcriptionLang ?? localSourceLang,
       hasForceLocalTranscription || hasForceLocalTranslation,
+      hasFallbackTranscription || localSourceLang !== null,
     )
 
     // Mirror the same set as typed AudioSubscription[]. When Soniox-direct is
@@ -5134,12 +5158,15 @@ class LocalMiniappRuntime {
       if (q.feature !== "transcription") return
       if (!q.blocked) {
         this.lastSttQuotaNoticeAt = 0
-        void localSttFallbackCoordinator.onSttQuotaBlockedChange(false)
+        void localSttFallbackCoordinator.onSttQuotaBlockedChange(false).then(() => this.broadcastCloudStatus())
         this.onQuotaRestored?.("transcription")
         return
       }
       console.warn(`${LOG_TAG}: cloud transcription blocked by plan quota`)
       void localSttFallbackCoordinator.onSttQuotaBlockedChange(true).then(() => {
+        // Miniapps learn of the block through the status snapshot; the lens
+        // notice below is for the wearer.
+        this.broadcastCloudStatus()
         const localSttActive = localSttFallbackCoordinator.isActive()
         const now = Date.now()
         if (now - this.lastSttQuotaNoticeAt >= STT_QUOTA_NOTICE_DEBOUNCE_MS) {
@@ -5383,7 +5410,7 @@ class LocalMiniappRuntime {
     )
   }
 
-  private currentCloudStatus(): CloudClientStatusSnapshot {
+  private currentCloudStatus(): MiniappCloudStatus {
     const base = cloudClientService.getStatus()
     const fallbackActive =
       (useSettingsStore.getState().getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive) as boolean | undefined) ===
@@ -5394,6 +5421,7 @@ class LocalMiniappRuntime {
     return {
       status: base.status,
       audioTransport: fallbackActive || sonioxDegraded ? "offline" : base.audioTransport,
+      quotaBlocked: localSttFallbackCoordinator.isSttQuotaBlocked() ? ["transcription"] : [],
     }
   }
 
@@ -5543,7 +5571,7 @@ class LocalMiniappRuntime {
 
     for (const packageName of matchedSubs) {
       if (only !== undefined && packageName !== only) continue
-      let transcriptionRoute: "default" | "forceLocal" | "all" | undefined
+      let transcriptionRoute: TranscriptionDeliveryRoute | undefined
       if (normalizedStream.startsWith("transcription:") && transcriptionSource) {
         const routes = this.appTranscriptionRoutesForEvent(packageName, normalizedStream)
         transcriptionRoute =
@@ -5638,17 +5666,12 @@ class LocalMiniappRuntime {
       })
   }
 
-  private appTranscriptionRoutesForSubscription(
-    packageName: string,
-    stream: string,
-  ): {
-    cloud: boolean
-    forceLocal: boolean
-  } {
+  private appTranscriptionRoutesForSubscription(packageName: string, stream: string): TranscriptionSubscriberRoutes {
     const app = this.connectedApps.get(packageName)
     return {
       cloud: app?.cloudTranscriptionStreams.has(stream) === true,
       forceLocal: app?.forceLocalTranscriptionStreams.has(stream) === true,
+      localFallback: app?.noFallbackTranscriptionStreams.has(stream) !== true,
     }
   }
 
@@ -5666,21 +5689,20 @@ class LocalMiniappRuntime {
     }
   }
 
-  private appTranscriptionRoutesForEvent(
-    packageName: string,
-    stream: string,
-  ): {
-    cloud: boolean
-    forceLocal: boolean
-  } {
+  private appTranscriptionRoutesForEvent(packageName: string, stream: string): TranscriptionSubscriberRoutes {
     const app = this.connectedApps.get(packageName)
     const autoStream = `${MiniappStreamType.TRANSCRIPTION}:auto`
+    const cloudExact = app?.cloudTranscriptionStreams.has(stream) === true
+    const cloudAuto = app?.cloudTranscriptionStreams.has(autoStream) === true
     return {
-      cloud:
-        app?.cloudTranscriptionStreams.has(stream) === true || app?.cloudTranscriptionStreams.has(autoStream) === true,
+      cloud: cloudExact || cloudAuto,
       forceLocal:
         app?.forceLocalTranscriptionStreams.has(stream) === true ||
         app?.forceLocalTranscriptionStreams.has(autoStream) === true,
+      // Fallback stays accepted while any matching default subscription takes it.
+      localFallback:
+        (cloudExact && app?.noFallbackTranscriptionStreams.has(stream) !== true) ||
+        (cloudAuto && app?.noFallbackTranscriptionStreams.has(autoStream) !== true),
     }
   }
 

package/src/services/LocalSttFallbackCoordinator.ts

@@ -8,7 +8,8 @@ import sttModelManager from "./STTModelManager"
  * Decides whether on-device Sherpa STT should produce transcripts for local
  * miniapps, instead of (or in addition to) cloud transcription. Local STT is
  * active when the cloud is unavailable (disconnected or refusing STT for plan
- * quota) or any subscription requires `forceLocal`.
+ * quota) and some subscriber accepts fallback text, or any subscription
+ * requires `forceLocal`.
  *
  * Activation flips `localSttFallbackActive` in the settings store, which
  * the bluetooth-sdk native side reads to decide whether to feed PCM into
@@ -37,6 +38,8 @@ class LocalSttFallbackCoordinator {
 
   private hasTranscriptionSubscription = false
   private hasForceLocalSubscription = false
+  /** Some default-route subscriber takes on-device text while the primary is down. */
+  private hasFallbackSubscription = false
   private activeLanguage: string | null = null
   /**
    * Default to "cloud is up" so we never accidentally activate local STT before
@@ -123,10 +126,18 @@ class LocalSttFallbackCoordinator {
     return this.localActive ? this.currentReason() : null
   }
 
-  onSubscriptionChange(hasTranscription: boolean, language: string | null, hasForceLocal = false): void {
-    this.log(`onSubscriptionChange(hasTx=${hasTranscription}, lang=${language}, forceLocal=${hasForceLocal})`)
+  onSubscriptionChange(
+    hasTranscription: boolean,
+    language: string | null,
+    hasForceLocal = false,
+    acceptsFallback = true,
+  ): void {
+    this.log(
+      `onSubscriptionChange(hasTx=${hasTranscription}, lang=${language}, forceLocal=${hasForceLocal}, fallback=${acceptsFallback})`,
+    )
     this.hasTranscriptionSubscription = hasTranscription
     this.hasForceLocalSubscription = hasTranscription && hasForceLocal
+    this.hasFallbackSubscription = hasTranscription && acceptsFallback
     this.activeLanguage = hasTranscription ? language : null
     void this.reconcile()
   }
@@ -256,7 +267,10 @@ class LocalSttFallbackCoordinator {
   }
 
   private shouldUseLocalStt(): boolean {
-    return this.hasTranscriptionSubscription && (this.hasForceLocalSubscription || !this.primaryUp())
+    return (
+      this.hasTranscriptionSubscription &&
+      (this.hasForceLocalSubscription || (this.hasFallbackSubscription && !this.primaryUp()))
+    )
   }
 
   private log(msg: string): void {

package/src/services/TranscriptionRouting.ts

@@ -1,14 +1,19 @@
 export interface TranscriptionRouteSummary {
   hasCloudSubscriber: boolean
   hasForceLocalSubscriber: boolean
+  /** A default-route subscriber that takes on-device text while the primary is down. */
+  hasFallbackSubscriber: boolean
 }
 
 export interface TranscriptionSubscriberRoutes {
   cloud: boolean
   forceLocal: boolean
+  /** Default-route fallback acceptance; absent means accepted. */
+  localFallback?: boolean
 }
 
-export type TranscriptionDeliveryRoute = "default" | "forceLocal" | "all"
+/** "fallback" is on-device text serving the default route because the primary is down. */
+export type TranscriptionDeliveryRoute = "default" | "forceLocal" | "fallback" | "all"
 
 export function summarizeTranscriptionRoutes<T>(
   subscribers: Iterable<T>,
@@ -16,12 +21,14 @@ export function summarizeTranscriptionRoutes<T>(
 ): TranscriptionRouteSummary {
   let hasCloudSubscriber = false
   let hasForceLocalSubscriber = false
+  let hasFallbackSubscriber = false
   for (const subscriber of subscribers) {
     const routes = routesFor(subscriber)
     if (routes.cloud) hasCloudSubscriber = true
     if (routes.forceLocal) hasForceLocalSubscriber = true
+    if (routes.cloud && routes.localFallback !== false) hasFallbackSubscriber = true
   }
-  return {hasCloudSubscriber, hasForceLocalSubscriber}
+  return {hasCloudSubscriber, hasForceLocalSubscriber, hasFallbackSubscriber}
 }
 
 /**
@@ -41,7 +48,8 @@ export function transcriptionDeliveryRoute(
 ): TranscriptionDeliveryRoute | null {
   if (source === "cloud") return routes.cloud ? "default" : null
   if (primaryUp) return routes.forceLocal ? "forceLocal" : null
-  if (routes.cloud && routes.forceLocal) return "all"
+  const fallback = routes.cloud && routes.localFallback !== false
+  if (fallback && routes.forceLocal) return "all"
   if (routes.forceLocal) return "forceLocal"
-  return routes.cloud ? "default" : null
+  return fallback ? "fallback" : null
 }

package/src/stores/settings.ts

@@ -742,19 +742,6 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: false,
     persist: true,
   },
-  // Debug: route the R1 ring through the glasses (advStart + RING_CONNECT_INFO,
-  // events forwarded by the CFW) instead of the phone link. OFF by default:
-  // the ring sends its touch reports to the glasses whenever it holds glasses
-  // targets, so the phone link only carries telemetry until those are erased.
-  // Synced to the Bluetooth SDK, which re-routes a connected ring on change.
-  // Device-local debug knob, so not saved on the server.
-  ring_via_glasses: {
-    key: "ring_via_glasses",
-    defaultValue: () => false,
-    writable: true,
-    saveOnServer: false,
-    persist: true,
-  },
   // Maximum rate at which display updates leave the phone for the glasses.
   // The Bluetooth SDK paces every scene frame and legacy layout to this,
   // coalescing faster updates last-wins; clears bypass it. No effect on G2
@@ -1107,7 +1094,6 @@ export const BLUETOOTH_SETTING_KEYS: string[] = [
   SETTINGS.cfw_scene_renderer.key,
   SETTINGS.cfw_fast_ble.key,
   SETTINGS.g2_perf_logs.key,
-  SETTINGS.ring_via_glasses.key,
   SETTINGS.display_max_fps.key,
   SETTINGS.twelve_hour_time.key,
   SETTINGS.metric_system.key,