@glassly/engine 0.3.8-dev.1015 → 0.3.8-dev.1016

package/src/services/LocalMiniappRuntime.ts

@@ -673,6 +673,9 @@ class LocalMiniappRuntime {
   private connectedApps: Map<string, ConnectedMiniapp> = new Map()
   /** Claimed gestures (CLAIM_GESTURES): a stack per gesture, the most recent claimer holding it. */
   private gestureClaims: GestureClaimState = emptyGestureClaims()
+  /** Apps with a text field open on the glasses (TEXT_FIELD). */
+  private textFieldApps = new Set<string>()
+  private textFieldListeners = new Set<(open: boolean) => void>()
   // The last main-view frame each app asked to render. Widget mode refuses
   // frames and demotion clears them, and an app that has drawn once may not
   // draw again on its own, so promotion replays this.
@@ -1295,6 +1298,7 @@ class LocalMiniappRuntime {
     }
     // Its gesture claims go back to whoever claimed them before.
     this.applyGestureClaimChanges(dropGestureClaims(this.gestureClaims, packageName))
+    this.setTextFieldOpen(packageName, false)
     this.retainedRenders.delete(packageName)
 
     // Warm-up is a request-owned camera lease. Cancel it even if the app record was already
@@ -1478,6 +1482,9 @@ class LocalMiniappRuntime {
       case MiniappRequestType.CLAIM_GESTURES:
         this.handleClaimGestures(packageName, payload, requestId)
         break
+      case MiniappRequestType.TEXT_FIELD:
+        this.handleTextField(packageName, payload, requestId)
+        break
       case MiniappRequestType.DISPLAY:
         this.handleDisplay(packageName, payload, requestId)
         break
@@ -2214,6 +2221,48 @@ class LocalMiniappRuntime {
     }
   }
 
+  /**
+   * TEXT_FIELD (session.input.setTextFieldOpen): the app has a text field on
+   * the glasses, or no longer does. Text entry that costs power to leave on
+   * (Neural Band handwriting) runs while any app has one. Cleared on app stop.
+   */
+  private handleTextField(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
+    if (typeof payload.open !== "boolean") {
+      this.sendResult(packageName, requestId, false, undefined, {
+        code: MiniappErrorCode.INVALID_ARGUMENT,
+        message: "Text field open must be a boolean",
+      })
+      return
+    }
+    if (this.connectedApps.has(packageName)) this.setTextFieldOpen(packageName, payload.open)
+    this.sendResult(packageName, requestId, true)
+  }
+
+  private setTextFieldOpen(packageName: string, open: boolean): void {
+    const before = this.textFieldApps.size > 0
+    if (open) this.textFieldApps.add(packageName)
+    else this.textFieldApps.delete(packageName)
+    const after = this.textFieldApps.size > 0
+    if (before === after) return
+    for (const cb of this.textFieldListeners) {
+      try {
+        cb(after)
+      } catch (err) {
+        console.warn(`${LOG_TAG}: text field listener failed:`, err)
+      }
+    }
+  }
+
+  /** True while any running miniapp has a text field open on the glasses. */
+  public hasOpenTextField(): boolean {
+    return this.textFieldApps.size > 0
+  }
+
+  public onTextFieldChange(cb: (open: boolean) => void): () => void {
+    this.textFieldListeners.add(cb)
+    return () => this.textFieldListeners.delete(cb)
+  }
+
   /** The app holding `gesture` (the most recent claimer), or null when it is unclaimed. */
   public gestureClaimant(gesture: string): string | null {
     const claimant = gestureClaimant(this.gestureClaims, gesture)

package/src/services/neuralBand/HandwritingGate.ts

@@ -0,0 +1,111 @@
+/**
+ * Decides when each connected Meta Neural Band runs its handwriting model. The
+ * model streams to the phone for as long as it runs, written on or not, so it
+ * runs only while it can type somewhere: the wearer has handwriting enabled
+ * and a miniapp has a text field open on the glasses. It outlives the last
+ * field by OFF_DELAY_MS, so moving between fields doesn't restart it. A session
+ * the band ends on its own (timeout, refused setting, silence) isn't retried
+ * until no field is open, so a band that can't run the model isn't asked again
+ * and again.
+ */
+import type {NeuralBandHandwritingStatus} from "./NeuralBandInput"
+
+type Phase = NeuralBandHandwritingStatus["phase"]
+
+export interface HandwritingGateDeps {
+  enabled: () => boolean
+  textFieldOpen: () => boolean
+  setHandwriting: (id: string, enabled: boolean) => Promise<void>
+  setTimeout: (fn: () => void, ms: number) => unknown
+  clearTimeout: (handle: unknown) => void
+}
+
+interface Band {
+  /** What we last asked this band for. */
+  requested: boolean
+  failed: boolean
+  phase: Phase
+}
+
+export const OFF_DELAY_MS = 5_000
+const ACTIVE = new Set<Phase>(["preparing", "ready", "restoring"])
+
+export class HandwritingGate {
+  private bands = new Map<string, Band>()
+  private offTimer: unknown = null
+
+  constructor(private readonly deps: HandwritingGateDeps) {}
+
+  /** The connected bands. A band keeps nothing across connections, so a reconnect starts fresh. */
+  setBands(ids: string[]): void {
+    for (const id of [...this.bands.keys()]) if (!ids.includes(id)) this.bands.delete(id)
+    for (const id of ids) if (!this.bands.has(id)) this.bands.set(id, {requested: false, failed: false, phase: "idle"})
+    this.update()
+  }
+
+  /** A band reported its handwriting phase. */
+  status(id: string, phase: Phase): void {
+    const band = this.bands.get(id)
+    if (!band) return
+    band.phase = phase
+    if (phase === "finished" && band.requested) {
+      band.requested = false
+      band.failed = true
+    }
+    this.update()
+  }
+
+  /** Re-evaluate after the setting or the open text fields changed. */
+  update(): void {
+    const enabled = this.deps.enabled()
+    if (enabled && this.deps.textFieldOpen()) {
+      this.clearOffTimer()
+      for (const [id, band] of this.bands) {
+        // A session still winding down (or one we didn't start) has to finish first.
+        if (band.requested || band.failed || ACTIVE.has(band.phase)) continue
+        this.request(id, band, true)
+      }
+      return
+    }
+    for (const band of this.bands.values()) band.failed = false
+    if (![...this.bands.values()].some((band) => band.requested)) {
+      this.clearOffTimer()
+      return
+    }
+    if (!enabled) {
+      this.clearOffTimer()
+      this.stopAll()
+    } else if (this.offTimer === null) {
+      this.offTimer = this.deps.setTimeout(() => {
+        this.offTimer = null
+        if (!this.deps.textFieldOpen() || !this.deps.enabled()) this.stopAll()
+      }, OFF_DELAY_MS)
+    }
+  }
+
+  dispose(): void {
+    this.clearOffTimer()
+    this.bands.clear()
+  }
+
+  private stopAll(): void {
+    for (const [id, band] of this.bands) if (band.requested) this.request(id, band, false)
+  }
+
+  private request(id: string, band: Band, enabled: boolean): void {
+    band.requested = enabled
+    this.deps.setHandwriting(id, enabled).catch((err) => {
+      console.warn(`HandwritingGate: turning handwriting ${enabled ? "on" : "off"} failed:`, err)
+      if (enabled && this.bands.get(id) === band && band.requested) {
+        band.requested = false
+        band.failed = true
+      }
+    })
+  }
+
+  private clearOffTimer(): void {
+    if (this.offTimer === null) return
+    this.deps.clearTimeout(this.offTimer)
+    this.offTimer = null
+  }
+}

package/src/services/neuralBand/NeuralBandSettingsSync.ts

@@ -1,16 +1,23 @@
 /**
- * Pushes the Neural Band user settings (air mouse, wearing hand) to every
- * connected band: to each band as it connects, and to all of them whenever
- * either setting changes. The native side keeps nothing across connections.
+ * Pushes the Neural Band user settings (air mouse, wearing hand, vibration) to
+ * every connected band: to each band as it connects, and to all of them
+ * whenever a setting changes. The native side keeps nothing across
+ * connections. Handwriting is gated, not pushed: HandwritingGate runs it while
+ * the setting is on and a miniapp has a text field open.
  */
 import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
 
 import {useGlassesStore} from "../../stores/glasses"
 import {SETTINGS, useSettingsStore} from "../../stores/settings"
 import {ControllerTypes} from "../../types/enums"
+import {BgTimer} from "../../utils/timers"
 import {readyControllers} from "../controllers"
+import localMiniappRuntime from "../LocalMiniappRuntime"
+import {HandwritingGate} from "./HandwritingGate"
+import neuralBandInput from "./NeuralBandInput"
 
 let unsubscribers: (() => void)[] = []
+let handwriting: HandwritingGate | null = null
 
 function connectedBandIds(): string[] {
   return readyControllers(useGlassesStore.getState().controllers)
@@ -39,10 +46,32 @@ function pushVibration(id: string): void {
   )
 }
 
+function startHandwritingGate(): (() => void)[] {
+  const gate = new HandwritingGate({
+    enabled: () => useSettingsStore.getState().getSetting(SETTINGS.neural_band_handwriting.key) !== false,
+    textFieldOpen: () => localMiniappRuntime.hasOpenTextField(),
+    setHandwriting: (id, enabled) => BluetoothSdk.neuralBandSetHandwriting(id, enabled),
+    setTimeout: (fn, ms) => BgTimer.setTimeout(fn, ms),
+    clearTimeout: (handle) => BgTimer.clearTimeout(handle as number),
+  })
+  handwriting = gate
+  gate.setBands(connectedBandIds())
+  return [
+    useSettingsStore.subscribe(
+      (s) => s.getSetting(SETTINGS.neural_band_handwriting.key),
+      () => gate.update(),
+    ),
+    localMiniappRuntime.onTextFieldChange(() => gate.update()),
+    neuralBandInput.onHandwritingStatus(({id, status}) => gate.status(id, status.phase)),
+    () => gate.dispose(),
+  ]
+}
+
 export function startNeuralBandSettingsSync(): void {
   if (unsubscribers.length > 0) return
   let connected = new Set(connectedBandIds())
   unsubscribers = [
+    ...startHandwritingGate(),
     useGlassesStore.subscribe(
       () => connectedBandIds().sort().join(","),
       () => {
@@ -54,6 +83,7 @@ export function startNeuralBandSettingsSync(): void {
           pushVibration(id)
         }
         connected = next
+        handwriting?.setBands([...next])
       },
     ),
     useSettingsStore.subscribe(
@@ -74,4 +104,5 @@ export function startNeuralBandSettingsSync(): void {
 export function stopNeuralBandSettingsSync(): void {
   for (const unsubscribe of unsubscribers) unsubscribe()
   unsubscribers = []
+  handwriting = null
 }

package/src/stores/settings.ts

@@ -848,6 +848,16 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: false,
     persist: true,
   },
+  // Meta Neural Band handwriting typing into miniapps. Enabled means the band runs
+  // its handwriting model while a miniapp has a text field open on the glasses
+  // (HandwritingGate); the model streams the whole time it runs, so never otherwise.
+  neural_band_handwriting: {
+    key: "neural_band_handwriting",
+    defaultValue: () => true,
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
   // The band's own buzz on each gesture (its haptics pattern 1; pattern 2 is silent).
   neural_band_vibration: {
     key: "neural_band_vibration",