@glassly/engine 0.3.2-beta.572 → 0.3.2-beta.597

package/src/services/neuralBand/NeuralBandInput.ts

@@ -0,0 +1,148 @@
+/**
+ * The Meta Neural Band's own event streams, past the gestures that ride `touch_event`:
+ * claim progress (for the pairing screens), the handwriting model (typed into the
+ * miniapp keyboard stream as synthesized key presses) and the air mouse (forwarded
+ * to miniapps on the pointer stream).
+ */
+import type {NeuralBandClaimEvent, NeuralBandHandwritingEvent, NeuralBandPointerEvent} from "@glassly/bluetooth-sdk"
+import {MiniappStreamType} from "@glassly/miniapp"
+
+import type {KeyboardEventPayload} from "../GlassesKeyboardInput"
+import localMiniappRuntime from "../LocalMiniappRuntime"
+
+export interface NeuralBandClaimStatus {
+  stage: "idle" | "progress" | "completed" | "failed"
+  message?: string
+  sessionExpired: boolean
+  bandName?: string
+}
+
+export interface NeuralBandHandwritingStatus {
+  phase: "idle" | "preparing" | "ready" | "restoring" | "finished"
+  message: string
+  problem?: string
+  /** The last text the band produced since handwriting was switched on. */
+  text: string
+}
+
+export interface NeuralBandInputOptions {
+  forward?: (stream: string, payload: unknown) => void
+  now?: () => number
+}
+
+const MODIFIERS = {shift: false, ctrl: false, alt: false, meta: false}
+
+/** One handwriting change as the key presses that would have typed it. */
+export function keyEventsForTextChange(previous: string, next: string, timestamp: number): KeyboardEventPayload[] {
+  let common = 0
+  while (common < previous.length && common < next.length && previous[common] === next[common]) common++
+  const events: KeyboardEventPayload[] = []
+  const press = (key: string, code: string, text?: string) => {
+    events.push({phase: "down", key, code, modifiers: MODIFIERS, text, repeat: false, timestamp})
+    events.push({phase: "up", key, code, modifiers: MODIFIERS, repeat: false, timestamp})
+  }
+  for (let i = common; i < previous.length; i++) press("Backspace", "Backspace")
+  for (const char of Array.from(next.slice(common))) {
+    if (char === "\n") press("Enter", "Enter")
+    else if (char === " ") press(" ", "Space", " ")
+    else press(char, codeFor(char), char)
+  }
+  return events
+}
+
+function codeFor(char: string): string {
+  if (/^[a-z]$/i.test(char)) return `Key${char.toUpperCase()}`
+  if (/^[0-9]$/.test(char)) return `Digit${char}`
+  return "Unidentified"
+}
+
+export class NeuralBandInput {
+  private claim: NeuralBandClaimStatus = {stage: "idle", sessionExpired: false}
+  private handwriting: NeuralBandHandwritingStatus = {phase: "idle", message: "", text: ""}
+  private claimListeners = new Set<(status: NeuralBandClaimStatus) => void>()
+  private handwritingListeners = new Set<(status: NeuralBandHandwritingStatus) => void>()
+  private readonly forward: (stream: string, payload: unknown) => void
+  private readonly now: () => number
+
+  constructor(options: NeuralBandInputOptions = {}) {
+    this.forward = options.forward ?? ((stream, payload) => localMiniappRuntime.forwardEvent(stream, payload))
+    this.now = options.now ?? (() => Date.now())
+  }
+
+  claimStatus(): NeuralBandClaimStatus {
+    return this.claim
+  }
+
+  onClaimStatus(cb: (status: NeuralBandClaimStatus) => void): () => void {
+    this.claimListeners.add(cb)
+    return () => this.claimListeners.delete(cb)
+  }
+
+  handwritingStatus(): NeuralBandHandwritingStatus {
+    return this.handwriting
+  }
+
+  onHandwritingStatus(cb: (status: NeuralBandHandwritingStatus) => void): () => void {
+    this.handwritingListeners.add(cb)
+    return () => this.handwritingListeners.delete(cb)
+  }
+
+  /** A new pairing attempt starts from a clean slate. */
+  resetClaim(): void {
+    this.setClaim({stage: "idle", sessionExpired: false})
+  }
+
+  handleClaim(event: NeuralBandClaimEvent): void {
+    this.setClaim({
+      stage: event.stage,
+      message: event.message,
+      sessionExpired: event.sessionExpired === true,
+      bandName: event.bandName,
+    })
+  }
+
+  handleHandwriting(event: NeuralBandHandwritingEvent): void {
+    if (event.kind === "state") {
+      const text = event.phase === "preparing" ? "" : this.handwriting.text
+      this.setHandwriting({phase: event.phase, message: event.message, problem: event.problem, text})
+      return
+    }
+    const previous = this.handwriting.text
+    if (event.text !== previous) {
+      for (const key of keyEventsForTextChange(previous, event.text, this.now())) {
+        this.forward(MiniappStreamType.KEYBOARD_EVENT, key)
+      }
+    }
+    this.setHandwriting({...this.handwriting, text: event.text})
+  }
+
+  handlePointer(event: NeuralBandPointerEvent): void {
+    if (event.kind === "move") {
+      this.forward(MiniappStreamType.POINTER_EVENT, {
+        kind: "move",
+        dx: event.dx,
+        dy: event.dy,
+        timestamp: event.timestamp,
+      })
+    } else {
+      this.forward(MiniappStreamType.POINTER_EVENT, {
+        kind: "buttons",
+        buttons: event.buttons,
+        timestamp: event.timestamp,
+      })
+    }
+  }
+
+  private setClaim(status: NeuralBandClaimStatus): void {
+    this.claim = status
+    for (const cb of this.claimListeners) cb(status)
+  }
+
+  private setHandwriting(status: NeuralBandHandwritingStatus): void {
+    this.handwriting = status
+    for (const cb of this.handwritingListeners) cb(status)
+  }
+}
+
+const neuralBandInput = new NeuralBandInput()
+export default neuralBandInput

package/src/services/neuralBand/NeuralBandSettingsSync.ts

@@ -0,0 +1,63 @@
+/**
+ * Pushes the Neural Band's user settings (air mouse, wearing hand) to the native
+ * controller: when the band connects, and whenever either setting changes while
+ * it is connected. The native side keeps nothing across connections.
+ */
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+
+import {useGlassesStore} from "../../stores/glasses"
+import {SETTINGS, useSettingsStore} from "../../stores/settings"
+import {ControllerTypes} from "../../types/enums"
+
+let unsubscribers: (() => void)[] = []
+
+function bandConnected(): boolean {
+  const glasses = useGlassesStore.getState()
+  const settings = useSettingsStore.getState()
+  return (
+    glasses.controllerConnected === true &&
+    glasses.controllerFullyBooted === true &&
+    settings.getSetting(SETTINGS.default_controller.key) === ControllerTypes.NEURAL_BAND
+  )
+}
+
+function pushAirMouse(): void {
+  if (!bandConnected()) return
+  const enabled = useSettingsStore.getState().getSetting(SETTINGS.neural_band_air_mouse.key) === true
+  void BluetoothSdk.neuralBandSetPointer(enabled).catch((err) =>
+    console.warn("NeuralBandSettingsSync: air mouse push failed:", err),
+  )
+}
+
+function pushHand(): void {
+  if (!bandConnected()) return
+  const hand = useSettingsStore.getState().getSetting(SETTINGS.neural_band_hand.key) === "left" ? "left" : "right"
+  void BluetoothSdk.neuralBandSetHand(hand).catch((err) =>
+    console.warn("NeuralBandSettingsSync: hand push failed:", err),
+  )
+}
+
+export function startNeuralBandSettingsSync(): void {
+  if (unsubscribers.length > 0) return
+  let wasConnected = bandConnected()
+  unsubscribers = [
+    useGlassesStore.subscribe(
+      (s) => s.controllerConnected === true && s.controllerFullyBooted === true,
+      () => {
+        const connected = bandConnected()
+        if (connected && !wasConnected) {
+          pushAirMouse()
+          pushHand()
+        }
+        wasConnected = connected
+      },
+    ),
+    useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.neural_band_air_mouse.key), pushAirMouse),
+    useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.neural_band_hand.key), pushHand),
+  ]
+}
+
+export function stopNeuralBandSettingsSync(): void {
+  for (const unsubscribe of unsubscribers) unsubscribe()
+  unsubscribers = []
+}

package/src/services/neuralBand/sha256.ts

@@ -0,0 +1,70 @@
+/**
+ * SHA-256 over a UTF-8 string, hex encoded. The Meta sign-in callback carries the
+ * first 16 hex digits of the request token's digest; nothing else here needs a
+ * hash, and the app ships no crypto dependency, so this stays small and local.
+ */
+
+const K = new Uint32Array([
+  0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98,
+  0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
+  0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8,
+  0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+  0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819,
+  0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
+  0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
+  0xc67178f2,
+])
+
+function rotr(x: number, n: number): number {
+  return (x >>> n) | (x << (32 - n))
+}
+
+export function sha256Hex(text: string): string {
+  const bytes = new TextEncoder().encode(text)
+  const bitLength = bytes.length * 8
+  const padded = new Uint8Array(Math.ceil((bytes.length + 9) / 64) * 64)
+  padded.set(bytes)
+  padded[bytes.length] = 0x80
+  const view = new DataView(padded.buffer)
+  view.setUint32(padded.length - 8, Math.floor(bitLength / 0x100000000))
+  view.setUint32(padded.length - 4, bitLength >>> 0)
+
+  const h = new Uint32Array([
+    0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
+  ])
+  const w = new Uint32Array(64)
+  for (let offset = 0; offset < padded.length; offset += 64) {
+    for (let i = 0; i < 16; i++) w[i] = view.getUint32(offset + i * 4)
+    for (let i = 16; i < 64; i++) {
+      const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >>> 3)
+      const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >>> 10)
+      w[i] = (w[i - 16] + s0 + w[i - 7] + s1) >>> 0
+    }
+    let [a, b, c, d, e, f, g, hh] = h
+    for (let i = 0; i < 64; i++) {
+      const s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25)
+      const ch = (e & f) ^ (~e & g)
+      const t1 = (hh + s1 + ch + K[i] + w[i]) >>> 0
+      const s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22)
+      const maj = (a & b) ^ (a & c) ^ (b & c)
+      const t2 = (s0 + maj) >>> 0
+      hh = g
+      g = f
+      f = e
+      e = (d + t1) >>> 0
+      d = c
+      c = b
+      b = a
+      a = (t1 + t2) >>> 0
+    }
+    h[0] = (h[0] + a) >>> 0
+    h[1] = (h[1] + b) >>> 0
+    h[2] = (h[2] + c) >>> 0
+    h[3] = (h[3] + d) >>> 0
+    h[4] = (h[4] + e) >>> 0
+    h[5] = (h[5] + f) >>> 0
+    h[6] = (h[6] + g) >>> 0
+    h[7] = (h[7] + hh) >>> 0
+  }
+  return Array.from(h, (word) => word.toString(16).padStart(8, "0")).join("")
+}

package/src/services/soniox/SonioxRealtimeClient.ts

@@ -10,8 +10,8 @@
  * machinery) or `@soniox/web` (MediaStream capture): the wire protocol is one
  * JSON config message, then binary PCM frames up / JSON token results down.
  * Protocol shape mirrors the vendored SDK's buildConfigMessage /
- * parseResultMessage (cloud-v2 runtime's provider is the reference
- * implementation: cloud-v2/packages/runtime/src/services/audio/providers/soniox.ts).
+ * parseResultMessage (cloud runtime's provider is the reference
+ * implementation: cloud/packages/runtime/src/services/audio/providers/soniox.ts).
  */
 
 import {BgTimer} from "../../utils/timers"

package/src/services/soniox/SonioxTokenAssembler.ts

@@ -2,7 +2,7 @@
  * SonioxTokenAssembler
  *
  * Pure port of the cloud Soniox provider's utterance assembly
- * (cloud-v2/packages/runtime/src/services/audio/providers/soniox.ts). Soniox
+ * (cloud/packages/runtime/src/services/audio/providers/soniox.ts). Soniox
  * emits a rolling token window that is re-finalized/compacted across results;
  * this class turns that stream into stable interim/final utterances:
  *

package/src/stores/cloudClientStatus.ts

@@ -1,6 +1,6 @@
 /**
  * Cloud-client runtime status store — moved into engine so engine owns the
- * cloud-v2 runtime status (connection status + audio transport). Re-exported
+ * cloud runtime status (connection status + audio transport). Re-exported
  * through the host's `@/stores/cloudClientStatus` shim so the app keeps its
  * imports, and surfaced as `engine.stores.cloudClientStatus`.
  */

package/src/stores/settings.ts

@@ -262,7 +262,7 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: false,
     persist: true,
   },
-  // Cloud V2 endpoint OVERRIDES. Empty = no override; cloudClient's resolveUrl
+  // Cloud endpoint OVERRIDES. Empty = no override; cloudClient's resolveUrl
   // owns the full precedence (override -> env -> Metro-derived dev default).
   // The value may be an explicit URL or the METRO_AUTO sentinel ("my dev
   // laptop", resolved live from Metro so it survives network changes). Never
@@ -284,7 +284,7 @@ export const SETTINGS: Record<string, Setting> = {
     persist: true,
     resetOnBuildEnvChange: true,
   },
-  // Bookmarked Cloud V2 endpoint pairs. Each entry is {label, coreUrl,
+  // Bookmarked Cloud endpoint pairs. Each entry is {label, coreUrl,
   // runtimeUrl} — core + runtime are saved together because they are always
   // applied as a matched set (presets fill both; Save & Test verifies both).
   saved_cloud_url_pairs: {
@@ -411,7 +411,7 @@ export const SETTINGS: Record<string, Setting> = {
     persist: true,
   },
   location_tier: {key: "location_tier", defaultValue: () => "", writable: true, saveOnServer: true, persist: true},
-  // Volatile bearer synced to Bluetooth for glasses-side Cloud V2 calls. The
+  // Volatile bearer synced to Bluetooth for glasses-side Cloud calls. The
   // key name is kept for BLE compatibility, but the value must never be loaded
   // from or saved to the legacy Cloud V1 settings store.
   core_token: {key: "core_token", defaultValue: () => "", writable: true, saveOnServer: false, persist: false},
@@ -744,10 +744,10 @@ export const SETTINGS: Record<string, Setting> = {
   },
   // Debug: route the R1 ring through the glasses (advStart + RING_CONNECT_INFO,
   // events forwarded by the CFW) instead of the phone link. OFF by default:
-  // the ring sends its touch reports to the glasses whenever it holds glasses
-  // targets, so the phone link only carries telemetry until those are erased.
-  // Synced to the Bluetooth SDK, which re-routes a connected ring on change.
-  // Device-local debug knob, so not saved on the server.
+  // the ring is a direct phone link whether or not a G2 is paired, which is
+  // how the Faceclaw reference client runs it. Synced to the Bluetooth SDK,
+  // which re-routes a connected ring on change. Device-local debug knob, so
+  // not saved on the server.
   ring_via_glasses: {
     key: "ring_via_glasses",
     defaultValue: () => false,
@@ -755,6 +755,24 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: false,
     persist: true,
   },
+  // Meta Neural Band: the air mouse streams the band's orientation at 128 Hz, so it
+  // stays off unless asked for. The band side is pushed when the band connects.
+  neural_band_air_mouse: {
+    key: "neural_band_air_mouse",
+    defaultValue: () => false,
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
+  // Which wrist wears the band: flips the air mouse's forearm axis and is written
+  // to the band's own hand setting on connect.
+  neural_band_hand: {
+    key: "neural_band_hand",
+    defaultValue: () => "right",
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
   // Maximum rate at which display updates leave the phone for the glasses.
   // The Bluetooth SDK paces every scene frame and legacy layout to this,
   // coalescing faster updates last-wins; clears bypass it. No effect on G2

package/src/types/enums.ts

@@ -17,4 +17,5 @@ export enum DeviceTypes {
 
 export enum ControllerTypes {
   R1 = "Even Realities R1",
+  NEURAL_BAND = "Meta Neural Band",
 }

package/src/utils/cloudClient/cloudSecureStore.ts

@@ -1,8 +1,8 @@
 /**
  * MMKV-backed KeyValueStore for @glassly/cloud-client — moved into engine so
- * engine owns the cloud-v2 credential storage (the v2 refresh token, so a
+ * engine owns the cloud credential storage (the v2 refresh token, so a
  * relaunch does not force a new login). A dedicated MMKV instance keeps
- * cloud-v2 credentials out of the app's general store. Re-exported through the
+ * cloud credentials out of the app's general store. Re-exported through the
  * host's `@/utils/cloudClient/MmkvSecureStore` shim.
  *
  * NOTE: MMKV is not hardware-backed. Before production handling of the