@glassly/engine 0.3.2-dev.586 → 0.3.2-dev.596

package/src/services/neuralBand/MetaAuth.ts

@@ -0,0 +1,192 @@
+/**
+ * The Meta account sign-in a Neural Band claim needs, ported from callbacked/kinesis
+ * (MIT, Copyright (c) 2026 callbacked): Sources/Kinesis/MetaAuth.swift.
+ *
+ * The chain: an anonymous tokens query, a WebView sign-in on auth.meta.com whose
+ * fb-viewapp://…frl_login redirect carries a token and a blob, a blob decrypt, and
+ * the ar-genai exchange that yields the user session. The session then goes to the
+ * native Bluetooth SDK (keychain / keystore), which runs the ownership exchanges;
+ * nothing here persists it.
+ */
+import {sha256Hex} from "./sha256"
+
+// Meta's own client identifiers, the same in every copy of Meta's apps. They name the
+// app that asks, not a person. They are not secrets, and no account or session is in them.
+const FRL_CLIENT = "FRL|388177446008673|083800dd7efbbd42eab18c9886d79c18"
+const AR_CLIENT = "AR|306760944872162|a919421a55a8ea18080ab2f10f57f1be"
+const TOKENS_QUERY_URL = "https://meta.graph.meta.com/webview_tokens_query"
+const BLOBS_DECRYPT_URL = "https://meta.graph.meta.com/webview_blobs_decrypt"
+const LOGIN_URL = "https://ar-genai.graph.meta.com/login"
+const AUTH_ENTRY_BASE =
+  "https://auth.meta.com/?native_app_id=388177446008673&source_app_id=388177446008673&native_sso_etoken="
+
+export class MetaAuthError extends Error {}
+
+export interface MetaSsoTokens {
+  nativeSsoToken: string
+  etoken: string
+}
+
+export interface MetaSession {
+  accessToken: string
+  userId: string
+}
+
+export function authEntryUrl(tokens: MetaSsoTokens): string {
+  return AUTH_ENTRY_BASE + encodeURIComponent(tokens.etoken)
+}
+
+/** The redirect that ends the sign-in: a custom scheme, or the frl_login marker on https. */
+export function isLoginCallback(url: string): boolean {
+  return url.toLowerCase().startsWith("fb-viewapp:") || url.includes("frl_login")
+}
+
+export function callbackQuery(url: string): {token?: string; blob?: string} {
+  const query = url.includes("?") ? url.slice(url.indexOf("?") + 1) : ""
+  const out: {token?: string; blob?: string} = {}
+  for (const pair of query.split("&")) {
+    const [key, value = ""] = pair.split("=")
+    if (key === "token") out.token = decodeURIComponent(value)
+    if (key === "blob") out.blob = decodeURIComponent(value)
+  }
+  return out
+}
+
+/** The first 16 hex digits of the request token's SHA-256, which the callback must present. */
+export function expectedCallbackToken(nativeSsoToken: string): string {
+  return sha256Hex(nativeSsoToken).slice(0, 16)
+}
+
+export function callbackMatches(token: string | undefined, nativeSsoToken: string): boolean {
+  if (!token || token.length !== 16) return false
+  const expected = expectedCallbackToken(nativeSsoToken)
+  let difference = 0
+  for (let i = 0; i < 16; i++) difference |= token.charCodeAt(i) ^ expected.charCodeAt(i)
+  return difference === 0
+}
+
+export function makeLsd(): string {
+  let digits = ""
+  for (let i = 0; i < 6; i++) digits += Math.floor(Math.random() * 10)
+  return "S0." + digits
+}
+
+export function jazoest(lsd: string): string {
+  let sum = 0
+  for (const char of lsd) sum += char.codePointAt(0) ?? 0
+  return "2" + sum
+}
+
+export function multipartBody(fields: [string, string][], boundary: string): string {
+  let body = ""
+  for (const [name, value] of fields) {
+    body += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`
+  }
+  return body + `--${boundary}--\r\n`
+}
+
+export function urlForm(fields: [string, string][]): string {
+  return fields.map(([name, value]) => `${name}=${encodeURIComponent(value)}`).join("&")
+}
+
+type Json = Record<string, unknown>
+
+async function post(url: string, headers: Record<string, string>, body: string): Promise<Json> {
+  const response = await fetch(url, {method: "POST", headers, body})
+  let json: Json | null = null
+  try {
+    json = (await response.json()) as Json
+  } catch {
+    json = null
+  }
+  if (response.status !== 200 || json === null) {
+    throw new MetaAuthError(`Meta sign-in failed (HTTP ${response.status}). Try again.`)
+  }
+  if (json.error !== undefined) {
+    const error = json.error as {code?: number; error_subcode?: number} | null
+    throw new MetaAuthError(
+      `Meta sign-in failed (code ${error?.code ?? "-"}, subcode ${error?.error_subcode ?? "-"}). Try again.`,
+    )
+  }
+  return json
+}
+
+async function postMultipart(url: string, fields: [string, string][]): Promise<Json> {
+  const boundary = `glassly.form.${Date.now().toString(36)}${Math.random().toString(36).slice(2)}`
+  return post(
+    url,
+    {
+      "Content-Type": `multipart/form-data; boundary=${boundary}`,
+      "Accept": "application/json",
+      "Origin": "https://auth.meta.com",
+    },
+    multipartBody(fields, boundary),
+  )
+}
+
+/** Step 1: the anonymous WebView tokens query. */
+export async function tokensQuery(): Promise<MetaSsoTokens> {
+  const lsd = makeLsd()
+  const json = await postMultipart(TOKENS_QUERY_URL, [
+    ["access_token", FRL_CLIENT],
+    ["lsd", lsd],
+    ["jazoest", jazoest(lsd)],
+  ])
+  const nativeSsoToken = json.native_sso_token
+  const etoken = json.native_sso_etoken
+  if (typeof nativeSsoToken !== "string" || !nativeSsoToken || typeof etoken !== "string" || !etoken) {
+    throw new MetaAuthError("Meta didn't return sign-in tokens. Try again.")
+  }
+  return {nativeSsoToken, etoken}
+}
+
+/** Step 3: decrypt the login blob into the FRL access token. */
+export async function decryptBlob(blob: string, requestToken: string): Promise<string> {
+  const lsd = makeLsd()
+  const json = await postMultipart(BLOBS_DECRYPT_URL, [
+    ["blob", blob],
+    ["request_token", requestToken],
+    ["access_token", FRL_CLIENT],
+    ["lsd", lsd],
+    ["jazoest", jazoest(lsd)],
+  ])
+  const token = json.access_token
+  if (typeof token !== "string" || !token) throw new MetaAuthError("Meta didn't confirm the sign-in. Try again.")
+  return token
+}
+
+function randomId(): string {
+  const hex = () =>
+    Math.floor(Math.random() * 0x10000)
+      .toString(16)
+      .padStart(4, "0")
+  return `${hex()}${hex()}-${hex()}-${hex()}-${hex()}-${hex()}${hex()}${hex()}`.toUpperCase()
+}
+
+/** Step 4: exchange the FRL token for the ar user session. */
+export async function login(frlAccessToken: string): Promise<MetaSession> {
+  const json = await post(
+    LOGIN_URL,
+    {
+      "Content-Type": "application/x-www-form-urlencoded",
+      "Accept": "application/json",
+      "Authorization": "OAuth " + AR_CLIENT,
+    },
+    urlForm([
+      ["frl_access_token", frlAccessToken],
+      ["logging_session_id", randomId()],
+      ["format", "json"],
+      ["device_id", randomId()],
+      ["generate_session_cookies", "1"],
+      ["generate_analytics_claim", "1"],
+      ["method", "POST"],
+    ]),
+  )
+  const accessToken = json.access_token
+  const rawUserId = json.user_id
+  const userId = typeof rawUserId === "string" ? rawUserId : typeof rawUserId === "number" ? String(rawUserId) : ""
+  if (typeof accessToken !== "string" || !accessToken || !userId) {
+    throw new MetaAuthError("Meta didn't return an account session. Try again.")
+  }
+  return {accessToken, userId}
+}

package/src/services/neuralBand/NeuralBandInput.ts

@@ -0,0 +1,148 @@
+/**
+ * The Meta Neural Band's own event streams, past the gestures that ride `touch_event`:
+ * claim progress (for the pairing screens), the handwriting model (typed into the
+ * miniapp keyboard stream as synthesized key presses) and the air mouse (forwarded
+ * to miniapps on the pointer stream).
+ */
+import type {NeuralBandClaimEvent, NeuralBandHandwritingEvent, NeuralBandPointerEvent} from "@glassly/bluetooth-sdk"
+import {MiniappStreamType} from "@glassly/miniapp"
+
+import type {KeyboardEventPayload} from "../GlassesKeyboardInput"
+import localMiniappRuntime from "../LocalMiniappRuntime"
+
+export interface NeuralBandClaimStatus {
+  stage: "idle" | "progress" | "completed" | "failed"
+  message?: string
+  sessionExpired: boolean
+  bandName?: string
+}
+
+export interface NeuralBandHandwritingStatus {
+  phase: "idle" | "preparing" | "ready" | "restoring" | "finished"
+  message: string
+  problem?: string
+  /** The last text the band produced since handwriting was switched on. */
+  text: string
+}
+
+export interface NeuralBandInputOptions {
+  forward?: (stream: string, payload: unknown) => void
+  now?: () => number
+}
+
+const MODIFIERS = {shift: false, ctrl: false, alt: false, meta: false}
+
+/** One handwriting change as the key presses that would have typed it. */
+export function keyEventsForTextChange(previous: string, next: string, timestamp: number): KeyboardEventPayload[] {
+  let common = 0
+  while (common < previous.length && common < next.length && previous[common] === next[common]) common++
+  const events: KeyboardEventPayload[] = []
+  const press = (key: string, code: string, text?: string) => {
+    events.push({phase: "down", key, code, modifiers: MODIFIERS, text, repeat: false, timestamp})
+    events.push({phase: "up", key, code, modifiers: MODIFIERS, repeat: false, timestamp})
+  }
+  for (let i = common; i < previous.length; i++) press("Backspace", "Backspace")
+  for (const char of Array.from(next.slice(common))) {
+    if (char === "\n") press("Enter", "Enter")
+    else if (char === " ") press(" ", "Space", " ")
+    else press(char, codeFor(char), char)
+  }
+  return events
+}
+
+function codeFor(char: string): string {
+  if (/^[a-z]$/i.test(char)) return `Key${char.toUpperCase()}`
+  if (/^[0-9]$/.test(char)) return `Digit${char}`
+  return "Unidentified"
+}
+
+export class NeuralBandInput {
+  private claim: NeuralBandClaimStatus = {stage: "idle", sessionExpired: false}
+  private handwriting: NeuralBandHandwritingStatus = {phase: "idle", message: "", text: ""}
+  private claimListeners = new Set<(status: NeuralBandClaimStatus) => void>()
+  private handwritingListeners = new Set<(status: NeuralBandHandwritingStatus) => void>()
+  private readonly forward: (stream: string, payload: unknown) => void
+  private readonly now: () => number
+
+  constructor(options: NeuralBandInputOptions = {}) {
+    this.forward = options.forward ?? ((stream, payload) => localMiniappRuntime.forwardEvent(stream, payload))
+    this.now = options.now ?? (() => Date.now())
+  }
+
+  claimStatus(): NeuralBandClaimStatus {
+    return this.claim
+  }
+
+  onClaimStatus(cb: (status: NeuralBandClaimStatus) => void): () => void {
+    this.claimListeners.add(cb)
+    return () => this.claimListeners.delete(cb)
+  }
+
+  handwritingStatus(): NeuralBandHandwritingStatus {
+    return this.handwriting
+  }
+
+  onHandwritingStatus(cb: (status: NeuralBandHandwritingStatus) => void): () => void {
+    this.handwritingListeners.add(cb)
+    return () => this.handwritingListeners.delete(cb)
+  }
+
+  /** A new pairing attempt starts from a clean slate. */
+  resetClaim(): void {
+    this.setClaim({stage: "idle", sessionExpired: false})
+  }
+
+  handleClaim(event: NeuralBandClaimEvent): void {
+    this.setClaim({
+      stage: event.stage,
+      message: event.message,
+      sessionExpired: event.sessionExpired === true,
+      bandName: event.bandName,
+    })
+  }
+
+  handleHandwriting(event: NeuralBandHandwritingEvent): void {
+    if (event.kind === "state") {
+      const text = event.phase === "preparing" ? "" : this.handwriting.text
+      this.setHandwriting({phase: event.phase, message: event.message, problem: event.problem, text})
+      return
+    }
+    const previous = this.handwriting.text
+    if (event.text !== previous) {
+      for (const key of keyEventsForTextChange(previous, event.text, this.now())) {
+        this.forward(MiniappStreamType.KEYBOARD_EVENT, key)
+      }
+    }
+    this.setHandwriting({...this.handwriting, text: event.text})
+  }
+
+  handlePointer(event: NeuralBandPointerEvent): void {
+    if (event.kind === "move") {
+      this.forward(MiniappStreamType.POINTER_EVENT, {
+        kind: "move",
+        dx: event.dx,
+        dy: event.dy,
+        timestamp: event.timestamp,
+      })
+    } else {
+      this.forward(MiniappStreamType.POINTER_EVENT, {
+        kind: "buttons",
+        buttons: event.buttons,
+        timestamp: event.timestamp,
+      })
+    }
+  }
+
+  private setClaim(status: NeuralBandClaimStatus): void {
+    this.claim = status
+    for (const cb of this.claimListeners) cb(status)
+  }
+
+  private setHandwriting(status: NeuralBandHandwritingStatus): void {
+    this.handwriting = status
+    for (const cb of this.handwritingListeners) cb(status)
+  }
+}
+
+const neuralBandInput = new NeuralBandInput()
+export default neuralBandInput

package/src/services/neuralBand/NeuralBandSettingsSync.ts

@@ -0,0 +1,63 @@
+/**
+ * Pushes the Neural Band's user settings (air mouse, wearing hand) to the native
+ * controller: when the band connects, and whenever either setting changes while
+ * it is connected. The native side keeps nothing across connections.
+ */
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+
+import {useGlassesStore} from "../../stores/glasses"
+import {SETTINGS, useSettingsStore} from "../../stores/settings"
+import {ControllerTypes} from "../../types/enums"
+
+let unsubscribers: (() => void)[] = []
+
+function bandConnected(): boolean {
+  const glasses = useGlassesStore.getState()
+  const settings = useSettingsStore.getState()
+  return (
+    glasses.controllerConnected === true &&
+    glasses.controllerFullyBooted === true &&
+    settings.getSetting(SETTINGS.default_controller.key) === ControllerTypes.NEURAL_BAND
+  )
+}
+
+function pushAirMouse(): void {
+  if (!bandConnected()) return
+  const enabled = useSettingsStore.getState().getSetting(SETTINGS.neural_band_air_mouse.key) === true
+  void BluetoothSdk.neuralBandSetPointer(enabled).catch((err) =>
+    console.warn("NeuralBandSettingsSync: air mouse push failed:", err),
+  )
+}
+
+function pushHand(): void {
+  if (!bandConnected()) return
+  const hand = useSettingsStore.getState().getSetting(SETTINGS.neural_band_hand.key) === "left" ? "left" : "right"
+  void BluetoothSdk.neuralBandSetHand(hand).catch((err) =>
+    console.warn("NeuralBandSettingsSync: hand push failed:", err),
+  )
+}
+
+export function startNeuralBandSettingsSync(): void {
+  if (unsubscribers.length > 0) return
+  let wasConnected = bandConnected()
+  unsubscribers = [
+    useGlassesStore.subscribe(
+      (s) => s.controllerConnected === true && s.controllerFullyBooted === true,
+      () => {
+        const connected = bandConnected()
+        if (connected && !wasConnected) {
+          pushAirMouse()
+          pushHand()
+        }
+        wasConnected = connected
+      },
+    ),
+    useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.neural_band_air_mouse.key), pushAirMouse),
+    useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.neural_band_hand.key), pushHand),
+  ]
+}
+
+export function stopNeuralBandSettingsSync(): void {
+  for (const unsubscribe of unsubscribers) unsubscribe()
+  unsubscribers = []
+}

package/src/services/neuralBand/sha256.ts

@@ -0,0 +1,70 @@
+/**
+ * SHA-256 over a UTF-8 string, hex encoded. The Meta sign-in callback carries the
+ * first 16 hex digits of the request token's digest; nothing else here needs a
+ * hash, and the app ships no crypto dependency, so this stays small and local.
+ */
+
+const K = new Uint32Array([
+  0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98,
+  0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
+  0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8,
+  0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+  0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819,
+  0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
+  0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
+  0xc67178f2,
+])
+
+function rotr(x: number, n: number): number {
+  return (x >>> n) | (x << (32 - n))
+}
+
+export function sha256Hex(text: string): string {
+  const bytes = new TextEncoder().encode(text)
+  const bitLength = bytes.length * 8
+  const padded = new Uint8Array(Math.ceil((bytes.length + 9) / 64) * 64)
+  padded.set(bytes)
+  padded[bytes.length] = 0x80
+  const view = new DataView(padded.buffer)
+  view.setUint32(padded.length - 8, Math.floor(bitLength / 0x100000000))
+  view.setUint32(padded.length - 4, bitLength >>> 0)
+
+  const h = new Uint32Array([
+    0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
+  ])
+  const w = new Uint32Array(64)
+  for (let offset = 0; offset < padded.length; offset += 64) {
+    for (let i = 0; i < 16; i++) w[i] = view.getUint32(offset + i * 4)
+    for (let i = 16; i < 64; i++) {
+      const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >>> 3)
+      const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >>> 10)
+      w[i] = (w[i - 16] + s0 + w[i - 7] + s1) >>> 0
+    }
+    let [a, b, c, d, e, f, g, hh] = h
+    for (let i = 0; i < 64; i++) {
+      const s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25)
+      const ch = (e & f) ^ (~e & g)
+      const t1 = (hh + s1 + ch + K[i] + w[i]) >>> 0
+      const s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22)
+      const maj = (a & b) ^ (a & c) ^ (b & c)
+      const t2 = (s0 + maj) >>> 0
+      hh = g
+      g = f
+      f = e
+      e = (d + t1) >>> 0
+      d = c
+      c = b
+      b = a
+      a = (t1 + t2) >>> 0
+    }
+    h[0] = (h[0] + a) >>> 0
+    h[1] = (h[1] + b) >>> 0
+    h[2] = (h[2] + c) >>> 0
+    h[3] = (h[3] + d) >>> 0
+    h[4] = (h[4] + e) >>> 0
+    h[5] = (h[5] + f) >>> 0
+    h[6] = (h[6] + g) >>> 0
+    h[7] = (h[7] + hh) >>> 0
+  }
+  return Array.from(h, (word) => word.toString(16).padStart(8, "0")).join("")
+}

package/src/stores/settings.ts

@@ -755,6 +755,24 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: false,
     persist: true,
   },
+  // Meta Neural Band: the air mouse streams the band's orientation at 128 Hz, so it
+  // stays off unless asked for. The band side is pushed when the band connects.
+  neural_band_air_mouse: {
+    key: "neural_band_air_mouse",
+    defaultValue: () => false,
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
+  // Which wrist wears the band: flips the air mouse's forearm axis and is written
+  // to the band's own hand setting on connect.
+  neural_band_hand: {
+    key: "neural_band_hand",
+    defaultValue: () => "right",
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
   // Maximum rate at which display updates leave the phone for the glasses.
   // The Bluetooth SDK paces every scene frame and legacy layout to this,
   // coalescing faster updates last-wins; clears bypass it. No effect on G2

package/src/types/enums.ts

@@ -17,4 +17,5 @@ export enum DeviceTypes {
 
 export enum ControllerTypes {
   R1 = "Even Realities R1",
+  NEURAL_BAND = "Meta Neural Band",
 }