@glassly/engine 0.3.5-beta.710 → 0.3.5-beta.760

package/src/services/appleWatch/AppleWatchLink.ts

@@ -0,0 +1,297 @@
+/**
+ * The Apple Watch remote's phone half: the watch app's status context (glasses, the miniapp in
+ * front, launchable miniapps, whether Assist takes questions, the mirror setting), mirror frames
+ * while the setting is on, and the commands the watch sends back. Native adds whether the active
+ * watch is paired and keeps the newest frame for when the watch app opens. Gestures ride
+ * `touch_event` like any controller's.
+ */
+import type {AppleWatchCommandEvent} from "@glassly/bluetooth-sdk"
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+import type {MirrorSnapshot} from "@glassly/miniapp"
+import {MiniappStreamType} from "@glassly/miniapp"
+import {Platform} from "react-native"
+
+import {useAppStatusStore} from "../../stores/apps"
+import {useGlassesStore} from "../../stores/glasses"
+import {SETTINGS, useSettingsStore} from "../../stores/settings"
+import type {ClientApp} from "../../types"
+import {BgTimer} from "../../utils/timers"
+import {isGlassesConnected} from "../GlassesReadiness"
+import localMiniappRuntime from "../LocalMiniappRuntime"
+import {mirrorService} from "../MirrorService"
+import {keyEventsForTextChange} from "../neuralBand/NeuralBandInput"
+
+export const ASSIST_PACKAGE = "com.glassly.assistant"
+export const ASSIST_ACTION = "ask"
+/** WatchConnectivity messages top out near 64 KB; frames past this are skipped, not truncated. */
+const MAX_FRAME_BYTES = 48_000
+const MAX_IMAGE_CHARS = 16_000
+const MAX_APPS = 24
+const STATE_DEBOUNCE_MS = 250
+const MIRROR_INTERVAL_MS = 500
+
+export interface WatchStateInput {
+  glasses: {connected: boolean; model: string; battery: number}
+  apps: readonly ClientApp[]
+  assist: boolean
+  mirror: boolean
+}
+
+/** Miniapps the watch can open: shown, compatible, and drawing on the glasses. */
+export function launchableApps(apps: readonly ClientApp[]): {packageName: string; name: string}[] {
+  return apps
+    .filter((app) => app.type === "standard" && !app.hidden && app.compatibility?.isCompatible !== false)
+    .filter((app) => app.packageName && !app.packageName.includes("@empty"))
+    .slice(0, MAX_APPS)
+    .map((app) => ({packageName: app.packageName, name: app.name}))
+}
+
+export function watchState({glasses, apps, assist, mirror}: WatchStateInput): Record<string, unknown> {
+  const front = apps.find((app) => app.foregrounded && app.running && app.type === "standard")
+  const state: Record<string, unknown> = {
+    glasses: {connected: glasses.connected, name: glasses.model || "Glasses", battery: glasses.battery},
+    apps: launchableApps(apps),
+    assist,
+    mirror,
+  }
+  if (front) state.app = {packageName: front.packageName, name: front.name}
+  return state
+}
+
+type WatchElement = Record<string, string | number | boolean | number[]>
+
+const level = (color: unknown, fallback = 15) =>
+  typeof color === "number" ? Math.max(0, Math.min(15, color)) : fallback
+
+/** The glasses frame in the watch's drawing vocabulary; null when there is nothing to draw. */
+export function mirrorFrame(snapshot: MirrorSnapshot): Record<string, unknown> | null {
+  const canvas = snapshot.canvas
+  if (!canvas) return null
+  const content = snapshot.content
+  if (content.kind === "bitmap") {
+    if (content.data.length > MAX_FRAME_BYTES) return null
+    return {width: canvas.width, height: canvas.height, bitmap: content.data, elements: []}
+  }
+  if (content.kind !== "scene") return null
+  const elements: WatchElement[] = []
+  for (const element of content.elements) {
+    const {x, y, w, h} = element.box
+    const style = element.style ?? {}
+    switch (element.type) {
+      case "text":
+        if (!element.text) break
+        elements.push({
+          type: "text",
+          x,
+          y,
+          w,
+          h,
+          text: element.text,
+          size: style.fontSize ?? 20,
+          color: level(style.color),
+        })
+        break
+      case "rect":
+        elements.push({
+          type: "rect",
+          x,
+          y,
+          w,
+          h,
+          fill: style.fill === true,
+          border: style.border ?? 1,
+          radius: style.radius ?? 0,
+          color: level(style.color),
+        })
+        break
+      case "circle":
+        elements.push({
+          type: "circle",
+          x,
+          y,
+          w,
+          h,
+          fill: style.fill === true,
+          border: style.border ?? 1,
+          color: level(style.color),
+        })
+        break
+      case "line":
+      case "triangle":
+      case "quad": {
+        const points = (element.points ?? []).flatMap((point) => [point.x, point.y])
+        if (points.length < 4) break
+        const closed = element.type !== "line"
+        elements.push({
+          type: "line",
+          points,
+          width: style.width ?? style.border ?? 1,
+          color: level(style.color),
+          closed,
+          fill: closed && style.fill === true,
+        })
+        break
+      }
+      case "image":
+        if (element.data && element.data.length <= MAX_IMAGE_CHARS)
+          elements.push({type: "image", x, y, w, h, data: element.data})
+        break
+    }
+  }
+  const frame = {width: canvas.width, height: canvas.height, elements}
+  return JSON.stringify(frame).length > MAX_FRAME_BYTES ? null : frame
+}
+
+export interface AppleWatchCommandDeps {
+  apps: () => readonly ClientApp[]
+  start: (app: ClientApp) => Promise<unknown>
+  stop: (packageName: string) => Promise<unknown>
+  ask: (query: string) => Promise<unknown>
+  type: (stream: string, payload: unknown) => void
+  now: () => number
+}
+
+const defaultDeps: AppleWatchCommandDeps = {
+  apps: () => useAppStatusStore.getState().apps,
+  start: (app) => useAppStatusStore.getState().start(app),
+  stop: (packageName) => useAppStatusStore.getState().stop(packageName),
+  ask: (query) => localMiniappRuntime.invokeActionFromHost(ASSIST_PACKAGE, ASSIST_ACTION, {query}),
+  type: (stream, payload) => localMiniappRuntime.forwardEvent(stream, payload),
+  now: () => Date.now(),
+}
+
+export async function handleAppleWatchCommand(
+  event: AppleWatchCommandEvent,
+  deps: AppleWatchCommandDeps = defaultDeps,
+): Promise<void> {
+  switch (event.command) {
+    case "launch": {
+      const app = launchableApps(deps.apps()).some((entry) => entry.packageName === event.packageName)
+        ? deps.apps().find((entry) => entry.packageName === event.packageName)
+        : undefined
+      if (!app) {
+        console.warn(`AppleWatch: launch of unknown miniapp ${event.packageName}`)
+        return
+      }
+      await deps.start(app)
+      return
+    }
+    case "stop": {
+      const front = deps.apps().find((app) => app.foregrounded && app.running && app.type === "standard")
+      if (front) await deps.stop(front.packageName)
+      return
+    }
+    case "assist": {
+      const query = event.text.trim()
+      if (query) await deps.ask(query)
+      return
+    }
+    case "text":
+      for (const key of keyEventsForTextChange("", event.text, deps.now()))
+        deps.type(MiniappStreamType.KEYBOARD_EVENT, key)
+      return
+  }
+}
+
+let stops: (() => void)[] = []
+let stopMirror: (() => void) | null = null
+
+function currentState(): Record<string, unknown> {
+  const glasses = useGlassesStore.getState()
+  return watchState({
+    glasses: {
+      connected: isGlassesConnected(glasses.connection),
+      model: glasses.deviceModel,
+      battery: glasses.batteryLevel,
+    },
+    apps: useAppStatusStore.getState().apps,
+    assist: localMiniappRuntime.hostActionAvailable(ASSIST_PACKAGE, ASSIST_ACTION),
+    mirror: useSettingsStore.getState().getSetting(SETTINGS.apple_watch_mirror.key) === true,
+  })
+}
+
+function syncMirror(enabled: boolean): void {
+  if (!enabled) {
+    if (stopMirror) {
+      stopMirror()
+      stopMirror = null
+      void BluetoothSdk.setWatchMirror(null).catch(() => {})
+    }
+    return
+  }
+  if (stopMirror) return
+  let lastSentAt = 0
+  let pending: MirrorSnapshot | null = null
+  let timer: ReturnType<typeof BgTimer.setTimeout> | null = null
+  const send = (snapshot: MirrorSnapshot) => {
+    lastSentAt = Date.now()
+    void BluetoothSdk.setWatchMirror(mirrorFrame(snapshot)).catch((err) =>
+      console.warn("AppleWatch: mirror push failed:", err),
+    )
+  }
+  const unsubscribe = mirrorService.subscribe((snapshot) => {
+    const wait = lastSentAt + MIRROR_INTERVAL_MS - Date.now()
+    if (wait <= 0 && !timer) {
+      send(snapshot)
+      return
+    }
+    pending = snapshot
+    timer ??= BgTimer.setTimeout(
+      () => {
+        timer = null
+        if (pending) send(pending)
+        pending = null
+      },
+      Math.max(wait, 0),
+    )
+  })
+  stopMirror = () => {
+    unsubscribe()
+    if (timer) BgTimer.clearTimeout(timer)
+    timer = null
+  }
+}
+
+export function startAppleWatchLink(): void {
+  if (Platform.OS !== "ios" || stops.length > 0) return
+  let lastKey = ""
+  let timer: ReturnType<typeof BgTimer.setTimeout> | null = null
+  const push = () => {
+    timer = null
+    const state = currentState()
+    const key = JSON.stringify(state)
+    if (key === lastKey) return
+    lastKey = key
+    void BluetoothSdk.setWatchState(state).catch((err) => console.warn("AppleWatch: state push failed:", err))
+  }
+  const schedule = () => {
+    timer ??= BgTimer.setTimeout(push, STATE_DEBOUNCE_MS)
+  }
+  const mirrorEnabled = () => useSettingsStore.getState().getSetting(SETTINGS.apple_watch_mirror.key) === true
+  const commands = BluetoothSdk.addListener("apple_watch_command", (event) => {
+    handleAppleWatchCommand(event).catch((err) => console.warn(`AppleWatch: ${event.command} failed:`, err))
+  })
+  stops = [
+    () => commands.remove(),
+    useGlassesStore.subscribe(schedule),
+    useAppStatusStore.subscribe(schedule),
+    useSettingsStore.subscribe(
+      (s) => s.getSetting(SETTINGS.apple_watch_mirror.key),
+      () => {
+        syncMirror(mirrorEnabled())
+        schedule()
+      },
+    ),
+    () => {
+      if (timer) BgTimer.clearTimeout(timer)
+      timer = null
+    },
+  ]
+  syncMirror(mirrorEnabled())
+  schedule()
+}
+
+export function stopAppleWatchLink(): void {
+  stops.splice(0).forEach((stop) => stop())
+  syncMirror(false)
+}

package/src/services/controllers.ts

@@ -26,9 +26,26 @@ const NEURAL_BAND_SURFACE: InputSurface = {
   ],
 }
 
+/** The Apple Watch trackpad's taps and four-way swipes, and the Digital Crown as the dial. */
+const APPLE_WATCH_SURFACE: InputSurface = {
+  type: "swipe2d",
+  events: [
+    "TAP",
+    "DOUBLE_TAP",
+    "PRESS_HOLD",
+    "SWIPE_UP",
+    "SWIPE_DOWN",
+    "SWIPE_LEFT",
+    "SWIPE_RIGHT",
+    "DIAL_UP",
+    "DIAL_DOWN",
+  ],
+}
+
 // The R1 swipes up/down like the temples it stands in for, so it adds nothing.
 const INPUT_SURFACES: Readonly<Record<string, readonly InputSurface[]>> = {
   [ControllerTypes.NEURAL_BAND]: [NEURAL_BAND_SURFACE],
+  [ControllerTypes.APPLE_WATCH]: [APPLE_WATCH_SURFACE],
   [ControllerTypes.R1]: [],
 }
 

package/src/services/settingsBridge.ts

@@ -6,10 +6,10 @@
  * be tested without the native import surface the runtime pulls in.
  *
  * A dashboard miniapp is the usual consumer: it edits the notification,
- * glasses-menu and display-position keys the phone's settings screens used
- * to, through the store, so persistence, server sync and the Bluetooth sync
- * of `menu_apps` and the position behave as if a phone screen wrote them. A dashboard's own look is not here; it keeps
- * that in its storage.
+ * glasses-menu, display-position, brightness and microphone keys the phone's
+ * settings screens do, through the store, so persistence, server sync and the
+ * Bluetooth sync behave as if a phone screen wrote them. A dashboard's own
+ * look is not here; it keeps that in its storage.
  */
 
 import type {ControllerStatus, PairedController} from "@glassly/bluetooth-sdk"
@@ -28,6 +28,11 @@ export const SETTINGS_BRIDGE_KEYS: Readonly<Record<string, SettingsBridgeAccess>
   // Where the glasses draw; each model's steps are its display.positionRange.
   dashboard_depth: "write",
   dashboard_height: "write",
+  // Device settings the dashboard's glasses Settings page edits.
+  brightness: "write",
+  auto_brightness: "write",
+  // Per paired model: the store resolves the indexed key.
+  preferred_mic: "write",
   twelve_hour_time: "write",
   menu_apps: "write",
   notifications_enabled: "write",

package/src/stores/apps.ts

@@ -256,6 +256,7 @@ function projectApps(previousState: AppStatusState, localApps: ClientApp[]): Cli
         // across both passes.
         foregrounded: app.packageName === previousState.foregroundedPackage,
         paused: app.local && app.running && isMiniappsPaused(),
+        widgetOnly: !!app.local && app.running && miniappWidgetMode.isWidgetOnly(app.packageName),
       }
       const prev = previousByPackage.get(app.packageName)
       return prev && shallowEqualApp(prev, next) ? prev : next
@@ -362,6 +363,11 @@ export const useAppStatusStore = create<AppStatusState>((set, get) => ({
       return false
     }
 
+    // An app running only for its widget is promoted before the host hook:
+    // the hook may foreground it, and setForeground sends a widget-only app
+    // back here.
+    if (app.widgetOnly) miniappWidgetMode.setWidgetOnly(packageName, false)
+
     // Host navigation / open-animation seam. Runs on EVERY accepted open —
     // including re-opening an already-running app — so the animation always
     // plays (BuiltInMiniappCatalog.navigateForApp: overlay apps setForeground
@@ -374,13 +380,12 @@ export const useAppStatusStore = create<AppStatusState>((set, get) => ({
 
     // If the tapped app is already running, this is a re-open: the nav/foreground
     // above already drove the animation, and there's nothing to spawn. Short-
-    // circuit before the spawn pipeline. An app running only for its widget
-    // is promoted instead: it keeps its context and takes the foreground.
+    // circuit before the spawn pipeline. A promoted app keeps its context but
+    // still takes the foreground below.
     if (app.running && !app.widgetOnly) {
       saveLastOpenTime(packageName)
       return true
     }
-    if (app.widgetOnly) miniappWidgetMode.setWidgetOnly(packageName, false)
 
     // Only-one rules: one standard app in the foreground, one dashboard
     // drawing the overlay. Starting another stops the one running. Widget-only
@@ -495,13 +500,10 @@ export const useAppStatusStore = create<AppStatusState>((set, get) => ({
       apps: s.apps.map((a) => ({...a, foregrounded: a.packageName === packageName})),
     }))
 
-    // Ensure the JSContext exists. start() is idempotent for an
-    // already-running app and enforces the foreground-only-one rule for
-    // standard apps. Fire-and-forget after the flag flip so it doesn't block
-    // paint.
-    // if (!app.running) {
-    //   get().start(app)
-    // }
+    // The wearer is in this app now: one running only for its widget becomes
+    // a normal run. start() promotes it, stops the standard peer and keeps the
+    // context. Fire-and-forget after the flag flip so it doesn't block paint.
+    if (app.running && app.widgetOnly) void get().start(app, {skipNavigation: true})
   },
 
   clearForeground: () => {

package/src/stores/bluetoothSettingKeys.ts

@@ -35,3 +35,23 @@ function isMentraLiveDevice(model: string | undefined): boolean {
 export function getBluetoothSettingKeysForDevice(deviceModel: string | undefined, fullKeys: string[]): string[] {
   return isMentraLiveDevice(deviceModel) ? MENTRA_LIVE_SETTING_KEYS : fullKeys
 }
+
+/**
+ * Glasses Menu entries the firmware's own menu cannot open: Settings is the
+ * dashboard's glasses page, which only the dashboard's Apps menu shows.
+ */
+const NATIVE_MENU_EXCLUDED_PACKAGES = new Set(["com.glassly.settings"])
+const nativeMenuAppsCache = new WeakMap<object, unknown[]>()
+
+/** `menu_apps` as the glasses get it; the same array in gives the same array out, so the change-push stays quiet. */
+export function nativeMenuApps(value: unknown): unknown {
+  if (!Array.isArray(value)) return value
+  let filtered = nativeMenuAppsCache.get(value)
+  if (!filtered) {
+    filtered = value.filter(
+      (item) => !NATIVE_MENU_EXCLUDED_PACKAGES.has((item as {packageName?: string} | null)?.packageName ?? ""),
+    )
+    nativeMenuAppsCache.set(value, filtered)
+  }
+  return filtered
+}

package/src/stores/settings.ts

@@ -4,7 +4,7 @@ import {create} from "zustand"
 import {subscribeWithSelector} from "zustand/middleware"
 
 import {storage} from "../utils/storage"
-import {getBluetoothSettingKeysForDevice} from "./bluetoothSettingKeys"
+import {getBluetoothSettingKeysForDevice, nativeMenuApps} from "./bluetoothSettingKeys"
 import {useGlassesStore} from "./glasses"
 
 export {MENTRA_LIVE_SETTING_KEYS, getBluetoothSettingKeysForDevice} from "./bluetoothSettingKeys"
@@ -831,6 +831,15 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: false,
     persist: true,
   },
+  // Apple Watch: draw the glasses' current frame behind the watch trackpad. Frames
+  // cross WatchConnectivity while the watch app is up, so it stays off unless asked for.
+  apple_watch_mirror: {
+    key: "apple_watch_mirror",
+    defaultValue: () => false,
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
   // Maximum rate at which display updates leave the phone for the glasses.
   // The Bluetooth SDK paces every scene frame and legacy layout to this,
   // coalescing faster updates last-wins; clears bypass it. No effect on G2
@@ -1577,6 +1586,29 @@ export const useSettingsStore = create<SettingsState>()(
           // desired value was applied so a transient failure retries on boot.
           if (migrationSucceeded) storage.save(CAMERA_FOV_MIGRATION_KEY, true)
         }
+
+        // Settings joined the Glasses Menu (the dashboard's glasses Settings
+        // page): add it once, last, to a menu the wearer already saved. An
+        // unsaved menu gets it from the default; a full one (10) is left alone.
+        const MENU_SETTINGS_MIGRATION_KEY = "migration:menu_apps_settings_v1"
+        const menuSettingsMigrationDone = storage.load<boolean>(MENU_SETTINGS_MIGRATION_KEY)
+        if (menuSettingsMigrationDone.is_error() || !menuSettingsMigrationDone.value) {
+          const menu = get().getSetting(SETTINGS.menu_apps.key) as Array<{packageName?: string}> | null
+          let menuMigrated = true
+          if (
+            Array.isArray(menu) &&
+            menu.length < 10 &&
+            !menu.some((item) => item?.packageName === "com.glassly.settings")
+          ) {
+            const next = [...menu, {packageName: "com.glassly.settings", name: "Settings"}]
+            const result = await get().setSetting(SETTINGS.menu_apps.key, next, true)
+            if (result.is_error()) {
+              console.log("SETTINGS: menu_apps Settings migration failed:", result.error)
+              menuMigrated = false
+            }
+          }
+          if (menuMigrated) storage.save(MENU_SETTINGS_MIGRATION_KEY, true)
+        }
       })
       loadAllSettingsInFlight = inFlight
       void Promise.resolve(inFlight).then((result) => {
@@ -1598,7 +1630,7 @@ export const useSettingsStore = create<SettingsState>()(
         if (key === SETTINGS.core_token.key && (typeof value !== "string" || value.trim().length === 0)) {
           continue
         }
-        bluetoothSettings[key] = value
+        bluetoothSettings[key] = key === SETTINGS.menu_apps.key ? nativeMenuApps(value) : value
       }
       return bluetoothSettings
     },

package/src/types/capabilities/rayneo-io.ts

@@ -32,6 +32,8 @@ export const rayneoIO: Capabilities = {
     fieldOfView: {horizontal: 30},
     maxTextLines: 5,
     adjustBrightness: true,
+    // Both drivers map these settings onto the launcher's three heights and two distances.
+    positionRange: {height: {min: 0, max: 8}, depth: {min: 1, max: 3}},
 
     // Scene display API — the Glassly CFW page canvas.
     width: 540,

package/src/types/enums.ts

@@ -18,4 +18,5 @@ export enum DeviceTypes {
 export enum ControllerTypes {
   R1 = "Even Realities R1",
   NEURAL_BAND = "Meta Neural Band",
+  APPLE_WATCH = "Apple Watch",
 }

package/src/utils/display/scene/index.ts

@@ -17,6 +17,7 @@ export type {
   SceneStrokeStyle,
   SceneTextStyle,
   SceneTransition,
+  SceneTransitionEdge,
 } from "./types"
 export {boxesEqual, contentHash, elementContentHash} from "./types"
 export {resolveAnchors} from "./anchor"

package/src/utils/display/scene/process.ts

@@ -21,6 +21,7 @@ import type {
   ScenePoint,
   SceneTextStyle,
   SceneTransition,
+  SceneTransitionEdge,
 } from "./types"
 import {MAX_SCENE_DEPTH, POINT_SHAPE_ARITY, POINT_SHAPE_TYPES, elementContentHash} from "./types"
 
@@ -66,11 +67,24 @@ function pointsBox(points: ScenePoint[]): SceneBox {
   return {x, y, w: Math.max(...xs) - x + 1, h: Math.max(...ys) - y + 1}
 }
 
+function validEdge(e: unknown): SceneTransitionEdge | undefined {
+  if (!e || typeof e !== "object") return undefined
+  const dx = Math.round(Number((e as SceneTransitionEdge).dx ?? 0))
+  const dy = Math.round(Number((e as SceneTransitionEdge).dy ?? 0))
+  if (!isFinite(dx) || !isFinite(dy) || Math.abs(dx) > 1024 || Math.abs(dy) > 1024) return undefined
+  return {dx, dy}
+}
+
 function validTransition(t: unknown): SceneTransition | undefined {
   if (!t || typeof t !== "object") return undefined
   const {durationMs, easing} = t as SceneTransition
   if (!isFinite(durationMs) || durationMs <= 0) return undefined
-  return easing ? {durationMs, easing} : {durationMs}
+  const out: SceneTransition = easing ? {durationMs, easing} : {durationMs}
+  const enter = validEdge((t as SceneTransition).enter)
+  const exit = validEdge((t as SceneTransition).exit)
+  if (enter) out.enter = enter
+  if (exit) out.exit = exit
+  return out
 }
 
 function boxShrunk(orig: SceneBox, clamped: SceneBox): boolean {
@@ -175,6 +189,8 @@ export function processScene(
 
   for (const {el, index, box, points} of valid) {
     const transition = validTransition((el as {transition?: unknown}).transition)
+    // Leaving elements only exist to tween away; a device that jumps never shows them.
+    if (transition?.exit && !caps.animation) continue
     const transform: {rotation?: number; pivot?: ScenePoint} = {}
     if (el.rotation !== undefined || el.pivot !== undefined) {
       const angle = el.rotation ?? 0

package/src/utils/display/scene/types.ts

@@ -18,6 +18,7 @@ export type {
   SceneStrokeStyle,
   SceneTextStyle,
   SceneTransition,
+  SceneTransitionEdge,
 } from "@glassly/miniapp/display"
 export {
   boxesEqual,