@glassly/engine 0.3.5-dev.745 → 0.3.5-dev.746

package/src/services/appleWatch/AppleWatchLink.ts

@@ -0,0 +1,297 @@
+/**
+ * The Apple Watch remote's phone half: the watch app's status context (glasses, the miniapp in
+ * front, launchable miniapps, whether Assist takes questions, the mirror setting), mirror frames
+ * while the setting is on, and the commands the watch sends back. Native adds whether the active
+ * watch is paired and keeps the newest frame for when the watch app opens. Gestures ride
+ * `touch_event` like any controller's.
+ */
+import type {AppleWatchCommandEvent} from "@glassly/bluetooth-sdk"
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+import type {MirrorSnapshot} from "@glassly/miniapp"
+import {MiniappStreamType} from "@glassly/miniapp"
+import {Platform} from "react-native"
+
+import {useAppStatusStore} from "../../stores/apps"
+import {useGlassesStore} from "../../stores/glasses"
+import {SETTINGS, useSettingsStore} from "../../stores/settings"
+import type {ClientApp} from "../../types"
+import {BgTimer} from "../../utils/timers"
+import {isGlassesConnected} from "../GlassesReadiness"
+import localMiniappRuntime from "../LocalMiniappRuntime"
+import {mirrorService} from "../MirrorService"
+import {keyEventsForTextChange} from "../neuralBand/NeuralBandInput"
+
+export const ASSIST_PACKAGE = "com.glassly.assistant"
+export const ASSIST_ACTION = "ask"
+/** WatchConnectivity messages top out near 64 KB; frames past this are skipped, not truncated. */
+const MAX_FRAME_BYTES = 48_000
+const MAX_IMAGE_CHARS = 16_000
+const MAX_APPS = 24
+const STATE_DEBOUNCE_MS = 250
+const MIRROR_INTERVAL_MS = 500
+
+export interface WatchStateInput {
+  glasses: {connected: boolean; model: string; battery: number}
+  apps: readonly ClientApp[]
+  assist: boolean
+  mirror: boolean
+}
+
+/** Miniapps the watch can open: shown, compatible, and drawing on the glasses. */
+export function launchableApps(apps: readonly ClientApp[]): {packageName: string; name: string}[] {
+  return apps
+    .filter((app) => app.type === "standard" && !app.hidden && app.compatibility?.isCompatible !== false)
+    .filter((app) => app.packageName && !app.packageName.includes("@empty"))
+    .slice(0, MAX_APPS)
+    .map((app) => ({packageName: app.packageName, name: app.name}))
+}
+
+export function watchState({glasses, apps, assist, mirror}: WatchStateInput): Record<string, unknown> {
+  const front = apps.find((app) => app.foregrounded && app.running && app.type === "standard")
+  const state: Record<string, unknown> = {
+    glasses: {connected: glasses.connected, name: glasses.model || "Glasses", battery: glasses.battery},
+    apps: launchableApps(apps),
+    assist,
+    mirror,
+  }
+  if (front) state.app = {packageName: front.packageName, name: front.name}
+  return state
+}
+
+type WatchElement = Record<string, string | number | boolean | number[]>
+
+const level = (color: unknown, fallback = 15) =>
+  typeof color === "number" ? Math.max(0, Math.min(15, color)) : fallback
+
+/** The glasses frame in the watch's drawing vocabulary; null when there is nothing to draw. */
+export function mirrorFrame(snapshot: MirrorSnapshot): Record<string, unknown> | null {
+  const canvas = snapshot.canvas
+  if (!canvas) return null
+  const content = snapshot.content
+  if (content.kind === "bitmap") {
+    if (content.data.length > MAX_FRAME_BYTES) return null
+    return {width: canvas.width, height: canvas.height, bitmap: content.data, elements: []}
+  }
+  if (content.kind !== "scene") return null
+  const elements: WatchElement[] = []
+  for (const element of content.elements) {
+    const {x, y, w, h} = element.box
+    const style = element.style ?? {}
+    switch (element.type) {
+      case "text":
+        if (!element.text) break
+        elements.push({
+          type: "text",
+          x,
+          y,
+          w,
+          h,
+          text: element.text,
+          size: style.fontSize ?? 20,
+          color: level(style.color),
+        })
+        break
+      case "rect":
+        elements.push({
+          type: "rect",
+          x,
+          y,
+          w,
+          h,
+          fill: style.fill === true,
+          border: style.border ?? 1,
+          radius: style.radius ?? 0,
+          color: level(style.color),
+        })
+        break
+      case "circle":
+        elements.push({
+          type: "circle",
+          x,
+          y,
+          w,
+          h,
+          fill: style.fill === true,
+          border: style.border ?? 1,
+          color: level(style.color),
+        })
+        break
+      case "line":
+      case "triangle":
+      case "quad": {
+        const points = (element.points ?? []).flatMap((point) => [point.x, point.y])
+        if (points.length < 4) break
+        const closed = element.type !== "line"
+        elements.push({
+          type: "line",
+          points,
+          width: style.width ?? style.border ?? 1,
+          color: level(style.color),
+          closed,
+          fill: closed && style.fill === true,
+        })
+        break
+      }
+      case "image":
+        if (element.data && element.data.length <= MAX_IMAGE_CHARS)
+          elements.push({type: "image", x, y, w, h, data: element.data})
+        break
+    }
+  }
+  const frame = {width: canvas.width, height: canvas.height, elements}
+  return JSON.stringify(frame).length > MAX_FRAME_BYTES ? null : frame
+}
+
+export interface AppleWatchCommandDeps {
+  apps: () => readonly ClientApp[]
+  start: (app: ClientApp) => Promise<unknown>
+  stop: (packageName: string) => Promise<unknown>
+  ask: (query: string) => Promise<unknown>
+  type: (stream: string, payload: unknown) => void
+  now: () => number
+}
+
+const defaultDeps: AppleWatchCommandDeps = {
+  apps: () => useAppStatusStore.getState().apps,
+  start: (app) => useAppStatusStore.getState().start(app),
+  stop: (packageName) => useAppStatusStore.getState().stop(packageName),
+  ask: (query) => localMiniappRuntime.invokeActionFromHost(ASSIST_PACKAGE, ASSIST_ACTION, {query}),
+  type: (stream, payload) => localMiniappRuntime.forwardEvent(stream, payload),
+  now: () => Date.now(),
+}
+
+export async function handleAppleWatchCommand(
+  event: AppleWatchCommandEvent,
+  deps: AppleWatchCommandDeps = defaultDeps,
+): Promise<void> {
+  switch (event.command) {
+    case "launch": {
+      const app = launchableApps(deps.apps()).some((entry) => entry.packageName === event.packageName)
+        ? deps.apps().find((entry) => entry.packageName === event.packageName)
+        : undefined
+      if (!app) {
+        console.warn(`AppleWatch: launch of unknown miniapp ${event.packageName}`)
+        return
+      }
+      await deps.start(app)
+      return
+    }
+    case "stop": {
+      const front = deps.apps().find((app) => app.foregrounded && app.running && app.type === "standard")
+      if (front) await deps.stop(front.packageName)
+      return
+    }
+    case "assist": {
+      const query = event.text.trim()
+      if (query) await deps.ask(query)
+      return
+    }
+    case "text":
+      for (const key of keyEventsForTextChange("", event.text, deps.now()))
+        deps.type(MiniappStreamType.KEYBOARD_EVENT, key)
+      return
+  }
+}
+
+let stops: (() => void)[] = []
+let stopMirror: (() => void) | null = null
+
+function currentState(): Record<string, unknown> {
+  const glasses = useGlassesStore.getState()
+  return watchState({
+    glasses: {
+      connected: isGlassesConnected(glasses.connection),
+      model: glasses.deviceModel,
+      battery: glasses.batteryLevel,
+    },
+    apps: useAppStatusStore.getState().apps,
+    assist: localMiniappRuntime.hostActionAvailable(ASSIST_PACKAGE, ASSIST_ACTION),
+    mirror: useSettingsStore.getState().getSetting(SETTINGS.apple_watch_mirror.key) === true,
+  })
+}
+
+function syncMirror(enabled: boolean): void {
+  if (!enabled) {
+    if (stopMirror) {
+      stopMirror()
+      stopMirror = null
+      void BluetoothSdk.setWatchMirror(null).catch(() => {})
+    }
+    return
+  }
+  if (stopMirror) return
+  let lastSentAt = 0
+  let pending: MirrorSnapshot | null = null
+  let timer: ReturnType<typeof BgTimer.setTimeout> | null = null
+  const send = (snapshot: MirrorSnapshot) => {
+    lastSentAt = Date.now()
+    void BluetoothSdk.setWatchMirror(mirrorFrame(snapshot)).catch((err) =>
+      console.warn("AppleWatch: mirror push failed:", err),
+    )
+  }
+  const unsubscribe = mirrorService.subscribe((snapshot) => {
+    const wait = lastSentAt + MIRROR_INTERVAL_MS - Date.now()
+    if (wait <= 0 && !timer) {
+      send(snapshot)
+      return
+    }
+    pending = snapshot
+    timer ??= BgTimer.setTimeout(
+      () => {
+        timer = null
+        if (pending) send(pending)
+        pending = null
+      },
+      Math.max(wait, 0),
+    )
+  })
+  stopMirror = () => {
+    unsubscribe()
+    if (timer) BgTimer.clearTimeout(timer)
+    timer = null
+  }
+}
+
+export function startAppleWatchLink(): void {
+  if (Platform.OS !== "ios" || stops.length > 0) return
+  let lastKey = ""
+  let timer: ReturnType<typeof BgTimer.setTimeout> | null = null
+  const push = () => {
+    timer = null
+    const state = currentState()
+    const key = JSON.stringify(state)
+    if (key === lastKey) return
+    lastKey = key
+    void BluetoothSdk.setWatchState(state).catch((err) => console.warn("AppleWatch: state push failed:", err))
+  }
+  const schedule = () => {
+    timer ??= BgTimer.setTimeout(push, STATE_DEBOUNCE_MS)
+  }
+  const mirrorEnabled = () => useSettingsStore.getState().getSetting(SETTINGS.apple_watch_mirror.key) === true
+  const commands = BluetoothSdk.addListener("apple_watch_command", (event) => {
+    handleAppleWatchCommand(event).catch((err) => console.warn(`AppleWatch: ${event.command} failed:`, err))
+  })
+  stops = [
+    () => commands.remove(),
+    useGlassesStore.subscribe(schedule),
+    useAppStatusStore.subscribe(schedule),
+    useSettingsStore.subscribe(
+      (s) => s.getSetting(SETTINGS.apple_watch_mirror.key),
+      () => {
+        syncMirror(mirrorEnabled())
+        schedule()
+      },
+    ),
+    () => {
+      if (timer) BgTimer.clearTimeout(timer)
+      timer = null
+    },
+  ]
+  syncMirror(mirrorEnabled())
+  schedule()
+}
+
+export function stopAppleWatchLink(): void {
+  stops.splice(0).forEach((stop) => stop())
+  syncMirror(false)
+}

package/src/services/controllers.ts

@@ -26,9 +26,26 @@ const NEURAL_BAND_SURFACE: InputSurface = {
   ],
 }
 
+/** The Apple Watch trackpad's taps and four-way swipes, and the Digital Crown as the dial. */
+const APPLE_WATCH_SURFACE: InputSurface = {
+  type: "swipe2d",
+  events: [
+    "TAP",
+    "DOUBLE_TAP",
+    "PRESS_HOLD",
+    "SWIPE_UP",
+    "SWIPE_DOWN",
+    "SWIPE_LEFT",
+    "SWIPE_RIGHT",
+    "DIAL_UP",
+    "DIAL_DOWN",
+  ],
+}
+
 // The R1 swipes up/down like the temples it stands in for, so it adds nothing.
 const INPUT_SURFACES: Readonly<Record<string, readonly InputSurface[]>> = {
   [ControllerTypes.NEURAL_BAND]: [NEURAL_BAND_SURFACE],
+  [ControllerTypes.APPLE_WATCH]: [APPLE_WATCH_SURFACE],
   [ControllerTypes.R1]: [],
 }
 

package/src/stores/settings.ts

@@ -831,6 +831,15 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: false,
     persist: true,
   },
+  // Apple Watch: draw the glasses' current frame behind the watch trackpad. Frames
+  // cross WatchConnectivity while the watch app is up, so it stays off unless asked for.
+  apple_watch_mirror: {
+    key: "apple_watch_mirror",
+    defaultValue: () => false,
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
   // Maximum rate at which display updates leave the phone for the glasses.
   // The Bluetooth SDK paces every scene frame and legacy layout to this,
   // coalescing faster updates last-wins; clears bypass it. No effect on G2

package/src/types/enums.ts

@@ -18,4 +18,5 @@ export enum DeviceTypes {
 export enum ControllerTypes {
   R1 = "Even Realities R1",
   NEURAL_BAND = "Meta Neural Band",
+  APPLE_WATCH = "Apple Watch",
 }