@glassly/engine 0.3.2-dev.617 → 0.3.3-beta.630

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glassly/engine",
-  "version": "0.3.2-dev.617",
+  "version": "0.3.3-beta.630",
   "description": "Glassly Engine — miniapp registry, WebView bridge, and miniapp store",
   "main": "build/index.js",
   "react-native": "src/index.ts",
@@ -97,17 +97,17 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "@glassly/bluetooth-sdk": "0.3.2-dev.617",
-    "@glassly/cloud-client": "0.3.2-dev.617",
-    "@glassly/cloud-protocol": "0.3.2-dev.617",
-    "@glassly/crust": "0.3.2-dev.617",
-    "@glassly/miniapp": "0.3.2-dev.617",
+    "@glassly/bluetooth-sdk": "0.3.3-beta.630",
+    "@glassly/cloud-client": "0.3.3-beta.630",
+    "@glassly/cloud-protocol": "0.3.3-beta.630",
+    "@glassly/crust": "0.3.3-beta.630",
+    "@glassly/miniapp": "0.3.3-beta.630",
     "buffer": "^6.0.3",
     "events": "^3.3.0",
     "react-native-marked": "8.1.1",
     "semver": "^7.7.4",
     "typesafe-ts": "^1.7.1",
-    "@glassly/llm-providers": "0.3.2-dev.617"
+    "@glassly/llm-providers": "0.3.3-beta.630"
   },
   "devDependencies": {
     "@glassly/llm-providers": "workspace:*",
@@ -132,6 +132,7 @@
     "react-native-reanimated": ">=4.2.1 <5",
     "react-native-worklets": ">=0.7.4 <1",
     "expo-file-system": ">=55.0.0",
+    "expo-image-manipulator": ">=55.0.0",
     "react-native-zip-archive": "7.1.1",
     "zustand": "*",
     "expo-battery": ">=55.0.0",

package/src/facades/glasses.ts

@@ -95,6 +95,7 @@ function projectInfo() {
     rightMac: s.rightMacAddress,
     sceneCanvasWidth: s.sceneCanvasWidth,
     sceneCanvasHeight: s.sceneCanvasHeight,
+    volumeGate: s.volumeGate === true,
     // Copy: the snapshot must not hand callers a mutable reference into the store.
     wifi: {...s.wifi},
   }

package/src/generated/releaseMetadata.ts

@@ -11,10 +11,10 @@ export interface EngineReleaseMetadata {
 
 export const ENGINE_RELEASE_METADATA: Readonly<EngineReleaseMetadata> = Object.freeze({
   "schemaVersion": 1,
-  "familyBaseVersion": "0.3.2",
-  "releaseIdentity": "0.3.2-dev.617",
-  "releaseSetId": "glassly-0.3.2-dev.617",
-  "sourceCommit": "28231d2694923368e949dded3f7e8c3e015af932",
+  "familyBaseVersion": "0.3.3",
+  "releaseIdentity": "0.3.3-beta.630",
+  "releaseSetId": "glassly-0.3.3-beta.630",
+  "sourceCommit": "5d41a8af4bb8c4353bb00670eb4913a05bd9e3c9",
   "otaManifestUrl": null,
   "otaManifestSha256": null
 })

package/src/index.ts

@@ -45,6 +45,7 @@ export type {
 // imperative access goes through engine.settings.
 export {SETTINGS, useSetting, AI_DATA_CONSENT_VERSION} from "./stores/settings"
 export {isCfwFirmwareVersion, isCustomDashboardEligible} from "./services/pages/pageEligibility"
+export {getEffectiveCapabilities} from "./services/effectiveCapabilities"
 
 // AI tier registry: the curated per-tier model dropdown options and helpers
 // the Settings > AI Models screen renders. Resolution itself runs inside

package/src/services/GestureDotsService.ts

@@ -3,7 +3,9 @@
  * glasses when a touch gesture arrives, while nothing else is on the display:
  * a filled circle for a single tap, two for a double tap, an arrow for a swipe
  * (cleared after a short pulse), a bullseye for a recognized long press.
- * The Neural Band's middle finger (index 1) draws the same shapes hollow. Extracted from
+ * Controllers that report press edges (touch_down … tap_end) show the press
+ * itself instead: a dot while pressed, the bullseye once held, cleared at
+ * release. The Neural Band's middle finger (index 1) draws the same shapes hollow. Extracted from
  * the CFW dashboard service; it renders through
  * the ordinary scene pipeline, so it works on any positioning device (stock
  * G2 included), and is gated by the gesture_dots debug setting.
@@ -24,6 +26,8 @@ import sceneRenderer from "./SceneRenderer"
 const DOTS_APP_ID = "system.gesture-dots"
 /** How long a gesture stays on the indicator. */
 const GESTURE_FLASH_MS = 700
+/** A press-edge device's tap and hold reports trail its last edge by its double-tap window. */
+const EDGE_REPORT_GRACE_MS = 1000
 const MARGIN_X = 16
 const MARGIN_TOP = 10
 /** Gestures that flash a glyph. */
@@ -45,6 +49,8 @@ class GestureDotsService {
   private flashTimer: number | null = null
   /** Raw controller holds survive hold recognition/dial events until their own release. */
   private presses = new Map<number, {raw: boolean; glyph: GestureDotGlyph}>()
+  /** When each finger last sent a press edge (touch_down, or tap_end after one). */
+  private lastEdgeAt = new Map<number, number>()
   /** Avoid sending identical images for successive phases of the same press. */
   private renderedGlyph: GestureDotGlyph | null = null
   /** Whether the indicator scene currently has anything on it (needs a clear when the flash ends). */
@@ -76,6 +82,7 @@ class GestureDotsService {
     for (const unsub of this.unsubs) unsub()
     this.unsubs = []
     this.relinquish()
+    this.lastEdgeAt.clear()
     this.started = false
   }
 
@@ -124,9 +131,12 @@ class GestureDotsService {
     const finger = fingerIndex ?? 0
     const press = this.presses.get(finger)
     if (gestureName === "touch_down" || gestureName === "tap_start") {
+      if (gestureName === "touch_down") this.lastEdgeAt.set(finger, Date.now())
       if (press) {
-        // Both reports describe one press. Never restart its visual feedback.
+        // Both reports describe one press: after touch_down, tap_start means it is held.
         if (gestureName === "touch_down") press.raw = true
+        else if (press.raw) press.glyph = "hold"
+        this.render()
         return
       }
       this.clearFlash()
@@ -137,21 +147,24 @@ class GestureDotsService {
     if (gestureName === "tap_end") {
       if (!press) return
       this.presses.delete(finger)
-      // The band's resolved tap arrives after release (double-tap window).
-      // Keep the glyph lit across that gap so one pinch never flashes twice.
-      if (press.raw && this.presses.size === 0) this.startFlash(press.glyph)
+      if (press.raw) this.lastEdgeAt.set(finger, Date.now())
       this.render()
       return
     }
     const shape = FLASH_GLYPHS[gestureName]
     const glyph = shape && finger === 1 ? MIDDLE_FINGER_GLYPHS[shape] ?? shape : shape
-    if (press?.raw && (!glyph || gestureName === "long_press")) {
+    // The press already showed its own tap and hold; the device's report of
+    // them follows after its double-tap window, often after release.
+    const shownByPress = gestureName === "single_tap" || gestureName === "long_press"
+    if (press?.raw && (!glyph || shownByPress)) {
       // Pinch-and-turn reports do not release a finger. A recognized hold
       // stays visible until tap_end instead of expiring halfway through it.
-      if (glyph) press.glyph = glyph
+      if (gestureName === "long_press") press.glyph = "hold"
       this.render()
       return
     }
+    const lastEdgeAt = this.lastEdgeAt.get(finger)
+    if (!press && shownByPress && lastEdgeAt !== undefined && Date.now() - lastEdgeAt < EDGE_REPORT_GRACE_MS) return
     // Completed taps/swipes become one continuous pulse, even if the device
     // omits tap_end. Older tap_start-only devices retain the same fallback.
     this.presses.delete(finger)

package/src/services/GlassesTouchInput.ts

@@ -7,6 +7,7 @@ import {useGlassesStore} from "../stores/glasses"
 import {BgTimer} from "../utils/timers"
 import dashboardOpenTrace from "./debug/DashboardOpenTrace"
 import gestureDotsService from "./GestureDotsService"
+import {isGlassesConnected} from "./GlassesReadiness"
 import localDisplayManager from "./LocalDisplayManager"
 import localMiniappRuntime from "./LocalMiniappRuntime"
 import {MiniappExitGesture} from "./MiniappExitGesture"
@@ -92,8 +93,9 @@ export function dispatchTouchEvent(event: TouchEvent): void {
   // Nobody claimed a double tap: the glasses' own dashboard. On eligible
   // Glassly CFW glasses the dashboard miniapp claims it while it runs
   // (ADR-0006); stock firmware keeps its native dashboard (ADR-0002). The
-  // modal nav UI owns the screen; showDashboard would fight it.
-  if (osDoubleTap && !nativeNavigationService.isActive) {
+  // modal nav UI owns the screen; showDashboard would fight it. Without
+  // glasses there is no dashboard to open, and the mirror keeps its frame.
+  if (osDoubleTap && !nativeNavigationService.isActive && isGlassesConnected(useGlassesStore.getState().connection)) {
     useDisplayStore.getState().setDisplayEvent({layout: {layoutType: "unavailable", reason: "firmware-dashboard"}})
     void Promise.resolve(BluetoothSdk.showDashboard()).catch((err) => {
       console.error("DeviceEventRouter: showDashboard failed:", err)

package/src/services/LocalMiniappRuntime.ts

@@ -72,6 +72,7 @@ import overlayPages from "./pages/OverlayPages"
 import dashboardOpenTrace from "./debug/DashboardOpenTrace"
 import {isAppForegrounded, isPhoneActive, onAppForegroundChange, onPhoneActivityChange} from "./PhoneActivity"
 import {installedNotificationApps} from "./PhoneNotificationsSync"
+import {MiniappIcons} from "./MiniappIcons"
 import {
   DERIVED_SETTINGS_KEYS,
   SETTINGS_BRIDGE_KEYS,
@@ -1719,7 +1720,7 @@ class LocalMiniappRuntime {
       // Inter-miniapp interop (lifecycle is SYSTEM-only; actions are open,
       // gated per-action by the target's declared visibility)
       case MiniappRequestType.MINIAPPS_LIST:
-        this.handleMiniappsList(packageName, payload, requestId)
+        void this.handleMiniappsList(packageName, payload, requestId)
         break
       case MiniappRequestType.MINIAPPS_START:
         void this.handleMiniappsStart(packageName, payload, requestId)
@@ -5330,12 +5331,14 @@ class LocalMiniappRuntime {
     if (this.capabilitiesWired) return
     this.capabilitiesWired = true
     const broadcast = () => this.broadcastCapabilities()
-    // Connection, model and firmware also decide the dashboard view's eligibility.
+    // Connection, model and firmware also decide the dashboard view's eligibility;
+    // a connected Neural Band adds its swipe2d surface to `button`.
     useGlassesStore.subscribe(
       (s) =>
-        `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}`,
+        `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}:${s.controllerConnected}`,
       broadcast,
     )
+    useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.default_controller.key), broadcast)
     // The debug knob flips hasNativeNavigation on custom firmware.
     useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.cfw_native_navigation.key), broadcast)
     // The dashboard master switch flips the dashboard view.
@@ -5902,18 +5905,28 @@ class LocalMiniappRuntime {
     }
   }
 
-  private handleMiniappsList(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
+  private miniappIcons = new MiniappIcons()
+
+  private async handleMiniappsList(
+    packageName: string,
+    payload: Record<string, unknown>,
+    requestId?: string,
+  ): Promise<void> {
     if (!this.requireMiniappsPermission(packageName, requestId, "list")) return
     const includeIncompatible = payload.includeIncompatible === true
-    const infos = this.interopApps()
-      .filter(
-        (a) =>
-          a.packageName &&
-          a.packageName !== packageName &&
-          !a.packageName.includes("@empty") &&
-          (includeIncompatible || a.compatibility?.isCompatible !== false),
-      )
-      .map((a) => this.buildMiniappInfo(a))
+    const apps = this.interopApps().filter(
+      (a) =>
+        a.packageName &&
+        a.packageName !== packageName &&
+        !a.packageName.includes("@empty") &&
+        (includeIncompatible || a.compatibility?.isCompatible !== false),
+    )
+    const infos = await Promise.all(
+      apps.map(async (app) => ({
+        ...this.buildMiniappInfo(app),
+        ...(payload.includeIcons === true ? {icon: await this.miniappIcons.read(app)} : {}),
+      })),
+    )
     this.sendResult(packageName, requestId, true, infos)
     this.auditInterop({caller: packageName, op: "list", ok: true})
   }

package/src/services/MiniappIcons.ts

@@ -0,0 +1,50 @@
+import {File} from "expo-file-system"
+import {ImageManipulator, SaveFormat} from "expo-image-manipulator"
+
+type IconApp = {packageName: string; version?: string; logoUrl: string}
+
+/** Small thumbnails keep the inter-miniapp bridge and every grid frame lightweight. */
+async function thumbnail(uri: string): Promise<string | undefined> {
+  const context = ImageManipulator.manipulate(uri)
+  let image: Awaited<ReturnType<typeof context.renderAsync>> | undefined
+  let savedUri: string | undefined
+  try {
+    context.resize({width: 32, height: 32})
+    image = await context.renderAsync()
+    const saved = await image.saveAsync({format: SaveFormat.PNG, base64: true})
+    savedUri = saved.uri
+    return saved.base64
+  } finally {
+    image?.release()
+    context.release()
+    if (savedUri) {
+      try {
+        new File(savedUri).delete()
+      } catch {
+        /* Already removed by cache cleanup. */
+      }
+    }
+  }
+}
+
+/** Cache by installed version and icon URL; failed icons never prevent listing apps. */
+export class MiniappIcons {
+  private cache = new Map<string, {key: string; value: Promise<string | undefined>}>()
+
+  constructor(private readonly load = thumbnail) {}
+
+  read(app: IconApp): Promise<string | undefined> {
+    if (!app.logoUrl) return Promise.resolve(undefined)
+    const key = JSON.stringify([app.version, app.logoUrl])
+    const cached = this.cache.get(app.packageName)
+    if (cached?.key === key) return cached.value
+    const value = this.load(app.logoUrl).catch(() => {
+      if (this.cache.get(app.packageName)?.value === value) this.cache.delete(app.packageName)
+      return undefined
+    })
+    this.cache.set(app.packageName, {key, value})
+    // Bound memory even if apps are repeatedly installed and removed.
+    if (this.cache.size > 64) this.cache.delete(this.cache.keys().next().value!)
+    return value
+  }
+}

package/src/services/effectiveCapabilities.ts

@@ -11,11 +11,22 @@
  * lifted alongside the canvas, bounded only to keep frames sane. Firmware
  * that draws vector shapes keeps 128 scene slots, so shape-heavy frames
  * (a game board, a chart) get a larger shared text/shape budget.
+ *
+ * A connected controller that swipes in two dimensions (the Meta Neural Band)
+ * adds its surface to `button.buttons` as a `swipe2d` entry, so a miniapp can
+ * tell four-way swipes from the temples' and the R1 ring's up/down.
  */
 import {useGlassesStore} from "../stores/glasses"
 import {SETTINGS, useSettingsStore} from "../stores/settings"
-import {ALL_DISPLAY_SHAPES, DeviceTypes, getModelCapabilities, type Capabilities} from "../types"
+import {
+  ALL_DISPLAY_SHAPES,
+  DeviceTypes,
+  getModelCapabilities,
+  type ButtonCapabilities,
+  type Capabilities,
+} from "../types"
 import {resolveNativeNavigation} from "./nativeNavigationGate"
+import {input2dFrom} from "./settingsBridge"
 
 const CFW_MAX_TEXT_ELEMENTS = 24
 const CFW_MAX_IMAGE_ELEMENTS = 16
@@ -36,17 +47,28 @@ function forceNativeNavigationOnCfw(): boolean {
   return useSettingsStore.getState().getSetting(SETTINGS.cfw_native_navigation.key) === true
 }
 
+/** The connected firmware gates microphone streaming on volume (G2 Glassly CFW revision 40+). */
+export function runtimeVolumeGate(): boolean {
+  return useGlassesStore.getState().volumeGate === true
+}
+
 export function getEffectiveCapabilities(model: DeviceTypes): Capabilities {
+  return withControllerInput(glassesCapabilities(model))
+}
+
+function glassesCapabilities(model: DeviceTypes): Capabilities {
   const base = getModelCapabilities(model)
-  // Only models with a Glassly CFW publish a canvas; never let a stale value
-  // reshape another model.
+  // Only models with a Glassly CFW publish a canvas or a volume gate; never
+  // let a stale value reshape another model.
   const canvas = base.hasCustomFirmware ? runtimeSceneCanvas() : null
-  if (!base.display || !canvas) return base
+  const volumeGate = base.hasCustomFirmware && base.hasMicrophone && runtimeVolumeGate()
+  if (!canvas) return volumeGate ? withVolumeGate(base) : base
+  if (!base.display) return volumeGate ? withVolumeGate(base) : base
   // Glassly CFW supports the complete vector contract, including SVG and rotation.
   // Upstream g2flash publishes a raster canvas with sceneVector=false.
   const vector = useGlassesStore.getState().sceneVector === true
   return {
-    ...base,
+    ...(volumeGate ? withVolumeGate(base) : base),
     hasNativeNavigation: resolveNativeNavigation(base.hasNativeNavigation, true, forceNativeNavigationOnCfw()),
     display: {
       ...base.display,
@@ -67,3 +89,40 @@ export function getEffectiveCapabilities(model: DeviceTypes): Capabilities {
     },
   }
 }
+
+type InputSurface = NonNullable<ButtonCapabilities["buttons"]>[number]
+
+/** The Neural Band's thumb swipes, pinches and dial. */
+const NEURAL_BAND_SURFACE: InputSurface = {
+  type: "swipe2d",
+  events: [
+    "TAP",
+    "DOUBLE_TAP",
+    "PRESS_HOLD",
+    "SWIPE_UP",
+    "SWIPE_DOWN",
+    "SWIPE_LEFT",
+    "SWIPE_RIGHT",
+    "DIAL_UP",
+    "DIAL_DOWN",
+  ],
+}
+
+function withControllerInput(caps: Capabilities): Capabilities {
+  const controller = useSettingsStore.getState().getSetting(SETTINGS.default_controller.key)
+  if (!input2dFrom(useGlassesStore.getState(), controller)) return caps
+  const buttons = caps.button?.buttons ?? []
+  return {
+    ...caps,
+    hasButton: true,
+    button: {
+      ...caps.button,
+      count: (caps.button?.count ?? buttons.length) + 1,
+      buttons: [...buttons, NEURAL_BAND_SURFACE],
+    },
+  }
+}
+
+function withVolumeGate(base: Capabilities): Capabilities {
+  return {...base, microphone: {...(base.microphone ?? {}), volumeGate: true}}
+}

package/src/stores/glasses.ts

@@ -145,6 +145,7 @@ const initialState: GlassesStore = {
   sceneCanvasHeight: 0,
   sceneVector: false,
   sceneSettle: false,
+  volumeGate: false,
   // wifi info
   wifi: {state: "disconnected"},
   wifiStatusKnown: false,

package/src/stores/settings.ts

@@ -611,6 +611,40 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: true,
     persist: true,
   },
+  // G2 Glassly CFW microphone volume gate (revision 40+): the temples stop
+  // streaming audio while nothing reaches the threshold. Off until the user
+  // turns it on; shown only when the connected firmware reports the capability.
+  volume_gate_enabled: {
+    key: "volume_gate_enabled",
+    defaultValue: () => false,
+    writable: true,
+    saveOnServer: true,
+    persist: true,
+  },
+  // dBFS RMS a 50 ms chunk must reach to count as sound.
+  volume_gate_threshold_db: {
+    key: "volume_gate_threshold_db",
+    defaultValue: () => -45,
+    writable: true,
+    saveOnServer: true,
+    persist: true,
+  },
+  // Quiet time before the gate closes.
+  volume_gate_hold_ms: {
+    key: "volume_gate_hold_ms",
+    defaultValue: () => 3000,
+    writable: true,
+    saveOnServer: true,
+    persist: true,
+  },
+  // Minimum time the gate stays open once sound opens it.
+  volume_gate_min_open_ms: {
+    key: "volume_gate_min_open_ms",
+    defaultValue: () => 2000,
+    writable: true,
+    saveOnServer: true,
+    persist: true,
+  },
   always_on_status_bar: {
     key: "always_on_status_bar",
     defaultValue: () => false,
@@ -1108,6 +1142,10 @@ export const BLUETOOTH_SETTING_KEYS: string[] = [
   SETTINGS.power_saving_mode.key,
   SETTINGS.voice_activity_detection_enabled.key,
   SETTINGS.loudness_gate_enabled.key,
+  SETTINGS.volume_gate_enabled.key,
+  SETTINGS.volume_gate_threshold_db.key,
+  SETTINGS.volume_gate_hold_ms.key,
+  SETTINGS.volume_gate_min_open_ms.key,
   SETTINGS.lc3_frame_size.key,
   SETTINGS.preferred_mic.key,
   SETTINGS.screen_disabled.key,

package/src/types/hardware.ts

@@ -95,6 +95,10 @@ export const ALL_DISPLAY_SHAPES: DisplayShape[] = ["rect", "circle", "arc", "pie
 export interface MicrophoneCapabilities {
   count?: number
   hasVAD?: boolean // Voice Activity Detection
+  // The connected firmware can stop streaming audio below a volume floor
+  // (G2 Glassly CFW revision 40+). Never set on a model profile: it is
+  // reported at runtime and layered on by getEffectiveCapabilities.
+  volumeGate?: boolean
 }
 
 /**