@glassly/miniapp 0.3.0-dev.1

package/src/modules/camera.ts

@@ -0,0 +1,258 @@
+/**
+ * @fileoverview CameraModule — glasses camera control and photo capture.
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type CameraRoiPosition = "center" | "bottom" | "top"
+export type CameraFovPreset = "narrow" | "standard" | "wide"
+
+export type CameraFovRequest =
+  | {
+      /** Horizontal FOV, degrees. */
+      fov: number
+      /** Crop/region position: center, bottom, or top. Defaults to center. */
+      roiPosition?: CameraRoiPosition
+    }
+  | {
+      /** Named FOV preset. Presets use center ROI. */
+      preset: CameraFovPreset
+    }
+
+export type SetCameraFovOptions = CameraFovRequest
+
+export interface CameraFovResult {
+  requestId: string
+  fov: number
+  roiPosition: CameraRoiPosition
+  timestamp: number
+}
+
+export interface TakePhotoOptions {
+  size?: "low" | "medium" | "high" | "max"
+  /** Capture at maximum source quality and optimize BLE delivery for readable text. */
+  mode?: "photo" | "text"
+  /**
+   * Image delivery path. `auto` tries direct Wi-Fi upload and falls back to
+   * BLE; `direct` disables BLE fallback; `ble` always relays through the phone.
+   */
+  transferMethod?: "auto" | "direct" | "ble"
+  compress?: "none" | "low" | "medium" | "high"
+  sound?: boolean
+  saveToGallery?: boolean
+  /**
+   * Manual shutter / exposure time in nanoseconds. Omit (or pass undefined)
+   * to let the glasses auto-expose. Honored only on cameras that support
+   * manual exposure; ignored otherwise.
+   */
+  exposureTimeNs?: number
+  /** Sensor ISO for this capture only. Only used when `exposureTimeNs` enables manual exposure. */
+  iso?: number | null
+  /** After AE convergence, divide the metered exposure by this factor (scan mode). */
+  aeExposureDivisor?: number
+  /** Cap ISO after AE metering (scan mode). */
+  isoCap?: number
+  /** Request per-capture noise reduction. Sent on the wire; glasses may log `not_implemented`. */
+  noiseReduction?: boolean
+  /** Request per-capture edge enhancement. Sent on the wire; glasses may log `not_implemented`. */
+  edgeEnhancement?: boolean
+  /**
+   * ZSL buffering. Manual and scan stills force it off because fixed sensor controls take priority.
+   */
+  zsl?: boolean
+  /**
+   * MFNR still capture. Manual and scan stills force it off because fixed sensor controls take priority.
+   */
+  mfnr?: boolean
+  /** ISP digital gain hint. */
+  ispDigitalGain?: number
+  /** ISP analog gain hint. */
+  ispAnalogGain?: string
+  /**
+   * Override the SDK's default 60s request timeout for this capture only.
+   * Leave unset for a normal user-triggered capture (the full 60s ceiling is
+   * appropriate there). Callers that speculatively fire a capture and may
+   * abandon it if the result isn't needed (e.g. a short-lived non-visual
+   * assistant turn racing a capture against a ~1s latency cap) should pass a
+   * short value here so the abandoned `takePhoto()` promise settles quickly
+   * instead of sitting on the default 60s ceiling.
+   */
+  timeoutMs?: number
+}
+
+export interface PhotoTaken {
+  /** Request id that correlates ASG status, upload, and phone/cloud logs. */
+  requestId: string
+  photoUrl: string
+  mimeType: string
+  size: number
+}
+
+export interface WarmUpCameraOptions {
+  size?: "low" | "medium" | "high" | "max"
+  /** Match the upcoming capture mode so warm-up and capture share one ASG camera session. */
+  mode?: "photo" | "text"
+  exposureTimeNs?: number
+  /** Ready-state hold in milliseconds. Defaults to 15 seconds and is capped at 60 seconds. */
+  durationMs?: number
+  /** ZSL preview buffering for the warm-up session. */
+  zsl?: boolean
+  /** MFNR still capture for the warm-up session. */
+  mfnr?: boolean
+}
+
+export interface StartVideoRecordingOptions {
+  /** Video width in pixels. Omit to use the device's saved button-video default. */
+  width?: number
+  /** Video height in pixels. Omit to use the device's saved button-video default. */
+  height?: number
+  /**
+   * Frames per second. Lower frame rates keep the glasses cooler and produce
+   * smaller files — useful for long recordings where smooth motion isn't needed.
+   * Omit to use the device default.
+   */
+  fps?: number
+  /** Play start/stop capture sounds. Defaults to true. */
+  sound?: boolean
+  /** Keep the recording on the glasses after it finishes. Defaults to false. */
+  save?: boolean
+}
+
+export interface VideoRecordingStarted {
+  /** Identifier for this recording session; pass to {@link stopVideoRecording}. */
+  recordingId: string
+}
+
+export interface StopVideoRecordingOptions {
+  /**
+   * Upload the finished recording to this URL with a multipart POST (the file is
+   * the `video` part). The glasses upload it directly. Omit to keep the recording
+   * on the glasses instead.
+   */
+  uploadUrl?: string
+  /** Bearer token sent as the `Authorization` header on the upload, if your endpoint needs one. */
+  uploadAuthToken?: string
+}
+
+export class CameraModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  /** True iff `CAMERA` is declared in the miniapp's manifest. */
+  get hasPermission(): boolean {
+    return this.session._hasManifestPermission("CAMERA")
+  }
+
+  /**
+   * Apply a session-owned camera FOV/ROI override on the glasses.
+   *
+   * Resolves after the ASG client reports that the setting was applied to camera
+   * hardware after the restart cooldown. The host restores the prior override or persistent base
+   * when this miniapp closes. Requires CAMERA permission declared in miniapp.json.
+   */
+  async setFov(request: CameraFovRequest): Promise<CameraFovResult> {
+    return this.session.sendRequest<CameraFovResult>({
+      type: MiniappRequestType.CAMERA_FOV,
+      ...request,
+    })
+  }
+
+  /**
+   * Take a photo via the glasses camera. Returns a URL to the captured image.
+   * Requires CAMERA permission declared in miniapp.json.
+   *
+   * The photo is uploaded to cloud storage; the returned URL is a short-TTL
+   * (~30 minute) signed download URL. If the glasses don't have a camera,
+   * the phone-side handler rejects with an error. Check
+   * `session.capabilities.hasCamera` before calling.
+   */
+  async takePhoto(options: TakePhotoOptions = {}): Promise<PhotoTaken> {
+    return this.session.sendRequest<PhotoTaken>(
+      {
+        type: MiniappRequestType.PHOTO,
+        size: options.size ?? "medium",
+        mode: options.mode ?? "photo",
+        ...(options.transferMethod !== undefined ? {transferMethod: options.transferMethod} : {}),
+        compress: options.compress ?? "none",
+        sound: options.sound ?? true,
+        saveToGallery: options.saveToGallery ?? false,
+        exposureTimeNs: options.exposureTimeNs,
+        iso: options.iso,
+        aeExposureDivisor: options.aeExposureDivisor,
+        isoCap: options.isoCap,
+        noiseReduction: options.noiseReduction,
+        edgeEnhancement: options.edgeEnhancement,
+        zsl: options.zsl,
+        mfnr: options.mfnr,
+        ispDigitalGain: options.ispDigitalGain,
+        ispAnalogGain: options.ispAnalogGain,
+      },
+      options.timeoutMs != null ? {timeoutMs: options.timeoutMs} : undefined,
+    )
+  }
+
+  /**
+   * Pre-warm the glasses camera so the next takePhoto() is near-instant.
+   * The camera stays warm for ~durationMs (default 15s, maximum 60s); call warmUp() again to
+   * extend it. Warm with the same `size` you'll capture with — a mismatched size
+   * forces the camera to reconfigure and loses the speedup. Requires CAMERA
+   * permission in miniapp.json. Resolves once the camera reports ready. The host automatically
+   * releases this miniapp's request-owned lease when the miniapp closes.
+   *
+   * Warm-ups are serialized: only one runs at a time and none may start while a
+   * photo is being captured. If the camera is busy (a capture is in flight, or
+   * another warm-up is still opening) this rejects with `camera_busy` — retry
+   * shortly, or just call takePhoto() (it works regardless, only slower). The
+   * normal warmUp() → takePhoto() sequence never hits this.
+   */
+  async warmUp(options: WarmUpCameraOptions = {}): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.CAMERA_WARM_UP,
+      size: options.size ?? "medium",
+      mode: options.mode ?? "photo",
+      exposureTimeNs: options.exposureTimeNs,
+      durationMs: options.durationMs ?? 15000,
+      zsl: options.zsl,
+      mfnr: options.mfnr,
+    })
+  }
+
+  /**
+   * Start recording video on the glasses camera. Returns a `recordingId` to pass
+   * to {@link stopVideoRecording} after the glasses report that recording
+   * started. Requires CAMERA permission declared in miniapp.json. Check
+   * `session.capabilities.hasCamera` before calling.
+   *
+   * Resolution and frame rate are optional — omit them to use the device's saved
+   * button-video settings. Lowering `fps` (e.g. to 5) keeps the glasses cooler
+   * during long recordings.
+   */
+  async startVideoRecording(options: StartVideoRecordingOptions = {}): Promise<VideoRecordingStarted> {
+    return this.session.sendRequest<VideoRecordingStarted>({
+      type: MiniappRequestType.VIDEO_RECORDING_START,
+      width: options.width,
+      height: options.height,
+      fps: options.fps,
+      sound: options.sound ?? true,
+      save: options.save ?? false,
+    })
+  }
+
+  /**
+   * Stop an in-progress video recording started with {@link startVideoRecording}.
+   *
+   * By default the recording stays on the glasses. Pass `uploadUrl` to have the
+   * glasses upload the finished clip to your own endpoint.
+   *
+   * @param recordingId The id returned from `startVideoRecording`.
+   * @param options     Optional upload destination for the finished recording.
+   */
+  async stopVideoRecording(recordingId: string, options: StopVideoRecordingOptions = {}): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.VIDEO_RECORDING_STOP,
+      recordingId,
+      uploadUrl: options.uploadUrl,
+      uploadAuthToken: options.uploadAuthToken,
+    })
+  }
+}

package/src/modules/cloud.ts

@@ -0,0 +1,94 @@
+/**
+ * @fileoverview CloudModule — local miniapp view of the phone-owned cloud client.
+ *
+ * Miniapps do not own or configure the cloud connection. The phone owns one
+ * cloud-client instance and pushes status snapshots here so miniapps can render
+ * honest UX such as online/offline captions indicators.
+ */
+
+import {EventEmitter} from "eventemitter3"
+
+import {MiniappSession} from "../session"
+import type {UnsubscribeFn} from "./events"
+
+export type CloudClientConnectionStatus = "connected" | "connecting" | "reconnecting" | "disconnected"
+export type CloudClientAudioTransport = "udp" | "ws" | "offline" | "none"
+
+export interface CloudClientStatus {
+  status: CloudClientConnectionStatus
+  audioTransport: CloudClientAudioTransport
+}
+
+type CloudEmitterEvents = {
+  status: (status: CloudClientStatus) => void
+}
+
+export const CLOUD_STATUS_STREAM = "_cloud_status"
+
+const DEFAULT_STATUS: CloudClientStatus = {
+  status: "disconnected",
+  audioTransport: "none",
+}
+
+function normalizeCloudStatus(data: unknown): CloudClientStatus | null {
+  if (!data || typeof data !== "object") return null
+  const raw = data as Record<string, unknown>
+  const status = raw.status
+  const audioTransport = raw.audioTransport
+
+  if (status !== "connected" && status !== "connecting" && status !== "reconnecting" && status !== "disconnected") {
+    return null
+  }
+  if (
+    audioTransport !== "udp" &&
+    audioTransport !== "ws" &&
+    audioTransport !== "offline" &&
+    audioTransport !== "none"
+  ) {
+    return null
+  }
+
+  return {status, audioTransport}
+}
+
+export class CloudModule {
+  private readonly emitter = new EventEmitter<CloudEmitterEvents>()
+  private current: CloudClientStatus = {...DEFAULT_STATUS}
+
+  constructor(private readonly session: MiniappSession) {
+    this.session._subscribe(CLOUD_STATUS_STREAM, (data) => {
+      const next = normalizeCloudStatus(data)
+      if (!next) return
+      if (next.status === this.current.status && next.audioTransport === this.current.audioTransport) return
+      this.current = next
+      this.emitter.emit("status", {...this.current})
+    })
+  }
+
+  /** Latest status snapshot pushed by the phone runtime. */
+  get status(): CloudClientStatus {
+    return {...this.current}
+  }
+
+  /** True when the phone's cloud-client runtime handshake is complete. */
+  get connected(): boolean {
+    return this.current.status === "connected"
+  }
+
+  /** Method alias for callers who prefer function-style state reads. */
+  isConnected(): boolean {
+    return this.connected
+  }
+
+  /**
+   * Subscribe to cloud-client status updates. The current value is delivered
+   * immediately so UI code can hydrate without waiting for the next transition.
+   */
+  onStatusChanged(handler: (status: CloudClientStatus) => void): UnsubscribeFn {
+    handler({...this.current})
+    this.emitter.on("status", handler)
+    return () => {
+      this.emitter.off("status", handler)
+    }
+  }
+}

package/src/modules/dashboard.ts

@@ -0,0 +1,51 @@
+/**
+ * @fileoverview DashboardAPI. setContent() is a noop in v1 (the cloud
+ * DashboardManager owns widget rendering); showNativeDashboard() is live.
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type DashboardMode = "main" | "expanded" | "always_on"
+
+/** "shown" = forwarded to the glasses; "unsupported" = the host refused (e.g. native nav mode owns the screen). */
+export interface ShowNativeDashboardResult {
+  status: "shown" | "unsupported"
+  reason?: string
+}
+
+export class DashboardAPI {
+  private warned = false
+
+  constructor(private readonly session: MiniappSession) {}
+
+  setContent(mode: DashboardMode, content: string): void {
+    if (!this.warned) {
+      console.warn("[@glassly/miniapp] dashboard.setContent() is deferred in v1.")
+      this.warned = true
+    }
+    // Still forward so the phone can log/ignore consistently.
+    this.session.sendOneShot({
+      type: MiniappRequestType.DASHBOARD_CONTENT_UPDATE,
+      mode,
+      content,
+    })
+  }
+
+  /**
+   * Bring the glasses' native dashboard (the OS-level idle screen) to the
+   * foreground. On the G2 this tears down the current page; the display
+   * returns when the wearer dismisses the dashboard or content is re-rendered.
+   */
+  showNativeDashboard(): Promise<ShowNativeDashboardResult> {
+    return this.session
+      .sendRequest<ShowNativeDashboardResult>({type: MiniappRequestType.SHOW_NATIVE_DASHBOARD})
+      .catch((err) => ({
+        status: "unsupported" as const,
+        reason:
+          typeof err === "object" && err !== null && "message" in err
+            ? String((err as {message: unknown}).message)
+            : String(err),
+      }))
+  }
+}

package/src/modules/display.ts

@@ -0,0 +1,207 @@
+/**
+ * @fileoverview DisplayManager — the glasses display, via `render()`.
+ *
+ * render(elements) is replace-the-frame scene rendering: each call describes
+ * everything that should be on screen, the host diffs it against the previous
+ * frame per device, and `render([])` clears. The legacy one-shot `show*`
+ * layout methods were removed once nothing first-party or known-external
+ * called them; hosts still accept the historical `miniapp_display` wire shape
+ * from bundles packed with older SDKs.
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type ViewType = "main" | "dashboard"
+
+export type DisplayBreakMode = "character" | "character-no-hyphen" | "word" | "strict-word"
+
+// ============================================================================
+// render() — the scene API
+// ============================================================================
+
+/** Canvas corner a box's `x`/`y` are measured from (insets grow inward). */
+export type RenderAnchor = "top-left" | "top-right" | "bottom-left" | "bottom-right"
+
+/**
+ * Pixel-space bounding box on the device's drawable canvas (see
+ * `capabilities.display`).
+ *
+ * With an `anchor`, `x`/`y` become insets from that corner, growing inward,
+ * and the host resolves the box against the connected device's real canvas —
+ * e.g. `{x: 0, y: 0, w: 120, h: 120, anchor: "bottom-right"}` sits flush in
+ * the bottom-right corner on every device, whatever its resolution. Corners
+ * only; there is no center/edge anchoring or scaling. Hosts older than the
+ * anchor feature ignore the field and place the box from the top-left.
+ */
+export interface RenderBox {
+  x: number
+  y: number
+  w: number
+  h: number
+  /** Which canvas corner x/y are measured from. Default "top-left". */
+  anchor?: RenderAnchor
+}
+
+export interface RenderTextStyle {
+  /** Border width in px (0/absent = none). */
+  border?: number
+  /** Border corner radius in px. */
+  radius?: number
+  /** What happens to text that doesn't fit the box after wrapping. Default "clip". */
+  overflow?: "clip" | "ellipsis"
+  /** Line-break policy for wrapping (host wraps; the box carries pre-wrapped text). */
+  breakMode?: DisplayBreakMode
+  /**
+   * "mono": fixed-pitch text drawn at `fontSize` px with a fixed line height
+   * of 1.2 × size, and NOT re-wrapped by the host — lines are taken as given.
+   * For grids (terminals, tables). Devices without a phone-side rasterizer
+   * (stock G2 firmware) fall back to their own proportional font.
+   */
+  font?: "default" | "mono"
+  /** Font size in px for `font: "mono"`. Default 20. */
+  fontSize?: number
+}
+
+export interface RenderRectStyle {
+  border?: number
+  radius?: number
+}
+
+/**
+ * One element of a rendered scene. `id` is optional but recommended for
+ * anything that updates over time: elements with a stable id update in place on
+ * the glasses (no flicker); the host matches unnamed elements by geometry.
+ */
+export type RenderElement =
+  | {type: "text"; id?: string; box: RenderBox; text: string; style?: RenderTextStyle}
+  | {type: "image"; id?: string; box: RenderBox; data: string}
+  | {type: "rect"; id?: string; box: RenderBox; style?: RenderRectStyle}
+
+export interface RenderOptions {
+  view?: ViewType
+  /** Auto-clear after this many ms (same semantics as legacy display options). */
+  durationMs?: number
+}
+
+// ============================================================================
+// Native navigation UI (capability `hasNativeNavigation`, e.g. G2)
+// ============================================================================
+
+/** Turn-by-turn card fields — pre-formatted display strings, rendered verbatim by the device. */
+export interface NativeNavInfo {
+  /** Firmware direction-icon index, 0…35. */
+  directionIcon: number
+  /** Distance to the next maneuver, e.g. "500 m". */
+  distance: string
+  roadName: string
+  /** Remaining trip duration, e.g. "11 min". */
+  spendTime: string
+  /** Distance to the destination, e.g. "0.9 km". */
+  remainDistance: string
+  /** Arrival time, e.g. "18:18". */
+  etaTime: string
+  /** Current speed, e.g. "5 km/h". */
+  speed: string
+}
+
+/**
+ * "sent" = forwarded to the device. "unsupported" = no native nav UI on this
+ * device / no active trip — callers should fall back to render().
+ */
+export interface NativeNavResult {
+  status: "sent" | "unsupported" | "blocked"
+  reason?: string
+}
+
+/**
+ * Outcome of a render request, resolved when the host's display arbitration
+ * settles it. "displayed" = accepted and sent to the device — NOT a
+ * render confirmation. "blocked" = another app owns the display (or the
+ * request failed); `reason` says why.
+ */
+export interface RenderResult {
+  status: "displayed" | "blocked"
+  /** True when the host adjusted the scene (clamped boxes, dropped elements, degraded for the device). */
+  degraded?: boolean
+  /** Ids of elements the host dropped (budget/bounds/device limits) — dropped is never silent. */
+  dropped?: string[]
+  reason?: string
+}
+
+export class DisplayManager {
+  constructor(private readonly session: MiniappSession) {}
+
+  /**
+   * Render a whole scene of positioned elements — replace-the-frame.
+   *
+   * Each call describes everything that should be on screen; the host diffs it
+   * against the previous frame per device (elements with a stable `id` update
+   * in place; elements you stop sending are removed). There is no lifecycle to
+   * manage and no remove calls — `render([])` clears.
+   *
+   * Coordinates are raw pixels on the device's drawable canvas — read
+   * `session.capabilities.display` (populated on the "ready" event) for the
+   * real width/height and element budgets. Out-of-bounds boxes are clamped and
+   * over-budget elements are dropped tail-first; both are reported via the
+   * result, never silently.
+   *
+   * Awaiting is OPT-IN — a plain fire-and-forget call is fine. The returned
+   * promise never rejects.
+   *
+   * @example
+   * const result = await session.display.render([
+   *   {type: "text",  id: "stats", box: {x: 12, y: 9, w: 200, h: 28}, text: "863 m  11 min"},
+   *   {type: "image", id: "map",   box: {x: 335, y: 14, w: 150, h: 150}, data: mapPng},
+   *   {type: "rect",  box: {x: 0, y: 0, w: 500, h: 220}, style: {radius: 4}},
+   * ])
+   * if (result.degraded) console.warn("dropped:", result.dropped)
+   */
+  render(elements: RenderElement[], options: RenderOptions = {}): Promise<RenderResult> {
+    return this.session
+      .sendRequest<RenderResult>({
+        type: MiniappRequestType.RENDER,
+        view: options.view ?? "main",
+        elements,
+        durationMs: options.durationMs,
+      })
+      .catch((err) => ({
+        status: "blocked" as const,
+        reason:
+          typeof err === "object" && err !== null && "message" in err
+            ? String((err as {message: unknown}).message)
+            : String(err),
+      }))
+  }
+
+  /** Update the native nav UI's turn-by-turn card. */
+  sendNativeNavInfo(info: NativeNavInfo): Promise<NativeNavResult> {
+    return this.sendNativeNav({action: "info", info})
+  }
+
+  /**
+   * Push the native nav UI's live mini map (base64 BMP/PNG; the host re-encodes
+   * and scales to the device's slot — 176×176 on G2). Draw north-up with the
+   * cursor centred; the device rotates the bitmap by `rotationDeg`.
+   */
+  sendNativeNavMiniMap(data: string, options: {rotationDeg?: number; border?: boolean} = {}): Promise<NativeNavResult> {
+    return this.sendNativeNav({action: "mini_map", map: {data, ...options}})
+  }
+
+  /** Push the native nav UI's overview (full-route) map — 576×188 on G2. */
+  sendNativeNavOverviewMap(data: string): Promise<NativeNavResult> {
+    return this.sendNativeNav({action: "overview_map", map: {data}})
+  }
+
+  private sendNativeNav(payload: Record<string, unknown>): Promise<NativeNavResult> {
+    return this.session
+      .sendRequest<NativeNavResult>({type: MiniappRequestType.NATIVE_NAV, ...payload})
+      .catch((err) => ({
+        status: "unsupported" as const,
+        reason:
+          typeof err === "object" && err !== null && "message" in err
+            ? String((err as {message: unknown}).message)
+            : String(err),
+      }))
+  }
+}