@mentra/engine 0.1.0-beta.0

package/src/utils/display/profiles/z100.ts

@@ -0,0 +1,198 @@
+import { DisplayProfile } from "./types";
+
+/**
+ * Vuzix Z100 glyph widths - extracted from NotoSans-Regular.ttf at 21px
+ *
+ * These are pixel widths at the default font size used by the Z100 display.
+ * The Z100 uses Noto Sans as its system font.
+ *
+ * Extracted using: node scripts/extract-font-metrics.js NotoSans-Regular.ttf 21
+ */
+const Z100_GLYPH_WIDTHS: Record<string, number> = {
+  // Space and punctuation
+  " ": 5,
+  "!": 6,
+  '"': 9,
+  "#": 14,
+  "$": 12,
+  "%": 17,
+  "&": 15,
+  "'": 5,
+  "(": 6,
+  ")": 6,
+  "*": 12,
+  "+": 12,
+  ",": 6,
+  "-": 7, // Hyphen width - important for line breaking
+  ".": 6,
+  "/": 8,
+
+  // Numbers
+  "0": 12,
+  "1": 12,
+  "2": 12,
+  "3": 12,
+  "4": 12,
+  "5": 12,
+  "6": 12,
+  "7": 12,
+  "8": 12,
+  "9": 12,
+
+  // More punctuation
+  ":": 6,
+  ";": 6,
+  "<": 12,
+  "=": 12,
+  ">": 12,
+  "?": 9,
+  "@": 19,
+
+  // Uppercase
+  "A": 13,
+  "B": 14,
+  "C": 13,
+  "D": 15,
+  "E": 12,
+  "F": 11,
+  "G": 15,
+  "H": 16,
+  "I": 7,
+  "J": 6,
+  "K": 13,
+  "L": 11,
+  "M": 19,
+  "N": 16,
+  "O": 16,
+  "P": 13,
+  "Q": 16,
+  "R": 13,
+  "S": 12,
+  "T": 12,
+  "U": 15,
+  "V": 13,
+  "W": 20,
+  "X": 12,
+  "Y": 12,
+  "Z": 12,
+
+  // Brackets & special
+  "[": 7,
+  "\\": 8,
+  "]": 7,
+  "^": 12,
+  "_": 9,
+  "`": 6,
+
+  // Lowercase
+  "a": 12,
+  "b": 13,
+  "c": 10,
+  "d": 13,
+  "e": 12,
+  "f": 7,
+  "g": 13,
+  "h": 13,
+  "i": 5,
+  "j": 5,
+  "k": 11,
+  "l": 5,
+  "m": 20,
+  "n": 13,
+  "o": 13,
+  "p": 13,
+  "q": 13,
+  "r": 9,
+  "s": 10,
+  "t": 8,
+  "u": 13,
+  "v": 11,
+  "w": 17,
+  "x": 11,
+  "y": 11,
+  "z": 10,
+
+  // More special
+  "{": 8,
+  "|": 12,
+  "}": 8,
+  "~": 12,
+};
+
+/**
+ * Vuzix Z100 Smart Glasses Display Profile
+ *
+ * The Z100 has a green monochrome display with 640x480 resolution.
+ * It uses the Vuzix Ultralite SDK and Noto Sans font family.
+ *
+ * Display specs:
+ * - Resolution: 640x480
+ * - Usable text width: ~390px (empirically tested - SDK applies margins)
+ * - Color: Green monochrome
+ * - Max text lines: 7
+ * - Font: Noto Sans Regular at 21px
+ * - No bitmap support via text commands
+ * - Adjustable brightness
+ *
+ * NOTE: The Vuzix Ultralite SDK handles text rendering internally and applies
+ * its own margins/padding. The usable text width is significantly less than
+ * the physical 640px resolution. Empirical testing shows ~42 characters of
+ * mixed text (387px measured) fits on one line before the SDK wraps.
+ */
+export const Z100_PROFILE: DisplayProfile = {
+  id: "vuzix-z100",
+  name: "Vuzix Z100",
+
+  // Display dimensions
+  // Empirically tested: ~42 chars of mixed text (387px) fits before SDK wraps
+  // Using 390px to account for measurement variance
+  displayWidthPx: 390,
+  maxLines: 7,
+
+  // BLE constraints
+  // Z100 uses Vuzix Ultralite SDK which handles chunking internally
+  // These are conservative values for direct BLE if needed
+  maxPayloadBytes: 512,
+  bleChunkSize: 180,
+
+  // Font metrics - Noto Sans Regular at 21px
+  fontMetrics: {
+    glyphWidths: new Map(Object.entries(Z100_GLYPH_WIDTHS)),
+    defaultGlyphWidth: 12, // Average width from extracted metrics
+    // Z100 uses direct pixel widths, no render formula needed
+    renderFormula: (glyphWidth: number) => glyphWidth,
+
+    // Uniform-width scripts for CJK and other non-Latin
+    // Noto Sans CJK characters are typically wider
+    uniformScripts: {
+      cjk: 21, // Full-width CJK characters
+      hiragana: 21,
+      katakana: 21,
+      korean: 21,
+      cyrillic: 14, // Cyrillic is similar to Latin in Noto Sans
+    },
+
+    fallback: {
+      latinMaxWidth: 20, // 'W' and 'm' are the widest at 20px
+      unknownBehavior: "useLatinMax",
+    },
+  },
+
+  constraints: {
+    minCharsBeforeHyphen: 3,
+    noStartChars: [".", ",", "!", "?", ":", ";", ")", "]", "}", "。", "，", "！", "？", "：", "；", "）", "】", "」"],
+    noEndChars: ["(", "[", "{", "（", "【", "「"],
+  },
+};
+
+/**
+ * Get the hyphen width for Z100 in pixels.
+ * Used for line breaking calculations.
+ */
+export const Z100_HYPHEN_WIDTH_PX = 7;
+
+/**
+ * Get the space width for Z100 in pixels.
+ * Used for column alignment calculations.
+ */
+export const Z100_SPACE_WIDTH_PX = 5;

package/src/utils/display/scene/degrade.ts

@@ -0,0 +1,80 @@
+/**
+ * Degrade — compile a scene for devices that can't position elements
+ * (`canPosition: false`: G1, Z100). Output is a LEGACY layout that rides the
+ * existing display path — DisplayProcessor wraps it there, exactly as today,
+ * which is what keeps sugar byte-identical on old hardware (spec §2/§5:
+ * degrade output must be UNWRAPPED; wrapping twice breaks the goldens).
+ *
+ * Rules (design doc §3.4.8): content-preserving, never-erroring,
+ * honestly-reported. Text collapses in reading order (y, then x) joined by
+ * blank lines; EXACTLY two text elements sharing a y-range with disjoint
+ * x-ranges become a double_text_wall (ColumnComposer path downstream); rects
+ * drop silently (decoration); images drop + report.
+ */
+
+import type {SceneElementInput} from "./types"
+
+export interface DegradedScene {
+  /** Legacy layout for the existing display path, or null for an empty scene (caller clears). */
+  layout: {layoutType: string; [key: string]: unknown} | null
+  degraded: boolean
+  dropped: string[]
+}
+
+type TextEl = Extract<SceneElementInput, {type: "text"}>
+
+function yOverlap(a: TextEl, b: TextEl): boolean {
+  return a.box.y < b.box.y + b.box.h && b.box.y < a.box.y + a.box.h
+}
+
+function xDisjoint(a: TextEl, b: TextEl): boolean {
+  return a.box.x + a.box.w <= b.box.x || b.box.x + b.box.w <= a.box.x
+}
+
+export function degradeScene(input: readonly SceneElementInput[]): DegradedScene {
+  const dropped: string[] = []
+  let degraded = false
+
+  const texts: TextEl[] = []
+  input.forEach((el, index) => {
+    if (!el || typeof el !== "object" || !el.box) {
+      // Malformed element — report it like the positioning pipeline does
+      // instead of silently pretending the render was clean.
+      dropped.push((el as {id?: string} | null)?.id ?? `element[${index}]`)
+      degraded = true
+      return
+    }
+    if (el.type === "text") {
+      texts.push(el)
+    } else if (el.type === "image") {
+      dropped.push(el.id ?? `image[${index}]`)
+      degraded = true
+    }
+    // rects: silent drop — pure decoration on a text wall.
+  })
+
+  if (texts.length === 0) {
+    return {layout: null, degraded, dropped}
+  }
+
+  // Row exception: exactly 2 columns with clean separation — the shipped
+  // double_text_wall mechanism (scope guard: nothing fancier; this is NOT a
+  // layout engine).
+  if (texts.length === 2 && yOverlap(texts[0], texts[1]) && xDisjoint(texts[0], texts[1])) {
+    const [left, right] = texts[0].box.x <= texts[1].box.x ? [texts[0], texts[1]] : [texts[1], texts[0]]
+    return {
+      layout: {layoutType: "double_text_wall", topText: left.text ?? "", bottomText: right.text ?? ""},
+      degraded,
+      dropped,
+    }
+  }
+
+  // Reading order: y, then x; join with blank lines. Downstream wrap clips
+  // overflow per the device profile.
+  const ordered = [...texts].sort((a, b) => a.box.y - b.box.y || a.box.x - b.box.x)
+  const text = ordered
+    .map((t) => t.text ?? "")
+    .filter((t) => t.length > 0)
+    .join("\n\n")
+  return {layout: {layoutType: "text_wall", text}, degraded, dropped}
+}

package/src/utils/display/scene/differ.ts

@@ -0,0 +1,128 @@
+/**
+ * Scene differ — matches elements between the last frame sent to the device and
+ * the next scene, and classifies each element's change. Pure set arithmetic:
+ * nothing in here is device-specific (spec §1).
+ *
+ * Matching rules (design doc §3.4.1, in order):
+ *   1. same explicit `id` + same `type`  ⇒ same element
+ *   2. no id: same `type` + identical box ⇒ same element
+ *   3. no id: same `type`, by order among the still-unmatched
+ *   4. anything unmatched in prev ⇒ removed; unmatched in next ⇒ created
+ *
+ * Matched no-id elements INHERIT the previous element's assigned id so the
+ * device-side id⇒firmware-component mapping stays stable; fresh elements get a
+ * synthesized id from the caller (SceneStore counter). Synthesized ids are
+ * prefixed "~" so they can never collide with app-supplied ids.
+ */
+
+import type {FrameElement, SceneBox, SceneChange, SceneElementType} from "./types"
+import {boxesEqual} from "./types"
+
+/** A processed (clamped/wrapped) element, pre-diff: id optional, no annotation yet. */
+export interface DiffableElement {
+  id?: string
+  type: SceneElementType
+  box: SceneBox
+  text?: string
+  data?: string
+  style?: FrameElement["style"]
+  contentHash: string
+}
+
+export interface DiffResult {
+  elements: FrameElement[]
+  removed: string[]
+}
+
+function classify(prev: FrameElement, next: DiffableElement): SceneChange {
+  if (!boxesEqual(prev.box, next.box)) return "moved"
+  if (prev.contentHash !== next.contentHash) return "updated"
+  return "unchanged"
+}
+
+/**
+ * Assemble the annotated FrameElement, OMITTING optional fields that are
+ * undefined. A present key with an `undefined` value cannot cross the Expo
+ * Android bridge — its `Map<String, Any>` converter rejects it ("Value is
+ * undefined, expected an Object"), which surfaces as ERR_UNEXPECTED from
+ * displayEvent. (iOS's `[String: Any]` bridge silently drops nils, so the
+ * defect was invisible there.) Keeping the keys absent — not undefined —
+ * matches the optional-field contract in types.ts.
+ */
+function buildFrameElement(el: DiffableElement, id: string, change: SceneChange): FrameElement {
+  const out: FrameElement = {id, type: el.type, box: el.box, contentHash: el.contentHash, change}
+  if (el.text !== undefined) out.text = el.text
+  if (el.data !== undefined) out.data = el.data
+  if (el.style !== undefined) out.style = el.style
+  return out
+}
+
+/**
+ * Diff `next` against `prev` (the last frame sent for this app/view/device;
+ * empty array for a fresh scene). `nextSyntheticId` mints ids for elements the
+ * app didn't name — must be stable per (app, view) across calls (SceneStore).
+ */
+export function diffScene(
+  prev: readonly FrameElement[],
+  next: readonly DiffableElement[],
+  nextSyntheticId: () => string,
+): DiffResult {
+  const prevUnmatched = new Set(prev.map((_, i) => i))
+  const prevByExplicitId = new Map<string, number>()
+  prev.forEach((el, i) => {
+    if (!el.id.startsWith("~")) prevByExplicitId.set(`${el.type}:${el.id}`, i)
+  })
+
+  // matches[nextIndex] = prevIndex | undefined
+  const matches: (number | undefined)[] = new Array(next.length).fill(undefined)
+
+  // Pass 1: explicit id + type.
+  next.forEach((el, i) => {
+    if (!el.id) return
+    const j = prevByExplicitId.get(`${el.type}:${el.id}`)
+    if (j !== undefined && prevUnmatched.has(j)) {
+      matches[i] = j
+      prevUnmatched.delete(j)
+    }
+  })
+
+  // Pass 2: no-id elements match a no-id prev element of the same type with an
+  // identical box.
+  next.forEach((el, i) => {
+    if (el.id || matches[i] !== undefined) return
+    for (const j of prevUnmatched) {
+      const p = prev[j]
+      if (p.id.startsWith("~") && p.type === el.type && boxesEqual(p.box, el.box)) {
+        matches[i] = j
+        prevUnmatched.delete(j)
+        break
+      }
+    }
+  })
+
+  // Pass 3: remaining no-id elements match remaining no-id prev of the same
+  // type in order (k-th unmatched text ↔ k-th unmatched text).
+  next.forEach((el, i) => {
+    if (el.id || matches[i] !== undefined) return
+    for (const j of prevUnmatched) {
+      const p = prev[j]
+      if (p.id.startsWith("~") && p.type === el.type) {
+        matches[i] = j
+        prevUnmatched.delete(j)
+        break
+      }
+    }
+  })
+
+  const elements: FrameElement[] = next.map((el, i) => {
+    const j = matches[i]
+    if (j !== undefined) {
+      const p = prev[j]
+      return buildFrameElement(el, el.id ?? p.id, classify(p, el))
+    }
+    return buildFrameElement(el, el.id ?? nextSyntheticId(), "created")
+  })
+
+  const removed = [...prevUnmatched].map((j) => prev[j].id)
+  return {elements, removed}
+}

package/src/utils/display/scene/index.ts

@@ -0,0 +1,21 @@
+export type {
+  FrameElement,
+  SceneBox,
+  SceneBreakMode,
+  SceneChange,
+  SceneDisplayCapabilities,
+  SceneElementInput,
+  SceneElementType,
+  SceneFrame,
+  SceneProcessResult,
+  SceneRectStyle,
+  SceneTextStyle,
+} from "./types"
+export {boxesEqual, contentHash, elementContentHash} from "./types"
+export type {DiffResult, DiffableElement} from "./differ"
+export {diffScene} from "./differ"
+export type {ProcessedScene} from "./process"
+export {processScene, profileLineHeightPx} from "./process"
+export type {DegradedScene} from "./degrade"
+export {degradeScene} from "./degrade"
+export {SceneStore} from "./store"

package/src/utils/display/scene/process.ts

@@ -0,0 +1,214 @@
+/**
+ * Scene processing — validate → clamp → budget → wrap. Everything here is
+ * generic; device variation enters only through the capabilities block and the
+ * display profile (data, not code). Spec §5.
+ *
+ * Never rejects a frame: offending elements are dropped per-element and
+ * reported via `dropped` / `degraded` (design doc §3.4.6; spec §4 refines the
+ * design doc's image "reject" to per-element drop).
+ */
+
+import {TextMeasurer} from "../measurer/TextMeasurer"
+import {TextWrapper} from "../wrapper/TextWrapper"
+import type {DisplayProfile} from "../profiles/types"
+import type {DiffableElement} from "./differ"
+import type {SceneBox, SceneDisplayCapabilities, SceneElementInput, SceneTextStyle} from "./types"
+import {elementContentHash} from "./types"
+
+export interface ProcessedScene {
+  elements: DiffableElement[]
+  degraded: boolean
+  dropped: string[]
+}
+
+/** Reporting id for an element the app may not have named. */
+function reportId(el: SceneElementInput, index: number): string {
+  return el.id ?? `${el.type}[${index}]`
+}
+
+function clampBox(box: SceneBox, width: number, height: number): SceneBox | null {
+  const x1 = Math.max(0, Math.floor(box.x))
+  const y1 = Math.max(0, Math.floor(box.y))
+  const x2 = Math.min(width, Math.floor(box.x) + Math.max(0, Math.floor(box.w)))
+  const y2 = Math.min(height, Math.floor(box.y) + Math.max(0, Math.floor(box.h)))
+  if (x2 <= x1 || y2 <= y1) return null
+  return {x: x1, y: y1, w: x2 - x1, h: y2 - y1}
+}
+
+function boxShrunk(orig: SceneBox, clamped: SceneBox): boolean {
+  return (
+    clamped.x !== Math.floor(orig.x) ||
+    clamped.y !== Math.floor(orig.y) ||
+    clamped.w !== Math.floor(orig.w) ||
+    clamped.h !== Math.floor(orig.h)
+  )
+}
+
+/**
+ * Line height for box-height→line-count math — ONLY when the profile declares
+ * a calibrated `lineHeightPx`. Deriving one from full-canvas numbers proved
+ * wrong on hardware (288/8=36px clipped the G2 nav instruction to one line;
+ * the real container line height is smaller). Without calibration we return
+ * null and the wrap step skips host-side height clipping entirely — text wraps
+ * to the box WIDTH and the firmware clips vertically in-box, which is exactly
+ * the legacy behavior.
+ */
+export function profileLineHeightPx(profile: DisplayProfile, _canvasHeight: number): number | null {
+  return profile.lineHeightPx ?? null
+}
+
+/**
+ * Process a raw scene against a device's capabilities + profile. Output is
+ * diff-ready (text pre-wrapped, boxes clamped, content hashed) and reflects
+ * exactly what will be sent — the diff baseline is post-processed by design
+ * (spec §4).
+ */
+export function processScene(
+  input: readonly SceneElementInput[],
+  caps: SceneDisplayCapabilities,
+  profile: DisplayProfile,
+): ProcessedScene {
+  const dropped: string[] = []
+  let degraded = false
+
+  const measurer = new TextMeasurer(profile)
+  const wrapper = new TextWrapper(measurer)
+  const lineHeight = profileLineHeightPx(profile, caps.height)
+
+  // Validate + dedupe explicit ids (first occurrence wins; dupes are dev error).
+  const seenIds = new Set<string>()
+  const valid: {el: SceneElementInput; index: number}[] = []
+  input.forEach((raw, index) => {
+    if (!raw || typeof raw !== "object" || !raw.box) {
+      dropped.push(reportId(raw ?? ({type: "text"} as SceneElementInput), index))
+      degraded = true
+      return
+    }
+    // Non-finite box numbers (NaN/±Infinity/non-number) would survive the
+    // Math.* clamp below and poison the frame all the way to native ints.
+    const b = raw.box
+    if (![b.x, b.y, b.w, b.h].every((n) => typeof n === "number" && Number.isFinite(n))) {
+      dropped.push(reportId(raw, index))
+      degraded = true
+      return
+    }
+    // Leading "~" is the differ's synthetic-id namespace — an app id there
+    // would never match across frames (re-created every push). Normalize by
+    // stripping; the rewrite is deterministic, so diffing stays stable.
+    const el = raw.id?.startsWith("~") ? {...raw, id: raw.id.replace(/^~+/, "") || undefined} : raw
+    if (el.id) {
+      const key = `${el.type}:${el.id}`
+      if (seenIds.has(key)) {
+        dropped.push(reportId(el, index))
+        degraded = true
+        return
+      }
+      seenIds.add(key)
+    }
+    valid.push({el, index})
+  })
+
+  // Clamp + per-type limits, then budget in array order.
+  let textBudget = caps.maxTextElements
+  let imageBudget = caps.maxImageElements
+  const out: DiffableElement[] = []
+
+  for (const {el, index} of valid) {
+    const clamped = clampBox(el.box, caps.width, caps.height)
+    if (!clamped) {
+      dropped.push(reportId(el, index))
+      degraded = true
+      continue
+    }
+    const shrunk = boxShrunk(el.box, clamped)
+    if (shrunk) degraded = true
+
+    if (el.type === "image") {
+      // Images: a shrunk box can't be honored (we don't crop pixels host-side)
+      // and per-image device limits are box-level (the firmware allocates the
+      // component from the box). SGCs scale pixels to the box (phone-side
+      // scaling — never on glasses).
+      if (shrunk) {
+        dropped.push(reportId(el, index))
+        continue
+      }
+      if (caps.maxImagePx && (clamped.w > caps.maxImagePx.width || clamped.h > caps.maxImagePx.height)) {
+        dropped.push(reportId(el, index))
+        degraded = true
+        continue
+      }
+      if (imageBudget <= 0) {
+        dropped.push(reportId(el, index))
+        degraded = true
+        continue
+      }
+      imageBudget--
+      out.push({
+        id: el.id,
+        type: "image",
+        box: clamped,
+        data: el.data,
+        contentHash: elementContentHash({type: "image", data: el.data}),
+      })
+      continue
+    }
+
+    // text + rect share the text-container budget (design doc §3.4.6).
+    if (textBudget <= 0) {
+      dropped.push(reportId(el, index))
+      degraded = true
+      continue
+    }
+    textBudget--
+
+    if (el.type === "rect") {
+      out.push({
+        id: el.id,
+        type: "rect",
+        box: clamped,
+        style: el.style,
+        contentHash: elementContentHash({type: "rect", style: el.style}),
+      })
+      continue
+    }
+
+    // Text: wrap on the phone into the (clamped) box. The box then carries
+    // pre-wrapped text; firmware in-box wrap is a fallback, not the mechanism.
+    // Height clipping only applies with a CALIBRATED line height — otherwise
+    // the firmware clips vertically in-box (legacy behavior).
+    const style: SceneTextStyle = el.style ?? {}
+    const maxLines = lineHeight ? Math.max(1, Math.floor(clamped.h / lineHeight)) : profile.maxLines
+    const result = wrapper.wrap(el.text ?? "", {
+      maxWidthPx: clamped.w,
+      maxLines,
+      ...(style.breakMode ? {breakMode: style.breakMode} : {}),
+    })
+    let lines = result.lines
+    if (result.truncated) {
+      degraded = true
+      if (style.overflow === "ellipsis" && lines.length > 0) {
+        // Trim the last line until it fits WITH the ellipsis appended —
+        // otherwise the extra glyph can overflow the clamped box width.
+        let last = lines[lines.length - 1]
+        while (last.length > 0 && !measurer.fitsInWidth(`${last}…`, clamped.w)) {
+          last = last.slice(0, -1)
+        }
+        // A box narrower than the ellipsis glyph itself gets an empty line —
+        // never emit a line wider than the clamped box.
+        const ellipsized = last.length > 0 || measurer.fitsInWidth("…", clamped.w) ? `${last}…` : ""
+        lines = [...lines.slice(0, -1), ellipsized]
+      }
+    }
+    const wrappedText = lines.join("\n")
+    out.push({
+      id: el.id,
+      type: "text",
+      box: clamped,
+      text: wrappedText,
+      style: el.style,
+      contentHash: elementContentHash({type: "text", text: wrappedText, style: el.style}),
+    })
+  }
+
+  return {elements: out, degraded, dropped}
+}