@mentra/engine 0.1.0-beta.0

package/src/services/SceneRenderer.ts

@@ -0,0 +1,292 @@
+/**
+ * SceneRenderer — the device-aware boundary of the scene display pipeline.
+ *
+ * LocalDisplayManager owns WHO may draw (arbitration/boot/expiry); this service
+ * owns WHAT gets sent for a scene: capabilities lookup, legacy-layout→scene
+ * conversion (sugar and old bundles ride the same pipeline), process → diff →
+ * SceneFrame emission, retained-scene replay, and the degrade branch for
+ * devices that can't position elements.
+ *
+ * Everything device-specific here is DATA (capability fields, display profile);
+ * the code paths are generic. Spec: notes/miniapp-display-render-implementation-spec.md
+ */
+
+import displayProcessor from "./DisplayProcessor"
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
+import {SETTINGS, useSettingsStore} from "../stores/settings"
+import {useGlassesStore} from "../stores/glasses"
+import {DeviceTypes, getModelCapabilities} from "../types"
+import {
+  degradeScene,
+  diffScene,
+  processScene,
+  SceneStore,
+  type FrameElement,
+  type SceneBreakMode,
+  type SceneDisplayCapabilities,
+  type SceneElementInput,
+  type SceneFrame,
+} from "../utils/display/scene"
+
+const LOG_TAG = "SCENE_RENDERER"
+
+/** Legacy layout shape (matches DisplayPayload.layout). */
+export type LegacyLayout = {layoutType: string; [key: string]: unknown}
+
+export type SceneEmitResult =
+  /** A SceneFrame went to native. */
+  | {kind: "scene"; degraded: boolean; dropped: string[]}
+  /** Degrade path: caller routes this legacy layout through the existing pipeline (null layout ⇒ clear). */
+  | {kind: "legacy"; layout: LegacyLayout | null; degraded: boolean; dropped: string[]}
+  /** No display capabilities on the connected device. */
+  | {kind: "no-display"}
+
+class SceneRenderer {
+  private static instance: SceneRenderer | null = null
+  private store = new SceneStore()
+
+  public static getInstance(): SceneRenderer {
+    if (!SceneRenderer.instance) {
+      SceneRenderer.instance = new SceneRenderer()
+    }
+    return SceneRenderer.instance
+  }
+
+  // ===========================================================================
+  // Capabilities
+  // ===========================================================================
+
+  /**
+   * Scene capabilities of the currently-selected device, or null when it has no
+   * display block. Read on demand — same model source the CONNECT_ACK uses.
+   */
+  public currentCapabilities(): SceneDisplayCapabilities | null {
+    // dev read these via the (since-removed) settings/glassesStatus runtime
+    // hooks; engine reads its own stores directly.
+    const model =
+      (useSettingsStore.getState().getSetting(SETTINGS.default_wearable.key) as string | undefined) ??
+      (useGlassesStore.getState().deviceModel as string | undefined)
+    const display = getModelCapabilities((model ?? DeviceTypes.NONE) as DeviceTypes)?.display
+    if (!display) return null
+    return {
+      width: display.width ?? display.resolution?.width ?? displayProcessor.getProfile().displayWidthPx,
+      height: display.height ?? display.resolution?.height ?? displayProcessor.getProfile().displayHeightPx ?? 288,
+      canPosition: display.canPosition === true,
+      maxTextElements: display.maxTextElements ?? 6,
+      maxImageElements: display.maxImageElements ?? 0,
+      maxImagePx: display.maxImagePx,
+      shapes: display.shapes ?? [],
+      intensityLevels: display.intensityLevels ?? 2,
+      partialUpdate: display.partialUpdate ?? false,
+    }
+  }
+
+  // ===========================================================================
+  // Legacy layout → scene conversion (sugar + old bundles on the scene path)
+  // ===========================================================================
+
+  /**
+   * Convert a legacy layout into scene elements for a positioning device.
+   * Returns null for layouts that must stay on the legacy path (dashboard_card,
+   * clear_view, unknown types). Sugar semantics: full-canvas / split boxes.
+   */
+  public convertLegacyLayout(layout: LegacyLayout, caps: SceneDisplayCapabilities): SceneElementInput[] | null {
+    const W = caps.width
+    const H = caps.height
+    const breakMode = layout.breakMode as SceneBreakMode | undefined
+
+    switch (layout.layoutType) {
+      case "text_wall":
+        return [
+          {
+            type: "text",
+            id: "sugar:wall",
+            box: {x: 0, y: 0, w: W, h: H},
+            text: String(layout.text ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+        ]
+      case "double_text_wall": {
+        const half = Math.floor(W / 2)
+        return [
+          {
+            type: "text",
+            id: "sugar:left",
+            box: {x: 0, y: 0, w: half, h: H},
+            text: String(layout.topText ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+          {
+            type: "text",
+            id: "sugar:right",
+            box: {x: half, y: 0, w: W - half, h: H},
+            text: String(layout.bottomText ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+        ]
+      }
+      case "reference_card": {
+        const titleH = Math.max(24, Math.floor(H / 5))
+        return [
+          {type: "text", id: "sugar:title", box: {x: 0, y: 0, w: W, h: titleH}, text: String(layout.title ?? "")},
+          {
+            type: "text",
+            id: "sugar:body",
+            box: {x: 0, y: titleH, w: W, h: H - titleH},
+            text: String(layout.text ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+        ]
+      }
+      case "bitmap_view": {
+        // Dimensionless legacy bitmaps historically landed in the device's
+        // default ~200×100 container (G2 seeded 200×100 at 188,44) — NOT the
+        // full canvas, which would tile a small image across the whole frame.
+        const w = (layout.width as number | undefined) ?? Math.min(200, W)
+        const h = (layout.height as number | undefined) ?? Math.min(100, H)
+        return [
+          {
+            type: "image",
+            id: "sugar:bmp",
+            box: {
+              x: (layout.x as number | undefined) ?? Math.max(0, Math.round((W - w) / 2)),
+              y: (layout.y as number | undefined) ?? Math.max(0, Math.round((H - h) / 4)),
+              w,
+              h,
+            },
+            data: String(layout.data ?? ""),
+          },
+        ]
+      }
+      case "positioned_text": {
+        const border = (layout.borderWidth as number | undefined) ?? 0
+        const radius = (layout.borderRadius as number | undefined) ?? 0
+        return [
+          {
+            type: "text",
+            id: "sugar:pos",
+            box: {
+              x: (layout.x as number | undefined) ?? 0,
+              y: (layout.y as number | undefined) ?? 0,
+              w: (layout.width as number | undefined) ?? W,
+              h: (layout.height as number | undefined) ?? H,
+            },
+            text: String(layout.text ?? ""),
+            ...(border || radius ? {style: {border, radius}} : {}),
+          },
+        ]
+      }
+      default:
+        return null
+    }
+  }
+
+  // ===========================================================================
+  // Emit
+  // ===========================================================================
+
+  /**
+   * Process + diff + send a scene for (app, view). On non-positioning devices
+   * this returns the degraded legacy layout instead — the caller routes it
+   * through the existing DisplayProcessor path (which wraps it, keeping G1/Z100
+   * output byte-identical to today).
+   */
+  public emitScene(appId: string, view: "main" | "dashboard", elements: SceneElementInput[]): SceneEmitResult {
+    const caps = this.currentCapabilities()
+    if (!caps) return {kind: "no-display"}
+
+    if (!caps.canPosition) {
+      const {layout, degraded, dropped} = degradeScene(elements)
+      return {kind: "legacy", layout, degraded, dropped}
+    }
+
+    const processed = processScene(elements, caps, displayProcessor.getProfile())
+    // A stale baseline means the glasses were cleared since the last commit —
+    // diff from empty so everything repaints (the retained frame stays valid
+    // as the replay source only).
+    const prevFrame = this.store.isBaselineStale(appId, view) ? [] : this.store.lastFrame(appId, view)
+    const {elements: diffed, removed} = diffScene(
+      prevFrame,
+      processed.elements,
+      this.store.nextSyntheticId(appId, view),
+    )
+    this.store.commit(appId, view, diffed)
+
+    const frame: SceneFrame = {
+      appId,
+      view,
+      sceneEpoch: this.store.currentEpoch(appId, view),
+      elements: diffed,
+      removed,
+    }
+    this.sendFrame(frame)
+    return {kind: "scene", degraded: processed.degraded, dropped: processed.dropped}
+  }
+
+  /**
+   * Re-emit an app's retained scene (reconnect / arbitration restore). Replay
+   * is always create-based — the device rebuilds from scratch (spec §4).
+   * Returns false when nothing is retained.
+   */
+  public replayApp(appId: string, view: "main" | "dashboard"): boolean {
+    const frame = this.store.buildReplayFrame(appId, view)
+    if (!frame) return false
+    this.sendFrame(frame)
+    return true
+  }
+
+  /** Forget an app's retained scenes (app stop). */
+  public clearApp(appId: string): void {
+    this.store.clear(appId)
+  }
+
+  /**
+   * Reset the diff baseline for (app, view) without touching the device.
+   * Used when a legacy layout bypasses the scene path (clear_view /
+   * dashboard_card / unknown layoutTypes go to native raw): after that, the
+   * retained frame no longer matches the glasses, so the next render() must
+   * diff from empty — otherwise "unchanged" elements would be skipped and
+   * never repainted.
+   */
+  public resetBaseline(appId: string, view: "main" | "dashboard"): void {
+    this.store.commit(appId, view, [])
+  }
+
+  /**
+   * A system-level clear wiped the view (durationMs expiry, unmount fallback,
+   * boot-window expiry): mark every app's baseline for that view stale. Unlike
+   * resetBaseline this KEEPS the retained frames — they remain the
+   * restore-as-replay source; only incremental diffing restarts from empty.
+   */
+  public markViewStale(view: "main" | "dashboard"): void {
+    this.store.markViewStale(view)
+  }
+
+  private sendFrame(frame: SceneFrame): void {
+    try {
+      // sendDisplayEvent hook -> direct BluetoothSdk.displayEvent (the exact
+      // wiring the deleted MantleManager hook provided).
+      void Promise.resolve(BluetoothSdk.displayEvent({view: frame.view, scene: frame})).catch((err) => {
+        console.error(`${LOG_TAG}: native scene send failed:`, err)
+      })
+    } catch (err) {
+      console.error(`${LOG_TAG}: native scene send failed:`, err)
+    }
+  }
+
+  // ===========================================================================
+  // Test hooks
+  // ===========================================================================
+
+  public _resetForTest(): void {
+    this.store.clearAll()
+  }
+
+  public _lastFrameForTest(appId: string, view: string): FrameElement[] {
+    return this.store.lastFrame(appId, view)
+  }
+}
+
+const sceneRenderer = SceneRenderer.getInstance()
+export default sceneRenderer
+export {SceneRenderer}

package/src/services/SentenceTtsPipeline.ts

@@ -0,0 +1,106 @@
+export interface SentenceAudio {
+  audioUrl: string
+  cleanup?: () => Promise<void> | void
+}
+
+export interface SentencePlaybackResult {
+  success: boolean
+  completed: boolean
+  error: string | null
+  duration: number | null
+}
+
+export interface SentencePipelineRun {
+  cancelled: boolean
+}
+
+interface SentenceTtsPipelineOptions {
+  sentences: string[]
+  firstAudio: SentenceAudio
+  run: SentencePipelineRun
+  synthesize: (sentence: string) => Promise<SentenceAudio>
+  play: (audio: SentenceAudio, index: number) => Promise<SentencePlaybackResult>
+  onCleanupError?: (error: unknown) => void
+}
+
+async function cleanupAudio(audio: SentenceAudio, onError?: (error: unknown) => void): Promise<void> {
+  try {
+    await Promise.resolve(audio.cleanup?.())
+  } catch (error) {
+    onError?.(error)
+  }
+}
+
+/**
+ * Play pre-segmented TTS with one-sentence lookahead.
+ *
+ * The next sentence begins synthesis before playback of the current sentence,
+ * minimizing the transition delay without requiring incremental waveform
+ * support from the TTS model.
+ */
+export async function runSentenceTtsPipeline({
+  sentences,
+  firstAudio,
+  run,
+  synthesize,
+  play,
+  onCleanupError,
+}: SentenceTtsPipelineOptions): Promise<SentencePlaybackResult> {
+  let current = firstAudio
+  let totalDuration = 0
+
+  for (let index = 0; index < sentences.length; index++) {
+    if (run.cancelled) {
+      await cleanupAudio(current, onCleanupError)
+      return {success: true, completed: false, error: null, duration: totalDuration}
+    }
+
+    // Attach both handlers immediately so a fast native rejection cannot
+    // become an unhandled promise while the current sentence is playing.
+    const nextOutcome =
+      index + 1 < sentences.length
+        ? synthesize(sentences[index + 1]).then(
+            (audio) => ({audio, error: null}),
+            (error: unknown) => ({audio: null, error}),
+          )
+        : null
+
+    const playback = await play(current, index)
+    totalDuration += playback.duration ?? 0
+    await cleanupAudio(current, onCleanupError)
+
+    if (run.cancelled) {
+      if (nextOutcome) {
+        void nextOutcome.then(({audio}) => {
+          if (audio) void cleanupAudio(audio, onCleanupError)
+        })
+      }
+      return {success: true, completed: false, error: null, duration: totalDuration}
+    }
+    if (!playback.success || !playback.completed) {
+      if (nextOutcome) {
+        void nextOutcome.then(({audio}) => {
+          if (audio) void cleanupAudio(audio, onCleanupError)
+        })
+      }
+      return playback
+    }
+    if (!nextOutcome) {
+      return {success: true, completed: true, error: null, duration: totalDuration}
+    }
+
+    const next = await nextOutcome
+    if (!next.audio) {
+      const message = next.error instanceof Error ? next.error.message : String(next.error)
+      return {
+        success: false,
+        completed: false,
+        error: `Offline TTS failed for sentence ${index + 2}: ${message}`,
+        duration: totalDuration,
+      }
+    }
+    current = next.audio
+  }
+
+  return {success: true, completed: true, error: null, duration: totalDuration}
+}

package/src/services/StreamLifecycleController.ts

@@ -0,0 +1,213 @@
+/**
+ * Drives RTMP stream liveliness checks via a keep-alive heartbeat:
+ * `setActive(true)` starts the timer; each tick sends a keep-alive with a
+ * fresh ackId; `maxMissedAcks` consecutive timeouts fire `onTimeout`.
+ *
+ * Mirror of `cloud/packages/cloud/src/services/streaming/StreamLifecycleController.ts`
+ * — kept byte-equivalent except for the `LifecycleLogger` interface (pino
+ * isn't a phone dep). Behavior changes must land in both files.
+ */
+
+export interface LifecycleLogger {
+  child(bindings: Record<string, unknown>): LifecycleLogger
+  debug(...args: unknown[]): void
+  warn(...args: unknown[]): void
+  error(...args: unknown[]): void
+}
+
+interface StreamLifecycleCallbacks {
+  sendKeepAlive: (ackId: string) => Promise<void> | void
+  onTimeout: () => Promise<void> | void
+  onKeepAliveSent?: (ackId: string) => void
+  onKeepAliveAcked?: (ackId: string, ageMs: number) => void
+  onKeepAliveMissed?: (ackId: string, ageMs: number, missedCount: number) => void
+}
+
+export interface StreamLifecycleOptions {
+  logger: LifecycleLogger
+  streamId: string
+  keepAliveIntervalMs: number
+  ackTimeoutMs: number
+  maxMissedAcks: number
+  shouldSendKeepAlive?: () => boolean
+  now?: () => number
+}
+
+interface PendingAckInfo {
+  sentAt: number
+  timeout: ReturnType<typeof setTimeout>
+}
+
+/**
+ * Drives RTMP stream liveliness checks. Activate to start the heartbeat;
+ * each tick sends a keep-alive with a fresh ackId and arms a timeout. When
+ * `maxMissedAcks` consecutive timeouts fire, `onTimeout` is invoked and the
+ * caller is expected to tear the stream down.
+ */
+export class StreamLifecycleController {
+  private keepAliveTimer?: ReturnType<typeof setInterval>
+  private pendingAcks: Map<string, PendingAckInfo> = new Map()
+  private missedAcks = 0
+  private lastActivityMs: number
+  private active = false
+  private disposed = false
+
+  private readonly logger: LifecycleLogger
+  private readonly streamId: string
+  private readonly keepAliveIntervalMs: number
+  private readonly ackTimeoutMs: number
+  private readonly maxMissedAcks: number
+  private readonly shouldSendKeepAlive?: () => boolean
+  private readonly now: () => number
+
+  constructor(
+    options: StreamLifecycleOptions,
+    private readonly callbacks: StreamLifecycleCallbacks,
+  ) {
+    this.logger = options.logger.child({component: "StreamLifecycle"})
+    this.streamId = options.streamId
+    this.keepAliveIntervalMs = options.keepAliveIntervalMs
+    this.ackTimeoutMs = options.ackTimeoutMs
+    this.maxMissedAcks = options.maxMissedAcks
+    this.shouldSendKeepAlive = options.shouldSendKeepAlive
+    this.now = options.now ?? (() => Date.now())
+    this.lastActivityMs = this.now()
+  }
+
+  setActive(active: boolean): void {
+    if (this.disposed || this.active === active) return
+
+    this.logger.debug({streamId: this.streamId, active}, "Updating lifecycle active state")
+
+    this.active = active
+    if (active) {
+      this.startTimer()
+    } else {
+      this.stopTimer()
+      this.clearPendingAcks()
+      this.missedAcks = 0
+    }
+  }
+
+  recordActivity(): void {
+    this.lastActivityMs = this.now()
+    this.missedAcks = 0
+  }
+
+  handleAck(ackId: string): void {
+    if (this.disposed) return
+
+    const ackInfo = this.pendingAcks.get(ackId)
+    if (!ackInfo) {
+      this.logger.warn({streamId: this.streamId, ackId}, "Received unknown keep-alive ACK")
+      return
+    }
+
+    clearTimeout(ackInfo.timeout)
+    this.pendingAcks.delete(ackId)
+    this.recordActivity()
+
+    this.callbacks.onKeepAliveAcked?.(ackId, this.now() - ackInfo.sentAt)
+  }
+
+  dispose(): void {
+    if (this.disposed) return
+
+    this.disposed = true
+    this.stopTimer()
+    this.clearPendingAcks()
+    this.logger.debug({streamId: this.streamId}, "Lifecycle disposed")
+  }
+
+  getLastActivityMs(): number {
+    return this.lastActivityMs
+  }
+
+  private startTimer(): void {
+    if (this.keepAliveTimer) return
+    this.keepAliveTimer = setInterval(() => {
+      void this.tick()
+    }, this.keepAliveIntervalMs)
+    this.logger.debug({streamId: this.streamId}, "Keep-alive timer started")
+  }
+
+  private stopTimer(): void {
+    if (this.keepAliveTimer) {
+      clearInterval(this.keepAliveTimer)
+      this.keepAliveTimer = undefined
+      this.logger.debug({streamId: this.streamId}, "Keep-alive timer stopped")
+    }
+  }
+
+  private async tick(): Promise<void> {
+    if (this.disposed || !this.active) return
+
+    if (this.shouldSendKeepAlive && !this.shouldSendKeepAlive()) {
+      this.logger.warn(
+        {streamId: this.streamId},
+        "Skipping keep-alive send because transport is unavailable",
+      )
+      return
+    }
+
+    const ackId = this.createAckId()
+    const sentAt = this.now()
+
+    const timeout = setTimeout(() => {
+      this.onAckTimeout(ackId, sentAt)
+    }, this.ackTimeoutMs)
+
+    this.pendingAcks.set(ackId, {sentAt, timeout})
+    this.callbacks.onKeepAliveSent?.(ackId)
+
+    try {
+      await this.callbacks.sendKeepAlive(ackId)
+    } catch (error) {
+      this.logger.error({streamId: this.streamId, ackId, error}, "Error sending keep-alive")
+    }
+  }
+
+  private onAckTimeout(ackId: string, sentAt: number): void {
+    if (this.disposed) return
+
+    // Race guard: handleAck may have processed the ACK between the timeout
+    // being scheduled and this callback running. In that case the ackId is
+    // no longer pending — counting it as missed would falsely escalate.
+    if (!this.pendingAcks.has(ackId)) return
+
+    this.pendingAcks.delete(ackId)
+    this.missedAcks += 1
+    const ageMs = this.now() - sentAt
+
+    this.logger.warn(
+      {streamId: this.streamId, ackId, missedAcks: this.missedAcks, ageMs},
+      "Keep-alive ACK timeout",
+    )
+
+    this.callbacks.onKeepAliveMissed?.(ackId, ageMs, this.missedAcks)
+
+    if (this.missedAcks >= this.maxMissedAcks) {
+      this.logger.error(
+        {
+          streamId: this.streamId,
+          missedAcks: this.missedAcks,
+          maxMissedAcks: this.maxMissedAcks,
+        },
+        "Maximum missed ACKs reached; triggering timeout",
+      )
+
+      void this.callbacks.onTimeout()
+    }
+  }
+
+  private clearPendingAcks(): void {
+    for (const {timeout} of this.pendingAcks.values()) {
+      clearTimeout(timeout)
+    }
+    this.pendingAcks.clear()
+  }
+
+  private createAckId(): string {
+    return `a${this.now().toString(36).slice(-5)}`
+  }
+}