@mentra/engine 0.1.0-beta.0

package/build/services/LocalDisplayManager.js

@@ -0,0 +1,579 @@
+/**
+ * LocalDisplayManager
+ *
+ * Phone-side arbiter for display requests originating from LOCAL miniapps
+ * (the @mentra/miniapp SDK). Mirrors the user-facing behavior of the cloud
+ * DisplayManager (cloud/packages/cloud/src/services/layout/DisplayManager6.1.ts)
+ * for the features that matter on the phone:
+ *
+ *   - boot message ("Starting <AppName>…") with a bounded window
+ *   - durationMs auto-clear
+ *   - core-app vs background-app arbitration with a background lock
+ *
+ * Scope is deliberately local-only. Cloud-originated displays are untouched
+ * and may race with local displays during dev; that's acceptable per plan
+ * (agents/local-display-manager-plan.md).
+ *
+ * Display pacing is NOT done here. A JS-side ~300ms throttle used to live in
+ * this class, but JS timers (and therefore the throttle's trailing flush) do
+ * not fire on schedule while the iOS app is suspended in the background — so
+ * under streaming captions, updates accumulated and then flushed in a burst on
+ * the next wake. Pacing + last-wins coalescing now live in the native per-device
+ * queue (e.g. G2's event-driven EvenHub queue), which drains off the same BLE
+ * events that wake the process. This class forwards every request straight
+ * through; the native layer collapses redundant frames per display target.
+ *
+ * The remaining BgTimer use (boot window, durationMs expiry) is timing that
+ * does not need sub-second background fidelity.
+ */
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal";
+import displayProcessor from "./DisplayProcessor";
+import { useDisplayStore } from "../stores/display";
+import sceneRenderer from "./SceneRenderer";
+import { isGlassesConnected } from "./GlassesReadiness";
+import { useGlassesStore } from "../stores/glasses";
+import { BgTimer } from "../utils/timers";
+// =============================================================================
+// Constants
+// =============================================================================
+const LOG_TAG = "LOCAL_DISPLAY";
+const BOOT_DURATION_MS = 1500;
+// Mirrors cloud lease for a bg app holding the display. The bg app has to
+// keep driving the display to hold the lock; if it goes quiet and the core
+// app wants the screen, we release.
+const BACKGROUND_LOCK_TIMEOUT_MS = 10_000;
+// Sentinel package name used as the "source" of the system boot message.
+const SYSTEM_BOOT_PKG = "system.boot";
+// =============================================================================
+// LocalDisplayManager
+// =============================================================================
+class LocalDisplayManager {
+    static instance = null;
+    coreApp = null;
+    coreAppDisplay = null;
+    currentDisplay = null;
+    backgroundLock = null;
+    bootingApp = null;
+    bootQueue = new Map();
+    expiryTimerId = null;
+    /** Reconnect-replay subscription state (attachToRuntime). */
+    runtimeAttached = false;
+    glassesWasConnected = false;
+    unsubscribeGlassesStatus = null;
+    /** Injectable for tests. */
+    now = () => Date.now();
+    constructor() { }
+    static getInstance() {
+        if (!LocalDisplayManager.instance) {
+            LocalDisplayManager.instance = new LocalDisplayManager();
+        }
+        return LocalDisplayManager.instance;
+    }
+    // ===========================================================================
+    // Lifecycle API (called from MiniappHost)
+    // ===========================================================================
+    /**
+     * A new local miniapp just mounted. Start the boot window.
+     * Calling this for a different pkg while another is booting cancels the
+     * previous boot.
+     */
+    onMount(packageName, displayName) {
+        console.log(`${LOG_TAG}: onMount(${packageName}, "${displayName}")`);
+        // Cancel any in-flight boot for a prior app.
+        this.cancelBoot();
+        // Snapshot whatever's on the glasses BEFORE the boot text overwrites it.
+        // registerApp fires onMount for every spawn — including a background app or
+        // a crash-respawn while another app is foreground — so a boot that times out
+        // without rendering must put the prior frame back rather than blank it.
+        const preBoot = this.currentDisplay;
+        // Send the boot message directly (bypass arbitration — this is a system
+        // display). Goes through sendNow so scene devices render it as a scene:
+        // otherwise every app start would be a legacy→scene transition on the
+        // glasses (which costs a blanking clear).
+        this.sendNow(SYSTEM_BOOT_PKG, {
+            view: "main",
+            layout: { layoutType: "text_wall", text: `Starting ${displayName}…` },
+        });
+        const timerId = BgTimer.setTimeout(() => {
+            this.endBoot(/* triggeredByFirstDisplay */ false);
+        }, BOOT_DURATION_MS);
+        this.bootingApp = {
+            packageName,
+            displayName,
+            startedAt: this.now(),
+            timerId,
+            preBoot,
+        };
+        this.bootQueue.clear();
+    }
+    /**
+     * Mark which miniapp is the "core" (foreground) app. Pass null when no local
+     * miniapp is foreground.
+     */
+    onCoreAppChange(packageName) {
+        if (this.coreApp === packageName)
+            return;
+        console.log(`${LOG_TAG}: onCoreAppChange(${packageName ?? "null"})`);
+        this.coreApp = packageName;
+        // No core app means no saved frame to restore on the next unmount.
+        // Without this, the previous core's last frame can flicker back onto
+        // the glasses when the user-initiated clear is followed by an
+        // onUnmount that still finds a populated coreAppDisplay.
+        if (packageName === null) {
+            this.coreAppDisplay = null;
+        }
+    }
+    /**
+     * A local miniapp is going away. Clean up all state associated with it.
+     */
+    onUnmount(packageName) {
+        console.log(`${LOG_TAG}: onUnmount(${packageName})`);
+        // If this was the booting app, cancel boot.
+        if (this.bootingApp?.packageName === packageName) {
+            this.cancelBoot();
+        }
+        this.bootQueue.get(packageName)?.resolve?.({ status: "blocked", reason: "app stopped" });
+        this.bootQueue.delete(packageName);
+        // Forget the app's retained scenes — nothing left to replay for it.
+        sceneRenderer.clearApp(packageName);
+        // Release bg lock if this app held it.
+        if (this.backgroundLock?.packageName === packageName) {
+            this.backgroundLock = null;
+        }
+        // Clear saved core app display if the core is going away.
+        if (this.coreApp === packageName) {
+            this.coreAppDisplay = null;
+            this.coreApp = null;
+        }
+        // If this app owned the current on-glasses display, clear it (and maybe
+        // restore the core app's display if it still has time left).
+        if (this.currentDisplay?.packageName === packageName) {
+            this.clearExpiryTimer();
+            this.currentDisplay = null;
+            this.tryRestoreCoreDisplay();
+            if (!this.currentDisplay) {
+                // Nothing to restore → clear the glasses.
+                this.sendClear();
+            }
+        }
+    }
+    // ===========================================================================
+    // Display request API (called from LocalMiniappRuntime.handleDisplay)
+    // ===========================================================================
+    /**
+     * A miniapp requested a display. Routes through boot queue, arbitration,
+     * and throttle. This is the only public entry point for miniapp display
+     * traffic on the local path.
+     *
+     * `resolve`, when provided (render() requests), is guaranteed to be called
+     * exactly once with the request's outcome — including for requests parked in
+     * the boot queue (resolved when drained or superseded).
+     */
+    request(packageName, payload, resolve) {
+        // Dashboard view: pass straight through (no throttle/arbitration). Local
+        // dashboard rendering is currently a stub on the phone anyway.
+        if (payload.view === "dashboard") {
+            this.sendNow(packageName, payload, resolve);
+            return;
+        }
+        // During boot, any app's requests go into the queue. If the booting app
+        // itself makes its first display call, end boot early and drain.
+        if (this.bootingApp) {
+            const superseded = this.bootQueue.get(packageName);
+            superseded?.resolve?.({ status: "blocked", reason: "superseded by a newer frame" });
+            this.bootQueue.set(packageName, { payload, resolve });
+            if (this.bootingApp.packageName === packageName) {
+                this.endBoot(/* triggeredByFirstDisplay */ true);
+            }
+            return;
+        }
+        this.arbitrateAndSend(packageName, payload, resolve);
+    }
+    // ===========================================================================
+    // Internals — arbitration
+    // ===========================================================================
+    arbitrateAndSend(packageName, payload, resolve) {
+        const now = this.now();
+        // Expire a stale bg lock before arbitrating.
+        if (this.backgroundLock && now > this.backgroundLock.expiresAt) {
+            this.backgroundLock = null;
+        }
+        const isCore = packageName === this.coreApp;
+        if (isCore) {
+            // Save so we can restore later.
+            this.coreAppDisplay = {
+                packageName,
+                processedEvent: {}, // filled by sendNow
+                scene: payload.scene,
+                expiresAt: payload.durationMs ? now + payload.durationMs : null,
+            };
+            // If a bg app holds the lock AND is currently on the glasses, core is
+            // blocked. The saved coreAppDisplay surfaces when the bg app expires
+            // or unmounts.
+            const bgHoldsAndDisplays = this.backgroundLock &&
+                this.currentDisplay &&
+                this.currentDisplay.packageName === this.backgroundLock.packageName;
+            if (bgHoldsAndDisplays) {
+                resolve?.({ status: "blocked", reason: "a background app holds the display" });
+                return;
+            }
+            // Core can display. If a bg lock exists but holder isn't actively on
+            // the glasses, release it.
+            if (this.backgroundLock && !bgHoldsAndDisplays) {
+                this.backgroundLock = null;
+            }
+            this.sendNow(packageName, payload, resolve);
+            return;
+        }
+        // Non-core: this is a background app.
+        if (this.backgroundLock && this.backgroundLock.packageName !== packageName) {
+            // Another bg app already holds the lock — drop.
+            resolve?.({ status: "blocked", reason: "another background app holds the display" });
+            return;
+        }
+        // Acquire or bump the lock.
+        if (!this.backgroundLock) {
+            this.backgroundLock = {
+                packageName,
+                acquiredAt: now,
+                expiresAt: now + BACKGROUND_LOCK_TIMEOUT_MS,
+                lastActiveTime: now,
+            };
+        }
+        else {
+            this.backgroundLock.lastActiveTime = now;
+            this.backgroundLock.expiresAt = now + BACKGROUND_LOCK_TIMEOUT_MS;
+        }
+        this.sendNow(packageName, payload, resolve);
+    }
+    // ===========================================================================
+    // Internals — send
+    // ===========================================================================
+    sendNow(packageName, payload, resolve) {
+        const expiresAt = payload.durationMs ? this.now() + payload.durationMs : null;
+        const view = (payload.view === "dashboard" ? "dashboard" : "main");
+        // Scene requests — and legacy layouts a positioning device can render as a
+        // scene (sugar, old bundles) — go through the scene pipeline. Everything
+        // else stays on the legacy path.
+        let sceneElements = payload.scene ?? null;
+        if (!sceneElements && payload.layout) {
+            const caps = sceneRenderer.currentCapabilities();
+            if (caps?.canPosition) {
+                sceneElements = sceneRenderer.convertLegacyLayout(payload.layout, caps);
+            }
+        }
+        if (sceneElements) {
+            this.sendScene(packageName, view, sceneElements, expiresAt, resolve);
+        }
+        else if (payload.layout) {
+            // This layout bypasses the scene pipeline (clear_view, dashboard_card,
+            // unknown types, or a non-positioning device). If this (app, view) had a
+            // retained scene, the glasses no longer match it — reset the diff
+            // baseline so the next render() repaints from empty instead of skipping
+            // "unchanged" elements.
+            sceneRenderer.resetBaseline(packageName, view);
+            const rawEvent = {
+                view: payload.view ?? "main",
+                layout: payload.layout,
+            };
+            if (payload.durationMs !== undefined) {
+                rawEvent.durationMs = payload.durationMs;
+            }
+            this.sendToNative(packageName, rawEvent, expiresAt);
+            resolve?.({ status: "displayed" });
+        }
+        else {
+            // A scene payload on a device with no display block at all.
+            resolve?.({ status: "blocked", reason: "no display connected" });
+            return;
+        }
+        // If core app fired, refresh the saved snapshot with the actual processed
+        // event so restore uses the wrapped text (legacy) / the retained scene.
+        if (packageName === this.coreApp && this.currentDisplay) {
+            this.coreAppDisplay = {
+                packageName,
+                processedEvent: this.currentDisplay.processedEvent,
+                scene: this.currentDisplay.scene,
+                expiresAt: this.currentDisplay.expiresAt,
+            };
+        }
+    }
+    /**
+     * Send a scene through SceneRenderer. On positioning devices this emits a
+     * SceneFrame; on degrade devices SceneRenderer hands back a legacy layout
+     * that rides the existing (unchanged) DisplayProcessor path.
+     */
+    sendScene(packageName, view, elements, expiresAt, resolve) {
+        const result = sceneRenderer.emitScene(packageName, view, elements);
+        if (result.kind === "no-display") {
+            resolve?.({ status: "blocked", reason: "no display connected" });
+            return;
+        }
+        if (result.kind === "legacy") {
+            if (result.layout) {
+                this.sendToNative(packageName, { view, layout: result.layout }, expiresAt);
+            }
+            else {
+                // Scene degraded to nothing (e.g. all-image scene on a text-only
+                // device). render() replaces the frame, so nothing means clear.
+                this.sendClear();
+            }
+            resolve?.({ status: "displayed", degraded: result.degraded, dropped: result.dropped });
+            return;
+        }
+        this.currentDisplay = { packageName, processedEvent: {}, scene: elements, expiresAt };
+        this.clearExpiryTimer();
+        if (expiresAt !== null) {
+            const delay = Math.max(0, expiresAt - this.now());
+            this.expiryTimerId = BgTimer.setTimeout(() => {
+                this.expiryTimerId = null;
+                this.handleExpiry(packageName);
+            }, delay);
+        }
+        resolve?.({ status: "displayed", degraded: result.degraded, dropped: result.dropped });
+    }
+    sendToNative(packageName, rawEvent, expiresAt) {
+        let processedEvent;
+        try {
+            processedEvent = displayProcessor.processDisplayEvent(rawEvent);
+        }
+        catch (err) {
+            console.error(`${LOG_TAG}: DisplayProcessor error, using raw event:`, err);
+            processedEvent = rawEvent;
+        }
+        try {
+            // The display output path is a direct btsdk call now (was a host sendDisplayEvent
+            // hook) so a bare OEM renders to the glasses without wiring it.
+            void Promise.resolve(BluetoothSdk.displayEvent(processedEvent)).catch((err) => {
+                console.error(`${LOG_TAG}: native display failed:`, err);
+            });
+            // The mirror store also lives in engine, so display requests update the
+            // phone-side preview without any host hook.
+            useDisplayStore.getState().setDisplayEvent(JSON.stringify(processedEvent));
+        }
+        catch (err) {
+            console.error(`${LOG_TAG}: native display failed:`, err);
+        }
+        this.currentDisplay = { packageName, processedEvent, expiresAt };
+        this.clearExpiryTimer();
+        if (expiresAt !== null) {
+            const delay = Math.max(0, expiresAt - this.now());
+            this.expiryTimerId = BgTimer.setTimeout(() => {
+                this.expiryTimerId = null;
+                this.handleExpiry(packageName);
+            }, delay);
+        }
+    }
+    handleExpiry(packageName) {
+        // Someone else is on the glasses now — ignore.
+        if (!this.currentDisplay || this.currentDisplay.packageName !== packageName) {
+            return;
+        }
+        // Release the lock if it belonged to this (bg) app.
+        if (this.backgroundLock?.packageName === packageName) {
+            this.backgroundLock = null;
+        }
+        this.currentDisplay = null;
+        this.tryRestoreCoreDisplay();
+        if (!this.currentDisplay) {
+            this.sendClear();
+        }
+    }
+    tryRestoreCoreDisplay() {
+        if (!this.coreAppDisplay)
+            return;
+        // If the core snapshot has a duration and it's already elapsed, skip.
+        if (this.coreAppDisplay.expiresAt !== null && this.now() >= this.coreAppDisplay.expiresAt) {
+            this.coreAppDisplay = null;
+            return;
+        }
+        // Re-send the core's saved display.
+        this.restoreDisplay(this.coreAppDisplay);
+    }
+    sendClear() {
+        const clearEvent = {
+            view: "main",
+            layout: { layoutType: "clear_view" },
+        };
+        this.sendToNative("system.clear", clearEvent, null);
+        this.currentDisplay = null;
+        // The glasses main view is now blank: every app's retained scene is
+        // replay-only until it re-renders — diffing against it would skip
+        // "unchanged" elements onto the blank screen.
+        sceneRenderer.markViewStale("main");
+    }
+    // ===========================================================================
+    // Internals — boot
+    // ===========================================================================
+    cancelBoot() {
+        if (!this.bootingApp)
+            return;
+        BgTimer.clearTimeout(this.bootingApp.timerId);
+        this.bootingApp = null;
+    }
+    endBoot(triggeredByFirstDisplay) {
+        if (!this.bootingApp)
+            return;
+        const preBoot = this.bootingApp.preBoot;
+        BgTimer.clearTimeout(this.bootingApp.timerId);
+        this.bootingApp = null;
+        // Drain the queue: core first, then everyone else. Per-app last wins
+        // because the map holds one payload per packageName.
+        const queued = Array.from(this.bootQueue.entries());
+        this.bootQueue.clear();
+        const corePayload = this.coreApp ? queued.find(([pkg]) => pkg === this.coreApp) : undefined;
+        if (corePayload) {
+            this.arbitrateAndSend(corePayload[0], corePayload[1].payload, corePayload[1].resolve);
+        }
+        for (const [pkg, entry] of queued) {
+            if (pkg === this.coreApp)
+                continue;
+            this.arbitrateAndSend(pkg, entry.payload, entry.resolve);
+        }
+        // Boot ended on the timeout with nothing queued and no first display — the
+        // app just never rendered. Restore whatever was on the glasses before the
+        // boot text so a background app or crash-respawn can't blank a foreground
+        // app's frame (registerApp fires onMount for every spawn, and the core app
+        // isn't wired on the local path yet). If the glasses were empty before,
+        // clear the lingering "Starting …" instead — otherwise a miniapp that never
+        // renders (e.g. audio-only) would strand it.
+        if (!triggeredByFirstDisplay && queued.length === 0) {
+            // Only restore a frame that's still valid — one whose duration elapsed
+            // during the boot window should clear, not come back.
+            if (preBoot && (preBoot.expiresAt === null || preBoot.expiresAt > this.now())) {
+                this.restoreDisplay(preBoot);
+            }
+            else {
+                this.sendClear();
+            }
+        }
+    }
+    /**
+     * Re-push a previously-captured on-glasses frame (e.g. the display a boot
+     * window overwrote). Mirrors tryRestoreCoreDisplay: the saved event is already
+     * processed, and any remaining duration is recomputed from its expiry.
+     */
+    restoreDisplay(saved) {
+        const remaining = saved.expiresAt !== null ? Math.max(0, saved.expiresAt - this.now()) : undefined;
+        // Scene restores must REPLAY (create-based, epoch-bumped): re-emitting the
+        // scene through the normal diff would come back all-"unchanged" and the
+        // device would skip every element (spec §4 — replay is always create-based).
+        if (saved.scene) {
+            const expiresAt = remaining !== undefined ? this.now() + remaining : null;
+            if (sceneRenderer.replayApp(saved.packageName, "main")) {
+                this.currentDisplay = { packageName: saved.packageName, processedEvent: {}, scene: saved.scene, expiresAt };
+                this.clearExpiryTimer();
+                if (expiresAt !== null) {
+                    const delay = Math.max(0, expiresAt - this.now());
+                    this.expiryTimerId = BgTimer.setTimeout(() => {
+                        this.expiryTimerId = null;
+                        this.handleExpiry(saved.packageName);
+                    }, delay);
+                }
+            }
+            else {
+                // Retained state gone (app stopped since) — re-render the saved
+                // elements as a fresh scene instead.
+                this.sendNow(saved.packageName, { view: "main", scene: saved.scene, durationMs: remaining });
+            }
+            return;
+        }
+        this.sendNow(saved.packageName, {
+            view: "main",
+            layout: saved.processedEvent.layout,
+            durationMs: remaining,
+        });
+    }
+    // ===========================================================================
+    // Reconnect replay (design doc §3.4.7 — recovery = replay, host-driven)
+    // ===========================================================================
+    /**
+     * Subscribe to glasses connection state so the current owner's frame is
+     * replayed after a reconnect. Safe to call repeatedly; attaches once. Wired
+     * from MantleManager next to displayProcessor.attachToRuntime().
+     */
+    attachToRuntime() {
+        if (this.runtimeAttached)
+            return;
+        // dev wired this through the (since-removed) glassesStatus runtime hooks;
+        // engine reads its own glasses store directly.
+        this.glassesWasConnected = isGlassesConnected(useGlassesStore.getState().connection);
+        this.unsubscribeGlassesStatus = useGlassesStore.subscribe(() => {
+            const connected = isGlassesConnected(useGlassesStore.getState().connection);
+            if (connected === this.glassesWasConnected)
+                return;
+            this.glassesWasConnected = connected;
+            if (connected) {
+                this.replayCurrent();
+            }
+        });
+        this.runtimeAttached = true;
+    }
+    /**
+     * Re-push the current on-glasses frame after a device recovery. Scenes replay
+     * from SceneRenderer's retained state (create-based, epoch-bumped); legacy
+     * frames re-send their processed event. Expired frames don't come back.
+     */
+    replayCurrent() {
+        const current = this.currentDisplay;
+        if (!current)
+            return;
+        if (current.expiresAt !== null && this.now() >= current.expiresAt)
+            return;
+        console.log(`${LOG_TAG}: replayCurrent(${current.packageName})`);
+        if (current.scene) {
+            if (!sceneRenderer.replayApp(current.packageName, "main")) {
+                // Retained state gone (app stopped between disconnect and reconnect).
+                this.sendClear();
+            }
+        }
+        else if (current.processedEvent.layout) {
+            this.sendToNative(current.packageName, current.processedEvent, current.expiresAt);
+        }
+    }
+    // ===========================================================================
+    // Internals — timer helpers
+    // ===========================================================================
+    clearExpiryTimer() {
+        if (this.expiryTimerId !== null) {
+            BgTimer.clearTimeout(this.expiryTimerId);
+            this.expiryTimerId = null;
+        }
+    }
+    // ===========================================================================
+    // Test hooks
+    // ===========================================================================
+    /** Injected clock — tests only. */
+    _setNowForTest(fn) {
+        this.now = fn;
+    }
+    /** Wipe state — tests only. */
+    _resetForTest() {
+        this.cancelBoot();
+        this.clearExpiryTimer();
+        this.coreApp = null;
+        this.coreAppDisplay = null;
+        this.currentDisplay = null;
+        this.backgroundLock = null;
+        this.bootQueue.clear();
+        this.now = () => Date.now();
+        this.unsubscribeGlassesStatus?.();
+        this.unsubscribeGlassesStatus = null;
+        this.runtimeAttached = false;
+        this.glassesWasConnected = false;
+        sceneRenderer._resetForTest();
+    }
+    /** Read-only state snapshot — tests only. */
+    _peekForTest() {
+        return {
+            coreApp: this.coreApp,
+            currentDisplayPkg: this.currentDisplay?.packageName ?? null,
+            bgLockPkg: this.backgroundLock?.packageName ?? null,
+            isBooting: this.bootingApp !== null,
+        };
+    }
+}
+const localDisplayManager = LocalDisplayManager.getInstance();
+export default localDisplayManager;
+export { LocalDisplayManager };
+//# sourceMappingURL=LocalDisplayManager.js.map