@mentra/engine 0.1.0-beta.0

package/build/services/SceneRenderer.js

@@ -0,0 +1,248 @@
+/**
+ * SceneRenderer — the device-aware boundary of the scene display pipeline.
+ *
+ * LocalDisplayManager owns WHO may draw (arbitration/boot/expiry); this service
+ * owns WHAT gets sent for a scene: capabilities lookup, legacy-layout→scene
+ * conversion (sugar and old bundles ride the same pipeline), process → diff →
+ * SceneFrame emission, retained-scene replay, and the degrade branch for
+ * devices that can't position elements.
+ *
+ * Everything device-specific here is DATA (capability fields, display profile);
+ * the code paths are generic. Spec: notes/miniapp-display-render-implementation-spec.md
+ */
+import displayProcessor from "./DisplayProcessor";
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal";
+import { SETTINGS, useSettingsStore } from "../stores/settings";
+import { useGlassesStore } from "../stores/glasses";
+import { DeviceTypes, getModelCapabilities } from "../types";
+import { degradeScene, diffScene, processScene, SceneStore, } from "../utils/display/scene";
+const LOG_TAG = "SCENE_RENDERER";
+class SceneRenderer {
+    static instance = null;
+    store = new SceneStore();
+    static getInstance() {
+        if (!SceneRenderer.instance) {
+            SceneRenderer.instance = new SceneRenderer();
+        }
+        return SceneRenderer.instance;
+    }
+    // ===========================================================================
+    // Capabilities
+    // ===========================================================================
+    /**
+     * Scene capabilities of the currently-selected device, or null when it has no
+     * display block. Read on demand — same model source the CONNECT_ACK uses.
+     */
+    currentCapabilities() {
+        // dev read these via the (since-removed) settings/glassesStatus runtime
+        // hooks; engine reads its own stores directly.
+        const model = useSettingsStore.getState().getSetting(SETTINGS.default_wearable.key) ??
+            useGlassesStore.getState().deviceModel;
+        const display = getModelCapabilities((model ?? DeviceTypes.NONE))?.display;
+        if (!display)
+            return null;
+        return {
+            width: display.width ?? display.resolution?.width ?? displayProcessor.getProfile().displayWidthPx,
+            height: display.height ?? display.resolution?.height ?? displayProcessor.getProfile().displayHeightPx ?? 288,
+            canPosition: display.canPosition === true,
+            maxTextElements: display.maxTextElements ?? 6,
+            maxImageElements: display.maxImageElements ?? 0,
+            maxImagePx: display.maxImagePx,
+            shapes: display.shapes ?? [],
+            intensityLevels: display.intensityLevels ?? 2,
+            partialUpdate: display.partialUpdate ?? false,
+        };
+    }
+    // ===========================================================================
+    // Legacy layout → scene conversion (sugar + old bundles on the scene path)
+    // ===========================================================================
+    /**
+     * Convert a legacy layout into scene elements for a positioning device.
+     * Returns null for layouts that must stay on the legacy path (dashboard_card,
+     * clear_view, unknown types). Sugar semantics: full-canvas / split boxes.
+     */
+    convertLegacyLayout(layout, caps) {
+        const W = caps.width;
+        const H = caps.height;
+        const breakMode = layout.breakMode;
+        switch (layout.layoutType) {
+            case "text_wall":
+                return [
+                    {
+                        type: "text",
+                        id: "sugar:wall",
+                        box: { x: 0, y: 0, w: W, h: H },
+                        text: String(layout.text ?? ""),
+                        ...(breakMode ? { style: { breakMode } } : {}),
+                    },
+                ];
+            case "double_text_wall": {
+                const half = Math.floor(W / 2);
+                return [
+                    {
+                        type: "text",
+                        id: "sugar:left",
+                        box: { x: 0, y: 0, w: half, h: H },
+                        text: String(layout.topText ?? ""),
+                        ...(breakMode ? { style: { breakMode } } : {}),
+                    },
+                    {
+                        type: "text",
+                        id: "sugar:right",
+                        box: { x: half, y: 0, w: W - half, h: H },
+                        text: String(layout.bottomText ?? ""),
+                        ...(breakMode ? { style: { breakMode } } : {}),
+                    },
+                ];
+            }
+            case "reference_card": {
+                const titleH = Math.max(24, Math.floor(H / 5));
+                return [
+                    { type: "text", id: "sugar:title", box: { x: 0, y: 0, w: W, h: titleH }, text: String(layout.title ?? "") },
+                    {
+                        type: "text",
+                        id: "sugar:body",
+                        box: { x: 0, y: titleH, w: W, h: H - titleH },
+                        text: String(layout.text ?? ""),
+                        ...(breakMode ? { style: { breakMode } } : {}),
+                    },
+                ];
+            }
+            case "bitmap_view": {
+                // Dimensionless legacy bitmaps historically landed in the device's
+                // default ~200×100 container (G2 seeded 200×100 at 188,44) — NOT the
+                // full canvas, which would tile a small image across the whole frame.
+                const w = layout.width ?? Math.min(200, W);
+                const h = layout.height ?? Math.min(100, H);
+                return [
+                    {
+                        type: "image",
+                        id: "sugar:bmp",
+                        box: {
+                            x: layout.x ?? Math.max(0, Math.round((W - w) / 2)),
+                            y: layout.y ?? Math.max(0, Math.round((H - h) / 4)),
+                            w,
+                            h,
+                        },
+                        data: String(layout.data ?? ""),
+                    },
+                ];
+            }
+            case "positioned_text": {
+                const border = layout.borderWidth ?? 0;
+                const radius = layout.borderRadius ?? 0;
+                return [
+                    {
+                        type: "text",
+                        id: "sugar:pos",
+                        box: {
+                            x: layout.x ?? 0,
+                            y: layout.y ?? 0,
+                            w: layout.width ?? W,
+                            h: layout.height ?? H,
+                        },
+                        text: String(layout.text ?? ""),
+                        ...(border || radius ? { style: { border, radius } } : {}),
+                    },
+                ];
+            }
+            default:
+                return null;
+        }
+    }
+    // ===========================================================================
+    // Emit
+    // ===========================================================================
+    /**
+     * Process + diff + send a scene for (app, view). On non-positioning devices
+     * this returns the degraded legacy layout instead — the caller routes it
+     * through the existing DisplayProcessor path (which wraps it, keeping G1/Z100
+     * output byte-identical to today).
+     */
+    emitScene(appId, view, elements) {
+        const caps = this.currentCapabilities();
+        if (!caps)
+            return { kind: "no-display" };
+        if (!caps.canPosition) {
+            const { layout, degraded, dropped } = degradeScene(elements);
+            return { kind: "legacy", layout, degraded, dropped };
+        }
+        const processed = processScene(elements, caps, displayProcessor.getProfile());
+        // A stale baseline means the glasses were cleared since the last commit —
+        // diff from empty so everything repaints (the retained frame stays valid
+        // as the replay source only).
+        const prevFrame = this.store.isBaselineStale(appId, view) ? [] : this.store.lastFrame(appId, view);
+        const { elements: diffed, removed } = diffScene(prevFrame, processed.elements, this.store.nextSyntheticId(appId, view));
+        this.store.commit(appId, view, diffed);
+        const frame = {
+            appId,
+            view,
+            sceneEpoch: this.store.currentEpoch(appId, view),
+            elements: diffed,
+            removed,
+        };
+        this.sendFrame(frame);
+        return { kind: "scene", degraded: processed.degraded, dropped: processed.dropped };
+    }
+    /**
+     * Re-emit an app's retained scene (reconnect / arbitration restore). Replay
+     * is always create-based — the device rebuilds from scratch (spec §4).
+     * Returns false when nothing is retained.
+     */
+    replayApp(appId, view) {
+        const frame = this.store.buildReplayFrame(appId, view);
+        if (!frame)
+            return false;
+        this.sendFrame(frame);
+        return true;
+    }
+    /** Forget an app's retained scenes (app stop). */
+    clearApp(appId) {
+        this.store.clear(appId);
+    }
+    /**
+     * Reset the diff baseline for (app, view) without touching the device.
+     * Used when a legacy layout bypasses the scene path (clear_view /
+     * dashboard_card / unknown layoutTypes go to native raw): after that, the
+     * retained frame no longer matches the glasses, so the next render() must
+     * diff from empty — otherwise "unchanged" elements would be skipped and
+     * never repainted.
+     */
+    resetBaseline(appId, view) {
+        this.store.commit(appId, view, []);
+    }
+    /**
+     * A system-level clear wiped the view (durationMs expiry, unmount fallback,
+     * boot-window expiry): mark every app's baseline for that view stale. Unlike
+     * resetBaseline this KEEPS the retained frames — they remain the
+     * restore-as-replay source; only incremental diffing restarts from empty.
+     */
+    markViewStale(view) {
+        this.store.markViewStale(view);
+    }
+    sendFrame(frame) {
+        try {
+            // sendDisplayEvent hook -> direct BluetoothSdk.displayEvent (the exact
+            // wiring the deleted MantleManager hook provided).
+            void Promise.resolve(BluetoothSdk.displayEvent({ view: frame.view, scene: frame })).catch((err) => {
+                console.error(`${LOG_TAG}: native scene send failed:`, err);
+            });
+        }
+        catch (err) {
+            console.error(`${LOG_TAG}: native scene send failed:`, err);
+        }
+    }
+    // ===========================================================================
+    // Test hooks
+    // ===========================================================================
+    _resetForTest() {
+        this.store.clearAll();
+    }
+    _lastFrameForTest(appId, view) {
+        return this.store.lastFrame(appId, view);
+    }
+}
+const sceneRenderer = SceneRenderer.getInstance();
+export default sceneRenderer;
+export { SceneRenderer };
+//# sourceMappingURL=SceneRenderer.js.map

package/build/services/SceneRenderer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SceneRenderer.js","sourceRoot":"","sources":["../../src/services/SceneRenderer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,gBAAgB,MAAM,oBAAoB,CAAA;AACjD,OAAO,YAAY,MAAM,gCAAgC,CAAA;AACzD,OAAO,EAAC,QAAQ,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EAAC,eAAe,EAAC,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAC,WAAW,EAAE,oBAAoB,EAAC,MAAM,UAAU,CAAA;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,UAAU,GAMX,MAAM,wBAAwB,CAAA;AAE/B,MAAM,OAAO,GAAG,gBAAgB,CAAA;AAahC,MAAM,aAAa;IACT,MAAM,CAAC,QAAQ,GAAyB,IAAI,CAAA;IAC5C,KAAK,GAAG,IAAI,UAAU,EAAE,CAAA;IAEzB,MAAM,CAAC,WAAW;QACvB,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;YAC5B,aAAa,CAAC,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAA;QAC9C,CAAC;QACD,OAAO,aAAa,CAAC,QAAQ,CAAA;IAC/B,CAAC;IAED,8EAA8E;IAC9E,eAAe;IACf,8EAA8E;IAE9E;;;OAGG;IACI,mBAAmB;QACxB,wEAAwE;QACxE,+CAA+C;QAC/C,MAAM,KAAK,GACR,gBAAgB,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAwB;YAC5F,eAAe,CAAC,QAAQ,EAAE,CAAC,WAAkC,CAAA;QAChE,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,KAAK,IAAI,WAAW,CAAC,IAAI,CAAgB,CAAC,EAAE,OAAO,CAAA;QACzF,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAA;QACzB,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,EAAE,KAAK,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,cAAc;YACjG,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,MAAM,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,eAAe,IAAI,GAAG;YAC5G,WAAW,EAAE,OAAO,CAAC,WAAW,KAAK,IAAI;YACzC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,CAAC;YAC7C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,CAAC;YAC/C,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE;YAC5B,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,CAAC;YAC7C,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,KAAK;SAC9C,CAAA;IACH,CAAC;IAED,8EAA8E;IAC9E,2EAA2E;IAC3E,8EAA8E;IAE9E;;;;OAIG;IACI,mBAAmB,CAAC,MAAoB,EAAE,IAA8B;QAC7E,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAA;QACpB,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QACrB,MAAM,SAAS,GAAG,MAAM,CAAC,SAAuC,CAAA;QAEhE,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,EAAE,EAAE,YAAY;wBAChB,GAAG,EAAE,EAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAC;wBAC7B,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;wBAC/B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,EAAC,SAAS,EAAC,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC3C;iBACF,CAAA;YACH,KAAK,kBAAkB,CAAC,CAAC,CAAC;gBACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9B,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,EAAE,EAAE,YAAY;wBAChB,GAAG,EAAE,EAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAC;wBAChC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;wBAClC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,EAAC,SAAS,EAAC,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC3C;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,EAAE,EAAE,aAAa;wBACjB,GAAG,EAAE,EAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC,EAAC;wBACvC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;wBACrC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,EAAC,SAAS,EAAC,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC3C;iBACF,CAAA;YACH,CAAC;YACD,KAAK,gBAAgB,CAAC,CAAC,CAAC;gBACtB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAC9C,OAAO;oBACL,EAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,EAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,EAAC;oBACvG;wBACE,IAAI,EAAE,MAAM;wBACZ,EAAE,EAAE,YAAY;wBAChB,GAAG,EAAE,EAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,EAAC;wBAC3C,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;wBAC/B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,EAAC,SAAS,EAAC,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC3C;iBACF,CAAA;YACH,CAAC;YACD,KAAK,aAAa,CAAC,CAAC,CAAC;gBACnB,mEAAmE;gBACnE,qEAAqE;gBACrE,sEAAsE;gBACtE,MAAM,CAAC,GAAI,MAAM,CAAC,KAA4B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;gBAClE,MAAM,CAAC,GAAI,MAAM,CAAC,MAA6B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;gBACnE,OAAO;oBACL;wBACE,IAAI,EAAE,OAAO;wBACb,EAAE,EAAE,WAAW;wBACf,GAAG,EAAE;4BACH,CAAC,EAAG,MAAM,CAAC,CAAwB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;4BAC3E,CAAC,EAAG,MAAM,CAAC,CAAwB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;4BAC3E,CAAC;4BACD,CAAC;yBACF;wBACD,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;qBAChC;iBACF,CAAA;YACH,CAAC;YACD,KAAK,iBAAiB,CAAC,CAAC,CAAC;gBACvB,MAAM,MAAM,GAAI,MAAM,CAAC,WAAkC,IAAI,CAAC,CAAA;gBAC9D,MAAM,MAAM,GAAI,MAAM,CAAC,YAAmC,IAAI,CAAC,CAAA;gBAC/D,OAAO;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,EAAE,EAAE,WAAW;wBACf,GAAG,EAAE;4BACH,CAAC,EAAG,MAAM,CAAC,CAAwB,IAAI,CAAC;4BACxC,CAAC,EAAG,MAAM,CAAC,CAAwB,IAAI,CAAC;4BACxC,CAAC,EAAG,MAAM,CAAC,KAA4B,IAAI,CAAC;4BAC5C,CAAC,EAAG,MAAM,CAAC,MAA6B,IAAI,CAAC;yBAC9C;wBACD,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;wBAC/B,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,EAAC,MAAM,EAAE,MAAM,EAAC,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;qBACvD;iBACF,CAAA;YACH,CAAC;YACD;gBACE,OAAO,IAAI,CAAA;QACf,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,OAAO;IACP,8EAA8E;IAE9E;;;;;OAKG;IACI,SAAS,CAAC,KAAa,EAAE,IAA0B,EAAE,QAA6B;QACvF,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAA;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO,EAAC,IAAI,EAAE,YAAY,EAAC,CAAA;QAEtC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,EAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAA;YAC1D,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAC,CAAA;QACpD,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAA;QAC7E,0EAA0E;QAC1E,yEAAyE;QACzE,8BAA8B;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QAClG,MAAM,EAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAC,GAAG,SAAS,CAC3C,SAAS,EACT,SAAS,CAAC,QAAQ,EAClB,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CACxC,CAAA;QACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAA;QAEtC,MAAM,KAAK,GAAe;YACxB,KAAK;YACL,IAAI;YACJ,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC;YAChD,QAAQ,EAAE,MAAM;YAChB,OAAO;SACR,CAAA;QACD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACrB,OAAO,EAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAC,CAAA;IAClF,CAAC;IAED;;;;OAIG;IACI,SAAS,CAAC,KAAa,EAAE,IAA0B;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;QACxB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACrB,OAAO,IAAI,CAAA;IACb,CAAC;IAED,kDAAkD;IAC3C,QAAQ,CAAC,KAAa;QAC3B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACzB,CAAC;IAED;;;;;;;OAOG;IACI,aAAa,CAAC,KAAa,EAAE,IAA0B;QAC5D,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;IACpC,CAAC;IAED;;;;;OAKG;IACI,aAAa,CAAC,IAA0B;QAC7C,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;IAChC,CAAC;IAEO,SAAS,CAAC,KAAiB;QACjC,IAAI,CAAC;YACH,uEAAuE;YACvE,mDAAmD;YACnD,KAAK,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC9F,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,6BAA6B,EAAE,GAAG,CAAC,CAAA;YAC7D,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,6BAA6B,EAAE,GAAG,CAAC,CAAA;QAC7D,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,aAAa;IACb,8EAA8E;IAEvE,aAAa;QAClB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAA;IACvB,CAAC;IAEM,iBAAiB,CAAC,KAAa,EAAE,IAAY;QAClD,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAC1C,CAAC;;AAGH,MAAM,aAAa,GAAG,aAAa,CAAC,WAAW,EAAE,CAAA;AACjD,eAAe,aAAa,CAAA;AAC5B,OAAO,EAAC,aAAa,EAAC,CAAA","sourcesContent":["/**\n * SceneRenderer — the device-aware boundary of the scene display pipeline.\n *\n * LocalDisplayManager owns WHO may draw (arbitration/boot/expiry); this service\n * owns WHAT gets sent for a scene: capabilities lookup, legacy-layout→scene\n * conversion (sugar and old bundles ride the same pipeline), process → diff →\n * SceneFrame emission, retained-scene replay, and the degrade branch for\n * devices that can't position elements.\n *\n * Everything device-specific here is DATA (capability fields, display profile);\n * the code paths are generic. Spec: notes/miniapp-display-render-implementation-spec.md\n */\n\nimport displayProcessor from \"./DisplayProcessor\"\nimport BluetoothSdk from \"@mentra/bluetooth-sdk/internal\"\nimport {SETTINGS, useSettingsStore} from \"../stores/settings\"\nimport {useGlassesStore} from \"../stores/glasses\"\nimport {DeviceTypes, getModelCapabilities} from \"../types\"\nimport {\n  degradeScene,\n  diffScene,\n  processScene,\n  SceneStore,\n  type FrameElement,\n  type SceneBreakMode,\n  type SceneDisplayCapabilities,\n  type SceneElementInput,\n  type SceneFrame,\n} from \"../utils/display/scene\"\n\nconst LOG_TAG = \"SCENE_RENDERER\"\n\n/** Legacy layout shape (matches DisplayPayload.layout). */\nexport type LegacyLayout = {layoutType: string; [key: string]: unknown}\n\nexport type SceneEmitResult =\n  /** A SceneFrame went to native. */\n  | {kind: \"scene\"; degraded: boolean; dropped: string[]}\n  /** Degrade path: caller routes this legacy layout through the existing pipeline (null layout ⇒ clear). */\n  | {kind: \"legacy\"; layout: LegacyLayout | null; degraded: boolean; dropped: string[]}\n  /** No display capabilities on the connected device. */\n  | {kind: \"no-display\"}\n\nclass SceneRenderer {\n  private static instance: SceneRenderer | null = null\n  private store = new SceneStore()\n\n  public static getInstance(): SceneRenderer {\n    if (!SceneRenderer.instance) {\n      SceneRenderer.instance = new SceneRenderer()\n    }\n    return SceneRenderer.instance\n  }\n\n  // ===========================================================================\n  // Capabilities\n  // ===========================================================================\n\n  /**\n   * Scene capabilities of the currently-selected device, or null when it has no\n   * display block. Read on demand — same model source the CONNECT_ACK uses.\n   */\n  public currentCapabilities(): SceneDisplayCapabilities | null {\n    // dev read these via the (since-removed) settings/glassesStatus runtime\n    // hooks; engine reads its own stores directly.\n    const model =\n      (useSettingsStore.getState().getSetting(SETTINGS.default_wearable.key) as string | undefined) ??\n      (useGlassesStore.getState().deviceModel as string | undefined)\n    const display = getModelCapabilities((model ?? DeviceTypes.NONE) as DeviceTypes)?.display\n    if (!display) return null\n    return {\n      width: display.width ?? display.resolution?.width ?? displayProcessor.getProfile().displayWidthPx,\n      height: display.height ?? display.resolution?.height ?? displayProcessor.getProfile().displayHeightPx ?? 288,\n      canPosition: display.canPosition === true,\n      maxTextElements: display.maxTextElements ?? 6,\n      maxImageElements: display.maxImageElements ?? 0,\n      maxImagePx: display.maxImagePx,\n      shapes: display.shapes ?? [],\n      intensityLevels: display.intensityLevels ?? 2,\n      partialUpdate: display.partialUpdate ?? false,\n    }\n  }\n\n  // ===========================================================================\n  // Legacy layout → scene conversion (sugar + old bundles on the scene path)\n  // ===========================================================================\n\n  /**\n   * Convert a legacy layout into scene elements for a positioning device.\n   * Returns null for layouts that must stay on the legacy path (dashboard_card,\n   * clear_view, unknown types). Sugar semantics: full-canvas / split boxes.\n   */\n  public convertLegacyLayout(layout: LegacyLayout, caps: SceneDisplayCapabilities): SceneElementInput[] | null {\n    const W = caps.width\n    const H = caps.height\n    const breakMode = layout.breakMode as SceneBreakMode | undefined\n\n    switch (layout.layoutType) {\n      case \"text_wall\":\n        return [\n          {\n            type: \"text\",\n            id: \"sugar:wall\",\n            box: {x: 0, y: 0, w: W, h: H},\n            text: String(layout.text ?? \"\"),\n            ...(breakMode ? {style: {breakMode}} : {}),\n          },\n        ]\n      case \"double_text_wall\": {\n        const half = Math.floor(W / 2)\n        return [\n          {\n            type: \"text\",\n            id: \"sugar:left\",\n            box: {x: 0, y: 0, w: half, h: H},\n            text: String(layout.topText ?? \"\"),\n            ...(breakMode ? {style: {breakMode}} : {}),\n          },\n          {\n            type: \"text\",\n            id: \"sugar:right\",\n            box: {x: half, y: 0, w: W - half, h: H},\n            text: String(layout.bottomText ?? \"\"),\n            ...(breakMode ? {style: {breakMode}} : {}),\n          },\n        ]\n      }\n      case \"reference_card\": {\n        const titleH = Math.max(24, Math.floor(H / 5))\n        return [\n          {type: \"text\", id: \"sugar:title\", box: {x: 0, y: 0, w: W, h: titleH}, text: String(layout.title ?? \"\")},\n          {\n            type: \"text\",\n            id: \"sugar:body\",\n            box: {x: 0, y: titleH, w: W, h: H - titleH},\n            text: String(layout.text ?? \"\"),\n            ...(breakMode ? {style: {breakMode}} : {}),\n          },\n        ]\n      }\n      case \"bitmap_view\": {\n        // Dimensionless legacy bitmaps historically landed in the device's\n        // default ~200×100 container (G2 seeded 200×100 at 188,44) — NOT the\n        // full canvas, which would tile a small image across the whole frame.\n        const w = (layout.width as number | undefined) ?? Math.min(200, W)\n        const h = (layout.height as number | undefined) ?? Math.min(100, H)\n        return [\n          {\n            type: \"image\",\n            id: \"sugar:bmp\",\n            box: {\n              x: (layout.x as number | undefined) ?? Math.max(0, Math.round((W - w) / 2)),\n              y: (layout.y as number | undefined) ?? Math.max(0, Math.round((H - h) / 4)),\n              w,\n              h,\n            },\n            data: String(layout.data ?? \"\"),\n          },\n        ]\n      }\n      case \"positioned_text\": {\n        const border = (layout.borderWidth as number | undefined) ?? 0\n        const radius = (layout.borderRadius as number | undefined) ?? 0\n        return [\n          {\n            type: \"text\",\n            id: \"sugar:pos\",\n            box: {\n              x: (layout.x as number | undefined) ?? 0,\n              y: (layout.y as number | undefined) ?? 0,\n              w: (layout.width as number | undefined) ?? W,\n              h: (layout.height as number | undefined) ?? H,\n            },\n            text: String(layout.text ?? \"\"),\n            ...(border || radius ? {style: {border, radius}} : {}),\n          },\n        ]\n      }\n      default:\n        return null\n    }\n  }\n\n  // ===========================================================================\n  // Emit\n  // ===========================================================================\n\n  /**\n   * Process + diff + send a scene for (app, view). On non-positioning devices\n   * this returns the degraded legacy layout instead — the caller routes it\n   * through the existing DisplayProcessor path (which wraps it, keeping G1/Z100\n   * output byte-identical to today).\n   */\n  public emitScene(appId: string, view: \"main\" | \"dashboard\", elements: SceneElementInput[]): SceneEmitResult {\n    const caps = this.currentCapabilities()\n    if (!caps) return {kind: \"no-display\"}\n\n    if (!caps.canPosition) {\n      const {layout, degraded, dropped} = degradeScene(elements)\n      return {kind: \"legacy\", layout, degraded, dropped}\n    }\n\n    const processed = processScene(elements, caps, displayProcessor.getProfile())\n    // A stale baseline means the glasses were cleared since the last commit —\n    // diff from empty so everything repaints (the retained frame stays valid\n    // as the replay source only).\n    const prevFrame = this.store.isBaselineStale(appId, view) ? [] : this.store.lastFrame(appId, view)\n    const {elements: diffed, removed} = diffScene(\n      prevFrame,\n      processed.elements,\n      this.store.nextSyntheticId(appId, view),\n    )\n    this.store.commit(appId, view, diffed)\n\n    const frame: SceneFrame = {\n      appId,\n      view,\n      sceneEpoch: this.store.currentEpoch(appId, view),\n      elements: diffed,\n      removed,\n    }\n    this.sendFrame(frame)\n    return {kind: \"scene\", degraded: processed.degraded, dropped: processed.dropped}\n  }\n\n  /**\n   * Re-emit an app's retained scene (reconnect / arbitration restore). Replay\n   * is always create-based — the device rebuilds from scratch (spec §4).\n   * Returns false when nothing is retained.\n   */\n  public replayApp(appId: string, view: \"main\" | \"dashboard\"): boolean {\n    const frame = this.store.buildReplayFrame(appId, view)\n    if (!frame) return false\n    this.sendFrame(frame)\n    return true\n  }\n\n  /** Forget an app's retained scenes (app stop). */\n  public clearApp(appId: string): void {\n    this.store.clear(appId)\n  }\n\n  /**\n   * Reset the diff baseline for (app, view) without touching the device.\n   * Used when a legacy layout bypasses the scene path (clear_view /\n   * dashboard_card / unknown layoutTypes go to native raw): after that, the\n   * retained frame no longer matches the glasses, so the next render() must\n   * diff from empty — otherwise \"unchanged\" elements would be skipped and\n   * never repainted.\n   */\n  public resetBaseline(appId: string, view: \"main\" | \"dashboard\"): void {\n    this.store.commit(appId, view, [])\n  }\n\n  /**\n   * A system-level clear wiped the view (durationMs expiry, unmount fallback,\n   * boot-window expiry): mark every app's baseline for that view stale. Unlike\n   * resetBaseline this KEEPS the retained frames — they remain the\n   * restore-as-replay source; only incremental diffing restarts from empty.\n   */\n  public markViewStale(view: \"main\" | \"dashboard\"): void {\n    this.store.markViewStale(view)\n  }\n\n  private sendFrame(frame: SceneFrame): void {\n    try {\n      // sendDisplayEvent hook -> direct BluetoothSdk.displayEvent (the exact\n      // wiring the deleted MantleManager hook provided).\n      void Promise.resolve(BluetoothSdk.displayEvent({view: frame.view, scene: frame})).catch((err) => {\n        console.error(`${LOG_TAG}: native scene send failed:`, err)\n      })\n    } catch (err) {\n      console.error(`${LOG_TAG}: native scene send failed:`, err)\n    }\n  }\n\n  // ===========================================================================\n  // Test hooks\n  // ===========================================================================\n\n  public _resetForTest(): void {\n    this.store.clearAll()\n  }\n\n  public _lastFrameForTest(appId: string, view: string): FrameElement[] {\n    return this.store.lastFrame(appId, view)\n  }\n}\n\nconst sceneRenderer = SceneRenderer.getInstance()\nexport default sceneRenderer\nexport {SceneRenderer}\n"]}

package/build/services/SentenceTtsPipeline.d.ts

@@ -0,0 +1,31 @@
+export interface SentenceAudio {
+    audioUrl: string;
+    cleanup?: () => Promise<void> | void;
+}
+export interface SentencePlaybackResult {
+    success: boolean;
+    completed: boolean;
+    error: string | null;
+    duration: number | null;
+}
+export interface SentencePipelineRun {
+    cancelled: boolean;
+}
+interface SentenceTtsPipelineOptions {
+    sentences: string[];
+    firstAudio: SentenceAudio;
+    run: SentencePipelineRun;
+    synthesize: (sentence: string) => Promise<SentenceAudio>;
+    play: (audio: SentenceAudio, index: number) => Promise<SentencePlaybackResult>;
+    onCleanupError?: (error: unknown) => void;
+}
+/**
+ * Play pre-segmented TTS with one-sentence lookahead.
+ *
+ * The next sentence begins synthesis before playback of the current sentence,
+ * minimizing the transition delay without requiring incremental waveform
+ * support from the TTS model.
+ */
+export declare function runSentenceTtsPipeline({ sentences, firstAudio, run, synthesize, play, onCleanupError, }: SentenceTtsPipelineOptions): Promise<SentencePlaybackResult>;
+export {};
+//# sourceMappingURL=SentenceTtsPipeline.d.ts.map

package/build/services/SentenceTtsPipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SentenceTtsPipeline.d.ts","sourceRoot":"","sources":["../../src/services/SentenceTtsPipeline.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrC;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,OAAO,CAAA;IAClB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,OAAO,CAAA;CACnB;AAED,UAAU,0BAA0B;IAClC,SAAS,EAAE,MAAM,EAAE,CAAA;IACnB,UAAU,EAAE,aAAa,CAAA;IACzB,GAAG,EAAE,mBAAmB,CAAA;IACxB,UAAU,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,CAAA;IACxD,IAAI,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAA;IAC9E,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAA;CAC1C;AAUD;;;;;;GAMG;AACH,wBAAsB,sBAAsB,CAAC,EAC3C,SAAS,EACT,UAAU,EACV,GAAG,EACH,UAAU,EACV,IAAI,EACJ,cAAc,GACf,EAAE,0BAA0B,GAAG,OAAO,CAAC,sBAAsB,CAAC,CA0D9D"}

package/build/services/SentenceTtsPipeline.js

@@ -0,0 +1,67 @@
+async function cleanupAudio(audio, onError) {
+    try {
+        await Promise.resolve(audio.cleanup?.());
+    }
+    catch (error) {
+        onError?.(error);
+    }
+}
+/**
+ * Play pre-segmented TTS with one-sentence lookahead.
+ *
+ * The next sentence begins synthesis before playback of the current sentence,
+ * minimizing the transition delay without requiring incremental waveform
+ * support from the TTS model.
+ */
+export async function runSentenceTtsPipeline({ sentences, firstAudio, run, synthesize, play, onCleanupError, }) {
+    let current = firstAudio;
+    let totalDuration = 0;
+    for (let index = 0; index < sentences.length; index++) {
+        if (run.cancelled) {
+            await cleanupAudio(current, onCleanupError);
+            return { success: true, completed: false, error: null, duration: totalDuration };
+        }
+        // Attach both handlers immediately so a fast native rejection cannot
+        // become an unhandled promise while the current sentence is playing.
+        const nextOutcome = index + 1 < sentences.length
+            ? synthesize(sentences[index + 1]).then((audio) => ({ audio, error: null }), (error) => ({ audio: null, error }))
+            : null;
+        const playback = await play(current, index);
+        totalDuration += playback.duration ?? 0;
+        await cleanupAudio(current, onCleanupError);
+        if (run.cancelled) {
+            if (nextOutcome) {
+                void nextOutcome.then(({ audio }) => {
+                    if (audio)
+                        void cleanupAudio(audio, onCleanupError);
+                });
+            }
+            return { success: true, completed: false, error: null, duration: totalDuration };
+        }
+        if (!playback.success || !playback.completed) {
+            if (nextOutcome) {
+                void nextOutcome.then(({ audio }) => {
+                    if (audio)
+                        void cleanupAudio(audio, onCleanupError);
+                });
+            }
+            return playback;
+        }
+        if (!nextOutcome) {
+            return { success: true, completed: true, error: null, duration: totalDuration };
+        }
+        const next = await nextOutcome;
+        if (!next.audio) {
+            const message = next.error instanceof Error ? next.error.message : String(next.error);
+            return {
+                success: false,
+                completed: false,
+                error: `Offline TTS failed for sentence ${index + 2}: ${message}`,
+                duration: totalDuration,
+            };
+        }
+        current = next.audio;
+    }
+    return { success: true, completed: true, error: null, duration: totalDuration };
+}
+//# sourceMappingURL=SentenceTtsPipeline.js.map

package/build/services/SentenceTtsPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SentenceTtsPipeline.js","sourceRoot":"","sources":["../../src/services/SentenceTtsPipeline.ts"],"names":[],"mappings":"AAyBA,KAAK,UAAU,YAAY,CAAC,KAAoB,EAAE,OAAkC;IAClF,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,CAAC,KAAK,CAAC,CAAA;IAClB,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAC3C,SAAS,EACT,UAAU,EACV,GAAG,EACH,UAAU,EACV,IAAI,EACJ,cAAc,GACa;IAC3B,IAAI,OAAO,GAAG,UAAU,CAAA;IACxB,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACtD,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,MAAM,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;YAC3C,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAC,CAAA;QAChF,CAAC;QAED,qEAAqE;QACrE,qEAAqE;QACrE,MAAM,WAAW,GACf,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM;YAC1B,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CACnC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,EACjC,CAAC,KAAc,EAAE,EAAE,CAAC,CAAC,EAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAC,CAAC,CAC3C;YACH,CAAC,CAAC,IAAI,CAAA;QAEV,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAC3C,aAAa,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAA;QACvC,MAAM,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;QAE3C,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,IAAI,WAAW,EAAE,CAAC;gBAChB,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC,EAAC,KAAK,EAAC,EAAE,EAAE;oBAChC,IAAI,KAAK;wBAAE,KAAK,YAAY,CAAC,KAAK,EAAE,cAAc,CAAC,CAAA;gBACrD,CAAC,CAAC,CAAA;YACJ,CAAC;YACD,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAC,CAAA;QAChF,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC7C,IAAI,WAAW,EAAE,CAAC;gBAChB,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC,EAAC,KAAK,EAAC,EAAE,EAAE;oBAChC,IAAI,KAAK;wBAAE,KAAK,YAAY,CAAC,KAAK,EAAE,cAAc,CAAC,CAAA;gBACrD,CAAC,CAAC,CAAA;YACJ,CAAC;YACD,OAAO,QAAQ,CAAA;QACjB,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAC,CAAA;QAC/E,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,WAAW,CAAA;QAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACrF,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE,mCAAmC,KAAK,GAAG,CAAC,KAAK,OAAO,EAAE;gBACjE,QAAQ,EAAE,aAAa;aACxB,CAAA;QACH,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAA;IACtB,CAAC;IAED,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAC,CAAA;AAC/E,CAAC","sourcesContent":["export interface SentenceAudio {\n  audioUrl: string\n  cleanup?: () => Promise<void> | void\n}\n\nexport interface SentencePlaybackResult {\n  success: boolean\n  completed: boolean\n  error: string | null\n  duration: number | null\n}\n\nexport interface SentencePipelineRun {\n  cancelled: boolean\n}\n\ninterface SentenceTtsPipelineOptions {\n  sentences: string[]\n  firstAudio: SentenceAudio\n  run: SentencePipelineRun\n  synthesize: (sentence: string) => Promise<SentenceAudio>\n  play: (audio: SentenceAudio, index: number) => Promise<SentencePlaybackResult>\n  onCleanupError?: (error: unknown) => void\n}\n\nasync function cleanupAudio(audio: SentenceAudio, onError?: (error: unknown) => void): Promise<void> {\n  try {\n    await Promise.resolve(audio.cleanup?.())\n  } catch (error) {\n    onError?.(error)\n  }\n}\n\n/**\n * Play pre-segmented TTS with one-sentence lookahead.\n *\n * The next sentence begins synthesis before playback of the current sentence,\n * minimizing the transition delay without requiring incremental waveform\n * support from the TTS model.\n */\nexport async function runSentenceTtsPipeline({\n  sentences,\n  firstAudio,\n  run,\n  synthesize,\n  play,\n  onCleanupError,\n}: SentenceTtsPipelineOptions): Promise<SentencePlaybackResult> {\n  let current = firstAudio\n  let totalDuration = 0\n\n  for (let index = 0; index < sentences.length; index++) {\n    if (run.cancelled) {\n      await cleanupAudio(current, onCleanupError)\n      return {success: true, completed: false, error: null, duration: totalDuration}\n    }\n\n    // Attach both handlers immediately so a fast native rejection cannot\n    // become an unhandled promise while the current sentence is playing.\n    const nextOutcome =\n      index + 1 < sentences.length\n        ? synthesize(sentences[index + 1]).then(\n            (audio) => ({audio, error: null}),\n            (error: unknown) => ({audio: null, error}),\n          )\n        : null\n\n    const playback = await play(current, index)\n    totalDuration += playback.duration ?? 0\n    await cleanupAudio(current, onCleanupError)\n\n    if (run.cancelled) {\n      if (nextOutcome) {\n        void nextOutcome.then(({audio}) => {\n          if (audio) void cleanupAudio(audio, onCleanupError)\n        })\n      }\n      return {success: true, completed: false, error: null, duration: totalDuration}\n    }\n    if (!playback.success || !playback.completed) {\n      if (nextOutcome) {\n        void nextOutcome.then(({audio}) => {\n          if (audio) void cleanupAudio(audio, onCleanupError)\n        })\n      }\n      return playback\n    }\n    if (!nextOutcome) {\n      return {success: true, completed: true, error: null, duration: totalDuration}\n    }\n\n    const next = await nextOutcome\n    if (!next.audio) {\n      const message = next.error instanceof Error ? next.error.message : String(next.error)\n      return {\n        success: false,\n        completed: false,\n        error: `Offline TTS failed for sentence ${index + 2}: ${message}`,\n        duration: totalDuration,\n      }\n    }\n    current = next.audio\n  }\n\n  return {success: true, completed: true, error: null, duration: totalDuration}\n}\n"]}

package/build/services/StreamLifecycleController.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Drives RTMP stream liveliness checks via a keep-alive heartbeat:
+ * `setActive(true)` starts the timer; each tick sends a keep-alive with a
+ * fresh ackId; `maxMissedAcks` consecutive timeouts fire `onTimeout`.
+ *
+ * Mirror of `cloud/packages/cloud/src/services/streaming/StreamLifecycleController.ts`
+ * — kept byte-equivalent except for the `LifecycleLogger` interface (pino
+ * isn't a phone dep). Behavior changes must land in both files.
+ */
+export interface LifecycleLogger {
+    child(bindings: Record<string, unknown>): LifecycleLogger;
+    debug(...args: unknown[]): void;
+    warn(...args: unknown[]): void;
+    error(...args: unknown[]): void;
+}
+interface StreamLifecycleCallbacks {
+    sendKeepAlive: (ackId: string) => Promise<void> | void;
+    onTimeout: () => Promise<void> | void;
+    onKeepAliveSent?: (ackId: string) => void;
+    onKeepAliveAcked?: (ackId: string, ageMs: number) => void;
+    onKeepAliveMissed?: (ackId: string, ageMs: number, missedCount: number) => void;
+}
+export interface StreamLifecycleOptions {
+    logger: LifecycleLogger;
+    streamId: string;
+    keepAliveIntervalMs: number;
+    ackTimeoutMs: number;
+    maxMissedAcks: number;
+    shouldSendKeepAlive?: () => boolean;
+    now?: () => number;
+}
+/**
+ * Drives RTMP stream liveliness checks. Activate to start the heartbeat;
+ * each tick sends a keep-alive with a fresh ackId and arms a timeout. When
+ * `maxMissedAcks` consecutive timeouts fire, `onTimeout` is invoked and the
+ * caller is expected to tear the stream down.
+ */
+export declare class StreamLifecycleController {
+    private readonly callbacks;
+    private keepAliveTimer?;
+    private pendingAcks;
+    private missedAcks;
+    private lastActivityMs;
+    private active;
+    private disposed;
+    private readonly logger;
+    private readonly streamId;
+    private readonly keepAliveIntervalMs;
+    private readonly ackTimeoutMs;
+    private readonly maxMissedAcks;
+    private readonly shouldSendKeepAlive?;
+    private readonly now;
+    constructor(options: StreamLifecycleOptions, callbacks: StreamLifecycleCallbacks);
+    setActive(active: boolean): void;
+    recordActivity(): void;
+    handleAck(ackId: string): void;
+    dispose(): void;
+    getLastActivityMs(): number;
+    private startTimer;
+    private stopTimer;
+    private tick;
+    private onAckTimeout;
+    private clearPendingAcks;
+    private createAckId;
+}
+export {};
+//# sourceMappingURL=StreamLifecycleController.d.ts.map

package/build/services/StreamLifecycleController.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StreamLifecycleController.d.ts","sourceRoot":"","sources":["../../src/services/StreamLifecycleController.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,eAAe,CAAA;IACzD,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/B,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC9B,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CAChC;AAED,UAAU,wBAAwB;IAChC,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACtD,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACrC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;IACzC,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;IACzD,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAA;CAChF;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,eAAe,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAA;IACnC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CACnB;AAOD;;;;;GAKG;AACH,qBAAa,yBAAyB;IAkBlC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAjB5B,OAAO,CAAC,cAAc,CAAC,CAAgC;IACvD,OAAO,CAAC,WAAW,CAAyC;IAC5D,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,cAAc,CAAQ;IAC9B,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAQ;IAExB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAC5C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAQ;IACtC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAe;IACpD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAc;gBAGhC,OAAO,EAAE,sBAAsB,EACd,SAAS,EAAE,wBAAwB;IAYtD,SAAS,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI;IAehC,cAAc,IAAI,IAAI;IAKtB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAgB9B,OAAO,IAAI,IAAI;IASf,iBAAiB,IAAI,MAAM;IAI3B,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,SAAS;YAQH,IAAI;IA4BlB,OAAO,CAAC,YAAY;IAiCpB,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,WAAW;CAGpB"}

package/build/services/StreamLifecycleController.js

@@ -0,0 +1,152 @@
+/**
+ * Drives RTMP stream liveliness checks via a keep-alive heartbeat:
+ * `setActive(true)` starts the timer; each tick sends a keep-alive with a
+ * fresh ackId; `maxMissedAcks` consecutive timeouts fire `onTimeout`.
+ *
+ * Mirror of `cloud/packages/cloud/src/services/streaming/StreamLifecycleController.ts`
+ * — kept byte-equivalent except for the `LifecycleLogger` interface (pino
+ * isn't a phone dep). Behavior changes must land in both files.
+ */
+/**
+ * Drives RTMP stream liveliness checks. Activate to start the heartbeat;
+ * each tick sends a keep-alive with a fresh ackId and arms a timeout. When
+ * `maxMissedAcks` consecutive timeouts fire, `onTimeout` is invoked and the
+ * caller is expected to tear the stream down.
+ */
+export class StreamLifecycleController {
+    callbacks;
+    keepAliveTimer;
+    pendingAcks = new Map();
+    missedAcks = 0;
+    lastActivityMs;
+    active = false;
+    disposed = false;
+    logger;
+    streamId;
+    keepAliveIntervalMs;
+    ackTimeoutMs;
+    maxMissedAcks;
+    shouldSendKeepAlive;
+    now;
+    constructor(options, callbacks) {
+        this.callbacks = callbacks;
+        this.logger = options.logger.child({ component: "StreamLifecycle" });
+        this.streamId = options.streamId;
+        this.keepAliveIntervalMs = options.keepAliveIntervalMs;
+        this.ackTimeoutMs = options.ackTimeoutMs;
+        this.maxMissedAcks = options.maxMissedAcks;
+        this.shouldSendKeepAlive = options.shouldSendKeepAlive;
+        this.now = options.now ?? (() => Date.now());
+        this.lastActivityMs = this.now();
+    }
+    setActive(active) {
+        if (this.disposed || this.active === active)
+            return;
+        this.logger.debug({ streamId: this.streamId, active }, "Updating lifecycle active state");
+        this.active = active;
+        if (active) {
+            this.startTimer();
+        }
+        else {
+            this.stopTimer();
+            this.clearPendingAcks();
+            this.missedAcks = 0;
+        }
+    }
+    recordActivity() {
+        this.lastActivityMs = this.now();
+        this.missedAcks = 0;
+    }
+    handleAck(ackId) {
+        if (this.disposed)
+            return;
+        const ackInfo = this.pendingAcks.get(ackId);
+        if (!ackInfo) {
+            this.logger.warn({ streamId: this.streamId, ackId }, "Received unknown keep-alive ACK");
+            return;
+        }
+        clearTimeout(ackInfo.timeout);
+        this.pendingAcks.delete(ackId);
+        this.recordActivity();
+        this.callbacks.onKeepAliveAcked?.(ackId, this.now() - ackInfo.sentAt);
+    }
+    dispose() {
+        if (this.disposed)
+            return;
+        this.disposed = true;
+        this.stopTimer();
+        this.clearPendingAcks();
+        this.logger.debug({ streamId: this.streamId }, "Lifecycle disposed");
+    }
+    getLastActivityMs() {
+        return this.lastActivityMs;
+    }
+    startTimer() {
+        if (this.keepAliveTimer)
+            return;
+        this.keepAliveTimer = setInterval(() => {
+            void this.tick();
+        }, this.keepAliveIntervalMs);
+        this.logger.debug({ streamId: this.streamId }, "Keep-alive timer started");
+    }
+    stopTimer() {
+        if (this.keepAliveTimer) {
+            clearInterval(this.keepAliveTimer);
+            this.keepAliveTimer = undefined;
+            this.logger.debug({ streamId: this.streamId }, "Keep-alive timer stopped");
+        }
+    }
+    async tick() {
+        if (this.disposed || !this.active)
+            return;
+        if (this.shouldSendKeepAlive && !this.shouldSendKeepAlive()) {
+            this.logger.warn({ streamId: this.streamId }, "Skipping keep-alive send because transport is unavailable");
+            return;
+        }
+        const ackId = this.createAckId();
+        const sentAt = this.now();
+        const timeout = setTimeout(() => {
+            this.onAckTimeout(ackId, sentAt);
+        }, this.ackTimeoutMs);
+        this.pendingAcks.set(ackId, { sentAt, timeout });
+        this.callbacks.onKeepAliveSent?.(ackId);
+        try {
+            await this.callbacks.sendKeepAlive(ackId);
+        }
+        catch (error) {
+            this.logger.error({ streamId: this.streamId, ackId, error }, "Error sending keep-alive");
+        }
+    }
+    onAckTimeout(ackId, sentAt) {
+        if (this.disposed)
+            return;
+        // Race guard: handleAck may have processed the ACK between the timeout
+        // being scheduled and this callback running. In that case the ackId is
+        // no longer pending — counting it as missed would falsely escalate.
+        if (!this.pendingAcks.has(ackId))
+            return;
+        this.pendingAcks.delete(ackId);
+        this.missedAcks += 1;
+        const ageMs = this.now() - sentAt;
+        this.logger.warn({ streamId: this.streamId, ackId, missedAcks: this.missedAcks, ageMs }, "Keep-alive ACK timeout");
+        this.callbacks.onKeepAliveMissed?.(ackId, ageMs, this.missedAcks);
+        if (this.missedAcks >= this.maxMissedAcks) {
+            this.logger.error({
+                streamId: this.streamId,
+                missedAcks: this.missedAcks,
+                maxMissedAcks: this.maxMissedAcks,
+            }, "Maximum missed ACKs reached; triggering timeout");
+            void this.callbacks.onTimeout();
+        }
+    }
+    clearPendingAcks() {
+        for (const { timeout } of this.pendingAcks.values()) {
+            clearTimeout(timeout);
+        }
+        this.pendingAcks.clear();
+    }
+    createAckId() {
+        return `a${this.now().toString(36).slice(-5)}`;
+    }
+}
+//# sourceMappingURL=StreamLifecycleController.js.map