@mentra/engine 0.1.0-beta.0

package/build/services/PhonePhotoCoordinator.js

@@ -0,0 +1,326 @@
+/**
+ * PhonePhotoCoordinator — owns local-miniapp takePhoto() end-to-end.
+ *
+ * Architecture:
+ *   miniapp → SDK → LocalMiniappRuntime → photo runtime hook
+ *          → coordinator.takePhoto(packageName, opts)
+ *            ├── (precheck) glasses connected + hasCamera
+ *            ├── cloudClientService.startManagedPhoto → {requestId, uploadUrl, readUrl}  (cloud-v2 runtime presign)
+ *            ├── BluetoothSdk.requestPhoto(requestId, size, uploadUrl, compress, sound)
+ *            └── race:
+ *                  - cloudClientService.awaitManagedPhotoReady(requestId) resolves on photo.ready push
+ *                  - BluetoothSdk.requestPhoto rejects if terminal photo_response is an error
+ *                  - handlePhotoError(requestId, code, message) rejects if MantleManager observes
+ *                    the same BLE photo_response error before the native promise crosses the bridge
+ *
+ * `activeRequests` lets MantleManager's gated `photo_response` listener
+ * short-circuit our long-poll with a typed error (CAMERA_BUSY, BATTERY_LOW,
+ * etc.) instead of waiting 30s for cloud's timeout.
+ */
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal";
+import { cloudClientService } from "./CloudClientService";
+import { isGlassesConnected } from "./GlassesReadiness";
+import { useGlassesStore } from "../stores/glasses";
+/** Map legacy/cloud size names onto the native take_photo enum. */
+function normalizePhotoSize(value) {
+    if (typeof value !== "string")
+        return "medium";
+    switch (value) {
+        case "small":
+            return "low";
+        case "large":
+            return "high";
+        case "full":
+            return "max";
+        default:
+            return ["low", "medium", "high", "max"].includes(value) ? value : "medium";
+    }
+}
+export class PhotoError extends Error {
+    code;
+    stage;
+    transport;
+    constructor(code, message, stage, transport) {
+        super(message);
+        this.code = code;
+        this.stage = stage;
+        this.transport = transport;
+        this.name = "PhotoError";
+    }
+}
+/**
+ * Last-resort ceiling for the complete capture + encode + transfer + upload
+ * pipeline. This must exceed the managed-photo service's 30s ready-push
+ * timeout so the cloud can return its more specific failure first.
+ *
+ * Text mode commonly spends several seconds detecting/cropping the region and
+ * can take another 10s+ to transfer over BLE. A 15s ceiling incorrectly
+ * rejected successful captures while their upload was still in progress.
+ */
+export const CAPTURE_PIPELINE_TIMEOUT_MS = 45_000;
+export const CAMERA_WARM_UP_DEFAULT_DURATION_MS = 15_000;
+export const CAMERA_WARM_UP_MAX_DURATION_MS = 60_000;
+let bleRequestCounter = 0;
+/** Short 4-hex-char correlation id sent over BLE instead of the full cloud
+ *  requestId UUID (wire v2 keeps BLE JSON small). */
+function mintBleRequestId() {
+    bleRequestCounter = (bleRequestCounter + 1) & 0xffff;
+    return bleRequestCounter.toString(16).padStart(4, "0");
+}
+function toNativeCompression(compress) {
+    if (compress === "high")
+        return "heavy";
+    if (compress === "low" || compress === "medium")
+        return "medium";
+    return "none";
+}
+export class PhonePhotoCoordinator {
+    // Cloud requestId → in-flight slot. The gated photo_response listener
+    // (DeviceEventRouter) resolves short BLE ids via bleIdToCloud before calling
+    // owns() / handlePhotoError().
+    activeRequests = new Map();
+    /** Short BLE correlation id (4-char hex) → full cloud requestId. */
+    bleIdToCloud = new Map();
+    activeWarmUps = new Map();
+    async takePhoto(packageName, opts) {
+        // Pre-check: if glasses aren't even connected, the BLE photo command
+        // would be sent into the void and we'd wait 30s for the cloud long-poll
+        // to time out. Fail fast with a typed error.
+        //
+        // We DON'T pre-check `hasCamera` here — the canonical capability data
+        // lives in `getModelCapabilities(deviceModel)` from @mentra/types and
+        // pulling that into this file would add a cross-package import. If a
+        // cameraless device receives the BLE photo command, the glasses-side
+        // handler will return a photo_response error within ~1s and the gated
+        // photo_response listener in MantleManager will short-circuit our
+        // long-poll. Slower but correct.
+        if (!isGlassesConnected(useGlassesStore.getState().connection)) {
+            throw new PhotoError("GLASSES_NOT_CONNECTED", "Glasses are not connected", "command", "ble");
+        }
+        // Text-mode sensor resolution is owned by ASG constants; keep cloud metadata on a stable
+        // high-capacity tier and let the glasses ignore the public size when mode=text.
+        const captureSize = opts.size ?? "medium";
+        // 1) Presign via the cloud-v2 managed-photo service. Local miniapps use
+        //    ONLY the cloud-v2 path: the runtime presigns upload+read URLs and the
+        //    phone (as the device controller) delivers the bytes; the legacy
+        //    backend_url mint is gone from this flow.
+        let requestId;
+        let uploadUrl;
+        let readUrl;
+        const flowStarted = performance.now();
+        try {
+            const presignStarted = performance.now();
+            const r = await cloudClientService.startManagedPhoto({
+                size: opts.mode === "text" ? "max" : captureSize,
+            });
+            const presignMs = Math.round(performance.now() - presignStarted);
+            if (typeof __DEV__ !== "undefined" && __DEV__) {
+                console.debug(`[PhonePhotoCoordinator] presign ${presignMs}ms size=${opts.mode === "text" ? "max" : captureSize} mode=${opts.mode ?? "photo"}`);
+            }
+            requestId = r.requestId;
+            uploadUrl = r.uploadUrl;
+            readUrl = r.readUrl;
+        }
+        catch (err) {
+            throw this.toPhotoError(err, "PHOTO_REQUEST_FAILED", "presign", "cloud-rest");
+        }
+        // 2) Build the outcome Promise FIRST so both resolve+reject handles
+        //    are wired into activeRequests BEFORE any code path can produce
+        //    an error. Without this, a fast BLE photo_response (BATTERY_LOW
+        //    etc.) racing with the Promise constructor could fire
+        //    handlePhotoError() against a no-op rejectFn and silently drop
+        //    the error.
+        // When the managed-photo upload URL is loopback (the local storage provider
+        // reached over `adb reverse`), the glasses cannot reach it over WiFi —
+        // localhost on the glasses is the glasses. Force BLE transfer so the phone
+        // (which CAN reach the reversed runtime) relays the bytes; a public r2/s3
+        // presigned URL stays on "auto".
+        const isLoopbackUpload = /^https?:\/\/(localhost|127\.0\.0\.1|10\.0\.2\.2)\b/.test(uploadUrl);
+        const abort = new AbortController();
+        const bleRequestId = mintBleRequestId();
+        this.bleIdToCloud.set(bleRequestId, requestId);
+        const outcome = new Promise((resolve, reject) => {
+            this.activeRequests.set(requestId, { packageName, abort, resolve, reject });
+        });
+        // Last-resort watchdog for a wedged pipeline. The managed-photo ready push
+        // normally resolves or rejects first; this only prevents an indefinite hang
+        // if both the native terminal response and cloud push disappear.
+        const captureWatchdog = setTimeout(() => {
+            const e = this.activeRequests.get(requestId);
+            if (!e)
+                return;
+            e.abort.abort();
+            e.reject(new PhotoError("CAPTURE_TIMEOUT", "Photo capture did not complete. The take_photo command, media processing, transfer, or upload may have stalled.", "capture", isLoopbackUpload ? "ble" : "wifi"));
+        }, CAPTURE_PIPELINE_TIMEOUT_MS);
+        // Clear the watchdog on BOTH arms. Not `.finally()`: that would mint a new
+        // promise that re-rejects unobserved whenever the photo fails (the caller
+        // only awaits `outcome` itself), i.e. an unhandled rejection per failure.
+        const clearWatchdog = () => clearTimeout(captureWatchdog);
+        void outcome.then(clearWatchdog, clearWatchdog);
+        // 3) Drive glasses over BLE. requestPhoto now resolves at terminal
+        //    photo_response success, so run it beside the cloud poll instead of
+        //    awaiting it before polling. Native defaults transferMethod to "auto"
+        //    (Wi-Fi direct with BLE fallback) unless the miniapp forces BLE.
+        try {
+            void BluetoothSdk.requestPhoto({
+                requestId: bleRequestId,
+                appId: packageName,
+                size: normalizePhotoSize(captureSize),
+                mode: opts.mode ?? "photo",
+                webhookUrl: uploadUrl,
+                authToken: null,
+                ...(isLoopbackUpload || opts.transferMethod === "ble" ? { transferMethod: "ble" } : {}),
+                compress: toNativeCompression(opts.compress),
+                save: opts.saveToGallery ?? false,
+                sound: opts.sound ?? true,
+                exposureTimeNs: opts.exposureTimeNs ?? null,
+                iso: opts.iso,
+                aeExposureDivisor: opts.aeExposureDivisor,
+                isoCap: opts.isoCap,
+                noiseReduction: opts.noiseReduction,
+                edgeEnhancement: opts.edgeEnhancement,
+                mfnr: opts.mfnr,
+                zsl: opts.zsl,
+                ispDigitalGain: opts.ispDigitalGain,
+                ispAnalogGain: opts.ispAnalogGain,
+            }).catch((err) => {
+                const e = this.activeRequests.get(requestId);
+                if (!e)
+                    return;
+                e.abort.abort();
+                e.reject(this.toPhotoError(err, "BLE_SEND_FAILED", "command", "ble"));
+            });
+        }
+        catch (err) {
+            this.activeRequests.delete(requestId);
+            this.bleIdToCloud.delete(bleRequestId);
+            throw this.toPhotoError(err, "BLE_SEND_FAILED", "command", "ble");
+        }
+        // 4) Await the runtime's photo.ready push (replaces the legacy long-poll).
+        //    handlePhotoError races against it and uses the same entry to reject
+        //    first.
+        cloudClientService
+            .awaitManagedPhotoReady(requestId)
+            .then((res) => {
+            const e = this.activeRequests.get(requestId);
+            if (!e)
+                return; // already settled by handlePhotoError
+            e.resolve({ photoUrl: res.readUrl ?? readUrl, mimeType: "image/jpeg", size: -1 });
+        })
+            .catch((err) => {
+            const e = this.activeRequests.get(requestId);
+            if (e)
+                e.reject(this.toPhotoError(err, "POLL_FAILED", "push", "ws"));
+        });
+        try {
+            const result = await outcome;
+            if (typeof __DEV__ !== "undefined" && __DEV__) {
+                console.debug(`[PhonePhotoCoordinator] takePhoto complete ${Math.round(performance.now() - flowStarted)}ms requestId=${requestId}`);
+            }
+            return {
+                photoUrl: result.photoUrl,
+                mimeType: result.mimeType,
+                size: result.size,
+                requestId,
+            };
+        }
+        finally {
+            this.activeRequests.delete(requestId);
+            this.bleIdToCloud.delete(bleRequestId);
+        }
+    }
+    /** Map a short BLE requestId from photo_response back to the cloud requestId. */
+    resolveCloudRequestId(bleOrCloudId) {
+        return this.bleIdToCloud.get(bleOrCloudId) ?? bleOrCloudId;
+    }
+    /**
+     * Pre-warm the glasses camera so the next takePhoto() is near-instant.
+     *
+     * Pure BLE — NO cloud presign, NO upload, NO long-poll. The phone mints and owns
+     * the requestId, sends the warm-up command, and resolves when the camera reports
+     * ready (the native promise resolves on the ready status event).
+     */
+    async warmUpCamera(packageName, opts) {
+        // Pre-check: if glasses aren't connected, the BLE warm-up command would be
+        // sent into the void. Fail fast with a typed error.
+        if (!isGlassesConnected(useGlassesStore.getState().connection)) {
+            throw new PhotoError("GLASSES_NOT_CONNECTED", "Glasses are not connected", "command", "ble");
+        }
+        let lease;
+        try {
+            await this.stopWarmUpForApp(packageName);
+            const requestId = mintBleRequestId();
+            const requestedDuration = typeof opts.durationMs === "number" && Number.isFinite(opts.durationMs) && opts.durationMs > 0
+                ? Math.round(opts.durationMs)
+                : CAMERA_WARM_UP_DEFAULT_DURATION_MS;
+            const durationMs = Math.min(requestedDuration, CAMERA_WARM_UP_MAX_DURATION_MS);
+            lease = { requestId, durationMs };
+            this.activeWarmUps.set(packageName, lease);
+            await BluetoothSdk.warmUpCamera({
+                requestId,
+                size: normalizePhotoSize(opts.size ?? "medium"),
+                mode: opts.mode ?? "photo",
+                exposureTimeNs: opts.exposureTimeNs ?? null,
+                durationMs,
+            });
+            if (this.activeWarmUps.get(packageName) === lease) {
+                lease.expiryTimer = setTimeout(() => {
+                    if (this.activeWarmUps.get(packageName) === lease) {
+                        this.activeWarmUps.delete(packageName);
+                    }
+                }, durationMs);
+            }
+        }
+        catch (err) {
+            if (lease && this.activeWarmUps.get(packageName) === lease) {
+                if (lease.expiryTimer)
+                    clearTimeout(lease.expiryTimer);
+                this.activeWarmUps.delete(packageName);
+            }
+            throw this.toPhotoError(err, "WARM_UP_FAILED", "command", "ble");
+        }
+    }
+    /** Release a miniapp's warm-up even if its original warmUpCamera promise is still opening. */
+    async stopWarmUpForApp(packageName) {
+        const active = this.activeWarmUps.get(packageName);
+        if (!active)
+            return;
+        await BluetoothSdk.stopCameraWarmUp(active.requestId);
+        if (this.activeWarmUps.get(packageName) === active) {
+            this.activeWarmUps.delete(packageName);
+            if (active.expiryTimer)
+                clearTimeout(active.expiryTimer);
+        }
+    }
+    /** True iff this requestId (short BLE id or cloud id) is one we're currently waiting on. */
+    owns(requestId) {
+        const cloudId = this.resolveCloudRequestId(requestId);
+        return this.activeRequests.has(cloudId);
+    }
+    /**
+     * Called by MantleManager's gated photo_response listener when glasses
+     * report an error (BATTERY_LOW, CAMERA_BUSY, etc.) for a phone-owned
+     * requestId. Rejects the in-flight takePhoto Promise immediately.
+     */
+    handlePhotoError(requestId, errorCode, errorMessage) {
+        const cloudId = this.resolveCloudRequestId(requestId);
+        const entry = this.activeRequests.get(cloudId);
+        if (!entry)
+            return;
+        this.bleIdToCloud.delete(requestId);
+        // Abort the in-flight long-poll first so we don't double-resolve.
+        entry.abort.abort();
+        entry.reject(new PhotoError(errorCode || "GLASSES_ERROR", errorMessage || "Glasses error", "capture", "ble"));
+        // cloud-v2 pending photo requests TTL out on their own; nothing to free.
+    }
+    toPhotoError(err, fallbackCode, stage, transport) {
+        if (err instanceof PhotoError)
+            return err;
+        const code = err?.code;
+        const message = err instanceof Error ? err.message : String(err);
+        return new PhotoError(code || fallbackCode, message, stage, transport);
+    }
+}
+// Singleton — coordinator state is process-wide (mirrors phoneStreamCoordinator).
+export const phonePhotoCoordinator = new PhonePhotoCoordinator();
+//# sourceMappingURL=PhonePhotoCoordinator.js.map

package/build/services/PhonePhotoCoordinator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PhonePhotoCoordinator.js","sourceRoot":"","sources":["../../src/services/PhonePhotoCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,YAAY,MAAM,gCAAgC,CAAA;AAEzD,OAAO,EAAC,kBAAkB,EAAC,MAAM,sBAAsB,CAAA;AACvD,OAAO,EAAC,kBAAkB,EAAC,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAC,eAAe,EAAC,MAAM,mBAAmB,CAAA;AAQjD,mEAAmE;AACnE,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAA;IAC9C,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,OAAO;YACV,OAAO,KAAK,CAAA;QACd,KAAK,OAAO;YACV,OAAO,MAAM,CAAA;QACf,KAAK,MAAM;YACT,OAAO,KAAK,CAAA;QACd;YACE,OAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAW,CAAC,QAAQ,CAAC,KAAkB,CAAC,CAAC,CAAC,CAAE,KAAmB,CAAC,CAAC,CAAC,QAAQ,CAAA;IACrH,CAAC;AACH,CAAC;AAoCD,MAAM,OAAO,UAAW,SAAQ,KAAK;IAEjB;IAEA;IACA;IAJlB,YACkB,IAAY,EAC5B,OAAe,EACC,KAAkB,EAClB,SAA0B;QAE1C,KAAK,CAAC,OAAO,CAAC,CAAA;QALE,SAAI,GAAJ,IAAI,CAAQ;QAEZ,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAiB;QAG1C,IAAI,CAAC,IAAI,GAAG,YAAY,CAAA;IAC1B,CAAC;CACF;AASD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,MAAM,CAAA;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,MAAM,CAAA;AACxD,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAA;AAEpD,IAAI,iBAAiB,GAAG,CAAC,CAAA;AAEzB;qDACqD;AACrD,SAAS,gBAAgB;IACvB,iBAAiB,GAAG,CAAC,iBAAiB,GAAG,CAAC,CAAC,GAAG,MAAM,CAAA;IACpD,OAAO,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;AACxD,CAAC;AAED,SAAS,mBAAmB,CAAC,QAA+B;IAC1D,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,OAAO,CAAA;IACvC,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAA;IAChE,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,OAAO,qBAAqB;IAChC,sEAAsE;IACtE,6EAA6E;IAC7E,+BAA+B;IACd,cAAc,GAAG,IAAI,GAAG,EAAyB,CAAA;IAClE,oEAAoE;IACnD,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,aAAa,GAAG,IAAI,GAAG,EAGrC,CAAA;IAEH,KAAK,CAAC,SAAS,CAAC,WAAmB,EAAE,IAAe;QAClD,qEAAqE;QACrE,wEAAwE;QACxE,6CAA6C;QAC7C,EAAE;QACF,sEAAsE;QACtE,sEAAsE;QACtE,qEAAqE;QACrE,qEAAqE;QACrE,sEAAsE;QACtE,kEAAkE;QAClE,iCAAiC;QACjC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,UAAU,CAAC,uBAAuB,EAAE,2BAA2B,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;QAC9F,CAAC;QAED,yFAAyF;QACzF,gFAAgF;QAChF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAA;QAEzC,wEAAwE;QACxE,2EAA2E;QAC3E,qEAAqE;QACrE,8CAA8C;QAC9C,IAAI,SAAiB,CAAA;QACrB,IAAI,SAAiB,CAAA;QACrB,IAAI,OAAe,CAAA;QACnB,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QACrC,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;YACxC,MAAM,CAAC,GAAG,MAAM,kBAAkB,CAAC,iBAAiB,CAAC;gBACnD,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW;aACjD,CAAC,CAAA;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,CAAA;YAChE,IAAI,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,EAAE,CAAC;gBAC9C,OAAO,CAAC,KAAK,CACX,mCAAmC,SAAS,WAAW,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,SAC/F,IAAI,CAAC,IAAI,IAAI,OACf,EAAE,CACH,CAAA;YACH,CAAC;YACD,SAAS,GAAG,CAAC,CAAC,SAAS,CAAA;YACvB,SAAS,GAAG,CAAC,CAAC,SAAS,CAAA;YACvB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAA;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,sBAAsB,EAAE,SAAS,EAAE,YAAY,CAAC,CAAA;QAC/E,CAAC;QAED,oEAAoE;QACpE,oEAAoE;QACpE,oEAAoE;QACpE,0DAA0D;QAC1D,mEAAmE;QACnE,gBAAgB;QAChB,4EAA4E;QAC5E,uEAAuE;QACvE,2EAA2E;QAC3E,0EAA0E;QAC1E,iCAAiC;QACjC,MAAM,gBAAgB,GAAG,oDAAoD,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAE7F,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAA;QACnC,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAA;QACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;QAE9C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,EAAC,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAC,CAAC,CAAA;QAC3E,CAAC,CAAC,CAAA;QAEF,2EAA2E;QAC3E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,eAAe,GAAG,UAAU,CAAC,GAAG,EAAE;YACtC,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAC5C,IAAI,CAAC,CAAC;gBAAE,OAAM;YACd,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YACf,CAAC,CAAC,MAAM,CACN,IAAI,UAAU,CACZ,iBAAiB,EACjB,iHAAiH,EACjH,SAAS,EACT,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAClC,CACF,CAAA;QACH,CAAC,EAAE,2BAA2B,CAAC,CAAA;QAC/B,2EAA2E;QAC3E,0EAA0E;QAC1E,0EAA0E;QAC1E,MAAM,aAAa,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,CAAC,CAAA;QACzD,KAAK,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,CAAA;QAE/C,mEAAmE;QACnE,wEAAwE;QACxE,0EAA0E;QAC1E,qEAAqE;QACrE,IAAI,CAAC;YACH,KAAK,YAAY,CAAC,YAAY,CAAC;gBAC7B,SAAS,EAAE,YAAY;gBACvB,KAAK,EAAE,WAAW;gBAClB,IAAI,EAAE,kBAAkB,CAAC,WAAW,CAAC;gBACrC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,OAAO;gBAC1B,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,IAAI;gBACf,GAAG,CAAC,gBAAgB,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,EAAC,cAAc,EAAE,KAAc,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9F,QAAQ,EAAE,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,KAAK;gBACjC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;gBACzB,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;gBAC3C,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,aAAa,EAAE,IAAI,CAAC,aAAa;aAClC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACf,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBAC5C,IAAI,CAAC,CAAC;oBAAE,OAAM;gBACd,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;gBACf,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,iBAAiB,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAA;YACvE,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;YACtC,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,iBAAiB,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;QACnE,CAAC;QAED,2EAA2E;QAC3E,yEAAyE;QACzE,YAAY;QACZ,kBAAkB;aACf,sBAAsB,CAAC,SAAS,CAAC;aACjC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;YACZ,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAC5C,IAAI,CAAC,CAAC;gBAAE,OAAM,CAAC,sCAAsC;YACrD,CAAC,CAAC,OAAO,CAAC,EAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,IAAI,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,EAAC,CAAC,CAAA;QACjF,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAC5C,IAAI,CAAC;gBAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAA;QACtE,CAAC,CAAC,CAAA;QAEJ,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAA;YAC5B,IAAI,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,EAAE,CAAC;gBAC9C,OAAO,CAAC,KAAK,CACX,8CAA8C,IAAI,CAAC,KAAK,CACtD,WAAW,CAAC,GAAG,EAAE,GAAG,WAAW,CAChC,gBAAgB,SAAS,EAAE,CAC7B,CAAA;YACH,CAAC;YACD,OAAO;gBACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS;aACV,CAAA;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;QACxC,CAAC;IACH,CAAC;IAED,iFAAiF;IACjF,qBAAqB,CAAC,YAAoB;QACxC,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,YAAY,CAAA;IAC5D,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY,CAChB,WAAmB,EACnB,IAKC;QAED,2EAA2E;QAC3E,oDAAoD;QACpD,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,UAAU,CAAC,uBAAuB,EAAE,2BAA2B,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;QAC9F,CAAC;QAED,IAAI,KAAuG,CAAA;QAC3G,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACxC,MAAM,SAAS,GAAG,gBAAgB,EAAE,CAAA;YACpC,MAAM,iBAAiB,GACrB,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC;gBAC5F,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC7B,CAAC,CAAC,kCAAkC,CAAA;YACxC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,8BAA8B,CAAC,CAAA;YAC9E,KAAK,GAAG,EAAC,SAAS,EAAE,UAAU,EAAC,CAAA;YAC/B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;YAC1C,MAAM,YAAY,CAAC,YAAY,CAAC;gBAC9B,SAAS;gBACT,IAAI,EAAE,kBAAkB,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC;gBAC/C,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,OAAO;gBAC1B,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;gBAC3C,UAAU;aACX,CAAC,CAAA;YACF,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,KAAK,EAAE,CAAC;gBAClD,KAAK,CAAC,WAAW,GAAG,UAAU,CAAC,GAAG,EAAE;oBAClC,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,KAAK,EAAE,CAAC;wBAClD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;oBACxC,CAAC;gBACH,CAAC,EAAE,UAAU,CAAC,CAAA;YAChB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,KAAK,EAAE,CAAC;gBAC3D,IAAI,KAAK,CAAC,WAAW;oBAAE,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;gBACtD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YACxC,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,gBAAgB,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;QAClE,CAAC;IACH,CAAC;IAED,8FAA8F;IAC9F,KAAK,CAAC,gBAAgB,CAAC,WAAmB;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QAClD,IAAI,CAAC,MAAM;YAAE,OAAM;QACnB,MAAM,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACrD,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,MAAM,EAAE,CAAC;YACnD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YACtC,IAAI,MAAM,CAAC,WAAW;gBAAE,YAAY,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QAC1D,CAAC;IACH,CAAC;IAED,4FAA4F;IAC5F,IAAI,CAAC,SAAiB;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAA;QACrD,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IACzC,CAAC;IAED;;;;OAIG;IACH,gBAAgB,CAAC,SAAiB,EAAE,SAAiB,EAAE,YAAoB;QACzE,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAA;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC9C,IAAI,CAAC,KAAK;YAAE,OAAM;QAClB,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACnC,kEAAkE;QAClE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACnB,KAAK,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,SAAS,IAAI,eAAe,EAAE,YAAY,IAAI,eAAe,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAA;QAC7G,yEAAyE;IAC3E,CAAC;IAEO,YAAY,CAAC,GAAY,EAAE,YAAoB,EAAE,KAAkB,EAAE,SAA0B;QACrG,IAAI,GAAG,YAAY,UAAU;YAAE,OAAO,GAAG,CAAA;QACzC,MAAM,IAAI,GAAI,GAAuB,EAAE,IAAI,CAAA;QAC3C,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAChE,OAAO,IAAI,UAAU,CAAC,IAAI,IAAI,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAA;IACxE,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,EAAE,CAAA","sourcesContent":["/**\n * PhonePhotoCoordinator — owns local-miniapp takePhoto() end-to-end.\n *\n * Architecture:\n *   miniapp → SDK → LocalMiniappRuntime → photo runtime hook\n *          → coordinator.takePhoto(packageName, opts)\n *            ├── (precheck) glasses connected + hasCamera\n *            ├── cloudClientService.startManagedPhoto → {requestId, uploadUrl, readUrl}  (cloud-v2 runtime presign)\n *            ├── BluetoothSdk.requestPhoto(requestId, size, uploadUrl, compress, sound)\n *            └── race:\n *                  - cloudClientService.awaitManagedPhotoReady(requestId) resolves on photo.ready push\n *                  - BluetoothSdk.requestPhoto rejects if terminal photo_response is an error\n *                  - handlePhotoError(requestId, code, message) rejects if MantleManager observes\n *                    the same BLE photo_response error before the native promise crosses the bridge\n *\n * `activeRequests` lets MantleManager's gated `photo_response` listener\n * short-circuit our long-poll with a typed error (CAMERA_BUSY, BATTERY_LOW,\n * etc.) instead of waiting 30s for cloud's timeout.\n */\n\nimport BluetoothSdk from \"@mentra/bluetooth-sdk/internal\"\nimport type {PhotoSize} from \"@mentra/bluetooth-sdk/internal\"\nimport {cloudClientService} from \"./CloudClientService\"\nimport {isGlassesConnected} from \"./GlassesReadiness\"\nimport {useGlassesStore} from \"../stores/glasses\"\n\ninterface PhotoResult {\n  photoUrl: string\n  mimeType: string\n  size: number\n}\n\n/** Map legacy/cloud size names onto the native take_photo enum. */\nfunction normalizePhotoSize(value: unknown): PhotoSize {\n  if (typeof value !== \"string\") return \"medium\"\n  switch (value) {\n    case \"small\":\n      return \"low\"\n    case \"large\":\n      return \"high\"\n    case \"full\":\n      return \"max\"\n    default:\n      return ([\"low\", \"medium\", \"high\", \"max\"] as const).includes(value as PhotoSize) ? (value as PhotoSize) : \"medium\"\n  }\n}\n\nexport interface PhotoOpts {\n  /** Legacy cloud size names are normalized before the native take_photo command. */\n  size?: \"low\" | \"medium\" | \"high\" | \"max\" | \"small\" | \"large\" | \"full\"\n  mode?: \"photo\" | \"text\"\n  /** Force phone-relayed BLE delivery, or leave native on its auto Wi-Fi/BLE policy. */\n  transferMethod?: \"auto\" | \"ble\"\n  compress?: \"none\" | \"low\" | \"medium\" | \"high\"\n  sound?: boolean\n  saveToGallery?: boolean\n  exposureTimeNs?: number\n  iso?: number | null\n  aeExposureDivisor?: number\n  isoCap?: number\n  noiseReduction?: boolean\n  edgeEnhancement?: boolean\n  mfnr?: boolean\n  zsl?: boolean\n  ispDigitalGain?: number\n  ispAnalogGain?: string\n}\n\nexport interface PhotoTaken {\n  photoUrl: string\n  mimeType: string\n  size: number\n  requestId: string\n}\n\n/** Pipeline stage a photo request failed at — surfaced to the miniapp so a dev\n *  sees exactly where it broke, not just a flattened message. */\nexport type PhotoStage = \"presign\" | \"command\" | \"capture\" | \"upload\" | \"push\"\n/** Transport in play at the point of failure. */\nexport type PhotoTransport = \"cloud-rest\" | \"ble\" | \"wifi\" | \"ws\"\n\nexport class PhotoError extends Error {\n  constructor(\n    public readonly code: string,\n    message: string,\n    public readonly stage?: PhotoStage,\n    public readonly transport?: PhotoTransport,\n  ) {\n    super(message)\n    this.name = \"PhotoError\"\n  }\n}\n\ninterface ActiveRequest {\n  packageName: string\n  abort: AbortController\n  resolve: (r: PhotoResult) => void\n  reject: (err: Error) => void\n}\n\n/**\n * Last-resort ceiling for the complete capture + encode + transfer + upload\n * pipeline. This must exceed the managed-photo service's 30s ready-push\n * timeout so the cloud can return its more specific failure first.\n *\n * Text mode commonly spends several seconds detecting/cropping the region and\n * can take another 10s+ to transfer over BLE. A 15s ceiling incorrectly\n * rejected successful captures while their upload was still in progress.\n */\nexport const CAPTURE_PIPELINE_TIMEOUT_MS = 45_000\nexport const CAMERA_WARM_UP_DEFAULT_DURATION_MS = 15_000\nexport const CAMERA_WARM_UP_MAX_DURATION_MS = 60_000\n\nlet bleRequestCounter = 0\n\n/** Short 4-hex-char correlation id sent over BLE instead of the full cloud\n *  requestId UUID (wire v2 keeps BLE JSON small). */\nfunction mintBleRequestId(): string {\n  bleRequestCounter = (bleRequestCounter + 1) & 0xffff\n  return bleRequestCounter.toString(16).padStart(4, \"0\")\n}\n\nfunction toNativeCompression(compress: PhotoOpts[\"compress\"]): \"none\" | \"medium\" | \"heavy\" {\n  if (compress === \"high\") return \"heavy\"\n  if (compress === \"low\" || compress === \"medium\") return \"medium\"\n  return \"none\"\n}\n\nexport class PhonePhotoCoordinator {\n  // Cloud requestId → in-flight slot. The gated photo_response listener\n  // (DeviceEventRouter) resolves short BLE ids via bleIdToCloud before calling\n  // owns() / handlePhotoError().\n  private readonly activeRequests = new Map<string, ActiveRequest>()\n  /** Short BLE correlation id (4-char hex) → full cloud requestId. */\n  private readonly bleIdToCloud = new Map<string, string>()\n  private readonly activeWarmUps = new Map<\n    string,\n    {requestId: string; durationMs: number; expiryTimer?: ReturnType<typeof setTimeout>}\n  >()\n\n  async takePhoto(packageName: string, opts: PhotoOpts): Promise<PhotoTaken> {\n    // Pre-check: if glasses aren't even connected, the BLE photo command\n    // would be sent into the void and we'd wait 30s for the cloud long-poll\n    // to time out. Fail fast with a typed error.\n    //\n    // We DON'T pre-check `hasCamera` here — the canonical capability data\n    // lives in `getModelCapabilities(deviceModel)` from @mentra/types and\n    // pulling that into this file would add a cross-package import. If a\n    // cameraless device receives the BLE photo command, the glasses-side\n    // handler will return a photo_response error within ~1s and the gated\n    // photo_response listener in MantleManager will short-circuit our\n    // long-poll. Slower but correct.\n    if (!isGlassesConnected(useGlassesStore.getState().connection)) {\n      throw new PhotoError(\"GLASSES_NOT_CONNECTED\", \"Glasses are not connected\", \"command\", \"ble\")\n    }\n\n    // Text-mode sensor resolution is owned by ASG constants; keep cloud metadata on a stable\n    // high-capacity tier and let the glasses ignore the public size when mode=text.\n    const captureSize = opts.size ?? \"medium\"\n\n    // 1) Presign via the cloud-v2 managed-photo service. Local miniapps use\n    //    ONLY the cloud-v2 path: the runtime presigns upload+read URLs and the\n    //    phone (as the device controller) delivers the bytes; the legacy\n    //    backend_url mint is gone from this flow.\n    let requestId: string\n    let uploadUrl: string\n    let readUrl: string\n    const flowStarted = performance.now()\n    try {\n      const presignStarted = performance.now()\n      const r = await cloudClientService.startManagedPhoto({\n        size: opts.mode === \"text\" ? \"max\" : captureSize,\n      })\n      const presignMs = Math.round(performance.now() - presignStarted)\n      if (typeof __DEV__ !== \"undefined\" && __DEV__) {\n        console.debug(\n          `[PhonePhotoCoordinator] presign ${presignMs}ms size=${opts.mode === \"text\" ? \"max\" : captureSize} mode=${\n            opts.mode ?? \"photo\"\n          }`,\n        )\n      }\n      requestId = r.requestId\n      uploadUrl = r.uploadUrl\n      readUrl = r.readUrl\n    } catch (err) {\n      throw this.toPhotoError(err, \"PHOTO_REQUEST_FAILED\", \"presign\", \"cloud-rest\")\n    }\n\n    // 2) Build the outcome Promise FIRST so both resolve+reject handles\n    //    are wired into activeRequests BEFORE any code path can produce\n    //    an error. Without this, a fast BLE photo_response (BATTERY_LOW\n    //    etc.) racing with the Promise constructor could fire\n    //    handlePhotoError() against a no-op rejectFn and silently drop\n    //    the error.\n    // When the managed-photo upload URL is loopback (the local storage provider\n    // reached over `adb reverse`), the glasses cannot reach it over WiFi —\n    // localhost on the glasses is the glasses. Force BLE transfer so the phone\n    // (which CAN reach the reversed runtime) relays the bytes; a public r2/s3\n    // presigned URL stays on \"auto\".\n    const isLoopbackUpload = /^https?:\\/\\/(localhost|127\\.0\\.0\\.1|10\\.0\\.2\\.2)\\b/.test(uploadUrl)\n\n    const abort = new AbortController()\n    const bleRequestId = mintBleRequestId()\n    this.bleIdToCloud.set(bleRequestId, requestId)\n\n    const outcome = new Promise<PhotoResult>((resolve, reject) => {\n      this.activeRequests.set(requestId, {packageName, abort, resolve, reject})\n    })\n\n    // Last-resort watchdog for a wedged pipeline. The managed-photo ready push\n    // normally resolves or rejects first; this only prevents an indefinite hang\n    // if both the native terminal response and cloud push disappear.\n    const captureWatchdog = setTimeout(() => {\n      const e = this.activeRequests.get(requestId)\n      if (!e) return\n      e.abort.abort()\n      e.reject(\n        new PhotoError(\n          \"CAPTURE_TIMEOUT\",\n          \"Photo capture did not complete. The take_photo command, media processing, transfer, or upload may have stalled.\",\n          \"capture\",\n          isLoopbackUpload ? \"ble\" : \"wifi\",\n        ),\n      )\n    }, CAPTURE_PIPELINE_TIMEOUT_MS)\n    // Clear the watchdog on BOTH arms. Not `.finally()`: that would mint a new\n    // promise that re-rejects unobserved whenever the photo fails (the caller\n    // only awaits `outcome` itself), i.e. an unhandled rejection per failure.\n    const clearWatchdog = () => clearTimeout(captureWatchdog)\n    void outcome.then(clearWatchdog, clearWatchdog)\n\n    // 3) Drive glasses over BLE. requestPhoto now resolves at terminal\n    //    photo_response success, so run it beside the cloud poll instead of\n    //    awaiting it before polling. Native defaults transferMethod to \"auto\"\n    //    (Wi-Fi direct with BLE fallback) unless the miniapp forces BLE.\n    try {\n      void BluetoothSdk.requestPhoto({\n        requestId: bleRequestId,\n        appId: packageName,\n        size: normalizePhotoSize(captureSize),\n        mode: opts.mode ?? \"photo\",\n        webhookUrl: uploadUrl,\n        authToken: null,\n        ...(isLoopbackUpload || opts.transferMethod === \"ble\" ? {transferMethod: \"ble\" as const} : {}),\n        compress: toNativeCompression(opts.compress),\n        save: opts.saveToGallery ?? false,\n        sound: opts.sound ?? true,\n        exposureTimeNs: opts.exposureTimeNs ?? null,\n        iso: opts.iso,\n        aeExposureDivisor: opts.aeExposureDivisor,\n        isoCap: opts.isoCap,\n        noiseReduction: opts.noiseReduction,\n        edgeEnhancement: opts.edgeEnhancement,\n        mfnr: opts.mfnr,\n        zsl: opts.zsl,\n        ispDigitalGain: opts.ispDigitalGain,\n        ispAnalogGain: opts.ispAnalogGain,\n      }).catch((err) => {\n        const e = this.activeRequests.get(requestId)\n        if (!e) return\n        e.abort.abort()\n        e.reject(this.toPhotoError(err, \"BLE_SEND_FAILED\", \"command\", \"ble\"))\n      })\n    } catch (err) {\n      this.activeRequests.delete(requestId)\n      this.bleIdToCloud.delete(bleRequestId)\n      throw this.toPhotoError(err, \"BLE_SEND_FAILED\", \"command\", \"ble\")\n    }\n\n    // 4) Await the runtime's photo.ready push (replaces the legacy long-poll).\n    //    handlePhotoError races against it and uses the same entry to reject\n    //    first.\n    cloudClientService\n      .awaitManagedPhotoReady(requestId)\n      .then((res) => {\n        const e = this.activeRequests.get(requestId)\n        if (!e) return // already settled by handlePhotoError\n        e.resolve({photoUrl: res.readUrl ?? readUrl, mimeType: \"image/jpeg\", size: -1})\n      })\n      .catch((err) => {\n        const e = this.activeRequests.get(requestId)\n        if (e) e.reject(this.toPhotoError(err, \"POLL_FAILED\", \"push\", \"ws\"))\n      })\n\n    try {\n      const result = await outcome\n      if (typeof __DEV__ !== \"undefined\" && __DEV__) {\n        console.debug(\n          `[PhonePhotoCoordinator] takePhoto complete ${Math.round(\n            performance.now() - flowStarted,\n          )}ms requestId=${requestId}`,\n        )\n      }\n      return {\n        photoUrl: result.photoUrl,\n        mimeType: result.mimeType,\n        size: result.size,\n        requestId,\n      }\n    } finally {\n      this.activeRequests.delete(requestId)\n      this.bleIdToCloud.delete(bleRequestId)\n    }\n  }\n\n  /** Map a short BLE requestId from photo_response back to the cloud requestId. */\n  resolveCloudRequestId(bleOrCloudId: string): string {\n    return this.bleIdToCloud.get(bleOrCloudId) ?? bleOrCloudId\n  }\n\n  /**\n   * Pre-warm the glasses camera so the next takePhoto() is near-instant.\n   *\n   * Pure BLE — NO cloud presign, NO upload, NO long-poll. The phone mints and owns\n   * the requestId, sends the warm-up command, and resolves when the camera reports\n   * ready (the native promise resolves on the ready status event).\n   */\n  async warmUpCamera(\n    packageName: string,\n    opts: {\n      size?: \"low\" | \"medium\" | \"high\" | \"max\"\n      mode?: \"photo\" | \"text\"\n      exposureTimeNs?: number\n      durationMs?: number\n    },\n  ): Promise<void> {\n    // Pre-check: if glasses aren't connected, the BLE warm-up command would be\n    // sent into the void. Fail fast with a typed error.\n    if (!isGlassesConnected(useGlassesStore.getState().connection)) {\n      throw new PhotoError(\"GLASSES_NOT_CONNECTED\", \"Glasses are not connected\", \"command\", \"ble\")\n    }\n\n    let lease: {requestId: string; durationMs: number; expiryTimer?: ReturnType<typeof setTimeout>} | undefined\n    try {\n      await this.stopWarmUpForApp(packageName)\n      const requestId = mintBleRequestId()\n      const requestedDuration =\n        typeof opts.durationMs === \"number\" && Number.isFinite(opts.durationMs) && opts.durationMs > 0\n          ? Math.round(opts.durationMs)\n          : CAMERA_WARM_UP_DEFAULT_DURATION_MS\n      const durationMs = Math.min(requestedDuration, CAMERA_WARM_UP_MAX_DURATION_MS)\n      lease = {requestId, durationMs}\n      this.activeWarmUps.set(packageName, lease)\n      await BluetoothSdk.warmUpCamera({\n        requestId,\n        size: normalizePhotoSize(opts.size ?? \"medium\"),\n        mode: opts.mode ?? \"photo\",\n        exposureTimeNs: opts.exposureTimeNs ?? null,\n        durationMs,\n      })\n      if (this.activeWarmUps.get(packageName) === lease) {\n        lease.expiryTimer = setTimeout(() => {\n          if (this.activeWarmUps.get(packageName) === lease) {\n            this.activeWarmUps.delete(packageName)\n          }\n        }, durationMs)\n      }\n    } catch (err) {\n      if (lease && this.activeWarmUps.get(packageName) === lease) {\n        if (lease.expiryTimer) clearTimeout(lease.expiryTimer)\n        this.activeWarmUps.delete(packageName)\n      }\n      throw this.toPhotoError(err, \"WARM_UP_FAILED\", \"command\", \"ble\")\n    }\n  }\n\n  /** Release a miniapp's warm-up even if its original warmUpCamera promise is still opening. */\n  async stopWarmUpForApp(packageName: string): Promise<void> {\n    const active = this.activeWarmUps.get(packageName)\n    if (!active) return\n    await BluetoothSdk.stopCameraWarmUp(active.requestId)\n    if (this.activeWarmUps.get(packageName) === active) {\n      this.activeWarmUps.delete(packageName)\n      if (active.expiryTimer) clearTimeout(active.expiryTimer)\n    }\n  }\n\n  /** True iff this requestId (short BLE id or cloud id) is one we're currently waiting on. */\n  owns(requestId: string): boolean {\n    const cloudId = this.resolveCloudRequestId(requestId)\n    return this.activeRequests.has(cloudId)\n  }\n\n  /**\n   * Called by MantleManager's gated photo_response listener when glasses\n   * report an error (BATTERY_LOW, CAMERA_BUSY, etc.) for a phone-owned\n   * requestId. Rejects the in-flight takePhoto Promise immediately.\n   */\n  handlePhotoError(requestId: string, errorCode: string, errorMessage: string): void {\n    const cloudId = this.resolveCloudRequestId(requestId)\n    const entry = this.activeRequests.get(cloudId)\n    if (!entry) return\n    this.bleIdToCloud.delete(requestId)\n    // Abort the in-flight long-poll first so we don't double-resolve.\n    entry.abort.abort()\n    entry.reject(new PhotoError(errorCode || \"GLASSES_ERROR\", errorMessage || \"Glasses error\", \"capture\", \"ble\"))\n    // cloud-v2 pending photo requests TTL out on their own; nothing to free.\n  }\n\n  private toPhotoError(err: unknown, fallbackCode: string, stage?: PhotoStage, transport?: PhotoTransport): PhotoError {\n    if (err instanceof PhotoError) return err\n    const code = (err as {code?: string})?.code\n    const message = err instanceof Error ? err.message : String(err)\n    return new PhotoError(code || fallbackCode, message, stage, transport)\n  }\n}\n\n// Singleton — coordinator state is process-wide (mirrors phoneStreamCoordinator).\nexport const phonePhotoCoordinator = new PhonePhotoCoordinator()\n"]}

package/build/services/PhoneStreamCoordinator.d.ts

@@ -0,0 +1,168 @@
+/**
+ * PhoneStreamCoordinator — owns all local-miniapp streaming on the phone.
+ *
+ * Architecture:
+ *   miniapp → SDK → LocalMiniappRuntime → coordinator
+ *           coordinator → BluetoothSdk (BLE → glasses publisher)
+ *           coordinator ↔ cloudStreamApi (managed only, cloud-v2 runtime provisioning)
+ *           coordinator ↔ StreamLifecycleController (keep-alive heartbeat)
+ *           coordinator → status listeners → routed back to miniapp(s)
+ *
+ * Single-stream constraint:
+ *   At most ONE stream is active across all miniapps. The exception is that
+ *   multiple miniapps can subscribe to a single managed stream that's already
+ *   running — Cloudflare muxes one ingest into many viewers, so subscribers
+ *   share the same playback URLs. Refcounted; teardown happens when the last
+ *   subscriber releases.
+ *
+ * Status routing:
+ *   Glasses publisher status arrives over BLE as `stream_status` events with
+ *   a `streamId`. The coordinator's `owns(streamId)` lookup lets MantleManager
+ *   route phone-owned status events here (not to cloud). For managed streams,
+ *   we additionally poll Cloudflare's live-input status every 5s to surface
+ *   what the OTHER end of the pipe sees.
+ *
+ * Important: this is the ONLY place that mints `streamId`s for phone-owned
+ * streams. We use a `phone-` prefix so they're trivially distinguishable from
+ * cloud-minted IDs in logs and from cloud-SDK app streams that flow through
+ * the legacy path.
+ */
+import type { KeepAliveAckEvent, StreamResolvedConfig, StreamStartRequest, StreamStatusEvent } from "@mentra/bluetooth-sdk/internal";
+import { type RestreamDestinationInput } from "./cloudStreamApi";
+/**
+ * Default cadence + thresholds. Exposed via {@link CoordinatorTimings} so tests
+ * can shorten them; production code should never override these.
+ */
+declare const DEFAULT_TIMINGS: {
+    readonly keepAliveIntervalMs: 15000;
+    readonly ackTimeoutMs: 10000;
+    readonly maxMissedAcks: 3;
+    readonly cloudflareStatusPollMs: 5000;
+    readonly hlsReadinessInitialDelayMs: 5000;
+    readonly hlsReadinessPollMs: 2000;
+    readonly hlsReadinessMaxAttempts: 30;
+};
+type TimingConfig = {
+    [K in keyof typeof DEFAULT_TIMINGS]: number;
+};
+export type CoordinatorTimings = Partial<TimingConfig>;
+export interface StartUnmanagedOptions {
+    streamUrl: string;
+    video?: StreamStartRequest["video"];
+    audio?: StreamStartRequest["audio"];
+    sound?: boolean;
+}
+export interface StartManagedOptions {
+    restreamDestinations?: RestreamDestinationInput[];
+    video?: StreamStartRequest["video"];
+    audio?: StreamStartRequest["audio"];
+    sound?: boolean;
+    /**
+     * Ingest protocol preference — a real latency/durability trade on Cloudflare:
+     *   - "srt" (default): SRT ingest -> LL-HLS playback (~10-20s glass-to-screen),
+     *     with shareable HLS/DASH URLs and automatic recording.
+     *   - "whip": WebRTC ingest -> WHEP playback (<1s glass-to-screen), but NO
+     *     HLS/DASH playback and NO recording (Cloudflare limitation; the returned
+     *     hlsUrl will serve 204 forever).
+     */
+    ingest?: "srt" | "whip";
+}
+export interface StreamPublisherStartResult {
+    streamId: string;
+    status: string;
+    resolvedConfig?: StreamResolvedConfig;
+}
+export interface ManagedStartResult extends StreamPublisherStartResult {
+    liveInputId: string;
+    /** Playback mode this stream supports: "hls" (SRT/RTMP ingest — use hlsUrl/
+     *  dashUrl, recording on) or "webrtc" (WHIP ingest — use webrtcUrl/WHEP,
+     *  sub-second, no HLS, no recording). */
+    mode: "hls" | "webrtc";
+    hlsUrl: string;
+    dashUrl: string;
+    webrtcUrl?: string;
+}
+export type StreamStatusUpdate = {
+    streamId: string;
+    source: "glasses" | "cloudflare" | "coordinator";
+    status: string;
+    data?: Record<string, unknown>;
+};
+export type StatusSubscriber = (packageName: string, update: StreamStatusUpdate) => void;
+export declare class StreamConflictError extends Error {
+    readonly code: string;
+    /** Pipeline stage that failed (provision | command | publish | playback). */
+    readonly stage?: string | undefined;
+    /** Transport in play at the failure (cloud-rest | ble | wifi). */
+    readonly transport?: string | undefined;
+    constructor(code: string, message: string, 
+    /** Pipeline stage that failed (provision | command | publish | playback). */
+    stage?: string | undefined, 
+    /** Transport in play at the failure (cloud-rest | ble | wifi). */
+    transport?: string | undefined);
+}
+export declare class PhoneStreamCoordinator {
+    private current;
+    private lifecycle;
+    private statusSubscriber;
+    private idCounter;
+    private readonly timings;
+    /**
+     * Serializes state transitions (start, stop, teardown). Without it, a
+     * second `start*` racing with the first can pass the `this.current === null`
+     * pre-check while the first is still awaiting its provision/BLE work, and
+     * end up provisioning two separate streams. A teardown racing with a start
+     * can clear `current` mid-stop and let the start fire BLE writes that
+     * collide with the in-flight stopStream.
+     */
+    private inFlight;
+    /** Drop identical stream_status fanouts within one session. */
+    private lastFanoutSignature;
+    /** Send full resolvedConfig only once per stream session. */
+    private resolvedConfigForwarded;
+    constructor(timings?: CoordinatorTimings);
+    /**
+     * Run `work` under the transition lock. Each call awaits the previous
+     * transition before re-evaluating preconditions, so e.g. two concurrent
+     * `startManaged` calls are observed sequentially.
+     */
+    private runExclusive;
+    /**
+     * Register the function that routes status updates to miniapps.
+     * LocalMiniappRuntime wires this so updates become EVENT envelopes on the
+     * `stream_status` stream for every subscribing miniapp.
+     */
+    setStatusSubscriber(cb: StatusSubscriber): void;
+    /**
+     * True when a phone-owned stream currently uses this streamId. For managed
+     * streams, all subscribers share the same streamId so a single equality
+     * check covers the multi-subscriber case.
+     */
+    owns(streamId: string): boolean;
+    startUnmanaged(packageName: string, opts: StartUnmanagedOptions): Promise<StreamPublisherStartResult>;
+    startManaged(packageName: string, opts: StartManagedOptions): Promise<ManagedStartResult>;
+    stop(packageName: string, streamId?: string): Promise<void>;
+    /**
+     * Called by MantleManager when a `stream_status` event arrives from glasses
+     * and the registry says it's phone-owned.
+     */
+    handleGlassesStatus(event: StreamStatusEvent): void;
+    /** Called by MantleManager when a phone-owned keep_alive_ack arrives. */
+    handleKeepAliveAck(event: KeepAliveAckEvent): void;
+    private mintId;
+    private startLifecycle;
+    private startCloudflareStatusPoll;
+    private startHlsReadinessPoll;
+    private fanout;
+    /**
+     * Run teardown of the currently-active stream. Caller must hold the
+     * transition lock (see {@link runExclusive}). Keeps `this.current`
+     * populated until BluetoothSdk.stopStream resolves so a concurrent caller
+     * waiting on the lock can't claim the slot mid-stop and have its
+     * startStream BLE write collide with our in-flight stopStream.
+     */
+    private teardownLocked;
+}
+export declare const phoneStreamCoordinator: PhoneStreamCoordinator;
+export {};
+//# sourceMappingURL=PhoneStreamCoordinator.d.ts.map

package/build/services/PhoneStreamCoordinator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PhoneStreamCoordinator.d.ts","sourceRoot":"","sources":["../../src/services/PhoneStreamCoordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,gCAAgC,CAAA;AAMvC,OAAO,EAML,KAAK,wBAAwB,EAC9B,MAAM,kBAAkB,CAAA;AAEzB;;;GAGG;AACH,QAAA,MAAM,eAAe;;;;;;;;CASX,CAAA;AAEV,KAAK,YAAY,GAAG;KAAE,CAAC,IAAI,MAAM,OAAO,eAAe,GAAG,MAAM;CAAC,CAAA;AACjE,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;AAetD,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,CAAC,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAA;IACnC,KAAK,CAAC,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAA;IACnC,KAAK,CAAC,EAAE,OAAO,CAAA;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,oBAAoB,CAAC,EAAE,wBAAwB,EAAE,CAAA;IACjD,KAAK,CAAC,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAA;IACnC,KAAK,CAAC,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAA;IACnC,KAAK,CAAC,EAAE,OAAO,CAAA;IACf;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,cAAc,CAAC,EAAE,oBAAoB,CAAA;CACtC;AAED,MAAM,WAAW,kBAAmB,SAAQ,0BAA0B;IACpE,WAAW,EAAE,MAAM,CAAA;IACnB;;6CAEyC;IACzC,IAAI,EAAE,KAAK,GAAG,QAAQ,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,GAAG,YAAY,GAAG,aAAa,CAAA;IAChD,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,KAAK,IAAI,CAAA;AAkCxF,qBAAa,mBAAoB,SAAQ,KAAK;aAE1B,IAAI,EAAE,MAAM;IAE5B,6EAA6E;aAC7D,KAAK,CAAC,EAAE,MAAM;IAC9B,kEAAkE;aAClD,SAAS,CAAC,EAAE,MAAM;gBALlB,IAAI,EAAE,MAAM,EAC5B,OAAO,EAAE,MAAM;IACf,6EAA6E;IAC7D,KAAK,CAAC,EAAE,MAAM,YAAA;IAC9B,kEAAkE;IAClD,SAAS,CAAC,EAAE,MAAM,YAAA;CAKrC;AAkBD,qBAAa,sBAAsB;IACjC,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,gBAAgB,CAAgC;IACxD,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAmC;IACnD,+DAA+D;IAC/D,OAAO,CAAC,mBAAmB,CAAsB;IACjD,6DAA6D;IAC7D,OAAO,CAAC,uBAAuB,CAAQ;gBAE3B,OAAO,GAAE,kBAAuB;IAI5C;;;;OAIG;YACW,YAAY;IAY1B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,EAAE,gBAAgB,GAAG,IAAI;IAI/C;;;;OAIG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIzB,cAAc,CAClB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,0BAA0B,CAAC;IA+ChC,YAAY,CAChB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC,kBAAkB,CAAC;IAuGxB,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBjE;;;OAGG;IACH,mBAAmB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IA2CnD,yEAAyE;IACzE,kBAAkB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IASlD,OAAO,CAAC,MAAM;IAKd,OAAO,CAAC,cAAc;IAqCtB,OAAO,CAAC,yBAAyB;IAoEjC,OAAO,CAAC,qBAAqB;IA+D7B,OAAO,CAAC,MAAM;IAed;;;;;;OAMG;YACW,cAAc;CA4C7B;AAgDD,eAAO,MAAM,sBAAsB,wBAA+B,CAAA"}