@mentra/engine 0.1.0-dev.0 → 3.1.0-dev.4

package/src/services/PhoneCameraFovCoordinator.ts

@@ -0,0 +1,210 @@
+import BluetoothSdk, {type CameraFovRequest, type CameraFovResult} from "@mentra/bluetooth-sdk/internal"
+
+export const CAMERA_FOV_OVERRIDE_TTL_MS = 300_000
+export const CAMERA_FOV_OVERRIDE_REFRESH_MS = 120_000
+// Legacy ASG clients restart the camera HAL for an FOV update but cannot acknowledge
+// when it is usable again. The ASG cooldown is 5s; retain a small settling margin so
+// miniapps cannot submit a photo against the stale camera service.
+const LEGACY_CAMERA_RESTART_SETTLE_MS = 7_000
+
+interface OverrideEntry {
+  leaseId: string
+  fov: number
+  roiPosition: "center" | "bottom" | "top"
+  order: number
+}
+
+let leaseCounter = 0
+
+function mintLeaseId(): string {
+  leaseCounter = (leaseCounter + 1) & 0xffff
+  return `fov-${leaseCounter.toString(16).padStart(4, "0")}`
+}
+
+function isLegacyFovCompatibilityError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error)
+  return /timed out waiting for glasses response|not available|unknown method/i.test(message)
+}
+
+/**
+ * Serializes miniapp-owned FOV/ROI overrides. The last live owner wins; releasing it applies the
+ * previous owner or asks ASG to restore its persistent base. ASG refreshes are no-restart TTL
+ * extensions for crash safety.
+ */
+export class PhoneCameraFovCoordinator {
+  private readonly overrides = new Map<string, OverrideEntry>()
+  private effectiveLeaseId: string | undefined
+  private sequence = 0
+  private queue: Promise<unknown> = Promise.resolve()
+  private refreshTimer: ReturnType<typeof setTimeout> | undefined
+  private legacyMode = false
+
+  constructor(private readonly legacyRestartSettleMs = LEGACY_CAMERA_RESTART_SETTLE_MS) {}
+
+  /** Report-safe FOV ownership snapshot for incident diagnostics. */
+  getDiagnosticSnapshot(): Record<string, unknown> {
+    return {
+      owners: [...this.overrides.entries()]
+        .sort(([, a], [, b]) => b.order - a.order)
+        .map(([packageName, entry]) => ({
+          packageName,
+          effective: entry.leaseId === this.effectiveLeaseId,
+          fov: entry.fov,
+          roiPosition: entry.roiPosition,
+        })),
+      legacyMode: this.legacyMode,
+    }
+  }
+
+  setOverride(packageName: string, request: CameraFovRequest): Promise<CameraFovResult> {
+    return this.enqueue(async () => {
+      const previous = this.overrides.get(packageName)
+      const leaseId = previous?.leaseId ?? mintLeaseId()
+      const order = ++this.sequence
+      try {
+        if (this.legacyMode) return this.setLegacyOverride(packageName, request, leaseId, order)
+        const result = await BluetoothSdk.setCameraFovOverride({
+          ...request,
+          leaseId,
+          ttlMs: CAMERA_FOV_OVERRIDE_TTL_MS,
+        })
+        this.overrides.set(packageName, {
+          leaseId,
+          fov: result.fov,
+          roiPosition: result.roiPosition,
+          order,
+        })
+        this.effectiveLeaseId = leaseId
+        this.scheduleRefresh()
+        return result
+      } catch (error) {
+        if (!isLegacyFovCompatibilityError(error)) throw error
+        console.warn("[PhoneCameraFovCoordinator] override unavailable; using legacy FOV command", error)
+        const result = await this.setLegacyOverride(packageName, request, leaseId, order)
+        // Only cache compatibility mode after the fallback itself succeeds. A
+        // disconnect or missing native bridge must not permanently downgrade
+        // future requests on a healthy/newer glasses connection.
+        this.legacyMode = true
+        return result
+      }
+    })
+  }
+
+  releaseForApp(packageName: string): Promise<void> {
+    return this.enqueue(async () => {
+      const removed = this.overrides.get(packageName)
+      if (!removed) return
+      if (removed.leaseId !== this.effectiveLeaseId) {
+        this.overrides.delete(packageName)
+        return
+      }
+
+      const next = [...this.overrides.entries()]
+        .filter(([candidatePackage]) => candidatePackage !== packageName)
+        .map(([, entry]) => entry)
+        .sort((a, b) => b.order - a.order)[0]
+      if (this.legacyMode) {
+        if (next) {
+          await this.applyLegacyFov({fov: next.fov, roiPosition: next.roiPosition})
+          this.overrides.delete(packageName)
+          this.effectiveLeaseId = next.leaseId
+        } else {
+          // Legacy commands are one-way and have no lease-release protocol.
+          // Ask native to replay its unchanged persistent base setting, then
+          // probe the modern override path again for the next ownership cycle.
+          await BluetoothSdk.restoreLegacyCameraFov()
+          await this.waitForLegacyCameraRestart()
+          this.overrides.delete(packageName)
+          this.clearRefresh()
+          this.effectiveLeaseId = undefined
+          this.legacyMode = false
+        }
+        return
+      }
+      if (next) {
+        await BluetoothSdk.setCameraFovOverride({
+          leaseId: next.leaseId,
+          fov: next.fov,
+          roiPosition: next.roiPosition,
+          ttlMs: CAMERA_FOV_OVERRIDE_TTL_MS,
+        })
+        // Commit phone-side ownership only after ASG accepted the replacement. On failure the
+        // closing app remains effective here, allowing unregister/reconnect cleanup to retry.
+        this.overrides.delete(packageName)
+        this.effectiveLeaseId = next.leaseId
+        this.scheduleRefresh()
+      } else {
+        await BluetoothSdk.releaseCameraFovOverride(removed.leaseId)
+        this.overrides.delete(packageName)
+        this.clearRefresh()
+        this.effectiveLeaseId = undefined
+      }
+    })
+  }
+
+  private scheduleRefresh(): void {
+    if (this.legacyMode) return
+    this.clearRefresh()
+    const leaseId = this.effectiveLeaseId
+    if (!leaseId) return
+    this.refreshTimer = setTimeout(() => {
+      void this.enqueue(async () => {
+        if (this.effectiveLeaseId !== leaseId) return
+        const entry = [...this.overrides.values()].find((candidate) => candidate.leaseId === leaseId)
+        if (!entry) return
+        await BluetoothSdk.setCameraFovOverride({
+          leaseId,
+          fov: entry.fov,
+          roiPosition: entry.roiPosition,
+          ttlMs: CAMERA_FOV_OVERRIDE_TTL_MS,
+        })
+        this.scheduleRefresh()
+      }).catch((error) => console.warn("[PhoneCameraFovCoordinator] failed to refresh override", error))
+    }, CAMERA_FOV_OVERRIDE_REFRESH_MS)
+  }
+
+  private clearRefresh(): void {
+    if (this.refreshTimer) clearTimeout(this.refreshTimer)
+    this.refreshTimer = undefined
+  }
+
+  private async setLegacyOverride(
+    packageName: string,
+    request: CameraFovRequest,
+    leaseId: string,
+    order: number,
+  ): Promise<CameraFovResult> {
+    const result = await this.applyLegacyFov(request)
+    this.overrides.set(packageName, {
+      leaseId,
+      fov: result.fov,
+      roiPosition: result.roiPosition,
+      order,
+    })
+    this.effectiveLeaseId = leaseId
+    this.clearRefresh()
+    return result
+  }
+
+  private async applyLegacyFov(request: CameraFovRequest): Promise<CameraFovResult> {
+    const result = await BluetoothSdk.setLegacyCameraFov(request)
+    await this.waitForLegacyCameraRestart()
+    return result
+  }
+
+  private waitForLegacyCameraRestart(): Promise<void> {
+    if (this.legacyRestartSettleMs <= 0) return Promise.resolve()
+    return new Promise<void>((resolve) => setTimeout(resolve, this.legacyRestartSettleMs))
+  }
+
+  private enqueue<T>(operation: () => Promise<T>): Promise<T> {
+    const result = this.queue.then(operation, operation)
+    this.queue = result.then(
+      () => undefined,
+      () => undefined,
+    )
+    return result
+  }
+}
+
+export const phoneCameraFovCoordinator = new PhoneCameraFovCoordinator()

package/src/services/PhoneNotificationsSync.ts

@@ -1,10 +1,15 @@
 /**
  * Phone-notifications sync — engine-owned. Pushes the notification-forwarding
- * config (`notifications_enabled` + `notifications_blocklist`) to the native
+ * operational kill switch + config (`android_notification_listener_enabled`
+ * and `notifications_blocklist`) to the native
  * NotificationListener via `CrustModule.setNotificationConfig`, so
- * `engine.phoneNotifications.setEnabled()/setBlocklist()` actually reach the
- * listener for ANY host — not just the Mentra app, where this used to live in
- * MantleManager. Android-only; a no-op elsewhere. Started by `engine.start()`.
+ * the blocklist reaches the listener for ANY host — not just the Mentra App,
+ * where this used to live in MantleManager. The Android service is desired by
+ * default now that its cold start is isolated from React Native. Native code
+ * keeps the component discoverable in Android's notification-access Settings,
+ * but only binds the service after access is granted. The debug setting remains
+ * a hidden emergency kill switch; Notify's running state gates presentation,
+ * not capture. Android-only; a no-op elsewhere. Started by `engine.start()`.
  */
 import {shallow} from "zustand/shallow"
 import CrustModule from "@mentra/crust"
@@ -15,12 +20,9 @@ let unsubscribe: (() => void) | null = null
 
 function pushConfig(): void {
   const s = useSettingsStore.getState()
-  const enabled = Boolean(s.getSetting(SETTINGS.notifications_enabled.key))
+  const listenerEnabled = Boolean(s.getSetting(SETTINGS.android_notification_listener_enabled.key))
   const blocklist = s.getSetting(SETTINGS.notifications_blocklist.key)
-  // Unconditional (matches the prior MantleManager sync) — the listener is
-  // Android-only, but CrustModule.setNotificationConfig is a safe no-op elsewhere;
-  // the Android-only gating lives on the facade's setEnabled/setBlocklist.
-  CrustModule.setNotificationConfig(enabled, Array.isArray(blocklist) ? blocklist : []).catch((err: unknown) =>
+  CrustModule.setNotificationConfig(listenerEnabled, Array.isArray(blocklist) ? blocklist : []).catch((err: unknown) =>
     console.warn(`PhoneNotificationsSync: setNotificationConfig failed: ${(err as Error)?.message ?? err}`),
   )
 }
@@ -30,7 +32,7 @@ export function startPhoneNotificationsSync(): void {
   pushConfig() // initial sync so the listener has the current config
   unsubscribe = useSettingsStore.subscribe(
     (state) => ({
-      enabled: state.getSetting(SETTINGS.notifications_enabled.key),
+      listenerEnabled: state.getSetting(SETTINGS.android_notification_listener_enabled.key),
       blocklist: state.getSetting(SETTINGS.notifications_blocklist.key),
     }),
     () => pushConfig(),

package/src/services/PhonePhotoCoordinator.ts

@@ -19,7 +19,7 @@
  */
 
 import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
-import type {PhotoSize} from "@mentra/bluetooth-sdk/internal"
+import type {PhotoSize, PhotoTransferMethod} from "@mentra/bluetooth-sdk/internal"
 import {cloudClientService} from "./CloudClientService"
 import {isGlassesConnected} from "./GlassesReadiness"
 import {useGlassesStore} from "../stores/glasses"
@@ -49,10 +49,23 @@ export interface PhotoOpts {
   /** Legacy cloud size names are normalized before the native take_photo command. */
   size?: "low" | "medium" | "high" | "max" | "small" | "large" | "full"
   mode?: "photo" | "text"
+  /** Select direct-only, phone-relayed BLE, or the default Wi-Fi/BLE fallback policy. */
+  transferMethod?: PhotoTransferMethod
   compress?: "none" | "low" | "medium" | "high"
   sound?: boolean
   saveToGallery?: boolean
   exposureTimeNs?: number
+  iso?: number | null
+  aeExposureDivisor?: number
+  isoCap?: number
+  noiseReduction?: boolean
+  edgeEnhancement?: boolean
+  /** ZSL preview buffering. */
+  zsl?: boolean
+  /** MFNR still capture. */
+  mfnr?: boolean
+  ispDigitalGain?: number
+  ispAnalogGain?: string
 }
 
 export interface PhotoTaken {
@@ -80,6 +93,15 @@ export class PhotoError extends Error {
   }
 }
 
+function parsePhotoTransferMethod(value: unknown): PhotoTransferMethod | undefined {
+  if (value === undefined) return undefined
+  if (value === "auto" || value === "direct" || value === "ble") return value
+  throw new PhotoError(
+    "INVALID_ARGUMENT",
+    `Invalid transferMethod ${JSON.stringify(value)}. Expected "auto", "direct", or "ble".`,
+  )
+}
+
 interface ActiveRequest {
   packageName: string
   abort: AbortController
@@ -97,6 +119,8 @@ interface ActiveRequest {
  * rejected successful captures while their upload was still in progress.
  */
 export const CAPTURE_PIPELINE_TIMEOUT_MS = 45_000
+export const CAMERA_WARM_UP_DEFAULT_DURATION_MS = 15_000
+export const CAMERA_WARM_UP_MAX_DURATION_MS = 60_000
 
 let bleRequestCounter = 0
 
@@ -120,8 +144,23 @@ export class PhonePhotoCoordinator {
   private readonly activeRequests = new Map<string, ActiveRequest>()
   /** Short BLE correlation id (4-char hex) → full cloud requestId. */
   private readonly bleIdToCloud = new Map<string, string>()
+  private readonly activeWarmUps = new Map<
+    string,
+    {requestId: string; durationMs: number; expiryTimer?: ReturnType<typeof setTimeout>}
+  >()
+
+  /** Report-safe camera ownership snapshot for incident diagnostics. */
+  getDiagnosticSnapshot(): Record<string, unknown> {
+    return {
+      captureOwners: [...new Set([...this.activeRequests.values()].map((request) => request.packageName))].sort(),
+      warmUpOwners: [...this.activeWarmUps.keys()].sort(),
+      activeCaptureCount: this.activeRequests.size,
+    }
+  }
 
   async takePhoto(packageName: string, opts: PhotoOpts): Promise<PhotoTaken> {
+    const transferMethod = parsePhotoTransferMethod(opts.transferMethod)
+
     // Pre-check: if glasses aren't even connected, the BLE photo command
     // would be sent into the void and we'd wait 30s for the cloud long-poll
     // to time out. Fail fast with a typed error.
@@ -137,6 +176,10 @@ export class PhonePhotoCoordinator {
       throw new PhotoError("GLASSES_NOT_CONNECTED", "Glasses are not connected", "command", "ble")
     }
 
+    // Text-mode sensor resolution is owned by ASG constants; keep cloud metadata on a stable
+    // high-capacity tier and let the glasses ignore the public size when mode=text.
+    const captureSize = opts.size ?? "medium"
+
     // 1) Presign via the cloud-v2 managed-photo service. Local miniapps use
     //    ONLY the cloud-v2 path: the runtime presigns upload+read URLs and the
     //    phone (as the device controller) delivers the bytes; the legacy
@@ -147,10 +190,16 @@ export class PhonePhotoCoordinator {
     const flowStarted = performance.now()
     try {
       const presignStarted = performance.now()
-      const r = await cloudClientService.startManagedPhoto({size: opts.size ?? "medium"})
+      const r = await cloudClientService.startManagedPhoto({
+        size: opts.mode === "text" ? "max" : captureSize,
+      })
       const presignMs = Math.round(performance.now() - presignStarted)
       if (typeof __DEV__ !== "undefined" && __DEV__) {
-        console.debug(`[PhonePhotoCoordinator] presign ${presignMs}ms size=${opts.size ?? "medium"}`)
+        console.debug(
+          `[PhonePhotoCoordinator] presign ${presignMs}ms size=${opts.mode === "text" ? "max" : captureSize} mode=${
+            opts.mode ?? "photo"
+          }`,
+        )
       }
       requestId = r.requestId
       uploadUrl = r.uploadUrl
@@ -204,21 +253,30 @@ export class PhonePhotoCoordinator {
 
     // 3) Drive glasses over BLE. requestPhoto now resolves at terminal
     //    photo_response success, so run it beside the cloud poll instead of
-    //    awaiting it before polling. iOS auto-injects transferMethod: "auto"
-    //    (WiFi direct with BLE fallback) — see MentraLive.swift.
+    //    awaiting it before polling. Native defaults transferMethod to "auto"
+    //    (Wi-Fi direct with BLE fallback) unless the miniapp forces BLE.
     try {
       void BluetoothSdk.requestPhoto({
         requestId: bleRequestId,
         appId: packageName,
-        size: normalizePhotoSize(opts.size ?? "medium"),
+        size: normalizePhotoSize(captureSize),
         mode: opts.mode ?? "photo",
         webhookUrl: uploadUrl,
         authToken: null,
-        ...(isLoopbackUpload ? {transferMethod: "ble" as const} : {}),
+        ...(isLoopbackUpload ? {transferMethod: "ble" as const} : transferMethod ? {transferMethod} : {}),
         compress: toNativeCompression(opts.compress),
         save: opts.saveToGallery ?? false,
         sound: opts.sound ?? true,
         exposureTimeNs: opts.exposureTimeNs ?? null,
+        iso: opts.iso,
+        aeExposureDivisor: opts.aeExposureDivisor,
+        isoCap: opts.isoCap,
+        noiseReduction: opts.noiseReduction,
+        edgeEnhancement: opts.edgeEnhancement,
+        ...(opts.zsl != null ? {zsl: opts.zsl} : {}),
+        ...(opts.mfnr != null ? {mfnr: opts.mfnr} : {}),
+        ispDigitalGain: opts.ispDigitalGain,
+        ispAnalogGain: opts.ispAnalogGain,
       }).catch((err) => {
         const e = this.activeRequests.get(requestId)
         if (!e) return
@@ -250,7 +308,9 @@ export class PhonePhotoCoordinator {
       const result = await outcome
       if (typeof __DEV__ !== "undefined" && __DEV__) {
         console.debug(
-          `[PhonePhotoCoordinator] takePhoto complete ${Math.round(performance.now() - flowStarted)}ms requestId=${requestId}`,
+          `[PhonePhotoCoordinator] takePhoto complete ${Math.round(
+            performance.now() - flowStarted,
+          )}ms requestId=${requestId}`,
         )
       }
       return {
@@ -273,13 +333,20 @@ export class PhonePhotoCoordinator {
   /**
    * Pre-warm the glasses camera so the next takePhoto() is near-instant.
    *
-   * Pure BLE — NO cloud presign, NO upload, NO long-poll. The SDK mints the
-   * requestId, sends the warm-up command, and resolves when the camera reports
+   * Pure BLE — NO cloud presign, NO upload, NO long-poll. The phone mints and owns
+   * the requestId, sends the warm-up command, and resolves when the camera reports
    * ready (the native promise resolves on the ready status event).
    */
   async warmUpCamera(
     packageName: string,
-    opts: {size?: "low" | "medium" | "high" | "max"; exposureTimeNs?: number; durationMs?: number},
+    opts: {
+      size?: "low" | "medium" | "high" | "max"
+      mode?: "photo" | "text"
+      exposureTimeNs?: number
+      durationMs?: number
+      zsl?: boolean
+      mfnr?: boolean
+    },
   ): Promise<void> {
     // Pre-check: if glasses aren't connected, the BLE warm-up command would be
     // sent into the void. Fail fast with a typed error.
@@ -287,17 +354,53 @@ export class PhonePhotoCoordinator {
       throw new PhotoError("GLASSES_NOT_CONNECTED", "Glasses are not connected", "command", "ble")
     }
 
+    let lease: {requestId: string; durationMs: number; expiryTimer?: ReturnType<typeof setTimeout>} | undefined
     try {
+      await this.stopWarmUpForApp(packageName)
+      const requestId = mintBleRequestId()
+      const requestedDuration =
+        typeof opts.durationMs === "number" && Number.isFinite(opts.durationMs) && opts.durationMs > 0
+          ? Math.round(opts.durationMs)
+          : CAMERA_WARM_UP_DEFAULT_DURATION_MS
+      const durationMs = Math.min(requestedDuration, CAMERA_WARM_UP_MAX_DURATION_MS)
+      lease = {requestId, durationMs}
+      this.activeWarmUps.set(packageName, lease)
       await BluetoothSdk.warmUpCamera({
+        requestId,
         size: normalizePhotoSize(opts.size ?? "medium"),
+        mode: opts.mode ?? "photo",
         exposureTimeNs: opts.exposureTimeNs ?? null,
-        durationMs: opts.durationMs ?? 15000,
+        durationMs,
+        ...(opts.zsl != null ? {zsl: opts.zsl} : {}),
+        ...(opts.mfnr != null ? {mfnr: opts.mfnr} : {}),
       })
+      if (this.activeWarmUps.get(packageName) === lease) {
+        lease.expiryTimer = setTimeout(() => {
+          if (this.activeWarmUps.get(packageName) === lease) {
+            this.activeWarmUps.delete(packageName)
+          }
+        }, durationMs)
+      }
     } catch (err) {
+      if (lease && this.activeWarmUps.get(packageName) === lease) {
+        if (lease.expiryTimer) clearTimeout(lease.expiryTimer)
+        this.activeWarmUps.delete(packageName)
+      }
       throw this.toPhotoError(err, "WARM_UP_FAILED", "command", "ble")
     }
   }
 
+  /** Release a miniapp's warm-up even if its original warmUpCamera promise is still opening. */
+  async stopWarmUpForApp(packageName: string): Promise<void> {
+    const active = this.activeWarmUps.get(packageName)
+    if (!active) return
+    await BluetoothSdk.stopCameraWarmUp(active.requestId)
+    if (this.activeWarmUps.get(packageName) === active) {
+      this.activeWarmUps.delete(packageName)
+      if (active.expiryTimer) clearTimeout(active.expiryTimer)
+    }
+  }
+
   /** True iff this requestId (short BLE id or cloud id) is one we're currently waiting on. */
   owns(requestId: string): boolean {
     const cloudId = this.resolveCloudRequestId(requestId)
@@ -320,12 +423,7 @@ export class PhonePhotoCoordinator {
     // cloud-v2 pending photo requests TTL out on their own; nothing to free.
   }
 
-  private toPhotoError(
-    err: unknown,
-    fallbackCode: string,
-    stage?: PhotoStage,
-    transport?: PhotoTransport,
-  ): PhotoError {
+  private toPhotoError(err: unknown, fallbackCode: string, stage?: PhotoStage, transport?: PhotoTransport): PhotoError {
     if (err instanceof PhotoError) return err
     const code = (err as {code?: string})?.code
     const message = err instanceof Error ? err.message : String(err)