@mentra/engine 0.1.0-dev.0 → 3.1.0-dev.10

package/build/services/AudioPlaybackService.js

@@ -1,13 +1,23 @@
 import { createAudioPlayer, setAudioModeAsync } from "expo-audio";
 import BluetoothSdk from "@mentra/bluetooth-sdk/internal";
+import { AppState, Platform } from "react-native";
 import { BgTimer } from "../utils/timers";
+import { setAudioCloudUplinkSuppressed } from "./AudioCloudUplink";
+import { SILENT_AUDIO_SOURCE } from "./audioPlaybackAssets";
 const RESTORE_GLASSES_VOLUME_AFTER_PLAYBACK = false;
 class AudioPlaybackService {
     static instance = null;
     // Reuse a single AudioPlayer to avoid AudioTrack exhaustion
     // Creating new ExoPlayer instances per request leads to -12 ENOMEM errors
     player = null;
+    // The source currently loaded into the reusable player. Keep object identity
+    // so a delayed tail cleanup from an older request cannot unload a newer one.
+    loadedPlayback = null;
     currentPlayback = null;
+    // Requests that entered play() but have not yet reached the native player.
+    // Keeping these addressable prevents a cancelled request from starting after
+    // an async audio-mode or stream-cleanup step finishes.
+    pendingPlaybacks = new Map();
     // Live PCM streams (speaker.createStream), keyed by streamId. The runtime
     // enforces one per app; stopOtherAudio interrupts across apps.
     streams = new Map();
@@ -16,10 +26,25 @@ class AudioPlaybackService {
     // Prevents mic toggle flicker when playing back-to-back audio
     // Uses BgTimer to work reliably when app is backgrounded on Android
     audioStopDebounceTimer = null;
+    // Mentra Live's A2DP route may go idle between prompts. Keep a short warm
+    // window after actual audio so we only pre-roll the first sound after idle.
+    audioRouteWarmUntil = 0;
+    audioRouteWarmupPromise = null;
+    // Playback that opted into suppression stays suppressed briefly while its
+    // A2DP tail drains. Track those completed sources so a later unsuppressed
+    // sound can resume listening immediately instead of inheriting that gate.
+    tailUplinkSuppressions = new Set();
     // Original glasses media volume captured before we bump it for A2DP playback.
     glassesVolumeRestoreLevel = null;
     static AUDIO_STOP_DEBOUNCE_MS = 1500;
-    static A2DP_TAIL_DRAIN_MS = 900;
+    static A2DP_TAIL_DRAIN_MS = 700;
+    static AUDIO_ROUTE_WARM_WINDOW_MS = 1500;
+    static AUDIO_ROUTE_PREWARM_MS = 200;
+    static AUDIO_ROUTE_PREWARM_SAMPLE_RATE = 16_000;
+    /** Break accidental same-app replay storms without affecting deliberate later replays. */
+    static DUPLICATE_PLAY_WINDOW_MS = 750;
+    static AUDIO_ROUTE_PREWARM_PCM_BASE64 = "AAAA".repeat(Math.floor((AudioPlaybackService.AUDIO_ROUTE_PREWARM_MS * AudioPlaybackService.AUDIO_ROUTE_PREWARM_SAMPLE_RATE * 2) /
+        (1000 * 3))) + "AA==";
     /** If glasses report step volume at or below this, bump to FLOOR before A2DP playback. */
     static GLASSES_VOLUME_LOW_THRESHOLD = 2;
     static GLASSES_VOLUME_FLOOR = 9;
@@ -77,6 +102,67 @@ class AudioPlaybackService {
         }
         return this.player;
     }
+    markAudioRouteWarm() {
+        this.audioRouteWarmUntil = Date.now() + AudioPlaybackService.AUDIO_ROUTE_WARM_WINDOW_MS;
+    }
+    isAudioRouteWarm() {
+        return this.currentPlayback !== null || this.streams.size > 0 || Date.now() < this.audioRouteWarmUntil;
+    }
+    /**
+     * Wake a cold Android A2DP route with silent PCM before a user-facing sound.
+     *
+     * Do not call pcmStreamClose here: some Bluetooth routes do not advance an
+     * idle AudioTrack playback head, so close waits for its drain timeout. After
+     * exactly 200ms of silence we abort the stream instead, which releases the
+     * pre-roll immediately and lets the real URL playback begin.
+     */
+    async prewarmAudioRouteIfCold() {
+        if (Platform.OS !== "android" || this.isAudioRouteWarm())
+            return;
+        // This silent PCM pre-roll is only an anti-clipping optimization. Some
+        // Android devices defer opening its AudioTrack until the host Activity
+        // resumes even though the real URL player supports background playback.
+        // Never let that optional pre-roll block a glasses-initiated prompt.
+        if (AppState.currentState !== "active") {
+            console.log(`AUDIO: Skipping cold-route prewarm while app is ${AppState.currentState}`);
+            return;
+        }
+        if (this.audioRouteWarmupPromise)
+            return this.audioRouteWarmupPromise;
+        const streamId = `audio-route-prewarm-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+        const warmup = (async () => {
+            let opened = false;
+            const startedAt = Date.now();
+            try {
+                console.log(`AUDIO: Cold route; prewarming with ${AudioPlaybackService.AUDIO_ROUTE_PREWARM_MS}ms silence`);
+                await BluetoothSdk.pcmStreamOpen(streamId, AudioPlaybackService.AUDIO_ROUTE_PREWARM_SAMPLE_RATE, 1, 1);
+                opened = true;
+                await BluetoothSdk.pcmStreamWrite(streamId, AudioPlaybackService.AUDIO_ROUTE_PREWARM_PCM_BASE64);
+                await new Promise((resolve) => {
+                    BgTimer.setTimeout(resolve, AudioPlaybackService.AUDIO_ROUTE_PREWARM_MS);
+                });
+                await BluetoothSdk.pcmStreamAbort(streamId);
+                opened = false;
+                this.markAudioRouteWarm();
+                console.log(`AUDIO: Cold-route prewarm completed in ${Date.now() - startedAt}ms`);
+            }
+            catch (error) {
+                console.warn("AUDIO: Cold-route prewarm failed; continuing with requested audio:", error);
+            }
+            finally {
+                if (opened)
+                    await BluetoothSdk.pcmStreamAbort(streamId).catch(() => { });
+            }
+        })();
+        this.audioRouteWarmupPromise = warmup;
+        try {
+            await warmup;
+        }
+        finally {
+            if (this.audioRouteWarmupPromise === warmup)
+                this.audioRouteWarmupPromise = null;
+        }
+    }
     async getGlassesMediaVolumeWithTiming() {
         const startedAt = Date.now();
         try {
@@ -170,40 +256,111 @@ class AudioPlaybackService {
             this.glassesVolumeRestoreLevel = null;
         }
     }
-    pauseFinishedPlayerAfterTail(playback) {
+    unloadPlaybackSource(playback, reason) {
+        if (this.loadedPlayback !== playback)
+            return;
+        const player = this.player;
+        if (!player) {
+            this.loadedPlayback = null;
+            return;
+        }
+        try {
+            player.pause();
+        }
+        catch (e) {
+            console.warn(`AUDIO: Error pausing ${playback.requestId} during ${reason}:`, e);
+        }
+        try {
+            // replace(null) fails native argument casting on iOS, while remove()
+            // destroys the ExoPlayer that Android must reuse to avoid AudioTrack
+            // exhaustion. Replacing with bundled silence detaches the previous media
+            // item on both platforms and leaves the native player alive.
+            player.replace(SILENT_AUDIO_SOURCE);
+            console.log(`AUDIO: Cleared source for ${playback.requestId} during ${reason}`);
+        }
+        catch (e) {
+            console.warn(`AUDIO: Error clearing ${playback.requestId} during ${reason}:`, e);
+        }
+        finally {
+            if (this.loadedPlayback === playback)
+                this.loadedPlayback = null;
+        }
+    }
+    clearFinishedPlayerAfterTail(playback) {
+        if (playback.suppressCloudUplink) {
+            this.tailUplinkSuppressions.add(playback.uplinkSuppressionId);
+        }
         BgTimer.setTimeout(() => {
-            if (this.currentPlayback)
-                return;
-            if (!this.player)
-                return;
-            try {
-                this.player.pause();
-                console.log(`AUDIO: Paused finished player for ${playback.requestId} after ${AudioPlaybackService.A2DP_TAIL_DRAIN_MS}ms tail drain`);
-            }
-            catch (e) {
-                console.warn("AUDIO: Error pausing player after tail drain:", e);
+            // ExoPlayer's completion event precedes the final A2DP audio reaching the
+            // glasses. Keep cloud STT gated until that audible tail has drained, then
+            // accept fresh microphone audio immediately without restarting LC3.
+            if (playback.suppressCloudUplink && this.tailUplinkSuppressions.delete(playback.uplinkSuppressionId)) {
+                setAudioCloudUplinkSuppressed(playback.uplinkSuppressionId, false);
             }
+            this.unloadPlaybackSource(playback, `${AudioPlaybackService.A2DP_TAIL_DRAIN_MS}ms A2DP tail drain`);
         }, AudioPlaybackService.A2DP_TAIL_DRAIN_MS);
     }
+    releaseTailUplinkSuppressions() {
+        for (const sourceId of this.tailUplinkSuppressions) {
+            setAudioCloudUplinkSuppressed(sourceId, false);
+        }
+        this.tailUplinkSuppressions.clear();
+    }
     /**
      * Play audio from a URL.
      * Returns a promise that resolves with playback result when audio finishes or errors.
      */
     async play(request, onComplete) {
-        const { requestId, audioUrl, appId, volume = 1.0, stopOtherAudio = true } = request;
+        const { requestId, audioUrl, appId, volume = 1.0, stopOtherAudio = true, suppressCloudUplink = false } = request;
+        const now = Date.now();
+        const activeDuplicate = this.currentPlayback &&
+            !this.currentPlayback.completed &&
+            this.currentPlayback.appId === appId &&
+            this.currentPlayback.audioUrl === audioUrl &&
+            now - this.currentPlayback.startTime < AudioPlaybackService.DUPLICATE_PLAY_WINDOW_MS;
+        const pendingDuplicate = [...this.pendingPlaybacks.values()].some((candidate) => !candidate.cancelled &&
+            candidate.appId === appId &&
+            candidate.audioUrl === audioUrl &&
+            now - candidate.createdAt < AudioPlaybackService.DUPLICATE_PLAY_WINDOW_MS);
+        if (activeDuplicate || pendingDuplicate) {
+            console.warn(`AUDIO: Suppressed duplicate play request ${requestId}${appId ? ` from ${appId}` : ""}`);
+            onComplete(requestId, false, "Duplicate audio request suppressed", 0, "error");
+            return;
+        }
+        const pending = { cancelled: false, audioUrl, appId, createdAt: now };
+        this.pendingPlaybacks.set(requestId, pending);
         console.log(`AUDIO: Play request ${requestId}${appId ? ` from ${appId}` : ""}: ${audioUrl}`);
         try {
             // Ensure audio mode is configured for background playback
             await this.ensureAudioModeConfigured();
-            // Stop current playback if any (notify previous callback)
-            if (stopOtherAudio && this.currentPlayback && !this.currentPlayback.completed) {
-                console.log(`AUDIO: Interrupting current playback for new request`);
+            if (pending.cancelled) {
+                onComplete(requestId, true, null, 0, "interrupted");
+                return;
+            }
+            await this.prewarmAudioRouteIfCold();
+            if (pending.cancelled) {
+                onComplete(requestId, true, null, 0, "interrupted");
+                return;
+            }
+            // URL playback reuses one native player, so replacing its source always
+            // interrupts the previous URL even when stopOtherAudio is false. Notify
+            // that request before replacing it so its promise cannot hang. The option
+            // still controls whether separate PCM streams are interrupted below.
+            if (this.currentPlayback && !this.currentPlayback.completed) {
+                console.log(`AUDIO: Interrupting current URL playback for new request`);
                 this.interruptCurrentPlayback(true);
             }
             // Live PCM streams count as "other audio" too.
             if (stopOtherAudio) {
                 await this.abortAllStreams();
             }
+            if (pending.cancelled) {
+                onComplete(requestId, true, null, 0, "interrupted");
+                return;
+            }
+            if (!suppressCloudUplink) {
+                this.releaseTailUplinkSuppressions();
+            }
             // Get or create the reusable player
             const player = this.ensurePlayer();
             // Set volume
@@ -211,15 +368,24 @@ class AudioPlaybackService {
             // Store the new playback state
             const playback = {
                 requestId,
+                audioUrl,
+                uplinkSuppressionId: `url:${requestId}`,
                 appId,
                 startTime: Date.now(),
                 completed: false,
+                suppressCloudUplink,
                 onComplete,
             };
             this.currentPlayback = playback;
+            this.pendingPlaybacks.delete(requestId);
+            this.markAudioRouteWarm();
+            if (playback.suppressCloudUplink) {
+                setAudioCloudUplinkSuppressed(playback.uplinkSuppressionId, true);
+            }
             // Replace the source and play
             // Using replace() reuses the existing ExoPlayer/AudioTrack instead of creating new ones
             player.replace({ uri: audioUrl });
+            this.loadedPlayback = playback;
             player.play();
             // Mentra Live volume reads can block up to 5s when the glasses don't
             // answer. Do not put that in front of playback; guard it against late
@@ -239,10 +405,41 @@ class AudioPlaybackService {
             console.log(`AUDIO: Started playback for ${requestId}`);
         }
         catch (error) {
+            if (suppressCloudUplink) {
+                setAudioCloudUplinkSuppressed(`url:${requestId}`, false);
+            }
+            const failedPlayback = this.currentPlayback?.requestId === requestId ? this.currentPlayback : null;
+            if (failedPlayback) {
+                failedPlayback.completed = true;
+                this.currentPlayback = null;
+                this.unloadPlaybackSource(failedPlayback, "playback start failure");
+            }
+            if (pending.cancelled) {
+                onComplete(requestId, true, null, 0, "interrupted");
+                return;
+            }
             const errorMessage = error instanceof Error ? error.message : "Unknown error loading audio";
             console.error(`AUDIO: Failed to play ${requestId}:`, errorMessage);
             void this.restoreGlassesMediaVolume();
-            onComplete(requestId, false, errorMessage, null);
+            onComplete(requestId, false, errorMessage, null, "error");
+        }
+        finally {
+            if (this.pendingPlaybacks.get(requestId) === pending) {
+                this.pendingPlaybacks.delete(requestId);
+            }
+        }
+    }
+    /**
+     * Cancel one URL playback without touching audio owned by another request.
+     * Pending requests are prevented from reaching the native player; active
+     * requests are paused synchronously before their completion callback fires.
+     */
+    cancelPlayback(requestId) {
+        const pending = this.pendingPlaybacks.get(requestId);
+        if (pending)
+            pending.cancelled = true;
+        if (this.currentPlayback?.requestId === requestId && !this.currentPlayback.completed) {
+            this.interruptCurrentPlayback();
         }
     }
     /**
@@ -254,15 +451,13 @@ class AudioPlaybackService {
         const playback = this.currentPlayback;
         playback.completed = true;
         this.currentPlayback = null;
-        // Stop the player
-        if (this.player) {
-            try {
-                this.player.pause();
-            }
-            catch (error) {
-                console.error("AUDIO: Error pausing player:", error);
-            }
+        if (playback.suppressCloudUplink) {
+            setAudioCloudUplinkSuppressed(playback.uplinkSuppressionId, false);
         }
+        this.markAudioRouteWarm();
+        // Pausing alone leaves the URI available to OS/Bluetooth media Play.
+        // Unload it unless a newer request has already replaced this source.
+        this.unloadPlaybackSource(playback, "interruption");
         // Notify native that our app stopped playing audio (debounced)
         this.notifyAudioStopDebounced();
         if (!preserveGlassesMediaVolume) {
@@ -270,7 +465,7 @@ class AudioPlaybackService {
         }
         // Notify that playback was interrupted
         const elapsedMs = Date.now() - playback.startTime;
-        playback.onComplete(playback.requestId, true, null, elapsedMs);
+        playback.onComplete(playback.requestId, true, null, elapsedMs, "interrupted");
         console.log(`AUDIO: Interrupted ${playback.requestId} after ${elapsedMs}ms`);
     }
     /**
@@ -289,10 +484,11 @@ class AudioPlaybackService {
             playback.completed = true;
             // ExoPlayer can report finished before an A2DP sink has audibly drained
             // the last buffered audio. Pausing immediately can clip the tail on
-            // Mentra Live; defer cleanup unless another playback has started.
-            this.pauseFinishedPlayerAfterTail(playback);
-            playback.onComplete(playback.requestId, true, null, durationMs);
+            // Mentra Live; defer cleanup and guard it against a newer loaded source.
+            this.clearFinishedPlayerAfterTail(playback);
             this.currentPlayback = null;
+            playback.onComplete(playback.requestId, true, null, durationMs, "completed");
+            this.markAudioRouteWarm();
             // Notify native that our app stopped playing audio (debounced)
             this.notifyAudioStopDebounced();
             void this.restoreGlassesMediaVolume();
@@ -307,8 +503,13 @@ class AudioPlaybackService {
             if (elapsedMs > 1500) {
                 console.error(`AUDIO: Playback failed for ${playback.requestId} (player went idle after ${elapsedMs}ms)`);
                 playback.completed = true;
-                playback.onComplete(playback.requestId, false, "Playback failed (player went idle)", null);
                 this.currentPlayback = null;
+                if (playback.suppressCloudUplink) {
+                    setAudioCloudUplinkSuppressed(playback.uplinkSuppressionId, false);
+                }
+                this.unloadPlaybackSource(playback, "playback failure");
+                playback.onComplete(playback.requestId, false, "Playback failed (player went idle)", null, "error");
+                this.markAudioRouteWarm();
                 this.notifyAudioStopDebounced();
                 void this.restoreGlassesMediaVolume();
             }
@@ -327,6 +528,11 @@ class AudioPlaybackService {
         // Uses BgTimer to work reliably when app is backgrounded on Android
         this.audioStopDebounceTimer = BgTimer.setTimeout(() => {
             this.audioStopDebounceTimer = null;
+            // URL playback and live PCM streams may overlap when stopOtherAudio is false.
+            // A URL that ends must not clear the native playback signal while another
+            // source is still active, or the mic watchdog can interrupt that source.
+            if (this.isPlaying())
+                return;
             BluetoothSdk.setOwnAppAudioPlaying(false).catch((e) => {
                 console.warn("AUDIO: Failed to notify native of audio stop:", e);
             });
@@ -342,6 +548,7 @@ class AudioPlaybackService {
         const { streamId, appId, sampleRate, channels, volume = 1.0, stopOtherAudio = true } = request;
         console.log(`AUDIO: Stream open ${streamId} from ${appId}: rate=${sampleRate} ch=${channels}`);
         await this.ensureAudioModeConfigured();
+        await this.prewarmAudioRouteIfCold();
         if (stopOtherAudio) {
             if (this.currentPlayback && !this.currentPlayback.completed) {
                 console.log("AUDIO: Interrupting current playback for new stream");
@@ -350,10 +557,18 @@ class AudioPlaybackService {
             await this.abortAllStreams(streamId);
         }
         await BluetoothSdk.pcmStreamOpen(streamId, sampleRate, channels, Math.max(0, Math.min(1, volume)));
-        const stream = { streamId, appId, startTime: Date.now(), ended: false, onEnded: request.onEnded };
+        this.releaseTailUplinkSuppressions();
+        const stream = {
+            streamId,
+            appId,
+            startTime: Date.now(),
+            ended: false,
+            onEnded: request.onEnded,
+        };
         this.streams.set(streamId, stream);
-        // Same LC3-mic-suspend dance as play(): cancel any pending "stopped"
-        // notification and mark our app as playing audio.
+        this.markAudioRouteWarm();
+        // Cancel any pending "stopped" notification and mark our app as playing
+        // audio. PCM streams intentionally do not suppress cloud STT.
         if (this.audioStopDebounceTimer !== null) {
             BgTimer.clearTimeout(this.audioStopDebounceTimer);
             this.audioStopDebounceTimer = null;
@@ -423,6 +638,7 @@ class AudioPlaybackService {
             return;
         stream.ended = true;
         this.streams.delete(stream.streamId);
+        this.markAudioRouteWarm();
         // Only signal "audio stopped" when nothing else is making noise.
         if (this.streams.size === 0 && (!this.currentPlayback || this.currentPlayback.completed)) {
             this.notifyAudioStopDebounced();
@@ -525,6 +741,7 @@ class AudioPlaybackService {
                 console.error("AUDIO: Error releasing player:", error);
             }
             this.player = null;
+            this.loadedPlayback = null;
         }
     }
 }