@mentra/engine 3.2.0-dev.261 → 3.2.0-dev.262

package/build/services/SoftapCallTransport.js

@@ -22,6 +22,13 @@
 import { softapTrace, softapTraceFailure, beginSoftapTrace, resetSoftapTrace } from "../utils/softapTrace";
 /** Steps in order. Also the teardown order, reversed. */
 export const SOFTAP_STEPS = ["hotspot", "scopedJoin", "acsJoin", "publish", "live"];
+/**
+ * SoftAP loss is a media outage, not call termination. The wearer has this long to
+ * come back in range before the host should give up rebuilding the hop.
+ */
+export const RETURN_DEADLINE_MS = 60_000;
+/** Budget to re-arm hotspot, scoped join, ingest, and publish after the glasses return. */
+export const REARM_BUDGET_MS = 45_000;
 /** A failure, named by the step that produced it so the UI and the logs agree on the cause. */
 export class SoftapCallError extends Error {
     step;
@@ -94,6 +101,16 @@ function isScopedJoinUnavailable(error) {
     const message = error instanceof Error ? error.message : String(error);
     return /SOFTAP_UNAVAILABLE|ScopedNetworkError\$Unavailable|SSID not in scan, Wi-Fi off, or the system join prompt/i.test(message);
 }
+/**
+ * The glasses never answered, as opposed to answering "disabled".
+ *
+ * A timeout is not evidence the command was lost: BLE can redeliver it after the phone has given
+ * up, so the AP may be coming up at the very moment the retry would ask for it to go down.
+ */
+function isHotspotAnswerTimeout(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return /timed out waiting for glasses response|glasses did not answer/i.test(message);
+}
 function freshSteps() {
     return SOFTAP_STEPS.map((step) => ({ step, status: "pending" }));
 }
@@ -187,12 +204,40 @@ export class SoftapCallTransport {
     republishing = null;
     /** Bumped by `stop` so an in-flight republish cannot start_stream after Leave. */
     republishGeneration = 0;
+    /**
+     * Media hop generation. Distinct from [generation]: ACS join is preserved across recoveries,
+     * but hotspot, scoped network, ingest, and publisher belong to this number.
+     */
+    mediaGeneration = 0;
+    /** Why the current recover was requested. Cleared when the hop is live again. */
+    recoveryReason;
+    /** In-flight [recover]. Concurrent recoveries share one rebuild. */
+    recovering = null;
+    /** A [stop] without preserveMeeting landed during recover; recover must not rebuild. */
+    recoveryAbort = false;
+    /** Skip `acsJoin` undo so the ACS session survives a media-only teardown. */
+    meetingPreserved = false;
+    /**
+     * Identity of the publisher the most recent `startPublishing` created.
+     * A cancelled republish may only `stopPublishing` if it still owns this token.
+     */
+    publisherToken = null;
     constructor(deps) {
         this.deps = deps;
     }
     currentPhase() {
         return this.phase;
     }
+    currentMediaGeneration() {
+        return this.mediaGeneration;
+    }
+    recoveryState() {
+        return {
+            phase: this.phase,
+            mediaGeneration: this.mediaGeneration,
+            ...(this.recoveryReason ? { reason: this.recoveryReason } : {}),
+        };
+    }
     /**
      * The glasses camera died after this call was already live. Re-issue `start_stream` at the
      * existing ingest URL — do not rebind the WHIP listener, or the glasses POST to a dead port.
@@ -212,11 +257,15 @@ export class SoftapCallTransport {
         if (this.republishing)
             return this.republishing;
         const generation = this.republishGeneration;
-        this.republishing = this.runRepublish(reason, generation).finally(() => {
-            if (this.republishGeneration === generation)
+        let run;
+        run = this.runRepublish(reason, generation).finally(() => {
+            // Clear when *this* run still owns the field, even if stop() bumped the generation.
+            // Gating on generation left a settled promise stranded and the next republish was a no-op.
+            if (this.republishing === run)
                 this.republishing = null;
         });
-        return this.republishing;
+        this.republishing = run;
+        return run;
     }
     async runRepublish(reason, generation) {
         const ingestUrl = this.ingestUrl;
@@ -230,13 +279,19 @@ export class SoftapCallTransport {
             this.phase === "live" &&
             this.ingestUrl === ingestUrl) {
             attempt += 1;
+            let startedPublisher = false;
+            const token = {};
             try {
                 await this.deps.stopPublishing();
                 if (generation !== this.republishGeneration || this.terminating)
                     return;
-                await this.deps.startPublishing({ ingestUrl, traceId: this.traceId });
+                await this.startPublishingOwned(token, { ingestUrl, traceId: this.traceId });
+                startedPublisher = true;
                 if (generation !== this.republishGeneration || this.terminating) {
-                    await this.deps.stopPublishing().catch(() => undefined);
+                    // Only stop a publisher this run started, and only if a successor has not taken it.
+                    if (startedPublisher && this.publisherToken === token) {
+                        await this.deps.stopPublishing().catch(() => undefined);
+                    }
                     return;
                 }
                 softapTrace("glasses_republish_sent", { attempt, ingestUrl });
@@ -262,6 +317,88 @@ export class SoftapCallTransport {
             }
         }
     }
+    /**
+     * Rebuild the glasses→phone hop after SoftAP loss. ACS, mute intent, and media endpoints stay.
+     *
+     * Phase flips to `recovering` before any await so [shouldRepublish] stands down immediately —
+     * not after a BLE wait that could still issue `start_stream` at the dying hop.
+     */
+    recover(reason, options = {}) {
+        if (this.recovering)
+            return this.recovering;
+        if (this.phase !== "live") {
+            return Promise.reject(new SoftapCallError("live", "NOT_RECOVERABLE", `Cannot recover a SoftAP call that is ${this.phase}`));
+        }
+        this.phase = "recovering";
+        this.recoveryReason = reason;
+        this.recoveryAbort = false;
+        this.mediaGeneration++;
+        this.emitProgress();
+        softapTrace("softap_media_recover", { reason, mediaGeneration: this.mediaGeneration });
+        let run;
+        run = this.runRecover(options).finally(() => {
+            if (this.recovering === run)
+                this.recovering = null;
+        });
+        this.recovering = run;
+        return run;
+    }
+    async runRecover(options) {
+        const onProgress = this.onProgress;
+        // Wait for BLE *before* `set_hotspot_state false`. Doing stop first while the glasses are
+        // out of range queues the off command; when BLE returns it fires into a still-up hotspot
+        // and races the media-only start.
+        if (options.wait)
+            await options.wait();
+        if (this.recoveryAbort) {
+            throw new SoftapCallError("hotspot", "CANCELLED", "SoftAP recovery was cancelled");
+        }
+        await this.stop({ preserveMeeting: true, keepProgress: true });
+        if (this.recoveryAbort) {
+            throw new SoftapCallError("hotspot", "CANCELLED", "SoftAP recovery was cancelled");
+        }
+        // The return deadline bounds [options.wait]; the rebuild gets its own budget from the moment
+        // the glasses are back. Charging the rebuild for the time the wearer spent away left a return
+        // at 48s with 12s to re-arm a hotspot that takes ~30s, and the call was dropped as it worked.
+        const rearmMs = REARM_BUDGET_MS;
+        let budgetTimer;
+        const budget = new Promise((_, reject) => {
+            budgetTimer = setTimeout(() => {
+                void this.stop({ preserveMeeting: true }).catch(() => undefined);
+                reject(new SoftapCallError("hotspot", "REARM_BUDGET", "SoftAP media rebuild exceeded its budget"));
+            }, rearmMs);
+        });
+        try {
+            await Promise.race([this.start({ mediaOnly: true, onProgress }), budget]);
+        }
+        finally {
+            if (budgetTimer !== undefined)
+                clearTimeout(budgetTimer);
+        }
+        this.completed = [...SOFTAP_STEPS];
+        this.recoveryReason = undefined;
+        this.emitProgress();
+    }
+    /** Issue `start_stream` and take publisher ownership so a stale republish cannot stop a successor. */
+    async startPublishingOwned(token, args, report) {
+        const mediaGeneration = this.mediaGeneration;
+        const stamped = report
+            ? (detail) => {
+                if (mediaGeneration !== this.mediaGeneration) {
+                    softapTrace("softap_stale_callback", {
+                        source: "startPublishing",
+                        mediaGeneration,
+                        current: this.mediaGeneration,
+                        detail,
+                    });
+                    return;
+                }
+                report(detail);
+            }
+            : undefined;
+        await this.deps.startPublishing({ ...args, mediaGeneration }, stamped);
+        this.publisherToken = token;
+    }
     /**
      * True once a teardown has been decided. Anything that would otherwise report a failure — a lost
      * hotspot, a dropped ACS call — must check this first: after the wearer asks to leave, those are
@@ -277,6 +414,7 @@ export class SoftapCallTransport {
             phase: this.phase,
             steps: this.steps.map((step) => ({ ...step })),
             elapsedMs: this.startedAt ? Date.now() - this.startedAt : 0,
+            mediaGeneration: this.mediaGeneration,
         };
     }
     emitProgress() {
@@ -317,11 +455,10 @@ export class SoftapCallTransport {
         return this.ingestUrl;
     }
     /**
-     * Steps whose undo threw during the last teardown, so the caller can refuse the next call.
+     * Steps whose undo threw during the last teardown.
      *
-     * Teardown deliberately swallows these to keep unwinding — but a hotspot that would not turn off
-     * is exactly the state the next call cannot be built on, and starting anyway is what produced
-     * "Cannot start glasses hotspot" followed by a scoped join that never found the SSID.
+     * Teardown swallows these so unwind can finish. The host refuses the next join for ingest /
+     * scoped-network leaks; a hotspot that missed its off-ack is leftover ON and is raised again.
      */
     lastTeardownFailures() {
         // Logged on read rather than only on write, because this is the moment the list turns into a
@@ -379,17 +516,22 @@ export class SoftapCallTransport {
             softapTraceFailure("softap_call_refused", { reason: "cancelled before start" });
             throw new SoftapCallError("hotspot", "CANCELLED", "SoftAP call was cancelled before it started");
         }
-        if (this.phase !== "idle" && this.phase !== "failed") {
+        const mediaOnly = options.mediaOnly === true;
+        const allowed = this.phase === "idle" || this.phase === "failed" || (mediaOnly && this.phase === "recovering");
+        if (!allowed) {
             softapTraceFailure("softap_call_refused", { reason: "already active", phase: this.phase });
             throw new SoftapCallError("hotspot", "ALREADY_ACTIVE", `A SoftAP call is already ${this.phase}`);
         }
         this.startedEver = true;
         const generation = ++this.generation;
-        this.phase = "starting";
+        if (mediaOnly)
+            this.mediaGeneration++;
+        this.phase = mediaOnly ? "recovering" : "starting";
         this.terminating = false;
         this.teardownMode = "leave";
         this.endFailure = null;
-        this.completed = [];
+        this.meetingPreserved = mediaOnly;
+        this.completed = mediaOnly ? this.completed.filter((step) => step === "acsJoin") : [];
         this.ingestUrl = null;
         this.hotspot = null;
         this.teardownFailures = [];
@@ -397,11 +539,16 @@ export class SoftapCallTransport {
         this.stepStartedAt.clear();
         this.startedAt = Date.now();
         this.onProgress = options.onProgress;
-        const traceId = beginSoftapTrace(options.traceId);
+        // Media-only rebuilds mint a fresh trace id: the previous hop is gone and its id must not
+        // be reused, even if the caller still holds it.
+        const traceId = beginSoftapTrace(mediaOnly ? undefined : options.traceId);
         this.traceId = traceId;
-        softapTrace("softap_call_start", { traceId });
+        softapTrace("softap_call_start", { traceId, mediaOnly, mediaGeneration: this.mediaGeneration });
         this.emitProgress();
         try {
+            if (mediaOnly && !this.deps.rebindIngest) {
+                throw new SoftapCallError("acsJoin", "ACS_JOIN_FAILED", "rebindIngest is required for a media-only rebuild");
+            }
             await this.preflightWifi();
             await this.step(generation, "hotspot", "HOTSPOT_FAILED", async (report) => {
                 report("Asking the glasses to turn on their hotspot");
@@ -428,6 +575,17 @@ export class SoftapCallTransport {
                     report(`Phone is ${bindAddress} on ${hotspot.ssid}`);
             });
             await this.step(generation, "acsJoin", "ACS_JOIN_FAILED", async (report) => {
+                if (mediaOnly) {
+                    report(bindAddress ? `Rebinding the video receiver on ${bindAddress}` : "Rebinding the video receiver");
+                    const { ingestUrl } = await this.deps.rebindIngest(report);
+                    if (!ingestUrl) {
+                        throw new Error("the meeting reported no ingest URL");
+                    }
+                    this.ingestUrl = ingestUrl;
+                    softapTrace("acs_receiver_rebound", { ingestUrl });
+                    report(`Receiver ready at ${ingestUrl}`);
+                    return;
+                }
                 report(bindAddress ? `Opening video receiver on ${bindAddress}, then joining Teams` : "Joining Teams");
                 const { ingestUrl } = await this.deps.joinMeeting({
                     ssid: hotspot.ssid,
@@ -446,13 +604,13 @@ export class SoftapCallTransport {
             const ingestUrl = this.requireIngestUrl();
             await this.step(generation, "publish", "PUBLISH_FAILED", async (report) => {
                 report("Telling the glasses to start the camera and publish to the phone");
-                await this.deps.startPublishing({ ingestUrl, traceId }, report);
+                await this.startPublishingOwned({}, { ingestUrl, traceId }, report);
                 softapTrace("glasses_publishing", { ingestUrl });
                 report("Glasses camera is streaming to the phone");
             });
             await this.step(generation, "live", "NO_FIRST_FRAME", async (report) => {
                 report("Waiting for the first glasses video frame on this phone");
-                await this.deps.awaitFirstFrame(report);
+                await this.deps.awaitFirstFrame(report, mediaOnly ? { fresh: true } : undefined);
                 softapTrace("first_glasses_frame_received");
                 report("Glasses video is reaching this phone");
             });
@@ -463,15 +621,20 @@ export class SoftapCallTransport {
                 softapTraceFailure("softap_call_abandoned_at_live", { generation, current: this.generation });
                 return;
             }
+            if (mediaOnly)
+                this.completed = [...SOFTAP_STEPS];
             this.phase = "live";
+            this.recoveryReason = undefined;
             softapTrace("softap_call_live");
             this.emitProgress();
         }
         catch (error) {
             // Unwind before rethrowing. A caller that sees a rejection is entitled to assume nothing was
             // left running, and a hotspot left up is both a battery cost and a second call's failure.
-            await this.stop({ keepProgress: true });
-            this.phase = "failed";
+            // Media-only failures keep ACS; a user Leave during recover does not (`recoveryAbort`).
+            await this.stop({ keepProgress: true, preserveMeeting: mediaOnly && !this.recoveryAbort });
+            if (!(mediaOnly && this.recoveryAbort))
+                this.phase = "failed";
             this.emitProgress();
             throw error;
         }
@@ -496,12 +659,18 @@ export class SoftapCallTransport {
         this.republishGeneration++;
         if (options.mode)
             this.teardownMode = options.mode;
+        if (!options.preserveMeeting)
+            this.recoveryAbort = true;
         if (this.stopping) {
-            softapTrace("softap_stop_joined_in_flight", { mode: this.teardownMode });
-            return this.stopping;
+            softapTrace("softap_stop_joined_in_flight", { mode: this.teardownMode, preserveMeeting: !!options.preserveMeeting });
+            await this.stopping;
+            // A Leave that joined a preserveMeeting teardown still has to drop ACS.
+            if (options.preserveMeeting || (!this.completed.includes("acsJoin") && this.completed.length === 0 && !this.running)) {
+                return;
+            }
         }
         const running = this.running;
-        if (this.completed.length === 0 && this.phase === "idle" && !running) {
+        if (this.completed.length === 0 && (this.phase === "idle" || this.phase === "failed") && !running) {
             // Nothing was built, so there is nothing to unwind — but a start() that has not run yet
             // still has to be refused, and a generation bump still has to invalidate anything holding
             // the old one.
@@ -517,11 +686,21 @@ export class SoftapCallTransport {
             return;
         }
         this.generation++;
-        this.phase = "stopping";
+        this.meetingPreserved = options.preserveMeeting === true;
+        this.phase = this.meetingPreserved ? "recovering" : "stopping";
         this.endFailure = null;
-        softapTrace("softap_call_stop", { steps: this.completed.join(","), mode: this.teardownMode });
+        softapTrace("softap_call_stop", {
+            steps: this.completed.join(","),
+            mode: this.teardownMode,
+            preserveMeeting: this.meetingPreserved,
+        });
         this.emitProgress();
         this.stopping = (async () => {
+            const inFlightRepublish = this.republishing;
+            if (inFlightRepublish) {
+                softapTrace("softap_stop_draining_republish");
+                await inFlightRepublish.catch(() => undefined);
+            }
             // The generation bump above has already told the in-flight step to release whatever it
             // produced. Waiting for that release is what makes a resolved `stop()` mean "nothing from
             // this call is still coming". Deliberately unbounded: a native call that never returns must
@@ -549,7 +728,13 @@ export class SoftapCallTransport {
             // The late step may have recorded a failed self-undo while we waited. Preserve it,
             // and any earlier teardown result, until start() explicitly begins a new attempt.
             const failures = [...this.teardownFailures];
+            const kept = [];
             for (const step of [...this.completed].reverse()) {
+                if (this.meetingPreserved && step === "acsJoin") {
+                    if (!kept.includes("acsJoin"))
+                        kept.unshift(step);
+                    continue;
+                }
                 const undoStartedAt = Date.now();
                 try {
                     await this.undo(step);
@@ -567,10 +752,10 @@ export class SoftapCallTransport {
                     });
                 }
             }
-            this.completed = [];
+            this.completed = kept;
             this.hotspot = null;
             this.ingestUrl = null;
-            this.phase = "idle";
+            this.phase = this.meetingPreserved && !this.recoveryAbort ? "recovering" : "idle";
             this.teardownFailures = failures;
             softapTrace("softap_call_stopped", { undoFailures: failures.join(",") });
             resetSoftapTrace();
@@ -602,6 +787,8 @@ export class SoftapCallTransport {
             case "publish":
                 return this.deps.stopPublishing();
             case "acsJoin":
+                if (this.meetingPreserved)
+                    return;
                 return this.leaveOrEndMeeting();
             case "scopedJoin":
                 return this.deps.leaveScopedNetwork();
@@ -690,7 +877,8 @@ export class SoftapCallTransport {
             await this.undoSafely(step);
             throw new SoftapCallError(step, "CANCELLED", `SoftAP call was cancelled during ${step}`);
         }
-        this.completed.push(step);
+        if (!this.completed.includes(step))
+            this.completed.push(step);
         softapTrace("softap_step_done", { step, durationMs: Date.now() - startedAt });
         this.setStep(step, { status: "done", durationMs: Date.now() - startedAt });
     }
@@ -756,11 +944,19 @@ export function createSoftapCallDeps(args) {
             catch (error) {
                 if (error instanceof Error && /no password/.test(error.message))
                     throw error;
+                const unanswered = isHotspotAnswerTimeout(error);
                 // Cancel-then-start races the previous disable: the glasses report disabled (or no SSID)
                 // and the UI said "Couldn't start glasses hotspot" before step 2 ran on a leftover AP.
                 softapTraceFailure("hotspot_enable_retry", {
                     reason: error instanceof Error ? error.message : String(error),
+                    unanswered,
                 });
+                if (unanswered) {
+                    // Enable is idempotent; disabling here is what turned a late-delivered enable back off
+                    // and cost the whole rebuild budget after a walk-away.
+                    report?.("Glasses did not answer; asking again without turning the hotspot off");
+                    return await enable();
+                }
                 report?.("Glasses hotspot did not start; turning it off and trying again");
                 await subsystems.setHotspotState(false);
                 if (hotspotBroadcastWaitMs > 0) {
@@ -859,7 +1055,8 @@ export function createSoftapCallDeps(args) {
         },
         leaveMeeting: () => subsystems.leaveMeeting(packageName),
         ...(subsystems.endMeeting ? { endMeeting: () => subsystems.endMeeting(packageName) } : {}),
-        startPublishing: async ({ ingestUrl, traceId }, report) => {
+        ...(subsystems.rebindIngest ? { rebindIngest: subsystems.rebindIngest } : {}),
+        startPublishing: async ({ ingestUrl, traceId, mediaGeneration }, report) => {
             // Narrate the glasses side while the BLE start command is in flight. `initializing` means
             // the glasses accepted the command and are opening the camera; `streaming` means the WHIP
             // offer was answered and ICE connected; anything else is the reason it did not.
@@ -907,6 +1104,7 @@ export function createSoftapCallDeps(args) {
                     ice: { stun: "" },
                     traceId,
                     captureAudio: !lc3Uplink,
+                    ...(typeof mediaGeneration === "number" ? { mediaGeneration } : {}),
                     ...(policy ? { video: policy } : {}),
                 });
             }
@@ -914,7 +1112,10 @@ export function createSoftapCallDeps(args) {
                 unsubscribe?.();
             }
         },
-        stopPublishing: () => subsystems.stopPublishing(packageName),
+        stopPublishing: async () => {
+            await subsystems.stopPublishing(packageName);
+            subsystems.discardPendingBleStop?.();
+        },
         awaitFirstFrame: args.awaitFirstFrame,
         waitUntilLive: args.waitUntilLive,
     };