@mentra/engine 0.1.0-beta.0

package/build/services/CloudClientService.js

@@ -0,0 +1,644 @@
+/**
+ * Owns the singleton `@mentra/cloud-client` (CloudClient) for the engine runtime
+ * — the keystone move that pulls backend comms INTO engine.
+ *
+ * engine constructs the client from engine-owned pieces (the native UDP socket,
+ * the MMKV secure store, the cloud-status store) plus the two things the host
+ * provides through the engine front door: `auth.getSubjectToken` (the one
+ * permanent seam — the OEM owns login) and the resolved cloud endpoints
+ * (`config.coreUrl/runtimeUrl`, computed host-side from dev/settings). On init it
+ * exposes cloud runtime methods directly from this service, so the host no longer
+ * injects a cloud adapter — engine owns construction and wiring.
+ *
+ * The local-miniapp path drives the cloud's transcription/translation through
+ * this service.
+ */
+import { CloudClient, setNativeHttp, setNativeUdp, setSecureStorage } from "@mentra/cloud-client/react-native";
+import { Platform } from "react-native";
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal";
+import CrustModule from "@mentra/crust";
+import { getAuth, getConfigValues } from "../runtime/bootstrap";
+import { useSettingsStore, SETTINGS } from "../stores/settings";
+import { createCloudUdpSocket } from "../utils/cloudClient/RnUdpAdapter";
+import { cloudSecureStore } from "../utils/cloudClient/cloudSecureStore";
+import { useCloudClientStatusStore } from "../stores/cloudClientStatus";
+import { islandNotifications } from "./NotificationsEmitter";
+import { BgTimer } from "../utils/timers";
+import { logCloudV2TranscriptMetric } from "./CloudTranscriptE2EMetrics";
+import { nativeHttpResponseBody } from "./NativeHttpResponse";
+const LOG_TAG = "cloudClient";
+// Persistent cloud-disconnect detector: the cloud-client reconnect loop is infinite
+// and never surfaces a "giving up" signal, so "persistent failure" = the session has
+// been continuously NOT connected for this long. A quick reconnect cancels it; brief
+// flaps that recover don't fire. Raised as engine.notifications(connection_failed_persistent).
+const CLOUD_PERSISTENT_FAILURE_MS = 60_000;
+/** Cancel the pending persistent-failure alarm + re-arm for the next outage. */
+function clearPersistentFailureAlarm() {
+    if (persistentFailureTimer) {
+        BgTimer.clearTimeout(persistentFailureTimer);
+        persistentFailureTimer = null;
+    }
+    persistentFailureNotified = false;
+}
+// Neutral last-ditch fallbacks (reachable under `adb reverse`) for when the host
+// passes no endpoints. The host normally resolves the real URLs (dev override /
+// Metro host / env) and hands them in via `engine.configure({config})`.
+const FALLBACK_CORE_URL = "http://localhost:3000";
+const FALLBACK_RUNTIME_URL = "http://localhost:3001";
+const LOCAL_AUTH_PORT = 3002;
+let client = null;
+let connected = false;
+let persistentFailureTimer = null;
+let persistentFailureNotified = false;
+let audioSubscriptions = [];
+let transportsReady = false;
+/** Endpoints to build with — seeded from config, overridable via reconnect(). */
+let endpointsOverride = null;
+let runtimeStatusUnsubscribe = null;
+let transcriptUnsubscribe = null;
+let translationUnsubscribe = null;
+let authStateUnsubscribe = null;
+let localDevRuntimeToken = null;
+let lc3FrameSizeUnsubscribe = null;
+let coreTokenSyncPromise = null;
+const transcriptListeners = new Set();
+const translationListeners = new Set();
+const statusListeners = new Set();
+const connectionListeners = new Set();
+function resolveEndpoints() {
+    if (endpointsOverride)
+        return endpointsOverride;
+    const cfg = getConfigValues();
+    return {
+        core: cfg.coreUrl?.trim() || FALLBACK_CORE_URL,
+        runtime: cfg.runtimeUrl?.trim() || FALLBACK_RUNTIME_URL,
+    };
+}
+function getCoreClient() {
+    if (!client)
+        construct();
+    const core = client?.core;
+    if (!core)
+        throw new Error("cloud client core not configured");
+    return core;
+}
+function syncCoreAccessTokenToBluetooth() {
+    if (coreTokenSyncPromise)
+        return coreTokenSyncPromise;
+    coreTokenSyncPromise = syncCoreAccessTokenToBluetoothInternal().finally(() => {
+        coreTokenSyncPromise = null;
+    });
+    return coreTokenSyncPromise;
+}
+async function syncCoreAccessTokenToBluetoothInternal() {
+    if (!client)
+        construct();
+    const c = client;
+    if (!c)
+        throw new Error("cloud client not initialized");
+    const token = await c.auth.getCoreToken();
+    if (c !== client) {
+        throw new Error("cloud client changed while syncing core token");
+    }
+    const result = await useSettingsStore.getState().setSetting(SETTINGS.core_token.key, token, false);
+    if (result.is_error()) {
+        throw result.error;
+    }
+    try {
+        await BluetoothSdk.updateBluetoothSettings({ [SETTINGS.core_token.key]: token });
+    }
+    catch (err) {
+        console.warn(`${LOG_TAG}: direct Bluetooth core_token sync failed: ${err?.message ?? err}`);
+    }
+    return token;
+}
+function frameSizeBytes() {
+    // Read LIVE from the engine settings store (the source of truth) so a
+    // reconnect after a glasses swap (G1=20 ↔ G2=40) picks up the new frame size,
+    // not the one-time engine.configure({config}) snapshot. Falls back to the
+    // config value (set at boot) then 20 when the setting is unset.
+    const fromSettings = useSettingsStore.getState().getSetting(SETTINGS.lc3_frame_size.key);
+    const size = fromSettings ?? getConfigValues().audioFrameSizeBytes;
+    return size === 20 || size === 40 || size === 60 ? size : 20;
+}
+async function getSubjectToken() {
+    const a = getAuth();
+    if (!a)
+        throw new Error("cloudClient: engine.configure({auth}) not called");
+    const r = await a.getSubjectToken();
+    // IslandAuth's SubjectTokenType is intentionally open (`string & {}`) so OEMs
+    // can use other token kinds; cloud-client's is a closed union. The host's
+    // actual value ("supabase") is valid — narrow at this boundary.
+    return { token: r.token, type: r.type };
+}
+function normalizeExpiresAt(expiresAt) {
+    return expiresAt < 10_000_000_000 ? expiresAt * 1000 : expiresAt;
+}
+function shouldUseLocalDevRuntimeToken(endpoints) {
+    if (!__DEV__)
+        return false;
+    try {
+        const host = new URL(endpoints.runtime).hostname;
+        return host === "localhost" || host === "127.0.0.1" || host === "10.0.2.2";
+    }
+    catch {
+        return false;
+    }
+}
+async function getLocalDevRuntimeToken(runtimeEndpoint, opts) {
+    const now = Date.now();
+    if (!opts?.forceRefresh &&
+        localDevRuntimeToken?.runtimeUrl === runtimeEndpoint &&
+        localDevRuntimeToken.expiresAtMs - now > 60_000) {
+        return localDevRuntimeToken.token;
+    }
+    const base = new URL(runtimeEndpoint);
+    base.port = String(LOCAL_AUTH_PORT);
+    base.pathname = "/api/dev/runtime-token";
+    base.search = new URLSearchParams({ userId: "local-phone-user", tenantId: "mentra" }).toString();
+    const res = await fetch(base.toString());
+    if (!res.ok) {
+        throw new Error(`cloudClient: local dev runtime token failed (${res.status})`);
+    }
+    const body = (await res.json());
+    if (!body.access_token) {
+        throw new Error("cloudClient: local dev runtime token response missing access_token");
+    }
+    localDevRuntimeToken = {
+        runtimeUrl: runtimeEndpoint,
+        token: body.access_token,
+        expiresAtMs: now + (body.expires_in ?? 300) * 1000,
+    };
+    return body.access_token;
+}
+function toCloudClientStatusSnapshot(snapshot) {
+    return { status: snapshot.status, audioTransport: snapshot.audioTransport };
+}
+function currentRuntimeStatus() {
+    const state = useCloudClientStatusStore.getState();
+    return { status: state.status, audioTransport: state.audioTransport };
+}
+function setRuntimeStatus(snapshot) {
+    useCloudClientStatusStore.getState().setSnapshot(snapshot);
+    emitStatus(toCloudClientStatusSnapshot(snapshot));
+}
+function resetRuntimeStatus() {
+    useCloudClientStatusStore.getState().reset();
+    emitStatus(currentRuntimeStatus());
+}
+function stringifyMeta(meta) {
+    if (!meta || typeof meta !== "object")
+        return "";
+    try {
+        return ` ${JSON.stringify(meta)}`;
+    }
+    catch {
+        return "";
+    }
+}
+const cloudLogger = {
+    debug: (msg, meta) => console.log(`${LOG_TAG}: debug: ${msg}${stringifyMeta(meta)}`),
+    info: (msg, meta) => console.log(`${LOG_TAG}: info: ${msg}${stringifyMeta(meta)}`),
+    warn: (msg, meta) => console.warn(`${LOG_TAG}: warn: ${msg}${stringifyMeta(meta)}`),
+    error: (msg, meta) => console.warn(`${LOG_TAG}: error: ${msg}${stringifyMeta(meta)}`),
+};
+function notifyConnectionListeners(next) {
+    for (const l of connectionListeners) {
+        try {
+            l(next);
+        }
+        catch (err) {
+            console.warn(`${LOG_TAG}: connection listener threw: ${err?.message ?? err}`);
+        }
+    }
+}
+function extractUnsubscribe(sub) {
+    const candidates = [];
+    const s = sub;
+    if (s && typeof s === "object") {
+        candidates.push(s);
+        candidates.push(s.value?.subscription);
+        candidates.push(s.data?.subscription);
+        candidates.push(s.subscription);
+    }
+    for (const c of candidates) {
+        const unsub = c?.unsubscribe;
+        if (typeof unsub === "function")
+            return unsub;
+    }
+    return null;
+}
+function ensureAuthWatch() {
+    if (authStateUnsubscribe)
+        return;
+    authStateUnsubscribe = () => { };
+    const auth = getAuth();
+    if (!auth?.onStateChange)
+        return;
+    let reconnectPending = false;
+    try {
+        const sub = auth.onStateChange((event, session) => {
+            if (!session?.token)
+                return;
+            if (connected || reconnectPending)
+                return;
+            reconnectPending = true;
+            console.log(`${LOG_TAG}: auth session available (${event}) while disconnected — reconnecting`);
+            try {
+                cloudClientService.reconnect();
+            }
+            finally {
+                setTimeout(() => {
+                    reconnectPending = false;
+                }, 0);
+            }
+        });
+        const unsub = extractUnsubscribe(sub);
+        if (unsub)
+            authStateUnsubscribe = unsub;
+    }
+    catch (err) {
+        console.warn(`${LOG_TAG}: auth watch subscribe failed: ${err?.message ?? err}`);
+    }
+}
+function stopAuthWatch() {
+    authStateUnsubscribe?.();
+    authStateUnsubscribe = null;
+}
+function emitTranscript(data) {
+    for (const l of transcriptListeners) {
+        try {
+            l(data);
+        }
+        catch (err) {
+            console.warn(`${LOG_TAG}: transcript listener threw: ${err?.message ?? err}`);
+        }
+    }
+}
+function emitTranslation(data) {
+    for (const l of translationListeners) {
+        try {
+            l(data);
+        }
+        catch (err) {
+            console.warn(`${LOG_TAG}: translation listener threw: ${err?.message ?? err}`);
+        }
+    }
+}
+function emitStatus(snapshot) {
+    for (const l of statusListeners) {
+        try {
+            l(snapshot);
+        }
+        catch (err) {
+            console.warn(`${LOG_TAG}: status listener threw: ${err?.message ?? err}`);
+        }
+    }
+}
+function ensureTransports() {
+    if (transportsReady)
+        return;
+    transportsReady = true;
+    setNativeUdp(() => createCloudUdpSocket());
+    setSecureStorage(cloudSecureStore);
+    if (Platform.OS !== "android")
+        return;
+    setNativeHttp(async (input, init) => {
+        if (typeof input !== "string" || (init?.body != null && typeof init.body !== "string")) {
+            return globalThis.fetch(input, init);
+        }
+        const headers = Object.fromEntries(new Headers(init?.headers).entries());
+        const result = await CrustModule.nativeHttpRequest(init?.method ?? "GET", input, headers, init?.body ?? null);
+        return new Response(nativeHttpResponseBody(result.status, result.body), {
+            status: result.status,
+            statusText: result.statusText,
+            headers: result.headers,
+        });
+    });
+}
+function clearRuntimeEventSubscriptions() {
+    runtimeStatusUnsubscribe?.();
+    runtimeStatusUnsubscribe = null;
+    transcriptUnsubscribe?.();
+    transcriptUnsubscribe = null;
+    translationUnsubscribe?.();
+    translationUnsubscribe = null;
+}
+function startLc3FrameSizeWatcher() {
+    if (lc3FrameSizeUnsubscribe)
+        return;
+    let last = frameSizeBytes();
+    lc3FrameSizeUnsubscribe = useSettingsStore.subscribe((state) => state.getSetting(SETTINGS.lc3_frame_size.key), () => {
+        const next = frameSizeBytes();
+        if (next === last)
+            return;
+        last = next;
+        if (!client)
+            return;
+        console.log(`${LOG_TAG}: LC3 frame size changed to ${next}; reconnecting runtime`);
+        cloudClientService.reconnect();
+    });
+}
+function stopLc3FrameSizeWatcher() {
+    lc3FrameSizeUnsubscribe?.();
+    lc3FrameSizeUnsubscribe = null;
+}
+function construct() {
+    ensureTransports();
+    const endpoints = resolveEndpoints();
+    console.log(`${LOG_TAG}: endpoints ${JSON.stringify(endpoints)}`);
+    const coreAuth = { getSubjectToken };
+    const auth = shouldUseLocalDevRuntimeToken(endpoints)
+        ? {
+            core: coreAuth,
+            runtime: { getToken: (opts) => getLocalDevRuntimeToken(endpoints.runtime, opts) },
+        }
+        : { core: coreAuth, runtime: { source: "core" } };
+    client = new CloudClient({
+        endpoints,
+        // The phone LC3-encodes mic audio (even in phone/simulated mode); announce
+        // LC3 at 16 kHz with the frame size the encoder emits.
+        audio: { codec: "lc3", sampleRate: 16000, frameSizeBytes: frameSizeBytes() },
+        auth,
+        logger: cloudLogger,
+    });
+    const c = client;
+    clearRuntimeEventSubscriptions();
+    runtimeStatusUnsubscribe = c.runtime.onStatusChanged((status) => {
+        if (c !== client)
+            return;
+        setRuntimeStatus(status);
+    });
+    transcriptUnsubscribe = c.runtime.onTranscript((data) => {
+        if (c !== client)
+            return;
+        logCloudV2TranscriptMetric(data);
+        emitTranscript(data);
+    });
+    translationUnsubscribe = c.runtime.onTranslation((data) => {
+        if (c !== client)
+            return;
+        emitTranslation(data);
+    });
+    setRuntimeStatus(c.runtime.getStatus());
+    c.runtime.onConnected(() => {
+        if (c !== client)
+            return;
+        connected = true;
+        console.log(`${LOG_TAG}: runtime connected`);
+        // Cloud is up — cancel any pending persistent-failure alarm and re-arm for the
+        // next outage.
+        clearPersistentFailureAlarm();
+        // Re-apply the authoritative subscription set, including the empty set, so
+        // a fresh phone launch clears any stale server-side audio subscriptions.
+        c.runtime
+            .setSubscriptions(audioSubscriptions)
+            .catch((err) => console.warn(`${LOG_TAG}: re-applying queued subscriptions failed: ${err?.message ?? err}`));
+        notifyConnectionListeners(true);
+    });
+    c.runtime.onDisconnected((info) => {
+        if (c !== client)
+            return;
+        connected = false;
+        console.log(`${LOG_TAG}: runtime disconnected (${info.reason})`);
+        // Arm the persistent-failure alarm once; if we're still down when it fires, raise
+        // the notification. A reconnect within the window cancels it (onConnected above).
+        if (!persistentFailureTimer && !persistentFailureNotified) {
+            persistentFailureTimer = BgTimer.setTimeout(() => {
+                persistentFailureTimer = null;
+                if (connected)
+                    return;
+                persistentFailureNotified = true;
+                islandNotifications.emit({
+                    kind: "connection_failed_persistent",
+                    reason: `Cloud session has been disconnected for over ${Math.round(CLOUD_PERSISTENT_FAILURE_MS / 1000)}s`,
+                    metadata: { downForMs: CLOUD_PERSISTENT_FAILURE_MS, lastReason: info.reason },
+                    timestamp: Date.now(),
+                });
+            }, CLOUD_PERSISTENT_FAILURE_MS);
+        }
+        notifyConnectionListeners(false);
+    });
+    c.runtime.onError((err) => {
+        console.warn(`${LOG_TAG}: runtime error: ${err.code}`);
+    });
+    // Best-effort connect. Do not crash the app if the dev cloud is unreachable.
+    c.runtime
+        .connect()
+        .then(() => console.log(`${LOG_TAG}: connect() resolved`))
+        .catch((err) => console.warn(`${LOG_TAG}: connect() failed: ${err?.message ?? err}`));
+    syncCoreAccessTokenToBluetooth().catch((err) => console.warn(`${LOG_TAG}: initial Bluetooth core_token sync failed: ${err?.message ?? err}`));
+    startLc3FrameSizeWatcher();
+}
+/**
+ * engine's cloud client. Owns the singleton CloudClient, its connection state,
+ * and the runtime-hook wiring.
+ */
+export const cloudClientService = {
+    /**
+     * Construct (once) + connect the client. Idempotent. Best-effort connect — a
+     * failure is logged and the app keeps running. Requires
+     * `engine.configure({auth, config})` first.
+     */
+    init() {
+        if (client)
+            return;
+        ensureAuthWatch();
+        construct();
+    },
+    /**
+     * Tear down + rebuild. Pass new endpoints to switch URLs; pass `null` to CLEAR
+     * a prior override and fall back to the boot config (so cleared/default cloud
+     * URLs don't keep reconnecting to a stale override); omit to keep the current.
+     */
+    reconnect(endpoints) {
+        if (endpoints !== undefined)
+            endpointsOverride = endpoints;
+        try {
+            client?.runtime.close();
+        }
+        catch (err) {
+            console.warn(`${LOG_TAG}: reconnect close() failed: ${err?.message ?? err}`);
+        }
+        clearRuntimeEventSubscriptions();
+        clearPersistentFailureAlarm();
+        const wasConnected = connected;
+        client = null;
+        localDevRuntimeToken = null;
+        connected = false;
+        resetRuntimeStatus();
+        if (wasConnected)
+            notifyConnectionListeners(false);
+        construct();
+    },
+    async getPreinstalledMiniappRegistry() {
+        if (!client)
+            this.init();
+        const c = client;
+        if (!c?.core)
+            throw new Error("cloud client core is unavailable");
+        return c.core.miniapps.getRegistry();
+    },
+    async getMiniappAuthToken(packageName, opts) {
+        if (!client)
+            this.init();
+        const c = client;
+        if (!c)
+            throw new Error("cloud client not initialized");
+        const { token, expiresAt } = await c.auth.getMiniappToken(packageName, opts);
+        const identity = c.auth.identity;
+        return {
+            mentraUserId: identity.mentraUserId,
+            tenantId: identity.tenantId,
+            token,
+            expiresAt: normalizeExpiresAt(expiresAt),
+        };
+    },
+    /** Tear down the client + connection (the engine.stop() lifecycle). */
+    stop() {
+        try {
+            client?.runtime.close();
+        }
+        catch (err) {
+            console.warn(`${LOG_TAG}: stop close() failed: ${err?.message ?? err}`);
+        }
+        stopLc3FrameSizeWatcher();
+        stopAuthWatch();
+        clearRuntimeEventSubscriptions();
+        clearPersistentFailureAlarm();
+        const wasConnected = connected;
+        client = null;
+        localDevRuntimeToken = null;
+        connected = false;
+        resetRuntimeStatus();
+        if (wasConnected)
+            notifyConnectionListeners(false);
+    },
+    /** Device-side managed photo (cloud-v2): presign now, deliver bytes, await ready. */
+    startManagedPhoto(opts = {}) {
+        if (!client)
+            throw new Error("cloud client not connected");
+        return client.runtime.startManagedPhoto(opts);
+    },
+    awaitManagedPhotoReady(requestId) {
+        if (!client)
+            throw new Error("cloud client not connected");
+        return client.runtime.awaitManagedPhotoReady(requestId);
+    },
+    /** Managed stream (cloud-v2): provision ingest+playback on the runtime. */
+    startManagedStream(opts = {}) {
+        if (!client)
+            throw new Error("cloud client not connected");
+        return client.runtime.startManagedStream(opts);
+    },
+    getManagedStreamStatus(streamId) {
+        if (!client)
+            throw new Error("cloud client not connected");
+        return client.runtime.getManagedStreamStatus(streamId);
+    },
+    stopManagedStream(streamId) {
+        if (!client)
+            throw new Error("cloud client not connected");
+        return client.runtime.stopManagedStream(streamId);
+    },
+    /** Current live-session connection state (handshake completed). */
+    isConnected() {
+        return connected;
+    },
+    /** Subscribe to connection-state transitions. Returns an unsubscribe fn. */
+    onConnectionChange(listener) {
+        connectionListeners.add(listener);
+        return () => {
+            connectionListeners.delete(listener);
+        };
+    },
+    /** Replace the v2 cloud's audio subscription set for the live session. */
+    async setSubscriptions(subs) {
+        // Cache unconditionally so the desired set survives a not-yet-connected
+        // session and reconnects; the onConnected handler re-applies the cached set.
+        audioSubscriptions = subs;
+        const c = client;
+        if (connected && c) {
+            await c.runtime.setSubscriptions(subs);
+        }
+    },
+    sendAudioFrame(frame) {
+        client?.runtime.sendAudioFrame(frame);
+    },
+    onTranscript(cb) {
+        transcriptListeners.add(cb);
+        return () => {
+            transcriptListeners.delete(cb);
+        };
+    },
+    onTranslation(cb) {
+        translationListeners.add(cb);
+        return () => {
+            translationListeners.delete(cb);
+        };
+    },
+    getStatus() {
+        return currentRuntimeStatus();
+    },
+    getCoreUrl() {
+        return resolveEndpoints().core;
+    },
+    syncCoreTokenToBluetooth() {
+        return syncCoreAccessTokenToBluetooth();
+    },
+    onStatusChanged(cb) {
+        statusListeners.add(cb);
+        return () => {
+            statusListeners.delete(cb);
+        };
+    },
+    tts: {
+        speak(text, options) {
+            if (!client)
+                throw new Error("cloud client not connected");
+            return client.runtime.tts.speak(text, options);
+        },
+    },
+    maps: {
+        directions(req) {
+            if (!client)
+                throw new Error("cloud client not connected");
+            return client.runtime.maps.directions(req);
+        },
+        reverseGeocode(coord) {
+            if (!client)
+                throw new Error("cloud client not connected");
+            return client.runtime.maps.reverseGeocode(coord);
+        },
+        placeAutocomplete(req) {
+            if (!client)
+                throw new Error("cloud client not connected");
+            return client.runtime.maps.placeAutocomplete(req);
+        },
+        placeDetails(req) {
+            if (!client)
+                throw new Error("cloud client not connected");
+            return client.runtime.maps.placeDetails(req);
+        },
+    },
+    core: {
+        reports: {
+            submit(...args) {
+                return getCoreClient().reports.submit(...args);
+            },
+            addLogs(...args) {
+                return getCoreClient().reports.addLogs(...args);
+            },
+            addScreenshots(...args) {
+                return getCoreClient().reports.addScreenshots(...args);
+            },
+            complete(...args) {
+                return getCoreClient().reports.complete(...args);
+            },
+        },
+    },
+    hasAudioSubscriptions() {
+        return audioSubscriptions.length > 0;
+    },
+};
+//# sourceMappingURL=CloudClientService.js.map