@glassly/engine 0.3.3-dev.670 → 0.3.4-dev.674

package/build/services/LocalMiniappRuntime.js

@@ -34,6 +34,9 @@ import { markTranscriptSourceRecent, useTranscriptionDebugStore } from "../store
 import { resolveMapboxRenderToken } from "./MapboxTokenService";
 import { createAnthropicProvider, createGeminiProvider, createOpenAiProvider, } from "@glassly/llm-providers";
 import { normalizeAiTier, resolveAiTier } from "./AiTierResolution";
+import { decideLlmRoute, localLlmReason } from "./LlmRouting";
+import localLlmModelManager from "./LocalLlmModelManager";
+import localLlmProvider from "./LocalLlmProvider";
 import { HttpError } from "@glassly/cloud-client";
 import localDisplayManager from "./LocalDisplayManager";
 import sttModelManager from "./STTModelManager";
@@ -45,7 +48,8 @@ import dashboardOpenTrace from "./debug/DashboardOpenTrace";
 import { isAppForegrounded, isPhoneActive, onAppForegroundChange, onPhoneActivityChange } from "./PhoneActivity";
 import { installedNotificationApps } from "./PhoneNotificationsSync";
 import { MiniappIcons } from "./MiniappIcons";
-import { DERIVED_SETTINGS_KEYS, SETTINGS_BRIDGE_KEYS, input2dFrom, isDerivedSettingsKey, ringBatteryFrom, selectSettingsBridgeKeys, validateSettingsSet, } from "./settingsBridge";
+import { pairedControllersFrom, readyControllersKey } from "./controllers";
+import { DERIVED_SETTINGS_KEYS, SETTINGS_BRIDGE_KEYS, controllersSettingFrom, isDerivedSettingsKey, selectSettingsBridgeKeys, validateSettingsSet, } from "./settingsBridge";
 import { mayUsePermission } from "./miniappPermissions";
 import { activeDashboard, addDashboardApp, displayViewsFor, isDashboardApp, isOverlayApp, onOverlayAppsChange, removeDashboardApp, setOverlayViewer, } from "./pages/overlayApps";
 import { pagesEligible } from "./pages/pageEligibility";
@@ -262,6 +266,7 @@ function directLlmProvider(name) {
     directLlmProviders.set(name, built);
     return built;
 }
+const QUOTA_EXCEEDED_MESSAGE = "Monthly AI credit is used up. Upgrade in Settings > Membership, or add your own API key in Settings > API Keys.";
 /** The user's saved key (Settings > API Keys) for an LLM provider, "" if none. */
 function userLlmKey(provider) {
     const raw = useSettingsStore.getState().getSetting(MINIAPP_READABLE_API_KEYS[provider]);
@@ -2201,11 +2206,10 @@ class LocalMiniappRuntime {
                 return isPhoneActive();
             case "foreground_app_on_screen":
                 return localDisplayManager.foregroundAppOnScreen();
-            case "ring_battery":
-                return ringBatteryFrom(useGlassesStore.getState());
-            case "input_2d": {
-                const controller = useSettingsStore.getState().getSetting(SETTINGS.default_controller.key);
-                return input2dFrom(useGlassesStore.getState(), controller);
+            case "controllers": {
+                const glasses = useGlassesStore.getState();
+                const records = pairedControllersFrom(useSettingsStore.getState().getSetting(SETTINGS.controllers.key));
+                return controllersSettingFrom(records, glasses.controllers, glasses);
             }
         }
     }
@@ -2254,7 +2258,7 @@ class LocalMiniappRuntime {
     /**
      * Push bridged settings as they change: stored keys from the store, the
      * derived keys whenever their inputs move (phone activity, the display,
-     * the apps store, the ring, the dashboard role). Subscribers declared
+     * the apps store, the controllers, the dashboard role). Subscribers declared
      * PLATFORM_SETTINGS (gated at SUBSCRIBE), so this is a no-op fan-out for
      * everyone else.
      */
@@ -2292,8 +2296,8 @@ class LocalMiniappRuntime {
         localDisplayManager.onActiveDisplayChange(pushDerived);
         useAppStatusStore.subscribe(pushDerived);
         onOverlayAppsChange(pushDerived);
-        useGlassesStore.subscribe((s) => `${s.ringConnected}:${s.ringBatteryLevel}:${s.controllerConnected}:${s.controllerBatteryLevel}`, pushDerived);
-        useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.default_controller.key), pushDerived);
+        useGlassesStore.subscribe((s) => `${s.ringConnected}:${s.ringBatteryLevel}:${s.ringCharging}:${JSON.stringify(s.controllers ?? {})}`, pushDerived);
+        useSettingsStore.subscribe((s) => JSON.stringify(s.getSetting(SETTINGS.controllers.key) ?? null), pushDerived);
     }
     /**
      * Native nav UI feed (display.sendNativeNav*). Gated on the device's nav
@@ -3529,15 +3533,29 @@ class LocalMiniappRuntime {
         // here (the user's Settings > AI Models dropdowns, or "auto" from which
         // keys are saved — see AiTierResolution). Legacy provider/model pins from
         // pre-tier miniapps are deliberately stripped, not honored.
-        const { tier: rawTier, provider: _legacyProvider, model: _legacyModel, ...neutralRequest } = request;
+        const { tier: rawTier, provider: _legacyProvider, model: _legacyModel, forceLocal: rawForceLocal, localFallback: rawLocalFallback, ...neutralRequest } = request;
         const tier = normalizeAiTier(rawTier);
         const resolved = resolveAiTier(tier, tierSettingValue(tier), savedLlmKeys());
+        // ON-DEVICE ROUTING — the request's forceLocal / localFallback flags
+        // against the primary path's health (see LlmRouting). The primary is the
+        // vendor-direct call when a user key is saved, else the cloud relay.
+        const directKey = resolved.provider ? userLlmKey(resolved.provider) : "";
+        const forceLocal = rawForceLocal === true;
+        const localFallback = rawLocalFallback === true;
+        const cloudConnected = cloudClientService.isConnected();
+        const cloudDisabled = cloudClientService.isCloudDisabled();
+        const primaryUp = directKey.length > 0 || (cloudConnected && !cloudDisabled && !this.llmQuotaBlocked);
+        const route = decideLlmRoute({ forceLocal, localFallback, primaryUp });
+        if (route === "local") {
+            const reason = localLlmReason({ forceLocal, cloudDisabled, cloudConnected, quotaBlocked: this.llmQuotaBlocked });
+            await this.serveLocalAi(packageName, requestId, neutralRequest, reason, null);
+            return;
+        }
         try {
             // Direct when the resolved provider has a user key (BYO bill, unmetered,
             // works with the cloud disabled); otherwise relay to the cloud LLM
             // service, forwarding the resolved pin so platform-funded calls land on
             // the same model the user chose.
-            const directKey = resolved.provider ? userLlmKey(resolved.provider) : "";
             if (resolved.provider && directKey) {
                 const result = await directLlmProvider(resolved.provider).complete({
                     ...neutralRequest,
@@ -3560,17 +3578,30 @@ class LocalMiniappRuntime {
             // is spent. Distinct code (miniapps may fall back to a local path) and
             // a host hook so the app can offer the upgrade paywall — the miniapp
             // itself must never own that conversation.
-            if (err instanceof HttpError && (err.code === "quota_exceeded" || err.status === 402)) {
+            const quotaExceeded = err instanceof HttpError && (err.code === "quota_exceeded" || err.status === 402);
+            if (quotaExceeded && !this.llmQuotaBlocked) {
                 console.warn(`${LOG_TAG}: ai.complete quota exceeded for ${packageName}`);
                 this.llmQuotaBlocked = true;
                 this.onQuotaExceeded?.("llm");
+            }
+            const message = err instanceof Error ? err.message : "ai.complete failed";
+            // localFallback: the same request, on-device. The primary's error
+            // stands only when the phone cannot serve it either.
+            if (route === "primary_then_local") {
+                console.warn(`${LOG_TAG}: ai.complete primary failed for ${packageName} (${message}); trying on-device`);
+                await this.serveLocalAi(packageName, requestId, neutralRequest, quotaExceeded ? "llm_quota" : "primary_failed", {
+                    code: quotaExceeded ? MiniappErrorCode.QUOTA_EXCEEDED : MiniappErrorCode.INTERNAL,
+                    message: quotaExceeded ? QUOTA_EXCEEDED_MESSAGE : message,
+                });
+                return;
+            }
+            if (quotaExceeded) {
                 this.sendResult(packageName, requestId, false, undefined, {
                     code: MiniappErrorCode.QUOTA_EXCEEDED,
-                    message: "Monthly AI credit is used up. Upgrade in Settings > Membership, or add your own API key in Settings > API Keys.",
+                    message: QUOTA_EXCEEDED_MESSAGE,
                 });
                 return;
             }
-            const message = err instanceof Error ? err.message : "ai.complete failed";
             console.warn(`${LOG_TAG}: ai.complete failed for ${packageName}: ${message}`);
             this.sendResult(packageName, requestId, false, undefined, {
                 code: MiniappErrorCode.INTERNAL,
@@ -3578,6 +3609,50 @@ class LocalMiniappRuntime {
             });
         }
     }
+    /**
+     * Serve one ai.complete on the phone's own model (LocalLlmProvider). With
+     * no model on disk the download starts (or continues) in the background
+     * and the miniapp gets AI_LOCAL_UNAVAILABLE naming the state — unless
+     * `primaryError` is given (a failed fallback), in which case the primary's
+     * own error is what the miniapp sees.
+     */
+    async serveLocalAi(packageName, requestId, request, reason, primaryError) {
+        const modelId = localLlmModelManager.selectedModelId();
+        if (!(await localLlmModelManager.ensureSelectedModel())) {
+            const status = localLlmModelManager.getStatus();
+            const state = status?.modelId === modelId && status.stage === "downloading"
+                ? `is downloading (${status.percent}%)`
+                : status?.modelId === modelId && status.stage === "failed"
+                    ? `failed to download (${status.error ?? "unknown error"})`
+                    : "is not downloaded yet";
+            console.warn(`${LOG_TAG}: ai.complete on-device for ${packageName} (${reason}): model ${modelId} ${state}`);
+            this.sendResult(packageName, requestId, false, undefined, primaryError ?? {
+                code: MiniappErrorCode.AI_LOCAL_UNAVAILABLE,
+                message: `The on-device model (${localLlmModelManager.model(modelId).label}) ${state}. Manage it in Settings > AI Models.`,
+            });
+            return;
+        }
+        try {
+            // webSearch is a vendor server-side grant; the phone has none.
+            const { webSearch: _webSearch, ...localRequest } = request;
+            const result = await localLlmProvider.complete(localRequest);
+            console.log(`${LOG_TAG}: ai.complete served on-device for ${packageName} (${reason}, ${result.modelId})`);
+            this.sendResult(packageName, requestId, true, {
+                text: result.text,
+                stopReason: result.stopReason,
+                ...(result.toolCalls ? { toolCalls: result.toolCalls } : {}),
+                usage: result.usage,
+                provider: "device",
+                model: result.modelId,
+                billedToUserKey: false,
+            });
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : "on-device completion failed";
+            console.warn(`${LOG_TAG}: ai.complete on-device failed for ${packageName}: ${message}`);
+            this.sendResult(packageName, requestId, false, undefined, primaryError ?? { code: MiniappErrorCode.AI_LOCAL_UNAVAILABLE, message });
+        }
+    }
     /**
      * ai.generate — agentic visual generation. Always relayed to the cloud
      * runtime (the server drives the provider's agentic loop; there is no
@@ -4640,9 +4715,8 @@ class LocalMiniappRuntime {
         this.capabilitiesWired = true;
         const broadcast = () => this.broadcastCapabilities();
         // Connection, model and firmware also decide the dashboard view's eligibility;
-        // a connected Neural Band adds its swipe2d surface to `button`.
-        useGlassesStore.subscribe((s) => `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}:${s.controllerConnected}`, broadcast);
-        useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.default_controller.key), broadcast);
+        // connected controllers add their input surfaces to `button`.
+        useGlassesStore.subscribe((s) => `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}:${readyControllersKey(s.controllers)}`, broadcast);
         // The debug knob flips hasNativeNavigation on custom firmware.
         useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.cfw_native_navigation.key), broadcast);
         // The dashboard master switch flips the dashboard view.