@glassly/engine 0.3.3-dev.669 → 0.3.4-dev.674

package/src/services/LocalLlmProvider.ts

@@ -0,0 +1,289 @@
+/**
+ * LocalLlmProvider — session.ai.complete served by the phone itself: an
+ * ExecuTorch runner (react-native-executorch) over the Qwen3 export that
+ * LocalLlmModelManager downloaded. Stateless per request like the vendor
+ * providers: every call renders the whole conversation through the model's
+ * chat template, resets the KV cache, and generates on ExecuTorch's worklet
+ * thread so the JS thread stays free.
+ *
+ * The runner is loaded on the first request and kept resident while calls
+ * keep coming (loading a 1–3 GB model is seconds, not milliseconds); it is
+ * dropped after an idle period and whenever the selected model changes.
+ * Requests are serialized — one model, one KV cache.
+ *
+ * react-native-executorch is imported lazily so the engine module graph
+ * loads without the native runtime (bun tests, hosts without the dependency).
+ */
+import * as RNFS from "@dr.pogodin/react-native-fs"
+import {Template} from "@huggingface/jinja"
+
+import type {LlmCompleteRequest} from "@glassly/cloud-protocol/llm"
+import localLlmModelManager, {type LocalLlmModelId} from "./LocalLlmModelManager"
+import {
+  buildTemplateMessages,
+  estimateTokens,
+  finalizeLocalResult,
+  fitToBudget,
+  toTemplateTools,
+  type LocalCompletion,
+} from "./LocalLlmPrompt"
+import {SETTINGS, useSettingsStore} from "../stores/settings"
+
+type Rne = typeof import("react-native-executorch")
+// The runner API lives on the `llm` namespace; derive its types from there so
+// the import stays a type-only reference (no native module at load time).
+type LLMRunner = ReturnType<Rne["llm"]["createLLMRunner"]>
+type LLMGenerationStats = ReturnType<LLMRunner["generate"]>
+type LLMGenerationConfig = NonNullable<Parameters<LLMRunner["generate"]>[1]>
+
+const LOG_TAG = "LOCAL_LLM"
+
+/** Drop the resident model after this long without a request. */
+const IDLE_UNLOAD_MS = 5 * 60_000
+/** Hard stop on one generation — the SDK request times out at 120 s anyway. */
+const GENERATION_DEADLINE_MS = 100_000
+const DEFAULT_MAX_NEW_TOKENS = 512
+const MAX_NEW_TOKENS_CEILING = 1_024
+/** Headroom between the estimated prompt and the context end; estimates are rough. */
+const CONTEXT_MARGIN_TOKENS = 96
+
+interface LoadedModel {
+  modelId: LocalLlmModelId
+  runner: LLMRunner
+  template: Template
+  stopTokens: string[]
+  maxSeqLen: number
+  generate: (
+    runner: LLMRunner,
+    prompt: string,
+    config: LLMGenerationConfig,
+    stopTokens: string[],
+  ) => Promise<{text: string; stats: LLMGenerationStats}>
+}
+
+export interface LocalLlmCompleteResult extends LocalCompletion {
+  modelId: LocalLlmModelId
+}
+
+/**
+ * Runs on ExecuTorch's worklet runtime: `runner.generate` blocks its thread
+ * until EOS, `maxNewTokens`, or `runner.stop()` from the JS thread.
+ */
+function generateWorklet(
+  runner: LLMRunner,
+  prompt: string,
+  config: LLMGenerationConfig,
+  stopTokens: string[],
+): {text: string; stats: LLMGenerationStats} {
+  "worklet"
+  let text = ""
+  const stats = runner.generate(prompt, config, (token: string) => {
+    if (stopTokens.includes(token)) return
+    text += token
+  })
+  return {text, stats}
+}
+
+class LocalLlmProvider {
+  private static instance: LocalLlmProvider
+
+  private loaded: LoadedModel | null = null
+  private loading: Promise<LoadedModel> | null = null
+  private queue: Promise<unknown> = Promise.resolve()
+  private idleTimer: ReturnType<typeof setTimeout> | null = null
+  private rne: Rne | null = null
+
+  private constructor() {
+    // A different model picked in Settings invalidates the resident runner.
+    useSettingsStore.subscribe(
+      (state) => state.settings[SETTINGS.local_llm_model.key],
+      () => {
+        void this.unload()
+      },
+    )
+  }
+
+  static getInstance(): LocalLlmProvider {
+    if (!LocalLlmProvider.instance) LocalLlmProvider.instance = new LocalLlmProvider()
+    return LocalLlmProvider.instance
+  }
+
+  /** Whether a runner is resident right now (diagnostics; not a readiness check). */
+  isLoaded(): boolean {
+    return this.loaded !== null
+  }
+
+  loadedModelId(): LocalLlmModelId | null {
+    return this.loaded?.modelId ?? null
+  }
+
+  /** Serve one neutral completion on-device. Throws when the model is missing or generation fails. */
+  complete(request: LlmCompleteRequest): Promise<LocalLlmCompleteResult> {
+    const run = this.queue.then(() => this.completeInner(request))
+    this.queue = run.catch(() => undefined)
+    return run
+  }
+
+  /** Release the resident runner (idle, model switch, memory pressure). Safe to call repeatedly. */
+  async unload(): Promise<void> {
+    this.clearIdleTimer()
+    // Never dispose under a running generation: let the queue drain first.
+    await this.queue
+    const pending = this.loading
+    if (pending) {
+      try {
+        await pending
+      } catch {
+        // A failed load left nothing to release.
+      }
+    }
+    this.releaseRunner()
+  }
+
+  /** Synchronous dispose; callers guarantee no generation is in flight. */
+  private releaseRunner(): void {
+    const loaded = this.loaded
+    this.loaded = null
+    if (!loaded) return
+    try {
+      loaded.runner.dispose()
+    } catch (error) {
+      console.warn(`${LOG_TAG}: runner dispose failed:`, error)
+    }
+  }
+
+  private async completeInner(request: LlmCompleteRequest): Promise<LocalLlmCompleteResult> {
+    this.clearIdleTimer()
+    try {
+      const model = await this.ensureLoaded()
+      const jsonMode = request.jsonMode === true
+      const tools = toTemplateTools(request.tools)
+      const toolTokens = tools ? estimateTokens(JSON.stringify(tools)) + 64 : 0
+
+      const maxNewTokens = Math.max(
+        16,
+        Math.min(request.maxTokens ?? DEFAULT_MAX_NEW_TOKENS, MAX_NEW_TOKENS_CEILING, Math.floor(model.maxSeqLen / 4)),
+      )
+      const promptBudget = model.maxSeqLen - maxNewTokens - CONTEXT_MARGIN_TOKENS
+      const messages = fitToBudget(buildTemplateMessages({...request, jsonMode}), promptBudget, toolTokens)
+
+      const prompt = model.template.render({
+        messages,
+        tools,
+        add_generation_prompt: true,
+        enable_thinking: false,
+      })
+
+      model.runner.reset(0)
+      const deadline = setTimeout(() => {
+        console.warn(`${LOG_TAG}: generation deadline hit, stopping`)
+        model.runner.stop()
+      }, GENERATION_DEADLINE_MS)
+      let generated: {text: string; stats: LLMGenerationStats}
+      const startedAt = Date.now()
+      try {
+        generated = await model.generate(
+          model.runner,
+          prompt,
+          {maxNewTokens, temperature: request.temperature ?? 0.7},
+          model.stopTokens,
+        )
+      } finally {
+        clearTimeout(deadline)
+      }
+
+      const elapsedMs = Date.now() - startedAt
+      console.log(
+        `${LOG_TAG}: ${model.modelId} prompt=${generated.stats.numPromptTokens} out=${generated.stats.numGeneratedTokens} tokens in ${elapsedMs} ms`,
+      )
+
+      const seed = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`
+      const completion = finalizeLocalResult(
+        {
+          text: generated.text,
+          promptTokens: generated.stats.numPromptTokens,
+          generatedTokens: generated.stats.numGeneratedTokens,
+          maxNewTokens,
+        },
+        {jsonMode, toolsOffered: Boolean(tools), makeToolCallId: (index) => `local_${seed}_${index}`},
+      )
+      return {...completion, modelId: model.modelId}
+    } finally {
+      this.armIdleTimer()
+    }
+  }
+
+  private async ensureLoaded(): Promise<LoadedModel> {
+    const modelId = localLlmModelManager.selectedModelId()
+    if (this.loaded?.modelId === modelId) return this.loaded
+    if (this.loading) return this.loading
+
+    this.loading = this.load(modelId)
+      .then((model) => {
+        this.loaded = model
+        return model
+      })
+      .finally(() => {
+        this.loading = null
+      })
+    return this.loading
+  }
+
+  private async load(modelId: LocalLlmModelId): Promise<LoadedModel> {
+    // Runs inside the request queue, so nothing is generating; unload() would
+    // wait on that same queue.
+    this.releaseRunner()
+    if (!(await localLlmModelManager.isModelAvailable(modelId))) {
+      throw new Error(`on-device model ${modelId} is not downloaded`)
+    }
+    const rne = await this.runtime()
+    const paths = localLlmModelManager.paths(modelId)
+
+    const tokenizerConfig = rne.llm.parseTokenizerConfig(JSON.parse(await RNFS.readFile(paths.tokenizerConfig, "utf8")))
+    const template = new Template(tokenizerConfig.chatTemplate)
+
+    const startedAt = Date.now()
+    const createRunner = rne.wrapAsync(rne.llm.createLLMRunner)
+    const runner = await createRunner(paths.model, paths.tokenizer, [])
+    const maxSeqLen = runner.getKVCacheState().maxSeqLen
+    console.log(`${LOG_TAG}: loaded ${modelId} (context ${maxSeqLen}) in ${Date.now() - startedAt} ms`)
+
+    return {
+      modelId,
+      runner,
+      template,
+      stopTokens: [...tokenizerConfig.stopTokens],
+      maxSeqLen,
+      generate: rne.wrapAsync(generateWorklet),
+    }
+  }
+
+  private async runtime(): Promise<Rne> {
+    if (this.rne) return this.rne
+    const rne: Rne = await import("react-native-executorch")
+    // Software Mansion's anonymous download analytics; the host has its own telemetry.
+    rne.setTelemetryEnabled(false)
+    this.rne = rne
+    return rne
+  }
+
+  private armIdleTimer(): void {
+    this.clearIdleTimer()
+    this.idleTimer = setTimeout(() => {
+      this.idleTimer = null
+      console.log(`${LOG_TAG}: idle, unloading ${this.loaded?.modelId ?? "nothing"}`)
+      void this.unload()
+    }, IDLE_UNLOAD_MS)
+  }
+
+  private clearIdleTimer(): void {
+    if (this.idleTimer) {
+      clearTimeout(this.idleTimer)
+      this.idleTimer = null
+    }
+  }
+}
+
+const localLlmProvider = LocalLlmProvider.getInstance()
+export {LocalLlmProvider}
+export default localLlmProvider

package/src/services/LocalMiniappRuntime.ts

@@ -53,8 +53,17 @@ import {
 
 import {normalizeAiTier, resolveAiTier} from "./AiTierResolution"
 import type {AiTier, AiTierProvider, SavedLlmKeys} from "./AiTierResolution"
+import {decideLlmRoute, localLlmReason, type LocalLlmReason} from "./LlmRouting"
+import localLlmModelManager from "./LocalLlmModelManager"
+import localLlmProvider from "./LocalLlmProvider"
 import {HttpError} from "@glassly/cloud-client"
-import type {QuotaStatusPayload} from "@glassly/cloud-protocol"
+import type {
+  QuotaStatusPayload,
+  AudioSubscription,
+  LanguageSource,
+  TranscriptionData,
+  TranslationData,
+} from "@glassly/cloud-protocol"
 import localDisplayManager from "./LocalDisplayManager"
 import sttModelManager from "./STTModelManager"
 import localTranslationService, {type LocalTranslationSubscription} from "./LocalTranslationService"
@@ -73,12 +82,12 @@ import dashboardOpenTrace from "./debug/DashboardOpenTrace"
 import {isAppForegrounded, isPhoneActive, onAppForegroundChange, onPhoneActivityChange} from "./PhoneActivity"
 import {installedNotificationApps} from "./PhoneNotificationsSync"
 import {MiniappIcons} from "./MiniappIcons"
+import {pairedControllersFrom, readyControllersKey} from "./controllers"
 import {
   DERIVED_SETTINGS_KEYS,
   SETTINGS_BRIDGE_KEYS,
-  input2dFrom,
+  controllersSettingFrom,
   isDerivedSettingsKey,
-  ringBatteryFrom,
   selectSettingsBridgeKeys,
   validateSettingsSet,
   type DerivedSettingsKey,
@@ -136,7 +145,6 @@ import {invokeScanQrSeam} from "../runtime/scanQrSeam"
 import {normalizeStreamAudioConfig, normalizeStreamVideoConfig} from "../runtime/streamConfig"
 import {visibleActionRows} from "./manifestActions"
 import {toLanguageHint} from "@glassly/cloud-protocol/languages"
-import type {AudioSubscription, LanguageSource, TranscriptionData, TranslationData} from "@glassly/cloud-protocol"
 import {buildMiniappManifestSnapshot, type MiniappRuntimeDiagnosticSnapshot} from "../utils/miniappDiagnostics"
 import ttsModelManager from "./TTSModelManager"
 import {NavigationHandlers} from "./NavigationHandlers"
@@ -416,6 +424,9 @@ function directLlmProvider(name: AiTierProvider): LlmProviderImpl {
   return built
 }
 
+const QUOTA_EXCEEDED_MESSAGE =
+  "Monthly AI credit is used up. Upgrade in Settings > Membership, or add your own API key in Settings > API Keys."
+
 /** The user's saved key (Settings > API Keys) for an LLM provider, "" if none. */
 function userLlmKey(provider: AiTierProvider): string {
   const raw = useSettingsStore.getState().getSetting(MINIAPP_READABLE_API_KEYS[provider])
@@ -2560,11 +2571,10 @@ class LocalMiniappRuntime {
         return isPhoneActive()
       case "foreground_app_on_screen":
         return localDisplayManager.foregroundAppOnScreen()
-      case "ring_battery":
-        return ringBatteryFrom(useGlassesStore.getState())
-      case "input_2d": {
-        const controller = useSettingsStore.getState().getSetting(SETTINGS.default_controller.key)
-        return input2dFrom(useGlassesStore.getState(), controller)
+      case "controllers": {
+        const glasses = useGlassesStore.getState()
+        const records = pairedControllersFrom(useSettingsStore.getState().getSetting(SETTINGS.controllers.key))
+        return controllersSettingFrom(records, glasses.controllers, glasses)
       }
     }
   }
@@ -2619,7 +2629,7 @@ class LocalMiniappRuntime {
   /**
    * Push bridged settings as they change: stored keys from the store, the
    * derived keys whenever their inputs move (phone activity, the display,
-   * the apps store, the ring, the dashboard role). Subscribers declared
+   * the apps store, the controllers, the dashboard role). Subscribers declared
    * PLATFORM_SETTINGS (gated at SUBSCRIBE), so this is a no-op fan-out for
    * everyone else.
    */
@@ -2659,10 +2669,10 @@ class LocalMiniappRuntime {
     useAppStatusStore.subscribe(pushDerived)
     onOverlayAppsChange(pushDerived)
     useGlassesStore.subscribe(
-      (s) => `${s.ringConnected}:${s.ringBatteryLevel}:${s.controllerConnected}:${s.controllerBatteryLevel}`,
+      (s) => `${s.ringConnected}:${s.ringBatteryLevel}:${s.ringCharging}:${JSON.stringify(s.controllers ?? {})}`,
       pushDerived,
     )
-    useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.default_controller.key), pushDerived)
+    useSettingsStore.subscribe((s) => JSON.stringify(s.getSetting(SETTINGS.controllers.key) ?? null), pushDerived)
   }
 
   /**
@@ -4063,16 +4073,39 @@ class LocalMiniappRuntime {
     // here (the user's Settings > AI Models dropdowns, or "auto" from which
     // keys are saved — see AiTierResolution). Legacy provider/model pins from
     // pre-tier miniapps are deliberately stripped, not honored.
-    const {tier: rawTier, provider: _legacyProvider, model: _legacyModel, ...neutralRequest} = request
+    const {
+      tier: rawTier,
+      provider: _legacyProvider,
+      model: _legacyModel,
+      forceLocal: rawForceLocal,
+      localFallback: rawLocalFallback,
+      ...neutralRequest
+    } = request
     const tier = normalizeAiTier(rawTier)
     const resolved = resolveAiTier(tier, tierSettingValue(tier), savedLlmKeys())
 
+    // ON-DEVICE ROUTING — the request's forceLocal / localFallback flags
+    // against the primary path's health (see LlmRouting). The primary is the
+    // vendor-direct call when a user key is saved, else the cloud relay.
+    const directKey = resolved.provider ? userLlmKey(resolved.provider) : ""
+    const forceLocal = rawForceLocal === true
+    const localFallback = rawLocalFallback === true
+    const cloudConnected = cloudClientService.isConnected()
+    const cloudDisabled = cloudClientService.isCloudDisabled()
+    const primaryUp = directKey.length > 0 || (cloudConnected && !cloudDisabled && !this.llmQuotaBlocked)
+    const route = decideLlmRoute({forceLocal, localFallback, primaryUp})
+
+    if (route === "local") {
+      const reason = localLlmReason({forceLocal, cloudDisabled, cloudConnected, quotaBlocked: this.llmQuotaBlocked})
+      await this.serveLocalAi(packageName, requestId, neutralRequest, reason, null)
+      return
+    }
+
     try {
       // Direct when the resolved provider has a user key (BYO bill, unmetered,
       // works with the cloud disabled); otherwise relay to the cloud LLM
       // service, forwarding the resolved pin so platform-funded calls land on
       // the same model the user chose.
-      const directKey = resolved.provider ? userLlmKey(resolved.provider) : ""
       if (resolved.provider && directKey) {
         const result = await directLlmProvider(resolved.provider).complete({
           ...neutralRequest,
@@ -4094,18 +4127,38 @@ class LocalMiniappRuntime {
       // is spent. Distinct code (miniapps may fall back to a local path) and
       // a host hook so the app can offer the upgrade paywall — the miniapp
       // itself must never own that conversation.
-      if (err instanceof HttpError && (err.code === "quota_exceeded" || err.status === 402)) {
+      const quotaExceeded = err instanceof HttpError && (err.code === "quota_exceeded" || err.status === 402)
+      if (quotaExceeded && !this.llmQuotaBlocked) {
         console.warn(`${LOG_TAG}: ai.complete quota exceeded for ${packageName}`)
         this.llmQuotaBlocked = true
         this.onQuotaExceeded?.("llm")
+      }
+      const message = err instanceof Error ? err.message : "ai.complete failed"
+
+      // localFallback: the same request, on-device. The primary's error
+      // stands only when the phone cannot serve it either.
+      if (route === "primary_then_local") {
+        console.warn(`${LOG_TAG}: ai.complete primary failed for ${packageName} (${message}); trying on-device`)
+        await this.serveLocalAi(
+          packageName,
+          requestId,
+          neutralRequest,
+          quotaExceeded ? "llm_quota" : "primary_failed",
+          {
+            code: quotaExceeded ? MiniappErrorCode.QUOTA_EXCEEDED : MiniappErrorCode.INTERNAL,
+            message: quotaExceeded ? QUOTA_EXCEEDED_MESSAGE : message,
+          },
+        )
+        return
+      }
+
+      if (quotaExceeded) {
         this.sendResult(packageName, requestId, false, undefined, {
           code: MiniappErrorCode.QUOTA_EXCEEDED,
-          message:
-            "Monthly AI credit is used up. Upgrade in Settings > Membership, or add your own API key in Settings > API Keys.",
+          message: QUOTA_EXCEEDED_MESSAGE,
         })
         return
       }
-      const message = err instanceof Error ? err.message : "ai.complete failed"
       console.warn(`${LOG_TAG}: ai.complete failed for ${packageName}: ${message}`)
       this.sendResult(packageName, requestId, false, undefined, {
         code: MiniappErrorCode.INTERNAL,
@@ -4114,6 +4167,70 @@ class LocalMiniappRuntime {
     }
   }
 
+  /**
+   * Serve one ai.complete on the phone's own model (LocalLlmProvider). With
+   * no model on disk the download starts (or continues) in the background
+   * and the miniapp gets AI_LOCAL_UNAVAILABLE naming the state — unless
+   * `primaryError` is given (a failed fallback), in which case the primary's
+   * own error is what the miniapp sees.
+   */
+  private async serveLocalAi(
+    packageName: string,
+    requestId: string | undefined,
+    request: Record<string, unknown>,
+    reason: LocalLlmReason,
+    primaryError: {code: MiniappErrorCode; message: string} | null,
+  ): Promise<void> {
+    const modelId = localLlmModelManager.selectedModelId()
+    if (!(await localLlmModelManager.ensureSelectedModel())) {
+      const status = localLlmModelManager.getStatus()
+      const state =
+        status?.modelId === modelId && status.stage === "downloading"
+          ? `is downloading (${status.percent}%)`
+          : status?.modelId === modelId && status.stage === "failed"
+            ? `failed to download (${status.error ?? "unknown error"})`
+            : "is not downloaded yet"
+      console.warn(`${LOG_TAG}: ai.complete on-device for ${packageName} (${reason}): model ${modelId} ${state}`)
+      this.sendResult(
+        packageName,
+        requestId,
+        false,
+        undefined,
+        primaryError ?? {
+          code: MiniappErrorCode.AI_LOCAL_UNAVAILABLE,
+          message: `The on-device model (${localLlmModelManager.model(modelId).label}) ${state}. Manage it in Settings > AI Models.`,
+        },
+      )
+      return
+    }
+
+    try {
+      // webSearch is a vendor server-side grant; the phone has none.
+      const {webSearch: _webSearch, ...localRequest} = request
+      const result = await localLlmProvider.complete(localRequest as never)
+      console.log(`${LOG_TAG}: ai.complete served on-device for ${packageName} (${reason}, ${result.modelId})`)
+      this.sendResult(packageName, requestId, true, {
+        text: result.text,
+        stopReason: result.stopReason,
+        ...(result.toolCalls ? {toolCalls: result.toolCalls} : {}),
+        usage: result.usage,
+        provider: "device",
+        model: result.modelId,
+        billedToUserKey: false,
+      })
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "on-device completion failed"
+      console.warn(`${LOG_TAG}: ai.complete on-device failed for ${packageName}: ${message}`)
+      this.sendResult(
+        packageName,
+        requestId,
+        false,
+        undefined,
+        primaryError ?? {code: MiniappErrorCode.AI_LOCAL_UNAVAILABLE, message},
+      )
+    }
+  }
+
   /**
    * ai.generate — agentic visual generation. Always relayed to the cloud
    * runtime (the server drives the provider's agentic loop; there is no
@@ -5318,13 +5435,12 @@ class LocalMiniappRuntime {
     this.capabilitiesWired = true
     const broadcast = () => this.broadcastCapabilities()
     // Connection, model and firmware also decide the dashboard view's eligibility;
-    // a connected Neural Band adds its swipe2d surface to `button`.
+    // connected controllers add their input surfaces to `button`.
     useGlassesStore.subscribe(
       (s) =>
-        `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}:${s.controllerConnected}`,
+        `${s.sceneCanvasWidth ?? 0}x${s.sceneCanvasHeight ?? 0}:${s.sceneVector === true}:${s.firmwareVersion}:${s.connection.state}:${s.deviceModel}:${readyControllersKey(s.controllers)}`,
       broadcast,
     )
-    useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.default_controller.key), broadcast)
     // The debug knob flips hasNativeNavigation on custom firmware.
     useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.cfw_native_navigation.key), broadcast)
     // The dashboard master switch flips the dashboard view.