@glassly/engine 0.3.8-dev.1009 → 0.3.8-dev.1014

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glassly/engine",
-  "version": "0.3.8-dev.1009",
+  "version": "0.3.8-dev.1014",
   "description": "Glassly Engine — miniapp registry, WebView bridge, and miniapp store",
   "main": "build/index.js",
   "react-native": "src/index.ts",
@@ -97,17 +97,17 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "@glassly/bluetooth-sdk": "0.3.8-dev.1009",
-    "@glassly/cloud-client": "0.3.8-dev.1009",
-    "@glassly/cloud-protocol": "0.3.8-dev.1009",
-    "@glassly/crust": "0.3.8-dev.1009",
-    "@glassly/miniapp": "0.3.8-dev.1009",
+    "@glassly/bluetooth-sdk": "0.3.8-dev.1014",
+    "@glassly/cloud-client": "0.3.8-dev.1014",
+    "@glassly/cloud-protocol": "0.3.8-dev.1014",
+    "@glassly/crust": "0.3.8-dev.1014",
+    "@glassly/miniapp": "0.3.8-dev.1014",
     "buffer": "^6.0.3",
     "events": "^3.3.0",
     "react-native-marked": "8.3.2",
     "semver": "^7.7.4",
     "typesafe-ts": "^1.7.1",
-    "@glassly/llm-providers": "0.3.8-dev.1009",
+    "@glassly/llm-providers": "0.3.8-dev.1014",
     "@huggingface/jinja": "^0.5.9",
     "jsonrepair": "^3.12.0"
   },

package/src/facades/ai.ts

@@ -21,6 +21,12 @@ export const ai = {
     /** The Settings > Offline Models pick. */
     selectedModelId: (): LocalLlmModelId => localLlmModelManager.selectedModelId(),
     select: (id: LocalLlmModelId): Promise<void> => localLlmModelManager.select(id),
+    /** Whether the model fits this phone's memory at all. */
+    fitsDevice: (id: LocalLlmModelId): boolean => localLlmModelManager.fitsDevice(id),
+    /** The OS killed the app while this model loaded; picking it again retries it. */
+    isBlocked: (id: LocalLlmModelId): boolean => localLlmModelManager.isBlocked(id),
+    /** Why the model will not load on this phone, or null when it can. */
+    unusableReason: (id: LocalLlmModelId): string | null => localLlmModelManager.unusableReason(id),
     isDownloaded: (id?: LocalLlmModelId): Promise<boolean> => localLlmModelManager.isModelAvailable(id),
     downloadedModelIds: (): Promise<LocalLlmModelId[]> => localLlmModelManager.downloadedModelIds(),
     download: (id?: LocalLlmModelId): Promise<void> => localLlmModelManager.downloadModel(id),

package/src/generated/releaseMetadata.ts

@@ -12,9 +12,9 @@ export interface EngineReleaseMetadata {
 export const ENGINE_RELEASE_METADATA: Readonly<EngineReleaseMetadata> = Object.freeze({
   "schemaVersion": 1,
   "familyBaseVersion": "0.3.8",
-  "releaseIdentity": "0.3.8-dev.1009",
-  "releaseSetId": "glassly-0.3.8-dev.1009",
-  "sourceCommit": "4983873da52b50b866925cee3b550fa9dd4b7c28",
+  "releaseIdentity": "0.3.8-dev.1014",
+  "releaseSetId": "glassly-0.3.8-dev.1014",
+  "sourceCommit": "91cc37d7b4c99dc750e22528c9487e33fb27c53c",
   "otaManifestUrl": null,
   "otaManifestSha256": null
 })

package/src/services/LocalLlmModelManager.ts

@@ -15,6 +15,7 @@ import {Platform} from "react-native"
 import * as RNFS from "@dr.pogodin/react-native-fs"
 
 import {SETTINGS, useSettingsStore} from "../stores/settings"
+import {storage} from "../utils/storage"
 
 export type LocalLlmModelId = "qwen3-0.6b" | "qwen3-1.7b" | "qwen3-4b"
 
@@ -28,7 +29,7 @@ export interface LocalLlmModelConfig {
   sizeBytes: number
   /** Exact size of the published .pte; the availability check compares against it. */
   modelBytes: number
-  /** Rough resident memory while loaded; the Settings copy warns above ~half of device RAM. */
+  /** Rough resident memory while loaded; `fitsDevice` holds it against the phone's RAM. */
   ramBytes: number
   urls: {
     model: string
@@ -106,6 +107,36 @@ export function isLocalLlmModelId(value: unknown): value is LocalLlmModelId {
   return typeof value === "string" && value in MODELS
 }
 
+// Without the increased-memory-limit entitlement iOS kills a foreground app
+// at roughly 60% of physical RAM (Android's low-memory killer is no kinder),
+// and the host holds about 1 GB besides the model: JS runtimes, WebViews,
+// the speech model. A 4 GB phone fits the 0.6B model and nothing larger.
+const APP_MEMORY_SHARE = 0.6
+const HOST_BASELINE_BYTES = 1 * GB
+
+export function modelFitsMemory(model: LocalLlmModelConfig, deviceMemoryBytes: number | null): boolean {
+  if (!deviceMemoryBytes) return true
+  return model.ramBytes + HOST_BASELINE_BYTES <= deviceMemoryBytes * APP_MEMORY_SHARE
+}
+
+let deviceMemory: number | null | undefined
+function deviceMemoryBytes(): number | null {
+  if (deviceMemory === undefined) {
+    try {
+      deviceMemory = (require("expo-device") as {totalMemory?: number | null}).totalMemory ?? null
+    } catch {
+      deviceMemory = null
+    }
+  }
+  return deviceMemory
+}
+
+// Written before a model's first load and generation, cleared after either
+// ends. A marker still there at the next launch means the OS killed the app
+// mid-load (out of memory), so that model stays off until picked again.
+const COLD_LOAD_KEY = "local_llm_cold_load"
+const BLOCKED_MODELS_KEY = "local_llm_blocked_models"
+
 type StatusListener = (status: LocalLlmDownloadStatus | null) => void
 
 class LocalLlmModelManager {
@@ -114,6 +145,7 @@ class LocalLlmModelManager {
   private downloadJobId?: number
   private inFlight: {modelId: LocalLlmModelId; promise: Promise<void>} | null = null
   private cancelled = false
+  private coldLoadRecovered = false
   private currentStatus: LocalLlmDownloadStatus | null = null
   private listeners = new Set<StatusListener>()
   private modelReadyListeners = new Set<(modelId: LocalLlmModelId) => void>()
@@ -135,16 +167,89 @@ class LocalLlmModelManager {
     return MODELS[id]
   }
 
-  /** The Settings > AI Models pick; garbage in the store resolves to the default. */
+  /**
+   * The model on-device requests use: the Settings > Offline Models pick
+   * (garbage resolves to the default) while it is usable, else the largest
+   * model that is. With nothing usable it stays the pick, and loading it
+   * fails with `unusableReason`.
+   */
   selectedModelId(): LocalLlmModelId {
     const raw = useSettingsStore.getState().getSetting(SETTINGS.local_llm_model.key)
-    return isLocalLlmModelId(raw) ? raw : DEFAULT_LOCAL_LLM_MODEL
+    const picked = isLocalLlmModelId(raw) ? raw : DEFAULT_LOCAL_LLM_MODEL
+    if (this.isUsable(picked)) return picked
+    const fallback = this.models()
+      .filter((m) => this.isUsable(m.id))
+      .sort((a, b) => b.ramBytes - a.ramBytes)[0]
+    return fallback?.id ?? picked
   }
 
+  /** Picking a model in Settings is also the way to retry one that crashed the app. */
   async select(id: LocalLlmModelId): Promise<void> {
+    this.unblock(id)
     await useSettingsStore.getState().setSetting(SETTINGS.local_llm_model.key, id)
   }
 
+  fitsDevice(id: LocalLlmModelId): boolean {
+    return modelFitsMemory(MODELS[id], deviceMemoryBytes())
+  }
+
+  /** The OS killed the app while this model loaded; it stays off until picked again. */
+  isBlocked(id: LocalLlmModelId): boolean {
+    return this.blockedModelIds().includes(id)
+  }
+
+  isUsable(id: LocalLlmModelId): boolean {
+    return this.fitsDevice(id) && !this.isBlocked(id)
+  }
+
+  /** Why `id` will not load on this phone, or null when it can. */
+  unusableReason(id: LocalLlmModelId): string | null {
+    const label = MODELS[id].label
+    if (!this.fitsDevice(id)) {
+      return `${label} needs more memory than this phone has. Pick a smaller model in Settings > Offline Models.`
+    }
+    if (this.isBlocked(id)) {
+      return `${label} closed the app the last time it loaded. Pick it again in Settings > Offline Models to retry, or pick a smaller model.`
+    }
+    return null
+  }
+
+  /** Mark the start of a model's first load and generation in this process. */
+  beginColdLoad(id: LocalLlmModelId): void {
+    this.recoverColdLoad()
+    storage.save(COLD_LOAD_KEY, id)
+  }
+
+  endColdLoad(): void {
+    storage.removeMultiple([COLD_LOAD_KEY])
+  }
+
+  private blockedModelIds(): LocalLlmModelId[] {
+    this.recoverColdLoad()
+    const stored = storage.load<unknown>(BLOCKED_MODELS_KEY)
+    const ids = stored.is_ok() ? stored.value : []
+    return Array.isArray(ids) ? ids.filter(isLocalLlmModelId) : []
+  }
+
+  private unblock(id: LocalLlmModelId): void {
+    const remaining = this.blockedModelIds().filter((blocked) => blocked !== id)
+    storage.save(BLOCKED_MODELS_KEY, remaining)
+  }
+
+  // A cold-load marker left by an earlier process: that load never finished.
+  private recoverColdLoad(): void {
+    if (this.coldLoadRecovered) return
+    this.coldLoadRecovered = true
+    const marker = storage.load<unknown>(COLD_LOAD_KEY)
+    if (!marker.is_ok() || !isLocalLlmModelId(marker.value)) return
+    const id = marker.value
+    console.warn(`LOCAL_LLM: the app was killed while ${id} loaded; keeping it off until it is picked again`)
+    const stored = storage.load<unknown>(BLOCKED_MODELS_KEY)
+    const blocked = stored.is_ok() && Array.isArray(stored.value) ? stored.value.filter(isLocalLlmModelId) : []
+    if (!blocked.includes(id)) storage.save(BLOCKED_MODELS_KEY, [...blocked, id])
+    storage.removeMultiple([COLD_LOAD_KEY])
+  }
+
   getModelDirectory(): string {
     return `${RNFS.DocumentDirectoryPath}/llm_models`
   }
@@ -210,6 +315,7 @@ class LocalLlmModelManager {
   async ensureSelectedModel(): Promise<boolean> {
     const id = this.selectedModelId()
     if (await this.isModelAvailable(id)) return true
+    if (!this.fitsDevice(id)) return false
     void this.downloadModel(id).catch((error) => {
       console.warn(`LOCAL_LLM: background download of ${id} failed:`, error)
     })
@@ -222,6 +328,7 @@ class LocalLlmModelManager {
    * the running one — the RNFS job slot is singular.
    */
   async downloadModel(id: LocalLlmModelId = this.selectedModelId()): Promise<void> {
+    if (!this.fitsDevice(id)) throw new Error(this.unusableReason(id) ?? `${MODELS[id].label} does not fit this phone.`)
     if (this.inFlight) {
       if (this.inFlight.modelId === id) return this.inFlight.promise
       await this.inFlight.promise.catch(() => undefined)

package/src/services/LocalLlmProvider.ts

@@ -187,6 +187,11 @@ class LocalLlmProvider {
 
   private async completeInner(request: LlmCompleteRequest): Promise<LocalLlmCompleteResult> {
     this.clearIdleTimer()
+    // Loading and the first generation are where memory peaks; if the OS kills
+    // the app in between, the marker keeps the next launch from looping on it.
+    const selectedId = localLlmModelManager.selectedModelId()
+    const coldModelId = this.loaded?.modelId === selectedId ? null : selectedId
+    if (coldModelId) localLlmModelManager.beginColdLoad(coldModelId)
     try {
       const model = await this.ensureLoaded()
       const jsonMode = request.jsonMode === true
@@ -248,6 +253,7 @@ class LocalLlmProvider {
       )
       return {...completion, modelId: model.modelId}
     } finally {
+      if (coldModelId) localLlmModelManager.endColdLoad()
       this.armIdleTimer()
     }
   }
@@ -272,6 +278,8 @@ class LocalLlmProvider {
     // Runs inside the request queue, so nothing is generating; unload() would
     // wait on that same queue.
     this.releaseRunner()
+    const unusable = localLlmModelManager.unusableReason(modelId)
+    if (unusable) throw new Error(unusable)
     if (!(await localLlmModelManager.isModelAvailable(modelId))) {
       throw new Error(`on-device model ${modelId} is not downloaded`)
     }

package/src/services/LocalMiniappRuntime.ts

@@ -4322,6 +4322,18 @@ class LocalMiniappRuntime {
     primaryError: {code: MiniappErrorCode; message: string} | null,
   ): Promise<void> {
     const modelId = localLlmModelManager.selectedModelId()
+    const unusable = localLlmModelManager.unusableReason(modelId)
+    if (unusable) {
+      console.warn(`${LOG_TAG}: ai.complete on-device for ${packageName} (${reason}): ${unusable}`)
+      this.sendResult(
+        packageName,
+        requestId,
+        false,
+        undefined,
+        primaryError ?? {code: MiniappErrorCode.AI_LOCAL_UNAVAILABLE, message: unusable},
+      )
+      return
+    }
     if (!(await localLlmModelManager.ensureSelectedModel())) {
       const status = localLlmModelManager.getStatus()
       const state =
@@ -4338,7 +4350,7 @@ class LocalMiniappRuntime {
         undefined,
         primaryError ?? {
           code: MiniappErrorCode.AI_LOCAL_UNAVAILABLE,
-          message: `The on-device model (${localLlmModelManager.model(modelId).label}) ${state}. Manage it in Settings > AI Models.`,
+          message: `The on-device model (${localLlmModelManager.model(modelId).label}) ${state}. Manage it in Settings > Offline Models.`,
         },
       )
       return