@glassly/engine 0.3.3-dev.670 → 0.3.4-beta.685

package/src/services/LocalLlmModelManager.ts

@@ -0,0 +1,327 @@
+/**
+ * LocalLlmModelManager — the on-device LLM model files: a catalog of
+ * ExecuTorch exports (Software Mansion's Qwen3 .pte builds on Hugging Face),
+ * their download into Documents/llm_models/<id>/, availability checks, and a
+ * status stream the Settings > AI Models screen renders. Modelled on
+ * STTModelManager; the download itself is RNFS so the Settings screen can
+ * list, cancel, and delete models with the same calls the STT rows use.
+ *
+ * Which model is "the" on-device model is the `local_llm_model` setting
+ * (Settings > AI Models). Downloads are on demand: the first request that
+ * routes on-device without a model present starts one (ensureSelectedModel),
+ * and the Settings screen can start one explicitly.
+ */
+import {Platform} from "react-native"
+import * as RNFS from "@dr.pogodin/react-native-fs"
+
+import {SETTINGS, useSettingsStore} from "../stores/settings"
+
+export type LocalLlmModelId = "qwen3-0.6b" | "qwen3-1.7b" | "qwen3-4b"
+
+export const DEFAULT_LOCAL_LLM_MODEL: LocalLlmModelId = "qwen3-1.7b"
+
+export interface LocalLlmModelConfig {
+  id: LocalLlmModelId
+  /** Human label for the Settings dropdown. */
+  label: string
+  /** Approximate download size in bytes (the .pte dominates). */
+  sizeBytes: number
+  /** Rough resident memory while loaded; the Settings copy warns above ~half of device RAM. */
+  ramBytes: number
+  urls: {
+    model: string
+    tokenizer: string
+    tokenizerConfig: string
+  }
+}
+
+/** The three files every model directory holds; names are fixed so paths never depend on the URL. */
+export interface LocalLlmModelPaths {
+  model: string
+  tokenizer: string
+  tokenizerConfig: string
+}
+
+export type LocalLlmDownloadStage = "idle" | "downloading" | "complete" | "failed"
+
+export interface LocalLlmDownloadStatus {
+  modelId: LocalLlmModelId
+  stage: LocalLlmDownloadStage
+  /** 0–100, meaningful while stage is "downloading". */
+  percent: number
+  error?: string
+}
+
+// XNNPACK 8da4w exports from software-mansion/react-native-executorch-qwen-3,
+// pinned to the v0.10.0 release tag that react-native-executorch 0.10.x
+// expects. Sizes are the .pte file sizes on the Hub (tokenizer adds ~11 MB).
+const QWEN3_BASE = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0"
+const MB = 1024 * 1024
+const GB = 1024 * MB
+
+const MODELS: Record<LocalLlmModelId, LocalLlmModelConfig> = {
+  "qwen3-0.6b": {
+    id: "qwen3-0.6b",
+    label: "Qwen3 0.6B",
+    sizeBytes: 517 * MB,
+    ramBytes: 0.9 * GB,
+    urls: {
+      model: `${QWEN3_BASE}/0_6b/xnnpack/qwen_3_0_6b_xnnpack_8da4w.pte`,
+      tokenizer: `${QWEN3_BASE}/tokenizer.json`,
+      tokenizerConfig: `${QWEN3_BASE}/tokenizer_config.json`,
+    },
+  },
+  "qwen3-1.7b": {
+    id: "qwen3-1.7b",
+    label: "Qwen3 1.7B",
+    sizeBytes: 1315 * MB,
+    ramBytes: 1.8 * GB,
+    urls: {
+      model: `${QWEN3_BASE}/1_7b/xnnpack/qwen_3_1_7b_xnnpack_8da4w.pte`,
+      tokenizer: `${QWEN3_BASE}/tokenizer.json`,
+      tokenizerConfig: `${QWEN3_BASE}/tokenizer_config.json`,
+    },
+  },
+  "qwen3-4b": {
+    id: "qwen3-4b",
+    label: "Qwen3 4B",
+    sizeBytes: 2693 * MB,
+    ramBytes: 3.4 * GB,
+    urls: {
+      model: `${QWEN3_BASE}/4b/xnnpack/qwen_3_4b_xnnpack_8da4w.pte`,
+      tokenizer: `${QWEN3_BASE}/tokenizer.json`,
+      tokenizerConfig: `${QWEN3_BASE}/tokenizer_config.json`,
+    },
+  },
+}
+
+export function isLocalLlmModelId(value: unknown): value is LocalLlmModelId {
+  return typeof value === "string" && value in MODELS
+}
+
+type StatusListener = (status: LocalLlmDownloadStatus | null) => void
+
+class LocalLlmModelManager {
+  private static instance: LocalLlmModelManager
+
+  private downloadJobId?: number
+  private inFlight: {modelId: LocalLlmModelId; promise: Promise<void>} | null = null
+  private cancelled = false
+  private currentStatus: LocalLlmDownloadStatus | null = null
+  private listeners = new Set<StatusListener>()
+  private modelReadyListeners = new Set<(modelId: LocalLlmModelId) => void>()
+
+  private constructor() {}
+
+  static getInstance(): LocalLlmModelManager {
+    if (!LocalLlmModelManager.instance) {
+      LocalLlmModelManager.instance = new LocalLlmModelManager()
+    }
+    return LocalLlmModelManager.instance
+  }
+
+  models(): LocalLlmModelConfig[] {
+    return Object.values(MODELS)
+  }
+
+  model(id: LocalLlmModelId): LocalLlmModelConfig {
+    return MODELS[id]
+  }
+
+  /** The Settings > AI Models pick; garbage in the store resolves to the default. */
+  selectedModelId(): LocalLlmModelId {
+    const raw = useSettingsStore.getState().getSetting(SETTINGS.local_llm_model.key)
+    return isLocalLlmModelId(raw) ? raw : DEFAULT_LOCAL_LLM_MODEL
+  }
+
+  async select(id: LocalLlmModelId): Promise<void> {
+    await useSettingsStore.getState().setSetting(SETTINGS.local_llm_model.key, id)
+  }
+
+  getModelDirectory(): string {
+    return `${RNFS.DocumentDirectoryPath}/llm_models`
+  }
+
+  paths(id: LocalLlmModelId = this.selectedModelId()): LocalLlmModelPaths {
+    const dir = `${this.getModelDirectory()}/${id}`
+    return {
+      model: `${dir}/model.pte`,
+      tokenizer: `${dir}/tokenizer.json`,
+      tokenizerConfig: `${dir}/tokenizer_config.json`,
+    }
+  }
+
+  async isModelAvailable(id: LocalLlmModelId = this.selectedModelId()): Promise<boolean> {
+    const paths = this.paths(id)
+    try {
+      for (const path of [paths.tokenizerConfig, paths.tokenizer, paths.model]) {
+        if (!(await RNFS.exists(path))) return false
+      }
+      // A truncated .pte loads as garbage or crashes the runtime: insist on
+      // most of the published size. Downloads land in a .part file and are
+      // renamed only once whole, so this only catches external tampering.
+      const stat = await RNFS.stat(paths.model)
+      return Number(stat.size) >= MODELS[id].sizeBytes * 0.95
+    } catch (error) {
+      console.warn("LOCAL_LLM: availability check failed:", error)
+      return false
+    }
+  }
+
+  async downloadedModelIds(): Promise<LocalLlmModelId[]> {
+    const ids: LocalLlmModelId[] = []
+    for (const id of Object.keys(MODELS) as LocalLlmModelId[]) {
+      if (await this.isModelAvailable(id)) ids.push(id)
+    }
+    return ids
+  }
+
+  /** Subscribe to download status changes. Fires immediately with the current status. */
+  subscribe(listener: StatusListener): () => void {
+    this.listeners.add(listener)
+    listener(this.currentStatus)
+    return () => {
+      this.listeners.delete(listener)
+    }
+  }
+
+  getStatus(): LocalLlmDownloadStatus | null {
+    return this.currentStatus
+  }
+
+  /** Fired once a model's files are complete on disk. */
+  onModelReady(listener: (modelId: LocalLlmModelId) => void): () => void {
+    this.modelReadyListeners.add(listener)
+    return () => this.modelReadyListeners.delete(listener)
+  }
+
+  /**
+   * True when the selected model is on disk. Otherwise starts (or joins) its
+   * download in the background and returns false — the caller reports
+   * "downloading" rather than waiting out a multi-GB transfer.
+   */
+  async ensureSelectedModel(): Promise<boolean> {
+    const id = this.selectedModelId()
+    if (await this.isModelAvailable(id)) return true
+    void this.downloadModel(id).catch((error) => {
+      console.warn(`LOCAL_LLM: background download of ${id} failed:`, error)
+    })
+    return false
+  }
+
+  /**
+   * Download one model (three files, model last). Concurrent calls for the
+   * same model share the transfer; a call for a different model waits for
+   * the running one — the RNFS job slot is singular.
+   */
+  async downloadModel(id: LocalLlmModelId = this.selectedModelId()): Promise<void> {
+    if (this.inFlight) {
+      if (this.inFlight.modelId === id) return this.inFlight.promise
+      await this.inFlight.promise.catch(() => undefined)
+    }
+    const promise = this.downloadInner(id).finally(() => {
+      this.inFlight = null
+    })
+    this.inFlight = {modelId: id, promise}
+    return promise
+  }
+
+  private async downloadInner(id: LocalLlmModelId): Promise<void> {
+    const config = MODELS[id]
+    const paths = this.paths(id)
+    const dir = `${this.getModelDirectory()}/${id}`
+    this.cancelled = false
+    this.setStatus({modelId: id, stage: "downloading", percent: 0})
+
+    try {
+      await RNFS.mkdir(dir, {NSURLIsExcludedFromBackupKey: true})
+      // Tokenizer files first (small), the weights last: a model directory
+      // is only "available" once model.pte exists, so an interrupted run
+      // never leaves a complete-looking directory behind.
+      await this.fetchFile(config.urls.tokenizerConfig, paths.tokenizerConfig)
+      await this.fetchFile(config.urls.tokenizer, paths.tokenizer)
+      await this.fetchFile(config.urls.model, paths.model, (percent) =>
+        this.setStatus({modelId: id, stage: "downloading", percent}),
+      )
+      this.setStatus({modelId: id, stage: "complete", percent: 100})
+      for (const listener of this.modelReadyListeners) {
+        try {
+          listener(id)
+        } catch (error) {
+          console.warn("LOCAL_LLM: model-ready listener failed:", error)
+        }
+      }
+    } catch (error) {
+      const message = this.cancelled ? "download cancelled" : error instanceof Error ? error.message : String(error)
+      this.setStatus({modelId: id, stage: this.cancelled ? "idle" : "failed", percent: 0, error: message})
+      try {
+        if (await RNFS.exists(`${paths.model}.part`)) await RNFS.unlink(`${paths.model}.part`)
+      } catch (cleanupError) {
+        console.warn("LOCAL_LLM: partial cleanup failed:", cleanupError)
+      }
+      throw error
+    } finally {
+      this.downloadJobId = undefined
+    }
+  }
+
+  private async fetchFile(url: string, dest: string, onPercent?: (percent: number) => void): Promise<void> {
+    if (this.cancelled) throw new Error("download cancelled")
+    const partial = `${dest}.part`
+    const download = RNFS.downloadFile({
+      fromUrl: url,
+      toFile: partial,
+      // iOS keeps a background URLSession transfer going while the app is
+      // suspended; the files here are large enough to need it.
+      background: Platform.OS === "ios",
+      discretionary: false,
+      progressDivider: 1,
+      connectionTimeout: 30_000,
+      readTimeout: 60_000,
+      progress: (res) => {
+        if (!onPercent || res.contentLength <= 0) return
+        onPercent(Math.min(99, Math.floor((res.bytesWritten / res.contentLength) * 100)))
+      },
+    })
+    this.downloadJobId = download.jobId
+    const result = await download.promise
+    if (result.statusCode !== 200) {
+      throw new Error(`download failed with status ${result.statusCode}`)
+    }
+    if (await RNFS.exists(dest)) await RNFS.unlink(dest)
+    await RNFS.moveFile(partial, dest)
+  }
+
+  async cancelDownload(): Promise<void> {
+    if (this.downloadJobId === undefined) return
+    this.cancelled = true
+    await RNFS.stopDownload(this.downloadJobId)
+    this.downloadJobId = undefined
+  }
+
+  async deleteModel(id: LocalLlmModelId = this.selectedModelId()): Promise<void> {
+    const dir = `${this.getModelDirectory()}/${id}`
+    if (await RNFS.exists(dir)) await RNFS.unlink(dir)
+    if (this.currentStatus?.modelId === id) this.setStatus(null)
+  }
+
+  private setStatus(status: LocalLlmDownloadStatus | null): void {
+    this.currentStatus = status
+    for (const listener of this.listeners) {
+      try {
+        listener(status)
+      } catch (error) {
+        console.warn("LOCAL_LLM: status listener threw:", error)
+      }
+    }
+  }
+
+  static formatBytes(bytes: number): string {
+    if (bytes >= GB) return `${(bytes / GB).toFixed(1)} GB`
+    return `${Math.round(bytes / MB)} MB`
+  }
+}
+
+const instance = LocalLlmModelManager.getInstance()
+export {LocalLlmModelManager}
+export default instance

package/src/services/LocalLlmPrompt.ts

@@ -0,0 +1,246 @@
+/**
+ * LocalLlmPrompt — the pure half of the on-device LLM path: turning a neutral
+ * session.ai request into chat-template messages, fitting them into the
+ * model's context, and reading the model's text back into the neutral result
+ * (tool calls, JSON mode, stop reason). No native imports, so it is
+ * bun-testable; LocalLlmProvider owns the runner.
+ *
+ * Written against Qwen3's chat template: tools are advertised by the
+ * template itself, calls come back as `<tool_call>{json}</tool_call>`, and
+ * thinking is a `<think>…</think>` block we never ask for but still strip.
+ */
+import {jsonrepair} from "jsonrepair"
+
+import type {LlmMessage, LlmStopReason, LlmTool, LlmToolCall} from "@glassly/cloud-protocol/llm"
+
+export interface TemplateToolCall {
+  id: string
+  type: "function"
+  function: {name: string; arguments: Record<string, unknown>}
+}
+
+export type TemplateMessage =
+  | {role: "system" | "user"; content: string}
+  | {role: "assistant"; content: string; tool_calls?: TemplateToolCall[]}
+  | {role: "tool"; content: string; name?: string; tool_call_id?: string}
+
+export interface TemplateTool {
+  type: "function"
+  function: {name: string; description: string; parameters: Record<string, unknown>}
+}
+
+/** Appended to the system prompt under `jsonMode`; small models need it said twice. */
+export const JSON_MODE_INSTRUCTION =
+  "Respond with exactly one JSON object and nothing else: no prose before or after it, no markdown fences."
+
+/**
+ * Neutral messages → template messages. A user turn carrying toolResults
+ * becomes one `tool` message per result (the template groups consecutive
+ * tool messages into a single user block), followed by the user text if any.
+ */
+export function buildTemplateMessages(request: {
+  messages: LlmMessage[]
+  system?: string
+  jsonMode?: boolean
+}): TemplateMessage[] {
+  const out: TemplateMessage[] = []
+  const systemParts = [request.system?.trim(), request.jsonMode ? JSON_MODE_INSTRUCTION : undefined].filter(
+    (part): part is string => Boolean(part),
+  )
+  if (systemParts.length) out.push({role: "system", content: systemParts.join("\n\n")})
+
+  for (const message of request.messages) {
+    if (message.role === "assistant") {
+      const toolCalls = message.toolCalls?.map<TemplateToolCall>((call) => ({
+        id: call.id,
+        type: "function",
+        function: {name: call.name, arguments: call.input},
+      }))
+      out.push(
+        toolCalls?.length
+          ? {role: "assistant", content: message.content, tool_calls: toolCalls}
+          : {role: "assistant", content: message.content},
+      )
+      continue
+    }
+    for (const result of message.toolResults ?? []) {
+      out.push({
+        role: "tool",
+        tool_call_id: result.toolCallId,
+        content: result.isError ? `Error: ${result.content}` : result.content,
+      })
+    }
+    if (message.content.trim().length > 0 || !message.toolResults?.length) {
+      out.push({role: "user", content: message.content})
+    }
+  }
+  return out
+}
+
+export function toTemplateTools(tools?: LlmTool[]): TemplateTool[] | undefined {
+  if (!tools?.length) return undefined
+  return tools.map((tool) => ({
+    type: "function",
+    function: {name: tool.name, description: tool.description, parameters: tool.inputSchema},
+  }))
+}
+
+/** Rough tokens-per-character for a BPE vocabulary over mixed prose/JSON. */
+const CHARS_PER_TOKEN = 3.2
+/** Template framing per message (`<|im_start|>role … <|im_end|>`). */
+const TOKENS_PER_MESSAGE = 6
+
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / CHARS_PER_TOKEN)
+}
+
+export function estimateMessageTokens(message: TemplateMessage): number {
+  let total = TOKENS_PER_MESSAGE + estimateTokens(message.content)
+  if (message.role === "assistant" && message.tool_calls) {
+    for (const call of message.tool_calls) total += estimateTokens(JSON.stringify(call.function)) + 8
+  }
+  return total
+}
+
+const MIN_TOOL_CONTENT_CHARS = 1_200
+
+/**
+ * Fit the conversation into `budgetTokens`. History is grouped into turns
+ * (a user message plus everything up to the next user message, so a tool
+ * round stays with the question it answers); the oldest turns go first and
+ * the last turn is never dropped — the model must at least see the question.
+ * When that last turn alone still does not fit, tool outputs are cut to
+ * their head.
+ */
+export function fitToBudget(messages: TemplateMessage[], budgetTokens: number, toolTokens = 0): TemplateMessage[] {
+  const total = (list: TemplateMessage[]) => list.reduce((sum, m) => sum + estimateMessageTokens(m), toolTokens)
+  const system = messages[0]?.role === "system" ? [messages[0]] : []
+  const turns: TemplateMessage[][] = []
+  for (const message of messages.slice(system.length)) {
+    if (message.role === "user" || turns.length === 0) turns.push([message])
+    else turns[turns.length - 1]!.push(message)
+  }
+
+  while (turns.length > 1 && total([...system, ...turns.flat()]) > budgetTokens) {
+    turns.shift()
+  }
+
+  let fitted = [...system, ...turns.flat()]
+  if (total(fitted) > budgetTokens) {
+    fitted = fitted.map((m) =>
+      m.role === "tool" && m.content.length > MIN_TOOL_CONTENT_CHARS
+        ? {...m, content: `${m.content.slice(0, MIN_TOOL_CONTENT_CHARS)}\n…[truncated]`}
+        : m,
+    )
+  }
+  return fitted
+}
+
+/** Remove a leading `<think>…</think>` block (or an unterminated one). */
+export function stripThinking(text: string): string {
+  const trimmed = text.replace(/^\s*<think>[\s\S]*?<\/think>\s*/u, "")
+  if (trimmed !== text) return trimmed
+  return text.replace(/^\s*<think>[\s\S]*$/u, "").trimStart()
+}
+
+const TOOL_CALL_RE = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/gu
+
+/** Pull `<tool_call>` blocks out of the reply. Unparseable blocks are dropped, not surfaced. */
+export function parseToolCalls(
+  text: string,
+  makeId: (index: number) => string,
+): {toolCalls: LlmToolCall[]; text: string} {
+  const toolCalls: LlmToolCall[] = []
+  let index = 0
+  const remaining = text
+    .replace(TOOL_CALL_RE, (_match, body: string) => {
+      const parsed = parseJsonLoose(body)
+      if (parsed && typeof parsed === "object" && typeof (parsed as {name?: unknown}).name === "string") {
+        const {name, arguments: args} = parsed as {name: string; arguments?: unknown}
+        toolCalls.push({
+          id: makeId(index++),
+          name,
+          input: args && typeof args === "object" && !Array.isArray(args) ? (args as Record<string, unknown>) : {},
+        })
+      }
+      return ""
+    })
+    .trim()
+  return {toolCalls, text: remaining}
+}
+
+function parseJsonLoose(body: string): unknown {
+  try {
+    return JSON.parse(body)
+  } catch {
+    try {
+      return JSON.parse(jsonrepair(body))
+    } catch {
+      return undefined
+    }
+  }
+}
+
+/**
+ * The first JSON object in the reply, repaired if the model left it slightly
+ * broken (trailing commas, single quotes, a dropped brace). Null when there
+ * is nothing object-shaped: the caller keeps the raw text and the miniapp's
+ * own parser decides.
+ */
+export function extractJsonObject(text: string): string | null {
+  const unfenced = text.replace(/```(?:json)?/giu, "").trim()
+  const start = unfenced.indexOf("{")
+  if (start < 0) return null
+  const end = unfenced.lastIndexOf("}")
+  const candidate = end > start ? unfenced.slice(start, end + 1) : unfenced.slice(start)
+  try {
+    JSON.parse(candidate)
+    return candidate
+  } catch {
+    try {
+      const repaired = jsonrepair(candidate)
+      JSON.parse(repaired)
+      return repaired
+    } catch {
+      return null
+    }
+  }
+}
+
+export interface LocalGeneration {
+  text: string
+  promptTokens: number
+  generatedTokens: number
+  maxNewTokens: number
+}
+
+export interface LocalCompletion {
+  text: string
+  stopReason: LlmStopReason
+  toolCalls?: LlmToolCall[]
+  usage: {inputTokens: number; outputTokens: number}
+}
+
+/** Raw generation → neutral completion. */
+export function finalizeLocalResult(
+  generation: LocalGeneration,
+  opts: {jsonMode: boolean; toolsOffered: boolean; makeToolCallId: (index: number) => string},
+): LocalCompletion {
+  const usage = {inputTokens: generation.promptTokens, outputTokens: generation.generatedTokens}
+  let text = stripThinking(generation.text)
+
+  if (opts.toolsOffered) {
+    const parsed = parseToolCalls(text, opts.makeToolCallId)
+    if (parsed.toolCalls.length) {
+      return {text: parsed.text, stopReason: "tool_use", toolCalls: parsed.toolCalls, usage}
+    }
+    text = parsed.text
+  }
+
+  if (opts.jsonMode) {
+    text = extractJsonObject(text) ?? text
+  }
+
+  const stopReason: LlmStopReason = generation.generatedTokens >= generation.maxNewTokens ? "max_tokens" : "end_turn"
+  return {text: text.trim(), stopReason, usage}
+}