@glassly/engine 0.3.5-dev.737 → 0.3.5-dev.740

package/build/generated/releaseMetadata.js

@@ -1,9 +1,9 @@
 export const ENGINE_RELEASE_METADATA = Object.freeze({
     "schemaVersion": 1,
     "familyBaseVersion": "0.3.5",
-    "releaseIdentity": "0.3.5-dev.737",
-    "releaseSetId": "glassly-0.3.5-dev.737",
-    "sourceCommit": "4b824a377398b67972cb0a07483b7be2d7e42824",
+    "releaseIdentity": "0.3.5-dev.740",
+    "releaseSetId": "glassly-0.3.5-dev.740",
+    "sourceCommit": "6c44d91e3f1ca0f107b0a955920d8ca6ac37b1f8",
     "otaManifestUrl": null,
     "otaManifestSha256": null
 });

package/build/generated/releaseMetadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"releaseMetadata.js","sourceRoot":"","sources":["../../src/generated/releaseMetadata.ts"],"names":[],"mappings":"AAWA,MAAM,CAAC,MAAM,uBAAuB,GAAoC,MAAM,CAAC,MAAM,CAAC;IACpF,eAAe,EAAE,CAAC;IAClB,mBAAmB,EAAE,OAAO;IAC5B,iBAAiB,EAAE,eAAe;IAClC,cAAc,EAAE,uBAAuB;IACvC,cAAc,EAAE,0CAA0C;IAC1D,gBAAgB,EAAE,IAAI;IACtB,mBAAmB,EAAE,IAAI;CAC1B,CAAC,CAAA","sourcesContent":["/** Generated by release CI. Do not edit in a release checkout. */\nexport interface EngineReleaseMetadata {\n  schemaVersion: 1\n  familyBaseVersion: string | null\n  releaseIdentity: string | null\n  releaseSetId: string | null\n  sourceCommit: string | null\n  otaManifestUrl: string | null\n  otaManifestSha256: string | null\n}\n\nexport const ENGINE_RELEASE_METADATA: Readonly<EngineReleaseMetadata> = Object.freeze({\n  \"schemaVersion\": 1,\n  \"familyBaseVersion\": \"0.3.5\",\n  \"releaseIdentity\": \"0.3.5-dev.737\",\n  \"releaseSetId\": \"glassly-0.3.5-dev.737\",\n  \"sourceCommit\": \"4b824a377398b67972cb0a07483b7be2d7e42824\",\n  \"otaManifestUrl\": null,\n  \"otaManifestSha256\": null\n})\n"]}
+{"version":3,"file":"releaseMetadata.js","sourceRoot":"","sources":["../../src/generated/releaseMetadata.ts"],"names":[],"mappings":"AAWA,MAAM,CAAC,MAAM,uBAAuB,GAAoC,MAAM,CAAC,MAAM,CAAC;IACpF,eAAe,EAAE,CAAC;IAClB,mBAAmB,EAAE,OAAO;IAC5B,iBAAiB,EAAE,eAAe;IAClC,cAAc,EAAE,uBAAuB;IACvC,cAAc,EAAE,0CAA0C;IAC1D,gBAAgB,EAAE,IAAI;IACtB,mBAAmB,EAAE,IAAI;CAC1B,CAAC,CAAA","sourcesContent":["/** Generated by release CI. Do not edit in a release checkout. */\nexport interface EngineReleaseMetadata {\n  schemaVersion: 1\n  familyBaseVersion: string | null\n  releaseIdentity: string | null\n  releaseSetId: string | null\n  sourceCommit: string | null\n  otaManifestUrl: string | null\n  otaManifestSha256: string | null\n}\n\nexport const ENGINE_RELEASE_METADATA: Readonly<EngineReleaseMetadata> = Object.freeze({\n  \"schemaVersion\": 1,\n  \"familyBaseVersion\": \"0.3.5\",\n  \"releaseIdentity\": \"0.3.5-dev.740\",\n  \"releaseSetId\": \"glassly-0.3.5-dev.740\",\n  \"sourceCommit\": \"6c44d91e3f1ca0f107b0a955920d8ca6ac37b1f8\",\n  \"otaManifestUrl\": null,\n  \"otaManifestSha256\": null\n})\n"]}

package/build/services/LocalLlmPrompt.d.ts

@@ -41,17 +41,18 @@ export declare function buildTemplateMessages(request: {
     jsonMode?: boolean;
 }): TemplateMessage[];
 export declare function toTemplateTools(tools?: LlmTool[]): TemplateTool[] | undefined;
-export declare function estimateTokens(text: string): number;
-export declare function estimateMessageTokens(message: TemplateMessage): number;
+export interface FittedLocalPrompt {
+    prompt: string;
+    promptTokens: number;
+    truncated: boolean;
+}
 /**
- * Fit the conversation into `budgetTokens`. History is grouped into turns
- * (a user message plus everything up to the next user message, so a tool
- * round stays with the question it answers); the oldest turns go first and
- * the last turn is never dropped — the model must at least see the question.
- * When that last turn alone still does not fit, tool outputs are cut to
- * their head.
+ * Measure the complete rendered prompt with the model's tokenizer, including
+ * template framing, system instructions and tool schemas. Drop old turns first,
+ * then shorten tool results / message text in the final turn. Never cut template
+ * delimiters, system instructions, or structured tool calls to make them fit.
  */
-export declare function fitToBudget(messages: TemplateMessage[], budgetTokens: number, toolTokens?: number): TemplateMessage[];
+export declare function fitPromptToContext(messages: TemplateMessage[], budgetTokens: number, render: (messages: TemplateMessage[]) => string, countTokens: (prompt: string) => Promise<number>): Promise<FittedLocalPrompt>;
 /** Remove a leading `<think>…</think>` block (or an unterminated one). */
 export declare function stripThinking(text: string): string;
 /** Pull `<tool_call>` blocks out of the reply. Unparseable blocks are dropped, not surfaced. */

package/build/services/LocalLlmPrompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LocalLlmPrompt.d.ts","sourceRoot":"","sources":["../../src/services/LocalLlmPrompt.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAC,UAAU,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAC,MAAM,6BAA6B,CAAA;AAEhG,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,UAAU,CAAA;IAChB,QAAQ,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAC,CAAA;CAC7D;AAED,MAAM,MAAM,eAAe,GACvB;IAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GAC1C;IAAC,IAAI,EAAE,WAAW,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,gBAAgB,EAAE,CAAA;CAAC,GACrE;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAC,CAAA;AAEzE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAA;IAChB,QAAQ,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAC,CAAA;CACnF;AAED,uFAAuF;AACvF,eAAO,MAAM,qBAAqB,4GACyE,CAAA;AAE3G;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE;IAC7C,QAAQ,EAAE,UAAU,EAAE,CAAA;IACtB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB,GAAG,eAAe,EAAE,CAiCpB;AAED,wBAAgB,eAAe,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,SAAS,CAM7E;AAOD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAMtE;AAID;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,SAAI,GAAG,eAAe,EAAE,CAsBhH;AAED,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIlD;AAID,gGAAgG;AAChG,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAChC;IAAC,SAAS,EAAE,WAAW,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAC,CAkB1C;AAcD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkB7D;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,aAAa,CAAA;IACzB,SAAS,CAAC,EAAE,WAAW,EAAE,CAAA;IACzB,KAAK,EAAE;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAC,CAAA;CACnD;AAED,2CAA2C;AAC3C,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,eAAe,EAC3B,IAAI,EAAE;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAA;CAAC,GAC1F,eAAe,CAkBjB"}
+{"version":3,"file":"LocalLlmPrompt.d.ts","sourceRoot":"","sources":["../../src/services/LocalLlmPrompt.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAC,UAAU,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAC,MAAM,6BAA6B,CAAA;AAEhG,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,UAAU,CAAA;IAChB,QAAQ,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAC,CAAA;CAC7D;AAED,MAAM,MAAM,eAAe,GACvB;IAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GAC1C;IAAC,IAAI,EAAE,WAAW,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,gBAAgB,EAAE,CAAA;CAAC,GACrE;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAC,CAAA;AAEzE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAA;IAChB,QAAQ,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAC,CAAA;CACnF;AAED,uFAAuF;AACvF,eAAO,MAAM,qBAAqB,4GACyE,CAAA;AAE3G;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE;IAC7C,QAAQ,EAAE,UAAU,EAAE,CAAA;IACtB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB,GAAG,eAAe,EAAE,CAiCpB;AAED,wBAAgB,eAAe,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,SAAS,CAM7E;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,OAAO,CAAA;CACnB;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,CAAC,QAAQ,EAAE,eAAe,EAAE,KAAK,MAAM,EAC/C,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAC/C,OAAO,CAAC,iBAAiB,CAAC,CAmD5B;AAED,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIlD;AAID,gGAAgG;AAChG,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAChC;IAAC,SAAS,EAAE,WAAW,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAC,CAkB1C;AAcD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkB7D;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,aAAa,CAAA;IACzB,SAAS,CAAC,EAAE,WAAW,EAAE,CAAA;IACzB,KAAK,EAAE;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAC,CAAA;CACnD;AAED,2CAA2C;AAC3C,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,eAAe,EAC3B,IAAI,EAAE;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAA;CAAC,GAC1F,eAAe,CAkBjB"}

package/build/services/LocalLlmPrompt.js

@@ -55,32 +55,16 @@ export function toTemplateTools(tools) {
         function: { name: tool.name, description: tool.description, parameters: tool.inputSchema },
     }));
 }
-/** Rough tokens-per-character for a BPE vocabulary over mixed prose/JSON. */
-const CHARS_PER_TOKEN = 3.2;
-/** Template framing per message (`<|im_start|>role … <|im_end|>`). */
-const TOKENS_PER_MESSAGE = 6;
-export function estimateTokens(text) {
-    return Math.ceil(text.length / CHARS_PER_TOKEN);
-}
-export function estimateMessageTokens(message) {
-    let total = TOKENS_PER_MESSAGE + estimateTokens(message.content);
-    if (message.role === "assistant" && message.tool_calls) {
-        for (const call of message.tool_calls)
-            total += estimateTokens(JSON.stringify(call.function)) + 8;
-    }
-    return total;
-}
-const MIN_TOOL_CONTENT_CHARS = 1_200;
 /**
- * Fit the conversation into `budgetTokens`. History is grouped into turns
- * (a user message plus everything up to the next user message, so a tool
- * round stays with the question it answers); the oldest turns go first and
- * the last turn is never dropped — the model must at least see the question.
- * When that last turn alone still does not fit, tool outputs are cut to
- * their head.
+ * Measure the complete rendered prompt with the model's tokenizer, including
+ * template framing, system instructions and tool schemas. Drop old turns first,
+ * then shorten tool results / message text in the final turn. Never cut template
+ * delimiters, system instructions, or structured tool calls to make them fit.
  */
-export function fitToBudget(messages, budgetTokens, toolTokens = 0) {
-    const total = (list) => list.reduce((sum, m) => sum + estimateMessageTokens(m), toolTokens);
+export async function fitPromptToContext(messages, budgetTokens, render, countTokens) {
+    if (!Number.isInteger(budgetTokens) || budgetTokens < 1) {
+        throw new Error("On-device model has no room for a prompt after reserving output tokens");
+    }
     const system = messages[0]?.role === "system" ? [messages[0]] : [];
     const turns = [];
     for (const message of messages.slice(system.length)) {
@@ -89,16 +73,43 @@ export function fitToBudget(messages, budgetTokens, toolTokens = 0) {
         else
             turns[turns.length - 1].push(message);
     }
-    while (turns.length > 1 && total([...system, ...turns.flat()]) > budgetTokens) {
-        turns.shift();
-    }
     let fitted = [...system, ...turns.flat()];
-    if (total(fitted) > budgetTokens) {
-        fitted = fitted.map((m) => m.role === "tool" && m.content.length > MIN_TOOL_CONTENT_CHARS
-            ? { ...m, content: `${m.content.slice(0, MIN_TOOL_CONTENT_CHARS)}\n…[truncated]` }
-            : m);
+    let truncated = false;
+    const marker = "\n…[truncated]…\n";
+    while (true) {
+        const prompt = render(fitted);
+        const promptTokens = await countTokens(prompt);
+        if (!Number.isInteger(promptTokens) || promptTokens < 1) {
+            throw new Error("On-device tokenizer returned an invalid prompt token count");
+        }
+        if (promptTokens <= budgetTokens)
+            return { prompt, promptTokens, truncated };
+        truncated = true;
+        if (turns.length > 1) {
+            turns.shift();
+            fitted = [...system, ...turns.flat()];
+            continue;
+        }
+        // Preserve the user's question while tool output can still be shortened.
+        const candidates = fitted
+            .map((message, index) => ({ message, index }))
+            .filter(({ message }) => message.role !== "system" && Array.from(message.content).length > marker.length * 2)
+            .sort((a, b) => Number(b.message.role === "tool") - Number(a.message.role === "tool") ||
+            b.message.content.length - a.message.content.length);
+        const candidate = candidates[0];
+        if (!candidate) {
+            throw new Error(`On-device prompt instructions and tool definitions exceed the context budget (${promptTokens} > ${budgetTokens} tokens)`);
+        }
+        // Keep both ends of long transcripts and questions, without splitting a
+        // Unicode code point. Each pass shrinks strictly, then remeasures the whole
+        // rendered prompt; character ratios are never used as a safety check.
+        const chars = Array.from(candidate.message.content);
+        const keep = Math.floor(chars.length / 4);
+        fitted[candidate.index] = {
+            ...candidate.message,
+            content: `${chars.slice(0, keep).join("")}${marker}${chars.slice(-keep).join("")}`,
+        };
     }
-    return fitted;
 }
 /** Remove a leading `<think>…</think>` block (or an unterminated one). */
 export function stripThinking(text) {

package/build/services/LocalLlmPrompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"LocalLlmPrompt.js","sourceRoot":"","sources":["../../src/services/LocalLlmPrompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAC,UAAU,EAAC,MAAM,YAAY,CAAA;AAoBrC,uFAAuF;AACvF,MAAM,CAAC,MAAM,qBAAqB,GAChC,yGAAyG,CAAA;AAE3G;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAIrC;IACC,MAAM,GAAG,GAAsB,EAAE,CAAA;IACjC,MAAM,WAAW,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CACvG,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CACxC,CAAA;IACD,IAAI,WAAW,CAAC,MAAM;QAAE,GAAG,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAC,CAAC,CAAA;IAErF,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACvC,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,GAAG,CAAmB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACpE,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,IAAI,EAAE,UAAU;gBAChB,QAAQ,EAAE,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,EAAC;aACnD,CAAC,CAAC,CAAA;YACH,GAAG,CAAC,IAAI,CACN,SAAS,EAAE,MAAM;gBACf,CAAC,CAAC,EAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAC;gBACtE,CAAC,CAAC,EAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAC,CAClD,CAAA;YACD,SAAQ;QACV,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;YAC/C,GAAG,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE,MAAM;gBACZ,YAAY,EAAE,MAAM,CAAC,UAAU;gBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO;aACtE,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;YACtE,GAAG,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAC,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,IAAI,CAAC,KAAK,EAAE,MAAM;QAAE,OAAO,SAAS,CAAA;IACpC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,EAAC;KACzF,CAAC,CAAC,CAAA;AACL,CAAC;AAED,6EAA6E;AAC7E,MAAM,eAAe,GAAG,GAAG,CAAA;AAC3B,sEAAsE;AACtE,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAwB;IAC5D,IAAI,KAAK,GAAG,kBAAkB,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAChE,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,UAAU;YAAE,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAA;IACnG,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,sBAAsB,GAAG,KAAK,CAAA;AAEpC;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CAAC,QAA2B,EAAE,YAAoB,EAAE,UAAU,GAAG,CAAC;IAC3F,MAAM,KAAK,GAAG,CAAC,IAAuB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,qBAAqB,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;IAC9G,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAClE,MAAM,KAAK,GAAwB,EAAE,CAAA;IACrC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;;YACnE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,YAAY,EAAE,CAAC;QAC9E,KAAK,CAAC,KAAK,EAAE,CAAA;IACf,CAAC;IAED,IAAI,MAAM,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACzC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,YAAY,EAAE,CAAC;QACjC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACxB,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,sBAAsB;YAC5D,CAAC,CAAC,EAAC,GAAG,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,EAAC;YAChF,CAAC,CAAC,CAAC,CACN,CAAA;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC,CAAA;IACpE,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,OAAO,CAAA;IACpC,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,SAAS,EAAE,CAAA;AAC7D,CAAC;AAED,MAAM,YAAY,GAAG,4CAA4C,CAAA;AAEjE,gGAAgG;AAChG,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,MAAiC;IAEjC,MAAM,SAAS,GAAkB,EAAE,CAAA;IACnC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,MAAM,SAAS,GAAG,IAAI;SACnB,OAAO,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,IAAY,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;QACnC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAQ,MAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClG,MAAM,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAC,GAAG,MAA6C,CAAA;YAC7E,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;gBACnB,IAAI;gBACJ,KAAK,EAAE,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAAgC,CAAC,CAAC,CAAC,EAAE;aACzG,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,EAAE,CAAA;IACX,CAAC,CAAC;SACD,IAAI,EAAE,CAAA;IACT,OAAO,EAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAC,CAAA;AACrC,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAA;QAClB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IACnC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IACrC,MAAM,SAAS,GAAG,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACtF,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACrB,OAAO,SAAS,CAAA;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YACpB,OAAO,QAAQ,CAAA;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAgBD,2CAA2C;AAC3C,MAAM,UAAU,mBAAmB,CACjC,UAA2B,EAC3B,IAA2F;IAE3F,MAAM,KAAK,GAAG,EAAC,WAAW,EAAE,UAAU,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,CAAC,eAAe,EAAC,CAAA;IAC9F,IAAI,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;IAEzC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAA;QACxD,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAC5B,OAAO,EAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAC,CAAA;QACxF,CAAC;QACD,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACpB,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;IACxC,CAAC;IAED,MAAM,UAAU,GAAkB,UAAU,CAAC,eAAe,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAA;IACnH,OAAO,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAC,CAAA;AAC/C,CAAC","sourcesContent":["/**\n * LocalLlmPrompt — the pure half of the on-device LLM path: turning a neutral\n * session.ai request into chat-template messages, fitting them into the\n * model's context, and reading the model's text back into the neutral result\n * (tool calls, JSON mode, stop reason). No native imports, so it is\n * bun-testable; LocalLlmProvider owns the runner.\n *\n * Written against Qwen3's chat template: tools are advertised by the\n * template itself, calls come back as `<tool_call>{json}</tool_call>`, and\n * thinking is a `<think>…</think>` block we never ask for but still strip.\n */\nimport {jsonrepair} from \"jsonrepair\"\n\nimport type {LlmMessage, LlmStopReason, LlmTool, LlmToolCall} from \"@glassly/cloud-protocol/llm\"\n\nexport interface TemplateToolCall {\n  id: string\n  type: \"function\"\n  function: {name: string; arguments: Record<string, unknown>}\n}\n\nexport type TemplateMessage =\n  | {role: \"system\" | \"user\"; content: string}\n  | {role: \"assistant\"; content: string; tool_calls?: TemplateToolCall[]}\n  | {role: \"tool\"; content: string; name?: string; tool_call_id?: string}\n\nexport interface TemplateTool {\n  type: \"function\"\n  function: {name: string; description: string; parameters: Record<string, unknown>}\n}\n\n/** Appended to the system prompt under `jsonMode`; small models need it said twice. */\nexport const JSON_MODE_INSTRUCTION =\n  \"Respond with exactly one JSON object and nothing else: no prose before or after it, no markdown fences.\"\n\n/**\n * Neutral messages → template messages. A user turn carrying toolResults\n * becomes one `tool` message per result (the template groups consecutive\n * tool messages into a single user block), followed by the user text if any.\n */\nexport function buildTemplateMessages(request: {\n  messages: LlmMessage[]\n  system?: string\n  jsonMode?: boolean\n}): TemplateMessage[] {\n  const out: TemplateMessage[] = []\n  const systemParts = [request.system?.trim(), request.jsonMode ? JSON_MODE_INSTRUCTION : undefined].filter(\n    (part): part is string => Boolean(part),\n  )\n  if (systemParts.length) out.push({role: \"system\", content: systemParts.join(\"\\n\\n\")})\n\n  for (const message of request.messages) {\n    if (message.role === \"assistant\") {\n      const toolCalls = message.toolCalls?.map<TemplateToolCall>((call) => ({\n        id: call.id,\n        type: \"function\",\n        function: {name: call.name, arguments: call.input},\n      }))\n      out.push(\n        toolCalls?.length\n          ? {role: \"assistant\", content: message.content, tool_calls: toolCalls}\n          : {role: \"assistant\", content: message.content},\n      )\n      continue\n    }\n    for (const result of message.toolResults ?? []) {\n      out.push({\n        role: \"tool\",\n        tool_call_id: result.toolCallId,\n        content: result.isError ? `Error: ${result.content}` : result.content,\n      })\n    }\n    if (message.content.trim().length > 0 || !message.toolResults?.length) {\n      out.push({role: \"user\", content: message.content})\n    }\n  }\n  return out\n}\n\nexport function toTemplateTools(tools?: LlmTool[]): TemplateTool[] | undefined {\n  if (!tools?.length) return undefined\n  return tools.map((tool) => ({\n    type: \"function\",\n    function: {name: tool.name, description: tool.description, parameters: tool.inputSchema},\n  }))\n}\n\n/** Rough tokens-per-character for a BPE vocabulary over mixed prose/JSON. */\nconst CHARS_PER_TOKEN = 3.2\n/** Template framing per message (`<|im_start|>role … <|im_end|>`). */\nconst TOKENS_PER_MESSAGE = 6\n\nexport function estimateTokens(text: string): number {\n  return Math.ceil(text.length / CHARS_PER_TOKEN)\n}\n\nexport function estimateMessageTokens(message: TemplateMessage): number {\n  let total = TOKENS_PER_MESSAGE + estimateTokens(message.content)\n  if (message.role === \"assistant\" && message.tool_calls) {\n    for (const call of message.tool_calls) total += estimateTokens(JSON.stringify(call.function)) + 8\n  }\n  return total\n}\n\nconst MIN_TOOL_CONTENT_CHARS = 1_200\n\n/**\n * Fit the conversation into `budgetTokens`. History is grouped into turns\n * (a user message plus everything up to the next user message, so a tool\n * round stays with the question it answers); the oldest turns go first and\n * the last turn is never dropped — the model must at least see the question.\n * When that last turn alone still does not fit, tool outputs are cut to\n * their head.\n */\nexport function fitToBudget(messages: TemplateMessage[], budgetTokens: number, toolTokens = 0): TemplateMessage[] {\n  const total = (list: TemplateMessage[]) => list.reduce((sum, m) => sum + estimateMessageTokens(m), toolTokens)\n  const system = messages[0]?.role === \"system\" ? [messages[0]] : []\n  const turns: TemplateMessage[][] = []\n  for (const message of messages.slice(system.length)) {\n    if (message.role === \"user\" || turns.length === 0) turns.push([message])\n    else turns[turns.length - 1]!.push(message)\n  }\n\n  while (turns.length > 1 && total([...system, ...turns.flat()]) > budgetTokens) {\n    turns.shift()\n  }\n\n  let fitted = [...system, ...turns.flat()]\n  if (total(fitted) > budgetTokens) {\n    fitted = fitted.map((m) =>\n      m.role === \"tool\" && m.content.length > MIN_TOOL_CONTENT_CHARS\n        ? {...m, content: `${m.content.slice(0, MIN_TOOL_CONTENT_CHARS)}\\n…[truncated]`}\n        : m,\n    )\n  }\n  return fitted\n}\n\n/** Remove a leading `<think>…</think>` block (or an unterminated one). */\nexport function stripThinking(text: string): string {\n  const trimmed = text.replace(/^\\s*<think>[\\s\\S]*?<\\/think>\\s*/u, \"\")\n  if (trimmed !== text) return trimmed\n  return text.replace(/^\\s*<think>[\\s\\S]*$/u, \"\").trimStart()\n}\n\nconst TOOL_CALL_RE = /<tool_call>\\s*([\\s\\S]*?)\\s*<\\/tool_call>/gu\n\n/** Pull `<tool_call>` blocks out of the reply. Unparseable blocks are dropped, not surfaced. */\nexport function parseToolCalls(\n  text: string,\n  makeId: (index: number) => string,\n): {toolCalls: LlmToolCall[]; text: string} {\n  const toolCalls: LlmToolCall[] = []\n  let index = 0\n  const remaining = text\n    .replace(TOOL_CALL_RE, (_match, body: string) => {\n      const parsed = parseJsonLoose(body)\n      if (parsed && typeof parsed === \"object\" && typeof (parsed as {name?: unknown}).name === \"string\") {\n        const {name, arguments: args} = parsed as {name: string; arguments?: unknown}\n        toolCalls.push({\n          id: makeId(index++),\n          name,\n          input: args && typeof args === \"object\" && !Array.isArray(args) ? (args as Record<string, unknown>) : {},\n        })\n      }\n      return \"\"\n    })\n    .trim()\n  return {toolCalls, text: remaining}\n}\n\nfunction parseJsonLoose(body: string): unknown {\n  try {\n    return JSON.parse(body)\n  } catch {\n    try {\n      return JSON.parse(jsonrepair(body))\n    } catch {\n      return undefined\n    }\n  }\n}\n\n/**\n * The first JSON object in the reply, repaired if the model left it slightly\n * broken (trailing commas, single quotes, a dropped brace). Null when there\n * is nothing object-shaped: the caller keeps the raw text and the miniapp's\n * own parser decides.\n */\nexport function extractJsonObject(text: string): string | null {\n  const unfenced = text.replace(/```(?:json)?/giu, \"\").trim()\n  const start = unfenced.indexOf(\"{\")\n  if (start < 0) return null\n  const end = unfenced.lastIndexOf(\"}\")\n  const candidate = end > start ? unfenced.slice(start, end + 1) : unfenced.slice(start)\n  try {\n    JSON.parse(candidate)\n    return candidate\n  } catch {\n    try {\n      const repaired = jsonrepair(candidate)\n      JSON.parse(repaired)\n      return repaired\n    } catch {\n      return null\n    }\n  }\n}\n\nexport interface LocalGeneration {\n  text: string\n  promptTokens: number\n  generatedTokens: number\n  maxNewTokens: number\n}\n\nexport interface LocalCompletion {\n  text: string\n  stopReason: LlmStopReason\n  toolCalls?: LlmToolCall[]\n  usage: {inputTokens: number; outputTokens: number}\n}\n\n/** Raw generation → neutral completion. */\nexport function finalizeLocalResult(\n  generation: LocalGeneration,\n  opts: {jsonMode: boolean; toolsOffered: boolean; makeToolCallId: (index: number) => string},\n): LocalCompletion {\n  const usage = {inputTokens: generation.promptTokens, outputTokens: generation.generatedTokens}\n  let text = stripThinking(generation.text)\n\n  if (opts.toolsOffered) {\n    const parsed = parseToolCalls(text, opts.makeToolCallId)\n    if (parsed.toolCalls.length) {\n      return {text: parsed.text, stopReason: \"tool_use\", toolCalls: parsed.toolCalls, usage}\n    }\n    text = parsed.text\n  }\n\n  if (opts.jsonMode) {\n    text = extractJsonObject(text) ?? text\n  }\n\n  const stopReason: LlmStopReason = generation.generatedTokens >= generation.maxNewTokens ? \"max_tokens\" : \"end_turn\"\n  return {text: text.trim(), stopReason, usage}\n}\n"]}
+{"version":3,"file":"LocalLlmPrompt.js","sourceRoot":"","sources":["../../src/services/LocalLlmPrompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAC,UAAU,EAAC,MAAM,YAAY,CAAA;AAoBrC,uFAAuF;AACvF,MAAM,CAAC,MAAM,qBAAqB,GAChC,yGAAyG,CAAA;AAE3G;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAIrC;IACC,MAAM,GAAG,GAAsB,EAAE,CAAA;IACjC,MAAM,WAAW,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CACvG,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CACxC,CAAA;IACD,IAAI,WAAW,CAAC,MAAM;QAAE,GAAG,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAC,CAAC,CAAA;IAErF,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACvC,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,GAAG,CAAmB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACpE,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,IAAI,EAAE,UAAU;gBAChB,QAAQ,EAAE,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,EAAC;aACnD,CAAC,CAAC,CAAA;YACH,GAAG,CAAC,IAAI,CACN,SAAS,EAAE,MAAM;gBACf,CAAC,CAAC,EAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAC;gBACtE,CAAC,CAAC,EAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAC,CAClD,CAAA;YACD,SAAQ;QACV,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;YAC/C,GAAG,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE,MAAM;gBACZ,YAAY,EAAE,MAAM,CAAC,UAAU;gBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO;aACtE,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;YACtE,GAAG,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAC,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,IAAI,CAAC,KAAK,EAAE,MAAM;QAAE,OAAO,SAAS,CAAA;IACpC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,EAAC;KACzF,CAAC,CAAC,CAAA;AACL,CAAC;AAQD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAA2B,EAC3B,YAAoB,EACpB,MAA+C,EAC/C,WAAgD;IAEhD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAA;IAC3F,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAClE,MAAM,KAAK,GAAwB,EAAE,CAAA;IACrC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;;YACnE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC7C,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACzC,IAAI,SAAS,GAAG,KAAK,CAAA;IACrB,MAAM,MAAM,GAAG,mBAAmB,CAAA;IAClC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;QAC7B,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAA;QAC9C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;QAC/E,CAAC;QACD,IAAI,YAAY,IAAI,YAAY;YAAE,OAAO,EAAC,MAAM,EAAE,YAAY,EAAE,SAAS,EAAC,CAAA;QAC1E,SAAS,GAAG,IAAI,CAAA;QAChB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,MAAM,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;YACrC,SAAQ;QACV,CAAC;QACD,yEAAyE;QACzE,MAAM,UAAU,GAAG,MAAM;aACtB,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAC,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;aAC3C,MAAM,CAAC,CAAC,EAAC,OAAO,EAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;aAC1G,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC;YACrE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CACtD,CAAA;QACH,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;QAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,iFAAiF,YAAY,MAAM,YAAY,UAAU,CAC1H,CAAA;QACH,CAAC;QACD,wEAAwE;QACxE,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QACzC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG;YACxB,GAAG,SAAS,CAAC,OAAO;YACpB,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;SACnF,CAAA;IACH,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC,CAAA;IACpE,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,OAAO,CAAA;IACpC,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,SAAS,EAAE,CAAA;AAC7D,CAAC;AAED,MAAM,YAAY,GAAG,4CAA4C,CAAA;AAEjE,gGAAgG;AAChG,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,MAAiC;IAEjC,MAAM,SAAS,GAAkB,EAAE,CAAA;IACnC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,MAAM,SAAS,GAAG,IAAI;SACnB,OAAO,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,IAAY,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;QACnC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAQ,MAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClG,MAAM,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAC,GAAG,MAA6C,CAAA;YAC7E,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;gBACnB,IAAI;gBACJ,KAAK,EAAE,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAAgC,CAAC,CAAC,CAAC,EAAE;aACzG,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,EAAE,CAAA;IACX,CAAC,CAAC;SACD,IAAI,EAAE,CAAA;IACT,OAAO,EAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAC,CAAA;AACrC,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAA;QAClB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IACnC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IACrC,MAAM,SAAS,GAAG,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACtF,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACrB,OAAO,SAAS,CAAA;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YACpB,OAAO,QAAQ,CAAA;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAgBD,2CAA2C;AAC3C,MAAM,UAAU,mBAAmB,CACjC,UAA2B,EAC3B,IAA2F;IAE3F,MAAM,KAAK,GAAG,EAAC,WAAW,EAAE,UAAU,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,CAAC,eAAe,EAAC,CAAA;IAC9F,IAAI,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;IAEzC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAA;QACxD,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAC5B,OAAO,EAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAC,CAAA;QACxF,CAAC;QACD,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACpB,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;IACxC,CAAC;IAED,MAAM,UAAU,GAAkB,UAAU,CAAC,eAAe,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAA;IACnH,OAAO,EAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAC,CAAA;AAC/C,CAAC","sourcesContent":["/**\n * LocalLlmPrompt — the pure half of the on-device LLM path: turning a neutral\n * session.ai request into chat-template messages, fitting them into the\n * model's context, and reading the model's text back into the neutral result\n * (tool calls, JSON mode, stop reason). No native imports, so it is\n * bun-testable; LocalLlmProvider owns the runner.\n *\n * Written against Qwen3's chat template: tools are advertised by the\n * template itself, calls come back as `<tool_call>{json}</tool_call>`, and\n * thinking is a `<think>…</think>` block we never ask for but still strip.\n */\nimport {jsonrepair} from \"jsonrepair\"\n\nimport type {LlmMessage, LlmStopReason, LlmTool, LlmToolCall} from \"@glassly/cloud-protocol/llm\"\n\nexport interface TemplateToolCall {\n  id: string\n  type: \"function\"\n  function: {name: string; arguments: Record<string, unknown>}\n}\n\nexport type TemplateMessage =\n  | {role: \"system\" | \"user\"; content: string}\n  | {role: \"assistant\"; content: string; tool_calls?: TemplateToolCall[]}\n  | {role: \"tool\"; content: string; name?: string; tool_call_id?: string}\n\nexport interface TemplateTool {\n  type: \"function\"\n  function: {name: string; description: string; parameters: Record<string, unknown>}\n}\n\n/** Appended to the system prompt under `jsonMode`; small models need it said twice. */\nexport const JSON_MODE_INSTRUCTION =\n  \"Respond with exactly one JSON object and nothing else: no prose before or after it, no markdown fences.\"\n\n/**\n * Neutral messages → template messages. A user turn carrying toolResults\n * becomes one `tool` message per result (the template groups consecutive\n * tool messages into a single user block), followed by the user text if any.\n */\nexport function buildTemplateMessages(request: {\n  messages: LlmMessage[]\n  system?: string\n  jsonMode?: boolean\n}): TemplateMessage[] {\n  const out: TemplateMessage[] = []\n  const systemParts = [request.system?.trim(), request.jsonMode ? JSON_MODE_INSTRUCTION : undefined].filter(\n    (part): part is string => Boolean(part),\n  )\n  if (systemParts.length) out.push({role: \"system\", content: systemParts.join(\"\\n\\n\")})\n\n  for (const message of request.messages) {\n    if (message.role === \"assistant\") {\n      const toolCalls = message.toolCalls?.map<TemplateToolCall>((call) => ({\n        id: call.id,\n        type: \"function\",\n        function: {name: call.name, arguments: call.input},\n      }))\n      out.push(\n        toolCalls?.length\n          ? {role: \"assistant\", content: message.content, tool_calls: toolCalls}\n          : {role: \"assistant\", content: message.content},\n      )\n      continue\n    }\n    for (const result of message.toolResults ?? []) {\n      out.push({\n        role: \"tool\",\n        tool_call_id: result.toolCallId,\n        content: result.isError ? `Error: ${result.content}` : result.content,\n      })\n    }\n    if (message.content.trim().length > 0 || !message.toolResults?.length) {\n      out.push({role: \"user\", content: message.content})\n    }\n  }\n  return out\n}\n\nexport function toTemplateTools(tools?: LlmTool[]): TemplateTool[] | undefined {\n  if (!tools?.length) return undefined\n  return tools.map((tool) => ({\n    type: \"function\",\n    function: {name: tool.name, description: tool.description, parameters: tool.inputSchema},\n  }))\n}\n\nexport interface FittedLocalPrompt {\n  prompt: string\n  promptTokens: number\n  truncated: boolean\n}\n\n/**\n * Measure the complete rendered prompt with the model's tokenizer, including\n * template framing, system instructions and tool schemas. Drop old turns first,\n * then shorten tool results / message text in the final turn. Never cut template\n * delimiters, system instructions, or structured tool calls to make them fit.\n */\nexport async function fitPromptToContext(\n  messages: TemplateMessage[],\n  budgetTokens: number,\n  render: (messages: TemplateMessage[]) => string,\n  countTokens: (prompt: string) => Promise<number>,\n): Promise<FittedLocalPrompt> {\n  if (!Number.isInteger(budgetTokens) || budgetTokens < 1) {\n    throw new Error(\"On-device model has no room for a prompt after reserving output tokens\")\n  }\n  const system = messages[0]?.role === \"system\" ? [messages[0]] : []\n  const turns: TemplateMessage[][] = []\n  for (const message of messages.slice(system.length)) {\n    if (message.role === \"user\" || turns.length === 0) turns.push([message])\n    else turns[turns.length - 1]!.push(message)\n  }\n  let fitted = [...system, ...turns.flat()]\n  let truncated = false\n  const marker = \"\\n…[truncated]…\\n\"\n  while (true) {\n    const prompt = render(fitted)\n    const promptTokens = await countTokens(prompt)\n    if (!Number.isInteger(promptTokens) || promptTokens < 1) {\n      throw new Error(\"On-device tokenizer returned an invalid prompt token count\")\n    }\n    if (promptTokens <= budgetTokens) return {prompt, promptTokens, truncated}\n    truncated = true\n    if (turns.length > 1) {\n      turns.shift()\n      fitted = [...system, ...turns.flat()]\n      continue\n    }\n    // Preserve the user's question while tool output can still be shortened.\n    const candidates = fitted\n      .map((message, index) => ({message, index}))\n      .filter(({message}) => message.role !== \"system\" && Array.from(message.content).length > marker.length * 2)\n      .sort(\n        (a, b) =>\n          Number(b.message.role === \"tool\") - Number(a.message.role === \"tool\") ||\n          b.message.content.length - a.message.content.length,\n      )\n    const candidate = candidates[0]\n    if (!candidate) {\n      throw new Error(\n        `On-device prompt instructions and tool definitions exceed the context budget (${promptTokens} > ${budgetTokens} tokens)`,\n      )\n    }\n    // Keep both ends of long transcripts and questions, without splitting a\n    // Unicode code point. Each pass shrinks strictly, then remeasures the whole\n    // rendered prompt; character ratios are never used as a safety check.\n    const chars = Array.from(candidate.message.content)\n    const keep = Math.floor(chars.length / 4)\n    fitted[candidate.index] = {\n      ...candidate.message,\n      content: `${chars.slice(0, keep).join(\"\")}${marker}${chars.slice(-keep).join(\"\")}`,\n    }\n  }\n}\n\n/** Remove a leading `<think>…</think>` block (or an unterminated one). */\nexport function stripThinking(text: string): string {\n  const trimmed = text.replace(/^\\s*<think>[\\s\\S]*?<\\/think>\\s*/u, \"\")\n  if (trimmed !== text) return trimmed\n  return text.replace(/^\\s*<think>[\\s\\S]*$/u, \"\").trimStart()\n}\n\nconst TOOL_CALL_RE = /<tool_call>\\s*([\\s\\S]*?)\\s*<\\/tool_call>/gu\n\n/** Pull `<tool_call>` blocks out of the reply. Unparseable blocks are dropped, not surfaced. */\nexport function parseToolCalls(\n  text: string,\n  makeId: (index: number) => string,\n): {toolCalls: LlmToolCall[]; text: string} {\n  const toolCalls: LlmToolCall[] = []\n  let index = 0\n  const remaining = text\n    .replace(TOOL_CALL_RE, (_match, body: string) => {\n      const parsed = parseJsonLoose(body)\n      if (parsed && typeof parsed === \"object\" && typeof (parsed as {name?: unknown}).name === \"string\") {\n        const {name, arguments: args} = parsed as {name: string; arguments?: unknown}\n        toolCalls.push({\n          id: makeId(index++),\n          name,\n          input: args && typeof args === \"object\" && !Array.isArray(args) ? (args as Record<string, unknown>) : {},\n        })\n      }\n      return \"\"\n    })\n    .trim()\n  return {toolCalls, text: remaining}\n}\n\nfunction parseJsonLoose(body: string): unknown {\n  try {\n    return JSON.parse(body)\n  } catch {\n    try {\n      return JSON.parse(jsonrepair(body))\n    } catch {\n      return undefined\n    }\n  }\n}\n\n/**\n * The first JSON object in the reply, repaired if the model left it slightly\n * broken (trailing commas, single quotes, a dropped brace). Null when there\n * is nothing object-shaped: the caller keeps the raw text and the miniapp's\n * own parser decides.\n */\nexport function extractJsonObject(text: string): string | null {\n  const unfenced = text.replace(/```(?:json)?/giu, \"\").trim()\n  const start = unfenced.indexOf(\"{\")\n  if (start < 0) return null\n  const end = unfenced.lastIndexOf(\"}\")\n  const candidate = end > start ? unfenced.slice(start, end + 1) : unfenced.slice(start)\n  try {\n    JSON.parse(candidate)\n    return candidate\n  } catch {\n    try {\n      const repaired = jsonrepair(candidate)\n      JSON.parse(repaired)\n      return repaired\n    } catch {\n      return null\n    }\n  }\n}\n\nexport interface LocalGeneration {\n  text: string\n  promptTokens: number\n  generatedTokens: number\n  maxNewTokens: number\n}\n\nexport interface LocalCompletion {\n  text: string\n  stopReason: LlmStopReason\n  toolCalls?: LlmToolCall[]\n  usage: {inputTokens: number; outputTokens: number}\n}\n\n/** Raw generation → neutral completion. */\nexport function finalizeLocalResult(\n  generation: LocalGeneration,\n  opts: {jsonMode: boolean; toolsOffered: boolean; makeToolCallId: (index: number) => string},\n): LocalCompletion {\n  const usage = {inputTokens: generation.promptTokens, outputTokens: generation.generatedTokens}\n  let text = stripThinking(generation.text)\n\n  if (opts.toolsOffered) {\n    const parsed = parseToolCalls(text, opts.makeToolCallId)\n    if (parsed.toolCalls.length) {\n      return {text: parsed.text, stopReason: \"tool_use\", toolCalls: parsed.toolCalls, usage}\n    }\n    text = parsed.text\n  }\n\n  if (opts.jsonMode) {\n    text = extractJsonObject(text) ?? text\n  }\n\n  const stopReason: LlmStopReason = generation.generatedTokens >= generation.maxNewTokens ? \"max_tokens\" : \"end_turn\"\n  return {text: text.trim(), stopReason, usage}\n}\n"]}

package/build/services/LocalLlmProvider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LocalLlmProvider.d.ts","sourceRoot":"","sources":["../../src/services/LocalLlmProvider.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,6BAA6B,CAAA;AAEnE,OAA6B,EAAC,KAAK,eAAe,EAAC,MAAM,wBAAwB,CAAA;AACjF,OAAO,EAML,KAAK,eAAe,EACrB,MAAM,kBAAkB,CAAA;AAmCzB,MAAM,WAAW,sBAAuB,SAAQ,eAAe;IAC7D,OAAO,EAAE,eAAe,CAAA;CACzB;AAqBD,cAAM,gBAAgB;IACpB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAkB;IAEzC,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,GAAG,CAAmB;IAE9B,OAAO;IAUP,MAAM,CAAC,WAAW,IAAI,gBAAgB;IAKtC,mFAAmF;IACnF,QAAQ,IAAI,OAAO;IAInB,aAAa,IAAI,eAAe,GAAG,IAAI;IAIvC,oGAAoG;IACpG,QAAQ,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMtE,kGAAkG;IAC5F,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAe7B,yEAAyE;IACzE,OAAO,CAAC,aAAa;YAWP,aAAa;YA6Db,YAAY;YAgBZ,IAAI;YA6BJ,OAAO;IASrB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,cAAc;CAMvB;AAED,QAAA,MAAM,gBAAgB,kBAAiC,CAAA;AACvD,OAAO,EAAC,gBAAgB,EAAC,CAAA;AACzB,eAAe,gBAAgB,CAAA"}
+{"version":3,"file":"LocalLlmProvider.d.ts","sourceRoot":"","sources":["../../src/services/LocalLlmProvider.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,6BAA6B,CAAA;AAEnE,OAA6B,EAAC,KAAK,eAAe,EAAC,MAAM,wBAAwB,CAAA;AACjF,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,kBAAkB,CAAA;AAsCzB,MAAM,WAAW,sBAAuB,SAAQ,eAAe;IAC7D,OAAO,EAAE,eAAe,CAAA;CACzB;AA0BD,cAAM,gBAAgB;IACpB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAkB;IAEzC,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,GAAG,CAAmB;IAE9B,OAAO;IAUP,MAAM,CAAC,WAAW,IAAI,gBAAgB;IAKtC,mFAAmF;IACnF,QAAQ,IAAI,OAAO;IAInB,aAAa,IAAI,eAAe,GAAG,IAAI;IAIvC,oGAAoG;IACpG,QAAQ,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAMtE,kGAAkG;IAC5F,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAe7B,yEAAyE;IACzE,OAAO,CAAC,aAAa;YAaP,aAAa;YAmEb,YAAY;YAgBZ,IAAI;YAyCJ,OAAO;IASrB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,cAAc;CAMvB;AAED,QAAA,MAAM,gBAAgB,kBAAiC,CAAA;AACvD,OAAO,EAAC,gBAAgB,EAAC,CAAA;AACzB,eAAe,gBAAgB,CAAA"}

package/build/services/LocalLlmProvider.js

@@ -18,7 +18,7 @@ import * as RNFS from "@dr.pogodin/react-native-fs";
 import { Template } from "@huggingface/jinja";
 import { BgTimer } from "../utils/timers";
 import localLlmModelManager from "./LocalLlmModelManager";
-import { buildTemplateMessages, estimateTokens, finalizeLocalResult, fitToBudget, toTemplateTools, } from "./LocalLlmPrompt";
+import { buildTemplateMessages, finalizeLocalResult, fitPromptToContext, toTemplateTools, } from "./LocalLlmPrompt";
 import { SETTINGS, useSettingsStore } from "../stores/settings";
 const LOG_TAG = "LOCAL_LLM";
 /** Drop the resident model after this long without a request. */
@@ -27,7 +27,7 @@ const IDLE_UNLOAD_MS = 5 * 60_000;
 const GENERATION_DEADLINE_MS = 100_000;
 const DEFAULT_MAX_NEW_TOKENS = 512;
 const MAX_NEW_TOKENS_CEILING = 1_024;
-/** Headroom between the estimated prompt and the context end; estimates are rough. */
+/** Reserve space for runner-added BOS/EOS tokens and context boundary bookkeeping. */
 const CONTEXT_MARGIN_TOKENS = 96;
 /**
  * Runs on ExecuTorch's worklet runtime: `runner.generate` blocks its thread
@@ -43,6 +43,10 @@ function generateWorklet(runner, prompt, config, stopTokens) {
     });
     return { text, stats };
 }
+function countTokensWorklet(tokenizer, prompt) {
+    "worklet";
+    return tokenizer.encode(prompt).length;
+}
 class LocalLlmProvider {
     static instance;
     loaded = null;
@@ -102,6 +106,9 @@ class LocalLlmProvider {
         catch (error) {
             console.warn(`${LOG_TAG}: runner dispose failed:`, error);
         }
+        finally {
+            loaded.tokenizer.dispose();
+        }
     }
     async completeInner(request) {
         this.clearIdleTimer();
@@ -109,16 +116,10 @@ class LocalLlmProvider {
             const model = await this.ensureLoaded();
             const jsonMode = request.jsonMode === true;
             const tools = toTemplateTools(request.tools);
-            const toolTokens = tools ? estimateTokens(JSON.stringify(tools)) + 64 : 0;
             const maxNewTokens = Math.max(16, Math.min(request.maxTokens ?? DEFAULT_MAX_NEW_TOKENS, MAX_NEW_TOKENS_CEILING, Math.floor(model.maxSeqLen / 4)));
             const promptBudget = model.maxSeqLen - maxNewTokens - CONTEXT_MARGIN_TOKENS;
-            const messages = fitToBudget(buildTemplateMessages({ ...request, jsonMode }), promptBudget, toolTokens);
-            const prompt = model.template.render({
-                messages,
-                tools,
-                add_generation_prompt: true,
-                enable_thinking: false,
-            });
+            const { prompt, promptTokens, truncated } = await fitPromptToContext(buildTemplateMessages({ ...request, jsonMode }), promptBudget, (messages) => model.template.render({ messages, tools, add_generation_prompt: true, enable_thinking: false }), (text) => model.countTokens(model.tokenizer, text));
+            console.log(`${LOG_TAG}: ${model.modelId} prompt=${promptTokens} budget=${promptBudget} context=${model.maxSeqLen} maxNewTokens=${maxNewTokens} truncated=${truncated}`);
             model.runner.reset(0);
             const deadline = BgTimer.setTimeout(() => {
                 console.warn(`${LOG_TAG}: generation deadline hit, stopping`);
@@ -129,6 +130,10 @@ class LocalLlmProvider {
             try {
                 generated = await model.generate(model.runner, prompt, { maxNewTokens, temperature: request.temperature ?? 0.7 }, model.stopTokens);
             }
+            catch (error) {
+                console.warn(`${LOG_TAG}: generation failed: model=${model.modelId} prompt=${promptTokens} context=${model.maxSeqLen} maxNewTokens=${maxNewTokens}`, error);
+                throw error;
+            }
             finally {
                 BgTimer.clearTimeout(deadline);
             }
@@ -177,16 +182,30 @@ class LocalLlmProvider {
         const startedAt = Date.now();
         const createRunner = rne.wrapAsync(rne.llm.createLLMRunner);
         const runner = await createRunner(paths.model, paths.tokenizer, []);
-        const maxSeqLen = runner.getKVCacheState().maxSeqLen;
-        console.log(`${LOG_TAG}: loaded ${modelId} (context ${maxSeqLen}) in ${Date.now() - startedAt} ms`);
-        return {
-            modelId,
-            runner,
-            template,
-            stopTokens: [...tokenizerConfig.stopTokens],
-            maxSeqLen,
-            generate: rne.wrapAsync(generateWorklet),
-        };
+        let tokenizer;
+        try {
+            const maxSeqLen = runner.getKVCacheState().maxSeqLen;
+            if (!Number.isInteger(maxSeqLen) || maxSeqLen <= CONTEXT_MARGIN_TOKENS + 16) {
+                throw new Error(`On-device model ${modelId} reported an invalid context size: ${maxSeqLen}`);
+            }
+            tokenizer = await rne.wrapAsync(rne.nlp.loadTokenizer)(paths.tokenizer);
+            console.log(`${LOG_TAG}: loaded ${modelId} (context ${maxSeqLen}) in ${Date.now() - startedAt} ms`);
+            return {
+                modelId,
+                runner,
+                tokenizer,
+                countTokens: rne.wrapAsync(countTokensWorklet),
+                template,
+                stopTokens: [...tokenizerConfig.stopTokens],
+                maxSeqLen,
+                generate: rne.wrapAsync(generateWorklet),
+            };
+        }
+        catch (error) {
+            tokenizer?.dispose();
+            runner.dispose();
+            throw error;
+        }
     }
     async runtime() {
         if (this.rne)

package/build/services/LocalLlmProvider.js.map

@@ -1 +1 @@
-{"version":3,"file":"LocalLlmProvider.js","sourceRoot":"","sources":["../../src/services/LocalLlmProvider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,KAAK,IAAI,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAA;AAG3C,OAAO,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAA;AACvC,OAAO,oBAA4C,MAAM,wBAAwB,CAAA;AACjF,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,eAAe,GAEhB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAC,QAAQ,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAA;AAS7D,MAAM,OAAO,GAAG,WAAW,CAAA;AAE3B,iEAAiE;AACjE,MAAM,cAAc,GAAG,CAAC,GAAG,MAAM,CAAA;AACjC,+EAA+E;AAC/E,MAAM,sBAAsB,GAAG,OAAO,CAAA;AACtC,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAClC,MAAM,sBAAsB,GAAG,KAAK,CAAA;AACpC,sFAAsF;AACtF,MAAM,qBAAqB,GAAG,EAAE,CAAA;AAoBhC;;;GAGG;AACH,SAAS,eAAe,CACtB,MAAiB,EACjB,MAAc,EACd,MAA2B,EAC3B,UAAoB;IAEpB,SAAS,CAAA;IACT,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;QAC9D,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAM;QACtC,IAAI,IAAI,KAAK,CAAA;IACf,CAAC,CAAC,CAAA;IACF,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,CAAA;AACtB,CAAC;AAED,MAAM,gBAAgB;IACZ,MAAM,CAAC,QAAQ,CAAkB;IAEjC,MAAM,GAAuB,IAAI,CAAA;IACjC,OAAO,GAAgC,IAAI,CAAA;IAC3C,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAA;IAC3C,SAAS,GAAkB,IAAI,CAAA;IAC/B,GAAG,GAAe,IAAI,CAAA;IAE9B;QACE,wEAAwE;QACxE,gBAAgB,CAAC,SAAS,CACxB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,EACvD,GAAG,EAAE;YACH,KAAK,IAAI,CAAC,MAAM,EAAE,CAAA;QACpB,CAAC,CACF,CAAA;IACH,CAAC;IAED,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,gBAAgB,CAAC,QAAQ;YAAE,gBAAgB,CAAC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAA;QAClF,OAAO,gBAAgB,CAAC,QAAQ,CAAA;IAClC,CAAC;IAED,mFAAmF;IACnF,QAAQ;QACN,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAA;IAC7B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI,CAAA;IACrC,CAAC;IAED,oGAAoG;IACpG,QAAQ,CAAC,OAA2B;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAA;QAC9D,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QACvC,OAAO,GAAG,CAAA;IACZ,CAAC;IAED,kGAAkG;IAClG,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,cAAc,EAAE,CAAA;QACrB,uEAAuE;QACvE,MAAM,IAAI,CAAC,KAAK,CAAA;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAA;QAC5B,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,MAAM,OAAO,CAAA;YACf,CAAC;YAAC,MAAM,CAAC;gBACP,yCAAyC;YAC3C,CAAC;QACH,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAA;IACtB,CAAC;IAED,yEAAyE;IACjE,aAAa;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QAClB,IAAI,CAAC,MAAM;YAAE,OAAM;QACnB,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,0BAA0B,EAAE,KAAK,CAAC,CAAA;QAC3D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA2B;QACrD,IAAI,CAAC,cAAc,EAAE,CAAA;QACrB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;YACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,IAAI,CAAA;YAC1C,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YAEzE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,EAAE,EACF,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,sBAAsB,EAAE,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAC/G,CAAA;YACD,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,GAAG,YAAY,GAAG,qBAAqB,CAAA;YAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,qBAAqB,CAAC,EAAC,GAAG,OAAO,EAAE,QAAQ,EAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAA;YAErG,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACnC,QAAQ;gBACR,KAAK;gBACL,qBAAqB,EAAE,IAAI;gBAC3B,eAAe,EAAE,KAAK;aACvB,CAAC,CAAA;YAEF,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YACrB,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE;gBACvC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,qCAAqC,CAAC,CAAA;gBAC7D,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;YACrB,CAAC,EAAE,sBAAsB,CAAC,CAAA;YAC1B,IAAI,SAAoD,CAAA;YACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,KAAK,CAAC,QAAQ,CAC9B,KAAK,CAAC,MAAM,EACZ,MAAM,EACN,EAAC,YAAY,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,GAAG,EAAC,EACvD,KAAK,CAAC,UAAU,CACjB,CAAA;YACH,CAAC;oBAAS,CAAC;gBACT,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;YAChC,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;YACxC,OAAO,CAAC,GAAG,CACT,GAAG,OAAO,KAAK,KAAK,CAAC,OAAO,WAAW,SAAS,CAAC,KAAK,CAAC,eAAe,QAAQ,SAAS,CAAC,KAAK,CAAC,kBAAkB,cAAc,SAAS,KAAK,CAC7I,CAAA;YAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAA;YAClF,MAAM,UAAU,GAAG,mBAAmB,CACpC;gBACE,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,eAAe;gBAC7C,eAAe,EAAE,SAAS,CAAC,KAAK,CAAC,kBAAkB;gBACnD,YAAY;aACb,EACD,EAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,IAAI,IAAI,KAAK,EAAE,EAAC,CAC9F,CAAA;YACD,OAAO,EAAC,GAAG,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAC,CAAA;QAChD,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,EAAE,CAAA;QACrB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,OAAO,GAAG,oBAAoB,CAAC,eAAe,EAAE,CAAA;QACtD,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC,MAAM,CAAA;QACxD,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,OAAO,CAAA;QAErC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;aAC9B,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACd,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;YACnB,OAAO,KAAK,CAAA;QACd,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACrB,CAAC,CAAC,CAAA;QACJ,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,OAAwB;QACzC,0EAA0E;QAC1E,2BAA2B;QAC3B,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,CAAC,MAAM,oBAAoB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,oBAAoB,CAAC,CAAA;QACjE,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAEjD,MAAM,eAAe,GAAG,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;QACpH,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,eAAe,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC5B,MAAM,YAAY,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;QAC3D,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;QACnE,MAAM,SAAS,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC,SAAS,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,YAAY,OAAO,aAAa,SAAS,QAAQ,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,KAAK,CAAC,CAAA;QAEnG,OAAO;YACL,OAAO;YACP,MAAM;YACN,QAAQ;YACR,UAAU,EAAE,CAAC,GAAG,eAAe,CAAC,UAAU,CAAC;YAC3C,SAAS;YACT,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC;SACzC,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,IAAI,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC,GAAG,CAAA;QAC7B,MAAM,GAAG,GAAQ,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAA;QACxD,mFAAmF;QACnF,GAAG,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAA;QACd,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,cAAc,EAAE,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE;YACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;YACrB,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,qBAAqB,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,SAAS,EAAE,CAAC,CAAA;YAC/E,KAAK,IAAI,CAAC,MAAM,EAAE,CAAA;QACpB,CAAC,EAAE,cAAc,CAAC,CAAA;IACpB,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACvB,CAAC;IACH,CAAC;CACF;AAED,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,WAAW,EAAE,CAAA;AACvD,OAAO,EAAC,gBAAgB,EAAC,CAAA;AACzB,eAAe,gBAAgB,CAAA","sourcesContent":["/**\n * LocalLlmProvider — session.ai.complete served by the phone itself: an\n * ExecuTorch runner (react-native-executorch) over the Qwen3 export that\n * LocalLlmModelManager downloaded. Stateless per request like the vendor\n * providers: every call renders the whole conversation through the model's\n * chat template, resets the KV cache, and generates on ExecuTorch's worklet\n * thread so the JS thread stays free.\n *\n * The runner is loaded on the first request and kept resident while calls\n * keep coming (loading a 1–3 GB model is seconds, not milliseconds); it is\n * dropped after an idle period and whenever the selected model changes.\n * Requests are serialized — one model, one KV cache.\n *\n * react-native-executorch is imported lazily so the engine module graph\n * loads without the native runtime (bun tests, hosts without the dependency).\n */\nimport * as RNFS from \"@dr.pogodin/react-native-fs\"\nimport {Template} from \"@huggingface/jinja\"\n\nimport type {LlmCompleteRequest} from \"@glassly/cloud-protocol/llm\"\nimport {BgTimer} from \"../utils/timers\"\nimport localLlmModelManager, {type LocalLlmModelId} from \"./LocalLlmModelManager\"\nimport {\n  buildTemplateMessages,\n  estimateTokens,\n  finalizeLocalResult,\n  fitToBudget,\n  toTemplateTools,\n  type LocalCompletion,\n} from \"./LocalLlmPrompt\"\nimport {SETTINGS, useSettingsStore} from \"../stores/settings\"\n\ntype Rne = typeof import(\"react-native-executorch\")\n// The runner API lives on the `llm` namespace; derive its types from there so\n// the import stays a type-only reference (no native module at load time).\ntype LLMRunner = ReturnType<Rne[\"llm\"][\"createLLMRunner\"]>\ntype LLMGenerationStats = ReturnType<LLMRunner[\"generate\"]>\ntype LLMGenerationConfig = NonNullable<Parameters<LLMRunner[\"generate\"]>[1]>\n\nconst LOG_TAG = \"LOCAL_LLM\"\n\n/** Drop the resident model after this long without a request. */\nconst IDLE_UNLOAD_MS = 5 * 60_000\n/** Hard stop on one generation — the SDK request times out at 120 s anyway. */\nconst GENERATION_DEADLINE_MS = 100_000\nconst DEFAULT_MAX_NEW_TOKENS = 512\nconst MAX_NEW_TOKENS_CEILING = 1_024\n/** Headroom between the estimated prompt and the context end; estimates are rough. */\nconst CONTEXT_MARGIN_TOKENS = 96\n\ninterface LoadedModel {\n  modelId: LocalLlmModelId\n  runner: LLMRunner\n  template: Template\n  stopTokens: string[]\n  maxSeqLen: number\n  generate: (\n    runner: LLMRunner,\n    prompt: string,\n    config: LLMGenerationConfig,\n    stopTokens: string[],\n  ) => Promise<{text: string; stats: LLMGenerationStats}>\n}\n\nexport interface LocalLlmCompleteResult extends LocalCompletion {\n  modelId: LocalLlmModelId\n}\n\n/**\n * Runs on ExecuTorch's worklet runtime: `runner.generate` blocks its thread\n * until EOS, `maxNewTokens`, or `runner.stop()` from the JS thread.\n */\nfunction generateWorklet(\n  runner: LLMRunner,\n  prompt: string,\n  config: LLMGenerationConfig,\n  stopTokens: string[],\n): {text: string; stats: LLMGenerationStats} {\n  \"worklet\"\n  let text = \"\"\n  const stats = runner.generate(prompt, config, (token: string) => {\n    if (stopTokens.includes(token)) return\n    text += token\n  })\n  return {text, stats}\n}\n\nclass LocalLlmProvider {\n  private static instance: LocalLlmProvider\n\n  private loaded: LoadedModel | null = null\n  private loading: Promise<LoadedModel> | null = null\n  private queue: Promise<unknown> = Promise.resolve()\n  private idleTimer: number | null = null\n  private rne: Rne | null = null\n\n  private constructor() {\n    // A different model picked in Settings invalidates the resident runner.\n    useSettingsStore.subscribe(\n      (state) => state.settings[SETTINGS.local_llm_model.key],\n      () => {\n        void this.unload()\n      },\n    )\n  }\n\n  static getInstance(): LocalLlmProvider {\n    if (!LocalLlmProvider.instance) LocalLlmProvider.instance = new LocalLlmProvider()\n    return LocalLlmProvider.instance\n  }\n\n  /** Whether a runner is resident right now (diagnostics; not a readiness check). */\n  isLoaded(): boolean {\n    return this.loaded !== null\n  }\n\n  loadedModelId(): LocalLlmModelId | null {\n    return this.loaded?.modelId ?? null\n  }\n\n  /** Serve one neutral completion on-device. Throws when the model is missing or generation fails. */\n  complete(request: LlmCompleteRequest): Promise<LocalLlmCompleteResult> {\n    const run = this.queue.then(() => this.completeInner(request))\n    this.queue = run.catch(() => undefined)\n    return run\n  }\n\n  /** Release the resident runner (idle, model switch, memory pressure). Safe to call repeatedly. */\n  async unload(): Promise<void> {\n    this.clearIdleTimer()\n    // Never dispose under a running generation: let the queue drain first.\n    await this.queue\n    const pending = this.loading\n    if (pending) {\n      try {\n        await pending\n      } catch {\n        // A failed load left nothing to release.\n      }\n    }\n    this.releaseRunner()\n  }\n\n  /** Synchronous dispose; callers guarantee no generation is in flight. */\n  private releaseRunner(): void {\n    const loaded = this.loaded\n    this.loaded = null\n    if (!loaded) return\n    try {\n      loaded.runner.dispose()\n    } catch (error) {\n      console.warn(`${LOG_TAG}: runner dispose failed:`, error)\n    }\n  }\n\n  private async completeInner(request: LlmCompleteRequest): Promise<LocalLlmCompleteResult> {\n    this.clearIdleTimer()\n    try {\n      const model = await this.ensureLoaded()\n      const jsonMode = request.jsonMode === true\n      const tools = toTemplateTools(request.tools)\n      const toolTokens = tools ? estimateTokens(JSON.stringify(tools)) + 64 : 0\n\n      const maxNewTokens = Math.max(\n        16,\n        Math.min(request.maxTokens ?? DEFAULT_MAX_NEW_TOKENS, MAX_NEW_TOKENS_CEILING, Math.floor(model.maxSeqLen / 4)),\n      )\n      const promptBudget = model.maxSeqLen - maxNewTokens - CONTEXT_MARGIN_TOKENS\n      const messages = fitToBudget(buildTemplateMessages({...request, jsonMode}), promptBudget, toolTokens)\n\n      const prompt = model.template.render({\n        messages,\n        tools,\n        add_generation_prompt: true,\n        enable_thinking: false,\n      })\n\n      model.runner.reset(0)\n      const deadline = BgTimer.setTimeout(() => {\n        console.warn(`${LOG_TAG}: generation deadline hit, stopping`)\n        model.runner.stop()\n      }, GENERATION_DEADLINE_MS)\n      let generated: {text: string; stats: LLMGenerationStats}\n      const startedAt = Date.now()\n      try {\n        generated = await model.generate(\n          model.runner,\n          prompt,\n          {maxNewTokens, temperature: request.temperature ?? 0.7},\n          model.stopTokens,\n        )\n      } finally {\n        BgTimer.clearTimeout(deadline)\n      }\n\n      const elapsedMs = Date.now() - startedAt\n      console.log(\n        `${LOG_TAG}: ${model.modelId} prompt=${generated.stats.numPromptTokens} out=${generated.stats.numGeneratedTokens} tokens in ${elapsedMs} ms`,\n      )\n\n      const seed = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`\n      const completion = finalizeLocalResult(\n        {\n          text: generated.text,\n          promptTokens: generated.stats.numPromptTokens,\n          generatedTokens: generated.stats.numGeneratedTokens,\n          maxNewTokens,\n        },\n        {jsonMode, toolsOffered: Boolean(tools), makeToolCallId: (index) => `local_${seed}_${index}`},\n      )\n      return {...completion, modelId: model.modelId}\n    } finally {\n      this.armIdleTimer()\n    }\n  }\n\n  private async ensureLoaded(): Promise<LoadedModel> {\n    const modelId = localLlmModelManager.selectedModelId()\n    if (this.loaded?.modelId === modelId) return this.loaded\n    if (this.loading) return this.loading\n\n    this.loading = this.load(modelId)\n      .then((model) => {\n        this.loaded = model\n        return model\n      })\n      .finally(() => {\n        this.loading = null\n      })\n    return this.loading\n  }\n\n  private async load(modelId: LocalLlmModelId): Promise<LoadedModel> {\n    // Runs inside the request queue, so nothing is generating; unload() would\n    // wait on that same queue.\n    this.releaseRunner()\n    if (!(await localLlmModelManager.isModelAvailable(modelId))) {\n      throw new Error(`on-device model ${modelId} is not downloaded`)\n    }\n    const rne = await this.runtime()\n    const paths = localLlmModelManager.paths(modelId)\n\n    const tokenizerConfig = rne.llm.parseTokenizerConfig(JSON.parse(await RNFS.readFile(paths.tokenizerConfig, \"utf8\")))\n    const template = new Template(tokenizerConfig.chatTemplate)\n\n    const startedAt = Date.now()\n    const createRunner = rne.wrapAsync(rne.llm.createLLMRunner)\n    const runner = await createRunner(paths.model, paths.tokenizer, [])\n    const maxSeqLen = runner.getKVCacheState().maxSeqLen\n    console.log(`${LOG_TAG}: loaded ${modelId} (context ${maxSeqLen}) in ${Date.now() - startedAt} ms`)\n\n    return {\n      modelId,\n      runner,\n      template,\n      stopTokens: [...tokenizerConfig.stopTokens],\n      maxSeqLen,\n      generate: rne.wrapAsync(generateWorklet),\n    }\n  }\n\n  private async runtime(): Promise<Rne> {\n    if (this.rne) return this.rne\n    const rne: Rne = await import(\"react-native-executorch\")\n    // Software Mansion's anonymous download analytics; the host has its own telemetry.\n    rne.setTelemetryEnabled(false)\n    this.rne = rne\n    return rne\n  }\n\n  private armIdleTimer(): void {\n    this.clearIdleTimer()\n    this.idleTimer = BgTimer.setTimeout(() => {\n      this.idleTimer = null\n      console.log(`${LOG_TAG}: idle, unloading ${this.loaded?.modelId ?? \"nothing\"}`)\n      void this.unload()\n    }, IDLE_UNLOAD_MS)\n  }\n\n  private clearIdleTimer(): void {\n    if (this.idleTimer) {\n      BgTimer.clearTimeout(this.idleTimer)\n      this.idleTimer = null\n    }\n  }\n}\n\nconst localLlmProvider = LocalLlmProvider.getInstance()\nexport {LocalLlmProvider}\nexport default localLlmProvider\n"]}
+{"version":3,"file":"LocalLlmProvider.js","sourceRoot":"","sources":["../../src/services/LocalLlmProvider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,KAAK,IAAI,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAA;AAG3C,OAAO,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAA;AACvC,OAAO,oBAA4C,MAAM,wBAAwB,CAAA;AACjF,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,GAEhB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAC,QAAQ,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAA;AAU7D,MAAM,OAAO,GAAG,WAAW,CAAA;AAE3B,iEAAiE;AACjE,MAAM,cAAc,GAAG,CAAC,GAAG,MAAM,CAAA;AACjC,+EAA+E;AAC/E,MAAM,sBAAsB,GAAG,OAAO,CAAA;AACtC,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAClC,MAAM,sBAAsB,GAAG,KAAK,CAAA;AACpC,sFAAsF;AACtF,MAAM,qBAAqB,GAAG,EAAE,CAAA;AAsBhC;;;GAGG;AACH,SAAS,eAAe,CACtB,MAAiB,EACjB,MAAc,EACd,MAA2B,EAC3B,UAAoB;IAEpB,SAAS,CAAA;IACT,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;QAC9D,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAM;QACtC,IAAI,IAAI,KAAK,CAAA;IACf,CAAC,CAAC,CAAA;IACF,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,CAAA;AACtB,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAoB,EAAE,MAAc;IAC9D,SAAS,CAAA;IACT,OAAO,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAA;AACxC,CAAC;AAED,MAAM,gBAAgB;IACZ,MAAM,CAAC,QAAQ,CAAkB;IAEjC,MAAM,GAAuB,IAAI,CAAA;IACjC,OAAO,GAAgC,IAAI,CAAA;IAC3C,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAA;IAC3C,SAAS,GAAkB,IAAI,CAAA;IAC/B,GAAG,GAAe,IAAI,CAAA;IAE9B;QACE,wEAAwE;QACxE,gBAAgB,CAAC,SAAS,CACxB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,EACvD,GAAG,EAAE;YACH,KAAK,IAAI,CAAC,MAAM,EAAE,CAAA;QACpB,CAAC,CACF,CAAA;IACH,CAAC;IAED,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,gBAAgB,CAAC,QAAQ;YAAE,gBAAgB,CAAC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAA;QAClF,OAAO,gBAAgB,CAAC,QAAQ,CAAA;IAClC,CAAC;IAED,mFAAmF;IACnF,QAAQ;QACN,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAA;IAC7B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI,CAAA;IACrC,CAAC;IAED,oGAAoG;IACpG,QAAQ,CAAC,OAA2B;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAA;QAC9D,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QACvC,OAAO,GAAG,CAAA;IACZ,CAAC;IAED,kGAAkG;IAClG,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,cAAc,EAAE,CAAA;QACrB,uEAAuE;QACvE,MAAM,IAAI,CAAC,KAAK,CAAA;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAA;QAC5B,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,MAAM,OAAO,CAAA;YACf,CAAC;YAAC,MAAM,CAAC;gBACP,yCAAyC;YAC3C,CAAC;QACH,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAA;IACtB,CAAC;IAED,yEAAyE;IACjE,aAAa;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QAClB,IAAI,CAAC,MAAM;YAAE,OAAM;QACnB,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAA;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,0BAA0B,EAAE,KAAK,CAAC,CAAA;QAC3D,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA2B;QACrD,IAAI,CAAC,cAAc,EAAE,CAAA;QACrB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;YACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,IAAI,CAAA;YAC1C,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAE5C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,EAAE,EACF,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,sBAAsB,EAAE,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAC/G,CAAA;YACD,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,GAAG,YAAY,GAAG,qBAAqB,CAAA;YAC3E,MAAM,EAAC,MAAM,EAAE,YAAY,EAAE,SAAS,EAAC,GAAG,MAAM,kBAAkB,CAChE,qBAAqB,CAAC,EAAC,GAAG,OAAO,EAAE,QAAQ,EAAC,CAAC,EAC7C,YAAY,EACZ,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAC,QAAQ,EAAE,KAAK,EAAE,qBAAqB,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAC,CAAC,EAC3G,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CACnD,CAAA;YACD,OAAO,CAAC,GAAG,CACT,GAAG,OAAO,KAAK,KAAK,CAAC,OAAO,WAAW,YAAY,WAAW,YAAY,YAAY,KAAK,CAAC,SAAS,iBAAiB,YAAY,cAAc,SAAS,EAAE,CAC5J,CAAA;YAED,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YACrB,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE;gBACvC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,qCAAqC,CAAC,CAAA;gBAC7D,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;YACrB,CAAC,EAAE,sBAAsB,CAAC,CAAA;YAC1B,IAAI,SAAoD,CAAA;YACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,KAAK,CAAC,QAAQ,CAC9B,KAAK,CAAC,MAAM,EACZ,MAAM,EACN,EAAC,YAAY,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,GAAG,EAAC,EACvD,KAAK,CAAC,UAAU,CACjB,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CACV,GAAG,OAAO,8BAA8B,KAAK,CAAC,OAAO,WAAW,YAAY,YAAY,KAAK,CAAC,SAAS,iBAAiB,YAAY,EAAE,EACtI,KAAK,CACN,CAAA;gBACD,MAAM,KAAK,CAAA;YACb,CAAC;oBAAS,CAAC;gBACT,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;YAChC,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;YACxC,OAAO,CAAC,GAAG,CACT,GAAG,OAAO,KAAK,KAAK,CAAC,OAAO,WAAW,SAAS,CAAC,KAAK,CAAC,eAAe,QAAQ,SAAS,CAAC,KAAK,CAAC,kBAAkB,cAAc,SAAS,KAAK,CAC7I,CAAA;YAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAA;YAClF,MAAM,UAAU,GAAG,mBAAmB,CACpC;gBACE,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,eAAe;gBAC7C,eAAe,EAAE,SAAS,CAAC,KAAK,CAAC,kBAAkB;gBACnD,YAAY;aACb,EACD,EAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,IAAI,IAAI,KAAK,EAAE,EAAC,CAC9F,CAAA;YACD,OAAO,EAAC,GAAG,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAC,CAAA;QAChD,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,EAAE,CAAA;QACrB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,OAAO,GAAG,oBAAoB,CAAC,eAAe,EAAE,CAAA;QACtD,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC,MAAM,CAAA;QACxD,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,OAAO,CAAA;QAErC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;aAC9B,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACd,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;YACnB,OAAO,KAAK,CAAA;QACd,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACrB,CAAC,CAAC,CAAA;QACJ,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,OAAwB;QACzC,0EAA0E;QAC1E,2BAA2B;QAC3B,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,CAAC,MAAM,oBAAoB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,oBAAoB,CAAC,CAAA;QACjE,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAEjD,MAAM,eAAe,GAAG,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;QACpH,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,eAAe,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC5B,MAAM,YAAY,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;QAC3D,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;QACnE,IAAI,SAAgC,CAAA;QACpC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC,SAAS,CAAA;YACpD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,qBAAqB,GAAG,EAAE,EAAE,CAAC;gBAC5E,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,sCAAsC,SAAS,EAAE,CAAC,CAAA;YAC9F,CAAC;YACD,SAAS,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YACvE,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,YAAY,OAAO,aAAa,SAAS,QAAQ,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,KAAK,CAAC,CAAA;YACnG,OAAO;gBACL,OAAO;gBACP,MAAM;gBACN,SAAS;gBACT,WAAW,EAAE,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC;gBAC9C,QAAQ;gBACR,UAAU,EAAE,CAAC,GAAG,eAAe,CAAC,UAAU,CAAC;gBAC3C,SAAS;gBACT,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC;aACzC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,EAAE,OAAO,EAAE,CAAA;YACpB,MAAM,CAAC,OAAO,EAAE,CAAA;YAChB,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,IAAI,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC,GAAG,CAAA;QAC7B,MAAM,GAAG,GAAQ,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAA;QACxD,mFAAmF;QACnF,GAAG,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAA;QACd,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,cAAc,EAAE,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE;YACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;YACrB,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,qBAAqB,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,SAAS,EAAE,CAAC,CAAA;YAC/E,KAAK,IAAI,CAAC,MAAM,EAAE,CAAA;QACpB,CAAC,EAAE,cAAc,CAAC,CAAA;IACpB,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACvB,CAAC;IACH,CAAC;CACF;AAED,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,WAAW,EAAE,CAAA;AACvD,OAAO,EAAC,gBAAgB,EAAC,CAAA;AACzB,eAAe,gBAAgB,CAAA","sourcesContent":["/**\n * LocalLlmProvider — session.ai.complete served by the phone itself: an\n * ExecuTorch runner (react-native-executorch) over the Qwen3 export that\n * LocalLlmModelManager downloaded. Stateless per request like the vendor\n * providers: every call renders the whole conversation through the model's\n * chat template, resets the KV cache, and generates on ExecuTorch's worklet\n * thread so the JS thread stays free.\n *\n * The runner is loaded on the first request and kept resident while calls\n * keep coming (loading a 1–3 GB model is seconds, not milliseconds); it is\n * dropped after an idle period and whenever the selected model changes.\n * Requests are serialized — one model, one KV cache.\n *\n * react-native-executorch is imported lazily so the engine module graph\n * loads without the native runtime (bun tests, hosts without the dependency).\n */\nimport * as RNFS from \"@dr.pogodin/react-native-fs\"\nimport {Template} from \"@huggingface/jinja\"\n\nimport type {LlmCompleteRequest} from \"@glassly/cloud-protocol/llm\"\nimport {BgTimer} from \"../utils/timers\"\nimport localLlmModelManager, {type LocalLlmModelId} from \"./LocalLlmModelManager\"\nimport {\n  buildTemplateMessages,\n  finalizeLocalResult,\n  fitPromptToContext,\n  toTemplateTools,\n  type LocalCompletion,\n} from \"./LocalLlmPrompt\"\nimport {SETTINGS, useSettingsStore} from \"../stores/settings\"\n\ntype Rne = typeof import(\"react-native-executorch\")\n// The runner API lives on the `llm` namespace; derive its types from there so\n// the import stays a type-only reference (no native module at load time).\ntype Tokenizer = ReturnType<Rne[\"nlp\"][\"loadTokenizer\"]>\ntype LLMRunner = ReturnType<Rne[\"llm\"][\"createLLMRunner\"]>\ntype LLMGenerationStats = ReturnType<LLMRunner[\"generate\"]>\ntype LLMGenerationConfig = NonNullable<Parameters<LLMRunner[\"generate\"]>[1]>\n\nconst LOG_TAG = \"LOCAL_LLM\"\n\n/** Drop the resident model after this long without a request. */\nconst IDLE_UNLOAD_MS = 5 * 60_000\n/** Hard stop on one generation — the SDK request times out at 120 s anyway. */\nconst GENERATION_DEADLINE_MS = 100_000\nconst DEFAULT_MAX_NEW_TOKENS = 512\nconst MAX_NEW_TOKENS_CEILING = 1_024\n/** Reserve space for runner-added BOS/EOS tokens and context boundary bookkeeping. */\nconst CONTEXT_MARGIN_TOKENS = 96\n\ninterface LoadedModel {\n  modelId: LocalLlmModelId\n  runner: LLMRunner\n  tokenizer: Tokenizer\n  countTokens: (tokenizer: Tokenizer, prompt: string) => Promise<number>\n  template: Template\n  stopTokens: string[]\n  maxSeqLen: number\n  generate: (\n    runner: LLMRunner,\n    prompt: string,\n    config: LLMGenerationConfig,\n    stopTokens: string[],\n  ) => Promise<{text: string; stats: LLMGenerationStats}>\n}\n\nexport interface LocalLlmCompleteResult extends LocalCompletion {\n  modelId: LocalLlmModelId\n}\n\n/**\n * Runs on ExecuTorch's worklet runtime: `runner.generate` blocks its thread\n * until EOS, `maxNewTokens`, or `runner.stop()` from the JS thread.\n */\nfunction generateWorklet(\n  runner: LLMRunner,\n  prompt: string,\n  config: LLMGenerationConfig,\n  stopTokens: string[],\n): {text: string; stats: LLMGenerationStats} {\n  \"worklet\"\n  let text = \"\"\n  const stats = runner.generate(prompt, config, (token: string) => {\n    if (stopTokens.includes(token)) return\n    text += token\n  })\n  return {text, stats}\n}\n\nfunction countTokensWorklet(tokenizer: Tokenizer, prompt: string): number {\n  \"worklet\"\n  return tokenizer.encode(prompt).length\n}\n\nclass LocalLlmProvider {\n  private static instance: LocalLlmProvider\n\n  private loaded: LoadedModel | null = null\n  private loading: Promise<LoadedModel> | null = null\n  private queue: Promise<unknown> = Promise.resolve()\n  private idleTimer: number | null = null\n  private rne: Rne | null = null\n\n  private constructor() {\n    // A different model picked in Settings invalidates the resident runner.\n    useSettingsStore.subscribe(\n      (state) => state.settings[SETTINGS.local_llm_model.key],\n      () => {\n        void this.unload()\n      },\n    )\n  }\n\n  static getInstance(): LocalLlmProvider {\n    if (!LocalLlmProvider.instance) LocalLlmProvider.instance = new LocalLlmProvider()\n    return LocalLlmProvider.instance\n  }\n\n  /** Whether a runner is resident right now (diagnostics; not a readiness check). */\n  isLoaded(): boolean {\n    return this.loaded !== null\n  }\n\n  loadedModelId(): LocalLlmModelId | null {\n    return this.loaded?.modelId ?? null\n  }\n\n  /** Serve one neutral completion on-device. Throws when the model is missing or generation fails. */\n  complete(request: LlmCompleteRequest): Promise<LocalLlmCompleteResult> {\n    const run = this.queue.then(() => this.completeInner(request))\n    this.queue = run.catch(() => undefined)\n    return run\n  }\n\n  /** Release the resident runner (idle, model switch, memory pressure). Safe to call repeatedly. */\n  async unload(): Promise<void> {\n    this.clearIdleTimer()\n    // Never dispose under a running generation: let the queue drain first.\n    await this.queue\n    const pending = this.loading\n    if (pending) {\n      try {\n        await pending\n      } catch {\n        // A failed load left nothing to release.\n      }\n    }\n    this.releaseRunner()\n  }\n\n  /** Synchronous dispose; callers guarantee no generation is in flight. */\n  private releaseRunner(): void {\n    const loaded = this.loaded\n    this.loaded = null\n    if (!loaded) return\n    try {\n      loaded.runner.dispose()\n    } catch (error) {\n      console.warn(`${LOG_TAG}: runner dispose failed:`, error)\n    } finally {\n      loaded.tokenizer.dispose()\n    }\n  }\n\n  private async completeInner(request: LlmCompleteRequest): Promise<LocalLlmCompleteResult> {\n    this.clearIdleTimer()\n    try {\n      const model = await this.ensureLoaded()\n      const jsonMode = request.jsonMode === true\n      const tools = toTemplateTools(request.tools)\n\n      const maxNewTokens = Math.max(\n        16,\n        Math.min(request.maxTokens ?? DEFAULT_MAX_NEW_TOKENS, MAX_NEW_TOKENS_CEILING, Math.floor(model.maxSeqLen / 4)),\n      )\n      const promptBudget = model.maxSeqLen - maxNewTokens - CONTEXT_MARGIN_TOKENS\n      const {prompt, promptTokens, truncated} = await fitPromptToContext(\n        buildTemplateMessages({...request, jsonMode}),\n        promptBudget,\n        (messages) => model.template.render({messages, tools, add_generation_prompt: true, enable_thinking: false}),\n        (text) => model.countTokens(model.tokenizer, text),\n      )\n      console.log(\n        `${LOG_TAG}: ${model.modelId} prompt=${promptTokens} budget=${promptBudget} context=${model.maxSeqLen} maxNewTokens=${maxNewTokens} truncated=${truncated}`,\n      )\n\n      model.runner.reset(0)\n      const deadline = BgTimer.setTimeout(() => {\n        console.warn(`${LOG_TAG}: generation deadline hit, stopping`)\n        model.runner.stop()\n      }, GENERATION_DEADLINE_MS)\n      let generated: {text: string; stats: LLMGenerationStats}\n      const startedAt = Date.now()\n      try {\n        generated = await model.generate(\n          model.runner,\n          prompt,\n          {maxNewTokens, temperature: request.temperature ?? 0.7},\n          model.stopTokens,\n        )\n      } catch (error) {\n        console.warn(\n          `${LOG_TAG}: generation failed: model=${model.modelId} prompt=${promptTokens} context=${model.maxSeqLen} maxNewTokens=${maxNewTokens}`,\n          error,\n        )\n        throw error\n      } finally {\n        BgTimer.clearTimeout(deadline)\n      }\n\n      const elapsedMs = Date.now() - startedAt\n      console.log(\n        `${LOG_TAG}: ${model.modelId} prompt=${generated.stats.numPromptTokens} out=${generated.stats.numGeneratedTokens} tokens in ${elapsedMs} ms`,\n      )\n\n      const seed = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`\n      const completion = finalizeLocalResult(\n        {\n          text: generated.text,\n          promptTokens: generated.stats.numPromptTokens,\n          generatedTokens: generated.stats.numGeneratedTokens,\n          maxNewTokens,\n        },\n        {jsonMode, toolsOffered: Boolean(tools), makeToolCallId: (index) => `local_${seed}_${index}`},\n      )\n      return {...completion, modelId: model.modelId}\n    } finally {\n      this.armIdleTimer()\n    }\n  }\n\n  private async ensureLoaded(): Promise<LoadedModel> {\n    const modelId = localLlmModelManager.selectedModelId()\n    if (this.loaded?.modelId === modelId) return this.loaded\n    if (this.loading) return this.loading\n\n    this.loading = this.load(modelId)\n      .then((model) => {\n        this.loaded = model\n        return model\n      })\n      .finally(() => {\n        this.loading = null\n      })\n    return this.loading\n  }\n\n  private async load(modelId: LocalLlmModelId): Promise<LoadedModel> {\n    // Runs inside the request queue, so nothing is generating; unload() would\n    // wait on that same queue.\n    this.releaseRunner()\n    if (!(await localLlmModelManager.isModelAvailable(modelId))) {\n      throw new Error(`on-device model ${modelId} is not downloaded`)\n    }\n    const rne = await this.runtime()\n    const paths = localLlmModelManager.paths(modelId)\n\n    const tokenizerConfig = rne.llm.parseTokenizerConfig(JSON.parse(await RNFS.readFile(paths.tokenizerConfig, \"utf8\")))\n    const template = new Template(tokenizerConfig.chatTemplate)\n\n    const startedAt = Date.now()\n    const createRunner = rne.wrapAsync(rne.llm.createLLMRunner)\n    const runner = await createRunner(paths.model, paths.tokenizer, [])\n    let tokenizer: Tokenizer | undefined\n    try {\n      const maxSeqLen = runner.getKVCacheState().maxSeqLen\n      if (!Number.isInteger(maxSeqLen) || maxSeqLen <= CONTEXT_MARGIN_TOKENS + 16) {\n        throw new Error(`On-device model ${modelId} reported an invalid context size: ${maxSeqLen}`)\n      }\n      tokenizer = await rne.wrapAsync(rne.nlp.loadTokenizer)(paths.tokenizer)\n      console.log(`${LOG_TAG}: loaded ${modelId} (context ${maxSeqLen}) in ${Date.now() - startedAt} ms`)\n      return {\n        modelId,\n        runner,\n        tokenizer,\n        countTokens: rne.wrapAsync(countTokensWorklet),\n        template,\n        stopTokens: [...tokenizerConfig.stopTokens],\n        maxSeqLen,\n        generate: rne.wrapAsync(generateWorklet),\n      }\n    } catch (error) {\n      tokenizer?.dispose()\n      runner.dispose()\n      throw error\n    }\n  }\n\n  private async runtime(): Promise<Rne> {\n    if (this.rne) return this.rne\n    const rne: Rne = await import(\"react-native-executorch\")\n    // Software Mansion's anonymous download analytics; the host has its own telemetry.\n    rne.setTelemetryEnabled(false)\n    this.rne = rne\n    return rne\n  }\n\n  private armIdleTimer(): void {\n    this.clearIdleTimer()\n    this.idleTimer = BgTimer.setTimeout(() => {\n      this.idleTimer = null\n      console.log(`${LOG_TAG}: idle, unloading ${this.loaded?.modelId ?? \"nothing\"}`)\n      void this.unload()\n    }, IDLE_UNLOAD_MS)\n  }\n\n  private clearIdleTimer(): void {\n    if (this.idleTimer) {\n      BgTimer.clearTimeout(this.idleTimer)\n      this.idleTimer = null\n    }\n  }\n}\n\nconst localLlmProvider = LocalLlmProvider.getInstance()\nexport {LocalLlmProvider}\nexport default localLlmProvider\n"]}

package/build/types/capabilities/rayneo-io.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rayneo-io.d.ts","sourceRoot":"","sources":["../../../src/types/capabilities/rayneo-io.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,aAAa,CAAA;AAE7C;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,YAsFtB,CAAA"}
+{"version":3,"file":"rayneo-io.d.ts","sourceRoot":"","sources":["../../../src/types/capabilities/rayneo-io.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,aAAa,CAAA;AAE7C;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,YAwFtB,CAAA"}

package/build/types/capabilities/rayneo-io.js

@@ -27,6 +27,8 @@ export const rayneoIO = {
         fieldOfView: { horizontal: 30 },
         maxTextLines: 5,
         adjustBrightness: true,
+        // Both drivers map these settings onto the launcher's three heights and two distances.
+        positionRange: { height: { min: 0, max: 8 }, depth: { min: 1, max: 3 } },
         // Scene display API — the Glassly CFW page canvas.
         width: 540,
         height: 180,

package/build/types/capabilities/rayneo-io.js.map

@@ -1 +1 @@
-{"version":3,"file":"rayneo-io.js","sourceRoot":"","sources":["../../../src/types/capabilities/rayneo-io.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAiB;IACpC,SAAS,EAAE,WAAW;IAEtB,kCAAkC;IAClC,SAAS,EAAE,KAAK;IAChB,MAAM,EAAE,IAAI;IAEZ,yDAAyD;IACzD,UAAU,EAAE,IAAI;IAChB,OAAO,EAAE;QACP,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,OAAO;QACd,uEAAuE;QACvE,6CAA6C;QAC7C,gBAAgB,EAAE,IAAI;QACtB,UAAU,EAAE,EAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAC;QACrC,WAAW,EAAE,EAAC,UAAU,EAAE,EAAE,EAAC;QAC7B,YAAY,EAAE,CAAC;QACf,gBAAgB,EAAE,IAAI;QAEtB,mDAAmD;QACnD,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,GAAG;QACX,WAAW,EAAE,IAAI;QACjB,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,eAAe,EAAE,EAAE;QACnB,aAAa,EAAE,IAAI;KACpB;IAED,uEAAuE;IACvE,aAAa,EAAE,IAAI;IACnB,UAAU,EAAE;QACV,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,KAAK;KACd;IAED,4DAA4D;IAC5D,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,IAAI;IAEb,0EAA0E;IAC1E,MAAM,EAAE,KAAK;IACb,GAAG,EAAE,IAAI;IAET,qEAAqE;IACrE,SAAS,EAAE,IAAI;IACf,MAAM,EAAE;QACN,KAAK,EAAE,CAAC;QACR,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,CAAC;gBACrE,YAAY,EAAE,IAAI;aACnB;SACF;KACF;IAED,iCAAiC;IACjC,QAAQ,EAAE,KAAK;IACf,KAAK,EAAE,IAAI;IAEX,qBAAqB;IACrB,KAAK,EAAE;QACL,kBAAkB,EAAE,KAAK;KAC1B;IAED,8BAA8B;IAC9B,OAAO,EAAE,KAAK;IAEd,0EAA0E;IAC1E,MAAM,EAAE,KAAK;IAEb,6EAA6E;IAC7E,iBAAiB,EAAE,IAAI;IAEvB,uEAAuE;IACvE,uCAAuC;IACvC,kBAAkB,EAAE,IAAI;IAExB,kCAAkC;IAClC,mBAAmB,EAAE,KAAK;IAE1B,uEAAuE;IACvE,sBAAsB,EAAE,IAAI;CAC7B,CAAA","sourcesContent":["/**\n * @fileoverview RayNeo iO Hardware Capabilities\n *\n * Capability profile for the RayNeo iO smart glasses. The stock firmware\n * exposes a single-line subtitle page over a framed BLE protocol; with the\n * Glassly CFW installed the glasses open a 540×180 page the phone draws into\n * through the Glassly CFW record protocol (positioned text, images, rects).\n */\n\nimport type {Capabilities} from \"../hardware\"\n\n/**\n * RayNeo iO capability profile\n */\nexport const rayneoIO: Capabilities = {\n  modelName: \"RayNeo iO\",\n\n  // Camera capabilities - no camera\n  hasCamera: false,\n  camera: null,\n\n  // Display capabilities - single monochrome (1-bit) panel\n  hasDisplay: true,\n  display: {\n    count: 1,\n    isColor: false,\n    color: \"green\",\n    // Bitmaps and positioned elements need the Glassly CFW; stock firmware\n    // only shows text through its subtitle page.\n    canDisplayBitmap: true,\n    resolution: {width: 540, height: 180},\n    fieldOfView: {horizontal: 30},\n    maxTextLines: 5,\n    adjustBrightness: true,\n\n    // Scene display API — the Glassly CFW page canvas.\n    width: 540,\n    height: 180,\n    canPosition: true,\n    maxTextElements: 6,\n    maxImageElements: 4,\n    intensityLevels: 16,\n    partialUpdate: true,\n  },\n\n  // Microphone capabilities - one Opus mic stream through the voice page\n  hasMicrophone: true,\n  microphone: {\n    count: 1,\n    hasVAD: false,\n  },\n\n  // Speaker capabilities - no speaker channel in the protocol\n  hasSpeaker: false,\n  speaker: null,\n\n  // IMU capabilities - stock firmware exposes no IMU or head-gesture stream\n  hasIMU: false,\n  imu: null,\n\n  // Button capabilities - temple touchpad/crown plus a physical button\n  hasButton: true,\n  button: {\n    count: 1,\n    buttons: [\n      {\n        type: \"swipe1d\",\n        events: [\"TAP\", \"DOUBLE_TAP\", \"PRESS_HOLD\", \"SWIPE_UP\", \"SWIPE_DOWN\"],\n        isCapacitive: true,\n      },\n    ],\n  },\n\n  // Light capabilities - no lights\n  hasLight: false,\n  light: null,\n\n  // Power capabilities\n  power: {\n    hasExternalBattery: false,\n  },\n\n  // WiFi capabilities - no WiFi\n  hasWifi: false,\n\n  // OTA capabilities - RayNeo iO updates are not driven by the ASG OTA flow\n  hasOta: false,\n\n  // Glassly CFW is bundled for the RayNeo iO and flashed over BLE from the app\n  hasCustomFirmware: true,\n\n  // Dashboard - the launcher renders its own status UI; Glassly does not\n  // manage the dashboard settings for it\n  hasNativeDashboard: true,\n\n  // Navigation - no firmware nav UI\n  hasNativeNavigation: false,\n\n  // Firmware notification popups: the phone uploads notification content\n  hasNativeNotifications: true,\n}\n"]}
+{"version":3,"file":"rayneo-io.js","sourceRoot":"","sources":["../../../src/types/capabilities/rayneo-io.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAiB;IACpC,SAAS,EAAE,WAAW;IAEtB,kCAAkC;IAClC,SAAS,EAAE,KAAK;IAChB,MAAM,EAAE,IAAI;IAEZ,yDAAyD;IACzD,UAAU,EAAE,IAAI;IAChB,OAAO,EAAE;QACP,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,OAAO;QACd,uEAAuE;QACvE,6CAA6C;QAC7C,gBAAgB,EAAE,IAAI;QACtB,UAAU,EAAE,EAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAC;QACrC,WAAW,EAAE,EAAC,UAAU,EAAE,EAAE,EAAC;QAC7B,YAAY,EAAE,CAAC;QACf,gBAAgB,EAAE,IAAI;QACtB,uFAAuF;QACvF,aAAa,EAAE,EAAC,MAAM,EAAE,EAAC,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAC,EAAE,KAAK,EAAE,EAAC,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAC,EAAC;QAElE,mDAAmD;QACnD,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,GAAG;QACX,WAAW,EAAE,IAAI;QACjB,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,eAAe,EAAE,EAAE;QACnB,aAAa,EAAE,IAAI;KACpB;IAED,uEAAuE;IACvE,aAAa,EAAE,IAAI;IACnB,UAAU,EAAE;QACV,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,KAAK;KACd;IAED,4DAA4D;IAC5D,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,IAAI;IAEb,0EAA0E;IAC1E,MAAM,EAAE,KAAK;IACb,GAAG,EAAE,IAAI;IAET,qEAAqE;IACrE,SAAS,EAAE,IAAI;IACf,MAAM,EAAE;QACN,KAAK,EAAE,CAAC;QACR,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,CAAC;gBACrE,YAAY,EAAE,IAAI;aACnB;SACF;KACF;IAED,iCAAiC;IACjC,QAAQ,EAAE,KAAK;IACf,KAAK,EAAE,IAAI;IAEX,qBAAqB;IACrB,KAAK,EAAE;QACL,kBAAkB,EAAE,KAAK;KAC1B;IAED,8BAA8B;IAC9B,OAAO,EAAE,KAAK;IAEd,0EAA0E;IAC1E,MAAM,EAAE,KAAK;IAEb,6EAA6E;IAC7E,iBAAiB,EAAE,IAAI;IAEvB,uEAAuE;IACvE,uCAAuC;IACvC,kBAAkB,EAAE,IAAI;IAExB,kCAAkC;IAClC,mBAAmB,EAAE,KAAK;IAE1B,uEAAuE;IACvE,sBAAsB,EAAE,IAAI;CAC7B,CAAA","sourcesContent":["/**\n * @fileoverview RayNeo iO Hardware Capabilities\n *\n * Capability profile for the RayNeo iO smart glasses. The stock firmware\n * exposes a single-line subtitle page over a framed BLE protocol; with the\n * Glassly CFW installed the glasses open a 540×180 page the phone draws into\n * through the Glassly CFW record protocol (positioned text, images, rects).\n */\n\nimport type {Capabilities} from \"../hardware\"\n\n/**\n * RayNeo iO capability profile\n */\nexport const rayneoIO: Capabilities = {\n  modelName: \"RayNeo iO\",\n\n  // Camera capabilities - no camera\n  hasCamera: false,\n  camera: null,\n\n  // Display capabilities - single monochrome (1-bit) panel\n  hasDisplay: true,\n  display: {\n    count: 1,\n    isColor: false,\n    color: \"green\",\n    // Bitmaps and positioned elements need the Glassly CFW; stock firmware\n    // only shows text through its subtitle page.\n    canDisplayBitmap: true,\n    resolution: {width: 540, height: 180},\n    fieldOfView: {horizontal: 30},\n    maxTextLines: 5,\n    adjustBrightness: true,\n    // Both drivers map these settings onto the launcher's three heights and two distances.\n    positionRange: {height: {min: 0, max: 8}, depth: {min: 1, max: 3}},\n\n    // Scene display API — the Glassly CFW page canvas.\n    width: 540,\n    height: 180,\n    canPosition: true,\n    maxTextElements: 6,\n    maxImageElements: 4,\n    intensityLevels: 16,\n    partialUpdate: true,\n  },\n\n  // Microphone capabilities - one Opus mic stream through the voice page\n  hasMicrophone: true,\n  microphone: {\n    count: 1,\n    hasVAD: false,\n  },\n\n  // Speaker capabilities - no speaker channel in the protocol\n  hasSpeaker: false,\n  speaker: null,\n\n  // IMU capabilities - stock firmware exposes no IMU or head-gesture stream\n  hasIMU: false,\n  imu: null,\n\n  // Button capabilities - temple touchpad/crown plus a physical button\n  hasButton: true,\n  button: {\n    count: 1,\n    buttons: [\n      {\n        type: \"swipe1d\",\n        events: [\"TAP\", \"DOUBLE_TAP\", \"PRESS_HOLD\", \"SWIPE_UP\", \"SWIPE_DOWN\"],\n        isCapacitive: true,\n      },\n    ],\n  },\n\n  // Light capabilities - no lights\n  hasLight: false,\n  light: null,\n\n  // Power capabilities\n  power: {\n    hasExternalBattery: false,\n  },\n\n  // WiFi capabilities - no WiFi\n  hasWifi: false,\n\n  // OTA capabilities - RayNeo iO updates are not driven by the ASG OTA flow\n  hasOta: false,\n\n  // Glassly CFW is bundled for the RayNeo iO and flashed over BLE from the app\n  hasCustomFirmware: true,\n\n  // Dashboard - the launcher renders its own status UI; Glassly does not\n  // manage the dashboard settings for it\n  hasNativeDashboard: true,\n\n  // Navigation - no firmware nav UI\n  hasNativeNavigation: false,\n\n  // Firmware notification popups: the phone uploads notification content\n  hasNativeNotifications: true,\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glassly/engine",
-  "version": "0.3.5-dev.737",
+  "version": "0.3.5-dev.740",
   "description": "Glassly Engine — miniapp registry, WebView bridge, and miniapp store",
   "main": "build/index.js",
   "react-native": "src/index.ts",
@@ -97,17 +97,17 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "@glassly/bluetooth-sdk": "0.3.5-dev.737",
-    "@glassly/cloud-client": "0.3.5-dev.737",
-    "@glassly/cloud-protocol": "0.3.5-dev.737",
-    "@glassly/crust": "0.3.5-dev.737",
-    "@glassly/miniapp": "0.3.5-dev.737",
+    "@glassly/bluetooth-sdk": "0.3.5-dev.740",
+    "@glassly/cloud-client": "0.3.5-dev.740",
+    "@glassly/cloud-protocol": "0.3.5-dev.740",
+    "@glassly/crust": "0.3.5-dev.740",
+    "@glassly/miniapp": "0.3.5-dev.740",
     "buffer": "^6.0.3",
     "events": "^3.3.0",
     "react-native-marked": "8.1.1",
     "semver": "^7.7.4",
     "typesafe-ts": "^1.7.1",
-    "@glassly/llm-providers": "0.3.5-dev.737",
+    "@glassly/llm-providers": "0.3.5-dev.740",
     "@huggingface/jinja": "^0.5.9",
     "jsonrepair": "^3.12.0"
   },

package/src/generated/releaseMetadata.ts

@@ -12,9 +12,9 @@ export interface EngineReleaseMetadata {
 export const ENGINE_RELEASE_METADATA: Readonly<EngineReleaseMetadata> = Object.freeze({
   "schemaVersion": 1,
   "familyBaseVersion": "0.3.5",
-  "releaseIdentity": "0.3.5-dev.737",
-  "releaseSetId": "glassly-0.3.5-dev.737",
-  "sourceCommit": "4b824a377398b67972cb0a07483b7be2d7e42824",
+  "releaseIdentity": "0.3.5-dev.740",
+  "releaseSetId": "glassly-0.3.5-dev.740",
+  "sourceCommit": "6c44d91e3f1ca0f107b0a955920d8ca6ac37b1f8",
   "otaManifestUrl": null,
   "otaManifestSha256": null
 })

package/src/services/LocalLlmPrompt.ts

@@ -85,55 +85,74 @@ export function toTemplateTools(tools?: LlmTool[]): TemplateTool[] | undefined {
   }))
 }
 
-/** Rough tokens-per-character for a BPE vocabulary over mixed prose/JSON. */
-const CHARS_PER_TOKEN = 3.2
-/** Template framing per message (`<|im_start|>role … <|im_end|>`). */
-const TOKENS_PER_MESSAGE = 6
-
-export function estimateTokens(text: string): number {
-  return Math.ceil(text.length / CHARS_PER_TOKEN)
-}
-
-export function estimateMessageTokens(message: TemplateMessage): number {
-  let total = TOKENS_PER_MESSAGE + estimateTokens(message.content)
-  if (message.role === "assistant" && message.tool_calls) {
-    for (const call of message.tool_calls) total += estimateTokens(JSON.stringify(call.function)) + 8
-  }
-  return total
+export interface FittedLocalPrompt {
+  prompt: string
+  promptTokens: number
+  truncated: boolean
 }
 
-const MIN_TOOL_CONTENT_CHARS = 1_200
-
 /**
- * Fit the conversation into `budgetTokens`. History is grouped into turns
- * (a user message plus everything up to the next user message, so a tool
- * round stays with the question it answers); the oldest turns go first and
- * the last turn is never dropped — the model must at least see the question.
- * When that last turn alone still does not fit, tool outputs are cut to
- * their head.
+ * Measure the complete rendered prompt with the model's tokenizer, including
+ * template framing, system instructions and tool schemas. Drop old turns first,
+ * then shorten tool results / message text in the final turn. Never cut template
+ * delimiters, system instructions, or structured tool calls to make them fit.
  */
-export function fitToBudget(messages: TemplateMessage[], budgetTokens: number, toolTokens = 0): TemplateMessage[] {
-  const total = (list: TemplateMessage[]) => list.reduce((sum, m) => sum + estimateMessageTokens(m), toolTokens)
+export async function fitPromptToContext(
+  messages: TemplateMessage[],
+  budgetTokens: number,
+  render: (messages: TemplateMessage[]) => string,
+  countTokens: (prompt: string) => Promise<number>,
+): Promise<FittedLocalPrompt> {
+  if (!Number.isInteger(budgetTokens) || budgetTokens < 1) {
+    throw new Error("On-device model has no room for a prompt after reserving output tokens")
+  }
   const system = messages[0]?.role === "system" ? [messages[0]] : []
   const turns: TemplateMessage[][] = []
   for (const message of messages.slice(system.length)) {
     if (message.role === "user" || turns.length === 0) turns.push([message])
     else turns[turns.length - 1]!.push(message)
   }
-
-  while (turns.length > 1 && total([...system, ...turns.flat()]) > budgetTokens) {
-    turns.shift()
-  }
-
   let fitted = [...system, ...turns.flat()]
-  if (total(fitted) > budgetTokens) {
-    fitted = fitted.map((m) =>
-      m.role === "tool" && m.content.length > MIN_TOOL_CONTENT_CHARS
-        ? {...m, content: `${m.content.slice(0, MIN_TOOL_CONTENT_CHARS)}\n…[truncated]`}
-        : m,
-    )
+  let truncated = false
+  const marker = "\n…[truncated]…\n"
+  while (true) {
+    const prompt = render(fitted)
+    const promptTokens = await countTokens(prompt)
+    if (!Number.isInteger(promptTokens) || promptTokens < 1) {
+      throw new Error("On-device tokenizer returned an invalid prompt token count")
+    }
+    if (promptTokens <= budgetTokens) return {prompt, promptTokens, truncated}
+    truncated = true
+    if (turns.length > 1) {
+      turns.shift()
+      fitted = [...system, ...turns.flat()]
+      continue
+    }
+    // Preserve the user's question while tool output can still be shortened.
+    const candidates = fitted
+      .map((message, index) => ({message, index}))
+      .filter(({message}) => message.role !== "system" && Array.from(message.content).length > marker.length * 2)
+      .sort(
+        (a, b) =>
+          Number(b.message.role === "tool") - Number(a.message.role === "tool") ||
+          b.message.content.length - a.message.content.length,
+      )
+    const candidate = candidates[0]
+    if (!candidate) {
+      throw new Error(
+        `On-device prompt instructions and tool definitions exceed the context budget (${promptTokens} > ${budgetTokens} tokens)`,
+      )
+    }
+    // Keep both ends of long transcripts and questions, without splitting a
+    // Unicode code point. Each pass shrinks strictly, then remeasures the whole
+    // rendered prompt; character ratios are never used as a safety check.
+    const chars = Array.from(candidate.message.content)
+    const keep = Math.floor(chars.length / 4)
+    fitted[candidate.index] = {
+      ...candidate.message,
+      content: `${chars.slice(0, keep).join("")}${marker}${chars.slice(-keep).join("")}`,
+    }
   }
-  return fitted
 }
 
 /** Remove a leading `<think>…</think>` block (or an unterminated one). */

package/src/services/LocalLlmProvider.ts

@@ -22,9 +22,8 @@ import {BgTimer} from "../utils/timers"
 import localLlmModelManager, {type LocalLlmModelId} from "./LocalLlmModelManager"
 import {
   buildTemplateMessages,
-  estimateTokens,
   finalizeLocalResult,
-  fitToBudget,
+  fitPromptToContext,
   toTemplateTools,
   type LocalCompletion,
 } from "./LocalLlmPrompt"
@@ -33,6 +32,7 @@ import {SETTINGS, useSettingsStore} from "../stores/settings"
 type Rne = typeof import("react-native-executorch")
 // The runner API lives on the `llm` namespace; derive its types from there so
 // the import stays a type-only reference (no native module at load time).
+type Tokenizer = ReturnType<Rne["nlp"]["loadTokenizer"]>
 type LLMRunner = ReturnType<Rne["llm"]["createLLMRunner"]>
 type LLMGenerationStats = ReturnType<LLMRunner["generate"]>
 type LLMGenerationConfig = NonNullable<Parameters<LLMRunner["generate"]>[1]>
@@ -45,12 +45,14 @@ const IDLE_UNLOAD_MS = 5 * 60_000
 const GENERATION_DEADLINE_MS = 100_000
 const DEFAULT_MAX_NEW_TOKENS = 512
 const MAX_NEW_TOKENS_CEILING = 1_024
-/** Headroom between the estimated prompt and the context end; estimates are rough. */
+/** Reserve space for runner-added BOS/EOS tokens and context boundary bookkeeping. */
 const CONTEXT_MARGIN_TOKENS = 96
 
 interface LoadedModel {
   modelId: LocalLlmModelId
   runner: LLMRunner
+  tokenizer: Tokenizer
+  countTokens: (tokenizer: Tokenizer, prompt: string) => Promise<number>
   template: Template
   stopTokens: string[]
   maxSeqLen: number
@@ -85,6 +87,11 @@ function generateWorklet(
   return {text, stats}
 }
 
+function countTokensWorklet(tokenizer: Tokenizer, prompt: string): number {
+  "worklet"
+  return tokenizer.encode(prompt).length
+}
+
 class LocalLlmProvider {
   private static instance: LocalLlmProvider
 
@@ -150,6 +157,8 @@ class LocalLlmProvider {
       loaded.runner.dispose()
     } catch (error) {
       console.warn(`${LOG_TAG}: runner dispose failed:`, error)
+    } finally {
+      loaded.tokenizer.dispose()
     }
   }
 
@@ -159,21 +168,21 @@ class LocalLlmProvider {
       const model = await this.ensureLoaded()
       const jsonMode = request.jsonMode === true
       const tools = toTemplateTools(request.tools)
-      const toolTokens = tools ? estimateTokens(JSON.stringify(tools)) + 64 : 0
 
       const maxNewTokens = Math.max(
         16,
         Math.min(request.maxTokens ?? DEFAULT_MAX_NEW_TOKENS, MAX_NEW_TOKENS_CEILING, Math.floor(model.maxSeqLen / 4)),
       )
       const promptBudget = model.maxSeqLen - maxNewTokens - CONTEXT_MARGIN_TOKENS
-      const messages = fitToBudget(buildTemplateMessages({...request, jsonMode}), promptBudget, toolTokens)
-
-      const prompt = model.template.render({
-        messages,
-        tools,
-        add_generation_prompt: true,
-        enable_thinking: false,
-      })
+      const {prompt, promptTokens, truncated} = await fitPromptToContext(
+        buildTemplateMessages({...request, jsonMode}),
+        promptBudget,
+        (messages) => model.template.render({messages, tools, add_generation_prompt: true, enable_thinking: false}),
+        (text) => model.countTokens(model.tokenizer, text),
+      )
+      console.log(
+        `${LOG_TAG}: ${model.modelId} prompt=${promptTokens} budget=${promptBudget} context=${model.maxSeqLen} maxNewTokens=${maxNewTokens} truncated=${truncated}`,
+      )
 
       model.runner.reset(0)
       const deadline = BgTimer.setTimeout(() => {
@@ -189,6 +198,12 @@ class LocalLlmProvider {
           {maxNewTokens, temperature: request.temperature ?? 0.7},
           model.stopTokens,
         )
+      } catch (error) {
+        console.warn(
+          `${LOG_TAG}: generation failed: model=${model.modelId} prompt=${promptTokens} context=${model.maxSeqLen} maxNewTokens=${maxNewTokens}`,
+          error,
+        )
+        throw error
       } finally {
         BgTimer.clearTimeout(deadline)
       }
@@ -246,16 +261,28 @@ class LocalLlmProvider {
     const startedAt = Date.now()
     const createRunner = rne.wrapAsync(rne.llm.createLLMRunner)
     const runner = await createRunner(paths.model, paths.tokenizer, [])
-    const maxSeqLen = runner.getKVCacheState().maxSeqLen
-    console.log(`${LOG_TAG}: loaded ${modelId} (context ${maxSeqLen}) in ${Date.now() - startedAt} ms`)
-
-    return {
-      modelId,
-      runner,
-      template,
-      stopTokens: [...tokenizerConfig.stopTokens],
-      maxSeqLen,
-      generate: rne.wrapAsync(generateWorklet),
+    let tokenizer: Tokenizer | undefined
+    try {
+      const maxSeqLen = runner.getKVCacheState().maxSeqLen
+      if (!Number.isInteger(maxSeqLen) || maxSeqLen <= CONTEXT_MARGIN_TOKENS + 16) {
+        throw new Error(`On-device model ${modelId} reported an invalid context size: ${maxSeqLen}`)
+      }
+      tokenizer = await rne.wrapAsync(rne.nlp.loadTokenizer)(paths.tokenizer)
+      console.log(`${LOG_TAG}: loaded ${modelId} (context ${maxSeqLen}) in ${Date.now() - startedAt} ms`)
+      return {
+        modelId,
+        runner,
+        tokenizer,
+        countTokens: rne.wrapAsync(countTokensWorklet),
+        template,
+        stopTokens: [...tokenizerConfig.stopTokens],
+        maxSeqLen,
+        generate: rne.wrapAsync(generateWorklet),
+      }
+    } catch (error) {
+      tokenizer?.dispose()
+      runner.dispose()
+      throw error
     }
   }
 

package/src/types/capabilities/rayneo-io.ts

@@ -32,6 +32,8 @@ export const rayneoIO: Capabilities = {
     fieldOfView: {horizontal: 30},
     maxTextLines: 5,
     adjustBrightness: true,
+    // Both drivers map these settings onto the launcher's three heights and two distances.
+    positionRange: {height: {min: 0, max: 8}, depth: {min: 1, max: 3}},
 
     // Scene display API — the Glassly CFW page canvas.
     width: 540,