@bendyline/gezel-inference 1.0.2 → 1.0.3

package/dist/engines/native-manifest.d.ts

@@ -40,12 +40,12 @@
  * but logs that it could not anchor to a baked-in digest.
  */
 /** Native release version this build pins. Placeholder until first public release. */
-export declare const NATIVE_ENGINE_RELEASE = "0.1.51";
+export declare const NATIVE_ENGINE_RELEASE = "0.1.52";
 /** sha256 of the pinned release's `SHA256SUMS` asset. All-zeros = unpinned. */
-export declare const SHA256SUMS_DIGEST = "499a87aa18e113cdb52ee277bff71365e9176b63b536dbea7f3c2aa5ff5dbe68";
-/** Exact SHA256 values for every engine archive published by native-v0.1.51. */
+export declare const SHA256SUMS_DIGEST = "1a2183ad785671be0f78a0f25816f7c802ce668ebe4fa58d640d336ebcbd7023";
+/** Exact SHA256 values for every engine archive published by native-v0.1.52. */
 export declare const NATIVE_ENGINE_ARCHIVE_SHA256: Readonly<Record<string, string>>;
-/** Download size in bytes of every engine archive published by native-v0.1.51. */
+/** Download size in bytes of every engine archive published by native-v0.1.52. */
 export declare const NATIVE_ENGINE_ARCHIVE_BYTES: Readonly<Record<string, number>>;
 /**
  * True only when the pinned release's standalone macOS archives were

package/dist/engines/native-manifest.js

@@ -40,45 +40,47 @@
  * but logs that it could not anchor to a baked-in digest.
  */
 /** Native release version this build pins. Placeholder until first public release. */
-export const NATIVE_ENGINE_RELEASE = '0.1.51';
+export const NATIVE_ENGINE_RELEASE = '0.1.52';
 /** sha256 of the pinned release's `SHA256SUMS` asset. All-zeros = unpinned. */
-export const SHA256SUMS_DIGEST = '499a87aa18e113cdb52ee277bff71365e9176b63b536dbea7f3c2aa5ff5dbe68';
+export const SHA256SUMS_DIGEST = '1a2183ad785671be0f78a0f25816f7c802ce668ebe4fa58d640d336ebcbd7023';
 // BEGIN PINNED NATIVE ARCHIVE HASHES
-/** Exact SHA256 values for every engine archive published by native-v0.1.51. */
+/** Exact SHA256 values for every engine archive published by native-v0.1.52. */
 export const NATIVE_ENGINE_ARCHIVE_SHA256 = Object.freeze({
-    'gezel-native-0.1.51-darwin-arm64-metal.tar.gz': '5abbfd156fbcc717da96c89dc5baa507a56d2b55b501f1a975803a763e6414c4',
-    'gezel-native-0.1.51-darwin-arm64.tar.gz': '799bca88b0ea10aeb2dad9ed3065c9396f42383a6fd5c7347ed76abf0706f128',
-    'gezel-native-0.1.51-linux-arm64-cpu.tar.gz': '1612e5928693262884bcc2c60ba3975af4dc4267c2bd75597ef91e83681fa96e',
-    'gezel-native-0.1.51-linux-arm64-cuda.tar.gz': '4a457df5dbb8475ca0eb2f1a2373f1960bbb3f86a0ecd938a407f884cd6026a2',
-    'gezel-native-0.1.51-linux-arm64.tar.gz': '2aaead65bfd42ffef9a1f45a70fa24095bc2814842750536989cbcb7a9a0db24',
-    'gezel-native-0.1.51-linux-x64-cpu.tar.gz': '7453d6a3ef2de7ece967a033d5bd634789e41edefa4a823b580329f0ad98abb0',
-    'gezel-native-0.1.51-linux-x64-cuda.tar.gz': 'ac1054a33339bcf6167c309b867f5671def58e036bb612978c4a434cd7ac39d6',
-    'gezel-native-0.1.51-linux-x64-vulkan.tar.gz': '6739c9dca6cbcf53fb5d69dbd342ac6be49fe2f0d86471d66975e61a64c95e9f',
-    'gezel-native-0.1.51-linux-x64.tar.gz': 'a176ac570131db874e0a81fdec939415dd161a639827219a82f2a610f5ebd8d3',
-    'gezel-native-0.1.51-win32-arm64-cpu.zip': 'f79c0c8490d46f2d60e73fd948bf74ae93dfc7c650a878f7db5e227bbdbb85b9',
-    'gezel-native-0.1.51-win32-arm64.zip': 'c829c9c1fd08756aff9e677ea1addcf80a8739f13df3c9b6788a6237e53efacb',
-    'gezel-native-0.1.51-win32-x64-cpu.zip': 'f8e0ffeacc8aeea1f2fc91024382bdcdaeeeb65b8a45935af422acb4d7027190',
-    'gezel-native-0.1.51-win32-x64-cuda.zip': '6bd7dc2c43443a847058a4c4f1c53281c99563c214394a92470281a70f725d1e',
-    'gezel-native-0.1.51-win32-x64-vulkan.zip': 'e20fb148c854f267aa50981525e337c38d3ee496fd483393a676251d9c0a496f',
-    'gezel-native-0.1.51-win32-x64.zip': 'ca5d7866fa2ff2e89da9267fcb409eb9aff01ef1c43ed726327e4967d3ed5e56',
+    'gezel-native-0.1.52-darwin-arm64-metal.tar.gz': '8c8619530592692ad9fd905823a5bc7772a652d4257905919dac088c5fa86347',
+    'gezel-native-0.1.52-darwin-arm64.tar.gz': '3088b7d97968ea206bf58a516da2a9c635d7622672da220d48d790fd5555a797',
+    'gezel-native-0.1.52-linux-arm64-cpu.tar.gz': 'ec4b992eb372e735156691950eadb2aa8f7e5ef29437b53a9da6fe8cc2a7912b',
+    'gezel-native-0.1.52-linux-arm64-cuda.tar.gz': '4f97fd70f0d132f0bfcaadfd113f63add7d8b63982edf8b1b7a46301702047f5',
+    'gezel-native-0.1.52-linux-arm64.tar.gz': '3fa2c58ff75bea7ec0f739821250307bfba78cb026779ec3ad1ef935286af534',
+    'gezel-native-0.1.52-linux-x64-cpu.tar.gz': '1fbfd06acfea1ef9e2629c121da2d7da13ee2e76a0cebb8cf07b779938817966',
+    'gezel-native-0.1.52-linux-x64-cuda.tar.gz': '5c3c05f285c3922845b72e470ecfcd5e510b874a35ae841cd655a7098b541ffd',
+    'gezel-native-0.1.52-linux-x64-vulkan.tar.gz': '33a6dd016bd2cf0aada7f34c364a457acc524373b52a1d3f1f6e83c1240e93a7',
+    'gezel-native-0.1.52-linux-x64.tar.gz': 'f17afac30f984da17b6098bf5f9c5e88e9f9e128b8fbea0b5ff67a360a494313',
+    'gezel-native-0.1.52-win32-arm64-cpu.zip': '49f011b05eec55cc3f60730fdd46e3d28de331f60b195eb29abaa04fe62835a3',
+    'gezel-native-0.1.52-win32-arm64-cuda.zip': '7dbc99fd9516ea5a5cc49ad2c157bb66a240ecc2b5a5e22fae33142a6615caec',
+    'gezel-native-0.1.52-win32-arm64.zip': 'dfe61d86eae932e03d92665aa42b81121e21cfa76a85424d993b5726ee5fa504',
+    'gezel-native-0.1.52-win32-x64-cpu.zip': 'b1147052367daf694d6001ef0d45a8897a39a1d74cb776a42f22529d5c21f919',
+    'gezel-native-0.1.52-win32-x64-cuda.zip': '3c9568591d19ba04ceb12ae2e9f9f2a4ac7ade4597458a8674d444c06407324f',
+    'gezel-native-0.1.52-win32-x64-vulkan.zip': '9cb1e7721d1dbebfa830ec8cba1bbbbcb41f0145d60461e4bf50c4e347feecb2',
+    'gezel-native-0.1.52-win32-x64.zip': '7c94605b0783695174773ac2327274d91bb9d7ae31e7d932a2b9df78d44424df',
 });
-/** Download size in bytes of every engine archive published by native-v0.1.51. */
+/** Download size in bytes of every engine archive published by native-v0.1.52. */
 export const NATIVE_ENGINE_ARCHIVE_BYTES = Object.freeze({
-    'gezel-native-0.1.51-darwin-arm64-metal.tar.gz': 7056609,
-    'gezel-native-0.1.51-darwin-arm64.tar.gz': 41556782,
-    'gezel-native-0.1.51-linux-arm64-cpu.tar.gz': 8836261,
-    'gezel-native-0.1.51-linux-arm64-cuda.tar.gz': 619480627,
-    'gezel-native-0.1.51-linux-arm64.tar.gz': 41979458,
-    'gezel-native-0.1.51-linux-x64-cpu.tar.gz': 12914417,
-    'gezel-native-0.1.51-linux-x64-cuda.tar.gz': 989771250,
-    'gezel-native-0.1.51-linux-x64-vulkan.tar.gz': 25672179,
-    'gezel-native-0.1.51-linux-x64.tar.gz': 58057226,
-    'gezel-native-0.1.51-win32-arm64-cpu.zip': 5618599,
-    'gezel-native-0.1.51-win32-arm64.zip': 42055408,
-    'gezel-native-0.1.51-win32-x64-cpu.zip': 8714418,
-    'gezel-native-0.1.51-win32-x64-cuda.zip': 749850276,
-    'gezel-native-0.1.51-win32-x64-vulkan.zip': 21123169,
-    'gezel-native-0.1.51-win32-x64.zip': 56558016,
+    'gezel-native-0.1.52-darwin-arm64-metal.tar.gz': 7056567,
+    'gezel-native-0.1.52-darwin-arm64.tar.gz': 42206061,
+    'gezel-native-0.1.52-linux-arm64-cpu.tar.gz': 8836310,
+    'gezel-native-0.1.52-linux-arm64-cuda.tar.gz': 620408821,
+    'gezel-native-0.1.52-linux-arm64.tar.gz': 42037958,
+    'gezel-native-0.1.52-linux-x64-cpu.tar.gz': 12914985,
+    'gezel-native-0.1.52-linux-x64-cuda.tar.gz': 993543364,
+    'gezel-native-0.1.52-linux-x64-vulkan.tar.gz': 25672744,
+    'gezel-native-0.1.52-linux-x64.tar.gz': 57183428,
+    'gezel-native-0.1.52-win32-arm64-cpu.zip': 5618616,
+    'gezel-native-0.1.52-win32-arm64-cuda.zip': 219555740,
+    'gezel-native-0.1.52-win32-arm64.zip': 42114295,
+    'gezel-native-0.1.52-win32-x64-cpu.zip': 8714442,
+    'gezel-native-0.1.52-win32-x64-cuda.zip': 749856660,
+    'gezel-native-0.1.52-win32-x64-vulkan.zip': 21123132,
+    'gezel-native-0.1.52-win32-x64.zip': 55662940,
 });
 // END PINNED NATIVE ARCHIVE HASHES
 /**

package/dist/engines/resolver.js

@@ -763,7 +763,9 @@ async function resolveGitHubToken() {
         return process.env.GITHUB_TOKEN;
     try {
         const { execFileSync } = await import('node:child_process');
-        const out = execFileSync('gh', ['auth', 'token'], {
+        // Named host: a bare `gh auth token` answers for gh's default host, which
+        // can be a GitHub Enterprise server whose token must not go to github.com.
+        const out = execFileSync('gh', ['auth', 'token', '--hostname', 'github.com'], {
             encoding: 'utf8',
             stdio: ['ignore', 'pipe', 'ignore'],
             ...windowsHeadlessSpawnOptions(),

package/dist/fs/atomic.d.ts

@@ -50,4 +50,7 @@ export declare function writeFileAtomic(absPath: string, content: string | Uint8
  * private staging file by their adapter, then copied to a sibling temporary
  * path and renamed into place.
  */
-export declare function copyFileAtomic(sourcePath: string, absPath: string): Promise<void>;
+export declare function copyFileAtomic(sourcePath: string, absPath: string, opts?: {
+    noReplace?: boolean;
+    signal?: AbortSignal;
+}): Promise<void>;

package/dist/fs/atomic.js

@@ -108,10 +108,16 @@ export async function writeFileAtomic(absPath, content, opts) {
  * private staging file by their adapter, then copied to a sibling temporary
  * path and renamed into place.
  */
-export async function copyFileAtomic(sourcePath, absPath) {
+export async function copyFileAtomic(sourcePath, absPath, opts) {
     const tmp = `${absPath}.tmp-${process.pid}-${randomUUID()}`;
     try {
         await copyFile(sourcePath, tmp);
+        opts?.signal?.throwIfAborted();
+        if (opts?.noReplace) {
+            await link(tmp, absPath);
+            await rm(tmp, { force: true }).catch(() => undefined);
+            return;
+        }
         await renameWithWindowsRetry(tmp, absPath);
     }
     catch (err) {

package/dist/http/openai-compat/chat-target.d.ts

@@ -59,7 +59,7 @@ export declare function resolvePinnedModelTarget(ctx: ServiceContext, model: str
  * Pick the gezel that guarantees a persona-backed fallback for public
  * inference requests.
  *
- * The explicit Connected Apps choice wins. When it is unset or stale,
+ * The explicit Integrations choice wins. When it is unset or stale,
  * the install's Meester is the natural front door; the first available
  * gezel is the final recovery path for older or partially migrated
  * homes. A healthy store always has at least a Meester, but returning

package/dist/http/openai-compat/chat-target.js

@@ -29,7 +29,7 @@ export async function resolvePinnedModelTarget(ctx, model) {
  * Pick the gezel that guarantees a persona-backed fallback for public
  * inference requests.
  *
- * The explicit Connected Apps choice wins. When it is unset or stale,
+ * The explicit Integrations choice wins. When it is unset or stale,
  * the install's Meester is the natural front door; the first available
  * gezel is the final recovery path for older or partially migrated
  * homes. A healthy store always has at least a Meester, but returning

package/dist/http/routes/v1-chat.js

@@ -289,7 +289,7 @@ export function v1ChatRoutes(ctx, opts = {}) {
             // instruct folds, per-gezel overrides for gezel: targets) applies
             // unconditionally — it's "what the model is" — with the caller's
             // per-request knobs overlaid on top. The behavior `profile` is the
-            // supporting layer the Connected Apps switch gates. Caller-owned
+            // supporting layer the Integrations switch gates. Caller-owned
             // inference keeps parsing, grammar, and transcript compatibility but
             // omits interventions which assume Gezel owns the action loop.
             // Best-effort: a resolver failure serves engine defaults rather than

package/dist/http/routes/v1-knowledge.js

@@ -222,6 +222,10 @@ export function v1KnowledgeRoutes(ctx) {
             maxResults: 24,
             queryEmbedBudgetMs: 5000,
             catalogs: enabled.map((entry) => entry.ref.catalogId),
+            // Let the reranker choose among an article's sections: one passage per
+            // article, picked by embedding similarity, sent an infobox in place of
+            // the history the person asked about.
+            chunksPerDocument: 3,
         });
         c.req.raw.signal.throwIfAborted();
         const candidates = [];

package/dist/http/routes/v1-responses.d.ts

@@ -9,7 +9,7 @@ import type { InferenceContext } from '../inference-context.js';
  * and returns assistant text/tool calls in Responses wire shapes. The route
  * intentionally does not create a Gezel chat session or attach Gezel MCP.
  *
- * In the full service, auth is mounted in `server.ts`: the Connected Apps
+ * In the full service, auth is mounted in `server.ts`: the Integrations
  * master switch, bearer authentication, and the `openai` scope all apply. Codex can therefore use
  * an app-scoped token without ever learning the daemon's root credential.
  */

package/dist/inference-service.d.ts

@@ -1,4 +1,4 @@
-import { type ProviderName } from '@bendyline/gezel';
+import { type LogOutput, type ProviderName } from '@bendyline/gezel';
 import type { LLMProvider } from './providers/types.js';
 import './http/auth-context.js';
 /** Same shape the full service reports, so one handler serves either host. */
@@ -15,6 +15,11 @@ export interface InferenceServiceOptions {
     /** Host-provided local providers, useful for testing without downloading weights. */
     providers?: ReadonlyMap<ProviderName, LLMProvider>;
     onUnexpectedHttpError?: (event: InferenceHttpErrorEvent) => void;
+    /**
+     * Where log records go while this host runs; restored on `stop()`. An app
+     * embedding the host passes `'stderr'` so its own stdout stays its own.
+     */
+    logOutput?: LogOutput;
 }
 export interface RunningInferenceService {
     readonly profile: 'embedded-inference';

package/dist/inference-service.js

@@ -1,6 +1,6 @@
 import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
 import { join } from 'node:path';
-import { GEZEL_API_GENERATION, GEZEL_API_GENERATION_FLOOR, GEZEL_VERSION, GezelConfigSchema, createLogger, nowIso, startSuspendMonitor, stopSuspendMonitor, } from '@bendyline/gezel';
+import { GEZEL_API_GENERATION, GEZEL_API_GENERATION_FLOOR, GEZEL_VERSION, GezelConfigSchema, createLogger, getLogOutput, nowIso, setLogOutput, startSuspendMonitor, stopSuspendMonitor, } from '@bendyline/gezel';
 import { BundledSource, CatalogService } from '@bendyline/gezel-catalog';
 import { discoverNativeBinaries } from '@bendyline/gezel/native';
 import { gezelHome } from '@bendyline/gezel/paths';
@@ -35,16 +35,27 @@ import './http/auth-context.js';
 const log = createLogger('inference');
 /** Direct Fetch host: no listener, product store, background downloads or daemon discovery. */
 export async function startService(options = {}) {
+    const previousLogOutput = getLogOutput();
+    const restoreLogOutput = () => setLogOutput(previousLogOutput);
+    if (options.logOutput)
+        setLogOutput(options.logOutput);
     const home = options.home ?? gezelHome();
-    await ensurePrivateUserHome(home);
-    // Model files must land in this home's managed cache: unset, transformers.js
-    // falls back to a module-relative cache that is read-only inside a packaged
-    // app, and the embed worker (which reads this env) has no other view of home.
-    process.env[HF_CACHE_DIR_ENV] ??= transformersCacheDir(home);
-    const startedAt = nowIso();
     const runtimeDir = join(home, 'runtime');
-    const lock = await acquireSingleInstanceLock({ runtimeDir, lockPath: join(runtimeDir, 'lock') });
-    const cleanup = [() => lock.release()];
+    let lock;
+    try {
+        await ensurePrivateUserHome(home);
+        // Model files must land in this home's managed cache: unset, transformers.js
+        // falls back to a module-relative cache that is read-only inside a packaged
+        // app, and the embed worker (which reads this env) has no other view of home.
+        process.env[HF_CACHE_DIR_ENV] ??= transformersCacheDir(home);
+        lock = await acquireSingleInstanceLock({ runtimeDir, lockPath: join(runtimeDir, 'lock') });
+    }
+    catch (error) {
+        restoreLogOutput();
+        throw error;
+    }
+    const startedAt = nowIso();
+    const cleanup = [restoreLogOutput, () => lock.release()];
     const requests = new InFlightRequests();
     let stopping;
     let closed = false;

package/dist/knowledge/manager.d.ts

@@ -260,6 +260,13 @@ export declare class KnowledgeManager {
          * their own. Explicit search asks; proactive injection does not.
          */
         media?: boolean;
+        /**
+         * Also return up to this many of a document's other passages that the
+         * search found, after its best one. Fusion otherwise keeps one passage
+         * per document, chosen by vector similarity alone, so an article's name-
+         * dense infobox can stand in for the section a reranker would choose.
+         */
+        chunksPerDocument?: number;
     }): Promise<UnifiedSearchResult[]>;
     /** A document the lexical arms named but no chunk carries: cite it with its lead paragraph. */
     private toDocumentResult;

package/dist/knowledge/manager.js

@@ -1315,6 +1315,20 @@ export class KnowledgeManager {
             }
             fused.set(key, entry);
         };
+        // Every text passage the arms found, per document, best vector match first,
+        // for `chunksPerDocument`. Fusion still ranks documents, not passages.
+        const passagesByDocument = new Map();
+        if ((opts.chunksPerDocument ?? 1) > 1) {
+            for (const hit of response.chunks
+                .filter((h) => !h.media)
+                .sort((a, b) => (b.cosine ?? -1) - (a.cosine ?? -1))) {
+                const key = `${hit.catalogKey}\u0000${hit.documentId}`;
+                const passages = passagesByDocument.get(key) ?? [];
+                if (!passages.some((known) => known.chunkUid === hit.chunkUid))
+                    passages.push(hit);
+                passagesByDocument.set(key, passages);
+            }
+        }
         // Vector arm: one rank per document, in rerank-cosine order. A hit under
         // its profile's floor is not evidence at all — KNN always returns its
         // nearest rows, and a rank-0 row fuses to relevance 1.0 however far away
@@ -1446,7 +1460,24 @@ export class KnowledgeManager {
                 continue;
             perCatalogCount.set(entry.catalogKey, count + 1);
             seenSpatial.add(spatialIdentity);
-            out.push({ ...result, ...matches.get(`${entry.catalogKey}\u0000${entry.documentId}`) });
+            const docKey = `${entry.catalogKey}\u0000${entry.documentId}`;
+            out.push({ ...result, ...matches.get(docKey) });
+            if (out.length >= Math.max(10, opts.maxResults))
+                break;
+            // The document's other passages, each labelled by its own evidence: a
+            // sibling is vector evidence only if it clears the floor by itself.
+            const siblings = (passagesByDocument.get(docKey) ?? [])
+                .filter((hit) => hit.chunkUid !== entry.chunk?.chunkUid)
+                .slice(0, Math.max(0, (opts.chunksPerDocument ?? 1) - 1));
+            const floor = this.vectorFloorFor(entry.catalogKey);
+            for (const sibling of siblings) {
+                const own = floor !== null && sibling.cosine !== undefined && sibling.cosine >= floor
+                    ? { arm: 'vector', similarity: sibling.cosine }
+                    : { arm: 'fts' };
+                out.push(this.toResult(info, sibling, fusedRankRelevance(out.length), own));
+                if (out.length >= Math.max(10, opts.maxResults))
+                    break;
+            }
             if (out.length >= Math.max(10, opts.maxResults))
                 break;
         }

package/dist/perf/responsiveness.js

@@ -8,9 +8,10 @@ import { redactPathSecrets } from './redact-path.js';
 /**
  * Main-thread responsiveness monitor.
  *
- * The daemon runs on one event loop. In embedded mode (`pnpm app`, and the
- * packaged fallback) that loop is Electron's main thread, so a synchronous
- * block here freezes the window itself, and Windows labels it "Not Responding"
+ * The daemon runs on one event loop. With an in-process embedded service (the
+ * packaged fallback, E2E, `GEZEL_SERVICE_HOST=in-process`) that loop is
+ * Electron's main thread, so a synchronous block here freezes the window
+ * itself, and Windows labels it "Not Responding"
  * after 5 s. Before this existed a block under the suspend monitor's 10 s floor
  * left no trace, and a longer one was logged as "host resumed after Ns
  * suspended": sleep that never happened (two of them during one catalog

package/dist/providers/audio/kokoro-frontend.js

@@ -75,7 +75,7 @@ export class KokoroFrontend {
                 // put it back, because a packaging slip is the likely cause.
                 throw new Error(`Kokoro pronunciation dictionary is missing at ${file}; rebuild the service (node scripts/build-kokoro-lexicon.mjs stages it).`, { cause });
             }
-            return parseKokoroLexicon(gunzipSync(packed).toString('utf8'));
+            return parseKokoroLexicon(gunzipSync(packed).toString('utf8'), { language });
         })();
         // A failed read must not be cached as a permanent failure.
         void pending.catch(() => this.loaded.delete(language));

package/dist/providers/ds4/build-provider.js

@@ -526,12 +526,18 @@ export async function buildDs4Provider(opts) {
             return { command: binary, args, baseUrl: `http://127.0.0.1:${port}`, cwd: ds4BundleDir };
         },
     });
+    // Sessions budget prompts, `max_tokens`, and compaction against this window,
+    // so it must be the `--ctx` ds4 actually launches with. Left unset, it fell
+    // back to the 64K llama default: GLM 5.3 (launched at 32K) overflowed with
+    // `context_length_exceeded` long before compaction triggered, and DeepSeek
+    // (launched at 128K) compacted at a third of its window.
     const ds4Inner = new LlamaCppProvider({
         supervisor,
         logFile: ds4LogFile,
         disableThinkingRequestShape: 'deepseek',
         classifyLine: classifyDs4Line,
         visionEnabled: vision.enabled,
+        numCtx,
         ...baseProviderOpts,
     });
     ds4ProviderHolder.current = ds4Inner;

package/dist/providers/llama-cpp/build-provider.js

@@ -24,7 +24,7 @@ import { readGgufSummaryAsync } from './gguf-metadata-async.js';
 import { effectiveLazyTensorBytes } from './gguf-metadata.js';
 import { GpuStartupWatch } from './gpu-startup-watch.js';
 import { isGemmaModel, planLlamaCppKv, resolveLlamaCppKvCacheType, } from './kv-cache-type.js';
-import { degradeDenseFfnOffloadDecision, degradeMoeOffloadDecision, estimateExactPerSlotKvBytesF16, estimateKvReserveBytes, estimateWindowedKvLinearization, planDenseFfnOffload, planMoeOffload, } from './offload-planner.js';
+import { degradeDenseFfnOffloadDecision, degradeMoeOffloadDecision, estimateExactPerSlotKvBytesF16, estimateKvReserveBytes, estimateWindowedKvLinearization, planDenseFfnOffload, planMoeOffload, shedSlotsForGpuResidency, } from './offload-planner.js';
 import { LlamaCppProvider } from './provider.js';
 import { reasoningLaunchOverridesFromEnv } from './reasoning-launch.js';
 import { resolveSpecDraft } from './spec-draft.js';
@@ -347,13 +347,6 @@ export async function buildLlamaCppProvider(opts) {
     // path still clamps to the native train context.
     const explicitCtx = envNumCtx ?? perModelCtxOverride ?? config.llamaCppNumCtx;
     const hostFloorTokens = minViableLocalContextTokens();
-    // A per-model override below the host floor is deliberate user intent —
-    // lower the admission floor to the override instead of silently raising
-    // the request back to 64K. Env / machine-wide values keep their
-    // historical floor semantics.
-    const minViableTokens = perModelCtxOverride !== undefined && explicitCtx === perModelCtxOverride
-        ? Math.min(hostFloorTokens, perModelCtxOverride)
-        : hostFloorTokens;
     const contextRequirement = resolveLlamaCppContextRequirement({
         modelContextWindow: modelCatalogInfo?.contextWindow,
         ...(explicitCtx !== undefined ? { explicitContextWindow: explicitCtx } : {}),
@@ -362,7 +355,7 @@ export async function buildLlamaCppProvider(opts) {
             ? { manifestContextSize: manifestEngineConfig.contextSize }
             : {}),
         contextSizing: config.llamaCppContextSizing ?? 'adaptive',
-        minViableContextTokens: minViableTokens,
+        minViableContextTokens: hostFloorTokens,
     });
     // `let`: RAM-aware admission below may lower this (but never below the
     // model-aware minimum) before anything launch-visible consumes it.
@@ -650,6 +643,10 @@ export async function buildLlamaCppProvider(opts) {
     // mode. Undefined when the GGUF is unreadable — the pool builder then
     // falls back to the legacy weights-multiplier reservation.
     let plannedReservationBytes;
+    // What prices that reservation, kept for the out-of-memory ladder: when it
+    // drops a slot, the relaunch is charged for the KV it actually allocates.
+    let reservationKvFor;
+    let reservationResidentBytes;
     // Graphics memory the MoE offload plan puts on the card. Device admission
     // charges this rather than the whole card; undefined keeps the old charge.
     let plannedGpuBytes;
@@ -979,9 +976,9 @@ export async function buildLlamaCppProvider(opts) {
                     ssmConvKernel: summary.ssmConvKernel,
                     kvCacheType,
                 });
-            const kvReserveBytes = windowedKvPlan
-                ? Math.round(windowedKvPlan.fixedBytes * slots +
-                    windowedKvPlan.bytesPerToken * effectiveNumCtx * slots)
+            const kvReserveFor = (slotCount) => windowedKvPlan
+                ? Math.round(windowedKvPlan.fixedBytes * slotCount +
+                    windowedKvPlan.bytesPerToken * effectiveNumCtx * slotCount)
                 : estimateKvReserveBytes({
                     blockCount: summary.blockCount,
                     embeddingLength: summary.embeddingLength,
@@ -999,43 +996,47 @@ export async function buildLlamaCppProvider(opts) {
                     ssmInnerSize: summary.ssmInnerSize,
                     ssmStateSize: summary.ssmStateSize,
                     ssmConvKernel: summary.ssmConvKernel,
-                    ctxTokens: effectiveNumCtx * slots,
+                    ctxTokens: effectiveNumCtx * slotCount,
                     kvCacheType,
                 });
-            // The broker-ledger reservation for this launch: resident weights
-            // plus the KV the engine will allocate at the granted window (M1 —
-            // the weights-only ledger under-reserved dense models whose KV
-            // rivals their weights: qwen3.5-4b at 64K carries more KV than
-            // parameters). Consumed by the pool builder via
-            // `plannedReservationBytes` so co-residency admission can see it.
-            if (kvReserveBytes !== undefined) {
-                plannedReservationBytes = residentBytes + kvReserveBytes;
-            }
+            // Plan against the card as it is now. Sized to the whole card, a plan
+            // counts memory other programs hold: MoE admission then charged the
+            // model the whole card and refused it (2026-10-11), and a dense
+            // 31B planned into 30.3 GiB of a 32 GB card that the desktop and its
+            // apps were already holding ~5 GB of died with a Vulkan out-of-memory
+            // (2026-10-19).
+            const freeGpuBytes = vramBytes > 0 ? await measureAvailableGpuBytes().catch(() => undefined) : undefined;
+            // A lane count the person chose is not the planner's to spend, and
+            // eval runs under GEZEL_LLAMA_NUM_CTX accept the memory consequences.
+            const mayShedSlots = configuredSlots === undefined && envNumCtx === undefined;
+            const placeSlots = (gpuBytes, plan) => mayShedSlots
+                ? shedSlotsForGpuResidency({ slots, vramBytes: gpuBytes, plan })
+                : { slots, decision: plan(slots, gpuBytes), shed: false };
+            let placement;
             if (isMoE) {
-                // Plan against the card as it is now. Sized to the whole card, the
-                // plan counted memory other programs held, and admission then had
-                // to charge the model the whole card and refuse it (2026-10-11).
-                const freeGpuBytes = vramBytes > 0 ? await measureAvailableGpuBytes().catch(() => undefined) : undefined;
-                const moePlan = planMoeOffload({
-                    isMoE,
-                    residentBytes,
-                    vramBytes: freeGpuBytes !== undefined ? Math.min(vramBytes, freeGpuBytes) : vramBytes,
-                    ...(split
-                        ? {
-                            split,
-                            ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
-                            ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
-                        }
-                        : {}),
+                placement = placeSlots(freeGpuBytes !== undefined ? Math.min(vramBytes, freeGpuBytes) : vramBytes, (slotCount, gpuBytes) => {
+                    const kvReserveBytes = kvReserveFor(slotCount);
+                    return planMoeOffload({
+                        isMoE,
+                        residentBytes,
+                        vramBytes: gpuBytes,
+                        ...(split
+                            ? {
+                                split,
+                                ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
+                                ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
+                            }
+                            : {}),
+                    });
                 });
-                offloadDecision = moePlan;
+                offloadDecision = placement.decision;
                 // An explicit offload setting shadows the planner's flags, so its
                 // footprint would describe a launch that is not the one made.
                 const plannerPlacesExperts = (config.llamaCppCpuMoe ?? resolvedManifestEngineConfig?.cpuMoe) === undefined &&
                     (config.llamaCppNCpuMoe ?? resolvedManifestEngineConfig?.nCpuMoe) === undefined &&
                     (config.llamaCppNGpuLayers ?? resolvedManifestEngineConfig?.nGpuLayers) === undefined;
                 if (plannerPlacesExperts)
-                    plannedGpuBytes = moePlan.gpuBytes;
+                    plannedGpuBytes = placement.decision.gpuBytes;
             }
             else {
                 const explicitNCpuFfn = config.llamaCppNCpuFfn ?? resolvedManifestEngineConfig?.nCpuFfn;
@@ -1044,16 +1045,21 @@ export async function buildLlamaCppProvider(opts) {
                 // A finite whole-layer GPU count is a different placement strategy;
                 // don't layer an automatic tensor override on top of it. `-1` means
                 // all and is compatible with the planner's own attention-on-GPU plan.
-                offloadDecision =
-                    explicitNCpuFfn === undefined &&
-                        (explicitNGpuLayers === undefined || explicitNGpuLayers === -1)
-                        ? planDenseFfnOffload({
+                if (explicitNCpuFfn === undefined &&
+                    (explicitNGpuLayers === undefined || explicitNGpuLayers === -1)) {
+                    placement = placeSlots(
+                    // Respect the usable accelerator pool, any explicit capacity
+                    // ceiling, co-resident reservations, and what other programs
+                    // hold on the card now. The raw probe still gates this to a real
+                    // discrete-GPU build rather than treating a CPU/unified fast pool
+                    // as VRAM.
+                    vramBytes > 0
+                        ? Math.max(0, Math.min(budgetBytes - committedOtherBytes, freeGpuBytes ?? Number.POSITIVE_INFINITY))
+                        : 0, (slotCount, gpuBytes) => {
+                        const kvReserveBytes = kvReserveFor(slotCount);
+                        return planDenseFfnOffload({
                             isMoE,
-                            // Respect the usable accelerator pool, any explicit capacity
-                            // ceiling, and co-resident reservations. The raw probe still
-                            // gates this to a real discrete-GPU build rather than treating
-                            // a CPU/unified fast pool as VRAM.
-                            vramBytes: vramBytes > 0 ? Math.max(0, budgetBytes - committedOtherBytes) : 0,
+                            vramBytes: gpuBytes,
                             ...(denseFfnSplit ? { split: denseFfnSplit } : {}),
                             ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
                             ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
@@ -1062,9 +1068,30 @@ export async function buildLlamaCppProvider(opts) {
                                 ? brokerSnap.pools.ramShareBytes
                                 : hostCapacity.ramShareBytes,
                             freeSystemRamBytes: availableSystemRamBytes(),
-                        })
-                        : undefined;
+                        });
+                    });
+                    offloadDecision = placement.decision;
+                }
+                else {
+                    offloadDecision = undefined;
+                }
+            }
+            if (placement?.shed) {
+                log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? 'model'}: reducing engine slots ${slots} -> 1 to keep the model on the GPU with room to spare (other programs' graphics memory moves)`);
+                slots = 1;
+            }
+            // The broker-ledger reservation for this launch: resident weights
+            // plus the KV the engine will allocate at the granted window (M1 —
+            // the weights-only ledger under-reserved dense models whose KV
+            // rivals their weights: qwen3.5-4b at 64K carries more KV than
+            // parameters). Consumed by the pool builder via
+            // `plannedReservationBytes` so co-residency admission can see it.
+            const kvReserveBytes = kvReserveFor(slots);
+            if (kvReserveBytes !== undefined) {
+                plannedReservationBytes = residentBytes + kvReserveBytes;
             }
+            reservationKvFor = kvReserveFor;
+            reservationResidentBytes = residentBytes;
             if (offloadDecision?.reason) {
                 log.info(`[llama-cpp] offload plan (${modelCatalogInfo?.id ?? 'model'}): ${offloadDecision.reason}`);
             }
@@ -1228,16 +1255,34 @@ export async function buildLlamaCppProvider(opts) {
             gpuWatch?.observe(line);
             providerHolder.current?.onStdoutLine(line);
         },
-        // GPU-OOM recovery ladder: when a start dies of VRAM exhaustion and the
-        // planner's offload decision was in play, degrade it one step
-        // (partial split → all expert/FFN weights to RAM → engine-owned fit) and let the
-        // supervisor retry — `resolveLaunch` below re-reads `offloadDecision`
-        // on every spawn. The degraded plan sticks for later restarts of this
-        // provider, so a recovered engine doesn't re-OOM on its next boot.
+        // GPU-OOM recovery ladder: when a start dies of VRAM exhaustion, first
+        // give back the extra slots' KV (every weight stays on the GPU), then, if
+        // the planner's offload decision was in play, degrade it one step
+        // (partial split → all expert/FFN weights to RAM → engine-owned fit) and
+        // let the supervisor retry — `resolveLaunch` below re-reads `slots` and
+        // `offloadDecision` on every spawn. The degraded plan sticks for later
+        // restarts of this provider, so a recovered engine doesn't re-OOM on its
+        // next boot.
         recoverStartup: ({ panicKind }) => {
             if (panicKind !== 'cuda-out-of-memory' && panicKind !== 'vulkan-out-of-memory') {
                 return false;
             }
+            // A full-residency plan has no offload rung below it, so before this a
+            // two-slot launch that missed by one KV allocation failed outright
+            // (2026-10-19, Gemma 4 31B on a 32 GB card). This rung spends one of
+            // the supervisor's two retries; an offload rung still has the other.
+            if (slots > 1 && configuredSlots === undefined) {
+                const from = slots;
+                if (providerHolder.current?.narrowEngineSlots(1)) {
+                    slots = 1;
+                    const kvReserveBytes = reservationKvFor?.(1);
+                    if (kvReserveBytes !== undefined && reservationResidentBytes !== undefined) {
+                        plannedReservationBytes = reservationResidentBytes + kvReserveBytes;
+                    }
+                    log.warn(`[llama-cpp] ${panicKind} while starting ${modelCatalogInfo?.id ?? 'model'} — retrying with one engine slot instead of ${from}`);
+                    return true;
+                }
+            }
             // Explicit config/manifest offload settings shadow the planner
             // per-field (see `buildLlamaCppEngineArgs`); when every field the
             // ladder would change is pinned, a retry replays the same argv.
@@ -1274,6 +1319,12 @@ export async function buildLlamaCppProvider(opts) {
         },
         resolveLaunch: async () => {
             gpuWatch = new GpuStartupWatch(binaryBackend);
+            // Again at every launch, not only at build: llama.cpp from the 0.1.51
+            // release refuses a --slot-save-path that does not exist ("not a
+            // directory") and exits before ready, and an eval preflight lost its
+            // build-time directory during a 50 s capacity wait (2026-10-18). The
+            // person saw "no model is installed yet".
+            await mkdirAsync(slotSavePath, { recursive: true }).catch(() => { });
             const port = cachedPort ?? (await pickFreePort());
             cachedPort = port;
             // Prepend the binary's own directory to PATH so the OS finds the