@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/cache/controller.js

@@ -0,0 +1,377 @@
+/**
+ * Engine-agnostic policy controller for prompt-cache reuse across local
+ * inference engines.
+ *
+ * Local engines (MLX via our wrapped server, llama.cpp via llama-server's
+ * slot system) preserve KV cache across requests so the per-turn prefill
+ * cost drops from O(full prompt) to O(new tokens). The mechanism differs
+ * per engine — see {@link EngineCacheAdapter}. This controller owns
+ * everything that's the same across both:
+ *
+ *   - Tracking which sessions are "warm" (have cache) per provider
+ *   - LRU eviction when total cache memory exceeds a configured budget
+ *   - Invalidation routing on compaction / archive / delete / reset
+ *   - Stats surface for telemetry (engine pill, queue meter, /api/cache)
+ *
+ * State model: a `Map<providerName, ProviderState>` where each provider
+ * holds its own `Map<sessionId, CacheEntry>`. Per-session entries record
+ * the bytes the controller *thinks* the engine has cached (an estimate
+ * the controller maintains based on `recordTurn`'s `approxPromptTokens`
+ * argument); periodic `adapter.reportUsage()` polls reconcile against
+ * engine reality.
+ *
+ * LRU is per-provider, not global — MLX and llama.cpp have independent
+ * memory budgets because they're independent processes.
+ */
+const DEFAULT_BUDGET_BYTES = 4 * 1024 * 1024 * 1024; // 4 GB; Phase 3 makes this RAM-aware.
+const RECONCILE_INTERVAL_MS = 5_000;
+const EVICTION_HIGH_WATERMARK = 1.0; // evict when usage ≥ budget
+const EVICTION_LOW_WATERMARK = 0.8; // stop evicting when usage drops below 80% of budget
+/**
+ * KV cache size estimate in bytes per token. Used only for the
+ * controller's pre-reconcile bookkeeping; the adapter's
+ * `reportUsage()` returns engine-truth bytes that take precedence.
+ *
+ * 100 KB/token is reasonable for sliding-window-attention models
+ * (Gemma 3/4 family) where most layers are windowed and don't grow
+ * linearly with token count. Higher estimates overcommit memory
+ * accounting and trigger spurious eviction; the in-engine bound is
+ * the actual safety net for real memory pressure.
+ */
+const ESTIMATED_BYTES_PER_TOKEN = 100 * 1024;
+function emptyHitsBySource() {
+    return { memory: 0, disk: 0, prefix: 0, fresh: 0 };
+}
+const RECENT_TURN_WINDOW = 50;
+export class SessionCacheController {
+    providers = new Map();
+    /**
+     * Pre-registration budget overrides. Operator config can set a
+     * budget at service boot — before any provider's adapter has been
+     * lazily registered. We stash the value here and apply it the
+     * moment the adapter shows up via `registerAdapter`. Without this,
+     * the boot-time `setBudget` would be a silent no-op and the engine
+     * would run with `DEFAULT_BUDGET_BYTES` until the first manual
+     * `setBudget` after construction.
+     */
+    pendingBudgets = new Map();
+    now;
+    logger;
+    reconcileTimer;
+    constructor(opts = {}) {
+        this.now = opts.now ?? Date.now;
+        this.logger = opts.logger ?? {};
+        if (!opts.disableReconcileTimer) {
+            this.reconcileTimer = setInterval(() => {
+                void this.reconcileAll().catch((err) => {
+                    this.logger.warn?.(`[cache] reconcile loop error: ${err}`);
+                });
+            }, RECONCILE_INTERVAL_MS);
+            this.reconcileTimer.unref?.();
+        }
+    }
+    /**
+     * Register an adapter for a provider. Idempotent — re-registering
+     * replaces the prior adapter and clears its tracked state (e.g.
+     * supervisor restart hands us a new adapter instance for the same
+     * provider name). Default budget is `DEFAULT_BUDGET_BYTES`; Phase 3
+     * sets a per-engine RAM-aware default via `setBudget`.
+     */
+    registerAdapter(adapter) {
+        // If the operator already set a budget for this provider before
+        // any adapter showed up (boot-time config), honor it now. Otherwise
+        // start at the conservative default.
+        const initialBudget = this.pendingBudgets.get(adapter.providerName) ?? DEFAULT_BUDGET_BYTES;
+        this.pendingBudgets.delete(adapter.providerName);
+        this.providers.set(adapter.providerName, {
+            adapter,
+            budgetBytes: initialBudget,
+            entries: new Map(),
+            hits: 0,
+            misses: 0,
+            hitsBySource: emptyHitsBySource(),
+            gezelOutcomes: new Map(),
+            recentOutcomes: [],
+        });
+    }
+    /**
+     * Set the memory budget for a provider. Operator-tunable; Phase 3
+     * exposes this in the Settings UI. Eviction kicks in immediately if
+     * the new budget is below current usage. Calls before the provider's
+     * adapter has registered are remembered and applied on registration
+     * — this lets `service.ts` set boot-time budgets from config without
+     * coordinating with lazy provider construction.
+     */
+    setBudget(providerName, bytes) {
+        const state = this.providers.get(providerName);
+        if (!state) {
+            this.pendingBudgets.set(providerName, bytes);
+            return;
+        }
+        state.budgetBytes = bytes;
+        void this.enforceBudget(providerName);
+    }
+    /**
+     * Record that a turn just ran for this session. Updates LRU position
+     * and tracks token count for byte accounting. Called from
+     * `ChatManager.runSend` after `sendAndWait` returns.
+     *
+     * `approxPromptTokens` is the controller's best estimate of cache
+     * size — derived from `prompt_chars / 4`. The next `reportUsage()`
+     * poll reconciles against the engine's actual numbers.
+     *
+     * `wasHit` is whether the engine reused cache from a prior turn.
+     * Adapters know this from their own bookkeeping (llama-server `/slots`
+     * cache_tokens vs prefill cost; MLX wrapper's `[cache] hit` log).
+     * Default `false` — recording a turn without a hit signal counts as a
+     * miss for hit-rate purposes.
+     *
+     * `hitSource` (optional, #6) attributes the hit to a layer — `memory`/
+     * `disk`/`prefix` all count as a hit (and override `wasHit`); `fresh`
+     * is a miss. When supplied it drives the per-source / per-gezel
+     * telemetry the layered-prefix A/B reads.
+     */
+    recordTurn(args) {
+        const state = this.providers.get(args.providerName);
+        if (!state)
+            return;
+        const existing = state.entries.get(args.sessionId);
+        const bytes = args.approxPromptTokens * ESTIMATED_BYTES_PER_TOKEN;
+        const entry = {
+            sessionId: args.sessionId,
+            ...(args.gezelId ? { gezelId: args.gezelId } : {}),
+            tokenCount: args.approxPromptTokens,
+            bytes,
+            lastUsedAt: this.now(),
+            evictionPriority: existing?.evictionPriority ?? 'normal',
+        };
+        state.entries.set(args.sessionId, entry);
+        // hitSource subsumes wasHit when provided: anything but `fresh` is a hit.
+        const wasHit = args.hitSource ? args.hitSource !== 'fresh' : (args.wasHit ?? false);
+        if (args.hitSource)
+            state.hitsBySource[args.hitSource]++;
+        if (wasHit) {
+            state.hits++;
+            state.recentOutcomes.push(true);
+        }
+        else {
+            state.misses++;
+            state.recentOutcomes.push(false);
+        }
+        if (args.gezelId) {
+            const g = state.gezelOutcomes.get(args.gezelId) ?? { hits: 0, misses: 0 };
+            if (wasHit)
+                g.hits++;
+            else
+                g.misses++;
+            state.gezelOutcomes.set(args.gezelId, g);
+        }
+        if (state.recentOutcomes.length > RECENT_TURN_WINDOW) {
+            state.recentOutcomes.shift();
+        }
+        void this.enforceBudget(args.providerName);
+    }
+    /**
+     * Invalidate a single session across all providers. Fired by
+     * ChatManager on `compactInFlight` (cache no longer matches the
+     * compacted message list), `archiveSession`, and `deleteSession`.
+     * Best-effort — adapter eviction is async and may be ignored by
+     * engines that have already lost the cache.
+     */
+    invalidate(sessionId) {
+        for (const [name, state] of this.providers) {
+            if (!state.entries.has(sessionId))
+                continue;
+            state.entries.delete(sessionId);
+            void state.adapter.evict([sessionId]).catch((err) => {
+                this.logger.warn?.(`[cache] evict failed for ${name}/${sessionId}: ${err}`);
+            });
+        }
+    }
+    /**
+     * Drop everything for one provider, in the engine too. For an operator's
+     * explicit clear and for a provider being removed for good.
+     */
+    invalidateProvider(providerName) {
+        const sessionIds = this.forgetProvider(providerName);
+        const state = this.providers.get(providerName);
+        if (state && sessionIds.length > 0) {
+            void state.adapter.evict(sessionIds).catch((err) => {
+                this.logger.warn?.(`[cache] provider-wide evict failed for ${providerName}: ${err}`);
+            });
+        }
+    }
+    /**
+     * Drop this controller's bookkeeping for one provider WITHOUT touching the
+     * engine's caches. Fired on `resetClient`: the provider object is rebuilt,
+     * so our view of it is stale, but the engine's entries are not — they are
+     * keyed by session and checked token by token on reuse, so a stale one
+     * costs nothing while a deleted one costs a full re-prefill. Evicting them
+     * here wiped, from memory AND disk, the cache a session was about to resume
+     * from whenever the engine outlived the reset (a live settings change, an
+     * external or machine-wide engine across a daemon restart).
+     */
+    forgetProvider(providerName) {
+        const state = this.providers.get(providerName);
+        if (!state)
+            return [];
+        const sessionIds = Array.from(state.entries.keys());
+        state.entries.clear();
+        state.recentOutcomes = [];
+        state.hits = 0;
+        state.misses = 0;
+        state.hitsBySource = emptyHitsBySource();
+        state.gezelOutcomes.clear();
+        return sessionIds;
+    }
+    /**
+     * Pin a session against eviction (Phase 4 — voorman / starred
+     * sessions). `low` priority means "evict me last"; `normal` is the
+     * default LRU treatment.
+     */
+    pin(sessionId, priority) {
+        for (const state of this.providers.values()) {
+            const entry = state.entries.get(sessionId);
+            if (entry)
+                entry.evictionPriority = priority;
+        }
+    }
+    /**
+     * Snapshot for telemetry. Cheap — pure read of in-memory state.
+     * Empty `sessions` array when the provider has no warm cache.
+     */
+    getStats(providerName) {
+        const state = this.providers.get(providerName);
+        if (!state)
+            return null;
+        let totalBytes = 0;
+        const sessions = [];
+        for (const entry of state.entries.values()) {
+            totalBytes += entry.bytes;
+            sessions.push({
+                sessionId: entry.sessionId,
+                ...(entry.gezelId ? { gezelId: entry.gezelId } : {}),
+                tokenCount: entry.tokenCount,
+                bytes: entry.bytes,
+                lastUsedAt: entry.lastUsedAt,
+                evictionPriority: entry.evictionPriority,
+            });
+        }
+        const recentHits = state.recentOutcomes.filter((x) => x).length;
+        const recentHitRate = state.recentOutcomes.length === 0 ? 0 : recentHits / state.recentOutcomes.length;
+        const gezels = Array.from(state.gezelOutcomes.entries()).map(([gezelId, o]) => ({ gezelId, hits: o.hits, misses: o.misses }));
+        return {
+            providerName,
+            totalBytes,
+            budgetBytes: state.budgetBytes,
+            warmSessionCount: state.entries.size,
+            hits: state.hits,
+            misses: state.misses,
+            recentHitRate,
+            hitsBySource: { ...state.hitsBySource },
+            gezels,
+            sessions,
+        };
+    }
+    /** All providers' stats in one shot. */
+    getAllStats() {
+        return Array.from(this.providers.keys())
+            .map((n) => this.getStats(n))
+            .filter((s) => s !== null);
+    }
+    /**
+     * Reconcile the controller's tracked usage against what the engine
+     * actually has cached. Replaces the controller's view with the
+     * adapter's truth — engines know their cache size more precisely than
+     * the controller's per-token estimate. Called on a timer; tests can
+     * call manually via `reconcileProvider`.
+     */
+    async reconcileProvider(providerName) {
+        const state = this.providers.get(providerName);
+        if (!state)
+            return;
+        let usage;
+        try {
+            usage = await state.adapter.reportUsage();
+        }
+        catch (err) {
+            this.logger.warn?.(`[cache] reportUsage failed for ${providerName}: ${err}`);
+            return;
+        }
+        // Replace the entries map — adapter's view wins. We preserve
+        // eviction priority across the rebuild (operator pinning shouldn't
+        // be lost on reconcile).
+        const oldPriorities = new Map();
+        for (const [id, entry] of state.entries) {
+            oldPriorities.set(id, entry.evictionPriority);
+        }
+        state.entries.clear();
+        for (const u of usage) {
+            state.entries.set(u.sessionId, {
+                sessionId: u.sessionId,
+                tokenCount: u.tokenCount,
+                bytes: u.estBytes,
+                lastUsedAt: u.lastUsedAt,
+                evictionPriority: oldPriorities.get(u.sessionId) ?? 'normal',
+            });
+        }
+        await this.enforceBudget(providerName);
+    }
+    /** Stop the background reconcile timer. Call on shutdown. */
+    stop() {
+        if (this.reconcileTimer)
+            clearInterval(this.reconcileTimer);
+        this.reconcileTimer = undefined;
+    }
+    /**
+     * Watermark-based LRU eviction. When usage ≥ budget, evict
+     * least-recently-used entries (skipping `low`-priority pins until
+     * nothing else remains) until usage drops below 80% of budget. The
+     * gap prevents thrashing when usage hovers right at the budget
+     * boundary.
+     */
+    async enforceBudget(providerName) {
+        const state = this.providers.get(providerName);
+        if (!state)
+            return;
+        let totalBytes = 0;
+        for (const entry of state.entries.values())
+            totalBytes += entry.bytes;
+        if (totalBytes < state.budgetBytes * EVICTION_HIGH_WATERMARK)
+            return;
+        const target = state.budgetBytes * EVICTION_LOW_WATERMARK;
+        // Sort: normal-priority first (oldest first), then low-priority
+        // (oldest first). We only touch the low-priority bucket if normal
+        // is exhausted and we're still over the watermark.
+        const candidates = Array.from(state.entries.values()).sort((a, b) => {
+            if (a.evictionPriority !== b.evictionPriority) {
+                return a.evictionPriority === 'low' ? 1 : -1;
+            }
+            return a.lastUsedAt - b.lastUsedAt;
+        });
+        const toEvict = [];
+        for (const entry of candidates) {
+            if (totalBytes < target)
+                break;
+            toEvict.push(entry.sessionId);
+            totalBytes -= entry.bytes;
+        }
+        if (toEvict.length === 0)
+            return;
+        for (const id of toEvict)
+            state.entries.delete(id);
+        this.logger.info?.(`[cache] evicted ${toEvict.length} session(s) from ${providerName} ` +
+            `(now ${(totalBytes / (1024 * 1024)).toFixed(1)} MB / ${(state.budgetBytes / (1024 * 1024)).toFixed(0)} MB)`);
+        try {
+            await state.adapter.evict(toEvict);
+        }
+        catch (err) {
+            this.logger.warn?.(`[cache] adapter.evict failed for ${providerName}: ${err}`);
+        }
+    }
+    async reconcileAll() {
+        for (const name of this.providers.keys()) {
+            await this.reconcileProvider(name);
+        }
+    }
+}

package/dist/catalog/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bendyline
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/catalog/chat-models/bt/btl4-35b-q4/manifest.json

@@ -0,0 +1,146 @@
+{
+  "schemaVersion": 1,
+  "kind": "chat-model",
+  "id": "btl4-35b-q4",
+  "name": "BTL-4 (35B-A3B, Q4)",
+  "description": "Bad Theory Labs' agentic build of Ornith 1.0 35B, tuned for tool-heavy coding and long stretches of unattended work. Native reasoning, native tool calling, and a 256K context window that stays light on memory even when it's full. Capable-tier on-device option for machines with 32GB+ memory — see BTL-4 Compact for a smaller build.",
+  "tags": [
+    "badtheorylabs",
+    "agentic",
+    "tools",
+    "code",
+    "reasoning",
+    "long-context",
+    "moe"
+  ],
+  "category": "coding",
+  "maintainer": {
+    "name": "Bad Theory Labs",
+    "url": "https://huggingface.co/badtheorylabs"
+  },
+  "version": "1.0.1",
+  "updatedAt": "2026-08-15T00:00:00Z",
+  "license": "Apache-2.0",
+  "licenseClass": "open",
+  "licenseShortName": "Apache 2.0",
+  "recoScore": 10,
+  "parameterSize": "35B",
+  "approxSizeBytes": 21391448192,
+  "supportsTools": true,
+  "contextWindow": 262144,
+  "kvBytesPerTokenF16": 20480,
+  "kvFixedBytesF16": 64389120,
+  "upstream": "https://huggingface.co/badtheorylabs/BTL-4",
+  "style": {
+    "family": "qwen",
+    "reasoningFormat": "think",
+    "toolCallFormat": "function-call"
+  },
+  "behaviors": [
+    "reasoning.strip-think-tags",
+    "prompt.private-reasoning-guidance",
+    "prompt.tool-cookbook-condensed",
+    "fabrication.detect-past-tense-no-tools",
+    "turn.ollama-num-predict-bumped",
+    "turn.preamble-folding",
+    "mcp.compact-tool-schemas"
+  ],
+  "tuning": {
+    "sampling": {
+      "temperature": 0.6,
+      "topP": 0.95,
+      "topK": 20,
+      "minP": 0,
+      "maxTokens": 16384
+    },
+    "samplingWhenThinking": {
+      "temperature": 0.6,
+      "topP": 0.95
+    },
+    "reasoning": {
+      "enableThinking": true,
+      "thinkingBudget": 8192
+    },
+    "promptTags": {
+      "enableThinkingTag": "/think",
+      "disableThinkingTag": "/no_think"
+    },
+    "profiles": {
+      "thinking-general": {
+        "sampling": {
+          "temperature": 0.6,
+          "topP": 0.95,
+          "topK": 20,
+          "minP": 0,
+          "repetitionPenalty": 1,
+          "maxTokens": 24576
+        },
+        "reasoning": {
+          "enableThinking": true,
+          "thinkingBudget": 8192
+        }
+      },
+      "thinking-coding": {
+        "sampling": {
+          "temperature": 0.6,
+          "topP": 0.95,
+          "topK": 20,
+          "minP": 0,
+          "repetitionPenalty": 1,
+          "maxTokens": 32768
+        },
+        "reasoning": {
+          "enableThinking": true,
+          "thinkingBudget": 12288
+        }
+      },
+      "thinking-precise": {
+        "sampling": {
+          "temperature": 0.4,
+          "topP": 0.9,
+          "topK": 20,
+          "minP": 0,
+          "seed": 0,
+          "maxTokens": 6144
+        },
+        "reasoning": {
+          "enableThinking": true,
+          "thinkingBudget": 4096
+        }
+      },
+      "instruct": {
+        "sampling": {
+          "temperature": 0.7,
+          "topP": 0.8,
+          "topK": 20,
+          "minP": 0,
+          "presencePenalty": 1.5,
+          "repetitionPenalty": 1,
+          "maxTokens": 4096
+        },
+        "reasoning": {
+          "enableThinking": false
+        }
+      },
+      "creative": {
+        "sampling": {
+          "temperature": 1.1,
+          "topP": 0.95,
+          "topK": 40,
+          "maxTokens": 8192
+        },
+        "reasoning": {
+          "enableThinking": false
+        }
+      }
+    }
+  },
+  "llamaCpp": {
+    "huggingfaceRepo": "bartowski/badtheorylabs_BTL-4-GGUF",
+    "revision": "799d011b498e3287f514eb463ea04d6624ec945f",
+    "filename": "badtheorylabs_BTL-4-Q4_K_M.gguf",
+    "sha256": "75ce22e4f603bdf3f94c2b32151bb6d3492d2ab8a34ad8a4bfa2eaef0125c50f",
+    "approxSizeBytes": 21391448192,
+    "quantization": "Q4_K_M"
+  }
+}

package/dist/catalog/chat-models/bt/btl4-35b-q4/versions/1.0.0/manifest.json

@@ -0,0 +1,15 @@
+{
+  "schemaVersion": 1,
+  "version": "1.0.0",
+  "releasedAt": "2026-08-07T00:00:00Z",
+  "approxSizeBytes": 21391448192,
+  "llamaCpp": {
+    "huggingfaceRepo": "bartowski/badtheorylabs_BTL-4-GGUF",
+    "revision": "799d011b498e3287f514eb463ea04d6624ec945f",
+    "filename": "badtheorylabs_BTL-4-Q4_K_M.gguf",
+    "sha256": "75ce22e4f603bdf3f94c2b32151bb6d3492d2ab8a34ad8a4bfa2eaef0125c50f",
+    "approxSizeBytes": 21391448192,
+    "quantization": "Q4_K_M",
+    "residentBytes": 25669737830
+  }
+}

package/dist/catalog/chat-models/bt/btl4-35b-q4/versions/1.0.1/manifest.json

@@ -0,0 +1,15 @@
+{
+  "schemaVersion": 1,
+  "version": "1.0.1",
+  "releasedAt": "2026-08-15T00:00:00Z",
+  "approxSizeBytes": 21391448192,
+  "llamaCpp": {
+    "huggingfaceRepo": "bartowski/badtheorylabs_BTL-4-GGUF",
+    "revision": "799d011b498e3287f514eb463ea04d6624ec945f",
+    "filename": "badtheorylabs_BTL-4-Q4_K_M.gguf",
+    "sha256": "75ce22e4f603bdf3f94c2b32151bb6d3492d2ab8a34ad8a4bfa2eaef0125c50f",
+    "approxSizeBytes": 21391448192,
+    "quantization": "Q4_K_M"
+  },
+  "minGezelVersion": "1.26227"
+}