@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bendyline
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,14 @@
-# Temporary Holding Version
+# @bendyline/gezel-inference
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+The shared on-device runtime used by Gezel's full service and by embedded applications. It owns native engine lifecycle and capacity admission, model preparation, OpenAI-compatible chat, knowledge retrieval, worker entry points, and native release verification. The full service supplies product state, tools, cloud providers, UI and background work. There is one implementation of the shared runtime.
+
+Use the application SDK with `host: { mode: 'in-process', inferenceOnly: true }`. Install this package alongside `@bendyline/gezel-app-sdk`. Startup opens no HTTP listener and returns an authenticated Fetch transport serving `/v1/chat`, `/v1/responses`, `/v1/embeddings`, `/v1/models`, `/v1/models/ensure`, `/v1/knowledge` and the engine routes; `model: "gezel:embeddings"` runs the on-device text embedder. It does not download models. Model preparation requires an explicit request. This is the only implementation of the inference-only profile: a child-hosted `gezeld` serves the same host behind the service's loopback listener.
+
+- `/packaging` stages the pinned native release for an Electron host.
+- `/native-release` exposes the release pins without starting inference.
+- `/speech` exposes the shared Kokoro frontend and dictionary resolver.
+- `/internal/*` is the coordinated workspace interface for Gezel's full service; external applications should use the SDK.
+
+The build copies only chat-model and knowledge-catalog metadata from the exact Gilde pin owned by the catalog package. This is generated content, never a separately edited catalog. `@bendyline/gezel-catalog` loads it through its ordinary source API. The full service separately installs all of Gilde.
+
+Build with `pnpm --filter @bendyline/gezel-inference build` under the repository dependency lease. Unit and integration coverage for the moved runtime remains in the service suite; this package additionally tests the isolated host boundary.

package/dist/cache/adapter.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Engine-side contract for the {@link SessionCacheController}.
+ *
+ * The controller owns *policy* — which sessions deserve cache, when to
+ * evict, what the operator sees. The adapter owns *mechanism* — how a
+ * specific engine actually preserves and reuses prompt-cache state.
+ *
+ * Two adapters exist today:
+ *   - `LlamaCppCacheAdapter` — talks to llama-server's slot-based cache
+ *     via `cache_prompt: true` + `id_slot` in the request body, scrapes
+ *     `/slots` for memory stats.
+ *   - `MlxCacheAdapter` — talks to our wrapped `gezel_mlx_server.py`
+ *     via a `cache_id` request field + `/v1/cache/{stats,evict,warm}`
+ *     endpoints. Phase 1 ships a no-op stub here so Phase 2 can drop
+ *     in the real implementation without touching the controller.
+ *
+ * All methods are best-effort: an adapter that fails to honor an
+ * `evict()` doesn't break the controller, it just means the engine's
+ * cache is slightly larger than the controller thinks. The controller
+ * is the source of truth for policy decisions; the adapter is the
+ * source of truth for what's actually loaded in engine memory.
+ */
+export interface EngineCacheAdapter {
+    /**
+     * Per-provider name (e.g. `'mlx'`, `'llama-cpp'`) — keyed in the
+     * controller's adapter registry. Mirrors `LLMProvider.name`.
+     */
+    readonly providerName: string;
+    /**
+     * Build the request-body extras that will tell the engine "reuse the
+     * cache for this session." For llama-server: `{ cache_prompt: true,
+     * id_slot: <allocated> }`. For our MLX wrapper: `{ cache_id: <stable
+     * per-session id> }`. Returns an empty object for engines that don't
+     * support cache reuse.
+     *
+     * Called on every send by the provider's session implementation.
+     * Cheap — must not do I/O.
+     */
+    buildRequestExtras(sessionId: string): Record<string, unknown>;
+    /**
+     * Tell the engine to drop these sessions' caches now. Fired by the
+     * controller when LRU evicts, when the operator clicks "clear", or
+     * when a session is archived/deleted. Best-effort: a failed evict
+     * is logged and forgotten — the controller assumes the cache is
+     * gone for accounting purposes either way.
+     */
+    evict(sessionIds: readonly string[]): Promise<void>;
+    /**
+     * Snapshot of what the engine currently has cached. Polled by the
+     * controller every few seconds for telemetry; also informs the
+     * watermark-based eviction (controller compares its tracked usage
+     * against engine reality and reconciles).
+     *
+     * Returns an empty array when the engine doesn't support inspection
+     * or when no sessions are warm. `estBytes` is best-effort — engines
+     * estimate per-token cache size from model dimensions.
+     */
+    reportUsage(): Promise<readonly EngineCacheUsage[]>;
+    /**
+     * Phase 4: pre-warm a session by running prefill without generating.
+     * For llama-server: send a prefill-only request. For our MLX wrapper:
+     * `POST /v1/cache/warm`. Optional — engines that don't support it
+     * leave this unimplemented and the controller skips the warm path.
+     *
+     * `messages` is the prompt to warm with — usually the session's
+     * current message list as the controller sees it.
+     */
+    warm?(sessionId: string, messages: readonly CacheWarmMessage[]): Promise<void>;
+}
+/**
+ * Cumulative stable layers of the system prompt, used by layered prefix
+ * caching (flag `layeredPrefixCache`). Each field is a *cumulative*
+ * leading substring of the stable system message, so `gezel` is a true
+ * byte-prefix of `project`:
+ *
+ *   gezel   = the gezel-identity leading portion (header, about, traits,
+ *             lessons) — the bytes BEFORE the project-context band.
+ *             Shared across projects/sessions of the same gezel.
+ *   project = the full STABLE system message (identity + project +
+ *             discipline + tools-prose) that gets sent as `messages[0]`.
+ *             Shared across sessions of the same (model, gezel, project).
+ *
+ * Adapters hash each into a `prefix-gezel-<h>` / `prefix-gp-<h>` id and
+ * try them longest-first on a per-session cache miss, so a brand-new
+ * session inherits the warmest matching layer (gezel+project > gezel)
+ * instead of cold-prefilling.
+ *
+ * Note there is intentionally NO model-universal layer: that would
+ * require front-loading the discipline/cookbook prose, which the eval
+ * history (manager.ts buildInstructions, the Phase-2.4 ordering note)
+ * shows causes small-model "stuck-planning" regressions. The discipline
+ * and tool-prose bands stay in their proven late positions; the cache
+ * win comes from moving the VOLATILE band out of the system message so
+ * `project` (and the engine-templated tool schemas that follow it) form
+ * a reusable wire prefix.
+ */
+export interface SystemPromptLayers {
+    /** Gezel-identity leading prefix (before the project-context band). */
+    gezel: string;
+    /** Full stable system message (= the bytes sent as messages[0]). */
+    project: string;
+}
+export interface EngineCacheUsage {
+    sessionId: string;
+    /** Approximate token count cached for this session. */
+    tokenCount: number;
+    /** Best-effort byte estimate. Used for budget enforcement. */
+    estBytes: number;
+    /** Last time this entry was hit, in epoch ms. Drives LRU. */
+    lastUsedAt: number;
+}
+export interface CacheWarmMessage {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string | null;
+}

package/dist/cache/adapter.js

@@ -0,0 +1,23 @@
+/**
+ * Engine-side contract for the {@link SessionCacheController}.
+ *
+ * The controller owns *policy* — which sessions deserve cache, when to
+ * evict, what the operator sees. The adapter owns *mechanism* — how a
+ * specific engine actually preserves and reuses prompt-cache state.
+ *
+ * Two adapters exist today:
+ *   - `LlamaCppCacheAdapter` — talks to llama-server's slot-based cache
+ *     via `cache_prompt: true` + `id_slot` in the request body, scrapes
+ *     `/slots` for memory stats.
+ *   - `MlxCacheAdapter` — talks to our wrapped `gezel_mlx_server.py`
+ *     via a `cache_id` request field + `/v1/cache/{stats,evict,warm}`
+ *     endpoints. Phase 1 ships a no-op stub here so Phase 2 can drop
+ *     in the real implementation without touching the controller.
+ *
+ * All methods are best-effort: an adapter that fails to honor an
+ * `evict()` doesn't break the controller, it just means the engine's
+ * cache is slightly larger than the controller thinks. The controller
+ * is the source of truth for policy decisions; the adapter is the
+ * source of truth for what's actually loaded in engine memory.
+ */
+export {};

package/dist/cache/budget.d.ts

@@ -0,0 +1,15 @@
+/**
+ * RAM-aware default cache-budget heuristic.
+ *
+ * Operators tune via `config.cacheBudgetMb.{mlx,llamaCpp}`; when those
+ * are unset the controller falls back to a tier picked from system
+ * RAM. Tiered (not continuous) so the default is predictable per
+ * machine class — a 32 GB Mac always sees 4 GB, a 16 GB Mac always
+ * sees 2 GB, etc. — rather than some weird fractional number that
+ * shifts when other apps reserve memory.
+ *
+ * Conservative on purpose. The auto-default never bites unsupervised:
+ * users with unusual setups (e.g. a 16 GB Mac dedicated to one chat)
+ * tune up via Settings.
+ */
+export declare function defaultCacheBudgetMb(systemRamBytes: number): number;

package/dist/cache/budget.js

@@ -0,0 +1,31 @@
+/**
+ * RAM-aware default cache-budget heuristic.
+ *
+ * Operators tune via `config.cacheBudgetMb.{mlx,llamaCpp}`; when those
+ * are unset the controller falls back to a tier picked from system
+ * RAM. Tiered (not continuous) so the default is predictable per
+ * machine class — a 32 GB Mac always sees 4 GB, a 16 GB Mac always
+ * sees 2 GB, etc. — rather than some weird fractional number that
+ * shifts when other apps reserve memory.
+ *
+ * Conservative on purpose. The auto-default never bites unsupervised:
+ * users with unusual setups (e.g. a 16 GB Mac dedicated to one chat)
+ * tune up via Settings.
+ */
+const MB = 1024 * 1024;
+const GB = 1024 * MB;
+export function defaultCacheBudgetMb(systemRamBytes) {
+    // Sized so a typical 25K-token Meester session (~2.5 GB at our
+    // 100 KB/token estimate) fits comfortably without triggering
+    // eviction on the smallest supported Macs. Overbudget single
+    // sessions are kept anyway (in-engine "don't evict the only entry"
+    // rule), but the auto-tier should still be generous enough that
+    // multi-session users don't see surprise eviction either.
+    if (systemRamBytes < 16 * GB)
+        return 2048; // <16 GB → 2 GB (one warm session)
+    if (systemRamBytes < 32 * GB)
+        return 4096; // 16–32 GB → 4 GB (two warm sessions)
+    if (systemRamBytes < 64 * GB)
+        return 8192; // 32–64 GB → 8 GB (several)
+    return 16384; // ≥64 GB → 16 GB (many)
+}

package/dist/cache/controller.d.ts

@@ -0,0 +1,205 @@
+/**
+ * Engine-agnostic policy controller for prompt-cache reuse across local
+ * inference engines.
+ *
+ * Local engines (MLX via our wrapped server, llama.cpp via llama-server's
+ * slot system) preserve KV cache across requests so the per-turn prefill
+ * cost drops from O(full prompt) to O(new tokens). The mechanism differs
+ * per engine — see {@link EngineCacheAdapter}. This controller owns
+ * everything that's the same across both:
+ *
+ *   - Tracking which sessions are "warm" (have cache) per provider
+ *   - LRU eviction when total cache memory exceeds a configured budget
+ *   - Invalidation routing on compaction / archive / delete / reset
+ *   - Stats surface for telemetry (engine pill, queue meter, /api/cache)
+ *
+ * State model: a `Map<providerName, ProviderState>` where each provider
+ * holds its own `Map<sessionId, CacheEntry>`. Per-session entries record
+ * the bytes the controller *thinks* the engine has cached (an estimate
+ * the controller maintains based on `recordTurn`'s `approxPromptTokens`
+ * argument); periodic `adapter.reportUsage()` polls reconcile against
+ * engine reality.
+ *
+ * LRU is per-provider, not global — MLX and llama.cpp have independent
+ * memory budgets because they're independent processes.
+ */
+import type { EngineCacheAdapter } from './adapter.js';
+export interface SessionCacheStats {
+    sessionId: string;
+    gezelId?: string;
+    /** Approximate token count cached. */
+    tokenCount: number;
+    /** Approximate bytes used. */
+    bytes: number;
+    /** Epoch ms of the last turn that used this session. */
+    lastUsedAt: number;
+    /** Eviction priority — `low` is pinned (last to evict). */
+    evictionPriority: EvictionPriority;
+}
+export type EvictionPriority = 'low' | 'normal';
+/**
+ * Where a turn's prompt cache came from, reported by the engine adapter
+ * (MLX wrapper stdout `[cache] hit/disk-hit/prefix-hit/miss`; llama-server
+ * `/slots` reuse inference). `memory`/`disk`/`prefix` all count as a hit
+ * for hit-rate; `fresh` is a miss. Drives the `hitsBySource` telemetry
+ * that the layered-prefix A/B reads to attribute reuse to the right
+ * layer.
+ */
+export type HitSource = 'memory' | 'disk' | 'prefix' | 'fresh';
+export interface HitsBySource {
+    memory: number;
+    disk: number;
+    prefix: number;
+    fresh: number;
+}
+export interface GezelCacheStats {
+    gezelId: string;
+    hits: number;
+    misses: number;
+}
+export interface ProviderCacheStats {
+    providerName: string;
+    totalBytes: number;
+    budgetBytes: number;
+    /** All entries, including reusable `prefix-*` entries; legacy field name. */
+    warmSessionCount: number;
+    hits: number;
+    misses: number;
+    /** Cache hit rate over the last `RECENT_TURN_WINDOW` turns. */
+    recentHitRate: number;
+    /** Cumulative hit counts split by where the cache came from (#6). */
+    hitsBySource: HitsBySource;
+    /** Per-gezel hit/miss breakdown — surfaces which gezels' prefixes pay off. */
+    gezels: GezelCacheStats[];
+    sessions: SessionCacheStats[];
+}
+export interface SessionCacheControllerOptions {
+    /** Replaceable clock for tests. Defaults to `Date.now`. */
+    now?: () => number;
+    /** Replaceable logger. Defaults to `console`. */
+    logger?: {
+        info?: (m: string) => void;
+        warn?: (m: string) => void;
+        debug?: (m: string) => void;
+    };
+    /** Skip the periodic reconcile timer (tests drive reconciliation manually). */
+    disableReconcileTimer?: boolean;
+}
+export declare class SessionCacheController {
+    private readonly providers;
+    /**
+     * Pre-registration budget overrides. Operator config can set a
+     * budget at service boot — before any provider's adapter has been
+     * lazily registered. We stash the value here and apply it the
+     * moment the adapter shows up via `registerAdapter`. Without this,
+     * the boot-time `setBudget` would be a silent no-op and the engine
+     * would run with `DEFAULT_BUDGET_BYTES` until the first manual
+     * `setBudget` after construction.
+     */
+    private readonly pendingBudgets;
+    private readonly now;
+    private readonly logger;
+    private reconcileTimer?;
+    constructor(opts?: SessionCacheControllerOptions);
+    /**
+     * Register an adapter for a provider. Idempotent — re-registering
+     * replaces the prior adapter and clears its tracked state (e.g.
+     * supervisor restart hands us a new adapter instance for the same
+     * provider name). Default budget is `DEFAULT_BUDGET_BYTES`; Phase 3
+     * sets a per-engine RAM-aware default via `setBudget`.
+     */
+    registerAdapter(adapter: EngineCacheAdapter): void;
+    /**
+     * Set the memory budget for a provider. Operator-tunable; Phase 3
+     * exposes this in the Settings UI. Eviction kicks in immediately if
+     * the new budget is below current usage. Calls before the provider's
+     * adapter has registered are remembered and applied on registration
+     * — this lets `service.ts` set boot-time budgets from config without
+     * coordinating with lazy provider construction.
+     */
+    setBudget(providerName: string, bytes: number): void;
+    /**
+     * Record that a turn just ran for this session. Updates LRU position
+     * and tracks token count for byte accounting. Called from
+     * `ChatManager.runSend` after `sendAndWait` returns.
+     *
+     * `approxPromptTokens` is the controller's best estimate of cache
+     * size — derived from `prompt_chars / 4`. The next `reportUsage()`
+     * poll reconciles against the engine's actual numbers.
+     *
+     * `wasHit` is whether the engine reused cache from a prior turn.
+     * Adapters know this from their own bookkeeping (llama-server `/slots`
+     * cache_tokens vs prefill cost; MLX wrapper's `[cache] hit` log).
+     * Default `false` — recording a turn without a hit signal counts as a
+     * miss for hit-rate purposes.
+     *
+     * `hitSource` (optional, #6) attributes the hit to a layer — `memory`/
+     * `disk`/`prefix` all count as a hit (and override `wasHit`); `fresh`
+     * is a miss. When supplied it drives the per-source / per-gezel
+     * telemetry the layered-prefix A/B reads.
+     */
+    recordTurn(args: {
+        providerName: string;
+        sessionId: string;
+        gezelId?: string;
+        approxPromptTokens: number;
+        wasHit?: boolean;
+        hitSource?: HitSource;
+    }): void;
+    /**
+     * Invalidate a single session across all providers. Fired by
+     * ChatManager on `compactInFlight` (cache no longer matches the
+     * compacted message list), `archiveSession`, and `deleteSession`.
+     * Best-effort — adapter eviction is async and may be ignored by
+     * engines that have already lost the cache.
+     */
+    invalidate(sessionId: string): void;
+    /**
+     * Drop everything for one provider, in the engine too. For an operator's
+     * explicit clear and for a provider being removed for good.
+     */
+    invalidateProvider(providerName: string): void;
+    /**
+     * Drop this controller's bookkeeping for one provider WITHOUT touching the
+     * engine's caches. Fired on `resetClient`: the provider object is rebuilt,
+     * so our view of it is stale, but the engine's entries are not — they are
+     * keyed by session and checked token by token on reuse, so a stale one
+     * costs nothing while a deleted one costs a full re-prefill. Evicting them
+     * here wiped, from memory AND disk, the cache a session was about to resume
+     * from whenever the engine outlived the reset (a live settings change, an
+     * external or machine-wide engine across a daemon restart).
+     */
+    forgetProvider(providerName: string): string[];
+    /**
+     * Pin a session against eviction (Phase 4 — voorman / starred
+     * sessions). `low` priority means "evict me last"; `normal` is the
+     * default LRU treatment.
+     */
+    pin(sessionId: string, priority: EvictionPriority): void;
+    /**
+     * Snapshot for telemetry. Cheap — pure read of in-memory state.
+     * Empty `sessions` array when the provider has no warm cache.
+     */
+    getStats(providerName: string): ProviderCacheStats | null;
+    /** All providers' stats in one shot. */
+    getAllStats(): ProviderCacheStats[];
+    /**
+     * Reconcile the controller's tracked usage against what the engine
+     * actually has cached. Replaces the controller's view with the
+     * adapter's truth — engines know their cache size more precisely than
+     * the controller's per-token estimate. Called on a timer; tests can
+     * call manually via `reconcileProvider`.
+     */
+    reconcileProvider(providerName: string): Promise<void>;
+    /** Stop the background reconcile timer. Call on shutdown. */
+    stop(): void;
+    /**
+     * Watermark-based LRU eviction. When usage ≥ budget, evict
+     * least-recently-used entries (skipping `low`-priority pins until
+     * nothing else remains) until usage drops below 80% of budget. The
+     * gap prevents thrashing when usage hovers right at the budget
+     * boundary.
+     */
+    private enforceBudget;
+    private reconcileAll;
+}