@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/providers/native/local-engine-runtime.d.ts

@@ -0,0 +1,412 @@
+import { type GezelConfig } from '@bendyline/gezel';
+import type { CatalogService } from '@bendyline/gezel-catalog';
+import type { MlxRuntimeStatusBus } from '../../python/mlx-runtime-status-bus.js';
+import { type LLMProvider, type ProviderName } from '../types.js';
+import { type CapacityCommitted } from './capacity-broker.js';
+import { type LocalProviderName } from './engine-key.js';
+import type { EngineWaitOpts } from './provider-pool.js';
+/** How a bind waits when its model must replace a busy one; see {@link EngineWaitOpts}. */
+export interface LocalEngineBindOpts {
+    engineDrainWaitMs?: number;
+    signal?: AbortSignal;
+    onDrainWait?: EngineWaitOpts['onDrainWait'];
+}
+export interface EngineConfigStore {
+    readonly homePath: string;
+    readConfig(): Promise<GezelConfig>;
+    writeConfig(config: Partial<Record<keyof GezelConfig, unknown>>): Promise<GezelConfig>;
+}
+export interface LocalEngineRuntimeOptions {
+    home: string;
+    store: EngineConfigStore;
+    catalog: CatalogService;
+    llamaCppModels?: import('../llama-cpp/index.js').LlamaCppModelManager;
+    ds4Models?: import('../llama-cpp/index.js').LlamaCppModelManager;
+    mlxModels?: import('../mlx/index.js').MlxModelManager;
+    uvRuntime?: import('../../python/uv-runtime.js').UvRuntime;
+    mlxRuntimeStatus?: MlxRuntimeStatusBus;
+    cacheController?: import('../../cache/controller.js').SessionCacheController;
+    gpuArbiter?: import('../gpu-arbiter.js').GpuArbiter;
+    engineBinaries?: import('../../engines/registry.js').EngineBinaryRegistry;
+    engineRouter?: import('./engine-router.js').EngineRouter;
+}
+/** Native engine lifetime, admission, and cache control. No product sessions or tools. */
+export declare class LocalEngineRuntime {
+    protected readonly store: EngineConfigStore;
+    constructor(opts: LocalEngineRuntimeOptions);
+    /** Product chat may prefill its own sessions; the engine runtime never resolves them. */
+    protected prefillCacheSession(_sessionId: string): Promise<boolean>;
+    /** Cached provider instances keyed by provider name. */
+    protected readonly providers: Map<"copilot" | "openai" | "anthropic" | "anthropic-cli" | "codex-cli" | "ollama" | "llama-cpp" | "mlx" | "ds4" | "apple-foundation-models" | "android-mlkit" | "remote", LLMProvider>;
+    protected readonly catalog: CatalogService;
+    protected readonly llamaCppModels?: import('../llama-cpp/index.js').LlamaCppModelManager;
+    protected readonly ds4Models?: import('../llama-cpp/index.js').LlamaCppModelManager;
+    protected readonly mlxModels?: import('../mlx/index.js').MlxModelManager;
+    protected readonly uvRuntime?: import('../../python/uv-runtime.js').UvRuntime;
+    protected readonly mlxRuntimeStatus?: MlxRuntimeStatusBus;
+    protected readonly home: string;
+    protected readonly cacheController?: import('../../cache/controller.js').SessionCacheController;
+    protected readonly gpuArbiter?: import('../gpu-arbiter.js').GpuArbiter;
+    protected readonly engineBinaries?: import('../../engines/registry.js').EngineBinaryRegistry;
+    readonly engineRouter?: import('./engine-router.js').EngineRouter;
+    /**
+     * Snapshot of every provider's prompt-cache state. Empty array when
+     * no controller is wired (cloud-only install, tests). Polled by the
+     * EngineStatusPill alongside `/api/queues` for the popover stats
+     * rows. Cheap — pure read of in-memory state.
+     */
+    getCacheStats(): import('../../cache/controller.js').ProviderCacheStats[];
+    /**
+     * Operator-driven invalidation hook. The /api/cache/evict route
+     * calls this to drop a single session's cache; same path the
+     * compaction / archive / delete hooks already use internally.
+     */
+    invalidateSessionCache(sessionId: string): void;
+    /**
+     * Operator-driven full-provider eviction. Same hook the resetClient
+     * path uses; surfaced on /api/cache/clear so an operator can punt
+     * stale state without waiting for credential rotation.
+     */
+    invalidateProviderCache(providerName: string): void;
+    /**
+     * Live-update the controller's per-provider budget. Called from the
+     * config PUT handler when the operator changes `cacheBudgetMb` —
+     * eviction kicks in immediately if the new budget is below current
+     * usage. No-op if no controller is wired.
+     */
+    setCacheBudget(providerName: string, bytes: number): void;
+    /**
+     * Live-update the capacity broker's total budget for resident local
+     * engines. Called from the config PUT handler when the operator moves the
+     * memory slider (`localEngineMemoryGb`); `null` reverts to the host's
+     * auto-derived value.
+     *
+     * No-op when no router exists yet — {@link buildEngineRouter} reads the
+     * config itself, so a router built later already picks the new value up.
+     * The in-flight case is the one that needs care: a build that read the
+     * config *before* this write would otherwise install the stale budget and
+     * keep it until shutdown, so chain onto the pending promise rather than
+     * only checking the resolved cache.
+     */
+    setLocalEngineMemoryBudget(bytes: number | null): Promise<void>;
+    /**
+     * Live-update whether co-resident local models may spill into system RAM.
+     * `null` reverts to the host's auto choice. Same in-flight care as
+     * {@link setLocalEngineMemoryBudget}, and the same non-eviction contract:
+     * turning spillover off doesn't unload anything, it changes what the next
+     * spawn is allowed to add.
+     */
+    setAllowRamSpillover(allow: boolean | null): Promise<void>;
+    /** Return the provider if it has already been initialized; never creates one. */
+    getProviderIfReady(name: ProviderName): LLMProvider | null;
+    /**
+     * Lazily construct the multi-engine router using the engine runtime’s
+     * already-resolved deps (store, catalog, llamaCppModels, ds4Models, mlxModels,
+     * uvRuntime). Once built, the router owns the {@link ProviderPool}
+     * and {@link CapacityBroker}; subsequent calls return the cached
+     * instance.
+     *
+     * Returns `null` when {@link LocalEngineRuntimeOptions.engineRouter} was
+     * explicitly injected — that path is owned by the caller (typically
+     * tests). The lazy build is only for production wiring through
+     * `service.ts`.
+     */
+    protected engineRouterCache: import('./engine-router.js').EngineRouter | null;
+    protected engineRouterInitPromise: Promise<import('./engine-router.js').EngineRouter | null> | null;
+    /**
+     * Shut down the production-owned lazy engine router, including a router
+     * whose construction is still in flight. Pooled local providers do not live
+     * in {@link providers}, so omitting this step lets their native children
+     * outlive gezeld and become PPID-1 orphans.
+     *
+     * An explicitly injected {@link engineRouter} remains caller-owned (mostly
+     * a test seam) and is deliberately left alone.
+     */
+    protected shutdownOwnedEngineRouter(): Promise<void>;
+    /**
+     * Free the memory a live reset can free without severing anything: evict
+     * every resident engine that is idle right now and leave the rest to the
+     * pool's ordinary lifecycle. The router itself stays — dropping it while an
+     * engine is still serving a turn would orphan that process outside the
+     * broker's accounting, which is the reason the old path force-killed
+     * everything instead.
+     *
+     * An explicitly injected {@link engineRouter} remains caller-owned and is
+     * left alone, same rule as {@link shutdownOwnedEngineRouter}. A router
+     * whose construction is still in flight owns no resident engines yet, so
+     * this deliberately does not await it.
+     */
+    protected releaseIdleOwnedEngines(): Promise<void>;
+    /**
+     * Resolve (or build) the router. Returns `null` only when no local
+     * provider is wired up at all (cloud-only installs); production
+     * installs with `llamaCppModels`, `ds4Models`, or `mlxModels` set always get a
+     * router. Callers MUST tolerate a `null` return — the legacy
+     * singleton path takes over.
+     */
+    protected getEngineRouter(): Promise<import('./engine-router.js').EngineRouter | null>;
+    protected buildEngineRouter(): Promise<import('./engine-router.js').EngineRouter | null>;
+    /**
+     * Per-replica provider initialization + cache-controller wiring.
+     * Mirrors the singleton-path post-construction work inside
+     * {@link ensureProvider}, factored out so the pool's builder
+     * closures can share it. Cache-controller registration is gated
+     * on replica index — only replica 0 registers, because the
+     * controller is keyed by provider name and N adapters would
+     * collide.
+     */
+    protected initLocalProvider(name: LocalProviderName, provider: LLMProvider, config: GezelConfig): Promise<void>;
+    /**
+     * Construct the engine's `EngineCacheAdapter`, register it with the
+     * shared `cacheController`, and attach it to the provider so every
+     * subsequent send carries cache extras. Idempotent at the controller
+     * level — `registerAdapter` overwrites by `providerName`, so calling
+     * this for two replicas of the same provider just keeps the
+     * most-recently-registered adapter in the controller's tracking.
+     * Each provider still gets its own per-replica adapter via
+     * `setCacheAdapter`. Returns early when no controller is wired
+     * (test paths) or the provider isn't a local engine.
+     */
+    protected wireLocalProviderCacheAdapter(name: LocalProviderName, provider: LLMProvider, config: GezelConfig): Promise<void>;
+    /**
+     * Cache of catalog `residentBytes` per `(provider:modelId)`.
+     * Populated synchronously by `prefetchResidentBytes` so the
+     * router's sync resolver works. Falls back to the broker estimator
+     * on miss.
+     */
+    protected readonly resolveResidentBytesCache: Map<string, number>;
+    /**
+     * Drop cached reservations so the next admission re-reads the catalog and
+     * config. ds4 prices its footprint at the launch context, so a per-model
+     * context override changes what the broker must reserve — leaving the old
+     * entry cached would keep billing the pre-override number for the life of
+     * the process. Call from any route that mutates context sizing.
+     */
+    invalidateResidentBytesCache(provider?: LocalProviderName, modelId?: string): void;
+    /** Extra fixed reservation for the exact ds4 vision encoder this launch uses. */
+    private resolveDs4VisionResidentBytes;
+    protected resolveResidentBytes(provider: LocalProviderName, modelId: string): Promise<number>;
+    /**
+     * Reconcile the pool's resident set to match a target clone-count
+     * map. Surfaces the Settings → Local Models picker's intent to the
+     * runtime: spawn missing replicas (up to capacity) and evict
+     * excess. No-op when no router is configured.
+     */
+    reconcileEnginePool(provider: LocalProviderName, target: Record<string, number>): Promise<void>;
+    /**
+     * Unload one already-resident local model replica without constructing the
+     * engine router as a side effect. The pool rejects the request if the model
+     * became busy after the UI's last lifecycle sample.
+     */
+    unloadIdleEngine(provider: LocalProviderName, modelId: string, replicaIdx: number): Promise<boolean>;
+    /**
+     * Snapshot of the live pool — committed bytes, budget, per-key
+     * resident set. Surfaced via `GET /api/engines/status`. Returns
+     * `null` for installs without a pool wired.
+     */
+    engineStatus(): Promise<import('./provider-pool.js').PoolSnapshot | null>;
+    /**
+     * Read the already-live local-engine pool without constructing the router.
+     * Hot status endpoints use this path so opening a telemetry popover never
+     * initializes local inference as a side effect.
+     */
+    peekEngineStatus(): import('./provider-pool.js').PoolSnapshot | null;
+    /**
+     * Live provider-queue summaries for pool-resident local engines, keyed
+     * by provider name — the data `/api/queues` needs to reflect pooled
+     * work the singleton {@link getProviderIfReady} path can't see (chat
+     * turns AND background one-shots). See
+     * {@link ProviderPool.queueSummaries} for why the blind spot existed.
+     *
+     * Synchronous, non-building peek: reads the already-resolved router and
+     * returns an empty map when none exists yet. A poll before any local
+     * work has run has nothing resident to report anyway — and a status
+     * endpoint must never spin the router up as a side effect.
+     */
+    localEngineQueueSummaries(): Map<LocalProviderName, import('./provider-pool.js').PooledQueueSummary>;
+    /**
+     * Launch provenance of every live local engine — the granted context
+     * window, slots, and KV dtype each engine ACTUALLY started with, from
+     * the supervisors' retained launch payloads. Non-building and cheap:
+     * walks resident providers only, never starts an engine. Feeds
+     * `/api/system/diagnostics` `localEngines` (Settings → About), so a
+     * "model is looping" report carries the grant without log spelunking.
+     */
+    localEngineLaunchSummaries(): Array<{
+        provider: LocalProviderName;
+        modelId?: string;
+        snapshot: import('../types.js').EngineLaunchSnapshot;
+    }>;
+    /**
+     * Non-building lookup for an already-created native provider. Unlike
+     * `getProviderForModel`, this cannot allocate a replica, evict another model,
+     * or start an engine. The caller still checks the provider's live base URL
+     * because a retained provider object may currently have no resident process.
+     */
+    peekResidentLocalProviders(name: ProviderName, modelId?: string): LLMProvider[];
+    /**
+     * Resolve the context window a native model would receive without binding a
+     * pool replica. This is the broker-side half of remote admission: focusing a
+     * chat needs the live memory clamp so the user daemon can size its prompt,
+     * but it must not load the model or evict somebody else's resident engine.
+     *
+     * `standalone` prices the target as the eventual sole resident: inventory
+     * callers use it for stable device fitness, while remote admission combines
+     * it with `liveSystemPressure` because a competing Gezel engine is
+     * evictable/queueable but memory held by other applications is not. See
+     * {@link previewLocalEnginePlan}'s `standalone` note.
+     */
+    previewContextWindowForModel(name: LocalProviderName, modelId: string, opts?: {
+        standalone?: boolean;
+        liveSystemPressure?: boolean;
+    }): Promise<number | undefined>;
+    /**
+     * Reservations held by models OTHER than the one being previewed.
+     *
+     * `committed()` totals every replica, the previewed model's own included.
+     * Feeding that back as `committedOtherBytes` prices the launch as if a
+     * second copy had to load beside the resident one, so the model currently
+     * serving chats reports "won't fit" against its own reservation — and on a
+     * host where one big model fills most of the budget, every OTHER row is
+     * denied by a reservation that eviction would release. Every consumer of
+     * `committedOtherBytes` means co-resident models, so subtract our own
+     * replicas here. A key the parser doesn't recognise stays counted: an
+     * unattributable reservation is real memory, and over-counting only makes
+     * the preview conservative.
+     */
+    protected committedOtherBytesFor(snapshot: CapacityCommitted | undefined, provider: LocalProviderName, modelId: string): number;
+    /**
+     * The memory picture every engine's launch preview prices against.
+     *
+     * One derivation because each engine branch below used to rebuild it, and
+     * they drifted: the llama.cpp branch pinned its admission clamp to usable
+     * VRAM alone on discrete hosts, so a 21 GB MoE the broker would gladly
+     * admit against a 30 GB budget reported "won't fit" — while the MLX branch,
+     * which never passed live RAM, priced the same question correctly. A 4.7 GB
+     * model was denied the same way. Adding an engine must not re-open that.
+     *
+     * Policy previews deliberately carry no `freeSystemRamBytes`: live free RAM
+     * is self-referential there — a warm engine can depress it and make every
+     * inventory row, itself included, fluctuate. Imminent placement callers may
+     * opt into `liveSystemPressure`; that is the broker `/admit` path and must
+     * use the same reclaimable-aware clamp as provider construction. Confining
+     * grown KV to fast memory is not lost for policy previews —
+     * `planAdaptiveContextGrowth` applies that clamp itself from `budgetKind` +
+     * `vramBytes`.
+     */
+    protected previewCapacityInputs(provider: LocalProviderName, modelId: string, opts: {
+        standalone?: boolean;
+        liveSystemPressure?: boolean;
+    }): Promise<{
+        /** Admission budget for the whole resident set. */
+        budgetBytes: number;
+        /** Fast (on-accelerator) pool — the ceiling adaptive growth may spend. */
+        fastBudgetBytes: number;
+        /** What slot COUNT is sized against; below `fastBudgetBytes` on big hosts. */
+        concurrencySizingBytes: number;
+        /** Usable VRAM on a discrete card; 0 on unified / CPU-only hosts. */
+        vramBytes: number;
+        budgetKind: CapacityCommitted['pools']['kind'];
+        /** Reservations held by models OTHER than this one. */
+        committedOtherBytes: number;
+        /** Reclaimable-aware RAM available for an imminent placement. */
+        freeSystemRamBytes?: number;
+    }>;
+    /**
+     * Full non-binding launch preview: the context window a native model
+     * would receive AND the resident footprint at that window. Powers the
+     * models-list "size in memory" column alongside
+     * {@link previewContextWindowForModel}.
+     *
+     * Two footprints, deliberately: `plannedResidentBytes` is weights plus
+     * ONE slot's KV — what serving a single chat costs, and the only figure
+     * that tracks measured peak RSS. `reservedResidentBytes` is weights plus
+     * `plannedSlots` slots' KV — what the broker actually holds. Quoting the
+     * fleet as "size in memory" reads as the cost of using the model at all,
+     * which overstates a multi-slot host by the slot count.
+     */
+    previewLocalEnginePlan(name: LocalProviderName, modelId: string, 
+    /**
+     * ds4 only: price a catalog entry that is NOT downloaded yet. The ds4 plan
+     * needs the catalog block and this device's RAM tier, never the GGUF, so
+     * the browse list can quote the window and footprint a download would land
+     * on — which is when the fit decision is actually made. Other engines keep
+     * throwing {@link ModelNotInstalledError}: their plans read the real
+     * header.
+     */
+    opts?: {
+        allowUninstalled?: boolean;
+        /**
+         * Price the model as the eventual only resident engine. Inventory rows
+         * use this for stable device fitness. Remote admission also uses it
+         * because another Gezel engine can be evicted once idle; paired with
+         * `liveSystemPressure`, non-Gezel system load still constrains the plan.
+         * Actual launch admission keeps default live-reservation behavior and
+         * may evict an idle model (or report that a busy one is blocking the
+         * swap).
+         */
+        standalone?: boolean;
+        /**
+         * Apply the same reclaimable-aware live RAM clamp as a real provider
+         * launch. Remote admission uses this immediately before inference;
+         * inventory and settings previews deliberately leave it off so a warm
+         * model does not make every catalog row fluctuate with system pressure.
+         */
+        liveSystemPressure?: boolean;
+    }): Promise<{
+        contextWindow?: number;
+        plannedResidentBytes?: number;
+        reservedResidentBytes?: number;
+        plannedSlots?: number;
+        /** GGUF/model-config advertised native window — the context slider's max. */
+        nativeContextWindow?: number;
+        /** Applied per-model context override (config.modelContextOverrides). */
+        overrideContextTokens?: number;
+        /**
+         * What automatic sizing would grant right now, computed only while an
+         * override is active (otherwise `contextWindow` IS the automatic value).
+         * The slider's "Auto" marker.
+         */
+        autoContextWindow?: number;
+        /**
+         * Post-quant single-slot KV linearization so the UI can price
+         * "~X GB in memory" live while the slider drags:
+         * `weightsResidentBytes + kvFixedBytesPerSlot + kvBytesPerTokenPerSlot × ctx`.
+         * ds4 reports it too, from its catalog-authored slope; absent there only
+         * for entries that have not been measured yet.
+         */
+        kvBytesPerTokenPerSlot?: number;
+        kvFixedBytesPerSlot?: number;
+        weightsResidentBytes?: number;
+        /**
+         * ds4 only: the slider's max — min(native window, catalog maxLaunchCtx).
+         * ds4 has no ctx-vs-memory admission, so the authored launch ceiling is
+         * the only guard against a window the engine cannot actually serve.
+         */
+        contextCeilingTokens?: number;
+    }>;
+    /**
+     * Cancel a pending provider-queue entry by (provider, id). Resolves
+     * the queue the SAME two ways {@link localEngineQueueSummaries} +
+     * `/api/queues` surface it: the singleton `providers` map first
+     * (cloud providers, seeded test mocks), then the engine pool for
+     * pool-routed local engines whose queue the singleton never sees.
+     * Returns true when an entry was removed, false when the id is
+     * unknown (already running, already cancelled, or no such provider).
+     */
+    cancelProviderQueueItem(name: ProviderName, id: number): boolean;
+    /** Reorder a pending provider-queue entry. Same singleton-then-pool
+     *  resolution as {@link cancelProviderQueueItem}. */
+    moveProviderQueueItem(name: ProviderName, id: number, direction: 'up' | 'down'): boolean;
+    protected sessionsPerEngineKey(): Map<string, number>;
+    protected bindLocalReplica(router: import('./engine-router.js').EngineRouter, name: LocalProviderName, modelId: string, opts: {
+        sessionId: string;
+        priorEngineKey?: string | undefined;
+    } & LocalEngineBindOpts): Promise<{
+        engineKey: string;
+        provider: LLMProvider;
+    }>;
+    protected getLocalProviderForModel(name: LocalProviderName, modelId: string | undefined, opts: {
+        engineDrainWaitMs?: number;
+    }, fallback: () => Promise<LLMProvider>): Promise<LLMProvider>;
+}