@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/native/engine-key.js

@@ -0,0 +1,41 @@
+/**
+ * EngineKey — the string handle that identifies a local-engine
+ * instance in the ProviderPool. Composite: `${provider}:${modelId}:${replicaIdx}`.
+ *
+ * Cloud providers (copilot, openai, anthropic, anthropic-cli,
+ * codex-cli, ollama, mock) are NOT pool-managed — they keep the
+ * bare-provider-name keying ChatManager's `cloudProviders` map uses.
+ */
+export const LOCAL_PROVIDERS = ['llama-cpp', 'mlx', 'ds4'];
+export function isLocalProvider(name) {
+    return LOCAL_PROVIDERS.includes(name);
+}
+export function makeEngineKey(provider, modelId, replicaIdx) {
+    if (replicaIdx < 0 || !Number.isInteger(replicaIdx)) {
+        throw new Error(`invalid replicaIdx: ${replicaIdx}`);
+    }
+    if (!modelId || modelId.includes(':')) {
+        throw new Error(`invalid modelId for engine key: ${JSON.stringify(modelId)}`);
+    }
+    return `${provider}:${modelId}:${replicaIdx}`;
+}
+export function parseEngineKey(key) {
+    // Split from the right so a modelId without `:` is unambiguous.
+    // (We already reject modelIds containing `:` in makeEngineKey.)
+    const lastColon = key.lastIndexOf(':');
+    if (lastColon < 0)
+        return null;
+    const replicaStr = key.slice(lastColon + 1);
+    const replicaIdx = Number.parseInt(replicaStr, 10);
+    if (!Number.isInteger(replicaIdx) || String(replicaIdx) !== replicaStr)
+        return null;
+    const head = key.slice(0, lastColon);
+    const firstColon = head.indexOf(':');
+    if (firstColon < 0)
+        return null;
+    const provider = head.slice(0, firstColon);
+    const modelId = head.slice(firstColon + 1);
+    if (!isLocalProvider(provider) || !modelId)
+        return null;
+    return { provider, modelId, replicaIdx };
+}

package/dist/providers/native/engine-router.d.ts

@@ -0,0 +1,145 @@
+/**
+ * EngineRouter — the glue between {@link ProviderPool} +
+ * {@link CapacityBroker} and the chat-layer concepts (sessions,
+ * gezels, modelIds). It answers two questions:
+ *
+ *   1. "What replica should this session run on?" — `bindForSession`
+ *      picks (and remembers) a replica index per session so KV cache
+ *      lives in one process for the session's lifetime.
+ *   2. "Give me the live LLMProvider for this engine key." — `ensure`
+ *      hands the pool a `(provider, modelId, replicaIdx, residentBytes)`
+ *      tuple and gets back the provider.
+ *
+ * The router does NOT decide which modelId a session uses — that's
+ * resolved upstream by the chat layer from gezel frontmatter +
+ * config defaults. The router takes the resolved modelId as input.
+ *
+ * Live consumers: ChatManager's `ensureProviderForSession` (internal
+ * `/api` chat turns, session-sticky via persisted `engineKey`) and
+ * `getProviderForModel` (`/v1` + Ollama-compat one-shots, record-free).
+ */
+import type { LLMProvider } from '../types.js';
+import { CapacityBroker } from './capacity-broker.js';
+import { type LocalProviderName } from './engine-key.js';
+import type { EngineWaitOpts, PoolSnapshot, ProviderBuilder, ProviderPool } from './provider-pool.js';
+/**
+ * Resolves the catalog `residentBytes` for a model. The chat layer
+ * supplies this with knowledge of the installed catalog; the router
+ * is provider-agnostic so it doesn't import the catalog directly.
+ * Returning `undefined` triggers the broker's
+ * `estimateResidentBytes` fallback.
+ */
+export type ResidentBytesResolver = (provider: LocalProviderName, modelId: string) => number | undefined;
+export interface EngineRouterOptions {
+    broker: CapacityBroker;
+    pool: ProviderPool;
+    /**
+     * Per-provider builders. The chat layer wraps
+     * `buildLlamaCppProvider` / `buildMlxProvider` here, threading
+     * through the `modelId` and `replicaIdx` so the underlying
+     * factories can produce per-replica isolation (unique port,
+     * isolated slot-save-path, etc.).
+     */
+    builders: Partial<Record<LocalProviderName, ProviderBuilder>>;
+    /**
+     * Catalog-driven residentBytes lookup. Falls back to
+     * {@link CapacityBroker.estimateResidentBytes} when the lookup
+     * returns undefined.
+     */
+    resolveResidentBytes: ResidentBytesResolver;
+    /**
+     * Fallback per-engine approx size in bytes used by the broker
+     * estimator when the catalog has no entry at all. Lets the router
+     * be useful in tests / dev installs before any models are pulled.
+     * Defaults to 8 GB which is a defensible middle-of-the-road
+     * working-set for an unknown 4-bit quant.
+     */
+    fallbackApproxBytes?: number;
+}
+export interface BindResult {
+    /** The engine key the session is bound to (now persisted). */
+    engineKey: string;
+    /** Live provider instance for the bound replica. */
+    provider: LLMProvider;
+}
+export interface BindContext extends EngineWaitOpts {
+    /** The session's id, used by the chat layer to weight `pickReplicaForBind`. */
+    sessionId: string;
+    /** Active session count per existing engine key — drives load balancing. */
+    sessionsPerKey?: ReadonlyMap<string, number>;
+}
+export declare class EngineRouter {
+    readonly broker: CapacityBroker;
+    readonly pool: ProviderPool;
+    private readonly builders;
+    private readonly resolveResidentBytes;
+    private readonly fallbackApproxBytes;
+    constructor(opts: EngineRouterOptions);
+    /**
+     * Compute the working-set bytes the broker should reserve for
+     * `(provider, modelId)`. Catalog lookup wins; missing entry →
+     * fallback via {@link CapacityBroker.estimateResidentBytes}.
+     */
+    resolveBytes(provider: LocalProviderName, modelId: string): number;
+    /**
+     * Bind a session to a specific replica. Idempotent — if a prior
+     * `engineKey` was supplied AND the replica still exists, returns
+     * the same key and ensures it. Otherwise picks the least-loaded
+     * replica of `(provider, modelId)` and returns its key. Caller
+     * is responsible for persisting the returned `engineKey` onto the
+     * session record.
+     */
+    bindForSession(provider: LocalProviderName, modelId: string, ctx: BindContext, priorEngineKey?: string): Promise<BindResult>;
+    /** Whether the least-loaded pick would land on another resident, non-draining replica. */
+    private hasResidentSibling;
+    /**
+     * Get-or-build the provider for `(provider, modelId, replicaIdx)`.
+     * Reserves the catalog `residentBytes` and triggers LRU eviction
+     * inside the pool when budget is tight.
+     */
+    ensure(provider: LocalProviderName, modelId: string, replicaIdx: number, opts?: EngineWaitOpts): Promise<LLMProvider>;
+    /**
+     * Resolve a persisted engine key back to a live provider, only if
+     * the replica is currently resident. Used by the chat layer to
+     * fast-path subsequent turns without re-running the bind logic.
+     * Returns null when the key is absent from the pool (caller falls
+     * back to a fresh bind).
+     */
+    ensureExisting(engineKey: string, opts?: EngineWaitOpts): Promise<LLMProvider | null>;
+    /**
+     * Reconcile the resident set to match a target clone-count map.
+     * `target` is keyed by modelId; values are the desired clone count
+     * (>= 0; 0 means "evict all resident replicas of this model").
+     * Provider is fixed to the caller's chosen engine.
+     */
+    reconcileClones(provider: LocalProviderName, target: Record<string, number>): Promise<void>;
+    /**
+     * Unload every resident replica that is not mid-turn. Returns the keys left
+     * resident because they were busy. Used by a live config reset, which needs
+     * the memory back but must not sever work in flight.
+     */
+    releaseIdle(): Promise<string[]>;
+    /** Unload one resident replica immediately, provided it is still idle. */
+    unloadIdle(provider: LocalProviderName, modelId: string, replicaIdx: number): Promise<boolean>;
+    snapshot(): PoolSnapshot;
+    /** Best-effort shutdown — evicts every entry. */
+    shutdown(): Promise<void>;
+    /** Gracefully drain every admitted turn, including background one-shots,
+     * and permanently prevent this user daemon from spawning another engine. */
+    retire(): Promise<void>;
+}
+/**
+ * Helper consumed by the chat layer to derive a provider+modelId
+ * from a session record / gezel frontmatter / config defaults. Pure
+ * function; kept on the router module for colocation with the rest
+ * of the routing logic.
+ */
+export declare function resolveLocalRoute(args: {
+    providerName: string;
+    frontmatterModel?: string;
+    recordModel?: string;
+    configDefaultModel?: string;
+}): {
+    provider: LocalProviderName;
+    modelId: string;
+} | null;

package/dist/providers/native/engine-router.js

@@ -0,0 +1,191 @@
+/**
+ * EngineRouter — the glue between {@link ProviderPool} +
+ * {@link CapacityBroker} and the chat-layer concepts (sessions,
+ * gezels, modelIds). It answers two questions:
+ *
+ *   1. "What replica should this session run on?" — `bindForSession`
+ *      picks (and remembers) a replica index per session so KV cache
+ *      lives in one process for the session's lifetime.
+ *   2. "Give me the live LLMProvider for this engine key." — `ensure`
+ *      hands the pool a `(provider, modelId, replicaIdx, residentBytes)`
+ *      tuple and gets back the provider.
+ *
+ * The router does NOT decide which modelId a session uses — that's
+ * resolved upstream by the chat layer from gezel frontmatter +
+ * config defaults. The router takes the resolved modelId as input.
+ *
+ * Live consumers: ChatManager's `ensureProviderForSession` (internal
+ * `/api` chat turns, session-sticky via persisted `engineKey`) and
+ * `getProviderForModel` (`/v1` + Ollama-compat one-shots, record-free).
+ */
+import { createLogger } from '@bendyline/gezel';
+import { AbortedWhileQueuedError } from '../queue.js';
+import { CapacityBroker, isEngineBusyError } from './capacity-broker.js';
+import { isLocalProvider, makeEngineKey, parseEngineKey, } from './engine-key.js';
+const log = createLogger('engine-router');
+function waitOptsOf(ctx) {
+    return {
+        ...(ctx.drainWaitMs !== undefined ? { drainWaitMs: ctx.drainWaitMs } : {}),
+        ...(ctx.signal ? { signal: ctx.signal } : {}),
+        ...(ctx.onDrainWait ? { onDrainWait: ctx.onDrainWait } : {}),
+    };
+}
+const DEFAULT_FALLBACK_APPROX_BYTES = 8 * 1024 ** 3;
+export class EngineRouter {
+    broker;
+    pool;
+    builders;
+    resolveResidentBytes;
+    fallbackApproxBytes;
+    constructor(opts) {
+        this.broker = opts.broker;
+        this.pool = opts.pool;
+        this.builders = opts.builders;
+        this.resolveResidentBytes = opts.resolveResidentBytes;
+        this.fallbackApproxBytes = opts.fallbackApproxBytes ?? DEFAULT_FALLBACK_APPROX_BYTES;
+    }
+    /**
+     * Compute the working-set bytes the broker should reserve for
+     * `(provider, modelId)`. Catalog lookup wins; missing entry →
+     * fallback via {@link CapacityBroker.estimateResidentBytes}.
+     */
+    resolveBytes(provider, modelId) {
+        const hit = this.resolveResidentBytes(provider, modelId);
+        if (hit !== undefined && hit > 0)
+            return hit;
+        return CapacityBroker.estimateResidentBytes(provider, this.fallbackApproxBytes);
+    }
+    /**
+     * Bind a session to a specific replica. Idempotent — if a prior
+     * `engineKey` was supplied AND the replica still exists, returns
+     * the same key and ensures it. Otherwise picks the least-loaded
+     * replica of `(provider, modelId)` and returns its key. Caller
+     * is responsible for persisting the returned `engineKey` onto the
+     * session record.
+     */
+    async bindForSession(provider, modelId, ctx, priorEngineKey) {
+        const waitOpts = waitOptsOf(ctx);
+        if (priorEngineKey) {
+            const parsed = parseEngineKey(priorEngineKey);
+            if (parsed && parsed.provider === provider && parsed.modelId === modelId) {
+                if (this.pool.has(priorEngineKey)) {
+                    this.pool.touch(priorEngineKey);
+                    const p = await this.ensureExisting(priorEngineKey, waitOpts);
+                    if (p)
+                        return { engineKey: priorEngineKey, provider: p };
+                }
+                // Prior key was valid but the replica is gone (LRU-evicted,
+                // clones reduced, etc.). Re-spawn at the same index if budget
+                // allows; otherwise fall through to least-loaded pick — at once
+                // when a sibling replica is resident, rather than waiting for room.
+                const fits = this.broker.canReserve(this.resolveBytes(provider, modelId));
+                if (fits || !this.hasResidentSibling(provider, modelId, parsed.replicaIdx, ctx)) {
+                    try {
+                        const provider = await this.ensure(parsed.provider, parsed.modelId, parsed.replicaIdx, waitOpts);
+                        return { engineKey: priorEngineKey, provider };
+                    }
+                    catch (err) {
+                        // With no resident sibling, the least-loaded pick needs the same
+                        // busy engine's memory and would only repeat the wait — and a
+                        // stopped caller is done.
+                        if (isEngineBusyError(err) || err instanceof AbortedWhileQueuedError)
+                            throw err;
+                        log.warn(`bindForSession: prior key ${priorEngineKey} unavailable (${err instanceof Error ? err.message : String(err)}); rebinding to least-loaded replica`);
+                    }
+                }
+            }
+        }
+        const replicaIdx = this.pool.pickReplicaForBind(provider, modelId, ctx.sessionsPerKey ?? new Map());
+        const engineKey = makeEngineKey(provider, modelId, replicaIdx);
+        const instance = await this.ensure(provider, modelId, replicaIdx, waitOpts);
+        log.info(`bind ${ctx.sessionId} → ${engineKey}`);
+        return { engineKey, provider: instance };
+    }
+    /** Whether the least-loaded pick would land on another resident, non-draining replica. */
+    hasResidentSibling(provider, modelId, replicaIdx, ctx) {
+        const idx = this.pool.pickReplicaForBind(provider, modelId, ctx.sessionsPerKey ?? new Map());
+        const key = makeEngineKey(provider, modelId, idx);
+        return idx !== replicaIdx && this.pool.has(key) && !this.pool.isDraining(key);
+    }
+    /**
+     * Get-or-build the provider for `(provider, modelId, replicaIdx)`.
+     * Reserves the catalog `residentBytes` and triggers LRU eviction
+     * inside the pool when budget is tight.
+     */
+    async ensure(provider, modelId, replicaIdx, opts = {}) {
+        if (!this.builders[provider]) {
+            throw new Error(`EngineRouter: no builder registered for provider '${provider}'`);
+        }
+        const bytes = this.resolveBytes(provider, modelId);
+        return this.pool.ensure(provider, modelId, replicaIdx, bytes, opts);
+    }
+    /**
+     * Resolve a persisted engine key back to a live provider, only if
+     * the replica is currently resident. Used by the chat layer to
+     * fast-path subsequent turns without re-running the bind logic.
+     * Returns null when the key is absent from the pool (caller falls
+     * back to a fresh bind).
+     */
+    async ensureExisting(engineKey, opts = {}) {
+        const parsed = parseEngineKey(engineKey);
+        if (!parsed)
+            return null;
+        if (!this.pool.has(engineKey))
+            return null;
+        // ensure on an existing entry is a fast O(1) hit — unless it is draining
+        // for a swap, when it waits on `opts` and may rebuild.
+        return this.pool.ensure(parsed.provider, parsed.modelId, parsed.replicaIdx, this.resolveBytes(parsed.provider, parsed.modelId), opts);
+    }
+    /**
+     * Reconcile the resident set to match a target clone-count map.
+     * `target` is keyed by modelId; values are the desired clone count
+     * (>= 0; 0 means "evict all resident replicas of this model").
+     * Provider is fixed to the caller's chosen engine.
+     */
+    async reconcileClones(provider, target) {
+        const items = Object.entries(target).map(([modelId, clones]) => ({
+            provider,
+            modelId,
+            clones,
+        }));
+        await this.pool.reconcile(items, (p, m) => this.resolveBytes(p, m));
+    }
+    /**
+     * Unload every resident replica that is not mid-turn. Returns the keys left
+     * resident because they were busy. Used by a live config reset, which needs
+     * the memory back but must not sever work in flight.
+     */
+    async releaseIdle() {
+        return this.pool.releaseIdle();
+    }
+    /** Unload one resident replica immediately, provided it is still idle. */
+    async unloadIdle(provider, modelId, replicaIdx) {
+        return this.pool.unloadIdle(makeEngineKey(provider, modelId, replicaIdx));
+    }
+    snapshot() {
+        return this.pool.snapshot();
+    }
+    /** Best-effort shutdown — evicts every entry. */
+    async shutdown() {
+        await this.pool.shutdown();
+    }
+    /** Gracefully drain every admitted turn, including background one-shots,
+     * and permanently prevent this user daemon from spawning another engine. */
+    async retire() {
+        await this.pool.retire();
+    }
+}
+/**
+ * Helper consumed by the chat layer to derive a provider+modelId
+ * from a session record / gezel frontmatter / config defaults. Pure
+ * function; kept on the router module for colocation with the rest
+ * of the routing logic.
+ */
+export function resolveLocalRoute(args) {
+    if (!isLocalProvider(args.providerName))
+        return null;
+    const modelId = args.recordModel ?? args.frontmatterModel ?? args.configDefaultModel;
+    if (!modelId)
+        return null;
+    return { provider: args.providerName, modelId };
+}

package/dist/providers/native/gpu-panic-guard.d.ts

@@ -0,0 +1,54 @@
+/**
+ * GpuPanicGuard — refuse to spawn a fresh local GPU engine right after a
+ * macOS GPU-driver kernel panic, so gezel doesn't re-trigger the crash loop.
+ *
+ * Background: heavy Metal use on Apple Silicon can trip a latent Apple GPU
+ * driver bug (`IOGPUMemory.cpp` "completeMemory() prepare count underflow").
+ * It is NOT OOM and NOT gezel code — a userspace process can't panic the
+ * kernel — but our model load/unload + KV churn triggers it, and once the
+ * machine reboots, autostart / scheduled jobs / a retried chat would happily
+ * spawn the same workload again and crash it a second time. No telemetry
+ * threshold catches a driver refcount underflow; the only defense is to stop
+ * re-triggering.
+ *
+ * macOS writes a `.panic` report on the boot after a panic
+ * ({@link findRecentGpuPanics}). When one is recent, this guard denies new
+ * local-engine spawns (mlx + llama.cpp + ds4 — all Metal on a Mac) with a
+ * clear, actionable message, for a cooldown window. The user can override
+ * (`GEZEL_GPU_PANIC_GUARD=off`) to use local models at their own risk, or
+ * update macOS (Apple's fix). No-op off macOS and for non-GPU providers.
+ *
+ * Wired at the single spawn chokepoint — {@link ProviderPool.buildEntry} —
+ * so interactive chat AND autonomous jobs are both covered.
+ */
+import { type GpuPanicRecord, findRecentGpuPanics } from '@bendyline/gezel/native';
+import type { LocalProviderName } from './engine-key.js';
+export interface GpuPanicDecision {
+    blocked: boolean;
+    reason?: string;
+    panic?: GpuPanicRecord;
+}
+/** Minimal shape the pool depends on — keeps ProviderPool decoupled + testable. */
+export interface GpuSpawnGuard {
+    check(engine: LocalProviderName): GpuPanicDecision;
+}
+export interface GpuPanicGuardOptions {
+    /** Force on/off. Default: on for macOS unless `GEZEL_GPU_PANIC_GUARD=off`. */
+    enabled?: boolean;
+    /** Block window after the most recent panic. Default 1h (env override:
+     * `GEZEL_GPU_PANIC_COOLDOWN_HOURS`). */
+    cooldownMs?: number;
+    /** Test seams. */
+    now?: () => number;
+    find?: typeof findRecentGpuPanics;
+}
+export declare class GpuPanicGuard implements GpuSpawnGuard {
+    private readonly enabled;
+    private readonly cooldownMs;
+    private readonly now;
+    private readonly find;
+    constructor(opts?: GpuPanicGuardOptions);
+    check(engine: LocalProviderName): GpuPanicDecision;
+}
+/** User-facing denial: what happened, why it's blocked, how to proceed. */
+export declare function formatSpawnDenial(panic: GpuPanicRecord, cooldownMs: number): string;

package/dist/providers/native/gpu-panic-guard.js

@@ -0,0 +1,59 @@
+/**
+ * GpuPanicGuard — refuse to spawn a fresh local GPU engine right after a
+ * macOS GPU-driver kernel panic, so gezel doesn't re-trigger the crash loop.
+ *
+ * Background: heavy Metal use on Apple Silicon can trip a latent Apple GPU
+ * driver bug (`IOGPUMemory.cpp` "completeMemory() prepare count underflow").
+ * It is NOT OOM and NOT gezel code — a userspace process can't panic the
+ * kernel — but our model load/unload + KV churn triggers it, and once the
+ * machine reboots, autostart / scheduled jobs / a retried chat would happily
+ * spawn the same workload again and crash it a second time. No telemetry
+ * threshold catches a driver refcount underflow; the only defense is to stop
+ * re-triggering.
+ *
+ * macOS writes a `.panic` report on the boot after a panic
+ * ({@link findRecentGpuPanics}). When one is recent, this guard denies new
+ * local-engine spawns (mlx + llama.cpp + ds4 — all Metal on a Mac) with a
+ * clear, actionable message, for a cooldown window. The user can override
+ * (`GEZEL_GPU_PANIC_GUARD=off`) to use local models at their own risk, or
+ * update macOS (Apple's fix). No-op off macOS and for non-GPU providers.
+ *
+ * Wired at the single spawn chokepoint — {@link ProviderPool.buildEntry} —
+ * so interactive chat AND autonomous jobs are both covered.
+ */
+import { createLogger } from '@bendyline/gezel';
+import { findRecentGpuPanics } from '@bendyline/gezel/native';
+const log = createLogger('gpu-panic');
+const DEFAULT_COOLDOWN_MS = 60 * 60 * 1000;
+export class GpuPanicGuard {
+    enabled;
+    cooldownMs;
+    now;
+    find;
+    constructor(opts = {}) {
+        const envOff = process.env.GEZEL_GPU_PANIC_GUARD === 'off';
+        this.enabled = opts.enabled ?? (process.platform === 'darwin' && !envOff);
+        const envHours = Number.parseFloat(process.env.GEZEL_GPU_PANIC_COOLDOWN_HOURS ?? '');
+        this.cooldownMs =
+            opts.cooldownMs ??
+                (Number.isFinite(envHours) && envHours > 0 ? envHours * 60 * 60 * 1000 : DEFAULT_COOLDOWN_MS);
+        this.now = opts.now ?? Date.now;
+        this.find = opts.find ?? findRecentGpuPanics;
+    }
+    check(engine) {
+        if (!this.enabled)
+            return { blocked: false };
+        const panics = this.find({ withinMs: this.cooldownMs, now: this.now() });
+        const panic = panics[0];
+        if (!panic)
+            return { blocked: false };
+        const reason = formatSpawnDenial(panic, this.cooldownMs);
+        log.warn(`[gpu-panic] denying ${engine} engine spawn — recent GPU kernel panic (${panic.signature}, ${panic.when.toISOString()})`);
+        return { blocked: true, reason, panic };
+    }
+}
+/** User-facing denial: what happened, why it's blocked, how to proceed. */
+export function formatSpawnDenial(panic, cooldownMs) {
+    const hours = Math.max(1, Math.round(cooldownMs / (60 * 60 * 1000)));
+    return `Local on-device models are paused because macOS recorded a GPU kernel panic at ${panic.when.toISOString()} (${panic.signature}). Heavy local-model use can trigger this kind of macOS GPU-driver panic, so loading another model now risks crashing and restarting your Mac again. Gezel pauses new local engines for ${hours}h after a panic; installing the latest macOS update is the best protection. Cloud providers are unaffected. (Advanced: setting GEZEL_GPU_PANIC_GUARD=off skips this pause at your own risk.)`;
+}

package/dist/providers/native/inference.d.ts

@@ -0,0 +1,33 @@
+import type { ModelInfo } from '@bendyline/gezel';
+import type { LLMProvider, ProviderName } from '../types.js';
+import { LocalEngineRuntime } from './local-engine-runtime.js';
+/** The HTTP inference surface has no product chat/session or tool methods. */
+export interface NativeInference extends Pick<LocalEngineRuntime, keyof LocalEngineRuntime> {
+    getProviderForModel(name: ProviderName, modelId?: string, opts?: {
+        engineDrainWaitMs?: number;
+    }): Promise<LLMProvider>;
+    listModelsForProvider(name: ProviderName, signal?: AbortSignal): Promise<ModelInfo[]>;
+    resetClient(opts?: {
+        deferBusy?: boolean;
+    }): Promise<void>;
+}
+/** Machine-only consumer of the same admission, provider pool and cache owner as product chat. */
+export declare class EngineInference extends LocalEngineRuntime implements NativeInference {
+    private closing;
+    beginShutdown(): void;
+    private readonly pending;
+    getProviderForModel(name: ProviderName, modelId?: string, opts?: {
+        engineDrainWaitMs?: number;
+    }): Promise<LLMProvider>;
+    listModelsForProvider(name: ProviderName): Promise<ModelInfo[]>;
+    resetClient(opts?: {
+        deferBusy?: boolean;
+    }): Promise<void>;
+    shutdown(): Promise<void>;
+}
+export declare function installedModelInfos(models: Array<{
+    id: string;
+    name: string;
+    approxSizeBytes: number;
+    contextWindow?: number;
+}>): ModelInfo[];

package/dist/providers/native/inference.js

@@ -0,0 +1,49 @@
+import { isLocalProvider } from './engine-key.js';
+import { LocalEngineRuntime } from './local-engine-runtime.js';
+/** Machine-only consumer of the same admission, provider pool and cache owner as product chat. */
+export class EngineInference extends LocalEngineRuntime {
+    closing = false;
+    beginShutdown() {
+        this.closing = true;
+    }
+    pending = new Set();
+    getProviderForModel(name, modelId, opts = {}) {
+        if (!isLocalProvider(name))
+            return Promise.reject(new Error(`The machine engine cannot serve provider ${name}`));
+        if (this.closing)
+            return Promise.reject(new Error('Engine inference is shutting down'));
+        const request = this.getLocalProviderForModel(name, modelId, opts, async () => {
+            throw new Error(`No installed ${name} model is selected`);
+        });
+        this.pending.add(request);
+        void request.then(() => this.pending.delete(request), () => this.pending.delete(request));
+        return request;
+    }
+    async listModelsForProvider(name) {
+        if (!isLocalProvider(name))
+            return [];
+        const manager = name === 'mlx' ? this.mlxModels : name === 'ds4' ? this.ds4Models : this.llamaCppModels;
+        return installedModelInfos((await manager?.listInstalled()) ?? []);
+    }
+    async resetClient(opts) {
+        await this.releaseIdleOwnedEngines();
+    }
+    async shutdown() {
+        this.closing = true;
+        await Promise.allSettled([...this.pending]);
+        await this.shutdownOwnedEngineRouter();
+    }
+}
+export function installedModelInfos(models) {
+    return models.map((model) => {
+        const sizeGb = model.approxSizeBytes / 1024 ** 3;
+        const sizeLabel = sizeGb >= 0.1 ? ` · ${sizeGb.toFixed(1)} GB` : '';
+        const ctxLabel = model.contextWindow ? ` · ${Math.round(model.contextWindow / 1024)}k ctx` : '';
+        return {
+            id: model.id,
+            name: `${model.name}${sizeLabel}${ctxLabel}`,
+            supportsTools: true,
+            ...(model.contextWindow ? { contextWindow: model.contextWindow } : {}),
+        };
+    });
+}