@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/ffi-streaming-runner.d.ts

@@ -0,0 +1,146 @@
+/**
+ * In-process streaming-LLM runner.
+ *
+ * FFI streaming-LLM ABI declared in `ffi-streaming-llm.h`. The
+ * token-by-token loop hands `onTextChunk` accepted chunks and surfaces
+ * verifier events from native MTP.
+ *
+ * This file deliberately does not own the FFI context or the binding
+ * itself. It takes a narrow `LlmStreamingBinding` (see
+ * `services/llm-streaming-binding.ts`) + an opaque `LlmCtxHandle` as
+ * constructor arguments — that way it can be driven by libelizainference
+ * (via `wrapElizaInferenceFfi`) or any desktop libllama shim adapter without
+ * dragging in TTS/ASR surfaces. A single context can host concurrent generation
+ * sessions (one per pinned slot); the runner serialises with
+ * `slotInFlight`.
+ *
+ * Single-flight: lock map keyed by slot id, slot id `-1` unlocked. Two concurrent generates
+ * against the same pinned slot would interleave KV cache state, so the
+ * runner serializes them at the JS layer.
+ */
+import type { LlmCtxHandle, LlmStreamingBinding } from "./llm-streaming-binding";
+import type { LlmStreamHandle, LlmStreamStep } from "./voice/ffi-bindings";
+import type { VerifierStreamEvent } from "./voice/types";
+export interface FfiStreamingGenerateArgs {
+    /** Pre-tokenized prompt — the runner does not detokenize. */
+    promptTokens: Int32Array;
+    /** Pinned slot id; -1 disables pinning (any free slot). */
+    slotId: number;
+    /** Optional prompt cache key used to derive a slot when `slotId === -1`. */
+    cacheKey?: string;
+    maxTokens: number;
+    temperature: number;
+    topP: number;
+    topK: number;
+    repeatPenalty: number;
+    draftMin: number;
+    draftMax: number;
+    /** Reserved for separate draft-model speculation; null for Eliza-1 MTP. */
+    draftModelPath: string | null;
+    /**
+     * Per-load GPU offload (ABI v8). Forwarded into the native session config
+     * on `llmStreamOpen`. The fused libelizainference path loads the text model
+     * once per ctx, so the FIRST session's value wins; later sessions reuse the
+     * resident model. `undefined` selects the runtime default (all layers).
+     * The desktop libllama path already applies gpuLayers at `loadModel()`, so
+     * it ignores this field — it is load-time config, threaded here only so the
+     * fused runner can mirror the libllama load decision.
+     */
+    gpuLayers?: number;
+    /**
+     * KV-cache K/V quant type names (ABI v8), e.g. "qjl1_256" / "q4_polar".
+     * Same load-time semantics as `gpuLayers`: forwarded into the fused
+     * session config so the first `llmStreamOpen` applies the quantized cache.
+     */
+    cacheTypeK?: string | null;
+    cacheTypeV?: string | null;
+    /**
+     * Runtime context window in tokens (ABI v9). Forwarded into the fused
+     * session config on `llmStreamOpen`; `undefined` keeps the native
+     * ELIZA_LLM_N_CTX/default fallback.
+     */
+    contextSize?: number;
+    /**
+     * GBNF grammar source forcing the structured-reply envelope. Passed to
+     * the native session's `llmStreamOpen` config so sampling is
+     * grammar-constrained. `null` disables the constraint (free generation).
+     */
+    gbnfGrammar?: string | null;
+    /** Cancellation signal — fires `llmStreamCancel` on the active session. */
+    signal?: AbortSignal;
+    /**
+     * Per-step token cap for the native decode loop. Lower values make the
+     * local UI stream in finer-grained jumps (smoother token-by-token render)
+     * at the cost of more JS↔FFI round-trips per reply; higher values batch
+     * more tokens per step. When omitted, falls back to
+     * `resolveMaxTokensPerStep()` (env `ELIZA_LOCAL_STREAM_TOKENS_PER_STEP`,
+     * else `DEFAULT_MAX_TOKENS_PER_STEP`). Clamped to
+     * `[MIN_MAX_TOKENS_PER_STEP, MAX_MAX_TOKENS_PER_STEP]`.
+     */
+    maxTokensPerStep?: number;
+    /** Per-chunk text callback. */
+    onTextChunk?: (chunk: string) => void | Promise<void>;
+    /** Speculative accept/reject events from MTP verification. */
+    onVerifierEvent?: (event: VerifierStreamEvent) => void | Promise<void>;
+}
+export interface FfiStreamingGenerateResult {
+    text: string;
+    slotId: number;
+    firstTokenMs: number | null;
+    drafted: number;
+    accepted: number;
+}
+/**
+ * Resolve the per-step token cap for the native decode loop. Override via the
+ * `ELIZA_LOCAL_STREAM_TOKENS_PER_STEP` env var (e.g. set to `8` for smoother
+ * local streaming, weighed against the extra JS↔FFI round-trips and the shared
+ * voice phrase-chunker). Falls back to `DEFAULT_MAX_TOKENS_PER_STEP` (32) when
+ * unset or invalid; clamped to `[MIN_MAX_TOKENS_PER_STEP, MAX_MAX_TOKENS_PER_STEP]`.
+ */
+export declare function resolveMaxTokensPerStep(): number;
+/**
+ * Backend used by the mobile and desktop FFI routes.
+ */
+export declare class FfiStreamingRunner {
+    private readonly ffi;
+    private readonly ctx;
+    private readonly slotInFlight;
+    /**
+     * Constructor takes the narrow `LlmStreamingBinding` (see
+     * `services/llm-streaming-binding.ts`) so both libelizainference (via
+     * `wrapElizaInferenceFfi`) and desktop libllama adapters can
+     * satisfy it. The runner never touches TTS/ASR/mmap surfaces.
+     */
+    constructor(ffi: LlmStreamingBinding, ctx: LlmCtxHandle);
+    /**
+     * Run one generation. Mirrors `MtpLlamaServer.generateWithUsage()`
+     * — same single-flight rule, same callback shape, same result block
+     * minus the metrics scrape (FFI does not have a `/metrics` endpoint).
+     */
+    generateWithUsage(args: FfiStreamingGenerateArgs): Promise<FfiStreamingGenerateResult>;
+    /**
+     * Async-iterable variant. Yields each accepted-token batch as it lands
+     * so callers that want token-grained control (e.g. the voice scheduler
+     * driving phrase-chunking off accept/reject events) don't have to
+     * register a callback. Internally still routes through `generateWithUsage`
+     * via a pump so the single-flight rule applies.
+     */
+    generateStream(args: FfiStreamingGenerateArgs): AsyncIterable<LlmStreamStep>;
+    /**
+     * Save the streaming slot KV state to disk. Best called between turns
+     * — calling mid-stream is racy and the FFI side is allowed to refuse.
+     * Surfaced here so the conversation registry can persist between
+     * mobile backgrounds the same way `MtpLlamaServer.persistSlot` does.
+     */
+    saveSlot(stream: LlmStreamHandle, filename: string): void;
+    /** Restore a previously-saved slot KV file into a fresh session. */
+    restoreSlot(stream: LlmStreamHandle, filename: string): void;
+    private runGenerate;
+    /**
+     * Shared inner loop. Opens the session, runs the prefill + next pump,
+     * forwards each step through `onStep` plus the optional caller
+     * callbacks, and wires abort + cancel.
+     */
+    private runGenerateInner;
+}
+//# sourceMappingURL=ffi-streaming-runner.d.ts.map

package/dist/services/ffi-streaming-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ffi-streaming-runner.d.ts","sourceRoot":"","sources":["../../src/services/ffi-streaming-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,OAAO,KAAK,EACX,YAAY,EACZ,mBAAmB,EACnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,KAAK,EAAa,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpE,MAAM,WAAW,wBAAwB;IACxC,6DAA6D;IAC7D,YAAY,EAAE,UAAU,CAAC;IACzB,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,2EAA2E;IAC3E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,8DAA8D;IAC9D,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvE;AAED,MAAM,WAAW,0BAA0B;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CACjB;AAsBD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,IAAI,MAAM,CAMhD;AAED;;GAEG;AACH,qBAAa,kBAAkB;IAU7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,GAAG;IAVrB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAoC;IAEjE;;;;;OAKG;gBAEe,GAAG,EAAE,mBAAmB,EACxB,GAAG,EAAE,YAAY;IAGnC;;;;OAIG;IACG,iBAAiB,CACtB,IAAI,EAAE,wBAAwB,GAC5B,OAAO,CAAC,0BAA0B,CAAC;IAsBtC;;;;;;OAMG;IACI,cAAc,CACpB,IAAI,EAAE,wBAAwB,GAC5B,aAAa,CAAC,aAAa,CAAC;IAmD/B;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IASzD,oEAAoE;IACpE,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;YAW9C,WAAW;IA2BzB;;;;OAIG;YACW,gBAAgB;CAyG9B"}

package/dist/services/gpu-autotune.d.ts

@@ -0,0 +1,150 @@
+/**
+ * Per-GPU llama.cpp autotune.
+ *
+ * Turns a detected GPU (`{ name, totalMemoryMiB }`) into a fully resolved
+ * `llama-server` flag set, optionally narrowed for a specific bundle id.
+ *
+ * The static profile defaults live in `@elizaos/shared`
+ * (`gpu-profiles.ts`); this module layers the per-GPU JSON configs at
+ * `packages/inference/configs/gpu/*.json` on top, and adds bundle-aware
+ * overrides + a VRAM-bucket fallback for cards we don't have a tuned
+ * profile for.
+ *
+ * Resolution order (later wins):
+ *
+ *   1. `GPU_PROFILES[id]` static defaults.
+ *   2. Bundled `GPU_CONFIGS[id]` JSON overlay (this file).
+ *   3. Bundle-specific overrides (`bundle_recommendations.<bundle>`).
+ *   4. Per-call `overrides` arg.
+ *
+ * Pure functions only; no FS access, no process.env reads — env-var
+ * application happens at the FFI runtime spawn site. Tests in
+ * `__tests__/gpu-autotune.test.ts`.
+ *
+ * Scope: single-GPU only — never split layers across cards in this
+ * tier.
+ */
+import { type GpuProfile, type GpuProfileId } from "@elizaos/shared";
+/** Minimum input the helper needs to make a choice. */
+export interface GpuInfo {
+    /** Raw GPU name from `nvidia-smi --query-gpu=name`. */
+    name: string;
+    /** Total VRAM in MiB (also from `nvidia-smi --query-gpu=memory.total`). */
+    totalMemoryMiB: number;
+}
+/** Resolved llama-server flag set returned by `selectGpuConfig`. */
+export interface LlamaServerFlags {
+    n_gpu_layers: number;
+    ctx_size: number;
+    batch_size: number;
+    ubatch_size: number;
+    n_parallel: number;
+    cache_type_k: string;
+    cache_type_v: string;
+    flash_attn: boolean;
+    split_mode: "none" | "layer" | "row";
+    main_gpu: number;
+    mlock: boolean;
+    no_mmap: boolean;
+    no_kv_offload: boolean;
+    ctx_checkpoints: number;
+    ctx_checkpoint_interval: number;
+    draft_max: number;
+    draft_min: number;
+    draft_p_min: number;
+}
+/** Per-bundle override block read from `bundle_recommendations`. */
+export interface BundleRecommendation {
+    ctx_size?: number;
+    max_parallel?: number;
+    batch_size?: number;
+    ubatch_size?: number;
+    cache_type_k?: string;
+    cache_type_v?: string;
+    no_kv_offload?: boolean;
+}
+/** Per-GPU expected metrics. All values flagged as extrapolated until measured. */
+export interface ExpectedMetrics {
+    ttfa_p50_ms: number;
+    ttfa_p95_ms: number;
+    rtf: number;
+    tokens_per_second_decode?: number;
+    _provenance: "measured" | "extrapolated";
+}
+/** Full per-GPU JSON config; mirrors `gpu-config.schema.json`. */
+export interface GpuConfig {
+    id: GpuProfileId;
+    name: string;
+    match_names: string[];
+    vram_gb: number;
+    compute_capability: string;
+    arch: "ampere" | "ada-lovelace" | "blackwell" | "hopper";
+    memory_bandwidth_gbs: number;
+    fp8: boolean;
+    fp4: boolean;
+    flash_attn_3: boolean;
+    llama_server_flags: LlamaServerFlags;
+    bundle_recommendations: Record<string, BundleRecommendation>;
+    expected_metrics: ExpectedMetrics;
+    known_limits: string[];
+}
+/** Fallback bucket — used when `matchGpuProfile` returns null. */
+export interface FallbackBucket {
+    max_vram_gb: number;
+    config_id: GpuProfileId | null;
+    label: string;
+    parallel_scale?: number;
+}
+/**
+ * Result of `selectGpuConfig`. `source` records why we chose this
+ * config so the runtime can log it (especially when a fallback bucket
+ * fired).
+ */
+export interface SelectedGpuConfig {
+    config: GpuConfig;
+    flags: LlamaServerFlags;
+    /** Bundle id we narrowed against, or `null` when unscoped. */
+    bundleId: string | null;
+    /** "match" = exact name match, "bucket" = VRAM fallback. */
+    source: "match" | "bucket";
+    /** Bucket label when `source === "bucket"`; undefined otherwise. */
+    bucketLabel?: string;
+}
+export declare const GPU_CONFIGS: Readonly<Record<GpuProfileId, GpuConfig>>;
+/**
+ * VRAM-bucket fallback table. Ordered ascending by `max_vram_gb`; the
+ * first row whose threshold the GPU does NOT exceed wins. `config_id`
+ * `null` means we have no useful profile (give up; let the caller fall
+ * through to the catalog defaults).
+ */
+export declare const FALLBACK_BUCKETS: ReadonlyArray<FallbackBucket>;
+/**
+ * Pick a `GpuConfig` for a detected GPU.
+ *
+ * 1. Try the exact-name matcher in `@elizaos/shared`.
+ * 2. If that fails, fall through to the first VRAM bucket whose
+ *    threshold the GPU does NOT exceed.
+ * 3. Return `null` only when no bucket applies.
+ */
+export declare function selectGpuConfig(gpu: GpuInfo, opts?: {
+    bundleId?: string;
+    overrides?: Partial<LlamaServerFlags>;
+}): SelectedGpuConfig | null;
+/** Pick the first bucket whose `max_vram_gb` the GPU does NOT exceed. */
+export declare function pickFallbackBucket(vramGb: number): FallbackBucket | null;
+/**
+ * Convert resolved `LlamaServerFlags` to the canonical `llama-server`
+ * argv list. Mirrors the flag names llama.cpp actually accepts —
+ * `ffi-streaming-backend.ts` is the existing producer of these flags and is
+ * the source of truth for naming; this helper exists for tests and for
+ * the voice-bench harness.
+ */
+export declare function flagsToLlamaServerArgv(flags: LlamaServerFlags): string[];
+/**
+ * Cross-check: return the static `GpuProfile` for a `GpuConfig`. Used
+ * by the FFI runtime spawn site to feed `applyGpuProfile()` with the
+ * matching `GpuProfile` while keeping the JSON the source of truth for
+ * per-bundle overrides.
+ */
+export declare function staticProfileFor(config: GpuConfig): GpuProfile;
+//# sourceMappingURL=gpu-autotune.d.ts.map

package/dist/services/gpu-autotune.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpu-autotune.d.ts","sourceRoot":"","sources":["../../src/services/gpu-autotune.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAEN,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,MAAM,iBAAiB,CAAC;AAEzB,uDAAuD;AACvD,MAAM,WAAW,OAAO;IACvB,uDAAuD;IACvD,IAAI,EAAE,MAAM,CAAC;IACb,2EAA2E;IAC3E,cAAc,EAAE,MAAM,CAAC;CACvB;AAED,oEAAoE;AACpE,MAAM,WAAW,gBAAgB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,oEAAoE;AACpE,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,mFAAmF;AACnF,MAAM,WAAW,eAAe;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,WAAW,EAAE,UAAU,GAAG,cAAc,CAAC;CACzC;AAED,kEAAkE;AAClE,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,YAAY,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,GAAG,QAAQ,CAAC;IACzD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,YAAY,EAAE,OAAO,CAAC;IACtB,kBAAkB,EAAE,gBAAgB,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IAC7D,gBAAgB,EAAE,eAAe,CAAC;IAClC,YAAY,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,GAAG,IAAI,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,SAAS,CAAC;IAClB,KAAK,EAAE,gBAAgB,CAAC;IACxB,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,4DAA4D;IAC5D,MAAM,EAAE,OAAO,GAAG,QAAQ,CAAC;IAC3B,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAWD,eAAO,MAAM,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAsOjE,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,aAAa,CAAC,cAAc,CAiB1D,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC9B,GAAG,EAAE,OAAO,EACZ,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;CAAO,GACrE,iBAAiB,GAAG,IAAI,CAuB1B;AAED,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAKxE;AA4DD;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,gBAAgB,GAAG,MAAM,EAAE,CAsBxE;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,CAE9D"}

package/dist/services/gpu-detect.d.ts

@@ -0,0 +1,56 @@
+/**
+ * NVIDIA GPU detection + profile mapping.
+ *
+ * Wraps `nvidia-smi` to identify the host GPU and maps it to a
+ * `GpuProfileId` from `@elizaos/shared/local-inference/gpu-profiles`.
+ *
+ * Single-GPU only: when multiple GPUs are present, we use the first one
+ * `nvidia-smi` reports (canonical CUDA device 0). We do NOT try to split
+ * the model across multiple cards — that is an explicit non-goal of the
+ * single-GPU profile system.
+ *
+ * Detection is cached after first call. Pass `force: true` to bypass the
+ * cache when a fresh probe is needed (e.g. after a GPU hot-swap on a
+ * laptop dock — unusual but possible).
+ */
+import { type SpawnSyncReturns, spawnSync } from "node:child_process";
+import { type GpuProfile, type GpuProfileId } from "@elizaos/shared";
+export interface DetectedGpu {
+    /** Raw GPU name from `nvidia-smi --query-gpu=name`. */
+    name: string;
+    /** Total VRAM in MiB as reported by `nvidia-smi`. */
+    totalMemoryMiB: number;
+    /** Matched profile id, or `null` when the card is not in the supported set. */
+    profileId: GpuProfileId | null;
+}
+export interface GpuDetectionResult {
+    /** `true` when `nvidia-smi` ran successfully (even if no GPU matched a profile). */
+    nvidiaPresent: boolean;
+    /** First GPU reported by `nvidia-smi`; `null` when no NVIDIA GPU is present. */
+    gpu: DetectedGpu | null;
+    /** Resolved profile, or `null` for unsupported / non-NVIDIA hosts. */
+    profile: GpuProfile | null;
+}
+/**
+ * Detect the primary NVIDIA GPU and resolve it to a profile. Returns
+ * `{ nvidiaPresent: false }` on hosts without `nvidia-smi` on PATH or
+ * without an NVIDIA GPU.
+ *
+ * The probe runs `nvidia-smi --query-gpu=name,memory.total --format=csv,noheader,nounits`
+ * with a 3-second timeout so a misbehaving driver cannot stall boot.
+ */
+export declare function detectGpu(opts?: {
+    force?: boolean;
+}): GpuDetectionResult;
+/** Clear the cached detection result. Used by tests. */
+export declare function __resetGpuDetectionCacheForTests(): void;
+/** Override the nvidia-smi runner. Used by tests without mutating ESM exports. */
+export declare function __setGpuDetectionSpawnSyncForTests(runner: ((command: string, args: string[], options: Parameters<typeof spawnSync>[2]) => SpawnSyncReturns<string>) | null): void;
+/**
+ * Recommend a `GpuProfileId` for a synthetic GPU descriptor — used by the
+ * recommender service when it already has a `HardwareProbe` and does not
+ * want to re-shell out to `nvidia-smi`. Returns `null` when nothing
+ * matches.
+ */
+export declare function recommendProfileFromName(name: string): GpuProfileId | null;
+//# sourceMappingURL=gpu-detect.d.ts.map

package/dist/services/gpu-detect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpu-detect.d.ts","sourceRoot":"","sources":["../../src/services/gpu-detect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,KAAK,gBAAgB,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAEN,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,WAAW;IAC3B,uDAAuD;IACvD,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,cAAc,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,SAAS,EAAE,YAAY,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB;IAClC,oFAAoF;IACpF,aAAa,EAAE,OAAO,CAAC;IACvB,gFAAgF;IAChF,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACxB,sEAAsE;IACtE,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;CAC3B;AAiBD;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,kBAAkB,CAI5E;AAED,wDAAwD;AACxD,wBAAgB,gCAAgC,IAAI,IAAI,CAGvD;AAED,kFAAkF;AAClF,wBAAgB,kCAAkC,CACjD,MAAM,EACH,CAAC,CACD,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE,UAAU,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,KACnC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAC9B,IAAI,GACL,IAAI,CAEN;AA0CD;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAE1E"}

package/dist/services/handler-registry.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Side-registry of model handlers registered on an AgentRuntime.
+ *
+ * The elizaOS core exposes `runtime.registerModel(type, handler, provider,
+ * priority)` but no way to list who registered what. This module intercepts
+ * `registerModel` at runtime to record every registration in a Map keyed by
+ * model type, plus fires status listeners so the UI can render a live
+ * [ModelType × Provider] routing table.
+ *
+ * Because we monkey-patch `registerModel` we also keep the original
+ * handler reference — the router-handler (see `router-handler.ts`) uses
+ * this to dispatch inference calls by policy without going through
+ * `runtime.useModel` (which would loop back to us and recurse).
+ */
+import { AgentRuntime, type IAgentRuntime } from "@elizaos/core";
+export interface HandlerRegistration {
+    modelType: string;
+    provider: string;
+    priority: number;
+    registeredAt: string;
+    /**
+     * The original handler function. Captured so the router-handler can
+     * dispatch to it directly, bypassing `runtime.useModel` which would
+     * re-enter the router itself.
+     */
+    handler: (runtime: IAgentRuntime, params: Record<string, unknown>) => Promise<unknown>;
+}
+type Listener = (registrations: HandlerRegistration[]) => void;
+declare class HandlerRegistry {
+    private readonly registrations;
+    private readonly listeners;
+    private installedOn;
+    /**
+     * Snapshot of all registrations grouped by model type, sorted by
+     * priority descending inside each group (matches core's selection
+     * order). Callers must not mutate the returned array.
+     */
+    getAll(): HandlerRegistration[];
+    /** All registrations for a given model type, sorted by priority desc. */
+    getForType(modelType: string): HandlerRegistration[];
+    /**
+     * Registrations excluding a specific provider. Used by the router-handler
+     * to find "all providers except me" when dispatching.
+     */
+    getForTypeExcluding(modelType: string, excludeProvider: string): HandlerRegistration[];
+    subscribe(listener: Listener): () => void;
+    private emit;
+    private record;
+    /**
+     * Install the interception on a runtime. Idempotent per runtime instance.
+     * For most boot paths the prototype-level patch below already covers the
+     * runtime before any plugin registers; this method is the belt-and-braces
+     * fallback for runtimes constructed before the patch ran.
+     */
+    installOn(runtime: AgentRuntime): void;
+    /** Exposed so the prototype patch can record through the singleton. */
+    recordFromPrototype(reg: HandlerRegistration): void;
+}
+export declare const handlerRegistry: HandlerRegistry;
+/**
+ * Public type used by the API/UI — omits the handler function for
+ * serialisation and to prevent UI code from accidentally calling it.
+ */
+export interface PublicRegistration {
+    modelType: string;
+    provider: string;
+    priority: number;
+    registeredAt: string;
+}
+export declare function toPublicRegistration(reg: HandlerRegistration): PublicRegistration;
+export {};
+//# sourceMappingURL=handler-registry.d.ts.map

package/dist/services/handler-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"handler-registry.d.ts","sourceRoot":"","sources":["../../src/services/handler-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,YAAY,EAAE,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAEjE,MAAM,WAAW,mBAAmB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,EAAE,CACR,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC3B,OAAO,CAAC,OAAO,CAAC,CAAC;CACtB;AAED,KAAK,QAAQ,GAAG,CAAC,aAAa,EAAE,mBAAmB,EAAE,KAAK,IAAI,CAAC;AAE/D,cAAM,eAAe;IACpB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA4C;IAC1E,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAuB;IACjD,OAAO,CAAC,WAAW,CAAkC;IAErD;;;;OAIG;IACH,MAAM,IAAI,mBAAmB,EAAE;IAQ/B,yEAAyE;IACzE,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,mBAAmB,EAAE;IAKpD;;;OAGG;IACH,mBAAmB,CAClB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,GACrB,mBAAmB,EAAE;IAMxB,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,IAAI;IAOzC,OAAO,CAAC,IAAI;IAWZ,OAAO,CAAC,MAAM;IAYd;;;;;OAKG;IACH,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IA2CtC,uEAAuE;IACvE,mBAAmB,CAAC,GAAG,EAAE,mBAAmB,GAAG,IAAI;CAGnD;AA8DD,eAAO,MAAM,eAAe,iBAAwB,CAAC;AAErD;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,oBAAoB,CACnC,GAAG,EAAE,mBAAmB,GACtB,kBAAkB,CAOpB"}

package/dist/services/hardware.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Hardware probe for local inference sizing.
+ *
+ * Uses `capacitor-llama` when available to read GPU backend + VRAM. Falls back
+ * to Node's `os` module when the binding isn't installed — we don't require
+ * the plugin to be loaded for the probe endpoint to return useful data.
+ *
+ * Dynamic import is intentional: the binding pulls a native prebuilt that we
+ * don't want eagerly required at module-load time (breaks CI environments
+ * without the trusted-dependency flag).
+ */
+import type { Eliza1DeviceCaps } from "./manifest";
+import type { CpuFeatureProbe, HardwareProbe, OpenVinoHardwareProbe } from "./types";
+interface CpuFeatureDetectionHost {
+    platform?: NodeJS.Platform;
+    arch?: NodeJS.Architecture;
+    readFileSync?: (path: string, encoding: BufferEncoding) => string;
+    execFileSync?: (file: string, args: readonly string[], options: {
+        encoding: BufferEncoding;
+        stdio: "pipe";
+    }) => string;
+}
+export declare function detectCpuFeatures(host?: CpuFeatureDetectionHost): CpuFeatureProbe | undefined;
+export declare function hasUsableArmCpuBackend(probe: HardwareProbe): boolean;
+interface OpenVinoDetectionHost {
+    platform?: NodeJS.Platform;
+    env?: NodeJS.ProcessEnv;
+    existsSync?: (path: string) => boolean;
+    readdirSync?: (path: string) => string[];
+}
+export declare function detectOpenVinoDevices(host?: OpenVinoDetectionHost): OpenVinoHardwareProbe;
+/**
+ * Read current system + GPU state. Cheap enough to call per-request; no
+ * internal caching so the UI always reflects live VRAM usage.
+ */
+export declare function probeHardware(): Promise<HardwareProbe>;
+/**
+ * Map a hardware probe onto the Eliza-1 device-capability snapshot used by
+ * the manifest validator and the bundle downloader.
+ *
+ * Backends: `cpu` is always present (the floor). A detected GPU backend is
+ * added when `capacitor-llama` reports one (`metal` on Apple Silicon, `cuda`
+ * on NVIDIA, `vulkan` on cross-vendor Linux/Android). We do not synthesize
+ * `rocm` from the probe — `capacitor-llama` reports AMD as `vulkan` on the
+ * builds we ship, and a bundle that only verified ROCm but not Vulkan is
+ * legitimately not installable here.
+ *
+ * `ramMb` is total system RAM. On Apple Silicon that is also the GPU's
+ * working memory; on discrete-GPU boxes the recommendation engine layers
+ * its own VRAM-vs-RAM heuristics on top, but the bundle's `ramBudgetMb.min`
+ * is a system-RAM floor in every manifest.
+ */
+export declare function deviceCapsFromProbe(probe: HardwareProbe): Eliza1DeviceCaps;
+/**
+ * Compatibility assessment for a specific model given current hardware.
+ *
+ * Green/fits: comfortable headroom (model < 70% of effective memory).
+ * Yellow/tight: will run but may swap or stutter under load.
+ * Red/wontfit: exceeds available memory.
+ */
+export declare function assessFit(probe: HardwareProbe, modelSizeGb: number, minRamGb: number): "fits" | "tight" | "wontfit";
+export {};
+//# sourceMappingURL=hardware.d.ts.map

package/dist/services/hardware.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hardware.d.ts","sourceRoot":"","sources":["../../src/services/hardware.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH,OAAO,KAAK,EAAiB,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAElE,OAAO,KAAK,EACX,eAAe,EACf,aAAa,EAGb,qBAAqB,EACrB,MAAM,SAAS,CAAC;AAuDjB,UAAU,uBAAuB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC;IAC3B,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,KAAK,MAAM,CAAC;IAClE,YAAY,CAAC,EAAE,CACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE;QAAE,QAAQ,EAAE,cAAc,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAChD,MAAM,CAAC;CACZ;AAmCD,wBAAgB,iBAAiB,CAChC,IAAI,GAAE,uBAA4B,GAChC,eAAe,GAAG,SAAS,CA+C7B;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAGpE;AAwCD,UAAU,qBAAqB;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IACvC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CACzC;AAoCD,wBAAgB,qBAAqB,CACpC,IAAI,GAAE,qBAA0B,GAC9B,qBAAqB,CAgFvB;AAED;;;GAGG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,aAAa,CAAC,CA+B5D;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,aAAa,GAAG,gBAAgB,CAiB1E;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CACxB,KAAK,EAAE,aAAa,EACpB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACd,MAAM,GAAG,OAAO,GAAG,SAAS,CAW9B"}

package/dist/services/image-description-runtime.d.ts

@@ -0,0 +1,14 @@
+export interface ImageDescriptionRuntimeOptions {
+    tier: string;
+    modelPath: string;
+}
+export interface ImageDescriptionRuntime {
+    describe(args: {
+        imagePath: string;
+        prompt: string;
+        maxTokens?: number;
+    }): Promise<string>;
+    cleanup?(): Promise<void>;
+}
+export declare function createImageDescriptionRuntime(args: ImageDescriptionRuntimeOptions): Promise<ImageDescriptionRuntime>;
+//# sourceMappingURL=image-description-runtime.d.ts.map

package/dist/services/image-description-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-description-runtime.d.ts","sourceRoot":"","sources":["../../src/services/image-description-runtime.ts"],"names":[],"mappings":"AAkBA,MAAM,WAAW,8BAA8B;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,uBAAuB;IACvC,QAAQ,CAAC,IAAI,EAAE;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,6BAA6B,CAClD,IAAI,EAAE,8BAA8B,GAClC,OAAO,CAAC,uBAAuB,CAAC,CA8BlC"}