@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/ensure-local-artifacts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ensure-local-artifacts.d.ts","sourceRoot":"","sources":["../../src/services/ensure-local-artifacts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAK7C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAGvD,KAAK,WAAW,GAAG,OAAO,GAAG,YAAY,GAAG,OAAO,GAAG,QAAQ,CAAC;AAgB/D;;;;;GAKG;AACH,MAAM,MAAM,uBAAuB,GAAG,WAAW,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;AAE3E,MAAM,MAAM,yBAAyB,GAClC,SAAS,GACT,mBAAmB,GACnB,SAAS,GACT,QAAQ,CAAC;AAEZ,MAAM,WAAW,0BAA0B;IAC1C,IAAI,EAAE,uBAAuB,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,yBAAyB,CAAC;IAClC,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,wBAAwB;IACxC,mEAAmE;IACnE,IAAI,EAAE,WAAW,CAAC;IAClB,iFAAiF;IACjF,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,mFAAmF;IACnF,aAAa,EAAE,OAAO,CAAC;IACvB,gDAAgD;IAChD,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,uBAAuB;IACvB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;QAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;KAAE,CAAC;CAC9E;AAED,MAAM,WAAW,0BAA0B;IAC1C,SAAS,EAAE,0BAA0B,EAAE,CAAC;IACxC;;;;OAIG;IACH,QAAQ,EAAE,OAAO,CAAC;CAClB;AA4ID;;;;GAIG;AACH,wBAAsB,oBAAoB,CACzC,IAAI,EAAE,wBAAwB,GAC5B,OAAO,CAAC,0BAA0B,CAAC,CAuGrC"}

package/dist/services/external-scanner.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Discover GGUF files already on disk from other local-inference tools.
+ *
+ * Users often have LM Studio, Jan, Ollama, or raw HuggingFace downloads
+ * lying around. We scan their default cache paths and surface those models
+ * in the Model Hub with `source: "external-scan"` so Eliza can load them
+ * without re-downloading. Eliza never modifies or deletes these files —
+ * the uninstall endpoint refuses when `source !== "eliza-download"`.
+ *
+ * Ollama is special: its blobs live under `models/blobs/sha256-*` with no
+ * `.gguf` extension, and the human name only exists in adjacent manifests.
+ * We parse the manifests to recover the mapping; blobs we can't map stay
+ * hidden rather than surfacing as opaque hashes.
+ */
+import type { InstalledModel } from "./types";
+export declare function scanExternalModels(): Promise<InstalledModel[]>;
+//# sourceMappingURL=external-scanner.d.ts.map

package/dist/services/external-scanner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"external-scanner.d.ts","sourceRoot":"","sources":["../../src/services/external-scanner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AA6P9C,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAuCpE"}

package/dist/services/ffi-llm-mock.d.ts

@@ -0,0 +1,90 @@
+/**
+ * Mock implementation of `FfiLlmStreamingAbi` for testing.
+ *
+ * This module provides a fully in-process fake of the C ABI surface
+ * declared in `ffi-llm-streaming-abi.ts`. No native library is loaded,
+ * no GGUF is read, and no real inference is performed. The mock is safe
+ * to use in any test environment regardless of OS or hardware.
+ *
+ * Usage
+ * ─────
+ * ```ts
+ * import { makeFfiLlmMock } from "./ffi-llm-mock";
+ *
+ * const mock = makeFfiLlmMock();
+ * const caps  = detectMobileCapabilities(mock.ffi);
+ * // caps.streamingLlm === true
+ *
+ * const handle = mock.ffi.eliza_inference_llm_stream_open("fake.gguf", 512, 4, 0)!;
+ * mock.ffi.eliza_inference_llm_stream_prefill(handle, new Int32Array([1, 2, 3]), 0);
+ *
+ * const tokens: string[] = [];
+ * await mock.ffi.eliza_inference_llm_stream_generate(handle, 32, 0.8, 0.95, (id, text, done) => {
+ *   if (!done) tokens.push(text);
+ * });
+ * // tokens === ["Hello", " world", "!"]
+ * ```
+ *
+ * Synthetic stream
+ * ────────────────
+ * `generate` fires the callback with three synthetic tokens:
+ *   1. tokenId=1, tokenText="Hello", isDone=false  (after ~1 ms)
+ *   2. tokenId=2, tokenText=" world", isDone=false (after ~1 ms)
+ *   3. tokenId=3, tokenText="!",     isDone=true   (after ~1 ms)
+ *
+ * If `cancel` is called before the stream finishes, the mock observes the
+ * cancellation flag between tokens and stops early. The last callback
+ * invocation before cancellation will have `isDone=true` so callers
+ * always receive a terminal event.
+ *
+ * State tracking
+ * ──────────────
+ * `makeFfiLlmMock()` returns a `MockState` alongside the ABI object. The
+ * state object exposes internal counters that tests can assert on (e.g.
+ * number of `open` / `close` / `cancel` calls, whether a handle is
+ * considered open).
+ */
+import type { FfiLlmStreamingAbi, FfiMtpStreamingAbi } from "./ffi-llm-streaming-abi";
+/**
+ * Observable state counters for the mock. Tests read these to verify
+ * that the ABI layer is called in the right order.
+ */
+export interface MockState {
+    /** Number of times `eliza_inference_llm_stream_open` was called. */
+    openCount: number;
+    /** Number of times `eliza_inference_llm_stream_prefill` was called. */
+    prefillCount: number;
+    /** Number of times `eliza_inference_llm_stream_generate` was called. */
+    generateCount: number;
+    /** Number of times `eliza_inference_llm_stream_cancel` was called. */
+    cancelCount: number;
+    /** Number of times `eliza_inference_llm_stream_close` was called. */
+    closeCount: number;
+    /**
+     * Set of handle ids that have been opened but not yet closed.
+     * Useful for detecting handle leaks in multi-generate tests.
+     */
+    openHandles: Set<number>;
+    /**
+     * True when cancel was called on the most recently opened handle
+     * before the synthetic stream finished. Resets on the next `open`.
+     */
+    cancelledMidStream: boolean;
+}
+/**
+ * Build a mock `FfiLlmStreamingAbi` (and a matching `FfiMtpStreamingAbi`
+ * that delegates to the same synthetic stream) together with an observable
+ * `MockState`.
+ *
+ * The returned `ffi` object satisfies both interface contracts so tests
+ * can pass it to `detectMobileCapabilities()` and to `FfiStreamingRunner`
+ * adapters without needing separate factories.
+ */
+export declare function makeFfiLlmMock(): {
+    ffi: FfiLlmStreamingAbi & FfiMtpStreamingAbi & {
+        llmStreamSupported(): boolean;
+        ttsStreamSupported(): boolean;
+    };
+    state: MockState;
+};
+//# sourceMappingURL=ffi-llm-mock.d.ts.map

package/dist/services/ffi-llm-mock.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ffi-llm-mock.d.ts","sourceRoot":"","sources":["../../src/services/ffi-llm-mock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,OAAO,KAAK,EAEX,kBAAkB,EAClB,kBAAkB,EAElB,MAAM,yBAAyB,CAAC;AAwCjC;;;GAGG;AACH,MAAM,WAAW,SAAS;IACzB,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,YAAY,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,aAAa,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,WAAW,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB;;;OAGG;IACH,kBAAkB,EAAE,OAAO,CAAC;CAC5B;AAMD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,IAAI;IACjC,GAAG,EAAE,kBAAkB,GACtB,kBAAkB,GAAG;QACpB,kBAAkB,IAAI,OAAO,CAAC;QAC9B,kBAAkB,IAAI,OAAO,CAAC;KAC9B,CAAC;IACH,KAAK,EAAE,SAAS,CAAC;CACjB,CAsNA"}

package/dist/services/ffi-llm-streaming-abi.d.ts

@@ -0,0 +1,318 @@
+/**
+ * TypeScript-side ABI surface for the in-process FFI streaming LLM path.
+ *
+ * This file mirrors the C header at
+ * `packages/inference/llama.cpp/omnivoice/src/ffi-streaming.h` — the
+ * function names are the same so that bun:ffi symbol resolution uses the
+ * exact C exports without any aliasing.
+ *
+ * Rationale for a separate ABI module
+ * ────────────────────────────────────
+ * `ffi-streaming-runner.ts` depends on the `ElizaInferenceFfi` handle from
+ * `voice/ffi-bindings.ts`, which in turn is tied to the omnivoice-fused
+ * build of `libelizainference`. That handle carries TTS, ASR, embedding,
+ * and streaming-LLM symbols together. The ABI declared here is the
+ * *streaming-LLM-only* slice that the mobile bootstrap needs to reason
+ * about independently — it does not assume the full fused binary is
+ * loaded. Callers that already have an `ElizaInferenceFfi` can implement
+ * `FfiLlmStreamingAbi` as a thin wrapper; callers that only have the
+ * llama.cpp-only `libelizainference.so` (e.g. the Android AOSP bootstrap
+ * before omnivoice ships) can implement it directly.
+ *
+ * MTP phasing
+ * ──────────────
+ * Phase 1 — target model only. The `FfiLlmStreamingAbi` alone is
+ * sufficient: open a single-model streaming session, prefill, generate,
+ * cancel, close. No drafter weights required.
+ *
+ * Phase 2 — speculative decoding. When `MobileInferenceCapabilities.
+ * mtpSupported` is `true`, swap to `FfiMtpStreamingAbi` which opens
+ * a paired drafter + verifier session and runs the speculative decode loop
+ * on-device. The two ABI surfaces share the same `FfiLlmHandle` brand so
+ * the dispatcher (`runtime-dispatcher.ts`) sees a uniform handle type.
+ *
+ * iOS XCFramework gap
+ * ───────────────────
+ * The ABI is defined here, the C header is frozen, but the iOS
+ * XCFramework that re-exports these symbols through the Swift bridge has
+ * not shipped yet. `loadIosStreamingLlmBinding()` in
+ * `ios-llama-streaming.ts` returns `null` until the XCFramework build
+ * lands. See `docs/inference/ffi-streaming.md` §iOS XCFramework gap for
+ * the current status.
+ */
+/**
+ * Opaque handle to an open streaming-LLM session. The underlying C value
+ * is a pointer to a heap-allocated session struct; we brand it at the TS
+ * layer to prevent accidental mixing with other handle types.
+ *
+ * Concrete implementations will typically alias this to `bigint` (the
+ * bun:ffi representation of a C pointer) — but callers should treat it as
+ * opaque.
+ */
+export interface FfiLlmHandle {
+    readonly _brand: "FfiLlmHandle";
+}
+/**
+ * Token callback fired from the generation background thread once per
+ * decoded token (or once per speculative-accept batch in MTP mode).
+ *
+ * `isDone` is `true` on the *last* invocation for a given generate call.
+ * After `isDone` the handle remains open but must not be passed to
+ * `generate` again until the caller re-prefills.
+ *
+ * The callback executes synchronously on the background thread the C
+ * library uses for decoding — callers must not call any FFI method
+ * back from inside the callback (the lock is not re-entrant).
+ */
+export type TokenCallback = (tokenId: number, tokenText: string, isDone: boolean) => void;
+/**
+ * C ABI surface for the in-process streaming LLM path.
+ *
+ * Function names match the C exports in `ffi-streaming.h` exactly; bun:ffi
+ * resolves them by string match against the shared library symbol table.
+ *
+ * All methods are synchronous from the JS perspective (bun:ffi calls are
+ * synchronous unless declared `nonblocking`). `generate` is the one
+ * exception: it returns immediately after scheduling the background decode
+ * loop and delivers results via `tokenCallback`.
+ */
+export interface FfiLlmStreamingAbi {
+    /**
+     * Open a streaming-LLM session against the model at `modelPath`.
+     *
+     * The model is memory-mapped into the process — this call may block
+     * briefly on a cold filesystem. Subsequent calls with the same path
+     * share the mmap region (the C library uses a ref-counted mmap cache).
+     *
+     * Returns an opaque handle on success, or `null` when:
+     *   - the model file does not exist or cannot be read,
+     *   - the device lacks the RAM required for `contextSizeTokens`,
+     *   - `gpuLayers > 0` and the Metal / Vulkan device is unavailable.
+     *
+     * @param modelPath          Absolute path to a GGUF model file.
+     * @param contextSizeTokens  KV cache size in tokens (must be power-of-two
+     *                           aligned; the library rounds up if needed).
+     * @param numThreads         CPU decode threads. 0 = auto-detect (uses
+     *                           `eliza_inference_default_thread_count()`).
+     * @param gpuLayers          Number of transformer layers to offload to
+     *                           GPU. 0 = CPU only.
+     */
+    eliza_inference_llm_stream_open(modelPath: string, contextSizeTokens: number, numThreads: number, gpuLayers: number): FfiLlmHandle | null;
+    /**
+     * Prefill the KV cache with the supplied token ids.
+     *
+     * Blocks until all tokens are evaluated. On a large prompt this can
+     * take several hundred milliseconds on CPU — callers should not invoke
+     * on the main thread.
+     *
+     * @param handle        Active session from `open`.
+     * @param promptTokens  Pre-tokenized prompt; row-major int32 ids.
+     * @param slotId        KV slot index (0-based). Use -1 to allocate a
+     *                      fresh slot; use 0..N-1 to pin a conversational
+     *                      turn for KV reuse across multi-turn sessions.
+     * @returns Number of tokens prefilled, or -1 on error (invalid handle,
+     *          OOM, or KV cache exhausted).
+     */
+    eliza_inference_llm_stream_prefill(handle: FfiLlmHandle, promptTokens: Int32Array, slotId: number): number;
+    /**
+     * Start async token generation.
+     *
+     * The library spins up an internal worker thread (or reuses a pooled
+     * one) and begins decoding. Each decoded token fires `tokenCallback`
+     * from that thread. The final callback invocation has `isDone = true`.
+     *
+     * This call is non-blocking from the C caller's perspective: the C
+     * function returns 0 as soon as the worker is scheduled. From the JS
+     * perspective, callers should await the returned Promise — it resolves
+     * after the final `isDone = true` callback fires so that the JS async
+     * iterator can drain cleanly without a separate synchronisation
+     * mechanism. Mock implementations fulfil this contract by resolving
+     * the Promise after the last synthetic token; native FFI wrappers wrap
+     * a completion event or condition variable.
+     *
+     * Calling `generate` on a handle that is already generating is a hard
+     * error (returns -1 / rejects). Callers must wait for the Promise to
+     * resolve (or call `cancel` and await the resulting `isDone` callback)
+     * before re-using the handle.
+     *
+     * @param handle        Active session from `open`.
+     * @param maxNewTokens  Budget cap. Generation stops at `maxNewTokens`
+     *                      even if no EOS token was produced.
+     * @param temperature   Softmax temperature. 0.0 = greedy.
+     * @param topP          Nucleus sampling threshold (0.0–1.0).
+     * @param tokenCallback Callback fired per token from the decode thread.
+     * @returns Promise resolving to 0 on success, -1 on error.
+     */
+    eliza_inference_llm_stream_generate(handle: FfiLlmHandle, maxNewTokens: number, temperature: number, topP: number, tokenCallback: TokenCallback): number | Promise<number>;
+    /**
+     * Signal the active generation to stop at the next safe cancellation
+     * point (after the current speculative batch is retired).
+     *
+     * This does NOT wait for the background thread to finish — the thread
+     * fires a final `tokenCallback` with `isDone = true` shortly after the
+     * cancel flag is observed. Callers that need to know the thread has
+     * stopped must wait for that final callback.
+     *
+     * Calling `cancel` on a handle that is not currently generating is a
+     * no-op.
+     *
+     * @param handle Active session from `open`.
+     */
+    eliza_inference_llm_stream_cancel(handle: FfiLlmHandle): void;
+    /**
+     * Release all resources associated with `handle`.
+     *
+     * Evicts the KV cache slots occupied by this session and releases the
+     * mmap reference. The model's mmap region stays mapped until the ref
+     * count reaches zero (i.e. all sessions against that path are closed).
+     *
+     * Calling `close` on a handle that is still generating is a hard error
+     * — cancel first and wait for `isDone` before closing.
+     *
+     * @param handle Active session from `open`.
+     */
+    eliza_inference_llm_stream_close(handle: FfiLlmHandle): void;
+}
+/**
+ * C ABI surface for paired drafter + verifier speculative decoding.
+ *
+ * Phase 2 only — the mobile runtime enables this path when
+ * `MobileInferenceCapabilities.mtpSupported` is `true`. Phase 1
+ * devices use `FfiLlmStreamingAbi` only (target model, no drafter).
+ *
+ * The MTP session holds two model contexts internally:
+ *   1. The *drafter* — a small, fast model that proposes `speculativeWindowSize`
+ *      candidate tokens per step.
+ *   2. The *verifier* — the full target model that accepts or rejects the
+ *      drafter's proposals in one parallel evaluation batch.
+ *
+ * The token callback fires once per *accepted* token (after the verifier's
+ * accept decision). Rejected tokens are silently discarded at the C layer;
+ * the JS consumer always sees a stream of accepted tokens identical to
+ * what a greedy target-only decode would have produced (assuming the
+ * drafter and verifier share vocabulary and a compatible chat template).
+ *
+ * Method signatures mirror `FfiLlmStreamingAbi` exactly — only the `open`
+ * argument list differs (adds drafter path + speculative window). This
+ * keeps the dispatcher (`runtime-dispatcher.ts`) agnostic to which ABI is
+ * in use.
+ */
+export interface FfiMtpStreamingAbi {
+    /**
+     * Open a paired drafter + verifier streaming session.
+     *
+     * Both models are mmap'd; the KV cache is sized to `contextSizeTokens`
+     * for the verifier and proportionally smaller for the drafter (the C
+     * library computes the drafter KV budget automatically from the
+     * `speculativeWindowSize`).
+     *
+     * @param drafterModelPath     Absolute path to the drafter GGUF.
+     * @param verifierModelPath    Absolute path to the verifier GGUF.
+     * @param contextSizeTokens    Verifier KV size in tokens.
+     * @param numThreads           CPU threads for verifier (drafter shares
+     *                             the same thread pool).
+     * @param gpuLayers            Verifier GPU layer count. The drafter
+     *                             always runs on CPU in Phase 2 to avoid
+     *                             competing for Metal/Vulkan resources.
+     * @param speculativeWindowSize Number of drafter candidate tokens per
+     *                             speculative step (1–16; 4 is a safe
+     *                             starting point for mobile).
+     * @returns Opaque session handle, or `null` on failure.
+     */
+    eliza_inference_mtp_stream_open(drafterModelPath: string, verifierModelPath: string, contextSizeTokens: number, numThreads: number, gpuLayers: number, speculativeWindowSize: number): FfiLlmHandle | null;
+    /**
+     * Prefill both the drafter and verifier KV caches in a single blocking
+     * call. The verifier is prefilled first (it owns the ground-truth KV
+     * state); the drafter is then fast-forwarded to match.
+     *
+     * Same contract as `FfiLlmStreamingAbi.eliza_inference_llm_stream_prefill`.
+     */
+    eliza_inference_mtp_stream_prefill(handle: FfiLlmHandle, promptTokens: Int32Array, slotId: number): number;
+    /**
+     * Start speculative-decoding generation. The token callback fires for
+     * each verifier-accepted token — not for each drafter proposal.
+     *
+     * Same contract as `FfiLlmStreamingAbi.eliza_inference_llm_stream_generate`.
+     */
+    eliza_inference_mtp_stream_generate(handle: FfiLlmHandle, maxNewTokens: number, temperature: number, topP: number, tokenCallback: TokenCallback): number | Promise<number>;
+    /**
+     * Cancel an active MTP generation at the next speculation boundary.
+     * Same contract as `FfiLlmStreamingAbi.eliza_inference_llm_stream_cancel`.
+     */
+    eliza_inference_mtp_stream_cancel(handle: FfiLlmHandle): void;
+    /**
+     * Release both drafter and verifier sessions.
+     * Same contract as `FfiLlmStreamingAbi.eliza_inference_llm_stream_close`.
+     */
+    eliza_inference_mtp_stream_close(handle: FfiLlmHandle): void;
+}
+/**
+ * Device-side inference capability snapshot used by the mobile bootstrap
+ * to decide which ABI path to activate at startup.
+ *
+ * Produced by `detectMobileCapabilities()`. The runtime re-probes on
+ * every foreground resume (thermal / memory state can change while the
+ * app is backgrounded).
+ */
+export type MobileInferenceCapabilities = {
+    /**
+     * True when the `eliza_inference_llm_stream_*` symbols are present in
+     * the loaded `libelizainference` and `llmStreamSupported()` returns 1.
+     * This is the gate for Phase 1 on-device inference.
+     */
+    streamingLlm: boolean;
+    /**
+     * True when `streamingLlm` is true AND the drafter GGUF is bundled AND
+     * the device's thermal state is below `serious`. Gate for Phase 2
+     * speculative decoding.
+     */
+    mtpSupported: boolean;
+    /**
+     * True when the `eliza_inference_tts_synthesize_stream` symbol is
+     * present and `ttsStreamSupported()` returns 1. Gate for the OmniVoice
+     * TTS streaming path.
+     */
+    omnivoiceStreaming: boolean;
+    /**
+     * Device-reported maximum KV context in tokens. Derived from available
+     * device RAM minus the model weights footprint. The runtime clamps
+     * user-configured context sizes to this value.
+     *
+     * 0 when `streamingLlm` is false (no context available).
+     */
+    maxContextTokens: number;
+    /**
+     * Number of transformer layers the device can offload to GPU/NPU at
+     * the current thermal state without risking thermal throttling. 0 means
+     * CPU-only execution. The runtime uses this as the initial `gpuLayers`
+     * argument to `open`; it can be reduced dynamically when the thermal
+     * state worsens mid-session.
+     */
+    recommendedGpuLayers: number;
+};
+/**
+ * Derive a `MobileInferenceCapabilities` snapshot from an FFI binding.
+ *
+ * When `ffi` is `null` (e.g. in test environments, cloud-only builds, or
+ * when the native library failed to load), all boolean flags are `false`
+ * and numeric fields take safe zero defaults. This keeps the downstream
+ * runtime uniform: it can always read the capability struct without
+ * branching on "was an FFI loaded".
+ *
+ * When `ffi` is non-null, the function:
+ *   1. Calls `llmStreamSupported()` to set `streamingLlm`.
+ *   2. Sets `mtpSupported = false` for Phase 1 (drafter support
+ *      detection requires a platform-specific bundle probe that is NOT
+ *      part of this function; callers that have done the probe should set
+ *      the field themselves after receiving the snapshot).
+ *   3. Calls `ttsStreamSupported()` to set `omnivoiceStreaming`.
+ *   4. Uses conservative device defaults for `maxContextTokens` and
+ *      `recommendedGpuLayers` when the underlying library does not
+ *      expose separate capability-query symbols (Phase 1 does not require
+ *      them).
+ *
+ * @param ffi  A loaded FFI binding, or `null` for an all-false defaults
+ *             snapshot.
+ */
+export declare function detectMobileCapabilities(ffi: FfiLlmStreamingAbi | null): MobileInferenceCapabilities;
+//# sourceMappingURL=ffi-llm-streaming-abi.d.ts.map

package/dist/services/ffi-llm-streaming-abi.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ffi-llm-streaming-abi.d.ts","sourceRoot":"","sources":["../../src/services/ffi-llm-streaming-abi.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAMH;;;;;;;;GAQG;AACH,MAAM,WAAW,YAAY;IAC5B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CAChC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,aAAa,GAAG,CAC3B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,OAAO,KACX,IAAI,CAAC;AAMV;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,+BAA+B,CAC9B,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GACf,YAAY,GAAG,IAAI,CAAC;IAEvB;;;;;;;;;;;;;;OAcG;IACH,kCAAkC,CACjC,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,UAAU,EACxB,MAAM,EAAE,MAAM,GACZ,MAAM,CAAC;IAEV;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,mCAAmC,CAClC,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,aAAa,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5B;;;;;;;;;;;;;OAaG;IACH,iCAAiC,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE9D;;;;;;;;;;;OAWG;IACH,gCAAgC,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;CAC7D;AAMD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,+BAA+B,CAC9B,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,qBAAqB,EAAE,MAAM,GAC3B,YAAY,GAAG,IAAI,CAAC;IAEvB;;;;;;OAMG;IACH,kCAAkC,CACjC,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,UAAU,EACxB,MAAM,EAAE,MAAM,GACZ,MAAM,CAAC;IAEV;;;;;OAKG;IACH,mCAAmC,CAClC,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,aAAa,GAC1B,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5B;;;OAGG;IACH,iCAAiC,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE9D;;;OAGG;IACH,gCAAgC,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;CAC7D;AAMD;;;;;;;GAOG;AACH,MAAM,MAAM,2BAA2B,GAAG;IACzC;;;;OAIG;IACH,YAAY,EAAE,OAAO,CAAC;IAEtB;;;;OAIG;IACH,YAAY,EAAE,OAAO,CAAC;IAEtB;;;;OAIG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAE5B;;;;;;OAMG;IACH,gBAAgB,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAMF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,wBAAwB,CACvC,GAAG,EAAE,kBAAkB,GAAG,IAAI,GAC5B,2BAA2B,CAoD7B"}

package/dist/services/ffi-streaming-backend.d.ts

@@ -0,0 +1,201 @@
+/**
+ * In-process FFI streaming backend adapter.
+ *
+ * Implements `LocalInferenceBackend` as the optimized in-process
+ * llama.cpp path used by Eliza-1 on desktop and mobile.
+ *
+ * What this class deliberately does NOT do:
+ *   - Own the FFI context. The runtime provider passed to this class owns
+ *     native load/unload and hands back the binding, context, and tokenizer.
+ *   - Decode image bytes or call mtmd directly. Vision requests are validated
+ *     here, then forwarded to runtimes that expose `describeImage`.
+ */
+import type { BackendPlan, GenerateArgs, GenerateResult, LocalGenerateWithUsageResult, LocalInferenceBackend } from "./backend";
+import type { FfiStreamingRunner } from "./ffi-streaming-runner";
+import type { LlmCtxHandle, LlmStreamingBinding } from "./llm-streaming-binding";
+/**
+ * Constructor-injected adapter that resolves the FFI binding, context, and
+ * tokenizer for a given load. Two responsibilities:
+ *
+ *   1. Decide whether the FFI path is viable on the current binding
+ *      (`supported()`). Mirrors `LlmStreamingBinding.llmStreamSupported()`
+ *      plus any higher-level constraints (e.g. dylib path exists, build
+ *      target matches the bundle's required kernels).
+ *   2. Lifecycle: `acquire(plan)` returns the FFI runner ready for
+ *      `generate()` against the requested model, plus a tokenizer that
+ *      matches that model's vocab. `release()` tears everything down.
+ *
+ * Production runtime implementation: the fused libelizainference path
+ * (`desktop-fused-ffi-backend-runtime.ts`), which wraps `ElizaInferenceFfi`
+ * via `wrapElizaInferenceFfi()` from `services/llm-streaming-binding.ts`.
+ * libllama has been retired — there is no second runtime behind this slot.
+ */
+export interface FfiBackendRuntime {
+    supported(): boolean;
+    acquire(plan: BackendPlan): Promise<FfiBackendSession>;
+    release(): Promise<void>;
+    /**
+     * Optional parallel-slot pool surface. When the runtime exposes a
+     * ctx pool (the desktop libllama path does), `parallelSlots()`
+     * reports the live count and `resizeParallel(N)` grows/shrinks it.
+     * Runtimes without a pool report 1 and ignore resize requests.
+     */
+    parallelSlots?(): number;
+    resizeParallel?(target: number): Promise<boolean>;
+}
+/**
+ * Result of `FfiBackendRuntime.acquire()` — a live FFI session bound to a
+ * specific loaded model.
+ */
+export interface FfiBackendSession {
+    readonly binding: LlmStreamingBinding;
+    readonly ctx: LlmCtxHandle;
+    readonly runner: FfiStreamingRunner;
+    /**
+     * Tokenize a prompt string into model token ids using the loaded model's
+     * tokenizer. The vocab MUST match the GGUF — mismatches produce gibberish
+     * silently. The runtime is responsible for asserting this at acquire
+     * time.
+     */
+    readonly tokenize: (prompt: string) => Int32Array;
+    /**
+     * Native MTP speculative-decoding policy from the catalog. `null`
+     * disables speculative decoding for this session.
+     */
+    readonly mtp: {
+        specType: "draft-mtp";
+        draftMin: number;
+        draftMax: number;
+        gpuLayers: number | "auto";
+    } | null;
+    /**
+     * Absolute path to a *separate* MTP drafter GGUF resolved during load.
+     * `null` means embedded-draft-head MTP: the draft head is embedded in
+     * the main text GGUF and the native runner activates `--spec-type
+     * draft-mtp` with no `-md`. Speculative decoding is governed by `mtp`,
+     * not by the presence of this path.
+     */
+    readonly draftModelPath: string | null;
+    /**
+     * Multimodal projector (mmproj) GGUF path for vision describe. Resolved
+     * from `plan.overrides.mmprojPath` at acquire time. `null` disables
+     * vision — `describeImage` then throws an actionable error.
+     */
+    readonly mmprojPath: string | null;
+    /**
+     * Per-load runtime config the fused libelizainference path applies at its
+     * first `llmStreamOpen` (context size, gpuLayers + KV-cache quant types). The desktop
+     * libllama runtime applies these at `loadModel()` instead and leaves this
+     * `null` — the backend forwards them into the runner's per-call config only
+     * when present, so the fused path mirrors the libllama load decision without
+     * the libllama path double-applying them.
+     */
+    readonly loadConfig?: {
+        contextSize?: number;
+        gpuLayers?: number;
+        cacheTypeK?: string | null;
+        cacheTypeV?: string | null;
+    } | null;
+}
+/**
+ * Adapter that satisfies `LocalInferenceBackend` by delegating to
+ * `FfiStreamingRunner`. The `id` is `"llama-cpp"` because this is the
+ * in-process variant of the optimized llama.cpp path.
+ */
+export declare class FfiStreamingBackend implements LocalInferenceBackend {
+    private readonly runtime;
+    readonly id: "llama-cpp";
+    private session;
+    private loadedPath;
+    constructor(runtime: FfiBackendRuntime);
+    available(): Promise<boolean>;
+    hasLoadedModel(): boolean;
+    currentModelPath(): string | null;
+    load(plan: BackendPlan): Promise<void>;
+    unload(): Promise<void>;
+    generate(args: GenerateArgs): Promise<GenerateResult>;
+    generateWithUsage(args: GenerateArgs & {
+        slotId?: number;
+    }): Promise<LocalGenerateWithUsageResult>;
+    /**
+     * Persist the active session's KV state to a per-conversation file.
+     * v1 uses `llama_state_seq_save_file` against seq_id=0. The on-disk file
+     * path mirrors `ffi-streaming-backend.ts`'s conversation-keyed slot layout
+     * (`<cacheDir>/<conversationId>/<slotId>.kv`) so a switch between
+     * FFI and subprocess can resume each other's slots — once both
+     * paths agree on the file format.
+     */
+    persistConversationKv(conversationId: string, slotId: number): Promise<void>;
+    /** Restore a previously persisted KV state. Mirror of `persistConversationKv`. */
+    restoreConversationKv(conversationId: string, slotId: number): Promise<boolean>;
+    /**
+     * Pre-decode `promptPrefix` so the next `generate` against the same
+     * `cacheKey` skips re-prefill. Returns `false` when the prefix is
+     * empty or no session is loaded. The FFI runner serializes by
+     * `cacheKey` internally via the `slotInFlight` map.
+     */
+    prewarmConversation(promptPrefix: string, opts: {
+        slotId: number;
+        cacheKey: string;
+    }): Promise<boolean>;
+    currentRuntimeLoadConfig(): {
+        modelId: null;
+        modelPath: string | null;
+        contextSize: number | null;
+        cacheTypeK: string | null;
+        cacheTypeV: string | null;
+        gpuLayers: number | null;
+        parallel: number;
+        binaryPath: null;
+        backend: "llama-cpp";
+        mtp: {
+            specType: "draft-mtp";
+            draftMin: number;
+            draftMax: number;
+        } | null;
+    } | null;
+    /**
+     * True when Eliza-1 native MTP is active for the loaded target model.
+     * Covers both shapes: embedded-draft-head MTP (draft head embedded in
+     * the text GGUF, `draftModelPath` null) and separate-drafter MTP.
+     */
+    mtpEnabled(): boolean;
+    /**
+     * Parallel-slot pool size. Routed to the runtime's ctx pool when one
+     * exists; defaults to 1 otherwise.
+     */
+    parallelSlots(): number;
+    /**
+     * Grow or shrink the runtime's ctx pool to `target` slots. Returns
+     * false when the runtime has no pool surface (in which case parallel
+     * resize is ignored — the conversation registry tolerates
+     * fixed 1-slot operation).
+     */
+    resizeParallel(target: number): Promise<boolean>;
+    /**
+     * Vision describe via mmproj. Requires:
+     *   - The shim built with `-DELIZA_ENABLE_VISION=1` (ELIZA_ENABLE_VISION=1
+     *     at the build script env). When absent the runtime throws an
+     *     actionable error.
+     *   - `plan.overrides.mmprojPath` was passed at load time so the
+     *     adapter knows which mmproj GGUF to feed clip.
+     */
+    describeImage(args: {
+        bytes: Uint8Array;
+        mimeType?: string;
+        prompt?: string;
+        maxTokens?: number;
+        temperature?: number;
+        signal?: AbortSignal;
+        /** Per-token callback — when set + the runtime supports streaming vision,
+         * the description is decoded token-by-token through the same pipe as chat. */
+        onTextChunk?: (chunk: string) => void | Promise<void>;
+        maxTokensPerStep?: number;
+    }): Promise<{
+        text: string;
+        projectorMs?: number;
+        decodeMs?: number;
+    }>;
+    currentMmprojPath(): string | null;
+}
+//# sourceMappingURL=ffi-streaming-backend.d.ts.map

package/dist/services/ffi-streaming-backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ffi-streaming-backend.d.ts","sourceRoot":"","sources":["../../src/services/ffi-streaming-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EACX,WAAW,EACX,YAAY,EACZ,cAAc,EACd,4BAA4B,EAC5B,qBAAqB,EACrB,MAAM,WAAW,CAAC;AACnB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,KAAK,EACX,YAAY,EACZ,mBAAmB,EACnB,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,iBAAiB;IACjC,SAAS,IAAI,OAAO,CAAC;IACrB,OAAO,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACvD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB;;;;;OAKG;IACH,aAAa,CAAC,IAAI,MAAM,CAAC;IACzB,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAClD;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;IACtC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,UAAU,CAAC;IAClD;;;OAGG;IACH,QAAQ,CAAC,GAAG,EAAE;QACb,QAAQ,EAAE,WAAW,CAAC;QACtB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;KAC3B,GAAG,IAAI,CAAC;IACT;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;;;;OAIG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;;;OAOG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC3B,GAAG,IAAI,CAAC;CACT;AAED;;;;GAIG;AACH,qBAAa,mBAAoB,YAAW,qBAAqB;IAMpD,OAAO,CAAC,QAAQ,CAAC,OAAO;IALpC,QAAQ,CAAC,EAAE,EAAG,WAAW,CAAU;IAEnC,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,UAAU,CAAuB;gBAEZ,OAAO,EAAE,iBAAiB;IAEjD,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IAInC,cAAc,IAAI,OAAO;IAIzB,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAI3B,IAAI,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBvB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC;IAKrD,iBAAiB,CACtB,IAAI,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GACtC,OAAO,CAAC,4BAA4B,CAAC;IAuDxC;;;;;;;OAOG;IACG,qBAAqB,CAC1B,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;IAYhB,kFAAkF;IAC5E,qBAAqB,CAC1B,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC;IASnB;;;;;OAKG;IACG,mBAAmB,CACxB,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GACxC,OAAO,CAAC,OAAO,CAAC;IAuBnB,wBAAwB;;;;;;;;;;;;;;;;IAwBxB;;;;OAIG;IACH,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACH,aAAa,IAAI,MAAM;IAIvB;;;;;OAKG;IACG,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKtD;;;;;;;OAOG;IACG,aAAa,CAAC,IAAI,EAAE;QACzB,KAAK,EAAE,UAAU,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB;sFAC8E;QAC9E,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACtD,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IA4CtE,iBAAiB,IAAI,MAAM,GAAG,IAAI;CAGlC"}