@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/conversation-registry.ts

@@ -0,0 +1,264 @@
+/**
+ * Conversation registry for the local-inference path.
+ *
+ * Today's slot allocation is purely a hash function: `deriveSlotId` maps a
+ * `promptCacheKey` (or any stable string) to `slot_id` in `[0, parallel)`.
+ * That works for one-shot calls but breaks for long agentic loops:
+ *
+ *   - Two distinct conversations whose cache keys hash to the same slot
+ *     evict each other's KV every turn (slot thrashing).
+ *   - The current high-water mark of concurrent conversations is invisible,
+ *     so `--parallel N` cannot be tuned to fit.
+ *   - There is no notion of an explicit "I am still using this slot" lease,
+ *     so eviction is purely best-effort.
+ *
+ * This registry keeps a per-conversation reservation. `openConversation`
+ * picks the lowest-loaded slot and pins the conversation to it; subsequent
+ * `generateInConversation` calls always land on the same slot. When the
+ * pool is full, slot reuse falls back to the same-as-before hash policy
+ * (two leases on the same slot still serialise correctly via the dispatcher's
+ * generation queue).
+ *
+ * The registry tracks the high-water mark of concurrently-open conversations
+ * so the engine can warn, or later restart llama-server with a higher
+ * --parallel, when the load outgrows the configured slot count.
+ */
+
+import { createHash } from "node:crypto";
+
+/**
+ * Opaque handle returned by `openConversation`. Callers MUST treat this as
+ * opaque — the registry owns the slot id and lifetime.
+ */
+export interface ConversationHandle {
+	readonly conversationId: string;
+	readonly modelId: string;
+	/**
+	 * Pinned slot id in `[0, parallel)`, or `-1` when slot pinning is disabled
+	 * (parallel <= 0). Used by both backends as the cache key:
+	 *   - llama-server: forwarded as `slot_id` in the request payload.
+	 *   - node-llama-cpp: combined with the conversation id to derive the
+	 *     session-pool key so identical conversations share a session.
+	 */
+	readonly slotId: number;
+	/** Wall-clock ms when the handle was opened. */
+	readonly openedAtMs: number;
+	/** Wall-clock ms when the handle was last touched (open or generate). */
+	lastUsedMs: number;
+	/** TTL after which the registry MAY auto-close on the next sweep. */
+	readonly ttlMs: number;
+	/** True when `closeConversation` has been called; further use is rejected. */
+	closed: boolean;
+}
+
+export interface OpenConversationArgs {
+	conversationId: string;
+	modelId: string;
+	/** Slot count from the running server (`--parallel N`). Defaults to 1. */
+	parallel?: number;
+	/**
+	 * TTL after which the handle is considered idle and may be auto-closed
+	 * by `evictIdle`. Defaults to 60 minutes — long enough for an LLM call
+	 * to finish even on a slow drafter, short enough to recover from forgotten
+	 * close calls within the long-cache window.
+	 */
+	ttlMs?: number;
+}
+
+const DEFAULT_HANDLE_TTL_MS = 60 * 60 * 1000;
+
+/**
+ * In-memory registry of open conversation handles. A single instance is
+ * shared by the engine; each backend reads from it on every generate to
+ * decide which slot to pin to.
+ */
+export class ConversationRegistry {
+	private readonly handles = new Map<string, ConversationHandle>();
+	/** Per-slot reference count; lowest-loaded slot wins on next open. */
+	private readonly slotLoad = new Map<number, number>();
+	/** Largest concurrent open count seen; the engine reads this for parallel auto-tune. */
+	private highWaterMark = 0;
+
+	/**
+	 * Lookup / open a conversation handle. Idempotent for the same
+	 * conversation id + model id; callers can call this on every turn
+	 * without leaking handles. When the call is reusing an existing handle,
+	 * `lastUsedMs` is bumped for LRU-style eviction tracking.
+	 */
+	open(args: OpenConversationArgs): ConversationHandle {
+		if (!args.conversationId) {
+			throw new Error("[conversation-registry] conversationId is required");
+		}
+		if (!args.modelId) {
+			throw new Error("[conversation-registry] modelId is required");
+		}
+		const compositeKey = this.compositeKey(args.conversationId, args.modelId);
+		const existing = this.handles.get(compositeKey);
+		if (existing && !existing.closed) {
+			existing.lastUsedMs = Date.now();
+			return existing;
+		}
+
+		const parallel =
+			typeof args.parallel === "number" && args.parallel > 0
+				? Math.floor(args.parallel)
+				: 1;
+		const slotId = this.pickLowestLoadedSlot(parallel, args.conversationId);
+		const now = Date.now();
+		const handle: ConversationHandle = {
+			conversationId: args.conversationId,
+			modelId: args.modelId,
+			slotId,
+			openedAtMs: now,
+			lastUsedMs: now,
+			ttlMs: args.ttlMs ?? DEFAULT_HANDLE_TTL_MS,
+			closed: false,
+		};
+		this.handles.set(compositeKey, handle);
+		this.slotLoad.set(slotId, (this.slotLoad.get(slotId) ?? 0) + 1);
+		if (this.handles.size > this.highWaterMark) {
+			this.highWaterMark = this.handles.size;
+		}
+		return handle;
+	}
+
+	/**
+	 * Lookup an open handle by conversation+model. Returns null when the
+	 * conversation has not been opened or has already been closed. Bumps
+	 * `lastUsedMs` so an LRU sweep treats reads as activity.
+	 */
+	get(conversationId: string, modelId: string): ConversationHandle | null {
+		const handle = this.handles.get(this.compositeKey(conversationId, modelId));
+		if (!handle || handle.closed) return null;
+		handle.lastUsedMs = Date.now();
+		return handle;
+	}
+
+	/**
+	 * Close + drop a handle. Idempotent — closing an unknown / already-closed
+	 * handle has no additional effect, so callers can call this from cleanup paths
+	 * unconditionally.
+	 */
+	close(conversationId: string, modelId: string): void {
+		const compositeKey = this.compositeKey(conversationId, modelId);
+		const handle = this.handles.get(compositeKey);
+		if (!handle) return;
+		handle.closed = true;
+		this.handles.delete(compositeKey);
+		const remaining = (this.slotLoad.get(handle.slotId) ?? 0) - 1;
+		if (remaining <= 0) {
+			this.slotLoad.delete(handle.slotId);
+		} else {
+			this.slotLoad.set(handle.slotId, remaining);
+		}
+	}
+
+	/**
+	 * Sweep handles whose `lastUsedMs` is older than their TTL. Returns the
+	 * conversation ids dropped so callers can persist final KV state to
+	 * disk, etc. Safe to call on a timer.
+	 */
+	evictIdle(now: number = Date.now()): string[] {
+		const dropped: string[] = [];
+		for (const [compositeKey, handle] of this.handles) {
+			if (now - handle.lastUsedMs > handle.ttlMs) {
+				handle.closed = true;
+				this.handles.delete(compositeKey);
+				const remaining = (this.slotLoad.get(handle.slotId) ?? 0) - 1;
+				if (remaining <= 0) {
+					this.slotLoad.delete(handle.slotId);
+				} else {
+					this.slotLoad.set(handle.slotId, remaining);
+				}
+				dropped.push(handle.conversationId);
+			}
+		}
+		return dropped;
+	}
+
+	/**
+	 * Snapshot every currently-open handle. Used by the shutdown path to
+	 * emit a save-state request per slot.
+	 */
+	snapshot(): readonly ConversationHandle[] {
+		return [...this.handles.values()];
+	}
+
+	/** Largest concurrent open count seen since the registry was created. */
+	highWater(): number {
+		return this.highWaterMark;
+	}
+
+	/** Number of currently-open handles. */
+	size(): number {
+		return this.handles.size;
+	}
+
+	/**
+	 * Recommended `--parallel` slot count given the observed high-water mark
+	 * of concurrently-open conversations plus a small headroom (max(2, 25%)).
+	 * The engine's auto-tune (J4) compares this against the running server's
+	 * slot count: when this is larger AND there's RAM headroom, it restarts
+	 * llama-server with the higher value so new conversations get their own
+	 * KV slots instead of thrashing.
+	 *
+	 * `running` is the currently-configured slot count; when the high-water
+	 * mark hasn't outgrown it, this returns `running` (no resize needed) so
+	 * callers can compare against equality without a second branch.
+	 */
+	recommendedParallel(running: number): number {
+		const headroom = Math.max(2, Math.ceil(this.highWaterMark * 0.25));
+		const desired = Math.max(1, this.highWaterMark + headroom);
+		return Math.max(running, desired);
+	}
+
+	/**
+	 * Drop every handle and reset the high-water mark + slot-load bookkeeping.
+	 * Test-only — the module singleton leaks state across files when the suite
+	 * runs together; call this in `beforeEach` to isolate. Not part of the
+	 * runtime contract.
+	 */
+	__resetForTests(): void {
+		for (const handle of this.handles.values()) handle.closed = true;
+		this.handles.clear();
+		this.slotLoad.clear();
+		this.highWaterMark = 0;
+	}
+
+	/**
+	 * Pick the slot with the fewest in-flight handles. Ties are broken by a
+	 * deterministic hash of the conversation id, which avoids consistently
+	 * loading slot 0 when N concurrent opens race.
+	 */
+	private pickLowestLoadedSlot(
+		parallel: number,
+		conversationId: string,
+	): number {
+		if (parallel <= 1) return 0;
+		let bestSlot = 0;
+		let bestLoad = Number.POSITIVE_INFINITY;
+		for (let slot = 0; slot < parallel; slot += 1) {
+			const load = this.slotLoad.get(slot) ?? 0;
+			if (load < bestLoad) {
+				bestLoad = load;
+				bestSlot = slot;
+			}
+		}
+		if (bestLoad === 0) return bestSlot;
+		// All slots are loaded equally — use the conversation hash for a
+		// deterministic tie-break. Same conversation, same slot when reopened.
+		const digest = createHash("sha256").update(conversationId).digest();
+		return digest.readUInt32BE(0) % parallel;
+	}
+
+	private compositeKey(conversationId: string, modelId: string): string {
+		return `${modelId}::${conversationId}`;
+	}
+}
+
+/**
+ * Module-singleton registry. The engine reads this on every generate; the
+ * conversation lifecycle API (`openConversation`, `closeConversation`)
+ * mutates it.
+ */
+export const conversationRegistry = new ConversationRegistry();

package/src/services/desktop-fused-ffi-backend-runtime.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Desktop production `FfiBackendRuntime` over the FUSED `libelizainference` —
+ * the SOLE desktop text runtime now that libllama has been retired.
+ *
+ * Desktop text generation runs through the fused library: the same
+ * `eliza_inference_llm_stream_*` ABI (v9) the voice subsystem already loads,
+ * so text + voice share one native lib, one GGML pin, and one resident text
+ * model.
+ *
+ *   - The fused lib's `eliza_inference_llm_stream_open` loads the bundle's text
+ *     GGUF (`<bundleRoot>/text/*.gguf`) and applies same-file MTP speculative
+ *     decoding + KV-cache quant + per-load GPU layers natively (ABI v9). The
+ *     path is gated on the capability probes
+ *     (`llmStreamSupported && llmMtpSupported && llmKvQuantSupported`).
+ *   - A fused lib that lacks MTP / KV-quant / native tokenize is REFUSED by
+ *     `supported()` → the engine raises LocalInferenceUnavailable. There is no
+ *     libllama fallback and never an unoptimized fused loop.
+ *
+ * Tokenization runs over the fused handle's resident text vocab via ABI-v9
+ * `eliza_inference_tokenize`: the fused `create()` + first `llmStreamOpen`
+ * already made the text vocab resident, so no second model is loaded.
+ * `tokenizeSupported()` gates this; a pre-v9 lib without the symbol is refused.
+ *
+ * Lifecycle: one fused context per loaded model; `acquire()` builds it,
+ * `release()` tears it down. A throwing native free poisons the runtime so no
+ * new allocation happens over leaked resources.
+ */
+import type { BackendPlan } from "./backend";
+import type { FfiBackendRuntime, FfiBackendSession } from "./ffi-streaming-backend";
+/**
+ * Resolve the on-disk path to the fused `libelizainference`. Precedence:
+ *   1. `ELIZA_INFERENCE_LIBRARY` — an explicit absolute path.
+ *   2. `<bundleRoot>/lib/<name>` — the bundle-local lib.
+ *   3. `ELIZA_INFERENCE_LIB_DIR/<name>` — an explicit lib directory.
+ *   4. `<stateDir>/local-inference/lib/<name>` — the default staging dir written
+ *      by `scripts/stage-desktop-fused-lib.mjs`, so a staged desktop build is
+ *      found with no env wiring.
+ * Returns null when none of the candidates exist on disk — `supported()` then
+ * reports unavailable and the engine raises LocalInferenceUnavailable.
+ */
+export declare function resolveFusedLibraryPath(bundleRoot: string | null, env?: NodeJS.ProcessEnv): string | null;
+export declare class DesktopFusedFfiBackendRuntime implements FfiBackendRuntime {
+    private active;
+    private poisonedError;
+    /** Cached `supported()` result so the engine gate and the dispatcher agree. */
+    private supportedCache;
+    /**
+     * Viable only when:
+     *   - bun:ffi resolves on the current runtime,
+     *   - the fused dylib is present AND reports ABI-v9 capability: the
+     *     streaming-LLM surface, same-file MTP, KV-cache quant, AND native
+     *     tokenization (`eliza_inference_tokenize`).
+     * A pre-v9 fused lib reports the probes as unsupported → refused, and the
+     * engine raises LocalInferenceUnavailable. libllama has been retired; there
+     * is no fallback runtime and no tokenizer sidecar.
+     */
+    supported(): boolean;
+    /** Clear the cached `supported()` result (tests / lib swaps). */
+    resetSupportedCache(): void;
+    private computeSupported;
+    acquire(plan: BackendPlan): Promise<FfiBackendSession>;
+    parallelSlots(): number;
+    /**
+     * Whether the LIVE session can describe images through the fused
+     * `eliza_inference_describe_image`. Mirrors the FfiStreamingBackend gate:
+     * true only when a session is bound and the fused lib exposes vision.
+     */
+    visionSupported(): boolean;
+    /**
+     * Vision describe through the fused `eliza_inference_describe_image`
+     * (ABI v9). Reuses the mtmd machinery linked for ASR over the bundle's text
+     * model + the passed mmproj projector. The `FfiStreamingBackend` forwards
+     * `describeImage`/`visionSupported` to this runtime by duck-typing.
+     */
+    describeImage(args: {
+        imageBytes: Uint8Array;
+        mmprojPath: string;
+        prompt?: string;
+        maxTokens?: number;
+        temperature?: number;
+        signal?: AbortSignal;
+    }): Promise<{
+        text: string;
+        projectorMs?: number;
+        decodeMs?: number;
+    }>;
+    release(): Promise<void>;
+}
+/**
+ * Process singleton — the engine wires this as the sole `FfiBackendRuntime` for
+ * the dispatcher's `"llama-cpp"` slot. The ABI-v9 capability probes in
+ * `supported()` gate whether the fused lib serves text at all.
+ */
+export declare const desktopFusedFfiBackendRuntime: DesktopFusedFfiBackendRuntime;
+//# sourceMappingURL=desktop-fused-ffi-backend-runtime.d.ts.map

package/src/services/desktop-fused-ffi-backend-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"desktop-fused-ffi-backend-runtime.d.ts","sourceRoot":"","sources":["desktop-fused-ffi-backend-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAOH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EACX,iBAAiB,EACjB,iBAAiB,EACjB,MAAM,yBAAyB,CAAC;AAsBjC;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CACtC,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,GAAG,GAAE,MAAM,CAAC,UAAwB,GAClC,MAAM,GAAG,IAAI,CAgBf;AAmBD,qBAAa,6BAA8B,YAAW,iBAAiB;IACtE,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,aAAa,CAAsB;IAC3C,+EAA+E;IAC/E,OAAO,CAAC,cAAc,CAAwB;IAE9C;;;;;;;;;OASG;IACH,SAAS,IAAI,OAAO;IAMpB,iEAAiE;IACjE,mBAAmB,IAAI,IAAI;IAI3B,OAAO,CAAC,gBAAgB;IAsClB,OAAO,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAgF5D,aAAa,IAAI,MAAM;IAMvB;;;;OAIG;IACH,eAAe,IAAI,OAAO;IAS1B;;;;;OAKG;IACG,aAAa,CAAC,IAAI,EAAE;QACzB,UAAU,EAAE,UAAU,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,WAAW,CAAC;KACrB,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IA4BhE,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB9B;AAED;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,+BACN,CAAC"}