@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/service.d.ts

@@ -0,0 +1,128 @@
+/**
+ * Public facade for the local-inference service.
+ *
+ * Single entry point used by the API routes, settings UI, and orchestration
+ * code. Holds singleton instances of the downloader
+ * and active-model coordinator so subscribers receive the same event
+ * stream across the process.
+ */
+import { type AgentRuntime } from "@elizaos/core";
+import { type LocalInferenceLoadOverrides } from "./active-model";
+import { MemoryArbiter } from "./memory-arbiter";
+import { type RecommendedModelSelection } from "./recommendation";
+import type { ActiveModelState, AgentModelSlot, CatalogModel, DownloadEvent, DownloadJob, HardwareProbe, LocalInferenceReadiness, ModelAssignments, ModelHubSnapshot, TextGenerationSlot } from "./types";
+import { type VerifyResult } from "./verify";
+export declare class LocalInferenceService {
+    private readonly downloader;
+    private readonly activeModel;
+    private bundledBootstrap;
+    /**
+     * Memory Arbiter (WS1). Lazily created on first access so the heavy
+     * pressure-source machinery doesn't run for processes that never load
+     * a local model (CI, dev shells, etc.). Once created, the arbiter is
+     * also published via `setMemoryArbiter` so cross-plugin consumers
+     * (plugin-vision, plugin-image-gen) can use `getMemoryArbiter()`.
+     */
+    private memoryArbiter;
+    /**
+     * Mobile pressure bridge — populated by the Capacitor host (iOS / Android
+     * onTrimMemory) so a native pressure callback can reach the arbiter.
+     * Stays null on desktop until WS2/WS8 wire the native side.
+     */
+    private mobilePressureBridge;
+    private imageGenCapabilityRegistered;
+    getCatalog(): CatalogModel[];
+    /**
+     * Register any bundled GGUF files staged by the AOSP build (or any
+     * other install path that drops a `manifest.json` next to the model
+     * files) into the registry. Runs at most once per process; the
+     * promise is cached so concurrent first callers wait on the same
+     * work.
+     */
+    private bootstrapBundled;
+    getInstalled(): Promise<import("@elizaos/shared").InstalledModel[]>;
+    getHardware(): Promise<HardwareProbe>;
+    getDownloads(): DownloadJob[];
+    getActive(): ActiveModelState;
+    getAssignments(): Promise<ModelAssignments>;
+    setSlotAssignment(slot: AgentModelSlot, modelId: string | null): Promise<ModelAssignments>;
+    snapshot(): Promise<ModelHubSnapshot>;
+    getTextReadiness(): Promise<LocalInferenceReadiness>;
+    getRecommendedModel(slot: TextGenerationSlot, hardware?: HardwareProbe): Promise<RecommendedModelSelection>;
+    getRecommendedModels(hardware?: HardwareProbe): Promise<Record<TextGenerationSlot, RecommendedModelSelection>>;
+    /**
+     * Kernel capability probing is now owned by the native FFI runtime. Null
+     * means "no static CAPABILITIES.json probe"; the dispatcher still enforces
+     * runtime-required kernels at load time.
+     */
+    private installedBinaryKernels;
+    startDownload(modelIdOrSpec: string | CatalogModel): Promise<DownloadJob>;
+    startSmallerFallbackDownload(currentModelId: string, slot?: TextGenerationSlot, hardware?: HardwareProbe): Promise<{
+        model: CatalogModel;
+        job: DownloadJob;
+    } | null>;
+    searchHuggingFace(query: string, limit?: number): Promise<CatalogModel[]>;
+    searchModelHub(query: string, hub: "huggingface" | "modelscope", limit?: number): Promise<CatalogModel[]>;
+    /**
+     * Verify an installed model's file integrity. When the model was a
+     * Eliza-download and there was no stored sha256 yet (legacy entry), the
+     * computed hash is persisted so subsequent verifies have a baseline.
+     */
+    verifyModel(id: string): Promise<VerifyResult>;
+    cancelDownload(modelId: string): boolean;
+    subscribeDownloads(listener: (event: DownloadEvent) => void): () => void;
+    subscribeActive(listener: (state: ActiveModelState) => void): () => void;
+    setActive(runtime: AgentRuntime | null, modelId: string, overrides?: LocalInferenceLoadOverrides): Promise<ActiveModelState>;
+    prewarmActiveVoice(modelId: string): Promise<boolean>;
+    /**
+     * Warm the Stage-1 stable prefix after an explicit model activation.
+     *
+     * `ensureLocalInferenceHandler` also attempts this at runtime boot, but
+     * desktop activation often happens later through `/api/local-inference/active`;
+     * at boot there may be no resident model, so that early warmup correctly
+     * stays inactive. Running it here closes that gap without blocking activation.
+     */
+    prewarmSystemPrefix(runtime: AgentRuntime): Promise<boolean>;
+    clearActive(runtime: AgentRuntime | null): Promise<ActiveModelState>;
+    /**
+     * Diagnostic snapshot of the local prefix-cache state. Returns:
+     *   - `engine`: in-process session-pool size and live cache keys.
+     * Used by the API layer to render a "local cache" debug panel.
+     */
+    getLocalCacheStats(): Promise<{
+        engine: {
+            size: number;
+            maxSize: number;
+            keys: string[];
+        } | null;
+    }>;
+    /**
+     * Memory Arbiter (WS1). Returns the process-wide arbiter, creating it on
+     * first call. The arbiter is constructed against the engine's existing
+     * `SharedResourceRegistry` so eviction policy is consistent across the
+     * voice/text paths and the cross-plugin handles.
+     *
+     * The pressure source is a composite of:
+     *   - `nodeOsPressureSource()` — desktop polling at 5 s.
+     *   - A `capacitorPressureSource()` bridge — populated by the Capacitor
+     *     host on iOS/Android. The native side calls `dispatchMobilePressure`
+     *     when the OS hands it a memory-warning callback.
+     */
+    getMemoryArbiter(): MemoryArbiter;
+    private registerImageGenCapability;
+    private loadImageGenBackend;
+    private resolveImageGenLoadArgs;
+    private imageGenFileExists;
+    /**
+     * Capacitor bridge entrypoint. The mobile host (iOS / Android) calls
+     * this from the native pressure callback. Safe to call before the
+     * arbiter has been created — we create it on demand.
+     */
+    dispatchMobilePressure(level: "nominal" | "low" | "critical", freeMb?: number): void;
+    uninstall(modelId: string): Promise<{
+        removed: boolean;
+        reason?: "external" | "not-found";
+    }>;
+}
+export declare const localInferenceService: LocalInferenceService;
+//# sourceMappingURL=service.d.ts.map

package/src/services/service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["service.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,OAAO,EACN,KAAK,YAAY,EAIjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEN,KAAK,2BAA2B,EAChC,MAAM,gBAAgB,CAAC;AAqBxB,OAAO,EACN,aAAa,EAGb,MAAM,kBAAkB,CAAC;AAS1B,OAAO,EAEN,KAAK,yBAAyB,EAG9B,MAAM,kBAAkB,CAAC;AAW1B,OAAO,KAAK,EACX,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,WAAW,EACX,aAAa,EACb,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,KAAK,YAAY,EAAwB,MAAM,UAAU,CAAC;AA0DnE,qBAAa,qBAAqB;IAKjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAExB;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgC;IAC5D,OAAO,CAAC,gBAAgB,CAA8B;IACtD;;;;;;OAMG;IACH,OAAO,CAAC,aAAa,CAA8B;IACnD;;;;OAIG;IACH,OAAO,CAAC,oBAAoB,CAEZ;IAChB,OAAO,CAAC,4BAA4B,CAAS;IAE7C,UAAU;IAIV;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IASlB,YAAY;IAKZ,WAAW,IAAI,OAAO,CAAC,aAAa,CAAC;IAI3C,YAAY,IAAI,WAAW,EAAE;IAI7B,SAAS,IAAI,gBAAgB;IAIvB,cAAc,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAI3C,iBAAiB,CACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,MAAM,GAAG,IAAI,GACpB,OAAO,CAAC,gBAAgB,CAAC;IAKtB,QAAQ,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAyBrC,gBAAgB,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAcpD,mBAAmB,CACxB,IAAI,EAAE,kBAAkB,EACxB,QAAQ,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,yBAAyB,CAAC;IAS/B,oBAAoB,CACzB,QAAQ,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,yBAAyB,CAAC,CAAC;IAQjE;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAIxB,aAAa,CAClB,aAAa,EAAE,MAAM,GAAG,YAAY,GAClC,OAAO,CAAC,WAAW,CAAC;IAIjB,4BAA4B,CACjC,cAAc,EAAE,MAAM,EACtB,IAAI,GAAE,kBAAiC,EACvC,QAAQ,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC;QAAE,KAAK,EAAE,YAAY,CAAC;QAAC,GAAG,EAAE,WAAW,CAAA;KAAE,GAAG,IAAI,CAAC;IActD,iBAAiB,CACtB,KAAK,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,YAAY,EAAE,CAAC;IAIpB,cAAc,CACnB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,aAAa,GAAG,YAAY,EACjC,KAAK,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,YAAY,EAAE,CAAC;IAI1B;;;;OAIG;IACG,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAoCpD,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAIxC,kBAAkB,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,GAAG,MAAM,IAAI;IAIxE,eAAe,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,MAAM,IAAI;IAIlE,SAAS,CACd,OAAO,EAAE,YAAY,GAAG,IAAI,EAC5B,OAAO,EAAE,MAAM,EACf,SAAS,CAAC,EAAE,2BAA2B,GACrC,OAAO,CAAC,gBAAgB,CAAC;IAyBtB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAI3D;;;;;;;OAOG;IACG,mBAAmB,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC;IAwB5D,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAI1E;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC;QACnC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,EAAE,CAAA;SAAE,GAAG,IAAI,CAAC;KACjE,CAAC;IAMF;;;;;;;;;;;OAWG;IACH,gBAAgB,IAAI,aAAa;IAoFjC,OAAO,CAAC,0BAA0B;YAiBpB,mBAAmB;YAyDnB,uBAAuB;IAmDrC,OAAO,CAAC,kBAAkB;IAI1B;;;;OAIG;IACH,sBAAsB,CACrB,KAAK,EAAE,SAAS,GAAG,KAAK,GAAG,UAAU,EACrC,MAAM,CAAC,EAAE,MAAM,GACb,IAAI;IAKD,SAAS,CACd,OAAO,EAAE,MAAM,GACb,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;KAAE,CAAC;CAQnE;AAwBD,eAAO,MAAM,qBAAqB,uBAA8B,CAAC"}

package/src/services/service.test.ts

@@ -0,0 +1,223 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { AgentRuntime } from "@elizaos/core";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const coreMocks = vi.hoisted(() => ({
+	debug: vi.fn(),
+	renderMessageHandlerStablePrefix: vi.fn(async () => "stable-stage1-prefix"),
+}));
+
+vi.mock("@elizaos/core", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("@elizaos/core")>();
+	return {
+		...actual,
+		logger: {
+			...actual.logger,
+			debug: coreMocks.debug,
+		},
+		renderMessageHandlerStablePrefix:
+			coreMocks.renderMessageHandlerStablePrefix,
+	};
+});
+
+import { ActiveModelCoordinator } from "./active-model";
+import { localInferenceEngine } from "./engine";
+import {
+	readRoutingPreferences,
+	writeRoutingPreferences,
+} from "./routing-preferences";
+import { LocalInferenceService } from "./service";
+import type { ActiveModelState, InstalledModel } from "./types";
+
+function makeInstalledModel(id: string): InstalledModel {
+	return {
+		id,
+		displayName: id,
+		path: `/tmp/${id}.gguf`,
+		sizeBytes: 1024,
+		installedAt: "2026-05-15T00:00:00.000Z",
+		lastUsedAt: null,
+		source: "external-scan",
+	};
+}
+
+function readyState(modelId: string): ActiveModelState {
+	return {
+		modelId,
+		loadedAt: "2026-05-15T00:00:01.000Z",
+		status: "ready",
+		loadedContextSize: 32768,
+		loadedCacheTypeK: "q8_0",
+		loadedCacheTypeV: "q8_0",
+		loadedGpuLayers: 99,
+	};
+}
+
+function makeRuntime(): AgentRuntime {
+	return {
+		agentId: "agent-test",
+	} as unknown as AgentRuntime;
+}
+
+describe("LocalInferenceService activation prewarm", () => {
+	let originalStateDir: string | undefined;
+	let tempStateDir: string | null = null;
+
+	beforeEach(async () => {
+		originalStateDir = process.env.ELIZA_STATE_DIR;
+		tempStateDir = await mkdtemp(join(tmpdir(), "eliza-service-test-"));
+		process.env.ELIZA_STATE_DIR = tempStateDir;
+	});
+
+	afterEach(async () => {
+		vi.restoreAllMocks();
+		coreMocks.debug.mockReset();
+		coreMocks.renderMessageHandlerStablePrefix.mockClear();
+		if (originalStateDir === undefined) {
+			delete process.env.ELIZA_STATE_DIR;
+		} else {
+			process.env.ELIZA_STATE_DIR = originalStateDir;
+		}
+		if (tempStateDir) {
+			await rm(tempStateDir, { recursive: true, force: true });
+			tempStateDir = null;
+		}
+	});
+
+	it("prewarms the Stage-1 system prefix after a model becomes active", async () => {
+		const service = new LocalInferenceService();
+		const runtime = makeRuntime();
+		const installed = makeInstalledModel("eliza-1-test");
+		const prewarm = vi
+			.spyOn(localInferenceEngine, "prewarmConversation")
+			.mockResolvedValue(true);
+		const voiceReady = vi
+			.spyOn(localInferenceEngine, "ensureActiveBundleVoiceReady")
+			.mockResolvedValue({} as never);
+		const transcribe = vi
+			.spyOn(localInferenceEngine, "transcribePcm")
+			.mockResolvedValue("");
+		const synthesize = vi
+			.spyOn(localInferenceEngine, "synthesizeSpeech")
+			.mockResolvedValue(new Uint8Array());
+
+		vi.spyOn(service, "getInstalled").mockResolvedValue([installed]);
+		vi.spyOn(ActiveModelCoordinator.prototype, "switchTo").mockResolvedValue(
+			readyState(installed.id),
+		);
+		vi.spyOn(localInferenceEngine, "hasLoadedModel").mockReturnValue(true);
+		vi.spyOn(localInferenceEngine, "activeBackendId").mockReturnValue(
+			"llama-cpp",
+		);
+
+		await expect(service.setActive(runtime, installed.id)).resolves.toEqual(
+			expect.objectContaining({ modelId: installed.id, status: "ready" }),
+		);
+
+		await vi.waitFor(() => {
+			expect(prewarm).toHaveBeenCalledWith(
+				"__system_prefix__",
+				"stable-stage1-prefix",
+			);
+		});
+		expect(voiceReady).toHaveBeenCalledOnce();
+		expect(transcribe).toHaveBeenCalledWith({
+			pcm: expect.any(Float32Array),
+			sampleRate: 16_000,
+		});
+		expect(synthesize).toHaveBeenCalledWith("Hello.");
+		expect(coreMocks.renderMessageHandlerStablePrefix).toHaveBeenCalledWith(
+			runtime,
+			"agent-test",
+		);
+	});
+
+	it("does not prewarm when activation is not ready", async () => {
+		const service = new LocalInferenceService();
+		const runtime = makeRuntime();
+		const installed = makeInstalledModel("eliza-1-test");
+		const prewarm = vi.spyOn(localInferenceEngine, "prewarmConversation");
+
+		vi.spyOn(service, "getInstalled").mockResolvedValue([installed]);
+		vi.spyOn(ActiveModelCoordinator.prototype, "switchTo").mockResolvedValue({
+			modelId: installed.id,
+			loadedAt: null,
+			status: "error",
+			error: "load failed",
+		});
+
+		await expect(service.setActive(runtime, installed.id)).resolves.toEqual(
+			expect.objectContaining({ modelId: installed.id, status: "error" }),
+		);
+
+		expect(prewarm).not.toHaveBeenCalled();
+	});
+
+	it("routes stale capacitor text preferences to eliza-local-inference after activation", async () => {
+		const service = new LocalInferenceService();
+		const installed = makeInstalledModel("eliza-1-test");
+
+		await writeRoutingPreferences({
+			preferredProvider: {
+				TEXT_SMALL: "capacitor-llama",
+				TEXT_LARGE: "capacitor-llama",
+			},
+			policy: {
+				TEXT_SMALL: "manual",
+				TEXT_LARGE: "manual",
+			},
+		});
+		vi.spyOn(service, "getInstalled").mockResolvedValue([installed]);
+		vi.spyOn(ActiveModelCoordinator.prototype, "switchTo").mockResolvedValue(
+			readyState(installed.id),
+		);
+
+		await service.setActive(null, installed.id);
+
+		await expect(readRoutingPreferences()).resolves.toMatchObject({
+			preferredProvider: {
+				TEXT_SMALL: "eliza-local-inference",
+				TEXT_LARGE: "eliza-local-inference",
+			},
+			policy: {
+				TEXT_SMALL: "manual",
+				TEXT_LARGE: "manual",
+			},
+		});
+	});
+
+	it("does not overwrite an explicit cloud text provider during activation", async () => {
+		const service = new LocalInferenceService();
+		const installed = makeInstalledModel("eliza-1-test");
+
+		await writeRoutingPreferences({
+			preferredProvider: {
+				TEXT_SMALL: "anthropic",
+				TEXT_LARGE: "anthropic",
+			},
+			policy: {
+				TEXT_SMALL: "manual",
+				TEXT_LARGE: "manual",
+			},
+		});
+		vi.spyOn(service, "getInstalled").mockResolvedValue([installed]);
+		vi.spyOn(ActiveModelCoordinator.prototype, "switchTo").mockResolvedValue(
+			readyState(installed.id),
+		);
+
+		await service.setActive(null, installed.id);
+
+		await expect(readRoutingPreferences()).resolves.toMatchObject({
+			preferredProvider: {
+				TEXT_SMALL: "anthropic",
+				TEXT_LARGE: "anthropic",
+			},
+			policy: {
+				TEXT_SMALL: "manual",
+				TEXT_LARGE: "manual",
+			},
+		});
+	});
+});