@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/src/services/memory-arbiter.test.ts

@@ -167,6 +167,78 @@ describe("MemoryArbiter — fit-to-budget LRU eviction", () => {
 		).toBe(true);
 	});
 
+	it("counts the engine's external footprint so evictToFit fires for the dominant roles (#8809 AC#1)", async () => {
+		const registry = new SharedResourceRegistry();
+		const events: ArbiterEvent[] = [];
+		// Budget 1000 MB. The active text/embedding bundle — the dominant
+		// resident consumer — is owned by the engine, NOT the arbiter's resident
+		// map, and is reported via externalFootprintMb=500. Two 400 MB capability
+		// models then can't both fit on top of it.
+		const arbiter = new MemoryArbiter({
+			registry,
+			now,
+			budgetMb: () => 1000,
+			externalFootprintMb: () => 500,
+		});
+		arbiter.onEvent((e) => events.push(e));
+		const embedding = makeCapability("embedding", {
+			residentRole: "embedding",
+			estimatedMb: 400,
+		});
+		const vision = makeCapability("vision-describe", {
+			residentRole: "vision",
+			estimatedMb: 400,
+		});
+		arbiter.registerCapability(embedding.registration);
+		arbiter.registerCapability(vision.registration);
+
+		clock = 1;
+		const e = await arbiter.acquire("embedding", "emb"); // 500 + 400 = 900 ≤ 1000
+		await e.release();
+		clock = 2;
+		const v = await arbiter.acquire("vision-describe", "vl"); // +400 → 1300 > 1000
+		await v.release();
+
+		// Without the external term this stays a silent no-op (800 ≤ 1000) and
+		// both models pile on top of the text bundle, overcommitting RAM. With
+		// it, the fit path evicts the LRU resident (emb).
+		expect(embedding.unloads).toEqual(["emb"]);
+		expect(arbiter.residentSnapshot().map((s) => s.modelKey)).toEqual(["vl"]);
+		expect(
+			events.some((e) => e.type === "eviction" && e.reason === "fit"),
+		).toBe(true);
+	});
+
+	it("keeps both models resident when no external footprint is reported", async () => {
+		const registry = new SharedResourceRegistry();
+		// externalFootprintMb defaults to 0 (the pre-#8809 accounting): 400 + 400
+		// ≤ 1000 → both fit, no eviction. This is the control for the test above.
+		const arbiter = new MemoryArbiter({ registry, now, budgetMb: () => 1000 });
+		const embedding = makeCapability("embedding", {
+			residentRole: "embedding",
+			estimatedMb: 400,
+		});
+		const vision = makeCapability("vision-describe", {
+			residentRole: "vision",
+			estimatedMb: 400,
+		});
+		arbiter.registerCapability(embedding.registration);
+		arbiter.registerCapability(vision.registration);
+
+		const e = await arbiter.acquire("embedding", "emb");
+		await e.release();
+		const v = await arbiter.acquire("vision-describe", "vl");
+		await v.release();
+
+		expect(embedding.unloads).toEqual([]);
+		expect(
+			arbiter
+				.residentSnapshot()
+				.map((s) => s.modelKey)
+				.sort(),
+		).toEqual(["emb", "vl"]);
+	});
+
 	it("never evicts the text target to make room", async () => {
 		const registry = new SharedResourceRegistry();
 		const arbiter = new MemoryArbiter({ registry, now, budgetMb: () => 1000 });
@@ -278,6 +350,73 @@ describe("MemoryArbiter — fit-to-budget LRU eviction", () => {
 });
 
 describe("MemoryArbiter — memory pressure", () => {
+	it("preloads under nominal pressure when the budget fits", async () => {
+		const registry = new SharedResourceRegistry();
+		const arbiter = new MemoryArbiter({ registry, now, budgetMb: () => 1000 });
+		const embedding = makeCapability("embedding", {
+			residentRole: "embedding",
+			estimatedMb: 400,
+		});
+		arbiter.registerCapability(embedding.registration);
+
+		const loaded = await arbiter.preload("embedding", "emb");
+
+		expect(loaded).toBe(true);
+		expect(embedding.loads).toEqual(["emb"]);
+		expect(arbiter.residentSnapshot()).toMatchObject([
+			{ capability: "embedding", modelKey: "emb", refCount: 0 },
+		]);
+		await arbiter.shutdown();
+	});
+
+	it("refuses preload under memory pressure", async () => {
+		const registry = new SharedResourceRegistry();
+		const source: CapacitorPressureSource = capacitorPressureSource({ now });
+		const arbiter = new MemoryArbiter({
+			registry,
+			pressureSource: source,
+			now,
+			budgetMb: () => 1000,
+		});
+		const embedding = makeCapability("embedding", {
+			residentRole: "embedding",
+			estimatedMb: 400,
+		});
+		arbiter.registerCapability(embedding.registration);
+		arbiter.start();
+
+		source.dispatch("low");
+		await flush();
+		expect(await arbiter.preload("embedding", "emb-low")).toBe(false);
+
+		source.dispatch("critical");
+		await flush();
+		expect(await arbiter.preload("embedding", "emb-critical")).toBe(false);
+		expect(embedding.loads).toEqual([]);
+		await arbiter.shutdown();
+	});
+
+	it("refuses preload when the resident set would exceed budget", async () => {
+		const registry = new SharedResourceRegistry();
+		const arbiter = new MemoryArbiter({ registry, now, budgetMb: () => 1000 });
+		const text = makeCapability("text", { estimatedMb: 700 });
+		const embedding = makeCapability("embedding", {
+			residentRole: "embedding",
+			estimatedMb: 400,
+		});
+		arbiter.registerCapability(text.registration);
+		arbiter.registerCapability(embedding.registration);
+
+		await (await arbiter.acquire("text", "txt")).release();
+
+		expect(await arbiter.preload("embedding", "emb")).toBe(false);
+		expect(embedding.loads).toEqual([]);
+		expect(arbiter.residentSnapshot()).toMatchObject([
+			{ capability: "text", modelKey: "txt" },
+		]);
+		await arbiter.shutdown();
+	});
+
 	it("critical evicts every non-text role but keeps the text target", async () => {
 		const registry = new SharedResourceRegistry();
 		const source: CapacitorPressureSource = capacitorPressureSource({ now });

package/src/services/memory-arbiter.ts

@@ -46,7 +46,7 @@
  *   arbiter.registerCapability({
  *     capability: "vision-describe",
  *     residentRole: "vision",
- *     load: async (modelKey) => loadQwen3VL(modelKey),
+ *     load: async (modelKey) => loadVisionModel(modelKey),
  *     unload: async (handle) => handle.dispose(),
  *     run: async (handle, req) => handle.describe(req.imageBytes),
  *   });
@@ -56,13 +56,13 @@
  *
  *   ```ts
  *   const result = await arbiter.requestVisionDescribe({
- *     modelKey: "qwen3-vl-4b",
+ *     modelKey: "gemma-vl-4b",
  *     imageBytes: pixels,
  *   });
  *   ```
  *
  * The arbiter handles:
- *   1. Acquiring (or reusing) the handle for `qwen3-vl-4b`.
+ *   1. Acquiring (or reusing) the handle for `gemma-vl-4b`.
  *   2. If a different capability holds the active model and we need to
  *      swap, evicting it first.
  *   3. Running the request.
@@ -110,6 +110,9 @@ export type ArbiterCapability =
 	| "image-gen"
 	| "transcribe";
 
+/** Identifies the arbiter as the registry's eviction-decision owner (#8809 AC#2). */
+const MEMORY_ARBITER_EVICTION_OWNER = "memory-arbiter";
+
 /**
  * Map a capability to the resident-role bucket the existing
  * `SharedResourceRegistry` already tracks. Adding a new capability MUST
@@ -254,6 +257,20 @@ export interface MemoryArbiterOptions {
 	 * Production wiring passes `os.totalmem()/MB - ramHeadroomReserveMb()`.
 	 */
 	budgetMb?: () => number | null;
+	/**
+	 * Resident footprint (MB) the arbiter does NOT own in its `resident` map
+	 * but which still consumes the host budget — the text-target and embedding
+	 * weights loaded by `LocalInferenceEngine` on its own
+	 * `SharedResourceRegistry` (#8809 M10b). Without this hook the arbiter's
+	 * `residentFootprintMb()` sees only vision/image-gen and the proactive
+	 * `evictToFit` path never trips (the two dominant resident consumers are
+	 * invisible to it). Production wiring (service.ts) reads the engine's
+	 * `text-target` + `embedding` resident estimates. These weights are owned by
+	 * the engine and are never the arbiter's eviction target — they are pure
+	 * budget accounting, exactly like the pinned text-target in the resident map.
+	 * Defaults to 0 (no external footprint known).
+	 */
+	externalFootprintMb?: () => number;
 }
 
 /**
@@ -268,6 +285,7 @@ export class MemoryArbiter {
 	private readonly log?: MemoryArbiterOptions["logger"];
 	private readonly now: () => number;
 	private readonly budgetMb: () => number | null;
+	private readonly externalFootprintMb: () => number;
 
 	private readonly capabilities = new Map<
 		ArbiterCapability,
@@ -308,6 +326,7 @@ export class MemoryArbiter {
 		this.log = opts.logger;
 		this.now = opts.now ?? (() => Date.now());
 		this.budgetMb = opts.budgetMb ?? (() => null);
+		this.externalFootprintMb = opts.externalFootprintMb ?? (() => 0);
 	}
 
 	/** Begin observing memory pressure. Idempotent. */
@@ -326,6 +345,9 @@ export class MemoryArbiter {
 				);
 			});
 		});
+		// This arbiter is now the single eviction-decision owner for the shared
+		// registry; the simpler MemoryMonitor poll defers to it (#8809 AC#2).
+		this.registry.claimEvictionOwnership(MEMORY_ARBITER_EVICTION_OWNER);
 	}
 
 	/** Stop observing pressure. Does NOT evict resident handles. */
@@ -335,6 +357,7 @@ export class MemoryArbiter {
 			this.pressureUnsubscribe = null;
 		}
 		this.pressureSource?.stop();
+		this.registry.releaseEvictionOwnership(MEMORY_ARBITER_EVICTION_OWNER);
 	}
 
 	/** Tear down: stop pressure observation and unload every resident handle. */
@@ -384,11 +407,7 @@ export class MemoryArbiter {
 		}
 		this.capabilities.set(
 			registration.capability,
-			registration as unknown as CapabilityRegistration<
-				unknown,
-				unknown,
-				unknown
-			>,
+			registration as CapabilityRegistration<unknown, unknown, unknown>,
 		);
 	}
 
@@ -422,6 +441,44 @@ export class MemoryArbiter {
 		return this.currentPressure;
 	}
 
+	/**
+	 * Predictive warm-load. Unlike `acquire`, preload never creates pressure to
+	 * make itself happen: it only loads when the system is nominal and the
+	 * configured budget proves the incoming resident footprint fits without
+	 * evicting anything. Returns `false` when the guard declines the preload.
+	 */
+	async preload(
+		capability: ArbiterCapability,
+		modelKey: string,
+	): Promise<boolean> {
+		const registration = this.capabilities.get(capability);
+		if (!registration) {
+			throw new Error(
+				`[memory-arbiter] no capability registered for "${capability}"`,
+			);
+		}
+		if (this.shuttingDown || this.currentPressure !== "nominal") {
+			return false;
+		}
+		const key = this.residentKey(capability, modelKey);
+		const existing = this.resident.get(key);
+		if (existing) {
+			existing.lastUsedAt = this.now();
+			return true;
+		}
+		const incomingMb = registration.estimatedMb ?? 0;
+		const budget = this.budgetMb();
+		if (budget === null || budget <= 0 || incomingMb <= 0) {
+			return false;
+		}
+		if (this.residentFootprintMb() + incomingMb > budget) {
+			return false;
+		}
+		const entry = await this.loadOrReuse(registration, modelKey);
+		entry.lastUsedAt = this.now();
+		return true;
+	}
+
 	/**
 	 * Acquire a handle for `(capability, modelKey)`. If the model is already
 	 * resident the refcount is bumped and we return immediately; otherwise we
@@ -682,19 +739,28 @@ export class MemoryArbiter {
 		if (budget === null || budget <= 0) return;
 		if (incomingMb <= 0) return;
 
-		const residentMb = (): number => {
-			let sum = 0;
-			for (const e of this.resident.values()) sum += e.estimatedMb;
-			return sum;
-		};
-
-		while (residentMb() + incomingMb > budget) {
+		while (this.residentFootprintMb() + incomingMb > budget) {
 			const candidate = this.lruEvictionCandidate();
 			if (!candidate) break;
 			await this.evictEntry(candidate, "fit");
 		}
 	}
 
+	/**
+	 * Total resident footprint counted against the budget: the arbiter's own
+	 * resident handles PLUS the engine-owned text-target + embedding weights
+	 * reported via `externalFootprintMb` (#8809 M10b). The external term is what
+	 * makes `evictToFit` actually fire — without it the two dominant resident
+	 * consumers (text, embedding) are invisible and the fit path is dead.
+	 */
+	private residentFootprintMb(): number {
+		let sum = 0;
+		for (const e of this.resident.values()) sum += e.estimatedMb;
+		const external = this.externalFootprintMb();
+		if (Number.isFinite(external) && external > 0) sum += external;
+		return sum;
+	}
+
 	/**
 	 * The next entry the fit path should drop: least-recently-used among
 	 * evictable residents (refcount 0, not the text target). Returns null when

package/src/services/memory-benchmark.test.ts

@@ -0,0 +1,91 @@
+import { MODEL_CATALOG } from "@elizaos/shared/local-inference";
+import { describe, expect, it } from "vitest";
+import {
+	buildMemoryBenchmarkPlan,
+	buildMemoryBenchmarkReport,
+	summarizeMemoryBenchmark,
+} from "./memory-benchmark";
+import type { HardwareProbe, InstalledModel } from "./types";
+
+function hardware(freeRamGb: number): HardwareProbe {
+	return {
+		totalRamGb: 16,
+		freeRamGb,
+		gpu: null,
+		cpuCores: 8,
+		platform: "darwin",
+		arch: "arm64",
+		appleSilicon: true,
+		recommendedBucket: "mid",
+		source: "os-fallback",
+	};
+}
+
+describe("memory benchmark report", () => {
+	it("marks the device-fit Eliza-1 tier and records curated resident estimates", () => {
+		const plan = buildMemoryBenchmarkPlan({
+			catalog: MODEL_CATALOG,
+			installed: [
+				{
+					id: "eliza-1-2b",
+					displayName: "Eliza-1 2B",
+					path: "/tmp/eliza-1-2b.bundle/text/eliza-1-2b-128k.gguf",
+					sizeBytes: 1_500_000_000,
+					bundleRoot: "/tmp/eliza-1-2b.bundle",
+					bundleSizeBytes: 2_000_000_000,
+					source: "eliza-download",
+					installedAt: "2026-06-22T00:00:00.000Z",
+					lastUsedAt: null,
+				} satisfies InstalledModel,
+			],
+			hardware: hardware(4.5),
+		});
+
+		const twoB = plan.find((model) => model.modelId === "eliza-1-2b");
+		expect(twoB?.installed).toBe(true);
+		expect(twoB?.selectedByDeviceFit).toBe(true);
+		// Gemma 4 cutover: KV is stock q8_0 (Gemma's MQA + windowed-SWA +
+		// shared-KV is already minimal; the head_dim=128 QJL kernel is retired).
+		expect(twoB?.plannedKvQuant).toBe("q8_0");
+		expect(twoB?.estimatedResidentMb).toBeGreaterThan(0);
+
+		const larger = plan.find((model) => model.modelId === "eliza-1-9b");
+		expect(larger?.fit).toBe("tight");
+		expect(larger?.selectedByDeviceFit).toBe(false);
+
+		const largest = plan.find((model) => model.modelId === "eliza-1-27b-256k");
+		expect(largest?.fit).toBe("wontfit");
+	});
+
+	it("summarizes load and telemetry counts", async () => {
+		const report = await buildMemoryBenchmarkReport(
+			{
+				catalog: MODEL_CATALOG,
+				installed: [],
+				hardware: hardware(2),
+			},
+			[
+				{
+					type: "model_load",
+					capability: "text",
+					modelKey: "m",
+					loadMs: 3,
+					atMs: 1,
+				},
+				{
+					type: "eviction",
+					capability: "text",
+					modelKey: "m",
+					reason: "fit",
+					estimatedMb: 512,
+					atMs: 2,
+				},
+			],
+		);
+
+		expect(report.deviceFit.modelId).toBeNull();
+		expect(report.telemetry.modelLoads).toBe(1);
+		expect(report.telemetry.evictions).toBe(1);
+		expect(summarizeMemoryBenchmark(report)).toContain("cloud fallback");
+	});
+});