@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/memory-pressure.ts

@@ -0,0 +1,414 @@
+/**
+ * Cross-platform memory-pressure sources for the Memory Arbiter (WS1).
+ *
+ * The arbiter consumes a `MemoryPressureSource` interface so it doesn't
+ * care whether the signal comes from polling `os.freemem()`, an Android
+ * `ComponentCallbacks2.onTrimMemory()` callback, or an iOS
+ * `UIApplicationDidReceiveMemoryWarningNotification` subscription.
+ *
+ * Three concrete implementations live here:
+ *
+ *   1. `nodeOsPressureSource` — desktop / server. Polls `os.freemem()` /
+ *      `os.totalmem()` on a fixed cadence (~5 s). Emits `nominal` /
+ *      `low` / `critical` from a two-threshold high-water mark over total
+ *      RAM. The polling timer is `unref`'d so it never holds the process
+ *      open.
+ *
+ *   2. `capacitorPressureSource` — Capacitor iOS/Android shim. The native
+ *      side (Android `ComponentCallbacks2.onTrimMemory`, iOS
+ *      `UIApplicationDidReceiveMemoryWarningNotification`) calls
+ *      `dispatch(level)` via the bridge, which fans the level out to
+ *      arbiter listeners. The JS contract is fixed here; the native
+ *      modules will be wired in WS2 / WS8.
+ *
+ *   3. `compositePressureSource` — combine sources (e.g. one OS poll +
+ *      one Capacitor bridge). The composite emits the *worst* reported
+ *      level so neither signal can mask the other.
+ *
+ * Pressure semantics — what each level means to the arbiter:
+ *   - `nominal`  → no action; the arbiter may load freely.
+ *   - `low`      → the arbiter SHOULD evict the lowest-priority resident
+ *                   role and may pause optimistic preloads.
+ *   - `critical` → the arbiter MUST evict every non-text-target role
+ *                   immediately; `acquire()` for non-text capabilities is
+ *                   allowed to throw.
+ *
+ * No fallback sludge:
+ *   - When a probe fails (e.g. `os.freemem()` returns NaN under sandboxing),
+ *     the source emits `nominal` and logs a warning. It does NOT pretend
+ *     the system is critical and start trashing models.
+ *   - `dispatch(level)` from a native bridge is trusted; the JS layer does
+ *     not second-guess it with its own poll.
+ */
+
+import { readSystemMemory } from "./system-memory";
+
+/** Pressure level the arbiter consumes. */
+export type MemoryPressureLevel = "nominal" | "low" | "critical";
+
+export interface MemoryPressureEvent {
+	level: MemoryPressureLevel;
+	/** Optional free RAM, MB. Present for poll-based sources; absent for OS-callback bridges. */
+	freeMb?: number;
+	/** Optional total RAM, MB. */
+	totalMb?: number;
+	/** Free as a fraction of total (0..1). Present when both above are present. */
+	freeFraction?: number;
+	/** Source identifier — handy for telemetry. `os-poll` / `capacitor` / `composite`. */
+	source: string;
+	/** Wall-clock ms when the event was generated. */
+	atMs: number;
+}
+
+export type MemoryPressureListener = (event: MemoryPressureEvent) => void;
+
+export interface MemoryPressureSource {
+	/** Stable identifier for the source (used in telemetry tags). */
+	readonly id: string;
+	/** Subscribe to pressure events. Returns the unsubscribe fn. */
+	subscribe(listener: MemoryPressureListener): () => void;
+	/** Begin observing. Idempotent. */
+	start(): void;
+	/** Stop observing. Idempotent. */
+	stop(): void;
+	/** Take a one-shot reading without subscribing. May reuse the cached level. */
+	current(): MemoryPressureEvent;
+}
+
+const BYTES_PER_MB = 1024 * 1024;
+
+export interface NodeOsPressureConfig {
+	/** Poll interval, ms. Default 5000 (5 s); min 500 ms. */
+	intervalMs: number;
+	/**
+	 * Below this fraction of total RAM, level becomes `low`. Default 0.15.
+	 */
+	lowWaterFraction: number;
+	/**
+	 * Below this fraction of total RAM, level becomes `critical`. Default 0.05.
+	 * Must be < lowWaterFraction; the source enforces this at construction.
+	 */
+	criticalWaterFraction: number;
+	/** Optional injected clock (testing). Defaults to `Date.now()`. */
+	now?: () => number;
+}
+
+export interface NodeOsPressureSources {
+	/**
+	 * Available/total memory in bytes. Defaults to `readSystemMemory()`
+	 * (`/proc/meminfo` `MemAvailable` on Linux/Android, `os.freemem()` elsewhere).
+	 */
+	osMemory?: () => { freeBytes: number; totalBytes: number };
+	/** Optional logger; warnings only. */
+	logger?: { warn?: (m: string) => void };
+}
+
+const NODE_OS_DEFAULTS: NodeOsPressureConfig = {
+	intervalMs: 5_000,
+	lowWaterFraction: 0.15,
+	criticalWaterFraction: 0.05,
+};
+
+/**
+ * Polling pressure source for desktop/server. The poll happens on an
+ * `unref`'d timer so the process can exit naturally. Listeners only fire
+ * when the level *changes* (or the first time after `start()`); raw
+ * readings are still available via `current()` for logging.
+ */
+export function nodeOsPressureSource(
+	overrides: Partial<NodeOsPressureConfig> = {},
+	sources: NodeOsPressureSources = {},
+): MemoryPressureSource {
+	const config: NodeOsPressureConfig = {
+		intervalMs: Math.max(
+			500,
+			overrides.intervalMs ?? NODE_OS_DEFAULTS.intervalMs,
+		),
+		lowWaterFraction: clampFraction(
+			overrides.lowWaterFraction,
+			NODE_OS_DEFAULTS.lowWaterFraction,
+		),
+		criticalWaterFraction: clampFraction(
+			overrides.criticalWaterFraction,
+			NODE_OS_DEFAULTS.criticalWaterFraction,
+		),
+		now: overrides.now,
+	};
+	if (config.criticalWaterFraction >= config.lowWaterFraction) {
+		throw new Error(
+			`[memory-pressure] criticalWaterFraction (${config.criticalWaterFraction}) must be < lowWaterFraction (${config.lowWaterFraction})`,
+		);
+	}
+	const probe = sources.osMemory ?? (() => readSystemMemory());
+	const now = config.now ?? (() => Date.now());
+	const listeners = new Set<MemoryPressureListener>();
+	let timer: NodeJS.Timeout | null = null;
+	let lastLevel: MemoryPressureLevel | null = null;
+	let lastEvent: MemoryPressureEvent = {
+		level: "nominal",
+		source: "os-poll",
+		atMs: now(),
+	};
+
+	const sample = (): MemoryPressureEvent => {
+		const { freeBytes, totalBytes } = probe();
+		const totalMb = Math.round(totalBytes / BYTES_PER_MB);
+		const freeMb = Math.round(freeBytes / BYTES_PER_MB);
+		if (!Number.isFinite(freeMb) || !Number.isFinite(totalMb) || totalMb <= 0) {
+			sources.logger?.warn?.(
+				`[memory-pressure] os memory probe returned invalid (free=${freeMb}, total=${totalMb}); reporting nominal`,
+			);
+			return { level: "nominal", source: "os-poll", atMs: now() };
+		}
+		const freeFraction = freeMb / totalMb;
+		const level: MemoryPressureLevel =
+			freeFraction <= config.criticalWaterFraction
+				? "critical"
+				: freeFraction <= config.lowWaterFraction
+					? "low"
+					: "nominal";
+		return {
+			level,
+			freeMb,
+			totalMb,
+			freeFraction,
+			source: "os-poll",
+			atMs: now(),
+		};
+	};
+
+	const tick = (): void => {
+		const event = sample();
+		lastEvent = event;
+		if (lastLevel === event.level) return;
+		lastLevel = event.level;
+		for (const listener of listeners) {
+			try {
+				listener(event);
+			} catch {
+				// Listener faults should never crash the poller. Drop it.
+				listeners.delete(listener);
+			}
+		}
+	};
+
+	return {
+		id: "os-poll",
+		subscribe(listener) {
+			listeners.add(listener);
+			// Fire the current level immediately so subscribers can react without
+			// waiting a full interval.
+			try {
+				listener(lastEvent);
+			} catch {
+				listeners.delete(listener);
+			}
+			return () => {
+				listeners.delete(listener);
+			};
+		},
+		start() {
+			if (timer) return;
+			tick();
+			const t = setInterval(tick, config.intervalMs);
+			t.unref();
+			timer = t;
+		},
+		stop() {
+			if (!timer) return;
+			clearInterval(timer);
+			timer = null;
+		},
+		current(): MemoryPressureEvent {
+			return sample();
+		},
+	};
+}
+
+function clampFraction(value: number | undefined, fallback: number): number {
+	if (value === undefined) return fallback;
+	if (!Number.isFinite(value) || value <= 0 || value >= 1) return fallback;
+	return value;
+}
+
+/**
+ * Capacitor-side bridge. The native module dispatches a level whenever
+ * the OS hands it a memory-pressure callback (Android: `onTrimMemory`,
+ * mapping `TRIM_MEMORY_RUNNING_LOW`/`TRIM_MEMORY_RUNNING_CRITICAL` to
+ * `low`/`critical`; iOS: `didReceiveMemoryWarning` → `critical`). The JS
+ * surface is `dispatch(level, freeMb?)`. The host wires this to the
+ * Capacitor plugin in WS2/WS8; here we own the contract + JS state.
+ */
+export interface CapacitorPressureSource extends MemoryPressureSource {
+	/** Called by the Capacitor native bridge whenever the OS signals pressure. */
+	dispatch(level: MemoryPressureLevel, freeMb?: number): void;
+}
+
+export function capacitorPressureSource(
+	opts: { now?: () => number } = {},
+): CapacitorPressureSource {
+	const now = opts.now ?? (() => Date.now());
+	const listeners = new Set<MemoryPressureListener>();
+	let lastEvent: MemoryPressureEvent = {
+		level: "nominal",
+		source: "capacitor",
+		atMs: now(),
+	};
+	let started = false;
+	return {
+		id: "capacitor",
+		subscribe(listener) {
+			listeners.add(listener);
+			try {
+				listener(lastEvent);
+			} catch {
+				listeners.delete(listener);
+			}
+			return () => {
+				listeners.delete(listener);
+			};
+		},
+		start() {
+			started = true;
+		},
+		stop() {
+			started = false;
+		},
+		current(): MemoryPressureEvent {
+			return lastEvent;
+		},
+		dispatch(level: MemoryPressureLevel, freeMb?: number): void {
+			if (!started) {
+				// Allow dispatch even before start() so the boot path can pre-load
+				// state from a recent OS callback; we just don't gate on it.
+			}
+			const event: MemoryPressureEvent = {
+				level,
+				source: "capacitor",
+				atMs: now(),
+				...(freeMb !== undefined ? { freeMb } : {}),
+			};
+			lastEvent = event;
+			for (const listener of listeners) {
+				try {
+					listener(event);
+				} catch {
+					listeners.delete(listener);
+				}
+			}
+		},
+	};
+}
+
+/**
+ * Combine multiple sources. Subscribers see the *worst* level reported by
+ * any underlying source — once a critical signal arrives, the composite
+ * stays critical until every source returns to nominal/low.
+ */
+export function compositePressureSource(
+	sources: ReadonlyArray<MemoryPressureSource>,
+	opts: { now?: () => number } = {},
+): MemoryPressureSource {
+	const now = opts.now ?? (() => Date.now());
+	// Index by position rather than by `source.id` — two underlying sources of
+	// the same kind (e.g. two `capacitor` bridges) must each get their own
+	// slot, otherwise the second one overwrites the first and we lose visibility
+	// into the first source's level.
+	const latestBySlot: (MemoryPressureEvent | null)[] = sources.map(() => null);
+	const listeners = new Set<MemoryPressureListener>();
+	const subs: Array<() => void> = [];
+
+	const worst = (): MemoryPressureEvent => {
+		let level: MemoryPressureLevel = "nominal";
+		let freeMb: number | undefined;
+		let totalMb: number | undefined;
+		for (const e of latestBySlot) {
+			if (!e) continue;
+			if (rank(e.level) > rank(level)) {
+				level = e.level;
+				freeMb = e.freeMb;
+				totalMb = e.totalMb;
+			}
+		}
+		const event: MemoryPressureEvent = {
+			level,
+			source: "composite",
+			atMs: now(),
+		};
+		if (freeMb !== undefined) event.freeMb = freeMb;
+		if (totalMb !== undefined) event.totalMb = totalMb;
+		if (freeMb !== undefined && totalMb !== undefined && totalMb > 0) {
+			event.freeFraction = freeMb / totalMb;
+		}
+		return event;
+	};
+
+	const fanout = (): void => {
+		const event = worst();
+		for (const listener of listeners) {
+			try {
+				listener(event);
+			} catch {
+				listeners.delete(listener);
+			}
+		}
+	};
+
+	let started = false;
+	let lastLevel: MemoryPressureLevel | null = null;
+
+	const handleAt = (slot: number) => (e: MemoryPressureEvent) => {
+		latestBySlot[slot] = e;
+		const w = worst();
+		if (lastLevel === w.level) return;
+		lastLevel = w.level;
+		fanout();
+	};
+
+	return {
+		id: "composite",
+		subscribe(listener) {
+			listeners.add(listener);
+			try {
+				listener(worst());
+			} catch {
+				listeners.delete(listener);
+			}
+			return () => {
+				listeners.delete(listener);
+			};
+		},
+		start() {
+			if (started) return;
+			started = true;
+			sources.forEach((s, idx) => {
+				s.start();
+				subs.push(s.subscribe(handleAt(idx)));
+			});
+		},
+		stop() {
+			if (!started) return;
+			started = false;
+			while (subs.length) {
+				const u = subs.pop();
+				try {
+					u?.();
+				} catch {
+					// Ignore — unsubscribe must never throw out.
+				}
+			}
+			for (const s of sources) s.stop();
+			for (let i = 0; i < latestBySlot.length; i++) latestBySlot[i] = null;
+			lastLevel = null;
+		},
+		current(): MemoryPressureEvent {
+			return worst();
+		},
+	};
+}
+
+function rank(level: MemoryPressureLevel): number {
+	if (level === "critical") return 2;
+	if (level === "low") return 1;
+	return 0;
+}

package/src/services/mtp-doctor.ts

@@ -0,0 +1,86 @@
+import {
+	ELIZA_1_HOSTED_MTP_TIER_IDS,
+	ELIZA_1_MTP_TIER_IDS,
+	MODEL_CATALOG,
+} from "./catalog";
+import { probeHardware } from "./hardware";
+
+export type MtpDoctorCheckStatus = "pass" | "warn" | "fail";
+
+export interface MtpDoctorCheck {
+	label: string;
+	status: MtpDoctorCheckStatus;
+	detail: string;
+	fix?: string;
+}
+
+export interface MtpDoctorReport {
+	ok: boolean;
+	checks: MtpDoctorCheck[];
+}
+
+export async function runMtpDoctor(): Promise<MtpDoctorReport> {
+	const checks: MtpDoctorCheck[] = [];
+	const mtpCatalogEntries = MODEL_CATALOG.filter(
+		(entry) => entry.runtime?.mtp !== undefined,
+	);
+
+	checks.push({
+		label: "catalog metadata",
+		status: mtpCatalogEntries.length > 0 ? "pass" : "fail",
+		detail:
+			mtpCatalogEntries.length > 0
+				? `${mtpCatalogEntries.length} MTP-capable model entries found`
+				: `No catalog entries advertise runtime.mtp; ${ELIZA_1_HOSTED_MTP_TIER_IDS.length}/${ELIZA_1_MTP_TIER_IDS.length} Gemma MTP drafter tiers have hosted GGUFs`,
+		fix:
+			mtpCatalogEntries.length > 0
+				? undefined
+				: "publish Gemma drafter GGUFs at bundles/<tier>/mtp/drafter-<tier>.gguf before enabling runtime MTP",
+	});
+
+	try {
+		const hardware = await probeHardware();
+		const backend = hardware.gpu?.backend ?? "cpu";
+		checks.push({
+			label: "hardware probe",
+			status: "pass",
+			detail: `${hardware.platform}/${hardware.arch}, backend=${backend}, ram=${hardware.totalRamGb}GB`,
+		});
+		if (backend === "cpu") {
+			checks.push({
+				label: "accelerator",
+				status: "warn",
+				detail:
+					"No GPU backend detected; MTP can run but may not improve latency",
+				fix: "install a Metal/CUDA/Vulkan-capable llama.cpp runtime",
+			});
+		}
+	} catch (error) {
+		checks.push({
+			label: "hardware probe",
+			status: "warn",
+			detail: error instanceof Error ? error.message : String(error),
+		});
+	}
+
+	checks.push({
+		label: "runtime mode",
+		status:
+			process.env.ELIZA_LOCAL_INFERENCE_BACKEND === "capacitor-llama"
+				? "warn"
+				: "pass",
+		detail:
+			process.env.ELIZA_LOCAL_INFERENCE_BACKEND === "capacitor-llama"
+				? "Backend override forces capacitor-llama, bypassing optimized llama.cpp MTP"
+				: "No backend override is blocking optimized llama.cpp selection",
+		fix:
+			process.env.ELIZA_LOCAL_INFERENCE_BACKEND === "capacitor-llama"
+				? "unset ELIZA_LOCAL_INFERENCE_BACKEND"
+				: undefined,
+	});
+
+	return {
+		ok: checks.every((check) => check.status !== "fail"),
+		checks,
+	};
+}