@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/lib-target.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, it } from "vitest";
+import {
+	libStagedName,
+	resolveHostLibTargets,
+	selectBundleLibFiles,
+} from "./lib-target";
+import { Eliza1LibFileEntrySchema } from "./manifest";
+
+const SHA = "a".repeat(64);
+const entry = (target: string, p: string, name?: string) => ({
+	target,
+	path: p,
+	sha256: SHA,
+	...(name ? { name } : {}),
+});
+
+describe("resolveHostLibTargets", () => {
+	it("pins to ELIZA_INFERENCE_LIB_TARGET when set", () => {
+		expect(
+			resolveHostLibTargets({
+				env: { ELIZA_INFERENCE_LIB_TARGET: "win-x64-cuda" },
+				platform: "linux",
+				arch: "x64",
+			}),
+		).toEqual(["win-x64-cuda"]);
+	});
+
+	it("returns [] on mobile (phones ship the lib natively)", () => {
+		expect(
+			resolveHostLibTargets({
+				env: { ELIZA_PLATFORM: "android" },
+				platform: "linux",
+				arch: "arm64",
+			}),
+		).toEqual([]);
+		expect(
+			resolveHostLibTargets({
+				env: { ELIZA_PLATFORM: "ios" },
+				platform: "darwin",
+				arch: "arm64",
+			}),
+		).toEqual([]);
+	});
+
+	it("maps darwin/arm64 to the metal set first (metal carries CPU fallback)", () => {
+		expect(
+			resolveHostLibTargets({ env: {}, platform: "darwin", arch: "arm64" }),
+		).toEqual(["darwin-arm64-metal", "darwin-arm64-cpu", "darwin-arm64"]);
+	});
+
+	it("prefers CPU on win32 by default, GPU when preferGpu", () => {
+		expect(
+			resolveHostLibTargets({ env: {}, platform: "win32", arch: "x64" }),
+		).toEqual(["win-x64-cpu", "win-x64", "win-x64-cuda"]);
+		expect(
+			resolveHostLibTargets({
+				env: {},
+				platform: "win32",
+				arch: "x64",
+				preferGpu: true,
+			}),
+		).toEqual(["win-x64-cuda", "win-x64-cpu", "win-x64"]);
+	});
+
+	it("maps linux/x64", () => {
+		expect(
+			resolveHostLibTargets({ env: {}, platform: "linux", arch: "x64" }),
+		).toEqual(["linux-x64-cpu", "linux-x64", "linux-x64-cuda"]);
+	});
+});
+
+describe("selectBundleLibFiles", () => {
+	it("returns null when the bundle has no lib[]", () => {
+		expect(selectBundleLibFiles({ files: {} }, ["win-x64-cpu"])).toBeNull();
+		expect(
+			selectBundleLibFiles({ files: { lib: [] } }, ["win-x64-cpu"]),
+		).toBeNull();
+	});
+
+	it("returns null when no host target matches", () => {
+		expect(
+			selectBundleLibFiles(
+				{ files: { lib: [entry("linux-x64-cpu", "lib/x.so")] } },
+				["win-x64-cpu", "win-x64"],
+			),
+		).toBeNull();
+	});
+
+	it("picks the first matching target in preference order, with all its files", () => {
+		const lib = [
+			entry("win-x64-cpu", "lib/cpu/elizainference.dll"),
+			entry("win-x64-cpu", "lib/cpu/ggml.dll"),
+			entry("win-x64-cuda", "lib/cuda/elizainference.dll"),
+		];
+		const cudaFirst = selectBundleLibFiles({ files: { lib } }, [
+			"win-x64-cuda",
+			"win-x64-cpu",
+		]);
+		expect(cudaFirst?.target).toBe("win-x64-cuda");
+		expect(cudaFirst?.files).toHaveLength(1);
+
+		const cpuFirst = selectBundleLibFiles({ files: { lib } }, [
+			"win-x64-cpu",
+			"win-x64-cuda",
+		]);
+		expect(cpuFirst?.target).toBe("win-x64-cpu");
+		expect(cpuFirst?.files).toHaveLength(2);
+	});
+});
+
+describe("libStagedName", () => {
+	it("uses the basename of path", () => {
+		expect(libStagedName(entry("t", "lib/win-x64/elizainference.dll"))).toBe(
+			"elizainference.dll",
+		);
+	});
+
+	it("prefers name when set, reduced to a basename (no path traversal)", () => {
+		expect(libStagedName(entry("t", "a/b.dll", "custom.dll"))).toBe(
+			"custom.dll",
+		);
+		expect(libStagedName(entry("t", "a/b.dll", "../../evil.dll"))).toBe(
+			"evil.dll",
+		);
+	});
+});
+
+describe("Eliza1LibFileEntrySchema", () => {
+	it("accepts a valid platform-targeted lib entry", () => {
+		const e = Eliza1LibFileEntrySchema.parse({
+			path: "lib/win-x64/elizainference.dll",
+			sha256: SHA,
+			target: "win-x64-cpu",
+			name: "elizainference.dll",
+		});
+		expect(e.target).toBe("win-x64-cpu");
+		expect(e.name).toBe("elizainference.dll");
+	});
+
+	it("requires a target", () => {
+		expect(() =>
+			Eliza1LibFileEntrySchema.parse({ path: "x", sha256: SHA }),
+		).toThrow();
+	});
+});

package/src/services/lib-target.ts

@@ -0,0 +1,102 @@
+/**
+ * Host → fused-lib target resolution for bundle delivery (#9105 / local-
+ * inference).
+ *
+ * An Eliza-1 bundle's `files.lib[]` carries the fused `libelizainference` SET
+ * (the fused lib + its ggml/llama/mtmd sibling backends) per platform `target`.
+ * The downloader fetches ONLY the set matching the host into `<bundleRoot>/lib/`,
+ * which the desktop FFI runtime resolves with no env wiring
+ * (`desktop-fused-ffi-backend-runtime.ts` `resolveFusedLibraryPath`, path #2).
+ *
+ * Pure + injectable (platform / arch / env are all overridable) so the mapping
+ * is unit-testable on any host.
+ */
+
+import type { Eliza1Files, Eliza1LibFileEntry } from "./manifest";
+
+export interface HostLibTargetOptions {
+	platform?: NodeJS.Platform;
+	arch?: string;
+	env?: NodeJS.ProcessEnv;
+	/** Prefer a GPU-accelerated target when the bundle hosts one. */
+	preferGpu?: boolean;
+}
+
+function platformKey(platform: NodeJS.Platform): string {
+	if (platform === "win32") return "win";
+	return platform; // "linux" | "darwin"
+}
+
+/**
+ * Ordered list of acceptable `target` keys for the host, best-first. The
+ * downloader picks the first target the bundle actually hosts.
+ *
+ *   - `ELIZA_INFERENCE_LIB_TARGET` pins a single target (power users / CI).
+ *   - Mobile (`ELIZA_PLATFORM=android|ios`) returns `[]` — phones ship the lib
+ *     natively (jniLibs / xcframework), never via bundle delivery.
+ *   - CPU is preferred by default: the fused lib always has GGML_CPU built in,
+ *     so the CPU set works on every host and is the smallest download. GPU sets
+ *     (`…-cuda`) are opt-in via `preferGpu` or the env pin.
+ *   - macOS arm64 maps to the `metal` set, which itself carries the CPU
+ *     fallback, so it is the canonical mac target.
+ */
+export function resolveHostLibTargets(
+	opts: HostLibTargetOptions = {},
+): string[] {
+	const platform = opts.platform ?? process.platform;
+	const arch = opts.arch ?? process.arch;
+	const env = opts.env ?? process.env;
+
+	const mobile = (env.ELIZA_PLATFORM ?? "").toLowerCase();
+	if (mobile === "android" || mobile === "ios") return [];
+
+	const pinned = env.ELIZA_INFERENCE_LIB_TARGET?.trim();
+	if (pinned) return [pinned];
+
+	const base = `${platformKey(platform)}-${arch}`;
+	const cpu = `${base}-cpu`;
+	const cuda = `${base}-cuda`;
+	const metal = `${base}-metal`;
+
+	if (platform === "darwin") {
+		// The metal set carries a CPU fallback, so it is the mac default.
+		return [metal, cpu, base];
+	}
+	return opts.preferGpu === true ? [cuda, cpu, base] : [cpu, base, cuda];
+}
+
+export interface SelectedLibTarget {
+	readonly target: string;
+	readonly files: ReadonlyArray<Eliza1LibFileEntry>;
+}
+
+/**
+ * Pick the bundle's lib file set for the first host target the manifest
+ * actually carries. Returns null when the bundle has no `lib[]` or hosts no
+ * target the host accepts (⇒ the runtime falls back to other resolution paths,
+ * ultimately cloud).
+ */
+export function selectBundleLibFiles(
+	manifest: { files: Pick<Eliza1Files, "lib"> },
+	targets: ReadonlyArray<string>,
+): SelectedLibTarget | null {
+	const lib = manifest.files.lib;
+	if (!lib || lib.length === 0) return null;
+	for (const target of targets) {
+		const files = lib.filter((e) => e.target === target);
+		if (files.length > 0) return { target, files };
+	}
+	return null;
+}
+
+/**
+ * Flat staged filename under `<bundleRoot>/lib/` for a lib entry. Always a
+ * basename (any directory component in `name`/`path` is stripped) so a manifest
+ * can never write outside the lib dir.
+ */
+export function libStagedName(entry: Eliza1LibFileEntry): string {
+	const raw =
+		entry.name && entry.name.trim().length > 0 ? entry.name.trim() : entry.path;
+	const parts = raw.split(/[\\/]/);
+	return parts[parts.length - 1] || raw;
+}

package/src/services/live-signals.test.ts

@@ -0,0 +1,132 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { inferenceTelemetry } from "./inference-telemetry";
+import {
+	estimateDecodeTokens,
+	type LiveDeviceSignals,
+	liveSignalsDemoteLocal,
+	MIN_DECODE_TPS_BUDGET,
+	readLiveDeviceSignals,
+	recordDecodeThroughput,
+	setLiveDeviceSignalsSource,
+} from "./live-signals";
+
+afterEach(() => {
+	setLiveDeviceSignalsSource(null);
+	inferenceTelemetry.reset();
+});
+
+const signals = (over: Partial<LiveDeviceSignals>): LiveDeviceSignals => ({
+	thermalState: null,
+	decodeTokensPerSecond: null,
+	...over,
+});
+
+describe("liveSignalsDemoteLocal", () => {
+	it("demotes on serious thermal", () => {
+		expect(liveSignalsDemoteLocal(signals({ thermalState: "serious" }))).toBe(
+			true,
+		);
+	});
+
+	it("demotes on critical thermal", () => {
+		expect(liveSignalsDemoteLocal(signals({ thermalState: "critical" }))).toBe(
+			true,
+		);
+	});
+
+	it("does not demote on nominal / fair thermal", () => {
+		expect(liveSignalsDemoteLocal(signals({ thermalState: "nominal" }))).toBe(
+			false,
+		);
+		expect(liveSignalsDemoteLocal(signals({ thermalState: "fair" }))).toBe(
+			false,
+		);
+	});
+
+	it("demotes when decode TPS is below budget", () => {
+		expect(
+			liveSignalsDemoteLocal(
+				signals({ decodeTokensPerSecond: MIN_DECODE_TPS_BUDGET - 1 }),
+			),
+		).toBe(true);
+	});
+
+	it("does not demote at or above the TPS budget", () => {
+		expect(
+			liveSignalsDemoteLocal(
+				signals({ decodeTokensPerSecond: MIN_DECODE_TPS_BUDGET }),
+			),
+		).toBe(false);
+		expect(
+			liveSignalsDemoteLocal(signals({ decodeTokensPerSecond: 100 })),
+		).toBe(false);
+	});
+
+	it("does not demote when both signals are unmeasured (null)", () => {
+		expect(liveSignalsDemoteLocal(signals({}))).toBe(false);
+	});
+});
+
+describe("readLiveDeviceSignals source injection", () => {
+	it("returns whatever the injected source provides", () => {
+		setLiveDeviceSignalsSource(() =>
+			signals({ thermalState: "serious", decodeTokensPerSecond: 3 }),
+		);
+		const out = readLiveDeviceSignals();
+		expect(out.thermalState).toBe("serious");
+		expect(out.decodeTokensPerSecond).toBe(3);
+	});
+
+	it("restores the default source when set to null", () => {
+		setLiveDeviceSignalsSource(() =>
+			signals({ thermalState: "critical", decodeTokensPerSecond: 1 }),
+		);
+		setLiveDeviceSignalsSource(null);
+		// Default source reads the (empty, in test) device bridge + telemetry:
+		// no device connected and no decode samples → both null.
+		const out = readLiveDeviceSignals();
+		expect(out.thermalState).toBeNull();
+		expect(out.decodeTokensPerSecond).toBeNull();
+	});
+});
+
+describe("estimateDecodeTokens", () => {
+	it("approximates ~4 characters per token", () => {
+		expect(estimateDecodeTokens("")).toBe(0);
+		expect(estimateDecodeTokens("abcd")).toBe(1);
+		expect(estimateDecodeTokens("a".repeat(40))).toBe(10);
+	});
+});
+
+describe("recordDecodeThroughput → default source → demotion (end-to-end)", () => {
+	it("a sub-budget generation makes the default source demote local", () => {
+		// 8 tokens in 2 s = 4 tok/s, below the 6 tok/s budget.
+		recordDecodeThroughput({ tokens: 8, elapsedMs: 2000 });
+
+		const out = readLiveDeviceSignals();
+		expect(out.decodeTokensPerSecond).toBeLessThan(MIN_DECODE_TPS_BUDGET);
+		expect(out.decodeTokensPerSecond).toBeCloseTo(4, 5);
+		expect(liveSignalsDemoteLocal(out)).toBe(true);
+	});
+
+	it("a healthy generation does not demote local", () => {
+		// 100 tokens in 1 s = 100 tok/s, comfortably above budget.
+		recordDecodeThroughput({ tokens: 100, elapsedMs: 1000 });
+
+		const out = readLiveDeviceSignals();
+		expect(out.decodeTokensPerSecond).toBeGreaterThanOrEqual(
+			MIN_DECODE_TPS_BUDGET,
+		);
+		expect(liveSignalsDemoteLocal(out)).toBe(false);
+	});
+
+	it("drops noise samples so the signal stays null", () => {
+		recordDecodeThroughput({ tokens: 2, elapsedMs: 1000 }); // too few tokens
+		recordDecodeThroughput({ tokens: 50, elapsedMs: 0 }); // clock didn't advance
+		recordDecodeThroughput({ tokens: 50, elapsedMs: -10 }); // negative span
+
+		const out = readLiveDeviceSignals();
+		expect(out.decodeTokensPerSecond).toBeNull();
+		expect(liveSignalsDemoteLocal(out)).toBe(false);
+	});
+});

package/src/services/live-signals.ts

@@ -0,0 +1,177 @@
+/**
+ * Live device signals for the capability-driven `auto` routing policy.
+ *
+ * `classifyDeviceTier()` is a *static* assessment — it scores the hardware once
+ * and caches the result. But a device that classifies MAX/GOOD at session start
+ * can still become a poor place to run local inference *right now*: a sustained
+ * workload pushes the SoC into thermal throttling, or decode throughput collapses
+ * under memory pressure. Those are live, time-varying signals the static tier
+ * cannot see.
+ *
+ * This module exposes the two live signals the `auto` branch demotes on:
+ *
+ *   - **thermal state** — read from the device bridge's most recent device
+ *     capabilities (`nominal | fair | serious | critical | unknown`). `serious`
+ *     or `critical` means the OS is actively throttling clocks; forcing local
+ *     inference there makes turns slower than cloud and worsens the throttle.
+ *   - **decode throughput** — the recent p50 decode tokens/sec from
+ *     `inferenceTelemetry`. When it falls below a usable budget the on-device
+ *     path is no longer competitive and `auto` routes to cloud.
+ *
+ * Both signals are *optional*: a quantity the host could not measure is reported
+ * as `null` (never a fabricated `0`), and a `null` signal never demotes — the
+ * static tier decision stands. The default source reads the real subsystems; a
+ * test or alternate host can inject its own via `setLiveDeviceSignalsSource`.
+ */
+
+import { deviceBridge } from "./device-bridge";
+import { inferenceTelemetry } from "./inference-telemetry";
+
+/** Thermal pressure reported by the OS. Mirrors the device-bridge wire enum. */
+export type ThermalState =
+	| "nominal"
+	| "fair"
+	| "serious"
+	| "critical"
+	| "unknown";
+
+/**
+ * The telemetry metric name carrying recent on-device decode throughput
+ * (tokens/sec). The backend records this after a generation completes; until
+ * then `summary()` returns a `count: 0` row and the signal is `null`.
+ */
+export const DECODE_TPS_METRIC = "inference.decode_tps";
+
+/**
+ * Minimum decode throughput (tokens/sec) below which the local path is no longer
+ * worth keeping under `auto`. Roughly the floor where on-device generation feels
+ * slower than a cloud round-trip on a constrained mobile SoC.
+ */
+export const MIN_DECODE_TPS_BUDGET = 6;
+
+/**
+ * Minimum decoded-token count a generation must produce before its throughput is
+ * trusted as a routing signal. A one- or two-token reply is dominated by prompt
+ * prefill + sampling fixed costs, so its tok/s is noise — recording it would
+ * smear the p50 the router demotes on. Real turns clear this floor easily.
+ */
+const MIN_TOKENS_FOR_TPS_SAMPLE = 4;
+
+/**
+ * Approximate decoded-token count for a generated string when an exact tokenizer
+ * count is not reachable from the call site. English averages ~4 characters per
+ * token; this is deliberately coarse — it only feeds a p50 the router compares
+ * against {@link MIN_DECODE_TPS_BUDGET}, never a billing or context-window count.
+ */
+export function estimateDecodeTokens(text: string): number {
+	return Math.ceil(text.length / 4);
+}
+
+/**
+ * Record one on-device decode-throughput observation into the telemetry ring the
+ * `auto` router reads via {@link DECODE_TPS_METRIC}. This is the producer that
+ * activates the dormant TPS demotion signal: backends call it after a real local
+ * generation completes with a known decoded-token count and wall-clock decode
+ * span.
+ *
+ * Drops the sample (records nothing) when it would be noise rather than signal:
+ * an empty/aborted generation, a sub-{@link MIN_TOKENS_FOR_TPS_SAMPLE} reply, or
+ * a non-positive elapsed time (a clock that did not advance). Never throws —
+ * `inferenceTelemetry.record` already guards the call site.
+ */
+export function recordDecodeThroughput(sample: {
+	tokens: number;
+	elapsedMs: number;
+}): void {
+	const { tokens, elapsedMs } = sample;
+	if (
+		!Number.isFinite(tokens) ||
+		tokens < MIN_TOKENS_FOR_TPS_SAMPLE ||
+		!Number.isFinite(elapsedMs) ||
+		elapsedMs <= 0
+	) {
+		return;
+	}
+	const tokPerSec = tokens / (elapsedMs / 1000);
+	if (!Number.isFinite(tokPerSec) || tokPerSec <= 0) {
+		return;
+	}
+	inferenceTelemetry.record(DECODE_TPS_METRIC, tokPerSec);
+}
+
+/** A point-in-time snapshot of the live device signals. */
+export interface LiveDeviceSignals {
+	/** Current OS thermal pressure, or `null` when no device reported it. */
+	thermalState: ThermalState | null;
+	/** Recent p50 decode throughput in tokens/sec, or `null` when unmeasured. */
+	decodeTokensPerSecond: number | null;
+}
+
+/** A source of live device signals — swappable for tests and alternate hosts. */
+export type LiveDeviceSignalsSource = () => LiveDeviceSignals;
+
+const VALID_THERMAL_STATES: ReadonlySet<string> = new Set([
+	"nominal",
+	"fair",
+	"serious",
+	"critical",
+	"unknown",
+]);
+
+function normalizeThermalState(value: unknown): ThermalState | null {
+	return typeof value === "string" && VALID_THERMAL_STATES.has(value)
+		? (value as ThermalState)
+		: null;
+}
+
+/**
+ * The default source: thermal from the device bridge's primary device, decode
+ * tok/s from the telemetry ring. Pure reads — never throws into the router.
+ */
+const defaultSource: LiveDeviceSignalsSource = () => {
+	const thermalState = normalizeThermalState(
+		deviceBridge.status().capabilities?.thermalState,
+	);
+	const decodeTokensPerSecond =
+		inferenceTelemetry.summary(DECODE_TPS_METRIC).p50;
+	return { thermalState, decodeTokensPerSecond };
+};
+
+let activeSource: LiveDeviceSignalsSource = defaultSource;
+
+/** Read the current live device signals. */
+export function readLiveDeviceSignals(): LiveDeviceSignals {
+	return activeSource();
+}
+
+/**
+ * Override the live-signals source. Pass `null` to restore the default
+ * (device-bridge + telemetry) source. Intended for tests and alternate hosts.
+ */
+export function setLiveDeviceSignalsSource(
+	source: LiveDeviceSignalsSource | null,
+): void {
+	activeSource = source ?? defaultSource;
+}
+
+/**
+ * Whether the live signals say the device should be demoted off local for now,
+ * even though the static tier favours local. True when the OS is throttling
+ * (`serious`/`critical`) or measured decode throughput sits below the budget.
+ * A `null` signal never demotes.
+ */
+export function liveSignalsDemoteLocal(signals: LiveDeviceSignals): boolean {
+	if (
+		signals.thermalState === "serious" ||
+		signals.thermalState === "critical"
+	) {
+		return true;
+	}
+	if (
+		signals.decodeTokensPerSecond !== null &&
+		signals.decodeTokensPerSecond < MIN_DECODE_TPS_BUDGET
+	) {
+		return true;
+	}
+	return false;
+}