@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/device-tier.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Device-tier classifier — maps a `HardwareProbe` to a `DeviceTier`
+ * (`MAX | GOOD | OKAY | POOR`) and produces the warning copy + the
+ * recommended local-voice policy the runtime + UI consume.
+ *
+ * Single-pass classifier (no async I/O). The host probe already happened in
+ * `hardware.ts`; this module is pure arithmetic on top of the probe.
+ *
+ * Numeric thresholds and tier definitions are sourced verbatim from R9
+ * (`.swarm/research/R9-memory.md` §3.1) — keep this file's per-tier table
+ * in sync with that document.
+ *
+ *   - **MAX** — all voice + LM models can be **parallelized** and held
+ *     resident at once (~24 GB effective model RAM AND ≥ 16 GB free at
+ *     session start AND a dGPU with ≥ 16 GB VRAM OR an Apple Silicon
+ *     Pro/Max/Ultra with ≥ 32 GB shared RAM).
+ *   - **GOOD** — all models can be loaded into memory and run, but
+ *     serialized (~12 GB effective model RAM AND ≥ 8 GB free at session
+ *     start AND a dGPU with ≥ 8 GB VRAM, OR Apple Silicon base ≥ 16 GB,
+ *     OR x86 CPU-only ≥ 32 GB).
+ *   - **OKAY** — models load and unload per turn; caching breaks across
+ *     swaps (~6 GB effective model RAM AND ≥ 3 GB free).
+ *   - **POOR** — severe issues; refuse local voice, recommend cloud.
+ *
+ * Mobile clamps to **OKAY** at best regardless of RAM because the OS
+ * background-task model breaks long-running local inference (iOS jetsam
+ * 3–4 GB ceiling; Android foreground-service requirement).
+ */
+import type { HardwareProbe } from "./types";
+/** The four device tiers used by the runtime + UI. */
+export type DeviceTier = "MAX" | "GOOD" | "OKAY" | "POOR";
+/** Tier ordering (higher index = better device). */
+export declare const DEVICE_TIER_ORDER: ReadonlyArray<DeviceTier>;
+/**
+ * Numeric thresholds. R9 §3.1 — keep in sync.
+ *
+ * "Effective model memory" is the memory the model can actually use, in GB.
+ * The math mirrors `recommendation.ts:effectiveMemoryGb()`:
+ *   - Apple Silicon: `totalRamGb`.
+ *   - Discrete GPU: `max(gpu.totalVramGb, totalRamGb * 0.5)`.
+ *   - CPU-only:     `totalRamGb * 0.5`.
+ */
+export declare const DEVICE_TIER_THRESHOLDS: {
+    readonly MAX: {
+        readonly effectiveModelMemoryGb: 24;
+        readonly freeRamGbAtSession: 16;
+        readonly dGpuMinVramGb: 16;
+        readonly appleSiliconMinMemoryGb: 32;
+    };
+    readonly GOOD: {
+        readonly effectiveModelMemoryGb: 12;
+        readonly freeRamGbAtSession: 8;
+        readonly dGpuMinVramGb: 8;
+        readonly appleSiliconMinMemoryGb: 16;
+        readonly x86CpuOnlyMinTotalGb: 32;
+    };
+    readonly OKAY: {
+        readonly effectiveModelMemoryGb: 6;
+        readonly freeRamGbAtSession: 3;
+        readonly minTotalRamGb: 16;
+        readonly mobileMinTotalRamGb: 12;
+    };
+};
+/** What the runtime should do by default given the tier classification. */
+export type RecommendedMode = "local" | "cloud-with-local-voice" | "cloud-only";
+/** A complete tier assessment — what the UI renders + what the runtime gates on. */
+export interface DeviceTierAssessment {
+    tier: DeviceTier;
+    /** Human-readable reasons for the decision. */
+    reasons: string[];
+    /** The top recommendation for the user, in plain text. */
+    topRecommendation: string;
+    /** True when local LM (eliza-1) is viable at the user's tier. */
+    canRunLocalLm: boolean;
+    /** True when the local voice stack (ASR + TTS) is viable. */
+    canRunLocalVoice: boolean;
+    /** Default backend mode for the user. */
+    recommendedMode: RecommendedMode;
+    /** Numeric snapshot to drive UI badges and first-run copy. */
+    numericContext: {
+        totalRamGb: number;
+        freeRamGb: number;
+        effectiveModelMemoryGb: number;
+        vramGb: number | null;
+        cpuCores: number;
+        appleSilicon: boolean;
+        mobile: boolean;
+    };
+}
+/**
+ * Compute the memory the model can actually use, in GB. Apple Silicon uses
+ * shared memory; discrete-GPU x86 weights VRAM; CPU-only halves total RAM.
+ * Mirrors `recommendation.ts:effectiveMemoryGb`.
+ */
+export declare function effectiveModelMemoryGb(probe: HardwareProbe): number;
+/**
+ * The single-pass classifier. Returns a complete assessment including the
+ * tier, the reasons, the recommended default mode, and the numeric context
+ * the UI needs to render the badge.
+ *
+ * The classifier is pure: same input → same output, no I/O, no clock.
+ */
+export declare function classifyDeviceTier(probe: HardwareProbe): DeviceTierAssessment;
+/**
+ * Warning-copy strings for each tier. The exact prose comes from R9 §7;
+ * I10 surfaces these via the `voice-tier.json` i18n bundle. Keep this in
+ * sync with `packages/ui/src/i18n/voice-tier.json`.
+ */
+export declare const TIER_WARNING_COPY: Readonly<Record<DeviceTier, {
+    header: string;
+    body: string;
+}>>;
+/** Convenience: total RAM in MB. */
+export declare function totalRamMb(probe: HardwareProbe): number;
+//# sourceMappingURL=device-tier.d.ts.map

package/src/services/device-tier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"device-tier.d.ts","sourceRoot":"","sources":["device-tier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,sDAAsD;AACtD,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAE1D,oDAAoD;AACpD,eAAO,MAAM,iBAAiB,EAAE,aAAa,CAAC,UAAU,CAKvD,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;CAoBzB,CAAC;AAEX,2EAA2E;AAC3E,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,wBAAwB,GAAG,YAAY,CAAC;AAEhF,oFAAoF;AACpF,MAAM,WAAW,oBAAoB;IACpC,IAAI,EAAE,UAAU,CAAC;IACjB,+CAA+C;IAC/C,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iEAAiE;IACjE,aAAa,EAAE,OAAO,CAAC;IACvB,6DAA6D;IAC7D,gBAAgB,EAAE,OAAO,CAAC;IAC1B,yCAAyC;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,8DAA8D;IAC9D,cAAc,EAAE;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,sBAAsB,EAAE,MAAM,CAAC;QAC/B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,OAAO,CAAC;QACtB,MAAM,EAAE,OAAO,CAAC;KAChB,CAAC;CACF;AAID;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,CAMnE;AA6CD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,aAAa,GAAG,oBAAoB,CA4E7E;AAoID;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CACvC,MAAM,CACL,UAAU,EACV;IACC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACb,CACD,CAkBD,CAAC;AAEF,oCAAoC;AACpC,wBAAgB,UAAU,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,CAEvD"}

package/src/services/device-tier.test.ts

@@ -0,0 +1,371 @@
+import { describe, expect, it } from "vitest";
+import {
+	classifyDeviceTier,
+	DEVICE_TIER_ORDER,
+	DEVICE_TIER_THRESHOLDS,
+	effectiveModelMemoryGb,
+	TIER_WARNING_COPY,
+} from "./device-tier";
+import type { HardwareProbe } from "./types";
+
+const baseProbe: HardwareProbe = {
+	totalRamGb: 16,
+	freeRamGb: 8,
+	gpu: null,
+	cpuCores: 8,
+	platform: "linux",
+	arch: "x64",
+	appleSilicon: false,
+	recommendedBucket: "mid",
+	source: "capacitor-llama",
+};
+
+function probe(overrides: Partial<HardwareProbe>): HardwareProbe {
+	return { ...baseProbe, ...overrides };
+}
+
+describe("classifyDeviceTier", () => {
+	describe("MAX tier", () => {
+		it("classifies a CUDA workstation with 24 GB VRAM as MAX", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 64,
+					freeRamGb: 48,
+					gpu: { backend: "cuda", totalVramGb: 24, freeVramGb: 22 },
+					cpuCores: 16,
+				}),
+			);
+			expect(result.tier).toBe("MAX");
+			expect(result.canRunLocalLm).toBe(true);
+			expect(result.canRunLocalVoice).toBe(true);
+			expect(result.recommendedMode).toBe("local");
+		});
+
+		it("classifies an Apple Silicon M3 Max 64 GB as MAX", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 64,
+					freeRamGb: 48,
+					gpu: null,
+					cpuCores: 16,
+					platform: "darwin",
+					arch: "arm64",
+					cpuFeatures: { neon: true, dotprod: true, i8mm: true },
+					appleSilicon: true,
+				}),
+			);
+			expect(result.tier).toBe("MAX");
+			expect(result.numericContext.effectiveModelMemoryGb).toBe(64);
+		});
+
+		it("includes reasons for MAX classification", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 64,
+					freeRamGb: 48,
+					gpu: { backend: "cuda", totalVramGb: 24, freeVramGb: 22 },
+					cpuCores: 16,
+				}),
+			);
+			expect(result.reasons.length).toBeGreaterThan(0);
+			expect(result.reasons[0]).toMatch(/effective model RAM/);
+		});
+	});
+
+	describe("GOOD tier — local dev box (RTX 5080 Laptop, 16 GB VRAM)", () => {
+		it("classifies a RTX 5080 Laptop (16 GB VRAM, 32 GB RAM, AVX-VNNI) as GOOD or MAX", () => {
+			// Per R9 §3 — the local dev box should land in MAX (>= 24 GB
+			// effective model RAM via VRAM-or-half-RAM math) OR GOOD when
+			// constrained by free RAM at session start. RTX 5080 Laptop = 16
+			// GB VRAM; system 32 GB RAM @ half = 16 GB effective; effective =
+			// max(VRAM, RAM/2) = 16 GB → GOOD.
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 32,
+					freeRamGb: 20,
+					gpu: { backend: "cuda", totalVramGb: 16, freeVramGb: 14 },
+					cpuCores: 16,
+					platform: "linux",
+					arch: "x64",
+				}),
+			);
+			expect(["GOOD", "MAX"]).toContain(result.tier);
+			expect(result.canRunLocalLm).toBe(true);
+			expect(result.canRunLocalVoice).toBe(true);
+		});
+
+		it("classifies a 16 GB VRAM dGPU + 16 GB RAM laptop as GOOD", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 16,
+					freeRamGb: 10,
+					gpu: { backend: "cuda", totalVramGb: 16, freeVramGb: 14 },
+					cpuCores: 12,
+				}),
+			);
+			expect(result.tier).toBe("GOOD");
+		});
+
+		it("classifies an Apple Silicon M3 base 16 GB as GOOD", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 16,
+					freeRamGb: 10,
+					gpu: null,
+					cpuCores: 8,
+					platform: "darwin",
+					arch: "arm64",
+					cpuFeatures: { neon: true, dotprod: true, i8mm: true },
+					appleSilicon: true,
+				}),
+			);
+			expect(result.tier).toBe("GOOD");
+		});
+
+		it("classifies an x86 CPU-only 32 GB box as GOOD", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 32,
+					freeRamGb: 20,
+					gpu: null,
+					cpuCores: 8,
+				}),
+			);
+			expect(result.tier).toBe("GOOD");
+		});
+	});
+
+	describe("OKAY tier", () => {
+		it("classifies a 16 GB CPU-only laptop as OKAY", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 16,
+					freeRamGb: 8,
+					gpu: null,
+					cpuCores: 8,
+				}),
+			);
+			expect(result.tier).toBe("OKAY");
+		});
+
+		it("clamps Apple Silicon 8 GB to OKAY", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 8,
+					freeRamGb: 4,
+					gpu: null,
+					cpuCores: 8,
+					platform: "darwin",
+					arch: "arm64",
+					cpuFeatures: { neon: true, dotprod: true, i8mm: true },
+					appleSilicon: true,
+				}),
+			);
+			// 8 GB ceiling makes this POOR by raw thresholds (effective < 6 GB
+			// after macOS reserve), but the AS8GB clamp ensures it does not
+			// claim above OKAY.
+			expect(["OKAY", "POOR"]).toContain(result.tier);
+			expect(result.tier === "OKAY" || result.tier === "POOR").toBe(true);
+		});
+	});
+
+	describe("POOR tier", () => {
+		it("classifies an 8 GB box without GPU as POOR", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 8,
+					freeRamGb: 3,
+					gpu: null,
+					cpuCores: 4,
+				}),
+			);
+			expect(result.tier).toBe("POOR");
+			expect(result.canRunLocalLm).toBe(false);
+			expect(result.recommendedMode).toBe("cloud-only");
+		});
+
+		it("classifies a 2-core CPU as POOR (no AVX2 baseline)", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 32,
+					freeRamGb: 24,
+					gpu: null,
+					cpuCores: 2,
+				}),
+			);
+			expect(result.tier).toBe("POOR");
+			expect(result.reasons.join(" ")).toMatch(/AVX2|cores/);
+		});
+
+		it("classifies ARM without CPU feature evidence as POOR", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 32,
+					freeRamGb: 24,
+					gpu: null,
+					cpuCores: 8,
+					platform: "linux",
+					arch: "arm64",
+					cpuFeatures: undefined,
+				}),
+			);
+			expect(result.tier).toBe("POOR");
+			expect(result.canRunLocalLm).toBe(false);
+		});
+	});
+
+	describe("mobile clamps", () => {
+		it("clamps iOS to OKAY even with 12 GB RAM", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 12,
+					freeRamGb: 6,
+					gpu: null,
+					cpuCores: 6,
+					platform: "darwin",
+					arch: "arm64",
+					cpuFeatures: { neon: true, dotprod: true, i8mm: true },
+					appleSilicon: true,
+					mobile: {
+						platform: "ios",
+						deviceModel: "iPhone 15 Pro",
+						availableRamGb: 6,
+					},
+				}),
+			);
+			expect(["OKAY", "POOR"]).toContain(result.tier);
+			expect(result.recommendedMode).toBe("cloud-with-local-voice");
+		});
+
+		it("returns POOR/cloud-only for low-end Android", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 4,
+					freeRamGb: 1.5,
+					gpu: null,
+					cpuCores: 4,
+					platform: "linux",
+					arch: "arm64",
+					cpuFeatures: { neon: true, dotprod: true, i8mm: false },
+					appleSilicon: false,
+					mobile: { platform: "android", availableRamGb: 1.5 },
+				}),
+			);
+			expect(result.tier).toBe("POOR");
+			expect(result.recommendedMode).toBe("cloud-only");
+		});
+
+		it("flagship Android (16 GB) clamps to OKAY", () => {
+			const result = classifyDeviceTier(
+				probe({
+					totalRamGb: 16,
+					freeRamGb: 8,
+					gpu: null,
+					cpuCores: 8,
+					platform: "linux",
+					arch: "arm64",
+					cpuFeatures: { neon: true, dotprod: true, i8mm: true },
+					mobile: { platform: "android", availableRamGb: 8 },
+				}),
+			);
+			expect(result.tier).toBe("OKAY");
+			// Mobile defaults to cloud TTS+ASR per R9 §6.3.
+			expect(result.recommendedMode).toBe("cloud-with-local-voice");
+		});
+	});
+
+	describe("free-RAM gate at session start", () => {
+		it("demotes one tier when free RAM is below 25% of total", () => {
+			const hot = classifyDeviceTier(
+				probe({
+					totalRamGb: 64,
+					freeRamGb: 48,
+					gpu: { backend: "cuda", totalVramGb: 24, freeVramGb: 22 },
+					cpuCores: 16,
+				}),
+			);
+			expect(hot.tier).toBe("MAX");
+			const constrained = classifyDeviceTier(
+				probe({
+					totalRamGb: 64,
+					freeRamGb: 8, // 12.5% — below the 25% gate
+					gpu: { backend: "cuda", totalVramGb: 24, freeVramGb: 22 },
+					cpuCores: 16,
+				}),
+			);
+			// 8 GB free fails MAX's freeRamGbAtSession threshold (16 GB) so it
+			// can't be MAX on threshold-grounds; the demote rule also fires
+			// and steps down one tier from MAX → GOOD.
+			expect(["GOOD", "OKAY"]).toContain(constrained.tier);
+		});
+	});
+});
+
+describe("effectiveModelMemoryGb", () => {
+	it("returns totalRamGb for Apple Silicon", () => {
+		expect(
+			effectiveModelMemoryGb(
+				probe({
+					totalRamGb: 32,
+					gpu: null,
+					appleSilicon: true,
+					platform: "darwin",
+					arch: "arm64",
+					cpuFeatures: { neon: true, dotprod: true, i8mm: true },
+				}),
+			),
+		).toBe(32);
+	});
+
+	it("returns max(vram, totalRam/2) for discrete GPU", () => {
+		expect(
+			effectiveModelMemoryGb(
+				probe({
+					totalRamGb: 32,
+					gpu: { backend: "cuda", totalVramGb: 24, freeVramGb: 24 },
+				}),
+			),
+		).toBe(24);
+		expect(
+			effectiveModelMemoryGb(
+				probe({
+					totalRamGb: 64,
+					gpu: { backend: "cuda", totalVramGb: 8, freeVramGb: 8 },
+				}),
+			),
+		).toBe(32);
+	});
+
+	it("returns totalRamGb / 2 for CPU-only", () => {
+		expect(effectiveModelMemoryGb(probe({ totalRamGb: 32, gpu: null }))).toBe(
+			16,
+		);
+	});
+});
+
+describe("TIER_WARNING_COPY", () => {
+	it("provides header + body for every tier", () => {
+		for (const tier of DEVICE_TIER_ORDER) {
+			expect(TIER_WARNING_COPY[tier]).toBeDefined();
+			expect(TIER_WARNING_COPY[tier].header).toMatch(/tier/i);
+			expect(TIER_WARNING_COPY[tier].body.length).toBeGreaterThan(20);
+		}
+	});
+});
+
+describe("DEVICE_TIER_THRESHOLDS", () => {
+	it("matches R9 §3.1 numbers", () => {
+		expect(DEVICE_TIER_THRESHOLDS.MAX.effectiveModelMemoryGb).toBe(24);
+		expect(DEVICE_TIER_THRESHOLDS.MAX.freeRamGbAtSession).toBe(16);
+		expect(DEVICE_TIER_THRESHOLDS.MAX.dGpuMinVramGb).toBe(16);
+		expect(DEVICE_TIER_THRESHOLDS.MAX.appleSiliconMinMemoryGb).toBe(32);
+
+		expect(DEVICE_TIER_THRESHOLDS.GOOD.effectiveModelMemoryGb).toBe(12);
+		expect(DEVICE_TIER_THRESHOLDS.GOOD.freeRamGbAtSession).toBe(8);
+		expect(DEVICE_TIER_THRESHOLDS.GOOD.dGpuMinVramGb).toBe(8);
+		expect(DEVICE_TIER_THRESHOLDS.GOOD.appleSiliconMinMemoryGb).toBe(16);
+
+		expect(DEVICE_TIER_THRESHOLDS.OKAY.effectiveModelMemoryGb).toBe(6);
+		expect(DEVICE_TIER_THRESHOLDS.OKAY.freeRamGbAtSession).toBe(3);
+	});
+});