@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/services/ffi-streaming-backend.ts

@@ -83,10 +83,10 @@ export interface FfiBackendSession {
 	} | null;
 	/**
 	 * Absolute path to a *separate* MTP drafter GGUF resolved during load.
-	 * `null` means same-file MTP: the NextN head is embedded in the main
-	 * text GGUF and the native runner activates `--spec-type draft-mtp`
-	 * with no `-md`. Speculative decoding is governed by `mtp`, not by the
-	 * presence of this path.
+	 * `null` means embedded-draft-head MTP: the draft head is embedded in
+	 * the main text GGUF and the native runner activates `--spec-type
+	 * draft-mtp` with no `-md`. Speculative decoding is governed by `mtp`,
+	 * not by the presence of this path.
 	 */
 	readonly draftModelPath: string | null;
 	/**
@@ -97,13 +97,14 @@ export interface FfiBackendSession {
 	readonly mmprojPath: string | null;
 	/**
 	 * Per-load runtime config the fused libelizainference path applies at its
-	 * first `llmStreamOpen` (gpuLayers + KV-cache quant types). The desktop
+	 * first `llmStreamOpen` (context size, gpuLayers + KV-cache quant types). The desktop
 	 * libllama runtime applies these at `loadModel()` instead and leaves this
 	 * `null` — the backend forwards them into the runner's per-call config only
 	 * when present, so the fused path mirrors the libllama load decision without
 	 * the libllama path double-applying them.
 	 */
 	readonly loadConfig?: {
+		contextSize?: number;
 		gpuLayers?: number;
 		cacheTypeK?: string | null;
 		cacheTypeV?: string | null;
@@ -195,7 +196,9 @@ export class FfiStreamingBackend implements LocalInferenceBackend {
 			gpuLayers: loadConfig?.gpuLayers,
 			cacheTypeK: loadConfig?.cacheTypeK,
 			cacheTypeV: loadConfig?.cacheTypeV,
+			contextSize: loadConfig?.contextSize,
 			signal: args.signal,
+			maxTokensPerStep: args.maxTokensPerStep,
 			onTextChunk: args.onTextChunk,
 			onVerifierEvent: args.onVerifierEvent,
 		});
@@ -280,14 +283,39 @@ export class FfiStreamingBackend implements LocalInferenceBackend {
 			gpuLayers: loadConfig?.gpuLayers,
 			cacheTypeK: loadConfig?.cacheTypeK,
 			cacheTypeV: loadConfig?.cacheTypeV,
+			contextSize: loadConfig?.contextSize,
 		});
 		return true;
 	}
 
+	currentRuntimeLoadConfig() {
+		if (!this.session) return null;
+		const loadConfig = this.session.loadConfig;
+		return {
+			modelId: null,
+			modelPath: this.loadedPath,
+			contextSize: loadConfig?.contextSize ?? null,
+			cacheTypeK: loadConfig?.cacheTypeK ?? null,
+			cacheTypeV: loadConfig?.cacheTypeV ?? null,
+			gpuLayers:
+				typeof loadConfig?.gpuLayers === "number" ? loadConfig.gpuLayers : null,
+			parallel: this.parallelSlots(),
+			binaryPath: null,
+			backend: this.id,
+			mtp: this.session.mtp
+				? {
+						specType: "draft-mtp" as const,
+						draftMin: this.session.mtp.draftMin,
+						draftMax: this.session.mtp.draftMax,
+					}
+				: null,
+		};
+	}
+
 	/**
 	 * True when Eliza-1 native MTP is active for the loaded target model.
-	 * Covers both shapes: same-file MTP (NextN head embedded in the text
-	 * GGUF, `draftModelPath` null) and separate-drafter MTP.
+	 * Covers both shapes: embedded-draft-head MTP (draft head embedded in
+	 * the text GGUF, `draftModelPath` null) and separate-drafter MTP.
 	 */
 	mtpEnabled(): boolean {
 		return Boolean(this.session?.mtp);
@@ -327,6 +355,10 @@ export class FfiStreamingBackend implements LocalInferenceBackend {
 		maxTokens?: number;
 		temperature?: number;
 		signal?: AbortSignal;
+		/** Per-token callback — when set + the runtime supports streaming vision,
+		 * the description is decoded token-by-token through the same pipe as chat. */
+		onTextChunk?: (chunk: string) => void | Promise<void>;
+		maxTokensPerStep?: number;
 	}): Promise<{ text: string; projectorMs?: number; decodeMs?: number }> {
 		if (!this.session) {
 			throw new Error(
@@ -342,7 +374,7 @@ export class FfiStreamingBackend implements LocalInferenceBackend {
 		// The runtime adapter has visionSupported() + describeImage(args).
 		// We re-shape `bytes` → `imageBytes` and merge in the resolved
 		// mmprojPath; the rest of args pass through unchanged.
-		const runtime = this.runtime as unknown as {
+		const runtime = this.runtime as FfiBackendRuntime & {
 			describeImage?: (args: {
 				imageBytes: Uint8Array;
 				mmprojPath: string;
@@ -350,6 +382,8 @@ export class FfiStreamingBackend implements LocalInferenceBackend {
 				maxTokens?: number;
 				temperature?: number;
 				signal?: AbortSignal;
+				onTextChunk?: (chunk: string) => void | Promise<void>;
+				maxTokensPerStep?: number;
 			}) => Promise<{ text: string; projectorMs?: number; decodeMs?: number }>;
 		};
 		if (!runtime.describeImage) {
@@ -364,6 +398,8 @@ export class FfiStreamingBackend implements LocalInferenceBackend {
 			maxTokens: args.maxTokens,
 			temperature: args.temperature,
 			signal: args.signal,
+			onTextChunk: args.onTextChunk,
+			maxTokensPerStep: args.maxTokensPerStep,
 		});
 	}
 

package/src/services/ffi-streaming-runner.test.ts

@@ -1,10 +1,63 @@
-import { describe, expect, it, vi } from "vitest";
-import { FfiStreamingRunner } from "./ffi-streaming-runner";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+	FfiStreamingRunner,
+	resolveMaxTokensPerStep,
+} from "./ffi-streaming-runner";
 import type {
 	LlmCtxHandle,
 	LlmStreamingBinding,
 } from "./llm-streaming-binding";
-import type { LlmStreamHandle } from "./voice/ffi-bindings";
+import type { LlmStreamHandle, LlmStreamStep } from "./voice/ffi-bindings";
+
+/**
+ * Build a binding whose `llmStreamNext` emits `steps.length` steps (the last
+ * with `done: true`), and that records the `maxTokensPerStep` passed on every
+ * call so tests can assert the resolved per-step cap.
+ */
+function makeStepBinding(steps: string[]): {
+	binding: LlmStreamingBinding;
+	stepCaps: number[];
+} {
+	const stream = 7n as LlmStreamHandle;
+	const stepCaps: number[] = [];
+	let i = 0;
+	const llmStreamNext = vi.fn(
+		(args: { maxTokensPerStep?: number }): LlmStreamStep => {
+			stepCaps.push(args.maxTokensPerStep ?? -1);
+			const text = steps[i] ?? "";
+			const done = i >= steps.length - 1;
+			i += 1;
+			return {
+				tokens: [i],
+				text,
+				done,
+				drafterDrafted: 0,
+				drafterAccepted: 0,
+			};
+		},
+	);
+	const binding: LlmStreamingBinding = {
+		llmStreamSupported: () => true,
+		llmStreamOpen: vi.fn().mockReturnValue(stream),
+		llmStreamPrefill: vi.fn(),
+		llmStreamNext,
+		llmStreamCancel: vi.fn(),
+		llmStreamClose: vi.fn(),
+	};
+	return { binding, stepCaps };
+}
+
+const BASE_ARGS = {
+	slotId: 0,
+	maxTokens: 64,
+	temperature: 0,
+	topP: 1,
+	topK: 0,
+	repeatPenalty: 1,
+	draftMin: 0,
+	draftMax: 0,
+	draftModelPath: null,
+} as const;
 
 describe("FfiStreamingRunner prewarm", () => {
 	it("treats maxTokens: 0 as prefill-only and never calls next-token generation", async () => {
@@ -38,10 +91,17 @@ describe("FfiStreamingRunner prewarm", () => {
 			draftMin: 0,
 			draftMax: 0,
 			draftModelPath: null,
+			contextSize: 32_768,
 			onTextChunk,
 		});
 
 		expect(binding.llmStreamOpen).toHaveBeenCalledTimes(1);
+		expect(binding.llmStreamOpen).toHaveBeenCalledWith({
+			ctx: 1n,
+			config: expect.objectContaining({
+				contextSize: 32_768,
+			}),
+		});
 		expect(binding.llmStreamPrefill).toHaveBeenCalledWith({
 			stream,
 			tokens: promptTokens,
@@ -58,3 +118,103 @@ describe("FfiStreamingRunner prewarm", () => {
 		});
 	});
 });
+
+describe("FfiStreamingRunner per-step granularity (#9174)", () => {
+	const ORIGINAL = process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+	afterEach(() => {
+		if (ORIGINAL === undefined) {
+			delete process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+		} else {
+			process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = ORIGINAL;
+		}
+	});
+
+	it("defaults to a 32-token per-step cap when no override is set", async () => {
+		delete process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+		const { binding, stepCaps } = makeStepBinding(["Hi ", "there"]);
+		const runner = new FfiStreamingRunner(binding, 1n as LlmCtxHandle);
+		await runner.generateWithUsage({
+			...BASE_ARGS,
+			promptTokens: new Int32Array([1]),
+		});
+		expect(stepCaps).toEqual([32, 32]);
+	});
+
+	it("forwards a per-call maxTokensPerStep to every llmStreamNext call", async () => {
+		delete process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+		const { binding, stepCaps } = makeStepBinding(["a", "b", "c"]);
+		const runner = new FfiStreamingRunner(binding, 1n as LlmCtxHandle);
+		await runner.generateWithUsage({
+			...BASE_ARGS,
+			promptTokens: new Int32Array([1]),
+			maxTokensPerStep: 4,
+		});
+		expect(stepCaps).toEqual([4, 4, 4]);
+	});
+
+	it("honors the ELIZA_LOCAL_STREAM_TOKENS_PER_STEP env override", async () => {
+		process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = "8";
+		const { binding, stepCaps } = makeStepBinding(["x", "y"]);
+		const runner = new FfiStreamingRunner(binding, 1n as LlmCtxHandle);
+		await runner.generateWithUsage({
+			...BASE_ARGS,
+			promptTokens: new Int32Array([1]),
+		});
+		expect(stepCaps).toEqual([8, 8]);
+	});
+
+	it("lets a per-call override win over the env var", async () => {
+		process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = "8";
+		const { binding, stepCaps } = makeStepBinding(["x"]);
+		const runner = new FfiStreamingRunner(binding, 1n as LlmCtxHandle);
+		await runner.generateWithUsage({
+			...BASE_ARGS,
+			promptTokens: new Int32Array([1]),
+			maxTokensPerStep: 1,
+		});
+		expect(stepCaps).toEqual([1]);
+	});
+
+	it("clamps out-of-range per-call overrides into the supported window", async () => {
+		delete process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+		const low = makeStepBinding(["x"]);
+		const runnerLow = new FfiStreamingRunner(low.binding, 1n as LlmCtxHandle);
+		await runnerLow.generateWithUsage({
+			...BASE_ARGS,
+			promptTokens: new Int32Array([1]),
+			maxTokensPerStep: 0,
+		});
+		// 0 floors to the minimum of 1, never disables generation.
+		expect(low.stepCaps).toEqual([1]);
+
+		const high = makeStepBinding(["x"]);
+		const runnerHigh = new FfiStreamingRunner(high.binding, 1n as LlmCtxHandle);
+		await runnerHigh.generateWithUsage({
+			...BASE_ARGS,
+			promptTokens: new Int32Array([1]),
+			maxTokensPerStep: 100_000,
+		});
+		expect(high.stepCaps).toEqual([512]);
+	});
+
+	describe("resolveMaxTokensPerStep", () => {
+		it("returns 32 when unset", () => {
+			delete process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+			expect(resolveMaxTokensPerStep()).toBe(32);
+		});
+
+		it("parses and clamps a valid env value", () => {
+			process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = "16";
+			expect(resolveMaxTokensPerStep()).toBe(16);
+			process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = "9999";
+			expect(resolveMaxTokensPerStep()).toBe(512);
+			process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = "0";
+			expect(resolveMaxTokensPerStep()).toBe(1);
+		});
+
+		it("falls back to 32 on a non-numeric env value", () => {
+			process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = "smooth";
+			expect(resolveMaxTokensPerStep()).toBe(32);
+		});
+	});
+});

package/src/services/ffi-streaming-runner.ts

@@ -61,6 +61,12 @@ export interface FfiStreamingGenerateArgs {
 	 */
 	cacheTypeK?: string | null;
 	cacheTypeV?: string | null;
+	/**
+	 * Runtime context window in tokens (ABI v9). Forwarded into the fused
+	 * session config on `llmStreamOpen`; `undefined` keeps the native
+	 * ELIZA_LLM_N_CTX/default fallback.
+	 */
+	contextSize?: number;
 	/**
 	 * GBNF grammar source forcing the structured-reply envelope. Passed to
 	 * the native session's `llmStreamOpen` config so sampling is
@@ -69,6 +75,16 @@ export interface FfiStreamingGenerateArgs {
 	gbnfGrammar?: string | null;
 	/** Cancellation signal — fires `llmStreamCancel` on the active session. */
 	signal?: AbortSignal;
+	/**
+	 * Per-step token cap for the native decode loop. Lower values make the
+	 * local UI stream in finer-grained jumps (smoother token-by-token render)
+	 * at the cost of more JS↔FFI round-trips per reply; higher values batch
+	 * more tokens per step. When omitted, falls back to
+	 * `resolveMaxTokensPerStep()` (env `ELIZA_LOCAL_STREAM_TOKENS_PER_STEP`,
+	 * else `DEFAULT_MAX_TOKENS_PER_STEP`). Clamped to
+	 * `[MIN_MAX_TOKENS_PER_STEP, MAX_MAX_TOKENS_PER_STEP]`.
+	 */
+	maxTokensPerStep?: number;
 	/** Per-chunk text callback. */
 	onTextChunk?: (chunk: string) => void | Promise<void>;
 	/** Speculative accept/reject events from MTP verification. */
@@ -86,6 +102,37 @@ export interface FfiStreamingGenerateResult {
 /** Default per-step caps. Match upstream llama-server's `n_predict` chunk size. */
 const DEFAULT_MAX_TOKENS_PER_STEP = 32;
 const DEFAULT_MAX_TEXT_BYTES = 1024;
+/**
+ * Sane bounds for the per-step token cap. The floor is 1 (true
+ * token-by-token); the ceiling guards against pathological values that would
+ * defeat streaming by emitting the whole reply in one step.
+ */
+const MIN_MAX_TOKENS_PER_STEP = 1;
+const MAX_MAX_TOKENS_PER_STEP = 512;
+
+/** Clamp a caller-supplied per-step cap into the supported range. */
+function clampMaxTokensPerStep(value: number): number {
+	if (!Number.isFinite(value)) return DEFAULT_MAX_TOKENS_PER_STEP;
+	return Math.min(
+		MAX_MAX_TOKENS_PER_STEP,
+		Math.max(MIN_MAX_TOKENS_PER_STEP, Math.trunc(value)),
+	);
+}
+
+/**
+ * Resolve the per-step token cap for the native decode loop. Override via the
+ * `ELIZA_LOCAL_STREAM_TOKENS_PER_STEP` env var (e.g. set to `8` for smoother
+ * local streaming, weighed against the extra JS↔FFI round-trips and the shared
+ * voice phrase-chunker). Falls back to `DEFAULT_MAX_TOKENS_PER_STEP` (32) when
+ * unset or invalid; clamped to `[MIN_MAX_TOKENS_PER_STEP, MAX_MAX_TOKENS_PER_STEP]`.
+ */
+export function resolveMaxTokensPerStep(): number {
+	const raw = process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP?.trim();
+	if (!raw) return DEFAULT_MAX_TOKENS_PER_STEP;
+	const parsed = Number.parseInt(raw, 10);
+	if (!Number.isFinite(parsed)) return DEFAULT_MAX_TOKENS_PER_STEP;
+	return clampMaxTokensPerStep(parsed);
+}
 
 /**
  * Backend used by the mobile and desktop FFI routes.
@@ -285,6 +332,7 @@ export class FfiStreamingRunner {
 				gpuLayers: args.gpuLayers,
 				cacheTypeK: args.cacheTypeK,
 				cacheTypeV: args.cacheTypeV,
+				contextSize: args.contextSize,
 			},
 		});
 
@@ -307,6 +355,11 @@ export class FfiStreamingRunner {
 				return;
 			}
 
+			const maxTokensPerStep =
+				args.maxTokensPerStep !== undefined
+					? clampMaxTokensPerStep(args.maxTokensPerStep)
+					: resolveMaxTokensPerStep();
+
 			let tokenIndex = 0;
 			while (true) {
 				if (args.signal?.aborted) {
@@ -315,7 +368,7 @@ export class FfiStreamingRunner {
 				}
 				const step = this.ffi.llmStreamNext({
 					stream,
-					maxTokensPerStep: DEFAULT_MAX_TOKENS_PER_STEP,
+					maxTokensPerStep,
 					maxTextBytes: DEFAULT_MAX_TEXT_BYTES,
 				});
 				onStep(step);

package/src/services/ffi-unload-ordering.test.ts

@@ -158,5 +158,9 @@ describe("DesktopFusedFfiBackendRuntime.release() ordering (#14)", () => {
 		// is explicitly poisoned so a new native model is not allocated over a
 		// failed cleanup state.
 		await expect(runtime.acquire(PLAN)).rejects.toThrow(/restart required/i);
-	});
+		// Heavy path (dynamic import + FFI acquire/release/acquire): fast in
+		// isolation but CPU-starved under the full 2122-test parallel suite, where
+		// it brushed the old 20s ceiling (20012ms). Headroom; a true hang still
+		// fails well within this bound.
+	}, 45_000);
 });

package/src/services/fused-eliza1-no-regression.test.ts

@@ -0,0 +1,144 @@
+/**
+ * #8808 acceptance criterion 4 — fused eliza-1 no-regression.
+ *
+ * The local stack is Eliza-1 only (#8808 cutover), so serving is deterministic.
+ * This test pins the invariants:
+ *   - `decideBackend` / `BackendDispatcher` route a `runtimeClass:"fused-eliza1"`
+ *     model to the fused `llama-cpp` runtime,
+ *   - the fused path retains its full-pipeline binding: the `BackendPlan` that
+ *     reaches the fused backend still carries the catalog entry and the
+ *     bundle-root override that `DesktopFusedFfiBackendRuntime.acquire()` reads
+ *     to anchor the fused context.
+ *
+ * It complements `backend-runtime-class.test.ts` (which proves the binary
+ * routing) by asserting that the FULL fused load contract is forwarded intact.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import {
+	BackendDispatcher,
+	type BackendPlan,
+	decideBackend,
+	type GenerateArgs,
+	type GenerateResult,
+	type LocalInferenceBackend,
+} from "./backend";
+import { findCatalogModel } from "./catalog";
+import type { CatalogModel } from "./types";
+
+const FUSED_TIER = findCatalogModel("eliza-1-4b") as CatalogModel;
+
+function makeBackend(id: LocalInferenceBackend["id"]): LocalInferenceBackend & {
+	loaded: BackendPlan[];
+} {
+	const loaded: BackendPlan[] = [];
+	return {
+		id,
+		loaded,
+		available: async () => true,
+		load: async (plan: BackendPlan) => {
+			loaded.push(plan);
+		},
+		unload: async () => {},
+		generate: async (_args: GenerateArgs): Promise<GenerateResult> => "ok",
+		hasLoadedModel: () => loaded.length > 0,
+		currentModelPath: () => loaded.at(-1)?.modelPath ?? null,
+	};
+}
+
+describe("fused eliza-1 no-regression (C4)", () => {
+	it("the catalog tier under test really is a fused-eliza1 tier", () => {
+		expect(FUSED_TIER).toBeTruthy();
+		expect(FUSED_TIER.runtimeClass).toBe("fused-eliza1");
+		expect(FUSED_TIER.runtime?.mtp).toBeUndefined();
+	});
+
+	it("decideBackend routes a fused Eliza-1 tier to the fused llama-cpp runtime", () => {
+		const decision = decideBackend({
+			override: "auto",
+			catalog: FUSED_TIER,
+			llamaCppAvailable: true,
+		});
+		expect(decision.backend).toBe("llama-cpp");
+	});
+
+	it("decideBackend routes everything to llama-cpp — the stack is Eliza-1 only", () => {
+		// Post-#8808 cutover: there is no generic-gguf backend; every model
+		// (even an unknown catalog entry) routes to the fused llama-cpp runtime.
+		const decision = decideBackend({
+			override: "auto",
+			catalog: undefined,
+			llamaCppAvailable: true,
+		});
+		expect(decision.backend).toBe("llama-cpp");
+	});
+
+	it("dispatcher forwards the fused full-pipeline binding (catalog + bundleRoot) to the fused backend", async () => {
+		const ffi = makeBackend("llama-cpp");
+		const dispatcher = new BackendDispatcher(
+			ffi,
+			() => true,
+			() => null,
+		);
+
+		const bundleRoot = "/models/eliza-1-4b";
+		const plan: BackendPlan = {
+			modelPath: `${bundleRoot}/text/eliza-1-4b-128k.gguf`,
+			modelId: "eliza-1-4b",
+			catalog: FUSED_TIER,
+			runtimeClass: "fused-eliza1",
+			overrides: {
+				bundleRoot,
+				draftModelPath: `${bundleRoot}/text/eliza-1-4b-mtp.gguf`,
+				gpuLayers: "max",
+				cacheTypeK: "tbq4_0",
+				cacheTypeV: "tbq3_0",
+			},
+		};
+
+		await dispatcher.load(plan);
+
+		// Routed to the fused runtime.
+		expect(ffi.loaded).toHaveLength(1);
+		expect(dispatcher.activeBackendId()).toBe("llama-cpp");
+
+		// The full-pipeline binding survives dispatch: the fused backend receives
+		// the same catalog entry plus the bundle-root and explicit drafter
+		// overrides that anchor the fused context and preserve fork KV-cache
+		// kernel settings.
+		const forwarded = ffi.loaded[0];
+		expect(forwarded.catalog).toBe(FUSED_TIER);
+		expect(forwarded.catalog?.runtime?.mtp).toBeUndefined();
+		expect(forwarded.overrides?.bundleRoot).toBe(bundleRoot);
+		expect(forwarded.overrides?.draftModelPath).toBe(
+			`${bundleRoot}/text/eliza-1-4b-mtp.gguf`,
+		);
+		expect(forwarded.overrides?.cacheTypeK).toBe("tbq4_0");
+		expect(forwarded.overrides?.cacheTypeV).toBe("tbq3_0");
+	});
+
+	it("env-override=llama-cpp keeps a fused tier on the fused path", async () => {
+		const prev = process.env.ELIZA_INFERENCE_BACKEND;
+		process.env.ELIZA_INFERENCE_BACKEND = "llama-cpp";
+		try {
+			const ffi = makeBackend("llama-cpp");
+			const dispatcher = new BackendDispatcher(
+				ffi,
+				() => true,
+				() => null,
+			);
+			const decision = dispatcher.decide({
+				modelPath: "/models/eliza-1-4b/text/eliza-1-4b-128k.gguf",
+				modelId: "eliza-1-4b",
+				catalog: FUSED_TIER,
+				runtimeClass: "fused-eliza1",
+			});
+			expect(decision.backend).toBe("llama-cpp");
+			expect(decision.reason).toBe("env-override");
+		} finally {
+			if (prev === undefined) delete process.env.ELIZA_INFERENCE_BACKEND;
+			else process.env.ELIZA_INFERENCE_BACKEND = prev;
+		}
+	});
+});

package/src/services/hardware.test.ts

@@ -213,8 +213,13 @@ describe("probeHardware GPU detection", () => {
 		expect(probe.gpu).not.toBeNull();
 		expect(probe.gpu?.backend).toBe("cuda");
 		expect(probe.gpu?.totalVramGb).toBeGreaterThanOrEqual(23);
-		// A 24 GB discrete GPU must not be mis-tiered as a CPU box.
-		expect(["MAX", "GOOD"]).toContain(classifyDeviceTier(probe).tier);
+		// A 24 GB discrete GPU must not be mis-tiered as a CPU box (POOR). MAX/GOOD
+		// additionally require >=16/>=8 GB *free* RAM at session start, which this test
+		// only mocks the GPU for; under parallel-suite memory pressure real free RAM
+		// can dip below those gates, legitimately tiering a 24 GB-GPU box to OKAY
+		// (still GPU-backed, not a CPU box). Assert "off CPU" (not POOR) -- the stated
+		// intent -- which is deterministic regardless of host free RAM.
+		expect(["MAX", "GOOD", "OKAY"]).toContain(classifyDeviceTier(probe).tier);
 	});
 
 	it("reports gpu:null when nvidia-smi is absent on a non-Apple host", async () => {

package/src/services/hardware.ts

@@ -13,8 +13,10 @@
 import { execFileSync } from "node:child_process";
 import fs from "node:fs";
 import os from "node:os";
+import path from "node:path";
 import { detectGpu } from "./gpu-detect";
 import type { Eliza1Backend, Eliza1DeviceCaps } from "./manifest";
+import { elizaModelsDir } from "./paths";
 import type {
 	CpuFeatureProbe,
 	HardwareProbe,
@@ -29,6 +31,30 @@ function bytesToGb(bytes: number): number {
 	return Math.round((bytes / BYTES_PER_GB) * 10) / 10;
 }
 
+/**
+ * Free disk space (GB) on the volume that holds the models directory. Walks up
+ * to the nearest existing ancestor before `statfs` so a not-yet-created models
+ * dir still resolves to its parent volume. Returns `undefined` when the volume
+ * cannot be stat'd (the fit check then falls back to RAM-only / mobile
+ * storage), never throws.
+ */
+async function probeFreeDiskGb(): Promise<number | undefined> {
+	try {
+		let dir = elizaModelsDir();
+		for (let i = 0; i < 12 && !fs.existsSync(dir); i += 1) {
+			const parent = path.dirname(dir);
+			if (parent === dir) break;
+			dir = parent;
+		}
+		const stats = await fs.promises.statfs(dir);
+		const available = stats.bavail * stats.bsize;
+		if (!Number.isFinite(available) || available < 0) return undefined;
+		return bytesToGb(available);
+	} catch {
+		return undefined;
+	}
+}
+
 /**
  * Pick a default bucket based on total available memory and architecture.
  *
@@ -331,9 +357,11 @@ export async function probeHardware(): Promise<HardwareProbe> {
 
 	const gpu = detectProbeGpu(appleSilicon, totalRamBytes, freeRamBytes);
 	const totalRamGb = bytesToGb(totalRamBytes);
+	const freeDiskGb = await probeFreeDiskGb();
 	return {
 		totalRamGb,
 		freeRamGb: bytesToGb(freeRamBytes),
+		...(freeDiskGb !== undefined ? { freeDiskGb } : {}),
 		gpu,
 		cpuCores,
 		cpuFeatures,