@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/runtime/embedding-warmup-policy.ts

@@ -0,0 +1,48 @@
+/**
+ * Whether to prefetch the local GGUF embedding model before runtime boot.
+ *
+ * Chat/inference provider (what you pick in first-run) is separate from
+ * **embeddings** (vector memory / RAG). By default the framework keeps
+ * `@elizaos/plugin-local-inference` loaded because API-based model plugins do
+ * not implement TEXT_EMBEDDING — so a local model was historically always
+ * warmed up. When Eliza Cloud is connected with **cloud embeddings** enabled,
+ * the cloud plugin handles embeddings instead; skipping warmup avoids a large
+ * download unrelated to “local inference” for chat.
+ */
+
+function isTruthyEnv(...names: string[]): boolean {
+	for (const name of names) {
+		const v = process.env[name]?.trim().toLowerCase();
+		if (v === "1" || v === "true" || v === "yes") return true;
+	}
+	return false;
+}
+
+export function isLocalEmbeddingDisabledByEnv(): boolean {
+	return isTruthyEnv("ELIZA_DISABLE_LOCAL_EMBEDDINGS");
+}
+
+export function shouldWarmupLocalEmbeddingModel(): boolean {
+	if (isTruthyEnv("ELIZA_SKIP_LOCAL_EMBEDDING_WARMUP")) {
+		return false;
+	}
+
+	if (isLocalEmbeddingDisabledByEnv()) {
+		return false;
+	}
+
+	const cloudEmbeddingsRoutedLocally = isTruthyEnv(
+		"ELIZA_CLOUD_EMBEDDINGS_DISABLED",
+	);
+
+	if (cloudEmbeddingsRoutedLocally) {
+		// User turned off cloud for embeddings — local plugin must serve TEXT_EMBEDDING.
+		return true;
+	}
+
+	if (isTruthyEnv("ELIZAOS_CLOUD_USE_EMBEDDINGS")) {
+		return false;
+	}
+
+	return true;
+}

package/src/runtime/ensure-local-inference-handler.d.ts

@@ -0,0 +1,62 @@
+import { type AgentRuntime, type IAgentRuntime, type UUID } from "@elizaos/core";
+export declare function shouldRegisterLocalInferenceHandlers(mode: string): boolean;
+/**
+ * AOSP / generic-FFI path: load the fused `libelizainference.so` into the bun
+ * process via `bun:ffi` (the AOSP plugin's loader; libllama is retired). The
+ * loader stays inactive at runtime when neither `ELIZA_LOCAL_LLAMA === "1"`
+ * (kept as the legacy opt-in env name) nor `process.arch === "riscv64"` is
+ * true (see `isAospEnabled` in `@elizaos/plugin-aosp-local-inference`), so the
+ * dynamic import below is safe on every platform; we only attempt registration
+ * when one of the triggers fires.
+ *
+ * riscv64 rationale: `capacitor-llama` ships prebuilts only for
+ * linux-{x64,arm64}, darwin-arm64, win-x64. Riscv64 hosts have no native NAPI
+ * binding option; the cross-built fused `libelizainference.so` is the only
+ * in-process llama.cpp path. The FFI loader satisfies the same
+ * `localInferenceLoader` service contract, so the rest of the engine —
+ * model handlers, embedding routing, response handler — works unchanged.
+ *
+ * The `try`/`catch` is justified because the AOSP build can ship the .so on
+ * one ABI but be invoked on another (e.g. cuttlefish_x86_64 reporting both
+ * x86_64 and arm64-v8a). When `ELIZA_LOCAL_LLAMA=1` is set but registration
+ * fails, the loader logs at `error` level — we must NOT silently fall
+ * through to the device-bridge or stock engine: the operator opted in and
+ * deserves the failure surfaced clearly. The riscv64 auto-trigger uses the
+ * same path; if the bundled `libelizainference.so` is missing the failure is
+ * logged but inference falls through to Cloud routing (per CLAUDE.md deployment
+ * topologies — local-only is supported but Cloud is an acceptable fallback
+ * when the on-device backend is unavailable).
+ */
+export declare function shouldAttemptAospLlamaLoader(env?: NodeJS.ProcessEnv, arch?: NodeJS.Architecture): boolean;
+/**
+ * Bionic-host delegation gate. On Android the app shell sets
+ * `ELIZA_BIONIC_HOST_DELEGATED=1` + `ELIZA_BIONIC_INFERENCE_SOCK=<name>` when a
+ * dynamic-Vulkan `libelizainference.so` is staged — meaning the GPU is reachable
+ * only from the bionic app process, never this musl agent. When set, the agent
+ * delegates inference to that in-process host over the abstract UDS instead of
+ * dlopen'ing the native lib itself (which would hit the Vulkan/HIDL wall).
+ */
+export declare function bionicInferenceSocketName(env?: NodeJS.ProcessEnv): string | null;
+/**
+ * Render the Stage-1 stable prefix for `roomId` and KV-prefill the
+ * local-inference slot that conversation pins to. Wire this from the
+ * voice turn controller (W9) on `speech-start` / voice-session-open so
+ * the response-handler prompt is hot before STT finishes — items I1/C1.
+ *
+ * Best-effort end to end: returns false (no throw) when there's no
+ * loaded local model, the active backend can't pre-warm (node-llama-cpp
+ * pins by cache key already), or rendering/pre-warm fails. A miss just
+ * means the real request cold-prefills.
+ */
+export declare function prewarmResponseHandler(runtime: IAgentRuntime, roomId: UUID): Promise<boolean>;
+/**
+ * Warm the Stage-1 stable prefix onto the deterministic
+ * `conv:__system_prefix__` slot at model-load / boot time, before any
+ * user message — item I3 (warm-on-load). The room id is irrelevant for
+ * the stable prefix (it carries no per-room state), so a fixed synthetic
+ * id is fine. No-op when no local model is loaded or the backend can't
+ * pre-warm. Best-effort: failures are logged at debug and swallowed.
+ */
+export declare function prewarmSystemPrefix(runtime: IAgentRuntime): Promise<boolean>;
+export declare function ensureLocalInferenceHandler(runtime: AgentRuntime): Promise<void>;
+//# sourceMappingURL=ensure-local-inference-handler.d.ts.map

package/src/runtime/ensure-local-inference-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ensure-local-inference-handler.d.ts","sourceRoot":"","sources":["ensure-local-inference-handler.ts"],"names":[],"mappings":"AA0BA,OAAO,EACN,KAAK,YAAY,EAEjB,KAAK,aAAa,EASlB,KAAK,IAAI,EACT,MAAM,eAAe,CAAC;AAiHvB,wBAAgB,oCAAoC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE1E;AAs0BD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,4BAA4B,CAC3C,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,IAAI,GAAE,MAAM,CAAC,YAA2B,GACtC,OAAO,CAKT;AAED;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CACxC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAClC,MAAM,GAAG,IAAI,CAIf;AA4FD;;;;;;;;;;GAUG;AACH,wBAAsB,sBAAsB,CAC3C,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,IAAI,GACV,OAAO,CAAC,OAAO,CAAC,CAiBlB;AAED;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,aAAa,GACpB,OAAO,CAAC,OAAO,CAAC,CAkBlB;AAED,wBAAsB,2BAA2B,CAChD,OAAO,EAAE,YAAY,GACnB,OAAO,CAAC,IAAI,CAAC,CAuPf"}

package/src/runtime/ensure-local-inference-handler.test.ts

@@ -0,0 +1,528 @@
+import { type AgentRuntime, ModelType } from "@elizaos/core";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const modeState = vi.hoisted(() => ({ mode: "local" }));
+const engineState = vi.hoisted(() => ({
+	activeBackendId: vi.fn(() => "llama-server"),
+	available: vi.fn(async () => true),
+	conversation: vi.fn(() => null),
+	currentModelPath: vi.fn(() => null),
+	ensureActiveBundleVoiceReady: vi.fn(async () => undefined),
+	generate: vi.fn(async () => "ok"),
+	generateInConversation: vi.fn(async () => ({
+		slotId: "slot-0",
+		text: "ok",
+		usage: {
+			input_tokens: 0,
+			output_tokens: 0,
+			cache_read_input_tokens: 0,
+			cache_creation_input_tokens: 0,
+		},
+	})),
+	hasLoadedModel: vi.fn(() => false),
+	load: vi.fn(async () => undefined),
+	openConversation: vi.fn(() => ({ id: "conversation" })),
+	prewarmConversation: vi.fn(async () => true),
+	synthesizeSpeech: vi.fn(async () => new Uint8Array([1, 2, 3])),
+	transcribePcm: vi.fn(async () => "transcribed"),
+	warnIfParallelTooLow: vi.fn(),
+}));
+const arbiterState = vi.hoisted(() => ({
+	hasCapability: vi.fn(
+		(capability: string) => capability === "vision-describe",
+	),
+	requestVisionDescribe: vi.fn(async () => ({
+		title: "A small image",
+		description: "A tiny synthetic image.",
+	})),
+}));
+vi.mock("../services/active-model", () => ({
+	resolveLocalInferenceLoadArgs: vi.fn(async (target) => target),
+}));
+
+vi.mock("../services/assignments", () => ({
+	autoAssignAtBoot: vi.fn(async () => null),
+	readEffectiveAssignments: vi.fn(async () => ({})),
+}));
+
+vi.mock("../services/cache-bridge", () => ({
+	extractConversationId: vi.fn(() => null),
+	extractPromptCacheKey: vi.fn(() => null),
+	resolveLocalCacheKey: vi.fn(() => null),
+}));
+
+vi.mock("../services/device-bridge", () => ({
+	deviceBridge: {
+		currentModelPath: vi.fn(() => null),
+		embed: vi.fn(),
+		generate: vi.fn(),
+		loadModel: vi.fn(),
+		unloadModel: vi.fn(),
+	},
+}));
+
+vi.mock("../services/engine", () => ({
+	localInferenceEngine: engineState,
+}));
+
+vi.mock("../services/handler-registry", () => ({
+	handlerRegistry: {
+		installOn: vi.fn(),
+	},
+}));
+
+vi.mock("../services/memory-arbiter", () => ({
+	tryGetMemoryArbiter: vi.fn(() => arbiterState),
+}));
+
+vi.mock("../services/registry", () => ({
+	listInstalledModels: vi.fn(async () => []),
+}));
+
+vi.mock("../services/router-handler", () => ({
+	installRouterHandler: vi.fn(),
+}));
+
+vi.mock("../services/voice", () => ({
+	decodeMonoPcm16Wav: vi.fn(() => ({
+		pcm: new Float32Array([0]),
+		sampleRate: 16_000,
+	})),
+}));
+
+import { installRouterHandler } from "../services/router-handler";
+import { VoiceStartupError } from "../services/voice/errors";
+import { ensureLocalInferenceHandler } from "./ensure-local-inference-handler";
+
+interface Registration {
+	modelType: string | number;
+	provider: string;
+	priority?: number;
+	handler: unknown;
+}
+
+function makeRuntime(): {
+	registrations: Registration[];
+	runtime: AgentRuntime;
+} {
+	const registrations: Registration[] = [];
+	const runtime = {
+		agentId: "agent-test",
+		getModel: vi.fn(() => undefined),
+		getSetting: vi.fn((key: string) =>
+			key === "ELIZA_RUNTIME_MODE" ? modeState.mode : undefined,
+		),
+		getService: vi.fn(() => null),
+		setSetting: vi.fn(),
+		registerModel: vi.fn(
+			(
+				modelType: string | number,
+				_handler: unknown,
+				provider: string,
+				priority?: number,
+			) => {
+				registrations.push({
+					modelType,
+					provider,
+					priority,
+					handler: _handler,
+				});
+			},
+		),
+		registerService: vi.fn(),
+	} as unknown as AgentRuntime;
+	return { registrations, runtime };
+}
+
+function findRegisteredHandler(
+	registrations: Registration[],
+	modelType: ModelType,
+): (runtime: AgentRuntime, params: Record<string, unknown>) => Promise<string> {
+	const registration = registrations.find(
+		(entry) => entry.modelType === modelType,
+	);
+	expect(registration).toBeDefined();
+	return registration?.handler as (
+		runtime: AgentRuntime,
+		params: Record<string, unknown>,
+	) => Promise<string>;
+}
+
+beforeEach(() => {
+	vi.clearAllMocks();
+	modeState.mode = "local";
+	delete process.env.ELIZA_LOCAL_LLAMA;
+	delete process.env.ELIZA_DEVICE_BRIDGE_ENABLED;
+	delete process.env.ELIZA_DISABLE_LOCAL_EMBEDDINGS;
+	engineState.available.mockResolvedValue(true);
+	engineState.currentModelPath.mockReturnValue(null);
+	engineState.hasLoadedModel.mockReturnValue(false);
+	arbiterState.hasCapability.mockImplementation(
+		(capability: string) => capability === "vision-describe",
+	);
+	arbiterState.requestVisionDescribe.mockResolvedValue({
+		title: "A small image",
+		description: "A tiny synthetic image.",
+	});
+});
+
+describe("ensureLocalInferenceHandler", () => {
+	it("registers Eliza-1 text, embedding, voice, and transcription handlers in local mode", async () => {
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+
+		expect(registrations).toEqual(
+			expect.arrayContaining([
+				expect.objectContaining({
+					modelType: ModelType.TEXT_SMALL,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.TEXT_LARGE,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.RESPONSE_HANDLER,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.ACTION_PLANNER,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.TEXT_COMPLETION,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.TEXT_EMBEDDING,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.TEXT_TO_SPEECH,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.TRANSCRIPTION,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+				expect.objectContaining({
+					modelType: ModelType.IMAGE_DESCRIPTION,
+					provider: "eliza-local-inference",
+					priority: 0,
+				}),
+			]),
+		);
+	});
+
+	it("honors ELIZA_DISABLE_LOCAL_EMBEDDINGS by leaving TEXT_EMBEDDING unregistered", async () => {
+		process.env.ELIZA_DISABLE_LOCAL_EMBEDDINGS = "1";
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+
+		expect(
+			registrations.some(
+				(entry) => entry.modelType === ModelType.TEXT_EMBEDDING,
+			),
+		).toBe(false);
+		expect(registrations).toEqual(
+			expect.arrayContaining([
+				expect.objectContaining({ modelType: ModelType.TEXT_SMALL }),
+				expect.objectContaining({ modelType: ModelType.TEXT_LARGE }),
+				expect.objectContaining({ modelType: ModelType.RESPONSE_HANDLER }),
+				expect.objectContaining({ modelType: ModelType.ACTION_PLANNER }),
+				expect.objectContaining({ modelType: ModelType.TEXT_COMPLETION }),
+				expect.objectContaining({ modelType: ModelType.TEXT_TO_SPEECH }),
+				expect.objectContaining({ modelType: ModelType.TRANSCRIPTION }),
+			]),
+		);
+		expect(installRouterHandler).toHaveBeenCalledWith(runtime, {
+			skipSlots: ["TEXT_EMBEDDING"],
+		});
+	});
+
+	it("skips handler registration outside local modes", async () => {
+		modeState.mode = "cloud";
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+
+		expect(registrations).toHaveLength(0);
+		expect(engineState.available).not.toHaveBeenCalled();
+	});
+
+	it("does not duplicate registrations on the same runtime", async () => {
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+		const firstCount = registrations.length;
+		await ensureLocalInferenceHandler(runtime);
+
+		expect(registrations).toHaveLength(firstCount);
+	});
+
+	it("renders v5 messages into a non-empty local prompt", async () => {
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(true);
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(registrations, ModelType.TEXT_SMALL);
+
+		await handler(runtime, {
+			messages: [
+				{ role: "system", content: "You are Eliza." },
+				{ role: "user", content: "hello. say hello back" },
+			],
+			maxTokens: 32,
+			temperature: 0.1,
+			topP: 0.9,
+		});
+
+		expect(engineState.generate).toHaveBeenCalledWith(
+			expect.objectContaining({
+				prompt: "system:\nYou are Eliza.\n\nuser:\nhello. say hello back",
+				maxTokens: 32,
+				temperature: 0.1,
+				topP: 0.9,
+			}),
+		);
+	});
+
+	it.each([
+		[ModelType.TEXT_SMALL, "TEXT_SMALL"],
+		[ModelType.TEXT_LARGE, "TEXT_LARGE"],
+		[ModelType.RESPONSE_HANDLER, "TEXT_SMALL"],
+	])("signals typed local unavailability for %s when no text model is loaded", async (modelType, slot) => {
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(false);
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(registrations, modelType);
+
+		await expect(
+			handler(runtime, {
+				messages: [{ role: "user", content: "hello" }],
+			}),
+		).rejects.toMatchObject({
+			code: "LOCAL_INFERENCE_UNAVAILABLE",
+			modelType: slot,
+			reason: "backend_unavailable",
+		});
+	});
+
+	it.each([
+		[ModelType.TEXT_SMALL, "TEXT_SMALL"],
+		[ModelType.TEXT_LARGE, "TEXT_LARGE"],
+		[ModelType.RESPONSE_HANDLER, "TEXT_SMALL"],
+	])("signals typed local unavailability for %s when the backend is unavailable", async (modelType, slot) => {
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(true);
+
+		// Register while the backend reports available (the pre-flight gate skips
+		// registration otherwise), then drop the binding to exercise the handler's
+		// runtime-defensive unavailability check — the real "binding went away
+		// after boot" scenario.
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(registrations, modelType);
+		engineState.available.mockResolvedValue(false);
+
+		await expect(
+			handler(runtime, {
+				messages: [{ role: "user", content: "hello" }],
+			}),
+		).rejects.toMatchObject({
+			code: "LOCAL_INFERENCE_UNAVAILABLE",
+			modelType: slot,
+			reason: "backend_unavailable",
+		});
+	});
+
+	it("routes image description through the Eliza-1 vision arbiter", async () => {
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+		const registration = registrations.find(
+			(entry) => entry.modelType === ModelType.IMAGE_DESCRIPTION,
+		);
+		const handler = registration?.handler as
+			| ((
+					runtime: AgentRuntime,
+					params: Record<string, unknown>,
+			  ) => Promise<{ title: string; description: string }>)
+			| undefined;
+		expect(handler).toBeDefined();
+
+		await expect(
+			handler?.(runtime, {
+				imageUrl: "data:image/png;base64,AAAA",
+				prompt: "describe this",
+			}),
+		).resolves.toEqual({
+			title: "A small image",
+			description: "A tiny synthetic image.",
+		});
+		expect(arbiterState.requestVisionDescribe).toHaveBeenCalledWith({
+			modelKey: "qwen3-vl",
+			payload: {
+				image: { kind: "dataUrl", dataUrl: "data:image/png;base64,AAAA" },
+				prompt: "describe this",
+			},
+		});
+		expect(runtime.setSetting).toHaveBeenCalledWith(
+			"ELIZA1_VISION_HANDLER_PRESENT",
+			"1",
+		);
+	});
+
+	it("arms the active voice bundle before TRANSCRIPTION", async () => {
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+		const registration = registrations.find(
+			(entry) => entry.modelType === ModelType.TRANSCRIPTION,
+		);
+		const handler = registration?.handler as
+			| ((
+					runtime: AgentRuntime,
+					params: Record<string, unknown>,
+			  ) => Promise<string>)
+			| undefined;
+		expect(handler).toBeDefined();
+
+		await expect(
+			handler?.(runtime, { audio: new Uint8Array([82, 73, 70, 70]) }),
+		).resolves.toBe("transcribed");
+
+		expect(engineState.ensureActiveBundleVoiceReady).toHaveBeenCalledTimes(1);
+		expect(engineState.transcribePcm).toHaveBeenCalledWith(
+			{ pcm: new Float32Array([0]), sampleRate: 16_000 },
+			undefined,
+		);
+	});
+
+	it("fails fast when the fused voice bundle is unavailable (no whisper fallback)", async () => {
+		// The fused libelizainference ASR runtime is the sole on-device
+		// transcriber. A startup failure must propagate (AGENTS.md §3) — there is
+		// no whisper.cpp second attempt and no silent empty transcript.
+		engineState.ensureActiveBundleVoiceReady.mockRejectedValueOnce(
+			new VoiceStartupError("missing-bundle-root", "no bundle"),
+		);
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+		const registration = registrations.find(
+			(entry) => entry.modelType === ModelType.TRANSCRIPTION,
+		);
+		const handler = registration?.handler as
+			| ((
+					runtime: AgentRuntime,
+					params: Record<string, unknown>,
+			  ) => Promise<string>)
+			| undefined;
+		expect(handler).toBeDefined();
+
+		await expect(
+			handler?.(runtime, { audio: new Uint8Array([82, 73, 70, 70]) }),
+		).rejects.toThrow(VoiceStartupError);
+
+		expect(engineState.transcribePcm).not.toHaveBeenCalled();
+	});
+
+	it("threads structured streaming callbacks through the RESPONSE_HANDLER registration", async () => {
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(true);
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(
+			registrations,
+			ModelType.RESPONSE_HANDLER,
+		);
+
+		const onStreamChunk = vi.fn();
+		await handler(runtime, {
+			messages: [{ role: "user", content: "hello" }],
+			streamStructured: true,
+			responseSkeleton: { spans: [] },
+			onStreamChunk,
+		});
+
+		expect(engineState.generate).toHaveBeenCalledWith(
+			expect.objectContaining({
+				prompt: "user:\nhello",
+				streamStructured: true,
+				onTextChunk: expect.any(Function),
+			}),
+		);
+	});
+
+	it("delivers engine onTextChunk tokens to the caller's onStreamChunk per token (chat streaming)", async () => {
+		// End-to-end guard for the local chat streaming regression: the registered
+		// RESPONSE_HANDLER handler must connect the runtime's `onStreamChunk` to the
+		// engine's `onTextChunk` so each generated token is delivered incrementally,
+		// not collapsed into one final chunk. The mocked engine fires onTextChunk
+		// per token (mirroring NodeLlamaCppBackend/FfiStreamingBackend), and we
+		// assert the caller saw multiple distinct chunks in order.
+		const tokens = ["On ", "it ", "now."];
+		engineState.generate.mockImplementationOnce(
+			async (args: { onTextChunk?: (chunk: string) => unknown }) => {
+				for (const token of tokens) {
+					await args.onTextChunk?.(token);
+				}
+				return tokens.join("");
+			},
+		);
+
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(true);
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(
+			registrations,
+			ModelType.RESPONSE_HANDLER,
+		);
+
+		const received: string[] = [];
+		await handler(runtime, {
+			messages: [{ role: "user", content: "hello" }],
+			streamStructured: true,
+			responseSkeleton: { spans: [] },
+			onStreamChunk: (chunk: string) => {
+				received.push(chunk);
+			},
+		});
+
+		expect(received).toEqual(tokens);
+		expect(received.length).toBeGreaterThan(1);
+	});
+
+	it("threads eliza thinking provider options into local engine args", async () => {
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(true);
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(
+			registrations,
+			ModelType.RESPONSE_HANDLER,
+		);
+
+		await handler(runtime, {
+			messages: [{ role: "user", content: "hello" }],
+			providerOptions: { eliza: { thinking: "off" } },
+		});
+
+		expect(engineState.generate).toHaveBeenCalledWith(
+			expect.objectContaining({
+				thinking: "off",
+			}),
+		);
+	});
+});