@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/bionic-host-loader.test.ts

@@ -0,0 +1,133 @@
+import net from "node:net";
+import { afterEach, describe, expect, it } from "vitest";
+import { BionicHostLoader } from "./bionic-host-loader";
+
+/**
+ * Real-IPC test: stand up an actual abstract-namespace AF_UNIX server (the same
+ * transport ElizaBionicInferenceServer.java binds on the device) and drive the
+ * loader against it. No mocks — this exercises the real node:net framing.
+ */
+
+function frame(json: string): Buffer {
+	const payload = Buffer.from(json, "utf8");
+	const out = Buffer.allocUnsafe(4 + payload.length);
+	out.writeUInt32BE(payload.length, 0);
+	payload.copy(out, 4);
+	return out;
+}
+
+/** A test host that decodes one request frame and replies with `respond(req)`. */
+function startHost(
+	name: string,
+	respond: (req: Record<string, unknown>) => string,
+): net.Server {
+	const server = net.createServer((sock) => {
+		let buf = Buffer.alloc(0);
+		let expected = -1;
+		sock.on("data", (d) => {
+			buf = Buffer.concat([buf, d]);
+			if (expected < 0 && buf.length >= 4) expected = buf.readUInt32BE(0);
+			if (expected >= 0 && buf.length >= 4 + expected) {
+				const req = JSON.parse(buf.subarray(4, 4 + expected).toString("utf8"));
+				sock.write(frame(respond(req)));
+			}
+		});
+	});
+	server.listen({ path: `\0${name}` });
+	return server;
+}
+
+let host: net.Server | null = null;
+afterEach(() => {
+	host?.close();
+	host = null;
+});
+
+const SOCK = `eliza-bionic-test-${process.pid}`;
+
+describe("BionicHostLoader (real abstract-UDS)", () => {
+	it("round-trips a buffered generate and returns the host completion", async () => {
+		let seen: Record<string, unknown> | null = null;
+		host = startHost(SOCK, (req) => {
+			seen = req;
+			return JSON.stringify({
+				ok: true,
+				text: "Two plus two equals four.",
+				tokens: 7,
+				ms: 500,
+				tokS: 14,
+			});
+		});
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({
+			modelPath: "/data/x/eliza-1/bundle/text/model.gguf",
+		});
+		expect(loader.currentModelPath()).toBe(
+			"/data/x/eliza-1/bundle/text/model.gguf",
+		);
+		const out = await loader.generate({
+			prompt: "what is 2+2?",
+			maxTokens: 32,
+		});
+		expect(out).toBe("Two plus two equals four.");
+		// bundleDir derived from the .../text/<model>.gguf layout.
+		expect(seen).toMatchObject({
+			op: "generate",
+			prompt: "what is 2+2?",
+			maxTokens: 32,
+			bundleDir: "/data/x/eliza-1/bundle",
+		});
+	});
+
+	it("forwards an empty bundleDir when the model is not in a text/ bundle", async () => {
+		let seen: Record<string, unknown> | null = null;
+		host = startHost(SOCK, (req) => {
+			seen = req;
+			return JSON.stringify({ ok: true, text: "hi" });
+		});
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({ modelPath: "/models/flat-model.gguf" });
+		await loader.generate({ prompt: "hi" });
+		expect((seen as { bundleDir?: string } | null)?.bundleDir).toBe("");
+	});
+
+	it("throws when the host returns ok:false", async () => {
+		host = startHost(SOCK, () =>
+			JSON.stringify({ ok: false, error: "no vulkan device" }),
+		);
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({ modelPath: "/m/text/x.gguf" });
+		await expect(loader.generate({ prompt: "x" })).rejects.toThrow(
+			/no vulkan device/,
+		);
+	});
+
+	it("survives a response split across multiple data chunks (multibyte safe)", async () => {
+		const text = `héllo 🌊 ünïcode ${"x".repeat(5000)}`;
+		host = net.createServer((sock) => {
+			let buf = Buffer.alloc(0);
+			let expected = -1;
+			sock.on("data", (d) => {
+				buf = Buffer.concat([buf, d]);
+				if (expected < 0 && buf.length >= 4) expected = buf.readUInt32BE(0);
+				if (expected >= 0 && buf.length >= 4 + expected) {
+					const full = frame(JSON.stringify({ ok: true, text }));
+					// Write in two pieces, splitting mid-buffer to exercise reassembly.
+					sock.write(full.subarray(0, 10));
+					setTimeout(() => sock.write(full.subarray(10)), 5);
+				}
+			});
+		});
+		host.listen({ path: `\0${SOCK}` });
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({ modelPath: "/m/text/x.gguf" });
+		const out = await loader.generate({ prompt: "x" });
+		expect(out).toBe(text);
+	});
+
+	it("rejects when the host is unreachable", async () => {
+		const loader = new BionicHostLoader(`eliza-bionic-absent-${process.pid}`);
+		await loader.loadModel({ modelPath: "/m/text/x.gguf" });
+		await expect(loader.generate({ prompt: "x" })).rejects.toThrow();
+	});
+});

package/src/services/bionic-host-loader.ts

@@ -0,0 +1,180 @@
+/**
+ * BionicHostLoader — the agent-side half of the on-device GPU delegation path.
+ *
+ * On Android the elizaOS agent runs as embedded bun under the musl loader, whose
+ * restricted linker namespace cannot load the bionic Android Vulkan driver (its
+ * HIDL/HAL closure) — so the musl agent can only run inference on the CPU. The
+ * GPU is reachable only from the normal bionic `ai.elizaos.app` process, where
+ * `ElizaBionicInferenceServer` (Java) has loaded `libelizainference.so` +
+ * `libggml-vulkan.so` and offloads the model to the Mali GPU.
+ *
+ * This loader implements the standard {@link LocalInferenceLoader} contract, so
+ * the TEXT_SMALL / TEXT_LARGE handlers in `ensure-local-inference-handler.ts`
+ * route through it transparently. `generate()` sends the prompt to the bionic
+ * host over an abstract-namespace `AF_UNIX` socket and gets the GPU completion
+ * back — the whole decode loop runs server-side, so there is no per-token
+ * two-process round trip.
+ *
+ * This is the buffered first slice (one GENERATE request → one full completion).
+ * Server-push per-step streaming, embed, and cancel are layered on later via the
+ * shared `LlmStreamingBinding`; the wire framing already carries an `op`
+ * discriminator for that.
+ */
+
+import net from "node:net";
+import path from "node:path";
+import { logger } from "@elizaos/core";
+import type {
+	LocalInferenceLoadArgs,
+	LocalInferenceLoader,
+} from "./active-model";
+
+/** Connect + full round-trip budget. A cold GPU decode of a long reply fits. */
+const REQUEST_TIMEOUT_MS = 120_000;
+/** Defensive ceiling on a single response frame (a full completion). */
+const MAX_FRAME_BYTES = 64 * 1024 * 1024;
+
+interface BionicGenerateResponse {
+	ok: boolean;
+	text?: string;
+	error?: string;
+	tokens?: number;
+	ms?: number;
+	tokS?: number;
+}
+
+/**
+ * Derive the fused-bundle root from a model GGUF path. The host's
+ * `eliza_inference_create(bundleDir)` expects the directory that contains
+ * `text/<model>.gguf`; when the installed model is laid out that way we forward
+ * it, otherwise we send empty and let the host fall back to its default bundle.
+ */
+function deriveBundleDir(modelPath: string): string {
+	if (!modelPath) return "";
+	const dir = path.dirname(modelPath);
+	if (path.basename(dir) === "text") return path.dirname(dir);
+	return "";
+}
+
+export class BionicHostLoader implements LocalInferenceLoader {
+	private modelPath: string | null = null;
+	private bundleDir = "";
+
+	/** @param socketName abstract-namespace socket name (no leading NUL). */
+	constructor(private readonly socketName: string) {}
+
+	async loadModel(args: LocalInferenceLoadArgs): Promise<void> {
+		this.modelPath = args.modelPath;
+		this.bundleDir = deriveBundleDir(args.modelPath);
+		logger.info(
+			`[BionicHostLoader] active model ${args.modelPath} (bundle ${this.bundleDir || "<host-default>"})`,
+		);
+	}
+
+	async unloadModel(): Promise<void> {
+		this.modelPath = null;
+	}
+
+	currentModelPath(): string | null {
+		return this.modelPath;
+	}
+
+	async generate(args: {
+		prompt: string;
+		stopSequences?: string[];
+		maxTokens?: number;
+		temperature?: number;
+		cacheKey?: string;
+	}): Promise<string> {
+		const res = await this.roundTrip<BionicGenerateResponse>({
+			op: "generate",
+			bundleDir: this.bundleDir,
+			prompt: args.prompt,
+			maxTokens: args.maxTokens ?? 256,
+			temperature: args.temperature ?? 0,
+		});
+		if (!res.ok) {
+			throw new Error(
+				`[BionicHostLoader] host generate failed: ${res.error ?? "unknown error"}`,
+			);
+		}
+		if (typeof res.tokS === "number") {
+			logger.debug(
+				`[BionicHostLoader] generated ${res.tokens ?? "?"} tok @ ${res.tokS.toFixed(1)} tok/s on the bionic GPU host`,
+			);
+		}
+		return res.text ?? "";
+	}
+
+	/**
+	 * One request → one response over a fresh connection. Length-prefixed frames:
+	 * `[int32 BE byte length][UTF-8 JSON]` in each direction.
+	 */
+	private roundTrip<T>(request: Record<string, unknown>): Promise<T> {
+		const payload = Buffer.from(JSON.stringify(request), "utf8");
+		const frame = Buffer.allocUnsafe(4 + payload.length);
+		frame.writeUInt32BE(payload.length, 0);
+		payload.copy(frame, 4);
+
+		return new Promise<T>((resolve, reject) => {
+			// Abstract-namespace socket: a leading NUL byte in the path.
+			const sock = net.connect({ path: `\0${this.socketName}` });
+			let settled = false;
+			let chunks: Buffer = Buffer.alloc(0);
+			let expected = -1;
+
+			const finish = (err: Error | null, value?: T) => {
+				if (settled) return;
+				settled = true;
+				clearTimeout(timer);
+				sock.destroy();
+				if (err) reject(err);
+				else resolve(value as T);
+			};
+
+			const timer = setTimeout(
+				() => finish(new Error("[BionicHostLoader] request timed out")),
+				REQUEST_TIMEOUT_MS,
+			);
+
+			sock.on("connect", () => sock.write(frame));
+			sock.on("data", (d: Buffer) => {
+				chunks = Buffer.concat([chunks, d]);
+				if (expected < 0 && chunks.length >= 4) {
+					expected = chunks.readUInt32BE(0);
+					if (expected < 0 || expected > MAX_FRAME_BYTES) {
+						finish(
+							new Error(
+								`[BionicHostLoader] bad response frame length ${expected}`,
+							),
+						);
+						return;
+					}
+				}
+				if (expected >= 0 && chunks.length >= 4 + expected) {
+					const json = chunks.subarray(4, 4 + expected).toString("utf8");
+					try {
+						finish(null, JSON.parse(json) as T);
+					} catch (e) {
+						finish(
+							new Error(
+								`[BionicHostLoader] malformed response: ${e instanceof Error ? e.message : String(e)}`,
+							),
+						);
+					}
+				}
+			});
+			sock.on("error", (e: Error) =>
+				finish(new Error(`[BionicHostLoader] socket error: ${e.message}`)),
+			);
+			sock.on("close", () => {
+				if (!settled)
+					finish(
+						new Error(
+							"[BionicHostLoader] host closed the connection before responding",
+						),
+					);
+			});
+		});
+	}
+}

package/src/services/bundled-models.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Bundled-models bootstrap for AOSP / on-device installs.
+ *
+ * The AOSP build pipeline stages Eliza-1 models into the APK at
+ * `assets/agent/models/{file}.gguf` plus a
+ * `manifest.json` describing each one (id, role, sha256, sizeBytes).
+ * `ElizaAgentService.extractAssetsIfNeeded()` copies those files into
+ * `$ELIZA_STATE_DIR/local-inference/models/` on first launch.
+ *
+ * This module reads the manifest at runtime startup and registers each
+ * file as a eliza-owned model in the local-inference registry, so the
+ * auto-assign pass picks them up for TEXT_LARGE / TEXT_SMALL /
+ * TEXT_EMBEDDING slots without needing the user to download anything.
+ *
+ * Idempotent: re-running with the registry already populated is a
+ * no-op for unchanged entries (`upsertElizaModel` overwrites entries
+ * with the same id, so updated sha256s on a later re-bundle replace
+ * the old metadata cleanly).
+ *
+ * Source classification: the runtime treats bundled models as
+ * `source: "eliza-download"` because Eliza ships the file and Eliza
+ * owns it on disk — same lifecycle as a user-initiated download
+ * (uninstall removes the file, the registry tracks the install). The
+ * only difference is the file arrived via APK extraction rather than
+ * an HTTP transfer.
+ */
+/**
+ * Walk the manifest and register every bundled GGUF file in the
+ * local-inference registry. Returns the number of entries successfully
+ * registered. A missing manifest is normal on Capacitor / desktop /
+ * non-AOSP installs and returns 0 silently.
+ */
+export declare function registerBundledModels(): Promise<number>;
+//# sourceMappingURL=bundled-models.d.ts.map

package/src/services/bundled-models.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundled-models.d.ts","sourceRoot":"","sources":["bundled-models.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AA2DH;;;;;GAKG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,MAAM,CAAC,CAsC7D"}

package/src/services/bundled-models.ts

@@ -0,0 +1,129 @@
+/**
+ * Bundled-models bootstrap for AOSP / on-device installs.
+ *
+ * The AOSP build pipeline stages Eliza-1 models into the APK at
+ * `assets/agent/models/{file}.gguf` plus a
+ * `manifest.json` describing each one (id, role, sha256, sizeBytes).
+ * `ElizaAgentService.extractAssetsIfNeeded()` copies those files into
+ * `$ELIZA_STATE_DIR/local-inference/models/` on first launch.
+ *
+ * This module reads the manifest at runtime startup and registers each
+ * file as a eliza-owned model in the local-inference registry, so the
+ * auto-assign pass picks them up for TEXT_LARGE / TEXT_SMALL /
+ * TEXT_EMBEDDING slots without needing the user to download anything.
+ *
+ * Idempotent: re-running with the registry already populated is a
+ * no-op for unchanged entries (`upsertElizaModel` overwrites entries
+ * with the same id, so updated sha256s on a later re-bundle replace
+ * the old metadata cleanly).
+ *
+ * Source classification: the runtime treats bundled models as
+ * `source: "eliza-download"` because Eliza ships the file and Eliza
+ * owns it on disk — same lifecycle as a user-initiated download
+ * (uninstall removes the file, the registry tracks the install). The
+ * only difference is the file arrived via APK extraction rather than
+ * an HTTP transfer.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import {
+	ensureDefaultAssignment,
+	readAssignments,
+	writeAssignments,
+} from "./assignments";
+import { elizaModelsDir } from "./paths";
+import { upsertElizaModel } from "./registry";
+import type { InstalledModel } from "./types";
+
+interface BundledModelEntry {
+	id: string;
+	displayName: string;
+	hfRepo: string;
+	ggufFile: string;
+	role: "chat" | "embedding";
+	sizeBytes: number;
+	sha256: string | null;
+}
+
+interface BundledModelManifest {
+	version: 1;
+	models: BundledModelEntry[];
+}
+
+function manifestPath(): string {
+	return path.join(elizaModelsDir(), "manifest.json");
+}
+
+async function readManifest(): Promise<BundledModelManifest | null> {
+	try {
+		const raw = await fs.readFile(manifestPath(), "utf8");
+		const parsed = JSON.parse(raw) as BundledModelManifest;
+		if (parsed.version !== 1 || !Array.isArray(parsed.models)) {
+			return null;
+		}
+		return parsed;
+	} catch {
+		return null;
+	}
+}
+
+async function ensureBundledAssignment(
+	modelId: string,
+	role: BundledModelEntry["role"],
+): Promise<void> {
+	if (role !== "embedding") {
+		await ensureDefaultAssignment(modelId);
+		return;
+	}
+
+	const current = await readAssignments();
+	if (current.TEXT_EMBEDDING) return;
+	await writeAssignments({ ...current, TEXT_EMBEDDING: modelId });
+}
+
+/**
+ * Walk the manifest and register every bundled GGUF file in the
+ * local-inference registry. Returns the number of entries successfully
+ * registered. A missing manifest is normal on Capacitor / desktop /
+ * non-AOSP installs and returns 0 silently.
+ */
+export async function registerBundledModels(): Promise<number> {
+	const manifest = await readManifest();
+	if (!manifest) return 0;
+	const dir = elizaModelsDir();
+	let registered = 0;
+	for (const entry of manifest.models) {
+		const filePath = path.join(dir, entry.ggufFile);
+		let sizeBytes = entry.sizeBytes;
+		try {
+			const stat = await fs.stat(filePath);
+			sizeBytes = stat.size;
+		} catch {
+			// File didn't extract — manifest references something the APK
+			// didn't ship. Skip this entry rather than registering a broken
+			// path. AOSP build's stage-default-models.mjs is the source of
+			// truth; if a file is missing the build is broken upstream.
+			continue;
+		}
+		const installed: InstalledModel = {
+			id: entry.id,
+			displayName: entry.displayName,
+			path: filePath,
+			sizeBytes,
+			hfRepo: entry.hfRepo,
+			installedAt: new Date().toISOString(),
+			lastUsedAt: null,
+			source: "eliza-download",
+			sha256: entry.sha256 ?? undefined,
+		};
+		await upsertElizaModel(installed);
+		// Auto-assign each bundled model to its manifest role if the user
+		// hasn't already assigned that slot. This keeps Eliza-1 chat models
+		// eligible for TEXT_EMBEDDING when the bundle explicitly marks them
+		// as the local embedding bootstrap model.
+		await ensureBundledAssignment(entry.id, entry.role);
+		registered += 1;
+	}
+	return registered;
+}

package/src/services/cache-bridge.d.ts

@@ -0,0 +1,206 @@
+/**
+ * Cache bridge for the local-inference path.
+ *
+ * Translates the runtime's `ProviderCachePlan` (a provider-neutral cache
+ * plan emitted by `@elizaos/core`'s `buildProviderCachePlan`) into
+ * concrete behaviour for the two local backends:
+ *
+ *   1. Out-of-process llama-server (MTP / buun-llama-cpp): stable
+ *      slot-id derivation + on-disk slot KV save/restore directory layout
+ *      + TTL-based eviction by mtime.
+ *   2. In-process node-llama-cpp: a session pool (see
+ *      `session-pool.ts`) keyed by `promptCacheKey`.
+ *
+ * This module is pure logic — no llama-server process management, no
+ * node-llama-cpp imports. All filesystem state is rooted under
+ * `local-inference/llama-cache/` so cleanup is easy and explicit.
+ */
+/**
+ * TTLs for cached prefix data, mirroring the cloud-side semantics:
+ *   - `short`: roughly the "default" Anthropic ephemeral cache window.
+ *   - `long`: roughly the "1h" Anthropic ephemeral cache window.
+ *   - `extended`: the OpenAI 24h prompt-cache retention window.
+ *
+ * Values are in milliseconds. Eviction uses file mtime, not access time,
+ * so a slot that is read repeatedly without being rewritten still ages
+ * out — which matches how llama-server writes the slot file each turn.
+ */
+export interface CacheTtls {
+    short: number;
+    long: number;
+    extended?: number;
+}
+export declare const DEFAULT_CACHE_TTLS: CacheTtls;
+/**
+ * Root directory for all local llama-cache state. Anything inside is
+ * Eliza-owned and safe to delete to reset the cache.
+ */
+export declare function llamaCacheRoot(): string;
+/**
+ * Per-model-hash cache directory. Slot save files for one model never
+ * collide with another model's; switching active model does not need to
+ * wipe the cache.
+ */
+export declare function cacheRoot(modelHash: string): string;
+/**
+ * llama-server `--slot-save-path` argument: the directory llama-server
+ * writes per-slot KV state into when a request includes
+ * `cache_prompt: true`. One directory per model hash.
+ */
+export declare function slotSavePath(modelHash: string): string;
+/**
+ * Stable model-fingerprint hash. Combines the absolute paths of the
+ * target / drafter GGUFs and the cache-type knobs so two distinct
+ * configurations don't share a slot directory.
+ */
+export declare function buildModelHash(input: {
+    targetModelPath: string;
+    drafterModelPath?: string | null;
+    cacheTypeK?: string | null;
+    cacheTypeV?: string | null;
+    /** Optional extra discriminator (context size, parallel, etc.). */
+    extra?: string | null;
+}): string;
+/**
+ * Map a `promptCacheKey` to a llama-server slot id in [0, parallel).
+ *
+ * llama-server's `--parallel N` flag pre-allocates N decoding slots and
+ * accepts a `slot_id` integer in `[0, N-1]` on each request. By hashing
+ * the cache key into that range we get:
+ *
+ *   - The same prefix hash always lands on the same slot, so the in-RAM
+ *     KV cache from the previous turn is reused.
+ *   - Different prefix hashes spread across slots and don't fight for
+ *     the same KV memory.
+ *
+ * Pass `parallel <= 0` to disable slot pinning (returns -1, the
+ * llama-server "any free slot" sentinel).
+ */
+export declare function deriveSlotId(promptCacheKey: string, parallel: number): number;
+/**
+ * Convert the runtime-side `CacheTTL` enum + OpenAI extended retention
+ * hint into a concrete TTL in milliseconds. This is what the eviction
+ * sweep uses when deciding whether a slot file is still live.
+ */
+export declare function ttlMsForKey(ttl: "short" | "long" | "extended" | undefined, ttls?: CacheTtls): number;
+/** TTL classes that can be encoded into a slot `.bin` filename. */
+export type SlotCacheTtlClass = "short" | "long" | "extended";
+/**
+ * Build the basename for a persisted slot/conversation `.bin` file with
+ * its TTL class encoded as a middle component: `<base>.<ttl>.bin`. The
+ * eviction sweep reads that component back via `parseSlotCacheTtlClass`
+ * so a slot persisted with the long retention window isn't deleted on
+ * the short horizon (and vice versa). Pass `"long"` for cross-restart
+ * conversation KV — that matches the prior global (long-only) behaviour
+ * for those files.
+ */
+export declare function slotCacheFileName(base: string, ttl: SlotCacheTtlClass): string;
+/**
+ * Parse the TTL class encoded into a slot `.bin` filename by
+ * `slotCacheFileName`. Returns `undefined` for legacy / hand-written
+ * filenames without an encoded class — those keep the `long` horizon
+ * (the prior global behaviour for persisted slot files).
+ */
+export declare function parseSlotCacheTtlClass(fileName: string): SlotCacheTtlClass | undefined;
+/**
+ * Sweep the slot-save directory and delete files older than their
+ * per-file TTL horizon. The TTL class is read from the filename
+ * (`<base>.<ttl>.bin` — see `slotCacheFileName`); files without an
+ * encoded class use the `long` horizon (the prior global behaviour).
+ * Mtime is the watermark; llama-server rewrites the slot file on every
+ * save, so a slot that's actively used keeps a fresh mtime.
+ *
+ * Returns the number of files deleted. Missing directories are not
+ * errors — eviction on a clean install just no-ops.
+ */
+export declare function evictExpired(rootDir: string, ttls?: CacheTtls, now?: number): Promise<number>;
+export interface CacheStatsEntry {
+    file: string;
+    sizeBytes: number;
+    mtimeMs: number;
+    ageMs: number;
+}
+/**
+ * Snapshot of the on-disk slot-save directory. Used by the public
+ * `getLocalCacheStats()` debugging endpoint.
+ */
+export declare function readCacheStats(rootDir: string, now?: number): Promise<CacheStatsEntry[]>;
+/**
+ * Resolve `promptCacheKey` from a `providerOptions` payload as emitted
+ * by `buildProviderCachePlan`. The runtime stuffs it under
+ * `providerOptions.eliza.promptCacheKey`. Returns `null` when the key is
+ * missing or not a non-empty string — callers fall back to the default
+ * "_default" session in that case.
+ */
+export declare function extractPromptCacheKey(providerOptions: unknown): string | null;
+/**
+ * Resolve `prefixHash` from `providerOptions.eliza.prefixHash`. Mirrors
+ * `extractPromptCacheKey` — returns null when missing or not a non-empty
+ * string. The prefix hash covers ONLY the stable prompt prefix (system
+ * prompt + tool definitions + large constant context), so a runtime
+ * timestamp in the unstable tail does not invalidate it.
+ *
+ * Local backends prefer this over `promptCacheKey` when available because
+ * it gives the strongest "same prefix → same slot" guarantee: two
+ * conversations with byte-identical stable prefixes will land on the same
+ * slot regardless of how their tail content differs.
+ */
+export declare function extractPrefixHash(providerOptions: unknown): string | null;
+/**
+ * Stable annotation describing a single segment of the prompt as it was
+ * emitted by the runtime planner. The cache-bridge consumes this to
+ * compute a stable-prefix-only hash for slot pinning, without having to
+ * look at the (timestamp-laden) tail.
+ *
+ * Mirrors `PromptSegment` in @elizaos/core/src/types/model.ts but is kept
+ * standalone so the cache-bridge can be imported by the local-inference
+ * backends without a hard dep on `@elizaos/core`.
+ */
+export interface AnnotatedPromptSegment {
+    content: string;
+    stable: boolean;
+}
+/**
+ * Hash the longest stable prefix of `segments`. Stops at the first
+ * unstable segment, so a runtime timestamp in the unstable tail never
+ * shifts the hash. Returns `null` when no stable segment exists, signaling
+ * to the caller that prefix-cache reuse cannot be derived purely from the
+ * prompt structure (fall back to the prompt-cache-key path instead).
+ *
+ * The hash is sha256-truncated to 16 hex chars, matching `buildModelHash`
+ * — short enough for log lines, wide enough that collision is not a
+ * realistic concern for any plausible number of concurrent prefixes.
+ */
+export declare function hashStablePrefix(segments: readonly AnnotatedPromptSegment[]): string | null;
+/**
+ * Extract the per-segment stable annotations from a `providerOptions`
+ * payload. The runtime emits these as `providerOptions.eliza.promptSegments`
+ * when a structured prompt is available — local backends use it to compute
+ * `hashStablePrefix` directly, without having to re-parse the prompt text.
+ *
+ * Returns `null` when the field is absent or malformed; callers fall back
+ * to `extractPromptCacheKey` / `extractPrefixHash`.
+ */
+export declare function extractAnnotatedSegments(providerOptions: unknown): AnnotatedPromptSegment[] | null;
+/**
+ * Resolve the conversation handle id from a `providerOptions` payload.
+ * The runtime stuffs it under `providerOptions.eliza.conversationId` when
+ * the calling context represents a long-lived conversation (chat handler,
+ * planner loop). When present, local backends should use it as the
+ * primary slot key — it's stable across turns regardless of prompt
+ * content drift, which gives the strongest possible cache reuse for
+ * agentic loops.
+ */
+export declare function extractConversationId(providerOptions: unknown): string | null;
+/**
+ * Resolve the stable per-call cache key for the local backends. Order of
+ * precedence:
+ *   1. Conversation id — strongest signal, identical across turns.
+ *   2. Annotated stable-prefix hash — survives unstable-tail drift.
+ *   3. `prefixHash` from the runtime cache plan — already stable-only via
+ *      `cachePrefixSegments` upstream.
+ *   4. `promptCacheKey` (`v5:<prefixHash>`) — back-compat fallback.
+ * Returns null when none are available.
+ */
+export declare function resolveLocalCacheKey(providerOptions: unknown): string | null;
+//# sourceMappingURL=cache-bridge.d.ts.map

package/src/services/cache-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache-bridge.d.ts","sourceRoot":"","sources":["cache-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,kBAAkB,EAAE,SAIhC,CAAC;AAEF;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,GAAG,MAAM,CAYT;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAU7E;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,EAC9C,IAAI,GAAE,SAA8B,GAClC,MAAM,CAIR;AAED,mEAAmE;AACnE,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9D;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,iBAAiB,GACpB,MAAM,CAER;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,MAAM,GACd,iBAAiB,GAAG,SAAS,CAgB/B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,YAAY,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,SAA8B,EACpC,GAAG,GAAE,MAAmB,GACtB,OAAO,CAAC,MAAM,CAAC,CA8BjB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,MAAM,EACf,GAAG,GAAE,MAAmB,GACtB,OAAO,CAAC,eAAe,EAAE,CAAC,CA2B5B;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAO7E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAOzE;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,sBAAsB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,SAAS,sBAAsB,EAAE,GACzC,MAAM,GAAG,IAAI,CAYf;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CACvC,eAAe,EAAE,OAAO,GACtB,sBAAsB,EAAE,GAAG,IAAI,CAejC;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAO7E;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAW5E"}