@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/bundled-models.ts

@@ -0,0 +1,129 @@
+/**
+ * Bundled-models bootstrap for AOSP / on-device installs.
+ *
+ * The AOSP build pipeline stages Eliza-1 models into the APK at
+ * `assets/agent/models/{file}.gguf` plus a
+ * `manifest.json` describing each one (id, role, sha256, sizeBytes).
+ * `ElizaAgentService.extractAssetsIfNeeded()` copies those files into
+ * `$ELIZA_STATE_DIR/local-inference/models/` on first launch.
+ *
+ * This module reads the manifest at runtime startup and registers each
+ * file as a eliza-owned model in the local-inference registry, so the
+ * auto-assign pass picks them up for TEXT_LARGE / TEXT_SMALL /
+ * TEXT_EMBEDDING slots without needing the user to download anything.
+ *
+ * Idempotent: re-running with the registry already populated is a
+ * no-op for unchanged entries (`upsertElizaModel` overwrites entries
+ * with the same id, so updated sha256s on a later re-bundle replace
+ * the old metadata cleanly).
+ *
+ * Source classification: the runtime treats bundled models as
+ * `source: "eliza-download"` because Eliza ships the file and Eliza
+ * owns it on disk — same lifecycle as a user-initiated download
+ * (uninstall removes the file, the registry tracks the install). The
+ * only difference is the file arrived via APK extraction rather than
+ * an HTTP transfer.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import {
+	ensureDefaultAssignment,
+	readAssignments,
+	writeAssignments,
+} from "./assignments";
+import { elizaModelsDir } from "./paths";
+import { upsertElizaModel } from "./registry";
+import type { InstalledModel } from "./types";
+
+interface BundledModelEntry {
+	id: string;
+	displayName: string;
+	hfRepo: string;
+	ggufFile: string;
+	role: "chat" | "embedding";
+	sizeBytes: number;
+	sha256: string | null;
+}
+
+interface BundledModelManifest {
+	version: 1;
+	models: BundledModelEntry[];
+}
+
+function manifestPath(): string {
+	return path.join(elizaModelsDir(), "manifest.json");
+}
+
+async function readManifest(): Promise<BundledModelManifest | null> {
+	try {
+		const raw = await fs.readFile(manifestPath(), "utf8");
+		const parsed = JSON.parse(raw) as BundledModelManifest;
+		if (parsed.version !== 1 || !Array.isArray(parsed.models)) {
+			return null;
+		}
+		return parsed;
+	} catch {
+		return null;
+	}
+}
+
+async function ensureBundledAssignment(
+	modelId: string,
+	role: BundledModelEntry["role"],
+): Promise<void> {
+	if (role !== "embedding") {
+		await ensureDefaultAssignment(modelId);
+		return;
+	}
+
+	const current = await readAssignments();
+	if (current.TEXT_EMBEDDING) return;
+	await writeAssignments({ ...current, TEXT_EMBEDDING: modelId });
+}
+
+/**
+ * Walk the manifest and register every bundled GGUF file in the
+ * local-inference registry. Returns the number of entries successfully
+ * registered. A missing manifest is normal on Capacitor / desktop /
+ * non-AOSP installs and returns 0 silently.
+ */
+export async function registerBundledModels(): Promise<number> {
+	const manifest = await readManifest();
+	if (!manifest) return 0;
+	const dir = elizaModelsDir();
+	let registered = 0;
+	for (const entry of manifest.models) {
+		const filePath = path.join(dir, entry.ggufFile);
+		let sizeBytes = entry.sizeBytes;
+		try {
+			const stat = await fs.stat(filePath);
+			sizeBytes = stat.size;
+		} catch {
+			// File didn't extract — manifest references something the APK
+			// didn't ship. Skip this entry rather than registering a broken
+			// path. AOSP build's stage-default-models.mjs is the source of
+			// truth; if a file is missing the build is broken upstream.
+			continue;
+		}
+		const installed: InstalledModel = {
+			id: entry.id,
+			displayName: entry.displayName,
+			path: filePath,
+			sizeBytes,
+			hfRepo: entry.hfRepo,
+			installedAt: new Date().toISOString(),
+			lastUsedAt: null,
+			source: "eliza-download",
+			sha256: entry.sha256 ?? undefined,
+		};
+		await upsertElizaModel(installed);
+		// Auto-assign each bundled model to its manifest role if the user
+		// hasn't already assigned that slot. This keeps Eliza-1 chat models
+		// eligible for TEXT_EMBEDDING when the bundle explicitly marks them
+		// as the local embedding bootstrap model.
+		await ensureBundledAssignment(entry.id, entry.role);
+		registered += 1;
+	}
+	return registered;
+}

package/src/services/cache-bridge.d.ts

@@ -0,0 +1,206 @@
+/**
+ * Cache bridge for the local-inference path.
+ *
+ * Translates the runtime's `ProviderCachePlan` (a provider-neutral cache
+ * plan emitted by `@elizaos/core`'s `buildProviderCachePlan`) into
+ * concrete behaviour for the two local backends:
+ *
+ *   1. Out-of-process llama-server (MTP / buun-llama-cpp): stable
+ *      slot-id derivation + on-disk slot KV save/restore directory layout
+ *      + TTL-based eviction by mtime.
+ *   2. In-process node-llama-cpp: a session pool (see
+ *      `session-pool.ts`) keyed by `promptCacheKey`.
+ *
+ * This module is pure logic — no llama-server process management, no
+ * node-llama-cpp imports. All filesystem state is rooted under
+ * `local-inference/llama-cache/` so cleanup is easy and explicit.
+ */
+/**
+ * TTLs for cached prefix data, mirroring the cloud-side semantics:
+ *   - `short`: roughly the "default" Anthropic ephemeral cache window.
+ *   - `long`: roughly the "1h" Anthropic ephemeral cache window.
+ *   - `extended`: the OpenAI 24h prompt-cache retention window.
+ *
+ * Values are in milliseconds. Eviction uses file mtime, not access time,
+ * so a slot that is read repeatedly without being rewritten still ages
+ * out — which matches how llama-server writes the slot file each turn.
+ */
+export interface CacheTtls {
+    short: number;
+    long: number;
+    extended?: number;
+}
+export declare const DEFAULT_CACHE_TTLS: CacheTtls;
+/**
+ * Root directory for all local llama-cache state. Anything inside is
+ * Eliza-owned and safe to delete to reset the cache.
+ */
+export declare function llamaCacheRoot(): string;
+/**
+ * Per-model-hash cache directory. Slot save files for one model never
+ * collide with another model's; switching active model does not need to
+ * wipe the cache.
+ */
+export declare function cacheRoot(modelHash: string): string;
+/**
+ * llama-server `--slot-save-path` argument: the directory llama-server
+ * writes per-slot KV state into when a request includes
+ * `cache_prompt: true`. One directory per model hash.
+ */
+export declare function slotSavePath(modelHash: string): string;
+/**
+ * Stable model-fingerprint hash. Combines the absolute paths of the
+ * target / drafter GGUFs and the cache-type knobs so two distinct
+ * configurations don't share a slot directory.
+ */
+export declare function buildModelHash(input: {
+    targetModelPath: string;
+    drafterModelPath?: string | null;
+    cacheTypeK?: string | null;
+    cacheTypeV?: string | null;
+    /** Optional extra discriminator (context size, parallel, etc.). */
+    extra?: string | null;
+}): string;
+/**
+ * Map a `promptCacheKey` to a llama-server slot id in [0, parallel).
+ *
+ * llama-server's `--parallel N` flag pre-allocates N decoding slots and
+ * accepts a `slot_id` integer in `[0, N-1]` on each request. By hashing
+ * the cache key into that range we get:
+ *
+ *   - The same prefix hash always lands on the same slot, so the in-RAM
+ *     KV cache from the previous turn is reused.
+ *   - Different prefix hashes spread across slots and don't fight for
+ *     the same KV memory.
+ *
+ * Pass `parallel <= 0` to disable slot pinning (returns -1, the
+ * llama-server "any free slot" sentinel).
+ */
+export declare function deriveSlotId(promptCacheKey: string, parallel: number): number;
+/**
+ * Convert the runtime-side `CacheTTL` enum + OpenAI extended retention
+ * hint into a concrete TTL in milliseconds. This is what the eviction
+ * sweep uses when deciding whether a slot file is still live.
+ */
+export declare function ttlMsForKey(ttl: "short" | "long" | "extended" | undefined, ttls?: CacheTtls): number;
+/** TTL classes that can be encoded into a slot `.bin` filename. */
+export type SlotCacheTtlClass = "short" | "long" | "extended";
+/**
+ * Build the basename for a persisted slot/conversation `.bin` file with
+ * its TTL class encoded as a middle component: `<base>.<ttl>.bin`. The
+ * eviction sweep reads that component back via `parseSlotCacheTtlClass`
+ * so a slot persisted with the long retention window isn't deleted on
+ * the short horizon (and vice versa). Pass `"long"` for cross-restart
+ * conversation KV — that matches the prior global (long-only) behaviour
+ * for those files.
+ */
+export declare function slotCacheFileName(base: string, ttl: SlotCacheTtlClass): string;
+/**
+ * Parse the TTL class encoded into a slot `.bin` filename by
+ * `slotCacheFileName`. Returns `undefined` for legacy / hand-written
+ * filenames without an encoded class — those keep the `long` horizon
+ * (the prior global behaviour for persisted slot files).
+ */
+export declare function parseSlotCacheTtlClass(fileName: string): SlotCacheTtlClass | undefined;
+/**
+ * Sweep the slot-save directory and delete files older than their
+ * per-file TTL horizon. The TTL class is read from the filename
+ * (`<base>.<ttl>.bin` — see `slotCacheFileName`); files without an
+ * encoded class use the `long` horizon (the prior global behaviour).
+ * Mtime is the watermark; llama-server rewrites the slot file on every
+ * save, so a slot that's actively used keeps a fresh mtime.
+ *
+ * Returns the number of files deleted. Missing directories are not
+ * errors — eviction on a clean install just no-ops.
+ */
+export declare function evictExpired(rootDir: string, ttls?: CacheTtls, now?: number): Promise<number>;
+export interface CacheStatsEntry {
+    file: string;
+    sizeBytes: number;
+    mtimeMs: number;
+    ageMs: number;
+}
+/**
+ * Snapshot of the on-disk slot-save directory. Used by the public
+ * `getLocalCacheStats()` debugging endpoint.
+ */
+export declare function readCacheStats(rootDir: string, now?: number): Promise<CacheStatsEntry[]>;
+/**
+ * Resolve `promptCacheKey` from a `providerOptions` payload as emitted
+ * by `buildProviderCachePlan`. The runtime stuffs it under
+ * `providerOptions.eliza.promptCacheKey`. Returns `null` when the key is
+ * missing or not a non-empty string — callers fall back to the default
+ * "_default" session in that case.
+ */
+export declare function extractPromptCacheKey(providerOptions: unknown): string | null;
+/**
+ * Resolve `prefixHash` from `providerOptions.eliza.prefixHash`. Mirrors
+ * `extractPromptCacheKey` — returns null when missing or not a non-empty
+ * string. The prefix hash covers ONLY the stable prompt prefix (system
+ * prompt + tool definitions + large constant context), so a runtime
+ * timestamp in the unstable tail does not invalidate it.
+ *
+ * Local backends prefer this over `promptCacheKey` when available because
+ * it gives the strongest "same prefix → same slot" guarantee: two
+ * conversations with byte-identical stable prefixes will land on the same
+ * slot regardless of how their tail content differs.
+ */
+export declare function extractPrefixHash(providerOptions: unknown): string | null;
+/**
+ * Stable annotation describing a single segment of the prompt as it was
+ * emitted by the runtime planner. The cache-bridge consumes this to
+ * compute a stable-prefix-only hash for slot pinning, without having to
+ * look at the (timestamp-laden) tail.
+ *
+ * Mirrors `PromptSegment` in @elizaos/core/src/types/model.ts but is kept
+ * standalone so the cache-bridge can be imported by the local-inference
+ * backends without a hard dep on `@elizaos/core`.
+ */
+export interface AnnotatedPromptSegment {
+    content: string;
+    stable: boolean;
+}
+/**
+ * Hash the longest stable prefix of `segments`. Stops at the first
+ * unstable segment, so a runtime timestamp in the unstable tail never
+ * shifts the hash. Returns `null` when no stable segment exists, signaling
+ * to the caller that prefix-cache reuse cannot be derived purely from the
+ * prompt structure (fall back to the prompt-cache-key path instead).
+ *
+ * The hash is sha256-truncated to 16 hex chars, matching `buildModelHash`
+ * — short enough for log lines, wide enough that collision is not a
+ * realistic concern for any plausible number of concurrent prefixes.
+ */
+export declare function hashStablePrefix(segments: readonly AnnotatedPromptSegment[]): string | null;
+/**
+ * Extract the per-segment stable annotations from a `providerOptions`
+ * payload. The runtime emits these as `providerOptions.eliza.promptSegments`
+ * when a structured prompt is available — local backends use it to compute
+ * `hashStablePrefix` directly, without having to re-parse the prompt text.
+ *
+ * Returns `null` when the field is absent or malformed; callers fall back
+ * to `extractPromptCacheKey` / `extractPrefixHash`.
+ */
+export declare function extractAnnotatedSegments(providerOptions: unknown): AnnotatedPromptSegment[] | null;
+/**
+ * Resolve the conversation handle id from a `providerOptions` payload.
+ * The runtime stuffs it under `providerOptions.eliza.conversationId` when
+ * the calling context represents a long-lived conversation (chat handler,
+ * planner loop). When present, local backends should use it as the
+ * primary slot key — it's stable across turns regardless of prompt
+ * content drift, which gives the strongest possible cache reuse for
+ * agentic loops.
+ */
+export declare function extractConversationId(providerOptions: unknown): string | null;
+/**
+ * Resolve the stable per-call cache key for the local backends. Order of
+ * precedence:
+ *   1. Conversation id — strongest signal, identical across turns.
+ *   2. Annotated stable-prefix hash — survives unstable-tail drift.
+ *   3. `prefixHash` from the runtime cache plan — already stable-only via
+ *      `cachePrefixSegments` upstream.
+ *   4. `promptCacheKey` (`v5:<prefixHash>`) — back-compat fallback.
+ * Returns null when none are available.
+ */
+export declare function resolveLocalCacheKey(providerOptions: unknown): string | null;
+//# sourceMappingURL=cache-bridge.d.ts.map

package/src/services/cache-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache-bridge.d.ts","sourceRoot":"","sources":["cache-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,kBAAkB,EAAE,SAIhC,CAAC;AAEF;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,GAAG,MAAM,CAYT;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAU7E;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,EAC9C,IAAI,GAAE,SAA8B,GAClC,MAAM,CAIR;AAED,mEAAmE;AACnE,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9D;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,iBAAiB,GACpB,MAAM,CAER;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,MAAM,GACd,iBAAiB,GAAG,SAAS,CAgB/B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,YAAY,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,SAA8B,EACpC,GAAG,GAAE,MAAmB,GACtB,OAAO,CAAC,MAAM,CAAC,CA8BjB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,MAAM,EACf,GAAG,GAAE,MAAmB,GACtB,OAAO,CAAC,eAAe,EAAE,CAAC,CA2B5B;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAO7E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAOzE;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,sBAAsB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,SAAS,sBAAsB,EAAE,GACzC,MAAM,GAAG,IAAI,CAYf;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CACvC,eAAe,EAAE,OAAO,GACtB,sBAAsB,EAAE,GAAG,IAAI,CAejC;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAO7E;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAW5E"}