@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/ram-budget.ts

@@ -0,0 +1,309 @@
+/**
+ * Resolve a model's RAM budget and decide whether it fits the host.
+ *
+ * For installed Eliza-1 tiers WITH a published `eliza-1.manifest.json`
+ * on disk we prefer the manifest's `ramBudgetMb.{min, recommended}`
+ * block — that's what packages/inference/AGENTS.md §3 + §6 designate
+ * as the source of truth for per-bundle memory expectations. For every
+ * other model (non-Eliza-1, uninstalled tiers, or Eliza-1 bundles that
+ * predate the manifest publish) we fall back to the catalog scalar
+ * `model.minRamGb` for the floor and synthesize a `recommendedMb` from
+ * the floor plus the bundle's per-token KV-cache footprint at its default
+ * context window — the same `KvGeometry` figure `kv-spill.ts` uses. That
+ * fixes the degenerate `minMb == recommendedMb` catalog case: a long
+ * session grows into the KV cache, so "boots" and "runs comfortably" are
+ * genuinely different lines.
+ *
+ * The manifest read is best-effort: a missing or malformed manifest
+ * never throws — recommendation runs at runtime and a broken manifest
+ * must not crash the dashboard. Build-time gates live in the publish
+ * script (packages/training/scripts/manifest/eliza1_manifest.py) and
+ * the validator (`./manifest/validator.ts`).
+ *
+ * This module is the single source of truth for the "does model X fit a
+ * host with N MB usable RAM" decision (`assessRamFit`). Both the Model
+ * Hub recommender (`recommendation.ts`) and the model-load admission gate
+ * (`active-model.ts`) call it — neither re-derives the math.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { ELIZA_1_TIER_IDS, type Eliza1TierId, MODEL_CATALOG } from "./catalog";
+import { estimateQuantizedKvBytesPerToken } from "./kv-spill";
+import { type Eliza1Manifest, validateManifest } from "./manifest";
+import type { CatalogModel, InstalledModel, RamBudget } from "./types";
+
+const MB_PER_GB = 1024;
+const BYTES_PER_MB = 1024 * 1024;
+
+/**
+ * RAM kept free for the OS, the dashboard, connectors, and the runtime
+ * itself when deciding whether a model load fits. The model's resident
+ * weights + KV cache must fit inside `(hostRamMb - RAM_HEADROOM_RESERVE_MB)`.
+ * Override via `ELIZA_LOCAL_RAM_HEADROOM_MB`.
+ */
+const DEFAULT_RAM_HEADROOM_RESERVE_MB = 1536;
+
+/**
+ * Default context window assumed for a catalog entry that doesn't declare
+ * `contextLength`. Used only to size the synthesized `recommendedMb` KV
+ * component; conservative on purpose.
+ */
+const FALLBACK_DEFAULT_CONTEXT_TOKENS = 32768;
+
+export function ramHeadroomReserveMb(): number {
+	const raw = process.env.ELIZA_LOCAL_RAM_HEADROOM_MB?.trim();
+	if (raw) {
+		const parsed = Number.parseInt(raw, 10);
+		if (Number.isFinite(parsed) && parsed >= 0) return parsed;
+	}
+	return DEFAULT_RAM_HEADROOM_RESERVE_MB;
+}
+
+export type { RamBudget } from "./types.js";
+
+/**
+ * Loader contract — keeps the helper testable without touching disk.
+ * Production callers pass `defaultManifestLoader`; tests inject a fake loader.
+ */
+export type ManifestLoader = (
+	modelId: string,
+	installed: InstalledModel | undefined,
+) => Eliza1Manifest | null;
+
+const ELIZA_1_TIER_ID_SET: ReadonlySet<string> = new Set(ELIZA_1_TIER_IDS);
+
+function isEliza1TierId(id: string): id is Eliza1TierId {
+	return ELIZA_1_TIER_ID_SET.has(id);
+}
+
+/**
+ * Memoizes the manifest read+parse+validate so repeated fit checks (the
+ * recommender scores the whole catalog on each refresh; the load gate re-checks
+ * per load) don't re-hit disk. Keyed on `modelId + manifestSha256`: the SHA is
+ * the validated manifest's content hash, so a re-downloaded bundle with a
+ * different manifest gets a fresh key automatically — no stale RAM budgets.
+ * Only positive results are cached, and only when a SHA is present; the no-SHA
+ * path (legacy installs) reads through unchanged.
+ */
+const MANIFEST_CACHE = new Map<string, Eliza1Manifest>();
+
+/** Test-only: drop memoized manifests so fixtures with reused SHAs don't leak. */
+export function __resetManifestCacheForTests(): void {
+	MANIFEST_CACHE.clear();
+}
+
+function manifestTierFromId(id: Eliza1TierId): string {
+	// Catalog id `eliza-1-<tier>` → manifest tier `<tier>`.
+	return id.slice("eliza-1-".length);
+}
+
+/**
+ * Production manifest loader — reads `eliza-1.manifest.json` from the
+ * installed bundle's directory. Two candidate paths are probed:
+ *
+ *   1. `dirname(dirname(model.path))` — the canonical bundle root when
+ *      the GGUF lives in a `text/` subdir per AGENTS.md §2.
+ *   2. `dirname(model.path)` — flat layout used by some test fixtures
+ *      and pre-bundle installs.
+ *
+ * Returns `null` for any failure: missing file, JSON parse error,
+ * manifest validation error, or tier mismatch.
+ */
+export function defaultManifestLoader(
+	modelId: string,
+	installed: InstalledModel | undefined,
+): Eliza1Manifest | null {
+	if (!installed?.path) return null;
+	if (!isEliza1TierId(modelId)) return null;
+
+	const cacheKey = installed.manifestSha256
+		? `${modelId}\x00${installed.manifestSha256}`
+		: null;
+	if (cacheKey) {
+		const cached = MANIFEST_CACHE.get(cacheKey);
+		if (cached) return cached;
+	}
+
+	const expectedTier = manifestTierFromId(modelId);
+	const candidates = [
+		path.join(
+			path.dirname(path.dirname(installed.path)),
+			"eliza-1.manifest.json",
+		),
+		path.join(path.dirname(installed.path), "eliza-1.manifest.json"),
+	];
+
+	for (const candidate of candidates) {
+		let raw: string;
+		try {
+			raw = fs.readFileSync(candidate, "utf8");
+		} catch {
+			continue;
+		}
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(raw);
+		} catch {
+			continue;
+		}
+		const result = validateManifest(parsed);
+		if (!result.ok) continue;
+		if (result.manifest.tier !== expectedTier) continue;
+		if (cacheKey) MANIFEST_CACHE.set(cacheKey, result.manifest);
+		return result.manifest;
+	}
+	return null;
+}
+
+/**
+ * Synthesized `recommendedMb` for the catalog fallback: the floor plus the
+ * bundle's compressed KV-cache footprint at its default context window.
+ * That's the figure a nominal long-running session actually grows into —
+ * a far better "fits vs tight" cutoff than reusing the floor verbatim.
+ */
+function catalogRecommendedMb(model: CatalogModel, minMb: number): number {
+	const ctxTokens =
+		typeof model.contextLength === "number" && model.contextLength > 0
+			? model.contextLength
+			: FALLBACK_DEFAULT_CONTEXT_TOKENS;
+	const kvBytes = estimateQuantizedKvBytesPerToken(model.params) * ctxTokens;
+	const kvMb = Math.ceil(kvBytes / BYTES_PER_MB);
+	return minMb + kvMb;
+}
+
+/**
+ * Resolve a `RamBudget` for `model`, optionally consulting the on-disk
+ * manifest of an installed Eliza-1 bundle.
+ *
+ * `installed` and `manifestLoader` are both optional — passing neither
+ * always returns the catalog-scalar fallback. The recommendation engine
+ * passes both at call sites where it has the installed-models list.
+ */
+export function resolveRamBudget(
+	model: CatalogModel,
+	installed?: InstalledModel,
+	manifestLoader: ManifestLoader = defaultManifestLoader,
+): RamBudget {
+	if (isEliza1TierId(model.id) && installed) {
+		const manifest = manifestLoader(model.id, installed);
+		if (manifest) {
+			return {
+				minMb: manifest.ramBudgetMb.min,
+				recommendedMb: manifest.ramBudgetMb.recommended,
+				source: "manifest",
+			};
+		}
+	}
+	const minMb = Math.round(model.minRamGb * MB_PER_GB);
+	return {
+		minMb,
+		recommendedMb: catalogRecommendedMb(model, minMb),
+		source: "catalog",
+	};
+}
+
+export interface RamFitOptions {
+	installed?: InstalledModel;
+	manifestLoader?: ManifestLoader;
+	/**
+	 * Override the headroom reserved for the OS/runtime. Defaults to
+	 * `ramHeadroomReserveMb()`. Pass 0 to assess against raw memory (the
+	 * recommender does this — it works in "effective memory available to
+	 * the model" terms, where the OS reserve is already discounted).
+	 */
+	reserveMb?: number;
+}
+
+export type RamFitLevel = "fits" | "tight" | "wontfit";
+
+export interface RamFitDecision {
+	level: RamFitLevel;
+	/** True for `fits` and `tight`; false only for `wontfit`. */
+	fits: boolean;
+	budget: RamBudget;
+	/** Memory after subtracting the headroom reserve, in MB. */
+	usableMb: number;
+	/** Headroom reserve applied, in MB. */
+	reserveMb: number;
+}
+
+/**
+ * The one fit decision. `hostRamMb` is the memory being assessed against,
+ * in megabytes (`os.totalmem() / 2**20`, or the probe's `totalRamGb * 1024`,
+ * or — for the recommender on a GPU host — the effective model-available
+ * memory in MB).
+ *
+ *   - `wontfit` : usable RAM (host minus headroom) is below the bundle's
+ *                 boot floor (`budget.minMb`). A load MUST be refused.
+ *   - `tight`   : boots, but usable RAM is below the recommended budget —
+ *                 a long session will swap or stutter under load.
+ *   - `fits`    : comfortable headroom.
+ */
+export function assessRamFit(
+	model: CatalogModel,
+	hostRamMb: number,
+	options: RamFitOptions = {},
+): RamFitDecision {
+	const budget = resolveRamBudget(
+		model,
+		options.installed,
+		options.manifestLoader ?? defaultManifestLoader,
+	);
+	const reserveMb = options.reserveMb ?? ramHeadroomReserveMb();
+	const usableMb = Math.max(0, hostRamMb - reserveMb);
+	let level: RamFitLevel;
+	if (usableMb < budget.minMb) level = "wontfit";
+	else if (usableMb < budget.recommendedMb) level = "tight";
+	else level = "fits";
+	return {
+		level,
+		fits: level !== "wontfit",
+		budget,
+		usableMb,
+		reserveMb,
+	};
+}
+
+/** Display name with a trailing `-<ctx>` window suffix stripped. */
+function contextVariantStem(model: CatalogModel): string {
+	return model.displayName.replace(/-(?:\d+k|\d+m)$/i, "");
+}
+
+/**
+ * Given a catalog entry, pick the variant of the same model "line" (same
+ * param count and display-name stem — e.g. `eliza-1-27b` / `eliza-1-27b-256k`)
+ * with the largest context window that still fits a
+ * host with `hostRamMb` of RAM. Returns `model` itself when it's already
+ * the best fit (or the only variant), or `null` when not even `model` fits
+ * — callers turn that into a refusal.
+ *
+ * Variants are matched by `(params, displayNameStem)` where the stem is the
+ * display name with any trailing `-<ctx>` suffix (`-256k`, `-1m`, `-128k`)
+ * stripped. This keeps `eliza-1-27b*` together without conflating `9b` and
+ * `27b`.
+ */
+export function pickFittingContextVariant(
+	model: CatalogModel,
+	hostRamMb: number,
+	options: RamFitOptions = {},
+	catalog: ReadonlyArray<CatalogModel> = MODEL_CATALOG,
+): CatalogModel | null {
+	const stem = contextVariantStem(model);
+	const variants = catalog.filter(
+		(candidate) =>
+			candidate.params === model.params &&
+			contextVariantStem(candidate) === stem,
+	);
+	// Largest context first; ties (shouldn't happen) keep catalog order.
+	const ranked = [...(variants.length > 0 ? variants : [model])].sort(
+		(left, right) => (right.contextLength ?? 0) - (left.contextLength ?? 0),
+	);
+	for (const candidate of ranked) {
+		const installed = candidate.id === model.id ? options.installed : undefined;
+		if (assessRamFit(candidate, hostRamMb, { ...options, installed }).fits) {
+			return candidate;
+		}
+	}
+	return null;
+}

package/src/services/readiness.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, it } from "vitest";
+import { buildTextGenerationReadiness } from "./readiness";
+import type { ActiveModelState, DownloadJob, InstalledModel } from "./types";
+
+const activeIdle: ActiveModelState = {
+	modelId: null,
+	loadedAt: null,
+	status: "idle",
+};
+
+describe("local inference text readiness", () => {
+	it("reports assigned download terminal error state", () => {
+		const installed: InstalledModel[] = [];
+		const failedDownload: DownloadJob = {
+			jobId: "job-1",
+			modelId: "eliza-1-2b",
+			state: "failed",
+			received: 128,
+			total: 512,
+			bytesPerSec: 0,
+			etaMs: null,
+			startedAt: new Date().toISOString(),
+			updatedAt: new Date().toISOString(),
+			error: "HTTP 503 from HuggingFace",
+		};
+
+		const readiness = buildTextGenerationReadiness({
+			assignments: {
+				TEXT_LARGE: "eliza-1-2b",
+			},
+			installed,
+			active: activeIdle,
+			downloads: [failedDownload],
+		});
+
+		expect(readiness.slots.TEXT_LARGE.assigned).toBe(true);
+		expect(readiness.slots.TEXT_LARGE.primaryDownloaded).toBe(false);
+		expect(readiness.slots.TEXT_LARGE.downloaded).toBe(false);
+		expect(readiness.slots.TEXT_LARGE.state).toBe("failed");
+		expect(readiness.slots.TEXT_LARGE.missingModelIds).toContain("eliza-1-2b");
+		expect(readiness.slots.TEXT_LARGE.download.percent).toBe(25);
+		expect(readiness.slots.TEXT_LARGE.errors).toContain(
+			"HTTP 503 from HuggingFace",
+		);
+	});
+
+	it("marks a downloaded active assignment ready", () => {
+		const installed: InstalledModel[] = [
+			{
+				id: "eliza-1-2b",
+				displayName: "eliza-1-2b",
+				path: "/tmp/eliza-1-2b.gguf",
+				sizeBytes: 2048,
+				installedAt: new Date().toISOString(),
+				lastUsedAt: null,
+				source: "eliza-download",
+			},
+			{
+				id: "eliza-1-2b-drafter",
+				displayName: "eliza-1-2b drafter",
+				path: "/tmp/eliza-1-2b-drafter.gguf",
+				sizeBytes: 512,
+				installedAt: new Date().toISOString(),
+				lastUsedAt: null,
+				source: "eliza-download",
+			},
+		];
+
+		const readiness = buildTextGenerationReadiness({
+			assignments: {
+				TEXT_SMALL: "eliza-1-2b",
+			},
+			installed,
+			active: {
+				modelId: "eliza-1-2b",
+				loadedAt: new Date().toISOString(),
+				status: "ready",
+			},
+			downloads: [],
+		});
+
+		expect(readiness.slots.TEXT_SMALL.downloaded).toBe(true);
+		expect(readiness.slots.TEXT_SMALL.active).toBe(true);
+		expect(readiness.slots.TEXT_SMALL.ready).toBe(true);
+		expect(readiness.slots.TEXT_SMALL.state).toBe("active");
+	});
+});

package/src/services/readiness.ts

@@ -0,0 +1,238 @@
+import { MODEL_CATALOG } from "./catalog";
+import { catalogDownloadSizeBytes } from "./recommendation";
+import type {
+	ActiveModelState,
+	CatalogModel,
+	DownloadJob,
+	DownloadState,
+	InstalledModel,
+	LocalInferenceDownloadStatus,
+	LocalInferenceReadiness,
+	LocalInferenceSlotReadiness,
+	ModelAssignments,
+	TextGenerationSlot,
+} from "./types";
+
+const TERMINAL_STATES = new Set<DownloadState>([
+	"completed",
+	"failed",
+	"cancelled",
+]);
+
+function catalogById(catalog: CatalogModel[]): Map<string, CatalogModel> {
+	return new Map(catalog.map((model) => [model.id, model]));
+}
+
+function installedById(
+	installed: InstalledModel[],
+): Map<string, InstalledModel> {
+	return new Map(installed.map((model) => [model.id, model]));
+}
+
+function latestJobByModel(downloads: DownloadJob[]): Map<string, DownloadJob> {
+	const jobs = new Map<string, DownloadJob>();
+	for (const job of downloads) {
+		const current = jobs.get(job.modelId);
+		if (!current || job.updatedAt.localeCompare(current.updatedAt) > 0) {
+			jobs.set(job.modelId, job);
+		}
+	}
+	return jobs;
+}
+
+function requiredModelIds(
+	assignedModelId: string,
+	catalog: Map<string, CatalogModel>,
+): string[] {
+	const model = catalog.get(assignedModelId);
+	return model ? [assignedModelId] : [assignedModelId];
+}
+
+function statusFromJobs(
+	jobs: DownloadJob[],
+	missingModelIds: string[],
+): LocalInferenceDownloadStatus {
+	const activeJobs = jobs.filter(
+		(job) => job.state === "queued" || job.state === "downloading",
+	);
+	const terminalJobs = jobs.filter((job) => TERMINAL_STATES.has(job.state));
+	const relevantJobs = activeJobs.length > 0 ? activeJobs : terminalJobs;
+	const state =
+		relevantJobs.find((job) => job.state === "failed")?.state ??
+		relevantJobs.find((job) => job.state === "cancelled")?.state ??
+		relevantJobs.find((job) => job.state === "downloading")?.state ??
+		relevantJobs.find((job) => job.state === "queued")?.state ??
+		relevantJobs.find((job) => job.state === "completed")?.state ??
+		(missingModelIds.length > 0 ? "missing" : "completed");
+	const receivedBytes = relevantJobs.reduce(
+		(total, job) => total + job.received,
+		0,
+	);
+	const totalBytes = relevantJobs.reduce((total, job) => total + job.total, 0);
+	const bytesPerSec = activeJobs.reduce(
+		(total, job) => total + job.bytesPerSec,
+		0,
+	);
+	const etaValues = activeJobs
+		.map((job) => job.etaMs)
+		.filter((etaMs): etaMs is number => etaMs !== null);
+	const etaMs =
+		etaValues.length > 0
+			? Math.max(...etaValues)
+			: (relevantJobs[0]?.etaMs ?? null);
+	const updatedAt =
+		relevantJobs
+			.map((job) => job.updatedAt)
+			.sort((left, right) => right.localeCompare(left))[0] ?? null;
+	const errors = relevantJobs.flatMap((job) => (job.error ? [job.error] : []));
+	return {
+		state,
+		receivedBytes,
+		totalBytes,
+		percent:
+			totalBytes > 0
+				? Math.max(
+						0,
+						Math.min(100, Math.round((receivedBytes / totalBytes) * 1000) / 10),
+					)
+				: null,
+		bytesPerSec,
+		etaMs,
+		updatedAt,
+		errors,
+	};
+}
+
+function slotReadiness(
+	slot: TextGenerationSlot,
+	assignments: ModelAssignments,
+	installed: Map<string, InstalledModel>,
+	catalog: Map<string, CatalogModel>,
+	catalogList: CatalogModel[],
+	downloads: Map<string, DownloadJob>,
+	active: ActiveModelState,
+): LocalInferenceSlotReadiness {
+	const assignedModelId = assignments[slot] ?? null;
+	if (!assignedModelId) {
+		return {
+			slot,
+			assigned: false,
+			assignedModelId: null,
+			displayName: null,
+			primaryDownloaded: false,
+			downloaded: false,
+			active: false,
+			ready: false,
+			state: "unassigned",
+			requiredModelIds: [],
+			missingModelIds: [],
+			installedBytes: 0,
+			expectedBytes: 0,
+			download: {
+				state: "missing",
+				receivedBytes: 0,
+				totalBytes: 0,
+				percent: null,
+				bytesPerSec: 0,
+				etaMs: null,
+				updatedAt: null,
+				errors: [],
+			},
+			errors: [],
+		};
+	}
+
+	const ids = requiredModelIds(assignedModelId, catalog);
+	const missingModelIds = ids.filter((id) => !installed.has(id));
+	const primaryDownloaded = installed.has(assignedModelId);
+	const downloaded = missingModelIds.length === 0;
+	const activeReady =
+		active.modelId === assignedModelId && active.status === "ready";
+	const model = catalog.get(assignedModelId);
+	const jobs = ids.flatMap((id) => {
+		const job = downloads.get(id);
+		return job ? [job] : [];
+	});
+	const download = statusFromJobs(jobs, missingModelIds);
+	const activeError =
+		active.modelId === assignedModelId &&
+		active.status === "error" &&
+		active.error
+			? [active.error]
+			: [];
+	const errors = [...download.errors, ...activeError];
+	const hasActiveJob = jobs.some(
+		(job) => job.state === "queued" || job.state === "downloading",
+	);
+	const terminalFailure = jobs.find(
+		(job) => job.state === "failed" || job.state === "cancelled",
+	);
+
+	return {
+		slot,
+		assigned: true,
+		assignedModelId,
+		displayName:
+			model?.displayName ?? installed.get(assignedModelId)?.displayName ?? null,
+		primaryDownloaded,
+		downloaded,
+		active: activeReady,
+		ready: downloaded && activeReady,
+		state: terminalFailure
+			? terminalFailure.state === "failed"
+				? "failed"
+				: "cancelled"
+			: hasActiveJob
+				? "downloading"
+				: activeReady
+					? "active"
+					: downloaded
+						? "downloaded"
+						: "missing",
+		requiredModelIds: ids,
+		missingModelIds,
+		installedBytes: ids.reduce(
+			(total, id) => total + (installed.get(id)?.sizeBytes ?? 0),
+			0,
+		),
+		expectedBytes: model ? catalogDownloadSizeBytes(model, catalogList) : 0,
+		download,
+		errors,
+	};
+}
+
+export function buildTextGenerationReadiness(input: {
+	assignments: ModelAssignments;
+	installed: InstalledModel[];
+	active: ActiveModelState;
+	downloads: DownloadJob[];
+	catalog?: CatalogModel[];
+}): LocalInferenceReadiness {
+	const catalogList = input.catalog ?? MODEL_CATALOG;
+	const catalog = catalogById(catalogList);
+	const installed = installedById(input.installed);
+	const downloads = latestJobByModel(input.downloads);
+	return {
+		updatedAt: new Date().toISOString(),
+		slots: {
+			TEXT_SMALL: slotReadiness(
+				"TEXT_SMALL",
+				input.assignments,
+				installed,
+				catalog,
+				catalogList,
+				downloads,
+				input.active,
+			),
+			TEXT_LARGE: slotReadiness(
+				"TEXT_LARGE",
+				input.assignments,
+				installed,
+				catalog,
+				catalogList,
+				downloads,
+				input.active,
+			),
+		},
+	};
+}