@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/ensure-local-artifacts.ts

@@ -0,0 +1,351 @@
+/**
+ * Orchestrator for the on-device artifact bundle the local-only / local
+ * runtime needs to operate: embedding, TTS, STT, and (when the user is not
+ * signed into Eliza Cloud) the text model. Called from:
+ *
+ *   - `runtime/ensure-local-inference-handler.ts` — fire-and-forget at agent
+ *     boot whenever the resolved runtime mode is `local` or `local-only`.
+ *   - `ui/src/first-run/auto-download-recommended.ts` — when the user picks
+ *     "Local" in the first-run runtime setup, after the local agent's
+ *     `/api/health` reports ready.
+ *   - `POST /api/local-inference/ensure` — on-demand re-trigger from the
+ *     settings UI / a CLI.
+ *
+ * Today every Eliza-1 tier ships every component (text + voice + asr +
+ * embedding + embedded-draft-head MTP metadata + cache) as one HuggingFace bundle. The
+ * orchestrator picks a tier per artifact kind (always the same tier today —
+ * tier resolution is centralised on the recommender) and triggers parallel
+ * `service.startDownload(modelId)` calls per kind. The downloader is
+ * idempotent on the same `modelId`, so when the same tier covers multiple
+ * kinds we end up with one bundle download and a per-kind audit trail.
+ *
+ * When the architecture splits the bundle into separately addressable
+ * sub-archives, the per-kind selection in {@link modelIdForKind} is where the
+ * per-modality model id diverges — the rest of the orchestrator already wraps
+ * each artifact in its own `Promise.allSettled` slot.
+ *
+ * `signedInCloud === true` skips the `text` artifact: cloud-routed inference
+ * handles TEXT_LARGE / TEXT_SMALL, and we don't want a multi-gig text weight
+ * download imposed on a user who chose cloud routing. Embedding / TTS / STT
+ * always download — the local runtime serves those locally regardless of
+ * whether the text model is cloud-routed.
+ */
+
+import type { Eliza1Tier } from "./manifest";
+import {
+	classifyRecommendationPlatform,
+	selectRecommendedModelForSlot,
+} from "./recommendation";
+import type { LocalInferenceService } from "./service";
+import type { CatalogModel, HardwareProbe, InstalledModel } from "./types";
+
+type RuntimeMode = "local" | "local-only" | "cloud" | "remote";
+
+/**
+ * Resolve the default service facade lazily. Static-importing
+ * `localInferenceService` from `./service` would instantiate the
+ * `LocalInferenceService` (and its inner `ActiveModelCoordinator`) at
+ * module-load time, which is heavier than this orchestrator needs and
+ * breaks any test that mocks `./active-model` without re-exporting
+ * `ActiveModelCoordinator`. The dynamic import keeps the runtime-default
+ * path while leaving test injection as the lighter-weight option.
+ */
+async function defaultService(): Promise<LocalInferenceService> {
+	const mod = await import("./service");
+	return mod.localInferenceService;
+}
+
+/**
+ * Artifact kinds the orchestrator can trigger. The label is consumer-facing
+ * (logs / API responses) and is intentionally distinct from the catalog's
+ * `ModelCategory` — the catalog labels a full chat tier as `"chat"`, while
+ * here we describe the on-device modality the artifact serves.
+ */
+export type EnsureLocalArtifactKind = "embedding" | "tts" | "stt" | "text";
+
+export type EnsureLocalArtifactStatus =
+	| "started"
+	| "already-installed"
+	| "skipped"
+	| "failed";
+
+export interface EnsureLocalArtifactOutcome {
+	kind: EnsureLocalArtifactKind;
+	modelId: string;
+	status: EnsureLocalArtifactStatus;
+	/** Free-text disambiguation. Populated for skipped / failed outcomes. */
+	reason?: string;
+}
+
+export interface EnsureLocalArtifactsArgs {
+	/** Resolved runtime mode (local | local-only | cloud | remote). */
+	mode: RuntimeMode;
+	/** Optional tier override; if omitted, picked from the device hardware probe. */
+	tier?: Eliza1Tier;
+	/** True when the user is signed into Eliza Cloud. Suppresses the text artifact. */
+	signedInCloud: boolean;
+	/** Optional service facade override (tests). */
+	service?: LocalInferenceService;
+	/** Optional logger. */
+	logger?: { info: (...a: unknown[]) => void; warn: (...a: unknown[]) => void };
+}
+
+export interface EnsureLocalArtifactsResult {
+	artifacts: EnsureLocalArtifactOutcome[];
+	/**
+	 * True iff every required artifact is now either installed or actively
+	 * downloading. False when any artifact slot failed; the per-artifact
+	 * outcome carries the reason.
+	 */
+	complete: boolean;
+}
+
+const LOG_TAG = "[ensureLocalArtifacts]";
+
+interface ResolvedLogger {
+	info: (...a: unknown[]) => void;
+	warn: (...a: unknown[]) => void;
+}
+
+function resolveLogger(
+	provided?: EnsureLocalArtifactsArgs["logger"],
+): ResolvedLogger {
+	if (provided) return provided;
+	return {
+		info: (...a: unknown[]) => console.info(LOG_TAG, ...a),
+		warn: (...a: unknown[]) => console.warn(LOG_TAG, ...a),
+	};
+}
+
+/**
+ * Map the recommender's selection for `TEXT_LARGE` (the canonical first-run
+ * slot) onto a tier id. The recommender already biases toward the smallest
+ * tier that fits the device's memory budget — that bundle is what carries
+ * the embedding / TTS / STT artifacts for the local runtime.
+ *
+ * Returns null when the recommender has no fitting catalog entry — usually
+ * means the device is below every tier's RAM floor. The caller falls back
+ * to recording a skipped outcome per artifact so the user sees an audited
+ * skip rather than a silent miss.
+ */
+function tierFromRecommendation(
+	hardware: HardwareProbe,
+): { model: CatalogModel; tier: Eliza1Tier } | null {
+	const selection = selectRecommendedModelForSlot("TEXT_LARGE", hardware);
+	const model = selection.model;
+	if (!model) return null;
+	const tier = tierFromCatalogId(model.id);
+	if (!tier) return null;
+	return { model, tier };
+}
+
+/** "eliza-1-2b" → "2b"; null if the id is not an Eliza-1 tier shape. */
+function tierFromCatalogId(id: string): Eliza1Tier | null {
+	if (!id.startsWith("eliza-1-")) return null;
+	const slug = id.slice("eliza-1-".length) as Eliza1Tier;
+	return slug;
+}
+
+/** "2b" → "eliza-1-2b" — inverse of {@link tierFromCatalogId}. */
+function catalogIdFromTier(tier: Eliza1Tier): string {
+	return `eliza-1-${tier}`;
+}
+
+/**
+ * The orchestrator picks a single bundle id today (each Eliza-1 tier ships
+ * embedding / voice / asr / drafter as one HF bundle). Each artifact "kind"
+ * therefore resolves to the same `modelId`; the downloader's idempotency
+ * collapses parallel `startDownload(modelId)` calls into one job.
+ *
+ * If the architecture later splits the bundle, this helper is where each
+ * kind would diverge to its own id — the orchestrator already wraps each
+ * kind in its own `Promise.allSettled` slot, so no surrounding code has to
+ * change.
+ */
+function modelIdForKind(
+	tier: Eliza1Tier,
+	_kind: EnsureLocalArtifactKind,
+): string {
+	return catalogIdFromTier(tier);
+}
+
+interface ArtifactPlan {
+	kind: EnsureLocalArtifactKind;
+	modelId: string;
+}
+
+function planArtifacts(
+	tier: Eliza1Tier,
+	signedInCloud: boolean,
+): ArtifactPlan[] {
+	const plan: ArtifactPlan[] = [
+		{ kind: "embedding", modelId: modelIdForKind(tier, "embedding") },
+		{ kind: "tts", modelId: modelIdForKind(tier, "tts") },
+		{ kind: "stt", modelId: modelIdForKind(tier, "stt") },
+	];
+	if (!signedInCloud) {
+		plan.push({ kind: "text", modelId: modelIdForKind(tier, "text") });
+	}
+	return plan;
+}
+
+/**
+ * Run one artifact slot: skip when already installed, otherwise trigger the
+ * download via the service facade. Never throws — errors are folded into the
+ * outcome record so {@link Promise.allSettled} stays a defensive safety net.
+ */
+async function runArtifact(
+	plan: ArtifactPlan,
+	service: LocalInferenceService,
+	installedIds: Set<string>,
+	logger: ResolvedLogger,
+): Promise<EnsureLocalArtifactOutcome> {
+	if (installedIds.has(plan.modelId)) {
+		logger.info(
+			`${plan.kind} model ${plan.modelId} is already installed; nothing to do`,
+		);
+		return {
+			kind: plan.kind,
+			modelId: plan.modelId,
+			status: "already-installed",
+		};
+	}
+	try {
+		await service.startDownload(plan.modelId);
+		logger.info(`${plan.kind} model ${plan.modelId} download started`);
+		return { kind: plan.kind, modelId: plan.modelId, status: "started" };
+	} catch (err) {
+		const reason = err instanceof Error ? err.message : String(err);
+		logger.warn(
+			`${plan.kind} model ${plan.modelId} download failed to start:`,
+			reason,
+		);
+		return {
+			kind: plan.kind,
+			modelId: plan.modelId,
+			status: "failed",
+			reason,
+		};
+	}
+}
+
+/**
+ * Build the skipped result for `cloud` / `remote` modes. Kept structurally
+ * identical to the local-mode result so callers don't have to branch on the
+ * runtime mode just to read the response.
+ */
+function skippedModeResult(): EnsureLocalArtifactsResult {
+	return { artifacts: [], complete: true };
+}
+
+/**
+ * Trigger the parallel downloads of every artifact the local runtime needs
+ * to operate on this device. See the module docstring for the per-mode /
+ * per-cloud-state contract.
+ */
+export async function ensureLocalArtifacts(
+	args: EnsureLocalArtifactsArgs,
+): Promise<EnsureLocalArtifactsResult> {
+	const logger = resolveLogger(args.logger);
+	const service = args.service ?? (await defaultService());
+
+	// Cloud / remote modes never need on-device artifacts — the runtime
+	// routes through cloud (cloud) or controls another local instance
+	// (remote). Either way the local artifact pipeline is irrelevant.
+	if (args.mode === "cloud" || args.mode === "remote") {
+		logger.info(`mode is ${args.mode}; no on-device artifacts to download`);
+		return skippedModeResult();
+	}
+
+	// Resolve the tier. Caller may pin one (settings override / test); else
+	// we project the hardware probe through the recommender's TEXT_LARGE
+	// ladder so the chosen tier matches what the user would have selected
+	// via the model hub. When the recommender returns null (no fitting
+	// catalog entry — device below every tier's floor) we record skipped
+	// outcomes for every kind so the caller sees a clear, audited skip
+	// rather than a silent miss.
+	let tier: Eliza1Tier | null = args.tier ?? null;
+	let recommendedModelLabel: string | null = null;
+	if (!tier) {
+		const hardware = await service.getHardware();
+		const recommendation = tierFromRecommendation(hardware);
+		if (recommendation) {
+			tier = recommendation.tier;
+			recommendedModelLabel = recommendation.model.id;
+			logger.info(
+				`tier ${tier} selected from recommender for platform ${classifyRecommendationPlatform(hardware)}`,
+			);
+		}
+	}
+
+	if (!tier) {
+		const reason =
+			"no Eliza-1 tier fits the device hardware probe; nothing to download";
+		logger.warn(reason);
+		const skipped: EnsureLocalArtifactOutcome[] = (
+			["embedding", "tts", "stt"] as EnsureLocalArtifactKind[]
+		).map((kind) => ({
+			kind,
+			modelId: "",
+			status: "skipped",
+			reason,
+		}));
+		if (!args.signedInCloud) {
+			skipped.push({
+				kind: "text",
+				modelId: "",
+				status: "skipped",
+				reason,
+			});
+		}
+		return { artifacts: skipped, complete: false };
+	}
+
+	if (recommendedModelLabel) {
+		logger.info(
+			`targeting bundle ${recommendedModelLabel} for tier ${tier}, signedInCloud=${args.signedInCloud}`,
+		);
+	} else {
+		logger.info(
+			`targeting tier ${tier} (override), signedInCloud=${args.signedInCloud}`,
+		);
+	}
+
+	// Read installed bundles once — every artifact runs through the same
+	// set, and the facade returns a fresh list each call.
+	let installed: ReadonlyArray<InstalledModel> = [];
+	try {
+		installed = await service.getInstalled();
+	} catch (err) {
+		logger.warn(
+			"failed to enumerate installed models; assuming nothing is installed:",
+			err instanceof Error ? err.message : String(err),
+		);
+	}
+	const installedIds = new Set(installed.map((m) => m.id));
+
+	const plan = planArtifacts(tier, args.signedInCloud);
+	const settled = await Promise.allSettled(
+		plan.map((entry) => runArtifact(entry, service, installedIds, logger)),
+	);
+
+	const artifacts: EnsureLocalArtifactOutcome[] = settled.map((result, idx) => {
+		if (result.status === "fulfilled") return result.value;
+		const reason =
+			result.reason instanceof Error
+				? result.reason.message
+				: String(result.reason);
+		return {
+			kind: plan[idx].kind,
+			modelId: plan[idx].modelId,
+			status: "failed",
+			reason,
+		};
+	});
+
+	const complete = artifacts.every(
+		(a) => a.status === "started" || a.status === "already-installed",
+	);
+
+	return { artifacts, complete };
+}

package/src/services/external-scanner.ts

@@ -0,0 +1,312 @@
+/**
+ * Discover GGUF files already on disk from other local-inference tools.
+ *
+ * Users often have LM Studio, Jan, Ollama, or raw HuggingFace downloads
+ * lying around. We scan their default cache paths and surface those models
+ * in the Model Hub with `source: "external-scan"` so Eliza can load them
+ * without re-downloading. Eliza never modifies or deletes these files —
+ * the uninstall endpoint refuses when `source !== "eliza-download"`.
+ *
+ * Ollama is special: its blobs live under `models/blobs/sha256-*` with no
+ * `.gguf` extension, and the human name only exists in adjacent manifests.
+ * We parse the manifests to recover the mapping; blobs we can't map stay
+ * hidden rather than surfacing as opaque hashes.
+ */
+
+import type { Dirent } from "node:fs";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import type { InstalledModel } from "./types";
+
+type ExternalOrigin = NonNullable<InstalledModel["externalOrigin"]>;
+
+interface ScanRoot {
+	origin: ExternalOrigin;
+	dir: string;
+	/** Extra logic for tools that don't just drop .gguf files by filename. */
+	kind: "flat" | "hf-snapshots" | "ollama";
+}
+
+function candidateRoots(): ScanRoot[] {
+	const home = os.homedir();
+	const platform = process.platform;
+	const roots: ScanRoot[] = [];
+
+	// ── LM Studio ──────────────────────────────────────────────────────
+	roots.push(
+		{
+			origin: "lm-studio",
+			dir: path.join(home, ".lmstudio", "models"),
+			kind: "flat",
+		},
+		{
+			origin: "lm-studio",
+			dir: path.join(home, ".cache", "lm-studio", "models"),
+			kind: "flat",
+		},
+	);
+
+	// ── Jan ─────────────────────────────────────────────────────────────
+	if (platform === "darwin") {
+		roots.push({
+			origin: "jan",
+			dir: path.join(
+				home,
+				"Library",
+				"Application Support",
+				"Jan",
+				"data",
+				"models",
+			),
+			kind: "flat",
+		});
+	} else if (platform === "win32") {
+		const appdata =
+			process.env.APPDATA ?? path.join(home, "AppData", "Roaming");
+		roots.push({
+			origin: "jan",
+			dir: path.join(appdata, "Jan", "data", "models"),
+			kind: "flat",
+		});
+	} else {
+		const xdg = process.env.XDG_CONFIG_HOME ?? path.join(home, ".config");
+		roots.push({
+			origin: "jan",
+			dir: path.join(xdg, "Jan", "data", "models"),
+			kind: "flat",
+		});
+	}
+	// Legacy Jan path, still seen on older installs.
+	roots.push({
+		origin: "jan",
+		dir: path.join(home, "jan", "models"),
+		kind: "flat",
+	});
+
+	// ── Ollama ──────────────────────────────────────────────────────────
+	const ollamaOverride = process.env.OLLAMA_MODELS?.trim();
+	if (ollamaOverride) {
+		roots.push({ origin: "ollama", dir: ollamaOverride, kind: "ollama" });
+	}
+	roots.push({
+		origin: "ollama",
+		dir: path.join(home, ".ollama", "models"),
+		kind: "ollama",
+	});
+	if (platform === "linux") {
+		roots.push(
+			{
+				origin: "ollama",
+				dir: "/usr/share/ollama/.ollama/models",
+				kind: "ollama",
+			},
+			{
+				origin: "ollama",
+				dir: "/var/lib/ollama/.ollama/models",
+				kind: "ollama",
+			},
+		);
+	}
+
+	// ── HuggingFace hub ────────────────────────────────────────────────
+	const hfOverride =
+		process.env.HF_HUB_CACHE?.trim() ||
+		(process.env.HF_HOME ? path.join(process.env.HF_HOME, "hub") : null);
+	const hfDefault = path.join(home, ".cache", "huggingface", "hub");
+	roots.push({
+		origin: "huggingface",
+		dir: hfOverride || hfDefault,
+		kind: "hf-snapshots",
+	});
+
+	// ── text-generation-webui ──────────────────────────────────────────
+	// Best-effort common install locations; users can symlink anything they
+	// want under here. We only pick up `.gguf` files.
+	roots.push(
+		{
+			origin: "text-gen-webui",
+			dir: path.join(home, "text-generation-webui", "user_data", "models"),
+			kind: "flat",
+		},
+		{
+			origin: "text-gen-webui",
+			dir: path.join(home, "text-generation-webui", "models"),
+			kind: "flat",
+		},
+	);
+
+	return roots;
+}
+
+async function dirExists(dir: string): Promise<boolean> {
+	try {
+		const stat = await fs.stat(dir);
+		return stat.isDirectory();
+	} catch {
+		return false;
+	}
+}
+
+async function* walkForGgufs(
+	root: string,
+	maxDepth = 6,
+): AsyncGenerator<{
+	absPath: string;
+	realPath: string;
+	size: number;
+	mtimeMs: number;
+}> {
+	const stack: Array<{ dir: string; depth: number }> = [
+		{ dir: root, depth: 0 },
+	];
+	while (stack.length > 0) {
+		const frame = stack.pop();
+		if (!frame) break;
+		const { dir, depth } = frame;
+		let entries: Dirent[];
+		try {
+			entries = await fs.readdir(dir, { withFileTypes: true });
+		} catch {
+			continue;
+		}
+		for (const entry of entries) {
+			const full = path.join(dir, entry.name);
+			if (entry.isDirectory()) {
+				if (depth < maxDepth) stack.push({ dir: full, depth: depth + 1 });
+				continue;
+			}
+			// HF snapshots are symlinks into ../blobs/; follow them.
+			const isLink = entry.isSymbolicLink();
+			if (!isLink && !entry.isFile()) continue;
+			if (!full.toLowerCase().endsWith(".gguf")) continue;
+			try {
+				const realPath = isLink ? await fs.realpath(full) : full;
+				const stat = await fs.stat(realPath);
+				if (!stat.isFile()) continue;
+				yield {
+					absPath: full,
+					realPath,
+					size: stat.size,
+					mtimeMs: stat.mtimeMs,
+				};
+			} catch {
+				// Broken symlink or permission issue; skip silently.
+			}
+		}
+	}
+}
+
+interface OllamaManifestLayer {
+	mediaType: string;
+	digest: string;
+	size: number;
+}
+
+interface OllamaManifest {
+	layers: OllamaManifestLayer[];
+}
+
+async function scanOllama(root: string): Promise<InstalledModel[]> {
+	const manifestsRoot = path.join(root, "manifests");
+	const blobsRoot = path.join(root, "blobs");
+
+	if (!(await dirExists(manifestsRoot)) || !(await dirExists(blobsRoot))) {
+		return [];
+	}
+
+	const results: InstalledModel[] = [];
+	const stack: string[] = [manifestsRoot];
+	while (stack.length > 0) {
+		const dir = stack.pop();
+		if (!dir) break;
+		let entries: Dirent[];
+		try {
+			entries = await fs.readdir(dir, { withFileTypes: true });
+		} catch {
+			continue;
+		}
+		for (const entry of entries) {
+			const full = path.join(dir, entry.name);
+			if (entry.isDirectory()) {
+				stack.push(full);
+				continue;
+			}
+			if (!entry.isFile()) continue;
+			let manifest: OllamaManifest;
+			try {
+				const raw = await fs.readFile(full, "utf8");
+				manifest = JSON.parse(raw) as OllamaManifest;
+			} catch {
+				continue;
+			}
+			const modelLayer = manifest.layers.find((l) =>
+				l.mediaType.includes("model"),
+			);
+			if (!modelLayer?.digest) continue;
+			const digest = modelLayer.digest.replace("sha256:", "sha256-");
+			const blobPath = path.join(blobsRoot, digest);
+			let size = modelLayer.size;
+			try {
+				const stat = await fs.stat(blobPath);
+				size = stat.size;
+			} catch {
+				continue;
+			}
+			const relativeManifest = path.relative(manifestsRoot, full);
+			const displayName = `ollama: ${relativeManifest.split(path.sep).slice(-2).join(":")}`;
+			results.push({
+				id: `external-ollama-${digest}`,
+				displayName,
+				path: blobPath,
+				sizeBytes: size,
+				installedAt: new Date().toISOString(),
+				lastUsedAt: null,
+				source: "external-scan",
+				externalOrigin: "ollama",
+			});
+		}
+	}
+	return results;
+}
+
+export async function scanExternalModels(): Promise<InstalledModel[]> {
+	const roots = candidateRoots();
+	const seenRealPaths = new Set<string>();
+	const results: InstalledModel[] = [];
+
+	await Promise.all(
+		roots.map(async (root) => {
+			if (!(await dirExists(root.dir))) return;
+
+			if (root.kind === "ollama") {
+				const ollamaModels = await scanOllama(root.dir);
+				for (const model of ollamaModels) {
+					if (seenRealPaths.has(model.path)) continue;
+					seenRealPaths.add(model.path);
+					results.push(model);
+				}
+				return;
+			}
+
+			for await (const found of walkForGgufs(root.dir)) {
+				if (seenRealPaths.has(found.realPath)) continue;
+				seenRealPaths.add(found.realPath);
+
+				const displayName = path.basename(found.absPath, ".gguf");
+				results.push({
+					id: `external-${root.origin}-${Buffer.from(found.realPath).toString("base64url").slice(0, 16)}`,
+					displayName: `${displayName} (${root.origin})`,
+					path: found.realPath,
+					sizeBytes: found.size,
+					installedAt: new Date(found.mtimeMs).toISOString(),
+					lastUsedAt: null,
+					source: "external-scan",
+					externalOrigin: root.origin,
+				});
+			}
+		}),
+	);
+
+	return results;
+}