@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/hf-search.ts

@@ -0,0 +1,420 @@
+/**
+ * HuggingFace Hub search for GGUF models.
+ *
+ * Calls `https://huggingface.co/api/models` with `filter=gguf` to narrow
+ * results to repos that actually ship GGUF quantisations. Each matching
+ * repo is expanded with `/api/models/<repo>` to pick a representative
+ * quant file (preferring Q4_K_M when present). Results are shaped like
+ * `CatalogModel` so the existing ModelCard renders them directly.
+ *
+ * We deliberately do not persist these — they're dynamic, and a curated
+ * entry with the same hfRepo always takes precedence in the UI.
+ */
+
+import type { CatalogModel, ModelBucket } from "./types";
+
+const HF_API = "https://huggingface.co/api";
+const SEARCH_TIMEOUT_MS = 10_000;
+
+interface HfSearchResultRaw {
+	id?: string;
+	modelId?: string;
+	author?: string;
+	likes?: number;
+	downloads?: number;
+	tags?: string[];
+	pipeline_tag?: string;
+}
+
+interface HfSiblingRaw {
+	rfilename?: string;
+	size?: number;
+}
+
+interface HfModelDetailRaw {
+	id?: string;
+	author?: string;
+	tags?: string[];
+	siblings?: HfSiblingRaw[];
+	pipeline_tag?: string;
+	likes?: number;
+	downloads?: number;
+}
+
+interface ModelScopeEnvelope<T> {
+	Code?: number;
+	Message?: string;
+	Data?: T;
+}
+
+interface ModelScopeModelRaw {
+	ModelId?: string;
+	model_id?: string;
+	Name?: string;
+	Path?: string;
+	Downloads?: number;
+	Tags?: string[];
+	Tasks?: string[];
+	Frameworks?: string[];
+}
+
+interface ModelScopeListRaw {
+	Models?: ModelScopeModelRaw[];
+}
+
+interface ModelScopeFileRaw {
+	Name?: string;
+	Path?: string;
+	Size?: number;
+}
+
+interface ModelScopeFilesRaw {
+	Files?: ModelScopeFileRaw[];
+}
+
+const QUANT_PREFERENCE = [
+	"Q4_K_M",
+	"Q5_K_M",
+	"Q4_0",
+	"Q5_0",
+	"Q3_K_M",
+	"Q8_0",
+	"Q2_K",
+];
+
+function pickQuantFile(siblings: HfSiblingRaw[]): HfSiblingRaw | null {
+	const ggufs = siblings.filter((s) =>
+		s.rfilename?.toLowerCase().endsWith(".gguf"),
+	);
+	if (ggufs.length === 0) return null;
+
+	for (const quant of QUANT_PREFERENCE) {
+		const match = ggufs.find((s) => s.rfilename?.toUpperCase().includes(quant));
+		if (match) return match;
+	}
+	// No preferred quant — pick the smallest GGUF so we don't surprise the
+	// user with a 40 GB file.
+	return [...ggufs].sort((a, b) => (a.size ?? 0) - (b.size ?? 0))[0] ?? null;
+}
+
+function extractQuantLabel(filename: string): string {
+	for (const quant of QUANT_PREFERENCE) {
+		if (filename.toUpperCase().includes(quant)) return quant;
+	}
+	return "GGUF";
+}
+
+/**
+ * Very rough parameter-count inference from model name / tags. We use this
+ * only to pick a bucket label — not for any hard memory check.
+ */
+function inferParams(
+	name: string,
+	tags: string[],
+): {
+	params: CatalogModel["params"];
+	bucket: ModelBucket;
+	parameterLabel?: string;
+} {
+	const lower = `${name} ${tags.join(" ")}`.toLowerCase();
+	const sizes: Array<[RegExp, CatalogModel["params"], ModelBucket, string?]> = [
+		[/\b32b\b/, "32B", "xl"],
+		[/\b27b\b/, "27B", "large"],
+		[/\b24b\b/, "24B", "large"],
+		[/\b22b\b/, "22B", "large"],
+		[/\b16b\b/, "16B", "large"],
+		[/\b14b\b/, "14B", "large"],
+		[/\b13b\b/, "14B", "large"],
+		[/\b9b\b/, "9B", "mid"],
+		[/(?:^|[^0-9])0\.8b\b/, "0.8B", "small"],
+		[/\b8b\b/, "8B", "mid"],
+		[/\b7b\b/, "7B", "mid"],
+		[/\b3b\b/, "3B", "small"],
+		[/\b2b\b/, "2B", "small"],
+		[/\b1b\b/, "1B", "small"],
+	];
+	for (const [re, params, bucket, parameterLabel] of sizes) {
+		if (re.test(lower)) {
+			return {
+				params,
+				bucket,
+				...(parameterLabel ? { parameterLabel } : {}),
+			};
+		}
+	}
+	return { params: "7B", bucket: "mid" };
+}
+
+function inferCategory(
+	tags: string[],
+	pipelineTag: string | undefined,
+): CatalogModel["category"] {
+	const lowerTags = tags.map((t) => t.toLowerCase());
+	if (lowerTags.some((t) => t.includes("code") || t.includes("coder"))) {
+		return "code";
+	}
+	if (
+		lowerTags.some(
+			(t) => t.includes("reasoning") || t === "math" || t.includes("r1"),
+		)
+	) {
+		return "reasoning";
+	}
+	if (
+		lowerTags.some(
+			(t) =>
+				t.includes("function") || t.includes("tool") || t.includes("hermes"),
+		)
+	) {
+		return "tools";
+	}
+	if (pipelineTag === "text-generation") {
+		return "chat";
+	}
+	return "chat";
+}
+
+async function fetchWithTimeout(
+	url: string,
+	init?: RequestInit,
+): Promise<Response> {
+	const controller = new AbortController();
+	const timer = setTimeout(() => controller.abort(), SEARCH_TIMEOUT_MS);
+	try {
+		return await fetch(url, { ...init, signal: controller.signal });
+	} finally {
+		clearTimeout(timer);
+	}
+}
+
+/**
+ * Search HuggingFace for GGUF repos matching `query`, returning
+ * catalog-shaped entries ready for the Model Hub UI.
+ */
+export async function searchHuggingFaceGguf(
+	query: string,
+	limit = 12,
+): Promise<CatalogModel[]> {
+	const trimmed = query.trim();
+	if (trimmed.length === 0) return [];
+
+	const searchUrl = new URL(`${HF_API}/models`);
+	searchUrl.searchParams.set("search", trimmed);
+	searchUrl.searchParams.set("filter", "gguf");
+	searchUrl.searchParams.set(
+		"limit",
+		String(Math.min(50, Math.max(1, limit * 2))),
+	);
+	searchUrl.searchParams.set("sort", "downloads");
+	searchUrl.searchParams.set("direction", "-1");
+
+	const searchRes = await fetchWithTimeout(searchUrl.toString(), {
+		headers: { accept: "application/json" },
+	});
+	if (!searchRes.ok) {
+		throw new Error(`HuggingFace search failed: HTTP ${searchRes.status}`);
+	}
+	const rawList = (await searchRes.json()) as HfSearchResultRaw[];
+	const candidates = rawList
+		.map((r) => r.id ?? r.modelId)
+		.filter((id): id is string => typeof id === "string" && id.length > 0)
+		.slice(0, limit);
+
+	// Parallel detail fetches; failures drop quietly so one bad repo doesn't
+	// take down the whole search.
+	const details = await Promise.all(
+		candidates.map(async (id) => {
+			try {
+				const res = await fetchWithTimeout(
+					`${HF_API}/models/${encodeURIComponent(id)}`,
+					{ headers: { accept: "application/json" } },
+				);
+				if (!res.ok) return null;
+				return (await res.json()) as HfModelDetailRaw;
+			} catch {
+				return null;
+			}
+		}),
+	);
+
+	const results: CatalogModel[] = [];
+	for (let i = 0; i < candidates.length; i++) {
+		const id = candidates[i];
+		const detail = details[i];
+		if (!id || !detail?.siblings) continue;
+		const sibling = pickQuantFile(detail.siblings);
+		if (!sibling?.rfilename) continue;
+
+		const sizeBytes = sibling.size ?? 0;
+		const sizeGb = sizeBytes > 0 ? sizeBytes / 1024 ** 3 : 4;
+		const { params, bucket, parameterLabel } = inferParams(
+			id,
+			detail.tags ?? [],
+		);
+		const quant = extractQuantLabel(sibling.rfilename);
+		const category = inferCategory(detail.tags ?? [], detail.pipeline_tag);
+		const displayName = id.split("/").pop() ?? id;
+
+		// minRam heuristic: 2x the file size, minimum 4 GB. Close enough for
+		// the fit badge; the ModelCard's fit computation still does the work.
+		const minRamGb = Math.max(4, Math.round(sizeGb * 2));
+
+		results.push({
+			id: `hf:${id}::${sibling.rfilename}`,
+			displayName,
+			hfRepo: id,
+			ggufFile: sibling.rfilename,
+			params,
+			...(parameterLabel ? { parameterLabel } : {}),
+			quant,
+			sizeGb: Math.round(sizeGb * 10) / 10,
+			minRamGb,
+			category,
+			bucket,
+			blurb:
+				(detail.tags ?? []).slice(0, 4).join(" · ") ||
+				`${detail.downloads ?? 0} downloads · ${detail.likes ?? 0} likes`,
+		});
+	}
+	return results;
+}
+
+function normalizeModelScopeRepoId(query: string): string | null {
+	const trimmed = query.trim().replace(/^modelscope:\/\//i, "");
+	const match = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.exec(trimmed);
+	return match ? trimmed : null;
+}
+
+function modelScopeOwnerQuery(query: string): string | null {
+	const trimmed = query.trim().replace(/^modelscope:\/\//i, "");
+	const first = trimmed.split(/\s+/)[0] ?? "";
+	if (/^[A-Za-z0-9_.-]+$/.test(first)) return first;
+	return null;
+}
+
+function pickModelScopeGguf(files: ModelScopeFileRaw[]): HfSiblingRaw | null {
+	return pickQuantFile(
+		files.map((file) => ({
+			rfilename: file.Path ?? file.Name,
+			size: file.Size,
+		})),
+	);
+}
+
+async function fetchModelScopeJson<T>(
+	url: string,
+	init?: RequestInit,
+): Promise<T> {
+	const headers = new Headers(init?.headers);
+	headers.set("accept", "application/json");
+	const res = await fetchWithTimeout(url, {
+		...init,
+		headers,
+		redirect: "follow",
+	});
+	if (!res.ok) {
+		throw new Error(`ModelScope request failed: HTTP ${res.status}`);
+	}
+	const envelope = (await res.json()) as ModelScopeEnvelope<T>;
+	if (envelope.Code !== 200 || envelope.Data === undefined) {
+		throw new Error(envelope.Message ?? "ModelScope request failed");
+	}
+	return envelope.Data;
+}
+
+async function listModelScopeOwnerModels(
+	owner: string,
+	limit: number,
+): Promise<string[]> {
+	const data = await fetchModelScopeJson<ModelScopeListRaw>(
+		"https://www.modelscope.cn/api/v1/models",
+		{
+			method: "PUT",
+			headers: { "content-type": "application/json" },
+			body: JSON.stringify({
+				Path: owner,
+				PageNumber: 1,
+				PageSize: Math.min(50, Math.max(1, limit * 3)),
+			}),
+		},
+	);
+	return (data.Models ?? [])
+		.map((m) => m.ModelId ?? m.model_id)
+		.filter((id): id is string => typeof id === "string" && id.length > 0);
+}
+
+async function modelScopeResultForRepo(
+	repoId: string,
+): Promise<CatalogModel | null> {
+	const filesData = await fetchModelScopeJson<ModelScopeFilesRaw>(
+		`https://www.modelscope.cn/api/v1/models/${repoId}/repo/files?Revision=master&Recursive=true`,
+	);
+	const sibling = pickModelScopeGguf(filesData.Files ?? []);
+	if (!sibling?.rfilename) return null;
+
+	const sizeBytes = sibling.size ?? 0;
+	const sizeGb = sizeBytes > 0 ? sizeBytes / 1024 ** 3 : 4;
+	const { params, bucket } = inferParams(repoId, []);
+	const quant = extractQuantLabel(sibling.rfilename);
+	const displayName = repoId.split("/").pop() ?? repoId;
+
+	return {
+		id: `modelscope:${repoId}::${sibling.rfilename}`,
+		displayName,
+		hub: "modelscope",
+		hfRepo: repoId,
+		ggufFile: sibling.rfilename,
+		params,
+		quant,
+		sizeGb: Math.round(sizeGb * 10) / 10,
+		minRamGb: Math.max(4, Math.round(sizeGb * 2)),
+		category: "chat",
+		bucket,
+		blurb: "ModelScope GGUF result · explicit opt-in only",
+	};
+}
+
+/**
+ * Search ModelScope for GGUF repos. ModelScope exposes reliable owner-list
+ * and repo-file APIs, so this accepts either `owner/model` or an owner name
+ * and filters the returned repos for GGUF files.
+ */
+export async function searchModelScopeGguf(
+	query: string,
+	limit = 12,
+): Promise<CatalogModel[]> {
+	const trimmed = query.trim();
+	if (trimmed.length === 0) return [];
+
+	const direct = normalizeModelScopeRepoId(trimmed);
+	const repoIds = direct
+		? [direct]
+		: await listModelScopeOwnerModels(
+				modelScopeOwnerQuery(trimmed) ?? trimmed,
+				limit,
+			);
+
+	const results = await Promise.all(
+		repoIds.slice(0, Math.min(50, limit * 3)).map(async (id) => {
+			try {
+				return await modelScopeResultForRepo(id);
+			} catch {
+				return null;
+			}
+		}),
+	);
+
+	return results
+		.filter((model): model is CatalogModel => Boolean(model))
+		.slice(0, limit);
+}
+
+export async function searchModelHubGguf(
+	query: string,
+	hub: "huggingface" | "modelscope" = "huggingface",
+	limit = 12,
+): Promise<CatalogModel[]> {
+	return hub === "modelscope"
+		? searchModelScopeGguf(query, limit)
+		: searchHuggingFaceGguf(query, limit);
+}

package/src/services/image-description-runtime.d.ts

@@ -0,0 +1,14 @@
+export interface ImageDescriptionRuntimeOptions {
+    tier: string;
+    modelPath: string;
+}
+export interface ImageDescriptionRuntime {
+    describe(args: {
+        imagePath: string;
+        prompt: string;
+        maxTokens?: number;
+    }): Promise<string>;
+    cleanup?(): Promise<void>;
+}
+export declare function createImageDescriptionRuntime(args: ImageDescriptionRuntimeOptions): Promise<ImageDescriptionRuntime>;
+//# sourceMappingURL=image-description-runtime.d.ts.map

package/src/services/image-description-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-description-runtime.d.ts","sourceRoot":"","sources":["image-description-runtime.ts"],"names":[],"mappings":"AAkBA,MAAM,WAAW,8BAA8B;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,uBAAuB;IACvC,QAAQ,CAAC,IAAI,EAAE;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,6BAA6B,CAClD,IAAI,EAAE,8BAA8B,GAClC,OAAO,CAAC,uBAAuB,CAAC,CA8BlC"}

package/src/services/image-description-runtime.test.ts

@@ -0,0 +1,61 @@
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+	load: vi.fn(async () => undefined),
+	describeImage: vi.fn(async () => ({ text: "ok" })),
+	unload: vi.fn(async () => undefined),
+}));
+
+vi.mock("./engine", () => ({
+	localInferenceEngine: {
+		load: mocks.load,
+		describeImage: mocks.describeImage,
+		unload: mocks.unload,
+	},
+}));
+
+import { createImageDescriptionRuntime } from "./image-description-runtime";
+
+afterEach(() => {
+	vi.clearAllMocks();
+});
+
+describe("createImageDescriptionRuntime", () => {
+	it("passes the manifest catalog id for direct bundle loads", async () => {
+		const bundleRoot = mkdtempSync(path.join(tmpdir(), "eliza-vision-"));
+		mkdirSync(path.join(bundleRoot, "text"), { recursive: true });
+		mkdirSync(path.join(bundleRoot, "vision"), { recursive: true });
+		const textPath = path.join(bundleRoot, "text", "eliza-1-9b-128k.gguf");
+		const mmprojPath = path.join(bundleRoot, "vision", "mmproj-9b.gguf");
+		writeFileSync(textPath, "GGUF", "utf8");
+		writeFileSync(mmprojPath, "GGUF", "utf8");
+		writeFileSync(
+			path.join(bundleRoot, "eliza-1.manifest.json"),
+			JSON.stringify({
+				id: "eliza-1-9b",
+				files: {
+					text: [{ path: "text/eliza-1-9b-128k.gguf", ctx: 131072 }],
+					vision: [{ path: "vision/mmproj-9b.gguf" }],
+				},
+			}),
+			"utf8",
+		);
+
+		const runtime = await createImageDescriptionRuntime({
+			tier: "eliza-1-9b",
+			modelPath: bundleRoot,
+		});
+
+		expect(mocks.load).toHaveBeenCalledWith(textPath, {
+			modelPath: textPath,
+			modelId: "eliza-1-9b",
+			mmprojPath,
+			contextSize: 4096,
+		});
+		await runtime.cleanup?.();
+		expect(mocks.unload).toHaveBeenCalled();
+	});
+});

package/src/services/image-description-runtime.ts

@@ -0,0 +1,118 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { localInferenceEngine } from "./engine";
+import type { LocalInferenceLoadArgs } from "./load-args";
+
+interface BundleManifestFile {
+	path: string;
+	ctx?: number;
+}
+
+interface BundleManifest {
+	id?: string;
+	files?: {
+		text?: BundleManifestFile[];
+		vision?: BundleManifestFile[];
+	};
+}
+
+export interface ImageDescriptionRuntimeOptions {
+	tier: string;
+	modelPath: string;
+}
+
+export interface ImageDescriptionRuntime {
+	describe(args: {
+		imagePath: string;
+		prompt: string;
+		maxTokens?: number;
+	}): Promise<string>;
+	cleanup?(): Promise<void>;
+}
+
+export async function createImageDescriptionRuntime(
+	args: ImageDescriptionRuntimeOptions,
+): Promise<ImageDescriptionRuntime> {
+	const bundleRoot = path.resolve(args.modelPath);
+	const manifest = await readBundleManifest(bundleRoot);
+	const textModelPath = resolveTextModelPath(bundleRoot, manifest, args.tier);
+	const mmprojPath = resolveVisionModelPath(bundleRoot, manifest, args.tier);
+	const loadArgs: LocalInferenceLoadArgs = {
+		modelPath: textModelPath,
+		modelId: manifest.id || args.tier,
+		mmprojPath,
+		contextSize: 4096,
+	};
+
+	await localInferenceEngine.load(textModelPath, loadArgs);
+
+	return {
+		async describe({ imagePath, prompt, maxTokens }) {
+			const bytes = await readImageInput(imagePath);
+			const result = await localInferenceEngine.describeImage({
+				bytes,
+				mimeType: mimeTypeFor(imagePath),
+				prompt,
+				maxTokens,
+				temperature: 0,
+			});
+			return result.text;
+		},
+		async cleanup() {
+			await localInferenceEngine.unload();
+		},
+	};
+}
+
+async function readBundleManifest(bundleRoot: string): Promise<BundleManifest> {
+	const manifestPath = path.join(bundleRoot, "eliza-1.manifest.json");
+	const raw = await readFile(manifestPath, "utf8");
+	return JSON.parse(raw) as BundleManifest;
+}
+
+function resolveTextModelPath(
+	bundleRoot: string,
+	manifest: BundleManifest,
+	tier: string,
+): string {
+	const textFiles = manifest.files?.text ?? [];
+	const selected = [...textFiles].sort(
+		(a, b) => (a.ctx ?? 0) - (b.ctx ?? 0),
+	)[0];
+	if (selected?.path) return path.join(bundleRoot, selected.path);
+	const slug = tier.replace(/^eliza-1-/, "");
+	return path.join(bundleRoot, "text", `eliza-1-${slug}-64k.gguf`);
+}
+
+function resolveVisionModelPath(
+	bundleRoot: string,
+	manifest: BundleManifest,
+	tier: string,
+): string {
+	const selected = manifest.files?.vision?.[0];
+	if (selected?.path) return path.join(bundleRoot, selected.path);
+	const slug = tier.replace(/^eliza-1-/, "");
+	return path.join(bundleRoot, "vision", `mmproj-${slug}.gguf`);
+}
+
+async function readImageInput(imagePath: string): Promise<Uint8Array> {
+	if (imagePath.startsWith("data:")) {
+		const comma = imagePath.indexOf(",");
+		if (comma < 0) throw new Error("invalid data URL image input");
+		const encoded = imagePath.slice(comma + 1);
+		return Uint8Array.from(Buffer.from(encoded, "base64"));
+	}
+	return Uint8Array.from(await readFile(imagePath));
+}
+
+function mimeTypeFor(imagePath: string): string | undefined {
+	if (imagePath.startsWith("data:")) {
+		const match = /^data:([^;,]+)/.exec(imagePath);
+		return match?.[1];
+	}
+	const ext = path.extname(imagePath).toLowerCase();
+	if (ext === ".png") return "image/png";
+	if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg";
+	if (ext === ".webp") return "image/webp";
+	return undefined;
+}