@valni/cli-darwin-x64 0.0.0-stage → 0.1.3

package/runtime/extensions/switchboard-local/extensions/local/engines/gguf.ts

@@ -0,0 +1,324 @@
+import { spawn } from "node:child_process";
+import { chmodSync, existsSync, mkdirSync, rmSync } from "node:fs";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { ChildProcess } from "node:child_process";
+import type { LocalEngine, LocalModelInfo, Progress } from "../engine.ts";
+import { llamaBinDir, llamaDir, llamaModelsDir } from "../paths.ts";
+import { claimOwnership, freePort, isAlive, portIsFree, readState, releaseOwnership, updateState } from "../store.ts";
+import { ggufQuants, repoFiles, type Quantization } from "../hf.ts";
+import { buildPlan, downloadUrl, latestBuild, type BuildChoice } from "./llamaBuild.ts";
+import { copyDir, detachChild, directorySize, findFile, healthy, killOnExit, killPid, run, stopChild, watchSpawn } from "./support.ts";
+
+const SERVER_NAME = process.platform === "win32" ? "llama-server.exe" : "llama-server";
+const HEALTH_TIMEOUT_MS = 180_000;
+const HEALTH_POLL_MS = 500;
+const DOWNLOAD_POLL_MS = 500;
+const CONTEXT_PER_MODEL = 32_768;
+const DEFAULT_CONTEXT_WINDOW = 8_192;
+
+interface RouterModel {
+	id: string;
+	status: string;
+	contextWindow: number;
+	vision: boolean;
+}
+
+function parseRouterModels(payload: unknown): RouterModel[] {
+	const data = (payload as { data?: unknown }).data;
+	if (!Array.isArray(data)) return [];
+	return data.flatMap((value) => {
+		if (typeof value !== "object" || value === null) return [];
+		const model = value as {
+			id?: unknown;
+			status?: { value?: unknown };
+			meta?: { n_ctx?: unknown; n_ctx_train?: unknown };
+			architecture?: { input_modalities?: unknown };
+		};
+		if (typeof model.id !== "string") return [];
+		const reported = [model.meta?.n_ctx, model.meta?.n_ctx_train].find(
+			(candidate) => typeof candidate === "number" && candidate > 0,
+		);
+		const modalities = model.architecture?.input_modalities;
+		return [
+			{
+				id: model.id,
+				status: typeof model.status?.value === "string" ? model.status.value : "unknown",
+				contextWindow: typeof reported === "number" ? reported : DEFAULT_CONTEXT_WINDOW,
+				vision: Array.isArray(modalities) && modalities.includes("image"),
+			},
+		];
+	});
+}
+
+export class GgufEngine implements LocalEngine {
+	readonly id = "gguf" as const;
+	readonly label = "GGUF";
+
+	child: ChildProcess | undefined;
+	private port: number | undefined;
+	private starting: Promise<void> | undefined;
+	private exitHookInstalled = false;
+	private adopted = false;
+
+	supported(): boolean {
+		return buildPlan(process.platform, process.arch) !== null;
+	}
+
+	unsupportedReason(): string {
+		return `llama.cpp ships no build for ${process.platform} ${process.arch}.`;
+	}
+
+	serverPath(): string {
+		return join(llamaBinDir(), SERVER_NAME);
+	}
+
+	isInstalled(): boolean {
+		return existsSync(this.serverPath());
+	}
+
+	isRunning(): boolean {
+		return this.port !== undefined;
+	}
+
+	baseUrl(): string {
+		if (this.port === undefined) throw new Error("the GGUF engine is not running");
+		return `http://127.0.0.1:${this.port}/v1`;
+	}
+
+	private routerUrl(path: string): string {
+		if (this.port === undefined) throw new Error("the GGUF engine is not running");
+		return `http://127.0.0.1:${this.port}${path}`;
+	}
+
+	async ensureReady(onProgress: (progress: Progress) => void, signal?: AbortSignal): Promise<void> {
+		if (!this.supported()) throw new Error(this.unsupportedReason());
+		if (this.port !== undefined) return;
+		this.starting ??= this.start(onProgress, signal).finally(() => {
+			this.starting = undefined;
+		});
+		await this.starting;
+	}
+
+	private async start(onProgress: (progress: Progress) => void, signal?: AbortSignal): Promise<void> {
+		const plan = buildPlan(process.platform, process.arch);
+		if (plan === null) throw new Error(this.unsupportedReason());
+		if (!this.isInstalled()) await this.install(plan.preferred, onProgress, signal);
+		const existing = readState().ggufPort;
+		if (existing !== undefined && (await healthy(`http://127.0.0.1:${existing}/health`))) {
+			this.port = existing;
+			this.adopted = true;
+			claimOwnership("ggufOwners", process.pid);
+			this.installExitHook();
+			return;
+		}
+		this.killStale();
+		try {
+			await this.launch(onProgress, signal);
+		} catch (error) {
+			if (plan.fallback === undefined) throw error;
+			onProgress({ label: `${plan.preferred.accelerator} build did not run, falling back to ${plan.fallback.accelerator}` });
+			rmSync(llamaBinDir(), { recursive: true, force: true });
+			await this.install(plan.fallback, onProgress, signal);
+			await this.launch(onProgress, signal);
+		}
+	}
+
+	private async install(
+		choice: BuildChoice,
+		onProgress: (progress: Progress) => void,
+		signal?: AbortSignal,
+	): Promise<void> {
+		const build = readState().ggufBuild ?? (await latestBuild(signal));
+		onProgress({ label: `Downloading the llama.cpp ${choice.accelerator} build ${build}` });
+		const url = downloadUrl(build, choice);
+		const timeout = AbortSignal.timeout(600_000);
+		const response = await fetch(url, { signal: signal ? AbortSignal.any([signal, timeout]) : timeout });
+		if (!response.ok) throw new Error(`could not download ${url}: HTTP ${response.status}`);
+		mkdirSync(llamaDir(), { recursive: true });
+		const archive = join(llamaDir(), "download");
+		await writeFile(archive, Buffer.from(await response.arrayBuffer()));
+		onProgress({ label: "Unpacking the llama.cpp build" });
+		const staging = join(llamaDir(), "staging");
+		rmSync(staging, { recursive: true, force: true });
+		mkdirSync(staging, { recursive: true });
+		await run("tar", ["-xf", archive, "-C", staging], { signal });
+		rmSync(archive, { force: true });
+		const server = findFile(staging, SERVER_NAME);
+		if (server === undefined) throw new Error(`${SERVER_NAME} was not in the llama.cpp build`);
+		rmSync(llamaBinDir(), { recursive: true, force: true });
+		mkdirSync(llamaBinDir(), { recursive: true });
+		copyDir(join(server, ".."), llamaBinDir());
+		rmSync(staging, { recursive: true, force: true });
+		chmodSync(this.serverPath(), 0o755);
+		updateState({ ggufBuild: build, ggufAccelerator: choice.accelerator });
+	}
+
+	private async launch(onProgress: (progress: Progress) => void, signal?: AbortSignal): Promise<void> {
+		const existing = readState().ggufPort;
+		const port = existing !== undefined && (await portIsFree(existing)) ? existing : await freePort();
+		mkdirSync(llamaModelsDir(), { recursive: true });
+		onProgress({ label: `Starting the GGUF engine on port ${port}` });
+		const child = spawn(
+			this.serverPath(),
+			[
+				"--models-dir",
+				llamaModelsDir(),
+				"--jinja",
+				"--host",
+				"127.0.0.1",
+				"--port",
+				String(port),
+				"-ngl",
+				"999",
+				"-c",
+				String(CONTEXT_PER_MODEL),
+			],
+			{
+				cwd: llamaDir(),
+				env: { ...process.env, LLAMA_CACHE: llamaModelsDir() },
+				detached: process.platform !== "win32",
+				stdio: ["ignore", "ignore", "pipe"],
+			},
+		);
+		let stderr = "";
+		child.stderr?.on("data", (chunk: Buffer) => {
+			stderr = `${stderr}${chunk.toString()}`.slice(-4000);
+		});
+		const spawnFailure = watchSpawn(child);
+		let exited = false;
+		child.once("exit", () => {
+			exited = true;
+			if (this.child === child) {
+				this.child = undefined;
+				this.port = undefined;
+			}
+		});
+		this.child = child;
+		detachChild(child);
+		updateState({ ggufPort: port, ggufPid: child.pid });
+		claimOwnership("ggufOwners", process.pid);
+		this.installExitHook();
+		const deadline = Date.now() + HEALTH_TIMEOUT_MS;
+		while (Date.now() < deadline) {
+			if (signal?.aborted) {
+				child.kill("SIGTERM");
+				throw new Error("cancelled");
+			}
+			if (exited) {
+				const failure = spawnFailure.failure();
+				if (failure !== undefined) throw failure;
+				const tail = stderr.trim().split("\n").slice(-4).join("\n");
+				throw new Error(`the GGUF engine stopped while starting${tail ? `\n${tail}` : ""}`);
+			}
+			if (await healthy(`http://127.0.0.1:${port}/health`)) {
+				this.port = port;
+				return;
+			}
+			await new Promise((resolve) => setTimeout(resolve, HEALTH_POLL_MS));
+		}
+		child.kill("SIGTERM");
+		throw new Error("the GGUF engine did not start in time");
+	}
+
+	private installExitHook(): void {
+		if (this.exitHookInstalled) return;
+		this.exitHookInstalled = true;
+		killOnExit(() => this.releaseAndMaybeKill());
+	}
+
+	private releaseAndMaybeKill(): number | undefined {
+		const remaining = releaseOwnership("ggufOwners", process.pid);
+		if (remaining.length > 0) return undefined;
+		const pid = this.child?.pid ?? readState().ggufPid;
+		if (pid === undefined || !isAlive(pid)) return undefined;
+		updateState({ ggufPid: undefined });
+		return pid;
+	}
+
+	private killStale(): void {
+		const pid = readState().ggufPid;
+		if (pid === undefined || pid === process.pid) return;
+		killPid(pid, "SIGTERM");
+		updateState({ ggufPid: undefined });
+	}
+
+	async addModel(
+		reference: string,
+		onProgress: (progress: Progress) => void,
+		signal?: AbortSignal,
+	): Promise<void> {
+		const [repoId, requested] = reference.split(":");
+		onProgress({ label: `Looking up ${repoId}` });
+		const quants = ggufQuants(await repoFiles(repoId, signal));
+		if (quants.length === 0) throw new Error(`${repoId} has no GGUF weights`);
+		const chosen = this.chooseQuant(quants, requested);
+		const modelId = `${repoId}:${chosen.name}`;
+		const before = directorySize(llamaModelsDir());
+		const response = await fetch(this.routerUrl("/models"), {
+			method: "POST",
+			headers: { "Content-Type": "application/json" },
+			body: JSON.stringify({ model: modelId }),
+			signal,
+		});
+		if (!response.ok) throw new Error(`the GGUF engine refused ${modelId}: HTTP ${response.status}`);
+		while (true) {
+			if (signal?.aborted) throw new Error("cancelled");
+			const models = await this.routerModels(signal);
+			const model = models.find((candidate) => candidate.id === modelId);
+			if (model !== undefined && model.status !== "downloading") return;
+			const written = Math.max(0, directorySize(llamaModelsDir()) - before);
+			const fraction = chosen.size > 0 ? Math.min(1, written / chosen.size) : undefined;
+			onProgress({ label: `Downloading ${modelId}`, fraction });
+			await new Promise((resolve) => setTimeout(resolve, DOWNLOAD_POLL_MS));
+		}
+	}
+
+	private chooseQuant(quants: readonly Quantization[], requested: string | undefined): Quantization {
+		if (requested === undefined) return quants[0];
+		const wanted = requested.toUpperCase();
+		const match = quants.find((quant) => quant.name === wanted);
+		if (match === undefined) {
+			throw new Error(`that repository has no ${wanted}. Available: ${quants.map((q) => q.name).join(", ")}`);
+		}
+		return match;
+	}
+
+	private async routerModels(signal?: AbortSignal): Promise<RouterModel[]> {
+		const timeout = AbortSignal.timeout(10_000);
+		const response = await fetch(this.routerUrl("/models"), {
+			signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+		});
+		if (!response.ok) throw new Error(`the GGUF engine returned HTTP ${response.status} for its model list`);
+		return parseRouterModels(await response.json());
+	}
+
+	async listModels(signal?: AbortSignal): Promise<LocalModelInfo[]> {
+		if (this.port === undefined) return [];
+		const baseUrl = this.baseUrl();
+		return (await this.routerModels(signal))
+			.filter((model) => model.status !== "downloading")
+			.map((model) => ({
+				id: model.id,
+				engine: this.id,
+				baseUrl,
+				contextWindow: model.contextWindow,
+				vision: model.vision,
+			}));
+	}
+
+	async stop(): Promise<void> {
+		const child = this.child;
+		const adopted = this.adopted;
+		this.child = undefined;
+		this.port = undefined;
+		this.adopted = false;
+		const remaining = releaseOwnership("ggufOwners", process.pid);
+		if (remaining.length > 0 || adopted) return;
+		if (!child || child.exitCode !== null) return;
+		updateState({ ggufPid: undefined });
+		await stopChild(child, 5_000);
+	}
+}
+
+export { parseRouterModels };

package/runtime/extensions/switchboard-local/extensions/local/engines/llamaBuild.ts

@@ -0,0 +1,62 @@
+export interface BuildChoice {
+	asset: string;
+	accelerator: string;
+}
+
+export interface BuildPlan {
+	preferred: BuildChoice;
+	fallback?: BuildChoice;
+}
+
+export function buildPlan(platform: NodeJS.Platform, arch: string): BuildPlan | null {
+	if (platform === "darwin" && arch === "arm64") {
+		return { preferred: { asset: "macos-arm64.tar.gz", accelerator: "Metal" } };
+	}
+	if (platform === "darwin" && arch === "x64") {
+		return { preferred: { asset: "macos-x64.tar.gz", accelerator: "Metal" } };
+	}
+	if (platform === "win32" && arch === "x64") {
+		return {
+			preferred: { asset: "win-vulkan-x64.zip", accelerator: "Vulkan" },
+			fallback: { asset: "win-cpu-x64.zip", accelerator: "CPU" },
+		};
+	}
+	if (platform === "win32" && arch === "arm64") {
+		return { preferred: { asset: "win-cpu-arm64.zip", accelerator: "CPU" } };
+	}
+	if (platform === "linux" && arch === "x64") {
+		return {
+			preferred: { asset: "ubuntu-vulkan-x64.tar.gz", accelerator: "Vulkan" },
+			fallback: { asset: "ubuntu-x64.tar.gz", accelerator: "CPU" },
+		};
+	}
+	if (platform === "linux" && arch === "arm64") {
+		return {
+			preferred: { asset: "ubuntu-vulkan-arm64.tar.gz", accelerator: "Vulkan" },
+			fallback: { asset: "ubuntu-arm64.tar.gz", accelerator: "CPU" },
+		};
+	}
+	return null;
+}
+
+export function assetName(build: string, choice: BuildChoice): string {
+	return `llama-${build}-bin-${choice.asset}`;
+}
+
+export async function latestBuild(signal?: AbortSignal): Promise<string> {
+	const timeout = AbortSignal.timeout(20_000);
+	const response = await fetch("https://api.github.com/repos/ggml-org/llama.cpp/releases?per_page=1", {
+		headers: { Accept: "application/vnd.github+json" },
+		signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+	});
+	if (!response.ok) throw new Error(`GitHub returned HTTP ${response.status} for llama.cpp releases`);
+	const payload: unknown = await response.json();
+	if (!Array.isArray(payload) || payload.length === 0) throw new Error("no llama.cpp releases were listed");
+	const tag = (payload[0] as { tag_name?: unknown }).tag_name;
+	if (typeof tag !== "string" || !tag) throw new Error("the newest llama.cpp release has no tag");
+	return tag;
+}
+
+export function downloadUrl(build: string, choice: BuildChoice): string {
+	return `https://github.com/ggml-org/llama.cpp/releases/download/${build}/${assetName(build, choice)}`;
+}

package/runtime/extensions/switchboard-local/extensions/local/engines/mlx.ts

@@ -0,0 +1,245 @@
+import { spawn } from "node:child_process";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { ChildProcess } from "node:child_process";
+import type { LocalEngine, LocalModelInfo, Progress } from "../engine.ts";
+import { managedDir, mlxVenvBin, mlxVenvDir, repoCacheDir } from "../paths.ts";
+import { claimOwnership, freePort, isAlive, portIsFree, readState, releaseOwnership, updateState } from "../store.ts";
+import { looksLikeMlxRepo, repoFiles, totalBytes } from "../hf.ts";
+import { detachChild, directorySize, healthy, killOnExit, killPid, run, stopChild, watchSpawn } from "./support.ts";
+
+const HEALTH_TIMEOUT_MS = 120_000;
+const HEALTH_POLL_MS = 250;
+const DOWNLOAD_POLL_MS = 500;
+const DEFAULT_CONTEXT_WINDOW = 32_768;
+
+function findContextWindow(value: unknown): number | undefined {
+	if (typeof value !== "object" || value === null) return undefined;
+	const record = value as Record<string, unknown>;
+	const direct = record.max_position_embeddings;
+	if (typeof direct === "number" && direct > 0) return direct;
+	for (const nested of Object.values(record)) {
+		const found = findContextWindow(nested);
+		if (found !== undefined) return found;
+	}
+	return undefined;
+}
+
+function contextWindowFor(repoId: string): number {
+	const snapshots = join(repoCacheDir(repoId), "snapshots");
+	let revisions: string[];
+	try {
+		revisions = readdirSync(snapshots);
+	} catch {
+		return DEFAULT_CONTEXT_WINDOW;
+	}
+	for (const revision of revisions) {
+		try {
+			const raw = readFileSync(join(snapshots, revision, "config.json"), "utf8");
+			const found = findContextWindow(JSON.parse(raw));
+			if (found !== undefined) return found;
+		} catch {
+			continue;
+		}
+	}
+	return DEFAULT_CONTEXT_WINDOW;
+}
+
+export class MlxEngine implements LocalEngine {
+	readonly id = "mlx" as const;
+	readonly label = "MLX";
+
+	child: ChildProcess | undefined;
+	private port: number | undefined;
+	private starting: Promise<void> | undefined;
+	private exitHookInstalled = false;
+	private adopted = false;
+
+	supported(): boolean {
+		return process.platform === "darwin" && process.arch === "arm64";
+	}
+
+	unsupportedReason(): string {
+		return "MLX runs on Apple Silicon only. Support for other machines arrives with the GGUF engine.";
+	}
+
+	isInstalled(): boolean {
+		return existsSync(mlxVenvBin("mlx_lm.server"));
+	}
+
+	isRunning(): boolean {
+		return this.port !== undefined;
+	}
+
+	baseUrl(): string {
+		if (this.port === undefined) throw new Error("the MLX engine is not running");
+		return `http://127.0.0.1:${this.port}/v1`;
+	}
+
+	async ensureReady(onProgress: (progress: Progress) => void, signal?: AbortSignal): Promise<void> {
+		if (!this.supported()) throw new Error(this.unsupportedReason());
+		if (this.port !== undefined) return;
+		this.starting ??= this.start(onProgress, signal).finally(() => {
+			this.starting = undefined;
+		});
+		await this.starting;
+	}
+
+	private async start(onProgress: (progress: Progress) => void, signal?: AbortSignal): Promise<void> {
+		await this.install(onProgress, signal);
+		const existing = readState().mlxPort;
+		if (existing !== undefined && (await healthy(`http://127.0.0.1:${existing}/health`))) {
+			this.port = existing;
+			this.adopted = true;
+			claimOwnership("mlxOwners", process.pid);
+			this.installExitHook();
+			return;
+		}
+		this.killStale();
+		const port = existing !== undefined && (await portIsFree(existing)) ? existing : await freePort();
+		onProgress({ label: `Starting the MLX engine on port ${port}` });
+		const child = spawn(mlxVenvBin("mlx_lm.server"), ["--host", "127.0.0.1", "--port", String(port)], {
+			cwd: managedDir(),
+			detached: process.platform !== "win32",
+			stdio: ["ignore", "ignore", "pipe"],
+		});
+		let stderr = "";
+		child.stderr?.on("data", (chunk: Buffer) => {
+			stderr = `${stderr}${chunk.toString()}`.slice(-4000);
+		});
+		const spawnFailure = watchSpawn(child);
+		let exited = false;
+		child.once("exit", () => {
+			exited = true;
+			if (this.child === child) {
+				this.child = undefined;
+				this.port = undefined;
+			}
+		});
+		this.child = child;
+		detachChild(child);
+		updateState({ mlxPort: port, mlxPid: child.pid });
+		claimOwnership("mlxOwners", process.pid);
+		this.installExitHook();
+		const deadline = Date.now() + HEALTH_TIMEOUT_MS;
+		while (Date.now() < deadline) {
+			if (signal?.aborted) {
+				child.kill("SIGTERM");
+				throw new Error("cancelled");
+			}
+			if (exited) {
+				const failure = spawnFailure.failure();
+				if (failure !== undefined) throw failure;
+				const tail = stderr.trim().split("\n").slice(-4).join("\n");
+				throw new Error(`the MLX engine stopped while starting${tail ? `\n${tail}` : ""}`);
+			}
+			if (await healthy(`http://127.0.0.1:${port}/health`)) {
+				this.port = port;
+				return;
+			}
+			await new Promise((resolve) => setTimeout(resolve, HEALTH_POLL_MS));
+		}
+		child.kill("SIGTERM");
+		throw new Error("the MLX engine did not start in time");
+	}
+
+	private async install(onProgress: (progress: Progress) => void, signal?: AbortSignal): Promise<void> {
+		if (this.isInstalled()) return;
+		if (!existsSync(mlxVenvBin("python"))) {
+			onProgress({ label: "Creating the Python environment" });
+			await run("python3", ["-m", "venv", mlxVenvDir()], { signal });
+		}
+		onProgress({ label: "Installing mlx-lm, this happens once" });
+		await run(mlxVenvBin("pip"), ["install", "--quiet", "--upgrade", "mlx-lm"], { signal });
+		if (!this.isInstalled()) throw new Error("mlx-lm installed but mlx_lm.server is missing");
+	}
+
+	private releaseAndMaybeKill(): number | undefined {
+		const remaining = releaseOwnership("mlxOwners", process.pid);
+		if (remaining.length > 0) return undefined;
+		const pid = this.child?.pid ?? readState().mlxPid;
+		if (pid === undefined || !isAlive(pid)) return undefined;
+		updateState({ mlxPid: undefined });
+		return pid;
+	}
+
+	private killStale(): void {
+		const pid = readState().mlxPid;
+		if (pid === undefined || pid === process.pid) return;
+		killPid(pid, "SIGTERM");
+		updateState({ mlxPid: undefined });
+	}
+
+	private installExitHook(): void {
+		if (this.exitHookInstalled) return;
+		this.exitHookInstalled = true;
+		killOnExit(() => this.releaseAndMaybeKill());
+	}
+
+	async addModel(
+		repoId: string,
+		onProgress: (progress: Progress) => void,
+		signal?: AbortSignal,
+	): Promise<void> {
+		if (!this.supported()) throw new Error(this.unsupportedReason());
+		onProgress({ label: `Looking up ${repoId}` });
+		const files = await repoFiles(repoId, signal);
+		if (!looksLikeMlxRepo(files)) {
+			throw new Error(
+				`${repoId} does not look like an MLX model. Look for a repository with safetensors weights, such as one under mlx-community.`,
+			);
+		}
+		const total = totalBytes(files);
+		const target = repoCacheDir(repoId);
+		const already = directorySize(target);
+		const download = run(
+			mlxVenvBin("python"),
+			["-c", "import sys;from huggingface_hub import snapshot_download;snapshot_download(sys.argv[1])", repoId],
+			{ signal },
+		);
+		let finished = false;
+		void download.finally(() => {
+			finished = true;
+		});
+		while (!finished) {
+			const current = directorySize(target);
+			const fraction = total > 0 ? Math.min(1, current / total) : undefined;
+			onProgress({ label: `Downloading ${repoId}`, fraction });
+			await new Promise((resolve) => setTimeout(resolve, DOWNLOAD_POLL_MS));
+		}
+		await download;
+		if (already === 0 && directorySize(target) === 0) throw new Error(`nothing was downloaded for ${repoId}`);
+	}
+
+	async listModels(signal?: AbortSignal): Promise<LocalModelInfo[]> {
+		if (this.port === undefined) return [];
+		const timeout = AbortSignal.timeout(10_000);
+		const response = await fetch(`http://127.0.0.1:${this.port}/v1/models`, {
+			signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+		});
+		if (!response.ok) throw new Error(`the MLX engine returned HTTP ${response.status} for its model list`);
+		const payload: unknown = await response.json();
+		const data = (payload as { data?: unknown }).data;
+		if (!Array.isArray(data)) return [];
+		const baseUrl = this.baseUrl();
+		return data.flatMap((value) => {
+			if (typeof value !== "object" || value === null) return [];
+			const id = (value as { id?: unknown }).id;
+			if (typeof id !== "string" || !id.includes("/")) return [];
+			return [{ id, engine: this.id, baseUrl, contextWindow: contextWindowFor(id) }];
+		});
+	}
+
+	async stop(): Promise<void> {
+		const child = this.child;
+		const adopted = this.adopted;
+		this.child = undefined;
+		this.port = undefined;
+		this.adopted = false;
+		const remaining = releaseOwnership("mlxOwners", process.pid);
+		if (remaining.length > 0 || adopted) return;
+		if (!child || child.exitCode !== null) return;
+		updateState({ mlxPid: undefined });
+		await stopChild(child, 3_000);
+	}
+}

package/runtime/extensions/switchboard-local/extensions/local/engines/select.ts

@@ -0,0 +1,38 @@
+import type { LocalEngine } from "../engine.ts";
+import { ggufQuants, looksLikeMlxRepo, parseModelRef, repoFiles } from "../hf.ts";
+
+export interface EngineChoice {
+	engine: LocalEngine;
+	reference: string;
+}
+
+export async function chooseEngine(
+	engines: readonly LocalEngine[],
+	input: string,
+	signal?: AbortSignal,
+): Promise<EngineChoice> {
+	const ref = parseModelRef(input);
+	if (ref === null) {
+		throw new Error(`"${input}" is not a Hugging Face model. Use a link, or owner/repository[:quant].`);
+	}
+	const byId = (id: string) => engines.find((engine) => engine.id === id);
+	if (ref.quant !== undefined) {
+		return { engine: usable(byId("gguf"), "GGUF"), reference: `${ref.repoId}:${ref.quant}` };
+	}
+	const files = await repoFiles(ref.repoId, signal);
+	if (ggufQuants(files).length > 0) {
+		return { engine: usable(byId("gguf"), "GGUF"), reference: ref.repoId };
+	}
+	if (looksLikeMlxRepo(files)) {
+		return { engine: usable(byId("mlx"), "MLX"), reference: ref.repoId };
+	}
+	throw new Error(
+		`${ref.repoId} has neither GGUF weights nor MLX safetensors, so nothing here can run it.`,
+	);
+}
+
+function usable(engine: LocalEngine | undefined, label: string): LocalEngine {
+	if (engine === undefined) throw new Error(`the ${label} engine is not available`);
+	if (!engine.supported()) throw new Error(engine.unsupportedReason());
+	return engine;
+}