@xynogen/pix-runtime 0.10.2 → 0.13.0

package/src/binaries/catalog.ts

@@ -0,0 +1,279 @@
+/**
+ * catalog.ts — every external command a pix package runs.
+ *
+ * One entry per executable: who uses it, where it matters, how to get it, and
+ * (for a few) a trusted GitHub release pix may download into `<agentDir>/bin`.
+ * User-defined commands (LSP servers, proc commands, skill scripts) are not
+ * catalogued; `which.ts` keeps serving those.
+ */
+
+import { join } from "node:path";
+import type { HostOs, HostPlatform } from "../platform.ts";
+
+export interface DownloadRecipe {
+	/** GitHub `owner/repo`. The latest release is found via the `/releases/latest` redirect. */
+	repo: string;
+	/** Release asset for this host and version (tag minus `v`/`release-`), or undefined if none. */
+	asset(host: HostPlatform, version: string): string | undefined;
+	/** Checksum manifest asset (`<sha256>  <name>` lines). Verified when set. */
+	checksums?: string;
+	/** Rough archive size shown before download. */
+	size?: string;
+}
+
+export interface BinarySpec {
+	/** Packages that run it. */
+	usedBy: readonly string[];
+	/** OSes where it is used. */
+	os: readonly HostOs[];
+	/** On Linux, used only under WSL (Windows interop). */
+	wslOnly?: boolean;
+	/** Alternative executable names tried in order (defaults to the key). */
+	names?: readonly string[];
+	/** Install hint per OS; `default` otherwise. */
+	hint: Partial<Record<HostOs | "default", string>>;
+	/** Nice-to-have: absence is informational, not an error. */
+	optional?: boolean;
+	/**
+	 * Standard install locations checked after `<agentDir>/bin` and before PATH
+	 * (e.g. Git Bash, which Git for Windows keeps off PATH). Empty values skipped.
+	 */
+	knownPaths?(env: NodeJS.ProcessEnv, host: HostPlatform): readonly string[];
+	/** Args printing a version, or null when the tool has no safe version probe. */
+	versionArgs?: readonly string[] | null;
+	download?: DownloadRecipe;
+}
+
+const ALL: readonly HostOs[] = ["win32", "linux", "darwin", "android"];
+const UNIX: readonly HostOs[] = ["linux", "darwin", "android"];
+const LINUX: readonly HostOs[] = ["linux"];
+
+/** Env lookup ignoring case: a copied Windows env loses process.env's case-insensitivity. */
+function envVar(env: NodeJS.ProcessEnv, key: string): string | undefined {
+	if (env[key]) return env[key];
+	const k = Object.keys(env).find((x) => x.toLowerCase() === key.toLowerCase());
+	return k ? env[k] : undefined;
+}
+
+const rustTriple = (h: HostPlatform): string | undefined => {
+	if (h.os === "win32") return "x86_64-pc-windows-msvc"; // x64 build also runs on Windows arm64
+	if (h.os === "darwin") return `${h.arch === "arm64" ? "aarch64" : "x86_64"}-apple-darwin`;
+	if (h.os !== "linux") return undefined;
+	if (h.arch === "x64") return "x86_64-unknown-linux-musl";
+	return h.libc === "glibc" ? "aarch64-unknown-linux-gnu" : undefined;
+};
+
+export const CATALOG = {
+	rtk: {
+		usedBy: ["pix-optimizer"],
+		os: ALL,
+		hint: {
+			win32: "winget install rtk-ai.rtk",
+			default: "download from github.com/rtk-ai/rtk/releases or `cargo install rtk`",
+		},
+		download: {
+			repo: "rtk-ai/rtk",
+			checksums: "checksums.txt",
+			size: "~5 MB",
+			asset: (h) => {
+				const t = rustTriple(h);
+				if (!t) return undefined;
+				return `rtk-${t}.${h.os === "win32" ? "zip" : "tar.gz"}`;
+			},
+		},
+	},
+	hunk: {
+		usedBy: ["pix-hunk"],
+		os: ALL,
+		hint: { default: "npm install -g hunkdiff" },
+		download: {
+			repo: "modem-dev/hunk",
+			checksums: "SHA256SUMS",
+			size: "~40 MB",
+			asset: (h) => {
+				if (h.os === "win32") return "hunkdiff-windows-x64.tar.gz";
+				if (h.os === "linux" || h.os === "darwin") return `hunkdiff-${h.os}-${h.arch}.tar.gz`;
+				return undefined;
+			},
+		},
+	},
+	aria2c: {
+		usedBy: ["pix-aria2"],
+		os: ALL,
+		hint: {
+			linux: "apt install aria2 · dnf install aria2 · pacman -S aria2",
+			darwin: "brew install aria2",
+			android: "pkg install aria2",
+			win32: "winget install aria2.aria2",
+			default: "download from github.com/aria2/aria2/releases",
+		},
+		download: {
+			repo: "aria2/aria2",
+			size: "~2.4 MB",
+			asset: (h, v) => (h.os === "win32" ? `aria2-${v}-win-64bit-build1.zip` : undefined),
+		},
+	},
+	ffmpeg: {
+		usedBy: ["pix-voice"],
+		os: ["linux", "win32", "darwin"],
+		versionArgs: ["-version"],
+		hint: {
+			linux: "apt install ffmpeg · dnf install ffmpeg · pacman -S ffmpeg",
+			darwin: "brew install ffmpeg",
+			win32: "winget install Gyan.FFmpeg",
+			default: "install ffmpeg",
+		},
+		download: {
+			repo: "BtbN/FFmpeg-Builds",
+			checksums: "checksums.sha256",
+			size: "~120 MB",
+			asset: (h) => {
+				const arm = h.arch === "arm64";
+				if (h.os === "win32") return `ffmpeg-master-latest-${arm ? "winarm64" : "win64"}-lgpl.zip`;
+				if (h.os === "linux")
+					return `ffmpeg-master-latest-${arm ? "linuxarm64" : "linux64"}-lgpl.tar.xz`;
+				return undefined;
+			},
+		},
+	},
+	rg: {
+		usedBy: ["pix-search", "pix-grep", "pix-graph"],
+		os: ALL,
+		hint: {
+			android: "pkg install ripgrep",
+			win32: "restart Pi (it downloads rg) or winget install BurntSushi.ripgrep.MSVC",
+			default: "restart Pi (it downloads rg) or install ripgrep",
+		},
+	},
+	git: {
+		usedBy: ["pix-footer", "pix-welcome", "pix-subagent", "pix-search"],
+		os: ALL,
+		hint: {
+			win32: "winget install Git.Git",
+			darwin: "xcode-select --install",
+			default: "install git",
+		},
+	},
+	ssh: {
+		usedBy: ["pix-ssh"],
+		os: ALL,
+		versionArgs: ["-V"],
+		hint: {
+			win32:
+				"enable the OpenSSH Client optional feature or winget install Microsoft.OpenSSH.Preview",
+			default: "install openssh-client",
+		},
+	},
+	scp: {
+		usedBy: ["pix-ssh"],
+		os: ALL,
+		versionArgs: null,
+		hint: {
+			win32:
+				"enable the OpenSSH Client optional feature or winget install Microsoft.OpenSSH.Preview",
+			default: "install openssh-client",
+		},
+	},
+	sshpass: {
+		usedBy: ["pix-ssh"],
+		os: UNIX,
+		versionArgs: ["-V"],
+		hint: { darwin: "brew install sshpass", default: "install sshpass" },
+	},
+	sudo: { usedBy: ["pix-sudo"], os: UNIX, hint: { default: "install sudo" } },
+	"wl-paste": { usedBy: ["pix-ask"], os: LINUX, hint: { default: "install wl-clipboard" } },
+	xclip: {
+		usedBy: ["pix-ask"],
+		os: LINUX,
+		versionArgs: ["-version"],
+		hint: { default: "install xclip" },
+	},
+	wslpath: {
+		usedBy: ["pix-ask"],
+		os: LINUX,
+		wslOnly: true,
+		versionArgs: null,
+		hint: { default: "WSL only" },
+	},
+	open: {
+		usedBy: ["pix-mcp"],
+		os: ["darwin"],
+		versionArgs: null,
+		hint: { default: "built into macOS" },
+	},
+	"xdg-open": { usedBy: ["pix-mcp"], os: LINUX, hint: { default: "install xdg-utils" } },
+	cmd: {
+		usedBy: ["pix-mcp"],
+		os: ["win32"],
+		versionArgs: null,
+		knownPaths: (env) => [`${envVar(env, "SystemRoot") ?? "C:\\Windows"}\\System32\\cmd.exe`],
+		hint: { default: "built into Windows" },
+	},
+	npm: {
+		usedBy: ["pix-mcp"],
+		os: ALL,
+		hint: { win32: "winget install OpenJS.NodeJS.LTS", default: "install Node.js" },
+	},
+	npx: {
+		usedBy: ["pix-mcp"],
+		os: ALL,
+		hint: { win32: "winget install OpenJS.NodeJS.LTS", default: "install Node.js" },
+	},
+	pi: {
+		usedBy: ["pix-update", "pix-models", "pix-welcome"],
+		os: ALL,
+		hint: { default: "npm install -g @earendil-works/pi-coding-agent" },
+	},
+	powershell: {
+		usedBy: ["pix-powershell", "pix-ask", "pix-voice"],
+		os: ["win32", "linux"],
+		wslOnly: true,
+		// WSL reaches Windows PowerShell through interop as powershell.exe.
+		names: ["powershell", "powershell.exe"],
+		versionArgs: null,
+		hint: { linux: "WSL only (Windows interop)", default: "built into Windows" },
+	},
+	wslview: {
+		usedBy: ["pix-mcp"],
+		os: LINUX,
+		wslOnly: true,
+		optional: true,
+		versionArgs: null,
+		hint: { default: "WSL only: install wslu" },
+	},
+	bash: {
+		usedBy: ["pix-bash", "pix-mcp"],
+		os: ALL,
+		hint: { win32: "winget install Git.Git (Git Bash)", default: "install bash" },
+		// Same locations Pi searches: Git for Windows puts only Git\cmd on PATH.
+		knownPaths: (env, h) =>
+			h.os === "win32"
+				? [envVar(env, "ProgramFiles"), envVar(env, "ProgramFiles(x86)")]
+						.filter((d): d is string => !!d)
+						// Native join: backslashes on real Windows, and the fake-win32 test runs anywhere.
+						.map((d) => join(d, "Git", "bin", "bash.exe"))
+				: [],
+	},
+} satisfies Record<string, BinarySpec>;
+
+export type BinaryName = keyof typeof CATALOG;
+
+export const BINARY_NAMES = Object.keys(CATALOG).sort() as BinaryName[];
+
+export function specOf(name: string): BinarySpec | undefined {
+	return (CATALOG as Record<string, BinarySpec>)[name];
+}
+
+export function hintFor(spec: BinarySpec, host: HostPlatform): string {
+	return spec.hint[host.os] ?? spec.hint.default ?? "";
+}
+
+/** Release asset pix would download on this host, or undefined when it only checks. */
+export function downloadAsset(
+	spec: BinarySpec,
+	host: HostPlatform,
+	version = "0",
+): string | undefined {
+	if (!spec.download || host.termux) return undefined;
+	return spec.download.asset(host, version);
+}

package/src/binaries/ensure.ts

@@ -0,0 +1,294 @@
+/**
+ * ensure.ts — resolve a binary, downloading it into `<agentDir>/bin` when the
+ * catalog has a trusted release for this host.
+ *
+ * Mirrors Pi's own fd/rg downloader (latest-release redirect, system tar/unzip,
+ * unique temp dir + rename) and adds SHA-256 verification when the release
+ * publishes a checksum manifest. Reports progress only through `onStatus`;
+ * callers decide where it appears (runtime cannot import UI code).
+ */
+
+import { spawnSync } from "node:child_process";
+import { createHash, randomBytes } from "node:crypto";
+import {
+	chmodSync,
+	createWriteStream,
+	existsSync,
+	mkdirSync,
+	readdirSync,
+	renameSync,
+	rmSync,
+} from "node:fs";
+import { join } from "node:path";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { binDir } from "../paths.ts";
+import { currentPlatform, type HostPlatform } from "../platform.ts";
+import { type BinaryName, type BinarySpec, downloadAsset, hintFor, specOf } from "./catalog.ts";
+import { BinaryMissingError, lookupTool, type ResolvedTool } from "./resolve.ts";
+
+export type ToolStatus =
+	| { kind: "downloading"; name: string; version: string; url: string; size?: string }
+	| { kind: "installed"; name: string; path: string; version: string; verified: boolean }
+	| { kind: "failed"; name: string; error: string; hint: string };
+
+export interface EnsureOptions {
+	onStatus?: (status: ToolStatus) => void;
+	signal?: AbortSignal;
+	env?: NodeJS.ProcessEnv;
+	host?: HostPlatform;
+	/** Injected fetch for tests. */
+	fetch?: typeof fetch;
+}
+
+const LOOKUP_TIMEOUT_MS = 15_000;
+const DOWNLOAD_TIMEOUT_MS = 10 * 60_000;
+const USER_AGENT = "pix-runtime";
+
+export function isOffline(env: NodeJS.ProcessEnv = process.env): boolean {
+	const v = env.PI_OFFLINE?.toLowerCase();
+	return v === "1" || v === "true" || v === "yes";
+}
+
+const inflight = new Map<string, Promise<ResolvedTool>>();
+
+/**
+ * Resolve `name`; if missing and downloadable here, download it. Concurrent
+ * calls for one name share a single download. Throws {@link BinaryMissingError}.
+ */
+export function ensureTool(
+	name: BinaryName | (string & {}),
+	opts: EnsureOptions = {},
+): Promise<ResolvedTool> {
+	const hit = lookupTool(name, { env: opts.env, host: opts.host });
+	if (hit.state === "ok" && hit.path && hit.source) {
+		return Promise.resolve({ name, path: hit.path, source: hit.source });
+	}
+	if (hit.state === "broken")
+		return Promise.reject(new BinaryMissingError(name, "broken", hit.hint, hit.error));
+	const spec = specOf(name);
+	const host = opts.host ?? currentPlatform();
+	if (!spec || !downloadAsset(spec, host)) {
+		return Promise.reject(new BinaryMissingError(name, hit.state, hit.hint));
+	}
+	if (isOffline(opts.env)) {
+		return Promise.reject(
+			new BinaryMissingError(name, "missing", hit.hint, "PI_OFFLINE is set; skipping download"),
+		);
+	}
+	const key = `${binDir(opts.env)}::${name}`;
+	const running = inflight.get(key);
+	if (running) return running;
+	const job = installFromRelease(name, spec, host, opts).finally(() => inflight.delete(key));
+	inflight.set(key, job);
+	return job;
+}
+
+/** Download + verify + extract one catalog recipe into binDir. Exported for tests. */
+export async function installFromRelease(
+	name: string,
+	spec: BinarySpec,
+	host: HostPlatform,
+	opts: EnsureOptions = {},
+): Promise<ResolvedTool> {
+	const recipe = spec.download;
+	const hint = hintFor(spec, host);
+	const doFetch = opts.fetch ?? fetch;
+	const fail = (error: unknown): never => {
+		const message = error instanceof Error ? error.message : String(error);
+		opts.onStatus?.({ kind: "failed", name, error: message, hint });
+		throw new BinaryMissingError(name, "missing", hint, `download failed: ${message}`);
+	};
+	if (!recipe) return fail("no download recipe");
+
+	const bin = binDir(opts.env);
+	const exeName = `${name}${host.exe}`;
+	const target = join(bin, exeName);
+	const work = join(bin, `.pix-dl-${name}-${process.pid}-${randomBytes(4).toString("hex")}`);
+	try {
+		const tag = await latestTag(recipe.repo, doFetch, opts.signal);
+		const version = tag.replace(/^(v|release-)/, "");
+		const asset = recipe.asset(host, version);
+		if (!asset) return fail(`no ${recipe.repo} asset for ${host.os}/${host.arch}`);
+		const base = `https://github.com/${recipe.repo}/releases/download/${tag}`;
+		const url = `${base}/${asset}`;
+		opts.onStatus?.({ kind: "downloading", name, version, url, size: recipe.size });
+
+		mkdirSync(work, { recursive: true });
+		const archive = join(work, asset);
+		const sha256 = await download(url, archive, doFetch, opts.signal);
+		// A recipe that declares a manifest must verify. A missing manifest fails the
+		// install, so a renamed or removed file cannot turn verification off quietly.
+		const verified = recipe.checksums !== undefined;
+		if (recipe.checksums) {
+			const expected = await expectedChecksum(
+				`${base}/${recipe.checksums}`,
+				asset,
+				doFetch,
+				opts.signal,
+			);
+			if (expected !== sha256)
+				throw new Error(`checksum mismatch for ${asset} (expected ${expected}, got ${sha256})`);
+		}
+		const out = join(work, "x");
+		mkdirSync(out);
+		extract(archive, out, host);
+		const found = findFile(out, exeName);
+		if (!found) throw new Error(`${exeName} not found in ${asset}`);
+		if (host.os !== "win32") chmodSync(found, 0o755);
+		mkdirSync(bin, { recursive: true });
+		try {
+			renameSync(found, target);
+		} catch (err) {
+			// A concurrent installer (another Pi process) may have won the rename,
+			// or Windows refuses to replace an in-use exe; either way a binary exists.
+			if (!existsSync(target)) throw err;
+		}
+		opts.onStatus?.({ kind: "installed", name, path: target, version, verified });
+		return { name, path: target, source: "download", version };
+	} catch (err) {
+		if (err instanceof BinaryMissingError) throw err;
+		return fail(err);
+	} finally {
+		rmSync(work, { recursive: true, force: true });
+	}
+}
+
+function withTimeout(ms: number, signal?: AbortSignal): AbortSignal {
+	const t = AbortSignal.timeout(ms);
+	return signal ? AbortSignal.any([signal, t]) : t;
+}
+
+/** Latest release tag via the `/releases/latest` redirect (no API quota). */
+async function latestTag(
+	repo: string,
+	doFetch: typeof fetch,
+	signal?: AbortSignal,
+): Promise<string> {
+	const res = await doFetch(`https://github.com/${repo}/releases/latest`, {
+		redirect: "manual",
+		headers: { "User-Agent": USER_AGENT },
+		signal: withTimeout(LOOKUP_TIMEOUT_MS, signal),
+	});
+	await res.body?.cancel().catch(() => {});
+	const location = res.status >= 300 && res.status < 400 ? res.headers.get("location") : null;
+	if (!location?.includes("/releases/tag/")) {
+		throw new Error(`cannot resolve latest ${repo} release (HTTP ${res.status})`);
+	}
+	const tag = new URL(location, "https://github.com").pathname.split("/").pop();
+	if (!tag) throw new Error(`unexpected redirect for ${repo}: ${location}`);
+	return decodeURIComponent(tag);
+}
+
+/** Stream `url` to `dest`, returning its SHA-256 hex. */
+async function download(
+	url: string,
+	dest: string,
+	doFetch: typeof fetch,
+	signal?: AbortSignal,
+): Promise<string> {
+	const res = await doFetch(url, {
+		headers: { "User-Agent": USER_AGENT },
+		signal: withTimeout(DOWNLOAD_TIMEOUT_MS, signal),
+	});
+	if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} for ${url}`);
+	const hash = createHash("sha256");
+	const tap = new Transform({
+		transform(chunk: Buffer, _enc, cb) {
+			hash.update(chunk);
+			cb(null, chunk);
+		},
+	});
+	await pipeline(
+		Readable.fromWeb(res.body as import("node:stream/web").ReadableStream),
+		tap,
+		createWriteStream(dest),
+	);
+	return hash.digest("hex");
+}
+
+/** Expected hash for `asset` from a `<sha256>  <name>` manifest. Throws when the manifest or entry is missing. */
+async function expectedChecksum(
+	url: string,
+	asset: string,
+	doFetch: typeof fetch,
+	signal?: AbortSignal,
+): Promise<string> {
+	const res = await doFetch(url, {
+		headers: { "User-Agent": USER_AGENT },
+		signal: withTimeout(LOOKUP_TIMEOUT_MS, signal),
+	});
+	if (res.status === 404) {
+		await res.body?.cancel().catch(() => {});
+		throw new Error(`checksum manifest missing (HTTP 404): ${url}`);
+	}
+	if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
+	const text = await res.text();
+	for (const line of text.split(/\r?\n/)) {
+		const m = line.trim().match(/^([0-9a-f]{64})\s+\*?(.+)$/i);
+		if (m?.[1] && m[2]?.trim() === asset) return m[1].toLowerCase();
+	}
+	throw new Error(`${asset} is not listed in the release checksums`);
+}
+
+function run(cmd: string, args: string[]): string | null {
+	const r = spawnSync(cmd, args, { stdio: "pipe", windowsHide: true });
+	if (!r.error && r.status === 0) return null;
+	const msg = r.error?.message || r.stderr?.toString().trim() || `exit ${r.status}`;
+	return `${cmd}: ${msg}`;
+}
+
+function windowsTar(): string {
+	const root = process.env.SystemRoot ?? process.env.WINDIR;
+	const sys = root ? join(root, "System32", "tar.exe") : "";
+	return sys && existsSync(sys) ? sys : "tar.exe";
+}
+
+/** Extract with system tools only (same approach as Pi): no npm deps. */
+function extract(archive: string, dest: string, host: HostPlatform): void {
+	const failures: string[] = [];
+	const attempt = (cmd: string, args: string[]): boolean => {
+		const err = run(cmd, args);
+		if (err) failures.push(err);
+		return !err;
+	};
+	if (host.os === "win32") {
+		if (attempt(windowsTar(), ["xf", archive, "-C", dest])) return;
+		if (archive.endsWith(".zip")) {
+			const ps =
+				"& { param($a, $d) $ErrorActionPreference = 'Stop'; Expand-Archive -LiteralPath $a -DestinationPath $d -Force }";
+			if (
+				attempt("powershell.exe", [
+					"-NoLogo",
+					"-NoProfile",
+					"-NonInteractive",
+					"-Command",
+					ps,
+					archive,
+					dest,
+				])
+			)
+				return;
+		}
+	} else if (archive.endsWith(".zip")) {
+		if (attempt("unzip", ["-q", archive, "-d", dest])) return;
+		if (attempt("tar", ["xf", archive, "-C", dest])) return;
+	} else {
+		const flag = archive.endsWith(".tar.xz") ? "xJf" : "xzf";
+		if (attempt("tar", [flag, archive, "-C", dest])) return;
+	}
+	throw new Error(`cannot extract ${archive}: ${failures.join("; ")}`);
+}
+
+function findFile(root: string, fileName: string): string | undefined {
+	const stack = [root];
+	while (stack.length) {
+		const dir = stack.pop() as string;
+		for (const entry of readdirSync(dir, { withFileTypes: true })) {
+			const full = join(dir, entry.name);
+			if (entry.isFile() && entry.name === fileName) return full;
+			if (entry.isDirectory()) stack.push(full);
+		}
+	}
+	return undefined;
+}

package/src/binaries/index.ts

@@ -0,0 +1,43 @@
+/**
+ * @xynogen/pix-runtime/binaries — one catalog, resolver, and downloader for
+ * every external command pix packages run. See README "Binaries".
+ */
+
+export {
+	BINARY_NAMES,
+	type BinaryName,
+	type BinarySpec,
+	CATALOG,
+	type DownloadRecipe,
+	downloadAsset,
+	hintFor,
+	specOf,
+} from "./catalog.ts";
+export {
+	type EnsureOptions,
+	ensureTool,
+	installFromRelease,
+	isOffline,
+	type ToolStatus,
+} from "./ensure.ts";
+export {
+	BinaryMissingError,
+	type LookupOptions,
+	listTools,
+	lookupTool,
+	type ResolvedTool,
+	requireTool,
+	resolveTool,
+	type ToolLookup,
+	type ToolSource,
+	type ToolState,
+	toolVersion,
+} from "./resolve.ts";
+export {
+	type BinaryChoices,
+	type BinaryStoreState,
+	binaryFilePath,
+	readBinaryStore,
+	setBinaryChoice,
+	syncBinaryStore,
+} from "./store.ts";