@danypops/pi-packed 0.19.7 → 0.19.10

package/service/src/adoption/commit-freshness.ts

@@ -0,0 +1,167 @@
+/**
+ * commit-freshness.ts — real commit-date lookups backing the `freshness`
+ * adoption dimension (score.ts). Two independent, best-effort sources:
+ * a local git checkout (free, no network) and GitHub's Commits API for a
+ * pre-install registry candidate (bounded, single attempt, GitHub-only).
+ * Both resolve to undefined on any failure -- never throw, never guess.
+ */
+import { runBounded } from "../publish/publish.ts";
+import {
+	GITHUB_MAX_TOTAL_BACKOFF_MS,
+	GITHUB_RETRY_MAX_ATTEMPTS,
+	GITHUB_SECONDARY_RATE_LIMIT_FALLBACK_MS,
+	GITHUB_TRANSIENT_BASE_DELAY_MS,
+} from "../shared/constants.ts";
+import { createLogger } from "../shared/log.ts";
+
+const log = createLogger("commit-freshness");
+
+const GITHUB_API_BASE = "https://api.github.com";
+const GITHUB_COMMITS_TIMEOUT_MS = 8_000;
+/** Matches github.com in every form Pi packages actually declare it in:
+ * https://, git+https://, git://, git@host:path (SSH shorthand), with or
+ * without a trailing .git. Any other host (GitLab, Bitbucket, sourcehut --
+ * all confirmed present in the real pi-extension ecosystem) never matches,
+ * so it never reaches the network call below. */
+const GITHUB_REPO_PATTERN = /^(?:git\+)?(?:https?:\/\/|git:\/\/|git@)(?:www\.)?github\.com[/:]([^/]+)\/([^/#?]+?)(?:\.git)?(?:[/#?].*)?$/i;
+
+function parseGithubRepo(repository: string | undefined): { owner: string; repo: string } | undefined {
+	if (!repository) return undefined;
+	const match = GITHUB_REPO_PATTERN.exec(repository.trim());
+	return match?.[1] && match[2] ? { owner: match[1], repo: match[2] } : undefined;
+}
+
+/** Seconds to wait before the next attempt, or undefined when the response
+ * isn't a rate-limit signal at all (a plain success or an unrelated error).
+ * Reads GitHub's two real signals -- confirmed against GitHub's own docs --
+ * never guesses a wait time neither header supports. */
+function githubRetryAfterSeconds(res: Response): number | undefined {
+	const retryAfter = res.headers.get("retry-after");
+	if (retryAfter !== null) {
+		const seconds = Number(retryAfter);
+		return Number.isFinite(seconds) && seconds > 0 ? seconds : GITHUB_SECONDARY_RATE_LIMIT_FALLBACK_MS / 1000;
+	}
+	if (res.headers.get("x-ratelimit-remaining") === "0") {
+		const reset = Number(res.headers.get("x-ratelimit-reset"));
+		if (Number.isFinite(reset)) return Math.max(0, reset - Date.now() / 1000);
+	}
+	return undefined;
+}
+
+/**
+ * Self-throttling wrapper for the single-candidate GitHub Commits API call:
+ * a rate-limit or transient failure backs off and retries rather than
+ * giving up on the first hiccup, but the total added wait across every
+ * retry is capped at GITHUB_MAX_TOTAL_BACKOFF_MS -- a real primary-limit
+ * exhaustion (X-RateLimit-Reset up to an hour away) is treated as
+ * immediately exhausted rather than blocking an interactive `packed score`
+ * call for anywhere near that long. "Exhaustion" means either the retry
+ * count or the total-wait budget runs out, whichever comes first; the
+ * caller then gets undefined, the same fail-open contract as before this
+ * task, never a thrown error.
+ */
+async function fetchGithubWithBackoff(url: string, init: RequestInit, timeoutMs: number): Promise<Response | undefined> {
+	let totalWaitMs = 0;
+	for (let attempt = 1; attempt <= GITHUB_RETRY_MAX_ATTEMPTS; attempt++) {
+		// Fresh per-attempt timeout, same as registry.ts's fetchWithRetry --
+		// reusing one AbortSignal.timeout() across retries would have it fire
+		// mid-backoff-sleep instead of timing each attempt independently.
+		const signal = init.signal ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs);
+		let res: Response;
+		try {
+			res = await fetch(url, { ...init, signal });
+		} catch (e) {
+			if (attempt === GITHUB_RETRY_MAX_ATTEMPTS) return undefined;
+			const delayMs = GITHUB_TRANSIENT_BASE_DELAY_MS * 2 ** (attempt - 1);
+			if (totalWaitMs + delayMs > GITHUB_MAX_TOTAL_BACKOFF_MS) return undefined;
+			totalWaitMs += delayMs;
+			log.warn("network error, retrying", { attempt, delayMs, error: e instanceof Error ? e.message : String(e) });
+			await Bun.sleep(delayMs);
+			continue;
+		}
+		if (res.ok) return res;
+		const rateLimitWaitSeconds = githubRetryAfterSeconds(res);
+		const isRetryable = rateLimitWaitSeconds !== undefined || res.status >= 500;
+		if (!isRetryable || attempt === GITHUB_RETRY_MAX_ATTEMPTS) return res;
+		const delayMs = rateLimitWaitSeconds !== undefined ? rateLimitWaitSeconds * 1000 : GITHUB_TRANSIENT_BASE_DELAY_MS * 2 ** (attempt - 1);
+		if (totalWaitMs + delayMs > GITHUB_MAX_TOTAL_BACKOFF_MS) {
+			log.warn("rate-limit wait exceeds total backoff budget, giving up", { attempt, delayMs, status: res.status });
+			return res;
+		}
+		totalWaitMs += delayMs;
+		log.warn("rate-limited, backing off", { attempt, delayMs, status: res.status });
+		await Bun.sleep(delayMs);
+	}
+	return undefined;
+}
+
+function packedUserAgent(): string {
+	const runtime = process.versions.bun ? `bun/${process.versions.bun}` : `node/${process.version}`;
+	return `packed (${process.platform}; ${runtime}; ${process.arch})`;
+}
+
+/**
+ * Reads a checkout's real last-commit timestamp via `git log`, scoped to
+ * `directory` when the package lives in a monorepo subdirectory. Bounded
+ * single call (runBounded's own timeout), no network at all. Undefined
+ * for a missing git binary, a non-git directory, or any parse failure --
+ * never a guess.
+ */
+export async function lastLocalCommitAt(root: string, directory?: string): Promise<string | undefined> {
+	const args = ["git", "-C", root, "log", "-1", "--format=%cI"];
+	if (directory) args.push("--", directory);
+	const result = await runBounded(args).catch((): { code: number; stdout: string; stderr: string } => ({
+		code: 1,
+		stdout: "",
+		stderr: "",
+	}));
+	if (result.code !== 0) return undefined;
+	const date = result.stdout.trim();
+	return date && Number.isFinite(Date.parse(date)) ? date : undefined;
+}
+
+export type FetchGithubLastCommitAt = (
+	repository: string | undefined,
+	directory?: string,
+	timeoutMs?: number,
+) => Promise<string | undefined>;
+
+/**
+ * Bounded, self-throttling, GitHub-only commit-date lookup for a candidate
+ * that hasn't been cloned yet (pre-install registry scoring). Any
+ * non-GitHub host, or a missing repository field, short-circuits to
+ * undefined with zero network calls. A transient failure or a short
+ * rate-limit wait is retried (see fetchGithubWithBackoff); a real
+ * primary-limit exhaustion or total-backoff-budget exhaustion still
+ * resolves to undefined rather than blocking. Never intended for a bulk
+ * sweep across every installed package -- see index/build-index.ts's own
+ * doc comment for why bulk generation never calls this function at all,
+ * regardless of how resilient a single call now is. `baseUrl` is
+ * test-only, matching this codebase's Bun.serve() HTTP fixture convention
+ * (see pi-version.ts's createFetchLatestPiRelease).
+ */
+export function createGithubLastCommitAt(baseUrl: string = GITHUB_API_BASE): FetchGithubLastCommitAt {
+	return async (repository, directory, timeoutMs = GITHUB_COMMITS_TIMEOUT_MS) => {
+		const parsed = parseGithubRepo(repository);
+		if (!parsed) return undefined;
+		const params = new URLSearchParams({ per_page: "1" });
+		if (directory) params.set("path", directory);
+		try {
+			const res = await fetchGithubWithBackoff(
+				`${baseUrl}/repos/${parsed.owner}/${parsed.repo}/commits?${params}`,
+				{
+					headers: { accept: "application/vnd.github+json", "user-agent": packedUserAgent() },
+				},
+				timeoutMs,
+			);
+			if (!res?.ok) return undefined;
+			const commits = (await res.json()) as Array<{ commit?: { committer?: { date?: unknown }; author?: { date?: unknown } } }>;
+			const date = commits[0]?.commit?.committer?.date ?? commits[0]?.commit?.author?.date;
+			return typeof date === "string" && Number.isFinite(Date.parse(date)) ? date : undefined;
+		} catch {
+			return undefined;
+		}
+	};
+}
+
+export const githubLastCommitAt: FetchGithubLastCommitAt = createGithubLastCommitAt();

package/service/src/adoption/doctor.ts

@@ -0,0 +1,135 @@
+/**
+ * doctor.ts — unions every currently-configured package's declared,
+ * enabled extensions across BOTH global and project scope, smoke-tests
+ * each in isolation, and reports any tool/command/shortcut/flag name
+ * claimed by more than one. Pi's own extension loader only discovers this
+ * kind of collision at actual startup, one package at a time; doctor
+ * answers the same question proactively, before pi ever runs.
+ */
+import { join } from "node:path";
+import { listPackageResources, type PackageResources, resolveInstalledDir } from "../packages/resources.ts";
+import { runExtensionSmoke, type SmokeOptions, type SmokeRegistrations } from "./smoke.ts";
+
+const REGISTRATION_KINDS = ["tools", "commands", "shortcuts", "flags"] as const;
+type RegistrationKind = (typeof REGISTRATION_KINDS)[number];
+const CONFLICT_KIND: Record<RegistrationKind, "tool" | "command" | "shortcut" | "flag"> = {
+	tools: "tool",
+	commands: "command",
+	shortcuts: "shortcut",
+	flags: "flag",
+};
+
+const MAX_EXTENSIONS_SCANNED = 50;
+
+export interface DoctorClaim {
+	name: string;
+	source: string;
+	scope: "global" | "project";
+	extension: string;
+}
+
+export interface DoctorConflict {
+	kind: "tool" | "command" | "shortcut" | "flag";
+	name: string;
+	claimants: DoctorClaim[];
+}
+
+export interface DoctorExtensionResult {
+	name: string;
+	source: string;
+	scope: "global" | "project";
+	extension: string;
+	status: string;
+	registrations: SmokeRegistrations;
+	message?: string;
+}
+
+export interface DoctorReport {
+	ok: boolean;
+	conflicts: DoctorConflict[];
+	extensions: DoctorExtensionResult[];
+	scanned: number;
+	truncated: boolean;
+}
+
+interface ScanJob {
+	group: PackageResources;
+	dir: string;
+	extension: string;
+}
+
+function collectJobs(piHome: string, projectRoot: string | undefined): ScanJob[] {
+	const { global, project } = listPackageResources(piHome, projectRoot);
+	const groups: { group: PackageResources; baseHome: string }[] = [
+		...global.map((group) => ({ group, baseHome: piHome })),
+		...(projectRoot ? project.map((group) => ({ group, baseHome: join(projectRoot, ".pi") })) : []),
+	];
+	const jobs: ScanJob[] = [];
+	for (const { group, baseHome } of groups) {
+		const dir = resolveInstalledDir(baseHome, group.source);
+		if (!dir) continue;
+		for (const item of group.extensions) {
+			if (item.enabled) jobs.push({ group, dir, extension: item.path });
+		}
+	}
+	return jobs;
+}
+
+export async function runDoctor(piHome: string, projectRoot?: string, options: SmokeOptions = {}): Promise<DoctorReport> {
+	const jobs = collectJobs(piHome, projectRoot);
+	const truncated = jobs.length > MAX_EXTENSIONS_SCANNED;
+	const bounded = jobs.slice(0, MAX_EXTENSIONS_SCANNED);
+
+	const claims = new Map<RegistrationKind, Map<string, DoctorClaim[]>>(REGISTRATION_KINDS.map((kind) => [kind, new Map()]));
+	const extensions: DoctorExtensionResult[] = [];
+	let anyNotOk = false;
+
+	for (const job of bounded) {
+		const result = await runExtensionSmoke(job.dir, join(job.dir, job.extension), options);
+		if (result.status !== "ok") anyNotOk = true;
+		extensions.push({
+			name: job.group.name,
+			source: job.group.source,
+			scope: job.group.scope,
+			extension: job.extension,
+			status: result.status,
+			registrations: result.registrations,
+			...(result.message ? { message: result.message } : {}),
+		});
+		if (result.status !== "ok") continue;
+		const claim: DoctorClaim = { name: job.group.name, source: job.group.source, scope: job.group.scope, extension: job.extension };
+		for (const kind of REGISTRATION_KINDS) {
+			const byName = claims.get(kind)!;
+			for (const name of result.registrations[kind]) byName.set(name, [...(byName.get(name) ?? []), claim]);
+		}
+	}
+
+	const conflicts: DoctorConflict[] = [];
+	for (const kind of REGISTRATION_KINDS) {
+		for (const [name, claimants] of claims.get(kind)!) {
+			if (claimants.length > 1) conflicts.push({ kind: CONFLICT_KIND[kind], name, claimants });
+		}
+	}
+	conflicts.sort((a, b) => a.kind.localeCompare(b.kind) || a.name.localeCompare(b.name));
+
+	return { ok: conflicts.length === 0 && !anyNotOk, conflicts, extensions, scanned: bounded.length, truncated };
+}
+
+function formatClaimant(claim: DoctorClaim): string {
+	return `${claim.name} (${claim.source}) [${claim.scope}] ${claim.extension}`;
+}
+
+export function formatDoctorReport(report: DoctorReport, json: boolean): string {
+	if (json) return `${JSON.stringify(report)}\n`;
+	let out = `${report.ok ? "PASS" : "FAIL"} — ${report.scanned} extension(s) scanned, ${report.conflicts.length} conflict(s)\n`;
+	for (const conflict of report.conflicts) {
+		out += `\nCONFLICT ${conflict.kind} "${conflict.name}" claimed by:\n`;
+		for (const claimant of conflict.claimants) out += `  - ${formatClaimant(claimant)}\n`;
+	}
+	for (const extension of report.extensions) {
+		if (extension.status === "ok") continue;
+		out += `\n${extension.status.toUpperCase()} ${formatClaimant({ name: extension.name, source: extension.source, scope: extension.scope, extension: extension.extension })}${extension.message ? `: ${extension.message}` : ""}\n`;
+	}
+	if (report.truncated) out += "\nOutput truncated: more enabled extensions exist than this run's bound.\n";
+	return out;
+}

package/service/src/adoption/install-validation.ts

@@ -0,0 +1,187 @@
+/**
+ * install-validation.ts — a headless "does this extension even load" gate
+ * for ExecInstaller.install(), so a package that crashes at registration
+ * time never gets fully wired into ~/.pi/npm and ~/.pi/agent in the first
+ * place. Stages the real npm tarball into a throwaway temp dir (never the
+ * live piHome), then runs @danypops/pi-extension-harness's own mock-pi-cli
+ * subprocess -- a real, isolated process exercising the same production
+ * jiti load path Pi's own binary uses -- against every declared
+ * pi.extensions entry.
+ */
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+export interface ExtensionLoadResult {
+	path: string;
+	ok: boolean;
+	message?: string;
+}
+
+export interface InstallValidationResult {
+	ok: boolean;
+	source: string;
+	extensions: ExtensionLoadResult[];
+	message?: string;
+}
+
+export interface InstallValidator {
+	validate(source: string): Promise<InstallValidationResult>;
+}
+
+const DEFAULT_LOAD_TIMEOUT_MS = 8_000;
+const MAX_LOAD_TIMEOUT_MS = 30_000;
+const PACK_TIMEOUT_MS = 30_000;
+const MAX_EXTENSIONS = 20;
+
+function bounded(value: number | undefined, fallback: number, maximum: number): number {
+	if (!Number.isFinite(value) || value === undefined || value <= 0) return fallback;
+	return Math.min(Math.floor(value), maximum);
+}
+
+/** Strips packed's own "npm:" scheme -- what npm pack itself expects is a
+ * bare registry spec (name, name@version) or a local path. Returns
+ * undefined for git:/https:/local sources: nothing here to stage from an
+ * npm tarball, so headless validation is out of scope for them. */
+export function bareNpmSpec(source: string): string | undefined {
+	return source.startsWith("npm:") ? source.slice(4) : undefined;
+}
+
+interface CommandResult {
+	code: number;
+	stdout: string;
+	stderr: string;
+	timedOut: boolean;
+}
+
+async function runCommand(command: string[], cwd: string, timeoutMs: number): Promise<CommandResult> {
+	const proc = Bun.spawn(command, { cwd, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
+	let timedOut = false;
+	const timer = setTimeout(() => {
+		timedOut = true;
+		try {
+			proc.kill();
+		} catch {
+			/* already exited */
+		}
+	}, timeoutMs);
+	const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
+	clearTimeout(timer);
+	return { code, stdout, stderr, timedOut };
+}
+
+type StageResult = { ok: true; root: string } | { ok: false; message: string };
+
+/** Downloads (or, for a local path spec, packs) the real tarball into
+ * stageDir and extracts it -- never touches the live piHome. */
+async function stageNpmTarball(spec: string, stageDir: string): Promise<StageResult> {
+	const pack = await runCommand(["npm", "pack", spec, "--json"], stageDir, PACK_TIMEOUT_MS);
+	if (pack.timedOut) return { ok: false, message: `npm pack exceeded ${PACK_TIMEOUT_MS}ms` };
+	if (pack.code !== 0) return { ok: false, message: (pack.stderr.trim() || `npm pack exited ${pack.code}`).slice(0, 2_000) };
+	let filename: string | undefined;
+	try {
+		const parsed = JSON.parse(pack.stdout) as Array<{ filename?: string }>;
+		filename = parsed[0]?.filename;
+	} catch (e) {
+		return { ok: false, message: `npm pack produced unparseable JSON: ${e instanceof Error ? e.message : e}` };
+	}
+	if (!filename) return { ok: false, message: "npm pack reported no tarball filename" };
+	const tarPath = join(stageDir, filename);
+	if (!existsSync(tarPath)) return { ok: false, message: `expected tarball ${filename} is missing after npm pack` };
+	const extract = await runCommand(["tar", "-xzf", tarPath, "-C", stageDir], stageDir, PACK_TIMEOUT_MS);
+	if (extract.code !== 0) return { ok: false, message: (extract.stderr.trim() || `tar exited ${extract.code}`).slice(0, 2_000) };
+	const root = join(stageDir, "package"); // npm's own tarball layout convention
+	if (!existsSync(join(root, "package.json"))) return { ok: false, message: "extracted tarball has no package.json at its expected root" };
+	return { ok: true, root };
+}
+
+/** Runs pi-extension-harness's mock-pi-cli against one extension entry
+ * point, in its own load-only mode (--tool omitted): a real, isolated
+ * subprocess exercising Pi's own production jiti load path. */
+export async function validateExtensionLoadsHeadless(entryPath: string, timeoutMs?: number): Promise<ExtensionLoadResult> {
+	const bound = bounded(timeoutMs, DEFAULT_LOAD_TIMEOUT_MS, MAX_LOAD_TIMEOUT_MS);
+	let cliPath: string;
+	try {
+		cliPath = createRequire(import.meta.url).resolve("@danypops/pi-extension-harness/mock-pi-cli");
+	} catch (e) {
+		return { path: entryPath, ok: false, message: `pi-extension-harness is not resolvable: ${e instanceof Error ? e.message : e}` };
+	}
+	const result = await runCommand(["node", cliPath, "--extension", entryPath], tmpdir(), bound);
+	if (result.timedOut) return { path: entryPath, ok: false, message: `extension load exceeded ${bound}ms` };
+	const lastEvent = result.stdout
+		.trim()
+		.split("\n")
+		.filter((line) => line.startsWith("{"))
+		.at(-1);
+	if (lastEvent) {
+		try {
+			const event = JSON.parse(lastEvent) as { type?: string; error?: string };
+			if (event.type === "load_ok") return { path: entryPath, ok: true };
+			if (event.type === "load_error") {
+				return {
+					path: entryPath,
+					ok: false,
+					message: (typeof event.error === "string" ? event.error : "extension failed to load").slice(0, 1_000),
+				};
+			}
+		} catch {
+			// Falls through to the generic failure below.
+		}
+	}
+	return {
+		path: entryPath,
+		ok: false,
+		message: (result.stderr.trim() || `mock-pi-cli exited ${result.code} with no recognizable event`).slice(0, 1_000),
+	};
+}
+
+/** Stages the real npm tarball in isolation and headlessly load-checks
+ * every pi.extensions entry it declares. A package with no pi.extensions
+ * (most npm packages) or a non-npm source has nothing to validate and
+ * passes through as ok. */
+export class HeadlessInstallValidator implements InstallValidator {
+	constructor(private readonly timeoutMs?: number) {}
+
+	async validate(source: string): Promise<InstallValidationResult> {
+		const spec = bareNpmSpec(source);
+		if (!spec) return { ok: true, source, extensions: [] };
+
+		const stageDir = mkdtempSync(join(tmpdir(), "packed-install-validate-"));
+		try {
+			const staged = await stageNpmTarball(spec, stageDir);
+			if (!staged.ok) return { ok: false, source, extensions: [], message: staged.message };
+
+			let manifest: Record<string, unknown>;
+			try {
+				manifest = JSON.parse(readFileSync(join(staged.root, "package.json"), "utf8")) as Record<string, unknown>;
+			} catch (e) {
+				return { ok: false, source, extensions: [], message: `package.json unreadable: ${e instanceof Error ? e.message : e}` };
+			}
+
+			const pi = typeof manifest.pi === "object" && manifest.pi !== null ? (manifest.pi as Record<string, unknown>) : undefined;
+			const declared = Array.isArray(pi?.extensions)
+				? pi.extensions.filter((entry): entry is string => typeof entry === "string").slice(0, MAX_EXTENSIONS)
+				: [];
+			if (declared.length === 0) return { ok: true, source, extensions: [] };
+
+			const extensions: ExtensionLoadResult[] = [];
+			for (const entry of declared) {
+				const entryPath = resolve(staged.root, entry);
+				if (!existsSync(entryPath)) {
+					extensions.push({ path: entry, ok: false, message: "declared pi.extensions entry is absent from the tarball" });
+					continue;
+				}
+				// Report the declared manifest entry, not the resolved absolute
+				// path into a throwaway temp stage dir -- meaningful to a caller,
+				// matches what package.json itself says.
+				const loadResult = await validateExtensionLoadsHeadless(entryPath, this.timeoutMs);
+				extensions.push({ ...loadResult, path: entry });
+			}
+			const ok = extensions.every((extension) => extension.ok);
+			return { ok, source, extensions, ...(ok ? {} : { message: "one or more declared extensions failed a headless load check" }) };
+		} finally {
+			rmSync(stageDir, { recursive: true, force: true });
+		}
+	}
+}