magus-cli 0.0.0-placeholder → 7.1.0

package/src/services/marketplace-fetcher.ts

@@ -0,0 +1,400 @@
+/**
+ * Marketplace.json fetcher with offline fallback.
+ *
+ * Extracted from `plugin-manager.ts` so the fetch + fallback logic can be
+ * unit-tested without dragging in claude-settings.ts's module-level
+ * `os.homedir()` constants (which contaminate the test suite when
+ * imported via plugin-manager.ts).
+ */
+
+import { isValidGitHubRepo } from "../utils/string-utils.js";
+import {
+	clearCachedCatalogs,
+	readCachedCatalog,
+	writeCachedCatalog,
+} from "./catalog-cache-store.js";
+import {
+	type GitHubHost,
+	cooldownFor,
+	describeCooldown,
+	hydrateGitHubBudget,
+	recordRateLimit,
+	recordSuccess,
+} from "./github-budget.js";
+import type { LocalMarketplace } from "./local-marketplace.js";
+import {
+	clearGitCatalogCache,
+	readCatalogFromRemote,
+} from "./marketplace-catalog-git.js";
+import {
+	type MarketplacePlugin,
+	clearMarketplaceVersionCache,
+	normalizeCatalogJson,
+} from "./marketplace-catalog.js";
+
+// Re-exported: these used to live here, and plugin-manager plus several screens
+// import them from this module.
+export {
+	getMarketplaceVersion,
+	normalizeCatalogJson,
+	type MarketplacePlugin,
+} from "./marketplace-catalog.js";
+
+// Session-level cache for fetched marketplace data (no TTL - persists until explicit refresh)
+const marketplaceCache = new Map<string, MarketplacePlugin[]>();
+
+/** Why a catalog fetch did not produce an authoritative plugin list. */
+export type MarketplaceFetchFailureKind =
+	| "rate-limited"
+	| "http"
+	| "timeout"
+	| "network"
+	| "invalid-repo"
+	| "invalid-content";
+
+export interface MarketplaceFetchFailure {
+	marketplace: string;
+	kind: MarketplaceFetchFailureKind;
+	httpStatus?: number;
+	/** Short, user-facing reason — rendered verbatim in the TUI. */
+	detail: string;
+	/**
+	 * Wall-clock ms when a retry is due, for rate limits. Lets the UI count down
+	 * and retry itself instead of telling the user to try again "later" and leaving
+	 * them to guess how much later.
+	 */
+	retryAt?: number;
+}
+
+export interface MarketplaceFetchResult {
+	plugins: MarketplacePlugin[];
+	/** Present exactly when `plugins` is not an authoritative remote answer. */
+	failure?: MarketplaceFetchFailure;
+}
+
+/**
+ * Where a resolved plugin list actually came from.
+ *
+ *  `remote`      raw.githubusercontent.com — authoritative.
+ *  `remote-git`  the clone's `origin/<branch>` ref, refreshed by `git fetch` —
+ *                also authoritative: it IS upstream's file. Reached when HTTP is
+ *                rate-limited, and it draws on neither GitHub HTTP budget.
+ *  `local-clone` the checked-out working tree — usable but NOT authoritative. It
+ *                is deliberately pinned (`autoUpdate: false`), so its versions can
+ *                be arbitrarily old.
+ *  `none`        nothing could be read.
+ */
+export type CatalogSource = "remote" | "remote-git" | "local-clone" | "none";
+
+/** Sources whose versions may be reported to the user as current. */
+export function isAuthoritative(source: CatalogSource): boolean {
+	return source === "remote" || source === "remote-git";
+}
+
+export interface MarketplaceResolution {
+	plugins: MarketplacePlugin[];
+	source: CatalogSource;
+	/**
+	 * The remote failure, if any. Set even when `source` is `local-clone` — the
+	 * fallback succeeded, but the versions in it are not known to be current.
+	 */
+	failure?: MarketplaceFetchFailure;
+}
+
+/**
+ * Failures are remembered, not just returned.
+ *
+ * Two reasons. First, so the TUI can render "4 of 6 catalogs unreachable"
+ * without every caller threading a result type through. Second, so a failure is
+ * cached like a success: an un-cached failure was re-attempted on every screen
+ * mount, and with a 10s timeout per marketplace that alone cost 30-45s of
+ * "Loading..." each time the user switched tabs and came back.
+ *
+ * The TTL is short and an explicit refresh clears it, so a transient blip never
+ * freezes the catalog for a whole session — which would be the same
+ * silent-staleness trap this module exists to close.
+ */
+const failureCache = new Map<
+	string,
+	{ failure: MarketplaceFetchFailure; at: number }
+>();
+const NEGATIVE_TTL_MS = 60_000;
+
+/** Monotonic clock, so a system clock change cannot expire the cache early. */
+function now(): number {
+	return performance.now();
+}
+
+export function clearMarketplaceCache(): void {
+	marketplaceCache.clear();
+	clearMarketplaceVersionCache();
+	clearGitCatalogCache();
+	failureCache.clear();
+}
+
+/**
+ * Drop the on-disk catalog cache as well.
+ *
+ * Separate and async because it must be AWAITED before the refetch that follows
+ * it: fire-and-forget would race the next `resolveMarketplacePlugins`, which would
+ * then read the cache it was told to discard and make `r` a no-op.
+ *
+ * Deliberately does NOT clear the host cooldown. That records GitHub's state, not
+ * ours, and retrying inside a known window cannot succeed — the screen counts down
+ * and refetches by itself when the window opens.
+ */
+export async function clearPersistedCatalogs(): Promise<void> {
+	await clearCachedCatalogs();
+}
+
+/**
+ * Every marketplace whose catalog could not be fetched, freshest answer first.
+ *
+ * A caller that renders plugin versions MUST consult this. An empty list is the
+ * only proof that the versions on screen were actually checked against GitHub.
+ */
+export function getMarketplaceFetchFailures(): MarketplaceFetchFailure[] {
+	const out: MarketplaceFetchFailure[] = [];
+	for (const [, entry] of failureCache) {
+		if (now() - entry.at < NEGATIVE_TTL_MS) out.push(entry.failure);
+	}
+	return out;
+}
+
+/** Record a failure and return it as the result, so callers cannot ignore it. */
+function fail(
+	marketplace: string,
+	kind: MarketplaceFetchFailureKind,
+	detail: string,
+	httpStatus?: number,
+	retryAt?: number,
+): MarketplaceFetchResult {
+	const failure: MarketplaceFetchFailure = {
+		marketplace,
+		kind,
+		detail,
+		httpStatus,
+		retryAt,
+	};
+	failureCache.set(marketplace, { failure, at: now() });
+	return { plugins: [], failure };
+}
+
+/**
+ * Turn a thrown fetch error into a classified failure.
+ *
+ * A timeout is distinguished from an outright network error because they mean
+ * different things to the user: the first is "GitHub is slow or unreachable
+ * right now", the second is usually "this machine has no route at all".
+ */
+function classifyThrown(
+	marketplace: string,
+	error: unknown,
+): MarketplaceFetchResult {
+	const name = (error as { name?: string } | null)?.name;
+	if (name === "TimeoutError" || name === "AbortError") {
+		return fail(
+			marketplace,
+			"timeout",
+			`no response in ${FETCH_TIMEOUT_MS / 1000}s`,
+		);
+	}
+	const message = error instanceof Error ? error.message : String(error);
+	return fail(marketplace, "network", message);
+}
+
+const FETCH_TIMEOUT_MS = 10_000;
+
+/**
+ * Fetch a marketplace catalog from GitHub.
+ *
+ * Returns a result, never a bare list, because "the fetch failed" and "this
+ * marketplace has no plugins" used to be the same value (`[]`) and the caller
+ * could not tell them apart. That single conflation is what let a rate-limited
+ * request render as "you are up to date": the empty list fell through to the
+ * on-disk clone, and a stale clone's version compared equal to the installed
+ * one. A failure must be a value the caller has to look at.
+ */
+export async function fetchMarketplacePlugins(
+	marketplaceName: string,
+	repo: string,
+): Promise<MarketplaceFetchResult> {
+	// Check cache first - session-level, no TTL
+	const cached = marketplaceCache.get(marketplaceName);
+	if (cached) {
+		return { plugins: cached };
+	}
+
+	// A fresh failure is reused rather than re-attempted: see `failureCache`.
+	const priorFailure = failureCache.get(marketplaceName);
+	if (priorFailure && now() - priorFailure.at < NEGATIVE_TTL_MS) {
+		return { plugins: [], failure: priorFailure.failure };
+	}
+
+	// Validate repo format to prevent SSRF
+	if (!isValidGitHubRepo(repo)) {
+		return fail(
+			marketplaceName,
+			"invalid-repo",
+			`not a valid GitHub repo: "${repo}"`,
+		);
+	}
+
+	const host: GitHubHost = "raw.githubusercontent.com";
+
+	// Cooldowns recorded by a PREVIOUS launch count too. Without this the gate is
+	// per-process, and a TUI's process lives for one screen session.
+	await hydrateGitHubBudget();
+
+	// Already known to be rate-limited: do not spend an attempt to rediscover it.
+	// The budget is per-host and shared with every other service, so six
+	// marketplaces used to trip the same limit six times over — and on some
+	// endpoints a rejected request extends the penalty.
+	const cooling = cooldownFor(host);
+	if (cooling) {
+		return fail(
+			marketplaceName,
+			"rate-limited",
+			describeCooldown(cooling),
+			429,
+			cooling.until,
+		);
+	}
+
+	try {
+		// Fetch marketplace.json from GitHub
+		const url = `https://raw.githubusercontent.com/${repo}/main/.claude-plugin/marketplace.json`;
+		const response = await fetch(url, {
+			signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+		});
+
+		if (!response.ok) {
+			// 429/403 are the rate-limit shapes GitHub returns for unauthenticated
+			// traffic. Called out separately because the fix is "wait, or
+			// authenticate", not "check your network".
+			const rateLimited = response.status === 429 || response.status === 403;
+			if (rateLimited) {
+				// Hand the response headers over: api.github.com states when the window
+				// reopens, raw.githubusercontent.com states nothing, and the budget
+				// module is what knows the difference.
+				const cooldown = recordRateLimit(host, response.headers);
+				return fail(
+					marketplaceName,
+					"rate-limited",
+					describeCooldown(cooldown),
+					response.status,
+					cooldown.until,
+				);
+			}
+			return fail(
+				marketplaceName,
+				"http",
+				`HTTP ${response.status} ${response.statusText}`.trim(),
+				response.status,
+			);
+		}
+
+		// Validate content-type
+		const contentType = response.headers.get("content-type");
+		if (
+			contentType &&
+			!contentType.includes("application/json") &&
+			!contentType.includes("text/plain")
+		) {
+			return fail(
+				marketplaceName,
+				"invalid-content",
+				`unexpected content-type "${contentType}"`,
+			);
+		}
+
+		const plugins = normalizeCatalogJson(
+			marketplaceName,
+			await response.json(),
+		);
+
+		// Cache the result (session-level)
+		marketplaceCache.set(marketplaceName, plugins);
+		// A success retires any recorded failure, so the banner clears itself.
+		failureCache.delete(marketplaceName);
+		recordSuccess(host);
+
+		return { plugins };
+	} catch (error) {
+		return classifyThrown(marketplaceName, error);
+	}
+}
+
+/**
+ * Resolve marketplace plugins with offline fallback.
+ *
+ * Calls `fetchMarketplacePlugins` (which hits GitHub) and, when it returns
+ * empty AND a local marketplace cache exists for the same name, falls back
+ * to the cached plugin manifest. This prevents every installed plugin from
+ * being flagged `isOrphaned: true` (and rendered "deprecated") whenever the
+ * machine can't reach `raw.githubusercontent.com` — the local cache already
+ * has the data, it just wasn't being consulted as a fallback.
+ *
+ * The two shapes are nearly identical; we drop a couple of LocalMarketplacePlugin
+ * fields (lspServers, agents, etc.) that MarketplacePlugin doesn't carry
+ * since the caller doesn't read them at this layer.
+ *
+ * The fallback stays — being offline should not make every plugin read as
+ * deprecated — but the return value now says WHICH source answered. The clone is
+ * a usable catalog, not an authoritative one: nothing refreshes it when a
+ * marketplace carries `autoUpdate: false`, so its versions can be arbitrarily
+ * old. A caller that reports "up to date" off a `local-clone` answer is
+ * reporting a guess as a fact.
+ */
+export async function resolveMarketplacePlugins(
+	mpName: string,
+	repo: string,
+	localMarketplaces: Map<string, LocalMarketplace>,
+): Promise<MarketplaceResolution> {
+	// Disk cache first, so a relaunch costs no network at all. Measured before this
+	// existed: every process spent six requests and ~13s re-fetching catalogs it had
+	// already read minutes earlier.
+	const cached = await readCachedCatalog(mpName);
+	if (cached) {
+		return {
+			plugins: cached.plugins,
+			source: cached.source as CatalogSource,
+		};
+	}
+
+	const remote = await fetchMarketplacePlugins(mpName, repo);
+	if (remote.plugins.length > 0) {
+		void writeCachedCatalog(mpName, remote.plugins, "remote");
+		return { plugins: remote.plugins, source: "remote" };
+	}
+
+	// HTTP failed — most often rate-limited. Before falling back to the pinned
+	// working tree, ask git for upstream's catalog. `git fetch` moves only remote
+	// refs, so this respects `autoUpdate: false` (the install source stays pinned)
+	// while still answering "what version exists upstream". It is also on a
+	// different transport, so a raw.githubusercontent.com rate limit does not
+	// affect it.
+	const viaGit = await readCatalogFromRemote(mpName);
+	if (viaGit && viaGit.length > 0) {
+		void writeCachedCatalog(mpName, viaGit, "remote-git");
+		return { plugins: viaGit, source: "remote-git", failure: remote.failure };
+	}
+
+	const local = localMarketplaces.get(mpName);
+	if (!local || local.plugins.length === 0) {
+		return { plugins: [], source: "none", failure: remote.failure };
+	}
+
+	return {
+		plugins: local.plugins.map((p) => ({
+			name: p.name,
+			version: p.version,
+			description: p.description,
+			category: p.category,
+			author: p.author,
+			releases: p.releases,
+		})),
+		source: "local-clone",
+		failure: remote.failure,
+	};
+}

package/src/services/marketplace-refresh.ts

@@ -0,0 +1,245 @@
+/**
+ * Keep on-disk marketplace clones current — safely.
+ *
+ * The plugin *list* magus shows is fetched fresh from GitHub, but a plugin's
+ * `setup` block and Claude Code's own installer read the on-disk marketplace
+ * clone (~/.claude/plugins/marketplaces/<name>). Nothing else keeps that clone
+ * current — recoverMarketplaceSettings only pulls entries it had to re-register —
+ * so a healthy marketplace's clone goes stale and its plugin metadata (e.g. a
+ * pinned dependency version in a plugin's setup) is never seen, even while the
+ * displayed version looks up to date.
+ *
+ * IMPORTANT: this refreshes the clone with a NON-DESTRUCTIVE `git pull --ff-only`,
+ * never `claude plugin marketplace update`. The latter (cacheMarketplaceFromGit)
+ * deletes the clone directory before re-cloning, so a failed clone leaves the
+ * marketplace permanently deleted — a documented hazard this codebase deliberately
+ * avoids (see marketplace-sync.ts and ai-docs/plugin-marketplace-bug-investigation.md).
+ * A failed `git pull --ff-only` leaves the clone exactly as it was: stale, but intact.
+ */
+
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import {
+	getConfiguredMarketplaces,
+	getGlobalClaudeDir,
+	getMarketplaceAutoUpdate,
+} from "./claude-settings.js";
+
+/** Per-git-invocation timeout. A hung pull must never block launching Claude. */
+const GIT_TIMEOUT_MS = 30_000;
+
+export interface RefreshResult {
+	/** Clones fast-forwarded to a newer commit (or already current). */
+	refreshed: string[];
+	/** Pull errored (network, diverged, timeout) — clone left intact and stale. */
+	failed: string[];
+	/** Nothing to do: no clone on disk, not a git repo, or dirty tree. */
+	skipped: string[];
+}
+
+function cloneDir(name: string): string {
+	return path.join(getGlobalClaudeDir(), "plugins", "marketplaces", name);
+}
+
+/** Run git in `cwd`, resolving stdout on exit 0. Killed after `timeoutMs`. */
+function runGit(
+	cwd: string,
+	args: string[],
+	timeoutMs: number,
+): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const proc = spawn("git", args, {
+			cwd,
+			timeout: timeoutMs,
+			killSignal: "SIGKILL",
+		});
+		let stdout = "";
+		let stderr = "";
+		proc.stdout?.on("data", (b) => {
+			stdout += b.toString();
+		});
+		proc.stderr?.on("data", (b) => {
+			stderr += b.toString();
+		});
+		proc.on("error", reject);
+		proc.on("close", (code, signal) => {
+			if (code === 0) {
+				resolve(stdout);
+			} else {
+				reject(
+					new Error(
+						`git ${args.join(" ")} ${
+							signal ? `killed (${signal})` : `exited ${code}`
+						}: ${stderr.trim()}`,
+					),
+				);
+			}
+		});
+	});
+}
+
+/** True if the clone has uncommitted changes we must not clobber. */
+async function isDirty(dir: string): Promise<boolean> {
+	const out = await runGit(dir, ["status", "--porcelain"], GIT_TIMEOUT_MS);
+	return out.trim().length > 0;
+}
+
+/**
+ * Outcome of fast-forwarding one clone:
+ *  - `absent`    no clone on disk (caller may create it via `marketplace add`)
+ *  - `skipped`   dirty tree — left untouched so local edits survive
+ *  - `refreshed` fast-forwarded, or already current
+ *  - `failed`    pull errored (network, diverged, timeout) — clone left intact
+ */
+export type CloneRefresh = "absent" | "skipped" | "refreshed" | "failed";
+
+/**
+ * Non-destructively fast-forward ONE marketplace clone. Never throws, never
+ * deletes: the worst case is `failed` (stale clone left intact). Shared by the
+ * batch refresh below and by claude-cli's "plugin not found" recovery, so both
+ * paths get the same safe semantics instead of the old `marketplace update`.
+ */
+export async function fastForwardCloneIfPresent(
+	name: string,
+): Promise<CloneRefresh> {
+	const dir = cloneDir(name);
+	// No clone on disk (claude never installed it) or not a git repo.
+	if (!fs.existsSync(path.join(dir, ".git"))) return "absent";
+	try {
+		// Never clobber a user's local edits to the clone.
+		if (await isDirty(dir)) return "skipped";
+		// --ff-only is the safety guarantee: it fetches and fast-forwards, and
+		// aborts without mutating the tree if it can't. It never deletes the dir,
+		// so a failure degrades to "stale clone", never "deleted marketplace".
+		await runGit(dir, ["pull", "--ff-only"], GIT_TIMEOUT_MS);
+		return "refreshed";
+	} catch {
+		return "failed";
+	}
+}
+
+/**
+ * Progress taps, so a caller can show what is happening during the pulls.
+ *
+ * This is the slowest thing on the update path — a git fetch per marketplace,
+ * over the network, with a 30s timeout each — and it used to report only after
+ * every clone had settled.
+ *
+ * Both callbacks are purely observational and CANNOT affect the refresh: each
+ * is invoked inside a try/catch that discards whatever it throws. `onSettled`
+ * in particular runs inside the `Promise.all` fan-out, so an unguarded throw
+ * there would reject the batch and turn a display error into a failed refresh —
+ * exactly the coupling this function's "never throws" contract exists to
+ * prevent.
+ */
+export interface RefreshProgress {
+	/** The eligible set, once selection is done and before any pull starts. */
+	onStart?: (names: string[]) => void;
+	/** One clone settled. Fires in completion order, not in `names` order. */
+	onSettled?: (name: string, status: CloneRefresh) => void;
+}
+
+/** Invoke an observer without letting it affect the caller. */
+function tap(fn: (() => void) | undefined): void {
+	if (!fn) return;
+	try {
+		fn();
+	} catch {
+		// A progress display is never worth failing a refresh over.
+	}
+}
+
+/**
+ * Non-destructively fast-forward the on-disk clone of every registered,
+ * auto-update-enabled, github-sourced marketplace.
+ *
+ * @param skip marketplaces already refreshed this run (e.g. re-registered ones),
+ *   so they aren't pulled twice.
+ * @param progress optional taps for a live progress display.
+ * @returns which marketplaces were refreshed, which failed (clone left intact),
+ *   and which were skipped. Failures and skips are collected rather than thrown —
+ *   one unreachable marketplace must not block launching Claude. Pulls run
+ *   concurrently, so a slow one doesn't serialize the rest onto the launch path.
+ */
+/**
+ * Select and report, writing nothing.
+ *
+ * Everything above the pull loop is a read — the marketplace registry and each
+ * entry's `autoUpdate` flag — so a read-only caller can have the full selection
+ * verdict for free. `--dry-run` and `--check` promise to write nothing and
+ * therefore cannot pull, but they must still be able to say "your magus clone
+ * has autoUpdate off, so its catalog will never advance". Gating the whole call
+ * off silently dropped that warning, which is the same "no signal reads as no
+ * problem" failure the flag exists to prevent.
+ */
+export async function reportRegisteredMarketplaces(
+	skip: Iterable<string> = [],
+): Promise<RefreshResult> {
+	return refreshRegisteredMarketplaces(skip, {}, { reportOnly: true });
+}
+
+export async function refreshRegisteredMarketplaces(
+	skip: Iterable<string> = [],
+	progress: RefreshProgress = {},
+	options: { reportOnly?: boolean } = {},
+): Promise<RefreshResult> {
+	const refreshed: string[] = [];
+	const failed: string[] = [];
+	const skipped: string[] = [];
+
+	const configured = await getConfiguredMarketplaces();
+	const skipSet = new Set(skip);
+
+	// Select eligible marketplaces first so the pulls can run concurrently.
+	const eligible: string[] = [];
+	for (const [name, cfg] of Object.entries(configured)) {
+		if (skipSet.has(name)) continue;
+		if (cfg.source?.source !== "github" || !cfg.source?.repo) {
+			skipped.push(name);
+			continue;
+		}
+		// Claude Code's own refresh policy. Honour it by not pulling, and say
+		// nothing: the version answer does not come from this clone.
+		// `readCatalogFromRemote` reads `origin/<branch>` with fetch + show, never
+		// the working tree — measured, a clone holding dev 7.4.0 with autoUpdate
+		// off still reported 7.4.1 available.
+		if ((await getMarketplaceAutoUpdate(name)) === false) {
+			skipped.push(name);
+			continue;
+		}
+		eligible.push(name);
+	}
+
+	// A read-only caller stops here, with the selection verdict intact and not
+	// one clone touched. `eligible` becomes `skipped`: nothing was refreshed, so
+	// claiming otherwise would be the false "all current" this module guards.
+	if (options.reportOnly) {
+		return {
+			refreshed: [],
+			failed: [],
+			skipped: [...skipped, ...eligible],
+		};
+	}
+
+	// Fast-forward every eligible clone in parallel so a slow one doesn't
+	// serialize the rest onto the launch path. fastForwardCloneIfPresent never
+	// throws, so the batch always resolves.
+	tap(() => progress.onStart?.([...eligible]));
+	const statuses = await Promise.all(
+		eligible.map(async (name) => {
+			const status = await fastForwardCloneIfPresent(name);
+			tap(() => progress.onSettled?.(name, status));
+			return status;
+		}),
+	);
+
+	statuses.forEach((status, i) => {
+		const name = eligible[i];
+		if (status === "refreshed") refreshed.push(name);
+		else if (status === "failed") failed.push(name);
+		else skipped.push(name); // absent | skipped
+	});
+
+	return { refreshed, failed, skipped };
+}