claudeup 4.36.0 → 4.37.0

package/src/services/marketplace-fetcher.ts

@@ -7,73 +7,288 @@
  * imported via plugin-manager.ts).
  */
 
-import type { PluginRelease } from "../types/index.js";
 import { isValidGitHubRepo } from "../utils/string-utils.js";
+import {
+  clearCachedCatalogs,
+  readCachedCatalog,
+  writeCachedCatalog,
+} from "./catalog-cache-store.js";
+import {
+  cooldownFor,
+  describeCooldown,
+  hydrateGitHubBudget,
+  recordRateLimit,
+  recordSuccess,
+  type GitHubHost,
+} from "./github-budget.js";
 import type { LocalMarketplace } from "./local-marketplace.js";
-import { normalizeReleases } from "./plugin-releases.js";
-
-export interface MarketplacePlugin {
-  name: string;
-  version: string | null;
-  description: string;
-  category?: string;
-  author?: { name: string; email?: string };
-  homepage?: string;
-  tags?: string[];
-  releases?: PluginRelease[];
-}
+import {
+  clearGitCatalogCache,
+  readCatalogFromRemote,
+} from "./marketplace-catalog-git.js";
+import {
+  clearMarketplaceVersionCache,
+  normalizeCatalogJson,
+  type MarketplacePlugin,
+} from "./marketplace-catalog.js";
+
+// Re-exported: these used to live here, and plugin-manager plus several screens
+// import them from this module.
+export {
+  getMarketplaceVersion,
+  normalizeCatalogJson,
+  type MarketplacePlugin,
+} from "./marketplace-catalog.js";
 
 // Session-level cache for fetched marketplace data (no TTL - persists until explicit refresh)
 const marketplaceCache = new Map<string, MarketplacePlugin[]>();
 
-// Session-level cache for each marketplace's declared version (from
-// marketplace.json's top-level `metadata.version`). Mirrors `marketplaceCache`
-// so the TUI can surface a marketplace's version without re-fetching. Only
-// populated for marketplaces that actually declare a version.
-const marketplaceVersionCache = new Map<string, string>();
+/** Why a catalog fetch did not produce an authoritative plugin list. */
+export type MarketplaceFetchFailureKind =
+  | "rate-limited"
+  | "http"
+  | "timeout"
+  | "network"
+  | "invalid-repo"
+  | "invalid-content";
+
+export interface MarketplaceFetchFailure {
+  marketplace: string;
+  kind: MarketplaceFetchFailureKind;
+  httpStatus?: number;
+  /** Short, user-facing reason — rendered verbatim in the TUI. */
+  detail: string;
+  /**
+   * Wall-clock ms when a retry is due, for rate limits. Lets the UI count down
+   * and retry itself instead of telling the user to try again "later" and leaving
+   * them to guess how much later.
+   */
+  retryAt?: number;
+}
+
+export interface MarketplaceFetchResult {
+  plugins: MarketplacePlugin[];
+  /** Present exactly when `plugins` is not an authoritative remote answer. */
+  failure?: MarketplaceFetchFailure;
+}
+
+/**
+ * Where a resolved plugin list actually came from.
+ *
+ *  `remote`      raw.githubusercontent.com — authoritative.
+ *  `remote-git`  the clone's `origin/<branch>` ref, refreshed by `git fetch` —
+ *                also authoritative: it IS upstream's file. Reached when HTTP is
+ *                rate-limited, and it draws on neither GitHub HTTP budget.
+ *  `local-clone` the checked-out working tree — usable but NOT authoritative. It
+ *                is deliberately pinned (`autoUpdate: false`), so its versions can
+ *                be arbitrarily old.
+ *  `none`        nothing could be read.
+ */
+export type CatalogSource = "remote" | "remote-git" | "local-clone" | "none";
+
+/** Sources whose versions may be reported to the user as current. */
+export function isAuthoritative(source: CatalogSource): boolean {
+  return source === "remote" || source === "remote-git";
+}
+
+export interface MarketplaceResolution {
+  plugins: MarketplacePlugin[];
+  source: CatalogSource;
+  /**
+   * The remote failure, if any. Set even when `source` is `local-clone` — the
+   * fallback succeeded, but the versions in it are not known to be current.
+   */
+  failure?: MarketplaceFetchFailure;
+}
+
+/**
+ * Failures are remembered, not just returned.
+ *
+ * Two reasons. First, so the TUI can render "4 of 6 catalogs unreachable"
+ * without every caller threading a result type through. Second, so a failure is
+ * cached like a success: an un-cached failure was re-attempted on every screen
+ * mount, and with a 10s timeout per marketplace that alone cost 30-45s of
+ * "Loading..." each time the user switched tabs and came back.
+ *
+ * The TTL is short and an explicit refresh clears it, so a transient blip never
+ * freezes the catalog for a whole session — which would be the same
+ * silent-staleness trap this module exists to close.
+ */
+const failureCache = new Map<string, { failure: MarketplaceFetchFailure; at: number }>();
+const NEGATIVE_TTL_MS = 60_000;
+
+/** Monotonic clock, so a system clock change cannot expire the cache early. */
+function now(): number {
+  return performance.now();
+}
 
 export function clearMarketplaceCache(): void {
   marketplaceCache.clear();
-  marketplaceVersionCache.clear();
+  clearMarketplaceVersionCache();
+  clearGitCatalogCache();
+  failureCache.clear();
 }
 
 /**
- * Returns the version a marketplace declared in its `metadata.version`, if one
- * was seen during a successful fetch this session. Returns `undefined` for
- * marketplaces that declare no version (or haven't been fetched yet).
+ * Drop the on-disk catalog cache as well.
+ *
+ * Separate and async because it must be AWAITED before the refetch that follows
+ * it: fire-and-forget would race the next `resolveMarketplacePlugins`, which would
+ * then read the cache it was told to discard and make `r` a no-op.
+ *
+ * Deliberately does NOT clear the host cooldown. That records GitHub's state, not
+ * ours, and retrying inside a known window cannot succeed — the screen counts down
+ * and refetches by itself when the window opens.
  */
-export function getMarketplaceVersion(
-  marketplaceName: string,
-): string | undefined {
-  return marketplaceVersionCache.get(marketplaceName);
+export async function clearPersistedCatalogs(): Promise<void> {
+  await clearCachedCatalogs();
+}
+
+/**
+ * Every marketplace whose catalog could not be fetched, freshest answer first.
+ *
+ * A caller that renders plugin versions MUST consult this. An empty list is the
+ * only proof that the versions on screen were actually checked against GitHub.
+ */
+export function getMarketplaceFetchFailures(): MarketplaceFetchFailure[] {
+  const out: MarketplaceFetchFailure[] = [];
+  for (const [, entry] of failureCache) {
+    if (now() - entry.at < NEGATIVE_TTL_MS) out.push(entry.failure);
+  }
+  return out;
+}
+
+/** Record a failure and return it as the result, so callers cannot ignore it. */
+function fail(
+  marketplace: string,
+  kind: MarketplaceFetchFailureKind,
+  detail: string,
+  httpStatus?: number,
+  retryAt?: number,
+): MarketplaceFetchResult {
+  const failure: MarketplaceFetchFailure = {
+    marketplace,
+    kind,
+    detail,
+    httpStatus,
+    retryAt,
+  };
+  failureCache.set(marketplace, { failure, at: now() });
+  return { plugins: [], failure };
+}
+
+/**
+ * Turn a thrown fetch error into a classified failure.
+ *
+ * A timeout is distinguished from an outright network error because they mean
+ * different things to the user: the first is "GitHub is slow or unreachable
+ * right now", the second is usually "this machine has no route at all".
+ */
+function classifyThrown(
+  marketplace: string,
+  error: unknown,
+): MarketplaceFetchResult {
+  const name = (error as { name?: string } | null)?.name;
+  if (name === "TimeoutError" || name === "AbortError") {
+    return fail(
+      marketplace,
+      "timeout",
+      `no response in ${FETCH_TIMEOUT_MS / 1000}s`,
+    );
+  }
+  const message = error instanceof Error ? error.message : String(error);
+  return fail(marketplace, "network", message);
 }
 
+const FETCH_TIMEOUT_MS = 10_000;
+
+/**
+ * Fetch a marketplace catalog from GitHub.
+ *
+ * Returns a result, never a bare list, because "the fetch failed" and "this
+ * marketplace has no plugins" used to be the same value (`[]`) and the caller
+ * could not tell them apart. That single conflation is what let a rate-limited
+ * request render as "you are up to date": the empty list fell through to the
+ * on-disk clone, and a stale clone's version compared equal to the installed
+ * one. A failure must be a value the caller has to look at.
+ */
 export async function fetchMarketplacePlugins(
   marketplaceName: string,
   repo: string,
-): Promise<MarketplacePlugin[]> {
+): Promise<MarketplaceFetchResult> {
   // Check cache first - session-level, no TTL
   const cached = marketplaceCache.get(marketplaceName);
   if (cached) {
-    return cached;
+    return { plugins: cached };
+  }
+
+  // A fresh failure is reused rather than re-attempted: see `failureCache`.
+  const priorFailure = failureCache.get(marketplaceName);
+  if (priorFailure && now() - priorFailure.at < NEGATIVE_TTL_MS) {
+    return { plugins: [], failure: priorFailure.failure };
   }
 
   // Validate repo format to prevent SSRF
   if (!isValidGitHubRepo(repo)) {
-    console.error(`Invalid GitHub repo format: ${repo}`);
-    return [];
+    return fail(
+      marketplaceName,
+      "invalid-repo",
+      `not a valid GitHub repo: "${repo}"`,
+    );
+  }
+
+  const host: GitHubHost = "raw.githubusercontent.com";
+
+  // Cooldowns recorded by a PREVIOUS launch count too. Without this the gate is
+  // per-process, and a TUI's process lives for one screen session.
+  await hydrateGitHubBudget();
+
+  // Already known to be rate-limited: do not spend an attempt to rediscover it.
+  // The budget is per-host and shared with every other service, so six
+  // marketplaces used to trip the same limit six times over — and on some
+  // endpoints a rejected request extends the penalty.
+  const cooling = cooldownFor(host);
+  if (cooling) {
+    return fail(
+      marketplaceName,
+      "rate-limited",
+      describeCooldown(cooling),
+      429,
+      cooling.until,
+    );
   }
 
   try {
     // Fetch marketplace.json from GitHub
     const url = `https://raw.githubusercontent.com/${repo}/main/.claude-plugin/marketplace.json`;
     const response = await fetch(url, {
-      signal: AbortSignal.timeout(10000), // 10s timeout
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
     });
 
     if (!response.ok) {
-      console.error(`Failed to fetch marketplace: ${response.status}`);
-      return [];
+      // 429/403 are the rate-limit shapes GitHub returns for unauthenticated
+      // traffic. Called out separately because the fix is "wait, or
+      // authenticate", not "check your network".
+      const rateLimited = response.status === 429 || response.status === 403;
+      if (rateLimited) {
+        // Hand the response headers over: api.github.com states when the window
+        // reopens, raw.githubusercontent.com states nothing, and the budget
+        // module is what knows the difference.
+        const cooldown = recordRateLimit(host, response.headers);
+        return fail(
+          marketplaceName,
+          "rate-limited",
+          describeCooldown(cooldown),
+          response.status,
+          cooldown.until,
+        );
+      }
+      return fail(
+        marketplaceName,
+        "http",
+        `HTTP ${response.status} ${response.statusText}`.trim(),
+        response.status,
+      );
     }
 
     // Validate content-type
@@ -83,55 +298,24 @@ export async function fetchMarketplacePlugins(
       !contentType.includes("application/json") &&
       !contentType.includes("text/plain")
     ) {
-      console.error(`Invalid content-type for marketplace: ${contentType}`);
-      return [];
+      return fail(
+        marketplaceName,
+        "invalid-content",
+        `unexpected content-type "${contentType}"`,
+      );
     }
 
-    interface RawPlugin {
-      name: string;
-      version?: string | null;
-      description?: string;
-      category?: string;
-      author?: { name: string; email?: string };
-      homepage?: string;
-      tags?: string[];
-      releases?: unknown;
-    }
-    const data = (await response.json()) as {
-      plugins?: RawPlugin[];
-      metadata?: { version?: string };
-    };
-    const plugins: MarketplacePlugin[] = [];
-
-    // Capture the marketplace's declared version (if any) so the TUI can
-    // display it. Absent/blank versions are simply not cached.
-    const version = data.metadata?.version;
-    if (typeof version === "string" && version.length > 0) {
-      marketplaceVersionCache.set(marketplaceName, version);
-    }
-
-    if (data.plugins && Array.isArray(data.plugins)) {
-      for (const plugin of data.plugins) {
-        plugins.push({
-          name: plugin.name,
-          version: plugin.version || null,
-          description: plugin.description || "",
-          category: plugin.category,
-          author: plugin.author,
-          homepage: plugin.homepage,
-          tags: plugin.tags,
-          releases: normalizeReleases(plugin.releases),
-        });
-      }
-    }
+    const plugins = normalizeCatalogJson(marketplaceName, await response.json());
 
     // Cache the result (session-level)
     marketplaceCache.set(marketplaceName, plugins);
+    // A success retires any recorded failure, so the banner clears itself.
+    failureCache.delete(marketplaceName);
+    recordSuccess(host);
 
-    return plugins;
+    return { plugins };
   } catch (error) {
-    console.error(`Error fetching marketplace ${marketplaceName}:`, error);
-    return [];
+    return classifyThrown(marketplaceName, error);
   }
 }
 
@@ -148,24 +332,63 @@ export async function fetchMarketplacePlugins(
  * The two shapes are nearly identical; we drop a couple of LocalMarketplacePlugin
  * fields (lspServers, agents, etc.) that MarketplacePlugin doesn't carry
  * since the caller doesn't read them at this layer.
+ *
+ * The fallback stays — being offline should not make every plugin read as
+ * deprecated — but the return value now says WHICH source answered. The clone is
+ * a usable catalog, not an authoritative one: nothing refreshes it when a
+ * marketplace carries `autoUpdate: false`, so its versions can be arbitrarily
+ * old. A caller that reports "up to date" off a `local-clone` answer is
+ * reporting a guess as a fact.
  */
 export async function resolveMarketplacePlugins(
   mpName: string,
   repo: string,
   localMarketplaces: Map<string, LocalMarketplace>,
-): Promise<MarketplacePlugin[]> {
+): Promise<MarketplaceResolution> {
+  // Disk cache first, so a relaunch costs no network at all. Measured before this
+  // existed: every process spent six requests and ~13s re-fetching catalogs it had
+  // already read minutes earlier.
+  const cached = await readCachedCatalog(mpName);
+  if (cached) {
+    return {
+      plugins: cached.plugins,
+      source: cached.source as CatalogSource,
+    };
+  }
+
   const remote = await fetchMarketplacePlugins(mpName, repo);
-  if (remote.length > 0) return remote;
+  if (remote.plugins.length > 0) {
+    void writeCachedCatalog(mpName, remote.plugins, "remote");
+    return { plugins: remote.plugins, source: "remote" };
+  }
+
+  // HTTP failed — most often rate-limited. Before falling back to the pinned
+  // working tree, ask git for upstream's catalog. `git fetch` moves only remote
+  // refs, so this respects `autoUpdate: false` (the install source stays pinned)
+  // while still answering "what version exists upstream". It is also on a
+  // different transport, so a raw.githubusercontent.com rate limit does not
+  // affect it.
+  const viaGit = await readCatalogFromRemote(mpName);
+  if (viaGit && viaGit.length > 0) {
+    void writeCachedCatalog(mpName, viaGit, "remote-git");
+    return { plugins: viaGit, source: "remote-git", failure: remote.failure };
+  }
 
   const local = localMarketplaces.get(mpName);
-  if (!local || local.plugins.length === 0) return remote;
-
-  return local.plugins.map((p) => ({
-    name: p.name,
-    version: p.version,
-    description: p.description,
-    category: p.category,
-    author: p.author,
-    releases: p.releases,
-  }));
+  if (!local || local.plugins.length === 0) {
+    return { plugins: [], source: "none", failure: remote.failure };
+  }
+
+  return {
+    plugins: local.plugins.map((p) => ({
+      name: p.name,
+      version: p.version,
+      description: p.description,
+      category: p.category,
+      author: p.author,
+      releases: p.releases,
+    })),
+    source: "local-clone",
+    failure: remote.failure,
+  };
 }