claudeup 4.41.0 → 4.42.0

package/src/services/skills-manager.ts

@@ -1,13 +1,21 @@
-import fs from "fs-extra";
-import path from "node:path";
 import os from "node:os";
+import path from "node:path";
+import fs from "fs-extra";
+import {
+	type RecommendedSkill,
+	classifyStarReliability,
+} from "../data/skill-repos.js";
 import type {
-	SkillSource,
-	SkillInfo,
-	SkillFrontmatter,
 	GitTreeResponse,
+	SkillFrontmatter,
+	SkillInfo,
+	SkillSource,
 } from "../types/index.js";
-import { classifyStarReliability, type RecommendedSkill } from "../data/skill-repos.js";
+import {
+	getMarketplaceClonePath,
+	scanMcpServers,
+} from "./local-marketplace.js";
+import { ensureCheckout } from "./plugin-checkout.js";
 
 const SKILLS_API_BASE =
 	"https://us-central1-claudish-6da10.cloudfunctions.net/skills";
@@ -106,7 +114,10 @@ export async function fetchSkillSetSkills(
 	const projectInstalled = await getInstalledSkillNames("project", projectPath);
 
 	const slugify = (name: string) =>
-		name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+		name
+			.toLowerCase()
+			.replace(/[^a-z0-9]+/g, "-")
+			.replace(/^-|-$/g, "");
 
 	const source: SkillSource = {
 		label: repo,
@@ -145,6 +156,507 @@ export async function fetchSkillSetSkills(
 	});
 }
 
+// ─── Plugin contents listing ─────────────────────────────────────────────────
+
+/** The four things a Claude Code plugin can ship. */
+export type PluginComponentKind = "skill" | "command" | "agent" | "mcp";
+
+/** One component found inside a plugin, for read-only display. */
+export interface PluginComponent {
+	kind: PluginComponentKind;
+	/** Skill directory, command/agent file stem, or MCP server key. */
+	name: string;
+	/**
+	 * Path within the repo. The SKILL.md / command .md / agent .md, or the
+	 * `.mcp.json` that declares the server.
+	 */
+	repoPath: string;
+	/**
+	 * Sub-directory between the component root and the component itself, when
+	 * the plugin groups them. "engineering" for skills/engineering/code-review.
+	 */
+	group?: string;
+	/** MCP only: the command the server runs, for the detail pane. */
+	mcpCommand?: string;
+}
+
+/**
+ * Strip a marketplace `source` string down to a repo-relative directory.
+ * "./" and "" both mean the repo root.
+ */
+function normalizePluginSource(source?: string): string {
+	if (!source) return "";
+	return source.replace(/^\.\//, "").replace(/\/+$/, "");
+}
+
+/** Sort key: group the four kinds, then alphabetise within each. */
+const KIND_ORDER: Record<PluginComponentKind, number> = {
+	skill: 0,
+	command: 1,
+	agent: 2,
+	mcp: 3,
+};
+
+/**
+ * Collect `<root>/**\/*.md` under one component directory.
+ *
+ * Commands and agents are flat `.md` files, sometimes one directory deep for
+ * grouping. README.md is excluded: several plugins document a directory from
+ * inside it, and listing that as a command invites the user to run it.
+ */
+function collectMarkdown(
+	paths: string[],
+	root: string,
+	kind: PluginComponentKind,
+): PluginComponent[] {
+	const out: PluginComponent[] = [];
+	for (const path of paths) {
+		if (!path.startsWith(root) || !path.endsWith(".md")) continue;
+		const inner = path.slice(root.length, -".md".length).split("/");
+		const name = inner[inner.length - 1];
+		if (!name || name.toLowerCase() === "readme") continue;
+		out.push({
+			kind,
+			name,
+			repoPath: path,
+			group: inner.length > 1 ? inner.slice(0, -1).join("/") : undefined,
+		});
+	}
+	return out;
+}
+
+/**
+ * List everything a plugin ships, from the repo tree.
+ *
+ * Skills, commands and agents come from one tree read. MCP servers cost a
+ * second request, made only when the plugin actually carries a `.mcp.json` —
+ * the file has to be opened to learn the server *names*, and a plugin whose
+ * only content is an MCP server otherwise renders as empty. mnemex is exactly
+ * that shape: no skills, no commands, no agents, one server, and it appeared
+ * to ship nothing at all.
+ *
+ * Skill discovery is scoped deliberately to `<source>/skills/**` plus a
+ * `<source>/SKILL.md` for a plugin that IS one skill. Repos in the wild mirror
+ * their skills into every harness directory they support and into per-language
+ * docs trees: everything-claude-code carries 898 SKILL.md files but only 286
+ * distinct skills, the rest living under `.kiro/`, `.agents/`, `.cursor/` and
+ * `docs/<lang>/`. An unscoped walk lists all 898 and reads as the plugin being
+ * three times its real size. ponytail is the same shape at small scale: 6
+ * skills, mirrored into `.openclaw/skills/`.
+ */
+export async function fetchPluginContents(
+	repo: string,
+	pluginSource?: string,
+	marketplaceName?: string,
+): Promise<PluginComponent[]> {
+	// Three sources, cheapest first.
+	//
+	// 1. Claude Code's marketplace clone. Free and complete — but only exists if
+	//    the marketplace was actually cloned. "✓ Added" does not imply that: the
+	//    badge means the catalog resolved, and a catalog resolves over HTTP.
+	// The short repo name is what a root-level SKILL.md is called, whichever
+	// source answers.
+	const rootSkillName = repo.split("/")[1] || repo;
+
+	if (marketplaceName) {
+		const local = await readLocalPluginContents(
+			getMarketplaceClonePath(marketplaceName),
+			pluginSource,
+			rootSkillName,
+		);
+		if (local) return local;
+	}
+
+	// 2. Our own read-only checkout. Unlimited and offline after the first fetch,
+	//    and explicitly not an install — see plugin-checkout.ts. This is what
+	//    keeps browsing working once the API budget is gone.
+	const checkout = await ensureCheckout(repo);
+	if (checkout) {
+		const cloned = await readLocalPluginContents(
+			checkout,
+			pluginSource,
+			rootSkillName,
+		);
+		if (cloned) return cloned;
+	}
+
+	// 3. The GitHub Tree API. Capped at 60 requests an hour across every repo on
+	//    screen combined, so it is the last resort rather than the first.
+	const tree = await fetchGitTree(repo);
+	const base = normalizePluginSource(pluginSource);
+	const prefix = base ? `${base}/` : "";
+	const skillsPrefix = `${prefix}skills/`;
+
+	const blobs = tree.tree
+		.filter((entry) => entry.type === "blob")
+		.map((entry) => entry.path);
+
+	const components: PluginComponent[] = [];
+	const seenSkills = new Set<string>();
+
+	for (const path of blobs) {
+		// A plugin that is itself a single skill (blader/humanizer): one
+		// SKILL.md at the plugin root and no skills/ directory at all.
+		if (path === `${prefix}SKILL.md`) {
+			const name = base ? base.split("/").pop() || repo : repo.split("/")[1];
+			if (!seenSkills.has(name)) {
+				seenSkills.add(name);
+				components.push({ kind: "skill", name, repoPath: path });
+			}
+			continue;
+		}
+
+		if (!path.startsWith(skillsPrefix) || !path.endsWith("/SKILL.md")) continue;
+
+		// "skills/engineering/code-review/SKILL.md" → ["engineering", "code-review"]
+		const inner = path
+			.slice(skillsPrefix.length, -"/SKILL.md".length)
+			.split("/");
+		const name = inner[inner.length - 1];
+		if (!name || seenSkills.has(name)) continue;
+		seenSkills.add(name);
+
+		components.push({
+			kind: "skill",
+			name,
+			repoPath: path,
+			group: inner.length > 1 ? inner.slice(0, -1).join("/") : undefined,
+		});
+	}
+
+	components.push(
+		...collectMarkdown(blobs, `${prefix}commands/`, "command"),
+		...collectMarkdown(blobs, `${prefix}agents/`, "agent"),
+	);
+
+	const mcpPath = `${prefix}.mcp.json`;
+	if (blobs.includes(mcpPath)) {
+		components.push(...(await fetchMcpServers(repo, mcpPath)));
+	}
+
+	components.sort((a, b) => {
+		if (a.kind !== b.kind) return KIND_ORDER[a.kind] - KIND_ORDER[b.kind];
+		const g = (a.group || "").localeCompare(b.group || "");
+		return g !== 0 ? g : a.name.localeCompare(b.name);
+	});
+	return components;
+}
+
+/**
+ * List a plugin's contents from the marketplace clone on disk.
+ *
+ * Returns null when there is no clone — the marketplace was never added, which
+ * is the normal state for a `featured` entry the user is only browsing. That
+ * case falls through to the tree API.
+ *
+ * The shape must match `fetchPluginContents`'s network path exactly, groups
+ * included, or a plugin would render differently depending on whether it
+ * happened to be installed.
+ */
+export async function readLocalPluginContents(
+	repoRoot: string,
+	pluginSource?: string,
+	/**
+	 * Name for a plugin that IS one skill at its root. Passed in rather than
+	 * derived from `repoRoot`, whose basename differs per source: a marketplace
+	 * clone is `magus`, a checkout is `blader__humanizer`. Deriving it made the
+	 * same plugin render under two different names depending on which source
+	 * answered.
+	 */
+	rootSkillName?: string,
+): Promise<PluginComponent[] | null> {
+	const base = normalizePluginSource(pluginSource);
+	const pluginDir = path.join(repoRoot, ...(base ? base.split("/") : []));
+	if (!(await fs.pathExists(pluginDir))) return null;
+
+	const fallbackName = rootSkillName || path.basename(repoRoot);
+	const components: PluginComponent[] = [];
+	const rel = (abs: string) => {
+		const suffix = path.relative(repoRoot, abs);
+		return suffix.split(path.sep).join("/");
+	};
+
+	// A plugin that is itself a single skill: one SKILL.md at its root.
+	const rootSkill = path.join(pluginDir, "SKILL.md");
+	if (await fs.pathExists(rootSkill)) {
+		components.push({
+			kind: "skill",
+			name: base ? base.split("/").pop() || fallbackName : fallbackName,
+			repoPath: rel(rootSkill),
+		});
+	}
+
+	for (const skill of await walkSkillDirs(path.join(pluginDir, "skills"))) {
+		components.push({
+			kind: "skill",
+			name: skill.name,
+			repoPath: rel(path.join(skill.dir, "SKILL.md")),
+			group: skill.group,
+		});
+	}
+
+	for (const [dir, kind] of [
+		["commands", "command"],
+		["agents", "agent"],
+	] as const) {
+		for (const file of await walkMarkdown(path.join(pluginDir, dir))) {
+			components.push({
+				kind,
+				name: file.name,
+				repoPath: rel(file.path),
+				group: file.group,
+			});
+		}
+	}
+
+	for (const name of await scanMcpServers(pluginDir)) {
+		components.push({
+			kind: "mcp",
+			name,
+			repoPath: rel(path.join(pluginDir, ".mcp.json")),
+		});
+	}
+
+	components.sort((a, b) => {
+		if (a.kind !== b.kind) return KIND_ORDER[a.kind] - KIND_ORDER[b.kind];
+		const g = (a.group || "").localeCompare(b.group || "");
+		return g !== 0 ? g : a.name.localeCompare(b.name);
+	});
+	return components;
+}
+
+/** Every directory under `root` that holds a SKILL.md, with its grouping path. */
+export async function walkSkillDirs(
+	root: string,
+	group = "",
+	depth = 0,
+): Promise<Array<{ name: string; dir: string; group?: string }>> {
+	if (depth > 3 || !(await fs.pathExists(root))) return [];
+	let entries: import("node:fs").Dirent[];
+	try {
+		entries = await fs.readdir(root, { withFileTypes: true });
+	} catch {
+		return [];
+	}
+
+	const found: Array<{ name: string; dir: string; group?: string }> = [];
+	for (const entry of entries) {
+		if (!entry.isDirectory()) continue;
+		const child = path.join(root, entry.name);
+		if (await fs.pathExists(path.join(child, "SKILL.md"))) {
+			found.push({ name: entry.name, dir: child, group: group || undefined });
+		} else {
+			found.push(
+				...(await walkSkillDirs(
+					child,
+					group ? `${group}/${entry.name}` : entry.name,
+					depth + 1,
+				)),
+			);
+		}
+	}
+	return found;
+}
+
+/** Markdown files under `root`, one level of grouping deep. README excluded. */
+export async function walkMarkdown(
+	root: string,
+	group = "",
+	depth = 0,
+): Promise<Array<{ name: string; path: string; group?: string }>> {
+	if (depth > 2 || !(await fs.pathExists(root))) return [];
+	let entries: import("node:fs").Dirent[];
+	try {
+		entries = await fs.readdir(root, { withFileTypes: true });
+	} catch {
+		return [];
+	}
+
+	const found: Array<{ name: string; path: string; group?: string }> = [];
+	for (const entry of entries) {
+		const child = path.join(root, entry.name);
+		if (entry.isDirectory()) {
+			found.push(
+				...(await walkMarkdown(
+					child,
+					group ? `${group}/${entry.name}` : entry.name,
+					depth + 1,
+				)),
+			);
+			continue;
+		}
+		if (!entry.name.endsWith(".md")) continue;
+		const name = entry.name.slice(0, -".md".length);
+		if (name.toLowerCase() === "readme") continue;
+		found.push({ name, path: child, group: group || undefined });
+	}
+	return found;
+}
+
+/**
+ * Read the server names out of a plugin's `.mcp.json`.
+ *
+ * Both shapes are in use: a bare `{ "<name>": {...} }` map, which mnemex
+ * writes, and the `{ "mcpServers": { "<name>": {...} } }` wrapper. A failure
+ * here degrades to a single unnamed entry rather than throwing — the plugin
+ * demonstrably has a server, and losing the whole listing over its name would
+ * be a worse answer than showing it without one.
+ */
+async function fetchMcpServers(
+	repo: string,
+	mcpPath: string,
+): Promise<PluginComponent[]> {
+	try {
+		const headers: Record<string, string> = {
+			Accept: "application/vnd.github.raw",
+			"X-GitHub-Api-Version": "2022-11-28",
+		};
+		const token =
+			process.env.GITHUB_TOKEN || process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
+		if (token) headers.Authorization = `Bearer ${token}`;
+
+		const response = await fetch(
+			`https://api.github.com/repos/${repo}/contents/${mcpPath}`,
+			{ headers, signal: AbortSignal.timeout(10000) },
+		);
+		if (!response.ok) throw new Error(String(response.status));
+
+		const raw = JSON.parse(await response.text()) as Record<string, unknown>;
+		const servers = (
+			raw.mcpServers && typeof raw.mcpServers === "object"
+				? raw.mcpServers
+				: raw
+		) as Record<string, { command?: string; args?: string[] }>;
+
+		const named = Object.entries(servers)
+			.filter(([, cfg]) => cfg && typeof cfg === "object")
+			.map(([name, cfg]) => ({
+				kind: "mcp" as const,
+				name,
+				repoPath: mcpPath,
+				mcpCommand: [cfg.command, ...(cfg.args ?? [])]
+					.filter(Boolean)
+					.join(" "),
+			}));
+
+		return named.length > 0
+			? named
+			: [{ kind: "mcp", name: "MCP server", repoPath: mcpPath }];
+	} catch {
+		return [{ kind: "mcp", name: "MCP server", repoPath: mcpPath }];
+	}
+}
+
+// ─── Plugin skill content ────────────────────────────────────────────────────
+
+/** A skill's own SKILL.md, split into its frontmatter and its body. */
+export interface PluginSkillDetail {
+	/** Every frontmatter key, in the order the file declares them. */
+	frontmatter: Array<{ key: string; value: string }>;
+	/** Markdown after the frontmatter block, trailing whitespace trimmed. */
+	body: string;
+	/** Total body lines, so a truncated render can say what it withheld. */
+	bodyLines: number;
+}
+
+/**
+ * Read one skill's SKILL.md.
+ *
+ * Served from the contents API rather than raw.githubusercontent.com, for two
+ * reasons: it is the host `fetchGitTree` already uses, so GITHUB_TOKEN raises
+ * the limit for both at once, and the raw host is the one a Tailscale MagicDNS
+ * search domain hijacks — which fails as an empty body rather than an error.
+ */
+export async function fetchPluginSkillDetail(
+	repo: string,
+	repoPath: string,
+): Promise<PluginSkillDetail> {
+	const headers: Record<string, string> = {
+		Accept: "application/vnd.github.raw",
+		"X-GitHub-Api-Version": "2022-11-28",
+	};
+	const token =
+		process.env.GITHUB_TOKEN || process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
+	if (token) headers.Authorization = `Bearer ${token}`;
+
+	const response = await fetch(
+		`https://api.github.com/repos/${repo}/contents/${repoPath}`,
+		{ headers, signal: AbortSignal.timeout(10000) },
+	);
+
+	if (response.status === 403 || response.status === 429) {
+		throw new Error(
+			"GitHub API rate limit exceeded. Set GITHUB_TOKEN to increase limits.",
+		);
+	}
+	if (!response.ok) {
+		throw new Error(
+			`GitHub API error: ${response.status} ${response.statusText}`,
+		);
+	}
+
+	return splitSkillDocument(await response.text());
+}
+
+/**
+ * Split a SKILL.md into ordered frontmatter pairs and its body.
+ *
+ * Ordered pairs rather than an object because the panel renders them as written
+ * — `name` and `description` first is the near-universal convention, and
+ * re-sorting them would bury the description under whatever keys sort earlier.
+ * List values are flattened to a comma-joined string for display.
+ */
+export function splitSkillDocument(content: string): PluginSkillDetail {
+	const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n?/);
+	const frontmatter: Array<{ key: string; value: string }> = [];
+
+	if (match) {
+		let pendingKey: string | null = null;
+		for (const line of match[1].split("\n")) {
+			// A "  - item" line continues the list opened by the previous key.
+			const listItem = line.match(/^\s*-\s+(.*)$/);
+			if (listItem && pendingKey) {
+				const last = frontmatter[frontmatter.length - 1];
+				last.value = last.value
+					? `${last.value}, ${stripQuotes(listItem[1])}`
+					: stripQuotes(listItem[1]);
+				continue;
+			}
+
+			const colonIdx = line.indexOf(":");
+			if (colonIdx === -1) continue;
+			const key = line.slice(0, colonIdx).trim();
+			if (!key || key.startsWith("#")) continue;
+
+			const raw = line.slice(colonIdx + 1).trim();
+			pendingKey = key;
+			frontmatter.push({
+				key,
+				value:
+					raw.startsWith("[") && raw.endsWith("]")
+						? raw
+								.slice(1, -1)
+								.split(",")
+								.map((s) => stripQuotes(s.trim()))
+								.filter(Boolean)
+								.join(", ")
+						: stripQuotes(raw),
+			});
+		}
+	}
+
+	const body = content.slice(match ? match[0].length : 0).trimEnd();
+	return {
+		frontmatter,
+		body,
+		bodyLines: body ? body.split("\n").length : 0,
+	};
+}
+
+function stripQuotes(value: string): string {
+	return value.replace(/^["']|["']$/g, "");
+}
+
 // ─── Frontmatter parser ───────────────────────────────────────────────────────
 
 function parseYamlFrontmatter(content: string): Partial<SkillFrontmatter> {
@@ -217,9 +729,7 @@ export async function getInstalledSkillNames(
 	projectPath?: string,
 ): Promise<Set<string>> {
 	const dir =
-		scope === "user"
-			? getUserSkillsDir()
-			: getProjectSkillsDir(projectPath);
+		scope === "user" ? getUserSkillsDir() : getProjectSkillsDir(projectPath);
 
 	const installed = new Set<string>();
 
@@ -292,12 +802,17 @@ export async function fetchAvailableSkills(
 	const projectInstalled = await getInstalledSkillNames("project", projectPath);
 
 	const slugify = (name: string) =>
-		name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+		name
+			.toLowerCase()
+			.replace(/[^a-z0-9]+/g, "-")
+			.replace(/^-|-$/g, "");
 
 	const markInstalled = (skill: SkillInfo): SkillInfo => {
 		const slug = slugify(skill.name);
-		const isUserInstalled = userInstalled.has(slug) || userInstalled.has(skill.name);
-		const isProjInstalled = projectInstalled.has(slug) || projectInstalled.has(skill.name);
+		const isUserInstalled =
+			userInstalled.has(slug) || userInstalled.has(skill.name);
+		const isProjInstalled =
+			projectInstalled.has(slug) || projectInstalled.has(skill.name);
 		const installed = isUserInstalled || isProjInstalled;
 		const installedScope: "user" | "project" | null = isProjInstalled
 			? "project"
@@ -333,7 +848,9 @@ export async function fetchAvailableSkills(
 
 	// 2. Fetch popular skills from Firebase API
 	const popular = await fetchPopularSkills(30);
-	const popularSkills = popular.map((s) => markInstalled({ ...s, isRecommended: false }));
+	const popularSkills = popular.map((s) =>
+		markInstalled({ ...s, isRecommended: false }),
+	);
 
 	// 3. Combine: recommended first, then popular (dedup by name)
 	const seen = new Set<string>(recommendedSkills.map((s) => s.name));