@rbbtsn0w/adg 0.1.0-alpha.1 → 0.1.0-beta.2

package/dist/vendor/skills/src/blob.js

@@ -0,0 +1,436 @@
+/**
+ * Blob-based skill download utilities.
+ *
+ * Enables fast skill installation by fetching pre-built skill snapshots
+ * from the skills.sh download API instead of cloning git repos.
+ *
+ * Flow:
+ *   1. GitHub Trees API → discover SKILL.md locations
+ *   2. raw.githubusercontent.com → fetch frontmatter to get skill names
+ *   3. skills.sh/api/download → fetch full file contents from cached blob
+ */
+import { parseFrontmatter } from "./frontmatter.js";
+import { sanitizeMetadata } from "./sanitize.js";
+// ─── Constants ───
+const DOWNLOAD_BASE_URL = process.env.SKILLS_DOWNLOAD_URL || 'https://skills.sh';
+// Repos that self-host their downloads on the blob fast-path
+export const BLOB_ALLOWED_REPOS = {
+    'zapier/connectors': {
+        downloadUrl: (slug) => `https://connectors-skills.zapier.com/download/${encodeURIComponent(slug)}/snapshot.json`,
+    },
+};
+/** Timeout for individual HTTP fetches (ms) */
+const FETCH_TIMEOUT = 10_000;
+// ─── Slug computation ───
+/**
+ * Convert a skill name to a URL-safe slug.
+ * Must match the server-side toSkillSlug() exactly.
+ */
+export function toSkillSlug(name) {
+    return name
+        .toLowerCase()
+        .replace(/[\s_]+/g, '-')
+        .replace(/[^a-z0-9-]/g, '')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '');
+}
+/**
+ * Within-process memo: once we've discovered that the GitHub API has
+ * rate-limited this IP, subsequent calls skip straight to the auth fallback
+ * instead of burning round trips on requests guaranteed to 403.
+ */
+let _rateLimitedThisSession = false;
+/** For tests only. */
+export function resetRepoTreeAuthState() {
+    _rateLimitedThisSession = false;
+}
+async function fetchTreeBranch(ownerRepo, branch, token) {
+    try {
+        const url = `https://api.github.com/repos/${ownerRepo}/git/trees/${encodeURIComponent(branch)}?recursive=1`;
+        const headers = {
+            Accept: 'application/vnd.github.v3+json',
+            'User-Agent': 'skills-cli',
+        };
+        if (token) {
+            headers['Authorization'] = `Bearer ${token}`;
+        }
+        const response = await fetch(url, {
+            headers,
+            signal: AbortSignal.timeout(FETCH_TIMEOUT),
+        });
+        if (response.ok) {
+            const data = (await response.json());
+            return {
+                tree: { sha: data.sha, branch, tree: data.tree },
+                rateLimited: false,
+                authMayHelp: false,
+            };
+        }
+        // GitHub signals rate-limit with 403 + X-RateLimit-Remaining: 0.
+        const rateLimited = response.status === 403 && response.headers.get('x-ratelimit-remaining') === '0';
+        // ADG patch: a 401/403/404 from an UNauthenticated request may just mean the
+        // repo is private — retry with a token before giving up. GitHub returns 404
+        // (not 403) for private repos to anonymous callers to avoid leaking existence.
+        const authMayHelp = !rateLimited &&
+            (response.status === 401 || response.status === 403 || response.status === 404);
+        return { tree: null, rateLimited, authMayHelp };
+    }
+    catch {
+        return { tree: null, rateLimited: false, authMayHelp: false };
+    }
+}
+/**
+ * Fetch the full recursive tree for a GitHub repo.
+ * Returns the tree data including all entries, or null on failure.
+ * Tries branches in order: ref (if specified), then main, then master.
+ *
+ * Authentication is lazy: by default the call goes out unauthenticated,
+ * which is enough for the vast majority of users (60 req/hr per IP).
+ * Only if GitHub responds with a rate-limit 403 do we ask the optional
+ * `getToken` callback for a token and retry. This avoids invoking
+ * `gh auth token` on every install, which corporate endpoint security
+ * tools flag as suspicious credential extraction. See issue #523.
+ */
+export async function fetchRepoTree(ownerRepo, ref, getToken) {
+    const branches = ref ? [ref] : ['HEAD', 'main', 'master'];
+    // Fast path: once we've seen a rate limit in this process, don't bother
+    // retrying unauth on subsequent calls. Go straight to auth.
+    if (_rateLimitedThisSession && getToken) {
+        const token = getToken();
+        if (!token)
+            return null;
+        for (const branch of branches) {
+            const result = await fetchTreeBranch(ownerRepo, branch, token);
+            if (result.tree)
+                return result.tree;
+        }
+        return null;
+    }
+    // First pass: unauthenticated.
+    let rateLimited = false;
+    let authMayHelp = false;
+    for (const branch of branches) {
+        const result = await fetchTreeBranch(ownerRepo, branch, null);
+        if (result.tree)
+            return result.tree;
+        if (result.rateLimited) {
+            // All branches share the same rate-limit bucket on this IP, so it's
+            // pointless to keep trying other branches in this pass.
+            rateLimited = true;
+            break;
+        }
+        // ADG patch: remember a possible-private signal, but keep trying the other
+        // candidate branches unauthenticated first (the repo may just lack `main`).
+        if (result.authMayHelp)
+            authMayHelp = true;
+    }
+    // ADG patch: fall back to an authenticated retry on EITHER a rate limit or a
+    // private/forbidden response. Upstream only retried on rate limit, so private
+    // repos never used the token and always failed.
+    if ((!rateLimited && !authMayHelp) || !getToken)
+        return null;
+    // Only a rate limit is a process-wide condition worth short-circuiting later.
+    if (rateLimited)
+        _rateLimitedThisSession = true;
+    const token = getToken();
+    if (!token)
+        return null;
+    for (const branch of branches) {
+        const result = await fetchTreeBranch(ownerRepo, branch, token);
+        if (result.tree)
+            return result.tree;
+    }
+    return null;
+}
+/**
+ * Extract the folder hash (tree SHA) for a specific skill path from a repo tree.
+ * This replaces the per-skill GitHub API call previously done in fetchSkillFolderHash().
+ */
+export function getSkillFolderHashFromTree(tree, skillPath) {
+    let folderPath = skillPath.replace(/\\/g, '/');
+    // Remove SKILL.md suffix to get folder path (case-insensitive)
+    if (folderPath.toLowerCase().endsWith('/skill.md')) {
+        folderPath = folderPath.slice(0, -9);
+    }
+    else if (folderPath.toLowerCase().endsWith('skill.md')) {
+        folderPath = folderPath.slice(0, -8);
+    }
+    if (folderPath.endsWith('/')) {
+        folderPath = folderPath.slice(0, -1);
+    }
+    // Root-level skill
+    if (!folderPath) {
+        return tree.sha;
+    }
+    const entry = tree.tree.find((e) => e.type === 'tree' && e.path === folderPath);
+    return entry?.sha ?? null;
+}
+// ─── Skill discovery from tree ───
+/** Known directories where SKILL.md files are commonly found (relative to repo root) */
+const PRIORITY_PREFIXES = [
+    '',
+    'skills/',
+    'skills/.curated/',
+    'skills/.experimental/',
+    'skills/.system/',
+    '.agents/skills/',
+    '.claude/skills/',
+    '.cline/skills/',
+    '.codebuddy/skills/',
+    '.codex/skills/',
+    '.commandcode/skills/',
+    '.continue/skills/',
+    '.github/skills/',
+    '.goose/skills/',
+    '.iflow/skills/',
+    '.junie/skills/',
+    '.kilocode/skills/',
+    '.kiro/skills/',
+    '.mux/skills/',
+    '.neovate/skills/',
+    '.opencode/skills/',
+    '.openhands/skills/',
+    '.pi/skills/',
+    '.qoder/skills/',
+    '.roo/skills/',
+    '.trae/skills/',
+    '.windsurf/skills/',
+    '.zencoder/skills/',
+];
+/**
+ * Find all SKILL.md file paths in a repo tree.
+ * Applies the same priority directory logic as discoverSkills().
+ * If subpath is set, only searches within that subtree.
+ */
+export function findSkillMdPaths(tree, subpath) {
+    // Find all blob entries that are SKILL.md files (case-insensitive)
+    const allSkillMds = tree.tree
+        .filter((e) => e.type === 'blob' && e.path.toLowerCase().endsWith('skill.md'))
+        .map((e) => e.path);
+    // Apply subpath filter
+    const prefix = subpath ? (subpath.endsWith('/') ? subpath : subpath + '/') : '';
+    const filtered = prefix
+        ? allSkillMds.filter((p) => p.startsWith(prefix) || p === prefix + 'SKILL.md')
+        : allSkillMds;
+    if (filtered.length === 0)
+        return [];
+    // Check priority directories first (same order as discoverSkills).
+    // Non-root prefixes also accept depth-2 paths so the blob fast path stays
+    // in sync with the on-disk walk's catalog-layout discovery.
+    const priorityResults = [];
+    const seen = new Set();
+    // Mirror of SKIP_DIRS at the top of src/skills.ts. Kept local to avoid
+    // a cross-file import; if these ever drift, update both.
+    const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'build', '__pycache__']);
+    const lowerSkillMdSet = new Set(filtered.map((p) => p.toLowerCase()));
+    for (const priorityPrefix of PRIORITY_PREFIXES) {
+        const fullPrefix = prefix + priorityPrefix;
+        const isContainer = priorityPrefix !== '';
+        for (const skillMd of filtered) {
+            // Check if this SKILL.md is directly inside the priority dir (one level deep)
+            if (!skillMd.startsWith(fullPrefix))
+                continue;
+            const rest = skillMd.slice(fullPrefix.length);
+            // Direct SKILL.md in the priority dir (e.g., "skills/SKILL.md")
+            if (rest.toLowerCase() === 'skill.md') {
+                if (!seen.has(skillMd)) {
+                    priorityResults.push(skillMd);
+                    seen.add(skillMd);
+                }
+                continue;
+            }
+            // SKILL.md one level deep (e.g., "skills/react-best-practices/SKILL.md")
+            const parts = rest.split('/');
+            if (parts.length === 2 && parts[1].toLowerCase() === 'skill.md') {
+                if (!seen.has(skillMd)) {
+                    priorityResults.push(skillMd);
+                    seen.add(skillMd);
+                }
+                continue;
+            }
+            // SKILL.md two levels deep under a known container prefix
+            // (e.g., "skills/<category>/<skill>/SKILL.md"). Skip if the parent
+            // child dir already has its own SKILL.md (no descent past), or if
+            // any path segment is an ignored directory.
+            if (isContainer &&
+                parts.length === 3 &&
+                parts[2].toLowerCase() === 'skill.md' &&
+                !SKIP_DIRS.has(parts[0]) &&
+                !SKIP_DIRS.has(parts[1])) {
+                const parentSkillMd = `${fullPrefix}${parts[0]}/SKILL.md`.toLowerCase();
+                if (!lowerSkillMdSet.has(parentSkillMd) && !seen.has(skillMd)) {
+                    priorityResults.push(skillMd);
+                    seen.add(skillMd);
+                }
+            }
+        }
+    }
+    // If we found skills in priority dirs, return those
+    if (priorityResults.length > 0)
+        return priorityResults;
+    // Fallback: return all SKILL.md files found (limited to 5 levels deep)
+    return filtered.filter((p) => {
+        const depth = p.split('/').length;
+        return depth <= 6; // 5 levels + the SKILL.md file itself
+    });
+}
+// ─── Fetching skill content ───
+/**
+ * Fetch a single SKILL.md from raw.githubusercontent.com to get frontmatter.
+ * Returns the raw content string, or null on failure.
+ */
+async function fetchSkillMdContent(ownerRepo, branch, skillMdPath) {
+    try {
+        const url = `https://raw.githubusercontent.com/${ownerRepo}/${branch}/${skillMdPath}`;
+        const response = await fetch(url, {
+            signal: AbortSignal.timeout(FETCH_TIMEOUT),
+        });
+        if (!response.ok)
+            return null;
+        return await response.text();
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Fetch a skill's full file contents from the skills.sh download API.
+ * Returns the files array and content hash, or null on failure.
+ */
+async function fetchSkillDownload(source, slug) {
+    try {
+        const [owner, repo] = source.split('/');
+        const defaultUrl = `${DOWNLOAD_BASE_URL}/api/download/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/${encodeURIComponent(slug)}`;
+        // Self-hosted repos build their own URL; otherwise fall back to the default.
+        const selfHosted = BLOB_ALLOWED_REPOS[source.toLowerCase()]?.downloadUrl(slug);
+        const url = selfHosted ?? defaultUrl;
+        const response = await fetch(url, {
+            signal: AbortSignal.timeout(FETCH_TIMEOUT),
+        });
+        if (!response.ok)
+            return null;
+        return (await response.json());
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Attempt to resolve skills from blob storage instead of cloning.
+ *
+ * Steps:
+ *   1. Fetch repo tree from GitHub Trees API
+ *   2. Discover SKILL.md paths from the tree
+ *   3. Fetch SKILL.md content from raw.githubusercontent.com (for frontmatter/name)
+ *   4. Compute slugs and fetch full snapshots from skills.sh download API
+ *
+ * Returns the resolved BlobSkills + tree data on success, or null on any failure
+ * (the caller should fall back to git clone).
+ *
+ * @param ownerRepo - e.g., "vercel-labs/agent-skills"
+ * @param options - subpath, skillFilter, ref, token
+ */
+export async function tryBlobInstall(ownerRepo, options = {}) {
+    // 1. Fetch the full repo tree
+    const tree = await fetchRepoTree(ownerRepo, options.ref, options.getToken);
+    if (!tree)
+        return null;
+    // 2. Discover SKILL.md paths in the tree
+    let skillMdPaths = findSkillMdPaths(tree, options.subpath);
+    if (skillMdPaths.length === 0)
+        return null;
+    // 3. If a skill filter is set (owner/repo@skill-name), try to narrow down
+    if (options.skillFilter) {
+        const filterSlug = toSkillSlug(options.skillFilter);
+        const filtered = skillMdPaths.filter((p) => {
+            // Match by folder name — e.g., "skills/react-best-practices/SKILL.md"
+            const parts = p.split('/');
+            if (parts.length < 2)
+                return false;
+            const folderName = parts[parts.length - 2];
+            return toSkillSlug(folderName) === filterSlug;
+        });
+        if (filtered.length > 0) {
+            skillMdPaths = filtered;
+        }
+        // If no match by folder name, we'll try matching by frontmatter name below
+    }
+    // 4. Fetch SKILL.md content from raw.githubusercontent.com in parallel
+    const mdFetches = await Promise.all(skillMdPaths.map(async (mdPath) => {
+        const content = await fetchSkillMdContent(ownerRepo, tree.branch, mdPath);
+        return { mdPath, content };
+    }));
+    // Parse frontmatter to get skill names
+    const parsedSkills = [];
+    for (const { mdPath, content } of mdFetches) {
+        if (!content)
+            continue;
+        const { data } = parseFrontmatter(content);
+        if (!data.name || !data.description)
+            continue;
+        if (typeof data.name !== 'string' || typeof data.description !== 'string')
+            continue;
+        // Skip internal skills unless explicitly requested
+        const isInternal = data.metadata?.internal === true;
+        if (isInternal && !options.includeInternal)
+            continue;
+        const safeName = sanitizeMetadata(data.name);
+        const safeDescription = sanitizeMetadata(data.description);
+        parsedSkills.push({
+            mdPath,
+            name: safeName,
+            description: safeDescription,
+            content,
+            slug: toSkillSlug(safeName),
+            metadata: data.metadata,
+        });
+    }
+    if (parsedSkills.length === 0)
+        return null;
+    // Apply skill filter by name if not already filtered by folder name
+    let filteredSkills = parsedSkills;
+    if (options.skillFilter) {
+        const filterSlug = toSkillSlug(options.skillFilter);
+        const nameFiltered = parsedSkills.filter((s) => s.slug === filterSlug);
+        if (nameFiltered.length > 0) {
+            filteredSkills = nameFiltered;
+        }
+        // If still no match, let the caller fall back to clone where
+        // filterSkills() does fuzzy matching
+        if (filteredSkills.length === 0)
+            return null;
+    }
+    // 5. Fetch full snapshots from skills.sh download API in parallel
+    const source = ownerRepo.toLowerCase();
+    const downloads = await Promise.all(filteredSkills.map(async (skill) => {
+        const download = await fetchSkillDownload(source, skill.slug);
+        return { skill, download };
+    }));
+    // If ANY download failed, fall back to clone — we don't do partial blob installs
+    const allSucceeded = downloads.every((d) => d.download !== null);
+    if (!allSucceeded)
+        return null;
+    // 6. Convert to BlobSkill objects
+    const blobSkills = downloads.map(({ skill, download }) => {
+        // Compute the folder path from the SKILL.md path (e.g., "skills/react-best-practices")
+        const mdPathLower = skill.mdPath.toLowerCase();
+        const folderPath = mdPathLower.endsWith('/skill.md')
+            ? skill.mdPath.slice(0, -9)
+            : mdPathLower === 'skill.md'
+                ? ''
+                : skill.mdPath.slice(0, -(1 + 'SKILL.md'.length));
+        return {
+            name: skill.name,
+            description: skill.description,
+            // BlobSkills don't have a disk path — set to empty string.
+            // The installer uses the files array directly.
+            path: '',
+            rawContent: skill.content,
+            metadata: skill.metadata,
+            files: download.files,
+            snapshotHash: download.hash,
+            repoPath: skill.mdPath,
+        };
+    });
+    return { skills: blobSkills, tree };
+}