akm-cli 0.9.0-rc.8 → 0.9.0

package/dist/registry/build-index.js

@@ -1,382 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-/**
- * Build the v2 JSON registry index consumed by the `static-index` registry
- * provider. This module emits artifacts that conform to the v2 schema; the
- * schema itself is the input contract owned by `src/registry/providers/static-index.ts`
- * (see v1 architecture spec §3.3 — "the v2 JSON index schema belongs to
- * static-index"). When the schema changes, the parser in `static-index.ts`
- * must be updated together with this builder.
- */
-import fs from "node:fs";
-import path from "node:path";
-import { fetchWithRetry, jsonWithByteCap } from "../core/common.js";
-import { getCacheDir } from "../core/paths.js";
-import { recognizeStashEntries } from "../indexer/scan/drain-dir.js";
-import { walkStashFlat } from "../indexer/walk/walker.js";
-import { asRecord, asString, GITHUB_API_BASE, githubHeaders } from "../integrations/github.js";
-import { writeResponseToFileCapped } from "../runtime.js";
-import { copyIncludedPaths, findNearestIncludeConfig } from "../sources/include.js";
-import { detectStashRoot } from "../sources/providers/provider-utils.js";
-import { extractTarGzSecure } from "../sources/providers/tar-utils.js";
-import { parseRegistryIndex } from "./providers/static-index.js";
-const DEFAULT_NPM_REGISTRY_BASE = "https://registry.npmjs.org";
-const REQUIRED_KEYWORDS = ["akm-stash"];
-const GITHUB_TOPICS = ["akm-stash"];
-const EXCLUDED_REPOS = new Set(["itlackey/akm"]);
-const EXCLUDED_NPM_PACKAGES = new Set(["akm-cli"]);
-const EMPTY_INSPECTION = {};
-function getDefaultRegistryBuildDir() {
-    return path.join(getCacheDir(), "registry-build");
-}
-export async function buildRegistryIndex(options) {
-    const manualEntriesPath = path.resolve(options?.manualEntriesPath ?? path.join(getDefaultRegistryBuildDir(), "manual-entries.json"));
-    const npmRegistryBase = trimTrailingSlash(options?.npmRegistryBase ?? DEFAULT_NPM_REGISTRY_BASE);
-    const githubApiBase = trimTrailingSlash(options?.githubApiBase ?? GITHUB_API_BASE);
-    const [manualKits, npmKits, githubKits] = await Promise.all([
-        loadManualEntries(manualEntriesPath),
-        scanNpm(npmRegistryBase),
-        scanGithub(githubApiBase),
-    ]);
-    const stashes = deduplicateStashes([...manualKits, ...npmKits, ...githubKits]).sort((a, b) => a.name.localeCompare(b.name));
-    const index = {
-        version: 3,
-        updatedAt: new Date().toISOString(),
-        stashes,
-    };
-    return {
-        index,
-        counts: {
-            manual: manualKits.length,
-            npm: npmKits.length,
-            github: githubKits.length,
-            total: stashes.length,
-        },
-        paths: {
-            manualEntriesPath,
-        },
-    };
-}
-export function writeRegistryIndex(index, outPath) {
-    const resolved = path.resolve(outPath ?? path.join(getDefaultRegistryBuildDir(), "index.json"));
-    fs.mkdirSync(path.dirname(resolved), { recursive: true });
-    fs.writeFileSync(resolved, `${JSON.stringify(index, null, 2)}\n`, "utf8");
-    return resolved;
-}
-async function scanNpm(npmRegistryBase) {
-    const stashes = [];
-    const seen = new Set();
-    for (const keyword of REQUIRED_KEYWORDS) {
-        let offset = 0;
-        const size = 250;
-        while (true) {
-            const url = `${npmRegistryBase}/-/v1/search?text=keywords:${encodeURIComponent(keyword)}&size=${size}&from=${offset}`;
-            const data = await fetchJson(url);
-            for (const obj of data.objects) {
-                const pkg = obj.package;
-                if (EXCLUDED_NPM_PACKAGES.has(pkg.name))
-                    continue;
-                const repoUrl = pkg.links?.repository ?? "";
-                const normalizedRepo = repoUrl.replace(/^https?:\/\/github\.com\//, "").replace(/\.git$/, "");
-                if (EXCLUDED_REPOS.has(normalizedRepo))
-                    continue;
-                const id = `npm:${pkg.name}`;
-                if (seen.has(id))
-                    continue;
-                seen.add(id);
-                const keywords = (pkg.keywords ?? []).map((value) => value.toLowerCase());
-                if (!keywords.some((value) => REQUIRED_KEYWORDS.includes(value)))
-                    continue;
-                let latestMetadata = {};
-                try {
-                    latestMetadata = await fetchJson(`${npmRegistryBase}/${encodeURIComponent(pkg.name)}/latest`);
-                }
-                catch {
-                    latestMetadata = {};
-                }
-                const inspection = await inspectNpmPackage(npmRegistryBase, latestMetadata).catch(() => EMPTY_INSPECTION);
-                const tags = mergeStrings((pkg.keywords ?? []).filter((value) => !REQUIRED_KEYWORDS.includes(value.toLowerCase())), inspection.tags);
-                stashes.push(normalizeStash({
-                    id,
-                    name: pkg.name,
-                    description: inspection.description ?? pkg.description,
-                    ref: pkg.name,
-                    source: "npm",
-                    homepage: pkg.links?.homepage ?? pkg.links?.npm,
-                    author: pkg.author?.name ?? pkg.author?.username ?? pkg.publisher?.username,
-                    latestVersion: inspection.latestVersion ?? pkg.version,
-                    license: asString(latestMetadata.license) ?? inspection.license,
-                    tags,
-                    assetTypes: inspection.assetTypes,
-                    assets: inspection.assets,
-                }));
-            }
-            if (data.objects.length < size)
-                break;
-            offset += size;
-        }
-    }
-    return stashes;
-}
-async function inspectNpmPackage(_npmRegistryBase, latestMetadata) {
-    const dist = asRecord(latestMetadata.dist);
-    const tarballUrl = asString(dist.tarball);
-    if (!tarballUrl)
-        return {};
-    const inspection = await inspectArchive(tarballUrl);
-    return {
-        description: asString(latestMetadata.description) ?? inspection.description,
-        latestVersion: asString(latestMetadata.version) ?? inspection.latestVersion,
-        license: asString(latestMetadata.license) ?? inspection.license,
-        tags: mergeStrings(extractNonReservedKeywords(latestMetadata.keywords), inspection.tags),
-        assetTypes: inspection.assetTypes,
-        assets: inspection.assets,
-    };
-}
-async function scanGithub(githubApiBase) {
-    const stashes = [];
-    const seen = new Set();
-    const headers = githubHeaders();
-    for (const topic of GITHUB_TOPICS) {
-        let page = 1;
-        const perPage = 100;
-        while (true) {
-            const q = encodeURIComponent(`topic:${topic}`);
-            const url = `${githubApiBase}/search/repositories?q=${q}&sort=updated&order=desc&per_page=${perPage}&page=${page}`;
-            const data = await fetchJson(url, headers);
-            for (const repo of data.items) {
-                if (EXCLUDED_REPOS.has(repo.full_name))
-                    continue;
-                const id = `github:${repo.full_name}`;
-                if (seen.has(id))
-                    continue;
-                seen.add(id);
-                const inspection = await inspectArchive(`${githubApiBase}/repos/${repo.full_name}/tarball/${encodeURIComponent(repo.default_branch)}`, headers).catch(() => EMPTY_INSPECTION);
-                const topics = repo.topics.filter((value) => !GITHUB_TOPICS.includes(value));
-                stashes.push(normalizeStash({
-                    id,
-                    name: repo.name,
-                    description: inspection.description ?? repo.description ?? undefined,
-                    ref: repo.full_name,
-                    source: "github",
-                    homepage: repo.html_url,
-                    author: repo.owner.login,
-                    latestVersion: inspection.latestVersion,
-                    license: repo.license?.spdx_id ?? inspection.license,
-                    tags: mergeStrings(topics, inspection.tags),
-                    assetTypes: inspection.assetTypes,
-                    assets: inspection.assets,
-                }));
-            }
-            if (data.items.length < perPage)
-                break;
-            page += 1;
-        }
-    }
-    return stashes;
-}
-async function inspectArchive(url, headers) {
-    // Route registry-build scratch space under the akm cache dir instead of
-    // the system temp directory. Long-running registry builds that crash
-    // mid-flight previously left orphan dirs under `/tmp`, which can fill
-    // the OS partition. Pinning to `${getCacheDir()}/registry-build/` keeps
-    // a single `rm -rf` of that directory sufficient to reclaim the space
-    // and leaves the operator's `/tmp` untouched. See #276 for the
-    // bench-tmp precedent and #284 for this non-bench rollout.
-    const tmpRoot = path.join(getCacheDir(), "registry-build");
-    fs.mkdirSync(tmpRoot, { recursive: true });
-    const tempDir = fs.mkdtempSync(path.join(tmpRoot, "build-"));
-    const archivePath = path.join(tempDir, "archive.tgz");
-    const extractDir = path.join(tempDir, "extract");
-    try {
-        const response = await fetchWithRetry(url, headers ? { headers } : undefined, { timeout: 120_000 });
-        if (!response.ok) {
-            throw new Error(`Failed to fetch archive (${response.status}) from ${url}`);
-        }
-        // Byte-capped + body-deadline streaming: the fetch timeout bounds only the
-        // header phase, so cap the streamed archive to guard against an unbounded or
-        // dribbling body during a registry build.
-        await writeResponseToFileCapped(archivePath, response);
-        // Reuse the secure extraction from registry-install which validates entries,
-        // uses --no-same-owner, strips components, and runs a post-extraction scan.
-        extractTarGzSecure(archivePath, extractDir);
-        const stashRoot = detectStashRoot(extractDir);
-        const inspectionRoot = applyIncludeConfigForInspection(stashRoot, tempDir, extractDir) ?? stashRoot;
-        const metadata = await enumerateAssets(inspectionRoot);
-        const packageMetadata = readNearestPackageJson(extractDir, inspectionRoot);
-        const assets = metadata.map((entry) => ({
-            type: entry.type,
-            name: entry.name,
-            ...(entry.description ? { description: entry.description } : {}),
-            ...(entry.tags && entry.tags.length > 0 ? { tags: entry.tags } : {}),
-            ...(typeof entry.fileSize === "number" ? { estimatedTokens: Math.round(entry.fileSize / 4) } : {}),
-        }));
-        return {
-            description: asString(packageMetadata.description),
-            latestVersion: asString(packageMetadata.version),
-            license: asString(packageMetadata.license),
-            tags: extractNonReservedKeywords(packageMetadata.keywords),
-            assetTypes: deriveAssetTypes(assets),
-            assets: assets.length > 0 ? assets : undefined,
-        };
-    }
-    finally {
-        fs.rmSync(tempDir, { recursive: true, force: true });
-    }
-}
-function readNearestPackageJson(extractDir, stashRoot) {
-    const candidates = [
-        path.join(stashRoot, "package.json"),
-        path.join(extractDir, "package.json"),
-        path.join(extractDir, "package", "package.json"),
-    ];
-    for (const candidate of candidates) {
-        try {
-            return asRecord(JSON.parse(fs.readFileSync(candidate, "utf8")));
-        }
-        catch { }
-    }
-    return {};
-}
-async function enumerateAssets(stashRoot) {
-    const fileContexts = walkStashFlat(stashRoot);
-    const dirGroups = new Map();
-    for (const ctx of fileContexts) {
-        const group = dirGroups.get(ctx.parentDirAbs);
-        if (group)
-            group.push(ctx.absPath);
-        else
-            dirGroups.set(ctx.parentDirAbs, [ctx.absPath]);
-    }
-    const entries = [];
-    for (const [dirPath, files] of dirGroups) {
-        // `.stash.json` sidecar overrides retired (#39): a published stash that
-        // still ships sidecars (pre-0.9.0, never migrated) contributes only its
-        // frontmatter-recognized entries.
-        const stash = recognizeStashEntries(stashRoot, files);
-        if (stash.entries.length === 0)
-            continue;
-        entries.push(...stash.entries.filter((entry) => !!entry.filename).map((entry) => attachFileSize(dirPath, entry)));
-    }
-    return entries.sort((a, b) => `${a.type}:${a.name}`.localeCompare(`${b.type}:${b.name}`));
-}
-function attachFileSize(dirPath, entry) {
-    if (typeof entry.fileSize === "number" || !entry.filename)
-        return entry;
-    try {
-        return { ...entry, fileSize: fs.statSync(path.join(dirPath, entry.filename)).size };
-    }
-    catch {
-        return entry;
-    }
-}
-function applyIncludeConfigForInspection(stashRoot, tempDir, searchRoot) {
-    const includeConfig = findNearestIncludeConfig(stashRoot, searchRoot);
-    if (!includeConfig)
-        return undefined;
-    const selectedDir = path.join(tempDir, "selected");
-    fs.rmSync(selectedDir, { recursive: true, force: true });
-    fs.mkdirSync(selectedDir, { recursive: true });
-    copyIncludedPaths(includeConfig.include, includeConfig.baseDir, selectedDir);
-    return selectedDir;
-}
-async function loadManualEntries(manualEntriesPath) {
-    try {
-        const raw = JSON.parse(fs.readFileSync(manualEntriesPath, "utf8"));
-        const candidateKits = Array.isArray(raw) ? raw : asRecord(raw).stashes;
-        const parsed = parseRegistryIndex({ version: 3, updatedAt: new Date().toISOString(), stashes: candidateKits });
-        if (!parsed)
-            return [];
-        // Legacy `curated` flag on input entries is ignored (v1 spec §4.2). The
-        // builder no longer emits it on the resulting index.
-        return parsed.stashes.map((stash) => normalizeStash(stash));
-    }
-    catch {
-        return [];
-    }
-}
-// npm / GitHub API JSON pages; 25 MB cap covers the largest realistic
-// search result set while still bounding memory against a malicious or
-// misconfigured upstream that streams unbounded JSON.
-const BUILD_INDEX_JSON_BYTE_CAP = 25 * 1024 * 1024;
-async function fetchJson(url, headers) {
-    const response = await fetchWithRetry(url, headers ? { headers } : undefined, { timeout: 30_000 });
-    if (!response.ok) {
-        // Error-body sampling is intentionally small; 4 KB is plenty to
-        // include upstream hints in the thrown error.
-        const body = await response.text().catch(() => "");
-        throw new Error(`HTTP ${response.status} from ${url}: ${body.slice(0, 200)}`);
-    }
-    return jsonWithByteCap(response, BUILD_INDEX_JSON_BYTE_CAP);
-}
-function deduplicateStashes(stashes) {
-    const byId = new Map();
-    for (const stash of stashes) {
-        const existing = byId.get(stash.id);
-        byId.set(stash.id, existing ? mergeEntries(existing, stash) : stash);
-    }
-    return [...byId.values()];
-}
-function mergeEntries(a, b) {
-    const assets = mergeAssets(a.assets, b.assets);
-    const assetTypes = mergeStrings(a.assetTypes, b.assetTypes, assets ? deriveAssetTypes(assets) : undefined);
-    return normalizeStash({
-        id: a.id,
-        name: a.name,
-        description: a.description ?? b.description,
-        ref: a.ref,
-        source: a.source,
-        homepage: a.homepage ?? b.homepage,
-        tags: mergeStrings(a.tags, b.tags),
-        assetTypes,
-        assets,
-        author: a.author ?? b.author,
-        license: a.license ?? b.license,
-        latestVersion: a.latestVersion ?? b.latestVersion,
-    });
-}
-function mergeAssets(a, b) {
-    if (!a && !b)
-        return undefined;
-    const merged = new Map();
-    for (const asset of [...(a ?? []), ...(b ?? [])]) {
-        const key = `${asset.type}:${asset.name}`;
-        if (!merged.has(key))
-            merged.set(key, asset);
-    }
-    const values = [...merged.values()];
-    return values.length > 0 ? sortAssets(values) : undefined;
-}
-function mergeStrings(...values) {
-    const merged = [...new Set(values.flatMap((value) => value ?? []).filter((value) => value.trim().length > 0))].sort();
-    return merged.length > 0 ? merged : undefined;
-}
-function deriveAssetTypes(assets) {
-    return mergeStrings(assets?.map((asset) => asset.type));
-}
-function extractNonReservedKeywords(value) {
-    if (!Array.isArray(value))
-        return undefined;
-    const filtered = value
-        .filter((item) => typeof item === "string")
-        .map((item) => item.trim())
-        .filter((item) => item.length > 0)
-        .filter((item) => !REQUIRED_KEYWORDS.includes(item.toLowerCase()));
-    return filtered.length > 0 ? filtered : undefined;
-}
-function normalizeStash(stash) {
-    const assets = stash.assets ? sortAssets(stash.assets) : undefined;
-    return {
-        ...stash,
-        ...(stash.tags && stash.tags.length > 0 ? { tags: stash.tags } : {}),
-        ...(stash.assetTypes && stash.assetTypes.length > 0 ? { assetTypes: stash.assetTypes } : {}),
-        ...(assets && assets.length > 0 ? { assets } : {}),
-    };
-}
-function sortAssets(assets) {
-    return [...assets].sort((a, b) => `${a.type}:${a.name}`.localeCompare(`${b.type}:${b.name}`));
-}
-function trimTrailingSlash(value) {
-    return value.replace(/\/+$/, "");
-}