akm-cli 0.9.0-beta.9 → 0.9.0-rc.1

package/dist/sources/providers/npm.js

@@ -36,18 +36,14 @@ class NpmSourceProvider {
         this.#config = config;
         this.name = config.name ?? config.url ?? "npm";
     }
-    async init(_ctx) {
-        // Resolution happens lazily in path(): until `sync()` runs there's no
-        // reliable on-disk path. Init is the registration handshake.
-    }
     path() {
         if (this.#config.path)
             return this.#config.path;
         throw new ConfigError(`npm source "${this.name}" has no resolved content path — run \`akm update\` to sync it before indexing.`);
     }
-    async sync() {
+    async sync(options) {
         const ref = npmRefFromConfig(this.#config);
-        await syncNpmRef(ref);
+        await syncNpmRef(ref, { force: options?.force });
     }
 }
 registerSourceProvider("npm", (config) => new NpmSourceProvider(config));

package/dist/sources/providers/provider-utils.js

@@ -91,14 +91,20 @@ export function copyDirectoryContents(sourceDir, destinationDir) {
             continue;
         const src = path.join(sourceDir, entry.name);
         const dest = path.join(destinationDir, entry.name);
-        fs.mkdirSync(path.dirname(dest), { recursive: true });
-        if (entry.isDirectory()) {
-            fs.cpSync(src, dest, { recursive: true, force: true });
-        }
-        else {
-            fs.copyFileSync(src, dest);
-        }
+        copyPathWithoutSymlinks(src, dest);
+    }
+}
+function copyPathWithoutSymlinks(sourcePath, destinationPath) {
+    const stat = fs.lstatSync(sourcePath);
+    if (stat.isSymbolicLink())
+        return;
+    fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
+    if (stat.isDirectory()) {
+        fs.mkdirSync(destinationPath, { recursive: true });
+        copyDirectoryContents(sourcePath, destinationPath);
+        return;
     }
+    fs.copyFileSync(sourcePath, destinationPath);
 }
 export function isDirectory(target) {
     try {

package/dist/sources/providers/sync-from-ref.js

@@ -1,6 +1,13 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Unified install-ref dispatcher.
+ *
+ * Replaces the historical `installRegistryRef()` entry point. Given an
+ * unparsed install ref, this resolves the right syncable provider and
+ * invokes its `sync()` method.
+ */
 import { UsageError } from "../../core/errors.js";
 import { parseRegistryRef } from "../../registry/resolve.js";
 import { detectStashRoot } from "./provider-utils.js";
@@ -18,7 +25,8 @@ export async function syncFromRef(ref, options) {
         return syncRegistryGitRef(ref, options);
     }
     // Exhaustiveness — `parseRegistryRef` only emits the four sources above.
-    throw new UsageError(`No syncable provider for ref: ${ref} (source=${parsed.source})`);
+    const _exhaustive = parsed;
+    throw new UsageError(`No syncable provider for ref: ${ref}`);
 }
 function syncLocalRef(parsed, options) {
     const stashRoot = detectStashRoot(parsed.sourcePath);

package/dist/sources/providers/website.js

@@ -2,22 +2,26 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { registerSourceProvider } from "../provider-factory.js";
-import { ensureWebsiteMirror, getWebsiteCachePaths, validateWebsiteUrl } from "../website-ingest.js";
+import { ensureWebsiteMirror, getWebsiteCachePaths, shouldAllowPrivateWebsiteUrlForTests, validateWebsiteUrl, } from "../website-ingest.js";
 /**
  * Website source provider — thin adapter over the shared website ingest module.
  */
 registerSourceProvider("website", (config) => {
-    const url = validateWebsiteUrl(config.url ?? "");
+    const allowPrivateHosts = shouldAllowPrivateWebsiteUrlForTests(config.url ?? "");
+    const url = validateWebsiteUrl(config.url ?? "", { allowPrivateHosts });
     const name = config.name ?? "website";
     return {
         kind: "website",
         name,
-        async init(_ctx) { },
         path() {
             return getWebsiteCachePaths(url).stashDir;
         },
-        async sync() {
-            await ensureWebsiteMirror(config, { requireStashDir: true });
+        async sync(options) {
+            await ensureWebsiteMirror(config, {
+                requireStashDir: true,
+                force: options?.force,
+                ...(allowPrivateHosts ? { allowPrivateHosts: true } : {}),
+            });
         },
     };
 });

package/dist/sources/website-ingest.js

@@ -3,12 +3,14 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { createHash } from "node:crypto";
 import fs from "node:fs";
+import { isIP } from "node:net";
 import path from "node:path";
-import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../core/common.js";
+import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir } from "../core/common.js";
 import { ConfigError, UsageError } from "../core/errors.js";
 import { getRegistryIndexCacheDir } from "../core/paths.js";
 import { warn } from "../core/warn.js";
 import { isExpired, sanitizeString } from "./providers/provider-utils.js";
+import { loadWikiSnapshotFetchers } from "./wiki-fetchers/registry.js";
 /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
 const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
 /** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
@@ -30,6 +32,30 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
  * whole crawl and return what we have when time runs out.
  */
 const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
+const WEBSITE_MAX_REDIRECTS = 8;
+export function shouldAllowPrivateWebsiteHostsForTests() {
+    return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
+}
+export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
+    if (!shouldAllowPrivateWebsiteHostsForTests())
+        return false;
+    try {
+        return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
+    }
+    catch {
+        return false;
+    }
+}
+function resolveFetcherStashDir(explicitStashDir) {
+    if (explicitStashDir)
+        return explicitStashDir;
+    try {
+        return resolveStashDir({ readOnly: true });
+    }
+    catch {
+        return null;
+    }
+}
 export function getWebsiteCachePaths(siteUrl) {
     const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
     const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
@@ -41,7 +67,7 @@ export function getWebsiteCachePaths(siteUrl) {
 }
 export async function ensureWebsiteMirror(config, options) {
     const rawUrl = config.url ?? "";
-    const normalizedUrl = validateWebsiteUrl(rawUrl);
+    const normalizedUrl = validateWebsiteUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
     const cachePaths = getWebsiteCachePaths(normalizedUrl);
     const requireStashDir = options?.requireStashDir === true;
     const force = options?.force === true;
@@ -63,6 +89,7 @@ export async function ensureWebsiteMirror(config, options) {
         await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
             maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
             maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
+            allowPrivateHosts: options?.allowPrivateHosts,
         });
         fs.writeFileSync(cachePaths.manifestPath, `${JSON.stringify({ url: normalizedUrl, fetchedAt: new Date().toISOString() }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
         return cachePaths;
@@ -114,20 +141,51 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
         fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
     }
 }
-export async function fetchWebsiteMarkdownSnapshot(rawUrl) {
-    const normalizedUrl = validateWebsiteInputUrl(rawUrl);
-    const fetched = await fetchWebsitePage(normalizedUrl);
+export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
+    const normalizedUrl = validateWebsiteInputUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
+    const parsedUrl = new URL(normalizedUrl);
+    const stashDir = resolveFetcherStashDir(options?.stashDir);
+    const context = {
+        stashDir: stashDir ?? "",
+        timeoutMs: options?.timeoutMs ?? 15_000,
+        signal: options?.signal,
+    };
+    for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
+        try {
+            if (!fetcher.matches(parsedUrl, context))
+                continue;
+            const snapshot = await fetcher.fetch(parsedUrl, context);
+            if (!snapshot)
+                continue;
+            return websiteMarkdownSnapshotFromResult(snapshot);
+        }
+        catch (error) {
+            warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
+        }
+    }
+    const fetched = await fetchWebsitePage(normalizedUrl, { allowPrivateHosts: options?.allowPrivateHosts });
     if (!fetched) {
         throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
     }
-    const preferredName = deriveImportPath(fetched.page.url);
-    const slug = preferredName.split("/").pop() ?? preferredName;
-    return {
+    return websiteMarkdownSnapshotFromResult({
         url: fetched.page.url,
         title: fetched.page.title,
         markdown: fetched.page.markdown,
+    });
+}
+function websiteMarkdownSnapshotFromResult(snapshot) {
+    const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
+    const slug = preferredName.split("/").pop() ?? preferredName;
+    return {
+        url: snapshot.url,
+        title: snapshot.title,
+        markdown: snapshot.markdown,
         preferredName,
-        content: buildMarkdownSnapshot(fetched.page, slug || "website"),
+        content: buildMarkdownSnapshot({
+            url: snapshot.url,
+            title: snapshot.title,
+            markdown: snapshot.markdown,
+        }, slug || "website", snapshot.tags),
     };
 }
 async function crawlWebsite(startUrl, options) {
@@ -147,7 +205,7 @@ async function crawlWebsite(startUrl, options) {
         if (!normalized || visited.has(normalized))
             continue;
         visited.add(normalized);
-        const fetched = await fetchWebsitePage(normalized);
+        const fetched = await fetchWebsitePage(normalized, { allowPrivateHosts: options.allowPrivateHosts });
         if (!fetched)
             continue;
         pages.push(fetched.page);
@@ -169,17 +227,8 @@ async function crawlWebsite(startUrl, options) {
     }
     return pages;
 }
-async function fetchWebsitePage(pageUrl) {
-    const parsedUrl = new URL(pageUrl);
-    if (parsedUrl.hostname.endsWith(".invalid")) {
-        throw new Error(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
-    }
-    const response = await fetchWithRetry(pageUrl, {
-        headers: {
-            Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
-            "User-Agent": "akm-cli website provider",
-        },
-    }, { timeout: 15_000, retries: 1 });
+async function fetchWebsitePage(pageUrl, options) {
+    const response = await fetchWebsiteResponse(pageUrl, 0, options);
     if (!response.ok) {
         if (response.status === 404)
             return null;
@@ -196,6 +245,7 @@ async function fetchWebsitePage(pageUrl) {
         throw err;
     }
     const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
+    assertWebsiteRequestUrl(finalUrl, Error, options);
     if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
         const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;
         return {
@@ -216,11 +266,35 @@ async function fetchWebsitePage(pageUrl) {
         links: [],
     };
 }
-function buildMarkdownSnapshot(page, slug) {
+async function fetchWebsiteResponse(pageUrl, redirectCount = 0, options) {
+    assertWebsiteRequestUrl(pageUrl, Error, options);
+    const response = await fetchWithRetry(pageUrl, {
+        headers: {
+            Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
+            "User-Agent": "akm-cli website provider",
+        },
+        redirect: "manual",
+    }, { timeout: 15_000, retries: 1 });
+    if (response.status >= 300 && response.status < 400) {
+        if (redirectCount >= WEBSITE_MAX_REDIRECTS) {
+            throw new Error(`Too many redirects while fetching ${pageUrl}`);
+        }
+        const location = response.headers.get("location");
+        if (!location) {
+            throw new Error(`Redirect response from ${pageUrl} did not include a Location header`);
+        }
+        const nextUrl = new URL(location, pageUrl).toString();
+        assertWebsiteRequestUrl(nextUrl, Error, options);
+        return fetchWebsiteResponse(nextUrl, redirectCount + 1, options);
+    }
+    return response;
+}
+function buildMarkdownSnapshot(page, slug, tags) {
     const title = sanitizeString(page.title, 200) || slug;
     const description = sanitizeString(`Snapshot of ${page.url}`, 500);
     const host = sanitizeString(new URL(page.url).hostname, 120);
     const content = page.markdown.trim() || `Source: ${page.url}`;
+    const normalizedTags = Array.from(new Set(["website", host, ...(tags ?? [])]));
     return [
         "---",
         `name: ${JSON.stringify(slug)}`,
@@ -228,8 +302,7 @@ function buildMarkdownSnapshot(page, slug) {
         `sourceUrl: ${JSON.stringify(page.url)}`,
         `title: ${JSON.stringify(title)}`,
         "tags:",
-        `  - ${JSON.stringify("website")}`,
-        `  - ${JSON.stringify(host)}`,
+        ...normalizedTags.map((tag) => `  - ${JSON.stringify(tag)}`),
         "---",
         "",
         `# ${title}`,
@@ -240,13 +313,13 @@ function buildMarkdownSnapshot(page, slug) {
         "",
     ].join("\n");
 }
-export function validateWebsiteUrl(rawUrl) {
-    return validateWebsiteUrlWithError(rawUrl, ConfigError);
+export function validateWebsiteUrl(rawUrl, options) {
+    return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
 }
-export function validateWebsiteInputUrl(rawUrl) {
-    return validateWebsiteUrlWithError(rawUrl, UsageError);
+export function validateWebsiteInputUrl(rawUrl, options) {
+    return validateWebsiteUrlWithError(rawUrl, UsageError, options);
 }
-function validateWebsiteUrlWithError(rawUrl, ErrorType) {
+function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
     if (!rawUrl) {
         throw new ErrorType("Website provider requires a URL");
     }
@@ -263,6 +336,7 @@ function validateWebsiteUrlWithError(rawUrl, ErrorType) {
     if (parsed.username || parsed.password) {
         throw new ErrorType("Website URL must not contain embedded credentials");
     }
+    assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
     parsed.hash = "";
     return normalizeSiteUrl(parsed.toString());
 }
@@ -461,6 +535,90 @@ function isAssetLikePath(pathname) {
 function isSafeLinkUrl(url) {
     return url.protocol === "http:" || url.protocol === "https:";
 }
+function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
+    const parsedUrl = new URL(rawUrl);
+    const hostname = parsedUrl.hostname.toLowerCase();
+    if (hostname.endsWith(".invalid")) {
+        throw new ErrorType(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
+    }
+    if (isForbiddenWebsiteHostname(hostname, options)) {
+        throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
+    }
+}
+// WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
+// node:net's isIP() only recognizes the bare address form and returns 0 for
+// anything bracketed — silently skipping all IPv6 forbidden-host checks
+// below for every hostname parsed off a URL. Strip the brackets before any
+// isIP()/isForbiddenIpv6() call so those checks actually run.
+function stripIpv6Brackets(hostname) {
+    return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+}
+function isForbiddenWebsiteHostname(hostname, options) {
+    if (options?.allowPrivateHosts === true)
+        return false;
+    if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
+        return true;
+    }
+    const bareHostname = stripIpv6Brackets(hostname);
+    const ipVersion = isIP(bareHostname);
+    if (ipVersion === 4)
+        return isForbiddenIpv4(bareHostname);
+    if (ipVersion === 6)
+        return isForbiddenIpv6(bareHostname);
+    return false;
+}
+function isLoopbackWebsiteHostname(hostname) {
+    if (hostname === "localhost" || hostname.endsWith(".localhost"))
+        return true;
+    const bareHostname = stripIpv6Brackets(hostname);
+    const ipVersion = isIP(bareHostname);
+    if (ipVersion === 4)
+        return bareHostname.startsWith("127.");
+    if (ipVersion === 6)
+        return bareHostname === "::1";
+    return false;
+}
+function isForbiddenIpv4(hostname) {
+    const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
+    if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
+        return true;
+    const [a, b] = parts;
+    return (a === 0 ||
+        a === 10 ||
+        a === 127 ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 168));
+}
+/**
+ * Extracts the embedded IPv4 address from an IPv4-mapped IPv6 literal
+ * (`::ffff:a.b.c.d` or its canonical hex form `::ffff:xxxx:yyyy`), or
+ * returns null if `hostname` isn't one.
+ */
+function extractIpv4MappedAddress(normalizedHostname) {
+    const match = normalizedHostname.match(/^::ffff:(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):([0-9a-f]{1,4}))$/);
+    if (!match)
+        return null;
+    if (match[1])
+        return match[1];
+    const high = Number.parseInt(match[2], 16);
+    const low = Number.parseInt(match[3], 16);
+    return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
+}
+function isForbiddenIpv6(hostname) {
+    const normalized = hostname.toLowerCase();
+    const mappedIpv4 = extractIpv4MappedAddress(normalized);
+    if (mappedIpv4)
+        return isForbiddenIpv4(mappedIpv4);
+    return (normalized === "::" ||
+        normalized === "::1" ||
+        normalized.startsWith("fc") ||
+        normalized.startsWith("fd") ||
+        normalized.startsWith("fe8") ||
+        normalized.startsWith("fe9") ||
+        normalized.startsWith("fea") ||
+        normalized.startsWith("feb"));
+}
 function stripDangerousBlockTag(value, tagName) {
     const pattern = new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}\\s*>`, "gi");
     return value.replace(pattern, "");

package/dist/sources/wiki-fetchers/registry.js

@@ -0,0 +1,53 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import fs from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { warn } from "../../core/warn.js";
+import youtubeFetcher from "./youtube.js";
+const FETCHER_DIR = path.join("scripts", "wiki-fetchers");
+const FETCHER_FILE_PATTERN = /\.(?:ts|js|mjs)$/i;
+const BUILTIN_FETCHERS = [youtubeFetcher];
+function isWikiSnapshotFetcher(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const candidate = value;
+    return (typeof candidate.name === "string" &&
+        typeof candidate.matches === "function" &&
+        typeof candidate.fetch === "function");
+}
+export async function loadWikiSnapshotFetchers(stashDir) {
+    const fetchers = [];
+    if (stashDir) {
+        const fetcherDir = path.join(stashDir, FETCHER_DIR);
+        try {
+            const entries = fs.readdirSync(fetcherDir).sort();
+            for (const entry of entries) {
+                if (!FETCHER_FILE_PATTERN.test(entry))
+                    continue;
+                try {
+                    const fileUrl = pathToFileURL(path.join(fetcherDir, entry)).toString();
+                    const mod = await import(fileUrl);
+                    if (isWikiSnapshotFetcher(mod.default)) {
+                        fetchers.push(mod.default);
+                    }
+                    else {
+                        warn("[akm] wiki-fetcher %s skipped: missing { name, matches, fetch }", entry);
+                    }
+                }
+                catch (error) {
+                    warn("[akm] wiki-fetcher %s failed to load: %s", entry, error instanceof Error ? error.message : String(error));
+                }
+            }
+        }
+        catch (error) {
+            const code = typeof error === "object" && error && "code" in error ? error.code : undefined;
+            if (code !== "ENOENT") {
+                warn("[akm] wiki-fetcher directory %s could not be read: %s", fetcherDir, error instanceof Error ? error.message : String(error));
+            }
+            // Missing directory means no custom fetchers.
+        }
+    }
+    return [...fetchers, ...BUILTIN_FETCHERS];
+}