akm-cli 0.3.0 → 0.3.1

package/dist/cli.js

@@ -553,17 +553,19 @@ const searchCommand = defineCommand({
 const addCommand = defineCommand({
     meta: {
         name: "add",
-        description: "Add a source (local directory, npm package, GitHub repo, git URL, or remote provider)",
+        description: "Add a source (local directory, website, npm package, GitHub repo, git URL, or remote provider)",
     },
     args: {
         ref: {
             type: "positional",
-            description: "Path, URL, or registry ref (npm package, owner/repo, git URL, or local directory)",
+            description: "Path, URL, or registry ref (website URL, npm package, owner/repo, git URL, or local directory)",
             required: true,
         },
         provider: { type: "string", description: "Provider type (e.g. openviking). Required for URL sources." },
         options: { type: "string", description: 'Provider options as JSON (e.g. \'{"apiKey":"key"}\').' },
         name: { type: "string", description: "Human-friendly name for the source" },
+        "max-pages": { type: "string", description: "Maximum pages to crawl for website sources (default: 50)" },
+        "max-depth": { type: "string", description: "Maximum crawl depth for website sources (default: 3)" },
     },
     async run({ args }) {
         await runWithJsonErrors(async () => {
@@ -580,7 +582,7 @@ const addCommand = defineCommand({
             }
             // URL with --provider → stash source (remote or git provider)
             if (args.provider) {
-                if (ref.startsWith("http://")) {
+                if (shouldWarnOnPlainHttp(ref)) {
                     warn("Warning: source URL uses plain HTTP (not HTTPS). For security, prefer https:// to protect against eavesdropping and tampering.");
                 }
                 let parsedOptions;
@@ -607,7 +609,19 @@ const addCommand = defineCommand({
                 output("stash-add", result);
                 return;
             }
-            const result = await akmAdd({ ref });
+            if (shouldWarnOnPlainHttp(ref)) {
+                warn("Warning: source URL uses plain HTTP (not HTTPS). For security, prefer https:// to protect against eavesdropping and tampering.");
+            }
+            const websiteOptions = {};
+            if (args["max-pages"])
+                websiteOptions.maxPages = args["max-pages"];
+            if (args["max-depth"])
+                websiteOptions.maxDepth = args["max-depth"];
+            const result = await akmAdd({
+                ref,
+                name: args.name,
+                options: Object.keys(websiteOptions).length > 0 ? websiteOptions : undefined,
+            });
             output("add", result);
         });
     },
@@ -624,6 +638,22 @@ function parseKindFilter(raw) {
     }
     return kinds;
 }
+function shouldWarnOnPlainHttp(ref) {
+    if (!ref.startsWith("http://"))
+        return false;
+    try {
+        const hostname = new URL(ref).hostname.toLowerCase();
+        return (hostname !== "localhost" &&
+            hostname !== "127.0.0.1" &&
+            hostname !== "0.0.0.0" &&
+            hostname !== "::1" &&
+            hostname !== "[::1]" &&
+            !hostname.endsWith(".localhost"));
+    }
+    catch {
+        return true;
+    }
+}
 const listCommand = defineCommand({
     meta: { name: "list", description: "List all sources (local directories, managed packages, remote providers)" },
     args: {

package/dist/indexer.js

@@ -18,8 +18,8 @@ export async function akmIndex(options) {
     const config = loadConfig();
     // Ensure git stash caches are extracted before resolving stash dirs,
     // so their content directories exist on disk for the walker to discover.
-    const { ensureGitCaches, resolveAllStashDirs } = await import("./search-source.js");
-    await ensureGitCaches(config);
+    const { ensureStashCaches, resolveAllStashDirs } = await import("./search-source.js");
+    await ensureStashCaches(config);
     const allStashDirs = resolveAllStashDirs(stashDir);
     const t0 = Date.now();
     // Open database — pass embedding dimension from config if available

package/dist/search-source.js

@@ -3,6 +3,7 @@ import path from "node:path";
 import { resolveStashDir } from "./common";
 import { loadConfig } from "./config";
 import { ensureGitMirror, getCachePaths, parseGitRepoUrl } from "./stash-providers/git";
+import { ensureWebsiteMirror, getCachePaths as getWebsiteCachePaths } from "./stash-providers/website";
 import { warn } from "./warn";
 // ── Resolution ──────────────────────────────────────────────────────────────
 /**
@@ -54,6 +55,19 @@ export function resolveStashSources(overrideStashDir, existingConfig) {
             }
         }
     }
+    // Website stash entries: resolve cache directory so the indexer can walk
+    // the scraped markdown snapshots.
+    for (const entry of config.stashes ?? []) {
+        if (entry.type === "website" && entry.url && entry.enabled !== false) {
+            try {
+                const cachePaths = getWebsiteCachePaths(entry.url);
+                addSource(cachePaths.stashDir, entry.name ?? entry.url);
+            }
+            catch (err) {
+                warn(`Warning: failed to resolve website stash cache for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
     // Installed kits (registry and local)
     for (const entry of config.installed ?? []) {
         addSource(entry.stashRoot, entry.id);
@@ -153,11 +167,12 @@ function isValidDirectory(dir) {
 // ── Git stash cache integration ──────────────────────────────────────────────
 const GIT_STASH_TYPES = new Set(["context-hub", "github", "git"]);
 /**
- * Ensure all git stash mirrors are refreshed so their cache directories
- * exist on disk. Must be called (async) before `resolveStashSources()` so
- * the content directories pass the `isValidDirectory()` check.
+ * Ensure all cache-backed stash providers are refreshed so their cache
+ * directories exist on disk. Must be called (async) before
+ * `resolveStashSources()` so the content directories pass the
+ * `isValidDirectory()` check.
  */
-export async function ensureGitCaches(config) {
+export async function ensureStashCaches(config) {
     const cfg = config ?? loadConfig();
     for (const entry of cfg.stashes ?? []) {
         if (!GIT_STASH_TYPES.has(entry.type) || !entry.url || entry.enabled === false)
@@ -171,6 +186,18 @@ export async function ensureGitCaches(config) {
             warn(`Warning: failed to refresh git mirror for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
         }
     }
+    for (const entry of cfg.stashes ?? []) {
+        if (entry.type !== "website" || !entry.url || entry.enabled === false)
+            continue;
+        try {
+            await ensureWebsiteMirror(entry, { requireStashDir: true });
+        }
+        catch (err) {
+            warn(`Warning: failed to refresh website stash for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
 }
-/** @deprecated Use ensureGitCaches instead. */
-export const ensureContextHubCaches = ensureGitCaches;
+/** @deprecated Use ensureStashCaches instead. */
+export const ensureGitCaches = ensureStashCaches;
+/** @deprecated Use ensureStashCaches instead. */
+export const ensureContextHubCaches = ensureStashCaches;

package/dist/stash-add.js

@@ -1,18 +1,22 @@
 import fs from "node:fs";
 import path from "node:path";
-import { resolveStashDir } from "./common";
+import { isHttpUrl, resolveStashDir } from "./common";
 import { loadConfig, saveConfig } from "./config";
 import { UsageError } from "./errors";
 import { akmIndex } from "./indexer";
 import { upsertLockEntry } from "./lockfile";
 import { detectStashRoot, installRegistryRef, upsertInstalledRegistryEntry } from "./registry-install";
 import { parseRegistryRef } from "./registry-resolve";
+import { ensureWebsiteMirror, validateWebsiteInputUrl } from "./stash-providers/website";
 export async function akmAdd(input) {
     const ref = input.ref.trim();
     if (!ref)
         throw new UsageError("Install ref or local directory is required. " +
             "Examples: `akm add @scope/kit`, `akm add github:owner/repo`, `akm add ./local/path`");
     const stashDir = resolveStashDir();
+    if (shouldAddAsWebsiteUrl(ref)) {
+        return addWebsiteStashSource(ref, stashDir, input.name, input.options);
+    }
     // Detect local directory refs and route them to stashes[] instead of installed[]
     try {
         const parsed = parseRegistryRef(ref);
@@ -69,6 +73,50 @@ async function addLocalStashSource(ref, sourcePath, stashDir) {
         },
     };
 }
+async function addWebsiteStashSource(ref, stashDir, name, options) {
+    const normalizedUrl = validateWebsiteInputUrl(ref);
+    const config = loadConfig();
+    const stashes = [...(config.stashes ?? [])];
+    let entry = stashes.find((stash) => stash.type === "website" && stash.url === normalizedUrl);
+    if (!entry) {
+        entry = {
+            type: "website",
+            url: normalizedUrl,
+            name: name ?? toWebsiteName(normalizedUrl),
+            ...(options && Object.keys(options).length > 0 ? { options } : {}),
+        };
+        stashes.push(entry);
+        saveConfig({ ...config, stashes });
+    }
+    else if (options && Object.keys(options).length > 0) {
+        entry.options = { ...entry.options, ...options };
+        saveConfig({ ...config, stashes });
+    }
+    const cachePaths = await ensureWebsiteMirror(entry, { requireStashDir: true });
+    const index = await akmIndex({ stashDir });
+    const updatedConfig = loadConfig();
+    return {
+        schemaVersion: 1,
+        stashDir,
+        ref,
+        stashSource: {
+            type: "website",
+            url: normalizedUrl,
+            name: entry.name,
+            stashRoot: cachePaths.stashDir,
+        },
+        config: {
+            stashCount: updatedConfig.stashes?.length ?? 0,
+            installedKitCount: updatedConfig.installed?.length ?? 0,
+        },
+        index: {
+            mode: index.mode,
+            totalEntries: index.totalEntries,
+            directoriesScanned: index.directoriesScanned,
+            directoriesSkipped: index.directoriesSkipped,
+        },
+    };
+}
 /**
  * Install a kit from a registry (npm, github, git).
  */
@@ -139,3 +187,26 @@ function toReadableId(resolvedPath) {
     }
     return resolvedPath;
 }
+// Keep this list limited to widely-used git hosts for the non-breaking
+// "repo-like URL" fast-path; everything else continues to default to website snapshots.
+const KNOWN_GIT_HOSTS = new Set(["github.com", "gitlab.com", "bitbucket.org", "codeberg.org", "git.sr.ht"]);
+export function shouldAddAsWebsiteUrl(ref) {
+    return isHttpUrl(ref) && !isLikelyGitRepositoryUrl(ref);
+}
+function isLikelyGitRepositoryUrl(ref) {
+    try {
+        const parsed = new URL(ref);
+        return KNOWN_GIT_HOSTS.has(parsed.hostname.toLowerCase()) || parsed.pathname.endsWith(".git");
+    }
+    catch {
+        return false;
+    }
+}
+function toWebsiteName(siteUrl) {
+    try {
+        return new URL(siteUrl).hostname;
+    }
+    catch {
+        return siteUrl;
+    }
+}

package/dist/stash-providers/index.js

@@ -8,3 +8,4 @@
 import "./filesystem";
 import "./git";
 import "./openviking";
+import "./website";

package/dist/stash-providers/website.js

@@ -0,0 +1,443 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { fetchWithRetry } from "../common";
+import { ConfigError, UsageError } from "../errors";
+import { getRegistryIndexCacheDir } from "../paths";
+import { registerStashProvider } from "../stash-provider-factory";
+import { isExpired, sanitizeString } from "./provider-utils";
+/** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
+const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
+/** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
+const CACHE_STALE_MS = 7 * 24 * 60 * 60 * 1000;
+/** Allow limited breadth-first expansion without letting the crawl queue grow unbounded. */
+const QUEUE_EXPANSION_FACTOR = 5;
+const MAX_PAGES_DEFAULT = 50;
+const MAX_DEPTH_DEFAULT = 3;
+class WebsiteStashProvider {
+    type = "website";
+    name;
+    constructor(config) {
+        this.name = config.name ?? "website";
+        validateWebsiteUrl(config.url ?? "");
+    }
+    /** Content is indexed through the standard FTS5 pipeline. */
+    async search(_options) {
+        return { hits: [] };
+    }
+    /** Content is local files, shown via showLocal. */
+    async show(_ref, _view) {
+        throw new Error("Website provider content is shown via local index");
+    }
+    /** Content is local; no remote show needed. */
+    canShow(_ref) {
+        return false;
+    }
+}
+registerStashProvider("website", (config) => new WebsiteStashProvider(config));
+function getCachePaths(siteUrl) {
+    const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
+    const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
+    return {
+        rootDir,
+        stashDir: path.join(rootDir, "stash"),
+        manifestPath: path.join(rootDir, "manifest.json"),
+    };
+}
+async function ensureWebsiteMirror(config, options) {
+    const rawUrl = config.url ?? "";
+    const normalizedUrl = validateWebsiteUrl(rawUrl);
+    const cachePaths = getCachePaths(normalizedUrl);
+    const requireStashDir = options?.requireStashDir === true;
+    let mtime = 0;
+    try {
+        mtime = fs.statSync(cachePaths.manifestPath).mtimeMs;
+    }
+    catch {
+        /* no cached manifest */
+    }
+    if (mtime &&
+        !isExpired(mtime, CACHE_REFRESH_INTERVAL_MS) &&
+        (!requireStashDir || hasExtractedSite(cachePaths.stashDir))) {
+        return cachePaths;
+    }
+    try {
+        fs.mkdirSync(cachePaths.rootDir, { recursive: true });
+        await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
+            maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
+            maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
+        });
+        fs.writeFileSync(cachePaths.manifestPath, `${JSON.stringify({ url: normalizedUrl, fetchedAt: new Date().toISOString() }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
+        return cachePaths;
+    }
+    catch (err) {
+        if (mtime && !isExpired(mtime, CACHE_STALE_MS) && (!requireStashDir || hasExtractedSite(cachePaths.stashDir))) {
+            return cachePaths;
+        }
+        throw err;
+    }
+}
+function hasExtractedSite(stashDir) {
+    try {
+        const knowledgeDir = path.join(stashDir, "knowledge");
+        if (!fs.statSync(stashDir).isDirectory() || !fs.statSync(knowledgeDir).isDirectory())
+            return false;
+        // Check top-level and one level of subdirectories for .md files
+        for (const entry of fs.readdirSync(knowledgeDir, { withFileTypes: true })) {
+            if (entry.isFile() && entry.name.endsWith(".md"))
+                return true;
+            if (entry.isDirectory()) {
+                const subEntries = fs.readdirSync(path.join(knowledgeDir, entry.name));
+                if (subEntries.some((e) => e.endsWith(".md")))
+                    return true;
+            }
+        }
+        return false;
+    }
+    catch {
+        return false;
+    }
+}
+async function scrapeWebsiteToStash(startUrl, stashDir, options) {
+    const pages = await crawlWebsite(startUrl, options);
+    if (pages.length === 0) {
+        throw new Error(`No content could be scraped from ${startUrl}`);
+    }
+    fs.rmSync(stashDir, { recursive: true, force: true });
+    const knowledgeDir = path.join(stashDir, "knowledge");
+    fs.mkdirSync(knowledgeDir, { recursive: true });
+    const usedPaths = new Set();
+    for (const page of pages) {
+        const relPath = urlToRelativePath(page.url);
+        const uniquePath = uniqueSlug(relPath, usedPaths);
+        const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
+        const dir = path.dirname(filePath);
+        if (dir !== knowledgeDir)
+            fs.mkdirSync(dir, { recursive: true });
+        const slug = uniquePath.split("/").pop() ?? "index";
+        fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
+    }
+}
+async function crawlWebsite(startUrl, options) {
+    const start = new URL(normalizeSiteUrl(startUrl));
+    const allowedOrigin = start.origin;
+    const queue = [{ url: start.toString(), depth: 0 }];
+    const visited = new Set();
+    const pages = [];
+    while (queue.length > 0 && pages.length < options.maxPages) {
+        const next = queue.shift();
+        if (!next)
+            break;
+        const normalized = normalizeCrawlUrl(next.url);
+        if (!normalized || visited.has(normalized))
+            continue;
+        visited.add(normalized);
+        const fetched = await fetchWebsitePage(normalized);
+        if (!fetched)
+            continue;
+        pages.push(fetched.page);
+        if (next.depth >= options.maxDepth)
+            continue;
+        for (const link of fetched.links) {
+            if (queue.length + pages.length >= options.maxPages * QUEUE_EXPANSION_FACTOR)
+                break;
+            if (link.origin !== allowedOrigin)
+                continue;
+            const candidate = normalizeCrawlUrl(link.toString());
+            if (!candidate || visited.has(candidate) || isAssetLikePath(link.pathname))
+                continue;
+            queue.push({ url: candidate, depth: next.depth + 1 });
+        }
+    }
+    return pages;
+}
+async function fetchWebsitePage(pageUrl) {
+    const response = await fetchWithRetry(pageUrl, {
+        headers: {
+            Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
+            "User-Agent": "akm-cli website provider",
+        },
+    }, { timeout: 15_000, retries: 1 });
+    if (!response.ok) {
+        if (response.status === 404)
+            return null;
+        throw new Error(`Failed to fetch website content (${response.status}) from ${pageUrl}`);
+    }
+    const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
+    const body = await response.text();
+    const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
+    if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
+        const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;
+        return {
+            page: {
+                url: finalUrl,
+                title,
+                markdown: htmlToMarkdown(body, finalUrl),
+            },
+            links: extractSameDocumentLinks(body, finalUrl),
+        };
+    }
+    return {
+        page: {
+            url: finalUrl,
+            title: extractTextTitle(body) || new URL(finalUrl).hostname,
+            markdown: body.trim(),
+        },
+        links: [],
+    };
+}
+function buildMarkdownSnapshot(page, slug) {
+    const title = sanitizeString(page.title, 200) || slug;
+    const description = sanitizeString(`Snapshot of ${page.url}`, 500);
+    const host = sanitizeString(new URL(page.url).hostname, 120);
+    const content = page.markdown.trim() || `Source: ${page.url}`;
+    return [
+        "---",
+        `name: ${JSON.stringify(slug)}`,
+        `description: ${JSON.stringify(description)}`,
+        `sourceUrl: ${JSON.stringify(page.url)}`,
+        `title: ${JSON.stringify(title)}`,
+        "tags:",
+        `  - ${JSON.stringify("website")}`,
+        `  - ${JSON.stringify(host)}`,
+        "---",
+        "",
+        `# ${title}`,
+        "",
+        `Source: ${page.url}`,
+        "",
+        content,
+        "",
+    ].join("\n");
+}
+function validateWebsiteUrl(rawUrl) {
+    return validateWebsiteUrlWithError(rawUrl, ConfigError);
+}
+function validateWebsiteInputUrl(rawUrl) {
+    return validateWebsiteUrlWithError(rawUrl, UsageError);
+}
+function validateWebsiteUrlWithError(rawUrl, ErrorType) {
+    if (!rawUrl) {
+        throw new ErrorType("Website provider requires a URL");
+    }
+    let parsed;
+    try {
+        parsed = new URL(rawUrl);
+    }
+    catch {
+        throw new ErrorType(`Website URL is not valid: "${rawUrl}"`);
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        throw new ErrorType(`Website URL must use http:// or https://, got "${parsed.protocol}" in "${rawUrl}"`);
+    }
+    if (parsed.username || parsed.password) {
+        throw new ErrorType("Website URL must not contain embedded credentials");
+    }
+    parsed.hash = "";
+    return normalizeSiteUrl(parsed.toString());
+}
+function normalizeSiteUrl(rawUrl) {
+    const parsed = new URL(rawUrl);
+    parsed.hash = "";
+    if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
+        parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+    }
+    return parsed.toString();
+}
+function normalizeCrawlUrl(rawUrl) {
+    try {
+        const parsed = new URL(rawUrl);
+        if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+            return null;
+        parsed.hash = "";
+        if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
+            parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+        }
+        return parsed.toString();
+    }
+    catch {
+        return null;
+    }
+}
+/** Convert a page URL into a relative file path preserving the URL hierarchy.
+ *  e.g. https://example.com/docs/guide → docs/guide
+ *       https://example.com/           → index
+ */
+function urlToRelativePath(rawUrl) {
+    const parsed = new URL(rawUrl);
+    const segments = parsed.pathname
+        .split("/")
+        .filter(Boolean)
+        .map((segment) => slugifySegment(segment))
+        .filter(Boolean);
+    if (parsed.search) {
+        const querySuffix = slugifySegment(parsed.search.slice(1));
+        if (querySuffix && segments.length > 0) {
+            segments[segments.length - 1] = `${segments[segments.length - 1]}_${querySuffix}`;
+        }
+    }
+    return segments.length > 0 ? segments.join("/") : "index";
+}
+function slugifySegment(value) {
+    return sanitizeString(value, 200)
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/g, "-")
+        .replace(/^-+|-+$/g, "");
+}
+function uniqueSlug(base, used) {
+    const seed = base || "website";
+    let candidate = seed;
+    let i = 2;
+    while (used.has(candidate)) {
+        candidate = `${seed}-${i}`;
+        i += 1;
+    }
+    used.add(candidate);
+    return candidate;
+}
+function coercePositiveInt(value, fallback) {
+    if (typeof value === "number" && Number.isInteger(value) && value > 0)
+        return value;
+    if (typeof value === "string") {
+        const parsed = Number.parseInt(value, 10);
+        if (Number.isInteger(parsed) && parsed > 0)
+            return parsed;
+    }
+    return fallback;
+}
+function looksLikeMarkup(body) {
+    return /<html[\s>]|<body[\s>]|<\/[a-z][\w:-]*>/i.test(body);
+}
+function extractHtmlTitle(html) {
+    const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+    if (title)
+        return decodeHtmlEntities(stripTags(title)).trim();
+    const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1];
+    if (h1)
+        return decodeHtmlEntities(stripTags(h1)).trim();
+    return undefined;
+}
+function extractTextTitle(text) {
+    for (const line of text.split(/\r?\n/)) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        if (trimmed.startsWith("#"))
+            return trimmed.replace(/^#+\s*/, "");
+        return trimmed.slice(0, 120);
+    }
+    return undefined;
+}
+function extractSameDocumentLinks(html, pageUrl) {
+    const links = [];
+    const hrefPattern = /<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>/gi;
+    for (const match of html.matchAll(hrefPattern)) {
+        const href = match[2]?.trim();
+        if (!href || href.startsWith("#"))
+            continue;
+        try {
+            const resolved = new URL(href, pageUrl);
+            if (!isSafeLinkUrl(resolved))
+                continue;
+            links.push(resolved);
+        }
+        catch {
+            /* ignore malformed links */
+        }
+    }
+    return links;
+}
+function htmlToMarkdown(html, pageUrl) {
+    let text = html;
+    text = stripDangerousBlockTag(text, "script");
+    text = stripDangerousBlockTag(text, "style");
+    text = stripDangerousBlockTag(text, "noscript");
+    text = stripDangerousBlockTag(text, "template");
+    text = text.replace(/<pre\b[^>]*><code\b[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_match, code) => {
+        const decoded = decodeHtmlEntities(stripTags(code)).trim();
+        return decoded ? `\n\n\`\`\`\n${decoded}\n\`\`\`\n\n` : "\n\n";
+    });
+    text = text.replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, (_match, code) => {
+        const decoded = decodeHtmlEntities(stripTags(code)).trim();
+        return decoded ? `\`${decoded}\`` : "";
+    });
+    text = text.replace(/<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi, (_match, _q, href, body) => {
+        const label = decodeHtmlEntities(stripTags(body)).trim();
+        if (!label)
+            return "";
+        try {
+            const resolved = new URL(href, pageUrl);
+            if (!isSafeLinkUrl(resolved))
+                return label;
+            return `[${label}](${resolved})`;
+        }
+        catch {
+            return label;
+        }
+    });
+    text = text.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level, body) => {
+        const heading = decodeHtmlEntities(stripTags(body)).trim();
+        return heading ? `\n\n${"#".repeat(Number(level))} ${heading}\n\n` : "\n\n";
+    });
+    text = text.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_match, body) => {
+        const item = decodeHtmlEntities(stripTags(body)).trim();
+        return item ? `\n- ${item}` : "";
+    });
+    text = text.replace(/<(p|div|section|article|main|header|footer|blockquote|table|tr)\b[^>]*>/gi, "\n\n");
+    text = text.replace(/<\/(p|div|section|article|main|header|footer|blockquote|table|tr)>/gi, "\n\n");
+    text = text.replace(/<br\s*\/?>/gi, "\n");
+    text = text.replace(/<\/?(ul|ol)\b[^>]*>/gi, "\n");
+    text = decodeHtmlEntities(stripTags(text));
+    text = text
+        .replace(/\r/g, "")
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    return text;
+}
+function stripTags(value) {
+    return value.replace(/<[^>]+>/g, " ");
+}
+function decodeHtmlEntities(value) {
+    const namedEntities = {
+        nbsp: " ",
+        amp: "&",
+        lt: "<",
+        gt: ">",
+        quot: '"',
+        apos: "'",
+    };
+    return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity) => {
+        const normalized = String(entity).toLowerCase();
+        if (normalized.startsWith("#x")) {
+            return safeCodePointToString(Number.parseInt(normalized.slice(2), 16)) ?? match;
+        }
+        if (normalized.startsWith("#")) {
+            return safeCodePointToString(Number.parseInt(normalized.slice(1), 10)) ?? match;
+        }
+        return namedEntities[normalized] ?? match;
+    });
+}
+function isAssetLikePath(pathname) {
+    // Keep this list intentionally conservative so docs paths are still crawled
+    // unless they clearly point at static assets/binaries.
+    return /\.(css|js|json|png|jpe?g|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|woff2?)$/i.test(pathname);
+}
+function isSafeLinkUrl(url) {
+    return url.protocol === "http:" || url.protocol === "https:";
+}
+function stripDangerousBlockTag(value, tagName) {
+    const pattern = new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}\\s*>`, "gi");
+    return value.replace(pattern, "");
+}
+function safeCodePointToString(value) {
+    if (!Number.isFinite(value) || value < 0 || value > 0x10ffff)
+        return undefined;
+    try {
+        return String.fromCodePoint(value);
+    }
+    catch {
+        return undefined;
+    }
+}
+export { ensureWebsiteMirror, getCachePaths, validateWebsiteInputUrl, validateWebsiteUrl, WebsiteStashProvider };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akm-cli",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "type": "module",
   "description": "akm (Agent Kit Manager) — A package manager for AI agent skills, commands, tools, and knowledge. Works with Claude Code, OpenCode, Cursor, and any AI coding assistant.",
   "keywords": [