@dynamik-dev/refdocs 0.3.0 → 0.5.0

package/dist/src/crawl.js

@@ -0,0 +1,222 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join, dirname, extname } from "node:path";
+import * as cheerio from "cheerio";
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+import { JSDOM } from "jsdom";
+const DEFAULT_CRAWL_OPTIONS = {
+    maxPages: 200,
+    depth: 3,
+    delayMs: 150,
+};
+const USER_AGENT = "refdocs-cli/0.4.0";
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function normalizeUrl(raw) {
+    try {
+        const u = new URL(raw);
+        u.hash = "";
+        // Strip trailing slash unless it's the root
+        if (u.pathname.length > 1 && u.pathname.endsWith("/")) {
+            u.pathname = u.pathname.slice(0, -1);
+        }
+        return u.toString();
+    }
+    catch {
+        return raw;
+    }
+}
+function isSubPath(candidate, scope) {
+    const candidateUrl = new URL(candidate);
+    const scopeUrl = new URL(scope);
+    if (candidateUrl.origin !== scopeUrl.origin)
+        return false;
+    const scopePath = scopeUrl.pathname.endsWith("/")
+        ? scopeUrl.pathname
+        : scopeUrl.pathname + "/";
+    return (candidateUrl.pathname === scopeUrl.pathname ||
+        candidateUrl.pathname.startsWith(scopePath));
+}
+function urlToFilePath(pageUrl) {
+    const u = new URL(pageUrl);
+    let path = u.pathname;
+    if (path.endsWith("/"))
+        path = path + "index";
+    if (path === "")
+        path = "/index";
+    // Strip leading slash
+    path = path.replace(/^\//, "");
+    // If it already has an extension, keep it but ensure .md for HTML pages
+    const ext = extname(path);
+    if (!ext || ext === ".html" || ext === ".htm") {
+        path = path.replace(/\.(html|htm)$/, "") + ".md";
+    }
+    // Ensure .md extension for extensionless paths
+    if (!extname(path)) {
+        path = path + ".md";
+    }
+    return path;
+}
+function htmlToMarkdown(html, url) {
+    const dom = new JSDOM(html, { url });
+    const reader = new Readability(dom.window.document);
+    const article = reader.parse();
+    const turndown = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        bulletListMarker: "-",
+    });
+    if (article?.content) {
+        let md = turndown.turndown(article.content);
+        // Prepend the title as an h1 if readability extracted one
+        if (article.title) {
+            md = `# ${article.title}\n\n${md}`;
+        }
+        return md;
+    }
+    // Fallback: convert the whole body
+    return turndown.turndown(html);
+}
+function discoverLinks(html, pageUrl, scope) {
+    const $ = cheerio.load(html);
+    const links = [];
+    const base = new URL(pageUrl);
+    $("a[href]").each((_, el) => {
+        const href = $(el).attr("href");
+        if (!href)
+            return;
+        // Skip anchors, javascript:, mailto:, etc.
+        if (href.startsWith("#") || href.startsWith("javascript:") || href.startsWith("mailto:")) {
+            return;
+        }
+        try {
+            const resolved = new URL(href, base).toString();
+            const normalized = normalizeUrl(resolved);
+            // Only follow links within scope and same origin
+            if (isSubPath(normalized, scope)) {
+                // Skip common non-page extensions
+                const ext = extname(new URL(normalized).pathname).toLowerCase();
+                if (ext && ![".html", ".htm", ""].includes(ext))
+                    return;
+                links.push(normalized);
+            }
+        }
+        catch {
+            // Invalid URL, skip
+        }
+    });
+    return [...new Set(links)];
+}
+async function fetchPage(url) {
+    const response = await fetch(url, {
+        headers: {
+            "User-Agent": USER_AGENT,
+            Accept: "text/html,text/plain,text/markdown,*/*",
+        },
+        redirect: "follow",
+    });
+    if (!response.ok) {
+        throw new Error(`HTTP ${response.status} fetching ${url}`);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    const html = await response.text();
+    return { html, contentType };
+}
+export function isTextFileUrl(url) {
+    try {
+        const pathname = new URL(url).pathname.toLowerCase();
+        return pathname.endsWith(".txt") || pathname.endsWith(".md");
+    }
+    catch {
+        return false;
+    }
+}
+export function isGitHubUrl(url) {
+    try {
+        return new URL(url).hostname === "github.com";
+    }
+    catch {
+        return false;
+    }
+}
+export function deriveLocalPath(url) {
+    const u = new URL(url);
+    const hostname = u.hostname;
+    let path = u.pathname.replace(/^\//, "");
+    if (!path)
+        path = "index";
+    return `ref-docs/${hostname}/${path}`;
+}
+export function deriveCrawlDir(url) {
+    const u = new URL(url);
+    const hostname = u.hostname;
+    let path = u.pathname.replace(/^\//, "").replace(/\/+$/, "");
+    if (!path)
+        path = "root";
+    return `ref-docs/${hostname}/${path}`;
+}
+export async function fetchSingleFile(url) {
+    const response = await fetch(url, {
+        headers: { "User-Agent": USER_AGENT },
+        redirect: "follow",
+    });
+    if (!response.ok) {
+        throw new Error(`HTTP ${response.status} fetching ${url}`);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    const content = await response.text();
+    return { content, contentType };
+}
+export async function crawlSite(startUrl, outputDir, options) {
+    const opts = { ...DEFAULT_CRAWL_OPTIONS, ...options };
+    const scope = normalizeUrl(startUrl);
+    const visited = new Set();
+    const queue = [{ url: scope, depth: 0 }];
+    const pages = [];
+    let filesWritten = 0;
+    while (queue.length > 0 && filesWritten < opts.maxPages) {
+        const item = queue.shift();
+        const normalized = normalizeUrl(item.url);
+        if (visited.has(normalized))
+            continue;
+        visited.add(normalized);
+        try {
+            const { html, contentType } = await fetchPage(normalized);
+            // Only process HTML pages
+            if (!contentType.includes("text/html"))
+                continue;
+            const markdown = htmlToMarkdown(html, normalized);
+            const filePath = urlToFilePath(normalized);
+            const fullPath = join(outputDir, filePath);
+            mkdirSync(dirname(fullPath), { recursive: true });
+            writeFileSync(fullPath, markdown, "utf-8");
+            filesWritten++;
+            pages.push(normalized);
+            // Discover links if we haven't hit max depth
+            if (item.depth < opts.depth) {
+                const links = discoverLinks(html, normalized, scope);
+                for (const link of links) {
+                    if (!visited.has(normalizeUrl(link))) {
+                        queue.push({ url: link, depth: item.depth + 1 });
+                    }
+                }
+            }
+            // Throttle between requests
+            if (queue.length > 0) {
+                await sleep(opts.delayMs);
+            }
+        }
+        catch (err) {
+            // Log but don't fail the whole crawl for one bad page
+            const msg = err instanceof Error ? err.message : String(err);
+            process.stderr.write(`Warning: skipping ${normalized} (${msg})\n`);
+        }
+    }
+    const hitLimit = filesWritten >= opts.maxPages && queue.length > 0;
+    if (hitLimit) {
+        process.stderr.write(`Reached ${opts.maxPages} page limit. Use --max-pages to increase.\n`);
+    }
+    return { filesWritten, pages };
+}
+//# sourceMappingURL=crawl.js.map

package/dist/src/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAkB9B,MAAM,qBAAqB,GAAiB;IAC1C,QAAQ,EAAE,GAAG;IACb,KAAK,EAAE,CAAC;IACR,OAAO,EAAE,GAAG;CACb,CAAC;AAEF,MAAM,UAAU,GAAG,mBAAmB,CAAC;AAEvC,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QACZ,4CAA4C;QAC5C,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtD,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,SAAiB,EAAE,KAAa;IACjD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,YAAY,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC/C,CAAC,CAAC,QAAQ,CAAC,QAAQ;QACnB,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC;IAC5B,OAAO,CACL,YAAY,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;QAC3C,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAC5C,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,IAAI,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC;IACtB,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC;IAC9C,IAAI,IAAI,KAAK,EAAE;QAAE,IAAI,GAAG,QAAQ,CAAC;IACjC,sBAAsB;IACtB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC/B,wEAAwE;IACxE,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QAC9C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC;IACnD,CAAC;IACD,+CAA+C;IAC/C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,GAAG,IAAI,GAAG,KAAK,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,GAAW;IAC/C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;IAE/B,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;KACtB,CAAC,CAAC;IAEH,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;QACrB,IAAI,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5C,0DAA0D;QAC1D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,EAAE,GAAG,KAAK,OAAO,CAAC,KAAK,OAAO,EAAE,EAAE,CAAC;QACrC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,mCAAmC;IACnC,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,OAAe,EAAE,KAAa;IACjE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAE9B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,2CAA2C;QAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACzF,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;YAE1C,iDAAiD;YACjD,IAAI,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC;gBACjC,kCAAkC;gBAClC,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBAChE,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;oBAAE,OAAO;gBACxD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,oBAAoB;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,GAAW;IAClC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,OAAO,EAAE;YACP,YAAY,EAAE,UAAU;YACxB,MAAM,EAAE,wCAAwC;SACjD;QACD,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACrD,OAAO,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,YAAY,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC5B,IAAI,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACzC,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,OAAO,CAAC;IAC1B,OAAO,YAAY,QAAQ,IAAI,IAAI,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC5B,IAAI,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC7D,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,MAAM,CAAC;IACzB,OAAO,YAAY,QAAQ,IAAI,IAAI,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAW;IAC/C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE;QACrC,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,SAAiB,EACjB,OAA+B;IAE/B,MAAM,IAAI,GAAG,EAAE,GAAG,qBAAqB,EAAE,GAAG,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAqC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC5B,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE1C,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,SAAS;QACtC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAExB,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,CAAC;YAE1D,0BAA0B;YAC1B,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAAE,SAAS;YAEjD,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAClD,MAAM,QAAQ,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAE3C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC3C,YAAY,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEvB,6CAA6C;YAC7C,IAAI,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;gBACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;wBACrC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;oBACnD,CAAC;gBACH,CAAC;YACH,CAAC;YAED,4BAA4B;YAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,sDAAsD;YACtD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,UAAU,KAAK,GAAG,KAAK,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,IAAI,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACnE,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,WAAW,IAAI,CAAC,QAAQ,6CAA6C,CACtE,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AACjC,CAAC"}

package/dist/src/index.js

@@ -1,22 +1,31 @@
 #!/usr/bin/env node
 import { Command } from "commander";
 import { join } from "node:path";
-import { loadConfig, configExists, initConfig } from "./config.js";
+import { existsSync } from "node:fs";
+import { loadConfig, configExists, initConfig, loadGlobalConfig, initGlobalConfig, getGlobalConfigDir } from "./config.js";
 import { buildIndex, loadPersistedIndex } from "./indexer.js";
-import { search } from "./search.js";
-import { addFromUrl, addLocalPath, removePath, updateSources } from "./add.js";
+import { search, mergeSearchResults } from "./search.js";
+import { addFromGitHub, addFromFileUrl, addFromCrawl, addLocalPath, removePath, updateSources } from "./add.js";
+import { isGitHubUrl } from "./crawl.js";
 const program = new Command();
 program
     .name("refdocs")
     .description("Local CLI tool for indexing and searching markdown documentation")
-    .version("0.3.0");
+    .version("0.5.0");
 program
     .command("init")
     .description("Create a .refdocs.json config file with defaults")
-    .action(() => {
+    .option("-g, --global", "initialize global config at ~/.refdocs/")
+    .action((opts) => {
     try {
-        initConfig(process.cwd());
-        console.log("Created .refdocs.json with default configuration.");
+        if (opts.global) {
+            initGlobalConfig();
+            console.log(`Created global config at ${getGlobalConfigDir()}/.refdocs.json`);
+        }
+        else {
+            initConfig(process.cwd());
+            console.log("Created .refdocs.json with default configuration.");
+        }
     }
     catch (err) {
         console.error(err.message);
@@ -26,13 +35,27 @@ program
 program
     .command("index")
     .description("Index all markdown files in configured paths")
-    .action(() => {
+    .option("-g, --global", "index the global config")
+    .action((opts) => {
     try {
-        const { config, configDir } = loadConfig();
-        const summary = buildIndex(config, configDir);
-        console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
-        console.log(`Index size: ${(summary.indexSizeBytes / 1024).toFixed(1)} KB`);
-        console.log(`Done in ${summary.elapsedMs}ms`);
+        if (opts.global) {
+            const globalResult = loadGlobalConfig();
+            if (!globalResult) {
+                console.error("No global config found. Run `refdocs init --global` first.");
+                process.exit(1);
+            }
+            const summary = buildIndex(globalResult.config, globalResult.configDir);
+            console.log(`[global] Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+            console.log(`Index size: ${(summary.indexSizeBytes / 1024).toFixed(1)} KB`);
+            console.log(`Done in ${summary.elapsedMs}ms`);
+        }
+        else {
+            const { config, configDir } = loadConfig();
+            const summary = buildIndex(config, configDir);
+            console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+            console.log(`Index size: ${(summary.indexSizeBytes / 1024).toFixed(1)} KB`);
+            console.log(`Done in ${summary.elapsedMs}ms`);
+        }
     }
     catch (err) {
         console.error(err.message);
@@ -48,14 +71,44 @@ program
     .option("--raw", "output chunk body only, no metadata")
     .action((query, opts) => {
     try {
-        const { config, configDir } = loadConfig();
-        const indexPath = join(configDir, config.index);
-        const { index } = loadPersistedIndex(indexPath, config);
         const maxResults = Math.min(Math.max(1, parseInt(opts.results, 10) || 3), 10);
-        const results = search(index, query, {
-            maxResults,
-            fileFilter: opts.file,
-        });
+        let localResults = [];
+        let hasLocalIndex = false;
+        let localError = null;
+        try {
+            const { config, configDir } = loadConfig();
+            const indexPath = join(configDir, config.index);
+            const { index } = loadPersistedIndex(indexPath, config);
+            hasLocalIndex = true;
+            localResults = search(index, query, { maxResults, fileFilter: opts.file });
+        }
+        catch (err) {
+            localError = err;
+        }
+        let globalResults = [];
+        let hasGlobalIndex = false;
+        const globalConfig = loadGlobalConfig();
+        if (globalConfig) {
+            const globalIndexPath = join(globalConfig.configDir, globalConfig.config.index);
+            if (existsSync(globalIndexPath)) {
+                try {
+                    const { index: globalIndex } = loadPersistedIndex(globalIndexPath, globalConfig.config);
+                    hasGlobalIndex = true;
+                    globalResults = search(globalIndex, query, { maxResults, fileFilter: opts.file });
+                    globalResults = globalResults.map((r) => ({
+                        ...r,
+                        file: `[global] ${r.file}`,
+                    }));
+                }
+                catch {
+                    // Global index exists but failed to load, skip
+                }
+            }
+        }
+        if (!hasLocalIndex && !hasGlobalIndex) {
+            throw localError ?? new Error("Index not found. Run `refdocs index` first.");
+        }
+        const results = mergeSearchResults(localResults, globalResults, maxResults);
         if (results.length === 0) {
             console.log("No results found.");
             return;
@@ -81,17 +134,31 @@ program
 program
     .command("list")
     .description("List all indexed files and their chunk counts")
-    .action(() => {
+    .option("-g, --global", "list global indexed files")
+    .action((opts) => {
     try {
-        const { config, configDir } = loadConfig();
+        let config, configDir;
+        if (opts.global) {
+            const globalResult = loadGlobalConfig();
+            if (!globalResult) {
+                console.error("No global config found. Run `refdocs init --global` first.");
+                process.exit(1);
+            }
+            config = globalResult.config;
+            configDir = globalResult.configDir;
+        }
+        else {
+            ({ config, configDir } = loadConfig());
+        }
         const indexPath = join(configDir, config.index);
         const { chunks } = loadPersistedIndex(indexPath, config);
         const byFile = new Map();
         for (const chunk of chunks) {
             byFile.set(chunk.file, (byFile.get(chunk.file) || 0) + 1);
         }
+        const label = opts.global ? "[global] " : "";
         for (const [file, count] of [...byFile.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
-            console.log(`${file} (${count} chunk${count !== 1 ? "s" : ""})`);
+            console.log(`${label}${file} (${count} chunk${count !== 1 ? "s" : ""})`);
         }
         console.log(`\n${byFile.size} files, ${chunks.length} chunks total`);
     }
@@ -103,9 +170,22 @@ program
 program
     .command("info <file>")
     .description("Show all chunks for a specific file")
-    .action((file) => {
+    .option("-g, --global", "show info from global index")
+    .action((file, opts) => {
     try {
-        const { config, configDir } = loadConfig();
+        let config, configDir;
+        if (opts.global) {
+            const globalResult = loadGlobalConfig();
+            if (!globalResult) {
+                console.error("No global config found. Run `refdocs init --global` first.");
+                process.exit(1);
+            }
+            config = globalResult.config;
+            configDir = globalResult.configDir;
+        }
+        else {
+            ({ config, configDir } = loadConfig());
+        }
         const indexPath = join(configDir, config.index);
         const { chunks } = loadPersistedIndex(indexPath, config);
         const fileChunks = chunks.filter((c) => c.file === file);
@@ -125,36 +205,78 @@ program
 });
 program
     .command("add <source>")
-    .description("Add a local path or download markdown docs from a GitHub URL")
+    .description("Add docs from a local path, GitHub URL, file URL, or crawled website")
     .option("--path <dir>", "override local storage directory")
     .option("--branch <branch>", "override branch detection from URL")
     .option("--no-index", "skip auto re-indexing after download")
+    .option("-g, --global", "store docs in global ~/.refdocs/ directory")
+    .option("--crawl", "crawl a website and convert pages to markdown")
+    .option("--max-pages <count>", "max pages to crawl (default: 200)")
+    .option("--depth <levels>", "max crawl depth (default: 3)")
     .action(async (source, opts) => {
     try {
-        const cwd = process.cwd();
-        if (!configExists(cwd)) {
-            initConfig(cwd);
-            console.log("Initialized .refdocs.json with default configuration.");
-        }
-        const { config, configDir } = loadConfig();
         const isUrl = source.startsWith("http://") || source.startsWith("https://");
-        if (isUrl) {
-            const result = await addFromUrl(source, { path: opts.path, branch: opts.branch }, configDir, config);
-            console.log(`Downloaded ${result.filesWritten} markdown files → ${result.localPath}/`);
-            console.log(`Source: ${result.source.owner}/${result.source.repo} (${result.source.branch})`);
+        if (opts.global && !isUrl) {
+            console.error("The --global flag can only be used with URLs, not local paths.");
+            process.exit(1);
+        }
+        let configDir;
+        let config;
+        if (opts.global) {
+            initGlobalConfig();
+            const globalResult = loadGlobalConfig();
+            if (!globalResult) {
+                console.error("Failed to initialize global config.");
+                process.exit(1);
+            }
+            configDir = globalResult.configDir;
+            config = globalResult.config;
+        }
+        else {
+            const cwd = process.cwd();
+            if (!configExists(cwd)) {
+                initConfig(cwd);
+                console.log("Initialized .refdocs.json with default configuration.");
+            }
+            ({ config, configDir } = loadConfig());
+        }
+        const label = opts.global ? "[global] " : "";
+        if (isUrl && opts.crawl) {
+            // Crawl mode: spider the site and convert to markdown
+            const maxPages = opts.maxPages ? parseInt(opts.maxPages, 10) : undefined;
+            const depth = opts.depth ? parseInt(opts.depth, 10) : undefined;
+            console.log(`Crawling ${source}...`);
+            const result = await addFromCrawl(source, { path: opts.path, maxPages, depth }, configDir, config);
+            console.log(`${label}Converted ${result.filesWritten} pages → ${result.localPath}/`);
+            if (opts.index && result.filesWritten > 0) {
+                reindex(opts.global, label);
+            }
+        }
+        else if (isUrl && isGitHubUrl(source)) {
+            // GitHub mode: download tarball
+            const result = await addFromGitHub(source, { path: opts.path, branch: opts.branch }, configDir, config);
+            console.log(`${label}Downloaded ${result.filesWritten} markdown files → ${result.localPath}/`);
+            if (result.source.type === "github") {
+                console.log(`Source: ${result.source.owner}/${result.source.repo} (${result.source.branch})`);
+            }
             if (opts.index && result.filesWritten > 0) {
-                const { config: freshConfig, configDir: freshDir } = loadConfig();
-                const summary = buildIndex(freshConfig, freshDir);
-                console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+                reindex(opts.global, label);
+            }
+        }
+        else if (isUrl) {
+            // Single file URL mode
+            const result = await addFromFileUrl(source, { path: opts.path }, configDir, config);
+            console.log(`${label}Downloaded 1 file → ${result.localPath}`);
+            if (opts.index && result.filesWritten > 0) {
+                reindex(opts.global, label);
             }
         }
         else {
+            // Local path mode
             const result = addLocalPath(source, configDir, config);
             console.log(`Added ${result.localPath} to paths`);
             if (opts.index) {
-                const { config: freshConfig, configDir: freshDir } = loadConfig();
-                const summary = buildIndex(freshConfig, freshDir);
-                console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+                reindex(opts.global, label);
             }
         }
     }
@@ -167,20 +289,33 @@ program
     .command("update")
     .description("Re-pull all tracked sources and re-index")
     .option("--no-index", "skip auto re-indexing after update")
+    .option("-g, --global", "update global sources")
     .action(async (opts) => {
     try {
-        const { config, configDir } = loadConfig();
+        let config, configDir;
+        if (opts.global) {
+            const globalResult = loadGlobalConfig();
+            if (!globalResult) {
+                console.error("No global config found. Run `refdocs init --global` first.");
+                process.exit(1);
+            }
+            config = globalResult.config;
+            configDir = globalResult.configDir;
+        }
+        else {
+            ({ config, configDir } = loadConfig());
+        }
         const token = process.env.GITHUB_TOKEN ?? undefined;
         const results = await updateSources(config, configDir, token);
+        const label = opts.global ? "[global] " : "";
         for (const r of results) {
-            console.log(`Updated ${r.source.owner}/${r.source.repo} → ${r.filesWritten} files`);
+            const desc = formatSourceDescription(r.source);
+            console.log(`${label}Updated ${desc} → ${r.filesWritten} files`);
         }
         const totalFiles = results.reduce((sum, r) => sum + r.filesWritten, 0);
         console.log(`\n${results.length} source${results.length !== 1 ? "s" : ""} updated (${totalFiles} files total)`);
         if (opts.index && totalFiles > 0) {
-            const { config: freshConfig, configDir: freshDir } = loadConfig();
-            const summary = buildIndex(freshConfig, freshDir);
-            console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+            reindex(opts.global, label);
         }
     }
     catch (err) {
@@ -192,22 +327,34 @@ program
     .command("remove <path>")
     .description("Remove a path from the index configuration")
     .option("--no-index", "skip auto re-indexing after removal")
+    .option("-g, --global", "remove from global config")
     .action((path, opts) => {
     try {
-        const { config, configDir } = loadConfig();
+        let config, configDir;
+        if (opts.global) {
+            const globalResult = loadGlobalConfig();
+            if (!globalResult) {
+                console.error("No global config found. Run `refdocs init --global` first.");
+                process.exit(1);
+            }
+            config = globalResult.config;
+            configDir = globalResult.configDir;
+        }
+        else {
+            ({ config, configDir } = loadConfig());
+        }
         const result = removePath(path, configDir, config);
         if (!result.removed) {
             console.error(`Path "${path}" not found in configured paths.`);
             process.exit(1);
         }
-        console.log(`Removed ${path} from paths`);
+        const label = opts.global ? "[global] " : "";
+        console.log(`${label}Removed ${path} from paths`);
         if (result.sourceRemoved) {
-            console.log(`Removed associated source`);
+            console.log(`${label}Removed associated source`);
         }
         if (opts.index) {
-            const { config: freshConfig, configDir: freshDir } = loadConfig();
-            const summary = buildIndex(freshConfig, freshDir);
-            console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+            reindex(opts.global, label);
         }
     }
     catch (err) {
@@ -215,6 +362,30 @@ program
         process.exit(1);
     }
 });
+function reindex(global, label) {
+    if (global) {
+        const freshGlobal = loadGlobalConfig();
+        if (freshGlobal) {
+            const summary = buildIndex(freshGlobal.config, freshGlobal.configDir);
+            console.log(`${label}Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+        }
+    }
+    else {
+        const { config: freshConfig, configDir: freshDir } = loadConfig();
+        const summary = buildIndex(freshConfig, freshDir);
+        console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
+    }
+}
+function formatSourceDescription(source) {
+    switch (source.type) {
+        case "github":
+            return `${source.owner}/${source.repo}`;
+        case "file":
+            return source.url;
+        case "crawl":
+            return source.url;
+    }
+}
 function formatResults(results) {
     results.forEach((r, i) => {
         console.log(`# [${i + 1}] ${r.file}:${r.lines[0]}-${r.lines[1]}`);