@ebowwa/markdown-docs-scraper 1.0.0 → 1.2.0

package/src/index.js

@@ -0,0 +1,487 @@
+"use strict";
+/**
+ * @ebowwa/markdown-docs-scraper
+ *
+ * Scrape and mirror markdown-based documentation sites
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarkdownDocsScraper = void 0;
+exports.scrapeMarkdownDocs = scrapeMarkdownDocs;
+// ============================================================================
+// SCRAPER
+// ============================================================================
+var MarkdownDocsScraper = /** @class */ (function () {
+    function MarkdownDocsScraper(options) {
+        this.options = {
+            baseUrl: options.baseUrl,
+            docsPath: options.docsPath || "/docs/en",
+            categories: options.categories || {},
+            outputDir: options.outputDir || "./docs",
+            concurrency: options.concurrency || 5,
+            onProgress: options.onProgress || (function () { }),
+        };
+    }
+    /**
+     * Fetch markdown content from a URL
+     */
+    MarkdownDocsScraper.prototype.fetchMarkdown = function (url) {
+        return __awaiter(this, void 0, void 0, function () {
+            var response, contentType, error_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        _a.trys.push([0, 3, , 4]);
+                        return [4 /*yield*/, fetch(url, {
+                                headers: {
+                                    Accept: "text/markdown, text/plain",
+                                    "User-Agent": "@ebowwa/markdown-docs-scraper",
+                                },
+                            })];
+                    case 1:
+                        response = _a.sent();
+                        if (!response.ok) {
+                            return [2 /*return*/, null];
+                        }
+                        contentType = response.headers.get("content-type") || "";
+                        if (!contentType.includes("markdown") && !contentType.includes("text/plain")) {
+                            // Try to parse anyway - some sites return incorrect content-type
+                        }
+                        return [4 /*yield*/, response.text()];
+                    case 2: return [2 /*return*/, _a.sent()];
+                    case 3:
+                        error_1 = _a.sent();
+                        console.error("Error fetching ".concat(url, ":"), error_1);
+                        return [2 /*return*/, null];
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Extract title from markdown content
+     */
+    MarkdownDocsScraper.prototype.extractTitle = function (markdown) {
+        var titleMatch = markdown.match(/^#\s+(.+)$/m);
+        return titleMatch ? titleMatch[1].trim() : "Untitled";
+    };
+    /**
+     * Sanitize filename from URL path
+     */
+    MarkdownDocsScraper.prototype.sanitizeFilename = function (path) {
+        return path
+            .toLowerCase()
+            .replace(/[^a-z0-9/]+/g, "-")
+            .replace(/^-|-$/g, "")
+            .replace(/\//g, "/");
+    };
+    /**
+     * Build URL for a documentation page
+     */
+    MarkdownDocsScraper.prototype.buildUrl = function (category, page) {
+        if (category) {
+            return "".concat(this.options.baseUrl).concat(this.options.docsPath, "/").concat(category, "/").concat(page, ".md");
+        }
+        else {
+            return "".concat(this.options.baseUrl).concat(this.options.docsPath, "/").concat(page, ".md");
+        }
+    };
+    /**
+     * Download a single documentation page
+     */
+    MarkdownDocsScraper.prototype.downloadPage = function (category, page) {
+        return __awaiter(this, void 0, void 0, function () {
+            var url, content;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        url = this.buildUrl(category, page);
+                        return [4 /*yield*/, this.fetchMarkdown(url)];
+                    case 1:
+                        content = _a.sent();
+                        if (!content) {
+                            return [2 /*return*/, null];
+                        }
+                        return [2 /*return*/, {
+                                url: url,
+                                title: this.extractTitle(content),
+                                content: content,
+                                category: category,
+                                pageName: page, // Store the page name for saving
+                            }];
+                }
+            });
+        });
+    };
+    /**
+     * Scrape pages discovered from llms.txt
+     */
+    MarkdownDocsScraper.prototype.scrapeFromLlms = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var startTime, downloaded, failed, pages, _loop_1, this_1, i, duration;
+            var _this = this;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        startTime = Date.now();
+                        downloaded = [];
+                        failed = [];
+                        return [4 /*yield*/, this.discoverPages()];
+                    case 1:
+                        pages = _a.sent();
+                        if (pages.length === 0) {
+                            console.log("No pages discovered, falling back to categories");
+                            return [2 /*return*/, this.scrape()];
+                        }
+                        console.log("Scraping ".concat(pages.length, " discovered pages..."));
+                        _loop_1 = function (i) {
+                            var batch, results;
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
+                                    case 0:
+                                        batch = pages.slice(i, i + this_1.options.concurrency);
+                                        return [4 /*yield*/, Promise.allSettled(batch.map(function (page) { return _this.downloadPage(page.category, page.page); }))];
+                                    case 1:
+                                        results = _b.sent();
+                                        results.forEach(function (result, index) {
+                                            var page = batch[index];
+                                            if (result.status === "fulfilled" && result.value) {
+                                                downloaded.push(result.value);
+                                            }
+                                            else {
+                                                failed.push({
+                                                    url: _this.buildUrl(page.category, page.page),
+                                                    error: result.status === "rejected" ? result.reason : "Not found",
+                                                });
+                                            }
+                                            _this.options.onProgress(downloaded.length + failed.length, pages.length);
+                                        });
+                                        return [2 /*return*/];
+                                }
+                            });
+                        };
+                        this_1 = this;
+                        i = 0;
+                        _a.label = 2;
+                    case 2:
+                        if (!(i < pages.length)) return [3 /*break*/, 5];
+                        return [5 /*yield**/, _loop_1(i)];
+                    case 3:
+                        _a.sent();
+                        _a.label = 4;
+                    case 4:
+                        i += this.options.concurrency;
+                        return [3 /*break*/, 2];
+                    case 5:
+                        duration = Date.now() - startTime;
+                        console.log("\u2705 Downloaded: ".concat(downloaded.length, " pages"));
+                        console.log("\u274C Failed: ".concat(failed.length, " pages"));
+                        console.log("\u23F1\uFE0F  Duration: ".concat((duration / 1000).toFixed(2), "s"));
+                        return [2 /*return*/, { downloaded: downloaded, failed: failed, duration: duration }];
+                }
+            });
+        });
+    };
+    /**
+     * Scrape all documentation pages
+     */
+    MarkdownDocsScraper.prototype.scrape = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var startTime, downloaded, failed, pages, total, _loop_2, this_2, i, duration;
+            var _this = this;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        startTime = Date.now();
+                        downloaded = [];
+                        failed = [];
+                        pages = this.getPagesToScrape();
+                        total = pages.length;
+                        console.log("Scraping ".concat(total, " pages from ").concat(this.options.baseUrl, "..."));
+                        _loop_2 = function (i) {
+                            var batch, results;
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
+                                    case 0:
+                                        batch = pages.slice(i, i + this_2.options.concurrency);
+                                        return [4 /*yield*/, Promise.allSettled(batch.map(function (page) { return _this.downloadPage(page.category, page.page); }))];
+                                    case 1:
+                                        results = _b.sent();
+                                        results.forEach(function (result, index) {
+                                            var page = batch[index];
+                                            if (result.status === "fulfilled" && result.value) {
+                                                downloaded.push(result.value);
+                                            }
+                                            else {
+                                                failed.push({
+                                                    url: _this.buildUrl(page.category, page.page),
+                                                    error: result.status === "rejected" ? result.reason : "Not found",
+                                                });
+                                            }
+                                            _this.options.onProgress(downloaded.length + failed.length, total);
+                                        });
+                                        return [2 /*return*/];
+                                }
+                            });
+                        };
+                        this_2 = this;
+                        i = 0;
+                        _a.label = 1;
+                    case 1:
+                        if (!(i < pages.length)) return [3 /*break*/, 4];
+                        return [5 /*yield**/, _loop_2(i)];
+                    case 2:
+                        _a.sent();
+                        _a.label = 3;
+                    case 3:
+                        i += this.options.concurrency;
+                        return [3 /*break*/, 1];
+                    case 4:
+                        duration = Date.now() - startTime;
+                        console.log("\u2705 Downloaded: ".concat(downloaded.length, " pages"));
+                        console.log("\u274C Failed: ".concat(failed.length, " pages"));
+                        console.log("\u23F1\uFE0F  Duration: ".concat((duration / 1000).toFixed(2), "s"));
+                        return [2 /*return*/, { downloaded: downloaded, failed: failed, duration: duration }];
+                }
+            });
+        });
+    };
+    /**
+     * Save scraped pages to disk
+     */
+    MarkdownDocsScraper.prototype.savePages = function (pages) {
+        return __awaiter(this, void 0, void 0, function () {
+            var fs, path, _i, pages_1, page, nameToUse, dir, filepath, header;
+            var _a;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0: return [4 /*yield*/, Promise.resolve().then(function () { return require("fs/promises"); })];
+                    case 1:
+                        fs = _b.sent();
+                        return [4 /*yield*/, Promise.resolve().then(function () { return require("path"); })];
+                    case 2:
+                        path = _b.sent();
+                        _i = 0, pages_1 = pages;
+                        _b.label = 3;
+                    case 3:
+                        if (!(_i < pages_1.length)) return [3 /*break*/, 7];
+                        page = pages_1[_i];
+                        nameToUse = page.pageName || ((_a = page.url.split("/").pop()) === null || _a === void 0 ? void 0 : _a.replace(".md", "")) || "untitled";
+                        dir = page.category
+                            ? path.join(this.options.outputDir, page.category)
+                            : this.options.outputDir;
+                        return [4 /*yield*/, fs.mkdir(dir, { recursive: true })];
+                    case 4:
+                        _b.sent();
+                        filepath = path.join(dir, "".concat(nameToUse, ".md"));
+                        header = "<!--\nSource: ".concat(page.url, "\nDownloaded: ").concat(new Date().toISOString(), "\n-->\n\n");
+                        return [4 /*yield*/, fs.writeFile(filepath, header + page.content, "utf-8")];
+                    case 5:
+                        _b.sent();
+                        _b.label = 6;
+                    case 6:
+                        _i++;
+                        return [3 /*break*/, 3];
+                    case 7: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get list of pages to scrape based on categories
+     */
+    MarkdownDocsScraper.prototype.getPagesToScrape = function () {
+        var pages = [];
+        for (var _i = 0, _a = Object.entries(this.options.categories); _i < _a.length; _i++) {
+            var _b = _a[_i], category = _b[0], pageList = _b[1];
+            for (var _c = 0, pageList_1 = pageList; _c < pageList_1.length; _c++) {
+                var page = pageList_1[_c];
+                pages.push({ category: category, page: page });
+            }
+        }
+        return pages;
+    };
+    /**
+     * Discover pages from llms.txt index
+     */
+    MarkdownDocsScraper.prototype.discoverPages = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var pages, llmsUrl, response, content, linkRegex, match, url, pagePath, pageName, pathParts, category, page, error_2;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        pages = [];
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 4, , 5]);
+                        llmsUrl = "".concat(this.options.baseUrl, "/docs/llms.txt");
+                        return [4 /*yield*/, fetch(llmsUrl, {
+                                headers: {
+                                    Accept: "text/plain",
+                                    "User-Agent": "@ebowwa/markdown-docs-scraper",
+                                },
+                            })];
+                    case 2:
+                        response = _a.sent();
+                        if (!response.ok) {
+                            console.warn("Could not fetch llms.txt from ".concat(llmsUrl));
+                            return [2 /*return*/, pages];
+                        }
+                        return [4 /*yield*/, response.text()];
+                    case 3:
+                        content = _a.sent();
+                        linkRegex = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
+                        match = void 0;
+                        while ((match = linkRegex.exec(content)) !== null) {
+                            url = match[2];
+                            pagePath = match[3];
+                            pageName = pagePath.replace(".md", "");
+                            pathParts = pageName.split("/");
+                            if (pathParts.length === 1) {
+                                // No category: just "page-name"
+                                pages.push({ category: "", page: pathParts[0] });
+                            }
+                            else if (pathParts.length === 2) {
+                                // Has category: "category/page-name"
+                                pages.push({ category: pathParts[0], page: pathParts[1] });
+                            }
+                            else {
+                                category = pathParts.slice(0, -1).join("/");
+                                page = pathParts[pathParts.length - 1];
+                                pages.push({ category: category, page: page });
+                            }
+                        }
+                        console.log("Discovered ".concat(pages.length, " pages from llms.txt"));
+                        return [3 /*break*/, 5];
+                    case 4:
+                        error_2 = _a.sent();
+                        console.error("Error discovering pages:", error_2);
+                        return [3 /*break*/, 5];
+                    case 5: return [2 /*return*/, pages];
+                }
+            });
+        });
+    };
+    /**
+     * Discover additional pages by parsing the docs index (fallback)
+     */
+    MarkdownDocsScraper.prototype.discoverPagesHtml = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var discovered, indexUrl, response, html, mdLinkRegex, match, path, error_3;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        discovered = [];
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 4, , 5]);
+                        indexUrl = "".concat(this.options.baseUrl).concat(this.options.docsPath);
+                        return [4 /*yield*/, fetch(indexUrl, {
+                                headers: {
+                                    Accept: "text/html",
+                                    "User-Agent": "@ebowwa/markdown-docs-scraper",
+                                },
+                            })];
+                    case 2:
+                        response = _a.sent();
+                        if (!response.ok) {
+                            return [2 /*return*/, discovered];
+                        }
+                        return [4 /*yield*/, response.text()];
+                    case 3:
+                        html = _a.sent();
+                        mdLinkRegex = /href="\/docs\/en\/([^"]+\.md)"/g;
+                        match = void 0;
+                        while ((match = mdLinkRegex.exec(html)) !== null) {
+                            path = match[1];
+                            if (!discovered.includes(path)) {
+                                discovered.push(path);
+                            }
+                        }
+                        console.log("Discovered ".concat(discovered.length, " additional pages from HTML"));
+                        return [3 /*break*/, 5];
+                    case 4:
+                        error_3 = _a.sent();
+                        console.error("Error discovering pages from HTML:", error_3);
+                        return [3 /*break*/, 5];
+                    case 5: return [2 /*return*/, discovered];
+                }
+            });
+        });
+    };
+    return MarkdownDocsScraper;
+}());
+exports.MarkdownDocsScraper = MarkdownDocsScraper;
+// ============================================================================
+// CONVENIENCE FUNCTION
+// ============================================================================
+/**
+ * Scrape markdown documentation with a single function call
+ */
+function scrapeMarkdownDocs(options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var scraper, result, _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    scraper = new MarkdownDocsScraper(options);
+                    if (!options.useLlms) return [3 /*break*/, 2];
+                    return [4 /*yield*/, scraper.scrapeFromLlms()];
+                case 1:
+                    _a = _b.sent();
+                    return [3 /*break*/, 4];
+                case 2: return [4 /*yield*/, scraper.scrape()];
+                case 3:
+                    _a = _b.sent();
+                    _b.label = 4;
+                case 4:
+                    result = _a;
+                    if (!options.outputDir) return [3 /*break*/, 6];
+                    return [4 /*yield*/, scraper.savePages(result.downloaded)];
+                case 5:
+                    _b.sent();
+                    _b.label = 6;
+                case 6: return [2 /*return*/, result];
+            }
+        });
+    });
+}
+// ============================================================================
+// EXPORTS
+// ============================================================================
+exports.default = MarkdownDocsScraper;