@ebowwa/markdown-docs-scraper 1.0.0 → 1.2.0

package/src/cli.js

@@ -0,0 +1,160 @@
+#!/usr/bin/env bun
+"use strict";
+/**
+ * CLI for @ebowwa/markdown-docs-scraper
+ */
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var commander_1 = require("commander");
+var index_js_1 = require("./index.js");
+commander_1.program
+    .name("markdown-docs-scraper")
+    .description("Scrape and mirror markdown-based documentation sites")
+    .version("1.0.0");
+commander_1.program
+    .command("scrape")
+    .description("Scrape documentation from a URL")
+    .requiredOption("-u, --url <url>", "Base URL of the documentation site")
+    .option("-o, --output <dir>", "Output directory", "./docs")
+    .option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en")
+    .option("-c, --concurrency <num>", "Concurrency level", "5")
+    .option("--discover", "Discover pages before scraping", false)
+    .action(function (options) { return __awaiter(void 0, void 0, void 0, function () {
+    var scraperOptions, result;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0:
+                scraperOptions = {
+                    baseUrl: options.url,
+                    docsPath: options.docsPath,
+                    outputDir: options.output,
+                    concurrency: parseInt(options.concurrency),
+                };
+                console.log("\uD83D\uDD0D Scraping ".concat(options.url, "..."));
+                console.log("\uD83D\uDCC1 Output: ".concat(options.output));
+                console.log();
+                return [4 /*yield*/, (0, index_js_1.scrapeMarkdownDocs)(scraperOptions)];
+            case 1:
+                result = _a.sent();
+                console.log();
+                console.log("Summary:");
+                console.log("  Downloaded: ".concat(result.downloaded.length));
+                console.log("  Failed: ".concat(result.failed.length));
+                console.log("  Duration: ".concat((result.duration / 1000).toFixed(2), "s"));
+                if (result.failed.length > 0) {
+                    console.log();
+                    console.log("Failed pages:");
+                    result.failed.slice(0, 10).forEach(function (f) {
+                        console.log("  \u274C ".concat(f.url, ": ").concat(f.error));
+                    });
+                    if (result.failed.length > 10) {
+                        console.log("  ... and ".concat(result.failed.length - 10, " more"));
+                    }
+                }
+                return [2 /*return*/];
+        }
+    });
+}); });
+commander_1.program
+    .command("discover")
+    .description("Discover all available documentation pages")
+    .requiredOption("-u, --url <url>", "Base URL of the documentation site")
+    .option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en")
+    .action(function (options) { return __awaiter(void 0, void 0, void 0, function () {
+    var scraper, pages;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0:
+                scraper = new index_js_1.MarkdownDocsScraper({
+                    baseUrl: options.url,
+                    docsPath: options.docsPath,
+                });
+                console.log("\uD83D\uDD0D Discovering pages from ".concat(options.url, "..."));
+                return [4 /*yield*/, scraper.discoverPages()];
+            case 1:
+                pages = _a.sent();
+                console.log("\nFound ".concat(pages.length, " pages:\n"));
+                pages.forEach(function (page) {
+                    console.log("  - ".concat(page));
+                });
+                return [2 /*return*/];
+        }
+    });
+}); });
+commander_1.program
+    .command("anthropic")
+    .description("Quick scrape of Anthropic Claude Code docs (uses llms.txt)")
+    .option("-o, --output <dir>", "Output directory", "./docs")
+    .action(function (options) { return __awaiter(void 0, void 0, void 0, function () {
+    var scraperOptions, result;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0:
+                scraperOptions = {
+                    baseUrl: "https://code.claude.com",
+                    docsPath: "/docs/en",
+                    outputDir: options.output,
+                };
+                console.log("🔍 Scraping Anthropic Claude Code documentation...");
+                console.log("📋 Using llms.txt index for complete page list");
+                console.log("\uD83D\uDCC1 Output: ".concat(options.output));
+                console.log();
+                return [4 /*yield*/, (0, index_js_1.scrapeMarkdownDocs)(__assign(__assign({}, scraperOptions), { useLlms: true }))];
+            case 1:
+                result = _a.sent();
+                console.log();
+                console.log("Summary:");
+                console.log("  Downloaded: ".concat(result.downloaded.length));
+                console.log("  Failed: ".concat(result.failed.length));
+                console.log("  Duration: ".concat((result.duration / 1000).toFixed(2), "s"));
+                return [2 /*return*/];
+        }
+    });
+}); });
+commander_1.program.parse();

package/src/cli.ts

@@ -19,12 +19,16 @@ program
   .option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en")
   .option("-c, --concurrency <num>", "Concurrency level", "5")
   .option("--discover", "Discover pages before scraping", false)
+  .option("--llms-paths <paths>", "Comma-separated llms.txt paths to try", "/llms.txt,/docs/llms.txt")
+  .option("--no-subdomain", "Disable docs/doc subdomain fallback", false)
   .action(async (options) => {
     const scraperOptions: ScraperOptions = {
       baseUrl: options.url,
       docsPath: options.docsPath,
       outputDir: options.output,
       concurrency: parseInt(options.concurrency),
+      llmsPaths: options.llmsPaths.split(","),
+      tryDocsSubdomain: !options.noSubdomain,
     };
 
     console.log(`🔍 Scraping ${options.url}...`);
@@ -56,10 +60,16 @@ program
   .description("Discover all available documentation pages")
   .requiredOption("-u, --url <url>", "Base URL of the documentation site")
   .option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en")
+  .option("--llms-paths <paths>", "Comma-separated llms.txt paths to try", "/llms.txt,/docs/llms.txt")
+  .option("--no-subdomain", "Disable docs/doc subdomain fallback", false)
   .action(async (options) => {
+    console.log("DEBUG CLI: Options received:", options);
+
     const scraper = new MarkdownDocsScraper({
       baseUrl: options.url,
       docsPath: options.docsPath,
+      llmsPaths: options.llmsPaths.split(","),
+      tryDocsSubdomain: !options.noSubdomain,
     });
 
     console.log(`🔍 Discovering pages from ${options.url}...`);
@@ -67,7 +77,8 @@ program
 
     console.log(`\nFound ${pages.length} pages:\n`);
     pages.forEach((page) => {
-      console.log(`  - ${page}`);
+      const path = page.category ? `${page.category}/${page.page}` : page.page;
+      console.log(`  - ${path}`);
     });
   });