@ebowwa/markdown-docs-scraper 1.0.0

package/dist/index.js

@@ -0,0 +1,247 @@
+import { createRequire } from "node:module";
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __toESM = (mod, isNodeMode, target) => {
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: () => mod[key],
+        enumerable: true
+      });
+  return to;
+};
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+
+// src/index.ts
+class MarkdownDocsScraper {
+  options;
+  constructor(options) {
+    this.options = {
+      baseUrl: options.baseUrl,
+      docsPath: options.docsPath || "/docs/en",
+      categories: options.categories || {},
+      outputDir: options.outputDir || "./docs",
+      concurrency: options.concurrency || 5,
+      onProgress: options.onProgress || (() => {})
+    };
+  }
+  async fetchMarkdown(url) {
+    try {
+      const response = await fetch(url, {
+        headers: {
+          Accept: "text/markdown, text/plain",
+          "User-Agent": "@ebowwa/markdown-docs-scraper"
+        }
+      });
+      if (!response.ok) {
+        return null;
+      }
+      const contentType = response.headers.get("content-type") || "";
+      if (!contentType.includes("markdown") && !contentType.includes("text/plain")) {}
+      return await response.text();
+    } catch (error) {
+      console.error(`Error fetching ${url}:`, error);
+      return null;
+    }
+  }
+  extractTitle(markdown) {
+    const titleMatch = markdown.match(/^#\s+(.+)$/m);
+    return titleMatch ? titleMatch[1].trim() : "Untitled";
+  }
+  sanitizeFilename(path) {
+    return path.toLowerCase().replace(/[^a-z0-9/]+/g, "-").replace(/^-|-$/g, "").replace(/\//g, "/");
+  }
+  buildUrl(category, page) {
+    if (category) {
+      return `${this.options.baseUrl}${this.options.docsPath}/${category}/${page}.md`;
+    } else {
+      return `${this.options.baseUrl}${this.options.docsPath}/${page}.md`;
+    }
+  }
+  async downloadPage(category, page) {
+    const url = this.buildUrl(category, page);
+    const content = await this.fetchMarkdown(url);
+    if (!content) {
+      return null;
+    }
+    return {
+      url,
+      title: this.extractTitle(content),
+      content,
+      category,
+      pageName: page
+    };
+  }
+  async scrapeFromLlms() {
+    const startTime = Date.now();
+    const downloaded = [];
+    const failed = [];
+    const pages = await this.discoverPages();
+    if (pages.length === 0) {
+      console.log("No pages discovered, falling back to categories");
+      return this.scrape();
+    }
+    console.log(`Scraping ${pages.length} discovered pages...`);
+    for (let i = 0;i < pages.length; i += this.options.concurrency) {
+      const batch = pages.slice(i, i + this.options.concurrency);
+      const results = await Promise.allSettled(batch.map((page) => this.downloadPage(page.category, page.page)));
+      results.forEach((result, index) => {
+        const page = batch[index];
+        if (result.status === "fulfilled" && result.value) {
+          downloaded.push(result.value);
+        } else {
+          failed.push({
+            url: this.buildUrl(page.category, page.page),
+            error: result.status === "rejected" ? result.reason : "Not found"
+          });
+        }
+        this.options.onProgress(downloaded.length + failed.length, pages.length);
+      });
+    }
+    const duration = Date.now() - startTime;
+    console.log(`✅ Downloaded: ${downloaded.length} pages`);
+    console.log(`❌ Failed: ${failed.length} pages`);
+    console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
+    return { downloaded, failed, duration };
+  }
+  async scrape() {
+    const startTime = Date.now();
+    const downloaded = [];
+    const failed = [];
+    const pages = this.getPagesToScrape();
+    const total = pages.length;
+    console.log(`Scraping ${total} pages from ${this.options.baseUrl}...`);
+    for (let i = 0;i < pages.length; i += this.options.concurrency) {
+      const batch = pages.slice(i, i + this.options.concurrency);
+      const results = await Promise.allSettled(batch.map((page) => this.downloadPage(page.category, page.page)));
+      results.forEach((result, index) => {
+        const page = batch[index];
+        if (result.status === "fulfilled" && result.value) {
+          downloaded.push(result.value);
+        } else {
+          failed.push({
+            url: this.buildUrl(page.category, page.page),
+            error: result.status === "rejected" ? result.reason : "Not found"
+          });
+        }
+        this.options.onProgress(downloaded.length + failed.length, total);
+      });
+    }
+    const duration = Date.now() - startTime;
+    console.log(`✅ Downloaded: ${downloaded.length} pages`);
+    console.log(`❌ Failed: ${failed.length} pages`);
+    console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
+    return { downloaded, failed, duration };
+  }
+  async savePages(pages) {
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    for (const page of pages) {
+      const nameToUse = page.pageName || page.url.split("/").pop()?.replace(".md", "") || "untitled";
+      const dir = page.category ? path.join(this.options.outputDir, page.category) : this.options.outputDir;
+      await fs.mkdir(dir, { recursive: true });
+      const filepath = path.join(dir, `${nameToUse}.md`);
+      const header = `<!--
+Source: ${page.url}
+Downloaded: ${new Date().toISOString()}
+-->
+
+`;
+      await fs.writeFile(filepath, header + page.content, "utf-8");
+    }
+  }
+  getPagesToScrape() {
+    const pages = [];
+    for (const [category, pageList] of Object.entries(this.options.categories)) {
+      for (const page of pageList) {
+        pages.push({ category, page });
+      }
+    }
+    return pages;
+  }
+  async discoverPages() {
+    const pages = [];
+    try {
+      const llmsUrl = `${this.options.baseUrl}/docs/llms.txt`;
+      const response = await fetch(llmsUrl, {
+        headers: {
+          Accept: "text/plain",
+          "User-Agent": "@ebowwa/markdown-docs-scraper"
+        }
+      });
+      if (!response.ok) {
+        console.warn(`Could not fetch llms.txt from ${llmsUrl}`);
+        return pages;
+      }
+      const content = await response.text();
+      const linkRegex = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
+      let match;
+      while ((match = linkRegex.exec(content)) !== null) {
+        const url = match[2];
+        const pagePath = match[3];
+        const pageName = pagePath.replace(".md", "");
+        const pathParts = pageName.split("/");
+        if (pathParts.length === 1) {
+          pages.push({ category: "", page: pathParts[0] });
+        } else if (pathParts.length === 2) {
+          pages.push({ category: pathParts[0], page: pathParts[1] });
+        } else {
+          const category = pathParts.slice(0, -1).join("/");
+          const page = pathParts[pathParts.length - 1];
+          pages.push({ category, page });
+        }
+      }
+      console.log(`Discovered ${pages.length} pages from llms.txt`);
+    } catch (error) {
+      console.error("Error discovering pages:", error);
+    }
+    return pages;
+  }
+  async discoverPagesHtml() {
+    const discovered = [];
+    try {
+      const indexUrl = `${this.options.baseUrl}${this.options.docsPath}`;
+      const response = await fetch(indexUrl, {
+        headers: {
+          Accept: "text/html",
+          "User-Agent": "@ebowwa/markdown-docs-scraper"
+        }
+      });
+      if (!response.ok) {
+        return discovered;
+      }
+      const html = await response.text();
+      const mdLinkRegex = /href="\/docs\/en\/([^"]+\.md)"/g;
+      let match;
+      while ((match = mdLinkRegex.exec(html)) !== null) {
+        const path = match[1];
+        if (!discovered.includes(path)) {
+          discovered.push(path);
+        }
+      }
+      console.log(`Discovered ${discovered.length} additional pages from HTML`);
+    } catch (error) {
+      console.error("Error discovering pages from HTML:", error);
+    }
+    return discovered;
+  }
+}
+async function scrapeMarkdownDocs(options) {
+  const scraper = new MarkdownDocsScraper(options);
+  const result = options.useLlms ? await scraper.scrapeFromLlms() : await scraper.scrape();
+  if (options.outputDir) {
+    await scraper.savePages(result.downloaded);
+  }
+  return result;
+}
+var src_default = MarkdownDocsScraper;
+export {
+  scrapeMarkdownDocs,
+  src_default as default,
+  MarkdownDocsScraper
+};

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@ebowwa/markdown-docs-scraper",
+  "version": "1.0.0",
+  "description": "Scrape and mirror markdown-based documentation sites",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "markdown-docs-scraper": "./dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./cli": {
+      "import": "./dist/cli.js",
+      "types": "./dist/cli.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "bun build src/index.ts --outdir dist --target node && bun build src/cli.ts --outdir dist --target node",
+    "dev": "bun run src/cli.ts",
+    "test": "bun test",
+    "prepublishOnly": "bun run build"
+  },
+  "dependencies": {
+    "commander": "^12.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.0",
+    "typescript": "^5.3.3"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "keywords": [
+    "markdown",
+    "documentation",
+    "scraper",
+    "mirror",
+    "cli"
+  ],
+  "author": "Ebowwa",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ebowwa/codespaces",
+    "directory": "packages/src/markdown-docs-scraper"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,99 @@
+#!/usr/bin/env bun
+/**
+ * CLI for @ebowwa/markdown-docs-scraper
+ */
+
+import { program } from "commander";
+import { scrapeMarkdownDocs, MarkdownDocsScraper, type ScraperOptions } from "./index.js";
+
+program
+  .name("markdown-docs-scraper")
+  .description("Scrape and mirror markdown-based documentation sites")
+  .version("1.0.0");
+
+program
+  .command("scrape")
+  .description("Scrape documentation from a URL")
+  .requiredOption("-u, --url <url>", "Base URL of the documentation site")
+  .option("-o, --output <dir>", "Output directory", "./docs")
+  .option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en")
+  .option("-c, --concurrency <num>", "Concurrency level", "5")
+  .option("--discover", "Discover pages before scraping", false)
+  .action(async (options) => {
+    const scraperOptions: ScraperOptions = {
+      baseUrl: options.url,
+      docsPath: options.docsPath,
+      outputDir: options.output,
+      concurrency: parseInt(options.concurrency),
+    };
+
+    console.log(`🔍 Scraping ${options.url}...`);
+    console.log(`📁 Output: ${options.output}`);
+    console.log();
+
+    const result = await scrapeMarkdownDocs(scraperOptions);
+
+    console.log();
+    console.log("Summary:");
+    console.log(`  Downloaded: ${result.downloaded.length}`);
+    console.log(`  Failed: ${result.failed.length}`);
+    console.log(`  Duration: ${(result.duration / 1000).toFixed(2)}s`);
+
+    if (result.failed.length > 0) {
+      console.log();
+      console.log("Failed pages:");
+      result.failed.slice(0, 10).forEach((f) => {
+        console.log(`  ❌ ${f.url}: ${f.error}`);
+      });
+      if (result.failed.length > 10) {
+        console.log(`  ... and ${result.failed.length - 10} more`);
+      }
+    }
+  });
+
+program
+  .command("discover")
+  .description("Discover all available documentation pages")
+  .requiredOption("-u, --url <url>", "Base URL of the documentation site")
+  .option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en")
+  .action(async (options) => {
+    const scraper = new MarkdownDocsScraper({
+      baseUrl: options.url,
+      docsPath: options.docsPath,
+    });
+
+    console.log(`🔍 Discovering pages from ${options.url}...`);
+    const pages = await scraper.discoverPages();
+
+    console.log(`\nFound ${pages.length} pages:\n`);
+    pages.forEach((page) => {
+      console.log(`  - ${page}`);
+    });
+  });
+
+program
+  .command("anthropic")
+  .description("Quick scrape of Anthropic Claude Code docs (uses llms.txt)")
+  .option("-o, --output <dir>", "Output directory", "./docs")
+  .action(async (options) => {
+    const scraperOptions: ScraperOptions = {
+      baseUrl: "https://code.claude.com",
+      docsPath: "/docs/en",
+      outputDir: options.output,
+    };
+
+    console.log("🔍 Scraping Anthropic Claude Code documentation...");
+    console.log("📋 Using llms.txt index for complete page list");
+    console.log(`📁 Output: ${options.output}`);
+    console.log();
+
+    const result = await scrapeMarkdownDocs({ ...scraperOptions, useLlms: true });
+
+    console.log();
+    console.log("Summary:");
+    console.log(`  Downloaded: ${result.downloaded.length}`);
+    console.log(`  Failed: ${result.failed.length}`);
+    console.log(`  Duration: ${(result.duration / 1000).toFixed(2)}s`);
+  });
+
+program.parse();