@ebowwa/markdown-docs-scraper 1.0.0

package/src/index.ts

@@ -0,0 +1,382 @@
+/**
+ * @ebowwa/markdown-docs-scraper
+ *
+ * Scrape and mirror markdown-based documentation sites
+ */
+
+// ============================================================================
+// TYPES
+// ============================================================================
+
+export interface DocPage {
+  url: string;
+  title: string;
+  content: string;
+  category?: string;
+  pageName?: string;  // The page filename (e.g., "agent-teams")
+}
+
+export interface ScraperOptions {
+  baseUrl: string;
+  docsPath?: string;
+  categories?: Record<string, string[]>;
+  outputDir?: string;
+  concurrency?: number;
+  onProgress?: (current: number, total: number) => void;
+}
+
+export interface ScraperResult {
+  downloaded: DocPage[];
+  failed: Array<{ url: string; error: string }>;
+  duration: number;
+}
+
+// ============================================================================
+// SCRAPER
+// ============================================================================
+
+export class MarkdownDocsScraper {
+  private options: Required<ScraperOptions>;
+
+  constructor(options: ScraperOptions) {
+    this.options = {
+      baseUrl: options.baseUrl,
+      docsPath: options.docsPath || "/docs/en",
+      categories: options.categories || {},
+      outputDir: options.outputDir || "./docs",
+      concurrency: options.concurrency || 5,
+      onProgress: options.onProgress || (() => {}),
+    };
+  }
+
+  /**
+   * Fetch markdown content from a URL
+   */
+  async fetchMarkdown(url: string): Promise<string | null> {
+    try {
+      const response = await fetch(url, {
+        headers: {
+          Accept: "text/markdown, text/plain",
+          "User-Agent": "@ebowwa/markdown-docs-scraper",
+        },
+      });
+
+      if (!response.ok) {
+        return null;
+      }
+
+      const contentType = response.headers.get("content-type") || "";
+      if (!contentType.includes("markdown") && !contentType.includes("text/plain")) {
+        // Try to parse anyway - some sites return incorrect content-type
+      }
+
+      return await response.text();
+    } catch (error) {
+      console.error(`Error fetching ${url}:`, error);
+      return null;
+    }
+  }
+
+  /**
+   * Extract title from markdown content
+   */
+  extractTitle(markdown: string): string {
+    const titleMatch = markdown.match(/^#\s+(.+)$/m);
+    return titleMatch ? titleMatch[1].trim() : "Untitled";
+  }
+
+  /**
+   * Sanitize filename from URL path
+   */
+  sanitizeFilename(path: string): string {
+    return path
+      .toLowerCase()
+      .replace(/[^a-z0-9/]+/g, "-")
+      .replace(/^-|-$/g, "")
+      .replace(/\//g, "/");
+  }
+
+  /**
+   * Build URL for a documentation page
+   */
+  buildUrl(category: string, page: string): string {
+    if (category) {
+      return `${this.options.baseUrl}${this.options.docsPath}/${category}/${page}.md`;
+    } else {
+      return `${this.options.baseUrl}${this.options.docsPath}/${page}.md`;
+    }
+  }
+
+  /**
+   * Download a single documentation page
+   */
+  async downloadPage(category: string, page: string): Promise<DocPage | null> {
+    const url = this.buildUrl(category, page);
+    const content = await this.fetchMarkdown(url);
+
+    if (!content) {
+      return null;
+    }
+
+    return {
+      url,
+      title: this.extractTitle(content),
+      content,
+      category,
+      pageName: page,  // Store the page name for saving
+    };
+  }
+
+  /**
+   * Scrape pages discovered from llms.txt
+   */
+  async scrapeFromLlms(): Promise<ScraperResult> {
+    const startTime = Date.now();
+    const downloaded: DocPage[] = [];
+    const failed: Array<{ url: string; error: string }> = [];
+
+    const pages = await this.discoverPages();
+
+    if (pages.length === 0) {
+      console.log("No pages discovered, falling back to categories");
+      return this.scrape();
+    }
+
+    console.log(`Scraping ${pages.length} discovered pages...`);
+
+    // Process pages in batches
+    for (let i = 0; i < pages.length; i += this.options.concurrency) {
+      const batch = pages.slice(i, i + this.options.concurrency);
+      const results = await Promise.allSettled(
+        batch.map((page) => this.downloadPage(page.category, page.page))
+      );
+
+      results.forEach((result, index) => {
+        const page = batch[index];
+        if (result.status === "fulfilled" && result.value) {
+          downloaded.push(result.value);
+        } else {
+          failed.push({
+            url: this.buildUrl(page.category, page.page),
+            error: result.status === "rejected" ? (result.reason as string) : "Not found",
+          });
+        }
+        this.options.onProgress(downloaded.length + failed.length, pages.length);
+      });
+    }
+
+    const duration = Date.now() - startTime;
+
+    console.log(`✅ Downloaded: ${downloaded.length} pages`);
+    console.log(`❌ Failed: ${failed.length} pages`);
+    console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
+
+    return { downloaded, failed, duration };
+  }
+
+  /**
+   * Scrape all documentation pages
+   */
+  async scrape(): Promise<ScraperResult> {
+    const startTime = Date.now();
+    const downloaded: DocPage[] = [];
+    const failed: Array<{ url: string; error: string }> = [];
+
+    const pages = this.getPagesToScrape();
+    const total = pages.length;
+
+    console.log(`Scraping ${total} pages from ${this.options.baseUrl}...`);
+
+    // Process pages in batches
+    for (let i = 0; i < pages.length; i += this.options.concurrency) {
+      const batch = pages.slice(i, i + this.options.concurrency);
+      const results = await Promise.allSettled(
+        batch.map((page) => this.downloadPage(page.category, page.page))
+      );
+
+      results.forEach((result, index) => {
+        const page = batch[index];
+        if (result.status === "fulfilled" && result.value) {
+          downloaded.push(result.value);
+        } else {
+          failed.push({
+            url: this.buildUrl(page.category, page.page),
+            error: result.status === "rejected" ? result.reason : "Not found",
+          });
+        }
+        this.options.onProgress(downloaded.length + failed.length, total);
+      });
+    }
+
+    const duration = Date.now() - startTime;
+
+    console.log(`✅ Downloaded: ${downloaded.length} pages`);
+    console.log(`❌ Failed: ${failed.length} pages`);
+    console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
+
+    return { downloaded, failed, duration };
+  }
+
+  /**
+   * Save scraped pages to disk
+   */
+  async savePages(pages: DocPage[]): Promise<void> {
+    const fs = await import("fs/promises");
+    const path = await import("path");
+
+    for (const page of pages) {
+      // Use pageName if available, otherwise extract from URL
+      const nameToUse = page.pageName || page.url.split("/").pop()?.replace(".md", "") || "untitled";
+
+      const dir = page.category
+        ? path.join(this.options.outputDir, page.category)
+        : this.options.outputDir;
+
+      await fs.mkdir(dir, { recursive: true });
+
+      const filepath = path.join(dir, `${nameToUse}.md`);
+
+      const header = `<!--\nSource: ${page.url}\nDownloaded: ${new Date().toISOString()}\n-->\n\n`;
+      await fs.writeFile(filepath, header + page.content, "utf-8");
+    }
+  }
+
+  /**
+   * Get list of pages to scrape based on categories
+   */
+  private getPagesToScrape(): Array<{ category: string; page: string }> {
+    const pages: Array<{ category: string; page: string }> = [];
+
+    for (const [category, pageList] of Object.entries(this.options.categories)) {
+      for (const page of pageList) {
+        pages.push({ category, page });
+      }
+    }
+
+    return pages;
+  }
+
+  /**
+   * Discover pages from llms.txt index
+   */
+  async discoverPages(): Promise<Array<{ category: string; page: string }>> {
+    const pages: Array<{ category: string; page: string }> = [];
+
+    try {
+      const llmsUrl = `${this.options.baseUrl}/docs/llms.txt`;
+      const response = await fetch(llmsUrl, {
+        headers: {
+          Accept: "text/plain",
+          "User-Agent": "@ebowwa/markdown-docs-scraper",
+        },
+      });
+
+      if (!response.ok) {
+        console.warn(`Could not fetch llms.txt from ${llmsUrl}`);
+        return pages;
+      }
+
+      const content = await response.text();
+
+      // Parse markdown links in format: [title](https://code.claude.com/docs/en/page.md)
+      const linkRegex = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
+      let match;
+
+      while ((match = linkRegex.exec(content)) !== null) {
+        const url = match[2];
+        const pagePath = match[3]; // e.g., "agent-teams.md" or "category/page.md"
+
+        // Remove .md extension
+        const pageName = pagePath.replace(".md", "");
+
+        // Check if there's a category in the path
+        const pathParts = pageName.split("/");
+
+        if (pathParts.length === 1) {
+          // No category: just "page-name"
+          pages.push({ category: "", page: pathParts[0] });
+        } else if (pathParts.length === 2) {
+          // Has category: "category/page-name"
+          pages.push({ category: pathParts[0], page: pathParts[1] });
+        } else {
+          // Deeper path: join everything except last as category
+          const category = pathParts.slice(0, -1).join("/");
+          const page = pathParts[pathParts.length - 1];
+          pages.push({ category, page });
+        }
+      }
+
+      console.log(`Discovered ${pages.length} pages from llms.txt`);
+    } catch (error) {
+      console.error("Error discovering pages:", error);
+    }
+
+    return pages;
+  }
+
+  /**
+   * Discover additional pages by parsing the docs index (fallback)
+   */
+  async discoverPagesHtml(): Promise<string[]> {
+    const discovered: string[] = [];
+
+    try {
+      const indexUrl = `${this.options.baseUrl}${this.options.docsPath}`;
+      const response = await fetch(indexUrl, {
+        headers: {
+          Accept: "text/html",
+          "User-Agent": "@ebowwa/markdown-docs-scraper",
+        },
+      });
+
+      if (!response.ok) {
+        return discovered;
+      }
+
+      const html = await response.text();
+      const mdLinkRegex = /href="\/docs\/en\/([^"]+\.md)"/g;
+      let match;
+
+      while ((match = mdLinkRegex.exec(html)) !== null) {
+        const path = match[1];
+        if (!discovered.includes(path)) {
+          discovered.push(path);
+        }
+      }
+
+      console.log(`Discovered ${discovered.length} additional pages from HTML`);
+    } catch (error) {
+      console.error("Error discovering pages from HTML:", error);
+    }
+
+    return discovered;
+  }
+}
+
+// ============================================================================
+// CONVENIENCE FUNCTION
+// ============================================================================
+
+/**
+ * Scrape markdown documentation with a single function call
+ */
+export async function scrapeMarkdownDocs(
+  options: ScraperOptions & { useLlms?: boolean }
+): Promise<ScraperResult> {
+  const scraper = new MarkdownDocsScraper(options);
+  const result = options.useLlms
+    ? await scraper.scrapeFromLlms()
+    : await scraper.scrape();
+
+  if (options.outputDir) {
+    await scraper.savePages(result.downloaded);
+  }
+
+  return result;
+}
+
+// ============================================================================
+// EXPORTS
+// ============================================================================
+
+export default MarkdownDocsScraper;