@ebowwa/markdown-docs-scraper 1.1.0 → 1.2.1

package/README.md

@@ -161,6 +161,88 @@ Downloaded: 2026-02-06T00:00:00.000Z
 Original markdown content...
 ```
 
+## Composable Scrapers Module
+
+The package includes a composable scraper architecture for multiple documentation source types.
+
+### Usage
+
+```typescript
+import {
+  scrapeSource,
+  registerScraper,
+  llmsTxtScraper,
+  githubRawScraper,
+  type SourceConfig,
+} from "@ebowwa/markdown-docs-scraper/scrapers";
+
+// Configure a source
+const config: SourceConfig = {
+  name: "My Docs",
+  sourceType: "llms-txt",
+  baseUrl: "https://docs.example.com",
+  docsPath: "/docs",
+  outputDir: "./docs/my-docs",
+  reportDir: "./reports/my-docs",
+};
+
+// Scrape using the registry (auto-selects scraper by sourceType)
+const result = await scrapeSource(config);
+```
+
+### Built-in Scrapers
+
+- **llms-txt**: Scrapes docs sites with llms.txt index files
+- **github-raw**: Downloads markdown directly from GitHub repos
+
+### Custom Scrapers
+
+```typescript
+import { registerScraper, type Scraper, type SourceType } from "@ebowwa/markdown-docs-scraper/scrapers";
+
+const myScraper: Scraper = {
+  type: "my-type" as SourceType,
+  async scrape(config) {
+    // Custom scraping logic
+    return {
+      downloaded: [],
+      failed: [],
+      duration: 0,
+    };
+  },
+};
+
+registerScraper(myScraper);
+```
+
+### Types
+
+```typescript
+type SourceType = "llms-txt" | "github-raw";
+
+interface SourceConfig {
+  name: string;
+  sourceType: SourceType;
+  baseUrl: string;
+  docsPath: string;
+  outputDir: string;
+  reportDir: string;
+  llmsTxtPath?: string;
+  linkPattern?: RegExp;
+  github?: {
+    repo: string;
+    includeCommits: boolean;
+    includeReleases: boolean;
+    includePRs: boolean;
+  };
+}
+
+interface Scraper {
+  type: SourceType;
+  scrape(config: SourceConfig): Promise<ScrapeResult>;
+}
+```
+
 ## License
 
 MIT

package/dist/cli.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env bun
+/**
+ * CLI for @ebowwa/markdown-docs-scraper
+ */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;GAEG"}

package/dist/cli.js

@@ -2426,6 +2426,7 @@ program.command("scrape").description("Scrape documentation from a URL").require
   }
 });
 program.command("discover").description("Discover all available documentation pages").requiredOption("-u, --url <url>", "Base URL of the documentation site").option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en").option("--llms-paths <paths>", "Comma-separated llms.txt paths to try", "/llms.txt,/docs/llms.txt").option("--no-subdomain", "Disable docs/doc subdomain fallback", false).action(async (options) => {
+  console.log("DEBUG CLI: Options received:", options);
   const scraper = new import__2.MarkdownDocsScraper({
     baseUrl: options.url,
     docsPath: options.docsPath,

package/dist/index.d.ts

@@ -0,0 +1,128 @@
+/**
+ * @ebowwa/markdown-docs-scraper
+ *
+ * Composable markdown documentation scraper.
+ * - Configurable llms.txt paths with fallbacks
+ * - Custom URL patterns for different doc sites
+ * - Works with any markdown documentation site
+ * - Uses full URLs from llms.txt directly
+ */
+export interface DocPage {
+    url: string;
+    title: string;
+    content: string;
+    category?: string;
+    pageName?: string;
+}
+export interface ScraperOptions {
+    baseUrl: string;
+    docsPath?: string;
+    categories?: Record<string, string[]>;
+    outputDir?: string;
+    concurrency?: number;
+    onProgress?: (current: number, total: number) => void;
+    /** Custom llms.txt paths to try (default: ["/llms.txt", "/docs/llms.txt"]) */
+    llmsPaths?: string[];
+    /** Also try docs subdomain variants (e.g., docs.example.com) */
+    tryDocsSubdomain?: boolean;
+    /** Custom regex pattern to extract pages from llms.txt (must have 3 capture groups: title, fullUrl, path) */
+    linkPattern?: RegExp;
+    /** Use full URLs from llms.txt directly (default: true for generic pattern) */
+    useDirectUrls?: boolean;
+}
+export interface ScraperResult {
+    downloaded: DocPage[];
+    failed: Array<{
+        url: string;
+        error: string;
+    }>;
+    duration: number;
+}
+/** Discovered page with full URL */
+interface DiscoveredPage {
+    category: string;
+    page: string;
+    fullUrl: string;
+}
+/** Extract title from markdown content */
+export declare function extractTitle(markdown: string): string;
+/** Parse page path into category and page name */
+export declare function parsePagePath(pagePath: string): {
+    category: string;
+    page: string;
+};
+/** Fetch markdown content from URL */
+export declare function fetchMarkdown(url: string, userAgent?: string): Promise<string | null>;
+export declare class MarkdownDocsScraper {
+    private options;
+    constructor(options: ScraperOptions);
+    /**
+     * Build URL for a documentation page (fallback when no direct URL)
+     */
+    buildUrl(category: string, page: string): string;
+    /**
+     * Download a page using either direct URL or built URL
+     */
+    downloadPage(pageInfo: DiscoveredPage): Promise<DocPage | null>;
+    /**
+     * Generate possible llms.txt URLs to try
+     */
+    private getLlmsUrls;
+    /**
+     * Fetch llms.txt from multiple possible URLs with fallback
+     */
+    private fetchLlmsTxt;
+    /**
+     * Discover pages from llms.txt index
+     */
+    discoverPages(): Promise<DiscoveredPage[]>;
+    /**
+     * Scrape pages discovered from llms.txt
+     */
+    scrapeFromLlms(): Promise<ScraperResult>;
+    /**
+     * Scrape all documentation pages (uses categories)
+     */
+    scrape(): Promise<ScraperResult>;
+    /**
+     * Extract body content from a file (strips header comment)
+     */
+    private extractBody;
+    /**
+     * Save scraped pages to disk (only writes if content changed)
+     */
+    savePages(pages: DocPage[]): Promise<{
+        updated: number;
+        skipped: number;
+    }>;
+    /**
+     * Get list of pages to scrape based on categories
+     */
+    private getPagesToScrape;
+}
+/**
+ * Scrape markdown documentation with a single function call
+ */
+export declare function scrapeMarkdownDocs(options: ScraperOptions & {
+    useLlms?: boolean;
+}): Promise<ScraperResult & {
+    saveStats?: {
+        updated: number;
+        skipped: number;
+    };
+}>;
+/** Pattern for Claude Code docs: /docs/en/page.md */
+export declare const CLAUDE_CODE_PATTERN: RegExp;
+/** Pattern for generic docs: any domain/path.md */
+export declare const GENERIC_PATTERN: RegExp;
+/** Create scraper options for Claude Code docs */
+export declare function claudeCodeOptions(outputDir: string): ScraperOptions;
+/** Create scraper options for Polymarket docs */
+export declare function polymarketOptions(outputDir: string): ScraperOptions;
+/**
+ * Re-export scrapers module for composable scraper architecture.
+ * This provides a registry-based system for different scraper implementations.
+ */
+export { type SourceType, type SourceConfig, type Scraper, type ScrapeResult as ScraperModuleResult, type DownloadResult, llmsTxtScraper, githubRawScraper, CLAUDE_CODE_PATTERN as SCRAPER_CLAUDE_CODE_PATTERN, GENERIC_PATTERN as SCRAPER_GENERIC_PATTERN, registerScraper, getScraper, scrapeSource, } from "./scrapers/index";
+export default MarkdownDocsScraper;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,MAAM,WAAW,OAAO;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACtD,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6GAA6G;IAC7G,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,OAAO,EAAE,CAAC;IACtB,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAYD,0CAA0C;AAC1C,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrD;AAED,kDAAkD;AAClD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAkBlF;AAED,sCAAsC;AACtC,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAkC,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAkBpH;AAMD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA2B;gBAE9B,OAAO,EAAE,cAAc;IAenC;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAUhD;;OAEG;IACG,YAAY,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAqBrE;;OAEG;IACH,OAAO,CAAC,WAAW;IA6BnB;;OAEG;YACW,YAAY;IA6B1B;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAuChD;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,aAAa,CAAC;IA+C9C;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,aAAa,CAAC;IAwCtC;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAyChF;;OAEG;IACH,OAAO,CAAC,gBAAgB;CAWzB;AAMD;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,cAAc,GAAG;IAAE,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,GAC9C,OAAO,CAAC,aAAa,GAAG;IAAE,SAAS,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC,CAe/E;AAMD,qDAAqD;AACrD,eAAO,MAAM,mBAAmB,QAAiE,CAAC;AAElG,mDAAmD;AACnD,eAAO,MAAM,eAAe,QAAuB,CAAC;AAEpD,kDAAkD;AAClD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAED,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAMD;;;GAGG;AACH,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,OAAO,EACZ,KAAK,YAAY,IAAI,mBAAmB,EACxC,KAAK,cAAc,EAGnB,cAAc,EACd,gBAAgB,EAChB,mBAAmB,IAAI,2BAA2B,EAClD,eAAe,IAAI,uBAAuB,EAG1C,eAAe,EACf,UAAU,EACV,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,eAAe,mBAAmB,CAAC"}

package/dist/index.js

@@ -17,9 +17,164 @@ var __toESM = (mod, isNodeMode, target) => {
 };
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
-// src/index.ts
-var GENERIC_LINK_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/([^)]+\.md))\)/g;
+// src/scrapers/llms-txt.ts
+var CLAUDE_CODE_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
+var GENERIC_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^/]+\/([^\s)]+\.md))\)/g;
+var llmsTxtScraper = {
+  type: "llms-txt",
+  async scrape(config) {
+    const options = getScraperOptions(config);
+    const result = await scrapeMarkdownDocs(options);
+    const downloaded = result.downloaded.map((page) => {
+      const category = page.category || "";
+      const filename = `${page.pageName || "untitled"}.md`;
+      const path = category ? `${category}/${filename}` : filename;
+      return {
+        success: true,
+        path,
+        title: page.title
+      };
+    });
+    return {
+      downloaded,
+      failed: result.failed,
+      duration: result.duration
+    };
+  }
+};
+function getScraperOptions(config) {
+  const baseOptions = {
+    baseUrl: config.baseUrl,
+    docsPath: config.docsPath,
+    outputDir: config.outputDir,
+    concurrency: 10,
+    useLlms: true,
+    tryDocsSubdomain: false
+  };
+  if (config.name === "Claude Code") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/docs/llms.txt"],
+      linkPattern: CLAUDE_CODE_PATTERN
+    };
+  }
+  if (config.name === "Polymarket") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/llms.txt"],
+      linkPattern: GENERIC_PATTERN
+    };
+  }
+  if (config.name === "Bun") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/docs/llms.txt", "/llms.txt"],
+      linkPattern: GENERIC_PATTERN
+    };
+  }
+  return {
+    ...baseOptions,
+    llmsPaths: config.llmsTxtPath ? [config.llmsTxtPath] : ["/llms.txt", "/docs/llms.txt"],
+    linkPattern: config.linkPattern || GENERIC_PATTERN
+  };
+}
+// src/scrapers/github-raw.ts
+var githubRawScraper = {
+  type: "github-raw",
+  async scrape(config) {
+    const startTime = Date.now();
+    const downloaded = [];
+    const failed = [];
+    if (!config.github?.repo) {
+      throw new Error(`GitHub source "${config.name}" missing github.repo config`);
+    }
+    const files = await fetchGitHubMarkdownFiles(config.github.repo, config.docsPath.replace(/^\//, ""));
+    for (const file of files) {
+      const content = await fetchGitHubRawContent(config.github.repo, file.path);
+      if (content) {
+        downloaded.push({
+          success: true,
+          path: file.name,
+          title: extractTitle(content) || file.name.replace(".md", "")
+        });
+        await saveFile(config.outputDir, file.name, content);
+      } else {
+        failed.push({
+          url: `https://raw.githubusercontent.com/${config.github.repo}/main/${file.path}`,
+          error: "Failed to fetch content"
+        });
+      }
+    }
+    return {
+      downloaded,
+      failed,
+      duration: Date.now() - startTime
+    };
+  }
+};
+async function fetchGitHubMarkdownFiles(repo, path) {
+  const url = `https://api.github.com/repos/${repo}/contents/${path}`;
+  const response = await fetch(url, {
+    headers: {
+      Accept: "application/vnd.github.v3+json",
+      "User-Agent": "@ebowwa/markdown-docs-scraper"
+    }
+  });
+  if (!response.ok) {
+    throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
+  }
+  const contents = await response.json();
+  return contents.filter((item) => item.type === "file" && item.name.endsWith(".md"));
+}
+async function fetchGitHubRawContent(repo, path) {
+  const url = `https://raw.githubusercontent.com/${repo}/main/${path}`;
+  try {
+    const response = await fetch(url, {
+      headers: {
+        Accept: "text/plain",
+        "User-Agent": "@ebowwa/markdown-docs-scraper"
+      }
+    });
+    if (!response.ok) {
+      return null;
+    }
+    return await response.text();
+  } catch (error) {
+    console.error(`Error fetching ${url}:`, error);
+    return null;
+  }
+}
 function extractTitle(markdown) {
+  const titleMatch = markdown.match(/^#\s+(.+)$/m);
+  return titleMatch ? titleMatch[1].trim() : null;
+}
+async function saveFile(outputDir, filename, content) {
+  const fs = await import("fs/promises");
+  const path = await import("path");
+  const outputPath = path.join(outputDir, filename);
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, content, "utf-8");
+}
+// src/scrapers/registry.ts
+var scrapers = new Map;
+function registerScraper(scraper) {
+  scrapers.set(scraper.type, scraper);
+}
+function getScraper(type) {
+  return scrapers.get(type);
+}
+async function scrapeSource(config) {
+  const scraper = scrapers.get(config.sourceType);
+  if (!scraper) {
+    throw new Error(`No scraper registered for type: ${config.sourceType}`);
+  }
+  return scraper.scrape(config);
+}
+registerScraper(llmsTxtScraper);
+registerScraper(githubRawScraper);
+// src/index.ts
+var GENERIC_LINK_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^/]+\/([^\s)]+\.md))\)/g;
+function extractTitle2(markdown) {
   const titleMatch = markdown.match(/^#\s+(.+)$/m);
   return titleMatch ? titleMatch[1].trim() : "Untitled";
 }
@@ -67,7 +222,8 @@ class MarkdownDocsScraper {
       onProgress: options.onProgress || (() => {}),
       llmsPaths: options.llmsPaths || ["/llms.txt", "/docs/llms.txt"],
       tryDocsSubdomain: options.tryDocsSubdomain ?? true,
-      linkPattern: options.linkPattern || GENERIC_LINK_PATTERN
+      linkPattern: options.linkPattern || GENERIC_LINK_PATTERN,
+      useDirectUrls: options.useDirectUrls ?? true
     };
   }
   buildUrl(category, page) {
@@ -79,18 +235,18 @@ class MarkdownDocsScraper {
       return `${this.options.baseUrl}/${page}.md`;
     }
   }
-  async downloadPage(category, page) {
-    const url = this.buildUrl(category, page);
+  async downloadPage(pageInfo) {
+    const url = this.options.useDirectUrls && pageInfo.fullUrl ? pageInfo.fullUrl : this.buildUrl(pageInfo.category, pageInfo.page);
     const content = await fetchMarkdown(url);
     if (!content) {
       return null;
     }
     return {
       url,
-      title: extractTitle(content),
+      title: extractTitle2(content),
       content,
-      category,
-      pageName: page
+      category: pageInfo.category,
+      pageName: pageInfo.page
     };
   }
   getLlmsUrls() {
@@ -114,20 +270,24 @@ class MarkdownDocsScraper {
   }
   async fetchLlmsTxt() {
     const urls = this.getLlmsUrls();
+    console.log(`DEBUG: Trying URLs: ${urls.join(", ")}`);
     for (const llmsUrl of urls) {
       try {
+        console.log(`DEBUG: Fetching ${llmsUrl}...`);
         const response = await fetch(llmsUrl, {
           headers: {
             Accept: "text/plain",
             "User-Agent": "@ebowwa/markdown-docs-scraper"
           }
         });
+        console.log(`DEBUG: Response status: ${response.status}`);
         if (response.ok) {
           const content = await response.text();
           console.log(`Found llms.txt at ${llmsUrl}`);
           return { content, url: llmsUrl };
         }
       } catch (error) {
+        console.log(`DEBUG: Error: ${error}`);
         continue;
       }
     }
@@ -143,13 +303,16 @@ class MarkdownDocsScraper {
         return pages;
       }
       const { content } = llmsResult;
-      const regex = new RegExp(this.options.linkPattern.source, this.options.linkPattern.flags);
+      const pattern = this.options.linkPattern;
+      const regex = new RegExp(pattern.source, pattern.flags);
       let match;
+      console.log(`DEBUG: Using pattern: ${pattern.source}`);
+      console.log(`DEBUG: Content length: ${content.length}`);
       while ((match = regex.exec(content)) !== null) {
-        const url = match[2];
+        const fullUrl = match[2];
         const pagePath = match[3];
         const { category, page } = parsePagePath(pagePath);
-        pages.push({ category, page });
+        pages.push({ category, page, fullUrl });
       }
       console.log(`Discovered ${pages.length} pages from llms.txt`);
     } catch (error) {
@@ -169,14 +332,15 @@ class MarkdownDocsScraper {
     console.log(`Scraping ${pages.length} discovered pages...`);
     for (let i = 0;i < pages.length; i += this.options.concurrency) {
       const batch = pages.slice(i, i + this.options.concurrency);
-      const results = await Promise.allSettled(batch.map((page) => this.downloadPage(page.category, page.page)));
+      const results = await Promise.allSettled(batch.map((page) => this.downloadPage(page)));
       results.forEach((result, index) => {
         const page = batch[index];
         if (result.status === "fulfilled" && result.value) {
           downloaded.push(result.value);
         } else {
+          const url = this.options.useDirectUrls && page.fullUrl ? page.fullUrl : this.buildUrl(page.category, page.page);
           failed.push({
-            url: this.buildUrl(page.category, page.page),
+            url,
             error: result.status === "rejected" ? result.reason : "Not found"
           });
         }
@@ -198,7 +362,7 @@ class MarkdownDocsScraper {
     console.log(`Scraping ${total} pages from ${this.options.baseUrl}...`);
     for (let i = 0;i < pages.length; i += this.options.concurrency) {
       const batch = pages.slice(i, i + this.options.concurrency);
-      const results = await Promise.allSettled(batch.map((page) => this.downloadPage(page.category, page.page)));
+      const results = await Promise.allSettled(batch.map((page) => this.downloadPage({ ...page, fullUrl: "" })));
       results.forEach((result, index) => {
         const page = batch[index];
         if (result.status === "fulfilled" && result.value) {
@@ -218,14 +382,28 @@ class MarkdownDocsScraper {
     console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
     return { downloaded, failed, duration };
   }
+  extractBody(content) {
+    const headerRegex = /^<!--\nSource: [^\n]+\nDownloaded: [^\n]+\n-->\n\n/;
+    return content.replace(headerRegex, "");
+  }
   async savePages(pages) {
     const fs = await import("fs/promises");
     const path = await import("path");
+    let updated = 0;
+    let skipped = 0;
     for (const page of pages) {
       const nameToUse = page.pageName || page.url.split("/").pop()?.replace(".md", "") || "untitled";
       const dir = page.category ? path.join(this.options.outputDir, page.category) : this.options.outputDir;
       await fs.mkdir(dir, { recursive: true });
       const filepath = path.join(dir, `${nameToUse}.md`);
+      try {
+        const existingContent = await fs.readFile(filepath, "utf-8");
+        const existingBody = this.extractBody(existingContent);
+        if (existingBody === page.content) {
+          skipped++;
+          continue;
+        }
+      } catch {}
       const header = `<!--
 Source: ${page.url}
 Downloaded: ${new Date().toISOString()}
@@ -233,13 +411,15 @@ Downloaded: ${new Date().toISOString()}
 
 `;
       await fs.writeFile(filepath, header + page.content, "utf-8");
+      updated++;
     }
+    return { updated, skipped };
   }
   getPagesToScrape() {
     const pages = [];
     for (const [category, pageList] of Object.entries(this.options.categories)) {
       for (const page of pageList) {
-        pages.push({ category, page });
+        pages.push({ category, page, fullUrl: "" });
       }
     }
     return pages;
@@ -248,22 +428,27 @@ Downloaded: ${new Date().toISOString()}
 async function scrapeMarkdownDocs(options) {
   const scraper = new MarkdownDocsScraper(options);
   const result = options.useLlms ? await scraper.scrapeFromLlms() : await scraper.scrape();
+  let saveStats;
   if (options.outputDir) {
-    await scraper.savePages(result.downloaded);
+    saveStats = await scraper.savePages(result.downloaded);
+    if (saveStats.updated > 0 || saveStats.skipped > 0) {
+      console.log(`  Saved: ${saveStats.updated} updated, ${saveStats.skipped} unchanged`);
+    }
   }
-  return result;
+  return { ...result, saveStats };
 }
-var CLAUDE_CODE_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
-var GENERIC_PATTERN = GENERIC_LINK_PATTERN;
+var CLAUDE_CODE_PATTERN2 = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
+var GENERIC_PATTERN2 = GENERIC_LINK_PATTERN;
 function claudeCodeOptions(outputDir) {
   return {
     baseUrl: "https://code.claude.com",
     docsPath: "/docs/en",
     llmsPaths: ["/docs/llms.txt"],
-    linkPattern: CLAUDE_CODE_PATTERN,
+    linkPattern: CLAUDE_CODE_PATTERN2,
     outputDir,
     concurrency: 10,
-    tryDocsSubdomain: false
+    tryDocsSubdomain: false,
+    useDirectUrls: false
   };
 }
 function polymarketOptions(outputDir) {
@@ -271,22 +456,30 @@ function polymarketOptions(outputDir) {
     baseUrl: "https://docs.polymarket.com",
     docsPath: "",
     llmsPaths: ["/llms.txt"],
-    linkPattern: GENERIC_PATTERN,
+    linkPattern: GENERIC_PATTERN2,
     outputDir,
     concurrency: 10,
-    tryDocsSubdomain: false
+    tryDocsSubdomain: false,
+    useDirectUrls: true
   };
 }
 var src_default = MarkdownDocsScraper;
 export {
+  scrapeSource,
   scrapeMarkdownDocs,
+  registerScraper,
   polymarketOptions,
   parsePagePath,
+  llmsTxtScraper,
+  githubRawScraper,
+  getScraper,
   fetchMarkdown,
-  extractTitle,
+  extractTitle2 as extractTitle,
   src_default as default,
   claudeCodeOptions,
+  GENERIC_PATTERN as SCRAPER_GENERIC_PATTERN,
+  CLAUDE_CODE_PATTERN as SCRAPER_CLAUDE_CODE_PATTERN,
   MarkdownDocsScraper,
-  GENERIC_PATTERN,
-  CLAUDE_CODE_PATTERN
+  GENERIC_PATTERN2 as GENERIC_PATTERN,
+  CLAUDE_CODE_PATTERN2 as CLAUDE_CODE_PATTERN
 };

package/dist/scrapers/github-raw.d.ts

@@ -0,0 +1,9 @@
+/**
+ * GitHub Raw Scraper
+ *
+ * Downloads markdown files directly from GitHub repositories via raw content URLs.
+ * Uses GitHub API to list files, then fetches each from raw.githubusercontent.com
+ */
+import type { Scraper } from "./types";
+export declare const githubRawScraper: Scraper;
+//# sourceMappingURL=github-raw.d.ts.map

package/dist/scrapers/github-raw.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-raw.d.ts","sourceRoot":"","sources":["../../src/scrapers/github-raw.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAA8C,MAAM,SAAS,CAAC;AAiBnF,eAAO,MAAM,gBAAgB,EAAE,OA6C9B,CAAC"}

package/dist/scrapers/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Scrapers Module
+ *
+ * Composable scraper architecture for multiple documentation source types.
+ * This module provides a registry-based system for different scraper implementations.
+ */
+export type { SourceType, SourceConfig, Scraper, ScrapeResult, DownloadResult } from "./types";
+export { llmsTxtScraper, CLAUDE_CODE_PATTERN, GENERIC_PATTERN } from "./llms-txt";
+export { githubRawScraper } from "./github-raw";
+export { registerScraper, getScraper, scrapeSource } from "./registry";
+//# sourceMappingURL=index.d.ts.map

package/dist/scrapers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scrapers/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAG/F,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClF,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGhD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC"}