@ebowwa/markdown-docs-scraper 1.0.0 → 1.2.0

package/README.md

@@ -52,6 +52,26 @@ Options:
   -o, --output <dir>       Output directory (default: "./docs")
   --docs-path <path>       Docs path (default: "/docs/en")
   -c, --concurrency <num>  Concurrency level (default: "5")
+  --llms-paths <paths>     Comma-separated llms.txt paths (default: "/llms.txt,/docs/llms.txt")
+  --no-subdomain           Disable docs/doc subdomain fallback
+```
+
+### llms.txt Discovery
+
+The scraper automatically tries multiple paths to find `llms.txt`:
+
+1. **Configured paths** (default: `/llms.txt`, `/docs/llms.txt`)
+2. **Docs subdomain** (e.g., `https://docs.example.com/llms.txt`)
+3. **Doc subdomain** (e.g., `https://doc.example.com/llms.txt`)
+
+Example with custom paths:
+```bash
+markdown-docs-scraper scrape -u https://example.com --llms-paths "/llms.txt,/api/llms.txt"
+```
+
+Disable subdomain fallback:
+```bash
+markdown-docs-scraper scrape -u https://example.com --no-subdomain
 ```
 
 ## Programmatic Usage
@@ -103,6 +123,8 @@ interface ScraperOptions {
   outputDir?: string;        // Output directory (default: "./docs")
   concurrency?: number;      // Concurrent downloads (default: 5)
   onProgress?: (current: number, total: number) => void;
+  llmsPaths?: string[];      // llms.txt paths to try (default: ["/llms.txt", "/docs/llms.txt"])
+  tryDocsSubdomain?: boolean; // Also try docs/doc subdomains (default: true)
 }
 ```
 
@@ -139,6 +161,88 @@ Downloaded: 2026-02-06T00:00:00.000Z
 Original markdown content...
 ```
 
+## Composable Scrapers Module
+
+The package includes a composable scraper architecture for multiple documentation source types.
+
+### Usage
+
+```typescript
+import {
+  scrapeSource,
+  registerScraper,
+  llmsTxtScraper,
+  githubRawScraper,
+  type SourceConfig,
+} from "@ebowwa/markdown-docs-scraper/scrapers";
+
+// Configure a source
+const config: SourceConfig = {
+  name: "My Docs",
+  sourceType: "llms-txt",
+  baseUrl: "https://docs.example.com",
+  docsPath: "/docs",
+  outputDir: "./docs/my-docs",
+  reportDir: "./reports/my-docs",
+};
+
+// Scrape using the registry (auto-selects scraper by sourceType)
+const result = await scrapeSource(config);
+```
+
+### Built-in Scrapers
+
+- **llms-txt**: Scrapes docs sites with llms.txt index files
+- **github-raw**: Downloads markdown directly from GitHub repos
+
+### Custom Scrapers
+
+```typescript
+import { registerScraper, type Scraper, type SourceType } from "@ebowwa/markdown-docs-scraper/scrapers";
+
+const myScraper: Scraper = {
+  type: "my-type" as SourceType,
+  async scrape(config) {
+    // Custom scraping logic
+    return {
+      downloaded: [],
+      failed: [],
+      duration: 0,
+    };
+  },
+};
+
+registerScraper(myScraper);
+```
+
+### Types
+
+```typescript
+type SourceType = "llms-txt" | "github-raw";
+
+interface SourceConfig {
+  name: string;
+  sourceType: SourceType;
+  baseUrl: string;
+  docsPath: string;
+  outputDir: string;
+  reportDir: string;
+  llmsTxtPath?: string;
+  linkPattern?: RegExp;
+  github?: {
+    repo: string;
+    includeCommits: boolean;
+    includeReleases: boolean;
+    includePRs: boolean;
+  };
+}
+
+interface Scraper {
+  type: SourceType;
+  scrape(config: SourceConfig): Promise<ScrapeResult>;
+}
+```
+
 ## License
 
 MIT

package/dist/cli.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env bun
+/**
+ * CLI for @ebowwa/markdown-docs-scraper
+ */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;GAEG"}

package/dist/cli.js

@@ -20,7 +20,7 @@ var __toESM = (mod, isNodeMode, target) => {
 var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
-// node_modules/commander/lib/error.js
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/lib/error.js
 var require_error = __commonJS((exports) => {
   class CommanderError extends Error {
     constructor(exitCode, code, message) {
@@ -44,7 +44,7 @@ var require_error = __commonJS((exports) => {
   exports.InvalidArgumentError = InvalidArgumentError;
 });
 
-// node_modules/commander/lib/argument.js
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/lib/argument.js
 var require_argument = __commonJS((exports) => {
   var { InvalidArgumentError } = require_error();
 
@@ -123,7 +123,7 @@ var require_argument = __commonJS((exports) => {
   exports.humanReadableArgName = humanReadableArgName;
 });
 
-// node_modules/commander/lib/help.js
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/lib/help.js
 var require_help = __commonJS((exports) => {
   var { humanReadableArgName } = require_argument();
 
@@ -372,7 +372,7 @@ var require_help = __commonJS((exports) => {
   exports.Help = Help;
 });
 
-// node_modules/commander/lib/option.js
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/lib/option.js
 var require_option = __commonJS((exports) => {
   var { InvalidArgumentError } = require_error();
 
@@ -523,7 +523,7 @@ var require_option = __commonJS((exports) => {
   exports.DualOptions = DualOptions;
 });
 
-// node_modules/commander/lib/suggestSimilar.js
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/lib/suggestSimilar.js
 var require_suggestSimilar = __commonJS((exports) => {
   var maxDistance = 3;
   function editDistance(a, b) {
@@ -596,7 +596,7 @@ var require_suggestSimilar = __commonJS((exports) => {
   exports.suggestSimilar = suggestSimilar;
 });
 
-// node_modules/commander/lib/command.js
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/lib/command.js
 var require_command = __commonJS((exports) => {
   var EventEmitter = __require("node:events").EventEmitter;
   var childProcess = __require("node:child_process");
@@ -1839,7 +1839,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
   exports.Command = Command;
 });
 
-// node_modules/commander/index.js
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/index.js
 var require_commander = __commonJS((exports) => {
   var { Argument } = require_argument();
   var { Command } = require_command();
@@ -2377,7 +2377,7 @@ Downloaded: `).concat(new Date().toISOString(), `
   exports.default = MarkdownDocsScraper;
 });
 
-// node_modules/commander/esm.mjs
+// ../../node_modules/.bun/commander@12.1.0/node_modules/commander/esm.mjs
 var import__ = __toESM(require_commander(), 1);
 var {
   program,
@@ -2396,12 +2396,14 @@ var {
 // src/cli.ts
 var import__2 = __toESM(require_src(), 1);
 program.name("markdown-docs-scraper").description("Scrape and mirror markdown-based documentation sites").version("1.0.0");
-program.command("scrape").description("Scrape documentation from a URL").requiredOption("-u, --url <url>", "Base URL of the documentation site").option("-o, --output <dir>", "Output directory", "./docs").option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en").option("-c, --concurrency <num>", "Concurrency level", "5").option("--discover", "Discover pages before scraping", false).action(async (options) => {
+program.command("scrape").description("Scrape documentation from a URL").requiredOption("-u, --url <url>", "Base URL of the documentation site").option("-o, --output <dir>", "Output directory", "./docs").option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en").option("-c, --concurrency <num>", "Concurrency level", "5").option("--discover", "Discover pages before scraping", false).option("--llms-paths <paths>", "Comma-separated llms.txt paths to try", "/llms.txt,/docs/llms.txt").option("--no-subdomain", "Disable docs/doc subdomain fallback", false).action(async (options) => {
   const scraperOptions = {
     baseUrl: options.url,
     docsPath: options.docsPath,
     outputDir: options.output,
-    concurrency: parseInt(options.concurrency)
+    concurrency: parseInt(options.concurrency),
+    llmsPaths: options.llmsPaths.split(","),
+    tryDocsSubdomain: !options.noSubdomain
   };
   console.log(`\uD83D\uDD0D Scraping ${options.url}...`);
   console.log(`\uD83D\uDCC1 Output: ${options.output}`);
@@ -2423,10 +2425,13 @@ program.command("scrape").description("Scrape documentation from a URL").require
     }
   }
 });
-program.command("discover").description("Discover all available documentation pages").requiredOption("-u, --url <url>", "Base URL of the documentation site").option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en").action(async (options) => {
+program.command("discover").description("Discover all available documentation pages").requiredOption("-u, --url <url>", "Base URL of the documentation site").option("--docs-path <path>", "Docs path (default: /docs/en)", "/docs/en").option("--llms-paths <paths>", "Comma-separated llms.txt paths to try", "/llms.txt,/docs/llms.txt").option("--no-subdomain", "Disable docs/doc subdomain fallback", false).action(async (options) => {
+  console.log("DEBUG CLI: Options received:", options);
   const scraper = new import__2.MarkdownDocsScraper({
     baseUrl: options.url,
-    docsPath: options.docsPath
+    docsPath: options.docsPath,
+    llmsPaths: options.llmsPaths.split(","),
+    tryDocsSubdomain: !options.noSubdomain
   });
   console.log(`\uD83D\uDD0D Discovering pages from ${options.url}...`);
   const pages = await scraper.discoverPages();
@@ -2434,7 +2439,8 @@ program.command("discover").description("Discover all available documentation pa
 Found ${pages.length} pages:
 `);
   pages.forEach((page) => {
-    console.log(`  - ${page}`);
+    const path = page.category ? `${page.category}/${page.page}` : page.page;
+    console.log(`  - ${path}`);
   });
 });
 program.command("anthropic").description("Quick scrape of Anthropic Claude Code docs (uses llms.txt)").option("-o, --output <dir>", "Output directory", "./docs").action(async (options) => {

package/dist/index.d.ts

@@ -0,0 +1,116 @@
+/**
+ * @ebowwa/markdown-docs-scraper
+ *
+ * Composable markdown documentation scraper.
+ * - Configurable llms.txt paths with fallbacks
+ * - Custom URL patterns for different doc sites
+ * - Works with any markdown documentation site
+ * - Uses full URLs from llms.txt directly
+ */
+export interface DocPage {
+    url: string;
+    title: string;
+    content: string;
+    category?: string;
+    pageName?: string;
+}
+export interface ScraperOptions {
+    baseUrl: string;
+    docsPath?: string;
+    categories?: Record<string, string[]>;
+    outputDir?: string;
+    concurrency?: number;
+    onProgress?: (current: number, total: number) => void;
+    /** Custom llms.txt paths to try (default: ["/llms.txt", "/docs/llms.txt"]) */
+    llmsPaths?: string[];
+    /** Also try docs subdomain variants (e.g., docs.example.com) */
+    tryDocsSubdomain?: boolean;
+    /** Custom regex pattern to extract pages from llms.txt (must have 3 capture groups: title, fullUrl, path) */
+    linkPattern?: RegExp;
+    /** Use full URLs from llms.txt directly (default: true for generic pattern) */
+    useDirectUrls?: boolean;
+}
+export interface ScraperResult {
+    downloaded: DocPage[];
+    failed: Array<{
+        url: string;
+        error: string;
+    }>;
+    duration: number;
+}
+/** Discovered page with full URL */
+interface DiscoveredPage {
+    category: string;
+    page: string;
+    fullUrl: string;
+}
+/** Extract title from markdown content */
+export declare function extractTitle(markdown: string): string;
+/** Parse page path into category and page name */
+export declare function parsePagePath(pagePath: string): {
+    category: string;
+    page: string;
+};
+/** Fetch markdown content from URL */
+export declare function fetchMarkdown(url: string, userAgent?: string): Promise<string | null>;
+export declare class MarkdownDocsScraper {
+    private options;
+    constructor(options: ScraperOptions);
+    /**
+     * Build URL for a documentation page (fallback when no direct URL)
+     */
+    buildUrl(category: string, page: string): string;
+    /**
+     * Download a page using either direct URL or built URL
+     */
+    downloadPage(pageInfo: DiscoveredPage): Promise<DocPage | null>;
+    /**
+     * Generate possible llms.txt URLs to try
+     */
+    private getLlmsUrls;
+    /**
+     * Fetch llms.txt from multiple possible URLs with fallback
+     */
+    private fetchLlmsTxt;
+    /**
+     * Discover pages from llms.txt index
+     */
+    discoverPages(): Promise<DiscoveredPage[]>;
+    /**
+     * Scrape pages discovered from llms.txt
+     */
+    scrapeFromLlms(): Promise<ScraperResult>;
+    /**
+     * Scrape all documentation pages (uses categories)
+     */
+    scrape(): Promise<ScraperResult>;
+    /**
+     * Save scraped pages to disk
+     */
+    savePages(pages: DocPage[]): Promise<void>;
+    /**
+     * Get list of pages to scrape based on categories
+     */
+    private getPagesToScrape;
+}
+/**
+ * Scrape markdown documentation with a single function call
+ */
+export declare function scrapeMarkdownDocs(options: ScraperOptions & {
+    useLlms?: boolean;
+}): Promise<ScraperResult>;
+/** Pattern for Claude Code docs: /docs/en/page.md */
+export declare const CLAUDE_CODE_PATTERN: RegExp;
+/** Pattern for generic docs: any domain/path.md */
+export declare const GENERIC_PATTERN: RegExp;
+/** Create scraper options for Claude Code docs */
+export declare function claudeCodeOptions(outputDir: string): ScraperOptions;
+/** Create scraper options for Polymarket docs */
+export declare function polymarketOptions(outputDir: string): ScraperOptions;
+/**
+ * Re-export scrapers module for composable scraper architecture.
+ * This provides a registry-based system for different scraper implementations.
+ */
+export { type SourceType, type SourceConfig, type Scraper, type ScrapeResult as ScraperModuleResult, type DownloadResult, llmsTxtScraper, githubRawScraper, CLAUDE_CODE_PATTERN as SCRAPER_CLAUDE_CODE_PATTERN, GENERIC_PATTERN as SCRAPER_GENERIC_PATTERN, registerScraper, getScraper, scrapeSource, } from "./scrapers/index";
+export default MarkdownDocsScraper;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,MAAM,WAAW,OAAO;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACtD,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6GAA6G;IAC7G,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,OAAO,EAAE,CAAC;IACtB,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAYD,0CAA0C;AAC1C,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrD;AAED,kDAAkD;AAClD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAkBlF;AAED,sCAAsC;AACtC,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAkC,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAkBpH;AAMD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA2B;gBAE9B,OAAO,EAAE,cAAc;IAenC;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAUhD;;OAEG;IACG,YAAY,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAqBrE;;OAEG;IACH,OAAO,CAAC,WAAW;IA6BnB;;OAEG;YACW,YAAY;IA6B1B;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAuChD;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,aAAa,CAAC;IA+C9C;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,aAAa,CAAC;IAwCtC;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBhD;;OAEG;IACH,OAAO,CAAC,gBAAgB;CAWzB;AAMD;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,cAAc,GAAG;IAAE,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,GAC9C,OAAO,CAAC,aAAa,CAAC,CAWxB;AAMD,qDAAqD;AACrD,eAAO,MAAM,mBAAmB,QAAiE,CAAC;AAElG,mDAAmD;AACnD,eAAO,MAAM,eAAe,QAAuB,CAAC;AAEpD,kDAAkD;AAClD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAED,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAMD;;;GAGG;AACH,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,OAAO,EACZ,KAAK,YAAY,IAAI,mBAAmB,EACxC,KAAK,cAAc,EAGnB,cAAc,EACd,gBAAgB,EAChB,mBAAmB,IAAI,2BAA2B,EAClD,eAAe,IAAI,uBAAuB,EAG1C,eAAe,EACf,UAAU,EACV,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,eAAe,mBAAmB,CAAC"}