@ebowwa/markdown-docs-scraper 1.0.0 → 1.2.0

package/src/scrapers/index.ts

@@ -0,0 +1,16 @@
+/**
+ * Scrapers Module
+ *
+ * Composable scraper architecture for multiple documentation source types.
+ * This module provides a registry-based system for different scraper implementations.
+ */
+
+// Types
+export type { SourceType, SourceConfig, Scraper, ScrapeResult, DownloadResult } from "./types";
+
+// Scrapers
+export { llmsTxtScraper, CLAUDE_CODE_PATTERN, GENERIC_PATTERN } from "./llms-txt";
+export { githubRawScraper } from "./github-raw";
+
+// Registry
+export { registerScraper, getScraper, scrapeSource } from "./registry";

package/src/scrapers/llms-txt.ts

@@ -0,0 +1,101 @@
+/**
+ * LLMS-TXT Scraper
+ *
+ * Scrapes documentation sites that provide llms.txt index files.
+ * Uses the core MarkdownDocsScraper under the hood.
+ */
+
+import { scrapeMarkdownDocs, type DocPage } from "../index";
+import type { Scraper, SourceConfig, ScrapeResult, DownloadResult } from "./types";
+
+// ============================================================================
+// URL PATTERNS
+// ============================================================================
+
+/** Pattern for Claude Code docs: /docs/en/page.md */
+export const CLAUDE_CODE_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
+
+/** Pattern for generic docs: any domain/path.md */
+export const GENERIC_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^/]+\/([^\s)]+\.md))\)/g;
+
+// ============================================================================
+// LLMS-TXT SCRAPER
+// ============================================================================
+
+export const llmsTxtScraper: Scraper = {
+  type: "llms-txt",
+
+  async scrape(config: SourceConfig): Promise<ScrapeResult> {
+    const options = getScraperOptions(config);
+    const result = await scrapeMarkdownDocs(options);
+
+    // Convert DocPage[] to DownloadResult[]
+    const downloaded: DownloadResult[] = result.downloaded.map((page: DocPage) => {
+      const category = page.category || "";
+      const filename = `${page.pageName || "untitled"}.md`;
+      const path = category ? `${category}/${filename}` : filename;
+
+      return {
+        success: true,
+        path,
+        title: page.title,
+      };
+    });
+
+    return {
+      downloaded,
+      failed: result.failed,
+      duration: result.duration,
+    };
+  },
+};
+
+// ============================================================================
+// OPTIONS BUILDER
+// ============================================================================
+
+/**
+ * Get scraper options based on source configuration
+ */
+function getScraperOptions(config: SourceConfig) {
+  const baseOptions = {
+    baseUrl: config.baseUrl,
+    docsPath: config.docsPath,
+    outputDir: config.outputDir,
+    concurrency: 10,
+    useLlms: true,
+    tryDocsSubdomain: false,
+  };
+
+  // Source-specific options
+  if (config.name === "Claude Code") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/docs/llms.txt"],
+      linkPattern: CLAUDE_CODE_PATTERN,
+    };
+  }
+
+  if (config.name === "Polymarket") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/llms.txt"],
+      linkPattern: GENERIC_PATTERN,
+    };
+  }
+
+  if (config.name === "Bun") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/docs/llms.txt", "/llms.txt"],
+      linkPattern: GENERIC_PATTERN,
+    };
+  }
+
+  // Default: use provided llmsTxtPath or try common paths
+  return {
+    ...baseOptions,
+    llmsPaths: config.llmsTxtPath ? [config.llmsTxtPath] : ["/llms.txt", "/docs/llms.txt"],
+    linkPattern: config.linkPattern || GENERIC_PATTERN,
+  };
+}

package/src/scrapers/registry.ts

@@ -0,0 +1,55 @@
+/**
+ * Scraper Registry
+ *
+ * Maps source types to scraper implementations.
+ * Allows registering new scrapers and looking them up by type.
+ */
+
+import type { Scraper, SourceType, SourceConfig, ScrapeResult } from "./types";
+import { llmsTxtScraper } from "./llms-txt";
+import { githubRawScraper } from "./github-raw";
+
+// ============================================================================
+// SCRAPER REGISTRY
+// ============================================================================
+
+/** Registry of all available scrapers keyed by type */
+const scrapers: Map<SourceType, Scraper> = new Map();
+
+/**
+ * Register a scraper implementation
+ */
+export function registerScraper(scraper: Scraper): void {
+  scrapers.set(scraper.type, scraper);
+}
+
+/**
+ * Get a scraper by type
+ */
+export function getScraper(type: SourceType): Scraper | undefined {
+  return scrapers.get(type);
+}
+
+/**
+ * Scrape a source using the appropriate scraper
+ */
+export async function scrapeSource(config: SourceConfig): Promise<ScrapeResult> {
+  const scraper = scrapers.get(config.sourceType);
+
+  if (!scraper) {
+    throw new Error(`No scraper registered for type: ${config.sourceType}`);
+  }
+
+  return scraper.scrape(config);
+}
+
+// ============================================================================
+// DEFAULT REGISTRATIONS
+// ============================================================================
+
+// Register built-in scrapers
+registerScraper(llmsTxtScraper);
+registerScraper(githubRawScraper);
+
+// Export scrapers for direct access if needed
+export { llmsTxtScraper, githubRawScraper };

package/src/scrapers/types.ts

@@ -0,0 +1,79 @@
+/**
+ * Scraper Types
+ *
+ * Core types for the composable scraper architecture.
+ * These types define the interface that all scrapers must implement.
+ */
+
+// ============================================================================
+// SOURCE TYPES
+// ============================================================================
+
+/** Supported documentation source types */
+export type SourceType = "llms-txt" | "github-raw";
+
+// ============================================================================
+// SCRAPER INTERFACE
+// ============================================================================
+
+/** Result from scraping a source */
+export interface ScrapeResult {
+  downloaded: DownloadResult[];
+  failed: Array<{ url: string; error: string }>;
+  duration?: number;
+}
+
+/** Individual download result */
+export interface DownloadResult {
+  success: boolean;
+  path: string;
+  title?: string;
+}
+
+/** Scraper interface - all scrapers must implement this */
+export interface Scraper {
+  /** Source type identifier */
+  type: SourceType;
+
+  /** Scrape documentation from a source */
+  scrape(config: SourceConfig): Promise<ScrapeResult>;
+}
+
+// ============================================================================
+// SOURCE CONFIG
+// ============================================================================
+
+/** Source configuration */
+export interface SourceConfig {
+  /** Display name */
+  name: string;
+
+  /** Source type - determines which scraper to use */
+  sourceType: SourceType;
+
+  /** Base URL for the documentation */
+  baseUrl: string;
+
+  /** Path to docs (e.g., /docs, /docs/en) */
+  docsPath: string;
+
+  /** Output directory for downloaded docs */
+  outputDir: string;
+
+  /** Output directory for daily reports */
+  reportDir: string;
+
+  /** llms.txt path (for llms-txt sources) */
+  llmsTxtPath?: string;
+
+  /** Custom link pattern for llms.txt parsing */
+  linkPattern?: RegExp;
+
+  /** GitHub config (for github-raw sources or GitHub API data) */
+  github?: {
+    repo: string;
+    includeCommits: boolean;
+    includeReleases: boolean;
+    includePRs: boolean;
+  };
+}