@ebowwa/markdown-docs-scraper 1.1.0 → 1.2.1

package/dist/scrapers/index.js

@@ -0,0 +1,448 @@
+import { createRequire } from "node:module";
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __toESM = (mod, isNodeMode, target) => {
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: () => mod[key],
+        enumerable: true
+      });
+  return to;
+};
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+// src/index.ts
+var GENERIC_LINK_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^/]+\/([^\s)]+\.md))\)/g;
+function extractTitle(markdown) {
+  const titleMatch = markdown.match(/^#\s+(.+)$/m);
+  return titleMatch ? titleMatch[1].trim() : "Untitled";
+}
+function parsePagePath(pagePath) {
+  const pageName = pagePath.replace(".md", "");
+  const pathParts = pageName.split("/");
+  if (pathParts.length === 1) {
+    return { category: "", page: pathParts[0] };
+  } else if (pathParts.length === 2) {
+    return { category: pathParts[0], page: pathParts[1] };
+  } else {
+    return {
+      category: pathParts.slice(0, -1).join("/"),
+      page: pathParts[pathParts.length - 1]
+    };
+  }
+}
+async function fetchMarkdown(url, userAgent = "@ebowwa/markdown-docs-scraper") {
+  try {
+    const response = await fetch(url, {
+      headers: {
+        Accept: "text/markdown, text/plain",
+        "User-Agent": userAgent
+      }
+    });
+    if (!response.ok) {
+      return null;
+    }
+    return await response.text();
+  } catch (error) {
+    console.error(`Error fetching ${url}:`, error);
+    return null;
+  }
+}
+
+class MarkdownDocsScraper {
+  options;
+  constructor(options) {
+    this.options = {
+      baseUrl: options.baseUrl,
+      docsPath: options.docsPath || "/docs/en",
+      categories: options.categories || {},
+      outputDir: options.outputDir || "./docs",
+      concurrency: options.concurrency || 5,
+      onProgress: options.onProgress || (() => {}),
+      llmsPaths: options.llmsPaths || ["/llms.txt", "/docs/llms.txt"],
+      tryDocsSubdomain: options.tryDocsSubdomain ?? true,
+      linkPattern: options.linkPattern || GENERIC_LINK_PATTERN,
+      useDirectUrls: options.useDirectUrls ?? true
+    };
+  }
+  buildUrl(category, page) {
+    if (category) {
+      return `${this.options.baseUrl}${this.options.docsPath}/${category}/${page}.md`;
+    } else if (this.options.docsPath) {
+      return `${this.options.baseUrl}${this.options.docsPath}/${page}.md`;
+    } else {
+      return `${this.options.baseUrl}/${page}.md`;
+    }
+  }
+  async downloadPage(pageInfo) {
+    const url = this.options.useDirectUrls && pageInfo.fullUrl ? pageInfo.fullUrl : this.buildUrl(pageInfo.category, pageInfo.page);
+    const content = await fetchMarkdown(url);
+    if (!content) {
+      return null;
+    }
+    return {
+      url,
+      title: extractTitle(content),
+      content,
+      category: pageInfo.category,
+      pageName: pageInfo.page
+    };
+  }
+  getLlmsUrls() {
+    const urls = [];
+    const baseUrl = this.options.baseUrl;
+    for (const path of this.options.llmsPaths) {
+      urls.push(`${baseUrl}${path}`);
+    }
+    if (this.options.tryDocsSubdomain) {
+      try {
+        const url = new URL(baseUrl);
+        const hostname = url.hostname;
+        if (!hostname.startsWith("docs.") && !hostname.startsWith("doc.")) {
+          const docsDomain = hostname.replace(/^www\./, "");
+          urls.push(`${url.protocol}//docs.${docsDomain}/llms.txt`);
+          urls.push(`${url.protocol}//docs.${docsDomain}/docs/llms.txt`);
+        }
+      } catch {}
+    }
+    return urls;
+  }
+  async fetchLlmsTxt() {
+    const urls = this.getLlmsUrls();
+    console.log(`DEBUG: Trying URLs: ${urls.join(", ")}`);
+    for (const llmsUrl of urls) {
+      try {
+        console.log(`DEBUG: Fetching ${llmsUrl}...`);
+        const response = await fetch(llmsUrl, {
+          headers: {
+            Accept: "text/plain",
+            "User-Agent": "@ebowwa/markdown-docs-scraper"
+          }
+        });
+        console.log(`DEBUG: Response status: ${response.status}`);
+        if (response.ok) {
+          const content = await response.text();
+          console.log(`Found llms.txt at ${llmsUrl}`);
+          return { content, url: llmsUrl };
+        }
+      } catch (error) {
+        console.log(`DEBUG: Error: ${error}`);
+        continue;
+      }
+    }
+    return null;
+  }
+  async discoverPages() {
+    const pages = [];
+    try {
+      const llmsResult = await this.fetchLlmsTxt();
+      if (!llmsResult) {
+        const attemptedUrls = this.getLlmsUrls();
+        console.warn(`Could not fetch llms.txt from any of: ${attemptedUrls.join(", ")}`);
+        return pages;
+      }
+      const { content } = llmsResult;
+      const pattern = this.options.linkPattern;
+      const regex = new RegExp(pattern.source, pattern.flags);
+      let match;
+      console.log(`DEBUG: Using pattern: ${pattern.source}`);
+      console.log(`DEBUG: Content length: ${content.length}`);
+      while ((match = regex.exec(content)) !== null) {
+        const fullUrl = match[2];
+        const pagePath = match[3];
+        const { category, page } = parsePagePath(pagePath);
+        pages.push({ category, page, fullUrl });
+      }
+      console.log(`Discovered ${pages.length} pages from llms.txt`);
+    } catch (error) {
+      console.error("Error discovering pages:", error);
+    }
+    return pages;
+  }
+  async scrapeFromLlms() {
+    const startTime = Date.now();
+    const downloaded = [];
+    const failed = [];
+    const pages = await this.discoverPages();
+    if (pages.length === 0) {
+      console.log("No pages discovered, falling back to categories");
+      return this.scrape();
+    }
+    console.log(`Scraping ${pages.length} discovered pages...`);
+    for (let i = 0;i < pages.length; i += this.options.concurrency) {
+      const batch = pages.slice(i, i + this.options.concurrency);
+      const results = await Promise.allSettled(batch.map((page) => this.downloadPage(page)));
+      results.forEach((result, index) => {
+        const page = batch[index];
+        if (result.status === "fulfilled" && result.value) {
+          downloaded.push(result.value);
+        } else {
+          const url = this.options.useDirectUrls && page.fullUrl ? page.fullUrl : this.buildUrl(page.category, page.page);
+          failed.push({
+            url,
+            error: result.status === "rejected" ? result.reason : "Not found"
+          });
+        }
+        this.options.onProgress(downloaded.length + failed.length, pages.length);
+      });
+    }
+    const duration = Date.now() - startTime;
+    console.log(`✅ Downloaded: ${downloaded.length} pages`);
+    console.log(`❌ Failed: ${failed.length} pages`);
+    console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
+    return { downloaded, failed, duration };
+  }
+  async scrape() {
+    const startTime = Date.now();
+    const downloaded = [];
+    const failed = [];
+    const pages = this.getPagesToScrape();
+    const total = pages.length;
+    console.log(`Scraping ${total} pages from ${this.options.baseUrl}...`);
+    for (let i = 0;i < pages.length; i += this.options.concurrency) {
+      const batch = pages.slice(i, i + this.options.concurrency);
+      const results = await Promise.allSettled(batch.map((page) => this.downloadPage({ ...page, fullUrl: "" })));
+      results.forEach((result, index) => {
+        const page = batch[index];
+        if (result.status === "fulfilled" && result.value) {
+          downloaded.push(result.value);
+        } else {
+          failed.push({
+            url: this.buildUrl(page.category, page.page),
+            error: result.status === "rejected" ? result.reason : "Not found"
+          });
+        }
+        this.options.onProgress(downloaded.length + failed.length, total);
+      });
+    }
+    const duration = Date.now() - startTime;
+    console.log(`✅ Downloaded: ${downloaded.length} pages`);
+    console.log(`❌ Failed: ${failed.length} pages`);
+    console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
+    return { downloaded, failed, duration };
+  }
+  extractBody(content) {
+    const headerRegex = /^<!--\nSource: [^\n]+\nDownloaded: [^\n]+\n-->\n\n/;
+    return content.replace(headerRegex, "");
+  }
+  async savePages(pages) {
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    let updated = 0;
+    let skipped = 0;
+    for (const page of pages) {
+      const nameToUse = page.pageName || page.url.split("/").pop()?.replace(".md", "") || "untitled";
+      const dir = page.category ? path.join(this.options.outputDir, page.category) : this.options.outputDir;
+      await fs.mkdir(dir, { recursive: true });
+      const filepath = path.join(dir, `${nameToUse}.md`);
+      try {
+        const existingContent = await fs.readFile(filepath, "utf-8");
+        const existingBody = this.extractBody(existingContent);
+        if (existingBody === page.content) {
+          skipped++;
+          continue;
+        }
+      } catch {}
+      const header = `<!--
+Source: ${page.url}
+Downloaded: ${new Date().toISOString()}
+-->
+
+`;
+      await fs.writeFile(filepath, header + page.content, "utf-8");
+      updated++;
+    }
+    return { updated, skipped };
+  }
+  getPagesToScrape() {
+    const pages = [];
+    for (const [category, pageList] of Object.entries(this.options.categories)) {
+      for (const page of pageList) {
+        pages.push({ category, page, fullUrl: "" });
+      }
+    }
+    return pages;
+  }
+}
+async function scrapeMarkdownDocs(options) {
+  const scraper = new MarkdownDocsScraper(options);
+  const result = options.useLlms ? await scraper.scrapeFromLlms() : await scraper.scrape();
+  let saveStats;
+  if (options.outputDir) {
+    saveStats = await scraper.savePages(result.downloaded);
+    if (saveStats.updated > 0 || saveStats.skipped > 0) {
+      console.log(`  Saved: ${saveStats.updated} updated, ${saveStats.skipped} unchanged`);
+    }
+  }
+  return { ...result, saveStats };
+}
+
+// src/scrapers/llms-txt.ts
+var CLAUDE_CODE_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
+var GENERIC_PATTERN = /\[([^\]]+)\]\((https?:\/\/[^/]+\/([^\s)]+\.md))\)/g;
+var llmsTxtScraper = {
+  type: "llms-txt",
+  async scrape(config) {
+    const options = getScraperOptions(config);
+    const result = await scrapeMarkdownDocs(options);
+    const downloaded = result.downloaded.map((page) => {
+      const category = page.category || "";
+      const filename = `${page.pageName || "untitled"}.md`;
+      const path = category ? `${category}/${filename}` : filename;
+      return {
+        success: true,
+        path,
+        title: page.title
+      };
+    });
+    return {
+      downloaded,
+      failed: result.failed,
+      duration: result.duration
+    };
+  }
+};
+function getScraperOptions(config) {
+  const baseOptions = {
+    baseUrl: config.baseUrl,
+    docsPath: config.docsPath,
+    outputDir: config.outputDir,
+    concurrency: 10,
+    useLlms: true,
+    tryDocsSubdomain: false
+  };
+  if (config.name === "Claude Code") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/docs/llms.txt"],
+      linkPattern: CLAUDE_CODE_PATTERN
+    };
+  }
+  if (config.name === "Polymarket") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/llms.txt"],
+      linkPattern: GENERIC_PATTERN
+    };
+  }
+  if (config.name === "Bun") {
+    return {
+      ...baseOptions,
+      llmsPaths: ["/docs/llms.txt", "/llms.txt"],
+      linkPattern: GENERIC_PATTERN
+    };
+  }
+  return {
+    ...baseOptions,
+    llmsPaths: config.llmsTxtPath ? [config.llmsTxtPath] : ["/llms.txt", "/docs/llms.txt"],
+    linkPattern: config.linkPattern || GENERIC_PATTERN
+  };
+}
+// src/scrapers/github-raw.ts
+var githubRawScraper = {
+  type: "github-raw",
+  async scrape(config) {
+    const startTime = Date.now();
+    const downloaded = [];
+    const failed = [];
+    if (!config.github?.repo) {
+      throw new Error(`GitHub source "${config.name}" missing github.repo config`);
+    }
+    const files = await fetchGitHubMarkdownFiles(config.github.repo, config.docsPath.replace(/^\//, ""));
+    for (const file of files) {
+      const content = await fetchGitHubRawContent(config.github.repo, file.path);
+      if (content) {
+        downloaded.push({
+          success: true,
+          path: file.name,
+          title: extractTitle2(content) || file.name.replace(".md", "")
+        });
+        await saveFile(config.outputDir, file.name, content);
+      } else {
+        failed.push({
+          url: `https://raw.githubusercontent.com/${config.github.repo}/main/${file.path}`,
+          error: "Failed to fetch content"
+        });
+      }
+    }
+    return {
+      downloaded,
+      failed,
+      duration: Date.now() - startTime
+    };
+  }
+};
+async function fetchGitHubMarkdownFiles(repo, path) {
+  const url = `https://api.github.com/repos/${repo}/contents/${path}`;
+  const response = await fetch(url, {
+    headers: {
+      Accept: "application/vnd.github.v3+json",
+      "User-Agent": "@ebowwa/markdown-docs-scraper"
+    }
+  });
+  if (!response.ok) {
+    throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
+  }
+  const contents = await response.json();
+  return contents.filter((item) => item.type === "file" && item.name.endsWith(".md"));
+}
+async function fetchGitHubRawContent(repo, path) {
+  const url = `https://raw.githubusercontent.com/${repo}/main/${path}`;
+  try {
+    const response = await fetch(url, {
+      headers: {
+        Accept: "text/plain",
+        "User-Agent": "@ebowwa/markdown-docs-scraper"
+      }
+    });
+    if (!response.ok) {
+      return null;
+    }
+    return await response.text();
+  } catch (error) {
+    console.error(`Error fetching ${url}:`, error);
+    return null;
+  }
+}
+function extractTitle2(markdown) {
+  const titleMatch = markdown.match(/^#\s+(.+)$/m);
+  return titleMatch ? titleMatch[1].trim() : null;
+}
+async function saveFile(outputDir, filename, content) {
+  const fs = await import("fs/promises");
+  const path = await import("path");
+  const outputPath = path.join(outputDir, filename);
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, content, "utf-8");
+}
+// src/scrapers/registry.ts
+var scrapers = new Map;
+function registerScraper(scraper) {
+  scrapers.set(scraper.type, scraper);
+}
+function getScraper(type) {
+  return scrapers.get(type);
+}
+async function scrapeSource(config) {
+  const scraper = scrapers.get(config.sourceType);
+  if (!scraper) {
+    throw new Error(`No scraper registered for type: ${config.sourceType}`);
+  }
+  return scraper.scrape(config);
+}
+registerScraper(llmsTxtScraper);
+registerScraper(githubRawScraper);
+export {
+  scrapeSource,
+  registerScraper,
+  llmsTxtScraper,
+  githubRawScraper,
+  getScraper,
+  GENERIC_PATTERN,
+  CLAUDE_CODE_PATTERN
+};

package/dist/scrapers/llms-txt.d.ts

@@ -0,0 +1,13 @@
+/**
+ * LLMS-TXT Scraper
+ *
+ * Scrapes documentation sites that provide llms.txt index files.
+ * Uses the core MarkdownDocsScraper under the hood.
+ */
+import type { Scraper } from "./types";
+/** Pattern for Claude Code docs: /docs/en/page.md */
+export declare const CLAUDE_CODE_PATTERN: RegExp;
+/** Pattern for generic docs: any domain/path.md */
+export declare const GENERIC_PATTERN: RegExp;
+export declare const llmsTxtScraper: Scraper;
+//# sourceMappingURL=llms-txt.d.ts.map

package/dist/scrapers/llms-txt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llms-txt.d.ts","sourceRoot":"","sources":["../../src/scrapers/llms-txt.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,OAAO,EAA8C,MAAM,SAAS,CAAC;AAMnF,qDAAqD;AACrD,eAAO,MAAM,mBAAmB,QAAiE,CAAC;AAElG,mDAAmD;AACnD,eAAO,MAAM,eAAe,QAAuD,CAAC;AAMpF,eAAO,MAAM,cAAc,EAAE,OA0B5B,CAAC"}

package/dist/scrapers/registry.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Scraper Registry
+ *
+ * Maps source types to scraper implementations.
+ * Allows registering new scrapers and looking them up by type.
+ */
+import type { Scraper, SourceType, SourceConfig, ScrapeResult } from "./types";
+import { llmsTxtScraper } from "./llms-txt";
+import { githubRawScraper } from "./github-raw";
+/**
+ * Register a scraper implementation
+ */
+export declare function registerScraper(scraper: Scraper): void;
+/**
+ * Get a scraper by type
+ */
+export declare function getScraper(type: SourceType): Scraper | undefined;
+/**
+ * Scrape a source using the appropriate scraper
+ */
+export declare function scrapeSource(config: SourceConfig): Promise<ScrapeResult>;
+export { llmsTxtScraper, githubRawScraper };
+//# sourceMappingURL=registry.d.ts.map

package/dist/scrapers/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/scrapers/registry.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC/E,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAShD;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAEtD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,GAAG,SAAS,CAEhE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAQ9E;AAWD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC"}

package/dist/scrapers/types.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Scraper Types
+ *
+ * Core types for the composable scraper architecture.
+ * These types define the interface that all scrapers must implement.
+ */
+/** Supported documentation source types */
+export type SourceType = "llms-txt" | "github-raw";
+/** Result from scraping a source */
+export interface ScrapeResult {
+    downloaded: DownloadResult[];
+    failed: Array<{
+        url: string;
+        error: string;
+    }>;
+    duration?: number;
+}
+/** Individual download result */
+export interface DownloadResult {
+    success: boolean;
+    path: string;
+    title?: string;
+}
+/** Scraper interface - all scrapers must implement this */
+export interface Scraper {
+    /** Source type identifier */
+    type: SourceType;
+    /** Scrape documentation from a source */
+    scrape(config: SourceConfig): Promise<ScrapeResult>;
+}
+/** Source configuration */
+export interface SourceConfig {
+    /** Display name */
+    name: string;
+    /** Source type - determines which scraper to use */
+    sourceType: SourceType;
+    /** Base URL for the documentation */
+    baseUrl: string;
+    /** Path to docs (e.g., /docs, /docs/en) */
+    docsPath: string;
+    /** Output directory for downloaded docs */
+    outputDir: string;
+    /** Output directory for daily reports */
+    reportDir: string;
+    /** llms.txt path (for llms-txt sources) */
+    llmsTxtPath?: string;
+    /** Custom link pattern for llms.txt parsing */
+    linkPattern?: RegExp;
+    /** GitHub config (for github-raw sources or GitHub API data) */
+    github?: {
+        repo: string;
+        includeCommits: boolean;
+        includeReleases: boolean;
+        includePRs: boolean;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/scrapers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/scrapers/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,YAAY,CAAC;AAMnD,oCAAoC;AACpC,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,iCAAiC;AACjC,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,2DAA2D;AAC3D,MAAM,WAAW,OAAO;IACtB,6BAA6B;IAC7B,IAAI,EAAE,UAAU,CAAC;IAEjB,yCAAyC;IACzC,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACrD;AAMD,2BAA2B;AAC3B,MAAM,WAAW,YAAY;IAC3B,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;IAEb,oDAAoD;IACpD,UAAU,EAAE,UAAU,CAAC;IAEvB,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAEhB,2CAA2C;IAC3C,QAAQ,EAAE,MAAM,CAAC;IAEjB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAElB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,+CAA+C;IAC/C,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,gEAAgE;IAChE,MAAM,CAAC,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,cAAc,EAAE,OAAO,CAAC;QACxB,eAAe,EAAE,OAAO,CAAC;QACzB,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;CACH"}

package/package.json

@@ -1,6 +1,10 @@
 {
   "name": "@ebowwa/markdown-docs-scraper",
-  "version": "1.1.0",
+  "version": "1.2.1",
+  "files": [
+    "dist",
+    "src"
+  ],
   "description": "Scrape and mirror markdown-based documentation sites",
   "type": "module",
   "main": "./dist/index.js",
@@ -13,13 +17,17 @@
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
     },
+    "./scrapers": {
+      "import": "./dist/scrapers/index.js",
+      "types": "./dist/scrapers/index.d.ts"
+    },
     "./cli": {
       "import": "./dist/cli.js",
       "types": "./dist/cli.d.ts"
     }
   },
   "scripts": {
-    "build": "bun build src/index.ts --outdir dist --target node && bun build src/cli.ts --outdir dist --target node",
+    "build": "bun build src/index.ts --outdir dist --target node && mkdir -p dist/scrapers && bun build src/scrapers/index.ts --outdir dist/scrapers --target node && bun build src/cli.ts --outdir dist --target node && tsc --build",
     "dev": "bun run src/cli.ts",
     "test": "bun test",
     "prepublishOnly": "bun run build"