@ebowwa/markdown-docs-scraper 1.2.0 → 1.2.1

package/dist/index.d.ts

@@ -85,9 +85,16 @@ export declare class MarkdownDocsScraper {
      */
     scrape(): Promise<ScraperResult>;
     /**
-     * Save scraped pages to disk
+     * Extract body content from a file (strips header comment)
      */
-    savePages(pages: DocPage[]): Promise<void>;
+    private extractBody;
+    /**
+     * Save scraped pages to disk (only writes if content changed)
+     */
+    savePages(pages: DocPage[]): Promise<{
+        updated: number;
+        skipped: number;
+    }>;
     /**
      * Get list of pages to scrape based on categories
      */
@@ -98,7 +105,12 @@ export declare class MarkdownDocsScraper {
  */
 export declare function scrapeMarkdownDocs(options: ScraperOptions & {
     useLlms?: boolean;
-}): Promise<ScraperResult>;
+}): Promise<ScraperResult & {
+    saveStats?: {
+        updated: number;
+        skipped: number;
+    };
+}>;
 /** Pattern for Claude Code docs: /docs/en/page.md */
 export declare const CLAUDE_CODE_PATTERN: RegExp;
 /** Pattern for generic docs: any domain/path.md */

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,MAAM,WAAW,OAAO;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACtD,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6GAA6G;IAC7G,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,OAAO,EAAE,CAAC;IACtB,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAYD,0CAA0C;AAC1C,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrD;AAED,kDAAkD;AAClD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAkBlF;AAED,sCAAsC;AACtC,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAkC,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAkBpH;AAMD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA2B;gBAE9B,OAAO,EAAE,cAAc;IAenC;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAUhD;;OAEG;IACG,YAAY,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAqBrE;;OAEG;IACH,OAAO,CAAC,WAAW;IA6BnB;;OAEG;YACW,YAAY;IA6B1B;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAuChD;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,aAAa,CAAC;IA+C9C;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,aAAa,CAAC;IAwCtC;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBhD;;OAEG;IACH,OAAO,CAAC,gBAAgB;CAWzB;AAMD;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,cAAc,GAAG;IAAE,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,GAC9C,OAAO,CAAC,aAAa,CAAC,CAWxB;AAMD,qDAAqD;AACrD,eAAO,MAAM,mBAAmB,QAAiE,CAAC;AAElG,mDAAmD;AACnD,eAAO,MAAM,eAAe,QAAuB,CAAC;AAEpD,kDAAkD;AAClD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAED,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAMD;;;GAGG;AACH,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,OAAO,EACZ,KAAK,YAAY,IAAI,mBAAmB,EACxC,KAAK,cAAc,EAGnB,cAAc,EACd,gBAAgB,EAChB,mBAAmB,IAAI,2BAA2B,EAClD,eAAe,IAAI,uBAAuB,EAG1C,eAAe,EACf,UAAU,EACV,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,eAAe,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,MAAM,WAAW,OAAO;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACtD,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6GAA6G;IAC7G,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,OAAO,EAAE,CAAC;IACtB,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oCAAoC;AACpC,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAYD,0CAA0C;AAC1C,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrD;AAED,kDAAkD;AAClD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAkBlF;AAED,sCAAsC;AACtC,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAkC,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAkBpH;AAMD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA2B;gBAE9B,OAAO,EAAE,cAAc;IAenC;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAUhD;;OAEG;IACG,YAAY,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAqBrE;;OAEG;IACH,OAAO,CAAC,WAAW;IA6BnB;;OAEG;YACW,YAAY;IA6B1B;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAuChD;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,aAAa,CAAC;IA+C9C;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,aAAa,CAAC;IAwCtC;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAyChF;;OAEG;IACH,OAAO,CAAC,gBAAgB;CAWzB;AAMD;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,cAAc,GAAG;IAAE,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,GAC9C,OAAO,CAAC,aAAa,GAAG;IAAE,SAAS,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC,CAe/E;AAMD,qDAAqD;AACrD,eAAO,MAAM,mBAAmB,QAAiE,CAAC;AAElG,mDAAmD;AACnD,eAAO,MAAM,eAAe,QAAuB,CAAC;AAEpD,kDAAkD;AAClD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAED,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,CAWnE;AAMD;;;GAGG;AACH,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,OAAO,EACZ,KAAK,YAAY,IAAI,mBAAmB,EACxC,KAAK,cAAc,EAGnB,cAAc,EACd,gBAAgB,EAChB,mBAAmB,IAAI,2BAA2B,EAClD,eAAe,IAAI,uBAAuB,EAG1C,eAAe,EACf,UAAU,EACV,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,eAAe,mBAAmB,CAAC"}

package/dist/index.js

@@ -382,14 +382,28 @@ class MarkdownDocsScraper {
     console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
     return { downloaded, failed, duration };
   }
+  extractBody(content) {
+    const headerRegex = /^<!--\nSource: [^\n]+\nDownloaded: [^\n]+\n-->\n\n/;
+    return content.replace(headerRegex, "");
+  }
   async savePages(pages) {
     const fs = await import("fs/promises");
     const path = await import("path");
+    let updated = 0;
+    let skipped = 0;
     for (const page of pages) {
       const nameToUse = page.pageName || page.url.split("/").pop()?.replace(".md", "") || "untitled";
       const dir = page.category ? path.join(this.options.outputDir, page.category) : this.options.outputDir;
       await fs.mkdir(dir, { recursive: true });
       const filepath = path.join(dir, `${nameToUse}.md`);
+      try {
+        const existingContent = await fs.readFile(filepath, "utf-8");
+        const existingBody = this.extractBody(existingContent);
+        if (existingBody === page.content) {
+          skipped++;
+          continue;
+        }
+      } catch {}
       const header = `<!--
 Source: ${page.url}
 Downloaded: ${new Date().toISOString()}
@@ -397,7 +411,9 @@ Downloaded: ${new Date().toISOString()}
 
 `;
       await fs.writeFile(filepath, header + page.content, "utf-8");
+      updated++;
     }
+    return { updated, skipped };
   }
   getPagesToScrape() {
     const pages = [];
@@ -412,10 +428,14 @@ Downloaded: ${new Date().toISOString()}
 async function scrapeMarkdownDocs(options) {
   const scraper = new MarkdownDocsScraper(options);
   const result = options.useLlms ? await scraper.scrapeFromLlms() : await scraper.scrape();
+  let saveStats;
   if (options.outputDir) {
-    await scraper.savePages(result.downloaded);
+    saveStats = await scraper.savePages(result.downloaded);
+    if (saveStats.updated > 0 || saveStats.skipped > 0) {
+      console.log(`  Saved: ${saveStats.updated} updated, ${saveStats.skipped} unchanged`);
+    }
   }
-  return result;
+  return { ...result, saveStats };
 }
 var CLAUDE_CODE_PATTERN2 = /\[([^\]]+)\]\((https?:\/\/[^\s)]+\/docs\/en\/([^)]+\.md))\)/g;
 var GENERIC_PATTERN2 = GENERIC_LINK_PATTERN;

package/dist/scrapers/index.js

@@ -226,14 +226,28 @@ class MarkdownDocsScraper {
     console.log(`⏱️  Duration: ${(duration / 1000).toFixed(2)}s`);
     return { downloaded, failed, duration };
   }
+  extractBody(content) {
+    const headerRegex = /^<!--\nSource: [^\n]+\nDownloaded: [^\n]+\n-->\n\n/;
+    return content.replace(headerRegex, "");
+  }
   async savePages(pages) {
     const fs = await import("fs/promises");
     const path = await import("path");
+    let updated = 0;
+    let skipped = 0;
     for (const page of pages) {
       const nameToUse = page.pageName || page.url.split("/").pop()?.replace(".md", "") || "untitled";
       const dir = page.category ? path.join(this.options.outputDir, page.category) : this.options.outputDir;
       await fs.mkdir(dir, { recursive: true });
       const filepath = path.join(dir, `${nameToUse}.md`);
+      try {
+        const existingContent = await fs.readFile(filepath, "utf-8");
+        const existingBody = this.extractBody(existingContent);
+        if (existingBody === page.content) {
+          skipped++;
+          continue;
+        }
+      } catch {}
       const header = `<!--
 Source: ${page.url}
 Downloaded: ${new Date().toISOString()}
@@ -241,7 +255,9 @@ Downloaded: ${new Date().toISOString()}
 
 `;
       await fs.writeFile(filepath, header + page.content, "utf-8");
+      updated++;
     }
+    return { updated, skipped };
   }
   getPagesToScrape() {
     const pages = [];
@@ -256,10 +272,14 @@ Downloaded: ${new Date().toISOString()}
 async function scrapeMarkdownDocs(options) {
   const scraper = new MarkdownDocsScraper(options);
   const result = options.useLlms ? await scraper.scrapeFromLlms() : await scraper.scrape();
+  let saveStats;
   if (options.outputDir) {
-    await scraper.savePages(result.downloaded);
+    saveStats = await scraper.savePages(result.downloaded);
+    if (saveStats.updated > 0 || saveStats.skipped > 0) {
+      console.log(`  Saved: ${saveStats.updated} updated, ${saveStats.skipped} unchanged`);
+    }
   }
-  return result;
+  return { ...result, saveStats };
 }
 
 // src/scrapers/llms-txt.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ebowwa/markdown-docs-scraper",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "files": [
     "dist",
     "src"

package/src/index.ts

@@ -367,12 +367,24 @@ export class MarkdownDocsScraper {
   }
 
   /**
-   * Save scraped pages to disk
+   * Extract body content from a file (strips header comment)
    */
-  async savePages(pages: DocPage[]): Promise<void> {
+  private extractBody(content: string): string {
+    // Match header comment and remove it
+    const headerRegex = /^<!--\nSource: [^\n]+\nDownloaded: [^\n]+\n-->\n\n/;
+    return content.replace(headerRegex, "");
+  }
+
+  /**
+   * Save scraped pages to disk (only writes if content changed)
+   */
+  async savePages(pages: DocPage[]): Promise<{ updated: number; skipped: number }> {
     const fs = await import("fs/promises");
     const path = await import("path");
 
+    let updated = 0;
+    let skipped = 0;
+
     for (const page of pages) {
       const nameToUse = page.pageName || page.url.split("/").pop()?.replace(".md", "") || "untitled";
 
@@ -384,9 +396,27 @@ export class MarkdownDocsScraper {
 
       const filepath = path.join(dir, `${nameToUse}.md`);
 
+      // Check if file exists and compare content
+      try {
+        const existingContent = await fs.readFile(filepath, "utf-8");
+        const existingBody = this.extractBody(existingContent);
+
+        // Skip if content unchanged
+        if (existingBody === page.content) {
+          skipped++;
+          continue;
+        }
+      } catch {
+        // File doesn't exist, will create it
+      }
+
+      // Content changed or new file - write it
       const header = `<!--\nSource: ${page.url}\nDownloaded: ${new Date().toISOString()}\n-->\n\n`;
       await fs.writeFile(filepath, header + page.content, "utf-8");
+      updated++;
     }
+
+    return { updated, skipped };
   }
 
   /**
@@ -414,17 +444,21 @@ export class MarkdownDocsScraper {
  */
 export async function scrapeMarkdownDocs(
   options: ScraperOptions & { useLlms?: boolean }
-): Promise<ScraperResult> {
+): Promise<ScraperResult & { saveStats?: { updated: number; skipped: number } }> {
   const scraper = new MarkdownDocsScraper(options);
   const result = options.useLlms
     ? await scraper.scrapeFromLlms()
     : await scraper.scrape();
 
+  let saveStats;
   if (options.outputDir) {
-    await scraper.savePages(result.downloaded);
+    saveStats = await scraper.savePages(result.downloaded);
+    if (saveStats.updated > 0 || saveStats.skipped > 0) {
+      console.log(`  Saved: ${saveStats.updated} updated, ${saveStats.skipped} unchanged`);
+    }
   }
 
-  return result;
+  return { ...result, saveStats };
 }
 
 // ============================================================================