@mintlify/scraping 3.0.189 → 4.0.1

package/bin/utils/hastComments.d.ts

@@ -0,0 +1,3 @@
+import type { Root as HastRoot } from 'hast';
+export declare function removeHastComments(root: HastRoot): void;
+export declare function rehypeRemoveHastComments(): (root: HastRoot) => void;

package/bin/utils/hastComments.js

@@ -0,0 +1,15 @@
+import { visit, CONTINUE } from 'unist-util-visit';
+export function removeHastComments(root) {
+    visit(root, 'comment', function (_, index, parent) {
+        if (parent && typeof index === 'number') {
+            parent.children.splice(index, 1);
+            return [CONTINUE, index];
+        }
+    });
+}
+export function rehypeRemoveHastComments() {
+    return function (root) {
+        return removeHastComments(root);
+    };
+}
+//# sourceMappingURL=hastComments.js.map

package/bin/utils/hastComments.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hastComments.js","sourceRoot":"","sources":["../../src/utils/hastComments.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEnD,MAAM,UAAU,kBAAkB,CAAC,IAAc;IAC/C,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM;QAC/C,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACxC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,wBAAwB;IACtC,OAAO,UAAU,IAAc;QAC7B,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC,CAAC;AACJ,CAAC"}

package/bin/utils/network.js

@@ -29,12 +29,17 @@ export async function startPuppeteer() {
     }
 }
 export async function getHtmlWithPuppeteer(browser, url) {
-    const page = await browser.newPage();
-    await page.goto(url.toString(), {
-        waitUntil: 'networkidle2',
-    });
-    const html = await exponentialBackoff(() => page.content());
-    return html;
+    try {
+        const page = await browser.newPage();
+        await page.goto(url.toString(), {
+            waitUntil: 'networkidle2',
+        });
+        return await exponentialBackoff(() => page.content());
+    }
+    catch (error) {
+        const errorMessage = getErrorMessage(error);
+        throw new Error(`Failed to download page from Puppeteer${errorMessage}`);
+    }
 }
 async function fetchPageResponse(url) {
     try {
@@ -46,23 +51,26 @@ async function fetchPageResponse(url) {
     }
     catch (error) {
         const errorMessage = getErrorMessage(error);
-        throw new Error(`${url}\n\t- failed to fetch page from source${errorMessage}`);
+        throw new Error(`Failed to fetch page from source${errorMessage}`);
     }
 }
 export async function fetchPageHtml(url, browser = undefined) {
-    if (browser) {
-        try {
-            const res = await getHtmlWithPuppeteer(browser, url);
-            if (res)
-                return res;
-            throw new Error('an unknown error occured');
+    try {
+        let res = undefined;
+        if (browser) {
+            res = await getHtmlWithPuppeteer(browser, url);
         }
-        catch (error) {
-            const errorMessage = getErrorMessage(error);
-            throw new Error(`${url}\n\t- Puppeteer failed to retrieve page from source${errorMessage}`);
+        else {
+            res = await exponentialBackoff(() => fetchPageResponse(url));
         }
+        if (res)
+            return res;
+        throw new Error('An unknown error occured.');
+    }
+    catch (error) {
+        const errorMessage = getErrorMessage(error);
+        throw new Error(`Error retrieving HTML for ${url.toString()}${errorMessage}`);
     }
-    return await exponentialBackoff(() => fetchPageResponse(url));
 }
 export async function fetchImage(url) {
     try {
@@ -76,7 +84,7 @@ export async function fetchImage(url) {
     }
     catch (error) {
         const errorMessage = getErrorMessage(error);
-        throw new Error(`${url}\n\t- failed to retrieve image from source${errorMessage}`);
+        throw new Error(`${url} - failed to retrieve image from source${errorMessage}`);
     }
 }
 //# sourceMappingURL=network.js.map

package/bin/utils/network.js.map

@@ -1 +1 @@
-{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,KAAK,UAAU,kBAAkB,CAC/B,SAA2B,EAC3B,UAAkB,CAAC,EACnB,QAAgB,IAAI,EACpB,SAAiB,CAAC;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;QAC9B,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,yCAAyC,YAAY,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACrD,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,sDAAsD,YAAY,EAAE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,OAAO,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,6CAA6C,YAAY,EAAE,CAAC,CAAC;IACrF,CAAC;AACH,CAAC"}
+{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,KAAK,UAAU,kBAAkB,CAC/B,SAA2B,EAC3B,UAAkB,CAAC,EACnB,QAAgB,IAAI,EACpB,SAAiB,CAAC;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC9B,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QACH,OAAO,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "3.0.189",
+  "version": "4.0.1",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -81,5 +81,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "e69843f13b6c803075baa3325cc70f0ab62062c8"
+  "gitHead": "4ef3ed8269c758723ad39cb522b831ee6db81067"
 }

package/src/cli.ts

@@ -3,10 +3,12 @@ import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
 import { FINAL_SUCCESS_MESSAGE } from './constants.js';
+import { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
 import { scrapePageGroup } from './scrapingPipeline/group.js';
 import { htmlToHast } from './scrapingPipeline/root.js';
 import { scrapeAllSiteTabs } from './scrapingPipeline/tabs.js';
 import { detectFramework, framework } from './utils/detectFramework.js';
+import { getErrorMessage } from './utils/errors.js';
 import { write } from './utils/file.js';
 import { log } from './utils/log.js';
 import { fetchPageHtml } from './utils/network.js';
@@ -27,6 +29,46 @@ await yargs(hideBin(process.argv))
     async ({ url }) => await site(url)
   )
 
+  .command(
+    'openapi-file <openapiFilename>',
+    'Creates MDX files from an OpenAPI spec',
+    (yargs) =>
+      yargs
+        .positional('openapiFilename', {
+          describe: 'The filename of the OpenAPI spec',
+          type: 'string',
+          demandOption: true,
+        })
+        .option('writeFiles', {
+          describe: 'Whether or not to write the frontmatter files',
+          default: true,
+          type: 'boolean',
+          alias: 'w',
+        })
+        .option('outDir', {
+          describe: 'The folder in which to write any created frontmatter files',
+          type: 'string',
+          alias: 'o',
+        }),
+    async (argv) => {
+      try {
+        const { nav } = await generateOpenApiPages(
+          argv.openapiFilename,
+          argv.writeFiles,
+          argv.outDir
+        );
+        console.log('navigation object suggestion:');
+        console.log(JSON.stringify(nav, undefined, 2));
+      } catch (error) {
+        if (error instanceof Error) {
+          console.error(error.message);
+        } else {
+          console.error(error);
+        }
+      }
+    }
+  )
+
   .strictCommands()
   .demandCommand(1, 'Unknown command. See above for the list of supported commands.')
   .alias('h', 'help')
@@ -34,37 +76,47 @@ await yargs(hideBin(process.argv))
   .parse();
 
 async function page(url: string) {
-  const urlObj = new URL(url);
-  const html = await fetchPageHtml(urlObj);
-  log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
+  try {
+    const urlObj = new URL(url);
+    const html = await fetchPageHtml(urlObj);
+    log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
 
-  const hast = htmlToHast(html);
-  detectFramework(hast);
+    const hast = htmlToHast(html);
+    detectFramework(hast);
 
-  const needsBrowser = framework.vendor === 'gitbook';
-  const results = await scrapePageGroup([urlObj], needsBrowser);
-  const result = results[0] || {
-    success: false,
-    message: `An unknown error occurred when scraping ${url}`,
-  };
+    const needsBrowser = framework.vendor === 'gitbook';
+    const results = await scrapePageGroup([urlObj], needsBrowser);
+    const result = results[0] || {
+      success: false,
+      message: `An unknown error occurred when scraping ${url}`,
+    };
 
-  if (result.success) {
-    log(`Successfully scraped ${url} ${result.data ? `into ${result.data[1]}` : ''}`);
-  } else {
-    log(result.message);
+    if (result.success) {
+      log(`Successfully scraped ${url} ${result.data ? `into ${result.data[1]}` : ''}`);
+    } else {
+      log(result.message);
+    }
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    log(errorMessage);
   }
 }
 
 async function site(url: string) {
-  const urlObj = new URL(url);
-  const html = await fetchPageHtml(urlObj);
-  log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
+  try {
+    const urlObj = new URL(url);
+    const html = await fetchPageHtml(urlObj);
+    log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
 
-  const result = await scrapeAllSiteTabs(html, urlObj);
-  if (result.success) {
-    write('mint.json', JSON.stringify(result.data, undefined, 2));
-    log(FINAL_SUCCESS_MESSAGE);
-  } else {
-    log(result.message);
+    const result = await scrapeAllSiteTabs(html, urlObj);
+    if (result.success) {
+      write('mint.json', JSON.stringify(result.data, undefined, 2));
+      log(FINAL_SUCCESS_MESSAGE);
+    } else {
+      log(result.message);
+    }
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    log(errorMessage);
   }
 }

package/src/scrapingPipeline/logo.ts

@@ -4,7 +4,9 @@ import { Browser } from 'puppeteer';
 import { EXIT, visit } from 'unist-util-visit';
 
 import { framework } from '../utils/detectFramework.js';
+import { getErrorMessage } from '../utils/errors.js';
 import { downloadImage } from '../utils/images.js';
+import { log } from '../utils/log.js';
 import { fetchPageHtml } from '../utils/network.js';
 import { htmlToHast } from './root.js';
 
@@ -104,12 +106,17 @@ export async function downloadLogos(
       })
     );
   } else {
-    const html = await fetchPageHtml(url);
-    await findLogosFromHtml(
-      html,
-      framework.vendor === 'gitbook' ? findGitBookLogoNodes : findDocusaurusLogoNodes,
-      filepaths
-    );
+    try {
+      const html = await fetchPageHtml(url);
+      await findLogosFromHtml(
+        html,
+        framework.vendor === 'gitbook' ? findGitBookLogoNodes : findDocusaurusLogoNodes,
+        filepaths
+      );
+    } catch (error) {
+      const errorMessage = getErrorMessage(error);
+      log(`Failed to retrieve logo from HTML: ${errorMessage}`);
+    }
   }
 
   if (browser) await browser.close();

package/src/scrapingPipeline/page.ts

@@ -27,6 +27,7 @@ import { unifiedRemoveEmptyParagraphs } from '../utils/emptyParagraphs.js';
 import { getErrorMessage, logErrorResults } from '../utils/errors.js';
 import { escapeCharactersOutsideCodeBlocks } from '../utils/escape.js';
 import { write, writePage } from '../utils/file.js';
+import { removeHastComments } from '../utils/hastComments.js';
 import { log } from '../utils/log.js';
 import { unifiedRemoveNestedRoots } from '../utils/nestedRoots.js';
 import { unifiedRemovePositions } from '../utils/position.js';
@@ -54,6 +55,7 @@ export async function scrapePage(
   }
 
   const hast = htmlToHast(html);
+  removeHastComments(hast);
 
   if (!framework.vendor) detectFramework(hast);
 

package/src/scrapingPipeline/root.ts

@@ -2,9 +2,16 @@ import type { Root as HastRoot } from 'hast';
 import rehypeParse from 'rehype-parse';
 import { unified } from 'unified';
 
+import { rehypeRemoveHastComments } from '../utils/hastComments.js';
 import { unifiedRemovePositions } from '../utils/position.js';
 
 export function htmlToHast(html: string): HastRoot {
-  // @ts-expect-error remarkStringify errors even if used for valid code from documentation examples
-  return unified().use(rehypeParse).use(unifiedRemovePositions).parse(html) as HastRoot;
+  return (
+    unified()
+      // @ts-expect-error remarkStringify errors even if used for valid code from documentation examples
+      .use(rehypeParse)
+      .use(unifiedRemovePositions)
+      .use(rehypeRemoveHastComments)
+      .parse(html) as HastRoot
+  );
 }

package/src/scrapingPipeline/tabs.ts

@@ -4,6 +4,7 @@ import { MintConfig, Tab } from '@mintlify/models';
 import { retrieveTabLinks } from '../tabs/retrieveReadme.js';
 import type { Result } from '../types/result.js';
 import { detectFramework, framework } from '../utils/detectFramework.js';
+import { getErrorMessage } from '../utils/errors.js';
 import { log } from '../utils/log.js';
 import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
 import { getTitleFromLink } from '../utils/title.js';
@@ -47,8 +48,12 @@ export async function scrapeAllSiteTabs(
       links.map(async (tabEntry) => {
         const newUrl = new URL(url);
         newUrl.pathname = tabEntry.url;
-        const newHtml = await fetchPageHtml(newUrl, undefined);
-        return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry] });
+        try {
+          const newHtml = await fetchPageHtml(newUrl, undefined);
+          return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry] });
+        } catch (error) {
+          return { success: false as const, message: getErrorMessage(error) };
+        }
       })
     );
 
@@ -64,7 +69,7 @@ export async function scrapeAllSiteTabs(
 
     const failures = results.filter((result) => !result.success);
     failures.forEach((result) => {
-      log('Failed to scrape tab: ' + result.message);
+      log('Failed to scrape tab' + result.message);
     });
 
     return {

package/src/utils/hastComments.ts

@@ -0,0 +1,17 @@
+import type { Root as HastRoot } from 'hast';
+import { visit, CONTINUE } from 'unist-util-visit';
+
+export function removeHastComments(root: HastRoot): void {
+  visit(root, 'comment', function (_, index, parent) {
+    if (parent && typeof index === 'number') {
+      parent.children.splice(index, 1);
+      return [CONTINUE, index];
+    }
+  });
+}
+
+export function rehypeRemoveHastComments() {
+  return function (root: HastRoot) {
+    return removeHastComments(root);
+  };
+}

package/src/utils/network.ts

@@ -38,12 +38,16 @@ export async function getHtmlWithPuppeteer(
   browser: Browser,
   url: string | URL
 ): Promise<string | undefined> {
-  const page = await browser.newPage();
-  await page.goto(url.toString(), {
-    waitUntil: 'networkidle2',
-  });
-  const html = await exponentialBackoff(() => page.content());
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url.toString(), {
+      waitUntil: 'networkidle2',
+    });
+    return await exponentialBackoff(() => page.content());
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    throw new Error(`Failed to download page from Puppeteer${errorMessage}`);
+  }
 }
 
 async function fetchPageResponse(url: string | URL): Promise<string> {
@@ -55,7 +59,7 @@ async function fetchPageResponse(url: string | URL): Promise<string> {
     return await res.text();
   } catch (error) {
     const errorMessage = getErrorMessage(error);
-    throw new Error(`${url}\n\t- failed to fetch page from source${errorMessage}`);
+    throw new Error(`Failed to fetch page from source${errorMessage}`);
   }
 }
 
@@ -63,18 +67,19 @@ export async function fetchPageHtml(
   url: string | URL,
   browser: Browser | undefined = undefined
 ): Promise<string> {
-  if (browser) {
-    try {
-      const res = await getHtmlWithPuppeteer(browser, url);
-      if (res) return res;
-      throw new Error('an unknown error occured');
-    } catch (error) {
-      const errorMessage = getErrorMessage(error);
-      throw new Error(`${url}\n\t- Puppeteer failed to retrieve page from source${errorMessage}`);
+  try {
+    let res: string | undefined = undefined;
+    if (browser) {
+      res = await getHtmlWithPuppeteer(browser, url);
+    } else {
+      res = await exponentialBackoff(() => fetchPageResponse(url));
     }
+    if (res) return res;
+    throw new Error('An unknown error occured.');
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    throw new Error(`Error retrieving HTML for ${url.toString()}${errorMessage}`);
   }
-
-  return await exponentialBackoff(() => fetchPageResponse(url));
 }
 
 export async function fetchImage(url: string): Promise<NodeJS.TypedArray> {
@@ -90,6 +95,6 @@ export async function fetchImage(url: string): Promise<NodeJS.TypedArray> {
     return imageData;
   } catch (error) {
     const errorMessage = getErrorMessage(error);
-    throw new Error(`${url}\n\t- failed to retrieve image from source${errorMessage}`);
+    throw new Error(`${url} - failed to retrieve image from source${errorMessage}`);
   }
 }