@mintlify/scraping 4.0.28 → 4.0.30

package/bin/utils/images.js

@@ -7,6 +7,9 @@ import { write } from './file.js';
 import { log } from './log.js';
 import { fetchImage } from './network.js';
 export async function downloadImage(src, rootPath) {
+    if (src.startsWith('data:image/')) {
+        return { success: true, data: [src, src] };
+    }
     try {
         let filename = await writeImageToFile(src, rootPath);
         filename = filename.replace(process.cwd(), '');

package/bin/utils/images.js.map

@@ -1 +1 @@
-{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/utils/images.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,QAAgB;IAEhB,IAAI,CAAC;QACH,IAAI,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QAE/C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QACxF,GAAG,CAAC,GAAG,iBAAiB,iCAAiC,EAAE,SAAS,CAAC,CAAC;QAEtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,GAAG,GAAG,qDAAqD;aACrE,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,QAAgB;IAC3D,MAAM,QAAQ,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExF,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,iCAAiC,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,CAAC;QACH,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,+BAA+B,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACxC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC/F,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAW,EAAE,GAAW;IAChE,MAAM,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC;IACtE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,0BAA0B,EAAE,CAAC;YAC7C,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC;gBAC5B,QAAQ,GAAG,yBAAyB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC9B,CAAC;QACD,QAAQ;YACN,kBAAkB,CAChB,GAAG;iBACA,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAC7B,CAAC,OAAO,CAAC,sCAAsC,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC;QACpE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACpD,CAAC"}
+{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/utils/images.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,QAAgB;IAEhB,IAAI,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,IAAI,CAAC;QACH,IAAI,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QAE/C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QACxF,GAAG,CAAC,GAAG,iBAAiB,iCAAiC,EAAE,SAAS,CAAC,CAAC;QAEtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,GAAG,GAAG,qDAAqD;aACrE,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,QAAgB;IAC3D,MAAM,QAAQ,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExF,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,iCAAiC,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,CAAC;QACH,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,+BAA+B,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACxC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC/F,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAW,EAAE,GAAW;IAChE,MAAM,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC;IACtE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,0BAA0B,EAAE,CAAC;YAC7C,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC;gBAC5B,QAAQ,GAAG,yBAAyB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC9B,CAAC;QACD,QAAQ;YACN,kBAAkB,CAChB,GAAG;iBACA,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAC7B,CAAC,OAAO,CAAC,sCAAsC,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC;QACpE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACpD,CAAC"}

package/bin/utils/network.js

@@ -1,6 +1,21 @@
 import { launch } from 'puppeteer';
+import { framework } from './detectFramework.js';
 import { getErrorMessage } from './errors.js';
 import { log } from './log.js';
+const userAgents = [
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+];
+const headers = {
+    'Accept-Language': 'en-US,en;q=0.9',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+    'Accept-Encoding': 'gzip, deflate, br, zstd',
+    Connection: 'keep-alive',
+};
 async function exponentialBackoff(operation, retries = 3, delay = 1000, factor = 2) {
     try {
         return await operation();
@@ -31,26 +46,44 @@ export async function startPuppeteer() {
 export async function getHtmlWithPuppeteer(browser, url) {
     try {
         const page = await browser.newPage();
-        await page.setExtraHTTPHeaders({
-            'Accept-Language': 'en-US,en;q=0.9',
-            Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
-            'Accept-Encoding': 'gzip, deflate, br, zstd',
-            Connection: 'keep-alive',
+        await page.setViewport({
+            width: 3072,
+            height: 2048,
+            deviceScaleFactor: 2,
+            isMobile: false,
+            hasTouch: false,
+            isLandscape: true,
         });
-        const userAgents = [
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
-        ];
+        await page.setExtraHTTPHeaders(headers);
         await page.setUserAgent(userAgents[Math.floor(Math.random() * userAgents.length)] || userAgents[0]);
         await page.setJavaScriptEnabled(true);
         await page.goto(url.toString(), {
             waitUntil: 'networkidle2',
             timeout: 30000,
         });
+        if (framework.vendor === 'docusaurus') {
+            await page.evaluate(() => {
+                const clickMenuItems = (parentElement = document) => {
+                    const menuItems = parentElement.getElementsByClassName('menu__link--sublist');
+                    for (const item of menuItems) {
+                        const clickEvent = new MouseEvent('click', {
+                            bubbles: true,
+                            cancelable: true,
+                            view: window,
+                        });
+                        item.dispatchEvent(clickEvent);
+                        const parentLi = item.parentElement;
+                        if (parentLi) {
+                            const nestedUl = parentLi.querySelector('ul');
+                            if (nestedUl) {
+                                clickMenuItems(nestedUl);
+                            }
+                        }
+                    }
+                };
+                clickMenuItems();
+            });
+        }
         const content = await exponentialBackoff(() => page.content());
         await page.close();
         return content;

package/bin/utils/network.js.map

@@ -1 +1 @@
-{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,KAAK,UAAU,kBAAkB,CAC/B,SAA2B,EAC3B,UAAkB,CAAC,EACnB,QAAgB,IAAI,EACpB,SAAiB,CAAC;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,mBAAmB,CAAC;YAC7B,iBAAiB,EAAE,gBAAgB;YACnC,MAAM,EACJ,yIAAyI;YAC3I,YAAY,EACV,uHAAuH;YACzH,iBAAiB,EAAE,yBAAyB;YAC5C,UAAU,EAAE,YAAY;SACzB,CAAC,CAAC;QACH,MAAM,UAAU,GAAG;YACjB,uHAAuH;YACvH,uHAAuH;YACvH,uHAAuH;YACvH,uHAAuH;YACvH,uHAAuH;SAC/G,CAAC;QAEX,MAAM,IAAI,CAAC,YAAY,CACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC9B,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC"}
+{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;CAC/G,CAAC;AAEX,MAAM,OAAO,GAAG;IACd,iBAAiB,EAAE,gBAAgB;IACnC,MAAM,EACJ,yIAAyI;IAC3I,YAAY,EACV,uHAAuH;IACzH,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,KAAK,UAAU,kBAAkB,CAC/B,SAA2B,EAC3B,UAAkB,CAAC,EACnB,QAAgB,IAAI,EACpB,SAAiB,CAAC;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,WAAW,CAAC;YACrB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,IAAI;YACZ,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,KAAK;YACf,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,YAAY,CACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC9B,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACvB,MAAM,cAAc,GAAG,CAAC,gBAAoC,QAAQ,EAAE,EAAE;oBACtE,MAAM,SAAS,GAAG,aAAa,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,CAAC;oBAE9E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE;4BACzC,OAAO,EAAE,IAAI;4BACb,UAAU,EAAE,IAAI;4BAChB,IAAI,EAAE,MAAM;yBACb,CAAC,CAAC;wBACH,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;wBAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC;wBACpC,IAAI,QAAQ,EAAE,CAAC;4BACb,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;4BAC9C,IAAI,QAAQ,EAAE,CAAC;gCACb,cAAc,CAAC,QAAQ,CAAC,CAAC;4BAC3B,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC;gBAEF,cAAc,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.28",
+  "version": "4.0.30",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -38,7 +38,7 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.187",
+    "@mintlify/common": "1.0.188",
     "@mintlify/openapi-parser": "^0.0.7",
     "fs-extra": "^11.1.1",
     "hast": "^1.0.0",
@@ -60,10 +60,10 @@
   },
   "devDependencies": {
     "@mintlify/eslint-config-typescript": "1.0.13",
-    "@mintlify/models": "0.0.146",
+    "@mintlify/models": "0.0.147",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.218",
+    "@mintlify/validation": "0.1.219",
     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
     "@tsconfig/recommended": "1.x",
     "@types/node": "^18.7.13",
@@ -78,5 +78,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "e8725b5e25908dc7d301bb50df5848017aac6bbd"
+  "gitHead": "03182511296c892727bca1ccb672bbd8deebe4c5"
 }

package/src/cli.ts

@@ -103,9 +103,11 @@ async function page(url: string) {
     } else {
       log(result.message);
     }
+    process.exit(0);
   } catch (error) {
     const errorMessage = getErrorMessage(error);
     log(errorMessage);
+    process.exit(1);
   }
 }
 
@@ -122,8 +124,10 @@ async function site(url: string) {
     } else {
       log(result.message);
     }
+    process.exit(0);
   } catch (error) {
     const errorMessage = getErrorMessage(error);
     log(errorMessage);
+    process.exit(1);
   }
 }

package/src/nav/listItems.ts

@@ -50,18 +50,19 @@ export function processListItem(
 
   const sectionHeader = findFirstChild(node, opts.sectionTagName);
   const childList = findFirstChild(node, opts.childListTagName);
+  if (!childList) {
+    return linkHref;
+  }
+
   let title = opts.title;
   if (!title) {
-    title = getText(link) || getText(sectionHeader) || '';
     if (framework.vendor === 'readme') {
       title = getText(sectionHeader) || getText(link) || '';
+    } else {
+      title = getText(link) || getText(sectionHeader) || '';
     }
   }
 
-  if (!childList) {
-    return linkHref;
-  }
-
   let childEntries = retrieveNavItems(childList);
   const newLink = childEntries.find(
     (child) => typeof child === 'string' && child.startsWith(linkHref)

package/src/nav/retrieve.ts

@@ -49,8 +49,9 @@ export function retrieveNavItems(rootNode: Element): Array<NavigationEntry> {
       node.children[0].tagName === 'div' &&
       node.children[0].children.filter((child) => child.type === 'text').length ===
         node.children[0].children.length
-    )
+    ) {
       title = findTitle(node.children[0], { delete: false });
+    }
 
     if (
       framework.vendor === 'readme' &&

package/src/scrapingPipeline/site.ts

@@ -9,7 +9,7 @@ import { retrieveRootNavElement } from '../nav/root.js';
 import type { Result } from '../types/result.js';
 import { detectFramework, framework } from '../utils/detectFramework.js';
 import { logErrorResults } from '../utils/errors.js';
-import { startPuppeteer } from '../utils/network.js';
+import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
 import { INDEX_NAMES, iterateThroughReservedNames } from '../utils/reservedNames.js';
 import { removeTrailingSlash, removeLeadingSlash } from '../utils/strings.js';
 import { downloadColors } from './color.js';
@@ -32,6 +32,13 @@ export async function scrapeSite(
 
   detectFramework(hast);
 
+  if (framework.vendor === 'docusaurus') {
+    const browser = await startPuppeteer();
+    html = await fetchPageHtml(url, browser);
+    hast = htmlToHast(html);
+    if (browser) await browser.close();
+  }
+
   const sidebar = retrieveRootNavElement(hast);
   if (!sidebar) return { success: false, message: `${url.toString()}: ${NAV_FAILURE_MSG}` };
 
@@ -118,8 +125,9 @@ export async function scrapeSite(
 
     navItems.forEach((navItem, index) => {
       if (typeof navItem !== 'string') return;
-      const name = navItem
-        .split('-')
+      const lastItemInPath = navItem.split('/').pop() || navItem;
+      const name = lastItemInPath
+        .split(/[-_]/)
         .map((str) => (str[0] ? `${str[0].toUpperCase()}${str.substring(1)}` : str))
         .join(' ');
 

package/src/utils/images.ts

@@ -13,6 +13,9 @@ export async function downloadImage(
   src: string,
   rootPath: string
 ): Promise<Result<[string, string]>> {
+  if (src.startsWith('data:image/')) {
+    return { success: true, data: [src, src] };
+  }
   try {
     let filename = await writeImageToFile(src, rootPath);
     filename = filename.replace(process.cwd(), '');

package/src/utils/network.ts

@@ -1,8 +1,27 @@
 import { Browser, launch } from 'puppeteer';
 
+import { framework } from './detectFramework.js';
 import { getErrorMessage } from './errors.js';
 import { log } from './log.js';
 
+const userAgents = [
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+] as const;
+
+const headers = {
+  'Accept-Language': 'en-US,en;q=0.9',
+  Accept:
+    'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+  'User-Agent':
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+  'Accept-Encoding': 'gzip, deflate, br, zstd',
+  Connection: 'keep-alive',
+} as const;
+
 async function exponentialBackoff<T>(
   operation: () => Promise<T>,
   retries: number = 3,
@@ -41,23 +60,15 @@ export async function getHtmlWithPuppeteer(
   try {
     const page = await browser.newPage();
 
-    await page.setExtraHTTPHeaders({
-      'Accept-Language': 'en-US,en;q=0.9',
-      Accept:
-        'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
-      'User-Agent':
-        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
-      'Accept-Encoding': 'gzip, deflate, br, zstd',
-      Connection: 'keep-alive',
+    await page.setViewport({
+      width: 3072,
+      height: 2048,
+      deviceScaleFactor: 2,
+      isMobile: false,
+      hasTouch: false,
+      isLandscape: true,
     });
-    const userAgents = [
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
-    ] as const;
-
+    await page.setExtraHTTPHeaders(headers);
     await page.setUserAgent(
       userAgents[Math.floor(Math.random() * userAgents.length)] || userAgents[0]
     );
@@ -68,6 +79,33 @@ export async function getHtmlWithPuppeteer(
       timeout: 30000,
     });
 
+    if (framework.vendor === 'docusaurus') {
+      await page.evaluate(() => {
+        const clickMenuItems = (parentElement: Element | Document = document) => {
+          const menuItems = parentElement.getElementsByClassName('menu__link--sublist');
+
+          for (const item of menuItems) {
+            const clickEvent = new MouseEvent('click', {
+              bubbles: true,
+              cancelable: true,
+              view: window,
+            });
+            item.dispatchEvent(clickEvent);
+
+            const parentLi = item.parentElement;
+            if (parentLi) {
+              const nestedUl = parentLi.querySelector('ul');
+              if (nestedUl) {
+                clickMenuItems(nestedUl);
+              }
+            }
+          }
+        };
+
+        clickMenuItems();
+      });
+    }
+
     const content = await exponentialBackoff(() => page.content());
     await page.close();
     return content;