@mintlify/scraping 4.0.35 → 4.0.37

package/bin/utils/images.js

@@ -4,7 +4,6 @@ import { SUPPORTED_MEDIA_EXTENSIONS } from '../constants.js';
 import { getErrorMessage } from './errors.js';
 import { getFileExtension } from './extension.js';
 import { write } from './file.js';
-import { log } from './log.js';
 import { fetchImage } from './network.js';
 export async function downloadImage(src, rootPath) {
     if (src.startsWith('data:image/')) {
@@ -13,8 +12,6 @@ export async function downloadImage(src, rootPath) {
     try {
         let filename = await writeImageToFile(src, rootPath);
         filename = filename.replace(process.cwd(), '');
-        const shortenedFilename = filename.length > 36 ? `...${filename.slice(-36)}` : filename;
-        log(`${shortenedFilename} downloaded and written to disk`, 'success');
         return { success: true, data: [src, filename] };
     }
     catch (error) {
@@ -59,9 +56,6 @@ export function isValidImageSrc(src) {
     if (!src) {
         return false;
     }
-    if (src.startsWith('data:')) {
-        return false;
-    }
     const ext = getFileExtension(src);
     if (ext && !SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
         return false;

package/bin/utils/images.js.map

@@ -1 +1 @@
-{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/utils/images.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,QAAgB;IAEhB,IAAI,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,IAAI,CAAC;QACH,IAAI,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QAE/C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QACxF,GAAG,CAAC,GAAG,iBAAiB,iCAAiC,EAAE,SAAS,CAAC,CAAC;QAEtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,GAAG,GAAG,qDAAqD;aACrE,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,QAAgB;IAC3D,MAAM,QAAQ,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExF,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,iCAAiC,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,CAAC;QACH,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,+BAA+B,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACxC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC/F,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAW,EAAE,GAAW;IAChE,MAAM,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC;IACtE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,0BAA0B,EAAE,CAAC;YAC7C,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC;gBAC5B,QAAQ,GAAG,yBAAyB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC9B,CAAC;QACD,QAAQ;YACN,kBAAkB,CAChB,GAAG;iBACA,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAC7B,CAAC,OAAO,CAAC,sCAAsC,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC;QACpE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACpD,CAAC"}
+{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/utils/images.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,QAAgB;IAEhB,IAAI,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,IAAI,CAAC;QACH,IAAI,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,GAAG,GAAG,qDAAqD;aACrE,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,QAAgB;IAC3D,MAAM,QAAQ,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExF,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,iCAAiC,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,CAAC;QACH,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,+BAA+B,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACxC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC/F,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAW,EAAE,GAAW;IAChE,MAAM,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC;IACtE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,0BAA0B,EAAE,CAAC;YAC7C,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC;gBAC5B,QAAQ,GAAG,yBAAyB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC9B,CAAC;QACD,QAAQ;YACN,kBAAkB,CAChB,GAAG;iBACA,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE;iBACd,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAC7B,CAAC,OAAO,CAAC,sCAAsC,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC;QACpE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACpD,CAAC"}

package/bin/utils/intoChunks.d.ts

@@ -0,0 +1 @@
+export declare function intoChunks<T>(values: T[], chunkSize?: number): Generator<T[], void, unknown>;

package/bin/utils/intoChunks.js

@@ -0,0 +1,7 @@
+export function* intoChunks(values, chunkSize = 16) {
+    for (let i = 0; i < values.length; i += chunkSize) {
+        const chunk = values.slice(i, i + chunkSize);
+        yield chunk;
+    }
+}
+//# sourceMappingURL=intoChunks.js.map

package/bin/utils/intoChunks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"intoChunks.js","sourceRoot":"","sources":["../../src/utils/intoChunks.ts"],"names":[],"mappings":"AAAA,MAAM,SAAS,CAAC,CAAC,UAAU,CAAI,MAAW,EAAE,SAAS,GAAG,EAAE;IACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC7C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}

package/bin/utils/network.js

@@ -57,31 +57,24 @@ export async function getHtmlWithPuppeteer(browser, url) {
         await page.setExtraHTTPHeaders(headers);
         await page.setUserAgent(userAgents[Math.floor(Math.random() * userAgents.length)] || userAgents[0]);
         await page.setJavaScriptEnabled(true);
-        await page.goto(url.toString(), {
+        await exponentialBackoff(() => page.goto(url.toString(), {
             waitUntil: 'networkidle2',
             timeout: 30000,
-        });
+        }));
         if (framework.vendor === 'docusaurus') {
             await page.evaluate(() => {
-                const clickMenuItems = (parentElement = document) => {
-                    const menuItems = parentElement.getElementsByClassName('menu__link--sublist');
+                document.addEventListener('click', (e) => {
+                    if (e.target instanceof Element && e.target.classList.contains('menu__link--sublist'))
+                        e.preventDefault();
+                }, true);
+                function clickItems(el) {
+                    const menuItems = el.getElementsByClassName('menu__link--sublist');
                     for (const item of menuItems) {
-                        const clickEvent = new MouseEvent('click', {
-                            bubbles: true,
-                            cancelable: true,
-                            view: window,
-                        });
-                        item.dispatchEvent(clickEvent);
-                        const parentLi = item.parentElement;
-                        if (parentLi) {
-                            const nestedUl = parentLi.querySelector('ul');
-                            if (nestedUl) {
-                                clickMenuItems(nestedUl);
-                            }
-                        }
+                        item.click();
+                        clickItems(item);
                     }
-                };
-                clickMenuItems();
+                }
+                clickItems(document);
             });
         }
         const content = await exponentialBackoff(() => page.content());

package/bin/utils/network.js.map

@@ -1 +1 @@
-{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;CAC/G,CAAC;AAEX,MAAM,OAAO,GAAG;IACd,iBAAiB,EAAE,gBAAgB;IACnC,MAAM,EACJ,yIAAyI;IAC3I,YAAY,EACV,uHAAuH;IACzH,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,KAAK,UAAU,kBAAkB,CAC/B,SAA2B,EAC3B,UAAkB,CAAC,EACnB,QAAgB,IAAI,EACpB,SAAiB,CAAC;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,WAAW,CAAC;YACrB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,IAAI;YACZ,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,KAAK;YACf,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,YAAY,CACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC9B,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACvB,MAAM,cAAc,GAAG,CAAC,gBAAoC,QAAQ,EAAE,EAAE;oBACtE,MAAM,SAAS,GAAG,aAAa,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,CAAC;oBAE9E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE;4BACzC,OAAO,EAAE,IAAI;4BACb,UAAU,EAAE,IAAI;4BAChB,IAAI,EAAE,MAAM;yBACb,CAAC,CAAC;wBACH,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;wBAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC;wBACpC,IAAI,QAAQ,EAAE,CAAC;4BACb,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;4BAC9C,IAAI,QAAQ,EAAE,CAAC;gCACb,cAAc,CAAC,QAAQ,CAAC,CAAC;4BAC3B,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC;gBAEF,cAAc,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC"}
+{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;CAC/G,CAAC;AAEX,MAAM,OAAO,GAAG;IACd,iBAAiB,EAAE,gBAAgB;IACnC,MAAM,EACJ,yIAAyI;IAC3I,YAAY,EACV,uHAAuH;IACzH,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,KAAK,UAAU,kBAAkB,CAC/B,SAA2B,EAC3B,UAAkB,CAAC,EACnB,QAAgB,IAAI,EACpB,SAAiB,CAAC;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,WAAW,CAAC;YACrB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,IAAI;YACZ,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,KAAK;YACf,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,YAAY,CACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxB,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CACH,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACvB,QAAQ,CAAC,gBAAgB,CACvB,OAAO,EACP,CAAC,CAAC,EAAE,EAAE;oBACJ,IAAI,CAAC,CAAC,MAAM,YAAY,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC;wBACnF,CAAC,CAAC,cAAc,EAAE,CAAC;gBACvB,CAAC,EACD,IAAI,CACL,CAAC;gBAEF,SAAS,UAAU,CAAC,EAA0B;oBAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,sBAAsB,CACzC,qBAAqB,CACW,CAAC;oBACnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;wBACb,UAAU,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.35",
+  "version": "4.0.37",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -38,7 +38,7 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.193",
+    "@mintlify/common": "1.0.194",
     "@mintlify/openapi-parser": "^0.0.7",
     "fs-extra": "^11.1.1",
     "hast": "^1.0.0",
@@ -78,5 +78,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "031c9f10a6db0931c74afe4da607e46fa81c3b59"
+  "gitHead": "2a2e6afdd346d5a13074540dfe326db6a23b8639"
 }

package/src/cli.ts

@@ -4,9 +4,9 @@ import { hideBin } from 'yargs/helpers';
 
 import { FINAL_SUCCESS_MESSAGE } from './constants.js';
 import { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
-import { scrapePageGroup } from './scrapingPipeline/group.js';
-import { htmlToHast } from './scrapingPipeline/root.js';
-import { scrapeAllSiteTabs } from './scrapingPipeline/tabs.js';
+import { scrapePageGroup } from './pipeline/group.js';
+import { htmlToHast } from './pipeline/root.js';
+import { scrapeAllSiteTabs } from './pipeline/tabs.js';
 import { detectFramework, framework } from './utils/detectFramework.js';
 import { getErrorMessage } from './utils/errors.js';
 import { write } from './utils/file.js';

package/src/constants.ts

@@ -1,5 +1,7 @@
 import { activeColors } from './utils/log.js';
 
+export const OVERVIEW_PAGE_SLUG = '/mintie_overview';
+
 export const SUPPORTED_MEDIA_EXTENSIONS = [
   'png',
   'jpeg',
@@ -8,13 +10,10 @@ export const SUPPORTED_MEDIA_EXTENSIONS = [
   'avif',
   'svg',
   'ico',
-  'gif',
-  'mp4',
   'jfif',
   'pjpeg',
   'pjp',
   'svgz',
-  'apng',
   'bmp',
 ];
 

package/src/nav/listItems.ts

@@ -2,6 +2,7 @@ import { NavigationEntry } from '@mintlify/models';
 import type { Element } from 'hast';
 import { EXIT, visit } from 'unist-util-visit';
 
+import { OVERVIEW_PAGE_SLUG } from '../constants.js';
 import { dedupedAppend } from '../utils/append.js';
 import { framework } from '../utils/detectFramework.js';
 import { findFirstChild } from '../utils/firstChild.js';
@@ -29,7 +30,7 @@ export function processListItem(
   let linkHref: string | undefined = undefined;
   linkHref = link.properties.href as string | undefined;
 
-  if (linkHref === undefined || linkHref === '#') {
+  if (linkHref === undefined || (framework.vendor !== 'docusaurus' && linkHref === '#')) {
     return undefined;
   }
 
@@ -50,7 +51,9 @@ export function processListItem(
 
   const sectionHeader = findFirstChild(node, opts.sectionTagName);
   const childList = findFirstChild(node, opts.childListTagName);
-  if (!childList) {
+  if (!childList && framework.vendor === 'docusaurus' && linkHref === '#') {
+    return undefined;
+  } else if (!childList) {
     return linkHref;
   }
 
@@ -64,19 +67,20 @@ export function processListItem(
   }
 
   let childEntries = retrieveNavItems(childList);
-  const newLink = childEntries.find(
-    (child) => typeof child === 'string' && child.startsWith(linkHref)
-  )
-    ? removeTrailingSlash(linkHref) + '/mintie_overview'
-    : linkHref;
+  const newLink =
+    linkHref !== '#' &&
+    childEntries.find((child) => typeof child === 'string' && child.startsWith(linkHref))
+      ? removeTrailingSlash(linkHref) + OVERVIEW_PAGE_SLUG
+      : linkHref;
 
-  if (childEntries.includes(linkHref)) {
+  if (linkHref !== '#' && childEntries.includes(linkHref)) {
     childEntries.forEach((child, index) => {
       if (child === linkHref) childEntries[index] = newLink;
     });
-  } else {
+    childEntries = dedupedAppend(newLink, childEntries, true);
+  } else if (linkHref !== '#') {
     childEntries = dedupedAppend(newLink, childEntries, true);
   }
 
-  return { group: title, pages: dedupedAppend(newLink, childEntries, true) };
+  return { group: title, pages: childEntries };
 }

package/src/nav/retrieve.ts

@@ -25,7 +25,7 @@ export function retrieveNavItems(rootNode: Element): Array<NavigationEntry> {
   let innerSectionTagName = 'div';
   switch (framework.vendor) {
     case 'docusaurus':
-      innerSectionTagName = 'li';
+      innerSectionTagName = 'div';
       break;
     case 'gitbook':
       innerSectionTagName = 'div';
@@ -43,12 +43,13 @@ export function retrieveNavItems(rootNode: Element): Array<NavigationEntry> {
     if (
       node.children[0] &&
       node.children[1] &&
-      node.children[1].type === 'element' &&
-      node.children[1].tagName === 'ul' &&
       node.children[0].type === 'element' &&
       node.children[0].tagName === 'div' &&
-      node.children[0].children.filter((child) => child.type === 'text').length ===
-        node.children[0].children.length
+      node.children[1].type === 'element' &&
+      node.children[1].tagName === 'ul' &&
+      node.children[0].children.filter(
+        (child) => framework.vendor === 'docusaurus' || child.type === 'text'
+      ).length === node.children[0].children.length
     ) {
       title = findTitle(node.children[0], { delete: false });
     }

package/src/pipeline/group.ts

@@ -0,0 +1,70 @@
+import type { Browser } from 'puppeteer';
+
+import { OVERVIEW_PAGE_SLUG } from '../constants.js';
+import type { Result } from '../types/result.js';
+import { getErrorMessage } from '../utils/errors.js';
+import { intoChunks } from '../utils/intoChunks.js';
+import { log } from '../utils/log.js';
+import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
+import { scrapePage } from './page.js';
+
+export async function scrapePageGroup(
+  navGroup: Array<URL>,
+  needsBrowser: boolean,
+  opts: {
+    externalLinks: boolean;
+    rootPaths?: Array<string>;
+  } = { externalLinks: false }
+): Promise<Array<Result<[string, string]>>> {
+  const browser: Browser | undefined = needsBrowser ? await startPuppeteer() : undefined;
+
+  const allResults: Array<Result<[string, string]>> = [];
+  try {
+    for (const chunk of intoChunks(navGroup)) {
+      const res = await Promise.all(
+        chunk.map(async (url, index) => {
+          try {
+            if (opts.externalLinks) {
+              const res = scrapePage(`external-link-${index}`, url, { externalLink: true });
+              return res;
+            }
+
+            let isOverviewPage = false;
+            if (url.toString().endsWith(OVERVIEW_PAGE_SLUG)) {
+              isOverviewPage = true;
+              url = new URL(url.toString().replace(OVERVIEW_PAGE_SLUG, ''));
+            }
+
+            const html = await fetchPageHtml(url, browser);
+            const res = scrapePage(html, url, {
+              externalLink: false,
+              isOverviewPage,
+              rootPath: opts.rootPaths ? opts.rootPaths[index] : undefined,
+            });
+            return res;
+          } catch (error) {
+            const errorMessage = getErrorMessage(error);
+            return {
+              success: false,
+              message: `We encountered an error when scraping ${url}${errorMessage}`,
+              data: [url.toString(), ''] as [string, string],
+            };
+          }
+        })
+      );
+      allResults.push(...res);
+    }
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    log(
+      `We encountered an error when scraping the page group from ${
+        navGroup[0]?.origin ?? 'the URL provided'
+      }${errorMessage}`
+    );
+    if (browser) await browser.close();
+    throw error;
+  } finally {
+    if (browser) await browser.close();
+    return allResults;
+  }
+}

package/src/{scrapingPipeline → pipeline}/logo.ts

@@ -101,11 +101,7 @@ export async function downloadLogos(
       htmls.push(await page.content());
       await page.click('.rm-ThemeToggle');
       htmls.push(await page.content());
-    } catch {
-      // do nothing, it just means there's no theme toggle
-    }
-
-    await browser.close();
+    } catch {}
 
     await Promise.all(
       htmls.map(async (html) => {

package/src/{scrapingPipeline → pipeline}/site.ts

@@ -3,6 +3,7 @@ import type { Root as HastRoot } from 'hast';
 import traverse from 'traverse';
 
 import { NAV_FAILURE_MSG } from '../constants.js';
+import { OVERVIEW_PAGE_SLUG } from '../constants.js';
 import { iterateOverNavItems } from '../nav/iterate.js';
 import { retrieveNavItems } from '../nav/retrieve.js';
 import { retrieveRootNavElement } from '../nav/root.js';
@@ -30,7 +31,7 @@ export async function scrapeSite(
   url = new URL(url);
   const origin = url.origin;
 
-  detectFramework(hast);
+  if (!framework.vendor) detectFramework(hast);
 
   if (framework.vendor === 'docusaurus') {
     const browser = await startPuppeteer();
@@ -113,11 +114,11 @@ export async function scrapeSite(
 
     traverse(navItems).forEach(function (value) {
       if (typeof value === 'string') {
-        this.update(value.replace('/mintie_overview', ''));
+        this.update(value.replace(OVERVIEW_PAGE_SLUG, ''));
       } else if (Array.isArray(value)) {
         this.update(
           value.map((item) =>
-            typeof item === 'string' ? item.replace('/mintie_overview', '') : item
+            typeof item === 'string' ? item.replace(OVERVIEW_PAGE_SLUG, '') : item
           )
         );
       }

package/src/{scrapingPipeline → pipeline}/tabs.ts

@@ -1,7 +1,7 @@
 import type { Colors, Navigation, NavigationEntry } from '@mintlify/models';
 import { MintConfig, Tab } from '@mintlify/models';
 
-import { retrieveTabLinks } from '../tabs/retrieveReadme.js';
+import { retrieveTabLinks } from '../tabs/retrieve.js';
 import type { Result } from '../types/result.js';
 import { detectFramework, framework } from '../utils/detectFramework.js';
 import { getErrorMessage } from '../utils/errors.js';
@@ -75,6 +75,7 @@ export async function scrapeAllSiteTabs(
     const browser = needsBrowser ? await startPuppeteer() : undefined;
     const logo = await downloadLogos(url, browser);
     const name = await downloadTitle(hast);
+    if (browser) await browser.close();
 
     return {
       success: true,

package/src/utils/images.ts

@@ -6,7 +6,6 @@ import type { Result } from '../types/result.js';
 import { getErrorMessage } from './errors.js';
 import { getFileExtension } from './extension.js';
 import { write } from './file.js';
-import { log } from './log.js';
 import { fetchImage } from './network.js';
 
 export async function downloadImage(
@@ -19,10 +18,6 @@ export async function downloadImage(
   try {
     let filename = await writeImageToFile(src, rootPath);
     filename = filename.replace(process.cwd(), '');
-
-    const shortenedFilename = filename.length > 36 ? `...${filename.slice(-36)}` : filename;
-    log(`${shortenedFilename} downloaded and written to disk`, 'success');
-
     return { success: true, data: [src, filename] };
   } catch (error) {
     if (error instanceof Error) {
@@ -70,10 +65,6 @@ export function isValidImageSrc(src: string) {
     return false;
   }
 
-  if (src.startsWith('data:')) {
-    return false;
-  }
-
   const ext = getFileExtension(src);
   if (ext && !SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
     return false;

package/src/utils/intoChunks.ts

@@ -0,0 +1,6 @@
+export function* intoChunks<T>(values: T[], chunkSize = 16) {
+  for (let i = 0; i < values.length; i += chunkSize) {
+    const chunk = values.slice(i, i + chunkSize);
+    yield chunk;
+  }
+}

package/src/utils/network.ts

@@ -74,35 +74,34 @@ export async function getHtmlWithPuppeteer(
     );
     await page.setJavaScriptEnabled(true);
 
-    await page.goto(url.toString(), {
-      waitUntil: 'networkidle2',
-      timeout: 30000,
-    });
+    await exponentialBackoff(() =>
+      page.goto(url.toString(), {
+        waitUntil: 'networkidle2',
+        timeout: 30000,
+      })
+    );
 
     if (framework.vendor === 'docusaurus') {
       await page.evaluate(() => {
-        const clickMenuItems = (parentElement: Element | Document = document) => {
-          const menuItems = parentElement.getElementsByClassName('menu__link--sublist');
-
+        document.addEventListener(
+          'click',
+          (e) => {
+            if (e.target instanceof Element && e.target.classList.contains('menu__link--sublist'))
+              e.preventDefault();
+          },
+          true
+        );
+
+        function clickItems(el: HTMLElement | Document) {
+          const menuItems = el.getElementsByClassName(
+            'menu__link--sublist'
+          ) as HTMLCollectionOf<HTMLElement>;
           for (const item of menuItems) {
-            const clickEvent = new MouseEvent('click', {
-              bubbles: true,
-              cancelable: true,
-              view: window,
-            });
-            item.dispatchEvent(clickEvent);
-
-            const parentLi = item.parentElement;
-            if (parentLi) {
-              const nestedUl = parentLi.querySelector('ul');
-              if (nestedUl) {
-                clickMenuItems(nestedUl);
-              }
-            }
+            item.click();
+            clickItems(item);
           }
-        };
-
-        clickMenuItems();
+        }
+        clickItems(document);
       });
     }
 

package/bin/scrapingPipeline/color.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"color.js","sourceRoot":"","sources":["../../src/scrapingPipeline/color.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAExD,SAAS,KAAK,CAAC,KAAa;IAC1B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,aAAa,CAAC,GAAuB;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,OAAO,oCAAoC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,cAAc,CAAC,GAAG,OAAsB;IAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,GAAG;YAAE,OAAO,KAAK,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEnC,IAAI,CAAqB,EAAE,CAAqB,EAAE,CAAqB,CAAC;IAExE,IAAI,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAE1E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAErC,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAE/C,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;AAC5D,CAAC;AAED,SAAS,WAAW,CAAC,SAAiB,EAAE,GAAW;IACjD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,uBAAuB,EAAE,GAAG,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrC,OAAO,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAc;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,aAAa,CAAC;IAE5D,IAAI,cAAc,GAAuB,SAAS,CAAC;IACnD,IAAI,YAAY,GAAuB,SAAS,CAAC;IACjD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YAAE,OAAO,QAAQ,CAAC;QAC9C,IACE,CAAC,SAAS,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;YACzE,CAAC,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,KAAK,eAAe,CAAC;YAE5E,OAAO,QAAQ,CAAC;QAElB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM;YACrF,OAAO,QAAQ,CAAC;QAElB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACtC,MAAM,eAAe,GACnB,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACjF,MAAM,aAAa,GACjB,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,qBAAqB,CAAC;QAEjF,MAAM,oBAAoB,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAClE,MAAM,kBAAkB,GAAG,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC9D,IAAI,CAAC,oBAAoB,IAAI,CAAC,kBAAkB;YAAE,OAAO,QAAQ,CAAC;QAElE,cAAc,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAChD,YAAY,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;QACnC,OAAO;YACL,OAAO,EAAE,cAAe;YACxB,KAAK,EAAE,YAAY;YACnB,IAAI,EAAE,cAAc;SACrB,CAAC;IACJ,CAAC;SAAM,IAAI,cAAc,EAAE,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,cAAe;YACxB,IAAI,EAAE,cAAc;SACrB,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,aAAa,CAAC;IACvB,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/group.js

@@ -1,44 +0,0 @@
-import { getErrorMessage } from '../utils/errors.js';
-import { log } from '../utils/log.js';
-import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
-import { scrapePage } from './page.js';
-export async function scrapePageGroup(navGroup, needsBrowser, opts = { externalLinks: false }) {
-    const browser = needsBrowser ? await startPuppeteer() : undefined;
-    try {
-        const res = await Promise.all(navGroup.map(async (url, index) => {
-            try {
-                if (opts.externalLinks) {
-                    const res = scrapePage(`external-link-${index}`, url, { externalLink: true });
-                    return res;
-                }
-                let isOverviewPage = false;
-                if (url.toString().endsWith('/mintie_overview')) {
-                    isOverviewPage = true;
-                    url = new URL(url.toString().replace('/mintie_overview', ''));
-                }
-                const html = await fetchPageHtml(url, browser);
-                const res = scrapePage(html, url, {
-                    externalLink: false,
-                    isOverviewPage,
-                    rootPath: opts.rootPaths ? opts.rootPaths[index] : undefined,
-                });
-                return res;
-            }
-            catch (error) {
-                const errorMessage = getErrorMessage(error);
-                throw new Error(`We encountered an error when scraping ${url}${errorMessage}`);
-            }
-        }));
-        return res;
-    }
-    catch (error) {
-        const errorMessage = getErrorMessage(error);
-        log(`We encountered an error when scraping the page group from ${navGroup[0]?.origin ?? 'the URL provided'}${errorMessage}`);
-        throw error;
-    }
-    finally {
-        if (browser)
-            await browser.close();
-    }
-}
-//# sourceMappingURL=group.js.map

package/bin/scrapingPipeline/group.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"group.js","sourceRoot":"","sources":["../../src/scrapingPipeline/group.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAoB,EACpB,YAAqB,EACrB,OAGI,EAAE,aAAa,EAAE,KAAK,EAAE;IAE5B,MAAM,OAAO,GAAwB,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;YAChC,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACvB,MAAM,GAAG,GAAG,UAAU,CAAC,iBAAiB,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC9E,OAAO,GAAG,CAAC;gBACb,CAAC;gBAED,IAAI,cAAc,GAAG,KAAK,CAAC;gBAC3B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;oBAChD,cAAc,GAAG,IAAI,CAAC;oBACtB,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;gBAChE,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE;oBAChC,YAAY,EAAE,KAAK;oBACnB,cAAc;oBACd,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7D,CAAC,CAAC;gBACH,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,GAAG,YAAY,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CACD,6DACE,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,kBACzB,GAAG,YAAY,EAAE,CAClB,CAAC;QACF,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/icon.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"icon.js","sourceRoot":"","sources":["../../src/scrapingPipeline/icon.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAc;IAClD,IAAI,GAAG,GAAW,EAAE,CAAC;IACrB,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,MAAM;YACvB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EACpC,CAAC;YACD,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,IAAc,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,cAAc,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,OAAO,cAAc,CAAC;IAErC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC"}

package/bin/scrapingPipeline/images.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/scrapingPipeline/images.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAe,EACf,GAAiB;IAEjB,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,KAAK,CAAC,IAAI,EAAE,UAAU,IAAI;QACxB,IAAI,QAAQ,GAAuB,SAAS,CAAC;QAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;QACtB,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YAC7C,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,CAC1F,EAAE,KAA2B,CAAC;QACjC,CAAC;QAED,IAAI,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE/B,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QACtD,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACnD,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE9D,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACjG,CAAC;IAEF,KAAK,CAAC,IAAI,EAAE,UAAU,IAAI,EAAE,KAAK,EAAE,MAAM;QACvC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC;YACnD,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;QACzE,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YAC7C,MAAM,OAAO,GAAI,IAAI,CAAC,UAAqC,CAAC,IAAI;YAC9D,uEAAuE;YACvE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,CAC1F,CAAC;YACF,IAAI,CAAC,OAAO;gBAAE,OAAO,QAAQ,CAAC;YAE9B,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,KAAe,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC;YAC5E,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;QACzE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/bin/scrapingPipeline/logo.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"logo.js","sourceRoot":"","sources":["../../src/scrapingPipeline/logo.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,SAAS,mBAAmB,CAAC,IAAc;IACzC,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC;YAEjD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAc;IAC1C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,MAAM,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAc;IAC7C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,EACnD,CAAC;YACD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK;oBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,IAAY,EACZ,UAA0D,EAC1D,SAAwB;IAExB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,QAAQ,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CACZ,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CACnB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,MAAM,aAAa,CAC7B,IAAI,CAAC,UAAU,CAAC,GAAa,EAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAC9B,CAAC;YAEF,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5B,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CACH,CAAC,CACH,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACpC,IAAI,CAAC,QAAQ;YAAE,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,OAA4B;IAE5B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;gBAC9B,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;YAEH,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACjC,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,oDAAoD;QACtD,CAAC;QAED,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEtB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,MAAM,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAC;QACvE,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,iBAAiB,CACrB,IAAI,EACJ,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,uBAAuB,EAC/E,SAAS,CACV,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5C,GAAG,CAAC,sCAAsC,YAAY,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzD,OAAO,eAAe,CAAC,MAAM,KAAK,CAAC;QACjC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;YAC1B,CAAC,CAAC;gBACE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAW;gBACnC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAW;aACnC;YACH,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC"}

package/bin/scrapingPipeline/page.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/scrapingPipeline/page.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,eAAe,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EACL,aAAa,EACb,UAAU,EACV,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,UAAU,EACV,eAAe,GAChB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAII,EAAE,YAAY,EAAE,KAAK,EAAE;IAE3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC;QACtB,MAAM,eAAe,GAAG,GAAG,QAAQ,MAAM,CAAC;QAC1C,SAAS,CAAC,eAAe,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAEzB,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,OAAO;QACV,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,mBAAmB,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAE9F,MAAM,aAAa,GAAa;QAC9B,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEF,MAAM,SAAS,GAAc,OAAO,EAAE;SACnC,GAAG,CAAC,mBAAmB,CAAC;SACxB,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,WAAW,CAAC;SAChB,GAAG,CAAC,aAAa,CAAC;SAClB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,oBAAoB,CAAC;SACzB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,uBAAuB,CAAC;SAC5B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,sBAAsB,CAAC;SAC3B,GAAG,CAAC,qBAAqB,CAAC;QAE3B,iDAAiD;QACjD,mDAAmD;QACnD,4BAA4B;SAC3B,GAAG,CAAC,8BAA8B,CAAC;SACnC,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,mBAAmB,CAAC;SACxB,GAAG,CAAC,0BAA0B,CAAC;SAC/B,GAAG,CAAC,qBAAqB,CAAC;SAC1B,GAAG,CAAC,yBAAyB,CAAC;SAC9B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,6BAA6B,CAAC;QACnC,6IAA6I;SAC5I,OAAO,CAAC,aAAa,CAAc,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClE,eAAe,CAAC,wBAAwB,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,yDAAyD,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QAC9F,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,EAAE;aACrB,GAAG,CAAC,SAAS,CAAC;aACd,GAAG,CAAC,SAAS,CAAC;YACf,kGAAkG;aACjG,GAAG,CAAC,eAAe,CAAC;aACpB,SAAS,CAAC,SAAS,CAAC,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC9D,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACjF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI,CAAC,QAAQ;gBACjB,CAAC,CAAC,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC;gBACpF,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,GAAG,MAAM,KAAK,iBAAiB,GAAG,YAAY,EAAE;YACzD,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;SACnB,CAAC;IACJ,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/root.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"root.js","sourceRoot":"","sources":["../../src/scrapingPipeline/root.ts"],"names":[],"mappings":"AACA,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OACE,OAAO,EAAE;QACP,kGAAkG;SACjG,GAAG,CAAC,WAAW,CAAC;SAChB,GAAG,CAAC,sBAAsB,CAAC;SAC3B,GAAG,CAAC,wBAAwB,CAAC;SAC7B,KAAK,CAAC,IAAI,CACd,CAAC;AACJ,CAAC"}

package/bin/scrapingPipeline/site.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"site.js","sourceRoot":"","sources":["../../src/scrapingPipeline/site.ts"],"names":[],"mappings":"AAEA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAA+C,EAAE;IAEjD,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACrB,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAEnC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAE1B,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,cAAc,EAAE,CAAC;QACvC,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,eAAe,EAAE,EAAE,CAAC;IAE1F,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,wCAAwC,GAAG,EAAE,EAAE,CAAC;IACpF,CAAC;IAED,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,8CAA8C,GAAG,EAAE,EAAE,CAAC;IAC1F,CAAC;IAED,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IACF,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7C,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;KAC1C,CAAC;IACF,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG,2BAA2B,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QACpE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE;YACzE,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE;YACjE,aAAa,EAAE,KAAK;YACpB,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CACpC,eAAe;aACZ,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,WAAW;aACR,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBAC1D,CAAC;qBAAM,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CACvE,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO;YACxC,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC;YAC3D,MAAM,IAAI,GAAG,cAAc;iBACxB,KAAK,CAAC,MAAM,CAAC;iBACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;iBAC3E,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,QAAQ,CAAC,KAAK,CAAC,GAAG;gBAChB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,CAAC,OAAO,CAAC;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG;YAChB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;SACnD,CAAC;QAEF,MAAM,eAAe,GAAG,SAAS;aAC9B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACd,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC9B,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC7E,OAAO,kBAAkB,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjE,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK;qBACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACf,OAAO,IAAI,KAAK,QAAQ,IAAI,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAC9E;qBACA,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,KAAK,GAAG,CAAC,CAAC;YACV,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;gBACxC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/D,KAAK,EAAE,CAAC;oBACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBAChB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC7D,CAAC;gBACD,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBACpB,KAAK,CAAC,IAAI,CACR,CAAC,GAAG,EAAE,EAAE,CACN,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CACvF,EACD,CAAC;gBACD,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,MAAM,CACV,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,CACC,OAAO,GAAG,KAAK,QAAQ;oBACvB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAC1D,CACJ,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,eAAe,CAAC,2BAA2B,EAAE,eAAe,CAAC,CAAC;QAC9D,eAAe,CAAC,oBAAoB,EAAE,CAAC,GAAG,eAAe,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC;QAE5E,MAAM,oBAAoB,GAAG,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC;QAC3D,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI;gBACJ,IAAI;gBACJ,MAAM;gBACN,OAAO;gBACP,UAAU,EAAE,QAAsB;gBAClC,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,sEAAsE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC"}