mintlify 1.1.1 → 1.1.2

package/bin/scraping/downloadAllImages.js

@@ -16,6 +16,10 @@ export default async function downloadAllImages($, content, origin, baseDir, mod
     ];
     // Wait to all images to download before continuing
     const origToNewArray = await Promise.all(imageSrcs.map(async (origImageSrc) => {
+        // We do not support downloading base64 in-line images.
+        if (origImageSrc.startsWith("data:")) {
+            return undefined;
+        }
         // Add origin if the image tags are using relative sources
         const imageHref = origImageSrc.startsWith("http")
             ? origImageSrc

package/bin/scraping/downloadAllImages.js.map

@@ -1 +1 @@
-{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAEhD,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,cAAoB;IAEpB,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;KACR;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,YAAoB,EAAE,EAAE;QAC3C,0DAA0D;QAC1D,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC;YAC/C,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,IAAI,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;QAEvC,IAAI,QAAQ,GAAG,2BAA2B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACrE,IAAI,cAAc,EAAE;YAClB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;SACrC;QAED,IAAI,CAAC,QAAQ,EAAE;YACb,OAAO,CAAC,KAAK,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAAC;YACjD,OAAO;SACR;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,CAAC;aACtC,IAAI,CAAC,GAAG,EAAE;YACT,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;gBACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;aACvD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,EAAE,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC;IACvC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAC1B,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnD,EAAE,CACH,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,GAAW;IAC9C,2BAA2B;IAC3B,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEnC,eAAe,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;QACxC,iFAAiF;QACjF,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,CAAC;AACb,CAAC"}
+{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAEhD,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,cAAoB;IAEpB,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;KACR;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,YAAoB,EAAE,EAAE;QAC3C,uDAAuD;QACvD,IAAI,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;YACpC,OAAO,SAAS,CAAC;SAClB;QAED,0DAA0D;QAC1D,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC;YAC/C,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,IAAI,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;QAEvC,IAAI,QAAQ,GAAG,2BAA2B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACrE,IAAI,cAAc,EAAE;YAClB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;SACrC;QAED,IAAI,CAAC,QAAQ,EAAE;YACb,OAAO,CAAC,KAAK,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAAC;YACjD,OAAO;SACR;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,CAAC;aACtC,IAAI,CAAC,GAAG,EAAE;YACT,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;gBACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;aACvD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,EAAE,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC;IACvC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAC1B,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnD,EAAE,CACH,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,GAAW;IAC9C,2BAA2B;IAC3B,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEnC,eAAe,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;QACxC,iFAAiF;QACjF,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,CAAC;AACb,CAAC"}

package/bin/scraping/scrapeSectionCommands.js

@@ -3,18 +3,11 @@ import { detectFramework, Frameworks } from "./detectFramework.js";
 import { getHrefFromArgs, getOrigin } from "../util.js";
 import { scrapeSection } from "./scrapeSection.js";
 import { scrapeDocusaurusSection } from "./site-scrapers/scrapeDocusaurusSection.js";
+import openNestedDocusaurusMenus from "./site-scrapers/openNestedDocusaurusMenus.js";
 import { scrapeGitBookSection } from "./site-scrapers/scrapeGitBookSection.js";
+import openNestedGitbookMenus from "./site-scrapers/openNestedGitbookMenus.js";
 import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
 import { startBrowser } from "../browser.js";
-function validateFramework(framework) {
-    if (!framework) {
-        console.log("Could not detect the framework automatically. Please use one of:");
-        console.log("scrape-page-docusaurus");
-        console.log("scrape-page-gitbook");
-        console.log("scrape-page-readme");
-        return process.exit(1);
-    }
-}
 export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
     const href = getHrefFromArgs(argv);
     const res = await axios.default.get(href);
@@ -22,50 +15,20 @@ export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
     await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
     process.exit(0);
 }
+export async function scrapeDocusaurusSectionCommand(argv) {
+    await scrapeSectionOpeningAllNested(argv, openNestedDocusaurusMenus, scrapeDocusaurusSection);
+}
 export async function scrapeGitbookSectionCommand(argv) {
-    await scrapeSectionGitBookWrapper(argv, scrapeGitBookSection);
+    await scrapeSectionOpeningAllNested(argv, openNestedGitbookMenus, scrapeGitBookSection);
 }
-async function scrapeSectionGitBookWrapper(argv, scrapeFunc) {
+async function scrapeSectionOpeningAllNested(argv, openLinks, scrapeFunc) {
     const href = getHrefFromArgs(argv);
     const browser = await startBrowser();
     const page = await browser.newPage();
     await page.goto(href, {
         waitUntil: "networkidle2",
     });
-    let prevEncountered = [];
-    let encounteredHref = ["fake"];
-    // Loop until we've encountered every link
-    while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
-        prevEncountered = encounteredHref;
-        encounteredHref = await page.evaluate((encounteredHref) => {
-            const icons = Array.from(document.querySelectorAll('path[d="M9 18l6-6-6-6"]'));
-            const linksFound = [];
-            icons.forEach(async (icon) => {
-                var _a, _b;
-                const toClick = (_a = icon === null || icon === void 0 ? void 0 : icon.parentElement) === null || _a === void 0 ? void 0 : _a.parentElement;
-                const link = (_b = toClick === null || toClick === void 0 ? void 0 : toClick.parentElement) === null || _b === void 0 ? void 0 : _b.parentElement;
-                // Skip icons not in the side navigation
-                if (!(link === null || link === void 0 ? void 0 : link.hasAttribute("href"))) {
-                    return;
-                }
-                const href = link.getAttribute("href");
-                // Should never occur but we keep it as a fail-safe
-                if ((href === null || href === void 0 ? void 0 : href.startsWith("https://")) || (href === null || href === void 0 ? void 0 : href.startsWith("http://"))) {
-                    return;
-                }
-                // Click any links we haven't seen before
-                if (href && !encounteredHref.includes(href)) {
-                    toClick === null || toClick === void 0 ? void 0 : toClick.click();
-                }
-                if (href) {
-                    linksFound.push(href);
-                }
-            });
-            return linksFound;
-        }, encounteredHref // Need to pass array into the browser
-        );
-    }
-    const html = await page.content();
+    const html = await openLinks(page);
     browser.close();
     await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
     process.exit(0);
@@ -78,7 +41,7 @@ export async function scrapeSectionAutomatically(argv) {
     validateFramework(framework);
     console.log("Detected framework: " + framework);
     if (framework === Frameworks.DOCUSAURUS) {
-        await scrapeSectionAxiosWrapper(argv, scrapeDocusaurusSection);
+        await scrapeDocusaurusSectionCommand(argv);
     }
     else if (framework === Frameworks.GITBOOK) {
         await scrapeGitbookSectionCommand(argv);
@@ -87,4 +50,13 @@ export async function scrapeSectionAutomatically(argv) {
         await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
     }
 }
+function validateFramework(framework) {
+    if (!framework) {
+        console.log("Could not detect the framework automatically. Please use one of:");
+        console.log("scrape-page-docusaurus");
+        console.log("scrape-page-gitbook");
+        console.log("scrape-page-readme");
+        return process.exit(1);
+    }
+}
 //# sourceMappingURL=scrapeSectionCommands.js.map

package/bin/scraping/scrapeSectionCommands.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAS,EAAE,UAAe;IACxE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,2BAA2B,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,IAAS,EAAE,UAAe;IACnE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,GAAG,CAAC,MAAM,CAAC,CAAC;IAE/B,0CAA0C;IAC1C,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE;QACvE,eAAe,GAAG,eAAe,CAAC;QAClC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CACnC,CAAC,eAAe,EAAE,EAAE;YAClB,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CACrC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CACrD,CAAC;YAEF,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAiB,EAAE,EAAE;;gBACxC,MAAM,OAAO,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,aAAa,0CAAE,aAAa,CAAC;gBACnD,MAAM,IAAI,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,aAAa,0CAAE,aAAa,CAAC;gBAEnD,wCAAwC;gBACxC,IAAI,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,YAAY,CAAC,MAAM,CAAC,CAAA,EAAE;oBAC/B,OAAO;iBACR;gBAED,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEvC,mDAAmD;gBACnD,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,UAAU,CAAC,MAAI,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,SAAS,CAAC,CAAA,EAAE;oBAC/D,OAAO;iBACR;gBAED,yCAAyC;gBACzC,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAC3C,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,EAAE,CAAC;iBAClB;gBACD,IAAI,IAAI,EAAE;oBACR,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACvB;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,UAAU,CAAC;QACpB,CAAC,EACD,eAAe,CAAC,sCAAsC;SACvD,CAAC;KACH;IAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAExC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,yBAAyB,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;KAChE;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;KAC5D;AACH,CAAC"}
+{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAS,EAAE,UAAe;IACxE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,IAAS;IAC5D,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,CACxB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CACjC,IAAI,EACJ,sBAAsB,EACtB,oBAAoB,CACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAAe;IAEf,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAExC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,8BAA8B,CAAC,IAAI,CAAC,CAAC;KAC5C;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;KAC5D;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC"}

package/bin/scraping/site-scrapers/alternateGroupTitle.js

@@ -0,0 +1,9 @@
+export default function alternateGroupTitle(firstLink, pages) {
+    // Only assign titles to nested navigation menus outside a section.
+    // Others should not have a title so we can merge them into one section.
+    if (pages.length > 0) {
+        return firstLink === null || firstLink === void 0 ? void 0 : firstLink.text();
+    }
+    return "";
+}
+//# sourceMappingURL=alternateGroupTitle.js.map

package/bin/scraping/site-scrapers/alternateGroupTitle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"alternateGroupTitle.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/alternateGroupTitle.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,SAAS,EAAE,KAAK;IAC1D,mEAAmE;IACnE,wEAAwE;IACxE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;QACpB,OAAO,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,IAAI,EAAE,CAAC;KAC1B;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/bin/scraping/site-scrapers/getLinksRecursively.js

@@ -1,4 +1,4 @@
-// Used by GitBook and ReadMe section scrapers
+// Used by Docusaurus, GitBook, and ReadMe section scrapers
 export default function getLinksRecursively(linkSections, $) {
     if (linkSections == null || linkSections.length === 0) {
         return [];
@@ -6,12 +6,17 @@ export default function getLinksRecursively(linkSections, $) {
     return linkSections
         .map((i, s) => {
         const subsection = $(s);
-        const link = subsection.children().first();
+        let link = subsection.children().first();
+        if (!link.attr("href")) {
+            // Docusaurus nests the <a> inside a <div>
+            link = link.find("a[href]").first();
+        }
         const linkHref = link.attr("href");
         // Skip missing links. For example, GitBook uses
         // empty divs are used for styling a line beside the nav.
         // Skip external links until Mintlify supports them
         if (!linkHref ||
+            linkHref === "#" ||
             linkHref.startsWith("https://") ||
             linkHref.startsWith("http://")) {
             return undefined;

package/bin/scraping/site-scrapers/getLinksRecursively.js.map

@@ -1 +1 @@
-{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/getLinksRecursively.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAC9C,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,YAAiB,EAAE,CAAM;IACnE,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QACrD,OAAO,EAAE,CAAC;KACX;IAED,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;YACA,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aACzD,CAAC;SACH;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
+{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/getLinksRecursively.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,YAAiB,EAAE,CAAM;IACnE,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QACrD,OAAO,EAAE,CAAC;KACX;IAED,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAEzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YACtB,0CAA0C;YAC1C,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;SACrC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,KAAK,GAAG;YAChB,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;YACA,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aACzD,CAAC;SACH;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}

package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js

@@ -0,0 +1,30 @@
+export default async function openNestedDocusaurusMenus(page) {
+    let prevEncountered = [];
+    let encounteredHref = ["fake-href-to-make-loop-run-at-least-once"];
+    // Loop until we've encountered every link
+    while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+        prevEncountered = encounteredHref;
+        encounteredHref = await page.evaluate((encounteredHref) => {
+            const collapsible = Array.from(document.querySelectorAll(".menu__link.menu__link--sublist"));
+            const linksFound = [];
+            collapsible.forEach(async (collapsibleItem) => {
+                const href = collapsibleItem === null || collapsibleItem === void 0 ? void 0 : collapsibleItem.getAttribute("href");
+                // Should never occur but we keep it as a fail-safe
+                if ((href === null || href === void 0 ? void 0 : href.startsWith("https://")) || (href === null || href === void 0 ? void 0 : href.startsWith("http://"))) {
+                    return;
+                }
+                // Click any links we haven't seen before
+                if (href && !encounteredHref.includes(href)) {
+                    collapsibleItem === null || collapsibleItem === void 0 ? void 0 : collapsibleItem.click();
+                }
+                if (href) {
+                    linksFound.push(href);
+                }
+            });
+            return linksFound;
+        }, encounteredHref // Need to pass array into the browser
+        );
+    }
+    return await page.content();
+}
+//# sourceMappingURL=openNestedDocusaurusMenus.js.map

package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openNestedDocusaurusMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedDocusaurusMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAU;IAChE,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,GAAG,CAAC,0CAA0C,CAAC,CAAC;IAEnE,0CAA0C;IAC1C,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE;QACvE,eAAe,GAAG,eAAe,CAAC;QAClC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CACnC,CAAC,eAAe,EAAE,EAAE;YAClB,MAAM,WAAW,GAAkB,KAAK,CAAC,IAAI,CAC3C,QAAQ,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAC7D,CAAC;YAEF,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,eAA4B,EAAE,EAAE;gBACzD,MAAM,IAAI,GAAG,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEnD,mDAAmD;gBACnD,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,UAAU,CAAC,MAAI,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,SAAS,CAAC,CAAA,EAAE;oBAC/D,OAAO;iBACR;gBAED,yCAAyC;gBACzC,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAC3C,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,KAAK,EAAE,CAAC;iBAC1B;gBAED,IAAI,IAAI,EAAE;oBACR,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACvB;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,UAAU,CAAC;QACpB,CAAC,EACD,eAAe,CAAC,sCAAsC;SACvD,CAAC;KACH;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}

package/bin/scraping/site-scrapers/openNestedGitbookMenus.js

@@ -0,0 +1,37 @@
+export default async function openNestedGitbookMenus(page) {
+    let prevEncountered = [];
+    let encounteredHref = ["fake-href-to-make-loop-run-at-least-once"];
+    // Loop until we've encountered every link
+    while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+        prevEncountered = encounteredHref;
+        encounteredHref = await page.evaluate((encounteredHref) => {
+            const icons = Array.from(document.querySelectorAll('path[d="M9 18l6-6-6-6"]'));
+            const linksFound = [];
+            icons.forEach(async (icon) => {
+                var _a, _b;
+                const toClick = (_a = icon === null || icon === void 0 ? void 0 : icon.parentElement) === null || _a === void 0 ? void 0 : _a.parentElement;
+                const link = (_b = toClick === null || toClick === void 0 ? void 0 : toClick.parentElement) === null || _b === void 0 ? void 0 : _b.parentElement;
+                // Skip icons not in the side navigation
+                if (!(link === null || link === void 0 ? void 0 : link.hasAttribute("href"))) {
+                    return;
+                }
+                const href = link.getAttribute("href");
+                // Should never occur but we keep it as a fail-safe
+                if ((href === null || href === void 0 ? void 0 : href.startsWith("https://")) || (href === null || href === void 0 ? void 0 : href.startsWith("http://"))) {
+                    return;
+                }
+                // Click any links we haven't seen before
+                if (href && !encounteredHref.includes(href)) {
+                    toClick === null || toClick === void 0 ? void 0 : toClick.click();
+                }
+                if (href) {
+                    linksFound.push(href);
+                }
+            });
+            return linksFound;
+        }, encounteredHref // Need to pass array into the browser
+        );
+    }
+    return await page.content();
+}
+//# sourceMappingURL=openNestedGitbookMenus.js.map

package/bin/scraping/site-scrapers/openNestedGitbookMenus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openNestedGitbookMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedGitbookMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAU;IAC7D,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,GAAG,CAAC,0CAA0C,CAAC,CAAC;IAEnE,0CAA0C;IAC1C,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE;QACvE,eAAe,GAAG,eAAe,CAAC;QAClC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CACnC,CAAC,eAAe,EAAE,EAAE;YAClB,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CACrC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CACrD,CAAC;YAEF,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAiB,EAAE,EAAE;;gBACxC,MAAM,OAAO,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,aAAa,0CAAE,aAAa,CAAC;gBACnD,MAAM,IAAI,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,aAAa,0CAAE,aAAa,CAAC;gBAEnD,wCAAwC;gBACxC,IAAI,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,YAAY,CAAC,MAAM,CAAC,CAAA,EAAE;oBAC/B,OAAO;iBACR;gBAED,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEvC,mDAAmD;gBACnD,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,UAAU,CAAC,MAAI,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,SAAS,CAAC,CAAA,EAAE;oBAC/D,OAAO;iBACR;gBAED,yCAAyC;gBACzC,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAC3C,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,EAAE,CAAC;iBAClB;gBACD,IAAI,IAAI,EAAE;oBACR,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACvB;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,UAAU,CAAC;QACpB,CAAC,EACD,eAAe,CAAC,sCAAsC;SACvD,CAAC;KACH;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js

@@ -1,48 +1,47 @@
 import cheerio from "cheerio";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
 import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+import alternateGroupTitle from "./alternateGroupTitle.js";
 export async function scrapeDocusaurusSection(html, origin, cliDir, overwrite) {
     const $ = cheerio.load(html);
     // Get all the navigation sections
     const navigationSections = $(".theme-doc-sidebar-menu").first().children();
     // Get all links per group
     const groupsConfig = navigationSections
-        .map((i, section) => {
-        const sectionComponent = $(section);
+        .map((i, s) => {
+        const section = $(s);
         // Links without a group
-        if (sectionComponent.hasClass("theme-doc-sidebar-item-link")) {
-            const linkHref = sectionComponent.find("a[href]").first().attr("href");
+        if (section.hasClass("theme-doc-sidebar-item-link")) {
+            const linkHref = section.find("a[href]").first().attr("href");
             return {
                 group: "",
                 pages: [linkHref],
             };
         }
-        const sectionTitle = sectionComponent
+        const firstLink = section
             .find(".menu__list-item-collapsible")
             .first()
-            .text();
-        // The category title can be a page too so we find from the
-        // section component instead of the more specific menu__list child
-        const linkPaths = sectionComponent
-            .find("a[href]")
-            .map((i, link) => {
-            return $(link).attr("href");
-        })
-            .filter((i, link) => link !== "#")
-            .toArray();
-        // Follows the same structure as mint.json
+            .find("a[href]");
+        const sectionTitle = firstLink.text();
+        const firstHref = firstLink.attr("href");
+        const linkSections = section.children().eq(1).children();
+        const pages = getLinksRecursively(linkSections, $);
         return {
-            group: sectionTitle,
-            pages: linkPaths,
+            group: sectionTitle || alternateGroupTitle(firstLink, pages),
+            pages: firstHref ? [firstHref, ...pages] : pages,
         };
     })
         .toArray();
+    // Merge groups with empty titles together
+    const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
     // Scrape each link in the navigation.
-    const groupsConfigCleanPaths = await Promise.all(groupsConfig.map(async (groupConfig) => {
-        groupConfig.pages = (await Promise.all(groupConfig.pages.map(async (pathname) => 
+    const groupsConfigCleanPaths = await Promise.all(reducedGroupsConfig.map(async (groupConfig) => {
+        groupConfig.pages = (await Promise.all(groupConfig.pages.map(async (navEntry) => 
         // Docusaurus requires a directory on all sections wheras we use root.
         // /docs is their default directory so we remove it
-        scrapeGettingFileNameFromUrl(pathname, cliDir, origin, overwrite, scrapeDocusaurusPage, false, "/docs"))))
+        scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapeDocusaurusPage, false, "/docs"))))
             // Remove skipped index pages (they return undefined from the above function)
             .filter(Boolean);
         return groupConfig;

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,gBAAgB,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAEpC,wBAAwB;QACxB,IAAI,gBAAgB,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YAC5D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvE,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,YAAY,GAAG,gBAAgB;aAClC,IAAI,CAAC,8BAA8B,CAAC;aACpC,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,2DAA2D;QAC3D,kEAAkE;QAClE,MAAM,SAAS,GAAG,gBAAgB;aAC/B,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC;aACjC,OAAO,EAAE,CAAC;QAEb,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QAC/C,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
+{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,wBAAwB;QACxB,IAAI,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,SAAS,GAAG,OAAO;aACtB,IAAI,CAAC,8BAA8B,CAAC;aACpC,KAAK,EAAE;aACP,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnB,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QAC5C,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QACxD,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookSection.js

@@ -3,6 +3,7 @@ import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js
 import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
 import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
 import getLinksRecursively from "./getLinksRecursively.js";
+import alternateGroupTitle from "./alternateGroupTitle.js";
 export async function scrapeGitBookSection(html, origin, cliDir, overwrite) {
     const $ = cheerio.load(html);
     // Get all the navigation sections
@@ -26,7 +27,7 @@ export async function scrapeGitBookSection(html, origin, cliDir, overwrite) {
         const linkSections = section.children().eq(1).children();
         const pages = getLinksRecursively(linkSections, $);
         return {
-            group: sectionTitle || alternateTitle(firstLink, pages),
+            group: sectionTitle || alternateGroupTitle(firstLink, pages),
             pages: firstHref ? [firstHref, ...pages] : pages,
         };
     })
@@ -40,12 +41,4 @@ export async function scrapeGitBookSection(html, origin, cliDir, overwrite) {
     }));
     return groupsConfigCleanPaths;
 }
-function alternateTitle(firstLink, pages) {
-    // Only assign titles to nested navigation menus outside a section.
-    // Others should not have a title so we can merge them into one section.
-    if (pages.length > 0) {
-        return firstLink === null || firstLink === void 0 ? void 0 : firstLink.text();
-    }
-    return "";
-}
 //# sourceMappingURL=scrapeGitBookSection.js.map

package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC;YACvD,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QAC1D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED,SAAS,cAAc,CAAC,SAAS,EAAE,KAAK;IACtC,mEAAmE;IACnE,wEAAwE;IACxE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;QACpB,OAAO,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,IAAI,EAAE,CAAC;KAC1B;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
+{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QAC1D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mintlify",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "The Mintlify CLI",
   "engines": {
     "node": ">=14.16"

package/src/index.ts

@@ -15,8 +15,8 @@ import {
   scrapeSectionAutomatically,
   scrapeSectionAxiosWrapper,
   scrapeGitbookSectionCommand,
+  scrapeDocusaurusSectionCommand,
 } from "./scraping/scrapeSectionCommands.js";
-import { scrapeDocusaurusSection } from "./scraping/site-scrapers/scrapeDocusaurusSection.js";
 import { scrapeReadMeSection } from "./scraping/site-scrapers/scrapeReadMeSection.js";
 import dev from "./local-preview/index.js";
 import installDepsCommand from "./local-preview/helper-commands/installDepsCommand.js";
@@ -86,7 +86,7 @@ yargs(hideBin(process.argv))
     "Scrapes the Docusaurus section",
     () => {},
     async (argv) => {
-      await scrapeSectionAxiosWrapper(argv, scrapeDocusaurusSection);
+      await scrapeDocusaurusSectionCommand(argv);
     }
   )
   .command(

package/src/local-preview/helper-commands/clearCommand.ts

@@ -2,10 +2,10 @@ import inquirer from "inquirer";
 import shell from "shelljs";
 import { CLIENT_PATH } from "../../constants.js";
 
-const clearCommand = () => {
+const clearCommand = async () => {
   shell.cd(CLIENT_PATH);
-  shell.exec("git clean -d -x -e node_modules -n");
-  inquirer
+  shell.exec("git clean -d -x -e node_modules -e last-invocation-path -n");
+  await inquirer
     .prompt([
       {
         type: "confirm",
@@ -16,7 +16,9 @@ const clearCommand = () => {
     ])
     .then(({ confirm }) => {
       if (confirm) {
-        shell.exec("git clean -d -x -e node_modules -f");
+        shell.exec(
+          "git clean -d -x -e node_modules -e last-invocation-path -f"
+        );
       } else {
         console.log("Clear cancelled.");
       }

package/src/local-preview/index.ts

@@ -2,6 +2,7 @@ import Chalk from "chalk";
 import open from "open";
 import { promises as _promises } from "fs";
 import fse, { pathExists } from "fs-extra";
+import inquirer from "inquirer";
 import { isInternetAvailable } from "is-internet-available";
 import path from "path";
 import shell from "shelljs";
@@ -18,6 +19,25 @@ import listener from "./utils/listener.js";
 import { createPage, createMetadataFileFromPages } from "./utils/metadata.js";
 import { updateConfigFile } from "./utils/mintConfigFile.js";
 import { buildLogger, ensureYarn } from "../util.js";
+import clearCommand from "./helper-commands/clearCommand.js";
+
+const saveInvocationPath = async () => {
+  await fse.outputFile(LAST_INVOCATION_PATH_FILE_LOCATION, CMD_EXEC_PATH);
+};
+
+const cleanOldFiles = async () => {
+  const lastInvocationPathExists = await pathExists(
+    LAST_INVOCATION_PATH_FILE_LOCATION
+  );
+  if (!lastInvocationPathExists) return;
+  const lastInvocationPath = (
+    await readFile(LAST_INVOCATION_PATH_FILE_LOCATION)
+  ).toString();
+  if (lastInvocationPath !== CMD_EXEC_PATH) {
+    // clean if invoked in new location
+    await clearCommand();
+  }
+};
 
 const { readFile } = _promises;
 
@@ -90,8 +110,33 @@ const shellExec = (cmd: string) => {
 const nodeModulesExists = async () => {
   return pathExists(path.join(DOT_MINTLIFY, "mint", "client", "node_modules"));
 };
+
+const promptForYarn = async () => {
+  const yarnInstalled = shell.which("yarn");
+  if (!yarnInstalled) {
+    await inquirer
+      .prompt([
+        {
+          type: "confirm",
+          name: "confirm",
+          message: "yarn must be globally installed. Install yarn?",
+          default: true,
+        },
+      ])
+      .then(({ confirm }) => {
+        if (confirm) {
+          shell.exec("npm install --global yarn");
+        } else {
+          console.log("Installation cancelled.");
+        }
+      });
+  }
+};
+
 const dev = async () => {
   shell.cd(HOME_DIR);
+  await cleanOldFiles();
+  await promptForYarn();
   const logger = buildLogger("Starting a local Mintlify instance...");
   await fse.ensureDir(path.join(DOT_MINTLIFY, "mint"));
   shell.cd(path.join(HOME_DIR, ".mintlify", "mint"));
@@ -130,8 +175,7 @@ const dev = async () => {
     runYarn = false;
   }
   shell.cd(CLIENT_PATH);
-  runYarn = !(await nodeModulesExists());
-  if (internet && runYarn) {
+  if (internet && (runYarn || !(await nodeModulesExists()))) {
     if (firstInstallation) {
       logger.succeed("Local Mintlify instance initialized");
     }
@@ -153,6 +197,7 @@ const dev = async () => {
     `);
     process.exit(1);
   }
+  await saveInvocationPath();
   await copyFiles(logger);
   run();
 };
@@ -161,7 +206,7 @@ const run = () => {
   shell.cd(CLIENT_PATH);
   console.log(
     `🌿 ${Chalk.green(
-      "Navigate to your local preview at https://localhost:3000"
+      "Navigate to your local preview at http://localhost:3000"
     )}`
   );
   shell.exec("npm run dev", { async: true });

package/src/local-preview/utils/listener.ts

@@ -10,7 +10,7 @@ const listener = () => {
   chokidar
     .watch(CMD_EXEC_PATH, {
       ignoreInitial: true,
-      ignored: "node_modules",
+      ignored: ["node_modules", ".git"],
       cwd: CMD_EXEC_PATH,
     })
     .on("all", async (event, filename) => {

package/src/scraping/downloadAllImages.ts

@@ -28,6 +28,11 @@ export default async function downloadAllImages(
   // Wait to all images to download before continuing
   const origToNewArray = await Promise.all(
     imageSrcs.map(async (origImageSrc: string) => {
+      // We do not support downloading base64 in-line images.
+      if (origImageSrc.startsWith("data:")) {
+        return undefined;
+      }
+
       // Add origin if the image tags are using relative sources
       const imageHref = origImageSrc.startsWith("http")
         ? origImageSrc