mintlify 1.0.7 → 1.0.8

package/bin/navigation.js

@@ -0,0 +1,4 @@
+export function isNavigation(navEntry) {
+    return typeof navEntry !== "string";
+}
+//# sourceMappingURL=navigation.js.map

package/bin/navigation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"navigation.js","sourceRoot":"","sources":["../src/navigation.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,YAAY,CAC1B,QAAyB;IAEzB,OAAO,OAAO,QAAQ,KAAK,QAAQ,CAAC;AACtC,CAAC"}

package/bin/pageTemplate.js

@@ -0,0 +1,30 @@
+import inquirer from "inquirer";
+import { createPage } from "./util.js";
+const generatePageTemplate = () => {
+    inquirer
+        .prompt([
+        {
+            type: "input",
+            name: "title",
+            message: "What is the title of the new page?",
+        },
+        {
+            type: "input",
+            name: "description",
+            message: "What is the description?",
+            default: "",
+        },
+    ])
+        .then((answers) => {
+        const { title, description } = answers;
+        createPage(title, description);
+        console.log("🌱 Created initial files for Mintlify docs");
+        process.exit(0);
+    })
+        .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });
+};
+export default generatePageTemplate;
+//# sourceMappingURL=pageTemplate.js.map

package/bin/pageTemplate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pageTemplate.js","sourceRoot":"","sources":["../src/pageTemplate.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,oBAAoB,GAAG,GAAG,EAAE;IAChC,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,oCAAoC;SAC9C;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,0BAA0B;YACnC,OAAO,EAAE,EAAE;SACZ;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEvC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,eAAe,oBAAoB,CAAC"}

package/bin/scraping/combineNavWithEmptyGroupTitles.js

@@ -0,0 +1,20 @@
+export default function combineNavWithEmptyGroupTitles(navArray) {
+    let newNavArray = [];
+    navArray.forEach((nav) => {
+        // The first run through the loop will always have -1 as the index.
+        // JavaScript returns undefined when we look for an index outside the size of the array.
+        const prev = newNavArray[newNavArray.length - 1];
+        if (prev == null) {
+            newNavArray.push(nav);
+        }
+        else if (!nav.group && !prev.group) {
+            // Joins multiple groups without a title together IF they occur side by side
+            prev.pages = prev.pages.concat(nav.pages);
+        }
+        else {
+            newNavArray.push(nav);
+        }
+    });
+    return newNavArray;
+}
+//# sourceMappingURL=combineNavWithEmptyGroupTitles.js.map

package/bin/scraping/combineNavWithEmptyGroupTitles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"combineNavWithEmptyGroupTitles.js","sourceRoot":"","sources":["../../src/scraping/combineNavWithEmptyGroupTitles.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,8BAA8B,CAAC,QAAsB;IAC3E,IAAI,WAAW,GAAG,EAAE,CAAC;IAErB,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAe,EAAE,EAAE;QACnC,mEAAmE;QACnE,wFAAwF;QACxF,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjD,IAAI,IAAI,IAAI,IAAI,EAAE;YAChB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACvB;aAAM,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACpC,4EAA4E;YAC5E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;SAC3C;aAAM;YACL,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACvB;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC"}

package/bin/scraping/getSitemapLinks.js

@@ -1,4 +1,6 @@
 import axios from "axios";
+// Not in use.
+// Gets all links in a sitemap.
 export const getSitemapLinks = async (url) => {
     const hostname = url.hostname.replace(".", "\\.");
     const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");

package/bin/scraping/getSitemapLinks.js.map

@@ -1 +1 @@
-{"version":3,"file":"getSitemapLinks.js","sourceRoot":"","sources":["../../src/scraping/getSitemapLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,GAAQ,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,cAAc,QAAQ,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAExE,IAAI;QACF,MAAM,SAAS,GAAG,CAAC,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAc,CAAC;QACrE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAoB,CAAC;QACxD,OAAO,KAAK,IAAI,EAAE,CAAC;KACpB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;QACvE,OAAO,EAAE,CAAC;KACX;AACH,CAAC,CAAC"}
+{"version":3,"file":"getSitemapLinks.js","sourceRoot":"","sources":["../../src/scraping/getSitemapLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,cAAc;AACd,+BAA+B;AAC/B,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,GAAQ,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,cAAc,QAAQ,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAExE,IAAI;QACF,MAAM,SAAS,GAAG,CAAC,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAc,CAAC;QACrE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAoB,CAAC;QACxD,OAAO,KAAK,IAAI,EAAE,CAAC;KACpB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;QACvE,OAAO,EAAE,CAAC;KACX;AACH,CAAC,CAAC"}

package/bin/scraping/scrapeFileGettingFileNameFromUrl.js

@@ -0,0 +1,43 @@
+import path from "path";
+import axios from "axios";
+import { getHtmlWithPuppeteer } from "../browser.js";
+import { createPage } from "../util.js";
+export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, baseToRemove) {
+    // Skip scraping external links
+    if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
+        return pathname;
+    }
+    // Removes file name from the end
+    const splitSubpath = pathname.split("/");
+    let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
+    // Remove base dir if passed in
+    if (baseToRemove && folders.startsWith(baseToRemove)) {
+        folders = folders.replace(baseToRemove, "");
+    }
+    // TO DO: Improve this by putting each page's images in a separate
+    // folder named after the title of the page.
+    const imageBaseDir = path.join(cliDir, "images", folders);
+    // Scrape each page separately
+    const href = new URL(pathname, origin).href;
+    let html;
+    if (puppeteer) {
+        html = await getHtmlWithPuppeteer(href);
+    }
+    else {
+        const res = await axios.default.get(href);
+        html = res.data;
+    }
+    const { title, description, markdown } = await scrapePageFunc(html, origin, cliDir, imageBaseDir);
+    // Check if page didn't have content
+    if (!title && !markdown) {
+        return undefined;
+    }
+    const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
+    // Default to introduction.mdx if we encountered index.html
+    const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
+    // Will create subfolders as needed
+    createPage(title, description, markdown, overwrite, newFileLocation, fileName);
+    // Removes first slash if we are in a folder, Mintlify doesn't need it
+    return folders ? path.join(folders, fileName).substring(1) : fileName;
+}
+//# sourceMappingURL=scrapeFileGettingFileNameFromUrl.js.map

package/bin/scraping/scrapeFileGettingFileNameFromUrl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cASE,EACF,SAAS,GAAG,KAAK,EACjB,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,CACb,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,SAAS,CAAC;KAClB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CACR,KAAK,EACL,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,QAAQ,CACT,CAAC;IAEF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}

package/bin/scraping/scrapeGettingFileNameFromUrl.js

@@ -1,43 +1,14 @@
-import path from "path";
-import axios from "axios";
-import { getHtmlWithPuppeteer } from "../browser.js";
-import { createPage } from "../util.js";
-export async function scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapePageFunc, puppeteer = false, baseToRemove) {
-    // Skip scraping external links
-    if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
-        return pathname;
-    }
-    // Removes file name from the end
-    const splitSubpath = pathname.split("/");
-    let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
-    // Remove base dir if passed in
-    if (baseToRemove && folders.startsWith(baseToRemove)) {
-        folders = folders.replace(baseToRemove, "");
-    }
-    // TO DO: Improve this by putting each page's images in a separate
-    // folder named after the title of the page.
-    const imageBaseDir = path.join(cliDir, "images", folders);
-    // Scrape each page separately
-    const href = new URL(pathname, origin).href;
-    let html;
-    if (puppeteer) {
-        html = await getHtmlWithPuppeteer(href);
-    }
-    else {
-        const res = await axios.default.get(href);
-        html = res.data;
-    }
-    const { title, description, markdown } = await scrapePageFunc(html, origin, cliDir, imageBaseDir);
-    // Check if page didn't have content
-    if (!title && !markdown) {
-        return undefined;
-    }
-    const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
-    // Default to introduction.mdx if we encountered index.html
-    const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
-    // Will create subfolders as needed
-    createPage(title, description, markdown, overwrite, newFileLocation, fileName);
-    // Removes first slash if we are in a folder, Mintlify doesn't need it
-    return folders ? path.join(folders, fileName).substring(1) : fileName;
+import { isNavigation } from "../navigation.js";
+import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
+export async function scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, baseToRemove) {
+    if (isNavigation(navEntry)) {
+        const newPages = [];
+        for (const nestedNavEntry of navEntry.pages) {
+            newPages.push(await scrapeGettingFileNameFromUrl(nestedNavEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer, baseToRemove));
+        }
+        navEntry.pages = newPages;
+        return navEntry;
+    }
+    return await scrapeFileGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer, baseToRemove);
 }
 //# sourceMappingURL=scrapeGettingFileNameFromUrl.js.map

package/bin/scraping/scrapeGettingFileNameFromUrl.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,MAAc,EACd,MAAc,EACd,QAAgB,EAChB,SAAkB,EAClB,cASE,EACF,SAAS,GAAG,KAAK,EACjB,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,CACb,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,SAAS,CAAC;KAClB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CACR,KAAK,EACL,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,QAAQ,CACT,CAAC;IAEF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}
+{"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AAEzF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,QAAyB,EACzB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cASE,EACF,SAAS,GAAG,KAAK,EACjB,YAAqB;IAErB,IAAI,YAAY,CAAC,QAAQ,CAAC,EAAE;QAC1B,MAAM,QAAQ,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,cAAc,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC3C,QAAQ,CAAC,IAAI,CACX,MAAM,4BAA4B,CAChC,cAAc,EACd,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,YAAY,CACb,CACF,CAAC;SACH;QACD,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC1B,OAAO,QAAQ,CAAC;KACjB;IAED,OAAO,MAAM,gCAAgC,CAC3C,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,YAAY,CACb,CAAC;AACJ,CAAC"}

package/bin/scraping/scrapePageCommands.js

@@ -0,0 +1,48 @@
+import axios from "axios";
+import { scrapePage } from "./scrapePage.js";
+import { scrapeDocusaurusPage } from "./site-scrapers/scrapeDocusaurusPage.js";
+import { scrapeGitBookPage } from "./site-scrapers/scrapeGitBookPage.js";
+import { scrapeReadMePage } from "./site-scrapers/scrapeReadMePage.js";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs } from "../util.js";
+import { getHtmlWithPuppeteer } from "../browser.js";
+function validateFramework(framework) {
+    if (!framework) {
+        console.log("Could not detect the framework automatically. Please use one of:");
+        console.log("scrape-page-docusaurus");
+        console.log("scrape-page-gitbook");
+        console.log("scrape-page-readme");
+        return process.exit(1);
+    }
+}
+export async function scrapePageWrapper(argv, scrapeFunc, puppeteer = false) {
+    const href = getHrefFromArgs(argv);
+    let html;
+    if (puppeteer) {
+        html = await getHtmlWithPuppeteer(href);
+    }
+    else {
+        const res = await axios.default.get(href);
+        html = res.data;
+    }
+    await scrapePage(scrapeFunc, href, html, argv.overwrite);
+    process.exit(0);
+}
+export async function scrapePageAutomatically(argv) {
+    const href = getHrefFromArgs(argv);
+    const res = await axios.default.get(href);
+    const html = res.data;
+    const framework = detectFramework(html);
+    validateFramework(framework);
+    console.log("Detected framework: " + framework);
+    if (framework === Frameworks.DOCUSAURUS) {
+        await scrapePageWrapper(argv, scrapeDocusaurusPage);
+    }
+    else if (framework === Frameworks.GITBOOK) {
+        await scrapePageWrapper(argv, scrapeGitBookPage, true);
+    }
+    else if (framework === Frameworks.README) {
+        await scrapePageWrapper(argv, scrapeReadMePage);
+    }
+}
+//# sourceMappingURL=scrapePageCommands.js.map

package/bin/scraping/scrapePageCommands.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,GAAG,KAAK;IACzE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAExC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;KACrD;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,IAAI,CAAC,CAAC;KACxD;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;KACjD;AACH,CAAC"}

package/bin/scraping/scrapeSectionCommands.js

@@ -0,0 +1,90 @@
+import axios from "axios";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs, getOrigin } from "../util.js";
+import { scrapeSection } from "./scrapeSection.js";
+import { scrapeDocusaurusSection } from "./site-scrapers/scrapeDocusaurusSection.js";
+import { scrapeGitBookSection } from "./site-scrapers/scrapeGitBookSection.js";
+import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
+import { startBrowser } from "../browser.js";
+function validateFramework(framework) {
+    if (!framework) {
+        console.log("Could not detect the framework automatically. Please use one of:");
+        console.log("scrape-page-docusaurus");
+        console.log("scrape-page-gitbook");
+        console.log("scrape-page-readme");
+        return process.exit(1);
+    }
+}
+export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
+    const href = getHrefFromArgs(argv);
+    const res = await axios.default.get(href);
+    const html = res.data;
+    await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+    process.exit(0);
+}
+export async function scrapeGitbookSectionCommand(argv) {
+    await scrapeSectionGitBookWrapper(argv, scrapeGitBookSection);
+}
+async function scrapeSectionGitBookWrapper(argv, scrapeFunc) {
+    const href = getHrefFromArgs(argv);
+    const browser = await startBrowser();
+    const page = await browser.newPage();
+    await page.goto(href, {
+        waitUntil: "networkidle2",
+    });
+    let prevEncountered = [];
+    let encounteredHref = ["fake"];
+    // Loop until we've encountered every link
+    while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+        prevEncountered = encounteredHref;
+        encounteredHref = await page.evaluate((encounteredHref) => {
+            const icons = Array.from(document.querySelectorAll('path[d="M9 18l6-6-6-6"]'));
+            const linksFound = [];
+            icons.forEach(async (icon) => {
+                var _a, _b;
+                const toClick = (_a = icon === null || icon === void 0 ? void 0 : icon.parentElement) === null || _a === void 0 ? void 0 : _a.parentElement;
+                const link = (_b = toClick === null || toClick === void 0 ? void 0 : toClick.parentElement) === null || _b === void 0 ? void 0 : _b.parentElement;
+                // Skip icons not in the side navigation
+                if (!(link === null || link === void 0 ? void 0 : link.hasAttribute("href"))) {
+                    return;
+                }
+                const href = link.getAttribute("href");
+                // Should never occur but we keep it as a fail-safe
+                if ((href === null || href === void 0 ? void 0 : href.startsWith("https://")) || (href === null || href === void 0 ? void 0 : href.startsWith("http://"))) {
+                    return;
+                }
+                // Click any links we haven't seen before
+                if (href && !encounteredHref.includes(href)) {
+                    toClick === null || toClick === void 0 ? void 0 : toClick.click();
+                }
+                if (href) {
+                    linksFound.push(href);
+                }
+            });
+            return linksFound;
+        }, encounteredHref // Need to pass array into the browser
+        );
+    }
+    const html = await page.content();
+    browser.close();
+    await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+    process.exit(0);
+}
+export async function scrapeSectionAutomatically(argv) {
+    const href = getHrefFromArgs(argv);
+    const res = await axios.default.get(href);
+    const html = res.data;
+    const framework = detectFramework(html);
+    validateFramework(framework);
+    console.log("Detected framework: " + framework);
+    if (framework === Frameworks.DOCUSAURUS) {
+        await scrapeSectionAxiosWrapper(argv, scrapeDocusaurusSection);
+    }
+    else if (framework === Frameworks.GITBOOK) {
+        await scrapeGitbookSectionCommand(argv);
+    }
+    else if (framework === Frameworks.README) {
+        await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
+    }
+}
+//# sourceMappingURL=scrapeSectionCommands.js.map

package/bin/scraping/scrapeSectionCommands.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAS,EAAE,UAAe;IACxE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,2BAA2B,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,IAAS,EAAE,UAAe;IACnE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,GAAG,CAAC,MAAM,CAAC,CAAC;IAE/B,0CAA0C;IAC1C,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE;QACvE,eAAe,GAAG,eAAe,CAAC;QAClC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CACnC,CAAC,eAAe,EAAE,EAAE;YAClB,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CACrC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CACrD,CAAC;YAEF,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAiB,EAAE,EAAE;;gBACxC,MAAM,OAAO,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,aAAa,0CAAE,aAAa,CAAC;gBACnD,MAAM,IAAI,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,aAAa,0CAAE,aAAa,CAAC;gBAEnD,wCAAwC;gBACxC,IAAI,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,YAAY,CAAC,MAAM,CAAC,CAAA,EAAE;oBAC/B,OAAO;iBACR;gBAED,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEvC,mDAAmD;gBACnD,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,UAAU,CAAC,MAAI,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,SAAS,CAAC,CAAA,EAAE;oBAC/D,OAAO;iBACR;gBAED,yCAAyC;gBACzC,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAC3C,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,EAAE,CAAC;iBAClB;gBACD,IAAI,IAAI,EAAE;oBACR,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACvB;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,UAAU,CAAC;QACpB,CAAC,EACD,eAAe,CAAC,sCAAsC;SACvD,CAAC;KACH;IAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAExC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,yBAAyB,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;KAChE;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;KAC5D;AACH,CAAC"}

package/bin/scraping/site-scrapers/getLinksRecursively.js

@@ -0,0 +1,33 @@
+// Used by GitBook and ReadMe section scrapers
+export default function getLinksRecursively(linkSections, $) {
+    if (linkSections == null || linkSections.length === 0) {
+        return [];
+    }
+    return linkSections
+        .map((i, s) => {
+        const subsection = $(s);
+        const link = subsection.children().first();
+        const linkHref = link.attr("href");
+        // Skip missing links. For example, GitBook uses
+        // empty divs are used for styling a line beside the nav.
+        // Skip external links until Mintlify supports them
+        if (!linkHref ||
+            linkHref.startsWith("https://") ||
+            linkHref.startsWith("http://")) {
+            return undefined;
+        }
+        const childLinks = subsection.children().eq(1).children();
+        if (childLinks.length > 0) {
+            // Put the section link in the list of pages.
+            // When we support the section itself being a link we should update this
+            return {
+                group: link.text(),
+                pages: [linkHref, ...getLinksRecursively(childLinks, $)],
+            };
+        }
+        return linkHref;
+    })
+        .toArray()
+        .filter(Boolean);
+}
+//# sourceMappingURL=getLinksRecursively.js.map

package/bin/scraping/site-scrapers/getLinksRecursively.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/getLinksRecursively.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAC9C,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,YAAiB,EAAE,CAAM;IACnE,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QACrD,OAAO,EAAE,CAAC;KACX;IAED,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;YACA,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aACzD,CAAC;SACH;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js

@@ -42,7 +42,7 @@ export async function scrapeDocusaurusSection(html, origin, cliDir, overwrite) {
         groupConfig.pages = (await Promise.all(groupConfig.pages.map(async (pathname) => 
         // Docusaurus requires a directory on all sections wheras we use root.
         // /docs is their default directory so we remove it
-        scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapeDocusaurusPage, false, "/docs"))))
+        scrapeGettingFileNameFromUrl(pathname, cliDir, origin, overwrite, scrapeDocusaurusPage, false, "/docs"))))
             // Remove skipped index pages (they return undefined from the above function)
             .filter(Boolean);
         return groupConfig;

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,gBAAgB,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAEpC,wBAAwB;QACxB,IAAI,gBAAgB,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YAC5D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvE,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,YAAY,GAAG,gBAAgB;aAClC,IAAI,CAAC,8BAA8B,CAAC;aACpC,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,2DAA2D;QAC3D,kEAAkE;QAClE,MAAM,SAAS,GAAG,gBAAgB;aAC/B,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC;aACjC,OAAO,EAAE,CAAC;QAEb,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QAC/C,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
+{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,gBAAgB,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAEpC,wBAAwB;QACxB,IAAI,gBAAgB,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YAC5D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvE,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,YAAY,GAAG,gBAAgB;aAClC,IAAI,CAAC,8BAA8B,CAAC;aACpC,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,2DAA2D;QAC3D,kEAAkE;QAClE,MAAM,SAAS,GAAG,gBAAgB;aAC/B,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC;aACjC,OAAO,EAAE,CAAC;QAEb,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QAC/C,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookSection.js

@@ -1,7 +1,8 @@
 import cheerio from "cheerio";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
-import { getSitemapLinks } from "../getSitemapLinks.js";
 import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
+import getLinksRecursively from "./getLinksRecursively.js";
 export async function scrapeGitBookSection(html, origin, cliDir, overwrite) {
     const $ = cheerio.load(html);
     // Get all the navigation sections
@@ -12,63 +13,39 @@ export async function scrapeGitBookSection(html, origin, cliDir, overwrite) {
         .first()
         .children();
     // Get all links per group
-    let allNavPathnames = [];
     const groupsConfig = navigationSections
-        .map((i, section) => {
+        .map((i, s) => {
+        const section = $(s);
         const sectionTitle = $(section)
             .find('div > div[dir="auto"]')
             .first()
             .text();
-        const linkPaths = $(section)
-            .find("a[href]")
-            .map((i, link) => {
-            const linkHref = $(link).attr("href");
-            // Skip external links until Mintlify supports them
-            if (linkHref.startsWith("https://") ||
-                linkHref.startsWith("http://")) {
-                return undefined;
-            }
-            return linkHref;
-        })
-            .toArray();
-        allNavPathnames = allNavPathnames.concat(linkPaths);
-        // Follows the same structure as mint.json
+        // Only present if the nested navigation is not in a group
+        const firstLink = section.children().eq(0);
+        const firstHref = firstLink.attr("href");
+        const linkSections = section.children().eq(1).children();
+        const pages = getLinksRecursively(linkSections, $);
         return {
-            group: sectionTitle,
-            pages: linkPaths,
+            group: sectionTitle || alternateTitle(firstLink, pages),
+            pages: firstHref ? [firstHref, ...pages] : pages,
         };
     })
-        .toArray();
-    // Scrape every link not in the navigation. Nested docs
-    // don't show up in navigation without clicking buttons,
-    // so this lets us download the files for the user to add
-    // manually to mint.json.
-    const sitemapPaths = (await getSitemapLinks(new URL("sitemap.xml", origin)))
-        .map((sitemapLinks) => {
-        return new URL(sitemapLinks).pathname;
-    })
-        .filter((pathname) => !allNavPathnames.includes(pathname));
-    const sitemapPathnamesForConfig = [];
-    for (const pathname of sitemapPaths) {
-        sitemapPathnamesForConfig.push(await scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapeGitBookPage, true));
-    }
+        .toArray()
+        .filter(Boolean);
+    // Merge groups with empty titles together
+    const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
     // Scrape each link in the navigation.
-    const groupsConfigCleanPaths = await Promise.all(groupsConfig.map(async (groupConfig) => {
-        const newPages = [];
-        for (const pathname of groupConfig.pages) {
-            newPages.push(await scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapeGitBookPage, true));
-        }
-        groupConfig.pages = newPages;
-        return groupConfig;
+    const groupsConfigCleanPaths = await Promise.all(reducedGroupsConfig.map(async (navEntry) => {
+        return await scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapeGitBookPage, true);
     }));
-    if (sitemapPathnamesForConfig.length > 0) {
-        return groupsConfigCleanPaths.concat([
-            {
-                group: "ATTENTION! WE CANNOT DETECT GROUPS FOR NESTED DOCS. PLEASE MOVE THEM INTO THEIR ORIGINAL GROUPS.",
-                pages: sitemapPathnamesForConfig,
-            },
-        ]);
-    }
     return groupsConfigCleanPaths;
 }
+function alternateTitle(firstLink, pages) {
+    // Only assign titles to nested navigation menus outside a section.
+    // Others should not have a title so we can merge them into one section.
+    if (pages.length > 0) {
+        return firstLink === null || firstLink === void 0 ? void 0 : firstLink.text();
+    }
+    return "";
+}
 //# sourceMappingURL=scrapeGitBookSection.js.map

package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC;aACzB,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEtC,mDAAmD;YACnD,IACE,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;gBACA,OAAO,SAAS,CAAC;aAClB;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEpD,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,uDAAuD;IACvD,wDAAwD;IACxD,yDAAyD;IACzD,yBAAyB;IACzB,MAAM,YAAY,GAAG,CAAC,MAAM,eAAe,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;SACzE,GAAG,CAAC,CAAC,YAAoB,EAAE,EAAE;QAC5B,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAErE,MAAM,yBAAyB,GAAG,EAAE,CAAC;IACrC,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE;QACnC,yBAAyB,CAAC,IAAI,CAC5B,MAAM,4BAA4B,CAChC,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CACF,CAAC;KACH;IAED,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,KAAK,EAAE;YACxC,QAAQ,CAAC,IAAI,CACX,MAAM,4BAA4B,CAChC,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CACF,CAAC;SACH;QACD,WAAW,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC7B,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE;QACxC,OAAO,sBAAsB,CAAC,MAAM,CAAC;YACnC;gBACE,KAAK,EACH,kGAAkG;gBACpG,KAAK,EAAE,yBAAyB;aACjC;SACF,CAAC,CAAC;KACJ;IAED,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
+{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC;YACvD,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QAC1D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED,SAAS,cAAc,CAAC,SAAS,EAAE,KAAK;IACtC,mEAAmE;IACnE,wEAAwE;IACxE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;QACpB,OAAO,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,IAAI,EAAE,CAAC;KAC1B;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMePage.js

@@ -14,7 +14,8 @@ export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir) {
     if (content.length === 0) {
         content = $(".rm-Article > .markdown-body");
     }
-    const contentHtml = content.html();
+    // API Pages don't have a markdown body in the same position so there's no HTML
+    let contentHtml = content.html() || "";
     const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir);
     const nhm = new NodeHtmlMarkdown();
     let markdown = nhm.translate(contentHtml);

package/bin/scraping/site-scrapers/scrapeReadMePage.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB;IAEpB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEnC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB;IAEpB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IAED,+EAA+E;IAC/E,IAAI,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEvC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMeSection.js

@@ -1,6 +1,7 @@
 import cheerio from "cheerio";
 import { scrapeReadMePage } from "./scrapeReadMePage.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import getLinksRecursively from "./getLinksRecursively.js";
 export async function scrapeReadMeSection(html, origin, cliDir, overwrite) {
     const $ = cheerio.load(html);
     // Get all the navigation sections, but only from the first
@@ -10,39 +11,28 @@ export async function scrapeReadMeSection(html, origin, cliDir, overwrite) {
         .first()
         .find(".rm-Sidebar-section");
     const groupsConfig = navigationSections
-        .map((i, section) => {
-        const sectionTitle = $(section).find("h3").first().text();
+        .map((i, s) => {
+        const section = $(s);
+        const sectionTitle = section.find("h3").first().text();
         // Get all links, then use filter to remove duplicates.
         // There are duplicates because of nested navigation, eg:
         // subgroupTitle -> /first-page
         // -- First Page -> /first-page   ** DUPLICATE **
         // -- Second Page -> /second-page
-        const linkPaths = $(section)
-            .find("a[href]")
-            .map((i, link) => {
-            const linkHref = $(link).attr("href");
-            // Skip external links until Mintlify supports them
-            if (linkHref.startsWith("https://") ||
-                linkHref.startsWith("http://")) {
-                return undefined;
-            }
-            return linkHref;
-        })
-            .toArray()
-            .filter((value, index, self) => self.indexOf(value) === index);
+        const linkSections = section.find(".rm-Sidebar-list").first().children();
+        const pages = getLinksRecursively(linkSections, $).filter((value, index, self) => self.indexOf(value) === index);
         // Follows the same structure as mint.json
         return {
             group: sectionTitle,
-            pages: linkPaths,
+            pages: pages,
         };
     })
         .toArray();
-    return await Promise.all(groupsConfig.map(async (groupConfig) => {
-        groupConfig.pages = await Promise.all(groupConfig.pages.map(async (pathname) => 
+    return await Promise.all(groupsConfig.map(async (navEntry) => {
+        return await scrapeGettingFileNameFromUrl(
         // ReadMe requires a directory on all sections wheras we use root.
         // /docs is their default directory so we remove it
-        scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapeReadMePage, false, "/docs")));
-        return groupConfig;
+        navEntry, cliDir, origin, overwrite, scrapeReadMePage, false, "/docs");
     }));
 }
 //# sourceMappingURL=scrapeReadMeSection.js.map