mintlify 1.0.6 → 1.0.7

package/bin/{scrapeGitBookPage.js → scraping/site-scrapers/scrapeGitBookPage.js}

@@ -1,18 +1,22 @@
-import axios from "axios";
 import cheerio from "cheerio";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-export async function scrapeGitBookPage(url) {
-    const res = await axios.default.get(url);
-    const $ = cheerio.load(res.data);
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir) {
+    const $ = cheerio.load(html);
     const titleComponent = $('[data-testid="page.title"]').first();
     const titleAndDescription = titleComponent.parent().parent().parent().text();
-    console.log(titleAndDescription);
     const description = titleAndDescription
         .replace(titleComponent.text(), "")
         .trim();
     const title = titleComponent.text().trim();
     const content = $('[data-testid="page.contentEditor"]').first();
     const contentHtml = $.html(content);
+    const modifyFileName = (fileName) => 
+    // Remove GitBook metadata from the start
+    // The first four %2F split metadata fields. Remaining ones are part of the file name.
+    fileName.split("%2F").slice(4).join("%2F");
+    const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, modifyFileName);
     const nhm = new NodeHtmlMarkdown();
     let markdown = nhm.translate(contentHtml);
     // Keep headers on one line and increase their depth by one
@@ -23,6 +27,7 @@ export async function scrapeGitBookPage(url) {
     markdown = markdown.replace(/\n\n\n/g, "\n\n");
     // Mintlify doesn't support bolded headers, remove the asterisks
     markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
     return { title, description, markdown };
 }
 //# sourceMappingURL=scrapeGitBookPage.js.map

package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB;IAEpB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,EAAE;IAClC,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookSection.js

@@ -0,0 +1,74 @@
+import cheerio from "cheerio";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import { getSitemapLinks } from "../getSitemapLinks.js";
+import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+export async function scrapeGitBookSection(html, origin, cliDir, overwrite) {
+    const $ = cheerio.load(html);
+    // Get all the navigation sections
+    const navigationSections = $('div[data-testid="page.desktopTableOfContents"] > div > div:first-child')
+        .children()
+        .first()
+        .children()
+        .first()
+        .children();
+    // Get all links per group
+    let allNavPathnames = [];
+    const groupsConfig = navigationSections
+        .map((i, section) => {
+        const sectionTitle = $(section)
+            .find('div > div[dir="auto"]')
+            .first()
+            .text();
+        const linkPaths = $(section)
+            .find("a[href]")
+            .map((i, link) => {
+            const linkHref = $(link).attr("href");
+            // Skip external links until Mintlify supports them
+            if (linkHref.startsWith("https://") ||
+                linkHref.startsWith("http://")) {
+                return undefined;
+            }
+            return linkHref;
+        })
+            .toArray();
+        allNavPathnames = allNavPathnames.concat(linkPaths);
+        // Follows the same structure as mint.json
+        return {
+            group: sectionTitle,
+            pages: linkPaths,
+        };
+    })
+        .toArray();
+    // Scrape every link not in the navigation. Nested docs
+    // don't show up in navigation without clicking buttons,
+    // so this lets us download the files for the user to add
+    // manually to mint.json.
+    const sitemapPaths = (await getSitemapLinks(new URL("sitemap.xml", origin)))
+        .map((sitemapLinks) => {
+        return new URL(sitemapLinks).pathname;
+    })
+        .filter((pathname) => !allNavPathnames.includes(pathname));
+    const sitemapPathnamesForConfig = [];
+    for (const pathname of sitemapPaths) {
+        sitemapPathnamesForConfig.push(await scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapeGitBookPage, true));
+    }
+    // Scrape each link in the navigation.
+    const groupsConfigCleanPaths = await Promise.all(groupsConfig.map(async (groupConfig) => {
+        const newPages = [];
+        for (const pathname of groupConfig.pages) {
+            newPages.push(await scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapeGitBookPage, true));
+        }
+        groupConfig.pages = newPages;
+        return groupConfig;
+    }));
+    if (sitemapPathnamesForConfig.length > 0) {
+        return groupsConfigCleanPaths.concat([
+            {
+                group: "ATTENTION! WE CANNOT DETECT GROUPS FOR NESTED DOCS. PLEASE MOVE THEM INTO THEIR ORIGINAL GROUPS.",
+                pages: sitemapPathnamesForConfig,
+            },
+        ]);
+    }
+    return groupsConfigCleanPaths;
+}
+//# sourceMappingURL=scrapeGitBookSection.js.map

package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC;aACzB,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEtC,mDAAmD;YACnD,IACE,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;gBACA,OAAO,SAAS,CAAC;aAClB;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEpD,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,uDAAuD;IACvD,wDAAwD;IACxD,yDAAyD;IACzD,yBAAyB;IACzB,MAAM,YAAY,GAAG,CAAC,MAAM,eAAe,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;SACzE,GAAG,CAAC,CAAC,YAAoB,EAAE,EAAE;QAC5B,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAErE,MAAM,yBAAyB,GAAG,EAAE,CAAC;IACrC,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE;QACnC,yBAAyB,CAAC,IAAI,CAC5B,MAAM,4BAA4B,CAChC,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CACF,CAAC;KACH;IAED,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,KAAK,EAAE;YACxC,QAAQ,CAAC,IAAI,CACX,MAAM,4BAA4B,CAChC,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CACF,CAAC;SACH;QACD,WAAW,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC7B,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE;QACxC,OAAO,sBAAsB,CAAC,MAAM,CAAC;YACnC;gBACE,KAAK,EACH,kGAAkG;gBACpG,KAAK,EAAE,yBAAyB;aACjC;SACF,CAAC,CAAC;KACJ;IAED,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/bin/{scrapeReadMePage.js → scraping/site-scrapers/scrapeReadMePage.js}

@@ -1,16 +1,21 @@
-import axios from "axios";
 import cheerio from "cheerio";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-export async function scrapeReadMePage(url) {
-    const res = await axios.default.get(url);
-    const $ = cheerio.load(res.data);
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir) {
+    const $ = cheerio.load(html);
     const titleComponent = $("h1").first();
     const title = titleComponent.text().trim();
-    const description = $(".markdown-body", titleComponent.parent())
-        .text()
-        .trim();
-    const content = $(".content-body .markdown-body").first();
-    const contentHtml = $.html(content);
+    let description = $(".markdown-body", titleComponent.parent()).text().trim();
+    if (!description) {
+        description = $(".rm-Article > header p").text().trim();
+    }
+    let content = $(".content-body .markdown-body").first();
+    if (content.length === 0) {
+        content = $(".rm-Article > .markdown-body");
+    }
+    const contentHtml = content.html();
+    const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir);
     const nhm = new NodeHtmlMarkdown();
     let markdown = nhm.translate(contentHtml);
     // Keep headers on one line and increase their depth by one
@@ -23,6 +28,7 @@ export async function scrapeReadMePage(url) {
     markdown = markdown.replace(/\n\n\n/g, "\n\n");
     // Mintlify doesn't support bolded headers, remove the asterisks
     markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
     return { title, description, markdown };
 }
 //# sourceMappingURL=scrapeReadMePage.js.map

package/bin/scraping/site-scrapers/scrapeReadMePage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB;IAEpB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEnC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMeSection.js

@@ -0,0 +1,48 @@
+import cheerio from "cheerio";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+export async function scrapeReadMeSection(html, origin, cliDir, overwrite) {
+    const $ = cheerio.load(html);
+    // Get all the navigation sections, but only from the first
+    // sidebar found. There are multiple in the HTML for mobile
+    // responsiveness but they all have the same links.
+    const navigationSections = $(".rm-Sidebar")
+        .first()
+        .find(".rm-Sidebar-section");
+    const groupsConfig = navigationSections
+        .map((i, section) => {
+        const sectionTitle = $(section).find("h3").first().text();
+        // Get all links, then use filter to remove duplicates.
+        // There are duplicates because of nested navigation, eg:
+        // subgroupTitle -> /first-page
+        // -- First Page -> /first-page   ** DUPLICATE **
+        // -- Second Page -> /second-page
+        const linkPaths = $(section)
+            .find("a[href]")
+            .map((i, link) => {
+            const linkHref = $(link).attr("href");
+            // Skip external links until Mintlify supports them
+            if (linkHref.startsWith("https://") ||
+                linkHref.startsWith("http://")) {
+                return undefined;
+            }
+            return linkHref;
+        })
+            .toArray()
+            .filter((value, index, self) => self.indexOf(value) === index);
+        // Follows the same structure as mint.json
+        return {
+            group: sectionTitle,
+            pages: linkPaths,
+        };
+    })
+        .toArray();
+    return await Promise.all(groupsConfig.map(async (groupConfig) => {
+        groupConfig.pages = await Promise.all(groupConfig.pages.map(async (pathname) => 
+        // ReadMe requires a directory on all sections wheras we use root.
+        // /docs is their default directory so we remove it
+        scrapeGettingFileNameFromUrl(cliDir, origin, pathname, overwrite, scrapeReadMePage, false, "/docs")));
+        return groupConfig;
+    }));
+}
+//# sourceMappingURL=scrapeReadMeSection.js.map

package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAElF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,2DAA2D;IAC3D,2DAA2D;IAC3D,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC;SACxC,KAAK,EAAE;SACP,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAE/B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAE1D,uDAAuD;QACvD,yDAAyD;QACzD,+BAA+B;QAC/B,iDAAiD;QACjD,iCAAiC;QACjC,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC;aACzB,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEtC,mDAAmD;YACnD,IACE,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;gBACA,OAAO,SAAS,CAAC;aAClB;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC;aACD,OAAO,EAAE;aACT,MAAM,CACL,CAAC,KAAa,EAAE,KAAa,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CACtE,CAAC;QAEJ,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,WAAW,CAAC,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QAC/C,kEAAkE;QAClE,mDAAmD;QACnD,4BAA4B,CAC1B,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,gBAAgB,EAChB,KAAK,EACL,OAAO,CACR,CACF,CACF,CAAC;QACF,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/bin/util.js

@@ -1,11 +1,10 @@
 import { mkdirSync, writeFileSync } from "fs";
-import { Page } from "./templates.js";
 import path from "path";
-export function getWebsite(url) {
-    // Gets the website
+import { Page } from "./templates.js";
+export function getOrigin(url) {
     // eg. https://google.com -> https://google.com
     // https://google.com/page -> https://google.com
-    return url.split("/").slice(0, 3).join("/");
+    return new URL(url).origin;
 }
 export function objToReadableString(objs) {
     // Two spaces as indentation
@@ -26,12 +25,32 @@ export const addMdx = (fileName) => {
     }
     return fileName + ".mdx";
 };
-export const createPage = (title, description, markdown, rootDir = "", fileName) => {
+export const createPage = (title, description, markdown, overwrite = false, rootDir = "", fileName) => {
+    const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
     // Create the folders needed if they're missing
     mkdirSync(rootDir, { recursive: true });
     // Write the page to memory
-    const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
-    writeFileSync(writePath, Page(title, description, markdown));
-    console.log("✏️ - " + writePath);
+    if (overwrite) {
+        writeFileSync(writePath, Page(title, description, markdown));
+        console.log("✏️ - " + writePath);
+    }
+    else {
+        try {
+            writeFileSync(writePath, Page(title, description, markdown), {
+                flag: "wx",
+            });
+            console.log("✏️ - " + writePath);
+        }
+        catch (e) {
+            // We do a try-catch instead of an if-statement to avoid a race condition
+            // of the file being created after we started writing.
+            if (e.code === "EEXIST") {
+                console.log(`❌ Skipping existing file ${writePath}`);
+            }
+            else {
+                console.error(e);
+            }
+        }
+    }
 };
 //# sourceMappingURL=util.js.map

package/bin/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,mBAAmB;IACnB,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAc;IAChD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,UAAkB,EAAE,EACpB,QAAiB,EACjB,EAAE;IACF,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5E,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;AACnC,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAc;IAChD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,YAAqB,KAAK,EAC1B,UAAkB,EAAE,EACpB,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE;QACb,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;KAClC;SAAM;QACL,IAAI;YACF,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;SAClC;QAAC,OAAO,CAAC,EAAE;YACV,yEAAyE;YACzE,sDAAsD;YACtD,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;gBACvB,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;aACtD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;SACF;KACF;AACH,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mintlify",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Mintlify CLI",
   "engines": {
     "node": ">=14.16"
@@ -19,8 +19,9 @@
     "axios": "^0.27.2",
     "cheerio": "^0.22.0",
     "inquirer": "^9.1.0",
+    "minimist-lite": "^2.2.1",
     "node-html-markdown": "^1.2.0",
-    "puppeteer": "^17.1.2"
+    "puppeteer": "^17.1.3"
   },
   "devDependencies": {
     "@types/inquirer": "^9.0.1",

package/src/browser.ts

@@ -0,0 +1,24 @@
+import { launch } from "puppeteer";
+
+export async function startBrowser() {
+  try {
+    return await launch({
+      headless: true,
+      ignoreHTTPSErrors: true,
+    });
+  } catch (err) {
+    console.log("Could not create a browser instance: ", err);
+    process.exit(1);
+  }
+}
+
+export async function getHtmlWithPuppeteer(href: string) {
+  const browser = await startBrowser();
+  const page = await browser.newPage();
+  await page.goto(href, {
+    waitUntil: "networkidle2",
+  });
+  const html = await page.content();
+  browser.close();
+  return html;
+}

package/src/downloadImage.ts

@@ -0,0 +1,35 @@
+import { existsSync, mkdirSync, createWriteStream } from "fs";
+import path from "path";
+import axios from "axios";
+
+export default async function downloadImage(
+  imageSrc: string,
+  writePath: string
+) {
+  // Avoid unnecessary downloads
+  if (existsSync(writePath)) {
+    return Promise.reject({
+      code: "EEXIST",
+    });
+  }
+
+  // Create the folders needed if they're missing
+  mkdirSync(path.dirname(writePath), { recursive: true });
+
+  const writer = createWriteStream(writePath);
+
+  const response = await axios.default.get(imageSrc, {
+    responseType: "stream",
+  });
+
+  // wx prevents overwriting an image with the exact same name
+  // being created in the time we were downloading
+  response.data.pipe(writer, {
+    flag: "wx",
+  });
+
+  return new Promise((resolve, reject) => {
+    writer.on("finish", resolve);
+    writer.on("error", reject);
+  });
+}

package/src/index.ts

@@ -1,23 +1,37 @@
 #! /usr/bin/env node
 
+import axios from "axios";
 import { writeFileSync } from "fs";
 import inquirer from "inquirer";
+import minimistLite from "minimist-lite";
 import { MintConfig } from "./templates.js";
-import { createPage, toFilename, objToReadableString } from "./util.js";
-import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
-import { scrapeReadMePage } from "./scrapeReadMePage.js";
-import { scrapeReadMe } from "./scrapeReadMe.js";
+import { scrapePage } from "./scraping/scrapePage.js";
+import { scrapeSection } from "./scraping/scrapeSection.js";
+import { createPage, toFilename, getOrigin } from "./util.js";
+import { scrapeDocusaurusPage } from "./scraping/site-scrapers/scrapeDocusaurusPage.js";
+import { scrapeDocusaurusSection } from "./scraping/site-scrapers/scrapeDocusaurusSection.js";
+import { scrapeGitBookPage } from "./scraping/site-scrapers/scrapeGitBookPage.js";
+import { scrapeGitBookSection } from "./scraping/site-scrapers/scrapeGitBookSection.js";
+import { scrapeReadMePage } from "./scraping/site-scrapers/scrapeReadMePage.js";
+import { scrapeReadMeSection } from "./scraping/site-scrapers/scrapeReadMeSection.js";
+import { detectFramework, Frameworks } from "./scraping/detectFramework.js";
+import { startBrowser, getHtmlWithPuppeteer } from "./browser.js";
 
-const args = process.argv.slice(2);
+const argv = minimistLite(process.argv.slice(2), {
+  boolean: ["overwrite"],
+  default: {
+    overwrite: false,
+  },
+});
 
-if (args.length === 0) {
+if (argv._.length === 0) {
   console.error(
     `No command specified. Here are is the list that you can use:\ninit: initialize a Mintlify documentation instance`
   );
   process.exit(1); //an error occurred
 }
 
-const command = args[0];
+const command = argv._[0];
 
 if (command === "init") {
   inquirer
@@ -55,7 +69,7 @@ if (command === "init") {
     .then((answers) => {
       const { name, color, ctaName, ctaUrl, title } = answers;
       writeFileSync(
-        "mint.config.json",
+        "mint.json",
         JSON.stringify(
           MintConfig(name, color, ctaName, ctaUrl, toFilename(title)),
           null,
@@ -100,26 +114,163 @@ if (command === "page") {
     });
 }
 
-if (command === "scrape-gitbook-page") {
-  const url = args[1];
-  const { title, description, markdown } = await scrapeGitBookPage(url);
-  createPage(title, description, markdown, process.cwd());
+function validateFramework(framework) {
+  if (!framework) {
+    console.log(
+      "Could not detect the framework automatically. Please use one of:"
+    );
+    console.log("scrape-page-docusaurus");
+    console.log("scrape-page-gitbook");
+    console.log("scrape-page-readme");
+    return process.exit(1);
+  }
+}
+
+async function scrapePageAutomatically() {
+  const href = argv._[1];
+  const res = await axios.default.get(href);
+  const html = res.data;
+  const framework = detectFramework(html);
+
+  validateFramework(framework);
+
+  console.log("Detected framework: " + framework);
+
+  if (framework === Frameworks.DOCUSAURUS) {
+    await scrapePageWrapper(scrapeDocusaurusPage);
+  } else if (framework === Frameworks.GITBOOK) {
+    await scrapePageWrapper(scrapeGitBookPage, true);
+  } else if (framework === Frameworks.README) {
+    await scrapePageWrapper(scrapeReadMePage);
+  }
+}
+
+async function scrapePageWrapper(scrapeFunc, puppeteer = false) {
+  const href = argv._[1];
+  let html;
+  if (puppeteer) {
+    html = await getHtmlWithPuppeteer(href);
+  } else {
+    const res = await axios.default.get(href);
+    html = res.data;
+  }
+  await scrapePage(scrapeFunc, href, html, argv.overwrite);
   process.exit(1);
 }
 
+if (command === "scrape-page") {
+  await scrapePageAutomatically();
+}
+
+if (command === "scrape-docusaurus-page") {
+  await scrapePageWrapper(scrapeDocusaurusPage);
+}
+
+if (command === "scrape-gitbook-page") {
+  await scrapePageWrapper(scrapeGitBookPage, true);
+}
+
 if (command === "scrape-readme-page") {
-  const url = args[1];
-  const { title, description, markdown } = await scrapeReadMePage(url);
-  createPage(title, description, markdown, process.cwd());
+  await scrapePageWrapper(scrapeReadMePage);
+}
+
+async function scrapeSectionAutomatically() {
+  const href = argv._[1];
+  const res = await axios.default.get(href);
+  const html = res.data;
+  const framework = detectFramework(html);
+
+  validateFramework(framework);
+
+  console.log("Detected framework: " + framework);
+
+  if (framework === Frameworks.DOCUSAURUS) {
+    await scrapeSectionAxiosWrapper(scrapeDocusaurusSection);
+  } else if (framework === Frameworks.GITBOOK) {
+    await scrapeSectionGitBookWrapper(scrapeGitBookSection);
+  } else if (framework === Frameworks.README) {
+    await scrapeSectionAxiosWrapper(scrapeReadMeSection);
+  }
+}
+
+async function scrapeSectionAxiosWrapper(scrapeFunc: any) {
+  const href = argv._[1];
+  const res = await axios.default.get(href);
+  const html = res.data;
+  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
   process.exit(1);
 }
 
-if (command === "scrape-readme-section") {
-  const url = args[1];
-  console.log("Started scraping...");
-  const groupsConfig = await scrapeReadMe(url, process.cwd());
-  console.log("Finished scraping.");
-  console.log("Add the following to your navigation in mint.config.js:");
-  console.log(objToReadableString(groupsConfig));
+async function scrapeSectionGitBookWrapper(scrapeFunc: any) {
+  const href = argv._[1];
+
+  const browser = await startBrowser();
+  const page = await browser.newPage();
+  await page.goto(href, {
+    waitUntil: "networkidle2",
+  });
+
+  let prevEncountered = [];
+  let encounteredHref = ["fake"];
+
+  // Loop until we've encountered every link
+  while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+    prevEncountered = encounteredHref;
+    encounteredHref = await page.evaluate(
+      (encounteredHref) => {
+        const icons = Array.from(
+          document.querySelectorAll('path[d="M9 18l6-6-6-6"]')
+        );
+
+        const linksFound = [];
+        icons.forEach(async (icon: HTMLElement) => {
+          const toClick = icon.parentElement.parentElement;
+          const link = toClick.parentElement.parentElement;
+
+          // Skip icons not in the side navigation
+          if (!link.hasAttribute("href")) {
+            return;
+          }
+
+          const href = link.getAttribute("href");
+
+          // Should never occur but we keep it as a fail-safe
+          if (href.startsWith("https://") || href.startsWith("http://")) {
+            return;
+          }
+
+          // Click any links we haven't seen before
+          if (!encounteredHref.includes(href)) {
+            toClick.click();
+          }
+
+          linksFound.push(href);
+        });
+
+        return linksFound;
+      },
+      encounteredHref // Need to pass array into the browser
+    );
+  }
+
+  const html = await page.content();
+  browser.close();
+  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
   process.exit(1);
 }
+
+if (command === "scrape-section") {
+  await scrapeSectionAutomatically();
+}
+
+if (command === "scrape-docusaurus-section") {
+  await scrapeSectionAxiosWrapper(scrapeDocusaurusSection);
+}
+
+if (command === "scrape-gitbook-section") {
+  await scrapeSectionGitBookWrapper(scrapeGitBookSection);
+}
+
+if (command === "scrape-readme-section") {
+  await scrapeSectionAxiosWrapper(scrapeReadMeSection);
+}

package/src/scraping/detectFramework.ts

@@ -0,0 +1,31 @@
+import cheerio from "cheerio";
+
+export enum Frameworks {
+  DOCUSAURUS = "DOCUSAURUS",
+  GITBOOK = "GITBOOK",
+  README = "README",
+}
+
+export function detectFramework(html) {
+  const $ = cheerio.load(html);
+  const docusaurusMeta = $('meta[name="generator"]');
+
+  if (
+    docusaurusMeta.length > 0 &&
+    docusaurusMeta.attr("content").includes("Docusaurus")
+  ) {
+    return Frameworks.DOCUSAURUS;
+  }
+
+  const isGitBook = $(".gitbook-root").length > 0;
+  if (isGitBook) {
+    return Frameworks.GITBOOK;
+  }
+
+  const isReadMe = $('meta[name="readme-deploy"]').length > 0;
+  if (isReadMe) {
+    return Frameworks.README;
+  }
+
+  return undefined;
+}