mintlify 1.0.6 → 1.0.8

package/bin/{scrapeGitBookPage.js → scraping/site-scrapers/scrapeGitBookPage.js}

@@ -1,18 +1,22 @@
-import axios from "axios";
 import cheerio from "cheerio";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-export async function scrapeGitBookPage(url) {
-    const res = await axios.default.get(url);
-    const $ = cheerio.load(res.data);
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir) {
+    const $ = cheerio.load(html);
     const titleComponent = $('[data-testid="page.title"]').first();
     const titleAndDescription = titleComponent.parent().parent().parent().text();
-    console.log(titleAndDescription);
     const description = titleAndDescription
         .replace(titleComponent.text(), "")
         .trim();
     const title = titleComponent.text().trim();
     const content = $('[data-testid="page.contentEditor"]').first();
     const contentHtml = $.html(content);
+    const modifyFileName = (fileName) => 
+    // Remove GitBook metadata from the start
+    // The first four %2F split metadata fields. Remaining ones are part of the file name.
+    fileName.split("%2F").slice(4).join("%2F");
+    const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, modifyFileName);
     const nhm = new NodeHtmlMarkdown();
     let markdown = nhm.translate(contentHtml);
     // Keep headers on one line and increase their depth by one
@@ -23,6 +27,7 @@ export async function scrapeGitBookPage(url) {
     markdown = markdown.replace(/\n\n\n/g, "\n\n");
     // Mintlify doesn't support bolded headers, remove the asterisks
     markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
     return { title, description, markdown };
 }
 //# sourceMappingURL=scrapeGitBookPage.js.map

package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB;IAEpB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,EAAE;IAClC,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookSection.js

@@ -0,0 +1,51 @@
+import cheerio from "cheerio";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+export async function scrapeGitBookSection(html, origin, cliDir, overwrite) {
+    const $ = cheerio.load(html);
+    // Get all the navigation sections
+    const navigationSections = $('div[data-testid="page.desktopTableOfContents"] > div > div:first-child')
+        .children()
+        .first()
+        .children()
+        .first()
+        .children();
+    // Get all links per group
+    const groupsConfig = navigationSections
+        .map((i, s) => {
+        const section = $(s);
+        const sectionTitle = $(section)
+            .find('div > div[dir="auto"]')
+            .first()
+            .text();
+        // Only present if the nested navigation is not in a group
+        const firstLink = section.children().eq(0);
+        const firstHref = firstLink.attr("href");
+        const linkSections = section.children().eq(1).children();
+        const pages = getLinksRecursively(linkSections, $);
+        return {
+            group: sectionTitle || alternateTitle(firstLink, pages),
+            pages: firstHref ? [firstHref, ...pages] : pages,
+        };
+    })
+        .toArray()
+        .filter(Boolean);
+    // Merge groups with empty titles together
+    const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
+    // Scrape each link in the navigation.
+    const groupsConfigCleanPaths = await Promise.all(reducedGroupsConfig.map(async (navEntry) => {
+        return await scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapeGitBookPage, true);
+    }));
+    return groupsConfigCleanPaths;
+}
+function alternateTitle(firstLink, pages) {
+    // Only assign titles to nested navigation menus outside a section.
+    // Others should not have a title so we can merge them into one section.
+    if (pages.length > 0) {
+        return firstLink === null || firstLink === void 0 ? void 0 : firstLink.text();
+    }
+    return "";
+}
+//# sourceMappingURL=scrapeGitBookSection.js.map

package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC;YACvD,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QAC1D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,CACL,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED,SAAS,cAAc,CAAC,SAAS,EAAE,KAAK;IACtC,mEAAmE;IACnE,wEAAwE;IACxE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;QACpB,OAAO,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,IAAI,EAAE,CAAC;KAC1B;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMePage.js

@@ -0,0 +1,35 @@
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir) {
+    const $ = cheerio.load(html);
+    const titleComponent = $("h1").first();
+    const title = titleComponent.text().trim();
+    let description = $(".markdown-body", titleComponent.parent()).text().trim();
+    if (!description) {
+        description = $(".rm-Article > header p").text().trim();
+    }
+    let content = $(".content-body .markdown-body").first();
+    if (content.length === 0) {
+        content = $(".rm-Article > .markdown-body");
+    }
+    // API Pages don't have a markdown body in the same position so there's no HTML
+    let contentHtml = content.html() || "";
+    const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir);
+    const nhm = new NodeHtmlMarkdown();
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line and increase their depth by one
+    markdown = markdown.replace(/# \n\n/g, "## ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Remove ReadMe anchor links
+    markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeReadMePage.js.map

package/bin/scraping/site-scrapers/scrapeReadMePage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB;IAEpB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IAED,+EAA+E;IAC/E,IAAI,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEvC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMeSection.js

@@ -0,0 +1,38 @@
+import cheerio from "cheerio";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+export async function scrapeReadMeSection(html, origin, cliDir, overwrite) {
+    const $ = cheerio.load(html);
+    // Get all the navigation sections, but only from the first
+    // sidebar found. There are multiple in the HTML for mobile
+    // responsiveness but they all have the same links.
+    const navigationSections = $(".rm-Sidebar")
+        .first()
+        .find(".rm-Sidebar-section");
+    const groupsConfig = navigationSections
+        .map((i, s) => {
+        const section = $(s);
+        const sectionTitle = section.find("h3").first().text();
+        // Get all links, then use filter to remove duplicates.
+        // There are duplicates because of nested navigation, eg:
+        // subgroupTitle -> /first-page
+        // -- First Page -> /first-page   ** DUPLICATE **
+        // -- Second Page -> /second-page
+        const linkSections = section.find(".rm-Sidebar-list").first().children();
+        const pages = getLinksRecursively(linkSections, $).filter((value, index, self) => self.indexOf(value) === index);
+        // Follows the same structure as mint.json
+        return {
+            group: sectionTitle,
+            pages: pages,
+        };
+    })
+        .toArray();
+    return await Promise.all(groupsConfig.map(async (navEntry) => {
+        return await scrapeGettingFileNameFromUrl(
+        // ReadMe requires a directory on all sections wheras we use root.
+        // /docs is their default directory so we remove it
+        navEntry, cliDir, origin, overwrite, scrapeReadMePage, false, "/docs");
+    }));
+}
+//# sourceMappingURL=scrapeReadMeSection.js.map

package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAG3D,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB;IAElB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,2DAA2D;IAC3D,2DAA2D;IAC3D,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC;SACxC,KAAK,EAAE;SACP,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAE/B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAEvD,uDAAuD;QACvD,yDAAyD;QACzD,+BAA+B;QAC/B,iDAAiD;QACjD,iCAAiC;QACjC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,MAAM,CACvD,CAAC,KAAa,EAAE,KAAa,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CACtE,CAAC;QAEF,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,KAAK;SACb,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QACnD,OAAO,MAAM,4BAA4B;QACvC,kEAAkE;QAClE,mDAAmD;QACnD,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,gBAAgB,EAChB,KAAK,EACL,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/bin/util.js

@@ -1,11 +1,12 @@
 import { mkdirSync, writeFileSync } from "fs";
-import { Page } from "./templates.js";
+import Ora from "ora";
 import path from "path";
-export function getWebsite(url) {
-    // Gets the website
+import { Page } from "./init-command/templates.js";
+import stopIfInvalidLink from "./validation/stopIfInvalidLink.js";
+export function getOrigin(url) {
     // eg. https://google.com -> https://google.com
     // https://google.com/page -> https://google.com
-    return url.split("/").slice(0, 3).join("/");
+    return new URL(url).origin;
 }
 export function objToReadableString(objs) {
     // Two spaces as indentation
@@ -26,12 +27,50 @@ export const addMdx = (fileName) => {
     }
     return fileName + ".mdx";
 };
-export const createPage = (title, description, markdown, rootDir = "", fileName) => {
+export const createPage = (title, description, markdown, overwrite = false, rootDir = "", fileName) => {
+    const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
     // Create the folders needed if they're missing
     mkdirSync(rootDir, { recursive: true });
     // Write the page to memory
-    const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
-    writeFileSync(writePath, Page(title, description, markdown));
-    console.log("✏️ - " + writePath);
+    if (overwrite) {
+        writeFileSync(writePath, Page(title, description, markdown));
+        console.log("✏️ - " + writePath);
+    }
+    else {
+        try {
+            writeFileSync(writePath, Page(title, description, markdown), {
+                flag: "wx",
+            });
+            console.log("✏️ - " + writePath);
+        }
+        catch (e) {
+            // We do a try-catch instead of an if-statement to avoid a race condition
+            // of the file being created after we started writing.
+            if (e.code === "EEXIST") {
+                console.log(`❌ Skipping existing file ${writePath}`);
+            }
+            else {
+                console.error(e);
+            }
+        }
+    }
+};
+export function getHrefFromArgs(argv) {
+    const href = argv.url;
+    stopIfInvalidLink(href);
+    return href;
+}
+export const buildLogger = (startText = "") => {
+    const logger = Ora().start(startText);
+    return logger;
+};
+export const getFileExtension = (filename) => {
+    return (filename.substring(filename.lastIndexOf(".") + 1, filename.length) ||
+        filename);
+};
+export const fileBelongsInPagesFolder = (filename) => {
+    const extension = getFileExtension(filename);
+    return (extension &&
+        (extension === "mdx" || extension === "md" || extension === "tsx"));
 };
 //# sourceMappingURL=util.js.map

package/bin/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,mBAAmB;IACnB,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAc;IAChD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,UAAkB,EAAE,EACpB,QAAiB,EACjB,EAAE;IACF,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5E,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;AACnC,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,6BAA6B,CAAC;AACnD,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAElE,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAc;IAChD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,YAAqB,KAAK,EAC1B,UAAkB,EAAE,EACpB,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE;QACb,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;KAClC;SAAM;QACL,IAAI;YACF,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;SAClC;QAAC,OAAO,CAAC,EAAE;YACV,yEAAyE;YACzE,sDAAsD;YACtD,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;gBACvB,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;aACtD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;SACF;KACF;AACH,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,IAAS;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;IACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,YAAoB,EAAE,EAAE,EAAE;IACpD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,OAAO,CACL,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC;QAClE,QAAQ,CACT,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,CACL,SAAS;QACT,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CACnE,CAAC;AACJ,CAAC,CAAC"}

package/bin/validation/isValidLink.js

@@ -0,0 +1,11 @@
+export default function isValidLink(href) {
+    // This checks the link is written correctly, not that the page exists.
+    try {
+        new URL(href);
+        return true;
+    }
+    catch (_) {
+        return false;
+    }
+}
+//# sourceMappingURL=isValidLink.js.map

package/bin/validation/isValidLink.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"isValidLink.js","sourceRoot":"","sources":["../../src/validation/isValidLink.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,WAAW,CAAC,IAAY;IAC9C,uEAAuE;IACvE,IAAI;QACF,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;KACb;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,KAAK,CAAC;KACd;AACH,CAAC"}

package/bin/validation/stopIfInvalidLink.js

@@ -0,0 +1,9 @@
+import isValidLink from "./isValidLink.js";
+export default function stopIfInvalidLink(href) {
+    if (!isValidLink(href)) {
+        console.log("Invalid link: " + href);
+        console.log("Make sure the link starts with http:// or https://");
+        process.exit(1);
+    }
+}
+//# sourceMappingURL=stopIfInvalidLink.js.map

package/bin/validation/stopIfInvalidLink.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopIfInvalidLink.js","sourceRoot":"","sources":["../../src/validation/stopIfInvalidLink.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C,MAAM,CAAC,OAAO,UAAU,iBAAiB,CAAC,IAAY;IACpD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC"}

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "mintlify",
-  "version": "1.0.6",
-  "description": "Mintlify CLI",
+  "version": "1.0.8",
+  "description": "The Mintlify CLI",
   "engines": {
     "node": ">=14.16"
   },
   "scripts": {
-    "build": "tsc"
+    "build": "tsc",
+    "local": "npm uninstall -g mintlify && npm i -g ."
   },
   "author": "Mintlify, Inc.",
   "license": "ISC",
@@ -16,15 +17,31 @@
   },
   "type": "module",
   "dependencies": {
+    "@apidevtools/swagger-parser": "^10.1.0",
     "axios": "^0.27.2",
+    "chalk": "^5.1.0",
     "cheerio": "^0.22.0",
+    "chokidar": "^3.5.3",
+    "favicons": "^7.0.1",
+    "fs-extra": "^10.1.0",
+    "gray-matter": "^4.0.3",
     "inquirer": "^9.1.0",
+    "is-internet-available": "^3.1.0",
+    "minimist-lite": "^2.2.1",
     "node-html-markdown": "^1.2.0",
-    "puppeteer": "^17.1.2"
+    "open": "^8.4.0",
+    "openapi-types": "^12.0.2",
+    "ora": "^6.1.2",
+    "puppeteer": "^17.1.3",
+    "shelljs": "^0.8.5",
+    "yargs": "^17.6.0"
   },
   "devDependencies": {
+    "@types/fs-extra": "^9.0.13",
     "@types/inquirer": "^9.0.1",
     "@types/node": "^18.7.13",
+    "@types/shelljs": "^0.8.11",
+    "@types/yargs": "^17.0.13",
     "typescript": "^4.8.2"
   }
 }

package/src/browser.ts

@@ -0,0 +1,24 @@
+import { launch } from "puppeteer";
+
+export async function startBrowser() {
+  try {
+    return await launch({
+      headless: true,
+      ignoreHTTPSErrors: true,
+    });
+  } catch (err) {
+    console.log("Could not create a browser instance: ", err);
+    process.exit(1);
+  }
+}
+
+export async function getHtmlWithPuppeteer(href: string) {
+  const browser = await startBrowser();
+  const page = await browser.newPage();
+  await page.goto(href, {
+    waitUntil: "networkidle2",
+  });
+  const html = await page.content();
+  browser.close();
+  return html;
+}

package/src/constants.ts

@@ -0,0 +1,10 @@
+import path from "path";
+import * as url from "url";
+
+// package installation location
+export const INSTALL_PATH = url.fileURLToPath(new URL(".", import.meta.url));
+
+// command execution location
+export const CMD_EXEC_PATH = process.cwd();
+
+export const CLIENT_PATH = path.join(INSTALL_PATH, "mint", "client");

package/src/downloadImage.ts

@@ -0,0 +1,35 @@
+import { existsSync, mkdirSync, createWriteStream } from "fs";
+import path from "path";
+import axios from "axios";
+
+export default async function downloadImage(
+  imageSrc: string,
+  writePath: string
+) {
+  // Avoid unnecessary downloads
+  if (existsSync(writePath)) {
+    return Promise.reject({
+      code: "EEXIST",
+    });
+  }
+
+  // Create the folders needed if they're missing
+  mkdirSync(path.dirname(writePath), { recursive: true });
+
+  const writer = createWriteStream(writePath);
+
+  const response = await axios.default.get(imageSrc, {
+    responseType: "stream",
+  });
+
+  // wx prevents overwriting an image with the exact same name
+  // being created in the time we were downloading
+  response.data.pipe(writer, {
+    flag: "wx",
+  });
+
+  return new Promise((resolve, reject) => {
+    writer.on("finish", resolve);
+    writer.on("error", reject);
+  });
+}

package/src/index.ts

@@ -1,125 +1,114 @@
-#! /usr/bin/env node
+#!/usr/bin/env node
 
-import { writeFileSync } from "fs";
-import inquirer from "inquirer";
-import { MintConfig } from "./templates.js";
-import { createPage, toFilename, objToReadableString } from "./util.js";
-import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
-import { scrapeReadMePage } from "./scrapeReadMePage.js";
-import { scrapeReadMe } from "./scrapeReadMe.js";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import initCommand from "./init-command/index.js";
+import generatePageTemplate from "./pageTemplate.js";
+import {
+  scrapePageAutomatically,
+  scrapePageWrapper,
+} from "./scraping/scrapePageCommands.js";
+import { scrapeDocusaurusPage } from "./scraping/site-scrapers/scrapeDocusaurusPage.js";
+import { scrapeGitBookPage } from "./scraping/site-scrapers/scrapeGitBookPage.js";
+import { scrapeReadMePage } from "./scraping/site-scrapers/scrapeReadMePage.js";
+import {
+  scrapeSectionAutomatically,
+  scrapeSectionAxiosWrapper,
+  scrapeGitbookSectionCommand,
+} from "./scraping/scrapeSectionCommands.js";
+import { scrapeDocusaurusSection } from "./scraping/site-scrapers/scrapeDocusaurusSection.js";
+import { scrapeReadMeSection } from "./scraping/site-scrapers/scrapeReadMeSection.js";
+import dev from "./local-preview/index.js";
 
-const args = process.argv.slice(2);
+// TODO - add descriptions to the command options https://github.com/yargs/yargs/blob/HEAD/docs/api.md#commandmodule
+yargs(hideBin(process.argv))
+  .command(
+    "dev",
+    "Runs Mintlify locally (Must run in directory with mint.json)",
+    () => {},
+    async () => {
+      await dev();
+    }
+  )
+  .command(
+    "init",
+    "Generate a mintlify template",
+    () => {},
+    () => {
+      initCommand();
+    }
+  )
+  .command(
+    "page",
+    "Generate a new page",
+    () => {},
+    () => {
+      generatePageTemplate();
+    }
+  )
+  .command(
+    "scrape-page [url]",
+    "Scrapes a page",
+    () => {},
+    async (argv) => {
+      await scrapePageAutomatically(argv);
+    }
+  )
+  .command(
+    "scrape-docusaurus-page [url]",
+    "Scrapes a Docusaurus page",
+    () => {},
+    async (argv) => {
+      await scrapePageWrapper(argv, scrapeDocusaurusPage);
+    }
+  )
+  .command(
+    "scrape-gitbook-page [url]",
+    "Scrapes a GitBook page",
+    () => {},
+    async (argv) => {
+      await scrapePageWrapper(argv, scrapeGitBookPage);
+    }
+  )
+  .command(
+    "scrape-readme-page [url]",
+    "Scrapes a ReadMe page",
+    () => {},
+    async (argv) => {
+      await scrapePageWrapper(argv, scrapeReadMePage);
+    }
+  )
+  .command(
+    "scrape-section [url]",
+    "Scrapes the docs in the section",
+    () => {},
+    async (argv) => {
+      await scrapeSectionAutomatically(argv);
+    }
+  )
+  .command(
+    "scrape-docusaurus-section [url]",
+    "Scrapes the Docusaurus section",
+    () => {},
+    async (argv) => {
+      await scrapeSectionAxiosWrapper(argv, scrapeDocusaurusSection);
+    }
+  )
+  .command(
+    "scrape-gitbook-section [url]",
+    "Scrapes the Gitbook section",
+    () => {},
+    async (argv) => {
+      await scrapeGitbookSectionCommand(argv);
+    }
+  )
+  .command(
+    "scrape-readme-section [url]",
+    "Scrapes the ReadMe section",
+    () => {},
+    async (argv) => {
+      await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
+    }
+  )
 
-if (args.length === 0) {
-  console.error(
-    `No command specified. Here are is the list that you can use:\ninit: initialize a Mintlify documentation instance`
-  );
-  process.exit(1); //an error occurred
-}
-
-const command = args[0];
-
-if (command === "init") {
-  inquirer
-    .prompt([
-      {
-        type: "input",
-        name: "name",
-        message: "What is the name of the organization?",
-      },
-      {
-        type: "input",
-        name: "color",
-        message: "What is the primary color of the brand?",
-        default: "#3b83f4",
-      },
-      {
-        type: "input",
-        name: "ctaName",
-        message: "What is the name of the call to action button?",
-        default: "Get Started",
-      },
-      {
-        type: "input",
-        name: "ctaUrl",
-        message: "What is the URL destination of the call to action button?",
-        default: "/",
-      },
-      {
-        type: "input",
-        name: "title",
-        message: "What is the title of the first page?",
-        default: "Introduction",
-      },
-    ])
-    .then((answers) => {
-      const { name, color, ctaName, ctaUrl, title } = answers;
-      writeFileSync(
-        "mint.config.json",
-        JSON.stringify(
-          MintConfig(name, color, ctaName, ctaUrl, toFilename(title)),
-          null,
-          "\t"
-        )
-      );
-      createPage(title);
-      console.log("🌱 Created initial files for Mintlify docs");
-      process.exit(1);
-    })
-    .catch((error) => {
-      console.error(error);
-      process.exit(1);
-    });
-}
-
-if (command === "page") {
-  inquirer
-    .prompt([
-      {
-        type: "input",
-        name: "title",
-        message: "What is the title of the new page?",
-      },
-      {
-        type: "input",
-        name: "description",
-        message: "What is the description?",
-        default: "",
-      },
-    ])
-    .then((answers) => {
-      const { title, description } = answers;
-
-      createPage(title, description);
-      console.log("🌱 Created initial files for Mintlify docs");
-      process.exit(1);
-    })
-    .catch((error) => {
-      console.error(error);
-      process.exit(1);
-    });
-}
-
-if (command === "scrape-gitbook-page") {
-  const url = args[1];
-  const { title, description, markdown } = await scrapeGitBookPage(url);
-  createPage(title, description, markdown, process.cwd());
-  process.exit(1);
-}
-
-if (command === "scrape-readme-page") {
-  const url = args[1];
-  const { title, description, markdown } = await scrapeReadMePage(url);
-  createPage(title, description, markdown, process.cwd());
-  process.exit(1);
-}
-
-if (command === "scrape-readme-section") {
-  const url = args[1];
-  console.log("Started scraping...");
-  const groupsConfig = await scrapeReadMe(url, process.cwd());
-  console.log("Finished scraping.");
-  console.log("Add the following to your navigation in mint.config.js:");
-  console.log(objToReadableString(groupsConfig));
-  process.exit(1);
-}
+  .parse();