mintlify 1.0.3 → 1.0.6

package/README.md

@@ -0,0 +1,7 @@
+# Mintlify CLI
+
+To test changes locally run:
+
+`npm run build`
+then
+`npm i -g .`

package/bin/index.js

@@ -2,12 +2,17 @@
 import { writeFileSync } from "fs";
 import inquirer from "inquirer";
 import { MintConfig } from "./templates.js";
+import { createPage, toFilename, objToReadableString } from "./util.js";
+import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { scrapeReadMe } from "./scrapeReadMe.js";
 const args = process.argv.slice(2);
 if (args.length === 0) {
     console.error(`No command specified. Here are is the list that you can use:\ninit: initialize a Mintlify documentation instance`);
     process.exit(1); //an error occurred
 }
-if (args[0] === "init") {
+const command = args[0];
+if (command === "init") {
     inquirer
         .prompt([
         {
@@ -42,11 +47,61 @@ if (args[0] === "init") {
     ])
         .then((answers) => {
         const { name, color, ctaName, ctaUrl, title } = answers;
-        writeFileSync("mint.config.json", JSON.stringify(MintConfig(name, color, ctaName, ctaUrl, title), null, "\t"));
+        writeFileSync("mint.config.json", JSON.stringify(MintConfig(name, color, ctaName, ctaUrl, toFilename(title)), null, "\t"));
+        createPage(title);
+        console.log("🌱 Created initial files for Mintlify docs");
+        process.exit(1);
+    })
+        .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });
+}
+if (command === "page") {
+    inquirer
+        .prompt([
+        {
+            type: "input",
+            name: "title",
+            message: "What is the title of the new page?",
+        },
+        {
+            type: "input",
+            name: "description",
+            message: "What is the description?",
+            default: "",
+        },
+    ])
+        .then((answers) => {
+        const { title, description } = answers;
+        createPage(title, description);
+        console.log("🌱 Created initial files for Mintlify docs");
+        process.exit(1);
     })
         .catch((error) => {
         console.error(error);
         process.exit(1);
     });
 }
+if (command === "scrape-gitbook-page") {
+    const url = args[1];
+    const { title, description, markdown } = await scrapeGitBookPage(url);
+    createPage(title, description, markdown, process.cwd());
+    process.exit(1);
+}
+if (command === "scrape-readme-page") {
+    const url = args[1];
+    const { title, description, markdown } = await scrapeReadMePage(url);
+    createPage(title, description, markdown, process.cwd());
+    process.exit(1);
+}
+if (command === "scrape-readme-section") {
+    const url = args[1];
+    console.log("Started scraping...");
+    const groupsConfig = await scrapeReadMe(url, process.cwd());
+    console.log("Finished scraping.");
+    console.log("Add the following to your navigation in mint.config.js:");
+    console.log(objToReadableString(groupsConfig));
+    process.exit(1);
+}
 //# sourceMappingURL=index.js.map

package/bin/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;IACrB,OAAO,CAAC,KAAK,CACX,kHAAkH,CACnH,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;CACrC;AAED,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,uCAAuC;SACjD;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,yCAAyC;YAClD,OAAO,EAAE,SAAS;SACnB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,gDAAgD;YACzD,OAAO,EAAE,aAAa;SACvB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,2DAA2D;YACpE,OAAO,EAAE,GAAG;SACb;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,sCAAsC;YAC/C,OAAO,EAAE,cAAc;SACxB;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAC/C,IAAI,EACJ,IAAI,CACL,CACF,CAAC;IACJ,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;IACrB,OAAO,CAAC,KAAK,CACX,kHAAkH,CACnH,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;CACrC;AAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;AAExB,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,uCAAuC;SACjD;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,yCAAyC;YAClD,OAAO,EAAE,SAAS;SACnB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,gDAAgD;YACzD,OAAO,EAAE,aAAa;SACvB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,2DAA2D;YACpE,OAAO,EAAE,GAAG;SACb;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,sCAAsC;YAC/C,OAAO,EAAE,cAAc;SACxB;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAC3D,IAAI,EACJ,IAAI,CACL,CACF,CAAC;QACF,UAAU,CAAC,KAAK,CAAC,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,oCAAoC;SAC9C;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,0BAA0B;YACnC,OAAO,EAAE,EAAE;SACZ;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEvC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,qBAAqB,EAAE;IACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACtE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,IAAI,OAAO,KAAK,oBAAoB,EAAE;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACrE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,IAAI,OAAO,KAAK,uBAAuB,EAAE;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACnC,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB"}

package/bin/scrapeGitBook.js

@@ -0,0 +1,28 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+export async function scrapeGitBook(url) {
+    const res = await axios.default.get(url);
+    const $ = cheerio.load(res.data);
+    const titleComponent = $('[data-testid="page.title"]').first();
+    const titleAndDescription = titleComponent.parent().parent().parent().text();
+    console.log(titleAndDescription);
+    const description = titleAndDescription
+        .replace(titleComponent.text(), "")
+        .trim();
+    const title = titleComponent.text().trim();
+    const content = $('[data-testid="page.contentEditor"]').first();
+    const contentHtml = $.html(content);
+    const nhm = new NodeHtmlMarkdown();
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line and increase their depth by one
+    markdown = markdown.replace(/# \n\n/g, "## ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeGitBook.js.map

package/bin/scrapeGitBook.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBook.js","sourceRoot":"","sources":["../src/scrapeGitBook.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAW;IAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scrapeGitBookPage.js

@@ -0,0 +1,28 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+export async function scrapeGitBookPage(url) {
+    const res = await axios.default.get(url);
+    const $ = cheerio.load(res.data);
+    const titleComponent = $('[data-testid="page.title"]').first();
+    const titleAndDescription = titleComponent.parent().parent().parent().text();
+    console.log(titleAndDescription);
+    const description = titleAndDescription
+        .replace(titleComponent.text(), "")
+        .trim();
+    const title = titleComponent.text().trim();
+    const content = $('[data-testid="page.contentEditor"]').first();
+    const contentHtml = $.html(content);
+    const nhm = new NodeHtmlMarkdown();
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line and increase their depth by one
+    markdown = markdown.replace(/# \n\n/g, "## ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeGitBookPage.js.map

package/bin/scrapeGitBookPage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../src/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IACjD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scrapeReadMe.js

@@ -0,0 +1,60 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import path from "path";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { createPage, getWebsite } from "./util.js";
+export async function scrapeReadMe(url, cliDir) {
+    const res = await axios.default.get(url);
+    const $ = cheerio.load(res.data);
+    const website = getWebsite(url);
+    // Get all the navigation sections
+    const navigationSections = $(".rm-Sidebar-section");
+    const groupsConfig = navigationSections
+        .map((i, section) => {
+        const sectionTitle = $(section).find("h3").first().text();
+        const linkPaths = $(section)
+            .find("a[href]")
+            .map((i, link) => {
+            const href = $(link).attr("href");
+            // Skip external links until Mintlify supports them
+            if (href.startsWith("https://") || href.startsWith("http://")) {
+                return undefined;
+            }
+            return href;
+        })
+            .toArray();
+        // Follows the same structure as mint.config.js
+        return {
+            group: sectionTitle,
+            pages: linkPaths,
+        };
+    })
+        .toArray();
+    return await Promise.all(groupsConfig.map(async (groupConfig) => {
+        groupConfig.pages = await Promise.all(groupConfig.pages.map(async (page) => {
+            // Scrape each page separately
+            const { title, description, markdown } = await scrapeReadMePage(website + page);
+            // Removes file name from the end
+            const splitSubpath = page.split("/");
+            let folders = splitSubpath
+                .slice(0, splitSubpath.length - 1)
+                .join("/");
+            // ReadMe requires a directory on all sections wheras we use root.
+            // docs/ is their home directory so we remove it
+            if (folders.startsWith("/docs")) {
+                folders = folders.replace("/docs", "");
+            }
+            const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
+            // Default to introduction.mdx if we encountered index.html
+            const fileName = splitSubpath.length === 2
+                ? "introduction"
+                : splitSubpath[splitSubpath.length - 1];
+            // Will create subfolders as needed
+            createPage(title, description, markdown, newFileLocation, fileName);
+            // Removes first slash if we are in a folder, Mintlify doesn't need it
+            return folders ? path.join(folders, fileName).substring(1) : fileName;
+        }));
+        return groupConfig;
+    }));
+}
+//# sourceMappingURL=scrapeReadMe.js.map

package/bin/scrapeReadMe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMe.js","sourceRoot":"","sources":["../src/scrapeReadMe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAU,MAAM,WAAW,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,MAAc;IAC5D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAEhC,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC;IAEpD,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC;aACzB,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAElC,mDAAmD;YACnD,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC7D,OAAO,SAAS,CAAC;aAClB;YAED,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,+CAA+C;QAC/C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,WAAW,CAAC,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC3C,8BAA8B;YAC9B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,gBAAgB,CAC7D,OAAO,GAAG,IAAI,CACf,CAAC;YAEF,iCAAiC;YACjC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,OAAO,GAAG,YAAY;iBACvB,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;iBACjC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,kEAAkE;YAClE,gDAAgD;YAChD,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC/B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;aACxC;YAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAEtE,2DAA2D;YAC3D,MAAM,QAAQ,GACZ,YAAY,CAAC,MAAM,KAAK,CAAC;gBACvB,CAAC,CAAC,cAAc;gBAChB,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAE5C,mCAAmC;YACnC,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,CAAC,CAAC;YAEpE,sEAAsE;YACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACxE,CAAC,CAAC,CACH,CAAC;QACF,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/bin/scrapeReadMePage.js

@@ -0,0 +1,28 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+export async function scrapeReadMePage(url) {
+    const res = await axios.default.get(url);
+    const $ = cheerio.load(res.data);
+    const titleComponent = $("h1").first();
+    const title = titleComponent.text().trim();
+    const description = $(".markdown-body", titleComponent.parent())
+        .text()
+        .trim();
+    const content = $(".content-body .markdown-body").first();
+    const contentHtml = $.html(content);
+    const nhm = new NodeHtmlMarkdown();
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line and increase their depth by one
+    markdown = markdown.replace(/# \n\n/g, "## ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Remove ReadMe anchor links
+    markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeReadMePage.js.map

package/bin/scrapeReadMePage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../src/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAChD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC;SAC7D,IAAI,EAAE;SACN,IAAI,EAAE,CAAC;IAEV,MAAM,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/templates.js

@@ -1,4 +1,4 @@
-export const MintConfig = (name, color, ctaName, ctaUrl, title) => {
+export const MintConfig = (name, color, ctaName, ctaUrl, filename) => {
     return {
         name,
         logo: "",
@@ -15,10 +15,27 @@ export const MintConfig = (name, color, ctaName, ctaUrl, title) => {
         navigation: [
             {
                 group: "Home",
-                pages: [title],
+                pages: [filename],
             },
         ],
         // footerSocials: {}, // support object type for footer tyoes
     };
 };
+export const Page = (title, description, markdown) => {
+    // If we are an empty String we want to add two quotes,
+    // if we added as we went we would detect the first quote
+    // as the closing quote.
+    const startsWithQuote = title.startsWith('"');
+    const endsWithQuote = title.startsWith('"');
+    if (!startsWithQuote) {
+        title = '"' + title;
+    }
+    if (!endsWithQuote) {
+        title = title + '"';
+    }
+    const optionalDescription = description
+        ? `\ndescription: "${description}"`
+        : "";
+    return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
+};
 //# sourceMappingURL=templates.js.map

package/bin/templates.js.map

@@ -1 +1 @@
-{"version":3,"file":"templates.js","sourceRoot":"","sources":["../src/templates.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,KAAa,EACb,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,KAAK,CAAC;aACf;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"templates.js","sourceRoot":"","sources":["../src/templates.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAClB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,EAAE;IACF,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE;QACpB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;KACrB;IACD,IAAI,CAAC,aAAa,EAAE;QAClB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;KACrB;IAED,MAAM,mBAAmB,GAAG,WAAW;QACrC,CAAC,CAAC,mBAAmB,WAAW,GAAG;QACnC,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC"}

package/bin/util.js

@@ -0,0 +1,37 @@
+import { mkdirSync, writeFileSync } from "fs";
+import { Page } from "./templates.js";
+import path from "path";
+export function getWebsite(url) {
+    // Gets the website
+    // eg. https://google.com -> https://google.com
+    // https://google.com/page -> https://google.com
+    return url.split("/").slice(0, 3).join("/");
+}
+export function objToReadableString(objs) {
+    // Two spaces as indentation
+    return objs.map((obj) => JSON.stringify(obj, null, 2)).join(",\n");
+}
+export const toFilename = (title) => {
+    // Gets rid of special characters at the start and end
+    // of the name by converting to spaces then using trim.
+    return title
+        .replace(/[^a-z0-9]/gi, " ")
+        .trim()
+        .replace(/ /g, "-")
+        .toLowerCase();
+};
+export const addMdx = (fileName) => {
+    if (fileName.endsWith(".mdx")) {
+        return fileName;
+    }
+    return fileName + ".mdx";
+};
+export const createPage = (title, description, markdown, rootDir = "", fileName) => {
+    // Create the folders needed if they're missing
+    mkdirSync(rootDir, { recursive: true });
+    // Write the page to memory
+    const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
+    writeFileSync(writePath, Page(title, description, markdown));
+    console.log("✏️ - " + writePath);
+};
+//# sourceMappingURL=util.js.map

package/bin/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,mBAAmB;IACnB,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAc;IAChD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,UAAkB,EAAE,EACpB,QAAiB,EACjB,EAAE;IACF,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5E,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;AACnC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mintlify",
-  "version": "1.0.3",
+  "version": "1.0.6",
   "description": "Mintlify CLI",
   "engines": {
     "node": ">=14.16"
@@ -16,7 +16,11 @@
   },
   "type": "module",
   "dependencies": {
-    "inquirer": "^9.1.0"
+    "axios": "^0.27.2",
+    "cheerio": "^0.22.0",
+    "inquirer": "^9.1.0",
+    "node-html-markdown": "^1.2.0",
+    "puppeteer": "^17.1.2"
   },
   "devDependencies": {
     "@types/inquirer": "^9.0.1",

package/src/index.ts

@@ -2,8 +2,11 @@
 
 import { writeFileSync } from "fs";
 import inquirer from "inquirer";
-import { MintConfig, Page } from "./templates.js";
-import { toFilename } from "./util.js";
+import { MintConfig } from "./templates.js";
+import { createPage, toFilename, objToReadableString } from "./util.js";
+import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { scrapeReadMe } from "./scrapeReadMe.js";
 
 const args = process.argv.slice(2);
 
@@ -59,6 +62,7 @@ if (command === "init") {
           "\t"
         )
       );
+      createPage(title);
       console.log("🌱 Created initial files for Mintlify docs");
       process.exit(1);
     })
@@ -86,7 +90,7 @@ if (command === "page") {
     .then((answers) => {
       const { title, description } = answers;
 
-      writeFileSync(toFilename(title), Page(title, description));
+      createPage(title, description);
       console.log("🌱 Created initial files for Mintlify docs");
       process.exit(1);
     })
@@ -95,3 +99,27 @@ if (command === "page") {
       process.exit(1);
     });
 }
+
+if (command === "scrape-gitbook-page") {
+  const url = args[1];
+  const { title, description, markdown } = await scrapeGitBookPage(url);
+  createPage(title, description, markdown, process.cwd());
+  process.exit(1);
+}
+
+if (command === "scrape-readme-page") {
+  const url = args[1];
+  const { title, description, markdown } = await scrapeReadMePage(url);
+  createPage(title, description, markdown, process.cwd());
+  process.exit(1);
+}
+
+if (command === "scrape-readme-section") {
+  const url = args[1];
+  console.log("Started scraping...");
+  const groupsConfig = await scrapeReadMe(url, process.cwd());
+  console.log("Finished scraping.");
+  console.log("Add the following to your navigation in mint.config.js:");
+  console.log(objToReadableString(groupsConfig));
+  process.exit(1);
+}

package/src/scrapeGitBookPage.ts

@@ -0,0 +1,36 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+
+export async function scrapeGitBookPage(url: string) {
+  const res = await axios.default.get(url);
+  const $ = cheerio.load(res.data);
+
+  const titleComponent = $('[data-testid="page.title"]').first();
+  const titleAndDescription = titleComponent.parent().parent().parent().text();
+  console.log(titleAndDescription);
+  const description = titleAndDescription
+    .replace(titleComponent.text(), "")
+    .trim();
+  const title = titleComponent.text().trim();
+
+  const content = $('[data-testid="page.contentEditor"]').first();
+  const contentHtml = $.html(content);
+
+  const nhm = new NodeHtmlMarkdown();
+  let markdown = nhm.translate(contentHtml);
+
+  // Keep headers on one line and increase their depth by one
+  markdown = markdown.replace(/# \n\n/g, "## ");
+
+  // Remove unnecessary nonwidth blank space characters
+  markdown = markdown.replace(/\u200b/g, "");
+
+  // Reduce unnecessary blank lines
+  markdown = markdown.replace(/\n\n\n/g, "\n\n");
+
+  // Mintlify doesn't support bolded headers, remove the asterisks
+  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+
+  return { title, description, markdown };
+}

package/src/scrapeReadMe.ts

@@ -0,0 +1,79 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import path from "path";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { createPage, getWebsite, addMdx } from "./util.js";
+
+export async function scrapeReadMe(url: string, cliDir: string) {
+  const res = await axios.default.get(url);
+  const $ = cheerio.load(res.data);
+  const website = getWebsite(url);
+
+  // Get all the navigation sections
+  const navigationSections = $(".rm-Sidebar-section");
+
+  const groupsConfig = navigationSections
+    .map((i, section) => {
+      const sectionTitle = $(section).find("h3").first().text();
+      const linkPaths = $(section)
+        .find("a[href]")
+        .map((i, link) => {
+          const href = $(link).attr("href");
+
+          // Skip external links until Mintlify supports them
+          if (href.startsWith("https://") || href.startsWith("http://")) {
+            return undefined;
+          }
+
+          return href;
+        })
+        .toArray();
+
+      // Follows the same structure as mint.config.js
+      return {
+        group: sectionTitle,
+        pages: linkPaths,
+      };
+    })
+    .toArray();
+
+  return await Promise.all(
+    groupsConfig.map(async (groupConfig) => {
+      groupConfig.pages = await Promise.all(
+        groupConfig.pages.map(async (page: string) => {
+          // Scrape each page separately
+          const { title, description, markdown } = await scrapeReadMePage(
+            website + page
+          );
+
+          // Removes file name from the end
+          const splitSubpath = page.split("/");
+          let folders = splitSubpath
+            .slice(0, splitSubpath.length - 1)
+            .join("/");
+
+          // ReadMe requires a directory on all sections wheras we use root.
+          // docs/ is their home directory so we remove it
+          if (folders.startsWith("/docs")) {
+            folders = folders.replace("/docs", "");
+          }
+
+          const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
+
+          // Default to introduction.mdx if we encountered index.html
+          const fileName =
+            splitSubpath.length === 2
+              ? "introduction"
+              : splitSubpath[splitSubpath.length - 1];
+
+          // Will create subfolders as needed
+          createPage(title, description, markdown, newFileLocation, fileName);
+
+          // Removes first slash if we are in a folder, Mintlify doesn't need it
+          return folders ? path.join(folders, fileName).substring(1) : fileName;
+        })
+      );
+      return groupConfig;
+    })
+  );
+}

package/src/scrapeReadMePage.ts

@@ -0,0 +1,37 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+
+export async function scrapeReadMePage(url: string) {
+  const res = await axios.default.get(url);
+  const $ = cheerio.load(res.data);
+
+  const titleComponent = $("h1").first();
+  const title = titleComponent.text().trim();
+  const description = $(".markdown-body", titleComponent.parent())
+    .text()
+    .trim();
+
+  const content = $(".content-body .markdown-body").first();
+  const contentHtml = $.html(content);
+
+  const nhm = new NodeHtmlMarkdown();
+  let markdown = nhm.translate(contentHtml);
+
+  // Keep headers on one line and increase their depth by one
+  markdown = markdown.replace(/# \n\n/g, "## ");
+
+  // Remove unnecessary nonwidth blank space characters
+  markdown = markdown.replace(/\u200b/g, "");
+
+  // Remove ReadMe anchor links
+  markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+
+  // Reduce unnecessary blank lines
+  markdown = markdown.replace(/\n\n\n/g, "\n\n");
+
+  // Mintlify doesn't support bolded headers, remove the asterisks
+  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+
+  return { title, description, markdown };
+}

package/src/templates.ts

@@ -28,7 +28,25 @@ export const MintConfig = (
   };
 };
 
-export const Page = (title: string, description?: string) => {
-  const optionalDescription = description ? `\n${description}` : "";
-  return `---\ntitle:${title}${optionalDescription}\n---`;
+export const Page = (
+  title: string,
+  description?: string,
+  markdown?: string
+) => {
+  // If we are an empty String we want to add two quotes,
+  // if we added as we went we would detect the first quote
+  // as the closing quote.
+  const startsWithQuote = title.startsWith('"');
+  const endsWithQuote = title.startsWith('"');
+  if (!startsWithQuote) {
+    title = '"' + title;
+  }
+  if (!endsWithQuote) {
+    title = title + '"';
+  }
+
+  const optionalDescription = description
+    ? `\ndescription: "${description}"`
+    : "";
+  return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
 };

package/src/util.ts

@@ -1,3 +1,48 @@
+import { mkdirSync, writeFileSync } from "fs";
+import { Page } from "./templates.js";
+import path from "path";
+
+export function getWebsite(url: string) {
+  // Gets the website
+  // eg. https://google.com -> https://google.com
+  // https://google.com/page -> https://google.com
+  return url.split("/").slice(0, 3).join("/");
+}
+
+export function objToReadableString(objs: Object[]) {
+  // Two spaces as indentation
+  return objs.map((obj) => JSON.stringify(obj, null, 2)).join(",\n");
+}
+
 export const toFilename = (title: string) => {
-  return title.replace(/[^a-z0-9]/gi, "-").toLowerCase();
+  // Gets rid of special characters at the start and end
+  // of the name by converting to spaces then using trim.
+  return title
+    .replace(/[^a-z0-9]/gi, " ")
+    .trim()
+    .replace(/ /g, "-")
+    .toLowerCase();
+};
+
+export const addMdx = (fileName: string) => {
+  if (fileName.endsWith(".mdx")) {
+    return fileName;
+  }
+  return fileName + ".mdx";
+};
+
+export const createPage = (
+  title: string,
+  description?: string,
+  markdown?: string,
+  rootDir: string = "",
+  fileName?: string
+) => {
+  // Create the folders needed if they're missing
+  mkdirSync(rootDir, { recursive: true });
+
+  // Write the page to memory
+  const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
+  writeFileSync(writePath, Page(title, description, markdown));
+  console.log("✏️ - " + writePath);
 };

package/tsconfig.json

@@ -3,7 +3,7 @@
     "module": "node16",
     "moduleResolution": "node16",
     "target": "es2017",
-    "lib": ["es2015"],
+    "lib": ["es2016", "dom"],
     "sourceMap": true,
     "outDir": "bin",
     "baseUrl": ".",