mintlify 1.0.4 → 1.0.5

package/README.md

@@ -0,0 +1,7 @@
+# Mintlify CLI
+
+To test changes locally run:
+
+`npm run build`
+then
+`npm i -g .`

package/bin/index.js

@@ -2,12 +2,16 @@
 import { writeFileSync } from "fs";
 import inquirer from "inquirer";
 import { MintConfig } from "./templates.js";
+import { createPage, toFilename } from "./util.js";
+import { scrapeGitBook } from "./scrapeGitBook.js";
+import { scrapeReadMe } from "./scrapeReadMe.js";
 const args = process.argv.slice(2);
 if (args.length === 0) {
     console.error(`No command specified. Here are is the list that you can use:\ninit: initialize a Mintlify documentation instance`);
     process.exit(1); //an error occurred
 }
-if (args[0] === "init") {
+const command = args[0];
+if (command === "init") {
     inquirer
         .prompt([
         {
@@ -42,11 +46,52 @@ if (args[0] === "init") {
     ])
         .then((answers) => {
         const { name, color, ctaName, ctaUrl, title } = answers;
-        writeFileSync("mint.config.json", JSON.stringify(MintConfig(name, color, ctaName, ctaUrl, title), null, "\t"));
+        writeFileSync("mint.config.json", JSON.stringify(MintConfig(name, color, ctaName, ctaUrl, toFilename(title)), null, "\t"));
+        createPage(title);
+        console.log("🌱 Created initial files for Mintlify docs");
+        process.exit(1);
     })
         .catch((error) => {
         console.error(error);
         process.exit(1);
     });
 }
+if (command === "page") {
+    inquirer
+        .prompt([
+        {
+            type: "input",
+            name: "title",
+            message: "What is the title of the new page?",
+        },
+        {
+            type: "input",
+            name: "description",
+            message: "What is the description?",
+            default: "",
+        },
+    ])
+        .then((answers) => {
+        const { title, description } = answers;
+        createPage(title, description);
+        console.log("🌱 Created initial files for Mintlify docs");
+        process.exit(1);
+    })
+        .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });
+}
+if (command === "scrape-gitbook") {
+    const url = args[1];
+    const { title, description, markdown } = await scrapeGitBook(url);
+    createPage(title, description, markdown, process.env.INIT_CWD);
+    process.exit(1);
+}
+if (command === "scrape-readme") {
+    const url = args[1];
+    const { title, description, markdown } = await scrapeReadMe(url);
+    createPage(title, description, markdown, process.env.INIT_CWD);
+    process.exit(1);
+}
 //# sourceMappingURL=index.js.map

package/bin/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;IACrB,OAAO,CAAC,KAAK,CACX,kHAAkH,CACnH,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;CACrC;AAED,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,uCAAuC;SACjD;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,yCAAyC;YAClD,OAAO,EAAE,SAAS;SACnB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,gDAAgD;YACzD,OAAO,EAAE,aAAa;SACvB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,2DAA2D;YACpE,OAAO,EAAE,GAAG;SACb;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,sCAAsC;YAC/C,OAAO,EAAE,cAAc;SACxB;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAC/C,IAAI,EACJ,IAAI,CACL,CACF,CAAC;IACJ,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;IACrB,OAAO,CAAC,KAAK,CACX,kHAAkH,CACnH,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;CACrC;AAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;AAExB,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,uCAAuC;SACjD;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,yCAAyC;YAClD,OAAO,EAAE,SAAS;SACnB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,gDAAgD;YACzD,OAAO,EAAE,aAAa;SACvB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,2DAA2D;YACpE,OAAO,EAAE,GAAG;SACb;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,sCAAsC;YAC/C,OAAO,EAAE,cAAc;SACxB;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAC3D,IAAI,EACJ,IAAI,CACL,CACF,CAAC;QACF,UAAU,CAAC,KAAK,CAAC,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,oCAAoC;SAC9C;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,0BAA0B;YACnC,OAAO,EAAE,EAAE;SACZ;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEvC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,gBAAgB,EAAE;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAClE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,IAAI,OAAO,KAAK,eAAe,EAAE;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IACjE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB"}

package/bin/scrapeGitBook.js

@@ -0,0 +1,28 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+export async function scrapeGitBook(url) {
+    const res = await axios.default.get(url);
+    const $ = cheerio.load(res.data);
+    const titleComponent = $('[data-testid="page.title"]').first();
+    const titleAndDescription = titleComponent.parent().parent().parent().text();
+    console.log(titleAndDescription);
+    const description = titleAndDescription
+        .replace(titleComponent.text(), "")
+        .trim();
+    const title = titleComponent.text().trim();
+    const content = $('[data-testid="page.contentEditor"]').first();
+    const contentHtml = $.html(content);
+    const nhm = new NodeHtmlMarkdown();
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line and increase their depth by one
+    markdown = markdown.replace(/# \n\n/g, "## ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeGitBook.js.map

package/bin/scrapeGitBook.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBook.js","sourceRoot":"","sources":["../src/scrapeGitBook.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAW;IAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scrapeReadMe.js

@@ -0,0 +1,28 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+export async function scrapeReadMe(url) {
+    const res = await axios.default.get(url);
+    const $ = cheerio.load(res.data);
+    const titleComponent = $("h1").first();
+    const title = titleComponent.text().trim();
+    const description = $(".markdown-body", titleComponent.parent())
+        .text()
+        .trim();
+    const content = $(".content-body .markdown-body").first();
+    const contentHtml = $.html(content);
+    const nhm = new NodeHtmlMarkdown();
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line and increase their depth by one
+    markdown = markdown.replace(/# \n\n/g, "## ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Remove ReadMe anchor links
+    markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeReadMe.js.map

package/bin/scrapeReadMe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMe.js","sourceRoot":"","sources":["../src/scrapeReadMe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW;IAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC;SAC7D,IAAI,EAAE;SACN,IAAI,EAAE,CAAC;IAEV,MAAM,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/templates.js

@@ -1,4 +1,4 @@
-export const MintConfig = (name, color, ctaName, ctaUrl, title) => {
+export const MintConfig = (name, color, ctaName, ctaUrl, filename) => {
     return {
         name,
         logo: "",
@@ -15,10 +15,27 @@ export const MintConfig = (name, color, ctaName, ctaUrl, title) => {
         navigation: [
             {
                 group: "Home",
-                pages: [title],
+                pages: [filename],
             },
         ],
         // footerSocials: {}, // support object type for footer tyoes
     };
 };
+export const Page = (title, description, markdown) => {
+    // If we are an empty String we want to add two quotes,
+    // if we added as we went we would detect the first quote
+    // as the closing quote.
+    const startsWithQuote = title.startsWith('"');
+    const endsWithQuote = title.startsWith('"');
+    if (!startsWithQuote) {
+        title = '"' + title;
+    }
+    if (!endsWithQuote) {
+        title = title + '"';
+    }
+    const optionalDescription = description
+        ? `\ndescription: "${description}"`
+        : "";
+    return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
+};
 //# sourceMappingURL=templates.js.map

package/bin/templates.js.map

@@ -1 +1 @@
-{"version":3,"file":"templates.js","sourceRoot":"","sources":["../src/templates.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,KAAa,EACb,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,KAAK,CAAC;aACf;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"templates.js","sourceRoot":"","sources":["../src/templates.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAClB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,EAAE;IACF,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE;QACpB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;KACrB;IACD,IAAI,CAAC,aAAa,EAAE;QAClB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;KACrB;IAED,MAAM,mBAAmB,GAAG,WAAW;QACrC,CAAC,CAAC,mBAAmB,WAAW,GAAG;QACnC,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC"}

package/bin/util.js

@@ -0,0 +1,15 @@
+import { writeFileSync } from "fs";
+import { Page } from "./templates.js";
+export const toFilename = (title) => {
+    // Gets rid of special characters at the start and end
+    // of the name by converting to spaces then using trim.
+    return title
+        .replace(/[^a-z0-9]/gi, " ")
+        .trim()
+        .replace(/ /g, "-")
+        .toLowerCase();
+};
+export const createPage = (title, description, markdown, rootDir = "") => {
+    return writeFileSync(rootDir + toFilename(title) + ".mdx", Page(title, description, markdown));
+};
+//# sourceMappingURL=util.js.map

package/bin/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,UAAkB,EAAE,EACpB,EAAE;IACF,OAAO,aAAa,CAClB,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,EACpC,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CACnC,CAAC;AACJ,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mintlify",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "Mintlify CLI",
   "engines": {
     "node": ">=14.16"
@@ -16,7 +16,11 @@
   },
   "type": "module",
   "dependencies": {
-    "inquirer": "^9.1.0"
+    "axios": "^0.27.2",
+    "cheerio": "^0.22.0",
+    "inquirer": "^9.1.0",
+    "node-html-markdown": "^1.2.0",
+    "puppeteer": "^17.1.2"
   },
   "devDependencies": {
     "@types/inquirer": "^9.0.1",

package/src/index.ts

@@ -4,6 +4,8 @@ import { writeFileSync } from "fs";
 import inquirer from "inquirer";
 import { MintConfig } from "./templates.js";
 import { createPage, toFilename } from "./util.js";
+import { scrapeGitBook } from "./scrapeGitBook.js";
+import { scrapeReadMe } from "./scrapeReadMe.js";
 
 const args = process.argv.slice(2);
 
@@ -96,3 +98,17 @@ if (command === "page") {
       process.exit(1);
     });
 }
+
+if (command === "scrape-gitbook") {
+  const url = args[1];
+  const { title, description, markdown } = await scrapeGitBook(url);
+  createPage(title, description, markdown, process.env.INIT_CWD);
+  process.exit(1);
+}
+
+if (command === "scrape-readme") {
+  const url = args[1];
+  const { title, description, markdown } = await scrapeReadMe(url);
+  createPage(title, description, markdown, process.env.INIT_CWD);
+  process.exit(1);
+}

package/src/scrapeGitBook.ts

@@ -0,0 +1,36 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+
+export async function scrapeGitBook(url: string) {
+  const res = await axios.default.get(url);
+  const $ = cheerio.load(res.data);
+
+  const titleComponent = $('[data-testid="page.title"]').first();
+  const titleAndDescription = titleComponent.parent().parent().parent().text();
+  console.log(titleAndDescription);
+  const description = titleAndDescription
+    .replace(titleComponent.text(), "")
+    .trim();
+  const title = titleComponent.text().trim();
+
+  const content = $('[data-testid="page.contentEditor"]').first();
+  const contentHtml = $.html(content);
+
+  const nhm = new NodeHtmlMarkdown();
+  let markdown = nhm.translate(contentHtml);
+
+  // Keep headers on one line and increase their depth by one
+  markdown = markdown.replace(/# \n\n/g, "## ");
+
+  // Remove unnecessary nonwidth blank space characters
+  markdown = markdown.replace(/\u200b/g, "");
+
+  // Reduce unnecessary blank lines
+  markdown = markdown.replace(/\n\n\n/g, "\n\n");
+
+  // Mintlify doesn't support bolded headers, remove the asterisks
+  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+
+  return { title, description, markdown };
+}

package/src/scrapeReadMe.ts

@@ -0,0 +1,37 @@
+import axios from "axios";
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+
+export async function scrapeReadMe(url: string) {
+  const res = await axios.default.get(url);
+  const $ = cheerio.load(res.data);
+
+  const titleComponent = $("h1").first();
+  const title = titleComponent.text().trim();
+  const description = $(".markdown-body", titleComponent.parent())
+    .text()
+    .trim();
+
+  const content = $(".content-body .markdown-body").first();
+  const contentHtml = $.html(content);
+
+  const nhm = new NodeHtmlMarkdown();
+  let markdown = nhm.translate(contentHtml);
+
+  // Keep headers on one line and increase their depth by one
+  markdown = markdown.replace(/# \n\n/g, "## ");
+
+  // Remove unnecessary nonwidth blank space characters
+  markdown = markdown.replace(/\u200b/g, "");
+
+  // Remove ReadMe anchor links
+  markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+
+  // Reduce unnecessary blank lines
+  markdown = markdown.replace(/\n\n\n/g, "\n\n");
+
+  // Mintlify doesn't support bolded headers, remove the asterisks
+  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+
+  return { title, description, markdown };
+}

package/src/templates.ts

@@ -28,7 +28,25 @@ export const MintConfig = (
   };
 };
 
-export const Page = (title: string, description?: string) => {
-  const optionalDescription = description ? `\n${description}` : "";
-  return `---\ntitle:${title}${optionalDescription}\n---`;
+export const Page = (
+  title: string,
+  description?: string,
+  markdown?: string
+) => {
+  // If we are an empty String we want to add two quotes,
+  // if we added as we went we would detect the first quote
+  // as the closing quote.
+  const startsWithQuote = title.startsWith('"');
+  const endsWithQuote = title.startsWith('"');
+  if (!startsWithQuote) {
+    title = '"' + title;
+  }
+  if (!endsWithQuote) {
+    title = title + '"';
+  }
+
+  const optionalDescription = description
+    ? `\ndescription: "${description}"`
+    : "";
+  return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
 };

package/src/util.ts

@@ -2,9 +2,23 @@ import { writeFileSync } from "fs";
 import { Page } from "./templates.js";
 
 export const toFilename = (title: string) => {
-  return title.replace(/[^a-z0-9]/gi, "-").toLowerCase();
+  // Gets rid of special characters at the start and end
+  // of the name by converting to spaces then using trim.
+  return title
+    .replace(/[^a-z0-9]/gi, " ")
+    .trim()
+    .replace(/ /g, "-")
+    .toLowerCase();
 };
 
-export const createPage = (title: string, description?: string) => {
-  return writeFileSync(toFilename(title), Page(title, description));
+export const createPage = (
+  title: string,
+  description?: string,
+  markdown?: string,
+  rootDir: string = ""
+) => {
+  return writeFileSync(
+    rootDir + toFilename(title) + ".mdx",
+    Page(title, description, markdown)
+  );
 };

package/tsconfig.json

@@ -3,7 +3,7 @@
     "module": "node16",
     "moduleResolution": "node16",
     "target": "es2017",
-    "lib": ["es2015"],
+    "lib": ["es2016", "dom"],
     "sourceMap": true,
     "outDir": "bin",
     "baseUrl": ".",