mintlify 1.0.5 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. package/bin/index.js +18 -8
  2. package/bin/index.js.map +1 -1
  3. package/bin/scrapeGitBookPage.js +28 -0
  4. package/bin/scrapeGitBookPage.js.map +1 -0
  5. package/bin/scrapeReadMe.js +54 -22
  6. package/bin/scrapeReadMe.js.map +1 -1
  7. package/bin/scrapeReadMePage.js +28 -0
  8. package/bin/scrapeReadMePage.js.map +1 -0
  9. package/bin/util.js +25 -3
  10. package/bin/util.js.map +1 -1
  11. package/package.json +1 -1
  12. package/src/index.ts +19 -8
  13. package/src/{scrapeGitBook.ts → scrapeGitBookPage.ts} +1 -1
  14. package/src/scrapeReadMe.ts +64 -22
  15. package/src/scrapeReadMePage.ts +37 -0
  16. package/src/util.ts +30 -6
package/bin/index.js CHANGED
@@ -2,8 +2,9 @@
2
2
  import { writeFileSync } from "fs";
3
3
  import inquirer from "inquirer";
4
4
  import { MintConfig } from "./templates.js";
5
- import { createPage, toFilename } from "./util.js";
6
- import { scrapeGitBook } from "./scrapeGitBook.js";
5
+ import { createPage, toFilename, objToReadableString } from "./util.js";
6
+ import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
7
+ import { scrapeReadMePage } from "./scrapeReadMePage.js";
7
8
  import { scrapeReadMe } from "./scrapeReadMe.js";
8
9
  const args = process.argv.slice(2);
9
10
  if (args.length === 0) {
@@ -82,16 +83,25 @@ if (command === "page") {
82
83
  process.exit(1);
83
84
  });
84
85
  }
85
- if (command === "scrape-gitbook") {
86
+ if (command === "scrape-gitbook-page") {
86
87
  const url = args[1];
87
- const { title, description, markdown } = await scrapeGitBook(url);
88
- createPage(title, description, markdown, process.env.INIT_CWD);
88
+ const { title, description, markdown } = await scrapeGitBookPage(url);
89
+ createPage(title, description, markdown, process.cwd());
89
90
  process.exit(1);
90
91
  }
91
- if (command === "scrape-readme") {
92
+ if (command === "scrape-readme-page") {
92
93
  const url = args[1];
93
- const { title, description, markdown } = await scrapeReadMe(url);
94
- createPage(title, description, markdown, process.env.INIT_CWD);
94
+ const { title, description, markdown } = await scrapeReadMePage(url);
95
+ createPage(title, description, markdown, process.cwd());
96
+ process.exit(1);
97
+ }
98
+ if (command === "scrape-readme-section") {
99
+ const url = args[1];
100
+ console.log("Started scraping...");
101
+ const groupsConfig = await scrapeReadMe(url, process.cwd());
102
+ console.log("Finished scraping.");
103
+ console.log("Add the following to your navigation in mint.config.js:");
104
+ console.log(objToReadableString(groupsConfig));
95
105
  process.exit(1);
96
106
  }
97
107
  //# sourceMappingURL=index.js.map
package/bin/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;IACrB,OAAO,CAAC,KAAK,CACX,kHAAkH,CACnH,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;CACrC;AAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;AAExB,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,uCAAuC;SACjD;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,yCAAyC;YAClD,OAAO,EAAE,SAAS;SACnB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,gDAAgD;YACzD,OAAO,EAAE,aAAa;SACvB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,2DAA2D;YACpE,OAAO,EAAE,GAAG;SACb;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,sCAAsC;YAC/C,OAAO,EAAE,cAAc;SACxB;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAC3D,IAAI,EACJ,IAAI,CACL,CACF,CAAC;QACF,UAAU,CAAC,KAAK,CAAC,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,oCAAoC;SAC9C;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,0BAA0B;YACnC,OAAO,EAAE,EAAE;SACZ;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEvC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,gBAAgB,EAAE;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAClE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,IAAI,OAAO,KAAK,eAAe,EAAE;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IACjE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;IACrB,OAAO,CAAC,KAAK,CACX,kHAAkH,CACnH,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;CACrC;AAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;AAExB,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,uCAAuC;SACjD;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,yCAAyC;YAClD,OAAO,EAAE,SAAS;SACnB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,gDAAgD;YACzD,OAAO,EAAE,aAAa;SACvB;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,2DAA2D;YACpE,OAAO,EAAE,GAAG;SACb;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,sCAAsC;YAC/C,OAAO,EAAE,cAAc;SACxB;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxD,aAAa,CACX,kBAAkB,EAClB,IAAI,CAAC,SAAS,CACZ,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAC3D,IAAI,EACJ,IAAI,CACL,CACF,CAAC;QACF,UAAU,CAAC,KAAK,CAAC,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,MAAM,EAAE;IACtB,QAAQ;SACL,MAAM,CAAC;QACN;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,oCAAoC;SAC9C;QACD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,0BAA0B;YACnC,OAAO,EAAE,EAAE;SACZ;KACF,CAAC;SACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAChB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEvC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;CACN;AAED,IAAI,OAAO,KAAK,qBAAqB,EAAE;IACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACtE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,IAAI,OAAO,KAAK,oBAAoB,EAAE;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACrE,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,IAAI,OAAO,KAAK,uBAAuB,EAAE;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACnC,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;CACjB"}
package/bin/scrapeGitBookPage.js ADDED
@@ -0,0 +1,28 @@
1
+ import axios from "axios";
2
+ import cheerio from "cheerio";
3
+ import { NodeHtmlMarkdown } from "node-html-markdown";
4
+ export async function scrapeGitBookPage(url) {
5
+ const res = await axios.default.get(url);
6
+ const $ = cheerio.load(res.data);
7
+ const titleComponent = $('[data-testid="page.title"]').first();
8
+ const titleAndDescription = titleComponent.parent().parent().parent().text();
9
+ console.log(titleAndDescription);
10
+ const description = titleAndDescription
11
+ .replace(titleComponent.text(), "")
12
+ .trim();
13
+ const title = titleComponent.text().trim();
14
+ const content = $('[data-testid="page.contentEditor"]').first();
15
+ const contentHtml = $.html(content);
16
+ const nhm = new NodeHtmlMarkdown();
17
+ let markdown = nhm.translate(contentHtml);
18
+ // Keep headers on one line and increase their depth by one
19
+ markdown = markdown.replace(/# \n\n/g, "## ");
20
+ // Remove unnecessary nonwidth blank space characters
21
+ markdown = markdown.replace(/\u200b/g, "");
22
+ // Reduce unnecessary blank lines
23
+ markdown = markdown.replace(/\n\n\n/g, "\n\n");
24
+ // Mintlify doesn't support bolded headers, remove the asterisks
25
+ markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
26
+ return { title, description, markdown };
27
+ }
28
+ //# sourceMappingURL=scrapeGitBookPage.js.map
package/bin/scrapeGitBookPage.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../src/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IACjD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
package/bin/scrapeReadMe.js CHANGED
@@ -1,28 +1,60 @@
1
1
  import axios from "axios";
2
2
  import cheerio from "cheerio";
3
- import { NodeHtmlMarkdown } from "node-html-markdown";
4
- export async function scrapeReadMe(url) {
3
+ import path from "path";
4
+ import { scrapeReadMePage } from "./scrapeReadMePage.js";
5
+ import { createPage, getWebsite } from "./util.js";
6
+ export async function scrapeReadMe(url, cliDir) {
5
7
  const res = await axios.default.get(url);
6
8
  const $ = cheerio.load(res.data);
7
- const titleComponent = $("h1").first();
8
- const title = titleComponent.text().trim();
9
- const description = $(".markdown-body", titleComponent.parent())
10
- .text()
11
- .trim();
12
- const content = $(".content-body .markdown-body").first();
13
- const contentHtml = $.html(content);
14
- const nhm = new NodeHtmlMarkdown();
15
- let markdown = nhm.translate(contentHtml);
16
- // Keep headers on one line and increase their depth by one
17
- markdown = markdown.replace(/# \n\n/g, "## ");
18
- // Remove unnecessary nonwidth blank space characters
19
- markdown = markdown.replace(/\u200b/g, "");
20
- // Remove ReadMe anchor links
21
- markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
22
- // Reduce unnecessary blank lines
23
- markdown = markdown.replace(/\n\n\n/g, "\n\n");
24
- // Mintlify doesn't support bolded headers, remove the asterisks
25
- markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
26
- return { title, description, markdown };
9
+ const website = getWebsite(url);
10
+ // Get all the navigation sections
11
+ const navigationSections = $(".rm-Sidebar-section");
12
+ const groupsConfig = navigationSections
13
+ .map((i, section) => {
14
+ const sectionTitle = $(section).find("h3").first().text();
15
+ const linkPaths = $(section)
16
+ .find("a[href]")
17
+ .map((i, link) => {
18
+ const href = $(link).attr("href");
19
+ // Skip external links until Mintlify supports them
20
+ if (href.startsWith("https://") || href.startsWith("http://")) {
21
+ return undefined;
22
+ }
23
+ return href;
24
+ })
25
+ .toArray();
26
+ // Follows the same structure as mint.config.js
27
+ return {
28
+ group: sectionTitle,
29
+ pages: linkPaths,
30
+ };
31
+ })
32
+ .toArray();
33
+ return await Promise.all(groupsConfig.map(async (groupConfig) => {
34
+ groupConfig.pages = await Promise.all(groupConfig.pages.map(async (page) => {
35
+ // Scrape each page separately
36
+ const { title, description, markdown } = await scrapeReadMePage(website + page);
37
+ // Removes file name from the end
38
+ const splitSubpath = page.split("/");
39
+ let folders = splitSubpath
40
+ .slice(0, splitSubpath.length - 1)
41
+ .join("/");
42
+ // ReadMe requires a directory on all sections wheras we use root.
43
+ // docs/ is their home directory so we remove it
44
+ if (folders.startsWith("/docs")) {
45
+ folders = folders.replace("/docs", "");
46
+ }
47
+ const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
48
+ // Default to introduction.mdx if we encountered index.html
49
+ const fileName = splitSubpath.length === 2
50
+ ? "introduction"
51
+ : splitSubpath[splitSubpath.length - 1];
52
+ // Will create subfolders as needed
53
+ createPage(title, description, markdown, newFileLocation, fileName);
54
+ // Removes first slash if we are in a folder, Mintlify doesn't need it
55
+ return folders ? path.join(folders, fileName).substring(1) : fileName;
56
+ }));
57
+ return groupConfig;
58
+ }));
27
59
  }
28
60
  //# sourceMappingURL=scrapeReadMe.js.map
package/bin/scrapeReadMe.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"scrapeReadMe.js","sourceRoot":"","sources":["../src/scrapeReadMe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW;IAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC;SAC7D,IAAI,EAAE;SACN,IAAI,EAAE,CAAC;IAEV,MAAM,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
1
+ {"version":3,"file":"scrapeReadMe.js","sourceRoot":"","sources":["../src/scrapeReadMe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAU,MAAM,WAAW,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,MAAc;IAC5D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAEhC,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC;IAEpD,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAClB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC;aACzB,IAAI,CAAC,SAAS,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACf,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAElC,mDAAmD;YACnD,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC7D,OAAO,SAAS,CAAC;aAClB;YAED,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,+CAA+C;QAC/C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE,CAAC;IAEb,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QACrC,WAAW,CAAC,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC3C,8BAA8B;YAC9B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,gBAAgB,CAC7D,OAAO,GAAG,IAAI,CACf,CAAC;YAEF,iCAAiC;YACjC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,OAAO,GAAG,YAAY;iBACvB,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;iBACjC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,kEAAkE;YAClE,gDAAgD;YAChD,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC/B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;aACxC;YAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAEtE,2DAA2D;YAC3D,MAAM,QAAQ,GACZ,YAAY,CAAC,MAAM,KAAK,CAAC;gBACvB,CAAC,CAAC,cAAc;gBAChB,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAE5C,mCAAmC;YACnC,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,CAAC,CAAC;YAEpE,sEAAsE;YACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACxE,CAAC,CAAC,CACH,CAAC;QACF,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
package/bin/scrapeReadMePage.js ADDED
@@ -0,0 +1,28 @@
1
+ import axios from "axios";
2
+ import cheerio from "cheerio";
3
+ import { NodeHtmlMarkdown } from "node-html-markdown";
4
+ export async function scrapeReadMePage(url) {
5
+ const res = await axios.default.get(url);
6
+ const $ = cheerio.load(res.data);
7
+ const titleComponent = $("h1").first();
8
+ const title = titleComponent.text().trim();
9
+ const description = $(".markdown-body", titleComponent.parent())
10
+ .text()
11
+ .trim();
12
+ const content = $(".content-body .markdown-body").first();
13
+ const contentHtml = $.html(content);
14
+ const nhm = new NodeHtmlMarkdown();
15
+ let markdown = nhm.translate(contentHtml);
16
+ // Keep headers on one line and increase their depth by one
17
+ markdown = markdown.replace(/# \n\n/g, "## ");
18
+ // Remove unnecessary nonwidth blank space characters
19
+ markdown = markdown.replace(/\u200b/g, "");
20
+ // Remove ReadMe anchor links
21
+ markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
22
+ // Reduce unnecessary blank lines
23
+ markdown = markdown.replace(/\n\n\n/g, "\n\n");
24
+ // Mintlify doesn't support bolded headers, remove the asterisks
25
+ markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
26
+ return { title, description, markdown };
27
+ }
28
+ //# sourceMappingURL=scrapeReadMePage.js.map
package/bin/scrapeReadMePage.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../src/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAChD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC;SAC7D,IAAI,EAAE;SACN,IAAI,EAAE,CAAC;IAEV,MAAM,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACnC,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
package/bin/util.js CHANGED
@@ -1,5 +1,16 @@
1
- import { writeFileSync } from "fs";
1
+ import { mkdirSync, writeFileSync } from "fs";
2
2
  import { Page } from "./templates.js";
3
+ import path from "path";
4
+ export function getWebsite(url) {
5
+ // Gets the website
6
+ // eg. https://google.com -> https://google.com
7
+ // https://google.com/page -> https://google.com
8
+ return url.split("/").slice(0, 3).join("/");
9
+ }
10
+ export function objToReadableString(objs) {
11
+ // Two spaces as indentation
12
+ return objs.map((obj) => JSON.stringify(obj, null, 2)).join(",\n");
13
+ }
3
14
  export const toFilename = (title) => {
4
15
  // Gets rid of special characters at the start and end
5
16
  // of the name by converting to spaces then using trim.
@@ -9,7 +20,18 @@ export const toFilename = (title) => {
9
20
  .replace(/ /g, "-")
10
21
  .toLowerCase();
11
22
  };
12
- export const createPage = (title, description, markdown, rootDir = "") => {
13
- return writeFileSync(rootDir + toFilename(title) + ".mdx", Page(title, description, markdown));
23
+ export const addMdx = (fileName) => {
24
+ if (fileName.endsWith(".mdx")) {
25
+ return fileName;
26
+ }
27
+ return fileName + ".mdx";
28
+ };
29
+ export const createPage = (title, description, markdown, rootDir = "", fileName) => {
30
+ // Create the folders needed if they're missing
31
+ mkdirSync(rootDir, { recursive: true });
32
+ // Write the page to memory
33
+ const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
34
+ writeFileSync(writePath, Page(title, description, markdown));
35
+ console.log("✏️ - " + writePath);
14
36
  };
15
37
  //# sourceMappingURL=util.js.map
package/bin/util.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,UAAkB,EAAE,EACpB,EAAE;IACF,OAAO,aAAa,CAClB,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,EACpC,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CACnC,CAAC;AACJ,CAAC,CAAC"}
1
+ {"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,mBAAmB;IACnB,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAc;IAChD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,UAAkB,EAAE,EACpB,QAAiB,EACjB,EAAE;IACF,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5E,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;AACnC,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mintlify",
3
- "version": "1.0.5",
3
+ "version": "1.0.6",
4
4
  "description": "Mintlify CLI",
5
5
  "engines": {
6
6
  "node": ">=14.16"
package/src/index.ts CHANGED
@@ -3,8 +3,9 @@
3
3
  import { writeFileSync } from "fs";
4
4
  import inquirer from "inquirer";
5
5
  import { MintConfig } from "./templates.js";
6
- import { createPage, toFilename } from "./util.js";
7
- import { scrapeGitBook } from "./scrapeGitBook.js";
6
+ import { createPage, toFilename, objToReadableString } from "./util.js";
7
+ import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
8
+ import { scrapeReadMePage } from "./scrapeReadMePage.js";
8
9
  import { scrapeReadMe } from "./scrapeReadMe.js";
9
10
 
10
11
  const args = process.argv.slice(2);
@@ -99,16 +100,26 @@ if (command === "page") {
99
100
  });
100
101
  }
101
102
 
102
- if (command === "scrape-gitbook") {
103
+ if (command === "scrape-gitbook-page") {
103
104
  const url = args[1];
104
- const { title, description, markdown } = await scrapeGitBook(url);
105
- createPage(title, description, markdown, process.env.INIT_CWD);
105
+ const { title, description, markdown } = await scrapeGitBookPage(url);
106
+ createPage(title, description, markdown, process.cwd());
106
107
  process.exit(1);
107
108
  }
108
109
 
109
- if (command === "scrape-readme") {
110
+ if (command === "scrape-readme-page") {
110
111
  const url = args[1];
111
- const { title, description, markdown } = await scrapeReadMe(url);
112
- createPage(title, description, markdown, process.env.INIT_CWD);
112
+ const { title, description, markdown } = await scrapeReadMePage(url);
113
+ createPage(title, description, markdown, process.cwd());
114
+ process.exit(1);
115
+ }
116
+
117
+ if (command === "scrape-readme-section") {
118
+ const url = args[1];
119
+ console.log("Started scraping...");
120
+ const groupsConfig = await scrapeReadMe(url, process.cwd());
121
+ console.log("Finished scraping.");
122
+ console.log("Add the following to your navigation in mint.config.js:");
123
+ console.log(objToReadableString(groupsConfig));
113
124
  process.exit(1);
114
125
  }
package/src/{scrapeGitBook.ts → scrapeGitBookPage.ts} RENAMED
@@ -2,7 +2,7 @@ import axios from "axios";
2
2
  import cheerio from "cheerio";
3
3
  import { NodeHtmlMarkdown } from "node-html-markdown";
4
4
 
5
- export async function scrapeGitBook(url: string) {
5
+ export async function scrapeGitBookPage(url: string) {
6
6
  const res = await axios.default.get(url);
7
7
  const $ = cheerio.load(res.data);
8
8
 
package/src/scrapeReadMe.ts CHANGED
@@ -1,37 +1,79 @@
1
1
  import axios from "axios";
2
2
  import cheerio from "cheerio";
3
- import { NodeHtmlMarkdown } from "node-html-markdown";
3
+ import path from "path";
4
+ import { scrapeReadMePage } from "./scrapeReadMePage.js";
5
+ import { createPage, getWebsite, addMdx } from "./util.js";
4
6
 
5
- export async function scrapeReadMe(url: string) {
7
+ export async function scrapeReadMe(url: string, cliDir: string) {
6
8
  const res = await axios.default.get(url);
7
9
  const $ = cheerio.load(res.data);
10
+ const website = getWebsite(url);
8
11
 
9
- const titleComponent = $("h1").first();
10
- const title = titleComponent.text().trim();
11
- const description = $(".markdown-body", titleComponent.parent())
12
- .text()
13
- .trim();
12
+ // Get all the navigation sections
13
+ const navigationSections = $(".rm-Sidebar-section");
14
14
 
15
- const content = $(".content-body .markdown-body").first();
16
- const contentHtml = $.html(content);
15
+ const groupsConfig = navigationSections
16
+ .map((i, section) => {
17
+ const sectionTitle = $(section).find("h3").first().text();
18
+ const linkPaths = $(section)
19
+ .find("a[href]")
20
+ .map((i, link) => {
21
+ const href = $(link).attr("href");
17
22
 
18
- const nhm = new NodeHtmlMarkdown();
19
- let markdown = nhm.translate(contentHtml);
23
+ // Skip external links until Mintlify supports them
24
+ if (href.startsWith("https://") || href.startsWith("http://")) {
25
+ return undefined;
26
+ }
20
27
 
21
- // Keep headers on one line and increase their depth by one
22
- markdown = markdown.replace(/# \n\n/g, "## ");
28
+ return href;
29
+ })
30
+ .toArray();
23
31
 
24
- // Remove unnecessary nonwidth blank space characters
25
- markdown = markdown.replace(/\u200b/g, "");
32
+ // Follows the same structure as mint.config.js
33
+ return {
34
+ group: sectionTitle,
35
+ pages: linkPaths,
36
+ };
37
+ })
38
+ .toArray();
26
39
 
27
- // Remove ReadMe anchor links
28
- markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
40
+ return await Promise.all(
41
+ groupsConfig.map(async (groupConfig) => {
42
+ groupConfig.pages = await Promise.all(
43
+ groupConfig.pages.map(async (page: string) => {
44
+ // Scrape each page separately
45
+ const { title, description, markdown } = await scrapeReadMePage(
46
+ website + page
47
+ );
29
48
 
30
- // Reduce unnecessary blank lines
31
- markdown = markdown.replace(/\n\n\n/g, "\n\n");
49
+ // Removes file name from the end
50
+ const splitSubpath = page.split("/");
51
+ let folders = splitSubpath
52
+ .slice(0, splitSubpath.length - 1)
53
+ .join("/");
32
54
 
33
- // Mintlify doesn't support bolded headers, remove the asterisks
34
- markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
55
+ // ReadMe requires a directory on all sections wheras we use root.
56
+ // docs/ is their home directory so we remove it
57
+ if (folders.startsWith("/docs")) {
58
+ folders = folders.replace("/docs", "");
59
+ }
35
60
 
36
- return { title, description, markdown };
61
+ const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
62
+
63
+ // Default to introduction.mdx if we encountered index.html
64
+ const fileName =
65
+ splitSubpath.length === 2
66
+ ? "introduction"
67
+ : splitSubpath[splitSubpath.length - 1];
68
+
69
+ // Will create subfolders as needed
70
+ createPage(title, description, markdown, newFileLocation, fileName);
71
+
72
+ // Removes first slash if we are in a folder, Mintlify doesn't need it
73
+ return folders ? path.join(folders, fileName).substring(1) : fileName;
74
+ })
75
+ );
76
+ return groupConfig;
77
+ })
78
+ );
37
79
  }
package/src/scrapeReadMePage.ts ADDED
@@ -0,0 +1,37 @@
1
+ import axios from "axios";
2
+ import cheerio from "cheerio";
3
+ import { NodeHtmlMarkdown } from "node-html-markdown";
4
+
5
+ export async function scrapeReadMePage(url: string) {
6
+ const res = await axios.default.get(url);
7
+ const $ = cheerio.load(res.data);
8
+
9
+ const titleComponent = $("h1").first();
10
+ const title = titleComponent.text().trim();
11
+ const description = $(".markdown-body", titleComponent.parent())
12
+ .text()
13
+ .trim();
14
+
15
+ const content = $(".content-body .markdown-body").first();
16
+ const contentHtml = $.html(content);
17
+
18
+ const nhm = new NodeHtmlMarkdown();
19
+ let markdown = nhm.translate(contentHtml);
20
+
21
+ // Keep headers on one line and increase their depth by one
22
+ markdown = markdown.replace(/# \n\n/g, "## ");
23
+
24
+ // Remove unnecessary nonwidth blank space characters
25
+ markdown = markdown.replace(/\u200b/g, "");
26
+
27
+ // Remove ReadMe anchor links
28
+ markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
29
+
30
+ // Reduce unnecessary blank lines
31
+ markdown = markdown.replace(/\n\n\n/g, "\n\n");
32
+
33
+ // Mintlify doesn't support bolded headers, remove the asterisks
34
+ markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
35
+
36
+ return { title, description, markdown };
37
+ }
package/src/util.ts CHANGED
@@ -1,5 +1,18 @@
1
- import { writeFileSync } from "fs";
1
+ import { mkdirSync, writeFileSync } from "fs";
2
2
  import { Page } from "./templates.js";
3
+ import path from "path";
4
+
5
+ export function getWebsite(url: string) {
6
+ // Gets the website
7
+ // eg. https://google.com -> https://google.com
8
+ // https://google.com/page -> https://google.com
9
+ return url.split("/").slice(0, 3).join("/");
10
+ }
11
+
12
+ export function objToReadableString(objs: Object[]) {
13
+ // Two spaces as indentation
14
+ return objs.map((obj) => JSON.stringify(obj, null, 2)).join(",\n");
15
+ }
3
16
 
4
17
  export const toFilename = (title: string) => {
5
18
  // Gets rid of special characters at the start and end
@@ -11,14 +24,25 @@ export const toFilename = (title: string) => {
11
24
  .toLowerCase();
12
25
  };
13
26
 
27
+ export const addMdx = (fileName: string) => {
28
+ if (fileName.endsWith(".mdx")) {
29
+ return fileName;
30
+ }
31
+ return fileName + ".mdx";
32
+ };
33
+
14
34
  export const createPage = (
15
35
  title: string,
16
36
  description?: string,
17
37
  markdown?: string,
18
- rootDir: string = ""
38
+ rootDir: string = "",
39
+ fileName?: string
19
40
  ) => {
20
- return writeFileSync(
21
- rootDir + toFilename(title) + ".mdx",
22
- Page(title, description, markdown)
23
- );
41
+ // Create the folders needed if they're missing
42
+ mkdirSync(rootDir, { recursive: true });
43
+
44
+ // Write the page to memory
45
+ const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
46
+ writeFileSync(writePath, Page(title, description, markdown));
47
+ console.log("✏️ - " + writePath);
24
48
  };