mintlify 2.0.12 → 2.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/constants.js +1 -1
- package/bin/local-preview/index.js +11 -15
- package/bin/local-preview/index.js.map +1 -1
- package/bin/local-preview/listener/utils/createPage.js +1 -1
- package/bin/local-preview/listener/utils/createPage.js.map +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js +2 -4
- package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js +15 -2
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js +2 -2
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js +3 -3
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map +1 -1
- package/package.json +3 -2
- package/src/constants.ts +1 -1
- package/src/local-preview/index.ts +11 -19
- package/src/local-preview/listener/utils/createPage.ts +1 -1
- package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts +23 -25
- package/src/scraping/site-scrapers/scrapeGitBookPage.ts +17 -2
- package/src/scraping/site-scrapers/scrapeGitBookSection.ts +2 -3
- package/src/scraping/site-scrapers/scrapeReadMeSection.ts +3 -3
package/bin/constants.js
CHANGED
|
@@ -2,7 +2,7 @@ import path from "path";
|
|
|
2
2
|
import * as url from "url";
|
|
3
3
|
import os from "os";
|
|
4
4
|
// Change this to bump to a newer version of mint's client
|
|
5
|
-
export const TARGET_MINT_VERSION = "v0.0.
|
|
5
|
+
export const TARGET_MINT_VERSION = "v0.0.6";
|
|
6
6
|
// package installation location
|
|
7
7
|
export const INSTALL_PATH = url.fileURLToPath(new URL(".", import.meta.url));
|
|
8
8
|
export const HOME_DIR = os.homedir();
|
|
@@ -11,9 +11,6 @@ import { CLIENT_PATH, HOME_DIR, DOT_MINTLIFY, CMD_EXEC_PATH, TARGET_MINT_VERSION
|
|
|
11
11
|
import { buildLogger, ensureYarn } from "../util.js";
|
|
12
12
|
import listener from "./listener/index.js";
|
|
13
13
|
import { getConfigPath } from "./listener/utils/mintConfigFile.js";
|
|
14
|
-
const shellExec = (cmd) => {
|
|
15
|
-
return shell.exec(cmd, { silent: true });
|
|
16
|
-
};
|
|
17
14
|
const nodeModulesExists = async () => {
|
|
18
15
|
return pathExists(path.join(DOT_MINTLIFY, "mint", "client", "node_modules"));
|
|
19
16
|
};
|
|
@@ -50,23 +47,22 @@ const downloadTargetMint = async (logger) => {
|
|
|
50
47
|
});
|
|
51
48
|
logger.text = "Extracting Mintlify framework...";
|
|
52
49
|
const TAR_PATH = path.join(MINT_PATH, "mint.tar.gz");
|
|
53
|
-
// Unzip tar file
|
|
54
50
|
fse.writeFileSync(TAR_PATH, Buffer.from(downloadRes.data));
|
|
55
|
-
|
|
56
|
-
//
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
51
|
+
// strip-components 1 removes the top level directory from the unzipped content
|
|
52
|
+
// which is a folder with the release sha
|
|
53
|
+
fse.mkdirSync(path.join(MINT_PATH, "mint-tmp"));
|
|
54
|
+
shell.exec("tar -xzf mint.tar.gz -C mint-tmp --strip-components 1", {
|
|
55
|
+
silent: true,
|
|
56
|
+
});
|
|
61
57
|
fse.removeSync(TAR_PATH);
|
|
62
|
-
fse.moveSync(path.join(MINT_PATH,
|
|
58
|
+
fse.moveSync(path.join(MINT_PATH, "mint-tmp", "client"), path.join(CLIENT_PATH));
|
|
63
59
|
fse.writeFileSync(VERSION_PATH, TARGET_MINT_VERSION);
|
|
64
|
-
// Delete unnecessary
|
|
65
|
-
fse.removeSync(path.join(MINT_PATH,
|
|
60
|
+
// Delete unnecessary content downloaded from GitHub
|
|
61
|
+
fse.removeSync(path.join(MINT_PATH, "mint-tmp"));
|
|
66
62
|
logger.text = "Installing dependencies...";
|
|
67
63
|
ensureYarn(logger);
|
|
68
64
|
shell.cd(CLIENT_PATH);
|
|
69
|
-
|
|
65
|
+
shell.exec("yarn", { silent: true });
|
|
70
66
|
};
|
|
71
67
|
const checkForMintJson = async (logger) => {
|
|
72
68
|
const configPath = await getConfigPath(CMD_EXEC_PATH);
|
|
@@ -120,7 +116,7 @@ const dev = async (argv) => {
|
|
|
120
116
|
await checkForMintJson(logger);
|
|
121
117
|
shell.cd(CLIENT_PATH);
|
|
122
118
|
const relativePath = path.relative(CLIENT_PATH, CMD_EXEC_PATH);
|
|
123
|
-
|
|
119
|
+
child_process.spawnSync("yarn preconfigure", [relativePath], { shell: true });
|
|
124
120
|
logger.succeed("Local Mintlify instance is ready. Launching your site...");
|
|
125
121
|
run(argv.port || "3000");
|
|
126
122
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/local-preview/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,aAAa,MAAM,eAAe,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EACL,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,mBAAmB,EACnB,YAAY,EACZ,SAAS,GACV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,QAAQ,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AAEnE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/local-preview/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,aAAa,MAAM,eAAe,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EACL,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,mBAAmB,EACnB,YAAY,EACZ,SAAS,GACV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,QAAQ,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AAEnE,MAAM,iBAAiB,GAAG,KAAK,IAAI,EAAE;IACnC,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;AAC/E,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;IAC/B,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,QAAQ;aACX,MAAM,CAAC;YACN;gBACE,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,gDAAgD;gBACzD,OAAO,EAAE,IAAI;aACd;SACF,CAAC;aACD,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;YACpB,IAAI,OAAO,EAAE;gBACX,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;aACzC;iBAAM;gBACL,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;aACxC;QACH,CAAC,CAAC,CAAC;KACN;AACH,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAAE,MAAM,EAAE,EAAE;IAC1C,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAE5B,MAAM,CAAC,IAAI,GAAG,mCAAmC,CAAC;IAElD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAC7D,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,mBAAmB;KACzB,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,GAAG,kCAAkC,CAAC;IACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACrD,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAW,CAAC,CAAC,CAAC;IAElE,+EAA+E;IAC/E,yCAAyC;IACzC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,KAAK,CAAC,IAAI,CAAC,uDAAuD,EAAE;QAClE,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IAEH,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEzB,GAAG,CAAC,QAAQ,CACV,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,EAC1C,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CACvB,CAAC;IAEF,GAAG,CAAC,aAAa,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;IAErD,oDAAoD;IACpD,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;IAEjD,MAAM,CAAC,IAAI,GAAG,4BAA4B,CAAC;IAE3C,UAAU,CAAC,MAAM,CAAC,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AACvC,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAM,EAAE,EAAE;IACxC,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,aAAa,CAAC,CAAC;IACtD,IAAI,UAAU,IAAI,IAAI,EAAE;QACtB,MAAM,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO;AACT,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,KAAK,EAAE,IAAwB,EAAE,EAAE;IAC7C,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IACnB,MAAM,aAAa,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,WAAW,CAAC,sCAAsC,CAAC,CAAC;IACnE,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC/B,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IAEpB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,EAAE,CAAC;IAC7C,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,CAAC,EAAE;QACjD,MAAM,CAAC,IAAI,CACT,0EAA0E,CAC3E,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,QAAQ,EAAE;QACZ,MAAM,iBAAiB,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;QAEzD,IAAI,wBAAwB,GAAG,CAAC,iBAAiB,CAAC;QAElD,IAAI,iBAAiB,EAAE;YACrB,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YAC3D,IAAI,WAAW,KAAK,mBAAmB,EAAE;gBACvC,wBAAwB,GAAG,IAAI,CAAC;aACjC;SACF;QAED,IAAI,wBAAwB,EAAE;YAC5B,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;SAClC;KACF;IAED,IAAI,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,EAAE;QAChC,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,CAAC,IAAI,CAAC;;;;OAIX,CAAC,CAAC;SACJ;aAAM;YACL,MAAM,CAAC,IAAI,CAAC;;;;OAIX,CAAC,CAAC;SACJ;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC/B,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IACtB,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAC/D,aAAa,CAAC,SAAS,CAAC,mBAAmB,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9E,MAAM,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC;IAC3E,GAAG,CAAE,IAAI,CAAC,IAAe,IAAI,MAAM,CAAC,CAAC;AACvC,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE;IAC3B,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IAEtB,4DAA4D;IAC5D,2DAA2D;IAC3D,MAAM,kBAAkB,GAAG,aAAa,CAAC,KAAK,CAAC,mBAAmB,EAAE;QAClE,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,IAAI,EAAE,IAAI;SACX;QACD,GAAG,EAAE,WAAW;QAChB,KAAK,EAAE,MAAM;QACb,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC;IACH,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpB,IAAI,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,EAAE;YACrD,OAAO,CAAC,GAAG,CACT,MAAM,KAAK,CAAC,KAAK,CACf,uDAAuD,IAAI,EAAE,CAC9D,EAAE,CACJ,CAAC;YACF,OAAO,CAAC,GAAG,CACT,MAAM,KAAK,CAAC,KAAK,CAAC,kDAAkD,CAAC,EAAE,CACxE,CAAC;YACF,IAAI,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;SAClC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC9B,QAAQ,EAAE,CAAC;AACb,CAAC,CAAC;AAEF,eAAe,GAAG,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createPage.js","sourceRoot":"","sources":["../../../../src/local-preview/listener/utils/createPage.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,iBAAiB,MAAM,oBAAoB,CAAC;AACnD,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGzC,MAAM,UAAU,GAAG,KAAK,EACtB,QAAgB,EAChB,WAAmB,EACnB,oBAA4B,EAC5B,YAA2B,EAC3B,EAAE;IACF,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;IAC7C,IAAI;QACF,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;QAC3E,WAAW,GAAG,aAAa,CAAC;KAC7B;IAAC,OAAO,KAAK,EAAE;QACd,WAAW,GAAG,uEAAuE,CAAC;KACvF;IAED,4DAA4D;IAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC7C,IAAI,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACrC,wCAAwC;IACxC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,6BAA6B,CAC1D,YAAY,EACZ,QAAQ,EAAE,OAAO,CAClB,CAAC;IAEF,IAAI,KAAK,EAAE;QACT,YAAY,GAAG,KAAK,CAAC;KACtB;IAED,MAAM,YAAY,GAAG;QACnB,KAAK,EAAE,YAAY;QACnB,WAAW;QACX,GAAG,QAAQ;QACX,IAAI,EAAE,yBAAyB,CAAC,IAAI,CAAC;KACtC,CAAC;IAEF,OAAO;QACL,YAAY;QACZ,WAAW;QACX,IAAI,EAAE,kBAAkB,CAAC,IAAI,CAAC;KAC/B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EAAE,WAAW,EAAE,oBAAoB,EAAE,EAAE;IAC9D,MAAM,0BAA0B,GAAG,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,uBAAuB,GAAG,aAAa,CAAC,oBAAoB,CAAC,CAAC;QACpE,uBAAuB,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACvD,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE;gBAC1B,OAAO,CAAC,GAAG,EAAE,CAAC;aACf;QACH,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"createPage.js","sourceRoot":"","sources":["../../../../src/local-preview/listener/utils/createPage.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,iBAAiB,MAAM,oBAAoB,CAAC;AACnD,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGzC,MAAM,UAAU,GAAG,KAAK,EACtB,QAAgB,EAChB,WAAmB,EACnB,oBAA4B,EAC5B,YAA2B,EAC3B,EAAE;IACF,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;IAC7C,IAAI;QACF,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;QAC3E,WAAW,GAAG,aAAa,CAAC;KAC7B;IAAC,OAAO,KAAK,EAAE;QACd,WAAW,GAAG,uEAAuE,CAAC;KACvF;IAED,4DAA4D;IAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC7C,IAAI,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACrC,wCAAwC;IACxC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,6BAA6B,CAC1D,YAAY,EACZ,QAAQ,EAAE,OAAO,CAClB,CAAC;IAEF,IAAI,KAAK,EAAE;QACT,YAAY,GAAG,KAAK,CAAC;KACtB;IAED,MAAM,YAAY,GAAG;QACnB,KAAK,EAAE,YAAY;QACnB,WAAW;QACX,GAAG,QAAQ;QACX,IAAI,EAAE,yBAAyB,CAAC,IAAI,CAAC;KACtC,CAAC;IAEF,OAAO;QACL,YAAY;QACZ,WAAW;QACX,IAAI,EAAE,kBAAkB,CAAC,IAAI,CAAC;KAC/B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EAAE,WAAW,EAAE,oBAAoB,EAAE,EAAE;IAC9D,MAAM,0BAA0B,GAAG,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,uBAAuB,GAAG,aAAa,CAAC,oBAAoB,CAAC,CAAC;QACpE,uBAAuB,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACvD,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE;gBAC1B,OAAO,CAAC,GAAG,EAAE,CAAC;aACf;QACH,CAAC,CAAC,CAAC;QACH,OAAO,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC;IAEF,MAAM,2BAA2B,GAAG,GAAG,EAAE;QACvC,OAAO,CAAC,IAAI,EAAE,EAAE;YACd,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACnB,IAAI,IAAI,IAAI,IAAI,EAAE;oBAChB,OAAO;iBACR;gBACD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE;oBACjD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAC5C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,KAAK,CAC/B,CAAC;oBACF,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC;oBACpD;oBACE,mBAAmB;oBACnB,YAAY,KAAK,CAAC,CAAC;wBACnB,CAAC,aAAa,CAAC,OAAO,CAAC;wBACvB,CAAC,YAAY,CAAC,OAAO,CAAC,EACtB;wBACA,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK;4BACjC,0BAA0B,CAAC,OAAO,CAAC,CAAC;qBACvC;iBACF;qBAAM;gBACL,eAAe;gBACf,IAAI,CAAC,IAAI,KAAK,OAAO;oBACrB,IAAI,CAAC,GAAG;oBACR,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;oBACxB,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EACvB;oBACA,IAAI,CAAC,GAAG,GAAG,0BAA0B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACjD;YACH,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;SACxB,GAAG,CAAC,SAAS,CAAC;SACd,GAAG,CAAC,SAAS,CAAC;SACd,GAAG,CAAC,iBAAiB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;SACxC,GAAG,CAAC,2BAA2B,CAAC;SAChC,OAAO,CAAC,WAAW,CAAC,CAAC;IACxB,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,EAAE;IACjC,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,EAAE;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;AACrD,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;AAEtD,MAAM,6BAA6B,GAAG,CAAC,YAAY,EAAE,gBAAgB,EAAE,EAAE;IACvE,IAAI,YAAY,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,gBAAgB,IAAI,IAAI,EAAE;QACzE,OAAO,EAAE,CAAC;KACX;IAED,MAAM,EAAE,SAAS,EAAE,GAAG,oCAAoC,CACxD,YAAY,EACZ,gBAAgB,CACjB,CAAC;IAEF,IAAI,SAAS,IAAI,IAAI,EAAE;QACrB,OAAO,EAAE,CAAC;KACX;IAED,OAAO;QACL,KAAK,EAAE,SAAS,CAAC,OAAO;QACxB,WAAW,EAAE,SAAS,CAAC,WAAW;KACnC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,oCAAoC,GAAG,CAC3C,YAAY,EACZ,gBAAgB,EAChB,EAAE;IACF,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,GAClC,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;IAE7C,IAAI,IAAI,CAAC;IAET,YAAY,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAC9B,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACrE,MAAM,gBAAgB,GAAG,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC;QACjE,IAAI,WAAW,IAAI,gBAAgB,EAAE;YACnC,IAAI,GAAG,WAAW,CAAC;SACpB;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,IAAI,IAAI,EAAE;QAChB,OAAO,EAAE,CAAC;KACX;IAED,IAAI,SAAS,CAAC;IACd,IAAI,MAAM,EAAE;QACV,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;KACxC;SAAM;QACL,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC;KACrC;IAED,OAAO;QACL,SAAS;QACT,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAAC,gBAAgB,EAAE,EAAE;IACpD,MAAM,WAAW,GAAG,gCAAgC,CAAC;IACrD,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,EAAE,CAAC;IACxC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAE/C,MAAM,kBAAkB,GAAG,WAAW;QACpC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,gBAAgB,GAAG,WAAW;QAClC,CAAC,CAAC,kBAAkB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC;QAChD,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,CAAC;IAE1E,OAAO;QACL,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QAC3E,QAAQ,EAAE,gBAAgB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE;QAC7D,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;KAC1C,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,yBAAyB,CAAC,IAAI;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QACjC,OAAO,IAAI,CAAC;KACb;IACD,OAAO,GAAG,GAAG,IAAI,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,EAAE;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB;IACzF,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,yBAAyB;IAC3E,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,yBAAyB;IACtG,OAAO,YAAY,CAAC;AACtB,CAAC,CAAC;AAEF,eAAe,UAAU,CAAC"}
|
|
@@ -7,8 +7,7 @@ export function getDocusaurusLinksPerGroup(navigationSections, $, version) {
|
|
|
7
7
|
return [];
|
|
8
8
|
}
|
|
9
9
|
function getDocusaurusLinksPerGroupLoop(navigationSections, $) {
|
|
10
|
-
return navigationSections
|
|
11
|
-
.map((i, s) => {
|
|
10
|
+
return navigationSections.toArray().map((s) => {
|
|
12
11
|
const section = $(s);
|
|
13
12
|
// Links without a group
|
|
14
13
|
if (section.hasClass("theme-doc-sidebar-item-link")) {
|
|
@@ -30,7 +29,6 @@ function getDocusaurusLinksPerGroupLoop(navigationSections, $) {
|
|
|
30
29
|
group: sectionTitle || alternateGroupTitle(firstLink, pages),
|
|
31
30
|
pages: firstHref ? [firstHref, ...pages] : pages,
|
|
32
31
|
};
|
|
33
|
-
})
|
|
34
|
-
.toArray();
|
|
32
|
+
});
|
|
35
33
|
}
|
|
36
34
|
//# sourceMappingURL=getDocusaurusLinksPerGroup.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getDocusaurusLinksPerGroup.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,MAAM,2BAA2B,CAAC;AAC5D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,UAAU,0BAA0B,CACxC,kBAAuB,EACvB,CAAM,EACN,OAA2B;IAE3B,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,GAAG,EAAE;QACtC,OAAO,8BAA8B,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;KAC9D;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,8BAA8B,CAAC,kBAAuB,EAAE,CAAM;IACrE,OAAO,kBAAkB
|
|
1
|
+
{"version":3,"file":"getDocusaurusLinksPerGroup.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,MAAM,2BAA2B,CAAC;AAC5D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,UAAU,0BAA0B,CACxC,kBAAuB,EACvB,CAAM,EACN,OAA2B;IAE3B,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,GAAG,EAAE;QACtC,OAAO,8BAA8B,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;KAC9D;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,8BAA8B,CAAC,kBAAuB,EAAE,CAAM;IACrE,OAAO,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE;QACpD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,wBAAwB;QACxB,IAAI,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,SAAS,GAAG,OAAO;aACtB,IAAI,CAAC,8BAA8B,CAAC;aACpC,KAAK,EAAE;aACP,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnB,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -12,6 +12,19 @@ export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, over
|
|
|
12
12
|
.trim();
|
|
13
13
|
const title = titleComponent.text().trim();
|
|
14
14
|
const content = $('[data-testid="page.contentEditor"]').first();
|
|
15
|
+
// Replace code blocks with parseable html
|
|
16
|
+
const codeBlocks = content.find('[spellcheck="false"] div');
|
|
17
|
+
codeBlocks.each((i, c) => {
|
|
18
|
+
const code = $(c);
|
|
19
|
+
code.find('[contenteditable="false"]').empty();
|
|
20
|
+
const codeContent = code
|
|
21
|
+
.children()
|
|
22
|
+
.toArray()
|
|
23
|
+
.map((d) => $(d).text())
|
|
24
|
+
.filter((text) => text !== "")
|
|
25
|
+
.join("\n");
|
|
26
|
+
code.replaceWith(`<pre><code>${codeContent}</code></pre>`);
|
|
27
|
+
});
|
|
15
28
|
const contentHtml = $.html(content);
|
|
16
29
|
const modifyFileName = (fileName) =>
|
|
17
30
|
// Remove GitBook metadata from the start
|
|
@@ -20,8 +33,8 @@ export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, over
|
|
|
20
33
|
const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite, modifyFileName);
|
|
21
34
|
const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
|
|
22
35
|
let markdown = nhm.translate(contentHtml);
|
|
23
|
-
// Keep headers on one line
|
|
24
|
-
markdown = markdown.replace(/# \n\n/g, "
|
|
36
|
+
// Keep headers on one line
|
|
37
|
+
markdown = markdown.replace(/# \n\n/g, "# ");
|
|
25
38
|
// Remove unnecessary nonwidth blank space characters
|
|
26
39
|
markdown = markdown.replace(/\u200b/g, "");
|
|
27
40
|
// Reduce unnecessary blank lines
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,0CAA0C;IAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC5D,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC;QAC/C,MAAM,WAAW,GAAG,IAAI;aACrB,QAAQ,EAAE;aACV,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;aAC7B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,cAAc,WAAW,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,EAAE;IAClC,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -23,7 +23,8 @@ export async function scrapeGitBookSection(html, origin, cliDir, imageBaseDir, o
|
|
|
23
23
|
.children();
|
|
24
24
|
// Get all links per group
|
|
25
25
|
const groupsConfig = navigationSections
|
|
26
|
-
.
|
|
26
|
+
.toArray()
|
|
27
|
+
.map((s) => {
|
|
27
28
|
const section = $(s);
|
|
28
29
|
const sectionTitle = $(section)
|
|
29
30
|
.find('div > div[dir="auto"]')
|
|
@@ -39,7 +40,6 @@ export async function scrapeGitBookSection(html, origin, cliDir, imageBaseDir, o
|
|
|
39
40
|
pages: firstHref ? [firstHref, ...pages] : pages,
|
|
40
41
|
};
|
|
41
42
|
})
|
|
42
|
-
.toArray()
|
|
43
43
|
.filter(Boolean);
|
|
44
44
|
// Merge groups with empty titles together
|
|
45
45
|
const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,4CAA4C,CAAC;SAC5D,KAAK,EAAE;SACP,IAAI,CAAC,KAAK,CAAC,CAAC;IACf,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,kCAAkC;IAClC,4FAA4F;IAC5F,yDAAyD;IACzD,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,0BAA0B,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC9D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,EACJ,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
|
|
@@ -15,7 +15,8 @@ export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, ov
|
|
|
15
15
|
.first()
|
|
16
16
|
.find(".rm-Sidebar-section");
|
|
17
17
|
const groupsConfig = navigationSections
|
|
18
|
-
.
|
|
18
|
+
.toArray()
|
|
19
|
+
.map((s) => {
|
|
19
20
|
const section = $(s);
|
|
20
21
|
const sectionTitle = section.find("h3").first().text();
|
|
21
22
|
// Get all links, then use filter to remove duplicates.
|
|
@@ -30,8 +31,7 @@ export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, ov
|
|
|
30
31
|
group: sectionTitle,
|
|
31
32
|
pages: pages,
|
|
32
33
|
};
|
|
33
|
-
})
|
|
34
|
-
.toArray();
|
|
34
|
+
});
|
|
35
35
|
// Scrape each link in the navigation.
|
|
36
36
|
const groupsConfigCleanPaths = await Promise.all(groupsConfig.map(async (navEntry) => {
|
|
37
37
|
return await scrapeGettingFileNameFromUrl(
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0CAA0C,CAAC;AAC3E,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,2DAA2D;IAC3D,2DAA2D;IAC3D,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC;SACxC,KAAK,EAAE;SACP,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAE/B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,GAAG,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0CAA0C,CAAC;AAC3E,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,2DAA2D;IAC3D,2DAA2D;IAC3D,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC;SACxC,KAAK,EAAE;SACP,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAE/B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAEvD,uDAAuD;QACvD,yDAAyD;QACzD,+BAA+B;QAC/B,iDAAiD;QACjD,iCAAiC;QACjC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,MAAM,CACvD,CAAC,KAAa,EAAE,KAAa,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CACtE,CAAC;QAEF,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,KAAK;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QACvD,OAAO,MAAM,4BAA4B;QACvC,mEAAmE;QACnE,mDAAmD;QACnD,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,gBAAgB,EAChB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mintlify",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.15",
|
|
4
4
|
"description": "The Mintlify CLI",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=18.0.0"
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"cheerio": "^0.22.0",
|
|
28
28
|
"chokidar": "^3.5.3",
|
|
29
29
|
"favicons": "^7.0.1",
|
|
30
|
-
"fs-extra": "^
|
|
30
|
+
"fs-extra": "^11.1.0",
|
|
31
31
|
"gray-matter": "^4.0.3",
|
|
32
32
|
"inquirer": "^9.1.0",
|
|
33
33
|
"is-absolute-url": "^4.0.1",
|
|
@@ -47,6 +47,7 @@
|
|
|
47
47
|
"yargs": "^17.6.0"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
+
"@types/cheerio": "^0.22.31",
|
|
50
51
|
"@types/fs-extra": "^9.0.13",
|
|
51
52
|
"@types/inquirer": "^9.0.1",
|
|
52
53
|
"@types/node": "^18.7.13",
|
package/src/constants.ts
CHANGED
|
@@ -3,7 +3,7 @@ import * as url from "url";
|
|
|
3
3
|
import os from "os";
|
|
4
4
|
|
|
5
5
|
// Change this to bump to a newer version of mint's client
|
|
6
|
-
export const TARGET_MINT_VERSION = "v0.0.
|
|
6
|
+
export const TARGET_MINT_VERSION = "v0.0.6";
|
|
7
7
|
|
|
8
8
|
// package installation location
|
|
9
9
|
export const INSTALL_PATH = url.fileURLToPath(new URL(".", import.meta.url));
|
|
@@ -21,10 +21,6 @@ import listener from "./listener/index.js";
|
|
|
21
21
|
import { ArgumentsCamelCase } from "yargs";
|
|
22
22
|
import { getConfigPath } from "./listener/utils/mintConfigFile.js";
|
|
23
23
|
|
|
24
|
-
const shellExec = (cmd: string) => {
|
|
25
|
-
return shell.exec(cmd, { silent: true });
|
|
26
|
-
};
|
|
27
|
-
|
|
28
24
|
const nodeModulesExists = async () => {
|
|
29
25
|
return pathExists(path.join(DOT_MINTLIFY, "mint", "client", "node_modules"));
|
|
30
26
|
};
|
|
@@ -65,36 +61,32 @@ const downloadTargetMint = async (logger) => {
|
|
|
65
61
|
|
|
66
62
|
logger.text = "Extracting Mintlify framework...";
|
|
67
63
|
const TAR_PATH = path.join(MINT_PATH, "mint.tar.gz");
|
|
68
|
-
|
|
69
|
-
// Unzip tar file
|
|
70
64
|
fse.writeFileSync(TAR_PATH, Buffer.from(downloadRes.data as any));
|
|
71
|
-
shellExec("tar -xzf mint.tar.gz");
|
|
72
65
|
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
).stdout.trim();
|
|
66
|
+
// strip-components 1 removes the top level directory from the unzipped content
|
|
67
|
+
// which is a folder with the release sha
|
|
68
|
+
fse.mkdirSync(path.join(MINT_PATH, "mint-tmp"));
|
|
69
|
+
shell.exec("tar -xzf mint.tar.gz -C mint-tmp --strip-components 1", {
|
|
70
|
+
silent: true,
|
|
71
|
+
});
|
|
80
72
|
|
|
81
73
|
fse.removeSync(TAR_PATH);
|
|
82
74
|
|
|
83
75
|
fse.moveSync(
|
|
84
|
-
path.join(MINT_PATH,
|
|
76
|
+
path.join(MINT_PATH, "mint-tmp", "client"),
|
|
85
77
|
path.join(CLIENT_PATH)
|
|
86
78
|
);
|
|
87
79
|
|
|
88
80
|
fse.writeFileSync(VERSION_PATH, TARGET_MINT_VERSION);
|
|
89
81
|
|
|
90
|
-
// Delete unnecessary
|
|
91
|
-
fse.removeSync(path.join(MINT_PATH,
|
|
82
|
+
// Delete unnecessary content downloaded from GitHub
|
|
83
|
+
fse.removeSync(path.join(MINT_PATH, "mint-tmp"));
|
|
92
84
|
|
|
93
85
|
logger.text = "Installing dependencies...";
|
|
94
86
|
|
|
95
87
|
ensureYarn(logger);
|
|
96
88
|
shell.cd(CLIENT_PATH);
|
|
97
|
-
|
|
89
|
+
shell.exec("yarn", { silent: true });
|
|
98
90
|
};
|
|
99
91
|
|
|
100
92
|
const checkForMintJson = async (logger) => {
|
|
@@ -157,7 +149,7 @@ const dev = async (argv: ArgumentsCamelCase) => {
|
|
|
157
149
|
await checkForMintJson(logger);
|
|
158
150
|
shell.cd(CLIENT_PATH);
|
|
159
151
|
const relativePath = path.relative(CLIENT_PATH, CMD_EXEC_PATH);
|
|
160
|
-
|
|
152
|
+
child_process.spawnSync("yarn preconfigure", [relativePath], { shell: true });
|
|
161
153
|
logger.succeed("Local Mintlify instance is ready. Launching your site...");
|
|
162
154
|
run((argv.port as string) || "3000");
|
|
163
155
|
};
|
|
@@ -13,34 +13,32 @@ export function getDocusaurusLinksPerGroup(
|
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
function getDocusaurusLinksPerGroupLoop(navigationSections: any, $: any) {
|
|
16
|
-
return navigationSections
|
|
17
|
-
|
|
18
|
-
const section = $(s);
|
|
16
|
+
return navigationSections.toArray().map((s: string) => {
|
|
17
|
+
const section = $(s);
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
19
|
+
// Links without a group
|
|
20
|
+
if (section.hasClass("theme-doc-sidebar-item-link")) {
|
|
21
|
+
const linkHref = section.find("a[href]").first().attr("href");
|
|
22
|
+
return {
|
|
23
|
+
group: "",
|
|
24
|
+
pages: [linkHref],
|
|
25
|
+
};
|
|
26
|
+
}
|
|
28
27
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
28
|
+
const firstLink = section
|
|
29
|
+
.find(".menu__list-item-collapsible")
|
|
30
|
+
.first()
|
|
31
|
+
.find("a[href]");
|
|
33
32
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
const sectionTitle = firstLink.text();
|
|
34
|
+
const firstHref = firstLink.attr("href");
|
|
35
|
+
const linkSections = section.children().eq(1).children();
|
|
37
36
|
|
|
38
|
-
|
|
37
|
+
const pages = getLinksRecursively(linkSections, $);
|
|
39
38
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
.toArray();
|
|
39
|
+
return {
|
|
40
|
+
group: sectionTitle || alternateGroupTitle(firstLink, pages),
|
|
41
|
+
pages: firstHref ? [firstHref, ...pages] : pages,
|
|
42
|
+
};
|
|
43
|
+
});
|
|
46
44
|
}
|
|
@@ -22,6 +22,21 @@ export async function scrapeGitBookPage(
|
|
|
22
22
|
const title = titleComponent.text().trim();
|
|
23
23
|
|
|
24
24
|
const content = $('[data-testid="page.contentEditor"]').first();
|
|
25
|
+
|
|
26
|
+
// Replace code blocks with parseable html
|
|
27
|
+
const codeBlocks = content.find('[spellcheck="false"] div');
|
|
28
|
+
codeBlocks.each((i, c) => {
|
|
29
|
+
const code = $(c);
|
|
30
|
+
code.find('[contenteditable="false"]').empty();
|
|
31
|
+
const codeContent = code
|
|
32
|
+
.children()
|
|
33
|
+
.toArray()
|
|
34
|
+
.map((d) => $(d).text())
|
|
35
|
+
.filter((text) => text !== "")
|
|
36
|
+
.join("\n");
|
|
37
|
+
code.replaceWith(`<pre><code>${codeContent}</code></pre>`);
|
|
38
|
+
});
|
|
39
|
+
|
|
25
40
|
const contentHtml = $.html(content);
|
|
26
41
|
|
|
27
42
|
const modifyFileName = (fileName) =>
|
|
@@ -41,8 +56,8 @@ export async function scrapeGitBookPage(
|
|
|
41
56
|
const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
|
|
42
57
|
let markdown = nhm.translate(contentHtml);
|
|
43
58
|
|
|
44
|
-
// Keep headers on one line
|
|
45
|
-
markdown = markdown.replace(/# \n\n/g, "
|
|
59
|
+
// Keep headers on one line
|
|
60
|
+
markdown = markdown.replace(/# \n\n/g, "# ");
|
|
46
61
|
|
|
47
62
|
// Remove unnecessary nonwidth blank space characters
|
|
48
63
|
markdown = markdown.replace(/\u200b/g, "");
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import cheerio from "cheerio";
|
|
2
|
-
import path from "path";
|
|
3
2
|
import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
|
|
4
3
|
import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
|
|
5
4
|
import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
|
|
@@ -37,7 +36,8 @@ export async function scrapeGitBookSection(
|
|
|
37
36
|
|
|
38
37
|
// Get all links per group
|
|
39
38
|
const groupsConfig: MintNavigation[] = navigationSections
|
|
40
|
-
.
|
|
39
|
+
.toArray()
|
|
40
|
+
.map((s: cheerio.Element) => {
|
|
41
41
|
const section = $(s);
|
|
42
42
|
const sectionTitle = $(section)
|
|
43
43
|
.find('div > div[dir="auto"]')
|
|
@@ -56,7 +56,6 @@ export async function scrapeGitBookSection(
|
|
|
56
56
|
pages: firstHref ? [firstHref, ...pages] : pages,
|
|
57
57
|
};
|
|
58
58
|
})
|
|
59
|
-
.toArray()
|
|
60
59
|
.filter(Boolean);
|
|
61
60
|
|
|
62
61
|
// Merge groups with empty titles together
|
|
@@ -26,7 +26,8 @@ export async function scrapeReadMeSection(
|
|
|
26
26
|
.find(".rm-Sidebar-section");
|
|
27
27
|
|
|
28
28
|
const groupsConfig: MintNavigation[] = navigationSections
|
|
29
|
-
.
|
|
29
|
+
.toArray()
|
|
30
|
+
.map((s: cheerio.Element) => {
|
|
30
31
|
const section = $(s);
|
|
31
32
|
const sectionTitle = section.find("h3").first().text();
|
|
32
33
|
|
|
@@ -45,8 +46,7 @@ export async function scrapeReadMeSection(
|
|
|
45
46
|
group: sectionTitle,
|
|
46
47
|
pages: pages,
|
|
47
48
|
};
|
|
48
|
-
})
|
|
49
|
-
.toArray();
|
|
49
|
+
});
|
|
50
50
|
|
|
51
51
|
// Scrape each link in the navigation.
|
|
52
52
|
const groupsConfigCleanPaths = await Promise.all(
|