mintlify 2.0.15 → 2.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/constants.js +1 -1
- package/bin/local-preview/listener/index.js +8 -3
- package/bin/local-preview/listener/index.js.map +1 -1
- package/bin/local-preview/listener/utils.js +6 -1
- package/bin/local-preview/listener/utils.js.map +1 -1
- package/package.json +2 -2
- package/scraper.md +120 -0
- package/src/constants.ts +1 -1
- package/src/local-preview/listener/index.ts +13 -5
- package/src/local-preview/listener/utils.ts +10 -1
package/bin/constants.js
CHANGED
|
@@ -2,7 +2,7 @@ import path from "path";
|
|
|
2
2
|
import * as url from "url";
|
|
3
3
|
import os from "os";
|
|
4
4
|
// Change this to bump to a newer version of mint's client
|
|
5
|
-
export const TARGET_MINT_VERSION = "v0.0.
|
|
5
|
+
export const TARGET_MINT_VERSION = "v0.0.8";
|
|
6
6
|
// package installation location
|
|
7
7
|
export const INSTALL_PATH = url.fileURLToPath(new URL(".", import.meta.url));
|
|
8
8
|
export const HOME_DIR = os.homedir();
|
|
@@ -3,7 +3,7 @@ import fse from "fs-extra";
|
|
|
3
3
|
import pathUtil from "path";
|
|
4
4
|
import Chalk from "chalk";
|
|
5
5
|
import mintValidation from "@mintlify/validation";
|
|
6
|
-
import { openApiCheck } from "./utils.js";
|
|
6
|
+
import { isFileSizeValid, openApiCheck } from "./utils.js";
|
|
7
7
|
import { updateGeneratedNav, updateOpenApiFiles } from "./update.js";
|
|
8
8
|
import { CLIENT_PATH, CMD_EXEC_PATH } from "../../constants.js";
|
|
9
9
|
import { promises as _promises } from "fs";
|
|
@@ -172,7 +172,7 @@ const onUpdateEvent = async (filename) => {
|
|
|
172
172
|
case "potentialYamlOpenApiSpec":
|
|
173
173
|
case "potentialJsonOpenApiSpec":
|
|
174
174
|
let isOpenApi = false;
|
|
175
|
-
const openApiInfo = await openApiCheck(
|
|
175
|
+
const openApiInfo = await openApiCheck(filePath);
|
|
176
176
|
isOpenApi = openApiInfo.isOpenApi;
|
|
177
177
|
if (isOpenApi) {
|
|
178
178
|
// TODO: Instead of re-generating all openApi files, optimize by just updating the specific file that changed.
|
|
@@ -182,7 +182,12 @@ const onUpdateEvent = async (filename) => {
|
|
|
182
182
|
}
|
|
183
183
|
break;
|
|
184
184
|
case "staticFile":
|
|
185
|
-
await
|
|
185
|
+
if (await isFileSizeValid(filePath, 5)) {
|
|
186
|
+
await fse.copy(filePath, targetPath);
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
console.error(Chalk.red(`🚨 The file at ${filename} is too big. The maximum file size is 5 mb.`));
|
|
190
|
+
}
|
|
186
191
|
break;
|
|
187
192
|
}
|
|
188
193
|
if (regenerateNav) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/local-preview/listener/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,QAAQ,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,cAAc,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/local-preview/listener/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,QAAQ,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,cAAc,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,MAAM,IAAI,CAAC;AAC3C,OAAO,UAAU,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;AAE/B,MAAM,QAAQ,GAAG,GAAG,EAAE;IACpB,QAAQ;SACL,KAAK,CAAC,aAAa,EAAE;QACpB,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,CAAC;QAC1C,GAAG,EAAE,aAAa;KACnB,CAAC;SACD,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE;QACpC,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC/C,QAAQ,QAAQ,EAAE;gBAChB,KAAK,MAAM;oBACT,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;oBAC7C,MAAM;gBACR,KAAK,SAAS;oBACZ,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,QAAQ,CAAC,CAAC;oBAChD,MAAM;gBACR,KAAK,YAAY;oBACf,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;oBAC5B,MAAM;gBACR,KAAK,SAAS;oBACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,QAAQ,CAAC,CAAC;oBAC9C,MAAM;gBACR,KAAK,YAAY;oBACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;oBAC7C,MAAM;aACT;SACF;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;SAC9B;IACH,CAAC,CAAC;SACD,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC/C,QAAQ,QAAQ,EAAE;gBAChB,KAAK,MAAM;oBACT,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;oBACvC,MAAM;gBACR,KAAK,SAAS;oBACZ,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC;oBAC1C,MAAM;gBACR,KAAK,YAAY;oBACf,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;oBAC7B,MAAM;gBACR,KAAK,SAAS;oBACZ,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;oBAC/C,MAAM;gBACR,KAAK,YAAY;oBACf,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,QAAQ,CAAC,CAAC;oBAC9C,MAAM;aACT;SACF;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;SAC9B;IACH,CAAC,CAAC;SACD,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI;YACF,MAAM,iBAAiB,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,aAAa,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC;YAC9D,IACE,iBAAiB,KAAK,MAAM;gBAC5B,iBAAiB,KAAK,SAAS;gBAC/B,iBAAiB,KAAK,YAAY;gBAClC,iBAAiB,KAAK,YAAY,EAClC;gBACA,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;aAC9B;YACD,QAAQ,iBAAiB,EAAE;gBACzB,KAAK,MAAM;oBACT,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;oBACzC,MAAM;gBACR,KAAK,SAAS;oBACZ,OAAO,CAAC,GAAG,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;oBAC5C,MAAM;gBACR,KAAK,YAAY;oBACf,OAAO,CAAC,GAAG,CACT,8EAA8E,CAC/E,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,KAAK,0BAA0B,CAAC;gBAChC,KAAK,0BAA0B;oBAC7B,MAAM,kBAAkB,EAAE,CAAC;oBAC3B,MAAM,kBAAkB,EAAE,CAAC;oBAC3B,MAAM;gBACR,KAAK,YAAY;oBACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;oBAC/C,MAAM;aACT;SACF;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;SAC9B;IACH,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CACpB,iBAAwC,EACxC,QAAgB,EACR,EAAE;IACV,QAAQ,iBAAiB,EAAE;QACzB,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS;YACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/D,KAAK,YAAY;YACf,OAAO,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAClE,KAAK,0BAA0B,CAAC;QAChC,KAAK,0BAA0B;YAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC;QAC1E,KAAK,YAAY;YACf,OAAO,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxD;YACE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;KACvC;AACH,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,aAAa,GAAG,KAAK,EAAE,QAAgB,EAAyB,EAAE;IACtE,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,aAAa,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC;IAC9D,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,IAAI,QAAQ,GACV,iBAAiB,KAAK,0BAA0B;QAChD,iBAAiB,KAAK,0BAA0B;QAC9C,CAAC,CAAC,YAAY;QACd,CAAC,CAAC,iBAAiB,CAAC;IAExB,QAAQ,iBAAiB,EAAE;QACzB,KAAK,MAAM;YACT,aAAa,GAAG,IAAI,CAAC;YAErB,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACzD,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,UAAU,CACtC,QAAQ,EACR,UAAU,EACV,aAAa,EACb,EAAE,CACH,CAAC;YACF,MAAM,GAAG,CAAC,UAAU,CAAC,UAAU,EAAE,WAAW,EAAE;gBAC5C,IAAI,EAAE,GAAG;aACV,CAAC,CAAC;YACH,MAAM;QACR,KAAK,SAAS;YACZ,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACrC,MAAM;QACR,KAAK,YAAY;YACf,aAAa,GAAG,IAAI,CAAC;YAErB,MAAM,mBAAmB,GAAG,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClE,IAAI;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;gBACnD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAChC,cAAc,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;gBAEhD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;oBACvB,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC1C,CAAC,CAAC,CAAC;gBAEH,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC3B,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC9C,CAAC,CAAC,CAAC;gBAEH,IAAI,MAAM,KAAK,SAAS,EAAE;oBACxB,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;iBACtC;aACF;YAAC,OAAO,KAAK,EAAE;gBACd,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE;oBAChC,OAAO,CAAC,KAAK,CACX,MAAM,KAAK,CAAC,GAAG,CACb,6KAA6K,CAC9K,EAAE,CACJ,CAAC;iBACH;qBAAM;oBACL,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;iBACjD;aACF;YAED,MAAM;QACR,KAAK,0BAA0B,CAAC;QAChC,KAAK,0BAA0B;YAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;YACtB,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;YACjD,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC;YAClC,IAAI,SAAS,EAAE;gBACb,8GAA8G;gBAC9G,MAAM,kBAAkB,EAAE,CAAC;gBAC3B,aAAa,GAAG,IAAI,CAAC;gBACrB,QAAQ,GAAG,SAAS,CAAC;aACtB;YACD,MAAM;QACR,KAAK,YAAY;YACf,IAAI,MAAM,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE;gBACtC,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;aACtC;iBAAM;gBACL,OAAO,CAAC,KAAK,CACX,KAAK,CAAC,GAAG,CACP,kBAAkB,QAAQ,6CAA6C,CACxE,CACF,CAAC;aACH;YACD,MAAM;KACT;IACD,IAAI,aAAa,EAAE;QACjB,2GAA2G;QAC3G,MAAM,kBAAkB,EAAE,CAAC;KAC5B;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,eAAe,QAAQ,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import SwaggerParser from "@apidevtools/swagger-parser";
|
|
2
2
|
import { promises as _promises } from "fs";
|
|
3
|
-
const { readdir } = _promises;
|
|
3
|
+
const { readdir, stat } = _promises;
|
|
4
4
|
export const getFileExtension = (filename) => {
|
|
5
5
|
return (filename.substring(filename.lastIndexOf(".") + 1, filename.length) ||
|
|
6
6
|
filename);
|
|
@@ -59,4 +59,9 @@ export const getFileList = async (dirName, og = dirName) => {
|
|
|
59
59
|
}
|
|
60
60
|
return files;
|
|
61
61
|
};
|
|
62
|
+
export const isFileSizeValid = async (path, maxFileSizeInMb) => {
|
|
63
|
+
const maxFileSizeBytes = maxFileSizeInMb * 1000000;
|
|
64
|
+
const stats = await stat(path);
|
|
65
|
+
return stats.size <= maxFileSizeBytes;
|
|
66
|
+
};
|
|
62
67
|
//# sourceMappingURL=utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/local-preview/listener/utils.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,MAAM,IAAI,CAAC;AAE3C,MAAM,EAAE,OAAO,EAAE,GAAG,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/local-preview/listener/utils.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,MAAM,IAAI,CAAC;AAE3C,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC;AAEpC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,EAAE;IAC3C,OAAO,CACL,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC;QAClE,QAAQ,CACT,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE;IACzC,IAAI,IAAI,CAAC;IACT,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI;QACF,IAAI,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC1C,SAAS,GAAG,IAAI,CAAC;KAClB;IAAC,MAAM;QACN,oBAAoB;KACrB;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC7B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,EAAE;IAC5C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG;QACf,GAAG,KAAK;QACR,KAAK,EAAE,QAAQ;KAChB,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,EAAE;IACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QACzB,OAAO,EAAE,CAAC;KACX;IACD,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACrB,IAAI,IAAI,IAAI,IAAI,EAAE;YAChB,OAAO;SACR;QACD,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE;YAChC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACzB;aAAM;YACL,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACrB;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,GAAG,OAAO,EAAE,EAAE;IACjE,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,CAAC,MAAM,WAAW,CAAC,GAAG,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;SAC3E;aAAM;YACL,MAAM,IAAI,GAAG,GAAG,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAClB;KACF;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,IAAY,EACZ,eAAuB,EACL,EAAE;IACpB,MAAM,gBAAgB,GAAG,eAAe,GAAG,OAAO,CAAC;IACnD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,IAAI,IAAI,gBAAgB,CAAC;AACxC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mintlify",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.17",
|
|
4
4
|
"description": "The Mintlify CLI",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=18.0.0"
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"type": "module",
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"@apidevtools/swagger-parser": "^10.1.0",
|
|
23
|
-
"@mintlify/validation": "^0.1.
|
|
23
|
+
"@mintlify/validation": "^0.1.5",
|
|
24
24
|
"@octokit/rest": "^19.0.5",
|
|
25
25
|
"axios": "^1.2.2",
|
|
26
26
|
"chalk": "^5.1.0",
|
package/scraper.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# CLI Scraping
|
|
2
|
+
|
|
3
|
+
The CLI has many commands. This doc focuses on how we coded scraping websites.
|
|
4
|
+
|
|
5
|
+
## User Interface
|
|
6
|
+
|
|
7
|
+
There are two main commands:
|
|
8
|
+
|
|
9
|
+
`mintlify scrape-page [url]`
|
|
10
|
+
|
|
11
|
+
and
|
|
12
|
+
|
|
13
|
+
`mintlify scrape-section [url]`
|
|
14
|
+
|
|
15
|
+
Scraping a page downloads a single page’s content. Scraping a section goes through the navigation and scrapes each page. The code for downloading a page’s content is shared between the two commands.
|
|
16
|
+
|
|
17
|
+
Important files: `scraping/scrapePageCommands.ts`, `scraping/scrapeSectionAutomatically.ts`
|
|
18
|
+
|
|
19
|
+
We have `scrape-gitbook-page` and similar commands for debugging. Ignore them, they just call internal functions directly. You should not need to use them unless you are debugging issues with Detecting Frameworks.
|
|
20
|
+
|
|
21
|
+
## Overwriting
|
|
22
|
+
|
|
23
|
+
The user has to add a `--overwrite` flag if they want to overwrite their current files.
|
|
24
|
+
|
|
25
|
+
## Sections vs Websites
|
|
26
|
+
|
|
27
|
+
We call the command `scrape-section` instead of `scrape-website` because we cannot scrape pages not in the navigation of the URL first passed in. For example, ReadMe has API Reference and other sections accessible through a separate top-navigation which we do not parse. We only scrape the navigation on the left: [https://docs.readme.com/main/docs](https://docs.readme.com/main/docs)
|
|
28
|
+
|
|
29
|
+
## Detecting Frameworks
|
|
30
|
+
|
|
31
|
+
The commands look in the page HTML to detect what framework scraper to use. For example, all Docusaurus sites have a metatag with the word Docusaurus in it. Some times, the metatag even has the Docusaurus version.
|
|
32
|
+
|
|
33
|
+
Each framework’s scrapers live in `scraping/site-scrapers/`
|
|
34
|
+
|
|
35
|
+
We currently support:
|
|
36
|
+
|
|
37
|
+
- Docusaurus
|
|
38
|
+
- GitBook
|
|
39
|
+
- ReadMe
|
|
40
|
+
|
|
41
|
+
## Terminal Output
|
|
42
|
+
|
|
43
|
+
We print a line in the terminal for every file we write. `util.ts` has a createPage function that takes care of writing the file and logging.
|
|
44
|
+
|
|
45
|
+
We use a pencil emoji when we successfully write a file. Images get a picture emoji. Likewise, we print a X emoji when we find a file that already exists and the user has not enabled overwriting files. We use emojis so you can tell what the command is doing without reading each file path.
|
|
46
|
+
|
|
47
|
+
We also print the file paths when scraping sections so the user can easily copy paste them into mint.json. Note that pages the user already added in Mintlify are not included in the printed example. We do not generate mint.json completely, we are just giving a small example to help users starting from scratch.
|
|
48
|
+
|
|
49
|
+
```jsx
|
|
50
|
+
Add the following to your navigation in mint.json:
|
|
51
|
+
|
|
52
|
+
{
|
|
53
|
+
"group": "Guides",
|
|
54
|
+
"pages": ["page-we-scraped"]
|
|
55
|
+
}
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
# Navigation Scraping
|
|
59
|
+
|
|
60
|
+
Most sites use JavaScript to open navigation menus which do not automatically include the menu buttons in the HTML. We use Puppeteer to click every nested menu so the site adds the menu buttons to the HTML. For example the original site’s HTML:
|
|
61
|
+
|
|
62
|
+
```jsx
|
|
63
|
+
<div>
|
|
64
|
+
<a id="my-nested-menu"></a>
|
|
65
|
+
</div>
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
can turn into this after opening the nested menu:
|
|
69
|
+
|
|
70
|
+
```jsx
|
|
71
|
+
<div>
|
|
72
|
+
<a id="my-nested-menu" aria-expanded=true></a>
|
|
73
|
+
<div>
|
|
74
|
+
<a href="/page"></a>
|
|
75
|
+
<a href="/other-page"></a>
|
|
76
|
+
</div>
|
|
77
|
+
</div>
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Ultimately, all section scrapers need to find an array of links to visit then call the scrape page function in a loop.
|
|
81
|
+
|
|
82
|
+
We use axios instead of Puppeteer if a site doesn’t hide links. Puppeteer is slow.
|
|
83
|
+
|
|
84
|
+
# Image File Locations
|
|
85
|
+
|
|
86
|
+
Images go in an `images/` folder because that’s what most users want. Scraping per section uses the same root-level images folder. Scraping per page downloads them to the current location. Thus, scraping a single page from a folder means the user always has to move the images themselves. That’s a trade-off we are comfortable with — trying to detect an existing images folder gets too complicated too fast.
|
|
87
|
+
|
|
88
|
+
# Cheerio
|
|
89
|
+
|
|
90
|
+
Cheerio is a library to scrape/handle the HTML after we have it in a string. Most of the work is using inspect-element to view a website and figure out where the content we want is, then writing the corresponding Cheerio code.
|
|
91
|
+
|
|
92
|
+
# HTML to MDX
|
|
93
|
+
|
|
94
|
+
We use an open-source library to convert HTML to Markdown: https://github.com/crosstype/node-html-markdown
|
|
95
|
+
|
|
96
|
+
The `util.ts` createPage function assembles the MDX metadata, we just need to return an object of the form `{ title, description, content }` from each page scraper.
|
|
97
|
+
|
|
98
|
+
## Parsing Issues
|
|
99
|
+
|
|
100
|
+
Parsing struggles when documentation websites are using non-standard HTML. For example, code blocks are supposed to use. `<pre><code></code></pre>` but GitBook just uses divs.
|
|
101
|
+
|
|
102
|
+
We can write custom translators for the library that determine how we parse certain objects.
|
|
103
|
+
|
|
104
|
+
In some cases, we will want custom translators even if parsing succeeds. For example, ReadMe callouts are using quote syntax
|
|
105
|
+
|
|
106
|
+
```jsx
|
|
107
|
+
> 💡
|
|
108
|
+
> Callout text
|
|
109
|
+
>
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
When we want to convert them to:
|
|
113
|
+
|
|
114
|
+
```jsx
|
|
115
|
+
<Tip>Callout text</Tip>
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Regex
|
|
119
|
+
|
|
120
|
+
You can use regex to make small changes where translators are overkill or there’s no obvious component to modify. For example, here’s the end of `scrapeDocusaurusPage.ts`:
|
package/src/constants.ts
CHANGED
|
@@ -3,7 +3,7 @@ import * as url from "url";
|
|
|
3
3
|
import os from "os";
|
|
4
4
|
|
|
5
5
|
// Change this to bump to a newer version of mint's client
|
|
6
|
-
export const TARGET_MINT_VERSION = "v0.0.
|
|
6
|
+
export const TARGET_MINT_VERSION = "v0.0.8";
|
|
7
7
|
|
|
8
8
|
// package installation location
|
|
9
9
|
export const INSTALL_PATH = url.fileURLToPath(new URL(".", import.meta.url));
|
|
@@ -3,7 +3,7 @@ import fse from "fs-extra";
|
|
|
3
3
|
import pathUtil from "path";
|
|
4
4
|
import Chalk from "chalk";
|
|
5
5
|
import mintValidation from "@mintlify/validation";
|
|
6
|
-
import { openApiCheck } from "./utils.js";
|
|
6
|
+
import { isFileSizeValid, openApiCheck } from "./utils.js";
|
|
7
7
|
import { updateGeneratedNav, updateOpenApiFiles } from "./update.js";
|
|
8
8
|
import { CLIENT_PATH, CMD_EXEC_PATH } from "../../constants.js";
|
|
9
9
|
import { promises as _promises } from "fs";
|
|
@@ -142,9 +142,11 @@ const onUpdateEvent = async (filename: string): Promise<FileCategory> => {
|
|
|
142
142
|
potentialCategory === "potentialJsonOpenApiSpec"
|
|
143
143
|
? "staticFile"
|
|
144
144
|
: potentialCategory;
|
|
145
|
+
|
|
145
146
|
switch (potentialCategory) {
|
|
146
147
|
case "page":
|
|
147
148
|
regenerateNav = true;
|
|
149
|
+
|
|
148
150
|
const contentStr = (await readFile(filePath)).toString();
|
|
149
151
|
const { pageContent } = await createPage(
|
|
150
152
|
filename,
|
|
@@ -195,9 +197,7 @@ const onUpdateEvent = async (filename: string): Promise<FileCategory> => {
|
|
|
195
197
|
case "potentialYamlOpenApiSpec":
|
|
196
198
|
case "potentialJsonOpenApiSpec":
|
|
197
199
|
let isOpenApi = false;
|
|
198
|
-
const openApiInfo = await openApiCheck(
|
|
199
|
-
pathUtil.join(CMD_EXEC_PATH, filename)
|
|
200
|
-
);
|
|
200
|
+
const openApiInfo = await openApiCheck(filePath);
|
|
201
201
|
isOpenApi = openApiInfo.isOpenApi;
|
|
202
202
|
if (isOpenApi) {
|
|
203
203
|
// TODO: Instead of re-generating all openApi files, optimize by just updating the specific file that changed.
|
|
@@ -207,7 +207,15 @@ const onUpdateEvent = async (filename: string): Promise<FileCategory> => {
|
|
|
207
207
|
}
|
|
208
208
|
break;
|
|
209
209
|
case "staticFile":
|
|
210
|
-
await
|
|
210
|
+
if (await isFileSizeValid(filePath, 5)) {
|
|
211
|
+
await fse.copy(filePath, targetPath);
|
|
212
|
+
} else {
|
|
213
|
+
console.error(
|
|
214
|
+
Chalk.red(
|
|
215
|
+
`🚨 The file at ${filename} is too big. The maximum file size is 5 mb.`
|
|
216
|
+
)
|
|
217
|
+
);
|
|
218
|
+
}
|
|
211
219
|
break;
|
|
212
220
|
}
|
|
213
221
|
if (regenerateNav) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import SwaggerParser from "@apidevtools/swagger-parser";
|
|
2
2
|
import { promises as _promises } from "fs";
|
|
3
3
|
|
|
4
|
-
const { readdir } = _promises;
|
|
4
|
+
const { readdir, stat } = _promises;
|
|
5
5
|
|
|
6
6
|
export const getFileExtension = (filename) => {
|
|
7
7
|
return (
|
|
@@ -67,3 +67,12 @@ export const getFileList = async (dirName: string, og = dirName) => {
|
|
|
67
67
|
|
|
68
68
|
return files;
|
|
69
69
|
};
|
|
70
|
+
|
|
71
|
+
export const isFileSizeValid = async (
|
|
72
|
+
path: string,
|
|
73
|
+
maxFileSizeInMb: number
|
|
74
|
+
): Promise<boolean> => {
|
|
75
|
+
const maxFileSizeBytes = maxFileSizeInMb * 1000000;
|
|
76
|
+
const stats = await stat(path);
|
|
77
|
+
return stats.size <= maxFileSizeBytes;
|
|
78
|
+
};
|