mintlify 2.0.4 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/constants.js +17 -0
- package/bin/constants.js.map +1 -1
- package/bin/downloadImage.js +47 -2
- package/bin/downloadImage.js.map +1 -1
- package/bin/local-preview/index.js +2 -1
- package/bin/local-preview/index.js.map +1 -1
- package/bin/local-preview/listener/index.js +1 -1
- package/bin/local-preview/listener/index.js.map +1 -1
- package/bin/scraping/combineNavWithEmptyGroupTitles.js.map +1 -1
- package/bin/scraping/downloadAllImages.js +9 -38
- package/bin/scraping/downloadAllImages.js.map +1 -1
- package/bin/scraping/downloadLogoImage.js +13 -0
- package/bin/scraping/downloadLogoImage.js.map +1 -0
- package/bin/scraping/scrapeFileGettingFileNameFromUrl.js +1 -1
- package/bin/scraping/scrapeFileGettingFileNameFromUrl.js.map +1 -1
- package/bin/scraping/scrapeGettingFileNameFromUrl.js +1 -2
- package/bin/scraping/scrapeGettingFileNameFromUrl.js.map +1 -1
- package/bin/scraping/scrapePage.js +3 -2
- package/bin/scraping/scrapePage.js.map +1 -1
- package/bin/scraping/scrapePageCommands.js +1 -1
- package/bin/scraping/scrapePageCommands.js.map +1 -1
- package/bin/scraping/scrapeSection.js +4 -1
- package/bin/scraping/scrapeSection.js.map +1 -1
- package/bin/scraping/scrapeSectionCommands.js +2 -2
- package/bin/scraping/scrapeSectionCommands.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js +2 -2
- package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js +5 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js +2 -2
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js +7 -1
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMePage.js +2 -2
- package/bin/scraping/site-scrapers/scrapeReadMePage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js +9 -3
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map +1 -1
- package/bin/util.js +4 -2
- package/bin/util.js.map +1 -1
- package/package.json +1 -1
- package/src/constants.ts +18 -0
- package/src/downloadImage.ts +61 -3
- package/src/local-preview/index.ts +2 -1
- package/src/local-preview/listener/index.ts +1 -1
- package/src/scraping/combineNavWithEmptyGroupTitles.ts +4 -4
- package/src/scraping/downloadAllImages.ts +12 -39
- package/src/scraping/downloadLogoImage.ts +24 -0
- package/src/scraping/scrapeFileGettingFileNameFromUrl.ts +3 -1
- package/src/scraping/scrapeGettingFileNameFromUrl.ts +4 -4
- package/src/scraping/scrapePage.ts +6 -9
- package/src/scraping/scrapePageCommands.ts +2 -2
- package/src/scraping/scrapeSection.ts +7 -2
- package/src/scraping/scrapeSectionCommands.ts +7 -4
- package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts +3 -1
- package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts +9 -4
- package/src/scraping/site-scrapers/scrapeGitBookPage.ts +2 -0
- package/src/scraping/site-scrapers/scrapeGitBookSection.ts +12 -4
- package/src/scraping/site-scrapers/scrapeReadMePage.ts +3 -1
- package/src/scraping/site-scrapers/scrapeReadMeSection.ts +14 -6
- package/src/types.d.ts +29 -0
- package/src/util.ts +6 -5
- package/bin/local-preview/utils/categorizeFiles.js +0 -63
- package/bin/local-preview/utils/categorizeFiles.js.map +0 -1
- package/bin/local-preview/utils/getOpenApiContext.js +0 -58
- package/bin/local-preview/utils/getOpenApiContext.js.map +0 -1
- package/bin/local-preview/utils/injectFavicons.js +0 -72
- package/bin/local-preview/utils/injectFavicons.js.map +0 -1
- package/bin/local-preview/utils/listener.js +0 -116
- package/bin/local-preview/utils/listener.js.map +0 -1
- package/bin/local-preview/utils/metadata.js +0 -118
- package/bin/local-preview/utils/metadata.js.map +0 -1
- package/bin/local-preview/utils/mintConfigFile.js +0 -43
- package/bin/local-preview/utils/mintConfigFile.js.map +0 -1
- package/bin/local-preview/utils/openApiCheck.js +0 -15
- package/bin/local-preview/utils/openApiCheck.js.map +0 -1
- package/bin/local-preview/utils/slugToTitle.js +0 -8
- package/bin/local-preview/utils/slugToTitle.js.map +0 -1
- package/bin/navigation.js +0 -4
- package/bin/navigation.js.map +0 -1
- package/src/navigation.ts +0 -12
package/bin/constants.js
CHANGED
|
@@ -8,4 +8,21 @@ export const DOT_MINTLIFY = path.join(HOME_DIR, ".mintlify");
|
|
|
8
8
|
export const CLIENT_PATH = path.join(DOT_MINTLIFY, "mint", "client");
|
|
9
9
|
// command execution location
|
|
10
10
|
export const CMD_EXEC_PATH = process.cwd();
|
|
11
|
+
export const SUPPORTED_MEDIA_EXTENSIONS = [
|
|
12
|
+
"jpeg",
|
|
13
|
+
"jpg",
|
|
14
|
+
"jfif",
|
|
15
|
+
"pjpeg",
|
|
16
|
+
"pjp",
|
|
17
|
+
"png",
|
|
18
|
+
"svg",
|
|
19
|
+
"svgz",
|
|
20
|
+
"ico",
|
|
21
|
+
"webp",
|
|
22
|
+
"gif",
|
|
23
|
+
"apng",
|
|
24
|
+
"avif",
|
|
25
|
+
"bmp",
|
|
26
|
+
"mp4",
|
|
27
|
+
];
|
|
11
28
|
//# sourceMappingURL=constants.js.map
|
package/bin/constants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,gCAAgC;AAChC,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;AAErC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AAE7D,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAErE,6BAA6B;AAC7B,MAAM,CAAC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,gCAAgC;AAChC,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;AAErC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AAE7D,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAErE,6BAA6B;AAC7B,MAAM,CAAC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;AAE3C,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;CACN,CAAC"}
|
package/bin/downloadImage.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, createWriteStream } from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import axios from "axios";
|
|
4
|
-
|
|
4
|
+
import { getFileExtension } from "./util.js";
|
|
5
|
+
import { SUPPORTED_MEDIA_EXTENSIONS } from "./constants.js";
|
|
6
|
+
async function writeImageToFile(imageSrc, writePath, overwrite) {
|
|
5
7
|
// Avoid unnecessary downloads
|
|
6
|
-
if (existsSync(writePath)) {
|
|
8
|
+
if (existsSync(writePath) && !overwrite) {
|
|
7
9
|
return Promise.reject({
|
|
8
10
|
code: "EEXIST",
|
|
9
11
|
});
|
|
@@ -24,4 +26,47 @@ export default async function downloadImage(imageSrc, writePath) {
|
|
|
24
26
|
writer.on("error", reject);
|
|
25
27
|
});
|
|
26
28
|
}
|
|
29
|
+
export function isValidImageSrc(src) {
|
|
30
|
+
if (!src) {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
// We do not support downloading base64 in-line images.
|
|
34
|
+
if (src.startsWith("data:")) {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
const imageHref = removeMetadataFromImageSrc(src);
|
|
38
|
+
const ext = getFileExtension(imageHref);
|
|
39
|
+
if (!SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
|
|
40
|
+
console.error("🚨 We do not support the file extension: " + ext);
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
export function removeMetadataFromImageSrc(src) {
|
|
46
|
+
// Part of the URL standard
|
|
47
|
+
const metadataSymbols = ["?", "#"];
|
|
48
|
+
metadataSymbols.forEach((dividerSymbol) => {
|
|
49
|
+
// Some frameworks add metadata after the file extension, we need to remove that.
|
|
50
|
+
src = src.split(dividerSymbol)[0];
|
|
51
|
+
});
|
|
52
|
+
return src;
|
|
53
|
+
}
|
|
54
|
+
export function cleanImageSrc(src, origin) {
|
|
55
|
+
// Add origin if the image tags are using relative sources
|
|
56
|
+
return src.startsWith("http") ? src : new URL(src, origin).href;
|
|
57
|
+
}
|
|
58
|
+
export default async function downloadImage(imageSrc, writePath, overwrite = false) {
|
|
59
|
+
await writeImageToFile(imageSrc, writePath, overwrite)
|
|
60
|
+
.then(() => {
|
|
61
|
+
console.log("🖼️ - " + writePath);
|
|
62
|
+
})
|
|
63
|
+
.catch((e) => {
|
|
64
|
+
if (e.code === "EEXIST") {
|
|
65
|
+
console.log(`❌ Skipping existing image ${writePath}`);
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
console.error(e);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
}
|
|
27
72
|
//# sourceMappingURL=downloadImage.js.map
|
package/bin/downloadImage.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAE5D,KAAK,UAAU,gBAAgB,CAC7B,QAAgB,EAChB,SAAiB,EACjB,SAAkB;IAElB,8BAA8B;IAC9B,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;KACJ;IAED,+CAA+C;IAC/C,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;QACzC,YAAY,EAAE,QAAQ;KACvB,CAAC,CAAC;IAEH,4DAA4D;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QACzB,IAAI,EAAE,IAAI;KACX,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE;QACR,OAAO,KAAK,CAAC;KACd;IAED,uDAAuD;IACvD,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAC3B,OAAO,KAAK,CAAC;KACd;IAED,MAAM,SAAS,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QAC7C,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACjE,OAAO,KAAK,CAAC;KACd;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,2BAA2B;IAC3B,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEnC,eAAe,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;QACxC,iFAAiF;QACjF,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAc;IACvD,0DAA0D;IAC1D,OAAO,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,aAAa,CACzC,QAAgB,EAChB,SAAiB,EACjB,YAAqB,KAAK;IAE1B,MAAM,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;SACnD,IAAI,CAAC,GAAG,EAAE;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;IACpC,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACX,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;SACvD;aAAM;YACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClB;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -100,7 +100,8 @@ const dev = async () => {
|
|
|
100
100
|
`);
|
|
101
101
|
process.exit(1);
|
|
102
102
|
}
|
|
103
|
-
|
|
103
|
+
const relativePath = path.relative(CLIENT_PATH, CMD_EXEC_PATH);
|
|
104
|
+
shellExec(`yarn preconfigure ${relativePath}`);
|
|
104
105
|
logger.succeed("Local Mintlify instance initialized");
|
|
105
106
|
run();
|
|
106
107
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/local-preview/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,EACL,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,QAAQ,MAAM,qBAAqB,CAAC;AAE3C,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,EAAE;IAChC,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,KAAK,IAAI,EAAE;IACnC,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;AAC/E,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;IAC/B,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,QAAQ;aACX,MAAM,CAAC;YACN;gBACE,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,gDAAgD;gBACzD,OAAO,EAAE,IAAI;aACd;SACF,CAAC;aACD,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;YACpB,IAAI,OAAO,EAAE;gBACX,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;aACzC;iBAAM;gBACL,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;aACxC;QACH,CAAC,CAAC,CAAC;KACN;AACH,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,KAAK,IAAI,EAAE;IACrB,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IACnB,MAAM,aAAa,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,WAAW,CAAC,uCAAuC,CAAC,CAAC;IACpE,MAAM,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAClD,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IACpB,MAAM,aAAa,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,UAAU,GAAG,SAAS,CAAC,iCAAiC,CAAC,CAAC,MAAM,CAAC;IACvE,IACE,aAAa,CAAC,UAAU,CACtB,iDAAiD,CAClD;QACD,UAAU,KAAK,0BAA0B,EACzC;QACA,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;KAC/B;IAED,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,MAAM,kBAAkB,GAAG,MAAM,UAAU,CACzC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CACxC,CAAC;IACF,IAAI,CAAC,kBAAkB,EAAE;QACvB,iBAAiB,GAAG,IAAI,CAAC;QACzB,IAAI,YAAY,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACxD,SAAS,CAAC,UAAU,CAAC,CAAC;YACtB,SAAS,CACP,oEAAoE,CACrE,CAAC;SACH;aAAM;YACL,MAAM,CAAC,IAAI,CACT,mEAAmE,CACpE,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;KACF;IAED,MAAM,QAAQ,GAAG,MAAM,mBAAmB,EAAE,CAAC;IAC7C,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,IAAI,QAAQ,IAAI,YAAY,EAAE;QAC5B,SAAS,CAAC,qCAAqC,CAAC,CAAC;QACjD,SAAS,CAAC,6CAA6C,CAAC,CAAC;QACzD,UAAU,GAAG,SAAS,CAAC,2BAA2B,CAAC,CAAC,MAAM,CAAC;QAC3D,SAAS,CAAC,sCAAsC,CAAC,CAAC;QAClD,SAAS,CAAC,8BAA8B,CAAC,CAAC;KAC3C;IACD,IAAI,UAAU,KAAK,uBAAuB,EAAE;QAC1C,OAAO,GAAG,KAAK,CAAC;KACjB;IACD,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IACtB,IAAI,QAAQ,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC,EAAE;QACzD,IAAI,iBAAiB,EAAE;YACrB,MAAM,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;SACvD;QACD,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACzC,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,SAAS,CAAC,MAAM,CAAC,CAAC;QAClB,IAAI,iBAAiB,EAAE;YACrB,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;SACzC;aAAM;YACL,MAAM,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;SACxC;KACF;IAED,IAAI,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,EAAE;QAChC,MAAM,CAAC,IAAI,CAAC;;;;KAIX,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/local-preview/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,EACL,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,QAAQ,MAAM,qBAAqB,CAAC;AAE3C,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,EAAE;IAChC,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,KAAK,IAAI,EAAE;IACnC,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;AAC/E,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;IAC/B,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,QAAQ;aACX,MAAM,CAAC;YACN;gBACE,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,gDAAgD;gBACzD,OAAO,EAAE,IAAI;aACd;SACF,CAAC;aACD,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;YACpB,IAAI,OAAO,EAAE;gBACX,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;aACzC;iBAAM;gBACL,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;aACxC;QACH,CAAC,CAAC,CAAC;KACN;AACH,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,KAAK,IAAI,EAAE;IACrB,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IACnB,MAAM,aAAa,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,WAAW,CAAC,uCAAuC,CAAC,CAAC;IACpE,MAAM,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAClD,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IACpB,MAAM,aAAa,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,UAAU,GAAG,SAAS,CAAC,iCAAiC,CAAC,CAAC,MAAM,CAAC;IACvE,IACE,aAAa,CAAC,UAAU,CACtB,iDAAiD,CAClD;QACD,UAAU,KAAK,0BAA0B,EACzC;QACA,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;KAC/B;IAED,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,MAAM,kBAAkB,GAAG,MAAM,UAAU,CACzC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CACxC,CAAC;IACF,IAAI,CAAC,kBAAkB,EAAE;QACvB,iBAAiB,GAAG,IAAI,CAAC;QACzB,IAAI,YAAY,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACxD,SAAS,CAAC,UAAU,CAAC,CAAC;YACtB,SAAS,CACP,oEAAoE,CACrE,CAAC;SACH;aAAM;YACL,MAAM,CAAC,IAAI,CACT,mEAAmE,CACpE,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;KACF;IAED,MAAM,QAAQ,GAAG,MAAM,mBAAmB,EAAE,CAAC;IAC7C,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,IAAI,QAAQ,IAAI,YAAY,EAAE;QAC5B,SAAS,CAAC,qCAAqC,CAAC,CAAC;QACjD,SAAS,CAAC,6CAA6C,CAAC,CAAC;QACzD,UAAU,GAAG,SAAS,CAAC,2BAA2B,CAAC,CAAC,MAAM,CAAC;QAC3D,SAAS,CAAC,sCAAsC,CAAC,CAAC;QAClD,SAAS,CAAC,8BAA8B,CAAC,CAAC;KAC3C;IACD,IAAI,UAAU,KAAK,uBAAuB,EAAE;QAC1C,OAAO,GAAG,KAAK,CAAC;KACjB;IACD,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IACtB,IAAI,QAAQ,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC,EAAE;QACzD,IAAI,iBAAiB,EAAE;YACrB,MAAM,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;SACvD;QACD,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACzC,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,SAAS,CAAC,MAAM,CAAC,CAAC;QAClB,IAAI,iBAAiB,EAAE;YACrB,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;SACzC;aAAM;YACL,MAAM,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;SACxC;KACF;IAED,IAAI,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,EAAE;QAChC,MAAM,CAAC,IAAI,CAAC;;;;KAIX,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAC/D,SAAS,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAC;IAC/C,MAAM,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;IACtD,GAAG,EAAE,CAAC;AACR,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,GAAG,EAAE;IACf,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IACtB,OAAO,CAAC,GAAG,CACT,MAAM,KAAK,CAAC,KAAK,CACf,yDAAyD,CAC1D,EAAE,CACJ,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC9B,QAAQ,EAAE,CAAC;AACb,CAAC,CAAC;AAEF,eAAe,GAAG,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/local-preview/listener/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,QAAQ,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,MAAM,IAAI,CAAC;AAC3C,OAAO,UAAU,MAAM,uBAAuB,CAAC;AAE/C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;AAE/B,MAAM,QAAQ,GAAG,GAAG,EAAE;IACpB,QAAQ;SACL,KAAK,CAAC,aAAa,EAAE;QACpB,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/local-preview/listener/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,QAAQ,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,MAAM,IAAI,CAAC;AAC3C,OAAO,UAAU,MAAM,uBAAuB,CAAC;AAE/C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;AAE/B,MAAM,QAAQ,GAAG,GAAG,EAAE;IACpB,QAAQ;SACL,KAAK,CAAC,aAAa,EAAE;QACpB,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,CAAC;QAC1C,GAAG,EAAE,aAAa;KACnB,CAAC;SACD,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACnC,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,WAAW,EAAE;YAC/C,IAAI,aAAa,CAAC,QAAQ,CAAC,EAAE;gBAC3B,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAC9B,WAAW,EACX,KAAK,EACL,QAAQ,EACR,QAAQ,CACT,CAAC;gBACF,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC7B,OAAO,CAAC,GAAG,CACT,GACE,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MACjD,YAAY,EACZ,QAAQ,CACT,CAAC;aACH;iBAAM,IAAI,QAAQ,KAAK,WAAW,EAAE;gBACnC,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAC9B,WAAW,EACX,KAAK,EACL,QAAQ,EACR,WAAW,CACZ,CAAC;gBACF,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC7B,OAAO,CAAC,GAAG,CACT,8EAA8E,CAC/E,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;iBAAM;gBACL,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACxD,IACE,SAAS;oBACT,CAAC,SAAS,KAAK,MAAM;wBACnB,SAAS,KAAK,MAAM;wBACpB,SAAS,KAAK,KAAK,CAAC,EACtB;oBACA,MAAM,kBAAkB,EAAE,CAAC;oBAC3B,MAAM,kBAAkB,EAAE,CAAC;oBAC3B,OAAO;iBACR;gBACD,kBAAkB;gBAClB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;aAChD;SACF;aAAM;YACL,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YACxD,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,IAAI,aAAa,CAAC,QAAQ,CAAC,EAAE;gBAC3B,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAC9B,WAAW,EACX,KAAK,EACL,QAAQ,EACR,QAAQ,CACT,CAAC;gBACF,IAAI,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE;oBACpC,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;oBAC1D,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;oBAC7B,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;oBACvC,OAAO;iBACR;gBACD,aAAa,GAAG,IAAI,CAAC;gBAErB,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACzD,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,UAAU,CACtC,QAAQ,EACR,UAAU,EACV,aAAa,EACb,EAAE,CACH,CAAC;gBACF,MAAM,GAAG,CAAC,UAAU,CAAC,UAAU,EAAE,WAAW,EAAE;oBAC5C,IAAI,EAAE,GAAG;iBACV,CAAC,CAAC;gBACH,QAAQ,KAAK,EAAE;oBACb,KAAK,KAAK,CAAC;oBACX,KAAK,QAAQ;wBACX,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;wBAC7C,MAAM;oBACR;wBACE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;wBACvC,MAAM;iBACT;aACF;iBAAM,IAAI,QAAQ,KAAK,WAAW,EAAE;gBACnC,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAC9B,WAAW,EACX,KAAK,EACL,QAAQ,EACR,WAAW,CACZ,CAAC;gBACF,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBACrC,QAAQ,KAAK,EAAE;oBACb,KAAK,KAAK,CAAC;oBACX,KAAK,QAAQ;wBACX,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAC5B,MAAM;oBACR;wBACE,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;wBAC7B,MAAM;iBACT;aACF;iBAAM;gBACL,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACxD,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,IACE,SAAS;oBACT,CAAC,SAAS,KAAK,MAAM;wBACnB,SAAS,KAAK,MAAM;wBACpB,SAAS,KAAK,KAAK,CAAC,EACtB;oBACA,MAAM,WAAW,GAAG,MAAM,YAAY,CACpC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CACvC,CAAC;oBACF,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC;oBAClC,IAAI,SAAS,EAAE;wBACb,MAAM,kBAAkB,EAAE,CAAC;wBAC3B,aAAa,GAAG,IAAI,CAAC;qBACtB;iBACF;gBACD,IAAI,CAAC,SAAS,EAAE;oBACd,kBAAkB;oBAClB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAClE,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;iBACtC;gBACD,QAAQ,KAAK,EAAE;oBACb,KAAK,KAAK,CAAC;oBACX,KAAK,QAAQ;wBACX,IAAI,SAAS,EAAE;4BACb,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,QAAQ,CAAC,CAAC;yBAC/C;6BAAM;4BACL,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;yBAC9C;wBACD,MAAM;oBACR;wBACE,IAAI,SAAS,EAAE;4BACb,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;yBAChD;6BAAM;4BACL,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,QAAQ,CAAC,CAAC;yBAC/C;wBACD,MAAM;iBACT;aACF;YACD,IAAI,aAAa,EAAE;gBACjB,2GAA2G;gBAC3G,MAAM,kBAAkB,EAAE,CAAC;aAC5B;SACF;IACH,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,eAAe,QAAQ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"combineNavWithEmptyGroupTitles.js","sourceRoot":"","sources":["../../src/scraping/combineNavWithEmptyGroupTitles.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"combineNavWithEmptyGroupTitles.js","sourceRoot":"","sources":["../../src/scraping/combineNavWithEmptyGroupTitles.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,8BAA8B,CACpD,QAA0B;IAE1B,IAAI,WAAW,GAAG,EAAE,CAAC;IAErB,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAmB,EAAE,EAAE;QACvC,mEAAmE;QACnE,wFAAwF;QACxF,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjD,IAAI,IAAI,IAAI,IAAI,EAAE;YAChB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACvB;aAAM,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACpC,4EAA4E;YAC5E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;SAC3C;aAAM;YACL,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACvB;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from "path";
|
|
2
|
-
import downloadImage from "../downloadImage.js";
|
|
2
|
+
import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from "../downloadImage.js";
|
|
3
3
|
// To Do: Use CheerioElement instead of any when we bump the cheerio version
|
|
4
|
-
export default async function downloadAllImages($, content, origin, baseDir, modifyFileName) {
|
|
4
|
+
export default async function downloadAllImages($, content, origin, baseDir, overwrite, modifyFileName) {
|
|
5
5
|
if (!baseDir) {
|
|
6
6
|
console.debug("Skipping image downloading");
|
|
7
7
|
return;
|
|
@@ -15,47 +15,18 @@ export default async function downloadAllImages($, content, origin, baseDir, mod
|
|
|
15
15
|
.toArray()),
|
|
16
16
|
];
|
|
17
17
|
// Wait to all images to download before continuing
|
|
18
|
-
const origToNewArray = await Promise.all(imageSrcs.map(async (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
// Add origin if the image tags are using relative sources
|
|
24
|
-
const imageHref = origImageSrc.startsWith("http")
|
|
25
|
-
? origImageSrc
|
|
26
|
-
: new URL(origImageSrc, origin).href;
|
|
27
|
-
let fileName = removeMetadataFromExtension(path.basename(imageHref));
|
|
18
|
+
const origToNewArray = await Promise.all(imageSrcs.map(async (imageSrc) => {
|
|
19
|
+
if (!isValidImageSrc(imageSrc))
|
|
20
|
+
return;
|
|
21
|
+
const imageHref = cleanImageSrc(imageSrc, origin);
|
|
22
|
+
let fileName = removeMetadataFromImageSrc(path.basename(imageHref));
|
|
28
23
|
if (modifyFileName) {
|
|
29
24
|
fileName = modifyFileName(fileName);
|
|
30
25
|
}
|
|
31
|
-
if (!fileName) {
|
|
32
|
-
console.error("Invalid image path " + imageHref);
|
|
33
|
-
return;
|
|
34
|
-
}
|
|
35
26
|
const writePath = path.join(baseDir, fileName);
|
|
36
|
-
await downloadImage(imageHref, writePath)
|
|
37
|
-
|
|
38
|
-
console.log("🖼️ - " + writePath);
|
|
39
|
-
})
|
|
40
|
-
.catch((e) => {
|
|
41
|
-
if (e.code === "EEXIST") {
|
|
42
|
-
console.log(`❌ Skipping existing image ${writePath}`);
|
|
43
|
-
}
|
|
44
|
-
else {
|
|
45
|
-
console.error(e);
|
|
46
|
-
}
|
|
47
|
-
});
|
|
48
|
-
return { [origImageSrc]: writePath };
|
|
27
|
+
await downloadImage(imageHref, writePath, overwrite);
|
|
28
|
+
return { [imageSrc]: writePath };
|
|
49
29
|
}));
|
|
50
30
|
return origToNewArray.reduce((result, current) => Object.assign(result, current), {});
|
|
51
31
|
}
|
|
52
|
-
function removeMetadataFromExtension(src) {
|
|
53
|
-
// Part of the URL standard
|
|
54
|
-
const metadataSymbols = ["?", "#"];
|
|
55
|
-
metadataSymbols.forEach((dividerSymbol) => {
|
|
56
|
-
// Some frameworks add metadata after the file extension, we need to remove that.
|
|
57
|
-
src = src.split(dividerSymbol)[0];
|
|
58
|
-
});
|
|
59
|
-
return src;
|
|
60
|
-
}
|
|
61
32
|
//# sourceMappingURL=downloadAllImages.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,SAAkB,EAClB,cAAoB;IAEpB,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;KACR;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAAE,OAAO;QAEvC,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE;YAClB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;SACrC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAC1B,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnD,EAAE,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from "../downloadImage.js";
|
|
3
|
+
import { getFileExtension } from "../util.js";
|
|
4
|
+
// To Do: Use CheerioElement instead of any when we bump the cheerio version
|
|
5
|
+
export default async function downloadLogoImage(imageSrc, imageBaseDir, origin, overwrite) {
|
|
6
|
+
if (!isValidImageSrc(imageSrc))
|
|
7
|
+
return;
|
|
8
|
+
const imageHref = cleanImageSrc(imageSrc, origin);
|
|
9
|
+
const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
|
|
10
|
+
const imagePath = path.join(imageBaseDir, "logo", "logo-light-mode." + ext);
|
|
11
|
+
await downloadImage(imageHref, imagePath, overwrite);
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=downloadLogoImage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,QAAgB,EAChB,YAAoB,EACpB,MAAc,EACd,SAAkB;IAElB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;QAAE,OAAO;IAEvC,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAE5E,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}
|
|
@@ -27,7 +27,7 @@ export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin,
|
|
|
27
27
|
const res = await axios.get(href);
|
|
28
28
|
html = res.data;
|
|
29
29
|
}
|
|
30
|
-
const { title, description, markdown } = await scrapePageFunc(html, origin, cliDir, imageBaseDir, version);
|
|
30
|
+
const { title, description, markdown } = await scrapePageFunc(html, origin, cliDir, imageBaseDir, overwrite, version);
|
|
31
31
|
// Check if page didn't have content
|
|
32
32
|
if (!title && !markdown) {
|
|
33
33
|
return undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,
|
|
1
|
+
{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,SAAS,CAAC;KAClB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CACR,KAAK,EACL,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,QAAQ,CACT,CAAC;IAEF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { isNavigation } from "../navigation.js";
|
|
2
1
|
import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
|
|
3
2
|
export async function scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, version, baseToRemove) {
|
|
4
|
-
if (
|
|
3
|
+
if (typeof navEntry !== "string") {
|
|
5
4
|
const newPages = [];
|
|
6
5
|
for (const nestedNavEntry of navEntry.pages) {
|
|
7
6
|
newPages.push(await scrapeGettingFileNameFromUrl(nestedNavEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer, version, baseToRemove));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AAEzF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,QAA6B,EAC7B,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,MAAM,QAAQ,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,cAAc,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC3C,QAAQ,CAAC,IAAI,CACX,MAAM,4BAA4B,CAChC,cAAc,EACd,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CACF,CAAC;SACH;QACD,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC1B,OAAO,QAAQ,CAAC;KACjB;IAED,OAAO,MAAM,gCAAgC,CAC3C,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CAAC;AACJ,CAAC"}
|
|
@@ -2,8 +2,9 @@ import path from "path";
|
|
|
2
2
|
import { createPage, getOrigin } from "../util.js";
|
|
3
3
|
export async function scrapePage(scrapeFunc, href, html, overwrite, version) {
|
|
4
4
|
const origin = getOrigin(href);
|
|
5
|
-
const
|
|
6
|
-
const
|
|
5
|
+
const cwd = process.cwd();
|
|
6
|
+
const imageBaseDir = path.join(cwd, "images");
|
|
7
|
+
const { title, description, markdown } = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
|
|
7
8
|
createPage(title, description, markdown, overwrite, process.cwd());
|
|
8
9
|
}
|
|
9
10
|
//# sourceMappingURL=scrapePage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapePage.js","sourceRoot":"","sources":["../../src/scraping/scrapePage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,
|
|
1
|
+
{"version":3,"file":"scrapePage.js","sourceRoot":"","sources":["../../src/scraping/scrapePage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAwB,EACxB,IAAY,EACZ,IAAY,EACZ,SAAkB,EAClB,OAA2B;IAE3B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CACvD,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AACrE,CAAC"}
|
|
@@ -25,7 +25,7 @@ export async function scrapePageWrapper(argv, scrapeFunc, options) {
|
|
|
25
25
|
const res = await axios.get(href);
|
|
26
26
|
html = res.data;
|
|
27
27
|
}
|
|
28
|
-
await scrapePage(scrapeFunc, href, html, argv.overwrite, options?.version);
|
|
28
|
+
await scrapePage(scrapeFunc, href, html, !!argv.overwrite, options?.version);
|
|
29
29
|
process.exit(0);
|
|
30
30
|
}
|
|
31
31
|
export async function scrapePageAutomatically(argv) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAS,EACT,
|
|
1
|
+
{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAS,EACT,UAAwB,EACxB,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,EAAE,SAAS,EAAE;QACtB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;KAClE;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;KACvE;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;KACjD;AACH,CAAC"}
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import path from "path";
|
|
1
2
|
import { objToReadableString } from "../util.js";
|
|
2
3
|
export async function scrapeSection(scrapeFunc, html, origin, overwrite, version) {
|
|
3
4
|
console.log(`Started scraping${overwrite ? ", overwrite mode is on" : ""}...`);
|
|
4
|
-
const
|
|
5
|
+
const cwd = process.cwd();
|
|
6
|
+
const imageBaseDir = path.join(cwd, "images");
|
|
7
|
+
const groupsConfig = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
|
|
5
8
|
console.log("Finished scraping.");
|
|
6
9
|
console.log("Add the following to your navigation in mint.json:");
|
|
7
10
|
console.log(objToReadableString(groupsConfig));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeSection.js","sourceRoot":"","sources":["../../src/scraping/scrapeSection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,
|
|
1
|
+
{"version":3,"file":"scrapeSection.js","sourceRoot":"","sources":["../../src/scraping/scrapeSection.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA2B,EAC3B,IAAY,EACZ,MAAc,EACd,SAAkB,EAClB,OAA2B;IAE3B,OAAO,CAAC,GAAG,CACT,mBAAmB,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,EAAE,KAAK,CAClE,CAAC;IACF,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;AACjD,CAAC"}
|
|
@@ -12,7 +12,7 @@ export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
|
|
|
12
12
|
const href = getHrefFromArgs(argv);
|
|
13
13
|
const res = await axios.get(href);
|
|
14
14
|
const html = res.data;
|
|
15
|
-
await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite, undefined);
|
|
15
|
+
await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, undefined);
|
|
16
16
|
process.exit(0);
|
|
17
17
|
}
|
|
18
18
|
export async function scrapeDocusaurusSectionCommand(argv, version // "1" | "2" | "3"
|
|
@@ -31,7 +31,7 @@ async function scrapeSectionOpeningAllNested(argv, openLinks, scrapeFunc, versio
|
|
|
31
31
|
});
|
|
32
32
|
const html = await openLinks(page);
|
|
33
33
|
browser.close();
|
|
34
|
-
await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite, version);
|
|
34
|
+
await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, version);
|
|
35
35
|
process.exit(0);
|
|
36
36
|
}
|
|
37
37
|
export async function scrapeSectionAutomatically(argv) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,yBAAyB,
|
|
1
|
+
{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAS,EACT,UAA2B;IAE3B,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,CAAC,CAAC,IAAI,CAAC,SAAS,EAChB,SAAS,CACV,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAS,EACT,OAAe,CAAC,kBAAkB;;IAElC,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CACjC,IAAI,EACJ,sBAAsB,EACtB,oBAAoB,CACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAA2B,EAC3B,OAAgB;IAEhB,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,CAAC,CAAC,IAAI,CAAC,SAAS,EAChB,OAAO,CACR,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;KACrD;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;KAC5D;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;KAChB;AACH,CAAC"}
|
|
@@ -2,7 +2,7 @@ import cheerio from "cheerio";
|
|
|
2
2
|
import { NodeHtmlMarkdown } from "node-html-markdown";
|
|
3
3
|
import downloadAllImages from "../downloadAllImages.js";
|
|
4
4
|
import replaceImagePaths from "../replaceImagePaths.js";
|
|
5
|
-
export async function scrapeDocusaurusPage(html, origin, cliDir, imageBaseDir, version // expects "2", or "3". Have not written support for "1" yet
|
|
5
|
+
export async function scrapeDocusaurusPage(html, origin, cliDir, imageBaseDir, overwrite, version // expects "2", or "3". Have not written support for "1" yet
|
|
6
6
|
) {
|
|
7
7
|
const $ = cheerio.load(html);
|
|
8
8
|
const article = version === "3" ? $(".theme-doc-markdown").first() : $("article").first();
|
|
@@ -15,7 +15,7 @@ export async function scrapeDocusaurusPage(html, origin, cliDir, imageBaseDir, v
|
|
|
15
15
|
// Do not include title in the content when we insert it in our metadata
|
|
16
16
|
titleComponent.remove();
|
|
17
17
|
const markdownContent = version === "3" ? article : article.find(".markdown").first();
|
|
18
|
-
const origToWritePath = await downloadAllImages($, markdownContent, origin, imageBaseDir);
|
|
18
|
+
const origToWritePath = await downloadAllImages($, markdownContent, origin, imageBaseDir, overwrite);
|
|
19
19
|
const markdownHtml = markdownContent.html();
|
|
20
20
|
const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
|
|
21
21
|
let markdown = nhm.translate(markdownHtml);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeDocusaurusPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,OAA2B,CAAC,4DAA4D;;IAExF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,OAAO,GACX,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IAE5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,oEAAoE;QACpE,OAAO,EAAE,CAAC;KACX;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,wEAAwE;IACxE,cAAc,CAAC,MAAM,EAAE,CAAC;IAExB,MAAM,eAAe,GACnB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,eAAe,EACf,MAAM,EACN,YAAY,
|
|
1
|
+
{"version":3,"file":"scrapeDocusaurusPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B,CAAC,4DAA4D;;IAExF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,OAAO,GACX,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IAE5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,oEAAoE;QACpE,OAAO,EAAE,CAAC;KACX;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,wEAAwE;IACxE,cAAc,CAAC,MAAM,EAAE,CAAC;IAExB,MAAM,eAAe,GACnB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,eAAe,EACf,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;IAE5C,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAE3C,IAAI,QAAQ,IAAI,IAAI,EAAE;QACpB,OAAO,CAAC,KAAK,CACX,6DAA6D,CAC9D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;KACnD;IAED,mGAAmG;IACnG,qFAAqF;IACrF,yFAAyF;IACzF,kFAAkF;IAClF,MAAM,WAAW,GAAG,IAAI,CAAC;IACzB,0EAA0E;IAC1E,0CAA0C;IAC1C,wBAAwB;IACxB,IAAI;IAEJ,uCAAuC;IACvC,wDAAwD;IACxD,sCAAsC;IACtC,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAEzD,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -3,8 +3,12 @@ import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js
|
|
|
3
3
|
import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
|
|
4
4
|
import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
|
|
5
5
|
import { getDocusaurusLinksPerGroup } from "./links-per-group/getDocusaurusLinksPerGroup.js";
|
|
6
|
-
|
|
6
|
+
import downloadLogoImage from "../downloadLogoImage.js";
|
|
7
|
+
export async function scrapeDocusaurusSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
|
|
7
8
|
const $ = cheerio.load(html);
|
|
9
|
+
// Download the logo
|
|
10
|
+
const logoSrc = $(".navbar__logo img").attr("src");
|
|
11
|
+
downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
|
|
8
12
|
// Get all the navigation sections
|
|
9
13
|
const navigationSections = $(".theme-doc-sidebar-menu").first().children();
|
|
10
14
|
// Get all links per group
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iDAAiD,CAAC;AAC7F,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAAe;IAEf,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,0BAA0B,CAC/D,kBAAkB,EAClB,CAAC,EACD,OAAO,CACR,CAAC;IAEF,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QAC5C,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC5D,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
|
|
@@ -2,7 +2,7 @@ import cheerio from "cheerio";
|
|
|
2
2
|
import { NodeHtmlMarkdown } from "node-html-markdown";
|
|
3
3
|
import downloadAllImages from "../downloadAllImages.js";
|
|
4
4
|
import replaceImagePaths from "../replaceImagePaths.js";
|
|
5
|
-
export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, _ // version
|
|
5
|
+
export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
|
|
6
6
|
) {
|
|
7
7
|
const $ = cheerio.load(html);
|
|
8
8
|
const titleComponent = $('[data-testid="page.title"]').first();
|
|
@@ -17,7 +17,7 @@ export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, _ //
|
|
|
17
17
|
// Remove GitBook metadata from the start
|
|
18
18
|
// The first four %2F split metadata fields. Remaining ones are part of the file name.
|
|
19
19
|
fileName.split("%2F").slice(4).join("%2F");
|
|
20
|
-
const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, modifyFileName);
|
|
20
|
+
const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite, modifyFileName);
|
|
21
21
|
const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
|
|
22
22
|
let markdown = nhm.translate(contentHtml);
|
|
23
23
|
// Keep headers on one line and increase their depth by one
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,EAAE;IAClC,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,EAAE;IAClC,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -4,8 +4,14 @@ import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
|
|
|
4
4
|
import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
|
|
5
5
|
import getLinksRecursivelyGitBook from "./links-per-group/getLinksRecursivelyGitBook.js";
|
|
6
6
|
import alternateGroupTitle from "./alternateGroupTitle.js";
|
|
7
|
-
|
|
7
|
+
import downloadLogoImage from "../downloadLogoImage.js";
|
|
8
|
+
export async function scrapeGitBookSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
|
|
8
9
|
const $ = cheerio.load(html);
|
|
10
|
+
// Download the logo
|
|
11
|
+
const logoSrc = $('a[data-testid="public.headerHomeLink"] img')
|
|
12
|
+
.first()
|
|
13
|
+
.attr("src");
|
|
14
|
+
downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
|
|
9
15
|
// Get all the navigation sections
|
|
10
16
|
// Some variants of the GitBook UI show the logo and search base in the side navigation bar,
|
|
11
17
|
// but the navigation sections are always the last value.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,4CAA4C,CAAC;SAC5D,KAAK,EAAE;SACP,IAAI,CAAC,KAAK,CAAC,CAAC;IACf,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,kCAAkC;IAClC,4FAA4F;IAC5F,yDAAyD;IACzD,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,0BAA0B,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC9D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,EACJ,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
|
|
@@ -2,7 +2,7 @@ import cheerio from "cheerio";
|
|
|
2
2
|
import { NodeHtmlMarkdown } from "node-html-markdown";
|
|
3
3
|
import downloadAllImages from "../downloadAllImages.js";
|
|
4
4
|
import replaceImagePaths from "../replaceImagePaths.js";
|
|
5
|
-
export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir, _ // version
|
|
5
|
+
export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
|
|
6
6
|
) {
|
|
7
7
|
const $ = cheerio.load(html);
|
|
8
8
|
const titleComponent = $("h1").first();
|
|
@@ -17,7 +17,7 @@ export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir, _ //
|
|
|
17
17
|
}
|
|
18
18
|
// API Pages don't have a markdown body in the same position so there's no HTML
|
|
19
19
|
let contentHtml = content.html() || "";
|
|
20
|
-
const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir);
|
|
20
|
+
const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite);
|
|
21
21
|
const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
|
|
22
22
|
let markdown = nhm.translate(contentHtml);
|
|
23
23
|
// Keep headers on one line and increase their depth by one
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IAED,+EAA+E;IAC/E,IAAI,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEvC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,
|
|
1
|
+
{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IAED,+EAA+E;IAC/E,IAAI,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEvC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -2,8 +2,12 @@ import cheerio from "cheerio";
|
|
|
2
2
|
import { scrapeReadMePage } from "./scrapeReadMePage.js";
|
|
3
3
|
import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
|
|
4
4
|
import getLinksRecursively from "./links-per-group/getLinksRecursively.js";
|
|
5
|
-
|
|
5
|
+
import downloadLogoImage from "../downloadLogoImage.js";
|
|
6
|
+
export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
|
|
6
7
|
const $ = cheerio.load(html);
|
|
8
|
+
// Download the logo
|
|
9
|
+
const logoSrc = $(".rm-Logo-img").first().attr("src");
|
|
10
|
+
downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
|
|
7
11
|
// Get all the navigation sections, but only from the first
|
|
8
12
|
// sidebar found. There are multiple in the HTML for mobile
|
|
9
13
|
// responsiveness but they all have the same links.
|
|
@@ -28,11 +32,13 @@ export async function scrapeReadMeSection(html, origin, cliDir, overwrite, versi
|
|
|
28
32
|
};
|
|
29
33
|
})
|
|
30
34
|
.toArray();
|
|
31
|
-
|
|
35
|
+
// Scrape each link in the navigation.
|
|
36
|
+
const groupsConfigCleanPaths = await Promise.all(groupsConfig.map(async (navEntry) => {
|
|
32
37
|
return await scrapeGettingFileNameFromUrl(
|
|
33
|
-
// ReadMe requires a directory on all sections
|
|
38
|
+
// ReadMe requires a directory on all sections whereas we use root.
|
|
34
39
|
// /docs is their default directory so we remove it
|
|
35
40
|
navEntry, cliDir, origin, overwrite, scrapeReadMePage, false, version, "/docs");
|
|
36
41
|
}));
|
|
42
|
+
return groupsConfigCleanPaths;
|
|
37
43
|
}
|
|
38
44
|
//# sourceMappingURL=scrapeReadMeSection.js.map
|