@mintlify/scraping 3.0.13 → 3.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierrc +1 -0
- package/bin/browser.js +3 -3
- package/bin/constants.js +23 -23
- package/bin/constants.js.map +1 -1
- package/bin/downloadImage.js +18 -18
- package/bin/downloadImage.js.map +1 -1
- package/bin/scraping/detectFramework.js +13 -13
- package/bin/scraping/detectFramework.js.map +1 -1
- package/bin/scraping/downloadAllImages.js +5 -5
- package/bin/scraping/downloadAllImages.js.map +1 -1
- package/bin/scraping/downloadLogoImage.js +4 -4
- package/bin/scraping/downloadLogoImage.js.map +1 -1
- package/bin/scraping/getSitemapLinks.js +4 -4
- package/bin/scraping/scrapeFileGettingFileNameFromUrl.js +10 -10
- package/bin/scraping/scrapeFileGettingFileNameFromUrl.js.map +1 -1
- package/bin/scraping/scrapeGettingFileNameFromUrl.js +2 -2
- package/bin/scraping/scrapeGettingFileNameFromUrl.js.map +1 -1
- package/bin/scraping/scrapePage.js +3 -3
- package/bin/scraping/scrapePage.js.map +1 -1
- package/bin/scraping/scrapePageCommands.d.ts +1 -1
- package/bin/scraping/scrapePageCommands.js +15 -15
- package/bin/scraping/scrapePageCommands.js.map +1 -1
- package/bin/scraping/scrapeSection.js +6 -6
- package/bin/scraping/scrapeSection.js.map +1 -1
- package/bin/scraping/scrapeSectionCommands.d.ts +1 -1
- package/bin/scraping/scrapeSectionCommands.js +14 -14
- package/bin/scraping/scrapeSectionCommands.js.map +1 -1
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js +11 -11
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js.map +1 -1
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js +12 -14
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js.map +1 -1
- package/bin/scraping/site-scrapers/alternateGroupTitle.js +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js +8 -11
- package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js.map +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js +6 -6
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js +4 -4
- package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.d.ts +1 -1
- package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js +4 -4
- package/bin/scraping/site-scrapers/openNestedGitbookMenus.d.ts +1 -1
- package/bin/scraping/site-scrapers/openNestedGitbookMenus.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js +14 -14
- package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js +9 -9
- package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js +12 -14
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js +10 -15
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMePage.js +15 -15
- package/bin/scraping/site-scrapers/scrapeReadMePage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js +11 -15
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map +1 -1
- package/bin/tsconfig.tsbuildinfo +1 -1
- package/bin/util.d.ts +1 -1
- package/bin/util.js +23 -26
- package/bin/util.js.map +1 -1
- package/bin/validation/stopIfInvalidLink.js +3 -3
- package/package.json +10 -3
- package/src/browser.ts +3 -3
- package/src/constants.ts +23 -23
- package/src/downloadImage.ts +21 -26
- package/src/scraping/detectFramework.ts +18 -18
- package/src/scraping/downloadAllImages.ts +7 -9
- package/src/scraping/downloadLogoImage.ts +5 -4
- package/src/scraping/getSitemapLinks.ts +4 -4
- package/src/scraping/scrapeFileGettingFileNameFromUrl.ts +12 -18
- package/src/scraping/scrapeGettingFileNameFromUrl.ts +7 -5
- package/src/scraping/scrapePage.ts +4 -3
- package/src/scraping/scrapePageCommands.ts +17 -18
- package/src/scraping/scrapeSection.ts +8 -16
- package/src/scraping/scrapeSectionCommands.ts +19 -34
- package/src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts +12 -11
- package/src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts +23 -24
- package/src/scraping/site-scrapers/alternateGroupTitle.ts +1 -1
- package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts +8 -11
- package/src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts +6 -6
- package/src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts +4 -4
- package/src/scraping/site-scrapers/openNestedDocusaurusMenus.ts +5 -5
- package/src/scraping/site-scrapers/openNestedGitbookMenus.ts +2 -4
- package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts +15 -18
- package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts +11 -14
- package/src/scraping/site-scrapers/scrapeGitBookPage.ts +13 -14
- package/src/scraping/site-scrapers/scrapeGitBookSection.ts +11 -15
- package/src/scraping/site-scrapers/scrapeReadMePage.ts +17 -22
- package/src/scraping/site-scrapers/scrapeReadMeSection.ts +27 -31
- package/src/util.ts +25 -36
- package/src/validation/stopIfInvalidLink.ts +3 -3
package/.prettierrc
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"@mintlify/prettier-config/config.js"
|
package/bin/browser.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { launch } from
|
|
1
|
+
import { launch } from 'puppeteer';
|
|
2
2
|
export async function startBrowser() {
|
|
3
3
|
try {
|
|
4
4
|
return await launch({
|
|
@@ -7,7 +7,7 @@ export async function startBrowser() {
|
|
|
7
7
|
});
|
|
8
8
|
}
|
|
9
9
|
catch (err) {
|
|
10
|
-
console.log(
|
|
10
|
+
console.log('Could not create a browser instance: ', err);
|
|
11
11
|
process.exit(1);
|
|
12
12
|
}
|
|
13
13
|
}
|
|
@@ -15,7 +15,7 @@ export async function getHtmlWithPuppeteer(href) {
|
|
|
15
15
|
const browser = await startBrowser();
|
|
16
16
|
const page = await browser.newPage();
|
|
17
17
|
await page.goto(href, {
|
|
18
|
-
waitUntil:
|
|
18
|
+
waitUntil: 'networkidle2',
|
|
19
19
|
});
|
|
20
20
|
const html = await page.content();
|
|
21
21
|
browser.close();
|
package/bin/constants.js
CHANGED
|
@@ -1,30 +1,30 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import
|
|
1
|
+
import os from 'os';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import * as url from 'url';
|
|
4
4
|
// package installation location
|
|
5
|
-
export const INSTALL_PATH = url.fileURLToPath(new URL(
|
|
5
|
+
export const INSTALL_PATH = url.fileURLToPath(new URL('.', import.meta.url));
|
|
6
6
|
export const HOME_DIR = os.homedir();
|
|
7
|
-
export const DOT_MINTLIFY = path.join(HOME_DIR,
|
|
8
|
-
export const VERSION_PATH = path.join(DOT_MINTLIFY,
|
|
9
|
-
export const CLIENT_PATH = path.join(DOT_MINTLIFY,
|
|
10
|
-
export const MINT_PATH = path.join(DOT_MINTLIFY,
|
|
7
|
+
export const DOT_MINTLIFY = path.join(HOME_DIR, '.mintlify');
|
|
8
|
+
export const VERSION_PATH = path.join(DOT_MINTLIFY, 'mint', 'mint-version.txt');
|
|
9
|
+
export const CLIENT_PATH = path.join(DOT_MINTLIFY, 'mint', 'client');
|
|
10
|
+
export const MINT_PATH = path.join(DOT_MINTLIFY, 'mint');
|
|
11
11
|
// command execution location
|
|
12
12
|
export const CMD_EXEC_PATH = process.cwd();
|
|
13
13
|
export const SUPPORTED_MEDIA_EXTENSIONS = [
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
14
|
+
'jpeg',
|
|
15
|
+
'jpg',
|
|
16
|
+
'jfif',
|
|
17
|
+
'pjpeg',
|
|
18
|
+
'pjp',
|
|
19
|
+
'png',
|
|
20
|
+
'svg',
|
|
21
|
+
'svgz',
|
|
22
|
+
'ico',
|
|
23
|
+
'webp',
|
|
24
|
+
'gif',
|
|
25
|
+
'apng',
|
|
26
|
+
'avif',
|
|
27
|
+
'bmp',
|
|
28
|
+
'mp4',
|
|
29
29
|
];
|
|
30
30
|
//# sourceMappingURL=constants.js.map
|
package/bin/constants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAE3B,gCAAgC;AAChC,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;AAErC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AAE7D,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAErE,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAEzD,6BAA6B;AAC7B,MAAM,CAAC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;AAE3C,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;CACN,CAAC"}
|
package/bin/downloadImage.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
1
|
+
import axios from 'axios';
|
|
2
|
+
import { existsSync, mkdirSync, createWriteStream } from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { SUPPORTED_MEDIA_EXTENSIONS } from './constants.js';
|
|
5
|
+
import { getFileExtension } from './util.js';
|
|
6
6
|
async function writeImageToFile(imageSrc, writePath, overwrite) {
|
|
7
7
|
// Avoid unnecessary downloads
|
|
8
8
|
if (existsSync(writePath) && !overwrite) {
|
|
9
9
|
return Promise.reject({
|
|
10
|
-
code:
|
|
10
|
+
code: 'EEXIST',
|
|
11
11
|
});
|
|
12
12
|
}
|
|
13
13
|
// Create the folders needed if they're missing
|
|
@@ -15,21 +15,21 @@ async function writeImageToFile(imageSrc, writePath, overwrite) {
|
|
|
15
15
|
const writer = createWriteStream(writePath);
|
|
16
16
|
try {
|
|
17
17
|
const response = await axios.get(imageSrc, {
|
|
18
|
-
responseType:
|
|
18
|
+
responseType: 'stream',
|
|
19
19
|
});
|
|
20
20
|
// wx prevents overwriting an image with the exact same name
|
|
21
21
|
// being created in the time we were downloading
|
|
22
22
|
response.data.pipe(writer, {
|
|
23
|
-
flag:
|
|
23
|
+
flag: 'wx',
|
|
24
24
|
});
|
|
25
25
|
return new Promise((resolve, reject) => {
|
|
26
|
-
writer.on(
|
|
27
|
-
writer.on(
|
|
26
|
+
writer.on('finish', resolve);
|
|
27
|
+
writer.on('error', reject);
|
|
28
28
|
});
|
|
29
29
|
}
|
|
30
30
|
catch (e) {
|
|
31
31
|
return Promise.reject({
|
|
32
|
-
code:
|
|
32
|
+
code: 'ENOTFOUND',
|
|
33
33
|
});
|
|
34
34
|
}
|
|
35
35
|
}
|
|
@@ -38,20 +38,20 @@ export function isValidImageSrc(src) {
|
|
|
38
38
|
return false;
|
|
39
39
|
}
|
|
40
40
|
// We do not support downloading base64 in-line images.
|
|
41
|
-
if (src.startsWith(
|
|
41
|
+
if (src.startsWith('data:')) {
|
|
42
42
|
return false;
|
|
43
43
|
}
|
|
44
44
|
const imageHref = removeMetadataFromImageSrc(src);
|
|
45
45
|
const ext = getFileExtension(imageHref);
|
|
46
46
|
if (ext && !SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
|
|
47
|
-
console.error(
|
|
47
|
+
console.error('🚨 We do not support the file extension: ' + ext);
|
|
48
48
|
return false;
|
|
49
49
|
}
|
|
50
50
|
return true;
|
|
51
51
|
}
|
|
52
52
|
export function removeMetadataFromImageSrc(src) {
|
|
53
53
|
// Part of the URL standard
|
|
54
|
-
const metadataSymbols = [
|
|
54
|
+
const metadataSymbols = ['?', '#'];
|
|
55
55
|
metadataSymbols.forEach((dividerSymbol) => {
|
|
56
56
|
// Some frameworks add metadata after the file extension, we need to remove that.
|
|
57
57
|
src = src.split(dividerSymbol)[0];
|
|
@@ -60,18 +60,18 @@ export function removeMetadataFromImageSrc(src) {
|
|
|
60
60
|
}
|
|
61
61
|
export function cleanImageSrc(src, origin) {
|
|
62
62
|
// Add origin if the image tags are using relative sources
|
|
63
|
-
return src.startsWith(
|
|
63
|
+
return src.startsWith('http') ? src : new URL(src, origin).href;
|
|
64
64
|
}
|
|
65
65
|
export default async function downloadImage(imageSrc, writePath, overwrite = false) {
|
|
66
66
|
await writeImageToFile(imageSrc, writePath, overwrite)
|
|
67
67
|
.then(() => {
|
|
68
|
-
console.log(
|
|
68
|
+
console.log('🖼️ - ' + writePath);
|
|
69
69
|
})
|
|
70
70
|
.catch((e) => {
|
|
71
|
-
if (e.code ===
|
|
71
|
+
if (e.code === 'EEXIST') {
|
|
72
72
|
console.log(`❌ Skipping existing image ${writePath}`);
|
|
73
73
|
}
|
|
74
|
-
else if (e.code ===
|
|
74
|
+
else if (e.code === 'ENOTFOUND') {
|
|
75
75
|
console.error(`🚨 Cannot download the image, address not found ${imageSrc}`);
|
|
76
76
|
}
|
|
77
77
|
else {
|
package/bin/downloadImage.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,SAAiB,EAAE,SAAkB;IACrF,8BAA8B;IAC9B,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;KACJ;IAED,+CAA+C;IAC/C,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;YACzC,YAAY,EAAE,QAAQ;SACvB,CAAC,CAAC;QACH,4DAA4D;QAC5D,gDAAgD;QAChD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACzB,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;KACJ;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,EAAE;QACR,OAAO,KAAK,CAAC;KACd;IACD,uDAAuD;IACvD,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAC3B,OAAO,KAAK,CAAC;KACd;IAED,MAAM,SAAS,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAExC,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACpD,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACjE,OAAO,KAAK,CAAC;KACd;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,GAAW;IACpD,2BAA2B;IAC3B,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEnC,eAAe,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;QACxC,iFAAiF;QACjF,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAc;IACvD,0DAA0D;IAC1D,OAAO,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,aAAa,CACzC,QAAgB,EAChB,SAAiB,EACjB,SAAS,GAAG,KAAK;IAEjB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;SACnD,IAAI,CAAC,GAAG,EAAE;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;IACpC,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACX,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;SACvD;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE;YACjC,OAAO,CAAC,KAAK,CAAC,mDAAmD,QAAQ,EAAE,CAAC,CAAC;SAC9E;aAAM;YACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClB;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import cheerio from
|
|
1
|
+
import cheerio from 'cheerio';
|
|
2
2
|
export var Frameworks;
|
|
3
3
|
(function (Frameworks) {
|
|
4
4
|
Frameworks["DOCUSAURUS"] = "DOCUSAURUS";
|
|
@@ -10,22 +10,22 @@ export function detectFramework(html) {
|
|
|
10
10
|
const $ = cheerio.load(html);
|
|
11
11
|
const docusaurusMeta = $('meta[name="generator"]');
|
|
12
12
|
if (docusaurusMeta.length > 0 &&
|
|
13
|
-
docusaurusMeta.attr(
|
|
14
|
-
typeof docusaurusMeta.attr(
|
|
15
|
-
docusaurusMeta.attr(
|
|
16
|
-
const metaAttrString = docusaurusMeta.attr(
|
|
17
|
-
if (metaAttrString.includes(
|
|
18
|
-
return { framework: Frameworks.DOCUSAURUS, version:
|
|
13
|
+
docusaurusMeta.attr('content') &&
|
|
14
|
+
typeof docusaurusMeta.attr('content') === 'string' &&
|
|
15
|
+
docusaurusMeta.attr('content').includes('Docusaurus')) {
|
|
16
|
+
const metaAttrString = docusaurusMeta.attr('content');
|
|
17
|
+
if (metaAttrString.includes('v3')) {
|
|
18
|
+
return { framework: Frameworks.DOCUSAURUS, version: '3' };
|
|
19
19
|
}
|
|
20
|
-
if (metaAttrString.includes(
|
|
21
|
-
return { framework: Frameworks.DOCUSAURUS, version:
|
|
20
|
+
if (metaAttrString.includes('v2')) {
|
|
21
|
+
return { framework: Frameworks.DOCUSAURUS, version: '2' };
|
|
22
22
|
}
|
|
23
|
-
else if (metaAttrString.includes(
|
|
24
|
-
console.warn(
|
|
25
|
-
return { framework: Frameworks.DOCUSAURUS, version:
|
|
23
|
+
else if (metaAttrString.includes('v1')) {
|
|
24
|
+
console.warn('WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.');
|
|
25
|
+
return { framework: Frameworks.DOCUSAURUS, version: '1' };
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
|
-
const isGitBook = $(
|
|
28
|
+
const isGitBook = $('.gitbook-root').length > 0;
|
|
29
29
|
if (isGitBook) {
|
|
30
30
|
return { framework: Frameworks.GITBOOK };
|
|
31
31
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/scraping/detectFramework.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,CAAN,IAAY,UAKX;AALD,WAAY,UAAU;IACpB,uCAAyB,CAAA;IACzB,iCAAmB,CAAA;IACnB,+BAAiB,CAAA;IACjB,mCAAqB,CAAA;AACvB,CAAC,EALW,UAAU,KAAV,UAAU,QAKrB;AAED,MAAM,UAAU,eAAe,CAAC,IAAI;IAClC,MAAM,CAAC,GAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEnD,IACE,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ;QACjD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EACjE;QACA,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAW,CAAC;QAChE,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;QACD,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACxC,OAAO,CAAC,IAAI,CACV,0FAA0F,CAC3F,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;KACF;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE;QACb,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;KAC1C;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5D,IAAI,QAAQ,EAAE;QACZ,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;KACzC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,IAAI,UAAU,EAAE;QACd,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;KAC3C;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,
|
|
1
|
+
{"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/scraping/detectFramework.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,CAAN,IAAY,UAKX;AALD,WAAY,UAAU;IACpB,uCAAyB,CAAA;IACzB,iCAAmB,CAAA;IACnB,+BAAiB,CAAA;IACjB,mCAAqB,CAAA;AACvB,CAAC,EALW,UAAU,KAAV,UAAU,QAKrB;AAED,MAAM,UAAU,eAAe,CAAC,IAAI;IAClC,MAAM,CAAC,GAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEnD,IACE,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ;QACjD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EACjE;QACA,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAW,CAAC;QAChE,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;QACD,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACxC,OAAO,CAAC,IAAI,CACV,0FAA0F,CAC3F,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;KACF;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE;QACb,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;KAC1C;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5D,IAAI,QAAQ,EAAE;QACZ,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;KACzC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,IAAI,UAAU,EAAE;QACd,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;KAC3C;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC"}
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import path from
|
|
2
|
-
import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from '../downloadImage.js';
|
|
3
3
|
// To Do: Use CheerioElement instead of any when we bump the cheerio version
|
|
4
4
|
export default async function downloadAllImages($, content, origin, baseDir, overwrite, modifyFileName) {
|
|
5
5
|
if (!baseDir) {
|
|
6
|
-
console.debug(
|
|
6
|
+
console.debug('Skipping image downloading');
|
|
7
7
|
return;
|
|
8
8
|
}
|
|
9
9
|
// We remove duplicates because some frameworks duplicate img tags
|
|
10
10
|
// to show the image larger when clicked on.
|
|
11
11
|
const imageSrcs = [
|
|
12
12
|
...new Set(content
|
|
13
|
-
.find(
|
|
14
|
-
.map((i, image) => $(image).attr(
|
|
13
|
+
.find('img[src]')
|
|
14
|
+
.map((i, image) => $(image).attr('src'))
|
|
15
15
|
.toArray()),
|
|
16
16
|
];
|
|
17
17
|
// Wait to all images to download before continuing
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,SAAkB,EAClB,cAAoB;IAEpB,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;KACR;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE;YAC3C,OAAO,EAAE,CAAC;SACX;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE;YAClB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;SACrC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;AACxF,CAAC"}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import path from
|
|
2
|
-
import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from
|
|
3
|
-
import { getFileExtension } from
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from '../downloadImage.js';
|
|
3
|
+
import { getFileExtension } from '../util.js';
|
|
4
4
|
// To Do: Use CheerioElement instead of any when we bump the cheerio version
|
|
5
5
|
export default async function downloadLogoImage(imageSrc, imageBaseDir, origin, overwrite) {
|
|
6
6
|
if (!imageSrc || !isValidImageSrc(imageSrc))
|
|
7
7
|
return;
|
|
8
8
|
const imageHref = cleanImageSrc(imageSrc, origin);
|
|
9
9
|
const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
|
|
10
|
-
const imagePath = path.join(imageBaseDir,
|
|
10
|
+
const imagePath = path.join(imageBaseDir, 'logo', 'logo-light-mode.' + ext);
|
|
11
11
|
await downloadImage(imageHref, imagePath, overwrite);
|
|
12
12
|
}
|
|
13
13
|
//# sourceMappingURL=downloadLogoImage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,QAA4B,EAC5B,YAAoB,EACpB,MAAc,EACd,SAAkB;IAElB,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;QAAE,OAAO;IAEpD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAE5E,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import axios from
|
|
1
|
+
import axios from 'axios';
|
|
2
2
|
// Not in use.
|
|
3
3
|
// Gets all links in a sitemap.
|
|
4
4
|
export const getSitemapLinks = async (url) => {
|
|
5
|
-
const hostname = url.hostname.replace(
|
|
6
|
-
const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`,
|
|
5
|
+
const hostname = url.hostname.replace('.', '\\.');
|
|
6
|
+
const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, 'gmi');
|
|
7
7
|
try {
|
|
8
8
|
const indexData = (await axios.get(url.href)).data;
|
|
9
9
|
const array = indexData.match(regex);
|
|
@@ -11,7 +11,7 @@ export const getSitemapLinks = async (url) => {
|
|
|
11
11
|
}
|
|
12
12
|
catch (err) {
|
|
13
13
|
console.error(err);
|
|
14
|
-
console.log(
|
|
14
|
+
console.log('Skipping sitemap links because we encountered an error.');
|
|
15
15
|
return [];
|
|
16
16
|
}
|
|
17
17
|
};
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import { getHtmlWithPuppeteer } from
|
|
4
|
-
import { createPage } from
|
|
1
|
+
import axios from 'axios';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { getHtmlWithPuppeteer } from '../browser.js';
|
|
4
|
+
import { createPage } from '../util.js';
|
|
5
5
|
export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, version, baseToRemove) {
|
|
6
6
|
// Skip scraping external links
|
|
7
|
-
if (pathname.startsWith(
|
|
7
|
+
if (pathname.startsWith('https://') || pathname.startsWith('http://')) {
|
|
8
8
|
return pathname;
|
|
9
9
|
}
|
|
10
10
|
// Removes file name from the end
|
|
11
|
-
const splitSubpath = pathname.split(
|
|
12
|
-
let folders = splitSubpath.slice(0, splitSubpath.length - 1).join(
|
|
11
|
+
const splitSubpath = pathname.split('/');
|
|
12
|
+
let folders = splitSubpath.slice(0, splitSubpath.length - 1).join('/');
|
|
13
13
|
// Remove base dir if passed in
|
|
14
14
|
if (baseToRemove && folders.startsWith(baseToRemove)) {
|
|
15
|
-
folders = folders.replace(baseToRemove,
|
|
15
|
+
folders = folders.replace(baseToRemove, '');
|
|
16
16
|
}
|
|
17
17
|
// TO DO: Improve this by putting each page's images in a separate
|
|
18
18
|
// folder named after the title of the page.
|
|
19
|
-
const imageBaseDir = path.join(cliDir,
|
|
19
|
+
const imageBaseDir = path.join(cliDir, 'images', folders);
|
|
20
20
|
// Scrape each page separately
|
|
21
21
|
const href = new URL(pathname, origin).href;
|
|
22
22
|
let html;
|
|
@@ -34,7 +34,7 @@ export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin,
|
|
|
34
34
|
}
|
|
35
35
|
const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
|
|
36
36
|
// Default to introduction.mdx if we encountered index.html
|
|
37
|
-
const fileName = splitSubpath[splitSubpath.length - 1] ||
|
|
37
|
+
const fileName = splitSubpath[splitSubpath.length - 1] || 'introduction';
|
|
38
38
|
// Will create subfolders as needed
|
|
39
39
|
createPage(title ?? '', description, markdown, overwrite, newFileLocation, fileName);
|
|
40
40
|
// Removes first slash if we are in a folder, Mintlify doesn't need it
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;KACtB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CAAC,KAAK,IAAI,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,CAAC,CAAC;IAErF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { scrapeFileGettingFileNameFromUrl } from
|
|
1
|
+
import { scrapeFileGettingFileNameFromUrl } from './scrapeFileGettingFileNameFromUrl.js';
|
|
2
2
|
export async function scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, version, baseToRemove) {
|
|
3
|
-
if (typeof navEntry !==
|
|
3
|
+
if (typeof navEntry !== 'string') {
|
|
4
4
|
const newPages = [];
|
|
5
5
|
for (const nestedNavEntry of navEntry.pages) {
|
|
6
6
|
newPages.push(await scrapeGettingFileNameFromUrl(nestedNavEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer, version, baseToRemove));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AAEzF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,QAA6B,EAC7B,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,MAAM,QAAQ,
|
|
1
|
+
{"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AAEzF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,QAA6B,EAC7B,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAC3C,KAAK,MAAM,cAAc,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC3C,QAAQ,CAAC,IAAI,CACX,MAAM,4BAA4B,CAChC,cAAc,EACd,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CACF,CAAC;SACH;QACD,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC1B,OAAO,QAAQ,CAAC;KACjB;IAED,OAAO,MAAM,gCAAgC,CAC3C,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CAAC;AACJ,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import path from
|
|
2
|
-
import { createPage, getOrigin } from
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { createPage, getOrigin } from '../util.js';
|
|
3
3
|
export async function scrapePage(scrapeFunc, href, html, overwrite, version) {
|
|
4
4
|
const origin = getOrigin(href);
|
|
5
5
|
const cwd = process.cwd();
|
|
6
|
-
const imageBaseDir = path.join(cwd,
|
|
6
|
+
const imageBaseDir = path.join(cwd, 'images');
|
|
7
7
|
const { title, description, markdown } = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
|
|
8
8
|
createPage(title, description, markdown, overwrite, process.cwd());
|
|
9
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapePage.js","sourceRoot":"","sources":["../../src/scraping/scrapePage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapePage.js","sourceRoot":"","sources":["../../src/scraping/scrapePage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAwB,EACxB,IAAY,EACZ,IAAY,EACZ,SAAkB,EAClB,OAA2B;IAE3B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CACvD,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AACrE,CAAC"}
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
import axios from
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
1
|
+
import axios from 'axios';
|
|
2
|
+
import { getHtmlWithPuppeteer } from '../browser.js';
|
|
3
|
+
import { getHrefFromArgs } from '../util.js';
|
|
4
|
+
import { detectFramework, Frameworks } from './detectFramework.js';
|
|
5
|
+
import { scrapePage } from './scrapePage.js';
|
|
6
|
+
import { scrapeIntercomPage } from './site-scrapers/Intercom/scrapeIntercomPage.js';
|
|
7
|
+
import { scrapeDocusaurusPage } from './site-scrapers/scrapeDocusaurusPage.js';
|
|
8
|
+
import { scrapeGitBookPage } from './site-scrapers/scrapeGitBookPage.js';
|
|
9
|
+
import { scrapeReadMePage } from './site-scrapers/scrapeReadMePage.js';
|
|
10
10
|
function validateFramework(framework) {
|
|
11
11
|
if (!framework) {
|
|
12
|
-
console.log(
|
|
13
|
-
console.log(
|
|
14
|
-
console.log(
|
|
15
|
-
console.log(
|
|
16
|
-
console.log(
|
|
12
|
+
console.log('Could not detect the framework automatically. Please use one of:');
|
|
13
|
+
console.log('scrape-page-docusaurus');
|
|
14
|
+
console.log('scrape-page-gitbook');
|
|
15
|
+
console.log('scrape-page-readme');
|
|
16
|
+
console.log('scrape-page-intercom');
|
|
17
17
|
return process.exit(1);
|
|
18
18
|
}
|
|
19
19
|
}
|
|
@@ -36,7 +36,7 @@ export async function scrapePageAutomatically(argv) {
|
|
|
36
36
|
const html = res.data;
|
|
37
37
|
const { framework, version } = detectFramework(html);
|
|
38
38
|
validateFramework(framework);
|
|
39
|
-
console.log(
|
|
39
|
+
console.log('Detected framework: ' + framework);
|
|
40
40
|
switch (framework) {
|
|
41
41
|
case Frameworks.DOCUSAURUS:
|
|
42
42
|
await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAEvE,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;QAChF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAwB,EACxB,UAAwB,EACxB,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,EAAE,SAAS,EAAE;QACtB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjE,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAChD,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAClD,MAAM;KACT;AACH,CAAC"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import path from
|
|
2
|
-
import { objToReadableString } from
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { objToReadableString } from '../util.js';
|
|
3
3
|
export async function scrapeSection(scrapeFunc, html, origin, overwrite, version) {
|
|
4
|
-
console.log(`Started scraping${overwrite ?
|
|
4
|
+
console.log(`Started scraping${overwrite ? ', overwrite mode is on' : ''}...`);
|
|
5
5
|
const cwd = process.cwd();
|
|
6
|
-
const imageBaseDir = path.join(cwd,
|
|
6
|
+
const imageBaseDir = path.join(cwd, 'images');
|
|
7
7
|
const groupsConfig = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
|
|
8
|
-
console.log(
|
|
9
|
-
console.log(
|
|
8
|
+
console.log('Finished scraping.');
|
|
9
|
+
console.log('Add the following to your navigation in mint.json:');
|
|
10
10
|
console.log(objToReadableString(groupsConfig));
|
|
11
11
|
}
|
|
12
12
|
//# sourceMappingURL=scrapeSection.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeSection.js","sourceRoot":"","sources":["../../src/scraping/scrapeSection.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeSection.js","sourceRoot":"","sources":["../../src/scraping/scrapeSection.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA2B,EAC3B,IAAY,EACZ,MAAc,EACd,SAAkB,EAClB,OAA2B;IAE3B,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;AACjD,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ArgumentsCamelCase } from
|
|
1
|
+
import { ArgumentsCamelCase } from 'yargs';
|
|
2
2
|
export declare function scrapeSectionAxiosWrapper(argv: ArgumentsCamelCase, scrapeFunc: ScrapeSectionFn): Promise<void>;
|
|
3
3
|
export declare function scrapeDocusaurusSectionCommand(argv: any, version: string | undefined): Promise<void>;
|
|
4
4
|
export declare function scrapeGitbookSectionCommand(argv: any): Promise<void>;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import axios from
|
|
2
|
-
import {
|
|
3
|
-
import { getHrefFromArgs, getOrigin } from
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import
|
|
7
|
-
import
|
|
8
|
-
import openNestedGitbookMenus from
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
1
|
+
import axios from 'axios';
|
|
2
|
+
import { startBrowser } from '../browser.js';
|
|
3
|
+
import { getHrefFromArgs, getOrigin } from '../util.js';
|
|
4
|
+
import { detectFramework, Frameworks } from './detectFramework.js';
|
|
5
|
+
import { scrapeSection } from './scrapeSection.js';
|
|
6
|
+
import { scrapeIntercomSection } from './site-scrapers/Intercom/scrapeIntercomSection.js';
|
|
7
|
+
import openNestedDocusaurusMenus from './site-scrapers/openNestedDocusaurusMenus.js';
|
|
8
|
+
import openNestedGitbookMenus from './site-scrapers/openNestedGitbookMenus.js';
|
|
9
|
+
import { scrapeDocusaurusSection } from './site-scrapers/scrapeDocusaurusSection.js';
|
|
10
|
+
import { scrapeGitBookSection } from './site-scrapers/scrapeGitBookSection.js';
|
|
11
|
+
import { scrapeReadMeSection } from './site-scrapers/scrapeReadMeSection.js';
|
|
12
12
|
export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
|
|
13
13
|
const href = getHrefFromArgs(argv);
|
|
14
14
|
const res = await axios.get(href);
|
|
@@ -28,7 +28,7 @@ async function scrapeSectionOpeningAllNested(argv, openLinks, scrapeFunc, versio
|
|
|
28
28
|
const browser = await startBrowser();
|
|
29
29
|
const page = await browser.newPage();
|
|
30
30
|
await page.goto(href, {
|
|
31
|
-
waitUntil:
|
|
31
|
+
waitUntil: 'networkidle2',
|
|
32
32
|
});
|
|
33
33
|
const html = await openLinks(page);
|
|
34
34
|
browser.close();
|
|
@@ -41,7 +41,7 @@ export async function scrapeSectionAutomatically(argv) {
|
|
|
41
41
|
const html = res.data;
|
|
42
42
|
const { framework, version } = detectFramework(html);
|
|
43
43
|
validateFramework(framework);
|
|
44
|
-
console.log(
|
|
44
|
+
console.log('Detected framework: ' + framework);
|
|
45
45
|
switch (framework) {
|
|
46
46
|
case Frameworks.DOCUSAURUS:
|
|
47
47
|
await scrapeDocusaurusSectionCommand(argv, version);
|
|
@@ -59,7 +59,7 @@ export async function scrapeSectionAutomatically(argv) {
|
|
|
59
59
|
}
|
|
60
60
|
function validateFramework(framework) {
|
|
61
61
|
if (!framework) {
|
|
62
|
-
console.log(
|
|
62
|
+
console.log('Could not detect the framework automatically. We only support Docusaurus (V2 and V3), GitBook, and ReadMe.');
|
|
63
63
|
process.exit();
|
|
64
64
|
}
|
|
65
65
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mDAAmD,CAAC;AAC1F,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAwB,EACxB,UAA2B;IAE3B,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAS,EACT,OAA2B,CAAC,kBAAkB;;IAE9C,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CAAC,IAAI,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC;AAC1F,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAA2B,EAC3B,OAAgB;IAEhB,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAC3D,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,yBAAyB,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;YAC7D,MAAM;KACT;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;KAChB;AACH,CAAC"}
|