@mintlify/scraping 3.0.141 → 3.0.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/browser.js +1 -1
- package/bin/browser.js.map +1 -1
- package/bin/checks.d.ts +8 -0
- package/bin/checks.js +24 -0
- package/bin/checks.js.map +1 -0
- package/bin/cli.js +49 -45
- package/bin/cli.js.map +1 -1
- package/bin/scraping/detectFramework.d.ts +7 -14
- package/bin/scraping/detectFramework.js +8 -14
- package/bin/scraping/detectFramework.js.map +1 -1
- package/bin/scraping/downloadAllImages.d.ts +2 -1
- package/bin/scraping/downloadAllImages.js +1 -2
- package/bin/scraping/downloadAllImages.js.map +1 -1
- package/bin/scraping/downloadLogoImage.js +0 -1
- package/bin/scraping/downloadLogoImage.js.map +1 -1
- package/bin/scraping/replaceImagePaths.d.ts +1 -1
- package/bin/scraping/replaceImagePaths.js +0 -3
- package/bin/scraping/replaceImagePaths.js.map +1 -1
- package/bin/scraping/scrapePageCommands.d.ts +3 -3
- package/bin/scraping/scrapePageCommands.js +22 -27
- package/bin/scraping/scrapePageCommands.js.map +1 -1
- package/bin/scraping/scrapeSectionCommands.d.ts +5 -5
- package/bin/scraping/scrapeSectionCommands.js +27 -30
- package/bin/scraping/scrapeSectionCommands.js.map +1 -1
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js +1 -1
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js.map +1 -1
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js +2 -2
- package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js.map +1 -1
- package/bin/scraping/site-scrapers/alternateGroupTitle.d.ts +3 -1
- package/bin/scraping/site-scrapers/alternateGroupTitle.js +1 -1
- package/bin/scraping/site-scrapers/alternateGroupTitle.js.map +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.d.ts +5 -1
- package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js.map +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.d.ts +3 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js +1 -4
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js.map +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.d.ts +3 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js +0 -3
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js.map +1 -1
- package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js +3 -3
- package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js.map +1 -1
- package/bin/scraping/site-scrapers/openNestedGitbookMenus.js +2 -2
- package/bin/scraping/site-scrapers/openNestedGitbookMenus.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js +1 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js +2 -2
- package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js +1 -1
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMePage.js +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMePage.js.map +1 -1
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js +2 -2
- package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map +1 -1
- package/bin/tsconfig.build.tsbuildinfo +1 -1
- package/bin/util.d.ts +0 -1
- package/bin/util.js +9 -26
- package/bin/util.js.map +1 -1
- package/package.json +4 -4
- package/src/browser.ts +1 -1
- package/src/checks.ts +32 -0
- package/src/cli.ts +48 -74
- package/src/scraping/detectFramework.ts +20 -15
- package/src/scraping/downloadAllImages.ts +7 -7
- package/src/scraping/downloadLogoImage.ts +0 -1
- package/src/scraping/replaceImagePaths.ts +1 -5
- package/src/scraping/scrapePageCommands.ts +32 -29
- package/src/scraping/scrapeSectionCommands.ts +38 -34
- package/src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts +1 -1
- package/src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts +2 -2
- package/src/scraping/site-scrapers/alternateGroupTitle.ts +5 -2
- package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts +7 -5
- package/src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts +8 -6
- package/src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts +7 -5
- package/src/scraping/site-scrapers/openNestedDocusaurusMenus.ts +3 -3
- package/src/scraping/site-scrapers/openNestedGitbookMenus.ts +3 -3
- package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts +1 -1
- package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts +2 -2
- package/src/scraping/site-scrapers/scrapeGitBookPage.ts +1 -1
- package/src/scraping/site-scrapers/scrapeGitBookSection.ts +3 -3
- package/src/scraping/site-scrapers/scrapeReadMePage.ts +1 -1
- package/src/scraping/site-scrapers/scrapeReadMeSection.ts +3 -3
- package/src/util.ts +10 -26
- package/tsconfig.json +0 -1
- package/bin/validation/isValidLink.d.ts +0 -1
- package/bin/validation/isValidLink.js +0 -11
- package/bin/validation/isValidLink.js.map +0 -1
- package/bin/validation/stopIfInvalidLink.d.ts +0 -1
- package/bin/validation/stopIfInvalidLink.js +0 -9
- package/bin/validation/stopIfInvalidLink.js.map +0 -1
- package/src/validation/isValidLink.ts +0 -9
- package/src/validation/stopIfInvalidLink.ts +0 -9
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeIntercomSection.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeIntercomSection.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,uCAAuC,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3B,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3D,KAAK,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAEjE,MAAM,eAAe,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAkB,EAAE,EAAE;QAChF,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,YAAY,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,YAAY,GAAG,CAAC,CAAC,sBAAsB,CAAC;aAC3C,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAC9C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAa,CAAC;QACpD,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAe,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAElE,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,KAAsB,EAAE,EAAE;QAC/C,OAAO,MAAM,4BAA4B,CACvC,KAAK,EACL,MAAM,EACN,MAAM,EACN,SAAS,EACT,kBAAkB,EAClB,KAAK,EACL,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1,3 @@
|
|
|
1
|
-
|
|
1
|
+
import { NavigationEntry } from '@mintlify/models';
|
|
2
|
+
import { Cheerio, Element } from 'cheerio';
|
|
3
|
+
export default function alternateGroupTitle(firstLink: Cheerio<Element>, pages: NavigationEntry[]): string;
|
|
@@ -2,7 +2,7 @@ export default function alternateGroupTitle(firstLink, pages) {
|
|
|
2
2
|
// Only assign titles to nested navigation menus outside a section.
|
|
3
3
|
// Others should not have a title so we can merge them into one section.
|
|
4
4
|
if (pages.length > 0) {
|
|
5
|
-
return firstLink
|
|
5
|
+
return firstLink.text();
|
|
6
6
|
}
|
|
7
7
|
return '';
|
|
8
8
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alternateGroupTitle.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/alternateGroupTitle.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"alternateGroupTitle.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/alternateGroupTitle.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,SAA2B,EAAE,KAAwB;IAC/F,mEAAmE;IACnE,wEAAwE;IACxE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -1 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import { Cheerio, CheerioAPI, Element } from 'cheerio';
|
|
2
|
+
export declare function getDocusaurusLinksPerGroup(navigationSections: Cheerio<Element>, $: CheerioAPI, version: string | undefined): {
|
|
3
|
+
group: string;
|
|
4
|
+
pages: import("@mintlify/models").NavigationEntry[];
|
|
5
|
+
}[];
|
|
@@ -14,7 +14,7 @@ function getDocusaurusLinksPerGroupLoop(navigationSections, $) {
|
|
|
14
14
|
const linkHref = section.find('a[href]').first().attr('href');
|
|
15
15
|
return {
|
|
16
16
|
group: '',
|
|
17
|
-
pages: [linkHref],
|
|
17
|
+
pages: linkHref !== undefined ? [linkHref] : [],
|
|
18
18
|
};
|
|
19
19
|
}
|
|
20
20
|
const firstLink = section.find('.menu__list-item-collapsible').first().find('a[href]');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getDocusaurusLinksPerGroup.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"getDocusaurusLinksPerGroup.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts"],"names":[],"mappings":"AAEA,OAAO,mBAAmB,MAAM,2BAA2B,CAAC;AAC5D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,UAAU,0BAA0B,CACxC,kBAAoC,EACpC,CAAa,EACb,OAA2B;IAE3B,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,GAAG,EAAE,CAAC;QACvC,OAAO,8BAA8B,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,8BAA8B,CAAC,kBAAoC,EAAE,CAAa;IACzF,OAAO,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5C,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,wBAAwB;QACxB,IAAI,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;aAChD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEvF,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +1,3 @@
|
|
|
1
|
-
|
|
1
|
+
import { NavigationEntry } from '@mintlify/models';
|
|
2
|
+
import { Cheerio, CheerioAPI, Element } from 'cheerio';
|
|
3
|
+
export default function getLinksRecursively(linkSections: Cheerio<Element>, $: CheerioAPI): NavigationEntry[];
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
// Used by Docusaurus and ReadMe section scrapers
|
|
2
2
|
export default function getLinksRecursively(linkSections, $) {
|
|
3
|
-
if (linkSections == null || linkSections.length === 0) {
|
|
4
|
-
return [];
|
|
5
|
-
}
|
|
6
3
|
return linkSections
|
|
7
|
-
.map((
|
|
4
|
+
.map((_, s) => {
|
|
8
5
|
const subsection = $(s);
|
|
9
6
|
let link = subsection.children().first();
|
|
10
7
|
if (!link.attr('href')) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts"],"names":[],"mappings":"AAGA,iDAAiD;AACjD,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,YAA8B,EAC9B,CAAa;IAEb,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAEzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACvB,0CAA0C;YAC1C,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;QACtC,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,KAAK,GAAG;YAChB,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aACzD,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
|
|
@@ -1 +1,3 @@
|
|
|
1
|
-
|
|
1
|
+
import { NavigationEntry } from '@mintlify/models';
|
|
2
|
+
import { Cheerio, CheerioAPI, Element } from 'cheerio';
|
|
3
|
+
export default function getLinksRecursivelyGitBook(linkSections: Cheerio<Element>, $: CheerioAPI): NavigationEntry[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getLinksRecursivelyGitBook.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"getLinksRecursivelyGitBook.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts"],"names":[],"mappings":"AAGA,kCAAkC;AAClC,MAAM,CAAC,OAAO,UAAU,0BAA0B,CAChD,YAA8B,EAC9B,CAAa;IAEb,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtB,yDAAyD;QACzD,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,KAAK,GAAG;YAChB,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,0BAA0B,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aAChE,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
|
|
@@ -7,15 +7,15 @@ export default async function openNestedDocusaurusMenus(page) {
|
|
|
7
7
|
encounteredHref = await page.evaluate((encounteredHref) => {
|
|
8
8
|
const collapsible = Array.from(document.querySelectorAll('.menu__link.menu__link--sublist'));
|
|
9
9
|
const linksFound = [];
|
|
10
|
-
collapsible.forEach(
|
|
11
|
-
const href = collapsibleItem
|
|
10
|
+
collapsible.forEach((collapsibleItem) => {
|
|
11
|
+
const href = collapsibleItem.getAttribute('href');
|
|
12
12
|
// Should never occur but we keep it as a fail-safe
|
|
13
13
|
if (href?.startsWith('https://') || href?.startsWith('http://')) {
|
|
14
14
|
return;
|
|
15
15
|
}
|
|
16
16
|
// Click any links we haven't seen before
|
|
17
17
|
if (href && !encounteredHref.includes(href)) {
|
|
18
|
-
collapsibleItem
|
|
18
|
+
collapsibleItem.click();
|
|
19
19
|
}
|
|
20
20
|
if (href) {
|
|
21
21
|
linksFound.push(href);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openNestedDocusaurusMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedDocusaurusMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAU;IAChE,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,GAAG,CAAC,0CAA0C,CAAC,CAAC;IAEnE,0CAA0C;IAC1C,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACxE,eAAe,GAAG,eAAe,CAAC;QAClC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CACnC,CAAC,eAAe,EAAE,EAAE;YAClB,MAAM,WAAW,GAAkB,KAAK,CAAC,IAAI,CAC3C,QAAQ,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAC7D,CAAC;YAEF,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,WAAW,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"openNestedDocusaurusMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedDocusaurusMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAU;IAChE,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,GAAG,CAAC,0CAA0C,CAAC,CAAC;IAEnE,0CAA0C;IAC1C,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACxE,eAAe,GAAG,eAAe,CAAC;QAClC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CACnC,CAAC,eAAe,EAAE,EAAE;YAClB,MAAM,WAAW,GAAkB,KAAK,CAAC,IAAI,CAC3C,QAAQ,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAC7D,CAAC;YAEF,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,WAAW,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,EAAE;gBACtC,MAAM,IAAI,GAAG,eAAe,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAElD,mDAAmD;gBACnD,IAAI,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChE,OAAO;gBACT,CAAC;gBAED,yCAAyC;gBACzC,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC5C,eAAe,CAAC,KAAK,EAAE,CAAC;gBAC1B,CAAC;gBAED,IAAI,IAAI,EAAE,CAAC;oBACT,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,UAAU,CAAC;QACpB,CAAC,EACD,eAAe,CAAC,sCAAsC;SACvD,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}
|
|
@@ -5,8 +5,8 @@ export default async function openNestedGitbookMenus(page) {
|
|
|
5
5
|
clickedAny = await page.evaluate(() => {
|
|
6
6
|
// Right pointing arrow. Only closed menus have this icon
|
|
7
7
|
const icons = Array.from(document.querySelectorAll('path[d="M9 18l6-6-6-6"]'));
|
|
8
|
-
icons.forEach(
|
|
9
|
-
const toClick = icon
|
|
8
|
+
icons.forEach((icon) => {
|
|
9
|
+
const toClick = icon.parentElement?.parentElement;
|
|
10
10
|
if (toClick) {
|
|
11
11
|
toClick.click();
|
|
12
12
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openNestedGitbookMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedGitbookMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAU;IAC7D,IAAI,UAAU,GAAG,IAAI,CAAC;IAEtB,iDAAiD;IACjD,OAAO,UAAU,EAAE,CAAC;QAClB,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACpC,yDAAyD;YACzD,MAAM,KAAK,
|
|
1
|
+
{"version":3,"file":"openNestedGitbookMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedGitbookMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAU;IAC7D,IAAI,UAAU,GAAG,IAAI,CAAC;IAEtB,iDAAiD;IACjD,OAAO,UAAU,EAAE,CAAC;QAClB,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACpC,yDAAyD;YACzD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,CAAC;YAE/E,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC;gBAClD,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeDocusaurusPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeDocusaurusPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusPage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B,CAAC,4DAA4D;;IAMxF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,OAAO,GAAG,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IAE1F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,oEAAoE;QACpE,OAAO;YACL,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,wEAAwE;IACxE,cAAc,CAAC,MAAM,EAAE,CAAC;IAExB,MAAM,eAAe,GAAG,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;IAEtF,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,eAAe,EACf,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;IAE5C,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEjE,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC7E,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzD,CAAC;IAED,mGAAmG;IACnG,qFAAqF;IACrF,yFAAyF;IACzF,kFAAkF;IAClF,MAAM,WAAW,GAAG,SAAS,CAAC;IAC9B,0EAA0E;IAC1E,0CAA0C;IAC1C,wBAAwB;IACxB,IAAI;IAEJ,uCAAuC;IACvC,wDAAwD;IACxD,sCAAsC;IACtC,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAEzD,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import cheerio from 'cheerio';
|
|
1
|
+
import * as cheerio from 'cheerio';
|
|
2
2
|
import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
|
|
3
3
|
import downloadLogoImage from '../downloadLogoImage.js';
|
|
4
4
|
import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
|
|
@@ -8,7 +8,7 @@ export async function scrapeDocusaurusSection(html, origin, cliDir, imageBaseDir
|
|
|
8
8
|
const $ = cheerio.load(html);
|
|
9
9
|
// Download the logo
|
|
10
10
|
const logoSrc = $('.navbar__logo img').attr('src');
|
|
11
|
-
downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
|
|
11
|
+
void downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
|
|
12
12
|
// Get all the navigation sections
|
|
13
13
|
const navigationSections = $('.theme-doc-sidebar-menu').first().children();
|
|
14
14
|
// Get all links per group
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AACA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,0BAA0B,EAAE,MAAM,iDAAiD,CAAC;AAC7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAAgB;IAEhB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,KAAK,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAEjE,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAe,0BAA0B,CAAC,kBAAkB,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAE5F,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QAC5C,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QACxD,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,0CAA0C;IAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC5D,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC;QAC/C,MAAM,WAAW,GAAG,IAAI;aACrB,QAAQ,EAAE;aACV,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;aAC7B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,cAAc,WAAW,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC1C,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import cheerio from 'cheerio';
|
|
1
|
+
import * as cheerio from 'cheerio';
|
|
2
2
|
import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
|
|
3
3
|
import downloadLogoImage from '../downloadLogoImage.js';
|
|
4
4
|
import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AACA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,4CAA4C,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpF,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAEjF,kCAAkC;IAClC,4FAA4F;IAC5F,yDAAyD;IACzD,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAe,kBAAkB;SAChD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAE7E,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,0BAA0B,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,OAAO,OAAO,CAAC,GAAG,CAChB,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QAC1D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,EACJ,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC1D,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;IAC9C,CAAC;IAED,+EAA+E;IAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEzC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import cheerio from 'cheerio';
|
|
1
|
+
import * as cheerio from 'cheerio';
|
|
2
2
|
import downloadLogoImage from '../downloadLogoImage.js';
|
|
3
3
|
import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
|
|
4
4
|
import getLinksRecursively from './links-per-group/getLinksRecursively.js';
|
|
@@ -21,7 +21,7 @@ export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, ov
|
|
|
21
21
|
// -- First Page -> /first-page ** DUPLICATE **
|
|
22
22
|
// -- Second Page -> /second-page
|
|
23
23
|
const linkSections = section.find('.rm-Sidebar-list').first().children();
|
|
24
|
-
const pages = getLinksRecursively(linkSections, $).filter((value, index,
|
|
24
|
+
const pages = getLinksRecursively(linkSections, $).filter((value, index, array) => array.indexOf(value) === index);
|
|
25
25
|
// Follows the same structure as mint.json
|
|
26
26
|
return {
|
|
27
27
|
group: sectionTitle,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AACA,OAAO,OAAO,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0CAA0C,CAAC;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAEjF,2DAA2D;IAC3D,2DAA2D;IAC3D,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAe,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACtE,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAEvD,uDAAuD;QACvD,yDAAyD;QACzD,+BAA+B;QAC/B,iDAAiD;QACjD,iCAAiC;QACjC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,MAAM,CACvD,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CACxD,CAAC;QAEF,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,KAAK;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,sCAAsC;IACtC,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QACnD,OAAO,MAAM,4BAA4B;QACvC,mEAAmE;QACnE,mDAAmD;QACnD,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,gBAAgB,EAChB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
|