mintlify 1.1.10 → 1.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/bin/downloadImage.js +1 -1
- package/bin/downloadImage.js.map +1 -1
- package/bin/scraping/getSitemapLinks.js +1 -1
- package/bin/scraping/getSitemapLinks.js.map +1 -1
- package/bin/scraping/scrapeFileGettingFileNameFromUrl.js +1 -1
- package/bin/scraping/scrapeFileGettingFileNameFromUrl.js.map +1 -1
- package/bin/scraping/scrapePageCommands.js +2 -2
- package/bin/scraping/scrapePageCommands.js.map +1 -1
- package/bin/scraping/scrapeSectionCommands.js +2 -2
- package/bin/scraping/scrapeSectionCommands.js.map +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js.map +1 -1
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js +38 -0
- package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js.map +1 -0
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js +2 -2
- package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map +1 -1
- package/package.json +5 -4
- package/src/downloadImage.ts +1 -1
- package/src/scraping/getSitemapLinks.ts +1 -1
- package/src/scraping/scrapeFileGettingFileNameFromUrl.ts +1 -1
- package/src/scraping/scrapePageCommands.ts +2 -2
- package/src/scraping/scrapeSectionCommands.ts +2 -2
- package/src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts +1 -1
- package/src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts +46 -0
- package/src/scraping/site-scrapers/scrapeGitBookSection.ts +2 -2
package/README.md
CHANGED
|
@@ -52,9 +52,12 @@ Note - `mintlify dev` requires `yarn` and it's recommended you install it as a g
|
|
|
52
52
|
|
|
53
53
|
#### Troubleshooting
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
Steps you can take if the dev CLI is not working (After each step try to run `mintlify dev` again):
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
- Make sure you are using Node v18 or higher.
|
|
58
|
+
- Run `mintlify install` to re-install dependencies.
|
|
59
|
+
- Navigate to the `.mintlify` folder in your home directory and delete its contents.
|
|
60
|
+
- If all else fails navigate to `~/.mintlify` and `git clone https://github.com/mintlify/mint.git --branch legacy-components-import`
|
|
58
61
|
|
|
59
62
|
### 🏃 Get Started
|
|
60
63
|
|
package/bin/downloadImage.js
CHANGED
|
@@ -11,7 +11,7 @@ export default async function downloadImage(imageSrc, writePath) {
|
|
|
11
11
|
// Create the folders needed if they're missing
|
|
12
12
|
mkdirSync(path.dirname(writePath), { recursive: true });
|
|
13
13
|
const writer = createWriteStream(writePath);
|
|
14
|
-
const response = await axios.
|
|
14
|
+
const response = await axios.get(imageSrc, {
|
|
15
15
|
responseType: "stream",
|
|
16
16
|
});
|
|
17
17
|
// wx prevents overwriting an image with the exact same name
|
package/bin/downloadImage.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,aAAa,CACzC,QAAgB,EAChB,SAAiB;IAEjB,8BAA8B;IAC9B,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE;QACzB,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;KACJ;IAED,+CAA+C;IAC/C,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"downloadImage.js","sourceRoot":"","sources":["../src/downloadImage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAC9D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,aAAa,CACzC,QAAgB,EAChB,SAAiB;IAEjB,8BAA8B;IAC9B,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE;QACzB,OAAO,OAAO,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;KACJ;IAED,+CAA+C;IAC/C,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;QACzC,YAAY,EAAE,QAAQ;KACvB,CAAC,CAAC;IAEH,4DAA4D;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QACzB,IAAI,EAAE,IAAI;KACX,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -5,7 +5,7 @@ export const getSitemapLinks = async (url) => {
|
|
|
5
5
|
const hostname = url.hostname.replace(".", "\\.");
|
|
6
6
|
const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");
|
|
7
7
|
try {
|
|
8
|
-
const indexData = (await axios.
|
|
8
|
+
const indexData = (await axios.get(url.href)).data;
|
|
9
9
|
const array = indexData.match(regex);
|
|
10
10
|
return array || [];
|
|
11
11
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getSitemapLinks.js","sourceRoot":"","sources":["../../src/scraping/getSitemapLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,cAAc;AACd,+BAA+B;AAC/B,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,GAAQ,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,cAAc,QAAQ,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAExE,IAAI;QACF,MAAM,SAAS,GAAG,CAAC,MAAM,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"getSitemapLinks.js","sourceRoot":"","sources":["../../src/scraping/getSitemapLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,cAAc;AACd,+BAA+B;AAC/B,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,GAAQ,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,cAAc,QAAQ,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAExE,IAAI;QACF,MAAM,SAAS,GAAG,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAc,CAAC;QAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAoB,CAAC;QACxD,OAAO,KAAK,IAAI,EAAE,CAAC;KACpB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;QACvE,OAAO,EAAE,CAAC;KACX;AACH,CAAC,CAAC"}
|
|
@@ -24,7 +24,7 @@ export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin,
|
|
|
24
24
|
html = await getHtmlWithPuppeteer(href);
|
|
25
25
|
}
|
|
26
26
|
else {
|
|
27
|
-
const res = await axios.
|
|
27
|
+
const res = await axios.get(href);
|
|
28
28
|
html = res.data;
|
|
29
29
|
}
|
|
30
30
|
const { title, description, markdown } = await scrapePageFunc(html, origin, cliDir, imageBaseDir, version);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAUE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAUE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,EACZ,OAAO,CACR,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,SAAS,CAAC;KAClB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CACR,KAAK,EACL,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,QAAQ,CACT,CAAC;IAEF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}
|
|
@@ -22,7 +22,7 @@ export async function scrapePageWrapper(argv, scrapeFunc, options) {
|
|
|
22
22
|
html = await getHtmlWithPuppeteer(href);
|
|
23
23
|
}
|
|
24
24
|
else {
|
|
25
|
-
const res = await axios.
|
|
25
|
+
const res = await axios.get(href);
|
|
26
26
|
html = res.data;
|
|
27
27
|
}
|
|
28
28
|
await scrapePage(scrapeFunc, href, html, argv.overwrite, options.version);
|
|
@@ -30,7 +30,7 @@ export async function scrapePageWrapper(argv, scrapeFunc, options) {
|
|
|
30
30
|
}
|
|
31
31
|
export async function scrapePageAutomatically(argv) {
|
|
32
32
|
const href = getHrefFromArgs(argv);
|
|
33
|
-
const res = await axios.
|
|
33
|
+
const res = await axios.get(href);
|
|
34
34
|
const html = res.data;
|
|
35
35
|
const { framework, version } = detectFramework(html);
|
|
36
36
|
validateFramework(framework);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAS,EACT,UAAe,EACf,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,CAAC,SAAS,EAAE;QACrB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAS,EACT,UAAe,EACf,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,CAAC,SAAS,EAAE;QACrB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;KAClE;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;KACvE;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;KACjD;AACH,CAAC"}
|
|
@@ -10,7 +10,7 @@ import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
|
|
|
10
10
|
import { startBrowser } from "../browser.js";
|
|
11
11
|
export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
|
|
12
12
|
const href = getHrefFromArgs(argv);
|
|
13
|
-
const res = await axios.
|
|
13
|
+
const res = await axios.get(href);
|
|
14
14
|
const html = res.data;
|
|
15
15
|
await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite, undefined);
|
|
16
16
|
process.exit(0);
|
|
@@ -36,7 +36,7 @@ async function scrapeSectionOpeningAllNested(argv, openLinks, scrapeFunc, versio
|
|
|
36
36
|
}
|
|
37
37
|
export async function scrapeSectionAutomatically(argv) {
|
|
38
38
|
const href = getHrefFromArgs(argv);
|
|
39
|
-
const res = await axios.
|
|
39
|
+
const res = await axios.get(href);
|
|
40
40
|
const html = res.data;
|
|
41
41
|
const { framework, version } = detectFramework(html);
|
|
42
42
|
validateFramework(framework);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAS,EAAE,UAAe;IACxE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAS,EAAE,UAAe;IACxE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,IAAI,CAAC,SAAS,EACd,SAAS,CACV,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAS,EACT,OAAe,CAAC,kBAAkB;;IAElC,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CACjC,IAAI,EACJ,sBAAsB,EACtB,oBAAoB,CACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAAe,EACf,OAAgB;IAEhB,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,IAAI,CAAC,SAAS,EACd,OAAO,CACR,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,IAAI,SAAS,KAAK,UAAU,CAAC,UAAU,EAAE;QACvC,MAAM,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;KACrD;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,OAAO,EAAE;QAC3C,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM,IAAI,SAAS,KAAK,UAAU,CAAC,MAAM,EAAE;QAC1C,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;KAC5D;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;KAChB;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,YAAiB,EAAE,CAAM;IACnE,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QACrD,OAAO,EAAE,CAAC;KACX;IAED,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAEzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YACtB,0CAA0C;YAC1C,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;SACrC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,KAAK,GAAG;YAChB,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;YACA,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aACzD,CAAC;SACH;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Used by GitBook section scraper
|
|
2
|
+
export default function getLinksRecursivelyGitBook(linkSections, $) {
|
|
3
|
+
if (linkSections == null || linkSections.length === 0) {
|
|
4
|
+
return [];
|
|
5
|
+
}
|
|
6
|
+
return linkSections
|
|
7
|
+
.map((i, s) => {
|
|
8
|
+
let subsection = $(s);
|
|
9
|
+
// GitBook has an extra div when more than one layer deep
|
|
10
|
+
if (subsection.children().length === 1) {
|
|
11
|
+
subsection = subsection.children().first();
|
|
12
|
+
}
|
|
13
|
+
const link = subsection.children().first();
|
|
14
|
+
const linkHref = link.attr("href");
|
|
15
|
+
// Skip missing links. For example, GitBook uses
|
|
16
|
+
// empty divs are used for styling a line beside the nav.
|
|
17
|
+
// Skip external links until Mintlify supports them
|
|
18
|
+
if (!linkHref ||
|
|
19
|
+
linkHref === "#" ||
|
|
20
|
+
linkHref.startsWith("https://") ||
|
|
21
|
+
linkHref.startsWith("http://")) {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
const childLinks = subsection.children().eq(1).children();
|
|
25
|
+
if (childLinks.length > 0) {
|
|
26
|
+
// Put the section link in the list of pages.
|
|
27
|
+
// When we support the section itself being a link we should update this
|
|
28
|
+
return {
|
|
29
|
+
group: link.text(),
|
|
30
|
+
pages: [linkHref, ...getLinksRecursivelyGitBook(childLinks, $)],
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
return linkHref;
|
|
34
|
+
})
|
|
35
|
+
.toArray()
|
|
36
|
+
.filter(Boolean);
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=getLinksRecursivelyGitBook.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getLinksRecursivelyGitBook.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,MAAM,CAAC,OAAO,UAAU,0BAA0B,CAAC,YAAiB,EAAE,CAAM;IAC1E,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QACrD,OAAO,EAAE,CAAC;KACX;IAED,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtB,yDAAyD;QACzD,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE;YACtC,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;SAC5C;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,KAAK,GAAG;YAChB,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;YACA,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,0BAA0B,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aAChE,CAAC;SACH;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
|
|
@@ -2,7 +2,7 @@ import cheerio from "cheerio";
|
|
|
2
2
|
import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
|
|
3
3
|
import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
|
|
4
4
|
import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
|
|
5
|
-
import
|
|
5
|
+
import getLinksRecursivelyGitBook from "./links-per-group/getLinksRecursivelyGitBook.js";
|
|
6
6
|
import alternateGroupTitle from "./alternateGroupTitle.js";
|
|
7
7
|
export async function scrapeGitBookSection(html, origin, cliDir, overwrite, version) {
|
|
8
8
|
const $ = cheerio.load(html);
|
|
@@ -27,7 +27,7 @@ export async function scrapeGitBookSection(html, origin, cliDir, overwrite, vers
|
|
|
27
27
|
const firstLink = section.children().eq(0);
|
|
28
28
|
const firstHref = firstLink.attr("href");
|
|
29
29
|
const linkSections = section.children().eq(1).children();
|
|
30
|
-
const pages =
|
|
30
|
+
const pages = getLinksRecursivelyGitBook(linkSections, $);
|
|
31
31
|
return {
|
|
32
32
|
group: sectionTitle || alternateGroupTitle(firstLink, pages),
|
|
33
33
|
pages: firstHref ? [firstHref, ...pages] : pages,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,
|
|
1
|
+
{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,kCAAkC;IAClC,4FAA4F;IAC5F,yDAAyD;IACzD,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAG,kBAAkB;SACpC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,0BAA0B,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QAC1D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,EACJ,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mintlify",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.12",
|
|
4
4
|
"description": "The Mintlify CLI",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=14.16"
|
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
"scripts": {
|
|
9
9
|
"prepare": "tsc",
|
|
10
10
|
"build": "tsc",
|
|
11
|
-
"local": "tsc && npm uninstall -g mintlify && npm i -g ."
|
|
11
|
+
"local": "tsc && npm uninstall -g mintlify && npm i -g .",
|
|
12
|
+
"publish-package": "npm run build && npm publish --access public"
|
|
12
13
|
},
|
|
13
14
|
"author": "Mintlify, Inc.",
|
|
14
15
|
"license": "ISC",
|
|
@@ -19,7 +20,7 @@
|
|
|
19
20
|
"type": "module",
|
|
20
21
|
"dependencies": {
|
|
21
22
|
"@apidevtools/swagger-parser": "^10.1.0",
|
|
22
|
-
"axios": "
|
|
23
|
+
"axios": "1.1.3",
|
|
23
24
|
"chalk": "^5.1.0",
|
|
24
25
|
"cheerio": "^0.22.0",
|
|
25
26
|
"chokidar": "^3.5.3",
|
|
@@ -33,7 +34,7 @@
|
|
|
33
34
|
"open": "^8.4.0",
|
|
34
35
|
"openapi-types": "^12.0.2",
|
|
35
36
|
"ora": "^6.1.2",
|
|
36
|
-
"puppeteer": "^
|
|
37
|
+
"puppeteer": "^19.4.0",
|
|
37
38
|
"shelljs": "^0.8.5",
|
|
38
39
|
"yargs": "^17.6.0"
|
|
39
40
|
},
|
package/src/downloadImage.ts
CHANGED
|
@@ -7,7 +7,7 @@ export const getSitemapLinks = async (url: URL) => {
|
|
|
7
7
|
const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");
|
|
8
8
|
|
|
9
9
|
try {
|
|
10
|
-
const indexData = (await axios.
|
|
10
|
+
const indexData = (await axios.get(url.href)).data as string;
|
|
11
11
|
const array = indexData.match(regex) as string[] | null;
|
|
12
12
|
return array || [];
|
|
13
13
|
} catch (err) {
|
|
@@ -29,7 +29,7 @@ export async function scrapePageWrapper(
|
|
|
29
29
|
if (options.puppeteer) {
|
|
30
30
|
html = await getHtmlWithPuppeteer(href);
|
|
31
31
|
} else {
|
|
32
|
-
const res = await axios.
|
|
32
|
+
const res = await axios.get(href);
|
|
33
33
|
html = res.data;
|
|
34
34
|
}
|
|
35
35
|
await scrapePage(scrapeFunc, href, html, argv.overwrite, options.version);
|
|
@@ -38,7 +38,7 @@ export async function scrapePageWrapper(
|
|
|
38
38
|
|
|
39
39
|
export async function scrapePageAutomatically(argv: any) {
|
|
40
40
|
const href = getHrefFromArgs(argv);
|
|
41
|
-
const res = await axios.
|
|
41
|
+
const res = await axios.get(href);
|
|
42
42
|
const html = res.data;
|
|
43
43
|
const { framework, version } = detectFramework(html);
|
|
44
44
|
|
|
@@ -11,7 +11,7 @@ import { startBrowser } from "../browser.js";
|
|
|
11
11
|
|
|
12
12
|
export async function scrapeSectionAxiosWrapper(argv: any, scrapeFunc: any) {
|
|
13
13
|
const href = getHrefFromArgs(argv);
|
|
14
|
-
const res = await axios.
|
|
14
|
+
const res = await axios.get(href);
|
|
15
15
|
const html = res.data;
|
|
16
16
|
await scrapeSection(
|
|
17
17
|
scrapeFunc,
|
|
@@ -71,7 +71,7 @@ async function scrapeSectionOpeningAllNested(
|
|
|
71
71
|
|
|
72
72
|
export async function scrapeSectionAutomatically(argv: any) {
|
|
73
73
|
const href = getHrefFromArgs(argv);
|
|
74
|
-
const res = await axios.
|
|
74
|
+
const res = await axios.get(href);
|
|
75
75
|
const html = res.data;
|
|
76
76
|
const { framework, version } = detectFramework(html);
|
|
77
77
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// Used by GitBook section scraper
|
|
2
|
+
export default function getLinksRecursivelyGitBook(linkSections: any, $: any) {
|
|
3
|
+
if (linkSections == null || linkSections.length === 0) {
|
|
4
|
+
return [];
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
return linkSections
|
|
8
|
+
.map((i, s) => {
|
|
9
|
+
let subsection = $(s);
|
|
10
|
+
|
|
11
|
+
// GitBook has an extra div when more than one layer deep
|
|
12
|
+
if (subsection.children().length === 1) {
|
|
13
|
+
subsection = subsection.children().first();
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const link = subsection.children().first();
|
|
17
|
+
const linkHref = link.attr("href");
|
|
18
|
+
|
|
19
|
+
// Skip missing links. For example, GitBook uses
|
|
20
|
+
// empty divs are used for styling a line beside the nav.
|
|
21
|
+
// Skip external links until Mintlify supports them
|
|
22
|
+
if (
|
|
23
|
+
!linkHref ||
|
|
24
|
+
linkHref === "#" ||
|
|
25
|
+
linkHref.startsWith("https://") ||
|
|
26
|
+
linkHref.startsWith("http://")
|
|
27
|
+
) {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const childLinks = subsection.children().eq(1).children();
|
|
32
|
+
|
|
33
|
+
if (childLinks.length > 0) {
|
|
34
|
+
// Put the section link in the list of pages.
|
|
35
|
+
// When we support the section itself being a link we should update this
|
|
36
|
+
return {
|
|
37
|
+
group: link.text(),
|
|
38
|
+
pages: [linkHref, ...getLinksRecursivelyGitBook(childLinks, $)],
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return linkHref;
|
|
43
|
+
})
|
|
44
|
+
.toArray()
|
|
45
|
+
.filter(Boolean);
|
|
46
|
+
}
|
|
@@ -3,7 +3,7 @@ import { NavigationEntry } from "../../navigation.js";
|
|
|
3
3
|
import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
|
|
4
4
|
import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
|
|
5
5
|
import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
|
|
6
|
-
import
|
|
6
|
+
import getLinksRecursivelyGitBook from "./links-per-group/getLinksRecursivelyGitBook.js";
|
|
7
7
|
import alternateGroupTitle from "./alternateGroupTitle.js";
|
|
8
8
|
|
|
9
9
|
export async function scrapeGitBookSection(
|
|
@@ -41,7 +41,7 @@ export async function scrapeGitBookSection(
|
|
|
41
41
|
const firstHref = firstLink.attr("href");
|
|
42
42
|
|
|
43
43
|
const linkSections = section.children().eq(1).children();
|
|
44
|
-
const pages =
|
|
44
|
+
const pages = getLinksRecursivelyGitBook(linkSections, $);
|
|
45
45
|
|
|
46
46
|
return {
|
|
47
47
|
group: sectionTitle || alternateGroupTitle(firstLink, pages),
|