@mintlify/scraping 4.0.1039 → 4.0.1041
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/bin/index.d.ts +0 -1
- package/bin/index.js +0 -1
- package/bin/index.js.map +1 -1
- package/bin/pipeline/logo.d.ts +1 -1
- package/bin/pipeline/logo.js +6 -1
- package/bin/pipeline/logo.js.map +1 -1
- package/bin/tsconfig.build.tsbuildinfo +1 -1
- package/bin/utils/network.d.ts +1 -1
- package/bin/utils/network.js +5 -2
- package/bin/utils/network.js.map +1 -1
- package/package.json +12 -4
- package/src/index.ts +0 -1
- package/src/pipeline/logo.ts +8 -7
- package/src/utils/network.ts +5 -2
package/bin/utils/network.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AsyncAPIDocumentInterface } from '@mintlify/common';
|
|
2
2
|
import { OpenAPI } from 'openapi-types';
|
|
3
|
-
import { Browser } from 'puppeteer';
|
|
3
|
+
import type { Browser } from 'puppeteer';
|
|
4
4
|
export declare function startPuppeteer(): Promise<Browser | undefined>;
|
|
5
5
|
export declare function getHtmlWithPuppeteer(browser: Browser, url: string | URL): Promise<string | undefined>;
|
|
6
6
|
export declare function fetchPageHtml(url: string | URL, browser?: Browser | undefined): Promise<string>;
|
package/bin/utils/network.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { exponentialBackoff, validateAsyncApi } from '@mintlify/common';
|
|
2
2
|
import yaml from 'js-yaml';
|
|
3
|
-
import { launch } from 'puppeteer';
|
|
4
3
|
import { framework } from './detectFramework.js';
|
|
5
4
|
import { getErrorMessage } from './errors.js';
|
|
6
5
|
import { log } from './log.js';
|
|
@@ -20,13 +19,17 @@ const headers = {
|
|
|
20
19
|
};
|
|
21
20
|
export async function startPuppeteer() {
|
|
22
21
|
try {
|
|
22
|
+
const { launch } = await import('puppeteer');
|
|
23
23
|
return await launch({
|
|
24
24
|
headless: true,
|
|
25
25
|
acceptInsecureCerts: true,
|
|
26
26
|
});
|
|
27
27
|
}
|
|
28
28
|
catch (error) {
|
|
29
|
-
if (error instanceof Error) {
|
|
29
|
+
if (error instanceof Error && 'code' in error && error.code === 'ERR_MODULE_NOT_FOUND') {
|
|
30
|
+
log('Could not create a browser instance: install puppeteer to scrape this site');
|
|
31
|
+
}
|
|
32
|
+
else if (error instanceof Error) {
|
|
30
33
|
log(`Could not create a browser instance: ${error.message}`);
|
|
31
34
|
}
|
|
32
35
|
}
|
package/bin/utils/network.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACnG,OAAO,IAAI,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACnG,OAAO,IAAI,MAAM,SAAS,CAAC;AAI3B,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;CAC/G,CAAC;AAEX,MAAM,OAAO,GAAG;IACd,iBAAiB,EAAE,gBAAgB;IACnC,MAAM,EACJ,yIAAyI;IAC3I,YAAY,EACV,uHAAuH;IACzH,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QAC7C,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACvF,GAAG,CAAC,4EAA4E,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAClC,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,WAAW,CAAC;YACrB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,IAAI;YACZ,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,KAAK;YACf,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,YAAY,CACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxB,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CACH,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACvB,QAAQ,CAAC,gBAAgB,CACvB,OAAO,EACP,CAAC,CAAC,EAAE,EAAE;oBACJ,IAAI,CAAC,CAAC,MAAM,YAAY,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC;wBACnF,CAAC,CAAC,cAAc,EAAE,CAAC;gBACvB,CAAC,EACD,IAAI,CACL,CAAC;gBAEF,SAAS,UAAU,CAAC,EAA0B;oBAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,sBAAsB,CACzC,qBAAqB,CACW,CAAC;oBACnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;wBACb,UAAU,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;oBAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;oBAC3D,IAAI,CAAC,KAAK;wBAAE,OAAO,CAAC,CAAC;oBACrB,IAAI,KAAK,GAAG,CAAC,CAAC;oBACd,MAAM,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;oBAC9D,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;wBACnB,MAAM,GAAG,GAAG,EAAE,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;wBAClD,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,YAAY,WAAW,CAAC;4BAAE,OAAO;wBAClD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAChC,IAAI,CAAC,MAAM;4BAAE,OAAO;wBACpB,MAAM,OAAO,GAAG,MAAM,CAAC,kBAAkB,CAAC;wBAC1C,IACE,OAAO,YAAY,WAAW;4BAC9B,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,GAAG;4BAC7B,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM;4BAE/B,OAAO;wBACT,GAAG,CAAC,KAAK,EAAE,CAAC;wBACZ,KAAK,EAAE,CAAC;oBACV,CAAC,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC,CAAC,CAAC;gBAEH,IAAI,YAAY,KAAK,CAAC;oBAAE,MAAM;gBAE9B,MAAM,IAAI;qBACP,eAAe,CACd,GAAG,EAAE;oBACH,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;oBAC3D,IAAI,CAAC,KAAK;wBAAE,OAAO,IAAI,CAAC;oBACxB,MAAM,OAAO,GAAG,KAAK,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC;oBACpE,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;wBACrC,MAAM,GAAG,GAAG,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;wBACtC,IAAI,CAAC,GAAG;4BAAE,OAAO,IAAI,CAAC;wBACtB,MAAM,OAAO,GAAG,CAAC,CAAC,kBAAkB,CAAC;wBACrC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,YAAY,WAAW,CAAC;4BAAE,OAAO,IAAI,CAAC;wBAC/D,OAAO,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;oBAC1E,CAAC,CAAC,CAAC;gBACL,CAAC,EACD,EAAE,OAAO,EAAE,IAAI,EAAE,CAClB;qBACA,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAQ;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC9C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAqB,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,iDAAiD,YAAY,EAAE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAQ;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC9C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uDAAuD,YAAY,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,qDAAqD,YAAY,EAAE,CAAC,CAAC;IAC7F,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mintlify/scraping",
|
|
3
|
-
"version": "4.0.
|
|
3
|
+
"version": "4.0.1041",
|
|
4
4
|
"description": "Scrape documentation frameworks to Mintlify docs",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=18.0.0"
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"format:check": "oxfmt --check"
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
|
-
"@mintlify/common": "1.0.
|
|
46
|
+
"@mintlify/common": "1.0.1171",
|
|
47
47
|
"fast-xml-parser": "5.7.3",
|
|
48
48
|
"fs-extra": "11.1.1",
|
|
49
49
|
"graphql": "16.11.0",
|
|
@@ -51,7 +51,6 @@
|
|
|
51
51
|
"js-yaml": "4.3.1",
|
|
52
52
|
"mdast-util-mdx-jsx": "3.1.3",
|
|
53
53
|
"neotraverse": "0.6.18",
|
|
54
|
-
"puppeteer": "24.3.1",
|
|
55
54
|
"rehype-parse": "9.0.1",
|
|
56
55
|
"remark-gfm": "4.0.0",
|
|
57
56
|
"remark-mdx": "3.0.1",
|
|
@@ -75,9 +74,18 @@
|
|
|
75
74
|
"oxfmt": "^0.51.0",
|
|
76
75
|
"oxlint": "1.66.0",
|
|
77
76
|
"oxlint-tsgolint": "^0.23.0",
|
|
77
|
+
"puppeteer": "24.3.1",
|
|
78
78
|
"rimraf": "5.0.1",
|
|
79
79
|
"typescript": "5.5.3",
|
|
80
80
|
"vitest": "2.1.9"
|
|
81
81
|
},
|
|
82
|
-
"
|
|
82
|
+
"peerDependencies": {
|
|
83
|
+
"puppeteer": "24.3.1"
|
|
84
|
+
},
|
|
85
|
+
"peerDependenciesMeta": {
|
|
86
|
+
"puppeteer": {
|
|
87
|
+
"optional": true
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
"gitHead": "8cb4865053ce68e70deaa4066d29c55c23bc7ae9"
|
|
83
91
|
}
|
package/src/index.ts
CHANGED
|
@@ -7,7 +7,6 @@ export { scrapePageGroup } from './pipeline/group.js';
|
|
|
7
7
|
export { scrapeAllSiteTabs } from './pipeline/tabs.js';
|
|
8
8
|
export { htmlToHast } from './pipeline/root.js';
|
|
9
9
|
export { detectFramework, framework } from './utils/detectFramework.js';
|
|
10
|
-
export { fetchPageHtml } from './utils/network.js';
|
|
11
10
|
export { write } from './utils/file.js';
|
|
12
11
|
export { getErrorMessage } from './utils/errors.js';
|
|
13
12
|
export { checkUrl } from './utils/url.js';
|
package/src/pipeline/logo.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Root as HastRoot, Element } from 'hast';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { Browser } from 'puppeteer';
|
|
3
|
+
import type { Browser } from 'puppeteer';
|
|
4
4
|
import { EXIT, visit } from 'unist-util-visit';
|
|
5
5
|
|
|
6
6
|
import { framework } from '../utils/detectFramework.js';
|
|
@@ -106,12 +106,13 @@ export async function downloadLogos(
|
|
|
106
106
|
} else {
|
|
107
107
|
try {
|
|
108
108
|
const html = await fetchPageHtml(url);
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
109
|
+
const findLogoNodes =
|
|
110
|
+
framework.vendor === 'readme'
|
|
111
|
+
? findReadmeLogoNodes
|
|
112
|
+
: framework.vendor === 'gitbook'
|
|
113
|
+
? findGitBookLogoNodes
|
|
114
|
+
: findDocusaurusLogoNodes;
|
|
115
|
+
await findLogosFromHtml(html, findLogoNodes, filepaths, url);
|
|
115
116
|
} catch (error) {
|
|
116
117
|
const errorMessage = getErrorMessage(error);
|
|
117
118
|
log(`Failed to retrieve logo from HTML: ${errorMessage}`);
|
package/src/utils/network.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AsyncAPIDocumentInterface, exponentialBackoff, validateAsyncApi } from '@mintlify/common';
|
|
2
2
|
import yaml from 'js-yaml';
|
|
3
3
|
import { OpenAPI } from 'openapi-types';
|
|
4
|
-
import { Browser
|
|
4
|
+
import type { Browser } from 'puppeteer';
|
|
5
5
|
|
|
6
6
|
import { framework } from './detectFramework.js';
|
|
7
7
|
import { getErrorMessage } from './errors.js';
|
|
@@ -27,12 +27,15 @@ const headers = {
|
|
|
27
27
|
|
|
28
28
|
export async function startPuppeteer() {
|
|
29
29
|
try {
|
|
30
|
+
const { launch } = await import('puppeteer');
|
|
30
31
|
return await launch({
|
|
31
32
|
headless: true,
|
|
32
33
|
acceptInsecureCerts: true,
|
|
33
34
|
});
|
|
34
35
|
} catch (error) {
|
|
35
|
-
if (error instanceof Error) {
|
|
36
|
+
if (error instanceof Error && 'code' in error && error.code === 'ERR_MODULE_NOT_FOUND') {
|
|
37
|
+
log('Could not create a browser instance: install puppeteer to scrape this site');
|
|
38
|
+
} else if (error instanceof Error) {
|
|
36
39
|
log(`Could not create a browser instance: ${error.message}`);
|
|
37
40
|
}
|
|
38
41
|
}
|