@mintlify/scraping 4.0.589 → 4.0.590
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +10 -3
- package/bin/cli.js.map +1 -1
- package/bin/constants.d.ts +1 -1
- package/bin/constants.js +1 -4
- package/bin/constants.js.map +1 -1
- package/bin/index.d.ts +9 -0
- package/bin/index.js +9 -0
- package/bin/index.js.map +1 -1
- package/bin/nav/retrieve.js +15 -0
- package/bin/nav/retrieve.js.map +1 -1
- package/bin/nav/root.js +7 -0
- package/bin/nav/root.js.map +1 -1
- package/bin/pipeline/site.d.ts +1 -0
- package/bin/pipeline/site.js +37 -8
- package/bin/pipeline/site.js.map +1 -1
- package/bin/pipeline/tabs.d.ts +3 -1
- package/bin/pipeline/tabs.js +4 -4
- package/bin/pipeline/tabs.js.map +1 -1
- package/bin/tsconfig.build.tsbuildinfo +1 -1
- package/bin/utils/network.js +46 -0
- package/bin/utils/network.js.map +1 -1
- package/bin/utils/strings.d.ts +1 -0
- package/bin/utils/strings.js +3 -0
- package/bin/utils/strings.js.map +1 -1
- package/bin/utils/text.js +7 -3
- package/bin/utils/text.js.map +1 -1
- package/package.json +2 -2
- package/src/cli.ts +13 -4
- package/src/constants.ts +1 -5
- package/src/index.ts +10 -0
- package/src/nav/retrieve.ts +22 -0
- package/src/nav/root.ts +11 -0
- package/src/pipeline/site.ts +42 -19
- package/src/pipeline/tabs.ts +5 -4
- package/src/utils/network.ts +47 -0
- package/src/utils/strings.ts +4 -0
- package/src/utils/text.ts +5 -3
package/bin/cli.js
CHANGED
|
@@ -15,7 +15,14 @@ import { fetchPageHtml } from './utils/network.js';
|
|
|
15
15
|
import { checkUrl } from './utils/url.js';
|
|
16
16
|
await yargs(hideBin(process.argv))
|
|
17
17
|
.command('page <url>', 'Scrapes the docs page for the URL provided', (yargs) => yargs.positional('url', { type: 'string', demandOption: true }).check(checkUrl), async ({ url }) => await page(url))
|
|
18
|
-
.command('section <url>', 'Scrapes the entire docs site based on the URL provided', (yargs) => yargs
|
|
18
|
+
.command('section <url>', 'Scrapes the entire docs site based on the URL provided', (yargs) => yargs
|
|
19
|
+
.positional('url', { type: 'string', demandOption: true })
|
|
20
|
+
.option('filter', {
|
|
21
|
+
describe: 'Only scrape URLs matching this path filter (e.g. /docs will match /docs and /docs/*)',
|
|
22
|
+
type: 'string',
|
|
23
|
+
alias: 'f',
|
|
24
|
+
})
|
|
25
|
+
.check(checkUrl), async ({ url, filter }) => await site(url, filter))
|
|
19
26
|
.command('openapi-file <openapiLocation>', 'Creates MDX files from an OpenAPI spec', (yargs) => yargs
|
|
20
27
|
.positional('openapiLocation', {
|
|
21
28
|
describe: 'The filename or URL location of the OpenAPI spec',
|
|
@@ -94,12 +101,12 @@ async function page(url) {
|
|
|
94
101
|
process.exit(1);
|
|
95
102
|
}
|
|
96
103
|
}
|
|
97
|
-
async function site(url) {
|
|
104
|
+
async function site(url, filter) {
|
|
98
105
|
try {
|
|
99
106
|
const urlObj = new URL(url);
|
|
100
107
|
const html = await fetchPageHtml(urlObj);
|
|
101
108
|
log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
|
|
102
|
-
const result = await scrapeAllSiteTabs(html, urlObj);
|
|
109
|
+
const result = await scrapeAllSiteTabs(html, urlObj, { filter });
|
|
103
110
|
if (result.success) {
|
|
104
111
|
const mintConfig = result.data;
|
|
105
112
|
const docsConfig = upgradeToDocsConfig(mintConfig, {
|
package/bin/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,4CAA4C,EAC5C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,eAAe,EACf,wDAAwD,EACxD,CAAC,KAAK,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,4CAA4C,EAC5C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC1F,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CACnC;KAEA,OAAO,CACN,eAAe,EACf,wDAAwD,EACxD,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;KACzD,MAAM,CAAC,QAAQ,EAAE;IAChB,QAAQ,EACN,sFAAsF;IACxF,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,GAAG;CACX,CAAC;KACD,KAAK,CAAC,QAAQ,CAAC,EACpB,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CACnD;KAEA,OAAO,CACN,gCAAgC,EAChC,wCAAwC,EACxC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,iBAAiB,EAAE;IAC7B,QAAQ,EAAE,kDAAkD;IAC5D,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,IAAI;CACnB,CAAC;KACD,MAAM,CAAC,YAAY,EAAE;IACpB,QAAQ,EAAE,+CAA+C;IACzD,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,QAAQ,EAAE;IAChB,QAAQ,EAAE,4DAA4D;IACtE,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,WAAW,EAAE;IACnB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,EAAE,KAAK;IACd,IAAI,EAAE,SAAS;CAChB,CAAC,EACN,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,eAAe,EAAE;YACtE,eAAe,EAAE,SAAS;YAC1B,OAAO,EAAE,SAAS;YAClB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,CACF;KAEA,cAAc,EAAE;KAChB,aAAa,CAAC,CAAC,EAAE,gEAAgE,CAAC;KAClF,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,KAAK,EAAE,CAAC;AAEX,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC9B,eAAe,CAAC,IAAI,CAAC,CAAC;QAEtB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI;YAC3B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,2CAA2C,GAAG,EAAE;SAC1D,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,GAAG,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,MAAe;IAC9C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,GAAG,CAAC,gDAAgD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACjE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAsB,CAAC;YACjD,MAAM,UAAU,GAAG,mBAAmB,CAAC,UAAU,EAAE;gBACjD,kBAAkB,EAAE,IAAI;aACzB,CAAC,CAAC;YACH,UAAU,CAAC,KAAK,GAAG,OAAO,CAAC;YAC3B,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7D,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
package/bin/constants.d.ts
CHANGED
|
@@ -5,4 +5,4 @@ export declare const SPACES: string;
|
|
|
5
5
|
export declare const CONTENT_FAILURE_MSG: string;
|
|
6
6
|
export declare const NAV_FAILURE_MSG: string;
|
|
7
7
|
export declare const MDAST_FAILURE_MSG = "failed to convert MDAST to Markdown string";
|
|
8
|
-
export declare const FINAL_SUCCESS_MESSAGE
|
|
8
|
+
export declare const FINAL_SUCCESS_MESSAGE = "We've successfully scraped your docs site. We've downloaded the `navigation` array (and if necessary, the `tabs` array) into `docs.json`.";
|
package/bin/constants.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { activeColors } from './utils/log.js';
|
|
2
1
|
export const OVERVIEW_PAGE_SLUG = '/mintie_overview';
|
|
3
2
|
export const SUPPORTED_MEDIA_EXTENSIONS = [
|
|
4
3
|
'png',
|
|
@@ -42,7 +41,5 @@ export const NAV_FAILURE_MSG = `failed to retrieve nav items from HTML.
|
|
|
42
41
|
${SPACES}Please double check your documentation provider and ensure they are supported.
|
|
43
42
|
${SPACES}We currently support: ReadMe, GitBook, and Docusaurus`;
|
|
44
43
|
export const MDAST_FAILURE_MSG = 'failed to convert MDAST to Markdown string';
|
|
45
|
-
export const FINAL_SUCCESS_MESSAGE = `We've successfully scraped your docs site.
|
|
46
|
-
${SPACES}We've downloaded the ${activeColors.cyan}\`navigation\`${activeColors.default} array (and if necessary, the ${activeColors.cyan}\`tabs\`${activeColors.default} array)
|
|
47
|
-
${SPACES}into ${activeColors.blue}\`docs.json\`${activeColors.default}.`;
|
|
44
|
+
export const FINAL_SUCCESS_MESSAGE = `We've successfully scraped your docs site. We've downloaded the \`navigation\` array (and if necessary, the \`tabs\` array) into \`docs.json\`.`;
|
|
48
45
|
//# sourceMappingURL=constants.js.map
|
package/bin/constants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,kBAAkB,GAAG,kBAAkB,CAAC;AAErD,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;CACN,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,WAAW;IACX,gBAAgB;IAChB,MAAM;IACN,SAAS;IACT,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,WAAW;IACX,WAAW;IACX,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,SAAS;CACD,CAAC;AAEX,MAAM,CAAC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAErC,MAAM,CAAC,MAAM,mBAAmB,GAAG;EACjC,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,eAAe,GAAG;EAC7B,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,iBAAiB,GAAG,4CAA4C,CAAC;AAE9E,MAAM,CAAC,MAAM,qBAAqB,GAAG,iJAAiJ,CAAC"}
|
package/bin/index.d.ts
CHANGED
|
@@ -2,3 +2,12 @@ export { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
|
|
|
2
2
|
export { generateOpenApiPagesForDocsConfig } from './openapi/generateOpenApiPagesForDocsConfig.js';
|
|
3
3
|
export * from './utils/log.js';
|
|
4
4
|
export { generateAsyncApiPagesForDocsConfig } from './asyncapi/generateAsyncApiPagesForDocsConfig.js';
|
|
5
|
+
export { scrapePageGroup } from './pipeline/group.js';
|
|
6
|
+
export { scrapeAllSiteTabs } from './pipeline/tabs.js';
|
|
7
|
+
export { htmlToHast } from './pipeline/root.js';
|
|
8
|
+
export { detectFramework, framework } from './utils/detectFramework.js';
|
|
9
|
+
export { fetchPageHtml } from './utils/network.js';
|
|
10
|
+
export { write } from './utils/file.js';
|
|
11
|
+
export { getErrorMessage } from './utils/errors.js';
|
|
12
|
+
export { checkUrl } from './utils/url.js';
|
|
13
|
+
export { FINAL_SUCCESS_MESSAGE } from './constants.js';
|
package/bin/index.js
CHANGED
|
@@ -2,4 +2,13 @@ export { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
|
|
|
2
2
|
export { generateOpenApiPagesForDocsConfig } from './openapi/generateOpenApiPagesForDocsConfig.js';
|
|
3
3
|
export * from './utils/log.js';
|
|
4
4
|
export { generateAsyncApiPagesForDocsConfig } from './asyncapi/generateAsyncApiPagesForDocsConfig.js';
|
|
5
|
+
export { scrapePageGroup } from './pipeline/group.js';
|
|
6
|
+
export { scrapeAllSiteTabs } from './pipeline/tabs.js';
|
|
7
|
+
export { htmlToHast } from './pipeline/root.js';
|
|
8
|
+
export { detectFramework, framework } from './utils/detectFramework.js';
|
|
9
|
+
export { fetchPageHtml } from './utils/network.js';
|
|
10
|
+
export { write } from './utils/file.js';
|
|
11
|
+
export { getErrorMessage } from './utils/errors.js';
|
|
12
|
+
export { checkUrl } from './utils/url.js';
|
|
13
|
+
export { FINAL_SUCCESS_MESSAGE } from './constants.js';
|
|
5
14
|
//# sourceMappingURL=index.js.map
|
package/bin/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,iCAAiC,EAAE,MAAM,gDAAgD,CAAC;AACnG,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,kCAAkC,EAAE,MAAM,kDAAkD,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,iCAAiC,EAAE,MAAM,gDAAgD,CAAC;AACnG,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,kCAAkC,EAAE,MAAM,kDAAkD,CAAC;AAEtG,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC"}
|
package/bin/nav/retrieve.js
CHANGED
|
@@ -33,6 +33,21 @@ export function retrieveNavItems(rootNode) {
|
|
|
33
33
|
node.tagName = 'li';
|
|
34
34
|
if (node.tagName !== 'li')
|
|
35
35
|
return CONTINUE;
|
|
36
|
+
const className = node.properties.className;
|
|
37
|
+
if (framework.vendor === 'gitbook' &&
|
|
38
|
+
Array.isArray(className) &&
|
|
39
|
+
className.includes('page-group-item')) {
|
|
40
|
+
const titleDiv = node.children.find((child) => child.type === 'element' && child.tagName === 'div');
|
|
41
|
+
const childList = node.children.find((child) => child.type === 'element' && child.tagName === 'ul');
|
|
42
|
+
if (titleDiv && titleDiv.type === 'element' && childList && childList.type === 'element') {
|
|
43
|
+
const title = findTitle(titleDiv, { delete: false });
|
|
44
|
+
const childEntries = retrieveNavItems(childList);
|
|
45
|
+
if (title && childEntries.length > 0) {
|
|
46
|
+
result.push({ group: title, pages: childEntries });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return SKIP;
|
|
50
|
+
}
|
|
36
51
|
let title = undefined;
|
|
37
52
|
if (node.children[0] &&
|
|
38
53
|
node.children[1] &&
|
package/bin/nav/retrieve.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../src/nav/retrieve.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEjD,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,IAAI,kBAAkB,GAAG,IAAI,CAAC;IAC9B,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,SAAS;YACZ,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,QAAQ;YACX,kBAAkB,GAAG,SAAS,CAAC;YAC/B,MAAM;IACV,CAAC;IAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,mBAAmB,GAAG,KAAK,CAAC;YAC5B,MAAM;QACR,KAAK,SAAS;YACZ,mBAAmB,GAAG,KAAK,CAAC;YAC5B,MAAM;QACR,KAAK,QAAQ;YACX,mBAAmB,GAAG,IAAI,CAAC;YAC3B,MAAM;IACV,CAAC;IAED,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI,EAAE,KAAK,EAAE,MAAM;QACtD,IAAI,IAAI,CAAC,OAAO,KAAK,kBAAkB;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAC7D,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;QAE3C,IAAI,KAAK,GAAuB,SAAS,CAAC;QAC1C,IACE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK;YAClC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,CACtE,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAC7C,CAAC;YACD,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,IACE,SAAS,CAAC,MAAM,KAAK,QAAQ;YAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,OAAO,KAAK,KAAK,QAAQ;YACzB,MAAM,EACN,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG;gBACd;oBACE,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACxB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,EAAE;YAClC,cAAc,EAAE,mBAAmB;YACnC,gBAAgB,EAAE,IAAI;YACtB,KAAK;SACN,CAAC,CAAC;QAEH,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
1
|
+
{"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../src/nav/retrieve.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEjD,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,IAAI,kBAAkB,GAAG,IAAI,CAAC;IAC9B,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,SAAS;YACZ,kBAAkB,GAAG,IAAI,CAAC;YAC1B,MAAM;QACR,KAAK,QAAQ;YACX,kBAAkB,GAAG,SAAS,CAAC;YAC/B,MAAM;IACV,CAAC;IAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,mBAAmB,GAAG,KAAK,CAAC;YAC5B,MAAM;QACR,KAAK,SAAS;YACZ,mBAAmB,GAAG,KAAK,CAAC;YAC5B,MAAM;QACR,KAAK,QAAQ;YACX,mBAAmB,GAAG,IAAI,CAAC;YAC3B,MAAM;IACV,CAAC;IAED,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI,EAAE,KAAK,EAAE,MAAM;QACtD,IAAI,IAAI,CAAC,OAAO,KAAK,kBAAkB;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAC7D,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;QAE3C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAC5C,IACE,SAAS,CAAC,MAAM,KAAK,SAAS;YAC9B,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;YACxB,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EACrC,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CACjC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAC/D,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAC9D,CAAC;YACF,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBACzF,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM,YAAY,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;gBACjD,IAAI,KAAK,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,KAAK,GAAuB,SAAS,CAAC;QAC1C,IACE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK;YAClC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,CACtE,CAAC,MAAM,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAC7C,CAAC;YACD,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,IACE,SAAS,CAAC,MAAM,KAAK,QAAQ;YAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI;YACjC,OAAO,KAAK,KAAK,QAAQ;YACzB,MAAM,EACN,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG;gBACd;oBACE,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACxB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,EAAE;YAClC,cAAc,EAAE,mBAAmB;YACnC,gBAAgB,EAAE,IAAI;YACtB,KAAK;SACN,CAAC,CAAC;QAEH,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/bin/nav/root.js
CHANGED
|
@@ -33,6 +33,13 @@ export function retrieveRootNavElement(rootNode) {
|
|
|
33
33
|
let element = undefined;
|
|
34
34
|
visit(rootNode, 'element', function (node) {
|
|
35
35
|
const { className } = node.properties;
|
|
36
|
+
if (framework.vendor === 'gitbook' &&
|
|
37
|
+
node.tagName === 'aside' &&
|
|
38
|
+
(node.properties.id === 'table-of-contents' ||
|
|
39
|
+
node.properties.dataTestid === 'table-of-contents')) {
|
|
40
|
+
element = node;
|
|
41
|
+
return EXIT;
|
|
42
|
+
}
|
|
36
43
|
if (node.tagName === rootTagName &&
|
|
37
44
|
Array.isArray(className) &&
|
|
38
45
|
!!intersection(className, rootSelectorSet).size) {
|
package/bin/nav/root.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"root.js","sourceRoot":"","sources":["../../src/nav/root.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD,MAAM,UAAU,sBAAsB,CAAC,QAAkB;IACvD,IAAI,WAAW,GAAG,OAAO,CAAC;IAC1B,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,WAAW,GAAG,KAAK,CAAC;YACpB,MAAM;QACR,KAAK,SAAS;YACZ,WAAW,GAAG,OAAO,CAAC;YACtB,MAAM;QACR,KAAK,QAAQ;YACX,WAAW,GAAG,KAAK,CAAC;YACpB,MAAM;IACV,CAAC;IAED,IAAI,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACtD,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YACpC,MAAM;QACR,KAAK,SAAS;YACZ,eAAe,GAAG,IAAI,GAAG,CAAC;gBACxB,uBAAuB;gBACvB,oBAAoB;gBACpB,uBAAuB;aACxB,CAAC,CAAC;YACH,MAAM;QACR,KAAK,QAAQ;YACX,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YAC1C,MAAM;IACV,CAAC;IAED,IAAI,OAAO,GAAwB,SAAS,CAAC;IAC7C,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI;QACvC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"root.js","sourceRoot":"","sources":["../../src/nav/root.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD,MAAM,UAAU,sBAAsB,CAAC,QAAkB;IACvD,IAAI,WAAW,GAAG,OAAO,CAAC;IAC1B,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,WAAW,GAAG,KAAK,CAAC;YACpB,MAAM;QACR,KAAK,SAAS;YACZ,WAAW,GAAG,OAAO,CAAC;YACtB,MAAM;QACR,KAAK,QAAQ;YACX,WAAW,GAAG,KAAK,CAAC;YACpB,MAAM;IACV,CAAC;IAED,IAAI,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACtD,QAAQ,SAAS,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,YAAY;YACf,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YACpC,MAAM;QACR,KAAK,SAAS;YACZ,eAAe,GAAG,IAAI,GAAG,CAAC;gBACxB,uBAAuB;gBACvB,oBAAoB;gBACpB,uBAAuB;aACxB,CAAC,CAAC;YACH,MAAM;QACR,KAAK,QAAQ;YACX,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YAC1C,MAAM;IACV,CAAC;IAED,IAAI,OAAO,GAAwB,SAAS,CAAC;IAC7C,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI;QACvC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;QAEtC,IACE,SAAS,CAAC,MAAM,KAAK,SAAS;YAC9B,IAAI,CAAC,OAAO,KAAK,OAAO;YACxB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,mBAAmB;gBACzC,IAAI,CAAC,UAAU,CAAC,UAAU,KAAK,mBAAmB,CAAC,EACrD,CAAC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IACE,IAAI,CAAC,OAAO,KAAK,WAAW;YAC5B,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;YACxB,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC,IAAI,EAC/C,CAAC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/bin/pipeline/site.d.ts
CHANGED
package/bin/pipeline/site.js
CHANGED
|
@@ -8,13 +8,18 @@ import { detectFramework, framework } from '../utils/detectFramework.js';
|
|
|
8
8
|
import { logErrorResults } from '../utils/errors.js';
|
|
9
9
|
import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
|
|
10
10
|
import { INDEX_NAMES, iterateThroughReservedNames } from '../utils/reservedNames.js';
|
|
11
|
-
import { removeTrailingSlash, removeLeadingSlash } from '../utils/strings.js';
|
|
11
|
+
import { removeTrailingSlash, removeLeadingSlash, optionallyAddLeadingSlash, } from '../utils/strings.js';
|
|
12
12
|
import { downloadColors } from './color.js';
|
|
13
13
|
import { scrapePageGroup } from './group.js';
|
|
14
14
|
import { downloadFavicon } from './icon.js';
|
|
15
15
|
import { downloadLogos } from './logo.js';
|
|
16
16
|
import { htmlToHast } from './root.js';
|
|
17
17
|
import { downloadTitle } from './title.js';
|
|
18
|
+
function matchesFilter(pathname, filter) {
|
|
19
|
+
const normalizedPathname = removeTrailingSlash(pathname);
|
|
20
|
+
const normalizedFilter = removeTrailingSlash(optionallyAddLeadingSlash(filter));
|
|
21
|
+
return (normalizedPathname === normalizedFilter || normalizedPathname.startsWith(normalizedFilter + '/'));
|
|
22
|
+
}
|
|
18
23
|
export async function scrapeSite(html, url, opts = {}) {
|
|
19
24
|
let hast = opts.hast;
|
|
20
25
|
if (!hast)
|
|
@@ -23,7 +28,7 @@ export async function scrapeSite(html, url, opts = {}) {
|
|
|
23
28
|
const origin = url.origin;
|
|
24
29
|
if (!framework.vendor)
|
|
25
30
|
detectFramework(hast);
|
|
26
|
-
if (framework.vendor === 'docusaurus') {
|
|
31
|
+
if (framework.vendor === 'docusaurus' || framework.vendor === 'gitbook') {
|
|
27
32
|
const browser = await startPuppeteer();
|
|
28
33
|
html = await fetchPageHtml(url, browser);
|
|
29
34
|
hast = htmlToHast(html);
|
|
@@ -43,8 +48,20 @@ export async function scrapeSite(html, url, opts = {}) {
|
|
|
43
48
|
}
|
|
44
49
|
const needsBrowser = framework.vendor === 'gitbook';
|
|
45
50
|
const externalLinks = listOfLinks.filter((url) => url.origin !== origin);
|
|
46
|
-
const internalLinks = listOfLinks.filter((url) =>
|
|
47
|
-
|
|
51
|
+
const internalLinks = listOfLinks.filter((url) => {
|
|
52
|
+
if (url.origin !== origin || removeTrailingSlash(url.toString()) === origin)
|
|
53
|
+
return false;
|
|
54
|
+
if (opts.filter && !matchesFilter(url.pathname, opts.filter))
|
|
55
|
+
return false;
|
|
56
|
+
return true;
|
|
57
|
+
});
|
|
58
|
+
const rootLinks = listOfLinks.filter((url) => {
|
|
59
|
+
if (url.origin !== origin || removeTrailingSlash(url.toString()) !== origin)
|
|
60
|
+
return false;
|
|
61
|
+
if (opts.filter && !matchesFilter('/', opts.filter))
|
|
62
|
+
return false;
|
|
63
|
+
return true;
|
|
64
|
+
});
|
|
48
65
|
const allPathnames = [
|
|
49
66
|
...internalLinks.map((url) => url.toString()),
|
|
50
67
|
...rootLinks.map((url) => url.toString()),
|
|
@@ -126,13 +143,22 @@ export async function scrapeSite(html, url, opts = {}) {
|
|
|
126
143
|
}
|
|
127
144
|
})
|
|
128
145
|
.filter(Boolean);
|
|
146
|
+
function filterErroredOrFilteredPaths(value) {
|
|
147
|
+
if (allErroredPaths.includes(value))
|
|
148
|
+
return true;
|
|
149
|
+
if (opts.filter && !matchesFilter('/' + value, opts.filter))
|
|
150
|
+
return true;
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
129
153
|
traverse(navItems).forEach(function (value) {
|
|
130
|
-
if (typeof value === 'string' &&
|
|
154
|
+
if (typeof value === 'string' &&
|
|
155
|
+
this.key !== 'group' &&
|
|
156
|
+
filterErroredOrFilteredPaths(value)) {
|
|
131
157
|
this.remove();
|
|
132
158
|
}
|
|
133
159
|
else if (Array.isArray(value)) {
|
|
134
160
|
this.update(value
|
|
135
|
-
.filter((item) => typeof item === 'string' &&
|
|
161
|
+
.filter((item) => typeof item === 'string' && filterErroredOrFilteredPaths(item) ? undefined : item)
|
|
136
162
|
.filter(Boolean));
|
|
137
163
|
}
|
|
138
164
|
});
|
|
@@ -158,8 +184,11 @@ export async function scrapeSite(html, url, opts = {}) {
|
|
|
158
184
|
}
|
|
159
185
|
else if (Array.isArray(value) &&
|
|
160
186
|
value.find((val) => typeof val === 'string' && (val.startsWith('https://') || val.startsWith('http://')))) {
|
|
161
|
-
|
|
162
|
-
|
|
187
|
+
const newPages = value.filter((val) => !(typeof val === 'string' && (val.startsWith('https://') || val.startsWith('http://'))));
|
|
188
|
+
if (newPages.length)
|
|
189
|
+
this.update(newPages);
|
|
190
|
+
else
|
|
191
|
+
this.parent?.remove();
|
|
163
192
|
}
|
|
164
193
|
});
|
|
165
194
|
logErrorResults('linking to external pages', externalResults);
|
package/bin/pipeline/site.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"site.js","sourceRoot":"","sources":["../../src/pipeline/site.ts"],"names":[],"mappings":"AAEA,OAAO,QAAQ,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,
|
|
1
|
+
{"version":3,"file":"site.js","sourceRoot":"","sources":["../../src/pipeline/site.ts"],"names":[],"mappings":"AAEA,OAAO,QAAQ,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,SAAS,aAAa,CAAC,QAAgB,EAAE,MAAc;IACrD,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC;IAChF,OAAO,CACL,kBAAkB,KAAK,gBAAgB,IAAI,kBAAkB,CAAC,UAAU,CAAC,gBAAgB,GAAG,GAAG,CAAC,CACjG,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAAgE,EAAE;IAElE,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACrB,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAEnC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAE1B,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAE7C,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACxE,MAAM,OAAO,GAAG,MAAM,cAAc,EAAE,CAAC;QACvC,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,eAAe,EAAE,EAAE,CAAC;IAE1F,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,wCAAwC,GAAG,EAAE,EAAE,CAAC;IACpF,CAAC;IAED,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,8CAA8C,GAAG,EAAE,EAAE,CAAC;IAC1F,CAAC;IAED,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/C,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1F,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QAC3E,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QAC3C,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1F,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QAClE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG;QACnB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7C,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;KAC1C,CAAC;IACF,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG,2BAA2B,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QACpE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE;YACzE,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE;YACjE,aAAa,EAAE,KAAK;YACpB,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CACpC,eAAe;aACZ,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,WAAW;aACR,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBAC1D,CAAC;qBAAM,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CACvE,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO;YACxC,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC;YAC3D,MAAM,IAAI,GAAG,cAAc;iBACxB,KAAK,CAAC,MAAM,CAAC;iBACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;iBAC3E,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,QAAQ,CAAC,KAAK,CAAC,GAAG;gBAChB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,CAAC,OAAO,CAAC;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG;YAChB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACtD,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;SACnD,CAAC;QAEF,MAAM,eAAe,GAAG,SAAS;aAC9B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACd,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC9B,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC7E,OAAO,kBAAkB,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,SAAS,4BAA4B,CAAC,KAAa;YACjD,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YACjD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;gBAAE,OAAO,IAAI,CAAC;YACzE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,IAAI,CAAC,GAAG,KAAK,OAAO;gBACpB,4BAA4B,CAAC,KAAK,CAAC,EACnC,CAAC;gBACD,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK;qBACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACf,OAAO,IAAI,KAAK,QAAQ,IAAI,4BAA4B,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAClF;qBACA,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,KAAK,GAAG,CAAC,CAAC;YACV,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;gBACxC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/D,KAAK,EAAE,CAAC;oBACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBAChB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IACE,OAAO,KAAK,KAAK,QAAQ;gBACzB,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC7D,CAAC;gBACD,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBACpB,KAAK,CAAC,IAAI,CACR,CAAC,GAAG,EAAE,EAAE,CACN,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CACvF,EACD,CAAC;gBACD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAC3B,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAC1F,CAAC;gBACF,IAAI,QAAQ,CAAC,MAAM;oBAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;;oBACtC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,eAAe,CAAC,2BAA2B,EAAE,eAAe,CAAC,CAAC;QAC9D,eAAe,CAAC,oBAAoB,EAAE,CAAC,GAAG,eAAe,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC;QAE5E,MAAM,oBAAoB,GAAG,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC;QAC3D,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI;gBACJ,IAAI;gBACJ,MAAM;gBACN,OAAO;gBACP,UAAU,EAAE,QAAsB;gBAClC,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,sEAAsE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC"}
|
package/bin/pipeline/tabs.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
import { MintConfig } from '@mintlify/models';
|
|
2
2
|
import type { Result } from '../types/result.js';
|
|
3
|
-
export declare function scrapeAllSiteTabs(html: string, url: string | URL
|
|
3
|
+
export declare function scrapeAllSiteTabs(html: string, url: string | URL, opts?: {
|
|
4
|
+
filter?: string;
|
|
5
|
+
}): Promise<Result<MintConfig>>;
|
package/bin/pipeline/tabs.js
CHANGED
|
@@ -9,7 +9,7 @@ import { downloadLogos } from './logo.js';
|
|
|
9
9
|
import { htmlToHast } from './root.js';
|
|
10
10
|
import { scrapeSite } from './site.js';
|
|
11
11
|
import { downloadTitle } from './title.js';
|
|
12
|
-
export async function scrapeAllSiteTabs(html, url) {
|
|
12
|
+
export async function scrapeAllSiteTabs(html, url, opts = {}) {
|
|
13
13
|
const hast = htmlToHast(html);
|
|
14
14
|
url = new URL(url);
|
|
15
15
|
detectFramework(hast);
|
|
@@ -20,7 +20,7 @@ export async function scrapeAllSiteTabs(html, url) {
|
|
|
20
20
|
if (!links ||
|
|
21
21
|
!links.length ||
|
|
22
22
|
(links.length === 1 && links[0] && links[0].url === url.pathname))
|
|
23
|
-
return scrapeSite(html, url, { hast });
|
|
23
|
+
return scrapeSite(html, url, { hast, filter: opts.filter });
|
|
24
24
|
if (!links.find((link) => url.pathname.startsWith(link.url))) {
|
|
25
25
|
links.push({
|
|
26
26
|
name: getTitleFromLink(url.pathname),
|
|
@@ -32,7 +32,7 @@ export async function scrapeAllSiteTabs(html, url) {
|
|
|
32
32
|
newUrl.pathname = tabEntry.url;
|
|
33
33
|
try {
|
|
34
34
|
const newHtml = await fetchPageHtml(newUrl, undefined);
|
|
35
|
-
return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry] });
|
|
35
|
+
return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry], filter: opts.filter });
|
|
36
36
|
}
|
|
37
37
|
catch (error) {
|
|
38
38
|
return { success: false, message: getErrorMessage(error) };
|
|
@@ -77,6 +77,6 @@ export async function scrapeAllSiteTabs(html, url) {
|
|
|
77
77
|
},
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
|
-
return scrapeSite(html, url, { hast });
|
|
80
|
+
return scrapeSite(html, url, { hast, filter: opts.filter });
|
|
81
81
|
}
|
|
82
82
|
//# sourceMappingURL=tabs.js.map
|
package/bin/pipeline/tabs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tabs.js","sourceRoot":"","sources":["../../src/pipeline/tabs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,GAAiB;
|
|
1
|
+
{"version":3,"file":"tabs.js","sourceRoot":"","sources":["../../src/pipeline/tabs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,GAAiB,EACjB,OAA4B,EAAE;IAE9B,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,IACE,SAAS,CAAC,MAAM,KAAK,QAAQ;QAC7B,SAAS,CAAC,MAAM,KAAK,YAAY;QACjC,SAAS,CAAC,MAAM,KAAK,SAAS,EAC9B,CAAC;QACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC1C,IACE,CAAC,KAAK;YACN,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,QAAQ,CAAC;YAEjE,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACpC,GAAG,EAAE,GAAG,CAAC,QAAQ;aAClB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACvD,OAAO,MAAM,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACtF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,EAAE,OAAO,EAAE,KAAc,EAAE,OAAO,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,WAAW,GAA2B,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,cAAc,CAAC;QAC7B,IAAI,MAAM,GAAW,aAAa,CAAC;QAEnC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,OAAO;YACzB,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,cAAc;gBAAE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1E,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,aAAa;gBAAE,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1B,GAAG,CAAC,sBAAsB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC;QACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI;gBACJ,IAAI;gBACJ,MAAM;gBACN,OAAO;gBACP,UAAU,EAAE,WAAyB;gBACrC,IAAI;aACL;SACF,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9D,CAAC"}
|