@mintlify/scraping 3.0.141 → 3.0.142

package/bin/browser.js

@@ -18,7 +18,7 @@ export async function getHtmlWithPuppeteer(href) {
         waitUntil: 'networkidle2',
     });
     const html = await page.content();
-    browser.close();
+    void browser.close();
     return html;
 }
 //# sourceMappingURL=browser.js.map

package/bin/browser.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAEnC,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,uCAAuC,EAAE,GAAG,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY;IACrD,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAEnC,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,uCAAuC,EAAE,GAAG,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY;IACrD,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC;IACrB,OAAO,IAAI,CAAC;AACd,CAAC"}

package/bin/checks.d.ts

@@ -0,0 +1,8 @@
+import { Framework } from './scraping/detectFramework.js';
+export declare function checkUrl({ url }: {
+    url: string;
+}): boolean;
+export declare function checkVersion({ tool, docusaurusVersion, }: {
+    tool: Framework | undefined;
+    docusaurusVersion: string | undefined;
+}): boolean;

package/bin/checks.js

@@ -0,0 +1,24 @@
+// This checks the link is written correctly, not that the page exists.
+export function checkUrl({ url }) {
+    try {
+        new URL(url);
+    }
+    catch {
+        throw Error(`Invalid link: ${url}\nMake sure the link starts with http:// or https://`);
+    }
+    return true;
+}
+export function checkVersion({ tool, docusaurusVersion, }) {
+    if (tool === 'docusaurus') {
+        if (docusaurusVersion === undefined) {
+            throw Error('When using Docusaurus, you must specify the version (1,2,3) using the --docusaurusVersion flag');
+        }
+    }
+    else {
+        if (docusaurusVersion !== undefined) {
+            throw Error('The --docusaurusVersion flag is only applicable when using Docusaurus.');
+        }
+    }
+    return true;
+}
+//# sourceMappingURL=checks.js.map

package/bin/checks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks.js","sourceRoot":"","sources":["../src/checks.ts"],"names":[],"mappings":"AAEA,uEAAuE;AACvE,MAAM,UAAU,QAAQ,CAAC,EAAE,GAAG,EAAmB;IAC/C,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,CAAC,iBAAiB,GAAG,sDAAsD,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,EAC3B,IAAI,EACJ,iBAAiB,GAIlB;IACC,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAC1B,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,KAAK,CACT,gGAAgG,CACjG,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,KAAK,CAAC,wEAAwE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/bin/cli.js

@@ -1,59 +1,63 @@
 #!/usr/bin/env node
-/* eslint-disable @typescript-eslint/no-empty-function */
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
+import { checkUrl, checkVersion } from './checks.js';
 import { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
-import { scrapePageAutomatically, scrapePageWrapper } from './scraping/scrapePageCommands.js';
-import { scrapeSectionAutomatically, scrapeSectionAxiosWrapper, scrapeGitbookSectionCommand, } from './scraping/scrapeSectionCommands.js';
-import { scrapeIntercomPage } from './scraping/site-scrapers/Intercom/scrapeIntercomPage.js';
-import { scrapeIntercomSection } from './scraping/site-scrapers/Intercom/scrapeIntercomSection.js';
-import { scrapeGitBookPage } from './scraping/site-scrapers/scrapeGitBookPage.js';
-import { scrapeReadMePage } from './scraping/site-scrapers/scrapeReadMePage.js';
-import { scrapeReadMeSection } from './scraping/site-scrapers/scrapeReadMeSection.js';
+import { frameworks } from './scraping/detectFramework.js';
+import { scrapePageAutomatically } from './scraping/scrapePageCommands.js';
+import { scrapeSectionAutomatically } from './scraping/scrapeSectionCommands.js';
 await yargs(hideBin(process.argv))
-    .command('page [url]', 'Scrapes a page', () => { }, async (argv) => {
-    await scrapePageAutomatically(argv);
+    .command('page <url>', 'Scrapes a page', (yargs) => yargs
+    .positional('url', { type: 'string', demandOption: true })
+    .check(checkUrl)
+    .option('overwrite', { alias: 'O', type: 'boolean', default: false })
+    .option('tool', { alias: 't', choices: frameworks })
+    .option('docusaurusVersion', {
+    alias: 'd',
+    type: 'string',
+    choices: ['1', '2', '3'],
 })
-    .command('gitbook-page [url]', 'Scrapes a GitBook page', () => { }, async (argv) => {
-    await scrapePageWrapper(argv, scrapeGitBookPage);
+    .check(checkVersion), async ({ url, overwrite, tool, docusaurusVersion }) => {
+    const frameworkHint = {
+        framework: tool,
+        version: docusaurusVersion ?? '3',
+    };
+    await scrapePageAutomatically(url, overwrite, frameworkHint);
 })
-    .command('readme-page [url]', 'Scrapes a ReadMe page', () => { }, async (argv) => {
-    await scrapePageWrapper(argv, scrapeReadMePage);
+    .command('section <url>', 'Scrapes the docs in the section', (yargs) => yargs
+    .positional('url', { type: 'string', demandOption: true })
+    .check(checkUrl)
+    .option('overwrite', { alias: 'O', type: 'boolean', default: false })
+    .option('tool', { alias: 't', choices: frameworks })
+    .option('docusaurusVersion', {
+    alias: 'd',
+    type: 'string',
+    choices: ['1', '2', '3'],
 })
-    .command('intercom-page [url]', 'Scrapes a Intercom page', () => { }, async (argv) => {
-    await scrapePageWrapper(argv, scrapeIntercomPage);
+    .check(checkVersion), async ({ url, overwrite, tool, docusaurusVersion }) => {
+    const frameworkHint = {
+        framework: tool,
+        version: docusaurusVersion ?? '3',
+    };
+    await scrapeSectionAutomatically(url, overwrite, frameworkHint);
 })
-    .command('section [url]', 'Scrapes the docs in the section', () => { }, async (argv) => {
-    await scrapeSectionAutomatically(argv);
+    .command('openapi-file <openapiFilename>', 'Creates MDX files from an OpenAPI spec', (yargs) => yargs
+    .positional('openapiFilename', {
+    describe: 'The filename of the OpenAPI spec',
+    type: 'string',
+    demandOption: true,
 })
-    .command('gitbook-section [url]', 'Scrapes the Gitbook section', () => { }, async (argv) => {
-    await scrapeGitbookSectionCommand(argv);
+    .option('writeFiles', {
+    describe: 'Whether or not to write the frontmatter files',
+    default: true,
+    type: 'boolean',
+    alias: 'w',
 })
-    .command('readme-section [url]', 'Scrapes the ReadMe section', () => { }, async (argv) => {
-    await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
-})
-    .command('intercom-section [url]', 'Scrapes the Intercom section', () => { }, async (argv) => {
-    await scrapeSectionAxiosWrapper(argv, scrapeIntercomSection);
-})
-    .command('openapi-file <openapiFilename>', 'Creates MDX files from an OpenAPI spec', (yargs) => {
-    return yargs
-        .positional('openapiFilename', {
-        describe: 'The filename of the OpenAPI spec',
-        type: 'string',
-    })
-        .option('writeFiles', {
-        describe: 'Whether or not to write the frontmatter files',
-        default: true,
-        type: 'boolean',
-        alias: 'w',
-    })
-        .option('outDir', {
-        describe: 'The folder in which to write any created frontmatter files',
-        type: 'string',
-        alias: 'o',
-    })
-        .demandOption('openapiFilename');
-}, async (argv) => {
+    .option('outDir', {
+    describe: 'The folder in which to write any created frontmatter files',
+    type: 'string',
+    alias: 'o',
+}), async (argv) => {
     try {
         const { nav } = await generateOpenApiPages(argv.openapiFilename, argv.writeFiles, argv.outDir);
         console.log('navigation object suggestion:');

package/bin/cli.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,yDAAyD;AACzD,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAC9F,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,EACzB,2BAA2B,GAC5B,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,yDAAyD,CAAC;AAC7F,OAAO,EAAE,qBAAqB,EAAE,MAAM,4DAA4D,CAAC;AACnG,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,gBAAgB,EAAE,MAAM,8CAA8C,CAAC;AAChF,OAAO,EAAE,mBAAmB,EAAE,MAAM,iDAAiD,CAAC;AAEtF,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,gBAAgB,EAChB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC,CACF;KACA,OAAO,CACN,oBAAoB,EACpB,wBAAwB,EACxB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;AACnD,CAAC,CACF;KACA,OAAO,CACN,mBAAmB,EACnB,uBAAuB,EACvB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;AAClD,CAAC,CACF;KACA,OAAO,CACN,qBAAqB,EACrB,yBAAyB,EACzB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;AACpD,CAAC,CACF;KACA,OAAO,CACN,eAAe,EACf,iCAAiC,EACjC,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC,CACF;KACA,OAAO,CACN,uBAAuB,EACvB,6BAA6B,EAC7B,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC,CACF;KACA,OAAO,CACN,sBAAsB,EACtB,4BAA4B,EAC5B,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;AAC7D,CAAC,CACF;KACA,OAAO,CACN,wBAAwB,EACxB,8BAA8B,EAC9B,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,MAAM,yBAAyB,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;AAC/D,CAAC,CACF;KACA,OAAO,CACN,gCAAgC,EAChC,wCAAwC,EACxC,CAAC,KAAK,EAAE,EAAE;IACR,OAAO,KAAK;SACT,UAAU,CAAC,iBAAiB,EAAE;QAC7B,QAAQ,EAAE,kCAAkC;QAC5C,IAAI,EAAE,QAAQ;KACf,CAAC;SACD,MAAM,CAAC,YAAY,EAAE;QACpB,QAAQ,EAAE,+CAA+C;QACzD,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,GAAG;KACX,CAAC;SACD,MAAM,CAAC,QAAQ,EAAE;QAChB,QAAQ,EAAE,4DAA4D;QACtE,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,GAAG;KACX,CAAC;SACD,YAAY,CAAC,iBAAiB,CAAC,CAAC;AACrC,CAAC,EACD,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,oBAAoB,CACxC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,CACF;IACD,+DAA+D;KAC9D,cAAc,EAAE;KAChB,aAAa,CAAC,CAAC,EAAE,gEAAgE,CAAC;IAEnF,iDAAiD;KAChD,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KAErB,KAAK,EAAE,CAAC"}
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAiB,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qCAAqC,CAAC;AAEjF,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KAC/B,OAAO,CACN,YAAY,EACZ,gBAAgB,EAChB,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;KACzD,KAAK,CAAC,QAAQ,CAAC;KACf,MAAM,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;KACpE,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;KACnD,MAAM,CAAC,mBAAmB,EAAE;IAC3B,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAU;CAClC,CAAC;KACD,KAAK,CAAC,YAAY,CAAC,EACxB,KAAK,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,EAAE;IACpD,MAAM,aAAa,GAAkB;QACnC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,iBAAiB,IAAI,GAAG;KAClC,CAAC;IAEF,MAAM,uBAAuB,CAAC,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;AAC/D,CAAC,CACF;KACA,OAAO,CACN,eAAe,EACf,iCAAiC,EACjC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;KACzD,KAAK,CAAC,QAAQ,CAAC;KACf,MAAM,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;KACpE,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;KACnD,MAAM,CAAC,mBAAmB,EAAE;IAC3B,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAU;CAClC,CAAC;KACD,KAAK,CAAC,YAAY,CAAC,EACxB,KAAK,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,EAAE;IACpD,MAAM,aAAa,GAAkB;QACnC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,iBAAiB,IAAI,GAAG;KAClC,CAAC;IAEF,MAAM,0BAA0B,CAAC,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;AAClE,CAAC,CACF;KACA,OAAO,CACN,gCAAgC,EAChC,wCAAwC,EACxC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK;KACF,UAAU,CAAC,iBAAiB,EAAE;IAC7B,QAAQ,EAAE,kCAAkC;IAC5C,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,IAAI;CACnB,CAAC;KACD,MAAM,CAAC,YAAY,EAAE;IACpB,QAAQ,EAAE,+CAA+C;IACzD,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,GAAG;CACX,CAAC;KACD,MAAM,CAAC,QAAQ,EAAE;IAChB,QAAQ,EAAE,4DAA4D;IACtE,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,GAAG;CACX,CAAC,EACN,KAAK,EAAE,IAAI,EAAE,EAAE;IACb,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,oBAAoB,CACxC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,CACF;IACD,+DAA+D;KAC9D,cAAc,EAAE;KAChB,aAAa,CAAC,CAAC,EAAE,gEAAgE,CAAC;IAEnF,iDAAiD;KAChD,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KAErB,KAAK,EAAE,CAAC"}

package/bin/scraping/detectFramework.d.ts

@@ -1,16 +1,9 @@
-export declare enum Frameworks {
-    DOCUSAURUS = "DOCUSAURUS",
-    GITBOOK = "GITBOOK",
-    README = "README",
-    INTERCOM = "INTERCOM"
-}
-export declare function detectFramework(html: any): {
-    framework: Frameworks;
-    version: string;
+export declare const frameworks: readonly ["docusaurus", "gitbook", "readme", "intercom"];
+export type Framework = (typeof frameworks)[number];
+export type FrameworkHint = {
+    framework: 'docusaurus';
+    version: '1' | '2' | '3';
 } | {
-    framework: Frameworks;
-    version?: undefined;
-} | {
-    framework: undefined;
-    version?: undefined;
+    framework: 'gitbook' | 'readme' | 'intercom' | undefined;
 };
+export declare function detectFramework(html: string): FrameworkHint;

package/bin/scraping/detectFramework.js

@@ -1,11 +1,5 @@
-import cheerio from 'cheerio';
-export var Frameworks;
-(function (Frameworks) {
-    Frameworks["DOCUSAURUS"] = "DOCUSAURUS";
-    Frameworks["GITBOOK"] = "GITBOOK";
-    Frameworks["README"] = "README";
-    Frameworks["INTERCOM"] = "INTERCOM";
-})(Frameworks || (Frameworks = {}));
+import * as cheerio from 'cheerio';
+export const frameworks = ['docusaurus', 'gitbook', 'readme', 'intercom'];
 export function detectFramework(html) {
     const $ = cheerio.load(html);
     const docusaurusMeta = $('meta[name="generator"]');
@@ -15,27 +9,27 @@ export function detectFramework(html) {
         docusaurusMeta.attr('content').includes('Docusaurus')) {
         const metaAttrString = docusaurusMeta.attr('content');
         if (metaAttrString.includes('v3')) {
-            return { framework: Frameworks.DOCUSAURUS, version: '3' };
+            return { framework: 'docusaurus', version: '3' };
         }
         if (metaAttrString.includes('v2')) {
-            return { framework: Frameworks.DOCUSAURUS, version: '2' };
+            return { framework: 'docusaurus', version: '2' };
         }
         else if (metaAttrString.includes('v1')) {
             console.warn('WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.');
-            return { framework: Frameworks.DOCUSAURUS, version: '1' };
+            return { framework: 'docusaurus', version: '1' };
         }
     }
     const isGitBook = $('.gitbook-root').length > 0;
     if (isGitBook) {
-        return { framework: Frameworks.GITBOOK };
+        return { framework: 'gitbook' };
     }
     const isReadMe = $('meta[name="readme-deploy"]').length > 0;
     if (isReadMe) {
-        return { framework: Frameworks.README };
+        return { framework: 'readme' };
     }
     const isIntercom = $("meta[name='intercom:trackingEvent']").length > 0;
     if (isIntercom) {
-        return { framework: Frameworks.INTERCOM };
+        return { framework: 'intercom' };
     }
     return { framework: undefined };
 }

package/bin/scraping/detectFramework.js.map

@@ -1 +1 @@
-{"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/scraping/detectFramework.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,CAAN,IAAY,UAKX;AALD,WAAY,UAAU;IACpB,uCAAyB,CAAA;IACzB,iCAAmB,CAAA;IACnB,+BAAiB,CAAA;IACjB,mCAAqB,CAAA;AACvB,CAAC,EALW,UAAU,KAAV,UAAU,QAKrB;AAED,MAAM,UAAU,eAAe,CAAC,IAAI;IAClC,MAAM,CAAC,GAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEnD,IACE,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ;QACjD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EACjE,CAAC;QACD,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAW,CAAC;QAChE,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QAC5D,CAAC;QACD,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QAC5D,CAAC;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,OAAO,CAAC,IAAI,CACV,0FAA0F,CAC3F,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5D,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;IAC5C,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC"}
+{"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/scraping/detectFramework.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,CAAU,CAAC;AAYnF,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,cAAc,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEnD,IACE,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ;QACjD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EACjE,CAAC;QACD,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAW,CAAC;QAChE,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QACnD,CAAC;QACD,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QACnD,CAAC;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,OAAO,CAAC,IAAI,CACV,0FAA0F,CAC3F,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;IAClC,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5D,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACjC,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;IACnC,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC"}

package/bin/scraping/downloadAllImages.d.ts

@@ -1,3 +1,4 @@
-export default function downloadAllImages($: any, content: any, origin: string, baseDir: string, overwrite: boolean, modifyFileName?: any): Promise<{
+import { Cheerio, CheerioAPI, Element } from 'cheerio';
+export default function downloadAllImages($: CheerioAPI, content: Cheerio<Element>, origin: string, baseDir: string, overwrite: boolean, modifyFileName?: (fileName: string) => string): Promise<{
     [x: string]: string;
 } | undefined>;

package/bin/scraping/downloadAllImages.js

@@ -1,6 +1,5 @@
 import path from 'path';
 import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from '../downloadImage.js';
-// To Do: Use CheerioElement instead of any when we bump the cheerio version
 export default async function downloadAllImages($, content, origin, baseDir, overwrite, modifyFileName) {
     if (!baseDir) {
         console.debug('Skipping image downloading');
@@ -11,7 +10,7 @@ export default async function downloadAllImages($, content, origin, baseDir, ove
     const imageSrcs = [
         ...new Set(content
             .find('img[src]')
-            .map((i, image) => $(image).attr('src'))
+            .map((_, image) => $(image).attr('src'))
             .toArray()),
     ];
     // Wait to all images to download before continuing

package/bin/scraping/downloadAllImages.js.map

@@ -1 +1 @@
-{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,SAAkB,EAClB,cAAoB;IAEpB,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;IACT,CAAC;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE,CAAC;YACnB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;AACxF,CAAC"}
+{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAa,EACb,OAAyB,EACzB,MAAc,EACd,OAAe,EACf,SAAkB,EAClB,cAA6C;IAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;IACT,CAAC;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/B,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE,CAAC;YACnB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;AACxF,CAAC"}

package/bin/scraping/downloadLogoImage.js

@@ -1,7 +1,6 @@
 import path from 'path';
 import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from '../downloadImage.js';
 import { getFileExtension } from '../util.js';
-// To Do: Use CheerioElement instead of any when we bump the cheerio version
 export default async function downloadLogoImage(imageSrc, imageBaseDir, origin, overwrite) {
     if (!imageSrc || !isValidImageSrc(imageSrc))
         return;

package/bin/scraping/downloadLogoImage.js.map

@@ -1 +1 @@
-{"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,QAA4B,EAC5B,YAAoB,EACpB,MAAc,EACd,SAAkB;IAElB,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;QAAE,OAAO;IAEpD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAE5E,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}
+{"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,QAA4B,EAC5B,YAAoB,EACpB,MAAc,EACd,SAAkB;IAElB,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;QAAE,OAAO;IAEpD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAE5E,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}

package/bin/scraping/replaceImagePaths.d.ts

@@ -1 +1 @@
-export default function replaceImagePaths(origToWritePath: object, cliDir: string, markdown: string): string;
+export default function replaceImagePaths(origToWritePath: Record<string, string>, cliDir: string, markdown: string): string;

package/bin/scraping/replaceImagePaths.js

@@ -1,7 +1,4 @@
 export default function replaceImagePaths(origToWritePath, cliDir, markdown) {
-    if (origToWritePath == null) {
-        return markdown;
-    }
     // Change image paths to use the downloaded locations
     for (const [origHref, writePath] of Object.entries(origToWritePath)) {
         // Use relative paths within the folder we are in

package/bin/scraping/replaceImagePaths.js.map

@@ -1 +1 @@
-{"version":3,"file":"replaceImagePaths.js","sourceRoot":"","sources":["../../src/scraping/replaceImagePaths.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,eAAuB,EACvB,MAAc,EACd,QAAgB;IAEhB,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;QAC5B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,qDAAqD;IACrD,KAAK,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACpE,iDAAiD;QACjD,IAAI,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
+{"version":3,"file":"replaceImagePaths.js","sourceRoot":"","sources":["../../src/scraping/replaceImagePaths.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,eAAuC,EACvC,MAAc,EACd,QAAgB;IAEhB,qDAAqD;IACrD,KAAK,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACpE,iDAAiD;QACjD,IAAI,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/bin/scraping/scrapePageCommands.d.ts

@@ -1,7 +1,7 @@
-import { ArgumentsCamelCase } from 'yargs';
+import { FrameworkHint } from './detectFramework.js';
 import { ScrapePageFn } from './scrapePage.js';
-export declare function scrapePageWrapper(argv: ArgumentsCamelCase, scrapeFunc: ScrapePageFn, options?: {
+export declare function scrapePageWrapper(url: string, overwrite: boolean, scrapeFunc: ScrapePageFn, options?: {
     version?: string;
     puppeteer?: boolean;
 }): Promise<void>;
-export declare function scrapePageAutomatically(argv: any): Promise<void>;
+export declare function scrapePageAutomatically(url: string, overwrite: boolean, frameworkHint: FrameworkHint): Promise<void>;

package/bin/scraping/scrapePageCommands.js

@@ -1,7 +1,6 @@
 import axios from 'axios';
 import { getHtmlWithPuppeteer } from '../browser.js';
-import { getHrefFromArgs } from '../util.js';
-import { detectFramework, Frameworks } from './detectFramework.js';
+import { detectFramework, frameworks } from './detectFramework.js';
 import { scrapePage } from './scrapePage.js';
 import { scrapeIntercomPage } from './site-scrapers/Intercom/scrapeIntercomPage.js';
 import { scrapeDocusaurusPage } from './site-scrapers/scrapeDocusaurusPage.js';
@@ -9,46 +8,42 @@ import { scrapeGitBookPage } from './site-scrapers/scrapeGitBookPage.js';
 import { scrapeReadMePage } from './site-scrapers/scrapeReadMePage.js';
 function validateFramework(framework) {
     if (!framework) {
-        console.log('Could not detect the framework automatically. Please use one of:');
-        console.log('scrape-page-docusaurus');
-        console.log('scrape-page-gitbook');
-        console.log('scrape-page-readme');
-        console.log('scrape-page-intercom');
+        console.log(`Could not detect the framework automatically. Please use the -t flag to specify one of: ${frameworks.join(', ')}`);
         return process.exit(1);
     }
 }
-export async function scrapePageWrapper(argv, scrapeFunc, options) {
-    const href = getHrefFromArgs(argv);
+export async function scrapePageWrapper(url, overwrite, scrapeFunc, options) {
     let html;
     if (options?.puppeteer) {
-        html = await getHtmlWithPuppeteer(href);
+        html = await getHtmlWithPuppeteer(url);
     }
     else {
-        const res = await axios.get(href);
+        const res = await axios.get(url);
         html = res.data;
     }
-    await scrapePage(scrapeFunc, href, html, !!argv.overwrite, options?.version);
+    await scrapePage(scrapeFunc, url, html, overwrite, options?.version);
     process.exit(0);
 }
-export async function scrapePageAutomatically(argv) {
-    const href = getHrefFromArgs(argv);
-    const res = await axios.get(href);
+export async function scrapePageAutomatically(url, overwrite, frameworkHint) {
+    const res = await axios.get(url);
     const html = res.data;
-    const { framework, version } = detectFramework(html);
-    validateFramework(framework);
-    console.log('Detected framework: ' + framework);
-    switch (framework) {
-        case Frameworks.DOCUSAURUS:
-            await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
+    frameworkHint = frameworkHint.framework ? frameworkHint : detectFramework(html);
+    validateFramework(frameworkHint.framework);
+    console.log('Detected framework: ' + frameworkHint.framework);
+    switch (frameworkHint.framework) {
+        case 'docusaurus':
+            await scrapePageWrapper(url, overwrite, scrapeDocusaurusPage, {
+                version: frameworkHint.version,
+            });
             break;
-        case Frameworks.GITBOOK:
-            await scrapePageWrapper(argv, scrapeGitBookPage, { puppeteer: true });
+        case 'gitbook':
+            await scrapePageWrapper(url, overwrite, scrapeGitBookPage, { puppeteer: true });
             break;
-        case Frameworks.README:
-            await scrapePageWrapper(argv, scrapeReadMePage);
+        case 'readme':
+            await scrapePageWrapper(url, overwrite, scrapeReadMePage);
             break;
-        case Frameworks.INTERCOM:
-            await scrapePageWrapper(argv, scrapeIntercomPage);
+        case 'intercom':
+            await scrapePageWrapper(url, overwrite, scrapeIntercomPage);
             break;
     }
 }

package/bin/scraping/scrapePageCommands.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAgB,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAEvE,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;QAChF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAwB,EACxB,UAAwB,EACxB,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IAClB,CAAC;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjE,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAChD,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAClD,MAAM;IACV,CAAC;AACH,CAAC"}
+{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,eAAe,EAA4B,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC7F,OAAO,EAAE,UAAU,EAAgB,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAEvE,SAAS,iBAAiB,CAAC,SAAgC;IACzD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CACT,2FAA2F,UAAU,CAAC,IAAI,CACxG,IAAI,CACL,EAAE,CACJ,CAAC;QACF,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,SAAkB,EAClB,UAAwB,EACxB,OAAmD;IAEnD,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,IAAI,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IAClB,CAAC;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAW,EACX,SAAkB,EAClB,aAA4B;IAE5B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAEhF,iBAAiB,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IAE3C,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAE9D,QAAQ,aAAa,CAAC,SAAS,EAAE,CAAC;QAChC,KAAK,YAAY;YACf,MAAM,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,oBAAoB,EAAE;gBAC5D,OAAO,EAAE,aAAa,CAAC,OAAO;aAC/B,CAAC,CAAC;YACH,MAAM;QACR,KAAK,SAAS;YACZ,MAAM,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChF,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAC1D,MAAM;QACR,KAAK,UAAU;YACb,MAAM,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,kBAAkB,CAAC,CAAC;YAC5D,MAAM;IACV,CAAC;AACH,CAAC"}

package/bin/scraping/scrapeSectionCommands.d.ts

@@ -1,6 +1,6 @@
-import { ArgumentsCamelCase } from 'yargs';
+import { FrameworkHint } from './detectFramework.js';
 import { ScrapeSectionFn } from './scrapeSection.js';
-export declare function scrapeSectionAxiosWrapper(argv: ArgumentsCamelCase, scrapeFunc: ScrapeSectionFn): Promise<void>;
-export declare function scrapeDocusaurusSectionCommand(argv: any, version: string | undefined): Promise<void>;
-export declare function scrapeGitbookSectionCommand(argv: any): Promise<void>;
-export declare function scrapeSectionAutomatically(argv: any): Promise<void>;
+export declare function scrapeSectionAxiosWrapper(url: string, overwrite: boolean, scrapeFunc: ScrapeSectionFn): Promise<void>;
+export declare function scrapeDocusaurusSectionCommand(url: string, overwrite: boolean, version: string | undefined): Promise<void>;
+export declare function scrapeGitbookSectionCommand(url: string, overwrite: boolean): Promise<void>;
+export declare function scrapeSectionAutomatically(url: string, overwrite: boolean, frameworkHint: FrameworkHint): Promise<void>;

package/bin/scraping/scrapeSectionCommands.js

@@ -1,7 +1,7 @@
 import axios from 'axios';
 import { startBrowser } from '../browser.js';
-import { getHrefFromArgs, getOrigin } from '../util.js';
-import { detectFramework, Frameworks } from './detectFramework.js';
+import { getOrigin } from '../util.js';
+import { detectFramework } from './detectFramework.js';
 import { scrapeSection } from './scrapeSection.js';
 import { scrapeIntercomSection } from './site-scrapers/Intercom/scrapeIntercomSection.js';
 import openNestedDocusaurusMenus from './site-scrapers/openNestedDocusaurusMenus.js';
@@ -9,51 +9,48 @@ import openNestedGitbookMenus from './site-scrapers/openNestedGitbookMenus.js';
 import { scrapeDocusaurusSection } from './site-scrapers/scrapeDocusaurusSection.js';
 import { scrapeGitBookSection } from './site-scrapers/scrapeGitBookSection.js';
 import { scrapeReadMeSection } from './site-scrapers/scrapeReadMeSection.js';
-export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
-    const href = getHrefFromArgs(argv);
-    const res = await axios.get(href);
+export async function scrapeSectionAxiosWrapper(url, overwrite, scrapeFunc) {
+    const res = await axios.get(url);
     const html = res.data;
-    await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, undefined);
+    await scrapeSection(scrapeFunc, html, getOrigin(url), overwrite, undefined);
     process.exit(0);
 }
-export async function scrapeDocusaurusSectionCommand(argv, version // "1" | "2" | "3"
+export async function scrapeDocusaurusSectionCommand(url, overwrite, version // "1" | "2" | "3"
 ) {
-    await scrapeSectionOpeningAllNested(argv, openNestedDocusaurusMenus, scrapeDocusaurusSection, version);
+    await scrapeSectionOpeningAllNested(url, overwrite, openNestedDocusaurusMenus, scrapeDocusaurusSection, version);
 }
-export async function scrapeGitbookSectionCommand(argv) {
-    await scrapeSectionOpeningAllNested(argv, openNestedGitbookMenus, scrapeGitBookSection);
+export async function scrapeGitbookSectionCommand(url, overwrite) {
+    await scrapeSectionOpeningAllNested(url, overwrite, openNestedGitbookMenus, scrapeGitBookSection);
 }
-async function scrapeSectionOpeningAllNested(argv, openLinks, scrapeFunc, version) {
-    const href = getHrefFromArgs(argv);
+async function scrapeSectionOpeningAllNested(url, overwrite, openLinks, scrapeFunc, version) {
     const browser = await startBrowser();
     const page = await browser.newPage();
-    await page.goto(href, {
+    await page.goto(url, {
         waitUntil: 'networkidle2',
     });
     const html = await openLinks(page);
-    browser.close();
-    await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, version);
+    void browser.close();
+    await scrapeSection(scrapeFunc, html, getOrigin(url), overwrite, version);
     process.exit(0);
 }
-export async function scrapeSectionAutomatically(argv) {
-    const href = getHrefFromArgs(argv);
-    const res = await axios.get(href);
+export async function scrapeSectionAutomatically(url, overwrite, frameworkHint) {
+    const res = await axios.get(url);
     const html = res.data;
-    const { framework, version } = detectFramework(html);
-    validateFramework(framework);
-    console.log('Detected framework: ' + framework);
-    switch (framework) {
-        case Frameworks.DOCUSAURUS:
-            await scrapeDocusaurusSectionCommand(argv, version);
+    frameworkHint = frameworkHint.framework ? frameworkHint : detectFramework(html);
+    validateFramework(frameworkHint.framework);
+    console.log('Detected framework: ' + frameworkHint.framework);
+    switch (frameworkHint.framework) {
+        case 'docusaurus':
+            await scrapeDocusaurusSectionCommand(url, overwrite, frameworkHint.version);
             break;
-        case Frameworks.GITBOOK:
-            await scrapeGitbookSectionCommand(argv);
+        case 'gitbook':
+            await scrapeGitbookSectionCommand(url, overwrite);
             break;
-        case Frameworks.README:
-            await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
+        case 'readme':
+            await scrapeSectionAxiosWrapper(url, overwrite, scrapeReadMeSection);
             break;
-        case Frameworks.INTERCOM:
-            await scrapeSectionAxiosWrapper(argv, scrapeIntercomSection);
+        case 'intercom':
+            await scrapeSectionAxiosWrapper(url, overwrite, scrapeIntercomSection);
             break;
     }
 }

package/bin/scraping/scrapeSectionCommands.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAmB,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,mDAAmD,CAAC;AAC1F,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAwB,EACxB,UAA2B;IAE3B,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAS,EACT,OAA2B,CAAC,kBAAkB;;IAE9C,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CAAC,IAAI,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC;AAC1F,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAA2B,EAC3B,OAAgB;IAEhB,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAC3D,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,yBAAyB,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;YAC7D,MAAM;IACV,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC;AACH,CAAC"}
+{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,eAAe,EAA4B,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAmB,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,mDAAmD,CAAC;AAC1F,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,GAAW,EACX,SAAkB,EAClB,UAA2B;IAE3B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,GAAW,EACX,SAAkB,EAClB,OAA2B,CAAC,kBAAkB;;IAE9C,MAAM,6BAA6B,CACjC,GAAG,EACH,SAAS,EACT,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,GAAW,EAAE,SAAkB;IAC/E,MAAM,6BAA6B,CAAC,GAAG,EAAE,SAAS,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC;AACpG,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,GAAW,EACX,SAAkB,EAClB,SAA0C,EAC1C,UAA2B,EAC3B,OAAgB;IAEhB,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;QACnB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,GAAW,EACX,SAAkB,EAClB,aAA4B;IAE5B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAEhF,iBAAiB,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAE9D,QAAQ,aAAa,CAAC,SAAS,EAAE,CAAC;QAChC,KAAK,YAAY;YACf,MAAM,8BAA8B,CAAC,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;YAC5E,MAAM;QACR,KAAK,SAAS;YACZ,MAAM,2BAA2B,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAClD,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,yBAAyB,CAAC,GAAG,EAAE,SAAS,EAAE,mBAAmB,CAAC,CAAC;YACrE,MAAM;QACR,KAAK,UAAU;YACb,MAAM,yBAAyB,CAAC,GAAG,EAAE,SAAS,EAAE,qBAAqB,CAAC,CAAC;YACvE,MAAM;IACV,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAgC;IACzD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC;AACH,CAAC"}

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import { NodeHtmlMarkdown } from 'node-html-markdown';
 import downloadAllImages from '../../downloadAllImages.js';
 import replaceImagePaths from '../../replaceImagePaths.js';

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeIntercomPage.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE/E,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"scrapeIntercomPage.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE/E,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js

@@ -1,12 +1,12 @@
 import axios from 'axios';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import downloadLogoImage from '../../downloadLogoImage.js';
 import { scrapeGettingFileNameFromUrl } from '../../scrapeGettingFileNameFromUrl.js';
 import { scrapeIntercomPage } from './scrapeIntercomPage.js';
 export async function scrapeIntercomSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
     let $ = cheerio.load(html);
     const logoSrc = $('.header__logo img').first().attr('src');
-    downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
+    void downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
     const collectionsLink = $('.section .g__space a');
     const collectionsMap = collectionsLink.toArray().map(async (s) => {
         const href = $(s).attr('href');