@mintlify/scraping 3.0.140 → 3.0.142

package/bin/util.js

@@ -1,7 +1,6 @@
-import { mkdirSync, writeFileSync } from 'fs';
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
 import Ora from 'ora';
 import path from 'path';
-import stopIfInvalidLink from './validation/stopIfInvalidLink.js';
 export const MintConfig = (name, color, ctaName, ctaUrl, filename) => {
     return {
         name,
@@ -68,35 +67,19 @@ export const createPage = (title, description, markdown, overwrite = false, root
     const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
     // Create the folders needed if they're missing
     mkdirSync(rootDir, { recursive: true });
-    // Write the page to memory
-    if (overwrite) {
+    if (!overwrite && existsSync(writePath)) {
+        console.log(`❌ Skipping existing file ${writePath}`);
+        return;
+    }
+    // Write the page to disk
+    try {
         writeFileSync(writePath, Page(title, description, markdown));
         console.log('✏️ - ' + writePath);
     }
-    else {
-        try {
-            writeFileSync(writePath, Page(title, description, markdown), {
-                flag: 'wx',
-            });
-            console.log('✏️ - ' + writePath);
-        }
-        catch (e) {
-            // We do a try-catch instead of an if-statement to avoid a race condition
-            // of the file being created after we started writing.
-            if (e?.code === 'EEXIST') {
-                console.log(`❌ Skipping existing file ${writePath}`);
-            }
-            else {
-                console.error(e);
-            }
-        }
+    catch (e) {
+        console.error(e);
     }
 };
-export function getHrefFromArgs(argv) {
-    const href = argv.url;
-    stopIfInvalidLink(href);
-    return href;
-}
 export const buildLogger = (startText = '') => {
     const logger = Ora().start(startText);
     return logger;

package/bin/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,GAAuB,MAAM,KAAK,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,WAAoB,EAAE,QAAiB,EAAE,EAAE;IAC7E,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;IACtB,CAAC;IACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;IACtB,CAAC;IAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,CAAC,CAAC,mBAAmB,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,SAAS,GAAG,KAAK,EACjB,OAAO,GAAG,EAAE,EACZ,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE,CAAC;QACd,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,sDAAsD;YACtD,IAAK,CAAsB,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,IAAS;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;IACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,GAAG,EAAE,EAAW,EAAE;IACrD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,SAAS,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CAAC;AACzF,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,GAAuB,MAAM,KAAK,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,WAAoB,EAAE,QAAiB,EAAE,EAAE;IAC7E,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;IACtB,CAAC;IACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;IACtB,CAAC;IAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,CAAC,CAAC,mBAAmB,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,SAAS,GAAG,KAAK,EACjB,OAAO,GAAG,EAAE,EACZ,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC;QACH,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,GAAG,EAAE,EAAW,EAAE;IACrD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,SAAS,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CAAC;AACzF,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "3.0.140",
+  "version": "3.0.142",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -39,22 +39,22 @@
   },
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.0",
-    "@mintlify/common": "1.0.124",
+    "@mintlify/common": "1.0.125",
     "axios": "^1.2.2",
-    "cheerio": "^0.22.0",
+    "cheerio": "^1.0.0-rc.12",
     "fs-extra": "^11.1.1",
     "node-html-markdown": "^1.3.0",
     "ora": "^6.1.2",
-    "puppeteer": "^19.4.0",
+    "puppeteer": "^22.14.0",
     "yargs": "^17.6.0"
   },
   "devDependencies": {
     "@mintlify/eslint-config": "1.0.5",
     "@mintlify/eslint-config-typescript": "1.0.10",
-    "@mintlify/models": "0.0.108",
+    "@mintlify/models": "0.0.109",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.170",
+    "@mintlify/validation": "0.1.171",
     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
     "@tsconfig/recommended": "1.x",
     "@types/cheerio": "^0.22.31",
@@ -71,5 +71,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "214320b7644d2fbc47d85d1f618f3f65e162f6a4"
+  "gitHead": "74d38f577b4f02953bfc0f9b0d493371cbb256d2"
 }

package/src/browser.ts

@@ -19,6 +19,6 @@ export async function getHtmlWithPuppeteer(href: string) {
     waitUntil: 'networkidle2',
   });
   const html = await page.content();
-  browser.close();
+  void browser.close();
   return html;
 }

package/src/checks.ts

@@ -0,0 +1,32 @@
+import { Framework } from './scraping/detectFramework.js';
+
+// This checks the link is written correctly, not that the page exists.
+export function checkUrl({ url }: { url: string }) {
+  try {
+    new URL(url);
+  } catch {
+    throw Error(`Invalid link: ${url}\nMake sure the link starts with http:// or https://`);
+  }
+  return true;
+}
+
+export function checkVersion({
+  tool,
+  docusaurusVersion,
+}: {
+  tool: Framework | undefined;
+  docusaurusVersion: string | undefined;
+}) {
+  if (tool === 'docusaurus') {
+    if (docusaurusVersion === undefined) {
+      throw Error(
+        'When using Docusaurus, you must specify the version (1,2,3) using the --docusaurusVersion flag'
+      );
+    }
+  } else {
+    if (docusaurusVersion !== undefined) {
+      throw Error('The --docusaurusVersion flag is only applicable when using Docusaurus.');
+    }
+  }
+  return true;
+}

package/src/cli.ts

@@ -1,95 +1,71 @@
 #!/usr/bin/env node
-
-/* eslint-disable @typescript-eslint/no-empty-function */
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
+import { checkUrl, checkVersion } from './checks.js';
 import { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
-import { scrapePageAutomatically, scrapePageWrapper } from './scraping/scrapePageCommands.js';
-import {
-  scrapeSectionAutomatically,
-  scrapeSectionAxiosWrapper,
-  scrapeGitbookSectionCommand,
-} from './scraping/scrapeSectionCommands.js';
-import { scrapeIntercomPage } from './scraping/site-scrapers/Intercom/scrapeIntercomPage.js';
-import { scrapeIntercomSection } from './scraping/site-scrapers/Intercom/scrapeIntercomSection.js';
-import { scrapeGitBookPage } from './scraping/site-scrapers/scrapeGitBookPage.js';
-import { scrapeReadMePage } from './scraping/site-scrapers/scrapeReadMePage.js';
-import { scrapeReadMeSection } from './scraping/site-scrapers/scrapeReadMeSection.js';
+import { FrameworkHint, frameworks } from './scraping/detectFramework.js';
+import { scrapePageAutomatically } from './scraping/scrapePageCommands.js';
+import { scrapeSectionAutomatically } from './scraping/scrapeSectionCommands.js';
 
 await yargs(hideBin(process.argv))
   .command(
-    'page [url]',
+    'page <url>',
     'Scrapes a page',
-    () => {},
-    async (argv) => {
-      await scrapePageAutomatically(argv);
-    }
-  )
-  .command(
-    'gitbook-page [url]',
-    'Scrapes a GitBook page',
-    () => {},
-    async (argv) => {
-      await scrapePageWrapper(argv, scrapeGitBookPage);
-    }
-  )
-  .command(
-    'readme-page [url]',
-    'Scrapes a ReadMe page',
-    () => {},
-    async (argv) => {
-      await scrapePageWrapper(argv, scrapeReadMePage);
-    }
-  )
-  .command(
-    'intercom-page [url]',
-    'Scrapes a Intercom page',
-    () => {},
-    async (argv) => {
-      await scrapePageWrapper(argv, scrapeIntercomPage);
+    (yargs) =>
+      yargs
+        .positional('url', { type: 'string', demandOption: true })
+        .check(checkUrl)
+        .option('overwrite', { alias: 'O', type: 'boolean', default: false })
+        .option('tool', { alias: 't', choices: frameworks })
+        .option('docusaurusVersion', {
+          alias: 'd',
+          type: 'string',
+          choices: ['1', '2', '3'] as const,
+        })
+        .check(checkVersion),
+    async ({ url, overwrite, tool, docusaurusVersion }) => {
+      const frameworkHint: FrameworkHint = {
+        framework: tool,
+        version: docusaurusVersion ?? '3',
+      };
+
+      await scrapePageAutomatically(url, overwrite, frameworkHint);
     }
   )
   .command(
-    'section [url]',
+    'section <url>',
     'Scrapes the docs in the section',
-    () => {},
-    async (argv) => {
-      await scrapeSectionAutomatically(argv);
-    }
-  )
-  .command(
-    'gitbook-section [url]',
-    'Scrapes the Gitbook section',
-    () => {},
-    async (argv) => {
-      await scrapeGitbookSectionCommand(argv);
-    }
-  )
-  .command(
-    'readme-section [url]',
-    'Scrapes the ReadMe section',
-    () => {},
-    async (argv) => {
-      await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
-    }
-  )
-  .command(
-    'intercom-section [url]',
-    'Scrapes the Intercom section',
-    () => {},
-    async (argv) => {
-      await scrapeSectionAxiosWrapper(argv, scrapeIntercomSection);
+    (yargs) =>
+      yargs
+        .positional('url', { type: 'string', demandOption: true })
+        .check(checkUrl)
+        .option('overwrite', { alias: 'O', type: 'boolean', default: false })
+        .option('tool', { alias: 't', choices: frameworks })
+        .option('docusaurusVersion', {
+          alias: 'd',
+          type: 'string',
+          choices: ['1', '2', '3'] as const,
+        })
+        .check(checkVersion),
+    async ({ url, overwrite, tool, docusaurusVersion }) => {
+      const frameworkHint: FrameworkHint = {
+        framework: tool,
+        version: docusaurusVersion ?? '3',
+      };
+
+      await scrapeSectionAutomatically(url, overwrite, frameworkHint);
     }
   )
   .command(
     'openapi-file <openapiFilename>',
     'Creates MDX files from an OpenAPI spec',
-    (yargs) => {
-      return yargs
+    (yargs) =>
+      yargs
         .positional('openapiFilename', {
           describe: 'The filename of the OpenAPI spec',
           type: 'string',
+          demandOption: true,
         })
         .option('writeFiles', {
           describe: 'Whether or not to write the frontmatter files',
@@ -101,9 +77,7 @@ await yargs(hideBin(process.argv))
           describe: 'The folder in which to write any created frontmatter files',
           type: 'string',
           alias: 'o',
-        })
-        .demandOption('openapiFilename');
-    },
+        }),
     async (argv) => {
       try {
         const { nav } = await generateOpenApiPages(

package/src/scraping/detectFramework.ts

@@ -1,14 +1,19 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
-export enum Frameworks {
-  DOCUSAURUS = 'DOCUSAURUS',
-  GITBOOK = 'GITBOOK',
-  README = 'README',
-  INTERCOM = 'INTERCOM',
-}
+export const frameworks = ['docusaurus', 'gitbook', 'readme', 'intercom'] as const;
+export type Framework = (typeof frameworks)[number];
+
+export type FrameworkHint =
+  | {
+      framework: 'docusaurus';
+      version: '1' | '2' | '3';
+    }
+  | {
+      framework: 'gitbook' | 'readme' | 'intercom' | undefined;
+    };
 
-export function detectFramework(html) {
-  const $: cheerio.Root = cheerio.load(html);
+export function detectFramework(html: string): FrameworkHint {
+  const $ = cheerio.load(html);
   const docusaurusMeta = $('meta[name="generator"]');
 
   if (
@@ -19,31 +24,31 @@ export function detectFramework(html) {
   ) {
     const metaAttrString = docusaurusMeta.attr('content') as string;
     if (metaAttrString.includes('v3')) {
-      return { framework: Frameworks.DOCUSAURUS, version: '3' };
+      return { framework: 'docusaurus', version: '3' };
     }
     if (metaAttrString.includes('v2')) {
-      return { framework: Frameworks.DOCUSAURUS, version: '2' };
+      return { framework: 'docusaurus', version: '2' };
     } else if (metaAttrString.includes('v1')) {
       console.warn(
         'WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.'
       );
-      return { framework: Frameworks.DOCUSAURUS, version: '1' };
+      return { framework: 'docusaurus', version: '1' };
     }
   }
 
   const isGitBook = $('.gitbook-root').length > 0;
   if (isGitBook) {
-    return { framework: Frameworks.GITBOOK };
+    return { framework: 'gitbook' };
   }
 
   const isReadMe = $('meta[name="readme-deploy"]').length > 0;
   if (isReadMe) {
-    return { framework: Frameworks.README };
+    return { framework: 'readme' };
   }
 
   const isIntercom = $("meta[name='intercom:trackingEvent']").length > 0;
   if (isIntercom) {
-    return { framework: Frameworks.INTERCOM };
+    return { framework: 'intercom' };
   }
 
   return { framework: undefined };

package/src/scraping/downloadAllImages.ts

@@ -1,3 +1,4 @@
+import { Cheerio, CheerioAPI, Element } from 'cheerio';
 import path from 'path';
 
 import downloadImage, {
@@ -6,14 +7,13 @@ import downloadImage, {
   removeMetadataFromImageSrc,
 } from '../downloadImage.js';
 
-// To Do: Use CheerioElement instead of any when we bump the cheerio version
 export default async function downloadAllImages(
-  $: any,
-  content: any,
+  $: CheerioAPI,
+  content: Cheerio<Element>,
   origin: string,
   baseDir: string,
   overwrite: boolean,
-  modifyFileName?: any
+  modifyFileName?: (fileName: string) => string
 ) {
   if (!baseDir) {
     console.debug('Skipping image downloading');
@@ -23,17 +23,17 @@ export default async function downloadAllImages(
   // We remove duplicates because some frameworks duplicate img tags
   // to show the image larger when clicked on.
   const imageSrcs = [
-    ...new Set<string>(
+    ...new Set(
       content
         .find('img[src]')
-        .map((i, image) => $(image).attr('src'))
+        .map((_, image) => $(image).attr('src'))
         .toArray()
     ),
   ];
 
   // Wait to all images to download before continuing
   const origToNewArray = await Promise.all(
-    imageSrcs.map(async (imageSrc: string) => {
+    imageSrcs.map(async (imageSrc) => {
       if (!imageSrc || !isValidImageSrc(imageSrc)) {
         return {};
       }

package/src/scraping/downloadLogoImage.ts

@@ -7,7 +7,6 @@ import downloadImage, {
 } from '../downloadImage.js';
 import { getFileExtension } from '../util.js';
 
-// To Do: Use CheerioElement instead of any when we bump the cheerio version
 export default async function downloadLogoImage(
   imageSrc: string | undefined,
   imageBaseDir: string,

package/src/scraping/replaceImagePaths.ts

@@ -1,12 +1,8 @@
 export default function replaceImagePaths(
-  origToWritePath: object,
+  origToWritePath: Record<string, string>,
   cliDir: string,
   markdown: string
 ) {
-  if (origToWritePath == null) {
-    return markdown;
-  }
-
   // Change image paths to use the downloaded locations
   for (const [origHref, writePath] of Object.entries(origToWritePath)) {
     // Use relative paths within the folder we are in

package/src/scraping/scrapePageCommands.ts

@@ -1,65 +1,68 @@
 import axios from 'axios';
-import { ArgumentsCamelCase } from 'yargs';
 
 import { getHtmlWithPuppeteer } from '../browser.js';
-import { getHrefFromArgs } from '../util.js';
-import { detectFramework, Frameworks } from './detectFramework.js';
+import { detectFramework, Framework, FrameworkHint, frameworks } from './detectFramework.js';
 import { scrapePage, ScrapePageFn } from './scrapePage.js';
 import { scrapeIntercomPage } from './site-scrapers/Intercom/scrapeIntercomPage.js';
 import { scrapeDocusaurusPage } from './site-scrapers/scrapeDocusaurusPage.js';
 import { scrapeGitBookPage } from './site-scrapers/scrapeGitBookPage.js';
 import { scrapeReadMePage } from './site-scrapers/scrapeReadMePage.js';
 
-function validateFramework(framework) {
+function validateFramework(framework: Framework | undefined) {
   if (!framework) {
-    console.log('Could not detect the framework automatically. Please use one of:');
-    console.log('scrape-page-docusaurus');
-    console.log('scrape-page-gitbook');
-    console.log('scrape-page-readme');
-    console.log('scrape-page-intercom');
+    console.log(
+      `Could not detect the framework automatically. Please use the -t flag to specify one of: ${frameworks.join(
+        ', '
+      )}`
+    );
     return process.exit(1);
   }
 }
 
 export async function scrapePageWrapper(
-  argv: ArgumentsCamelCase,
+  url: string,
+  overwrite: boolean,
   scrapeFunc: ScrapePageFn,
   options?: { version?: string; puppeteer?: boolean }
 ) {
-  const href = getHrefFromArgs(argv);
   let html: string;
   if (options?.puppeteer) {
-    html = await getHtmlWithPuppeteer(href);
+    html = await getHtmlWithPuppeteer(url);
   } else {
-    const res = await axios.get(href);
+    const res = await axios.get(url);
     html = res.data;
   }
-  await scrapePage(scrapeFunc, href, html, !!argv.overwrite, options?.version);
+  await scrapePage(scrapeFunc, url, html, overwrite, options?.version);
   process.exit(0);
 }
 
-export async function scrapePageAutomatically(argv: any) {
-  const href = getHrefFromArgs(argv);
-  const res = await axios.get(href);
+export async function scrapePageAutomatically(
+  url: string,
+  overwrite: boolean,
+  frameworkHint: FrameworkHint
+) {
+  const res = await axios.get(url);
   const html = res.data;
-  const { framework, version } = detectFramework(html);
+  frameworkHint = frameworkHint.framework ? frameworkHint : detectFramework(html);
 
-  validateFramework(framework);
+  validateFramework(frameworkHint.framework);
 
-  console.log('Detected framework: ' + framework);
+  console.log('Detected framework: ' + frameworkHint.framework);
 
-  switch (framework) {
-    case Frameworks.DOCUSAURUS:
-      await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
+  switch (frameworkHint.framework) {
+    case 'docusaurus':
+      await scrapePageWrapper(url, overwrite, scrapeDocusaurusPage, {
+        version: frameworkHint.version,
+      });
       break;
-    case Frameworks.GITBOOK:
-      await scrapePageWrapper(argv, scrapeGitBookPage, { puppeteer: true });
+    case 'gitbook':
+      await scrapePageWrapper(url, overwrite, scrapeGitBookPage, { puppeteer: true });
       break;
-    case Frameworks.README:
-      await scrapePageWrapper(argv, scrapeReadMePage);
+    case 'readme':
+      await scrapePageWrapper(url, overwrite, scrapeReadMePage);
       break;
-    case Frameworks.INTERCOM:
-      await scrapePageWrapper(argv, scrapeIntercomPage);
+    case 'intercom':
+      await scrapePageWrapper(url, overwrite, scrapeIntercomPage);
       break;
   }
 }