@mintlify/scraping 3.0.140 → 3.0.142

package/src/scraping/scrapeSectionCommands.ts

@@ -1,9 +1,9 @@
 import axios from 'axios';
-import { ArgumentsCamelCase } from 'yargs';
+import { Page } from 'puppeteer';
 
 import { startBrowser } from '../browser.js';
-import { getHrefFromArgs, getOrigin } from '../util.js';
-import { detectFramework, Frameworks } from './detectFramework.js';
+import { getOrigin } from '../util.js';
+import { detectFramework, Framework, FrameworkHint } from './detectFramework.js';
 import { ScrapeSectionFn, scrapeSection } from './scrapeSection.js';
 import { scrapeIntercomSection } from './site-scrapers/Intercom/scrapeIntercomSection.js';
 import openNestedDocusaurusMenus from './site-scrapers/openNestedDocusaurusMenus.js';
@@ -13,78 +13,82 @@ import { scrapeGitBookSection } from './site-scrapers/scrapeGitBookSection.js';
 import { scrapeReadMeSection } from './site-scrapers/scrapeReadMeSection.js';
 
 export async function scrapeSectionAxiosWrapper(
-  argv: ArgumentsCamelCase,
+  url: string,
+  overwrite: boolean,
   scrapeFunc: ScrapeSectionFn
 ) {
-  const href = getHrefFromArgs(argv);
-  const res = await axios.get(href);
+  const res = await axios.get(url);
   const html = res.data;
-  await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, undefined);
+  await scrapeSection(scrapeFunc, html, getOrigin(url), overwrite, undefined);
   process.exit(0);
 }
 
 export async function scrapeDocusaurusSectionCommand(
-  argv: any,
+  url: string,
+  overwrite: boolean,
   version: string | undefined // "1" | "2" | "3"
 ) {
   await scrapeSectionOpeningAllNested(
-    argv,
+    url,
+    overwrite,
     openNestedDocusaurusMenus,
     scrapeDocusaurusSection,
     version
   );
 }
 
-export async function scrapeGitbookSectionCommand(argv: any) {
-  await scrapeSectionOpeningAllNested(argv, openNestedGitbookMenus, scrapeGitBookSection);
+export async function scrapeGitbookSectionCommand(url: string, overwrite: boolean) {
+  await scrapeSectionOpeningAllNested(url, overwrite, openNestedGitbookMenus, scrapeGitBookSection);
 }
 
 async function scrapeSectionOpeningAllNested(
-  argv: any,
-  openLinks: any,
+  url: string,
+  overwrite: boolean,
+  openLinks: (page: Page) => Promise<string>,
   scrapeFunc: ScrapeSectionFn,
   version?: string
 ) {
-  const href = getHrefFromArgs(argv);
-
   const browser = await startBrowser();
   const page = await browser.newPage();
-  await page.goto(href, {
+  await page.goto(url, {
     waitUntil: 'networkidle2',
   });
 
   const html = await openLinks(page);
-  browser.close();
-  await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, version);
+  void browser.close();
+  await scrapeSection(scrapeFunc, html, getOrigin(url), overwrite, version);
   process.exit(0);
 }
 
-export async function scrapeSectionAutomatically(argv: any) {
-  const href = getHrefFromArgs(argv);
-  const res = await axios.get(href);
+export async function scrapeSectionAutomatically(
+  url: string,
+  overwrite: boolean,
+  frameworkHint: FrameworkHint
+) {
+  const res = await axios.get(url);
   const html = res.data;
-  const { framework, version } = detectFramework(html);
+  frameworkHint = frameworkHint.framework ? frameworkHint : detectFramework(html);
 
-  validateFramework(framework);
-  console.log('Detected framework: ' + framework);
+  validateFramework(frameworkHint.framework);
+  console.log('Detected framework: ' + frameworkHint.framework);
 
-  switch (framework) {
-    case Frameworks.DOCUSAURUS:
-      await scrapeDocusaurusSectionCommand(argv, version);
+  switch (frameworkHint.framework) {
+    case 'docusaurus':
+      await scrapeDocusaurusSectionCommand(url, overwrite, frameworkHint.version);
       break;
-    case Frameworks.GITBOOK:
-      await scrapeGitbookSectionCommand(argv);
+    case 'gitbook':
+      await scrapeGitbookSectionCommand(url, overwrite);
       break;
-    case Frameworks.README:
-      await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
+    case 'readme':
+      await scrapeSectionAxiosWrapper(url, overwrite, scrapeReadMeSection);
       break;
-    case Frameworks.INTERCOM:
-      await scrapeSectionAxiosWrapper(argv, scrapeIntercomSection);
+    case 'intercom':
+      await scrapeSectionAxiosWrapper(url, overwrite, scrapeIntercomSection);
       break;
   }
 }
 
-function validateFramework(framework: Frameworks | undefined) {
+function validateFramework(framework: Framework | undefined) {
   if (!framework) {
     console.log(
       'Could not detect the framework automatically. We only support Docusaurus (V2 and V3), GitBook, and ReadMe.'

package/src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import { NodeHtmlMarkdown } from 'node-html-markdown';
 
 import downloadAllImages from '../../downloadAllImages.js';

package/src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts

@@ -1,6 +1,6 @@
 import { Navigation, NavigationEntry } from '@mintlify/models';
 import axios from 'axios';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 import downloadLogoImage from '../../downloadLogoImage.js';
 import { scrapeGettingFileNameFromUrl } from '../../scrapeGettingFileNameFromUrl.js';
@@ -17,7 +17,7 @@ export async function scrapeIntercomSection(
   let $ = cheerio.load(html);
 
   const logoSrc = $('.header__logo img').first().attr('src');
-  downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
+  void downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
 
   const collectionsLink = $('.section .g__space a');
   const collectionsMap = collectionsLink.toArray().map(async (s: cheerio.Element) => {

package/src/scraping/site-scrapers/alternateGroupTitle.ts

@@ -1,8 +1,11 @@
-export default function alternateGroupTitle(firstLink: cheerio.Cheerio, pages) {
+import { NavigationEntry } from '@mintlify/models';
+import { Cheerio, Element } from 'cheerio';
+
+export default function alternateGroupTitle(firstLink: Cheerio<Element>, pages: NavigationEntry[]) {
   // Only assign titles to nested navigation menus outside a section.
   // Others should not have a title so we can merge them into one section.
   if (pages.length > 0) {
-    return firstLink?.text();
+    return firstLink.text();
   }
   return '';
 }

package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts

@@ -1,9 +1,11 @@
+import { Cheerio, CheerioAPI, Element } from 'cheerio';
+
 import alternateGroupTitle from '../alternateGroupTitle.js';
 import getLinksRecursively from './getLinksRecursively.js';
 
 export function getDocusaurusLinksPerGroup(
-  navigationSections: any,
-  $: any,
+  navigationSections: Cheerio<Element>,
+  $: CheerioAPI,
   version: string | undefined
 ) {
   if (version === '3' || version === '2') {
@@ -12,8 +14,8 @@ export function getDocusaurusLinksPerGroup(
   return [];
 }
 
-function getDocusaurusLinksPerGroupLoop(navigationSections: any, $: any) {
-  return navigationSections.toArray().map((s: string) => {
+function getDocusaurusLinksPerGroupLoop(navigationSections: Cheerio<Element>, $: CheerioAPI) {
+  return navigationSections.toArray().map((s) => {
     const section = $(s);
 
     // Links without a group
@@ -21,7 +23,7 @@ function getDocusaurusLinksPerGroupLoop(navigationSections: any, $: any) {
       const linkHref = section.find('a[href]').first().attr('href');
       return {
         group: '',
-        pages: [linkHref],
+        pages: linkHref !== undefined ? [linkHref] : [],
       };
     }
 

package/src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts

@@ -1,11 +1,13 @@
-// Used by Docusaurus and ReadMe section scrapers
-export default function getLinksRecursively(linkSections: any, $: any) {
-  if (linkSections == null || linkSections.length === 0) {
-    return [];
-  }
+import { NavigationEntry } from '@mintlify/models';
+import { Cheerio, CheerioAPI, Element } from 'cheerio';
 
+// Used by Docusaurus and ReadMe section scrapers
+export default function getLinksRecursively(
+  linkSections: Cheerio<Element>,
+  $: CheerioAPI
+): NavigationEntry[] {
   return linkSections
-    .map((i, s) => {
+    .map((_, s) => {
       const subsection = $(s);
       let link = subsection.children().first();
 

package/src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts

@@ -1,9 +1,11 @@
-// Used by GitBook section scraper
-export default function getLinksRecursivelyGitBook(linkSections: any, $: any) {
-  if (linkSections == null || linkSections.length === 0) {
-    return [];
-  }
+import { NavigationEntry } from '@mintlify/models';
+import { Cheerio, CheerioAPI, Element } from 'cheerio';
 
+// Used by GitBook section scraper
+export default function getLinksRecursivelyGitBook(
+  linkSections: Cheerio<Element>,
+  $: CheerioAPI
+): NavigationEntry[] {
   return linkSections
     .map((_, s) => {
       let subsection = $(s);

package/src/scraping/site-scrapers/openNestedDocusaurusMenus.ts

@@ -14,8 +14,8 @@ export default async function openNestedDocusaurusMenus(page: Page) {
         );
 
         const linksFound: string[] = [];
-        collapsible.forEach(async (collapsibleItem: HTMLElement) => {
-          const href = collapsibleItem?.getAttribute('href');
+        collapsible.forEach((collapsibleItem) => {
+          const href = collapsibleItem.getAttribute('href');
 
           // Should never occur but we keep it as a fail-safe
           if (href?.startsWith('https://') || href?.startsWith('http://')) {
@@ -24,7 +24,7 @@ export default async function openNestedDocusaurusMenus(page: Page) {
 
           // Click any links we haven't seen before
           if (href && !encounteredHref.includes(href)) {
-            collapsibleItem?.click();
+            collapsibleItem.click();
           }
 
           if (href) {

package/src/scraping/site-scrapers/openNestedGitbookMenus.ts

@@ -7,10 +7,10 @@ export default async function openNestedGitbookMenus(page: Page) {
   while (clickedAny) {
     clickedAny = await page.evaluate(() => {
       // Right pointing arrow. Only closed menus have this icon
-      const icons: HTMLElement[] = Array.from(document.querySelectorAll('path[d="M9 18l6-6-6-6"]'));
+      const icons = Array.from(document.querySelectorAll('path[d="M9 18l6-6-6-6"]'));
 
-      icons.forEach(async (icon: HTMLElement) => {
-        const toClick = icon?.parentElement?.parentElement;
+      icons.forEach((icon) => {
+        const toClick = icon.parentElement?.parentElement;
         if (toClick) {
           toClick.click();
         }

package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import { NodeHtmlMarkdown } from 'node-html-markdown';
 
 import downloadAllImages from '../downloadAllImages.js';

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -1,5 +1,5 @@
 import { Navigation, NavigationEntry } from '@mintlify/models';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
 import downloadLogoImage from '../downloadLogoImage.js';
@@ -19,7 +19,7 @@ export async function scrapeDocusaurusSection(
 
   // Download the logo
   const logoSrc = $('.navbar__logo img').attr('src');
-  downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
+  void downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
 
   // Get all the navigation sections
   const navigationSections = $('.theme-doc-sidebar-menu').first().children();

package/src/scraping/site-scrapers/scrapeGitBookPage.ts

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import { NodeHtmlMarkdown } from 'node-html-markdown';
 
 import downloadAllImages from '../downloadAllImages.js';

package/src/scraping/site-scrapers/scrapeGitBookSection.ts

@@ -1,5 +1,5 @@
 import { Navigation, NavigationEntry } from '@mintlify/models';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
 import downloadLogoImage from '../downloadLogoImage.js';
@@ -25,7 +25,7 @@ export async function scrapeGitBookSection(
   // Get all the navigation sections
   // Some variants of the GitBook UI show the logo and search base in the side navigation bar,
   // but the navigation sections are always the last value.
-  const navigationSections: cheerio.Cheerio = $(
+  const navigationSections = $(
     'div[data-testid="page.desktopTableOfContents"] > nav > div:first-child'
   )
     .children()
@@ -45,7 +45,7 @@ export async function scrapeGitBookSection(
       const firstLink = section.children().eq(0);
       const firstHref = firstLink.attr('href');
 
-      const linkSections: cheerio.Cheerio = section.children().eq(1).children();
+      const linkSections = section.children().eq(1).children();
       const pages = getLinksRecursivelyGitBook(linkSections, $);
 
       return {

package/src/scraping/site-scrapers/scrapeReadMePage.ts

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import { NodeHtmlMarkdown } from 'node-html-markdown';
 
 import downloadAllImages from '../downloadAllImages.js';

package/src/scraping/site-scrapers/scrapeReadMeSection.ts

@@ -1,5 +1,5 @@
 import { Navigation, NavigationEntry } from '@mintlify/models';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 import downloadLogoImage from '../downloadLogoImage.js';
 import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
@@ -25,7 +25,7 @@ export async function scrapeReadMeSection(
   // responsiveness but they all have the same links.
   const navigationSections = $('.rm-Sidebar').first().find('.rm-Sidebar-section');
 
-  const groupsConfig: Navigation = navigationSections.toArray().map((s: cheerio.Element) => {
+  const groupsConfig: Navigation = navigationSections.toArray().map((s) => {
     const section = $(s);
     const sectionTitle = section.find('h3').first().text();
 
@@ -36,7 +36,7 @@ export async function scrapeReadMeSection(
     // -- Second Page -> /second-page
     const linkSections = section.find('.rm-Sidebar-list').first().children();
     const pages = getLinksRecursively(linkSections, $).filter(
-      (value: string, index: number, self: any) => self.indexOf(value) === index
+      (value, index, array) => array.indexOf(value) === index
     );
 
     // Follows the same structure as mint.json

package/src/util.ts

@@ -1,10 +1,8 @@
 import { NavigationEntry } from '@mintlify/models';
-import { mkdirSync, writeFileSync } from 'fs';
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
 import Ora, { Ora as OraType } from 'ora';
 import path from 'path';
 
-import stopIfInvalidLink from './validation/stopIfInvalidLink.js';
-
 export const MintConfig = (
   name: string,
   color: string,
@@ -93,34 +91,20 @@ export const createPage = (
   // Create the folders needed if they're missing
   mkdirSync(rootDir, { recursive: true });
 
-  // Write the page to memory
-  if (overwrite) {
+  if (!overwrite && existsSync(writePath)) {
+    console.log(`❌ Skipping existing file ${writePath}`);
+    return;
+  }
+
+  // Write the page to disk
+  try {
     writeFileSync(writePath, Page(title, description, markdown));
     console.log('✏️ - ' + writePath);
-  } else {
-    try {
-      writeFileSync(writePath, Page(title, description, markdown), {
-        flag: 'wx',
-      });
-      console.log('✏️ - ' + writePath);
-    } catch (e) {
-      // We do a try-catch instead of an if-statement to avoid a race condition
-      // of the file being created after we started writing.
-      if ((e as { code: string })?.code === 'EEXIST') {
-        console.log(`❌ Skipping existing file ${writePath}`);
-      } else {
-        console.error(e);
-      }
-    }
+  } catch (e) {
+    console.error(e);
   }
 };
 
-export function getHrefFromArgs(argv: any) {
-  const href = argv.url;
-  stopIfInvalidLink(href);
-  return href;
-}
-
 export const buildLogger = (startText = ''): OraType => {
   const logger = Ora().start(startText);
   return logger;

package/tsconfig.json

@@ -5,7 +5,6 @@
     "sourceMap": true,
     "outDir": "bin",
     "declaration": true,
-    "noImplicitAny": false,
     "types": ["vitest/globals"]
   },
   "include": ["**/*.ts"],

package/bin/validation/isValidLink.d.ts

@@ -1 +0,0 @@
-export default function isValidLink(href: string): boolean;

package/bin/validation/isValidLink.js

@@ -1,11 +0,0 @@
-export default function isValidLink(href) {
-    // This checks the link is written correctly, not that the page exists.
-    try {
-        new URL(href);
-        return true;
-    }
-    catch (_) {
-        return false;
-    }
-}
-//# sourceMappingURL=isValidLink.js.map

package/bin/validation/isValidLink.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"isValidLink.js","sourceRoot":"","sources":["../../src/validation/isValidLink.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,WAAW,CAAC,IAAY;IAC9C,uEAAuE;IACvE,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/bin/validation/stopIfInvalidLink.d.ts

@@ -1 +0,0 @@
-export default function stopIfInvalidLink(href: string): void;

package/bin/validation/stopIfInvalidLink.js

@@ -1,9 +0,0 @@
-import isValidLink from './isValidLink.js';
-export default function stopIfInvalidLink(href) {
-    if (!isValidLink(href)) {
-        console.log('Invalid link: ' + href);
-        console.log('Make sure the link starts with http:// or https://');
-        process.exit(1);
-    }
-}
-//# sourceMappingURL=stopIfInvalidLink.js.map

package/bin/validation/stopIfInvalidLink.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"stopIfInvalidLink.js","sourceRoot":"","sources":["../../src/validation/stopIfInvalidLink.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C,MAAM,CAAC,OAAO,UAAU,iBAAiB,CAAC,IAAY;IACpD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}

package/src/validation/isValidLink.ts

@@ -1,9 +0,0 @@
-export default function isValidLink(href: string) {
-  // This checks the link is written correctly, not that the page exists.
-  try {
-    new URL(href);
-    return true;
-  } catch (_) {
-    return false;
-  }
-}

package/src/validation/stopIfInvalidLink.ts

@@ -1,9 +0,0 @@
-import isValidLink from './isValidLink.js';
-
-export default function stopIfInvalidLink(href: string) {
-  if (!isValidLink(href)) {
-    console.log('Invalid link: ' + href);
-    console.log('Make sure the link starts with http:// or https://');
-    process.exit(1);
-  }
-}