@mintlify/scraping 3.0.187 → 3.0.189

package/src/scraping/scrapePage.ts

@@ -1,40 +0,0 @@
-import path from 'path';
-
-import { createPage, getOrigin } from '../util.js';
-
-type ScrapePageResult = {
-  title: string;
-  description?: string;
-  markdown?: string;
-};
-
-export type ScrapePageFn = (
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined
-) => Promise<ScrapePageResult>;
-
-export async function scrapePage(
-  scrapeFunc: ScrapePageFn,
-  href: string,
-  html: string,
-  overwrite: boolean,
-  version: string | undefined
-) {
-  const origin = getOrigin(href);
-  const cwd = process.cwd();
-  const imageBaseDir = path.join(cwd, 'images');
-
-  const { title, description, markdown } = await scrapeFunc(
-    html,
-    origin,
-    cwd,
-    imageBaseDir,
-    overwrite,
-    version
-  );
-  createPage(title, description, markdown, overwrite, process.cwd());
-}

package/src/scraping/scrapePageCommands.ts

@@ -1,68 +0,0 @@
-import axios from 'axios';
-
-import { getHtmlWithPuppeteer } from '../browser.js';
-import { detectFramework, Framework, FrameworkHint, frameworks } from './detectFramework.js';
-import { scrapePage, ScrapePageFn } from './scrapePage.js';
-import { scrapeIntercomPage } from './site-scrapers/Intercom/scrapeIntercomPage.js';
-import { scrapeDocusaurusPage } from './site-scrapers/scrapeDocusaurusPage.js';
-import { scrapeGitBookPage } from './site-scrapers/scrapeGitBookPage.js';
-import { scrapeReadMePage } from './site-scrapers/scrapeReadMePage.js';
-
-function validateFramework(framework: Framework | undefined) {
-  if (!framework) {
-    console.log(
-      `Could not detect the framework automatically. Please use the -t flag to specify one of: ${frameworks.join(
-        ', '
-      )}`
-    );
-    return process.exit(1);
-  }
-}
-
-export async function scrapePageWrapper(
-  url: string,
-  overwrite: boolean,
-  scrapeFunc: ScrapePageFn,
-  options?: { version?: string; puppeteer?: boolean }
-) {
-  let html: string;
-  if (options?.puppeteer) {
-    html = await getHtmlWithPuppeteer(url);
-  } else {
-    const res = await axios.get(url);
-    html = res.data;
-  }
-  await scrapePage(scrapeFunc, url, html, overwrite, options?.version);
-  process.exit(0);
-}
-
-export async function scrapePageAutomatically(
-  url: string,
-  overwrite: boolean,
-  frameworkHint: FrameworkHint
-) {
-  const res = await axios.get(url);
-  const html = res.data;
-  frameworkHint = frameworkHint.framework ? frameworkHint : detectFramework(html);
-
-  validateFramework(frameworkHint.framework);
-
-  console.log('Detected framework: ' + frameworkHint.framework);
-
-  switch (frameworkHint.framework) {
-    case 'docusaurus':
-      await scrapePageWrapper(url, overwrite, scrapeDocusaurusPage, {
-        version: frameworkHint.version,
-      });
-      break;
-    case 'gitbook':
-      await scrapePageWrapper(url, overwrite, scrapeGitBookPage, { puppeteer: true });
-      break;
-    case 'readme':
-      await scrapePageWrapper(url, overwrite, scrapeReadMePage);
-      break;
-    case 'intercom':
-      await scrapePageWrapper(url, overwrite, scrapeIntercomPage);
-      break;
-  }
-}

package/src/scraping/scrapeSection.ts

@@ -1,30 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-import path from 'path';
-
-import { objToReadableString } from '../util.js';
-
-export type ScrapeSectionFn = (
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined
-) => Promise<NavigationEntry[]>;
-
-export async function scrapeSection(
-  scrapeFunc: ScrapeSectionFn,
-  html: string,
-  origin: string,
-  overwrite: boolean,
-  version: string | undefined
-) {
-  console.log(`Started scraping${overwrite ? ', overwrite mode is on' : ''}...`);
-  const cwd = process.cwd();
-  const imageBaseDir = path.join(cwd, 'images');
-
-  const groupsConfig = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
-  console.log('Finished scraping.');
-  console.log('Add the following to your navigation in mint.json:');
-  console.log(objToReadableString(groupsConfig));
-}

package/src/scraping/scrapeSectionCommands.ts

@@ -1,98 +0,0 @@
-import axios from 'axios';
-import { Page } from 'puppeteer';
-
-import { startBrowser } from '../browser.js';
-import { getOrigin } from '../util.js';
-import { detectFramework, Framework, FrameworkHint } from './detectFramework.js';
-import { ScrapeSectionFn, scrapeSection } from './scrapeSection.js';
-import { scrapeIntercomSection } from './site-scrapers/Intercom/scrapeIntercomSection.js';
-import openNestedDocusaurusMenus from './site-scrapers/openNestedDocusaurusMenus.js';
-import openNestedGitbookMenus from './site-scrapers/openNestedGitbookMenus.js';
-import { scrapeDocusaurusSection } from './site-scrapers/scrapeDocusaurusSection.js';
-import { scrapeGitBookSection } from './site-scrapers/scrapeGitBookSection.js';
-import { scrapeReadMeSection } from './site-scrapers/scrapeReadMeSection.js';
-
-export async function scrapeSectionAxiosWrapper(
-  url: string,
-  overwrite: boolean,
-  scrapeFunc: ScrapeSectionFn
-) {
-  const res = await axios.get(url);
-  const html = res.data;
-  await scrapeSection(scrapeFunc, html, getOrigin(url), overwrite, undefined);
-  process.exit(0);
-}
-
-export async function scrapeDocusaurusSectionCommand(
-  url: string,
-  overwrite: boolean,
-  version: string | undefined // "1" | "2" | "3"
-) {
-  await scrapeSectionOpeningAllNested(
-    url,
-    overwrite,
-    openNestedDocusaurusMenus,
-    scrapeDocusaurusSection,
-    version
-  );
-}
-
-export async function scrapeGitbookSectionCommand(url: string, overwrite: boolean) {
-  await scrapeSectionOpeningAllNested(url, overwrite, openNestedGitbookMenus, scrapeGitBookSection);
-}
-
-async function scrapeSectionOpeningAllNested(
-  url: string,
-  overwrite: boolean,
-  openLinks: (page: Page) => Promise<string>,
-  scrapeFunc: ScrapeSectionFn,
-  version?: string
-) {
-  const browser = await startBrowser();
-  const page = await browser.newPage();
-  await page.goto(url, {
-    waitUntil: 'networkidle2',
-  });
-
-  const html = await openLinks(page);
-  void browser.close();
-  await scrapeSection(scrapeFunc, html, getOrigin(url), overwrite, version);
-  process.exit(0);
-}
-
-export async function scrapeSectionAutomatically(
-  url: string,
-  overwrite: boolean,
-  frameworkHint: FrameworkHint
-) {
-  const res = await axios.get(url);
-  const html = res.data;
-  frameworkHint = frameworkHint.framework ? frameworkHint : detectFramework(html);
-
-  validateFramework(frameworkHint.framework);
-  console.log('Detected framework: ' + frameworkHint.framework);
-
-  switch (frameworkHint.framework) {
-    case 'docusaurus':
-      await scrapeDocusaurusSectionCommand(url, overwrite, frameworkHint.version);
-      break;
-    case 'gitbook':
-      await scrapeGitbookSectionCommand(url, overwrite);
-      break;
-    case 'readme':
-      await scrapeSectionAxiosWrapper(url, overwrite, scrapeReadMeSection);
-      break;
-    case 'intercom':
-      await scrapeSectionAxiosWrapper(url, overwrite, scrapeIntercomSection);
-      break;
-  }
-}
-
-function validateFramework(framework: Framework | undefined) {
-  if (!framework) {
-    console.log(
-      'Could not detect the framework automatically. We only support Docusaurus (V2 and V3), GitBook, and ReadMe.'
-    );
-    process.exit();
-  }
-}

package/src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts

@@ -1,52 +0,0 @@
-import * as cheerio from 'cheerio';
-import { NodeHtmlMarkdown } from 'node-html-markdown';
-
-import downloadAllImages from '../../downloadAllImages.js';
-import replaceImagePaths from '../../replaceImagePaths.js';
-
-export async function scrapeIntercomPage(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  _: string | undefined // version
-) {
-  const $ = cheerio.load(html);
-
-  const titleComponent = $('.t__h1').first();
-  const title = titleComponent.text().trim();
-  const description = $('.article__desc', titleComponent.parent()).text().trim();
-
-  const content = $('article').first();
-  const contentHtml = $.html(content);
-
-  const origToWritePath = await downloadAllImages(
-    $,
-    content,
-    origin,
-    imageBaseDir,
-    overwrite,
-    undefined
-  );
-
-  const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
-  let markdown = nhm.translate(contentHtml);
-
-  // Keep headers on one line
-  markdown = markdown.replace(/# \n\n/g, '# ');
-
-  // Remove unnecessary nonwidth blank space characters
-  markdown = markdown.replace(/\u200b/g, '');
-
-  // Reduce unnecessary blank lines
-  markdown = markdown.replace(/\n\n\n/g, '\n\n');
-
-  // Mintlify doesn't support bolded headers, remove the asterisks
-  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
-  if (origToWritePath) {
-    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
-  }
-
-  return { title, description, markdown };
-}

package/src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts

@@ -1,54 +0,0 @@
-import { Navigation, NavigationEntry } from '@mintlify/models';
-import axios from 'axios';
-import * as cheerio from 'cheerio';
-
-import downloadLogoImage from '../../downloadLogoImage.js';
-import { scrapeGettingFileNameFromUrl } from '../../scrapeGettingFileNameFromUrl.js';
-import { scrapeIntercomPage } from './scrapeIntercomPage.js';
-
-export async function scrapeIntercomSection(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined
-): Promise<NavigationEntry[]> {
-  let $ = cheerio.load(html);
-
-  const logoSrc = $('.header__logo img').first().attr('src');
-  void downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
-
-  const collectionsLink = $('.section .g__space a');
-  const collectionsMap = collectionsLink.toArray().map(async (s: cheerio.Element) => {
-    const href = $(s).attr('href');
-    const res = await axios.get(`${origin}${href}`);
-    const html = res.data;
-    $ = cheerio.load(html);
-    const sectionTitle = $('.collection h1').first().text().trim();
-    const sectionPages = $('.section .g__space a')
-      .toArray()
-      .map((s: cheerio.Element) => $(s).attr('href'))
-      .filter((page) => page !== undefined) as string[];
-    return {
-      group: sectionTitle,
-      pages: sectionPages,
-    };
-  });
-
-  const collections: Navigation = await Promise.all(collectionsMap);
-
-  return await Promise.all(
-    collections.map(async (entry: NavigationEntry) => {
-      return await scrapeGettingFileNameFromUrl(
-        entry,
-        cliDir,
-        origin,
-        overwrite,
-        scrapeIntercomPage,
-        false,
-        version
-      );
-    })
-  );
-}

package/src/scraping/site-scrapers/alternateGroupTitle.ts

@@ -1,11 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-import { Cheerio, Element } from 'cheerio';
-
-export default function alternateGroupTitle(firstLink: Cheerio<Element>, pages: NavigationEntry[]) {
-  // Only assign titles to nested navigation menus outside a section.
-  // Others should not have a title so we can merge them into one section.
-  if (pages.length > 0) {
-    return firstLink.text();
-  }
-  return '';
-}

package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts

@@ -1,45 +0,0 @@
-import { Cheerio, CheerioAPI, Element } from 'cheerio';
-
-import alternateGroupTitle from '../alternateGroupTitle.js';
-import getLinksRecursively from './getLinksRecursively.js';
-
-export function getDocusaurusLinksPerGroup(
-  navigationSections: Cheerio<Element>,
-  $: CheerioAPI,
-  version: string | undefined
-) {
-  if (version === '3' || version === '2') {
-    return getDocusaurusLinksPerGroupLoop(navigationSections, $);
-  }
-  return [];
-}
-
-function getDocusaurusLinksPerGroupLoop(navigationSections: Cheerio<Element>, $: CheerioAPI) {
-  return navigationSections.toArray().map((s) => {
-    const section = $(s);
-
-    // Links without a group
-    if (section.hasClass('theme-doc-sidebar-item-link') || section.hasClass('menu__link')) {
-      const linkHref = section.find('a[href]').first().attr('href');
-      return {
-        group: '',
-        pages: linkHref !== undefined ? [linkHref] : [],
-      };
-    }
-
-    const firstLink = !section.find('.menu__list-item-collapsible').first().find('a[href]').length
-      ? section.find('.menu__link--sublist').first().find('a[href]')
-      : section.find('.menu__list-item-collapsible').first().find('a[href]');
-
-    const sectionTitle = firstLink.text();
-    const firstHref = firstLink.attr('href');
-    const linkSections = section.children().eq(1).children();
-
-    const pages = getLinksRecursively(linkSections, $);
-
-    return {
-      group: sectionTitle || alternateGroupTitle(firstLink, pages),
-      pages: firstHref ? [firstHref, ...pages] : pages,
-    };
-  });
-}

package/src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts

@@ -1,47 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-import { Cheerio, CheerioAPI, Element } from 'cheerio';
-
-// Used by Docusaurus and ReadMe section scrapers
-export default function getLinksRecursively(
-  linkSections: Cheerio<Element>,
-  $: CheerioAPI
-): NavigationEntry[] {
-  return linkSections
-    .map((_, s) => {
-      const subsection = $(s);
-      let link = subsection.children().first();
-
-      if (!link.attr('href')) {
-        // Docusaurus nests the <a> inside a <div>
-        link = link.find('a[href]').first();
-      }
-      const linkHref = link.attr('href');
-
-      // Skip missing links. For example, GitBook uses
-      // empty divs are used for styling a line beside the nav.
-      // Skip external links until Mintlify supports them
-      if (
-        !linkHref ||
-        linkHref === '#' ||
-        linkHref.startsWith('https://') ||
-        linkHref.startsWith('http://')
-      ) {
-        return undefined;
-      }
-
-      const childLinks = subsection.children().eq(1).children();
-
-      if (childLinks.length > 0) {
-        // Put the section link in the list of pages.
-        // When we support the section itself being a link we should update this
-        return {
-          group: link.text(),
-          pages: [linkHref, ...getLinksRecursively(childLinks, $)],
-        };
-      }
-
-      return linkHref;
-    })
-    .toArray()
-    .filter(Boolean);
-}

package/src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts

@@ -1,44 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-import { Cheerio, CheerioAPI, Element } from 'cheerio';
-
-// Used by GitBook section scraper
-export default function getLinksRecursivelyGitBook(
-  linkSections: Cheerio<Element>,
-  $: CheerioAPI
-): NavigationEntry[] {
-  return linkSections
-    .map((_, s) => {
-      const subsection = $(s);
-      const sectionHeader = subsection.find('div').first();
-      const link = subsection.find('a').first();
-      const linkHref = link.attr('href');
-
-      // Skip missing links. For example, GitBook uses
-      // empty divs are used for styling a line beside the nav.
-      // Skip external links until Mintlify supports them
-      if (
-        !linkHref ||
-        linkHref === '#' ||
-        linkHref.startsWith('https://') ||
-        linkHref.startsWith('http://')
-      ) {
-        return undefined;
-      }
-
-      const childLinks = subsection.find('ul').first().children();
-      const title = link.text() ? link.text() : sectionHeader.text() ? sectionHeader.text() : '';
-
-      if (childLinks.length > 0) {
-        // Put the section link in the list of pages.
-        // When we support the section itself being a link we should update this
-        return {
-          group: title,
-          pages: [linkHref, ...getLinksRecursivelyGitBook(childLinks, $)],
-        };
-      }
-
-      return linkHref;
-    })
-    .toArray()
-    .filter(Boolean);
-}

package/src/scraping/site-scrapers/openNestedDocusaurusMenus.ts

@@ -1,42 +0,0 @@
-import { Page } from 'puppeteer';
-
-export default async function openNestedDocusaurusMenus(page: Page) {
-  let prevEncountered: string[] = [];
-  let encounteredHref = ['fake-href-to-make-loop-run-at-least-once'];
-
-  // Loop until we've encountered every link
-  while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
-    prevEncountered = encounteredHref;
-    encounteredHref = await page.evaluate(
-      (encounteredHref) => {
-        const collapsible: HTMLElement[] = Array.from(
-          document.querySelectorAll('.menu__link.menu__link--sublist')
-        );
-
-        const linksFound: string[] = [];
-        collapsible.forEach((collapsibleItem) => {
-          const href = collapsibleItem.getAttribute('href');
-
-          // Should never occur but we keep it as a fail-safe
-          if (href?.startsWith('https://') || href?.startsWith('http://')) {
-            return;
-          }
-
-          // Click any links we haven't seen before
-          if (href && !encounteredHref.includes(href)) {
-            collapsibleItem.click();
-          }
-
-          if (href) {
-            linksFound.push(href);
-          }
-        });
-
-        return linksFound;
-      },
-      encounteredHref // Need to pass array into the browser
-    );
-  }
-
-  return await page.content();
-}

package/src/scraping/site-scrapers/openNestedGitbookMenus.ts

@@ -1,27 +0,0 @@
-import { Page } from 'puppeteer';
-
-export default async function openNestedGitbookMenus(page: Page) {
-  let clickedAny = true;
-
-  // Loop until we've encountered every closed menu
-  while (clickedAny) {
-    clickedAny = await page.evaluate(() => {
-      let clicked = false;
-      // Right pointing arrow. Only menus have this icon
-      const icons = document.querySelectorAll(
-        'div > a > span > svg[style*="mask-image:url(https://ka-p.fontawesome.com/releases/v6.6.0/svgs/regular/chevron-right.svg?v=1&token=a463935e93)"]'
-      );
-
-      icons.forEach((icon) => {
-        const span = icon.parentElement;
-        if (span && span.className.includes('rotate-0')) {
-          span.click();
-          clicked = true;
-        }
-      });
-      return clicked;
-    });
-  }
-
-  return await page.content();
-}

package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts

@@ -1,85 +0,0 @@
-import * as cheerio from 'cheerio';
-import { NodeHtmlMarkdown } from 'node-html-markdown';
-
-import downloadAllImages from '../downloadAllImages.js';
-import replaceImagePaths from '../replaceImagePaths.js';
-
-export async function scrapeDocusaurusPage(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined // expects "2", or "3". Have not written support for "1" yet
-): Promise<{
-  title: string;
-  description?: string;
-  markdown?: string;
-}> {
-  const $ = cheerio.load(html);
-
-  const article = version === '3' ? $('.theme-doc-markdown').first() : $('article').first();
-
-  if (article.length === 0) {
-    // Index pages with no additional text don't have the markdown class
-    return {
-      title: '',
-    };
-  }
-
-  const titleComponent = article.find('h1');
-  const title = titleComponent.text().trim();
-
-  // Do not include title in the content when we insert it in our metadata
-  titleComponent.remove();
-
-  const markdownContent = version === '3' ? article : article.find('.markdown').first();
-
-  const origToWritePath = await downloadAllImages(
-    $,
-    markdownContent,
-    origin,
-    imageBaseDir,
-    overwrite
-  );
-
-  const markdownHtml = markdownContent.html();
-
-  const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
-  let markdown = markdownHtml ? nhm.translate(markdownHtml) : null;
-
-  if (markdown == null) {
-    console.error('We do not support scraping this page. Content will be empty');
-    return { title, description: undefined, markdown: '' };
-  }
-
-  // Description only exists in meta tags. The code is commented out because its prone to incorrectly
-  // including a description if the first line of text had markdown annotations like `.
-  // The commented out alternative is to ignore description if it's the first line of text,
-  // this means it was not set in the metadata and Docusaurus defaulted to the text.
-  const description = undefined;
-  // let description = $('meta[property="og:description"]').attr("content");
-  // if (markdown.startsWith(description)) {
-  //   description = null;
-  // }
-
-  // Remove Docusaurus links from headers
-  // When we parse their HTML the parser adds things like:
-  // [](#setup "Direct link to heading")
-  // to the end of each header.
-  markdown = markdown.replace(/\[\]\(#.+ ".+"\)\n/g, '\n');
-
-  // Remove unnecessary nonwidth blank space characters
-  markdown = markdown.replace(/\u200b/g, '');
-
-  // Reduce unnecessary blank lines
-  markdown = markdown.replace(/\n\n\n/g, '\n\n');
-
-  // Mintlify doesn't support bolded headers, remove the asterisks
-  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
-  if (origToWritePath) {
-    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
-  }
-
-  return { title, description, markdown };
-}