@mintlify/scraping 3.0.187 → 3.0.189

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -1,63 +0,0 @@
-import { Navigation, NavigationEntry } from '@mintlify/models';
-import * as cheerio from 'cheerio';
-
-import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
-import downloadLogoImage from '../downloadLogoImage.js';
-import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
-import { getDocusaurusLinksPerGroup } from './links-per-group/getDocusaurusLinksPerGroup.js';
-import { scrapeDocusaurusPage } from './scrapeDocusaurusPage.js';
-
-export async function scrapeDocusaurusSection(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version?: string
-): Promise<Navigation> {
-  const $ = cheerio.load(html);
-
-  // Download the logo
-  const logoSrc = $('.navbar__logo img').attr('src');
-  void downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
-
-  // Get all the navigation sections
-  const navigationSections = !$('.theme-doc-sidebar-menu').first().children().length
-    ? $('.main-wrapper').first().find('.menu__list').first().children()
-    : $('.theme-doc-sidebar-menu').first().children();
-
-  // Get all links per group
-  const groupsConfig: Navigation = getDocusaurusLinksPerGroup(navigationSections, $, version);
-
-  // Merge groups with empty titles together
-  const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
-
-  // Scrape each link in the navigation.
-  const groupsConfigCleanPaths = await Promise.all(
-    reducedGroupsConfig.map(async (groupConfig) => {
-      groupConfig.pages = (
-        await Promise.all(
-          groupConfig.pages.map(async (navEntry: NavigationEntry) =>
-            // Docusaurus requires a directory on all sections wheras we use root.
-            // /docs is their default directory so we remove it
-            scrapeGettingFileNameFromUrl(
-              navEntry,
-              cliDir,
-              origin,
-              overwrite,
-              scrapeDocusaurusPage,
-              false,
-              version,
-              '/docs'
-            )
-          )
-        )
-      )
-        // Remove skipped index pages (they return undefined from the above function)
-        .filter(Boolean);
-      return groupConfig;
-    })
-  );
-
-  return groupsConfigCleanPaths;
-}

package/src/scraping/site-scrapers/scrapeGitBookPage.ts

@@ -1,82 +0,0 @@
-import * as cheerio from 'cheerio';
-import { NodeHtmlMarkdown } from 'node-html-markdown';
-
-import { SUPPORTED_MEDIA_EXTENSIONS } from '../../constants.js';
-import { getLengthUntilMetadata } from '../../downloadImage.js';
-import downloadAllImages from '../downloadAllImages.js';
-import replaceImagePaths from '../replaceImagePaths.js';
-
-export async function scrapeGitBookPage(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  _: string | undefined // version
-) {
-  const $ = cheerio.load(html);
-
-  const mainContent = $('body > div > div > div > div > main');
-  const titleAndDescription = mainContent.children('header');
-  const titleComponent = titleAndDescription.children('h1');
-  const description = titleAndDescription.text().replace(titleComponent.text(), '').trim();
-  const title = titleComponent.text().trim();
-
-  const content = titleAndDescription.next('div');
-
-  // Replace code blocks with parseable html
-  const codeBlocks = content.find('pre > code');
-  codeBlocks.each((_, c) => {
-    const code = $(c);
-    const codeContent = code
-      .children()
-      .toArray()
-      .map((d) => $(d).text())
-      .filter((text) => text !== '')
-      .join('\n');
-    code.replaceWith(`<pre><code>${codeContent}</code></pre>`);
-  });
-
-  const contentHtml = $.html(content);
-
-  const modifyFileName = (fileName: string) => {
-    // Remove GitBook metadata from the start
-    // The first four %2F split metadata fields. Remaining ones are part of the file name.
-    for (const ext of SUPPORTED_MEDIA_EXTENSIONS) {
-      if (fileName.includes(`.${ext}`)) {
-        const splitFileName = fileName.split('%2F').slice(4).join('%2F');
-        return getLengthUntilMetadata(splitFileName, ext);
-      }
-    }
-    return fileName.split('%2F').slice(4).join('%2F');
-  };
-
-  const origToWritePath = await downloadAllImages(
-    $,
-    content,
-    origin,
-    imageBaseDir,
-    overwrite,
-    modifyFileName
-  );
-
-  const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
-  let markdown = nhm.translate(contentHtml);
-
-  // Keep headers on one line
-  markdown = markdown.replace(/# \n\n/g, '# ');
-
-  // Remove unnecessary nonwidth blank space characters
-  markdown = markdown.replace(/\u200b/g, '');
-
-  // Reduce unnecessary blank lines
-  markdown = markdown.replace(/\n\n\n/g, '\n\n');
-
-  // Mintlify doesn't support bolded headers, remove the asterisks
-  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
-  if (origToWritePath) {
-    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
-  }
-
-  return { title, description, markdown };
-}

package/src/scraping/site-scrapers/scrapeGitBookSection.ts

@@ -1,69 +0,0 @@
-import { Navigation, NavigationEntry } from '@mintlify/models';
-import * as cheerio from 'cheerio';
-
-import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
-import downloadLogoImage from '../downloadLogoImage.js';
-import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
-import alternateGroupTitle from './alternateGroupTitle.js';
-import getLinksRecursivelyGitBook from './links-per-group/getLinksRecursivelyGitBook.js';
-import { scrapeGitBookPage } from './scrapeGitBookPage.js';
-
-export async function scrapeGitBookSection(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined
-): Promise<NavigationEntry[]> {
-  const $ = cheerio.load(html);
-
-  // Download the logo
-  const logoSrc = $('body > header > div > div > div > a > img').first().attr('src');
-  downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite).catch(console.error);
-
-  // Get all the navigation sections
-  // Some variants of the GitBook UI show the logo and search base in the side navigation bar,
-  // but the navigation sections are always the last value.
-  const navigationSections = $('body > div > div > aside > div > ul > li');
-
-  // Get all links per group
-  const groupsConfig: Navigation = navigationSections
-    .toArray()
-    .map((s: cheerio.Element) => {
-      const section = $(s);
-      const sectionHeader = section.children('div').first();
-      const sectionTitle = sectionHeader.text();
-
-      // Only present if the nested navigation is not in a group
-      const firstLink = section.find('li > div > a').first();
-      const firstHref = firstLink.attr('href') || '/';
-
-      const linkSections = section.find('ul').first().children();
-      const pages = getLinksRecursivelyGitBook(linkSections, $);
-
-      return {
-        group: sectionTitle || alternateGroupTitle(firstLink, pages),
-        pages: firstHref ? [firstHref, ...pages] : pages,
-      };
-    })
-    .filter(Boolean);
-
-  // Merge groups with empty titles together
-  const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
-
-  // Scrape each link in the navigation.
-  return Promise.all(
-    reducedGroupsConfig.map(async (navEntry: NavigationEntry) => {
-      return await scrapeGettingFileNameFromUrl(
-        navEntry,
-        cliDir,
-        origin,
-        overwrite,
-        scrapeGitBookPage,
-        true,
-        version
-      );
-    })
-  );
-}

package/src/scraping/site-scrapers/scrapeReadMePage.ts

@@ -1,56 +0,0 @@
-import * as cheerio from 'cheerio';
-import { NodeHtmlMarkdown } from 'node-html-markdown';
-
-import downloadAllImages from '../downloadAllImages.js';
-import replaceImagePaths from '../replaceImagePaths.js';
-
-export async function scrapeReadMePage(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  _: string | undefined // version
-) {
-  const $ = cheerio.load(html);
-
-  const titleComponent = $('h1').first();
-  const title = titleComponent.text().trim();
-  let description = $('.markdown-body', titleComponent.parent()).text().trim();
-  if (!description) {
-    description = $('.rm-Article > header p').text().trim();
-  }
-
-  let content = $('.content-body .markdown-body').first();
-  if (content.length === 0) {
-    content = $('.rm-Article > .markdown-body');
-  }
-
-  // API Pages don't have a markdown body in the same position so there's no HTML
-  const contentHtml = content.html() || '';
-
-  const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite);
-
-  const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
-  let markdown = nhm.translate(contentHtml);
-
-  // Keep headers on one line and increase their depth by one
-  markdown = markdown.replace(/# \n\n/g, '## ');
-
-  // Remove unnecessary nonwidth blank space characters
-  markdown = markdown.replace(/\u200b/g, '');
-
-  // Remove ReadMe anchor links
-  markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, '\n');
-
-  // Reduce unnecessary blank lines
-  markdown = markdown.replace(/\n\n\n/g, '\n\n');
-
-  // Mintlify doesn't support bolded headers, remove the asterisks
-  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
-  if (origToWritePath) {
-    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
-  }
-
-  return { title, description, markdown };
-}

package/src/scraping/site-scrapers/scrapeReadMeSection.ts

@@ -1,66 +0,0 @@
-import { Navigation, NavigationEntry } from '@mintlify/models';
-import * as cheerio from 'cheerio';
-
-import downloadLogoImage from '../downloadLogoImage.js';
-import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
-import getLinksRecursively from './links-per-group/getLinksRecursively.js';
-import { scrapeReadMePage } from './scrapeReadMePage.js';
-
-export async function scrapeReadMeSection(
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined
-): Promise<NavigationEntry[]> {
-  const $ = cheerio.load(html);
-
-  // Download the logo
-  const logoSrc = $('.rm-Logo-img').first().attr('src');
-  downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite).catch(console.error);
-
-  // Get all the navigation sections, but only from the first
-  // sidebar found. There are multiple in the HTML for mobile
-  // responsiveness but they all have the same links.
-  const navigationSections = $('.rm-Sidebar').first().find('.rm-Sidebar-section');
-
-  const groupsConfig: Navigation = navigationSections.toArray().map((s) => {
-    const section = $(s);
-    const sectionTitle = section.find('h3').first().text();
-
-    // Get all links, then use filter to remove duplicates.
-    // There are duplicates because of nested navigation, eg:
-    // subgroupTitle -> /first-page
-    // -- First Page -> /first-page   ** DUPLICATE **
-    // -- Second Page -> /second-page
-    const linkSections = section.find('.rm-Sidebar-list').first().children();
-    const pages = getLinksRecursively(linkSections, $).filter(
-      (value, index, array) => array.indexOf(value) === index
-    );
-
-    // Follows the same structure as mint.json
-    return {
-      group: sectionTitle,
-      pages: pages,
-    };
-  });
-
-  // Scrape each link in the navigation.
-  return Promise.all(
-    groupsConfig.map(async (navEntry: NavigationEntry) => {
-      return await scrapeGettingFileNameFromUrl(
-        // ReadMe requires a directory on all sections whereas we use root.
-        // /docs is their default directory so we remove it
-        navEntry,
-        cliDir,
-        origin,
-        overwrite,
-        scrapeReadMePage,
-        false,
-        version,
-        '/docs'
-      );
-    })
-  );
-}

package/src/util.ts

@@ -1,122 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-import { existsSync, mkdirSync, writeFileSync } from 'fs';
-import Ora, { Ora as OraType } from 'ora';
-import path from 'path';
-
-export const MintConfig = (
-  name: string,
-  color: string,
-  ctaName: string,
-  ctaUrl: string,
-  filename: string
-) => {
-  return {
-    name,
-    logo: '',
-    favicon: '',
-    colors: {
-      primary: color,
-    },
-    topbarLinks: [],
-    topbarCtaButton: {
-      name: ctaName,
-      url: ctaUrl,
-    },
-    anchors: [],
-    navigation: [
-      {
-        group: 'Home',
-        pages: [filename],
-      },
-    ],
-    // footerSocials: {}, // support object type for footer tyoes
-  };
-};
-
-export const Page = (title: string, description?: string, markdown?: string) => {
-  // If we are an empty String we want to add two quotes,
-  // if we added as we went we would detect the first quote
-  // as the closing quote.
-  const startsWithQuote = title.startsWith('"');
-  const endsWithQuote = title.startsWith('"');
-  if (!startsWithQuote) {
-    title = '"' + title;
-  }
-  if (!endsWithQuote) {
-    title = title + '"';
-  }
-
-  const optionalDescription = description ? `\ndescription: "${description}"` : '';
-  return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
-};
-
-export function getOrigin(url: string) {
-  // eg. https://google.com -> https://google.com
-  // https://google.com/page -> https://google.com
-  return new URL(url).origin;
-}
-
-export function objToReadableString(objs: NavigationEntry[]) {
-  // Two spaces as indentation
-  return objs.map((obj) => JSON.stringify(obj, null, 2)).join(',\n');
-}
-
-export const toFilename = (title: string) => {
-  // Gets rid of special characters at the start and end
-  // of the name by converting to spaces then using trim.
-  return title
-    .replace(/[^a-z0-9]/gi, ' ')
-    .trim()
-    .replace(/ /g, '-')
-    .toLowerCase();
-};
-
-export const addMdx = (fileName: string) => {
-  if (fileName.endsWith('.mdx')) {
-    return fileName;
-  }
-  return fileName + '.mdx';
-};
-
-export const createPage = (
-  title: string,
-  description?: string,
-  markdown?: string,
-  overwrite = false,
-  rootDir = '',
-  fileName?: string
-) => {
-  const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
-
-  // Create the folders needed if they're missing
-  mkdirSync(rootDir, { recursive: true });
-
-  if (!overwrite && existsSync(writePath)) {
-    console.log(`❌ Skipping existing file ${writePath}`);
-    return;
-  }
-
-  // Write the page to disk
-  try {
-    writeFileSync(writePath, Page(title, description, markdown));
-    console.log('✏️ - ' + writePath);
-  } catch (e) {
-    console.error(e);
-  }
-};
-
-export const buildLogger = (startText = ''): OraType => {
-  const logger = Ora().start(startText);
-  return logger;
-};
-
-export const getFileExtension = (filename: string) => {
-  const ext = filename.substring(filename.lastIndexOf('.') + 1, filename.length);
-  if (filename === ext) return undefined;
-  return ext.toLowerCase();
-};
-
-export const fileBelongsInPagesFolder = (filename: string) => {
-  const extension = getFileExtension(filename);
-  return extension && (extension === 'mdx' || extension === 'md' || extension === 'tsx');
-};