@mintlify/scraping 4.0.5 → 4.0.6

package/src/scrapingPipeline/color.ts

@@ -0,0 +1,107 @@
+import { Colors } from '@mintlify/models';
+import type { Root as HastRoot } from 'hast';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+import { framework } from '../utils/detectFramework.js';
+
+function toHex(value: number) {
+  Math.round(value).toString(16).padStart(2, '0');
+}
+
+function checkValidHex(str: string | undefined): boolean {
+  if (!str) return false;
+  return /^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$/.test(str);
+}
+
+function checkRgbBounds(...numbers: Array<number>): boolean {
+  for (const num of numbers) {
+    if (num < 0 || num > 255) return false;
+  }
+  return true;
+}
+
+function rgbToHex(color: string): string | undefined {
+  if (checkValidHex(color)) return color;
+  color = color.trim().toLowerCase();
+
+  let r: number | undefined, g: number | undefined, b: number | undefined;
+
+  if (/^\d+\s+\d+\s+\d+(\s+[0-9.]+)?$/.test(color)) {
+    [r, g, b] = color.split(/\s+/).map(Number);
+  } else {
+    const values = color.match(/^rgba?\((\d+),(\d+),(\d+)(?:,([0-9.]+))?\)$/);
+
+    if (!values) {
+      return undefined;
+    }
+
+    [, r, g, b] = values.map(Number);
+  }
+
+  if (!r || !g || !b) return undefined;
+
+  if (!checkRgbBounds(r, g, b)) return undefined;
+
+  return `#${toHex(r)}${toHex(g)}${toHex(b)}`.toUpperCase();
+}
+
+function getCssValue(cssString: string, key: string): string | undefined {
+  const regex = new RegExp(`${key}\\s*[:|,]\\s*([^;)]+)`, 'i');
+  const match = cssString.match(regex);
+  return match && match[1] ? match[1].trim() : undefined;
+}
+
+export const defaultColors = {
+  primary: '#0D9373',
+  light: '#55D799',
+  dark: '#0D9373',
+};
+
+export async function downloadColors(hast: HastRoot): Promise<Colors> {
+  if (framework.vendor === 'docusaurus') return defaultColors;
+
+  let primaryHexCode: string | undefined = undefined;
+  let lightHexCode: string | undefined = undefined;
+  visit(hast, 'element', function (node) {
+    if (node.tagName !== 'style') return CONTINUE;
+    if (
+      (framework.vendor === 'gitbook' && !!Object.keys(node.properties).length) ||
+      (framework.vendor === 'readme' && node.properties.title !== 'rm-custom-css')
+    )
+      return CONTINUE;
+
+    if (node.children.length !== 1 || !node.children[0] || node.children[0].type !== 'text')
+      return CONTINUE;
+
+    const cssStr = node.children[0].value;
+    const primaryColorKey =
+      framework.vendor === 'readme' ? '--color-link-primary' : '--primary-color-600';
+    const lightColorKey =
+      framework.vendor === 'readme' ? '--color-link-primary' : '--primary-color-400';
+
+    const primaryCssColorValue = getCssValue(cssStr, primaryColorKey);
+    const lightCssColorValue = getCssValue(cssStr, lightColorKey);
+    if (!primaryCssColorValue || !lightCssColorValue) return CONTINUE;
+
+    primaryHexCode = rgbToHex(primaryCssColorValue);
+    lightHexCode = rgbToHex(lightCssColorValue);
+  });
+
+  const isPrimaryValid = checkValidHex(primaryHexCode);
+  const isLightValid = checkValidHex(lightHexCode);
+
+  if (isPrimaryValid && isLightValid) {
+    return {
+      primary: primaryHexCode!,
+      light: lightHexCode,
+      dark: primaryHexCode,
+    };
+  } else if (isPrimaryValid) {
+    return {
+      primary: primaryHexCode!,
+      dark: primaryHexCode,
+    };
+  } else {
+    return defaultColors;
+  }
+}

package/src/scrapingPipeline/group.ts

@@ -52,11 +52,8 @@ export async function scrapePageGroup(
         navGroup[0]?.origin ?? 'the URL provided'
       }${errorMessage}`
     );
-    console.error(error);
     throw error;
   } finally {
-    if (browser) {
-      await browser.close();
-    }
+    if (browser) await browser.close();
   }
 }

package/src/scrapingPipeline/icon.ts

@@ -3,24 +3,26 @@ import { EXIT, visit } from 'unist-util-visit';
 
 import { downloadImage } from '../utils/images.js';
 
-export async function downloadFavicon(hast: HastRoot): Promise<string | undefined> {
+export async function downloadFavicon(hast: HastRoot): Promise<string> {
   let src: string = '';
   visit(hast, 'element', function (node) {
     if (
       node.tagName === 'link' &&
-      typeof node.properties.rel === 'string' &&
-      (node.properties.rel === 'icon' || node.properties.rel === 'shortcut icon')
+      Array.isArray(node.properties.rel) &&
+      node.properties.rel.includes('icon')
     ) {
       src = node.properties.href as string;
       return EXIT;
     }
   });
 
-  if (!src) return undefined;
+  if (!src) {
+    return '/favicon.svg';
+  }
 
   const res = await downloadImage(src, process.cwd());
-  if (!res.success) return undefined;
-  if (!res.data) return undefined;
+  if (!res.success) return '/favicon.svg';
+  if (!res.data) return '/favicon.svg';
 
   return res.data[1];
 }

package/src/scrapingPipeline/logo.ts

@@ -88,17 +88,24 @@ export async function downloadLogos(
 ): Promise<string | { light: string; dark: string } | undefined> {
   url = new URL(url);
   const filepaths: Array<string> = [];
+
   if (browser) {
     const htmls: Array<string> = [];
 
-    const page = await browser.newPage();
-    await page.goto(url.toString(), {
-      waitUntil: 'networkidle2',
-    });
+    try {
+      const page = await browser.newPage();
+      await page.goto(url.toString(), {
+        waitUntil: 'networkidle2',
+      });
+
+      htmls.push(await page.content());
+      await page.click('.rm-ThemeToggle');
+      htmls.push(await page.content());
+    } catch {
+      // do nothing, it just means there's no theme toggle
+    }
 
-    htmls.push(await page.content());
-    await page.click('.rm-ThemeToggle');
-    htmls.push(await page.content());
+    await browser.close();
 
     await Promise.all(
       htmls.map(async (html) => {
@@ -119,8 +126,6 @@ export async function downloadLogos(
     }
   }
 
-  if (browser) await browser.close();
-
   const uniqueFilepaths = [...new Set(filepaths).values()];
 
   return uniqueFilepaths.length === 1

package/src/scrapingPipeline/page.ts

@@ -21,18 +21,25 @@ import { rehypeToRemarkCustomComponents } from '../customComponents/plugin.js';
 import { selectiveRehypeRemark } from '../customComponents/selective.js';
 import { retrieveRootContent } from '../root/retrieve.js';
 import type { Result } from '../types/result.js';
+import { unifiedRemoveBreaks } from '../utils/breaks.js';
 import { unifiedRemoveClassNames } from '../utils/className.js';
+import { unifiedRemoveCopyButtons } from '../utils/copyButton.js';
 import { detectFramework, framework } from '../utils/detectFramework.js';
+import { remarkRemoveEmptyEmphases } from '../utils/emptyEmphasis.js';
 import { unifiedRemoveEmptyParagraphs } from '../utils/emptyParagraphs.js';
 import { getErrorMessage, logErrorResults } from '../utils/errors.js';
-import { escapeCharactersOutsideCodeBlocks } from '../utils/escape.js';
-import { write, writePage } from '../utils/file.js';
+import { writePage } from '../utils/file.js';
+import { remarkProperlyFormatEmphasis } from '../utils/formatEmphasis.js';
 import { removeHastComments } from '../utils/hastComments.js';
+import { remarkSpaceListsOut } from '../utils/lists.js';
 import { log } from '../utils/log.js';
+import { remarkRemoveBottomMetadata } from '../utils/metadata.js';
 import { unifiedRemoveNestedRoots } from '../utils/nestedRoots.js';
 import { unifiedRemovePositions } from '../utils/position.js';
 import { removeLeadingSlash, removeTrailingSlash } from '../utils/strings.js';
+import { remarkRemoveCodeBlocksInCells } from '../utils/tableCells.js';
 import { getDescriptionFromRoot, getTitleFromHeading } from '../utils/title.js';
+import { remarkRemoveUpdatedAt } from '../utils/updatedAt.js';
 import { downloadImagesFromFile } from './images.js';
 import { htmlToHast } from './root.js';
 
@@ -61,7 +68,8 @@ export async function scrapePage(
 
   const urlStr = url.toString();
   const content = retrieveRootContent(hast);
-  if (!content) return { success: false, message: `${urlStr}: ${CONTENT_FAILURE_MSG}` };
+  if (!content)
+    return { success: false, message: `${urlStr}: ${CONTENT_FAILURE_MSG}`, data: [urlStr, ''] };
 
   const contentAsRoot: HastRoot = {
     type: 'root',
@@ -69,18 +77,19 @@ export async function scrapePage(
   };
 
   const mdastTree: MdastRoot = unified()
+    .use(unifiedRemoveBreaks)
+    .use(unifiedRemoveCopyButtons)
     .use(createCard)
     .use(createAccordion)
     .use(createFrame)
-    .use(createTabs)
     .use(createCallout)
     .use(createCardGroup)
     .use(createAccordionGroup)
     .use(createCodeGroup)
+    .use(createTabs)
     .use(unifiedRemoveClassNames)
-    .use(unifiedRemovePositions)
     .use(unifiedRemoveEmptyParagraphs)
-    .use(escapeCharactersOutsideCodeBlocks)
+    .use(unifiedRemovePositions)
     .use(selectiveRehypeRemark)
 
     // Cleans up any nested components left untouched
@@ -89,6 +98,13 @@ export async function scrapePage(
     .use(rehypeToRemarkCustomComponents)
     .use(convertHeaderLinksToText)
     .use(unifiedRemoveNestedRoots)
+    .use(remarkSpaceListsOut)
+    .use(remarkRemoveBottomMetadata)
+    .use(remarkRemoveUpdatedAt)
+    .use(remarkRemoveEmptyEmphases)
+    .use(remarkProperlyFormatEmphasis)
+    .use(remarkRemoveCodeBlocksInCells)
+    // @ts-expect-error moving some of the pipeline around results in contentAsRoot being treated differently than its type which is Root Element
     .runSync(contentAsRoot) as MdastRoot;
 
   try {
@@ -111,13 +127,15 @@ export async function scrapePage(
       .use(remarkStringify)
       .stringify(mdastTree);
 
+    const resultStr = String(result).replace(/\n{3,}/g, '\n\n');
+
     if (opts.rootPath) {
       url = new URL(opts.rootPath, url.origin);
     } else if (url.origin === removeTrailingSlash(url.toString())) {
       url = new URL('home', new URL(url).origin);
     }
 
-    writePage(url, opts.isOverviewPage ? 'Overview' : title, description, String(result));
+    writePage(url, opts.isOverviewPage ? 'Overview' : title, description, resultStr);
     return {
       success: true,
       data: opts.rootPath
@@ -125,8 +143,11 @@ export async function scrapePage(
         : undefined,
     };
   } catch (error) {
-    write('error.json', JSON.stringify(mdastTree, undefined, 2));
     const errorMessage = getErrorMessage(error);
-    return { success: false, message: `${urlStr}: ${MDAST_FAILURE_MSG}${errorMessage}` };
+    return {
+      success: false,
+      message: `${urlStr}: ${MDAST_FAILURE_MSG}${errorMessage}`,
+      data: [urlStr, ''],
+    };
   }
 }

package/src/scrapingPipeline/site.ts

@@ -11,11 +11,13 @@ import { detectFramework, framework } from '../utils/detectFramework.js';
 import { logErrorResults } from '../utils/errors.js';
 import { startPuppeteer } from '../utils/network.js';
 import { INDEX_NAMES, iterateThroughReservedNames } from '../utils/reservedNames.js';
-import { removeTrailingSlash } from '../utils/strings.js';
+import { removeTrailingSlash, removeLeadingSlash } from '../utils/strings.js';
+import { downloadColors } from './color.js';
 import { scrapePageGroup } from './group.js';
 import { downloadFavicon } from './icon.js';
 import { downloadLogos } from './logo.js';
 import { htmlToHast } from './root.js';
+import { downloadTitle } from './title.js';
 
 export async function scrapeSite(
   html: string,
@@ -127,24 +129,98 @@ export async function scrapeSite(
       };
     });
 
+    const allErrors = [
+      ...externalResults.filter((result) => !result.success),
+      ...internalResults.filter((result) => !result.success),
+      ...rootResults.filter((result) => !result.success),
+    ];
+
+    const allErroredPaths = allErrors
+      .map((result) => {
+        if (result.data) {
+          const url = new URL(result.data[0]);
+          const pathname = url.pathname;
+          const normalizedPathname = removeLeadingSlash(removeTrailingSlash(pathname));
+          return normalizedPathname;
+        } else {
+          return '';
+        }
+      })
+      .filter(Boolean);
+
+    traverse(navItems).forEach(function (value) {
+      if (typeof value === 'string' && allErroredPaths.includes(value)) {
+        this.remove();
+      } else if (Array.isArray(value)) {
+        this.update(
+          value
+            .filter((item) =>
+              typeof item === 'string' && allErroredPaths.includes(item) ? undefined : item
+            )
+            .filter(Boolean)
+        );
+      }
+    });
+
+    let count = 1;
+    while (count > 0) {
+      count = 0;
+      traverse(navItems).forEach(function (value) {
+        if (Array.isArray(value) && value.filter(Boolean).length === 0) {
+          count++;
+          if (this.parent) {
+            this.parent.remove();
+          } else {
+            this.remove();
+          }
+        }
+      });
+    }
+
+    traverse(navItems).forEach(function (value) {
+      if (
+        typeof value === 'string' &&
+        (value.startsWith('https://') || value.startsWith('http://'))
+      ) {
+        this.remove();
+      } else if (
+        Array.isArray(value) &&
+        value.find(
+          (val) =>
+            typeof val === 'string' && (val.startsWith('https://') || val.startsWith('http://'))
+        )
+      ) {
+        this.update(
+          value.filter(
+            (val) =>
+              !(
+                typeof val === 'string' &&
+                (val.startsWith('https://') || val.startsWith('http://'))
+              )
+          )
+        );
+      }
+    });
+
     logErrorResults('linking to external pages', externalResults);
     logErrorResults('scraping your docs', [...internalResults, ...rootResults]);
 
-    const browser = needsBrowser ? await startPuppeteer() : undefined;
+    const needsBrowserForLogos = framework.vendor === 'readme';
+    const browser = needsBrowserForLogos ? await startPuppeteer() : undefined;
 
     const favicon = await downloadFavicon(hast);
+    const colors = await downloadColors(hast);
     const logo = await downloadLogos(url, browser);
+    const name = await downloadTitle(hast);
 
     return {
       success: true,
       data: {
         $schema: 'https://mintlify.com/schema.json',
-        name: '',
+        name,
         logo,
-        colors: {
-          primary: '',
-        },
-        favicon: favicon ?? '',
+        colors,
+        favicon,
         navigation: navItems as Navigation,
         tabs: opts.tabs,
       },

package/src/scrapingPipeline/tabs.ts

@@ -1,4 +1,4 @@
-import type { Navigation, NavigationEntry } from '@mintlify/models';
+import type { Colors, Navigation, NavigationEntry } from '@mintlify/models';
 import { MintConfig, Tab } from '@mintlify/models';
 
 import { retrieveTabLinks } from '../tabs/retrieveReadme.js';
@@ -8,10 +8,11 @@ import { getErrorMessage } from '../utils/errors.js';
 import { log } from '../utils/log.js';
 import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
 import { getTitleFromLink } from '../utils/title.js';
-import { downloadFavicon } from './icon.js';
+import { defaultColors } from './color.js';
 import { downloadLogos } from './logo.js';
 import { htmlToHast } from './root.js';
 import { scrapeSite } from './site.js';
+import { downloadTitle } from './title.js';
 
 export async function scrapeAllSiteTabs(
   html: string,
@@ -22,12 +23,6 @@ export async function scrapeAllSiteTabs(
 
   detectFramework(hast);
 
-  const needsBrowser = framework.vendor === 'gitbook';
-  const browser = needsBrowser ? await startPuppeteer() : undefined;
-
-  const favicon = await downloadFavicon(hast);
-  const logo = await downloadLogos(url, browser);
-
   if (framework.vendor === 'readme' || framework.vendor === 'docusaurus') {
     const links = retrieveTabLinks(hast);
     if (
@@ -59,12 +54,16 @@ export async function scrapeAllSiteTabs(
 
     const navigations: Array<NavigationEntry> = [];
     const tabs: Array<Tab> = [];
+    let favicon = '/favicon.svg';
+    let colors: Colors = defaultColors;
 
     const successes = results.filter((result) => result.success);
     successes.forEach((result) => {
       if (!result.data) return;
       navigations.push(...result.data.navigation);
       if (result.data.tabs) tabs.push(...result.data.tabs);
+      if (result.data.favicon !== '/favicon.svg') favicon = result.data.favicon;
+      if (result.data.colors !== defaultColors) colors = result.data.colors;
     });
 
     const failures = results.filter((result) => !result.success);
@@ -72,16 +71,19 @@ export async function scrapeAllSiteTabs(
       log('Failed to scrape tab' + result.message);
     });
 
+    const needsBrowser = framework.vendor === 'readme';
+    const browser = needsBrowser ? await startPuppeteer() : undefined;
+    const logo = await downloadLogos(url, browser);
+    const name = await downloadTitle(hast);
+
     return {
       success: true,
       data: {
         $schema: 'https://mintlify.com/schema.json',
-        name: '',
+        name,
         logo,
-        colors: {
-          primary: '',
-        },
-        favicon: favicon ?? '',
+        colors,
+        favicon,
         navigation: navigations as Navigation,
         tabs,
       },

package/src/scrapingPipeline/title.ts

@@ -0,0 +1,38 @@
+import type { Root as HastRoot } from 'hast';
+import { CONTINUE, EXIT, visit } from 'unist-util-visit';
+
+const defaultTitle = 'Enter name here';
+
+export async function downloadTitle(hast: HastRoot): Promise<string> {
+  let text: string | undefined = undefined as string | undefined;
+
+  visit(hast, 'element', function (node) {
+    if (node.tagName !== 'title') return CONTINUE;
+
+    visit(node, 'text', function (subNode) {
+      text = subNode.value;
+      return EXIT;
+    });
+
+    if (text) {
+      return EXIT;
+    }
+  });
+
+  if (!text) return defaultTitle;
+
+  const title = text as string;
+  let siteGroupTitle = '';
+
+  if (title.includes('|')) {
+    siteGroupTitle = (title.split('|').at(-1) ?? '').trim() as string;
+  } else if (title.includes('–')) {
+    siteGroupTitle = (title.split('–').at(-1) ?? '').trim() as string;
+  } else if (title.includes('-')) {
+    siteGroupTitle = (title.split('-').at(-1) ?? '').trim() as string;
+  } else {
+    siteGroupTitle = title.trim();
+  }
+
+  return siteGroupTitle ? siteGroupTitle : defaultTitle;
+}

package/src/tabs/retrieveReadme.ts

@@ -8,7 +8,7 @@ import { findTitle, getTitleFromLink } from '../utils/title.js';
 export function retrieveTabLinks(rootNode: HastRoot): Array<Tab> | undefined {
   if (framework.vendor !== 'readme' && framework.vendor !== 'docusaurus') return undefined;
 
-  let element: Element | undefined = undefined;
+  let element: Element | undefined = undefined as Element | undefined;
   visit(rootNode, 'element', function (node) {
     if (framework.vendor === 'readme') {
       if (
@@ -35,7 +35,6 @@ export function retrieveTabLinks(rootNode: HastRoot): Array<Tab> | undefined {
     }
   });
 
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
   if (!element) return undefined;
 
   const links: Array<Tab> = [];

package/src/types/result.ts

@@ -1 +1 @@
-export type Result<T> = { success: true; data?: T } | { success: false; message: string };
+export type Result<T> = { success: true; data?: T } | { success: false; message: string; data?: T };

package/src/utils/breaks.ts

@@ -0,0 +1,19 @@
+import type { Element } from 'hast';
+import { visit } from 'unist-util-visit';
+
+export function unifiedRemoveBreaks() {
+  return function (node: Element) {
+    return removeBreaks(node);
+  };
+}
+
+// ReadMe-specific function since they use breaks in between
+// every element, but either way our parser adds whitespace
+// automatically
+export function removeBreaks(node: Element) {
+  return visit(node, 'element', function (subNode, index, parent) {
+    if (subNode.tagName === 'br' && parent && typeof index === 'number') {
+      parent.children.splice(index, 1);
+    }
+  });
+}

package/src/utils/children.ts

@@ -11,6 +11,7 @@ import type { State, Handle } from 'hast-util-to-mdast';
 import type { RootContent as MdastRootContent, Root as MdastRoot } from 'mdast';
 import { unified } from 'unified';
 
+import { ESCAPED_COMPONENTS } from '../constants.js';
 import { mdxJsxFlowElementHandler } from '../customComponents/selective.js';
 
 export function turnChildrenIntoMdx(
@@ -19,19 +20,25 @@ export function turnChildrenIntoMdx(
 ): Array<MdastRootContent> {
   const hast: HastRoot = {
     type: 'root',
-    children: children,
+    children,
   };
 
   const handlers: Record<string, Handle> = { ...defaultHandlers };
   if (opts.jsxImages) {
-    handlers['img'] = function (_: State, node: Element) {
+    handlers['img'] = function (h: State, node: Element) {
       Object.keys(node.properties).forEach((key) => {
         if (key !== 'src') delete node.properties[key];
       });
-      return mdxJsxFlowElementHandler(_, node);
+      return mdxJsxFlowElementHandler(h, node);
     };
   }
 
+  ESCAPED_COMPONENTS.forEach((component) => {
+    handlers[component] = function (h: State, node: Element) {
+      return mdxJsxFlowElementHandler(h, node);
+    };
+  });
+
   const mdxAst = unified()
     .use(function () {
       return function (tree: HastRoot): MdastRoot {

package/src/utils/className.ts

@@ -7,7 +7,7 @@ export function unifiedRemoveClassNames() {
   };
 }
 
-export function removeClassNames(node: Element) {
+function removeClassNames(node: Element) {
   return visit(node, 'element', function (subNode) {
     if ('properties' in subNode) delete subNode.properties.className;
   });

package/src/utils/copyButton.ts

@@ -0,0 +1,35 @@
+import type { Root as HastRoot } from 'hast';
+import { CONTINUE, EXIT, visit } from 'unist-util-visit';
+
+export function unifiedRemoveCopyButtons() {
+  return function (root: HastRoot) {
+    return removeCopyButtons(root);
+  };
+}
+
+// GitBook specific, since they have a 'Copy' button in every
+// code block which can't be not scraped since it's included
+// in every HTML output
+export function removeCopyButtons(root: HastRoot) {
+  visit(root, 'element', function (node, index, parent) {
+    if (
+      node.tagName !== 'button' ||
+      !Array.isArray(node.properties.className) ||
+      !node.properties.className.includes('group-hover/codeblock:opacity-[1]')
+    )
+      return CONTINUE;
+
+    let isCopyButton = false as boolean;
+    visit(node, 'text', function (textNode) {
+      if (textNode.value === 'Copy' || textNode.value === 'copy') {
+        isCopyButton = true;
+        return EXIT;
+      }
+    });
+
+    if (isCopyButton) {
+      if (!parent || typeof index !== 'number') return CONTINUE;
+      parent.children.splice(index, 1);
+    }
+  });
+}

package/src/utils/emptyEmphasis.ts

@@ -0,0 +1,18 @@
+import type { Root as MdastRoot } from 'mdast';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+export function remarkRemoveEmptyEmphases() {
+  return function (root: MdastRoot) {
+    return removeEmptyEmphases(root);
+  };
+}
+
+function removeEmptyEmphases(root: MdastRoot) {
+  visit(root, function (node, index, parent) {
+    if (node.type !== 'emphasis' && node.type !== 'strong') return CONTINUE;
+    if (node.children.length === 0) {
+      if (!parent || typeof index !== 'number') return CONTINUE;
+      parent.children.splice(index, 1);
+    }
+  });
+}