@mintlify/scraping 3.0.187 → 3.0.188

package/src/customComponents/plugin.ts

@@ -0,0 +1,31 @@
+import type { Root as HastRoot, Element } from 'hast';
+import type { BlockContent, DefinitionContent } from 'mdast';
+import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import { visit } from 'unist-util-visit';
+
+import { ESCAPED_COMPONENTS } from '../constants.js';
+import type { EscapedComponent } from '../types/components.js';
+
+export function rehypeToRemarkCustomComponents() {
+  return function (tree: HastRoot) {
+    visit(tree, 'element', function (node: Element, index, parent) {
+      if (ESCAPED_COMPONENTS.includes(node.tagName as unknown as EscapedComponent)) {
+        const newNode: MdxJsxFlowElement = {
+          type: 'mdxJsxFlowElement',
+          name: node.tagName,
+          // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+          attributes: Object.entries(node.properties || {}).map(([key, value]) => ({
+            type: 'mdxJsxAttribute',
+            name: key,
+            value: value as string,
+          })),
+          children: node.children as Array<BlockContent | DefinitionContent>,
+        };
+        if (parent && typeof index === 'number') {
+          parent.children[index] = newNode as unknown as Element;
+        }
+      }
+    });
+    return tree;
+  };
+}

package/src/customComponents/selective.ts

@@ -0,0 +1,37 @@
+import type { Root as HastRoot, Element } from 'hast';
+import { toMdast, defaultHandlers } from 'hast-util-to-mdast';
+import type { State, Handle } from 'hast-util-to-mdast';
+import type { Root as MdastRoot } from 'mdast';
+import type { BlockContent, DefinitionContent } from 'mdast';
+import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+
+import { ESCAPED_COMPONENTS } from '../constants.js';
+
+export function mdxJsxFlowElementHandler(_: State, node: Element): MdxJsxFlowElement {
+  return {
+    type: 'mdxJsxFlowElement',
+    name: node.tagName,
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+    attributes: Object.entries(node.properties ?? {}).map(([key, value]) => ({
+      type: 'mdxJsxAttribute',
+      name: key,
+      value: value as string,
+    })),
+    children: node.children as Array<BlockContent | DefinitionContent>,
+  };
+}
+
+export function selectiveRehypeRemark() {
+  const handlers: Record<string, Handle> = { ...defaultHandlers };
+  ESCAPED_COMPONENTS.forEach((tagName) => {
+    handlers[tagName] = mdxJsxFlowElementHandler;
+  });
+  handlers.mdxJsxFlowElement = mdxJsxFlowElementHandler;
+
+  return function (tree: HastRoot) {
+    const newTree = toMdast(tree, {
+      handlers,
+    }) as MdastRoot;
+    return newTree;
+  };
+}

package/src/nav/iterate.ts

@@ -0,0 +1,18 @@
+import { NavigationEntry } from '@mintlify/models';
+
+export function iterateOverNavItems(navItems: Array<NavigationEntry>, origin: string): Array<URL> {
+  return navItems.flatMap((navItem) => recurseOverGroup(navItem, origin));
+}
+
+function recurseOverGroup(group: NavigationEntry, origin: string): Array<URL> {
+  if (typeof group === 'string') {
+    return [new URL(group, origin)];
+  }
+
+  return group.pages.flatMap((pageOrGroup: string | NavigationEntry) => {
+    if (typeof pageOrGroup === 'string') {
+      return new URL(pageOrGroup, origin);
+    }
+    return recurseOverGroup(pageOrGroup, origin);
+  });
+}

package/src/nav/listItems.ts

@@ -0,0 +1,82 @@
+import { NavigationEntry } from '@mintlify/models';
+import type { Element } from 'hast';
+import { EXIT, visit } from 'unist-util-visit';
+
+import { dedupedAppend } from '../utils/append.js';
+import { framework } from '../utils/detectFramework.js';
+import { findFirstChild } from '../utils/firstChild.js';
+import { removeTrailingSlash } from '../utils/strings.js';
+import { getText } from '../utils/text.js';
+import { retrieveNavItems } from './retrieve.js';
+
+export type ListItemOptions = {
+  sectionTagName: string;
+  childListTagName: string;
+  title?: string;
+};
+
+export function processListItem(
+  node: Element,
+  opts: ListItemOptions = {
+    sectionTagName: 'div',
+    childListTagName: 'ul',
+    title: undefined,
+  }
+): NavigationEntry | undefined {
+  const link = findFirstChild(node, 'a');
+  if (!link) return undefined;
+
+  let linkHref: string | undefined = undefined;
+  linkHref = link.properties.href as string | undefined;
+
+  if (linkHref === undefined || linkHref === '#') {
+    return undefined;
+  }
+
+  let isApiReferenceLink = false;
+  visit(link, 'element', function (subNode) {
+    if (
+      subNode.tagName === 'span' &&
+      Array.isArray(subNode.properties.className) &&
+      subNode.properties.className.includes('rm-APIMethod')
+    ) {
+      isApiReferenceLink = true;
+      return EXIT;
+    }
+  });
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (isApiReferenceLink) return undefined;
+
+  if (linkHref.startsWith('/')) linkHref = linkHref.substring(1);
+
+  const sectionHeader = findFirstChild(node, opts.sectionTagName);
+  const childList = findFirstChild(node, opts.childListTagName);
+  let title = opts.title;
+  if (!title) {
+    title = getText(link) || getText(sectionHeader) || '';
+    if (framework.vendor === 'readme') {
+      title = getText(sectionHeader) || getText(link) || '';
+    }
+  }
+
+  if (!childList) {
+    return linkHref;
+  }
+
+  let childEntries = retrieveNavItems(childList);
+  const newLink = childEntries.find(
+    (child) => typeof child === 'string' && child.startsWith(linkHref)
+  )
+    ? removeTrailingSlash(linkHref) + '/mintie_overview'
+    : linkHref;
+
+  if (childEntries.includes(linkHref)) {
+    childEntries.forEach((child, index) => {
+      if (child === linkHref) childEntries[index] = newLink;
+    });
+  } else {
+    childEntries = dedupedAppend(newLink, childEntries, true);
+  }
+
+  return { group: title, pages: dedupedAppend(newLink, childEntries, true) };
+}

package/src/nav/retrieve.ts

@@ -0,0 +1,88 @@
+import { NavigationEntry } from '@mintlify/models';
+import type { Element } from 'hast';
+import { visit, CONTINUE, SKIP } from 'unist-util-visit';
+
+import { framework } from '../utils/detectFramework.js';
+import { findTitle } from '../utils/title.js';
+import { processListItem } from './listItems.js';
+
+export function retrieveNavItems(rootNode: Element): Array<NavigationEntry> {
+  const result: Array<NavigationEntry> = [];
+
+  let rootSectionTagName = 'li';
+  switch (framework.vendor) {
+    case 'docusaurus':
+      rootSectionTagName = 'li';
+      break;
+    case 'gitbook':
+      rootSectionTagName = 'li';
+      break;
+    case 'readme':
+      rootSectionTagName = 'section';
+      break;
+  }
+
+  let innerSectionTagName = 'div';
+  switch (framework.vendor) {
+    case 'docusaurus':
+      innerSectionTagName = 'li';
+      break;
+    case 'gitbook':
+      innerSectionTagName = 'div';
+      break;
+    case 'readme':
+      innerSectionTagName = 'h2';
+      break;
+  }
+
+  visit(rootNode, 'element', function (node, index, parent) {
+    if (node.tagName === rootSectionTagName) node.tagName = 'li';
+    if (node.tagName !== 'li') return CONTINUE;
+
+    let title: string | undefined = undefined;
+    if (
+      node.children[0] &&
+      node.children[1] &&
+      node.children[1].type === 'element' &&
+      node.children[1].tagName === 'ul' &&
+      node.children[0].type === 'element' &&
+      node.children[0].tagName === 'div' &&
+      node.children[0].children.filter((child) => child.type === 'text').length ===
+        node.children[0].children.length
+    )
+      title = findTitle(node.children[0], { delete: false });
+
+    if (
+      framework.vendor === 'readme' &&
+      node.children.length === 2 &&
+      node.children[1] &&
+      node.children[1].type === 'element' &&
+      node.children[1].tagName === 'ul' &&
+      typeof index === 'number' &&
+      parent
+    ) {
+      node.children = [
+        {
+          type: 'element',
+          tagName: 'div',
+          properties: {},
+          children: node.children,
+        },
+      ];
+    }
+
+    const entry = processListItem(node, {
+      sectionTagName: innerSectionTagName,
+      childListTagName: 'ul',
+      title,
+    });
+
+    if (entry !== undefined) {
+      result.push(entry);
+      return SKIP;
+    }
+    return CONTINUE;
+  });
+
+  return result;
+}

package/src/nav/root.ts

@@ -0,0 +1,47 @@
+import type { Root as HastRoot, Element } from 'hast';
+import { visit, EXIT } from 'unist-util-visit';
+
+import { framework } from '../utils/detectFramework.js';
+
+export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined {
+  let rootTagName = 'aside';
+  switch (framework.vendor) {
+    case 'docusaurus':
+      rootTagName = 'nav';
+      break;
+    case 'gitbook':
+      rootTagName = 'aside';
+      break;
+    case 'readme':
+      rootTagName = 'nav';
+      break;
+  }
+
+  let rootSelector = 'page-no-toc:hidden';
+  switch (framework.vendor) {
+    case 'docusaurus':
+      rootSelector = 'menu';
+      break;
+    case 'gitbook':
+      rootSelector = 'page-no-toc:hidden';
+      break;
+    case 'readme':
+      rootSelector = 'rm-Sidebar';
+      break;
+  }
+
+  let element: Element | undefined = undefined;
+  visit(rootNode, 'element', function (node) {
+    if (
+      node.tagName === rootTagName &&
+      node.properties.className &&
+      Array.isArray(node.properties.className) &&
+      node.properties.className.includes(rootSelector)
+    ) {
+      element = node;
+      return EXIT;
+    }
+  });
+
+  return element;
+}

package/src/openapi/generateOpenApiPages.ts

@@ -167,8 +167,8 @@ const createOpenApiFrontmatter = async (
   path: string
 ) => {
   const data = `---
-openapi: ${method} ${path}
----`;
+ openapi: ${method} ${path}
+ ---`;
 
   await outputFile(filename, data);
 };

package/src/root/retrieve.ts

@@ -0,0 +1,52 @@
+import type { Root as HastRoot, Element } from 'hast';
+import { visit, EXIT } from 'unist-util-visit';
+
+import { framework } from '../utils/detectFramework.js';
+
+export function retrieveRootContent(rootNode: HastRoot): Element | undefined {
+  let rootTagName = 'main';
+  switch (framework.vendor) {
+    case 'docusaurus':
+      rootTagName = 'article';
+      break;
+    case 'gitbook':
+      rootTagName = 'main';
+      break;
+    case 'readme':
+      rootTagName = 'section';
+      break;
+  }
+
+  let rootSelector = 'break-anywhere';
+  switch (framework.vendor) {
+    case 'docusaurus':
+      rootSelector = '';
+      break;
+    case 'gitbook':
+      rootSelector = '';
+      break;
+    case 'readme':
+      rootSelector = 'content-body';
+      break;
+  }
+
+  let element: Element | undefined = undefined;
+  visit(rootNode, 'element', function (node) {
+    if (node.tagName === rootTagName) {
+      if (rootSelector) {
+        if (
+          node.properties.className &&
+          Array.isArray(node.properties.className) &&
+          node.properties.className.includes(rootSelector)
+        ) {
+          element = node;
+        }
+      } else {
+        element = node;
+        return EXIT;
+      }
+    }
+  });
+
+  return element;
+}

package/src/scrapingPipeline/group.ts

@@ -0,0 +1,62 @@
+import type { Browser } from 'puppeteer';
+
+import type { Result } from '../types/result.js';
+import { getErrorMessage } from '../utils/errors.js';
+import { log } from '../utils/log.js';
+import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
+import { scrapePage } from './page.js';
+
+export async function scrapePageGroup(
+  navGroup: Array<URL>,
+  needsBrowser: boolean,
+  opts: {
+    externalLinks: boolean;
+    rootPaths?: Array<string>;
+  } = { externalLinks: false }
+): Promise<Array<Result<[string, string]>>> {
+  const browser: Browser | undefined = needsBrowser ? await startPuppeteer() : undefined;
+
+  try {
+    const res = await Promise.all(
+      navGroup.map(async (url, index) => {
+        try {
+          if (opts.externalLinks) {
+            const res = scrapePage(`external-link-${index}`, url, { externalLink: true });
+            return res;
+          }
+
+          let isOverviewPage = false;
+          if (url.toString().endsWith('/mintie_overview')) {
+            isOverviewPage = true;
+            url = new URL(url.toString().replace('/mintie_overview', ''));
+          }
+
+          const html = await fetchPageHtml(url, browser);
+          const res = scrapePage(html, url, {
+            externalLink: false,
+            isOverviewPage,
+            rootPath: opts.rootPaths ? opts.rootPaths[index] : undefined,
+          });
+          return res;
+        } catch (error) {
+          const errorMessage = getErrorMessage(error);
+          throw new Error(`We encountered an error when scraping ${url}${errorMessage}`);
+        }
+      })
+    );
+    return res;
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    log(
+      `We encountered an error when scraping the page group from ${
+        navGroup[0]?.origin ?? 'the URL provided'
+      }${errorMessage}`
+    );
+    console.error(error);
+    throw error;
+  } finally {
+    if (browser) {
+      await browser.close();
+    }
+  }
+}

package/src/scrapingPipeline/icon.ts

@@ -0,0 +1,26 @@
+import type { Root as HastRoot } from 'hast';
+import { EXIT, visit } from 'unist-util-visit';
+
+import { downloadImage } from '../utils/images.js';
+
+export async function downloadFavicon(hast: HastRoot): Promise<string | undefined> {
+  let src: string = '';
+  visit(hast, 'element', function (node) {
+    if (
+      node.tagName === 'link' &&
+      typeof node.properties.rel === 'string' &&
+      (node.properties.rel === 'icon' || node.properties.rel === 'shortcut icon')
+    ) {
+      src = node.properties.href as string;
+      return EXIT;
+    }
+  });
+
+  if (!src) return undefined;
+
+  const res = await downloadImage(src, process.cwd());
+  if (!res.success) return undefined;
+  if (!res.data) return undefined;
+
+  return res.data[1];
+}

package/src/scrapingPipeline/images.ts

@@ -0,0 +1,67 @@
+import type { Root as MdastRoot } from 'mdast';
+import type { MdxJsxAttribute } from 'mdast-util-mdx-jsx';
+import { join, dirname } from 'node:path';
+import { CONTINUE, visit } from 'unist-util-visit';
+
+import type { Result } from '../types/result.js';
+import { downloadImage } from '../utils/images.js';
+import { createFilename } from '../utils/path.js';
+
+export async function downloadImagesFromFile(
+  root: MdastRoot,
+  url: string | URL
+): Promise<Array<Result<[string, string]>>> {
+  url = new URL(url);
+  const imageUrls: Array<string> = [];
+
+  visit(root, function (node) {
+    let imageUrl: string | undefined = undefined;
+    if (node.type === 'image') {
+      imageUrl = node.url;
+    } else if (node.type === 'mdxJsxFlowElement') {
+      imageUrl = node.attributes.find(
+        (attr) => attr.type === 'mdxJsxAttribute' && (attr.name === 'src' || attr.name === 'img')
+      )?.value as string | undefined;
+    }
+
+    if (!imageUrl) return CONTINUE;
+
+    if (imageUrl.startsWith('/')) {
+      imageUrl = new URL(imageUrl, url.origin).toString();
+    }
+    imageUrls.push(imageUrl);
+  });
+
+  const rootPath = join(process.cwd(), '/images');
+  const filename = createFilename(rootPath, url, '');
+  const localRootPath = filename ? dirname(filename) : rootPath;
+
+  const imageResults = await Promise.all(
+    imageUrls.map(async (imageUrl) => {
+      const result = await downloadImage(imageUrl, localRootPath);
+      return result;
+    })
+  );
+
+  const imagePathsMap = new Map<string, string>(
+    imageResults.filter((result) => result.success).map((result) => result.data as [string, string])
+  );
+
+  visit(root, function (node, index, parent) {
+    if (node.type === 'image') {
+      node.url = imagePathsMap.get(node.url) ?? node.url;
+      if (parent && typeof index === 'number') parent.children[index] = node;
+    } else if (node.type === 'mdxJsxFlowElement') {
+      const urlAttr = (node.attributes as Array<MdxJsxAttribute>).find(
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        (attr) => attr.type === 'mdxJsxAttribute' && (attr.name === 'src' || attr.name === 'img')
+      );
+      if (!urlAttr) return CONTINUE;
+
+      urlAttr.value = imagePathsMap.get(urlAttr.value as string) ?? urlAttr.value;
+      if (parent && typeof index === 'number') parent.children[index] = node;
+    }
+  });
+
+  return imageResults;
+}

package/src/scrapingPipeline/logo.ts

@@ -0,0 +1,127 @@
+import type { Root as HastRoot, Element } from 'hast';
+import { join } from 'node:path';
+import { Browser } from 'puppeteer';
+import { EXIT, visit } from 'unist-util-visit';
+
+import { framework } from '../utils/detectFramework.js';
+import { downloadImage } from '../utils/images.js';
+import { fetchPageHtml } from '../utils/network.js';
+import { htmlToHast } from './root.js';
+
+function findReadmeLogoNodes(root: HastRoot): Array<Element> | undefined {
+  const elements: Array<Element> = [];
+  visit(root, 'element', function (node) {
+    if (
+      node.tagName === 'img' &&
+      Array.isArray(node.properties.className) &&
+      node.properties.className.includes('rm-Logo-img')
+    )
+      elements.push(node);
+  });
+  return elements.length ? elements : undefined;
+}
+
+function findGitBookLogoNodes(root: HastRoot): Array<Element> | undefined {
+  const elements: Array<Element> = [];
+  visit(root, 'element', function (node) {
+    if (node.tagName === 'img' && node.properties.alt === 'Logo') {
+      elements.push(node);
+    }
+  });
+  return elements.length ? elements : undefined;
+}
+
+function findDocusaurusLogoNodes(root: HastRoot): Array<Element> | undefined {
+  const elements: Array<Element> = [];
+  visit(root, 'element', function (node) {
+    if (
+      node.tagName === 'div' &&
+      Array.isArray(node.properties.className) &&
+      node.properties.className.includes('navbar__brand')
+    ) {
+      visit(node, 'element', function (subNode) {
+        if (subNode.tagName === 'img') elements.push(subNode);
+      });
+      return EXIT;
+    }
+  });
+  return elements.length ? elements : undefined;
+}
+
+async function findLogosFromHtml(
+  html: string,
+  downloadFn: (root: HastRoot) => Array<Element> | undefined,
+  filepaths: Array<string>
+) {
+  const hast = htmlToHast(html);
+  const imgNodes = downloadFn(hast);
+
+  if (imgNodes) {
+    filepaths.push(
+      ...(await Promise.all(
+        imgNodes.map(async (node) => {
+          const res = await downloadImage(
+            node.properties.src as string,
+            join(process.cwd(), 'images')
+          );
+
+          if (res.success && res.data) {
+            return res.data[1];
+          } else {
+            return '';
+          }
+        })
+      ))
+    );
+  }
+
+  filepaths.forEach((filepath, index) => {
+    if (!filepath) filepaths.splice(index, 1);
+  });
+}
+
+export async function downloadLogos(
+  url: string | URL,
+  browser: Browser | undefined
+): Promise<string | { light: string; dark: string } | undefined> {
+  url = new URL(url);
+  const filepaths: Array<string> = [];
+  if (browser) {
+    const htmls: Array<string> = [];
+
+    const page = await browser.newPage();
+    await page.goto(url.toString(), {
+      waitUntil: 'networkidle2',
+    });
+
+    htmls.push(await page.content());
+    await page.click('.rm-ThemeToggle');
+    htmls.push(await page.content());
+
+    await Promise.all(
+      htmls.map(async (html) => {
+        return await findLogosFromHtml(html, findReadmeLogoNodes, filepaths);
+      })
+    );
+  } else {
+    const html = await fetchPageHtml(url);
+    await findLogosFromHtml(
+      html,
+      framework.vendor === 'gitbook' ? findGitBookLogoNodes : findDocusaurusLogoNodes,
+      filepaths
+    );
+  }
+
+  if (browser) await browser.close();
+
+  const uniqueFilepaths = [...new Set(filepaths).values()];
+
+  return uniqueFilepaths.length === 1
+    ? uniqueFilepaths[0]
+    : uniqueFilepaths.length > 1
+      ? {
+          light: uniqueFilepaths[0] as string,
+          dark: uniqueFilepaths[1] as string,
+        }
+      : undefined;
+}