@mintlify/scraping 3.0.187 → 3.0.188

package/src/scrapingPipeline/page.ts

@@ -0,0 +1,130 @@
+import type { Root as HastRoot } from 'hast';
+import type { Root as MdastRoot } from 'mdast';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import remarkStringify from 'remark-stringify';
+import { unified } from 'unified';
+
+import { convertHeaderLinksToText } from '../components/link.js';
+import { CONTENT_FAILURE_MSG, MDAST_FAILURE_MSG } from '../constants.js';
+import {
+  createCallout,
+  createCard,
+  createAccordion,
+  createAccordionGroup,
+  createFrame,
+  createCodeGroup,
+  createTabs,
+  createCardGroup,
+} from '../customComponents/create.js';
+import { rehypeToRemarkCustomComponents } from '../customComponents/plugin.js';
+import { selectiveRehypeRemark } from '../customComponents/selective.js';
+import { retrieveRootContent } from '../root/retrieve.js';
+import type { Result } from '../types/result.js';
+import { unifiedRemoveClassNames } from '../utils/className.js';
+import { detectFramework, framework } from '../utils/detectFramework.js';
+import { unifiedRemoveEmptyParagraphs } from '../utils/emptyParagraphs.js';
+import { getErrorMessage, logErrorResults } from '../utils/errors.js';
+import { escapeCharactersOutsideCodeBlocks } from '../utils/escape.js';
+import { write, writePage } from '../utils/file.js';
+import { log } from '../utils/log.js';
+import { unifiedRemoveNestedRoots } from '../utils/nestedRoots.js';
+import { unifiedRemovePositions } from '../utils/position.js';
+import { removeLeadingSlash, removeTrailingSlash } from '../utils/strings.js';
+import { getDescriptionFromRoot, getTitleFromHeading } from '../utils/title.js';
+import { downloadImagesFromFile } from './images.js';
+import { htmlToHast } from './root.js';
+
+export async function scrapePage(
+  html: string,
+  url: string | URL,
+  opts: {
+    externalLink: boolean;
+    isOverviewPage?: boolean;
+    rootPath?: string;
+  } = { externalLink: false }
+): Promise<Result<[string, string]>> {
+  url = new URL(url);
+
+  if (opts.externalLink) {
+    const filename = html;
+    const filenameWithExt = `${filename}.mdx`;
+    writePage(filenameWithExt, '', '', '', url.toString());
+    return { success: true, data: [url.toString(), filename] };
+  }
+
+  const hast = htmlToHast(html);
+
+  if (!framework.vendor) detectFramework(hast);
+
+  const urlStr = url.toString();
+  const content = retrieveRootContent(hast);
+  if (!content) return { success: false, message: `${urlStr}: ${CONTENT_FAILURE_MSG}` };
+
+  const contentAsRoot: HastRoot = {
+    type: 'root',
+    children: [content],
+  };
+
+  const mdastTree: MdastRoot = unified()
+    .use(createCard)
+    .use(createAccordion)
+    .use(createFrame)
+    .use(createTabs)
+    .use(createCallout)
+    .use(createCardGroup)
+    .use(createAccordionGroup)
+    .use(createCodeGroup)
+    .use(unifiedRemoveClassNames)
+    .use(unifiedRemovePositions)
+    .use(unifiedRemoveEmptyParagraphs)
+    .use(escapeCharactersOutsideCodeBlocks)
+    .use(selectiveRehypeRemark)
+
+    // Cleans up any nested components left untouched
+    // by `selectiveRehypeRemark`, and converts them to
+    // MDX compatible components
+    .use(rehypeToRemarkCustomComponents)
+    .use(convertHeaderLinksToText)
+    .use(unifiedRemoveNestedRoots)
+    .runSync(contentAsRoot) as MdastRoot;
+
+  try {
+    const imageResults = await downloadImagesFromFile(mdastTree, url);
+    logErrorResults(`scraping images from ${url.toString()}`, imageResults);
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    log(`We encountered an error when scraping the images from ${url.toString()}${errorMessage}`);
+    throw error;
+  }
+
+  const title = getTitleFromHeading(mdastTree);
+  const description = getDescriptionFromRoot(mdastTree);
+
+  try {
+    const result = unified()
+      .use(remarkMdx)
+      .use(remarkGfm)
+      // @ts-expect-error remarkStringify errors even if used for valid code from documentation examples
+      .use(remarkStringify)
+      .stringify(mdastTree);
+
+    if (opts.rootPath) {
+      url = new URL(opts.rootPath, url.origin);
+    } else if (url.origin === removeTrailingSlash(url.toString())) {
+      url = new URL('home', new URL(url).origin);
+    }
+
+    writePage(url, opts.isOverviewPage ? 'Overview' : title, description, String(result));
+    return {
+      success: true,
+      data: opts.rootPath
+        ? [removeLeadingSlash(removeTrailingSlash(new URL(urlStr).pathname)), opts.rootPath]
+        : undefined,
+    };
+  } catch (error) {
+    write('error.json', JSON.stringify(mdastTree, undefined, 2));
+    const errorMessage = getErrorMessage(error);
+    return { success: false, message: `${urlStr}: ${MDAST_FAILURE_MSG}${errorMessage}` };
+  }
+}

package/src/scrapingPipeline/root.ts

@@ -0,0 +1,10 @@
+import type { Root as HastRoot } from 'hast';
+import rehypeParse from 'rehype-parse';
+import { unified } from 'unified';
+
+import { unifiedRemovePositions } from '../utils/position.js';
+
+export function htmlToHast(html: string): HastRoot {
+  // @ts-expect-error remarkStringify errors even if used for valid code from documentation examples
+  return unified().use(rehypeParse).use(unifiedRemovePositions).parse(html) as HastRoot;
+}

package/src/scrapingPipeline/site.ts

@@ -0,0 +1,161 @@
+import { MintConfig, Navigation, Tab } from '@mintlify/models';
+import type { Root as HastRoot } from 'hast';
+import traverse from 'traverse';
+
+import { NAV_FAILURE_MSG } from '../constants.js';
+import { iterateOverNavItems } from '../nav/iterate.js';
+import { retrieveNavItems } from '../nav/retrieve.js';
+import { retrieveRootNavElement } from '../nav/root.js';
+import type { Result } from '../types/result.js';
+import { detectFramework, framework } from '../utils/detectFramework.js';
+import { logErrorResults } from '../utils/errors.js';
+import { startPuppeteer } from '../utils/network.js';
+import { INDEX_NAMES, iterateThroughReservedNames } from '../utils/reservedNames.js';
+import { removeTrailingSlash } from '../utils/strings.js';
+import { scrapePageGroup } from './group.js';
+import { downloadFavicon } from './icon.js';
+import { downloadLogos } from './logo.js';
+import { htmlToHast } from './root.js';
+
+export async function scrapeSite(
+  html: string,
+  url: string | URL,
+  opts: { hast?: HastRoot; tabs?: Array<Tab> } = {}
+): Promise<Result<MintConfig>> {
+  let hast = opts.hast;
+  if (!hast) hast = htmlToHast(html);
+
+  url = new URL(url);
+  const origin = url.origin;
+
+  detectFramework(hast);
+
+  const sidebar = retrieveRootNavElement(hast);
+  if (!sidebar) return { success: false, message: `${url.toString()}: ${NAV_FAILURE_MSG}` };
+
+  const navItems = retrieveNavItems(sidebar);
+
+  if (origin === '') {
+    return { success: false, message: `invalid URL provided to scrape site: ${url}` };
+  }
+
+  const listOfLinks = iterateOverNavItems(navItems, origin);
+  if (listOfLinks.length === 0) {
+    return { success: false, message: `no navigation links were able to be found: ${url}` };
+  }
+
+  const needsBrowser = framework.vendor === 'gitbook';
+
+  const externalLinks = listOfLinks.filter((url) => url.origin !== origin);
+  const internalLinks = listOfLinks.filter(
+    (url) => url.origin === origin && removeTrailingSlash(url.toString()) !== origin
+  );
+  const rootLinks = listOfLinks.filter(
+    (url) => url.origin === origin && removeTrailingSlash(url.toString()) === origin
+  );
+
+  const allPathnames = [
+    ...internalLinks.map((url) => url.toString()),
+    ...rootLinks.map((url) => url.toString()),
+  ];
+  const rootPaths = rootLinks.map(() => {
+    const name = iterateThroughReservedNames(INDEX_NAMES, allPathnames);
+    allPathnames.push(name);
+    return name;
+  });
+
+  try {
+    const externalResults = await scrapePageGroup(externalLinks, needsBrowser, {
+      externalLinks: true,
+    });
+    const internalResults = await scrapePageGroup(internalLinks, needsBrowser);
+    const rootResults = await scrapePageGroup(rootLinks, needsBrowser, {
+      externalLinks: false,
+      rootPaths,
+    });
+
+    const externalLinkReplaceMap = new Map<string, string>(
+      externalResults
+        .filter((result) => result.success)
+        .map((result) => result.data as [string, string])
+    );
+
+    const rootPathReplaceMap = new Map<string, string>(
+      rootResults
+        .filter((result) => result.success)
+        .map((result) => result.data as [string, string])
+    );
+
+    traverse(navItems).forEach(function (value) {
+      if (typeof value === 'string') {
+        if (externalLinkReplaceMap.has(value)) {
+          this.update(externalLinkReplaceMap.get(value) ?? value);
+        } else if (rootPathReplaceMap.has(value)) {
+          this.update(rootPathReplaceMap.get(value) ?? value);
+        }
+      } else if (Array.isArray(value)) {
+        if (value.find((item) => externalLinkReplaceMap.has(item))) {
+          this.update(value.map((item) => externalLinkReplaceMap.get(item) ?? item));
+        } else if (value.find((item) => rootPathReplaceMap.has(item))) {
+          this.update(value.map((item) => rootPathReplaceMap.get(item) ?? item));
+        }
+      }
+    });
+
+    traverse(navItems).forEach(function (value) {
+      if (typeof value === 'string') {
+        this.update(value.replace('/mintie_overview', ''));
+      } else if (Array.isArray(value)) {
+        this.update(
+          value.map((item) =>
+            typeof item === 'string' ? item.replace('/mintie_overview', '') : item
+          )
+        );
+      }
+    });
+
+    navItems.forEach((navItem, index) => {
+      if (typeof navItem !== 'string') return;
+      const name = navItem
+        .split('-')
+        .map((str) => (str[0] ? `${str[0].toUpperCase()}${str.substring(1)}` : str))
+        .join(' ');
+
+      navItems[index] = {
+        group: name,
+        pages: [navItem],
+      };
+    });
+
+    logErrorResults('linking to external pages', externalResults);
+    logErrorResults('scraping your docs', [...internalResults, ...rootResults]);
+
+    const browser = needsBrowser ? await startPuppeteer() : undefined;
+
+    const favicon = await downloadFavicon(hast);
+    const logo = await downloadLogos(url, browser);
+
+    return {
+      success: true,
+      data: {
+        $schema: 'https://mintlify.com/schema.json',
+        name: '',
+        logo,
+        colors: {
+          primary: '',
+        },
+        favicon: favicon ?? '',
+        navigation: navItems as Navigation,
+        tabs: opts.tabs,
+      },
+    };
+  } catch (error) {
+    if (error instanceof Error) {
+      return { success: false, message: error.message };
+    }
+    return {
+      success: false,
+      message: 'An unknown error occurred when scraping this site. Please try again.',
+    };
+  }
+}

package/src/scrapingPipeline/tabs.ts

@@ -0,0 +1,87 @@
+import type { Navigation, NavigationEntry } from '@mintlify/models';
+import { MintConfig, Tab } from '@mintlify/models';
+
+import { retrieveTabLinks } from '../tabs/retrieveReadme.js';
+import type { Result } from '../types/result.js';
+import { detectFramework, framework } from '../utils/detectFramework.js';
+import { log } from '../utils/log.js';
+import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
+import { getTitleFromLink } from '../utils/title.js';
+import { downloadFavicon } from './icon.js';
+import { downloadLogos } from './logo.js';
+import { htmlToHast } from './root.js';
+import { scrapeSite } from './site.js';
+
+export async function scrapeAllSiteTabs(
+  html: string,
+  url: string | URL
+): Promise<Result<MintConfig>> {
+  const hast = htmlToHast(html);
+  url = new URL(url);
+
+  detectFramework(hast);
+
+  const needsBrowser = framework.vendor === 'gitbook';
+  const browser = needsBrowser ? await startPuppeteer() : undefined;
+
+  const favicon = await downloadFavicon(hast);
+  const logo = await downloadLogos(url, browser);
+
+  if (framework.vendor === 'readme' || framework.vendor === 'docusaurus') {
+    const links = retrieveTabLinks(hast);
+    if (
+      !links ||
+      !links.length ||
+      (links.length === 1 && links[0] && links[0].url === url.pathname)
+    )
+      return scrapeSite(html, url, { hast });
+
+    if (!links.find((link) => url.pathname.startsWith(link.url))) {
+      links.push({
+        name: getTitleFromLink(url.pathname),
+        url: url.pathname,
+      });
+    }
+
+    const results = await Promise.all(
+      links.map(async (tabEntry) => {
+        const newUrl = new URL(url);
+        newUrl.pathname = tabEntry.url;
+        const newHtml = await fetchPageHtml(newUrl, undefined);
+        return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry] });
+      })
+    );
+
+    const navigations: Array<NavigationEntry> = [];
+    const tabs: Array<Tab> = [];
+
+    const successes = results.filter((result) => result.success);
+    successes.forEach((result) => {
+      if (!result.data) return;
+      navigations.push(...result.data.navigation);
+      if (result.data.tabs) tabs.push(...result.data.tabs);
+    });
+
+    const failures = results.filter((result) => !result.success);
+    failures.forEach((result) => {
+      log('Failed to scrape tab: ' + result.message);
+    });
+
+    return {
+      success: true,
+      data: {
+        $schema: 'https://mintlify.com/schema.json',
+        name: '',
+        logo,
+        colors: {
+          primary: '',
+        },
+        favicon: favicon ?? '',
+        navigation: navigations as Navigation,
+        tabs,
+      },
+    };
+  }
+
+  return scrapeSite(html, url, { hast });
+}

package/src/tabs/retrieveReadme.ts

@@ -0,0 +1,99 @@
+import { Tab } from '@mintlify/models';
+import type { Root as HastRoot, Element } from 'hast';
+import { visit, EXIT, CONTINUE } from 'unist-util-visit';
+
+import { framework } from '../utils/detectFramework.js';
+import { findTitle, getTitleFromLink } from '../utils/title.js';
+
+export function retrieveTabLinks(rootNode: HastRoot): Array<Tab> | undefined {
+  if (framework.vendor !== 'readme' && framework.vendor !== 'docusaurus') return undefined;
+
+  let element: Element | undefined = undefined;
+  visit(rootNode, 'element', function (node) {
+    if (framework.vendor === 'readme') {
+      if (
+        node.tagName === 'header' &&
+        node.properties.className &&
+        Array.isArray(node.properties.className) &&
+        node.properties.className.includes('rm-Header')
+      ) {
+        element = node;
+        return EXIT;
+      }
+    }
+
+    if (framework.vendor === 'docusaurus') {
+      if (
+        node.tagName === 'nav' &&
+        node.properties.className &&
+        Array.isArray(node.properties.className) &&
+        node.properties.className.includes('navbar')
+      ) {
+        element = node;
+        return EXIT;
+      }
+    }
+  });
+
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!element) return undefined;
+
+  const links: Array<Tab> = [];
+  visit(element as Element, 'element', function (node) {
+    if (framework.vendor === 'readme') {
+      if (
+        node.tagName !== 'nav' &&
+        !(
+          node.tagName === 'div' &&
+          node.properties.className &&
+          Array.isArray(node.properties.className) &&
+          node.properties.className.includes('rm-Header-right')
+        )
+      )
+        return CONTINUE;
+
+      visit(node, 'element', function (subNode) {
+        if (
+          subNode.tagName !== 'a' ||
+          !subNode.properties.href ||
+          typeof subNode.properties.href !== 'string' ||
+          subNode.properties.href.startsWith('http')
+        )
+          return CONTINUE;
+        const title = findTitle(subNode);
+        links.push({
+          name: title || getTitleFromLink(subNode.properties.href),
+          url: subNode.properties.href,
+        });
+      });
+    }
+
+    if (framework.vendor === 'docusaurus') {
+      if (node.tagName !== 'nav') return CONTINUE;
+
+      visit(node, 'element', function (subNode, _, parent) {
+        if (
+          subNode.tagName !== 'a' ||
+          !subNode.properties.href ||
+          typeof subNode.properties.href !== 'string' ||
+          subNode.properties.href.startsWith('http') ||
+          !parent ||
+          parent.type !== 'element' ||
+          !Array.isArray(parent.properties.className) ||
+          parent.properties.className.length !== 1 ||
+          parent.properties.className[0] !== 'navbar__items' ||
+          parent.properties.className.includes('navbar__items--right')
+        )
+          return CONTINUE;
+
+        const title = findTitle(subNode);
+        links.push({
+          name: title || getTitleFromLink(subNode.properties.href),
+          url: subNode.properties.href,
+        });
+      });
+    }
+  });
+
+  return links;
+}

package/src/types/components.ts

@@ -0,0 +1,3 @@
+import { ESCAPED_COMPONENTS } from '../constants.js';
+
+export type EscapedComponent = (typeof ESCAPED_COMPONENTS)[number];

package/src/types/framework.ts

@@ -0,0 +1,10 @@
+export const docVendors = ['gitbook', 'readme', 'docusaurus', undefined] as const;
+export const docusaurusVersions = [2, 3] as const;
+
+export type DocusaurusVersion = (typeof docusaurusVersions)[number];
+export type FrameworkVendor = (typeof docVendors)[number];
+
+export type Framework = {
+  vendor: FrameworkVendor | undefined;
+  version: DocusaurusVersion | undefined;
+};

package/src/types/hast.ts

@@ -0,0 +1,12 @@
+import type { Root as HastRoot, Element, ElementContent } from 'hast';
+import type { MdxJsxFlowElementHast, MdxJsxTextElementHast } from 'mdast-util-mdx-jsx';
+
+export type HastNode = Element;
+export type HastNodeIndex = number | undefined;
+export type HastNodeParent =
+  | Element
+  | MdxJsxTextElementHast
+  | MdxJsxFlowElementHast
+  | HastRoot
+  | undefined;
+export type HastChildrenType = Array<ElementContent>;

package/src/types/result.ts

@@ -0,0 +1 @@
+export type Result<T> = { success: true; data?: T } | { success: false; message: string };

package/src/types/scrapeFunc.ts

@@ -0,0 +1,9 @@
+import type { Element } from 'hast';
+
+import type { HastNode, HastNodeIndex, HastNodeParent } from './hast.js';
+
+export type ScrapeFuncType = (
+  node: HastNode,
+  index: HastNodeIndex,
+  parent: HastNodeParent
+) => Element | undefined;

package/src/utils/append.ts

@@ -0,0 +1,9 @@
+export function dedupedAppend<T>(item: T, arr: Array<T>, prepend?: boolean): Array<T> {
+  if (arr.includes(item)) return arr;
+  if (prepend) {
+    arr.unshift(item);
+  } else {
+    arr.push(item);
+  }
+  return arr;
+}

package/src/utils/children.ts

@@ -0,0 +1,51 @@
+import {
+  type ElementContent,
+  type Root as HastRoot,
+  type Comment,
+  type Text,
+  type Element,
+  type RootContent as HastRootContent,
+} from 'hast';
+import { toMdast, defaultHandlers } from 'hast-util-to-mdast';
+import type { State, Handle } from 'hast-util-to-mdast';
+import type { RootContent as MdastRootContent, Root as MdastRoot } from 'mdast';
+import { unified } from 'unified';
+
+import { mdxJsxFlowElementHandler } from '../customComponents/selective.js';
+
+export function turnChildrenIntoMdx(
+  children: Array<HastRootContent | ElementContent | Element | Comment | Text>,
+  opts: { jsxImages: boolean } = { jsxImages: false }
+): Array<MdastRootContent> {
+  const hast: HastRoot = {
+    type: 'root',
+    children: children,
+  };
+
+  const handlers: Record<string, Handle> = { ...defaultHandlers };
+  if (opts.jsxImages) {
+    handlers['img'] = function (_: State, node: Element) {
+      Object.keys(node.properties).forEach((key) => {
+        if (key !== 'src') delete node.properties[key];
+      });
+      return mdxJsxFlowElementHandler(_, node);
+    };
+  }
+
+  const mdxAst = unified()
+    .use(function () {
+      return function (tree: HastRoot): MdastRoot {
+        const newTree = toMdast(tree, {
+          handlers,
+        }) as MdastRoot;
+        return newTree;
+      };
+    })
+    .runSync(hast);
+
+  mdxAst.children.forEach((child, index) => {
+    if (child.type === 'html') mdxAst.children.splice(index, 1);
+  });
+
+  return mdxAst.children;
+}

package/src/utils/className.ts

@@ -0,0 +1,14 @@
+import type { Element } from 'hast';
+import { visit } from 'unist-util-visit';
+
+export function unifiedRemoveClassNames() {
+  return function (node: Element) {
+    return removeClassNames(node);
+  };
+}
+
+export function removeClassNames(node: Element) {
+  return visit(node, 'element', function (subNode) {
+    if ('properties' in subNode) delete subNode.properties.className;
+  });
+}

package/src/utils/detectFramework.ts

@@ -0,0 +1,72 @@
+import type { Root } from 'hast';
+import { visit, EXIT, CONTINUE } from 'unist-util-visit';
+
+import type { Framework } from '../types/framework.js';
+import { log } from './log.js';
+
+export const framework: Framework = {
+  vendor: undefined,
+  version: undefined,
+};
+
+export function detectFramework(rootHast: Root): void {
+  visit(rootHast, 'element', function (node) {
+    if (
+      node.tagName === 'link' &&
+      Array.isArray(node.properties.rel) &&
+      node.properties.rel.includes('preconnect') &&
+      node.properties.href === 'https://api.gitbook.com'
+    ) {
+      framework.vendor = 'gitbook';
+      framework.version = undefined;
+      return EXIT;
+    }
+
+    // All of the other docs vendors rely on
+    // the `meta` element as well as a `name`
+    // property which should be a string
+    if (node.tagName !== 'meta' || typeof node.properties.name !== 'string') return CONTINUE;
+
+    switch (node.properties.name) {
+      case 'readme-deploy':
+        framework.vendor = 'readme';
+        framework.version = undefined;
+        return EXIT;
+
+      // case "intercom: trackingEvent":
+      //   framework.vendor = "intercom";
+      //   framework.version = undefined;
+      //   return EXIT;
+
+      case 'generator':
+        if (
+          typeof node.properties.content === 'string' &&
+          node.properties.content.includes('Docusaurus')
+        ) {
+          framework.vendor = 'docusaurus';
+          const meta = node.properties.content;
+          if (meta.includes('v3')) {
+            framework.version = 3;
+          } else if (meta.includes('v2')) {
+            framework.version = 2;
+          } else if (meta.includes('v1')) {
+            log(
+              'We detected Docusaurus version 1 but we only support scraping versions 2 and 3',
+              'error'
+            );
+            framework.vendor = undefined;
+            framework.version = undefined;
+          }
+          return EXIT;
+        }
+    }
+  });
+
+  if (framework.vendor) {
+    log('Successfully detected documentation vendor: ' + framework.vendor);
+  } else {
+    log('Failed to detect documentation vendor; please contact support@mintlify.com');
+    framework.version = undefined;
+    process.exit(1);
+  }
+}