@mintlify/scraping 3.0.186 → 3.0.188

package/src/components/CardGroup.ts

@@ -0,0 +1,69 @@
+import type { Element } from 'hast';
+
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+
+export function gitBookScrapeCardGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'Card') return undefined;
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!parent || !parent.children) return undefined;
+
+  let cardCount = 0;
+  for (const child of parent.children) {
+    if (child.type === 'element' && child.tagName === 'Card') cardCount++;
+  }
+
+  if (cardCount === parent.children.length) {
+    parent.type = 'element';
+    (parent as Element).tagName = 'CardGroup';
+  }
+
+  return undefined;
+}
+
+export function readmeScrapeCardGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'Card') return undefined;
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!parent || !parent.children) return undefined;
+
+  let cardCount = 0;
+  for (const child of parent.children) {
+    if (child.type === 'element' && child.tagName === 'Card') cardCount++;
+  }
+
+  if (cardCount === parent.children.length) {
+    parent.type = 'element';
+    (parent as Element).tagName = 'CardGroup';
+  }
+
+  return undefined;
+}
+
+export function docusaurusScrapeCardGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'Card') return undefined;
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!parent || !parent.children) return undefined;
+
+  let cardCount = 0;
+  for (const child of parent.children) {
+    if (child.type === 'element' && child.tagName === 'Card') cardCount++;
+  }
+
+  if (cardCount === parent.children.length) {
+    parent.type = 'element';
+    (parent as Element).tagName = 'CardGroup';
+  }
+
+  return undefined;
+}

package/src/components/CodeGroup.ts

@@ -0,0 +1,209 @@
+import type { Element, ElementContent } from 'hast';
+import type { Code } from 'mdast';
+import { visit, CONTINUE, EXIT } from 'unist-util-visit';
+
+import { assertIsDefined } from '../assert.js';
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+import { turnChildrenIntoMdx } from '../utils/children.js';
+
+function tabContainsOnlyCode(node: Element): boolean {
+  let tabsCount = 0;
+  let onlyCodeCount = 0;
+
+  visit(node, 'element', function (subNode) {
+    if (subNode.properties.role && subNode.properties.role === 'tabpanel') tabsCount++;
+    visit(subNode, 'element', function (tabNode, index, parent) {
+      if (tabNode.tagName === 'button' && typeof index === 'number' && parent) {
+        parent.children.splice(index, 1);
+      }
+    });
+    if (
+      subNode.children[0] &&
+      subNode.children[1] &&
+      subNode.children[2] &&
+      subNode.children[0].type === 'element' &&
+      subNode.children[0].children.length === 1 &&
+      subNode.children[1].type === 'element' &&
+      subNode.children[1].children.length === 1 &&
+      subNode.children[2].type === 'element' &&
+      subNode.children[2].children.length > 1 &&
+      subNode.children[2].children.length < 4 &&
+      subNode.children[2].children.find(
+        (child) => child.type === 'element' && (child.tagName === 'pre' || child.tagName === 'code')
+      )
+    ) {
+      return onlyCodeCount++;
+    }
+  });
+  return onlyCodeCount === tabsCount;
+}
+
+export function gitBookScrapeCodeGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'div' || !node.properties.role || node.properties.role !== 'tablist') {
+    return undefined;
+  }
+
+  if (!tabContainsOnlyCode(node)) return undefined;
+
+  const titles: Array<string> = [];
+  visit(node, 'element', function (subNode) {
+    if (subNode.tagName !== 'button') return CONTINUE;
+    visit(subNode, 'text', function (textNode) {
+      titles.push(textNode.value);
+      return EXIT;
+    });
+  });
+
+  assertIsDefined(parent);
+  parent.children.shift();
+
+  let lang = '';
+  visit(node, 'element', function (subNode) {
+    if (subNode.tagName === 'code' && 'id' in subNode.properties) {
+      lang = subNode.properties.id as string;
+      return EXIT;
+    }
+  });
+
+  const children = turnChildrenIntoMdx(parent.children) as Array<ElementContent>;
+  const tabChildren: Array<ElementContent> = [];
+  for (let childIndex = 0; childIndex < children.length; childIndex++) {
+    const child = children[childIndex];
+    tabChildren.push({
+      type: 'code',
+      lang: lang,
+      meta: titles[childIndex],
+      value: (child as unknown as Code).value,
+    } as unknown as ElementContent);
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'CodeGroup',
+    properties: {},
+    children: tabChildren as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function readmeScrapeCodeGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (
+    node.tagName !== 'div' ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    !node.properties.className.includes('CodeTabs')
+  ) {
+    return undefined;
+  }
+
+  if (node.children.length !== 2 || !node.children[0] || !node.children[1]) return undefined;
+  const tabTitles = node.children[0];
+
+  const titles: Array<string> = [];
+  visit(tabTitles, 'element', function (subNode) {
+    visit(subNode, 'text', function (textNode) {
+      titles.push(textNode.value);
+      return EXIT;
+    });
+  });
+
+  assertIsDefined(parent);
+  node.children.shift();
+
+  let lang = '';
+  visit(node, 'element', function (subNode, _, subParent) {
+    if (
+      subNode.tagName === 'code' &&
+      subParent &&
+      subParent.type === 'element' &&
+      subParent.tagName === 'pre' &&
+      'dataLang' in subNode.properties
+    ) {
+      lang = subNode.properties.dataLang as string;
+      return EXIT;
+    }
+  });
+
+  const children = turnChildrenIntoMdx(node.children) as Array<ElementContent>;
+  const tabChildren: Array<ElementContent> = [];
+  for (let childIndex = 0; childIndex < children.length; childIndex++) {
+    const child = children[childIndex];
+    tabChildren.push({
+      type: 'code',
+      lang: lang,
+      meta: titles[childIndex],
+      value: (child as unknown as Code).value,
+    } as unknown as ElementContent);
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'CodeGroup',
+    properties: {},
+    children: tabChildren as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function docusaurusScrapeCodeGroup(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'div' || !node.properties.role || node.properties.role !== 'tablist') {
+    return undefined;
+  }
+
+  if (!tabContainsOnlyCode(node)) return undefined;
+
+  const titles: Array<string> = [];
+  visit(node, 'element', function (subNode) {
+    if (subNode.tagName !== 'li') return CONTINUE;
+    visit(subNode, 'text', function (textNode) {
+      titles.push(textNode.value);
+      return EXIT;
+    });
+  });
+
+  assertIsDefined(parent);
+  parent.children.shift();
+
+  let lang = '';
+  visit(node, 'element', function (subNode) {
+    if (Array.isArray(subNode.properties.className) && subNode.properties.className.length > 0) {
+      lang = subNode.properties.className[0] as string;
+      lang = lang.replace('language-', '');
+    }
+  });
+
+  const children = turnChildrenIntoMdx(parent.children) as Array<ElementContent>;
+  const tabChildren: Array<ElementContent> = [];
+  for (let childIndex = 0; childIndex < children.length; childIndex++) {
+    const child = children[childIndex];
+    tabChildren.push({
+      type: 'code',
+      lang: lang,
+      meta: titles[childIndex],
+      value: (child as unknown as Code).value,
+    } as unknown as ElementContent);
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'CodeGroup',
+    properties: {},
+    children: tabChildren as Array<ElementContent>,
+  };
+
+  return newNode;
+}

package/src/components/Frame.ts

@@ -0,0 +1,86 @@
+import type { Element, ElementContent } from 'hast';
+
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+import { turnChildrenIntoMdx } from '../utils/children.js';
+import { findTitle } from '../utils/title.js';
+
+export function gitBookScrapeFrame(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (
+    (node.tagName === 'figure' &&
+      node.children.find(
+        (subNode) => subNode.type === 'element' && subNode.tagName === 'picture'
+      )) ||
+    (node.tagName !== 'picture' && node.tagName !== 'figure')
+  )
+    return undefined;
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Frame',
+    properties: {},
+    children: turnChildrenIntoMdx(node.children, { jsxImages: true }) as Array<ElementContent>,
+  };
+
+  const caption = findTitle(newNode);
+  if (caption) newNode.properties.caption = caption;
+
+  return newNode;
+}
+
+export function readmeScrapeFrame(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (
+    (node.tagName === 'figure' &&
+      node.children.find(
+        (subNode) => subNode.type === 'element' && subNode.tagName === 'picture'
+      )) ||
+    (node.tagName !== 'picture' && node.tagName !== 'figure')
+  )
+    return undefined;
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Frame',
+    properties: {},
+    children: turnChildrenIntoMdx(node.children, { jsxImages: true }) as Array<ElementContent>,
+  };
+
+  const caption = findTitle(newNode);
+  if (caption) newNode.properties.caption = caption;
+
+  return newNode;
+}
+
+export function docusaurusScrapeFrame(
+  node: HastNode,
+  _: HastNodeIndex,
+  __: HastNodeParent
+): Element | undefined {
+  if (
+    (node.tagName === 'figure' &&
+      node.children.find(
+        (subNode) => subNode.type === 'element' && subNode.tagName === 'picture'
+      )) ||
+    (node.tagName !== 'picture' && node.tagName !== 'figure')
+  )
+    return undefined;
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Frame',
+    properties: {},
+    children: turnChildrenIntoMdx(node.children, { jsxImages: true }) as Array<ElementContent>,
+  };
+
+  const caption = findTitle(newNode);
+  if (caption) newNode.properties.caption = caption;
+
+  return newNode;
+}

package/src/components/Tabs.ts

@@ -0,0 +1,154 @@
+import type { Element, ElementContent } from 'hast';
+import { visit, CONTINUE, EXIT } from 'unist-util-visit';
+
+import { assertIsDefined } from '../assert.js';
+import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
+import { turnChildrenIntoMdx } from '../utils/children.js';
+
+export function gitBookScrapeTabs(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'div' || !node.properties.role || node.properties.role !== 'tablist') {
+    return undefined;
+  }
+
+  const titles: Array<string> = [];
+  visit(node, 'element', function (subNode) {
+    if (subNode.tagName !== 'button') return CONTINUE;
+    visit(subNode, 'text', function (textNode) {
+      titles.push(textNode.value);
+      return EXIT;
+    });
+  });
+
+  assertIsDefined(parent);
+  parent.children.shift();
+
+  const children = turnChildrenIntoMdx(parent.children) as Array<ElementContent>;
+  const tabChildren: Array<ElementContent> = [];
+  for (let childIndex = 0; childIndex < children.length; childIndex++) {
+    const child = children[childIndex];
+    if (child) {
+      tabChildren.push({
+        type: 'element',
+        tagName: 'Tab',
+        properties: {
+          title: titles[childIndex],
+        },
+        children: [child],
+      });
+    }
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Tabs',
+    properties: {},
+    children: tabChildren as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function readmeScrapeTabs(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (
+    (node.tagName !== 'div' && node.tagName !== 'a') ||
+    !node.properties.className ||
+    !Array.isArray(node.properties.className) ||
+    !node.properties.className.includes('tabbed-component')
+  ) {
+    return undefined;
+  }
+
+  if (node.children.length < 2 || !node.children[0] || !node.children[1]) return undefined;
+  const tabTitles = node.children[0];
+
+  const titles: Array<string> = [];
+  visit(tabTitles, 'element', function (subNode) {
+    visit(subNode, 'text', function (textNode) {
+      titles.push(textNode.value);
+      return EXIT;
+    });
+  });
+
+  assertIsDefined(parent);
+  node.children.shift();
+
+  const children = turnChildrenIntoMdx(node.children) as Array<ElementContent>;
+  const tabChildren: Array<ElementContent> = [];
+  for (let childIndex = 0; childIndex < children.length; childIndex++) {
+    const child = children[childIndex];
+    if (child) {
+      tabChildren.push({
+        type: 'element',
+        tagName: 'Tab',
+        properties: {
+          title: titles[childIndex],
+        },
+        children: [child],
+      });
+    }
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Tabs',
+    properties: {},
+    children: tabChildren as Array<ElementContent>,
+  };
+
+  return newNode;
+}
+
+export function docusaurusScrapeTabs(
+  node: HastNode,
+  _: HastNodeIndex,
+  parent: HastNodeParent
+): Element | undefined {
+  if (node.tagName !== 'div' || !node.properties.role || node.properties.role !== 'tablist') {
+    return undefined;
+  }
+
+  const titles: Array<string> = [];
+  visit(node, 'element', function (subNode) {
+    if (subNode.tagName !== 'li') return CONTINUE;
+    visit(subNode, 'text', function (textNode) {
+      titles.push(textNode.value);
+      return EXIT;
+    });
+  });
+
+  assertIsDefined(parent);
+  parent.children.shift();
+
+  const children = turnChildrenIntoMdx(parent.children) as Array<ElementContent>;
+  const tabChildren: Array<ElementContent> = [];
+  for (let childIndex = 0; childIndex < children.length; childIndex++) {
+    const child = children[childIndex];
+    if (child) {
+      tabChildren.push({
+        type: 'element',
+        tagName: 'Tab',
+        properties: {
+          title: titles[childIndex],
+        },
+        children: [child],
+      });
+    }
+  }
+
+  const newNode: Element = {
+    type: 'element',
+    tagName: 'Tabs',
+    properties: {},
+    children: tabChildren as Array<ElementContent>,
+  };
+
+  return newNode;
+}

package/src/components/link.ts

@@ -0,0 +1,17 @@
+import type { Root as MdastRoot } from 'mdast';
+import { visit } from 'unist-util-visit';
+
+export function convertHeaderLinksToText() {
+  return function (tree: MdastRoot) {
+    visit(tree, 'heading', function (node) {
+      visit(node, 'link', function (subNode, index, parent) {
+        if (subNode.url.startsWith('#') && subNode.title == null) {
+          if (parent && typeof index === 'number') {
+            parent.children.splice(index, 1);
+          }
+        }
+      });
+    });
+    return tree;
+  };
+}

package/src/constants.ts

@@ -1,22 +1,4 @@
-import os from 'os';
-import path from 'path';
-import * as url from 'url';
-
-// package installation location
-export const INSTALL_PATH = url.fileURLToPath(new URL('.', import.meta.url));
-
-export const HOME_DIR = os.homedir();
-
-export const DOT_MINTLIFY = path.join(HOME_DIR, '.mintlify');
-
-export const VERSION_PATH = path.join(DOT_MINTLIFY, 'mint', 'mint-version.txt');
-
-export const CLIENT_PATH = path.join(DOT_MINTLIFY, 'mint', 'client');
-
-export const MINT_PATH = path.join(DOT_MINTLIFY, 'mint');
-
-// command execution location
-export const CMD_EXEC_PATH = process.cwd();
+import { colors } from './utils/log.js';
 
 export const SUPPORTED_MEDIA_EXTENSIONS = [
   'png',
@@ -35,3 +17,39 @@ export const SUPPORTED_MEDIA_EXTENSIONS = [
   'apng',
   'bmp',
 ];
+
+export const ESCAPED_COMPONENTS = [
+  'Accordion',
+  'AccordionGroup',
+  'Note',
+  'Warning',
+  'Info',
+  'Tip',
+  'Check',
+  'Card',
+  'CardGroup',
+  'CodeGroup',
+  'Frame',
+  'Icon',
+  'Steps',
+  'Step',
+  'Tabs',
+  'Tab',
+  'Tooltip',
+] as const;
+
+export const SPACES = ' '.repeat(13);
+
+export const CONTENT_FAILURE_MSG = `failed to retrieve root content from HTML.
+${SPACES}Please double check your documentation provider and ensure they are supported.
+${SPACES}We currently support: ReadMe, GitBook, and Docusaurus`;
+
+export const NAV_FAILURE_MSG = `failed to retrieve nav items from HTML.
+${SPACES}Please double check your documentation provider and ensure they are supported.
+${SPACES}We currently support: ReadMe, GitBook, and Docusaurus`;
+
+export const MDAST_FAILURE_MSG = 'failed to convert MDAST to Markdown string';
+
+export const FINAL_SUCCESS_MESSAGE = `We've successfully scraped your docs site.
+${SPACES}We've downloaded the ${colors.cyan}\`navigation\`${colors.default} array (and if necessary, the ${colors.cyan}\`tabs\`${colors.cyan} array)
+${SPACES}into ${colors.blue}\`mint.json\`${colors.default}.`;

package/src/customComponents/create.ts

@@ -0,0 +1,106 @@
+import type { Root as HastRoot, Element } from 'hast';
+import { visit, SKIP, CONTINUE } from 'unist-util-visit';
+
+import {
+  gitBookScrapeAccordion,
+  readmeScrapeAccordion,
+  docusaurusScrapeAccordion,
+} from '../components/Accordion.js';
+import {
+  gitBookScrapeAccordionGroup,
+  readmeScrapeAccordionGroup,
+  docusaurusScrapeAccordionGroup,
+} from '../components/AccordionGroup.js';
+import {
+  gitBookScrapeCallout,
+  readmeScrapeCallout,
+  docusaurusScrapeCallout,
+} from '../components/Callout.js';
+import { gitBookScrapeCard, readmeScrapeCard, docusaurusScrapeCard } from '../components/Card.js';
+import {
+  gitBookScrapeCardGroup,
+  readmeScrapeCardGroup,
+  docusaurusScrapeCardGroup,
+} from '../components/CardGroup.js';
+import {
+  gitBookScrapeCodeGroup,
+  readmeScrapeCodeGroup,
+  docusaurusScrapeCodeGroup,
+} from '../components/CodeGroup.js';
+import {
+  gitBookScrapeFrame,
+  readmeScrapeFrame,
+  docusaurusScrapeFrame,
+} from '../components/Frame.js';
+import { gitBookScrapeTabs, readmeScrapeTabs, docusaurusScrapeTabs } from '../components/Tabs.js';
+import type { ScrapeFuncType } from '../types/scrapeFunc.js';
+import { framework } from '../utils/detectFramework.js';
+import { log } from '../utils/log.js';
+
+function createComponent(
+  gitBookScrapeFunc: ScrapeFuncType,
+  readmeScrapeFunc: ScrapeFuncType,
+  docusaurusScrapeFunc: ScrapeFuncType
+) {
+  return function (tree: HastRoot) {
+    return visit(tree, 'element', function (node, index, parent) {
+      if (node.tagName === 'code' || node.tagName === 'pre') return SKIP;
+      let result: Element | undefined = undefined;
+
+      switch (framework.vendor) {
+        case 'gitbook':
+          result = gitBookScrapeFunc(node, index, parent);
+          break;
+        case 'readme':
+          result = readmeScrapeFunc(node, index, parent);
+          break;
+        case 'docusaurus':
+          result = docusaurusScrapeFunc(node, index, parent);
+          break;
+        default:
+          log('Invalid documentation vendor requested: ' + framework.vendor);
+          return SKIP;
+      }
+
+      if (!result) return CONTINUE;
+
+      if (parent && typeof index === 'number') {
+        parent.children[index] = result;
+        return SKIP;
+      }
+      return CONTINUE;
+    });
+  };
+}
+
+export function createLinks() {
+  return createComponent(gitBookScrapeCard, readmeScrapeCard, docusaurusScrapeCard);
+}
+export function createCard() {
+  return createComponent(gitBookScrapeCard, readmeScrapeCard, docusaurusScrapeCard);
+}
+export function createAccordion() {
+  return createComponent(gitBookScrapeAccordion, readmeScrapeAccordion, docusaurusScrapeAccordion);
+}
+export function createAccordionGroup() {
+  return createComponent(
+    gitBookScrapeAccordionGroup,
+    readmeScrapeAccordionGroup,
+    docusaurusScrapeAccordionGroup
+  );
+}
+export function createFrame() {
+  return createComponent(gitBookScrapeFrame, readmeScrapeFrame, docusaurusScrapeFrame);
+}
+export function createCodeGroup() {
+  return createComponent(gitBookScrapeCodeGroup, readmeScrapeCodeGroup, docusaurusScrapeCodeGroup);
+}
+export function createTabs() {
+  return createComponent(gitBookScrapeTabs, readmeScrapeTabs, docusaurusScrapeTabs);
+}
+export function createCallout() {
+  return createComponent(gitBookScrapeCallout, readmeScrapeCallout, docusaurusScrapeCallout);
+}
+export function createCardGroup() {
+  return createComponent(gitBookScrapeCardGroup, readmeScrapeCardGroup, docusaurusScrapeCardGroup);
+}