@mintlify/scraping 4.0.5 → 4.0.6

package/src/components/AccordionGroup.ts

@@ -1,24 +1,34 @@
-import type { Element } from 'hast';
+import type { Element, ElementContent } from 'hast';
 
 import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
 
 export function gitBookScrapeAccordionGroup(
   node: HastNode,
-  _: HastNodeIndex,
+  index: HastNodeIndex,
   parent: HastNodeParent
 ): Element | undefined {
   if (node.tagName !== 'Accordion') return undefined;
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-  if (!parent || !parent.children) return undefined;
+  if (!parent || typeof index !== 'number') return undefined;
 
   let accordionCount = 0;
-  for (const child of parent.children) {
-    if (child.type === 'element' && child.tagName === 'Accordion') accordionCount++;
+  while (parent.children[index]) {
+    const child = parent.children[index];
+    if (!child || child.type !== 'element' || child.tagName !== 'Accordion') break;
+    accordionCount++;
+    index++;
   }
 
-  if (accordionCount === parent.children.length) {
-    parent.type = 'element';
-    (parent as Element).tagName = 'AccordionGroup';
+  index -= accordionCount;
+
+  if (accordionCount > 1) {
+    const children = parent.children.splice(index, accordionCount);
+    const newNode = {
+      type: 'element' as const,
+      tagName: 'AccordionGroup',
+      properties: {},
+      children: children as Array<ElementContent>,
+    };
+    parent.children.splice(index, 0, newNode);
   }
 
   return undefined;
@@ -26,21 +36,31 @@ export function gitBookScrapeAccordionGroup(
 
 export function readmeScrapeAccordionGroup(
   node: HastNode,
-  _: HastNodeIndex,
+  index: HastNodeIndex,
   parent: HastNodeParent
 ): Element | undefined {
   if (node.tagName !== 'Accordion') return undefined;
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-  if (!parent || !parent.children) return undefined;
+  if (!parent || typeof index !== 'number') return undefined;
 
   let accordionCount = 0;
-  for (const child of parent.children) {
-    if (child.type === 'element' && child.tagName === 'Accordion') accordionCount++;
+  while (parent.children[index]) {
+    const child = parent.children[index];
+    if (!child || child.type !== 'element' || child.tagName !== 'Accordion') break;
+    accordionCount++;
+    index++;
   }
 
-  if (accordionCount === parent.children.length) {
-    parent.type = 'element';
-    (parent as Element).tagName = 'AccordionGroup';
+  index -= accordionCount;
+
+  if (accordionCount > 1) {
+    const children = parent.children.splice(index, accordionCount);
+    const newNode = {
+      type: 'element' as const,
+      tagName: 'AccordionGroup',
+      properties: {},
+      children: children as Array<ElementContent>,
+    };
+    parent.children.splice(index, 0, newNode);
   }
 
   return undefined;
@@ -48,21 +68,31 @@ export function readmeScrapeAccordionGroup(
 
 export function docusaurusScrapeAccordionGroup(
   node: HastNode,
-  _: HastNodeIndex,
+  index: HastNodeIndex,
   parent: HastNodeParent
 ): Element | undefined {
   if (node.tagName !== 'Accordion') return undefined;
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-  if (!parent || !parent.children) return undefined;
+  if (!parent || typeof index !== 'number') return undefined;
 
   let accordionCount = 0;
-  for (const child of parent.children) {
-    if (child.type === 'element' && child.tagName === 'Accordion') accordionCount++;
+  while (parent.children[index]) {
+    const child = parent.children[index];
+    if (!child || child.type !== 'element' || child.tagName !== 'Accordion') break;
+    accordionCount++;
+    index++;
   }
 
-  if (accordionCount === parent.children.length) {
-    parent.type = 'element';
-    (parent as Element).tagName = 'AccordionGroup';
+  index -= accordionCount;
+
+  if (accordionCount > 1) {
+    const children = parent.children.splice(index, accordionCount);
+    const newNode = {
+      type: 'element' as const,
+      tagName: 'AccordionGroup',
+      properties: {},
+      children: children as Array<ElementContent>,
+    };
+    parent.children.splice(index, 0, newNode);
   }
 
   return undefined;

package/src/components/Card.ts

@@ -72,6 +72,7 @@ export function readmeScrapeCard(
       !node.properties.className.includes('card') &&
       !node.properties.className.includes('Card') &&
       !node.properties.className.includes('docs-card') &&
+      !node.properties.className.includes('next-steps__step') &&
       !node.properties.className.join(' ').includes('_card') &&
       !node.properties.className.join(' ').includes('-card'))
   ) {
@@ -107,7 +108,7 @@ export function readmeScrapeCard(
       title: title,
       href: href,
     },
-    children: turnChildrenIntoMdx(parent.children as Array<Element>) as Array<ElementContent>,
+    children: turnChildrenIntoMdx(node.children as Array<Element>) as Array<ElementContent>,
   };
 
   return newNode;
@@ -161,7 +162,7 @@ export function docusaurusScrapeCard(
       title: title,
       href: href,
     },
-    children: turnChildrenIntoMdx(parent.children as Array<Element>) as Array<ElementContent>,
+    children: turnChildrenIntoMdx(node.children as Array<Element>) as Array<ElementContent>,
   };
 
   return newNode;

package/src/components/CardGroup.ts

@@ -8,8 +8,7 @@ export function gitBookScrapeCardGroup(
   parent: HastNodeParent
 ): Element | undefined {
   if (node.tagName !== 'Card') return undefined;
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-  if (!parent || !parent.children) return undefined;
+  if (!parent) return undefined;
 
   let cardCount = 0;
   for (const child of parent.children) {
@@ -30,8 +29,7 @@ export function readmeScrapeCardGroup(
   parent: HastNodeParent
 ): Element | undefined {
   if (node.tagName !== 'Card') return undefined;
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-  if (!parent || !parent.children) return undefined;
+  if (!parent) return undefined;
 
   let cardCount = 0;
   for (const child of parent.children) {
@@ -52,8 +50,7 @@ export function docusaurusScrapeCardGroup(
   parent: HastNodeParent
 ): Element | undefined {
   if (node.tagName !== 'Card') return undefined;
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-  if (!parent || !parent.children) return undefined;
+  if (!parent) return undefined;
 
   let cardCount = 0;
   for (const child of parent.children) {

package/src/components/CodeGroup.ts

@@ -6,36 +6,30 @@ import { assertIsDefined } from '../assert.js';
 import type { HastNode, HastNodeIndex, HastNodeParent } from '../types/hast.js';
 import { turnChildrenIntoMdx } from '../utils/children.js';
 
-function tabContainsOnlyCode(node: Element): boolean {
+function tabContainsOnlyCode(node: Element | undefined): boolean {
+  if (!node) return false;
+
   let tabsCount = 0;
   let onlyCodeCount = 0;
 
   visit(node, 'element', function (subNode) {
-    if (subNode.properties.role && subNode.properties.role === 'tabpanel') tabsCount++;
-    visit(subNode, 'element', function (tabNode, index, parent) {
-      if (tabNode.tagName === 'button' && typeof index === 'number' && parent) {
-        parent.children.splice(index, 1);
-      }
-    });
+    if (subNode.properties.role !== 'tabpanel') return CONTINUE;
+    tabsCount++;
     if (
       subNode.children[0] &&
-      subNode.children[1] &&
-      subNode.children[2] &&
       subNode.children[0].type === 'element' &&
       subNode.children[0].children.length === 1 &&
-      subNode.children[1].type === 'element' &&
-      subNode.children[1].children.length === 1 &&
-      subNode.children[2].type === 'element' &&
-      subNode.children[2].children.length > 1 &&
-      subNode.children[2].children.length < 4 &&
-      subNode.children[2].children.find(
+      subNode.children[0].children[0] &&
+      subNode.children[0].children[0].type === 'element' &&
+      subNode.children[0].children[0].children.length > 1 &&
+      subNode.children[0].children[0].children.find(
         (child) => child.type === 'element' && (child.tagName === 'pre' || child.tagName === 'code')
-      )
+      ) !== undefined
     ) {
-      return onlyCodeCount++;
+      onlyCodeCount++;
     }
   });
-  return onlyCodeCount === tabsCount;
+  return onlyCodeCount === tabsCount && tabsCount > 0;
 }
 
 export function gitBookScrapeCodeGroup(
@@ -43,11 +37,46 @@ export function gitBookScrapeCodeGroup(
   _: HastNodeIndex,
   parent: HastNodeParent
 ): Element | undefined {
-  if (node.tagName !== 'div' || !node.properties.role || node.properties.role !== 'tablist') {
-    return undefined;
+  if (
+    node.tagName === 'div' &&
+    Array.isArray(node.properties.className) &&
+    node.properties.className.includes('group/codeblock') &&
+    node.children.length === 2 &&
+    node.children[0] &&
+    node.children[1] &&
+    node.children[1].type === 'element' &&
+    (node.children[1].tagName === 'pre' || node.children[1].tagName === 'code')
+  ) {
+    let title = '';
+    visit(node.children[0], 'text', function (subNode) {
+      title = subNode.value;
+      return EXIT;
+    });
+
+    if (!title) return undefined;
+
+    const children = turnChildrenIntoMdx([node.children[1]]);
+    const code = {
+      type: 'code',
+      lang: 'bash',
+      meta: title,
+      value: (children[0] as unknown as Code).value,
+    };
+
+    const newNode: Element = {
+      type: 'element',
+      tagName: 'CodeGroup',
+      properties: {},
+      children: [code] as Array<ElementContent>,
+    };
+
+    return newNode;
   }
 
-  if (!tabContainsOnlyCode(node)) return undefined;
+  if (node.tagName !== 'div' || node.properties.role !== 'tablist') return undefined;
+
+  if (!tabContainsOnlyCode(parent as Element | undefined)) return undefined;
+  assertIsDefined(parent);
 
   const titles: Array<string> = [];
   visit(node, 'element', function (subNode) {
@@ -58,28 +87,31 @@ export function gitBookScrapeCodeGroup(
     });
   });
 
-  assertIsDefined(parent);
   parent.children.shift();
 
-  let lang = '';
-  visit(node, 'element', function (subNode) {
-    if (subNode.tagName === 'code' && 'id' in subNode.properties) {
-      lang = subNode.properties.id as string;
-      return EXIT;
+  const langs: Array<string> = [];
+  visit(parent, 'element', function (subNode) {
+    if (
+      subNode.tagName === 'div' &&
+      'id' in subNode.properties &&
+      subNode.properties.role === 'tabpanel'
+    ) {
+      langs.push(subNode.properties.id as string);
     }
   });
 
   const children = turnChildrenIntoMdx(parent.children) as Array<ElementContent>;
   const tabChildren: Array<ElementContent> = [];
-  for (let childIndex = 0; childIndex < children.length; childIndex++) {
-    const child = children[childIndex];
+  children.forEach((child, index) => {
+    const lang = langs[index] || 'bash';
+    const title = titles[index] || lang;
     tabChildren.push({
       type: 'code',
       lang: lang,
-      meta: titles[childIndex],
+      meta: title,
       value: (child as unknown as Code).value,
     } as unknown as ElementContent);
-  }
+  });
 
   const newNode: Element = {
     type: 'element',
@@ -88,13 +120,15 @@ export function gitBookScrapeCodeGroup(
     children: tabChildren as Array<ElementContent>,
   };
 
+  parent.children.length = 0;
+
   return newNode;
 }
 
 export function readmeScrapeCodeGroup(
   node: HastNode,
   _: HastNodeIndex,
-  parent: HastNodeParent
+  __: HastNodeParent
 ): Element | undefined {
   if (
     node.tagName !== 'div' ||
@@ -105,52 +139,53 @@ export function readmeScrapeCodeGroup(
     return undefined;
   }
 
-  if (node.children.length !== 2 || !node.children[0] || !node.children[1]) return undefined;
-  const tabTitles = node.children[0];
-
-  const titles: Array<string> = [];
-  visit(tabTitles, 'element', function (subNode) {
-    visit(subNode, 'text', function (textNode) {
-      titles.push(textNode.value);
-      return EXIT;
-    });
-  });
-
-  assertIsDefined(parent);
-  node.children.shift();
-
-  let lang = '';
-  visit(node, 'element', function (subNode, _, subParent) {
+  let newNode: Element | undefined = undefined;
+  visit(node, 'element', function (node) {
     if (
-      subNode.tagName === 'code' &&
-      subParent &&
-      subParent.type === 'element' &&
-      subParent.tagName === 'pre' &&
-      'dataLang' in subNode.properties
+      node.tagName !== 'div' ||
+      !node.properties.className ||
+      !Array.isArray(node.properties.className) ||
+      !node.properties.className.includes('CodeTabs-inner')
     ) {
-      lang = subNode.properties.dataLang as string;
-      return EXIT;
+      return CONTINUE;
     }
-  });
 
-  const children = turnChildrenIntoMdx(node.children) as Array<ElementContent>;
-  const tabChildren: Array<ElementContent> = [];
-  for (let childIndex = 0; childIndex < children.length; childIndex++) {
-    const child = children[childIndex];
-    tabChildren.push({
-      type: 'code',
-      lang: lang,
-      meta: titles[childIndex],
-      value: (child as unknown as Code).value,
-    } as unknown as ElementContent);
-  }
+    const langs: Array<string> = [];
+    const titles: Array<string> = [];
+    visit(node, 'element', function (subNode) {
+      if (
+        subNode.tagName !== 'code' ||
+        !Array.isArray(subNode.properties.className) ||
+        !subNode.properties.className.includes('rdmd-code')
+      )
+        return CONTINUE;
 
-  const newNode: Element = {
-    type: 'element',
-    tagName: 'CodeGroup',
-    properties: {},
-    children: tabChildren as Array<ElementContent>,
-  };
+      langs.push((subNode.properties.dataLang as string | undefined) ?? '');
+      titles.push((subNode.properties.name as string | undefined) ?? '');
+    });
+
+    const children = turnChildrenIntoMdx(node.children) as Array<ElementContent>;
+    const tabChildren: Array<ElementContent> = [];
+    children.forEach((child, index) => {
+      const lang = langs[index] || 'bash';
+      const title = titles[index] || lang;
+      tabChildren.push({
+        type: 'code',
+        lang: lang,
+        meta: title,
+        value: (child as unknown as Code).value,
+      } as unknown as ElementContent);
+    });
+
+    newNode = {
+      type: 'element',
+      tagName: 'CodeGroup',
+      properties: {},
+      children: tabChildren as Array<ElementContent>,
+    };
+
+    return EXIT;
+  });
 
   return newNode;
 }
@@ -165,6 +200,7 @@ export function docusaurusScrapeCodeGroup(
   }
 
   if (!tabContainsOnlyCode(node)) return undefined;
+  assertIsDefined(parent);
 
   const titles: Array<string> = [];
   visit(node, 'element', function (subNode) {
@@ -175,28 +211,34 @@ export function docusaurusScrapeCodeGroup(
     });
   });
 
-  assertIsDefined(parent);
   parent.children.shift();
 
-  let lang = '';
-  visit(node, 'element', function (subNode) {
-    if (Array.isArray(subNode.properties.className) && subNode.properties.className.length > 0) {
-      lang = subNode.properties.className[0] as string;
-      lang = lang.replace('language-', '');
+  const langs: Array<string> = [];
+  visit(parent, 'element', function (subNode) {
+    if (
+      subNode.tagName === 'div' &&
+      Array.isArray(subNode.properties.className) &&
+      subNode.properties.className.find((className) => className.toString().includes('language-'))
+    ) {
+      const lang = subNode.properties.className.find((className) =>
+        className.toString().includes('language-')
+      );
+      langs.push(lang?.toString().replace('language-', '') ?? '');
     }
   });
 
   const children = turnChildrenIntoMdx(parent.children) as Array<ElementContent>;
   const tabChildren: Array<ElementContent> = [];
-  for (let childIndex = 0; childIndex < children.length; childIndex++) {
-    const child = children[childIndex];
+  children.forEach((child, index) => {
+    const lang = langs[index] || 'bash';
+    const title = titles[index] || lang;
     tabChildren.push({
       type: 'code',
       lang: lang,
-      meta: titles[childIndex],
+      meta: title,
       value: (child as unknown as Code).value,
     } as unknown as ElementContent);
-  }
+  });
 
   const newNode: Element = {
     type: 'element',
@@ -205,5 +247,7 @@ export function docusaurusScrapeCodeGroup(
     children: tabChildren as Array<ElementContent>,
   };
 
+  parent.children.length = 0;
+
   return newNode;
 }

package/src/components/Tabs.ts

@@ -55,47 +55,80 @@ export function gitBookScrapeTabs(
 export function readmeScrapeTabs(
   node: HastNode,
   _: HastNodeIndex,
-  parent: HastNodeParent
+  __: HastNodeParent
 ): Element | undefined {
   if (
     (node.tagName !== 'div' && node.tagName !== 'a') ||
     !node.properties.className ||
     !Array.isArray(node.properties.className) ||
-    !node.properties.className.includes('tabbed-component')
+    (!node.properties.className.includes('tabbed-component') &&
+      !node.properties.className.includes('tabs') &&
+      !node.properties.className.includes('Tabs'))
   ) {
     return undefined;
   }
 
-  if (node.children.length < 2 || !node.children[0] || !node.children[1]) return undefined;
-  const tabTitles = node.children[0];
+  if (!node.children[0] || !node.children[1]) return undefined;
 
   const titles: Array<string> = [];
-  visit(tabTitles, 'element', function (subNode) {
-    visit(subNode, 'text', function (textNode) {
-      titles.push(textNode.value);
-      return EXIT;
-    });
-  });
+  const tabContents: Array<Element> = [];
 
-  assertIsDefined(parent);
-  node.children.shift();
+  if (node.children.length !== 2) {
+    visit(node, 'element', function (subNode) {
+      if (subNode.tagName !== 'label' && subNode.tagName !== 'button') return CONTINUE;
 
-  const children = turnChildrenIntoMdx(node.children) as Array<ElementContent>;
-  const tabChildren: Array<ElementContent> = [];
-  for (let childIndex = 0; childIndex < children.length; childIndex++) {
-    const child = children[childIndex];
-    if (child) {
-      tabChildren.push({
-        type: 'element',
-        tagName: 'Tab',
-        properties: {
-          title: titles[childIndex],
-        },
-        children: [child],
+      let title = '';
+      visit(subNode, 'text', function (textNode) {
+        title += textNode.value;
+      });
+
+      titles.push(title.trim().replace('\n', ''));
+    });
+
+    tabContents.push(
+      ...(node.children.filter((subNode) => {
+        if (
+          subNode.type === 'element' &&
+          Array.isArray(subNode.properties.className) &&
+          (subNode.properties.className.includes('tab') ||
+            subNode.properties.className.includes('Tab') ||
+            subNode.properties.className.includes('tabbed-content') ||
+            subNode.properties.className.includes('tab-content'))
+        )
+          return true;
+        return false;
+      }) as Array<Element>)
+    );
+  } else {
+    const tabTitles = node.children[0];
+
+    visit(tabTitles, 'element', function (subNode) {
+      visit(subNode, 'text', function (textNode) {
+        titles.push(textNode.value);
+        return EXIT;
       });
+    });
+
+    node.children.shift();
+    if (node.children[0].type === 'element') {
+      tabContents.push(...(node.children[0].children as Array<Element>));
     }
   }
 
+  const tabChildren: Array<ElementContent> = [];
+  tabContents.forEach((tab, index) => {
+    if (!titles[index]) return;
+    const children = turnChildrenIntoMdx([tab]) as Array<ElementContent>;
+    tabChildren.push({
+      type: 'element',
+      tagName: 'Tab',
+      properties: {
+        title: titles[index],
+      },
+      children,
+    });
+  });
+
   const newNode: Element = {
     type: 'element',
     tagName: 'Tabs',

package/src/constants.ts

@@ -1,4 +1,4 @@
-import { colors } from './utils/log.js';
+import { activeColors } from './utils/log.js';
 
 export const SUPPORTED_MEDIA_EXTENSIONS = [
   'png',
@@ -51,5 +51,5 @@ ${SPACES}We currently support: ReadMe, GitBook, and Docusaurus`;
 export const MDAST_FAILURE_MSG = 'failed to convert MDAST to Markdown string';
 
 export const FINAL_SUCCESS_MESSAGE = `We've successfully scraped your docs site.
-${SPACES}We've downloaded the ${colors.cyan}\`navigation\`${colors.default} array (and if necessary, the ${colors.cyan}\`tabs\`${colors.cyan} array)
-${SPACES}into ${colors.blue}\`mint.json\`${colors.default}.`;
+${SPACES}We've downloaded the ${activeColors.cyan}\`navigation\`${activeColors.default} array (and if necessary, the ${activeColors.cyan}\`tabs\`${activeColors.default} array)
+${SPACES}into ${activeColors.blue}\`mint.json\`${activeColors.default}.`;

package/src/nav/listItems.ts

@@ -33,7 +33,7 @@ export function processListItem(
     return undefined;
   }
 
-  let isApiReferenceLink = false;
+  let isApiReferenceLink = false as boolean;
   visit(link, 'element', function (subNode) {
     if (
       subNode.tagName === 'span' &&
@@ -44,7 +44,6 @@ export function processListItem(
       return EXIT;
     }
   });
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
   if (isApiReferenceLink) return undefined;
 
   if (linkHref.startsWith('/')) linkHref = linkHref.substring(1);