@mintlify/scraping 4.0.204 → 4.0.205

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.204",
+  "version": "4.0.205",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -44,13 +44,13 @@
     "hast-util-to-mdast": "^10.1.0",
     "js-yaml": "^4.1.0",
     "mdast-util-mdx-jsx": "^3.1.3",
+    "neotraverse": "^0.6.18",
     "puppeteer": "^22.14.0",
     "rehype-parse": "^9.0.0",
     "remark-gfm": "^4.0.0",
     "remark-mdx": "^3.0.1",
     "remark-parse": "^11.0.0",
     "remark-stringify": "^11.0.0",
-    "traverse": "^0.6.10",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",
     "yargs": "^17.6.0",
@@ -67,7 +67,6 @@
     "@types/hast": "^3.0.4",
     "@types/mdast": "^4.0.4",
     "@types/node": "^18.7.13",
-    "@types/traverse": "^0.6.37",
     "@types/yargs": "^17.0.13",
     "@typescript-eslint/eslint-plugin": "6.x",
     "@typescript-eslint/parser": "6.x",
@@ -78,5 +77,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "d2f72c3ad5d1a63bc43deb817b33d3d0e92f6353"
+  "gitHead": "78cfff36cf9563d70d2a9a2033a23a49633c73be"
 }

package/src/nav/root.ts

@@ -23,7 +23,7 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
       rootSelector = 'menu';
       break;
     case 'gitbook':
-      rootSelector = 'page-no-toc:hidden';
+      rootSelector = 'page-no-toc:lg:hidden';
       break;
     case 'readme':
       rootSelector = 'rm-Sidebar';

package/src/pipeline/group.ts

@@ -1,3 +1,4 @@
+import { convertStrToTitle } from '@mintlify/common';
 import type { Browser } from 'puppeteer';
 
 import { OVERVIEW_PAGE_SLUG } from '../constants.js';
@@ -25,7 +26,11 @@ export async function scrapePageGroup(
         chunk.map(async (url, index) => {
           try {
             if (opts.externalLinks) {
-              const res = scrapePage(`external-link-${index}`, url, { externalLink: true });
+              let externalLinkTitle =
+                convertStrToTitle(url.pathname.split('/').at(-1) ?? url.pathname) ||
+                `external-link-${index}`;
+              externalLinkTitle = externalLinkTitle.replace(/\s+/g, '-').toLowerCase();
+              const res = scrapePage(externalLinkTitle, url, { externalLink: true });
               return res;
             }
 

package/src/pipeline/site.ts

@@ -1,6 +1,6 @@
 import { MintConfig, Navigation, Tab } from '@mintlify/models';
 import type { Root as HastRoot } from 'hast';
-import traverse from 'traverse';
+import traverse from 'neotraverse';
 
 import { NAV_FAILURE_MSG } from '../constants.js';
 import { OVERVIEW_PAGE_SLUG } from '../constants.js';

package/src/pipeline/tabs.ts

@@ -23,8 +23,12 @@ export async function scrapeAllSiteTabs(
 
   detectFramework(hast);
 
-  if (framework.vendor === 'readme' || framework.vendor === 'docusaurus') {
-    const links = retrieveTabLinks(hast);
+  if (
+    framework.vendor === 'readme' ||
+    framework.vendor === 'docusaurus' ||
+    framework.vendor === 'gitbook'
+  ) {
+    const links = retrieveTabLinks(hast, url);
     if (
       !links ||
       !links.length ||

package/src/tabs/retrieve.ts

@@ -5,8 +5,14 @@ import { visit, EXIT, CONTINUE } from 'unist-util-visit';
 import { framework } from '../utils/detectFramework.js';
 import { findTitle, getTitleFromLink } from '../utils/title.js';
 
-export function retrieveTabLinks(rootNode: HastRoot): Array<Tab> | undefined {
-  if (framework.vendor !== 'readme' && framework.vendor !== 'docusaurus') return undefined;
+export function retrieveTabLinks(rootNode: HastRoot, url: URL): Array<Tab> | undefined {
+  if (
+    framework.vendor !== 'readme' &&
+    framework.vendor !== 'docusaurus' &&
+    framework.vendor !== 'gitbook'
+  ) {
+    return undefined;
+  }
 
   let element: Element | undefined = undefined as Element | undefined;
   visit(rootNode, 'element', function (node) {
@@ -33,64 +39,102 @@ export function retrieveTabLinks(rootNode: HastRoot): Array<Tab> | undefined {
         return EXIT;
       }
     }
+
+    if (framework.vendor === 'gitbook') {
+      if (
+        node.tagName === 'nav' &&
+        node.properties.id === 'sections' &&
+        node.properties.ariaLabel === 'Sections'
+      ) {
+        element = node;
+        return EXIT;
+      }
+    }
   });
 
   if (!element) return undefined;
 
   const links: Array<Tab> = [];
-  visit(element as Element, 'element', function (node) {
-    if (framework.vendor === 'readme') {
-      if (
-        node.tagName !== 'nav' &&
-        !(
-          node.tagName === 'div' &&
-          node.properties.className &&
-          Array.isArray(node.properties.className) &&
-          node.properties.className.includes('rm-Header-right')
-        )
-      )
-        return CONTINUE;
-
-      visit(node, 'element', function (subNode) {
+  visit(element, 'element', function (node) {
+    switch (framework.vendor) {
+      case 'readme':
         if (
-          subNode.tagName !== 'a' ||
-          !subNode.properties.href ||
-          typeof subNode.properties.href !== 'string' ||
-          subNode.properties.href.startsWith('http')
+          node.tagName !== 'nav' &&
+          !(
+            node.tagName === 'div' &&
+            node.properties.className &&
+            Array.isArray(node.properties.className) &&
+            node.properties.className.includes('rm-Header-right')
+          )
         )
           return CONTINUE;
-        const title = findTitle(subNode);
-        links.push({
-          name: title || getTitleFromLink(subNode.properties.href),
-          url: subNode.properties.href,
+
+        visit(node, 'element', function (subNode) {
+          if (
+            subNode.tagName !== 'a' ||
+            !subNode.properties.href ||
+            typeof subNode.properties.href !== 'string' ||
+            subNode.properties.href.startsWith('http')
+          )
+            return CONTINUE;
+          const title = findTitle(subNode);
+          links.push({
+            name: title || getTitleFromLink(subNode.properties.href),
+            url: subNode.properties.href,
+          });
         });
-      });
-    }
+        break;
 
-    if (framework.vendor === 'docusaurus') {
-      if (node.tagName !== 'nav') return CONTINUE;
+      case 'docusaurus':
+        if (node.tagName !== 'nav') return CONTINUE;
 
-      visit(node, 'element', function (subNode, _, parent) {
-        if (
-          subNode.tagName !== 'a' ||
-          !subNode.properties.href ||
-          typeof subNode.properties.href !== 'string' ||
-          subNode.properties.href.startsWith('http') ||
-          !parent ||
-          parent.type !== 'element' ||
-          !Array.isArray(parent.properties.className) ||
-          parent.properties.className.length !== 1 ||
-          parent.properties.className[0] !== 'navbar__items' ||
-          parent.properties.className.includes('navbar__items--right')
-        )
-          return CONTINUE;
+        visit(node, 'element', function (subNode, _, parent) {
+          if (
+            subNode.tagName !== 'a' ||
+            !subNode.properties.href ||
+            typeof subNode.properties.href !== 'string' ||
+            subNode.properties.href.startsWith('http') ||
+            !parent ||
+            parent.type !== 'element' ||
+            !Array.isArray(parent.properties.className) ||
+            parent.properties.className.length !== 1 ||
+            parent.properties.className[0] !== 'navbar__items' ||
+            parent.properties.className.includes('navbar__items--right')
+          )
+            return CONTINUE;
 
-        const title = findTitle(subNode);
-        links.push({
-          name: title || getTitleFromLink(subNode.properties.href),
-          url: subNode.properties.href,
+          const title = findTitle(subNode);
+          links.push({
+            name: title || getTitleFromLink(subNode.properties.href),
+            url: subNode.properties.href,
+          });
         });
-      });
+        break;
+
+      case 'gitbook':
+        if (node.tagName !== 'nav') return CONTINUE;
+
+        visit(node, 'element', function (subNode, _, parent) {
+          if (
+            subNode.tagName !== 'a' ||
+            !subNode.properties.href ||
+            typeof subNode.properties.href !== 'string' ||
+            !parent ||
+            parent.type !== 'element'
+          )
+            return CONTINUE;
+
+          const title = findTitle(subNode);
+          const link = new URL(subNode.properties.href);
+          links.push({
+            name: title || getTitleFromLink(subNode.properties.href),
+            url: link.origin === url.origin ? link.pathname : link.toString(),
+          });
+        });
+        break;
+
+      default:
+        break;
     }
   });