@mintlify/scraping 4.0.39 → 4.0.40

package/bin/utils/breadcrumbs.d.ts

@@ -0,0 +1,8 @@
+import type { Element } from 'hast';
+export declare function unifiedRemoveBreadCrumbs(): (node: Element) => undefined;
+/**
+ * Docusaurus-specific function since their breadcrumbs
+ * are within the content in the `article` itself instead
+ * of outside of the `article` element
+ */
+export declare function removeBreadCrumbs(node: Element): undefined;

package/bin/utils/breadcrumbs.js

@@ -0,0 +1,26 @@
+import { visit } from 'unist-util-visit';
+import { framework } from './detectFramework.js';
+export function unifiedRemoveBreadCrumbs() {
+    return function (node) {
+        return removeBreadCrumbs(node);
+    };
+}
+/**
+ * Docusaurus-specific function since their breadcrumbs
+ * are within the content in the `article` itself instead
+ * of outside of the `article` element
+ */
+export function removeBreadCrumbs(node) {
+    return visit(node, 'element', function (subNode, index, parent) {
+        if (framework.vendor === 'docusaurus' &&
+            subNode.tagName === 'nav' &&
+            subNode.properties.className &&
+            Array.isArray(subNode.properties.className) &&
+            subNode.properties.className.includes('theme-doc-breadcrumbs') &&
+            parent &&
+            typeof index === 'number') {
+            parent.children.splice(index, 1);
+        }
+    });
+}
+//# sourceMappingURL=breadcrumbs.js.map

package/bin/utils/breadcrumbs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"breadcrumbs.js","sourceRoot":"","sources":["../../src/utils/breadcrumbs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,UAAU,wBAAwB;IACtC,OAAO,UAAU,IAAa;QAC5B,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC7C,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IACE,SAAS,CAAC,MAAM,KAAK,YAAY;YACjC,OAAO,CAAC,OAAO,KAAK,KAAK;YACzB,OAAO,CAAC,UAAU,CAAC,SAAS;YAC5B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAC9D,MAAM;YACN,OAAO,KAAK,KAAK,QAAQ,EACzB,CAAC;YACD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/bin/utils/breaks.d.ts

@@ -1,3 +1,8 @@
 import type { Element } from 'hast';
 export declare function unifiedRemoveBreaks(): (node: Element) => undefined;
+/**
+ * ReadMe-specific function since they use breaks in between
+ * every element, but either way our parser adds whitespace
+ * automatically
+ */
 export declare function removeBreaks(node: Element): undefined;

package/bin/utils/breaks.js

@@ -4,9 +4,11 @@ export function unifiedRemoveBreaks() {
         return removeBreaks(node);
     };
 }
-// ReadMe-specific function since they use breaks in between
-// every element, but either way our parser adds whitespace
-// automatically
+/**
+ * ReadMe-specific function since they use breaks in between
+ * every element, but either way our parser adds whitespace
+ * automatically
+ */
 export function removeBreaks(node) {
     return visit(node, 'element', function (subNode, index, parent) {
         if (subNode.tagName === 'br' && parent && typeof index === 'number') {

package/bin/utils/breaks.js.map

@@ -1 +1 @@
-{"version":3,"file":"breaks.js","sourceRoot":"","sources":["../../src/utils/breaks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,MAAM,UAAU,mBAAmB;IACjC,OAAO,UAAU,IAAa;QAC5B,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,2DAA2D;AAC3D,gBAAgB;AAChB,MAAM,UAAU,YAAY,CAAC,IAAa;IACxC,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"breaks.js","sourceRoot":"","sources":["../../src/utils/breaks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,MAAM,UAAU,mBAAmB;IACjC,OAAO,UAAU,IAAa;QAC5B,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAa;IACxC,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/bin/utils/toc.d.ts

@@ -0,0 +1,8 @@
+import type { Element } from 'hast';
+export declare function unifiedRemoveTableOfContents(): (node: Element) => undefined;
+/**
+ * Docusaurus-specific function since their mobile ToC
+ * is within the content in the `article` itself instead
+ * of outside of the `article` element
+ */
+export declare function removeTableOfContents(node: Element): undefined;

package/bin/utils/toc.js

@@ -0,0 +1,26 @@
+import { visit } from 'unist-util-visit';
+import { framework } from './detectFramework.js';
+export function unifiedRemoveTableOfContents() {
+    return function (node) {
+        return removeTableOfContents(node);
+    };
+}
+/**
+ * Docusaurus-specific function since their mobile ToC
+ * is within the content in the `article` itself instead
+ * of outside of the `article` element
+ */
+export function removeTableOfContents(node) {
+    return visit(node, 'element', function (subNode, index, parent) {
+        if (framework.vendor === 'docusaurus' &&
+            subNode.tagName === 'div' &&
+            subNode.properties.className &&
+            Array.isArray(subNode.properties.className) &&
+            subNode.properties.className.includes('theme-doc-toc-mobile') &&
+            parent &&
+            typeof index === 'number') {
+            parent.children.splice(index, 1);
+        }
+    });
+}
+//# sourceMappingURL=toc.js.map

package/bin/utils/toc.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"toc.js","sourceRoot":"","sources":["../../src/utils/toc.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,UAAU,4BAA4B;IAC1C,OAAO,UAAU,IAAa;QAC5B,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAa;IACjD,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IACE,SAAS,CAAC,MAAM,KAAK,YAAY;YACjC,OAAO,CAAC,OAAO,KAAK,KAAK;YACzB,OAAO,CAAC,UAAU,CAAC,SAAS;YAC5B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAC7D,MAAM;YACN,OAAO,KAAK,KAAK,QAAQ,EACzB,CAAC;YACD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.39",
+  "version": "4.0.40",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -78,5 +78,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "bfcb32c90db7521bbfa54a7c31d8fe77594425e7"
+  "gitHead": "f1d4cc0ef3018924aff0a5c69f4664df24824c37"
 }

package/src/pipeline/images.ts

@@ -37,10 +37,7 @@ export async function downloadImagesFromFile(
   const localRootPath = filename ? dirname(filename) : rootPath;
 
   const imageResults = await Promise.all(
-    imageUrls.map(async (imageUrl) => {
-      const result = await downloadImage(imageUrl, localRootPath);
-      return result;
-    })
+    imageUrls.map(async (imageUrl) => await downloadImage(imageUrl, localRootPath))
   );
 
   const imagePathsMap = new Map<string, string>(
@@ -49,7 +46,11 @@ export async function downloadImagesFromFile(
 
   visit(root, function (node, index, parent) {
     if (node.type === 'image') {
-      node.url = imagePathsMap.get(node.url) ?? node.url;
+      if (node.url.startsWith('/')) {
+        node.url = imagePathsMap.get(new URL(node.url, url.origin).toString()) ?? node.url;
+      } else {
+        node.url = imagePathsMap.get(node.url) ?? node.url;
+      }
       if (parent && typeof index === 'number') parent.children[index] = node;
     } else if (node.type === 'mdxJsxFlowElement') {
       const urlAttr = (node.attributes as Array<MdxJsxAttribute>).find(

package/src/pipeline/page.ts

@@ -21,6 +21,7 @@ import { rehypeToRemarkCustomComponents } from '../customComponents/plugin.js';
 import { selectiveRehypeRemark } from '../customComponents/selective.js';
 import { retrieveRootContent } from '../root/retrieve.js';
 import type { Result } from '../types/result.js';
+import { unifiedRemoveBreadCrumbs } from '../utils/breadcrumbs.js';
 import { unifiedRemoveBreaks } from '../utils/breaks.js';
 import { unifiedRemoveClassNames } from '../utils/className.js';
 import { unifiedRemoveCopyButtons } from '../utils/copyButton.js';
@@ -39,6 +40,7 @@ import { unifiedRemovePositions } from '../utils/position.js';
 import { removeLeadingSlash, removeTrailingSlash } from '../utils/strings.js';
 import { remarkRemoveCodeBlocksInCells } from '../utils/tableCells.js';
 import { getDescriptionFromRoot, getTitleFromHeading } from '../utils/title.js';
+import { unifiedRemoveTableOfContents } from '../utils/toc.js';
 import { remarkRemoveUpdatedAt } from '../utils/updatedAt.js';
 import { downloadImagesFromFile } from './images.js';
 import { htmlToHast } from './root.js';
@@ -78,6 +80,8 @@ export async function scrapePage(
 
   const mdastTree: MdastRoot = unified()
     .use(unifiedRemoveBreaks)
+    .use(unifiedRemoveBreadCrumbs)
+    .use(unifiedRemoveTableOfContents)
     .use(unifiedRemoveCopyButtons)
     .use(createCard)
     .use(createAccordion)

package/src/utils/breadcrumbs.ts

@@ -0,0 +1,31 @@
+import type { Element } from 'hast';
+import { visit } from 'unist-util-visit';
+
+import { framework } from './detectFramework.js';
+
+export function unifiedRemoveBreadCrumbs() {
+  return function (node: Element) {
+    return removeBreadCrumbs(node);
+  };
+}
+
+/**
+ * Docusaurus-specific function since their breadcrumbs
+ * are within the content in the `article` itself instead
+ * of outside of the `article` element
+ */
+export function removeBreadCrumbs(node: Element) {
+  return visit(node, 'element', function (subNode, index, parent) {
+    if (
+      framework.vendor === 'docusaurus' &&
+      subNode.tagName === 'nav' &&
+      subNode.properties.className &&
+      Array.isArray(subNode.properties.className) &&
+      subNode.properties.className.includes('theme-doc-breadcrumbs') &&
+      parent &&
+      typeof index === 'number'
+    ) {
+      parent.children.splice(index, 1);
+    }
+  });
+}

package/src/utils/breaks.ts

@@ -7,9 +7,11 @@ export function unifiedRemoveBreaks() {
   };
 }
 
-// ReadMe-specific function since they use breaks in between
-// every element, but either way our parser adds whitespace
-// automatically
+/**
+ * ReadMe-specific function since they use breaks in between
+ * every element, but either way our parser adds whitespace
+ * automatically
+ */
 export function removeBreaks(node: Element) {
   return visit(node, 'element', function (subNode, index, parent) {
     if (subNode.tagName === 'br' && parent && typeof index === 'number') {

package/src/utils/toc.ts

@@ -0,0 +1,31 @@
+import type { Element } from 'hast';
+import { visit } from 'unist-util-visit';
+
+import { framework } from './detectFramework.js';
+
+export function unifiedRemoveTableOfContents() {
+  return function (node: Element) {
+    return removeTableOfContents(node);
+  };
+}
+
+/**
+ * Docusaurus-specific function since their mobile ToC
+ * is within the content in the `article` itself instead
+ * of outside of the `article` element
+ */
+export function removeTableOfContents(node: Element) {
+  return visit(node, 'element', function (subNode, index, parent) {
+    if (
+      framework.vendor === 'docusaurus' &&
+      subNode.tagName === 'div' &&
+      subNode.properties.className &&
+      Array.isArray(subNode.properties.className) &&
+      subNode.properties.className.includes('theme-doc-toc-mobile') &&
+      parent &&
+      typeof index === 'number'
+    ) {
+      parent.children.splice(index, 1);
+    }
+  });
+}