@mintlify/scraping 4.0.439 → 4.0.441

package/bin/utils/breadcrumbs.js

@@ -14,7 +14,6 @@ export function removeBreadCrumbs(node) {
     return visit(node, 'element', function (subNode, index, parent) {
         if (framework.vendor === 'docusaurus' &&
             subNode.tagName === 'nav' &&
-            subNode.properties.className &&
             Array.isArray(subNode.properties.className) &&
             subNode.properties.className.includes('theme-doc-breadcrumbs') &&
             parent &&

package/bin/utils/breadcrumbs.js.map

@@ -1 +1 @@
-{"version":3,"file":"breadcrumbs.js","sourceRoot":"","sources":["../../src/utils/breadcrumbs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,UAAU,wBAAwB;IACtC,OAAO,UAAU,IAAa;QAC5B,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC7C,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IACE,SAAS,CAAC,MAAM,KAAK,YAAY;YACjC,OAAO,CAAC,OAAO,KAAK,KAAK;YACzB,OAAO,CAAC,UAAU,CAAC,SAAS;YAC5B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAC9D,MAAM;YACN,OAAO,KAAK,KAAK,QAAQ,EACzB,CAAC;YACD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"breadcrumbs.js","sourceRoot":"","sources":["../../src/utils/breadcrumbs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,UAAU,wBAAwB;IACtC,OAAO,UAAU,IAAa;QAC5B,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC7C,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IACE,SAAS,CAAC,MAAM,KAAK,YAAY;YACjC,OAAO,CAAC,OAAO,KAAK,KAAK;YACzB,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAC9D,MAAM;YACN,OAAO,KAAK,KAAK,QAAQ,EACzB,CAAC;YACD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/bin/utils/intersection.d.ts

@@ -0,0 +1 @@
+export declare function intersection<T>(set1: Set<T> | Array<T>, set2: Set<T> | Array<T>): Set<T>;

package/bin/utils/intersection.js

@@ -0,0 +1,13 @@
+export function intersection(set1, set2) {
+    if (Array.isArray(set1))
+        set1 = new Set(set1);
+    if (Array.isArray(set2))
+        set2 = new Set(set2);
+    const intersectedSet = new Set();
+    for (const el of set1) {
+        if (set2.has(el))
+            intersectedSet.add(el);
+    }
+    return intersectedSet;
+}
+//# sourceMappingURL=intersection.js.map

package/bin/utils/intersection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"intersection.js","sourceRoot":"","sources":["../../src/utils/intersection.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY,CAAI,IAAuB,EAAE,IAAuB;IAC9E,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAE9C,MAAM,cAAc,GAAG,IAAI,GAAG,EAAK,CAAC;IACpC,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC"}

package/bin/utils/path.js

@@ -1,7 +1,6 @@
 import { join } from 'path';
 import { addMdx } from './extension.js';
 import { toFilename } from './file.js';
-import { log } from './log.js';
 export function createFilename(rootPath = process.cwd(), filename, title) {
     if (typeof filename === 'string' && filename.startsWith('http')) {
         const url = new URL(filename);
@@ -10,13 +9,9 @@ export function createFilename(rootPath = process.cwd(), filename, title) {
     else if (typeof filename === 'object') {
         filename = filename.pathname;
     }
-    else {
-        filename = filename;
+    if (filename.endsWith('/')) {
+        filename += 'index';
     }
-    if (filename === '') {
-        log(`Invalid file name provided: ${filename}`, 'error');
-        return undefined;
-    }
-    return join(rootPath, addMdx(filename || toFilename(title ?? '')));
+    return join(rootPath, addMdx(filename || toFilename(title || 'index')));
 }
 //# sourceMappingURL=path.js.map

package/bin/utils/path.js.map

@@ -1 +1 @@
-{"version":3,"file":"path.js","sourceRoot":"","sources":["../../src/utils/path.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,cAAc,CAC5B,WAAmB,OAAO,CAAC,GAAG,EAAE,EAChC,QAAsB,EACtB,KAAc;IAEd,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAChE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,CAAC;SAAM,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxC,QAAQ,GAAI,QAAgB,CAAC,QAAQ,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,QAAkB,CAAC;IAChC,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,GAAG,CAAC,+BAA+B,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;QACxD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC"}
+{"version":3,"file":"path.js","sourceRoot":"","sources":["../../src/utils/path.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,UAAU,cAAc,CAC5B,WAAmB,OAAO,CAAC,GAAG,EAAE,EAChC,QAAsB,EACtB,KAAc;IAEd,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAChE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,CAAC;SAAM,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxC,QAAQ,GAAI,QAAgB,CAAC,QAAQ,CAAC;IACxC,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,IAAI,OAAO,CAAC;IACtB,CAAC;IAED,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;AAC1E,CAAC"}

package/bin/utils/toc.js

@@ -14,7 +14,6 @@ export function removeTableOfContents(node) {
     return visit(node, 'element', function (subNode, index, parent) {
         if (framework.vendor === 'docusaurus' &&
             subNode.tagName === 'div' &&
-            subNode.properties.className &&
             Array.isArray(subNode.properties.className) &&
             subNode.properties.className.includes('theme-doc-toc-mobile') &&
             parent &&

package/bin/utils/toc.js.map

@@ -1 +1 @@
-{"version":3,"file":"toc.js","sourceRoot":"","sources":["../../src/utils/toc.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,UAAU,4BAA4B;IAC1C,OAAO,UAAU,IAAa;QAC5B,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAa;IACjD,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IACE,SAAS,CAAC,MAAM,KAAK,YAAY;YACjC,OAAO,CAAC,OAAO,KAAK,KAAK;YACzB,OAAO,CAAC,UAAU,CAAC,SAAS;YAC5B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAC7D,MAAM;YACN,OAAO,KAAK,KAAK,QAAQ,EACzB,CAAC;YACD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"toc.js","sourceRoot":"","sources":["../../src/utils/toc.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,UAAU,4BAA4B;IAC1C,OAAO,UAAU,IAAa;QAC5B,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAa;IACjD,OAAO,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QAC5D,IACE,SAAS,CAAC,MAAM,KAAK,YAAY;YACjC,OAAO,CAAC,OAAO,KAAK,KAAK;YACzB,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAC7D,MAAM;YACN,OAAO,KAAK,KAAK,QAAQ,EACzB,CAAC;YACD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.439",
+  "version": "4.0.441",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -38,7 +38,7 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.580",
+    "@mintlify/common": "1.0.581",
     "@mintlify/openapi-parser": "^0.0.8",
     "fs-extra": "^11.1.1",
     "hast-util-to-mdast": "^10.1.0",
@@ -61,7 +61,7 @@
     "@mintlify/models": "0.0.235",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.504",
+    "@mintlify/validation": "0.1.505",
     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
     "@tsconfig/recommended": "1.x",
     "@types/hast": "^3.0.4",
@@ -77,5 +77,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "d24a7506b0477ac367cc6b501035d9b159c105fc"
+  "gitHead": "58cf3c084aa0428be062febf942f1033623f1e71"
 }

package/src/cli.ts

@@ -129,6 +129,7 @@ async function site(url: string) {
       const docsConfig = upgradeToDocsConfig(mintConfig, {
         shouldUpgradeTheme: true,
       });
+      docsConfig.theme = 'aspen';
       write('docs.json', JSON.stringify(docsConfig, undefined, 2));
       log(FINAL_SUCCESS_MESSAGE);
     } else {

package/src/components/Callout.ts

@@ -11,7 +11,6 @@ export function gitBookScrapeCallout(
 ): Element | undefined {
   if (
     node.tagName !== 'div' ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     !node.properties.className
       .join(' ')
@@ -63,7 +62,6 @@ export function readmeScrapeCallout(
 ): Element | undefined {
   if (
     node.tagName !== 'blockquote' ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     !node.properties.className.includes('callout')
   ) {
@@ -113,7 +111,6 @@ export function docusaurusScrapeCallout(
 ): Element | undefined {
   if (
     node.tagName !== 'div' ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     (!node.properties.className.includes('admonition') &&
       !node.properties.className.includes('theme-admonition'))

package/src/components/Card.ts

@@ -14,7 +14,6 @@ export function gitBookScrapeCard(
 ): Element | undefined {
   if (
     (node.tagName !== 'a' && node.tagName !== 'div') ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     !node.properties.className
       .join(' ')
@@ -28,7 +27,6 @@ export function gitBookScrapeCard(
   let firstTextElement: Element | undefined = undefined;
   visit(node, 'element', function (subNode, index, parent) {
     if (
-      !subNode.properties.className ||
       !Array.isArray(subNode.properties.className) ||
       subNode.properties.className.join(' ') !== 'w-full space-y-2 lg:space-y-3 leading-normal'
     )
@@ -66,7 +64,6 @@ export function readmeScrapeCard(
 ): Element | undefined {
   if (
     (node.tagName !== 'div' && node.tagName !== 'a') ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     (!node.properties.className.includes('Tile') &&
       !node.properties.className.includes('card') &&
@@ -121,7 +118,6 @@ export function docusaurusScrapeCard(
 ): Element | undefined {
   if (
     (node.tagName !== 'div' && node.tagName !== 'a') ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     (!node.properties.className.includes('Tile') &&
       !node.properties.className.includes('card') &&

package/src/components/CardGroup.ts

@@ -57,7 +57,7 @@ export function docusaurusScrapeCardGroup(
     if (child.type === 'element' && child.tagName === 'Card') cardCount++;
   }
 
-  if (cardCount === parent.children.length) {
+  if (cardCount === parent.children.length && cardCount > 1) {
     parent.type = 'element';
     (parent as Element).tagName = 'CardGroup';
   }

package/src/components/CodeGroup.ts

@@ -132,7 +132,6 @@ export function readmeScrapeCodeGroup(
 ): Element | undefined {
   if (
     node.tagName !== 'div' ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     !node.properties.className.includes('CodeTabs')
   ) {
@@ -143,7 +142,6 @@ export function readmeScrapeCodeGroup(
   visit(node, 'element', function (node) {
     if (
       node.tagName !== 'div' ||
-      !node.properties.className ||
       !Array.isArray(node.properties.className) ||
       !node.properties.className.includes('CodeTabs-inner')
     ) {
@@ -195,7 +193,7 @@ export function docusaurusScrapeCodeGroup(
   _: HastNodeIndex,
   parent: HastNodeParent
 ): Element | undefined {
-  if (node.tagName !== 'div' || !node.properties.role || node.properties.role !== 'tablist') {
+  if ((node.tagName !== 'div' && node.tagName !== 'ul') || node.properties.role !== 'tablist') {
     return undefined;
   }
 

package/src/components/Tabs.ts

@@ -59,7 +59,6 @@ export function readmeScrapeTabs(
 ): Element | undefined {
   if (
     (node.tagName !== 'div' && node.tagName !== 'a') ||
-    !node.properties.className ||
     !Array.isArray(node.properties.className) ||
     (!node.properties.className.includes('tabbed-component') &&
       !node.properties.className.includes('tabs') &&
@@ -144,7 +143,7 @@ export function docusaurusScrapeTabs(
   _: HastNodeIndex,
   parent: HastNodeParent
 ): Element | undefined {
-  if (node.tagName !== 'div' || !node.properties.role || node.properties.role !== 'tablist') {
+  if ((node.tagName !== 'div' && node.tagName !== 'ul') || node.properties.role !== 'tablist') {
     return undefined;
   }
 

package/src/constants.ts

@@ -15,6 +15,7 @@ export const SUPPORTED_MEDIA_EXTENSIONS = [
   'pjp',
   'svgz',
   'bmp',
+  'gif',
 ];
 
 export const ESCAPED_COMPONENTS = [

package/src/nav/root.ts

@@ -2,6 +2,7 @@ import type { Root as HastRoot, Element } from 'hast';
 import { visit, EXIT } from 'unist-util-visit';
 
 import { framework } from '../utils/detectFramework.js';
+import { intersection } from '../utils/intersection.js';
 
 export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined {
   let rootTagName = 'aside';
@@ -17,10 +18,10 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
       break;
   }
 
-  let rootSelectorSet = new Set('page-no-toc:hidden');
+  let rootSelectorSet = new Set(['page-no-toc:hidden']);
   switch (framework.vendor) {
     case 'docusaurus':
-      rootSelectorSet = new Set('menu');
+      rootSelectorSet = new Set(['menu']);
       break;
     case 'gitbook':
       rootSelectorSet = new Set([
@@ -30,7 +31,7 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
       ]);
       break;
     case 'readme':
-      rootSelectorSet = new Set('rm-Sidebar');
+      rootSelectorSet = new Set(['rm-Sidebar']);
       break;
   }
 
@@ -39,7 +40,6 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
     const { className } = node.properties;
     if (
       node.tagName === rootTagName &&
-      className &&
       Array.isArray(className) &&
       !!intersection(className, rootSelectorSet).size
     ) {
@@ -50,15 +50,3 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
 
   return element;
 }
-
-function intersection<T>(set1: Set<T> | Array<T>, set2: Set<T> | Array<T>): Set<T> {
-  if (Array.isArray(set1)) set1 = new Set(set1);
-  if (Array.isArray(set2)) set2 = new Set(set2);
-
-  const intersectedSet = new Set<T>();
-  for (const el of set1) {
-    if (set2.has(el)) intersectedSet.add(el);
-  }
-
-  return intersectedSet;
-}

package/src/pipeline/icon.ts

@@ -1,13 +1,15 @@
 import type { Root as HastRoot } from 'hast';
 import { EXIT, visit } from 'unist-util-visit';
 
+import { framework } from '../utils/detectFramework.js';
 import { downloadImage } from '../utils/images.js';
 
 export async function downloadFavicon(hast: HastRoot): Promise<string> {
   let src: string = '';
+  const tagName = framework.vendor === 'docusaurus' ? 'meta' : 'link';
   visit(hast, 'element', function (node) {
     if (
-      node.tagName === 'link' &&
+      node.tagName === tagName &&
       Array.isArray(node.properties.rel) &&
       node.properties.rel.includes('icon')
     ) {
@@ -21,8 +23,7 @@ export async function downloadFavicon(hast: HastRoot): Promise<string> {
   }
 
   const res = await downloadImage(src, process.cwd());
-  if (!res.success) return '/favicon.svg';
-  if (!res.data) return '/favicon.svg';
+  if (!res.success || !res.data) return '/favicon.svg';
 
   return res.data[1];
 }

package/src/pipeline/logo.ts

@@ -37,7 +37,6 @@ function findDocusaurusLogoNodes(root: HastRoot): Array<Element> | undefined {
   const elements: Array<Element> = [];
   visit(root, 'element', function (node) {
     if (
-      node.tagName === 'div' &&
       Array.isArray(node.properties.className) &&
       node.properties.className.includes('navbar__brand')
     ) {
@@ -53,30 +52,26 @@ function findDocusaurusLogoNodes(root: HastRoot): Array<Element> | undefined {
 async function findLogosFromHtml(
   html: string,
   downloadFn: (root: HastRoot) => Array<Element> | undefined,
-  filepaths: Array<string>
-) {
+  filepaths: Array<string>,
+  url: URL
+): Promise<void> {
   const hast = htmlToHast(html);
   const imgNodes = downloadFn(hast);
+  if (!imgNodes) return;
+  const imagePaths = await Promise.all(
+    imgNodes.map(async (node) => {
+      const src = node.properties.src as string;
+      const imageUrl = new URL(src, url.origin).toString();
+      const res = await downloadImage(imageUrl, join(process.cwd(), 'images'));
+      if (res.success && res.data) {
+        return res.data[1];
+      } else {
+        return '';
+      }
+    })
+  );
 
-  if (imgNodes) {
-    filepaths.push(
-      ...(await Promise.all(
-        imgNodes.map(async (node) => {
-          const res = await downloadImage(
-            node.properties.src as string,
-            join(process.cwd(), 'images')
-          );
-
-          if (res.success && res.data) {
-            return res.data[1];
-          } else {
-            return '';
-          }
-        })
-      ))
-    );
-  }
-
+  filepaths.push(...imagePaths.filter(Boolean));
   filepaths.forEach((filepath, index) => {
     if (!filepath) filepaths.splice(index, 1);
   });
@@ -105,7 +100,7 @@ export async function downloadLogos(
 
     await Promise.all(
       htmls.map(async (html) => {
-        return await findLogosFromHtml(html, findReadmeLogoNodes, filepaths);
+        return await findLogosFromHtml(html, findReadmeLogoNodes, filepaths, url);
       })
     );
   } else {
@@ -114,7 +109,8 @@ export async function downloadLogos(
       await findLogosFromHtml(
         html,
         framework.vendor === 'gitbook' ? findGitBookLogoNodes : findDocusaurusLogoNodes,
-        filepaths
+        filepaths,
+        url
       );
     } catch (error) {
       const errorMessage = getErrorMessage(error);

package/src/pipeline/page.ts

@@ -57,7 +57,8 @@ export async function scrapePage(
   url = new URL(url);
 
   if (opts.externalLink) {
-    const filename = html;
+    let filename = html || 'index';
+    if (filename.endsWith('/')) filename += 'index';
     const filenameWithExt = `${filename}.mdx`;
     writePage(filenameWithExt, '', '', '', url.toString());
     return { success: true, data: [url.toString(), filename] };

package/src/root/retrieve.ts

@@ -1,50 +1,36 @@
 import type { Root as HastRoot, Element } from 'hast';
-import { visit, EXIT } from 'unist-util-visit';
+import { visit, EXIT, CONTINUE } from 'unist-util-visit';
 
 import { framework } from '../utils/detectFramework.js';
 
 export function retrieveRootContent(rootNode: HastRoot): Element | undefined {
-  let rootTagName = 'main';
+  let rootSelector: Map<string, string | undefined> = new Map([['main', 'break-anywhere']]);
   switch (framework.vendor) {
     case 'docusaurus':
-      rootTagName = 'article';
+      rootSelector = new Map([
+        ['article', undefined],
+        ['div', 'index-page'],
+      ]);
       break;
     case 'gitbook':
-      rootTagName = 'main';
+      rootSelector = new Map([['main', undefined]]);
       break;
     case 'readme':
-      rootTagName = 'article';
-      break;
-  }
-
-  let rootSelector = 'break-anywhere';
-  switch (framework.vendor) {
-    case 'docusaurus':
-      rootSelector = '';
-      break;
-    case 'gitbook':
-      rootSelector = '';
-      break;
-    case 'readme':
-      rootSelector = 'rm-Article';
+      rootSelector = new Map([['article', 'rm-Article']]);
       break;
   }
 
   let element: Element | undefined = undefined;
   visit(rootNode, 'element', function (node) {
-    if (node.tagName === rootTagName) {
-      if (rootSelector) {
-        if (
-          node.properties.className &&
-          Array.isArray(node.properties.className) &&
-          node.properties.className.includes(rootSelector)
-        ) {
-          element = node;
-        }
-      } else {
-        element = node;
-        return EXIT;
-      }
+    if (!rootSelector.has(node.tagName)) {
+      return CONTINUE;
+    }
+
+    const classNameSelector = rootSelector.get(node.tagName);
+    const { className } = node.properties;
+    if (!classNameSelector || (Array.isArray(className) && className.includes(classNameSelector))) {
+      element = node;
+      return EXIT;
     }
   });
 

package/src/tabs/retrieve.ts

@@ -19,7 +19,6 @@ export function retrieveTabLinks(rootNode: HastRoot, url: URL): Array<Tab> | und
     if (framework.vendor === 'readme') {
       if (
         node.tagName === 'header' &&
-        node.properties.className &&
         Array.isArray(node.properties.className) &&
         node.properties.className.includes('rm-Header')
       ) {
@@ -31,7 +30,6 @@ export function retrieveTabLinks(rootNode: HastRoot, url: URL): Array<Tab> | und
     if (framework.vendor === 'docusaurus') {
       if (
         node.tagName === 'nav' &&
-        node.properties.className &&
         Array.isArray(node.properties.className) &&
         node.properties.className.includes('navbar')
       ) {
@@ -62,7 +60,6 @@ export function retrieveTabLinks(rootNode: HastRoot, url: URL): Array<Tab> | und
           node.tagName !== 'nav' &&
           !(
             node.tagName === 'div' &&
-            node.properties.className &&
             Array.isArray(node.properties.className) &&
             node.properties.className.includes('rm-Header-right')
           )

package/src/utils/breadcrumbs.ts

@@ -19,7 +19,6 @@ export function removeBreadCrumbs(node: Element) {
     if (
       framework.vendor === 'docusaurus' &&
       subNode.tagName === 'nav' &&
-      subNode.properties.className &&
       Array.isArray(subNode.properties.className) &&
       subNode.properties.className.includes('theme-doc-breadcrumbs') &&
       parent &&

package/src/utils/intersection.ts

@@ -0,0 +1,11 @@
+export function intersection<T>(set1: Set<T> | Array<T>, set2: Set<T> | Array<T>): Set<T> {
+  if (Array.isArray(set1)) set1 = new Set(set1);
+  if (Array.isArray(set2)) set2 = new Set(set2);
+
+  const intersectedSet = new Set<T>();
+  for (const el of set1) {
+    if (set2.has(el)) intersectedSet.add(el);
+  }
+
+  return intersectedSet;
+}

package/src/utils/path.ts

@@ -2,7 +2,6 @@ import { join } from 'path';
 
 import { addMdx } from './extension.js';
 import { toFilename } from './file.js';
-import { log } from './log.js';
 
 export function createFilename(
   rootPath: string = process.cwd(),
@@ -14,14 +13,11 @@ export function createFilename(
     filename = url.pathname;
   } else if (typeof filename === 'object') {
     filename = (filename as URL).pathname;
-  } else {
-    filename = filename as string;
   }
 
-  if (filename === '') {
-    log(`Invalid file name provided: ${filename}`, 'error');
-    return undefined;
+  if (filename.endsWith('/')) {
+    filename += 'index';
   }
 
-  return join(rootPath, addMdx(filename || toFilename(title ?? '')));
+  return join(rootPath, addMdx(filename || toFilename(title || 'index')));
 }

package/src/utils/toc.ts

@@ -19,7 +19,6 @@ export function removeTableOfContents(node: Element) {
     if (
       framework.vendor === 'docusaurus' &&
       subNode.tagName === 'div' &&
-      subNode.properties.className &&
       Array.isArray(subNode.properties.className) &&
       subNode.properties.className.includes('theme-doc-toc-mobile') &&
       parent &&