@mintlify/scraping 4.0.5 → 4.0.6

package/bin/components/Tabs.js

@@ -39,40 +39,67 @@ export function gitBookScrapeTabs(node, _, parent) {
     };
     return newNode;
 }
-export function readmeScrapeTabs(node, _, parent) {
+export function readmeScrapeTabs(node, _, __) {
     if ((node.tagName !== 'div' && node.tagName !== 'a') ||
         !node.properties.className ||
         !Array.isArray(node.properties.className) ||
-        !node.properties.className.includes('tabbed-component')) {
+        (!node.properties.className.includes('tabbed-component') &&
+            !node.properties.className.includes('tabs') &&
+            !node.properties.className.includes('Tabs'))) {
         return undefined;
     }
-    if (node.children.length < 2 || !node.children[0] || !node.children[1])
+    if (!node.children[0] || !node.children[1])
         return undefined;
-    const tabTitles = node.children[0];
     const titles = [];
-    visit(tabTitles, 'element', function (subNode) {
-        visit(subNode, 'text', function (textNode) {
-            titles.push(textNode.value);
-            return EXIT;
+    const tabContents = [];
+    if (node.children.length !== 2) {
+        visit(node, 'element', function (subNode) {
+            if (subNode.tagName !== 'label' && subNode.tagName !== 'button')
+                return CONTINUE;
+            let title = '';
+            visit(subNode, 'text', function (textNode) {
+                title += textNode.value;
+            });
+            titles.push(title.trim().replace('\n', ''));
         });
-    });
-    assertIsDefined(parent);
-    node.children.shift();
-    const children = turnChildrenIntoMdx(node.children);
-    const tabChildren = [];
-    for (let childIndex = 0; childIndex < children.length; childIndex++) {
-        const child = children[childIndex];
-        if (child) {
-            tabChildren.push({
-                type: 'element',
-                tagName: 'Tab',
-                properties: {
-                    title: titles[childIndex],
-                },
-                children: [child],
+        tabContents.push(...node.children.filter((subNode) => {
+            if (subNode.type === 'element' &&
+                Array.isArray(subNode.properties.className) &&
+                (subNode.properties.className.includes('tab') ||
+                    subNode.properties.className.includes('Tab') ||
+                    subNode.properties.className.includes('tabbed-content') ||
+                    subNode.properties.className.includes('tab-content')))
+                return true;
+            return false;
+        }));
+    }
+    else {
+        const tabTitles = node.children[0];
+        visit(tabTitles, 'element', function (subNode) {
+            visit(subNode, 'text', function (textNode) {
+                titles.push(textNode.value);
+                return EXIT;
             });
+        });
+        node.children.shift();
+        if (node.children[0].type === 'element') {
+            tabContents.push(...node.children[0].children);
         }
     }
+    const tabChildren = [];
+    tabContents.forEach((tab, index) => {
+        if (!titles[index])
+            return;
+        const children = turnChildrenIntoMdx([tab]);
+        tabChildren.push({
+            type: 'element',
+            tagName: 'Tab',
+            properties: {
+                title: titles[index],
+            },
+            children,
+        });
+    });
     const newNode = {
         type: 'element',
         tagName: 'Tabs',

package/bin/components/Tabs.js.map

@@ -1 +1 @@
-{"version":3,"file":"Tabs.js","sourceRoot":"","sources":["../../src/components/Tabs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,UAAU,iBAAiB,CAC/B,IAAc,EACd,CAAgB,EAChB,MAAsB;IAEtB,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1F,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAClD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ;YACvC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,MAAM,CAAC,CAAC;IACxB,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAExB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAA0B,CAAC;IAC/E,MAAM,WAAW,GAA0B,EAAE,CAAC;IAC9C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACV,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC;iBAC1B;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAY;QACvB,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,MAAM;QACf,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,WAAoC;KAC/C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAc,EACd,CAAgB,EAChB,MAAsB;IAEtB,IACE,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,GAAG,CAAC;QAChD,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS;QAC1B,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACzC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EACvD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,OAAO;QAC3C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ;YACvC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,MAAM,CAAC,CAAC;IACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAEtB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAA0B,CAAC;IAC7E,MAAM,WAAW,GAA0B,EAAE,CAAC;IAC9C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACV,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC;iBAC1B;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAY;QACvB,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,MAAM;QACf,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,WAAoC;KAC/C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAc,EACd,CAAgB,EAChB,MAAsB;IAEtB,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1F,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;QAC9C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ;YACvC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,MAAM,CAAC,CAAC;IACxB,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAExB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAA0B,CAAC;IAC/E,MAAM,WAAW,GAA0B,EAAE,CAAC;IAC9C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACV,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC;iBAC1B;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAY;QACvB,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,MAAM;QACf,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,WAAoC;KAC/C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"Tabs.js","sourceRoot":"","sources":["../../src/components/Tabs.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,UAAU,iBAAiB,CAC/B,IAAc,EACd,CAAgB,EAChB,MAAsB;IAEtB,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1F,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAClD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ;YACvC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,MAAM,CAAC,CAAC;IACxB,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAExB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAA0B,CAAC;IAC/E,MAAM,WAAW,GAA0B,EAAE,CAAC;IAC9C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACV,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC;iBAC1B;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAY;QACvB,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,MAAM;QACf,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,WAAoC;KAC/C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAc,EACd,CAAgB,EAChB,EAAkB;IAElB,IACE,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,GAAG,CAAC;QAChD,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS;QAC1B,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACzC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YACtD,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAC9C,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAE7D,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,WAAW,GAAmB,EAAE,CAAC;IAEvC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;YACtC,IAAI,OAAO,CAAC,OAAO,KAAK,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ;gBAAE,OAAO,QAAQ,CAAC;YAEjF,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ;gBACvC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,WAAW,CAAC,IAAI,CACd,GAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YACnC,IACE,OAAO,CAAC,IAAI,KAAK,SAAS;gBAC1B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;gBAC3C,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC5C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC;oBACvD,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;gBAEvD,OAAO,IAAI,CAAC;YACd,OAAO,KAAK,CAAC;QACf,CAAC,CAAoB,CACtB,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAEnC,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,OAAO;YAC3C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ;gBACvC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAC5B,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACxC,WAAW,CAAC,IAAI,CAAC,GAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAA2B,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAA0B,EAAE,CAAC;IAC9C,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,OAAO;QAC3B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAA0B,CAAC;QACrE,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,UAAU,EAAE;gBACV,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;aACrB;YACD,QAAQ;SACT,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAY;QACvB,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,MAAM;QACf,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,WAAoC;KAC/C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAc,EACd,CAAgB,EAChB,MAAsB;IAEtB,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1F,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;QAC9C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ;YACvC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,eAAe,CAAC,MAAM,CAAC,CAAC;IACxB,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAExB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAA0B,CAAC;IAC/E,MAAM,WAAW,GAA0B,EAAE,CAAC;IAC9C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACV,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC;iBAC1B;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAY;QACvB,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,MAAM;QACf,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,WAAoC;KAC/C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/bin/constants.js

@@ -1,4 +1,4 @@
-import { colors } from './utils/log.js';
+import { activeColors } from './utils/log.js';
 export const SUPPORTED_MEDIA_EXTENSIONS = [
     'png',
     'jpeg',
@@ -44,6 +44,6 @@ ${SPACES}Please double check your documentation provider and ensure they are sup
 ${SPACES}We currently support: ReadMe, GitBook, and Docusaurus`;
 export const MDAST_FAILURE_MSG = 'failed to convert MDAST to Markdown string';
 export const FINAL_SUCCESS_MESSAGE = `We've successfully scraped your docs site.
-${SPACES}We've downloaded the ${colors.cyan}\`navigation\`${colors.default} array (and if necessary, the ${colors.cyan}\`tabs\`${colors.cyan} array)
-${SPACES}into ${colors.blue}\`mint.json\`${colors.default}.`;
+${SPACES}We've downloaded the ${activeColors.cyan}\`navigation\`${activeColors.default} array (and if necessary, the ${activeColors.cyan}\`tabs\`${activeColors.default} array)
+${SPACES}into ${activeColors.blue}\`mint.json\`${activeColors.default}.`;
 //# sourceMappingURL=constants.js.map

package/bin/constants.js.map

@@ -1 +1 @@
-{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;CACN,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,WAAW;IACX,gBAAgB;IAChB,MAAM;IACN,SAAS;IACT,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,WAAW;IACX,WAAW;IACX,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,SAAS;CACD,CAAC;AAEX,MAAM,CAAC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAErC,MAAM,CAAC,MAAM,mBAAmB,GAAG;EACjC,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,eAAe,GAAG;EAC7B,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,iBAAiB,GAAG,4CAA4C,CAAC;AAE9E,MAAM,CAAC,MAAM,qBAAqB,GAAG;EACnC,MAAM,wBAAwB,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,OAAO,iCAAiC,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,IAAI;EAC1I,MAAM,QAAQ,MAAM,CAAC,IAAI,gBAAgB,MAAM,CAAC,OAAO,GAAG,CAAC"}
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;CACN,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,WAAW;IACX,gBAAgB;IAChB,MAAM;IACN,SAAS;IACT,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,WAAW;IACX,WAAW;IACX,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,SAAS;CACD,CAAC;AAEX,MAAM,CAAC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAErC,MAAM,CAAC,MAAM,mBAAmB,GAAG;EACjC,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,eAAe,GAAG;EAC7B,MAAM;EACN,MAAM,uDAAuD,CAAC;AAEhE,MAAM,CAAC,MAAM,iBAAiB,GAAG,4CAA4C,CAAC;AAE9E,MAAM,CAAC,MAAM,qBAAqB,GAAG;EACnC,MAAM,wBAAwB,YAAY,CAAC,IAAI,iBAAiB,YAAY,CAAC,OAAO,iCAAiC,YAAY,CAAC,IAAI,WAAW,YAAY,CAAC,OAAO;EACrK,MAAM,QAAQ,YAAY,CAAC,IAAI,gBAAgB,YAAY,CAAC,OAAO,GAAG,CAAC"}

package/bin/nav/listItems.js

@@ -27,7 +27,6 @@ export function processListItem(node, opts = {
             return EXIT;
         }
     });
-    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
     if (isApiReferenceLink)
         return undefined;
     if (linkHref.startsWith('/'))

package/bin/nav/listItems.js.map

@@ -1 +1 @@
-{"version":3,"file":"listItems.js","sourceRoot":"","sources":["../../src/nav/listItems.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAQjD,MAAM,UAAU,eAAe,CAC7B,IAAa,EACb,OAAwB;IACtB,cAAc,EAAE,KAAK;IACrB,gBAAgB,EAAE,IAAI;IACtB,KAAK,EAAE,SAAS;CACjB;IAED,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAE5B,IAAI,QAAQ,GAAuB,SAAS,CAAC;IAC7C,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAA0B,CAAC;IAEtD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QAC/C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,kBAAkB,GAAG,KAAK,CAAC;IAC/B,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IACE,OAAO,CAAC,OAAO,KAAK,MAAM;YAC1B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,EACrD,CAAC;YACD,kBAAkB,GAAG,IAAI,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,uEAAuE;IACvE,IAAI,kBAAkB;QAAE,OAAO,SAAS,CAAC;IAEzC,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAE/D,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9D,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACxD,CAAC;IACH,CAAC;IAED,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,YAAY,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAC/B,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CACnE;QACC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,GAAG,kBAAkB;QACpD,CAAC,CAAC,QAAQ,CAAC;IAEb,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACpC,IAAI,KAAK,KAAK,QAAQ;gBAAE,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,YAAY,GAAG,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,CAAC;AAC7E,CAAC"}
+{"version":3,"file":"listItems.js","sourceRoot":"","sources":["../../src/nav/listItems.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAQjD,MAAM,UAAU,eAAe,CAC7B,IAAa,EACb,OAAwB;IACtB,cAAc,EAAE,KAAK;IACrB,gBAAgB,EAAE,IAAI;IACtB,KAAK,EAAE,SAAS;CACjB;IAED,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAE5B,IAAI,QAAQ,GAAuB,SAAS,CAAC;IAC7C,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAA0B,CAAC;IAEtD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QAC/C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,kBAAkB,GAAG,KAAgB,CAAC;IAC1C,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;QACtC,IACE,OAAO,CAAC,OAAO,KAAK,MAAM;YAC1B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YAC3C,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,EACrD,CAAC;YACD,kBAAkB,GAAG,IAAI,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,kBAAkB;QAAE,OAAO,SAAS,CAAC;IAEzC,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAE/D,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9D,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACxD,CAAC;IACH,CAAC;IAED,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,YAAY,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAC/B,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CACnE;QACC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,GAAG,kBAAkB;QACpD,CAAC,CAAC,QAAQ,CAAC;IAEb,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACpC,IAAI,KAAK,KAAK,QAAQ;gBAAE,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,YAAY,GAAG,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,CAAC;AAC7E,CAAC"}

package/bin/scrapingPipeline/color.d.ts

@@ -0,0 +1,8 @@
+import { Colors } from '@mintlify/models';
+import type { Root as HastRoot } from 'hast';
+export declare const defaultColors: {
+    primary: string;
+    light: string;
+    dark: string;
+};
+export declare function downloadColors(hast: HastRoot): Promise<Colors>;

package/bin/scrapingPipeline/color.js

@@ -0,0 +1,91 @@
+import { CONTINUE, visit } from 'unist-util-visit';
+import { framework } from '../utils/detectFramework.js';
+function toHex(value) {
+    Math.round(value).toString(16).padStart(2, '0');
+}
+function checkValidHex(str) {
+    if (!str)
+        return false;
+    return /^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$/.test(str);
+}
+function checkRgbBounds(...numbers) {
+    for (const num of numbers) {
+        if (num < 0 || num > 255)
+            return false;
+    }
+    return true;
+}
+function rgbToHex(color) {
+    if (checkValidHex(color))
+        return color;
+    color = color.trim().toLowerCase();
+    let r, g, b;
+    if (/^\d+\s+\d+\s+\d+(\s+[0-9.]+)?$/.test(color)) {
+        [r, g, b] = color.split(/\s+/).map(Number);
+    }
+    else {
+        const values = color.match(/^rgba?\((\d+),(\d+),(\d+)(?:,([0-9.]+))?\)$/);
+        if (!values) {
+            return undefined;
+        }
+        [, r, g, b] = values.map(Number);
+    }
+    if (!r || !g || !b)
+        return undefined;
+    if (!checkRgbBounds(r, g, b))
+        return undefined;
+    return `#${toHex(r)}${toHex(g)}${toHex(b)}`.toUpperCase();
+}
+function getCssValue(cssString, key) {
+    const regex = new RegExp(`${key}\\s*[:|,]\\s*([^;)]+)`, 'i');
+    const match = cssString.match(regex);
+    return match && match[1] ? match[1].trim() : undefined;
+}
+export const defaultColors = {
+    primary: '#0D9373',
+    light: '#55D799',
+    dark: '#0D9373',
+};
+export async function downloadColors(hast) {
+    if (framework.vendor === 'docusaurus')
+        return defaultColors;
+    let primaryHexCode = undefined;
+    let lightHexCode = undefined;
+    visit(hast, 'element', function (node) {
+        if (node.tagName !== 'style')
+            return CONTINUE;
+        if ((framework.vendor === 'gitbook' && !!Object.keys(node.properties).length) ||
+            (framework.vendor === 'readme' && node.properties.title !== 'rm-custom-css'))
+            return CONTINUE;
+        if (node.children.length !== 1 || !node.children[0] || node.children[0].type !== 'text')
+            return CONTINUE;
+        const cssStr = node.children[0].value;
+        const primaryColorKey = framework.vendor === 'readme' ? '--color-link-primary' : '--primary-color-600';
+        const lightColorKey = framework.vendor === 'readme' ? '--color-link-primary' : '--primary-color-400';
+        const primaryCssColorValue = getCssValue(cssStr, primaryColorKey);
+        const lightCssColorValue = getCssValue(cssStr, lightColorKey);
+        if (!primaryCssColorValue || !lightCssColorValue)
+            return CONTINUE;
+        primaryHexCode = rgbToHex(primaryCssColorValue);
+        lightHexCode = rgbToHex(lightCssColorValue);
+    });
+    const isPrimaryValid = checkValidHex(primaryHexCode);
+    const isLightValid = checkValidHex(lightHexCode);
+    if (isPrimaryValid && isLightValid) {
+        return {
+            primary: primaryHexCode,
+            light: lightHexCode,
+            dark: primaryHexCode,
+        };
+    }
+    else if (isPrimaryValid) {
+        return {
+            primary: primaryHexCode,
+            dark: primaryHexCode,
+        };
+    }
+    else {
+        return defaultColors;
+    }
+}
+//# sourceMappingURL=color.js.map

package/bin/scrapingPipeline/color.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"color.js","sourceRoot":"","sources":["../../src/scrapingPipeline/color.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAExD,SAAS,KAAK,CAAC,KAAa;IAC1B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,aAAa,CAAC,GAAuB;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,OAAO,oCAAoC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,cAAc,CAAC,GAAG,OAAsB;IAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,GAAG;YAAE,OAAO,KAAK,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,aAAa,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEnC,IAAI,CAAqB,EAAE,CAAqB,EAAE,CAAqB,CAAC;IAExE,IAAI,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAE1E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAErC,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAE/C,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;AAC5D,CAAC;AAED,SAAS,WAAW,CAAC,SAAiB,EAAE,GAAW;IACjD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,uBAAuB,EAAE,GAAG,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrC,OAAO,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAc;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,aAAa,CAAC;IAE5D,IAAI,cAAc,GAAuB,SAAS,CAAC;IACnD,IAAI,YAAY,GAAuB,SAAS,CAAC;IACjD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YAAE,OAAO,QAAQ,CAAC;QAC9C,IACE,CAAC,SAAS,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;YACzE,CAAC,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,KAAK,eAAe,CAAC;YAE5E,OAAO,QAAQ,CAAC;QAElB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM;YACrF,OAAO,QAAQ,CAAC;QAElB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACtC,MAAM,eAAe,GACnB,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACjF,MAAM,aAAa,GACjB,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,qBAAqB,CAAC;QAEjF,MAAM,oBAAoB,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAClE,MAAM,kBAAkB,GAAG,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC9D,IAAI,CAAC,oBAAoB,IAAI,CAAC,kBAAkB;YAAE,OAAO,QAAQ,CAAC;QAElE,cAAc,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAChD,YAAY,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;QACnC,OAAO;YACL,OAAO,EAAE,cAAe;YACxB,KAAK,EAAE,YAAY;YACnB,IAAI,EAAE,cAAc;SACrB,CAAC;IACJ,CAAC;SAAM,IAAI,cAAc,EAAE,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,cAAe;YACxB,IAAI,EAAE,cAAc;SACrB,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,aAAa,CAAC;IACvB,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/group.js

@@ -34,13 +34,11 @@ export async function scrapePageGroup(navGroup, needsBrowser, opts = { externalL
     catch (error) {
         const errorMessage = getErrorMessage(error);
         log(`We encountered an error when scraping the page group from ${navGroup[0]?.origin ?? 'the URL provided'}${errorMessage}`);
-        console.error(error);
         throw error;
     }
     finally {
-        if (browser) {
+        if (browser)
             await browser.close();
-        }
     }
 }
 //# sourceMappingURL=group.js.map

package/bin/scrapingPipeline/group.js.map

@@ -1 +1 @@
-{"version":3,"file":"group.js","sourceRoot":"","sources":["../../src/scrapingPipeline/group.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAoB,EACpB,YAAqB,EACrB,OAGI,EAAE,aAAa,EAAE,KAAK,EAAE;IAE5B,MAAM,OAAO,GAAwB,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;YAChC,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACvB,MAAM,GAAG,GAAG,UAAU,CAAC,iBAAiB,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC9E,OAAO,GAAG,CAAC;gBACb,CAAC;gBAED,IAAI,cAAc,GAAG,KAAK,CAAC;gBAC3B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;oBAChD,cAAc,GAAG,IAAI,CAAC;oBACtB,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;gBAChE,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE;oBAChC,YAAY,EAAE,KAAK;oBACnB,cAAc;oBACd,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7D,CAAC,CAAC;gBACH,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,GAAG,YAAY,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CACD,6DACE,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,kBACzB,GAAG,YAAY,EAAE,CAClB,CAAC;QACF,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"group.js","sourceRoot":"","sources":["../../src/scrapingPipeline/group.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAoB,EACpB,YAAqB,EACrB,OAGI,EAAE,aAAa,EAAE,KAAK,EAAE;IAE5B,MAAM,OAAO,GAAwB,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;YAChC,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACvB,MAAM,GAAG,GAAG,UAAU,CAAC,iBAAiB,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC9E,OAAO,GAAG,CAAC;gBACb,CAAC;gBAED,IAAI,cAAc,GAAG,KAAK,CAAC;gBAC3B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;oBAChD,cAAc,GAAG,IAAI,CAAC;oBACtB,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;gBAChE,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE;oBAChC,YAAY,EAAE,KAAK;oBACnB,cAAc;oBACd,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7D,CAAC,CAAC;gBACH,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,GAAG,YAAY,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CACD,6DACE,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,kBACzB,GAAG,YAAY,EAAE,CAClB,CAAC;QACF,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/icon.d.ts

@@ -1,2 +1,2 @@
 import type { Root as HastRoot } from 'hast';
-export declare function downloadFavicon(hast: HastRoot): Promise<string | undefined>;
+export declare function downloadFavicon(hast: HastRoot): Promise<string>;

package/bin/scrapingPipeline/icon.js

@@ -4,19 +4,20 @@ export async function downloadFavicon(hast) {
     let src = '';
     visit(hast, 'element', function (node) {
         if (node.tagName === 'link' &&
-            typeof node.properties.rel === 'string' &&
-            (node.properties.rel === 'icon' || node.properties.rel === 'shortcut icon')) {
+            Array.isArray(node.properties.rel) &&
+            node.properties.rel.includes('icon')) {
             src = node.properties.href;
             return EXIT;
         }
     });
-    if (!src)
-        return undefined;
+    if (!src) {
+        return '/favicon.svg';
+    }
     const res = await downloadImage(src, process.cwd());
     if (!res.success)
-        return undefined;
+        return '/favicon.svg';
     if (!res.data)
-        return undefined;
+        return '/favicon.svg';
     return res.data[1];
 }
 //# sourceMappingURL=icon.js.map

package/bin/scrapingPipeline/icon.js.map

@@ -1 +1 @@
-{"version":3,"file":"icon.js","sourceRoot":"","sources":["../../src/scrapingPipeline/icon.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAc;IAClD,IAAI,GAAG,GAAW,EAAE,CAAC;IACrB,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,MAAM;YACvB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ;YACvC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,eAAe,CAAC,EAC3E,CAAC;YACD,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,IAAc,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAE3B,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IACnC,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAEhC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC"}
+{"version":3,"file":"icon.js","sourceRoot":"","sources":["../../src/scrapingPipeline/icon.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAc;IAClD,IAAI,GAAG,GAAW,EAAE,CAAC;IACrB,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,MAAM;YACvB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EACpC,CAAC;YACD,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,IAAc,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,cAAc,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,OAAO,cAAc,CAAC;IAErC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC"}

package/bin/scrapingPipeline/logo.js

@@ -64,13 +64,19 @@ export async function downloadLogos(url, browser) {
     const filepaths = [];
     if (browser) {
         const htmls = [];
-        const page = await browser.newPage();
-        await page.goto(url.toString(), {
-            waitUntil: 'networkidle2',
-        });
-        htmls.push(await page.content());
-        await page.click('.rm-ThemeToggle');
-        htmls.push(await page.content());
+        try {
+            const page = await browser.newPage();
+            await page.goto(url.toString(), {
+                waitUntil: 'networkidle2',
+            });
+            htmls.push(await page.content());
+            await page.click('.rm-ThemeToggle');
+            htmls.push(await page.content());
+        }
+        catch {
+            // do nothing, it just means there's no theme toggle
+        }
+        await browser.close();
         await Promise.all(htmls.map(async (html) => {
             return await findLogosFromHtml(html, findReadmeLogoNodes, filepaths);
         }));
@@ -85,8 +91,6 @@ export async function downloadLogos(url, browser) {
             log(`Failed to retrieve logo from HTML: ${errorMessage}`);
         }
     }
-    if (browser)
-        await browser.close();
     const uniqueFilepaths = [...new Set(filepaths).values()];
     return uniqueFilepaths.length === 1
         ? uniqueFilepaths[0]

package/bin/scrapingPipeline/logo.js.map

@@ -1 +1 @@
-{"version":3,"file":"logo.js","sourceRoot":"","sources":["../../src/scrapingPipeline/logo.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,SAAS,mBAAmB,CAAC,IAAc;IACzC,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC;YAEjD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAc;IAC1C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,MAAM,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAc;IAC7C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,EACnD,CAAC;YACD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK;oBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,IAAY,EACZ,UAA0D,EAC1D,SAAwB;IAExB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,QAAQ,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CACZ,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CACnB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,MAAM,aAAa,CAC7B,IAAI,CAAC,UAAU,CAAC,GAAa,EAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAC9B,CAAC;YAEF,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5B,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CACH,CAAC,CACH,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACpC,IAAI,CAAC,QAAQ;YAAE,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,OAA4B;IAE5B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC9B,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACjC,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAEjC,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,MAAM,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAC;QACvE,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,iBAAiB,CACrB,IAAI,EACJ,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,uBAAuB,EAC/E,SAAS,CACV,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5C,GAAG,CAAC,sCAAsC,YAAY,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,IAAI,OAAO;QAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEnC,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzD,OAAO,eAAe,CAAC,MAAM,KAAK,CAAC;QACjC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;YAC1B,CAAC,CAAC;gBACE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAW;gBACnC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAW;aACnC;YACH,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC"}
+{"version":3,"file":"logo.js","sourceRoot":"","sources":["../../src/scrapingPipeline/logo.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,SAAS,mBAAmB,CAAC,IAAc;IACzC,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC;YAEjD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAc;IAC1C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,MAAM,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAc;IAC7C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,EACnD,CAAC;YACD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK;oBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,IAAY,EACZ,UAA0D,EAC1D,SAAwB;IAExB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,QAAQ,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CACZ,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CACnB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,MAAM,aAAa,CAC7B,IAAI,CAAC,UAAU,CAAC,GAAa,EAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAC9B,CAAC;YAEF,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5B,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CACH,CAAC,CACH,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACpC,IAAI,CAAC,QAAQ;YAAE,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,OAA4B;IAE5B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;gBAC9B,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;YAEH,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACjC,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,oDAAoD;QACtD,CAAC;QAED,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEtB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,MAAM,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAC;QACvE,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,iBAAiB,CACrB,IAAI,EACJ,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,uBAAuB,EAC/E,SAAS,CACV,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5C,GAAG,CAAC,sCAAsC,YAAY,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzD,OAAO,eAAe,CAAC,MAAM,KAAK,CAAC;QACjC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;YAC1B,CAAC,CAAC;gBACE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAW;gBACnC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAW;aACnC;YACH,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC"}

package/bin/scrapingPipeline/page.js

@@ -8,18 +8,25 @@ import { createCallout, createCard, createAccordion, createAccordionGroup, creat
 import { rehypeToRemarkCustomComponents } from '../customComponents/plugin.js';
 import { selectiveRehypeRemark } from '../customComponents/selective.js';
 import { retrieveRootContent } from '../root/retrieve.js';
+import { unifiedRemoveBreaks } from '../utils/breaks.js';
 import { unifiedRemoveClassNames } from '../utils/className.js';
+import { unifiedRemoveCopyButtons } from '../utils/copyButton.js';
 import { detectFramework, framework } from '../utils/detectFramework.js';
+import { remarkRemoveEmptyEmphases } from '../utils/emptyEmphasis.js';
 import { unifiedRemoveEmptyParagraphs } from '../utils/emptyParagraphs.js';
 import { getErrorMessage, logErrorResults } from '../utils/errors.js';
-import { escapeCharactersOutsideCodeBlocks } from '../utils/escape.js';
-import { write, writePage } from '../utils/file.js';
+import { writePage } from '../utils/file.js';
+import { remarkProperlyFormatEmphasis } from '../utils/formatEmphasis.js';
 import { removeHastComments } from '../utils/hastComments.js';
+import { remarkSpaceListsOut } from '../utils/lists.js';
 import { log } from '../utils/log.js';
+import { remarkRemoveBottomMetadata } from '../utils/metadata.js';
 import { unifiedRemoveNestedRoots } from '../utils/nestedRoots.js';
 import { unifiedRemovePositions } from '../utils/position.js';
 import { removeLeadingSlash, removeTrailingSlash } from '../utils/strings.js';
+import { remarkRemoveCodeBlocksInCells } from '../utils/tableCells.js';
 import { getDescriptionFromRoot, getTitleFromHeading } from '../utils/title.js';
+import { remarkRemoveUpdatedAt } from '../utils/updatedAt.js';
 import { downloadImagesFromFile } from './images.js';
 import { htmlToHast } from './root.js';
 export async function scrapePage(html, url, opts = { externalLink: false }) {
@@ -37,24 +44,25 @@ export async function scrapePage(html, url, opts = { externalLink: false }) {
     const urlStr = url.toString();
     const content = retrieveRootContent(hast);
     if (!content)
-        return { success: false, message: `${urlStr}: ${CONTENT_FAILURE_MSG}` };
+        return { success: false, message: `${urlStr}: ${CONTENT_FAILURE_MSG}`, data: [urlStr, ''] };
     const contentAsRoot = {
         type: 'root',
         children: [content],
     };
     const mdastTree = unified()
+        .use(unifiedRemoveBreaks)
+        .use(unifiedRemoveCopyButtons)
         .use(createCard)
         .use(createAccordion)
         .use(createFrame)
-        .use(createTabs)
         .use(createCallout)
         .use(createCardGroup)
         .use(createAccordionGroup)
         .use(createCodeGroup)
+        .use(createTabs)
         .use(unifiedRemoveClassNames)
-        .use(unifiedRemovePositions)
         .use(unifiedRemoveEmptyParagraphs)
-        .use(escapeCharactersOutsideCodeBlocks)
+        .use(unifiedRemovePositions)
         .use(selectiveRehypeRemark)
         // Cleans up any nested components left untouched
         // by `selectiveRehypeRemark`, and converts them to
@@ -62,6 +70,13 @@ export async function scrapePage(html, url, opts = { externalLink: false }) {
         .use(rehypeToRemarkCustomComponents)
         .use(convertHeaderLinksToText)
         .use(unifiedRemoveNestedRoots)
+        .use(remarkSpaceListsOut)
+        .use(remarkRemoveBottomMetadata)
+        .use(remarkRemoveUpdatedAt)
+        .use(remarkRemoveEmptyEmphases)
+        .use(remarkProperlyFormatEmphasis)
+        .use(remarkRemoveCodeBlocksInCells)
+        // @ts-expect-error moving some of the pipeline around results in contentAsRoot being treated differently than its type which is Root Element
         .runSync(contentAsRoot);
     try {
         const imageResults = await downloadImagesFromFile(mdastTree, url);
@@ -81,13 +96,14 @@ export async function scrapePage(html, url, opts = { externalLink: false }) {
             // @ts-expect-error remarkStringify errors even if used for valid code from documentation examples
             .use(remarkStringify)
             .stringify(mdastTree);
+        const resultStr = String(result).replace(/\n{3,}/g, '\n\n');
         if (opts.rootPath) {
             url = new URL(opts.rootPath, url.origin);
         }
         else if (url.origin === removeTrailingSlash(url.toString())) {
             url = new URL('home', new URL(url).origin);
         }
-        writePage(url, opts.isOverviewPage ? 'Overview' : title, description, String(result));
+        writePage(url, opts.isOverviewPage ? 'Overview' : title, description, resultStr);
         return {
             success: true,
             data: opts.rootPath
@@ -96,9 +112,12 @@ export async function scrapePage(html, url, opts = { externalLink: false }) {
         };
     }
     catch (error) {
-        write('error.json', JSON.stringify(mdastTree, undefined, 2));
         const errorMessage = getErrorMessage(error);
-        return { success: false, message: `${urlStr}: ${MDAST_FAILURE_MSG}${errorMessage}` };
+        return {
+            success: false,
+            message: `${urlStr}: ${MDAST_FAILURE_MSG}${errorMessage}`,
+            data: [urlStr, ''],
+        };
     }
 }
 //# sourceMappingURL=page.js.map

package/bin/scrapingPipeline/page.js.map

@@ -1 +1 @@
-{"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/scrapingPipeline/page.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,eAAe,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EACL,aAAa,EACb,UAAU,EACV,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,UAAU,EACV,eAAe,GAChB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,iCAAiC,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAII,EAAE,YAAY,EAAE,KAAK,EAAE;IAE3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC;QACtB,MAAM,eAAe,GAAG,GAAG,QAAQ,MAAM,CAAC;QAC1C,SAAS,CAAC,eAAe,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAEzB,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,mBAAmB,EAAE,EAAE,CAAC;IAEtF,MAAM,aAAa,GAAa;QAC9B,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEF,MAAM,SAAS,GAAc,OAAO,EAAE;SACnC,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,WAAW,CAAC;SAChB,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,aAAa,CAAC;SAClB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,oBAAoB,CAAC;SACzB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,uBAAuB,CAAC;SAC5B,GAAG,CAAC,sBAAsB,CAAC;SAC3B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,iCAAiC,CAAC;SACtC,GAAG,CAAC,qBAAqB,CAAC;QAE3B,iDAAiD;QACjD,mDAAmD;QACnD,4BAA4B;SAC3B,GAAG,CAAC,8BAA8B,CAAC;SACnC,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,wBAAwB,CAAC;SAC7B,OAAO,CAAC,aAAa,CAAc,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClE,eAAe,CAAC,wBAAwB,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,yDAAyD,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QAC9F,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,EAAE;aACrB,GAAG,CAAC,SAAS,CAAC;aACd,GAAG,CAAC,SAAS,CAAC;YACf,kGAAkG;aACjG,GAAG,CAAC,eAAe,CAAC;aACpB,SAAS,CAAC,SAAS,CAAC,CAAC;QAExB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC9D,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACtF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI,CAAC,QAAQ;gBACjB,CAAC,CAAC,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC;gBACpF,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,iBAAiB,GAAG,YAAY,EAAE,EAAE,CAAC;IACvF,CAAC;AACH,CAAC"}
+{"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/scrapingPipeline/page.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,eAAe,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EACL,aAAa,EACb,UAAU,EACV,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,UAAU,EACV,eAAe,GAChB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAII,EAAE,YAAY,EAAE,KAAK,EAAE;IAE3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC;QACtB,MAAM,eAAe,GAAG,GAAG,QAAQ,MAAM,CAAC;QAC1C,SAAS,CAAC,eAAe,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAEzB,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,OAAO;QACV,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,mBAAmB,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAE9F,MAAM,aAAa,GAAa;QAC9B,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEF,MAAM,SAAS,GAAc,OAAO,EAAE;SACnC,GAAG,CAAC,mBAAmB,CAAC;SACxB,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,WAAW,CAAC;SAChB,GAAG,CAAC,aAAa,CAAC;SAClB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,oBAAoB,CAAC;SACzB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,uBAAuB,CAAC;SAC5B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,sBAAsB,CAAC;SAC3B,GAAG,CAAC,qBAAqB,CAAC;QAE3B,iDAAiD;QACjD,mDAAmD;QACnD,4BAA4B;SAC3B,GAAG,CAAC,8BAA8B,CAAC;SACnC,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,mBAAmB,CAAC;SACxB,GAAG,CAAC,0BAA0B,CAAC;SAC/B,GAAG,CAAC,qBAAqB,CAAC;SAC1B,GAAG,CAAC,yBAAyB,CAAC;SAC9B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,6BAA6B,CAAC;QACnC,6IAA6I;SAC5I,OAAO,CAAC,aAAa,CAAc,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClE,eAAe,CAAC,wBAAwB,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,yDAAyD,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QAC9F,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,EAAE;aACrB,GAAG,CAAC,SAAS,CAAC;aACd,GAAG,CAAC,SAAS,CAAC;YACf,kGAAkG;aACjG,GAAG,CAAC,eAAe,CAAC;aACpB,SAAS,CAAC,SAAS,CAAC,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC9D,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACjF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI,CAAC,QAAQ;gBACjB,CAAC,CAAC,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC;gBACpF,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,GAAG,MAAM,KAAK,iBAAiB,GAAG,YAAY,EAAE;YACzD,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;SACnB,CAAC;IACJ,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/site.js

@@ -7,11 +7,13 @@ import { detectFramework, framework } from '../utils/detectFramework.js';
 import { logErrorResults } from '../utils/errors.js';
 import { startPuppeteer } from '../utils/network.js';
 import { INDEX_NAMES, iterateThroughReservedNames } from '../utils/reservedNames.js';
-import { removeTrailingSlash } from '../utils/strings.js';
+import { removeTrailingSlash, removeLeadingSlash } from '../utils/strings.js';
+import { downloadColors } from './color.js';
 import { scrapePageGroup } from './group.js';
 import { downloadFavicon } from './icon.js';
 import { downloadLogos } from './logo.js';
 import { htmlToHast } from './root.js';
+import { downloadTitle } from './title.js';
 export async function scrapeSite(html, url, opts = {}) {
     let hast = opts.hast;
     if (!hast)
@@ -96,21 +98,76 @@ export async function scrapeSite(html, url, opts = {}) {
                 pages: [navItem],
             };
         });
+        const allErrors = [
+            ...externalResults.filter((result) => !result.success),
+            ...internalResults.filter((result) => !result.success),
+            ...rootResults.filter((result) => !result.success),
+        ];
+        const allErroredPaths = allErrors
+            .map((result) => {
+            if (result.data) {
+                const url = new URL(result.data[0]);
+                const pathname = url.pathname;
+                const normalizedPathname = removeLeadingSlash(removeTrailingSlash(pathname));
+                return normalizedPathname;
+            }
+            else {
+                return '';
+            }
+        })
+            .filter(Boolean);
+        traverse(navItems).forEach(function (value) {
+            if (typeof value === 'string' && allErroredPaths.includes(value)) {
+                this.remove();
+            }
+            else if (Array.isArray(value)) {
+                this.update(value
+                    .filter((item) => typeof item === 'string' && allErroredPaths.includes(item) ? undefined : item)
+                    .filter(Boolean));
+            }
+        });
+        let count = 1;
+        while (count > 0) {
+            count = 0;
+            traverse(navItems).forEach(function (value) {
+                if (Array.isArray(value) && value.filter(Boolean).length === 0) {
+                    count++;
+                    if (this.parent) {
+                        this.parent.remove();
+                    }
+                    else {
+                        this.remove();
+                    }
+                }
+            });
+        }
+        traverse(navItems).forEach(function (value) {
+            if (typeof value === 'string' &&
+                (value.startsWith('https://') || value.startsWith('http://'))) {
+                this.remove();
+            }
+            else if (Array.isArray(value) &&
+                value.find((val) => typeof val === 'string' && (val.startsWith('https://') || val.startsWith('http://')))) {
+                this.update(value.filter((val) => !(typeof val === 'string' &&
+                    (val.startsWith('https://') || val.startsWith('http://')))));
+            }
+        });
         logErrorResults('linking to external pages', externalResults);
         logErrorResults('scraping your docs', [...internalResults, ...rootResults]);
-        const browser = needsBrowser ? await startPuppeteer() : undefined;
+        const needsBrowserForLogos = framework.vendor === 'readme';
+        const browser = needsBrowserForLogos ? await startPuppeteer() : undefined;
         const favicon = await downloadFavicon(hast);
+        const colors = await downloadColors(hast);
         const logo = await downloadLogos(url, browser);
+        const name = await downloadTitle(hast);
         return {
             success: true,
             data: {
                 $schema: 'https://mintlify.com/schema.json',
-                name: '',
+                name,
                 logo,
-                colors: {
-                    primary: '',
-                },
-                favicon: favicon ?? '',
+                colors,
+                favicon,
                 navigation: navItems,
                 tabs: opts.tabs,
             },