@mintlify/scraping 4.0.589 → 4.0.591

package/bin/utils/network.js

@@ -65,6 +65,52 @@ export async function getHtmlWithPuppeteer(browser, url) {
                 clickItems(document);
             });
         }
+        if (framework.vendor === 'gitbook') {
+            for (let round = 0; round < 10; round++) {
+                const clickedCount = await page.evaluate(() => {
+                    const tocEl = document.getElementById('table-of-contents');
+                    if (!tocEl)
+                        return 0;
+                    let count = 0;
+                    const items = tocEl.querySelectorAll('li.page-document-item');
+                    items.forEach((li) => {
+                        const btn = li.querySelector(':scope > a button');
+                        if (!btn || !(btn instanceof HTMLElement))
+                            return;
+                        const anchor = btn.closest('a');
+                        if (!anchor)
+                            return;
+                        const sibling = anchor.nextElementSibling;
+                        if (sibling instanceof HTMLElement &&
+                            sibling.style.opacity === '1' &&
+                            sibling.style.height === 'auto')
+                            return;
+                        btn.click();
+                        count++;
+                    });
+                    return count;
+                });
+                if (clickedCount === 0)
+                    break;
+                await page
+                    .waitForFunction(() => {
+                    const tocEl = document.getElementById('table-of-contents');
+                    if (!tocEl)
+                        return true;
+                    const anchors = tocEl.querySelectorAll('li.page-document-item > a');
+                    return Array.from(anchors).every((a) => {
+                        const btn = a.querySelector('button');
+                        if (!btn)
+                            return true;
+                        const sibling = a.nextElementSibling;
+                        if (!sibling || !(sibling instanceof HTMLElement))
+                            return true;
+                        return sibling.style.opacity === '1' && sibling.style.height === 'auto';
+                    });
+                }, { timeout: 5000 })
+                    .catch(() => { });
+            }
+        }
         const content = await exponentialBackoff(() => page.content());
         await page.close();
         return content;

package/bin/utils/network.js.map

@@ -1 +1 @@
-{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACnG,OAAO,IAAI,MAAM,SAAS,CAAC;AAE3B,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;CAC/G,CAAC;AAEX,MAAM,OAAO,GAAG;IACd,iBAAiB,EAAE,gBAAgB;IACnC,MAAM,EACJ,yIAAyI;IAC3I,YAAY,EACV,uHAAuH;IACzH,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,WAAW,CAAC;YACrB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,IAAI;YACZ,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,KAAK;YACf,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,YAAY,CACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxB,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CACH,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACvB,QAAQ,CAAC,gBAAgB,CACvB,OAAO,EACP,CAAC,CAAC,EAAE,EAAE;oBACJ,IAAI,CAAC,CAAC,MAAM,YAAY,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC;wBACnF,CAAC,CAAC,cAAc,EAAE,CAAC;gBACvB,CAAC,EACD,IAAI,CACL,CAAC;gBAEF,SAAS,UAAU,CAAC,EAA0B;oBAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,sBAAsB,CACzC,qBAAqB,CACW,CAAC;oBACnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;wBACb,UAAU,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAQ;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC9C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAqB,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,iDAAiD,YAAY,EAAE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAQ;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC9C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uDAAuD,YAAY,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,qDAAqD,YAAY,EAAE,CAAC,CAAC;IAC7F,CAAC;AACH,CAAC"}
+{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACnG,OAAO,IAAI,MAAM,SAAS,CAAC;AAE3B,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;IACvH,uHAAuH;CAC/G,CAAC;AAEX,MAAM,OAAO,GAAG;IACd,iBAAiB,EAAE,gBAAgB;IACnC,MAAM,EACJ,yIAAyI;IAC3I,YAAY,EACV,uHAAuH;IACzH,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,MAAM,IAAI,CAAC,WAAW,CAAC;YACrB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,IAAI;YACZ,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,KAAK;YACf,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,YAAY,CACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxB,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK;SACf,CAAC,CACH,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACvB,QAAQ,CAAC,gBAAgB,CACvB,OAAO,EACP,CAAC,CAAC,EAAE,EAAE;oBACJ,IAAI,CAAC,CAAC,MAAM,YAAY,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC;wBACnF,CAAC,CAAC,cAAc,EAAE,CAAC;gBACvB,CAAC,EACD,IAAI,CACL,CAAC;gBAEF,SAAS,UAAU,CAAC,EAA0B;oBAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,sBAAsB,CACzC,qBAAqB,CACW,CAAC;oBACnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;wBACb,UAAU,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;oBAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;oBAC3D,IAAI,CAAC,KAAK;wBAAE,OAAO,CAAC,CAAC;oBACrB,IAAI,KAAK,GAAG,CAAC,CAAC;oBACd,MAAM,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;oBAC9D,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;wBACnB,MAAM,GAAG,GAAG,EAAE,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;wBAClD,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,YAAY,WAAW,CAAC;4BAAE,OAAO;wBAClD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAChC,IAAI,CAAC,MAAM;4BAAE,OAAO;wBACpB,MAAM,OAAO,GAAG,MAAM,CAAC,kBAAkB,CAAC;wBAC1C,IACE,OAAO,YAAY,WAAW;4BAC9B,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,GAAG;4BAC7B,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM;4BAE/B,OAAO;wBACT,GAAG,CAAC,KAAK,EAAE,CAAC;wBACZ,KAAK,EAAE,CAAC;oBACV,CAAC,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC,CAAC,CAAC;gBAEH,IAAI,YAAY,KAAK,CAAC;oBAAE,MAAM;gBAE9B,MAAM,IAAI;qBACP,eAAe,CACd,GAAG,EAAE;oBACH,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;oBAC3D,IAAI,CAAC,KAAK;wBAAE,OAAO,IAAI,CAAC;oBACxB,MAAM,OAAO,GAAG,KAAK,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC;oBACpE,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;wBACrC,MAAM,GAAG,GAAG,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;wBACtC,IAAI,CAAC,GAAG;4BAAE,OAAO,IAAI,CAAC;wBACtB,MAAM,OAAO,GAAG,CAAC,CAAC,kBAAkB,CAAC;wBACrC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,YAAY,WAAW,CAAC;4BAAE,OAAO,IAAI,CAAC;wBAC/D,OAAO,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;oBAC1E,CAAC,CAAC,CAAC;gBACL,CAAC,EACD,EAAE,OAAO,EAAE,IAAI,EAAE,CAClB;qBACA,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,CAAC;QACH,IAAI,GAAG,GAAuB,SAAS,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAQ;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC9C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAqB,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,iDAAiD,YAAY,EAAE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAQ;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE;YAC9C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uDAAuD,YAAY,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,qDAAqD,YAAY,EAAE,CAAC,CAAC;IAC7F,CAAC;AACH,CAAC"}

package/bin/utils/strings.d.ts

@@ -1,2 +1,3 @@
 export declare function removeTrailingSlash(str: string): string;
 export declare function removeLeadingSlash(str: string): string;
+export declare function optionallyAddLeadingSlash(str: string): string;

package/bin/utils/strings.js

@@ -4,4 +4,7 @@ export function removeTrailingSlash(str) {
 export function removeLeadingSlash(str) {
     return str.startsWith('/') ? str.substring(1) : str;
 }
+export function optionallyAddLeadingSlash(str) {
+    return str.startsWith('/') ? str : '/' + str;
+}
 //# sourceMappingURL=strings.js.map

package/bin/utils/strings.js.map

@@ -1 +1 @@
-{"version":3,"file":"strings.js","sourceRoot":"","sources":["../../src/utils/strings.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,CAAC"}
+{"version":3,"file":"strings.js","sourceRoot":"","sources":["../../src/utils/strings.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAW;IACnD,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC;AAC/C,CAAC"}

package/bin/utils/text.js

@@ -1,10 +1,14 @@
-import { visit } from 'unist-util-visit';
+import { CONTINUE, SKIP, visit } from 'unist-util-visit';
 export function getText(element) {
     if (!element)
         return '';
     let text = '';
-    visit(element, 'text', function (node) {
-        text += node.value;
+    visit(element, function (node) {
+        if (node.type === 'element' && node.tagName === 'svg')
+            return SKIP;
+        if (node.type === 'text')
+            text += node.value;
+        return CONTINUE;
     });
     return text;
 }

package/bin/utils/text.js.map

@@ -1 +1 @@
-{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/utils/text.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,MAAM,UAAU,OAAO,CAAC,OAA4B;IAClD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/utils/text.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzD,MAAM,UAAU,OAAO,CAAC,OAA4B;IAClD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,CAAC,OAAO,EAAE,UAAU,IAAI;QAC3B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;QACnE,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;YAAE,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;QAC7C,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.589",
+  "version": "4.0.591",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -43,7 +43,7 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.728",
+    "@mintlify/common": "1.0.729",
     "@mintlify/openapi-parser": "^0.0.8",
     "fs-extra": "11.1.1",
     "hast-util-to-mdast": "10.1.0",
@@ -66,7 +66,7 @@
     "@mintlify/models": "0.0.272",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.594",
+    "@mintlify/validation": "0.1.595",
     "@trivago/prettier-plugin-sort-imports": "4.3.0",
     "@tsconfig/recommended": "1.0.2",
     "@types/hast": "3.0.4",
@@ -82,5 +82,5 @@
     "typescript": "5.5.3",
     "vitest": "2.0.4"
   },
-  "gitHead": "fdbcfc062b0d175a55530b0021aca2acaa98906b"
+  "gitHead": "c40f23d742edb39a24f4259970f9cd5f0a506f3c"
 }

package/src/cli.ts

@@ -27,8 +27,17 @@ await yargs(hideBin(process.argv))
   .command(
     'section <url>',
     'Scrapes the entire docs site based on the URL provided',
-    (yargs) => yargs.positional('url', { type: 'string', demandOption: true }).check(checkUrl),
-    async ({ url }) => await site(url)
+    (yargs) =>
+      yargs
+        .positional('url', { type: 'string', demandOption: true })
+        .option('filter', {
+          describe:
+            'Only scrape URLs matching this path filter (e.g. /docs will match /docs and /docs/*)',
+          type: 'string',
+          alias: 'f',
+        })
+        .check(checkUrl),
+    async ({ url, filter }) => await site(url, filter)
   )
 
   .command(
@@ -117,13 +126,13 @@ async function page(url: string) {
   }
 }
 
-async function site(url: string) {
+async function site(url: string, filter?: string) {
   try {
     const urlObj = new URL(url);
     const html = await fetchPageHtml(urlObj);
     log('Successfully retrieved initial HTML from src: ' + urlObj.toString());
 
-    const result = await scrapeAllSiteTabs(html, urlObj);
+    const result = await scrapeAllSiteTabs(html, urlObj, { filter });
     if (result.success) {
       const mintConfig = result.data as MintConfigType;
       const docsConfig = upgradeToDocsConfig(mintConfig, {

package/src/constants.ts

@@ -1,5 +1,3 @@
-import { activeColors } from './utils/log.js';
-
 export const OVERVIEW_PAGE_SLUG = '/mintie_overview';
 
 export const SUPPORTED_MEDIA_EXTENSIONS = [
@@ -50,6 +48,4 @@ ${SPACES}We currently support: ReadMe, GitBook, and Docusaurus`;
 
 export const MDAST_FAILURE_MSG = 'failed to convert MDAST to Markdown string';
 
-export const FINAL_SUCCESS_MESSAGE = `We've successfully scraped your docs site.
-${SPACES}We've downloaded the ${activeColors.cyan}\`navigation\`${activeColors.default} array (and if necessary, the ${activeColors.cyan}\`tabs\`${activeColors.default} array)
-${SPACES}into ${activeColors.blue}\`docs.json\`${activeColors.default}.`;
+export const FINAL_SUCCESS_MESSAGE = `We've successfully scraped your docs site. We've downloaded the \`navigation\` array (and if necessary, the \`tabs\` array) into \`docs.json\`.`;

package/src/index.ts

@@ -2,3 +2,13 @@ export { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
 export { generateOpenApiPagesForDocsConfig } from './openapi/generateOpenApiPagesForDocsConfig.js';
 export * from './utils/log.js';
 export { generateAsyncApiPagesForDocsConfig } from './asyncapi/generateAsyncApiPagesForDocsConfig.js';
+
+export { scrapePageGroup } from './pipeline/group.js';
+export { scrapeAllSiteTabs } from './pipeline/tabs.js';
+export { htmlToHast } from './pipeline/root.js';
+export { detectFramework, framework } from './utils/detectFramework.js';
+export { fetchPageHtml } from './utils/network.js';
+export { write } from './utils/file.js';
+export { getErrorMessage } from './utils/errors.js';
+export { checkUrl } from './utils/url.js';
+export { FINAL_SUCCESS_MESSAGE } from './constants.js';

package/src/nav/retrieve.ts

@@ -39,6 +39,28 @@ export function retrieveNavItems(rootNode: Element): Array<NavigationEntry> {
     if (node.tagName === rootSectionTagName) node.tagName = 'li';
     if (node.tagName !== 'li') return CONTINUE;
 
+    const className = node.properties.className;
+    if (
+      framework.vendor === 'gitbook' &&
+      Array.isArray(className) &&
+      className.includes('page-group-item')
+    ) {
+      const titleDiv = node.children.find(
+        (child) => child.type === 'element' && child.tagName === 'div'
+      );
+      const childList = node.children.find(
+        (child) => child.type === 'element' && child.tagName === 'ul'
+      );
+      if (titleDiv && titleDiv.type === 'element' && childList && childList.type === 'element') {
+        const title = findTitle(titleDiv, { delete: false });
+        const childEntries = retrieveNavItems(childList);
+        if (title && childEntries.length > 0) {
+          result.push({ group: title, pages: childEntries });
+        }
+      }
+      return SKIP;
+    }
+
     let title: string | undefined = undefined;
     if (
       node.children[0] &&

package/src/nav/root.ts

@@ -38,6 +38,17 @@ export function retrieveRootNavElement(rootNode: HastRoot): Element | undefined
   let element: Element | undefined = undefined;
   visit(rootNode, 'element', function (node) {
     const { className } = node.properties;
+
+    if (
+      framework.vendor === 'gitbook' &&
+      node.tagName === 'aside' &&
+      (node.properties.id === 'table-of-contents' ||
+        node.properties.dataTestid === 'table-of-contents')
+    ) {
+      element = node;
+      return EXIT;
+    }
+
     if (
       node.tagName === rootTagName &&
       Array.isArray(className) &&

package/src/pipeline/site.ts

@@ -12,7 +12,11 @@ import { detectFramework, framework } from '../utils/detectFramework.js';
 import { logErrorResults } from '../utils/errors.js';
 import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
 import { INDEX_NAMES, iterateThroughReservedNames } from '../utils/reservedNames.js';
-import { removeTrailingSlash, removeLeadingSlash } from '../utils/strings.js';
+import {
+  removeTrailingSlash,
+  removeLeadingSlash,
+  optionallyAddLeadingSlash,
+} from '../utils/strings.js';
 import { downloadColors } from './color.js';
 import { scrapePageGroup } from './group.js';
 import { downloadFavicon } from './icon.js';
@@ -20,10 +24,18 @@ import { downloadLogos } from './logo.js';
 import { htmlToHast } from './root.js';
 import { downloadTitle } from './title.js';
 
+function matchesFilter(pathname: string, filter: string): boolean {
+  const normalizedPathname = removeTrailingSlash(pathname);
+  const normalizedFilter = removeTrailingSlash(optionallyAddLeadingSlash(filter));
+  return (
+    normalizedPathname === normalizedFilter || normalizedPathname.startsWith(normalizedFilter + '/')
+  );
+}
+
 export async function scrapeSite(
   html: string,
   url: string | URL,
-  opts: { hast?: HastRoot; tabs?: Array<Tab> } = {}
+  opts: { hast?: HastRoot; tabs?: Array<Tab>; filter?: string } = {}
 ): Promise<Result<MintConfig>> {
   let hast = opts.hast;
   if (!hast) hast = htmlToHast(html);
@@ -33,7 +45,7 @@ export async function scrapeSite(
 
   if (!framework.vendor) detectFramework(hast);
 
-  if (framework.vendor === 'docusaurus') {
+  if (framework.vendor === 'docusaurus' || framework.vendor === 'gitbook') {
     const browser = await startPuppeteer();
     html = await fetchPageHtml(url, browser);
     hast = htmlToHast(html);
@@ -57,12 +69,16 @@ export async function scrapeSite(
   const needsBrowser = framework.vendor === 'gitbook';
 
   const externalLinks = listOfLinks.filter((url) => url.origin !== origin);
-  const internalLinks = listOfLinks.filter(
-    (url) => url.origin === origin && removeTrailingSlash(url.toString()) !== origin
-  );
-  const rootLinks = listOfLinks.filter(
-    (url) => url.origin === origin && removeTrailingSlash(url.toString()) === origin
-  );
+  const internalLinks = listOfLinks.filter((url) => {
+    if (url.origin !== origin || removeTrailingSlash(url.toString()) === origin) return false;
+    if (opts.filter && !matchesFilter(url.pathname, opts.filter)) return false;
+    return true;
+  });
+  const rootLinks = listOfLinks.filter((url) => {
+    if (url.origin !== origin || removeTrailingSlash(url.toString()) !== origin) return false;
+    if (opts.filter && !matchesFilter('/', opts.filter)) return false;
+    return true;
+  });
 
   const allPathnames = [
     ...internalLinks.map((url) => url.toString()),
@@ -157,14 +173,24 @@ export async function scrapeSite(
       })
       .filter(Boolean);
 
+    function filterErroredOrFilteredPaths(value: string) {
+      if (allErroredPaths.includes(value)) return true;
+      if (opts.filter && !matchesFilter('/' + value, opts.filter)) return true;
+      return false;
+    }
+
     traverse(navItems).forEach(function (value) {
-      if (typeof value === 'string' && allErroredPaths.includes(value)) {
+      if (
+        typeof value === 'string' &&
+        this.key !== 'group' &&
+        filterErroredOrFilteredPaths(value)
+      ) {
         this.remove();
       } else if (Array.isArray(value)) {
         this.update(
           value
             .filter((item) =>
-              typeof item === 'string' && allErroredPaths.includes(item) ? undefined : item
+              typeof item === 'string' && filterErroredOrFilteredPaths(item) ? undefined : item
             )
             .filter(Boolean)
         );
@@ -199,15 +225,12 @@ export async function scrapeSite(
             typeof val === 'string' && (val.startsWith('https://') || val.startsWith('http://'))
         )
       ) {
-        this.update(
-          value.filter(
-            (val) =>
-              !(
-                typeof val === 'string' &&
-                (val.startsWith('https://') || val.startsWith('http://'))
-              )
-          )
+        const newPages = value.filter(
+          (val) =>
+            !(typeof val === 'string' && (val.startsWith('https://') || val.startsWith('http://')))
         );
+        if (newPages.length) this.update(newPages);
+        else this.parent?.remove();
       }
     });
 

package/src/pipeline/tabs.ts

@@ -16,7 +16,8 @@ import { downloadTitle } from './title.js';
 
 export async function scrapeAllSiteTabs(
   html: string,
-  url: string | URL
+  url: string | URL,
+  opts: { filter?: string } = {}
 ): Promise<Result<MintConfig>> {
   const hast = htmlToHast(html);
   url = new URL(url);
@@ -34,7 +35,7 @@ export async function scrapeAllSiteTabs(
       !links.length ||
       (links.length === 1 && links[0] && links[0].url === url.pathname)
     )
-      return scrapeSite(html, url, { hast });
+      return scrapeSite(html, url, { hast, filter: opts.filter });
 
     if (!links.find((link) => url.pathname.startsWith(link.url))) {
       links.push({
@@ -49,7 +50,7 @@ export async function scrapeAllSiteTabs(
         newUrl.pathname = tabEntry.url;
         try {
           const newHtml = await fetchPageHtml(newUrl, undefined);
-          return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry] });
+          return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry], filter: opts.filter });
         } catch (error) {
           return { success: false as const, message: getErrorMessage(error) };
         }
@@ -95,5 +96,5 @@ export async function scrapeAllSiteTabs(
     };
   }
 
-  return scrapeSite(html, url, { hast });
+  return scrapeSite(html, url, { hast, filter: opts.filter });
 }

package/src/utils/network.ts

@@ -90,6 +90,53 @@ export async function getHtmlWithPuppeteer(
       });
     }
 
+    if (framework.vendor === 'gitbook') {
+      for (let round = 0; round < 10; round++) {
+        const clickedCount = await page.evaluate(() => {
+          const tocEl = document.getElementById('table-of-contents');
+          if (!tocEl) return 0;
+          let count = 0;
+          const items = tocEl.querySelectorAll('li.page-document-item');
+          items.forEach((li) => {
+            const btn = li.querySelector(':scope > a button');
+            if (!btn || !(btn instanceof HTMLElement)) return;
+            const anchor = btn.closest('a');
+            if (!anchor) return;
+            const sibling = anchor.nextElementSibling;
+            if (
+              sibling instanceof HTMLElement &&
+              sibling.style.opacity === '1' &&
+              sibling.style.height === 'auto'
+            )
+              return;
+            btn.click();
+            count++;
+          });
+          return count;
+        });
+
+        if (clickedCount === 0) break;
+
+        await page
+          .waitForFunction(
+            () => {
+              const tocEl = document.getElementById('table-of-contents');
+              if (!tocEl) return true;
+              const anchors = tocEl.querySelectorAll('li.page-document-item > a');
+              return Array.from(anchors).every((a) => {
+                const btn = a.querySelector('button');
+                if (!btn) return true;
+                const sibling = a.nextElementSibling;
+                if (!sibling || !(sibling instanceof HTMLElement)) return true;
+                return sibling.style.opacity === '1' && sibling.style.height === 'auto';
+              });
+            },
+            { timeout: 5000 }
+          )
+          .catch(() => {});
+      }
+    }
+
     const content = await exponentialBackoff(() => page.content());
     await page.close();
     return content;

package/src/utils/strings.ts

@@ -5,3 +5,7 @@ export function removeTrailingSlash(str: string): string {
 export function removeLeadingSlash(str: string): string {
   return str.startsWith('/') ? str.substring(1) : str;
 }
+
+export function optionallyAddLeadingSlash(str: string): string {
+  return str.startsWith('/') ? str : '/' + str;
+}

package/src/utils/text.ts

@@ -1,11 +1,13 @@
 import type { Element } from 'hast';
-import { visit } from 'unist-util-visit';
+import { CONTINUE, SKIP, visit } from 'unist-util-visit';
 
 export function getText(element: Element | undefined): string {
   if (!element) return '';
   let text = '';
-  visit(element, 'text', function (node) {
-    text += node.value;
+  visit(element, function (node) {
+    if (node.type === 'element' && node.tagName === 'svg') return SKIP;
+    if (node.type === 'text') text += node.value;
+    return CONTINUE;
   });
   return text;
 }