@mintlify/scraping 3.0.187 → 3.0.189

package/bin/scrapingPipeline/logo.js

@@ -0,0 +1,92 @@
+import { join } from 'node:path';
+import { EXIT, visit } from 'unist-util-visit';
+import { framework } from '../utils/detectFramework.js';
+import { downloadImage } from '../utils/images.js';
+import { fetchPageHtml } from '../utils/network.js';
+import { htmlToHast } from './root.js';
+function findReadmeLogoNodes(root) {
+    const elements = [];
+    visit(root, 'element', function (node) {
+        if (node.tagName === 'img' &&
+            Array.isArray(node.properties.className) &&
+            node.properties.className.includes('rm-Logo-img'))
+            elements.push(node);
+    });
+    return elements.length ? elements : undefined;
+}
+function findGitBookLogoNodes(root) {
+    const elements = [];
+    visit(root, 'element', function (node) {
+        if (node.tagName === 'img' && node.properties.alt === 'Logo') {
+            elements.push(node);
+        }
+    });
+    return elements.length ? elements : undefined;
+}
+function findDocusaurusLogoNodes(root) {
+    const elements = [];
+    visit(root, 'element', function (node) {
+        if (node.tagName === 'div' &&
+            Array.isArray(node.properties.className) &&
+            node.properties.className.includes('navbar__brand')) {
+            visit(node, 'element', function (subNode) {
+                if (subNode.tagName === 'img')
+                    elements.push(subNode);
+            });
+            return EXIT;
+        }
+    });
+    return elements.length ? elements : undefined;
+}
+async function findLogosFromHtml(html, downloadFn, filepaths) {
+    const hast = htmlToHast(html);
+    const imgNodes = downloadFn(hast);
+    if (imgNodes) {
+        filepaths.push(...(await Promise.all(imgNodes.map(async (node) => {
+            const res = await downloadImage(node.properties.src, join(process.cwd(), 'images'));
+            if (res.success && res.data) {
+                return res.data[1];
+            }
+            else {
+                return '';
+            }
+        }))));
+    }
+    filepaths.forEach((filepath, index) => {
+        if (!filepath)
+            filepaths.splice(index, 1);
+    });
+}
+export async function downloadLogos(url, browser) {
+    url = new URL(url);
+    const filepaths = [];
+    if (browser) {
+        const htmls = [];
+        const page = await browser.newPage();
+        await page.goto(url.toString(), {
+            waitUntil: 'networkidle2',
+        });
+        htmls.push(await page.content());
+        await page.click('.rm-ThemeToggle');
+        htmls.push(await page.content());
+        await Promise.all(htmls.map(async (html) => {
+            return await findLogosFromHtml(html, findReadmeLogoNodes, filepaths);
+        }));
+    }
+    else {
+        const html = await fetchPageHtml(url);
+        await findLogosFromHtml(html, framework.vendor === 'gitbook' ? findGitBookLogoNodes : findDocusaurusLogoNodes, filepaths);
+    }
+    if (browser)
+        await browser.close();
+    const uniqueFilepaths = [...new Set(filepaths).values()];
+    return uniqueFilepaths.length === 1
+        ? uniqueFilepaths[0]
+        : uniqueFilepaths.length > 1
+            ? {
+                light: uniqueFilepaths[0],
+                dark: uniqueFilepaths[1],
+            }
+            : undefined;
+}
+//# sourceMappingURL=logo.js.map

package/bin/scrapingPipeline/logo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logo.js","sourceRoot":"","sources":["../../src/scrapingPipeline/logo.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,SAAS,mBAAmB,CAAC,IAAc;IACzC,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC;YAEjD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAc;IAC1C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,KAAK,MAAM,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAc;IAC7C,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,IAAI;QACnC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,EACnD,CAAC;YACD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK;oBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,IAAY,EACZ,UAA0D,EAC1D,SAAwB;IAExB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,QAAQ,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CACZ,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CACnB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,MAAM,aAAa,CAC7B,IAAI,CAAC,UAAU,CAAC,GAAa,EAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAC9B,CAAC;YAEF,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5B,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CACH,CAAC,CACH,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACpC,IAAI,CAAC,QAAQ;YAAE,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,OAA4B;IAE5B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC9B,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACjC,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAEjC,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,MAAM,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAC;QACvE,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,iBAAiB,CACrB,IAAI,EACJ,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,uBAAuB,EAC/E,SAAS,CACV,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEnC,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzD,OAAO,eAAe,CAAC,MAAM,KAAK,CAAC;QACjC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;YAC1B,CAAC,CAAC;gBACE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAW;gBACnC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAW;aACnC;YACH,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC"}

package/bin/scrapingPipeline/page.d.ts

@@ -0,0 +1,6 @@
+import type { Result } from '../types/result.js';
+export declare function scrapePage(html: string, url: string | URL, opts?: {
+    externalLink: boolean;
+    isOverviewPage?: boolean;
+    rootPath?: string;
+}): Promise<Result<[string, string]>>;

package/bin/scrapingPipeline/page.js

@@ -0,0 +1,102 @@
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import remarkStringify from 'remark-stringify';
+import { unified } from 'unified';
+import { convertHeaderLinksToText } from '../components/link.js';
+import { CONTENT_FAILURE_MSG, MDAST_FAILURE_MSG } from '../constants.js';
+import { createCallout, createCard, createAccordion, createAccordionGroup, createFrame, createCodeGroup, createTabs, createCardGroup, } from '../customComponents/create.js';
+import { rehypeToRemarkCustomComponents } from '../customComponents/plugin.js';
+import { selectiveRehypeRemark } from '../customComponents/selective.js';
+import { retrieveRootContent } from '../root/retrieve.js';
+import { unifiedRemoveClassNames } from '../utils/className.js';
+import { detectFramework, framework } from '../utils/detectFramework.js';
+import { unifiedRemoveEmptyParagraphs } from '../utils/emptyParagraphs.js';
+import { getErrorMessage, logErrorResults } from '../utils/errors.js';
+import { escapeCharactersOutsideCodeBlocks } from '../utils/escape.js';
+import { write, writePage } from '../utils/file.js';
+import { log } from '../utils/log.js';
+import { unifiedRemoveNestedRoots } from '../utils/nestedRoots.js';
+import { unifiedRemovePositions } from '../utils/position.js';
+import { removeLeadingSlash, removeTrailingSlash } from '../utils/strings.js';
+import { getDescriptionFromRoot, getTitleFromHeading } from '../utils/title.js';
+import { downloadImagesFromFile } from './images.js';
+import { htmlToHast } from './root.js';
+export async function scrapePage(html, url, opts = { externalLink: false }) {
+    url = new URL(url);
+    if (opts.externalLink) {
+        const filename = html;
+        const filenameWithExt = `${filename}.mdx`;
+        writePage(filenameWithExt, '', '', '', url.toString());
+        return { success: true, data: [url.toString(), filename] };
+    }
+    const hast = htmlToHast(html);
+    if (!framework.vendor)
+        detectFramework(hast);
+    const urlStr = url.toString();
+    const content = retrieveRootContent(hast);
+    if (!content)
+        return { success: false, message: `${urlStr}: ${CONTENT_FAILURE_MSG}` };
+    const contentAsRoot = {
+        type: 'root',
+        children: [content],
+    };
+    const mdastTree = unified()
+        .use(createCard)
+        .use(createAccordion)
+        .use(createFrame)
+        .use(createTabs)
+        .use(createCallout)
+        .use(createCardGroup)
+        .use(createAccordionGroup)
+        .use(createCodeGroup)
+        .use(unifiedRemoveClassNames)
+        .use(unifiedRemovePositions)
+        .use(unifiedRemoveEmptyParagraphs)
+        .use(escapeCharactersOutsideCodeBlocks)
+        .use(selectiveRehypeRemark)
+        // Cleans up any nested components left untouched
+        // by `selectiveRehypeRemark`, and converts them to
+        // MDX compatible components
+        .use(rehypeToRemarkCustomComponents)
+        .use(convertHeaderLinksToText)
+        .use(unifiedRemoveNestedRoots)
+        .runSync(contentAsRoot);
+    try {
+        const imageResults = await downloadImagesFromFile(mdastTree, url);
+        logErrorResults(`scraping images from ${url.toString()}`, imageResults);
+    }
+    catch (error) {
+        const errorMessage = getErrorMessage(error);
+        log(`We encountered an error when scraping the images from ${url.toString()}${errorMessage}`);
+        throw error;
+    }
+    const title = getTitleFromHeading(mdastTree);
+    const description = getDescriptionFromRoot(mdastTree);
+    try {
+        const result = unified()
+            .use(remarkMdx)
+            .use(remarkGfm)
+            // @ts-expect-error remarkStringify errors even if used for valid code from documentation examples
+            .use(remarkStringify)
+            .stringify(mdastTree);
+        if (opts.rootPath) {
+            url = new URL(opts.rootPath, url.origin);
+        }
+        else if (url.origin === removeTrailingSlash(url.toString())) {
+            url = new URL('home', new URL(url).origin);
+        }
+        writePage(url, opts.isOverviewPage ? 'Overview' : title, description, String(result));
+        return {
+            success: true,
+            data: opts.rootPath
+                ? [removeLeadingSlash(removeTrailingSlash(new URL(urlStr).pathname)), opts.rootPath]
+                : undefined,
+        };
+    }
+    catch (error) {
+        write('error.json', JSON.stringify(mdastTree, undefined, 2));
+        const errorMessage = getErrorMessage(error);
+        return { success: false, message: `${urlStr}: ${MDAST_FAILURE_MSG}${errorMessage}` };
+    }
+}
+//# sourceMappingURL=page.js.map

package/bin/scrapingPipeline/page.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/scrapingPipeline/page.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,eAAe,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EACL,aAAa,EACb,UAAU,EACV,eAAe,EACf,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,UAAU,EACV,eAAe,GAChB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,iCAAiC,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAII,EAAE,YAAY,EAAE,KAAK,EAAE;IAE3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC;QACtB,MAAM,eAAe,GAAG,GAAG,QAAQ,MAAM,CAAC;QAC1C,SAAS,CAAC,eAAe,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAE9B,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,mBAAmB,EAAE,EAAE,CAAC;IAEtF,MAAM,aAAa,GAAa;QAC9B,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEF,MAAM,SAAS,GAAc,OAAO,EAAE;SACnC,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,WAAW,CAAC;SAChB,GAAG,CAAC,UAAU,CAAC;SACf,GAAG,CAAC,aAAa,CAAC;SAClB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,oBAAoB,CAAC;SACzB,GAAG,CAAC,eAAe,CAAC;SACpB,GAAG,CAAC,uBAAuB,CAAC;SAC5B,GAAG,CAAC,sBAAsB,CAAC;SAC3B,GAAG,CAAC,4BAA4B,CAAC;SACjC,GAAG,CAAC,iCAAiC,CAAC;SACtC,GAAG,CAAC,qBAAqB,CAAC;QAE3B,iDAAiD;QACjD,mDAAmD;QACnD,4BAA4B;SAC3B,GAAG,CAAC,8BAA8B,CAAC;SACnC,GAAG,CAAC,wBAAwB,CAAC;SAC7B,GAAG,CAAC,wBAAwB,CAAC;SAC7B,OAAO,CAAC,aAAa,CAAc,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClE,eAAe,CAAC,wBAAwB,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,GAAG,CAAC,yDAAyD,GAAG,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;QAC9F,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,EAAE;aACrB,GAAG,CAAC,SAAS,CAAC;aACd,GAAG,CAAC,SAAS,CAAC;YACf,kGAAkG;aACjG,GAAG,CAAC,eAAe,CAAC;aACpB,SAAS,CAAC,SAAS,CAAC,CAAC;QAExB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC9D,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACtF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI,CAAC,QAAQ;gBACjB,CAAC,CAAC,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC;gBACpF,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,iBAAiB,GAAG,YAAY,EAAE,EAAE,CAAC;IACvF,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/root.d.ts

@@ -0,0 +1,2 @@
+import type { Root as HastRoot } from 'hast';
+export declare function htmlToHast(html: string): HastRoot;

package/bin/scrapingPipeline/root.js

@@ -0,0 +1,8 @@
+import rehypeParse from 'rehype-parse';
+import { unified } from 'unified';
+import { unifiedRemovePositions } from '../utils/position.js';
+export function htmlToHast(html) {
+    // @ts-expect-error remarkStringify errors even if used for valid code from documentation examples
+    return unified().use(rehypeParse).use(unifiedRemovePositions).parse(html);
+}
+//# sourceMappingURL=root.js.map

package/bin/scrapingPipeline/root.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"root.js","sourceRoot":"","sources":["../../src/scrapingPipeline/root.ts"],"names":[],"mappings":"AACA,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,kGAAkG;IAClG,OAAO,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAa,CAAC;AACxF,CAAC"}

package/bin/scrapingPipeline/site.d.ts

@@ -0,0 +1,7 @@
+import { MintConfig, Tab } from '@mintlify/models';
+import type { Root as HastRoot } from 'hast';
+import type { Result } from '../types/result.js';
+export declare function scrapeSite(html: string, url: string | URL, opts?: {
+    hast?: HastRoot;
+    tabs?: Array<Tab>;
+}): Promise<Result<MintConfig>>;

package/bin/scrapingPipeline/site.js

@@ -0,0 +1,129 @@
+import traverse from 'traverse';
+import { NAV_FAILURE_MSG } from '../constants.js';
+import { iterateOverNavItems } from '../nav/iterate.js';
+import { retrieveNavItems } from '../nav/retrieve.js';
+import { retrieveRootNavElement } from '../nav/root.js';
+import { detectFramework, framework } from '../utils/detectFramework.js';
+import { logErrorResults } from '../utils/errors.js';
+import { startPuppeteer } from '../utils/network.js';
+import { INDEX_NAMES, iterateThroughReservedNames } from '../utils/reservedNames.js';
+import { removeTrailingSlash } from '../utils/strings.js';
+import { scrapePageGroup } from './group.js';
+import { downloadFavicon } from './icon.js';
+import { downloadLogos } from './logo.js';
+import { htmlToHast } from './root.js';
+export async function scrapeSite(html, url, opts = {}) {
+    let hast = opts.hast;
+    if (!hast)
+        hast = htmlToHast(html);
+    url = new URL(url);
+    const origin = url.origin;
+    detectFramework(hast);
+    const sidebar = retrieveRootNavElement(hast);
+    if (!sidebar)
+        return { success: false, message: `${url.toString()}: ${NAV_FAILURE_MSG}` };
+    const navItems = retrieveNavItems(sidebar);
+    if (origin === '') {
+        return { success: false, message: `invalid URL provided to scrape site: ${url}` };
+    }
+    const listOfLinks = iterateOverNavItems(navItems, origin);
+    if (listOfLinks.length === 0) {
+        return { success: false, message: `no navigation links were able to be found: ${url}` };
+    }
+    const needsBrowser = framework.vendor === 'gitbook';
+    const externalLinks = listOfLinks.filter((url) => url.origin !== origin);
+    const internalLinks = listOfLinks.filter((url) => url.origin === origin && removeTrailingSlash(url.toString()) !== origin);
+    const rootLinks = listOfLinks.filter((url) => url.origin === origin && removeTrailingSlash(url.toString()) === origin);
+    const allPathnames = [
+        ...internalLinks.map((url) => url.toString()),
+        ...rootLinks.map((url) => url.toString()),
+    ];
+    const rootPaths = rootLinks.map(() => {
+        const name = iterateThroughReservedNames(INDEX_NAMES, allPathnames);
+        allPathnames.push(name);
+        return name;
+    });
+    try {
+        const externalResults = await scrapePageGroup(externalLinks, needsBrowser, {
+            externalLinks: true,
+        });
+        const internalResults = await scrapePageGroup(internalLinks, needsBrowser);
+        const rootResults = await scrapePageGroup(rootLinks, needsBrowser, {
+            externalLinks: false,
+            rootPaths,
+        });
+        const externalLinkReplaceMap = new Map(externalResults
+            .filter((result) => result.success)
+            .map((result) => result.data));
+        const rootPathReplaceMap = new Map(rootResults
+            .filter((result) => result.success)
+            .map((result) => result.data));
+        traverse(navItems).forEach(function (value) {
+            if (typeof value === 'string') {
+                if (externalLinkReplaceMap.has(value)) {
+                    this.update(externalLinkReplaceMap.get(value) ?? value);
+                }
+                else if (rootPathReplaceMap.has(value)) {
+                    this.update(rootPathReplaceMap.get(value) ?? value);
+                }
+            }
+            else if (Array.isArray(value)) {
+                if (value.find((item) => externalLinkReplaceMap.has(item))) {
+                    this.update(value.map((item) => externalLinkReplaceMap.get(item) ?? item));
+                }
+                else if (value.find((item) => rootPathReplaceMap.has(item))) {
+                    this.update(value.map((item) => rootPathReplaceMap.get(item) ?? item));
+                }
+            }
+        });
+        traverse(navItems).forEach(function (value) {
+            if (typeof value === 'string') {
+                this.update(value.replace('/mintie_overview', ''));
+            }
+            else if (Array.isArray(value)) {
+                this.update(value.map((item) => typeof item === 'string' ? item.replace('/mintie_overview', '') : item));
+            }
+        });
+        navItems.forEach((navItem, index) => {
+            if (typeof navItem !== 'string')
+                return;
+            const name = navItem
+                .split('-')
+                .map((str) => (str[0] ? `${str[0].toUpperCase()}${str.substring(1)}` : str))
+                .join(' ');
+            navItems[index] = {
+                group: name,
+                pages: [navItem],
+            };
+        });
+        logErrorResults('linking to external pages', externalResults);
+        logErrorResults('scraping your docs', [...internalResults, ...rootResults]);
+        const browser = needsBrowser ? await startPuppeteer() : undefined;
+        const favicon = await downloadFavicon(hast);
+        const logo = await downloadLogos(url, browser);
+        return {
+            success: true,
+            data: {
+                $schema: 'https://mintlify.com/schema.json',
+                name: '',
+                logo,
+                colors: {
+                    primary: '',
+                },
+                favicon: favicon ?? '',
+                navigation: navItems,
+                tabs: opts.tabs,
+            },
+        };
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            return { success: false, message: error.message };
+        }
+        return {
+            success: false,
+            message: 'An unknown error occurred when scraping this site. Please try again.',
+        };
+    }
+}
+//# sourceMappingURL=site.js.map

package/bin/scrapingPipeline/site.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"site.js","sourceRoot":"","sources":["../../src/scrapingPipeline/site.ts"],"names":[],"mappings":"AAEA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,GAAiB,EACjB,OAA+C,EAAE;IAEjD,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACrB,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAEnC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAE1B,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,eAAe,EAAE,EAAE,CAAC;IAE1F,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,wCAAwC,GAAG,EAAE,EAAE,CAAC;IACpF,CAAC;IAED,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,8CAA8C,GAAG,EAAE,EAAE,CAAC;IAC1F,CAAC;IAED,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IACF,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,MAAM,CACjF,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7C,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;KAC1C,CAAC;IACF,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG,2BAA2B,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QACpE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE;YACzE,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE;YACjE,aAAa,EAAE,KAAK;YACpB,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CACpC,eAAe;aACZ,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,WAAW;aACR,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;aAClC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAwB,CAAC,CACpD,CAAC;QAEF,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBAC1D,CAAC;qBAAM,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK;YACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,MAAM,CACT,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CACvE,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO;YACxC,MAAM,IAAI,GAAG,OAAO;iBACjB,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;iBAC3E,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,QAAQ,CAAC,KAAK,CAAC,GAAG;gBAChB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,CAAC,OAAO,CAAC;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,eAAe,CAAC,2BAA2B,EAAE,eAAe,CAAC,CAAC;QAC9D,eAAe,CAAC,oBAAoB,EAAE,CAAC,GAAG,eAAe,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC;QAE5E,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAElE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAE/C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI,EAAE,EAAE;gBACR,IAAI;gBACJ,MAAM,EAAE;oBACN,OAAO,EAAE,EAAE;iBACZ;gBACD,OAAO,EAAE,OAAO,IAAI,EAAE;gBACtB,UAAU,EAAE,QAAsB;gBAClC,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,sEAAsE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC"}

package/bin/scrapingPipeline/tabs.d.ts

@@ -0,0 +1,3 @@
+import { MintConfig } from '@mintlify/models';
+import type { Result } from '../types/result.js';
+export declare function scrapeAllSiteTabs(html: string, url: string | URL): Promise<Result<MintConfig>>;

package/bin/scrapingPipeline/tabs.js

@@ -0,0 +1,67 @@
+import { retrieveTabLinks } from '../tabs/retrieveReadme.js';
+import { detectFramework, framework } from '../utils/detectFramework.js';
+import { log } from '../utils/log.js';
+import { fetchPageHtml, startPuppeteer } from '../utils/network.js';
+import { getTitleFromLink } from '../utils/title.js';
+import { downloadFavicon } from './icon.js';
+import { downloadLogos } from './logo.js';
+import { htmlToHast } from './root.js';
+import { scrapeSite } from './site.js';
+export async function scrapeAllSiteTabs(html, url) {
+    const hast = htmlToHast(html);
+    url = new URL(url);
+    detectFramework(hast);
+    const needsBrowser = framework.vendor === 'gitbook';
+    const browser = needsBrowser ? await startPuppeteer() : undefined;
+    const favicon = await downloadFavicon(hast);
+    const logo = await downloadLogos(url, browser);
+    if (framework.vendor === 'readme' || framework.vendor === 'docusaurus') {
+        const links = retrieveTabLinks(hast);
+        if (!links ||
+            !links.length ||
+            (links.length === 1 && links[0] && links[0].url === url.pathname))
+            return scrapeSite(html, url, { hast });
+        if (!links.find((link) => url.pathname.startsWith(link.url))) {
+            links.push({
+                name: getTitleFromLink(url.pathname),
+                url: url.pathname,
+            });
+        }
+        const results = await Promise.all(links.map(async (tabEntry) => {
+            const newUrl = new URL(url);
+            newUrl.pathname = tabEntry.url;
+            const newHtml = await fetchPageHtml(newUrl, undefined);
+            return await scrapeSite(newHtml, newUrl, { tabs: [tabEntry] });
+        }));
+        const navigations = [];
+        const tabs = [];
+        const successes = results.filter((result) => result.success);
+        successes.forEach((result) => {
+            if (!result.data)
+                return;
+            navigations.push(...result.data.navigation);
+            if (result.data.tabs)
+                tabs.push(...result.data.tabs);
+        });
+        const failures = results.filter((result) => !result.success);
+        failures.forEach((result) => {
+            log('Failed to scrape tab: ' + result.message);
+        });
+        return {
+            success: true,
+            data: {
+                $schema: 'https://mintlify.com/schema.json',
+                name: '',
+                logo,
+                colors: {
+                    primary: '',
+                },
+                favicon: favicon ?? '',
+                navigation: navigations,
+                tabs,
+            },
+        };
+    }
+    return scrapeSite(html, url, { hast });
+}
+//# sourceMappingURL=tabs.js.map

package/bin/scrapingPipeline/tabs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tabs.js","sourceRoot":"","sources":["../../src/scrapingPipeline/tabs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,GAAiB;IAEjB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnB,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;IACpD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAElE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAE/C,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACvE,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACrC,IACE,CAAC,KAAK;YACN,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,QAAQ,CAAC;YAEjE,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACpC,GAAG,EAAE,GAAG,CAAC,QAAQ;aAClB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACvD,OAAO,MAAM,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,WAAW,GAA2B,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,OAAO;YACzB,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1B,GAAG,CAAC,wBAAwB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,OAAO,EAAE,kCAAkC;gBAC3C,IAAI,EAAE,EAAE;gBACR,IAAI;gBACJ,MAAM,EAAE;oBACN,OAAO,EAAE,EAAE;iBACZ;gBACD,OAAO,EAAE,OAAO,IAAI,EAAE;gBACtB,UAAU,EAAE,WAAyB;gBACrC,IAAI;aACL;SACF,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC"}

package/bin/tabs/retrieveReadme.d.ts

@@ -0,0 +1,3 @@
+import { Tab } from '@mintlify/models';
+import type { Root as HastRoot } from 'hast';
+export declare function retrieveTabLinks(rootNode: HastRoot): Array<Tab> | undefined;

package/bin/tabs/retrieveReadme.js

@@ -0,0 +1,78 @@
+import { visit, EXIT, CONTINUE } from 'unist-util-visit';
+import { framework } from '../utils/detectFramework.js';
+import { findTitle, getTitleFromLink } from '../utils/title.js';
+export function retrieveTabLinks(rootNode) {
+    if (framework.vendor !== 'readme' && framework.vendor !== 'docusaurus')
+        return undefined;
+    let element = undefined;
+    visit(rootNode, 'element', function (node) {
+        if (framework.vendor === 'readme') {
+            if (node.tagName === 'header' &&
+                node.properties.className &&
+                Array.isArray(node.properties.className) &&
+                node.properties.className.includes('rm-Header')) {
+                element = node;
+                return EXIT;
+            }
+        }
+        if (framework.vendor === 'docusaurus') {
+            if (node.tagName === 'nav' &&
+                node.properties.className &&
+                Array.isArray(node.properties.className) &&
+                node.properties.className.includes('navbar')) {
+                element = node;
+                return EXIT;
+            }
+        }
+    });
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+    if (!element)
+        return undefined;
+    const links = [];
+    visit(element, 'element', function (node) {
+        if (framework.vendor === 'readme') {
+            if (node.tagName !== 'nav' &&
+                !(node.tagName === 'div' &&
+                    node.properties.className &&
+                    Array.isArray(node.properties.className) &&
+                    node.properties.className.includes('rm-Header-right')))
+                return CONTINUE;
+            visit(node, 'element', function (subNode) {
+                if (subNode.tagName !== 'a' ||
+                    !subNode.properties.href ||
+                    typeof subNode.properties.href !== 'string' ||
+                    subNode.properties.href.startsWith('http'))
+                    return CONTINUE;
+                const title = findTitle(subNode);
+                links.push({
+                    name: title || getTitleFromLink(subNode.properties.href),
+                    url: subNode.properties.href,
+                });
+            });
+        }
+        if (framework.vendor === 'docusaurus') {
+            if (node.tagName !== 'nav')
+                return CONTINUE;
+            visit(node, 'element', function (subNode, _, parent) {
+                if (subNode.tagName !== 'a' ||
+                    !subNode.properties.href ||
+                    typeof subNode.properties.href !== 'string' ||
+                    subNode.properties.href.startsWith('http') ||
+                    !parent ||
+                    parent.type !== 'element' ||
+                    !Array.isArray(parent.properties.className) ||
+                    parent.properties.className.length !== 1 ||
+                    parent.properties.className[0] !== 'navbar__items' ||
+                    parent.properties.className.includes('navbar__items--right'))
+                    return CONTINUE;
+                const title = findTitle(subNode);
+                links.push({
+                    name: title || getTitleFromLink(subNode.properties.href),
+                    url: subNode.properties.href,
+                });
+            });
+        }
+    });
+    return links;
+}
+//# sourceMappingURL=retrieveReadme.js.map

package/bin/tabs/retrieveReadme.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieveReadme.js","sourceRoot":"","sources":["../../src/tabs/retrieveReadme.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAEhE,MAAM,UAAU,gBAAgB,CAAC,QAAkB;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,SAAS,CAAC;IAEzF,IAAI,OAAO,GAAwB,SAAS,CAAC;IAC7C,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,IAAI;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,IACE,IAAI,CAAC,OAAO,KAAK,QAAQ;gBACzB,IAAI,CAAC,UAAU,CAAC,SAAS;gBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,EAC/C,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;gBACtB,IAAI,CAAC,UAAU,CAAC,SAAS;gBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC5C,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,uEAAuE;IACvE,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAE/B,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,CAAC,OAAkB,EAAE,SAAS,EAAE,UAAU,IAAI;QACjD,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClC,IACE,IAAI,CAAC,OAAO,KAAK,KAAK;gBACtB,CAAC,CACC,IAAI,CAAC,OAAO,KAAK,KAAK;oBACtB,IAAI,CAAC,UAAU,CAAC,SAAS;oBACzB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;oBACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CACtD;gBAED,OAAO,QAAQ,CAAC;YAElB,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO;gBACtC,IACE,OAAO,CAAC,OAAO,KAAK,GAAG;oBACvB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI;oBACxB,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;oBAC3C,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAE1C,OAAO,QAAQ,CAAC;gBAClB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;oBACxD,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI;iBAC7B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;gBAAE,OAAO,QAAQ,CAAC;YAE5C,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,CAAC,EAAE,MAAM;gBACjD,IACE,OAAO,CAAC,OAAO,KAAK,GAAG;oBACvB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI;oBACxB,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;oBAC3C,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC1C,CAAC,MAAM;oBACP,MAAM,CAAC,IAAI,KAAK,SAAS;oBACzB,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC3C,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACxC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,eAAe;oBAClD,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBAE5D,OAAO,QAAQ,CAAC;gBAElB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;oBACxD,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI;iBAC7B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}