@mintlify/scraping 3.0.187 → 3.0.189

package/bin/utils/log.js

@@ -0,0 +1,68 @@
+export const colors = {
+    red: '\x1b[31m',
+    green: '\x1b[32m',
+    yellow: '\x1b[33m',
+    blue: '\x1b[34m',
+    magenta: '\x1b[35m',
+    cyan: '\x1b[36m',
+    default: '\x1b[0m',
+};
+const statuses = ['success', 'failure', 'error', 'warn', 'warning', 'info'];
+export const checkIcon = '✔ ';
+export const xIcon = '✘ ';
+export const infoIcon = 'ⓘ ';
+export const warningIcon = '⚠ ';
+export function log(message, statusOrColor) {
+    let color = colors.blue;
+    let statusMsg = 'INFO   ';
+    let icon = infoIcon;
+    const msg = typeof message === 'string' ? message.toLowerCase() : '';
+    if (!statusOrColor) {
+        statusOrColor =
+            msg.includes('fail') ||
+                msg.includes('error') ||
+                msg.includes('could not') ||
+                msg.includes("couldn't") ||
+                msg.includes('did not') ||
+                msg.includes('invalid') ||
+                msg.includes("didn't")
+                ? 'error'
+                : msg.includes('success') ||
+                    msg.includes('downloaded') ||
+                    msg.includes('written') ||
+                    msg.includes('added')
+                    ? 'success'
+                    : msg.includes('warn')
+                        ? 'warn'
+                        : undefined;
+    }
+    switch (statusOrColor) {
+        case undefined:
+        case 'info':
+            break;
+        case 'success':
+            color = colors.green;
+            statusMsg = 'SUCCESS';
+            icon = checkIcon;
+            break;
+        case 'warn':
+        case 'warning':
+            color = colors.yellow;
+            statusMsg = 'WARNING';
+            icon = warningIcon;
+            break;
+        case 'failure':
+        case 'error':
+            color = colors.red;
+            statusMsg = 'ERROR  ';
+            icon = xIcon;
+            break;
+    }
+    console.log(`${color}${icon} ${statusMsg}${colors.default} - ${typeof message === 'string' ||
+        typeof message === 'bigint' ||
+        typeof message === 'number' ||
+        typeof message === 'boolean'
+        ? message
+        : JSON.stringify(message, undefined, 2)}`);
+}
+//# sourceMappingURL=log.js.map

package/bin/utils/log.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.js","sourceRoot":"","sources":["../../src/utils/log.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,GAAG,EAAE,UAAU;IACf,KAAK,EAAE,UAAU;IACjB,MAAM,EAAE,UAAU;IAClB,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE,UAAU;IACnB,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE,SAAS;CACV,CAAC;AACX,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAU,CAAC;AAKrF,MAAM,CAAC,MAAM,SAAS,GAAG,IAAa,CAAC;AACvC,MAAM,CAAC,MAAM,KAAK,GAAG,IAAa,CAAC;AACnC,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAa,CAAC;AACtC,MAAM,CAAC,MAAM,WAAW,GAAG,IAAa,CAAC;AAEzC,MAAM,UAAU,GAAG,CAAC,OAAyB,EAAE,aAAsB;IACnE,IAAI,KAAK,GAA2B,MAAM,CAAC,IAAI,CAAC;IAChD,IAAI,SAAS,GAAW,SAAS,CAAC;IAClC,IAAI,IAAI,GAAW,QAAQ,CAAC;IAE5B,MAAM,GAAG,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,aAAa;YACX,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACpB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACrB,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACzB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACxB,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACvB,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACvB,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACpB,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;oBACrB,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;oBAC1B,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;oBACvB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;oBACvB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;wBACpB,CAAC,CAAC,MAAM;wBACR,CAAC,CAAC,SAAS,CAAC;IACtB,CAAC;IAED,QAAQ,aAAa,EAAE,CAAC;QACtB,KAAK,SAAS,CAAC;QACf,KAAK,MAAM;YACT,MAAM;QAER,KAAK,SAAS;YACZ,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YACrB,SAAS,GAAG,SAAS,CAAC;YACtB,IAAI,GAAG,SAAS,CAAC;YACjB,MAAM;QAER,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS;YACZ,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;YACtB,SAAS,GAAG,SAAS,CAAC;YACtB,IAAI,GAAG,WAAW,CAAC;YACnB,MAAM;QAER,KAAK,SAAS,CAAC;QACf,KAAK,OAAO;YACV,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC;YACnB,SAAS,GAAG,SAAS,CAAC;YACtB,IAAI,GAAG,KAAK,CAAC;YACb,MAAM;IACV,CAAC;IAED,OAAO,CAAC,GAAG,CACT,GAAG,KAAK,GAAG,IAAI,IAAI,SAAS,GAAG,MAAM,CAAC,OAAO,MAC3C,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,OAAO,KAAK,SAAS;QAC1B,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAC1C,EAAE,CACH,CAAC;AACJ,CAAC"}

package/bin/utils/nestedRoots.d.ts

@@ -0,0 +1,7 @@
+import type { Root as MdastRoot } from 'mdast';
+/** There should only be one root; this replaces any inner roots
+ * that we might get because of us converting the content manually
+ * to `contentAsRoot` from `content`
+ */
+export declare function unifiedRemoveNestedRoots(): (node: MdastRoot) => void;
+export declare function removeNestedRoots(root: MdastRoot): void;

package/bin/utils/nestedRoots.js

@@ -0,0 +1,19 @@
+import { CONTINUE, visit } from 'unist-util-visit';
+/** There should only be one root; this replaces any inner roots
+ * that we might get because of us converting the content manually
+ * to `contentAsRoot` from `content`
+ */
+export function unifiedRemoveNestedRoots() {
+    return function (node) {
+        return removeNestedRoots(node);
+    };
+}
+export function removeNestedRoots(root) {
+    visit(root, 'root', function (node, _, parent) {
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        if (!parent)
+            return CONTINUE;
+        parent.children = node.children;
+    });
+}
+//# sourceMappingURL=nestedRoots.js.map

package/bin/utils/nestedRoots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"nestedRoots.js","sourceRoot":"","sources":["../../src/utils/nestedRoots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;GAGG;AACH,MAAM,UAAU,wBAAwB;IACtC,OAAO,UAAU,IAAe;QAC9B,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAe;IAC/C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,IAAI,EAAE,CAAC,EAAE,MAAM;QAC3C,uEAAuE;QACvE,IAAI,CAAC,MAAM;YAAE,OAAO,QAAQ,CAAC;QAC5B,MAAoB,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC"}

package/bin/utils/network.d.ts

@@ -0,0 +1,5 @@
+import { Browser } from 'puppeteer';
+export declare function startPuppeteer(): Promise<Browser | undefined>;
+export declare function getHtmlWithPuppeteer(browser: Browser, url: string | URL): Promise<string | undefined>;
+export declare function fetchPageHtml(url: string | URL, browser?: Browser | undefined): Promise<string>;
+export declare function fetchImage(url: string): Promise<NodeJS.TypedArray>;

package/bin/utils/network.js

@@ -0,0 +1,82 @@
+import { launch } from 'puppeteer';
+import { getErrorMessage } from './errors.js';
+import { log } from './log.js';
+async function exponentialBackoff(operation, retries = 3, delay = 1000, factor = 2) {
+    try {
+        return await operation();
+    }
+    catch (error) {
+        if (retries > 0) {
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            return exponentialBackoff(operation, retries - 1, delay * factor, factor);
+        }
+        else {
+            throw error;
+        }
+    }
+}
+export async function startPuppeteer() {
+    try {
+        return await launch({
+            headless: true,
+            ignoreHTTPSErrors: true,
+        });
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            log(`Could not create a browser instance: ${error.message}`);
+        }
+    }
+}
+export async function getHtmlWithPuppeteer(browser, url) {
+    const page = await browser.newPage();
+    await page.goto(url.toString(), {
+        waitUntil: 'networkidle2',
+    });
+    const html = await exponentialBackoff(() => page.content());
+    return html;
+}
+async function fetchPageResponse(url) {
+    try {
+        const res = await fetch(url);
+        if (!res.ok) {
+            throw new Error(`${res.status} ${res.statusText}`);
+        }
+        return await res.text();
+    }
+    catch (error) {
+        const errorMessage = getErrorMessage(error);
+        throw new Error(`${url}\n\t- failed to fetch page from source${errorMessage}`);
+    }
+}
+export async function fetchPageHtml(url, browser = undefined) {
+    if (browser) {
+        try {
+            const res = await getHtmlWithPuppeteer(browser, url);
+            if (res)
+                return res;
+            throw new Error('an unknown error occured');
+        }
+        catch (error) {
+            const errorMessage = getErrorMessage(error);
+            throw new Error(`${url}\n\t- Puppeteer failed to retrieve page from source${errorMessage}`);
+        }
+    }
+    return await exponentialBackoff(() => fetchPageResponse(url));
+}
+export async function fetchImage(url) {
+    try {
+        const res = await exponentialBackoff(() => fetch(url));
+        if (!res.ok) {
+            throw new Error(`${res.status} ${res.statusText}`);
+        }
+        const imageBuffer = await res.arrayBuffer();
+        const imageData = new Uint8Array(imageBuffer);
+        return imageData;
+    }
+    catch (error) {
+        const errorMessage = getErrorMessage(error);
+        throw new Error(`${url}\n\t- failed to retrieve image from source${errorMessage}`);
+    }
+}
+//# sourceMappingURL=network.js.map

package/bin/utils/network.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,KAAK,UAAU,kBAAkB,CAC/B,SAA2B,EAC3B,UAAkB,CAAC,EACnB,QAAgB,IAAI,EACpB,SAAiB,CAAC;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC;YAClB,QAAQ,EAAE,IAAI;YACd,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,GAAG,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgB,EAChB,GAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;QAC9B,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAiB;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,yCAAyC,YAAY,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAiB,EACjB,UAA+B,SAAS;IAExC,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACrD,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,sDAAsD,YAAY,EAAE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,OAAO,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAE9C,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,6CAA6C,YAAY,EAAE,CAAC,CAAC;IACrF,CAAC;AACH,CAAC"}

package/bin/utils/path.d.ts

@@ -0,0 +1 @@
+export declare function createFilename(rootPath: string | undefined, filename: string | URL, title?: string): string | undefined;

package/bin/utils/path.js

@@ -0,0 +1,22 @@
+import { join } from 'path';
+import { addMdx } from './extension.js';
+import { toFilename } from './file.js';
+import { log } from './log.js';
+export function createFilename(rootPath = process.cwd(), filename, title) {
+    if (typeof filename === 'string' && filename.startsWith('http')) {
+        const url = new URL(filename);
+        filename = url.pathname;
+    }
+    else if (typeof filename === 'object') {
+        filename = filename.pathname;
+    }
+    else {
+        filename = filename;
+    }
+    if (filename === '') {
+        log(`Invalid file name provided: ${filename}`, 'error');
+        return undefined;
+    }
+    return join(rootPath, addMdx(filename || toFilename(title ?? '')));
+}
+//# sourceMappingURL=path.js.map

package/bin/utils/path.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"path.js","sourceRoot":"","sources":["../../src/utils/path.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,MAAM,UAAU,cAAc,CAC5B,WAAmB,OAAO,CAAC,GAAG,EAAE,EAChC,QAAsB,EACtB,KAAc;IAEd,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAChE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,CAAC;SAAM,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxC,QAAQ,GAAI,QAAgB,CAAC,QAAQ,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,QAAkB,CAAC;IAChC,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,GAAG,CAAC,+BAA+B,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;QACxD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC"}

package/bin/utils/position.d.ts

@@ -0,0 +1,3 @@
+import type { Element } from 'hast';
+export declare function unifiedRemovePositions(): (node: Element) => undefined;
+export declare function removePositions(node: Element): undefined;

package/bin/utils/position.js

@@ -0,0 +1,12 @@
+import { visit } from 'unist-util-visit';
+export function unifiedRemovePositions() {
+    return function (node) {
+        return removePositions(node);
+    };
+}
+export function removePositions(node) {
+    return visit(node, function (subNode) {
+        delete subNode.position;
+    });
+}
+//# sourceMappingURL=position.js.map

package/bin/utils/position.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"position.js","sourceRoot":"","sources":["../../src/utils/position.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,MAAM,UAAU,sBAAsB;IACpC,OAAO,UAAU,IAAa;QAC5B,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAa;IAC3C,OAAO,KAAK,CAAC,IAAI,EAAE,UAAU,OAAO;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC"}

package/bin/utils/reservedNames.d.ts

@@ -0,0 +1,4 @@
+export declare const blacklistedNames: Map<string, string>;
+export declare const INDEX_NAMES: string[];
+export declare const GROUP_NAMES: string[];
+export declare function iterateThroughReservedNames(namesToUse: Array<string>, namesInUse: Array<string>): string;

package/bin/utils/reservedNames.js

@@ -0,0 +1,27 @@
+export const blacklistedNames = new Map([['api', 'api-reference']]);
+export const INDEX_NAMES = [
+    'home',
+    'introduction',
+    'getting-started',
+    'get-started',
+    'welcome',
+    'start',
+];
+export const GROUP_NAMES = [
+    'Home',
+    'Introduction',
+    'Getting Started',
+    'Get Started',
+    'Welcome',
+    'Start',
+    'Docs',
+];
+export function iterateThroughReservedNames(namesToUse, namesInUse) {
+    for (const name of namesToUse) {
+        if (namesInUse.includes(name))
+            continue;
+        return name;
+    }
+    return '';
+}
+//# sourceMappingURL=reservedNames.js.map

package/bin/utils/reservedNames.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reservedNames.js","sourceRoot":"","sources":["../../src/utils/reservedNames.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAiB,CAAC,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;AAEpF,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,MAAM;IACN,cAAc;IACd,iBAAiB;IACjB,aAAa;IACb,SAAS;IACT,OAAO;CACR,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,MAAM;IACN,cAAc;IACd,iBAAiB;IACjB,aAAa;IACb,SAAS;IACT,OAAO;IACP,MAAM;CACP,CAAC;AAEF,MAAM,UAAU,2BAA2B,CACzC,UAAyB,EACzB,UAAyB;IAEzB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/bin/utils/strings.d.ts

@@ -0,0 +1,2 @@
+export declare function removeTrailingSlash(str: string): string;
+export declare function removeLeadingSlash(str: string): string;

package/bin/utils/strings.js

@@ -0,0 +1,7 @@
+export function removeTrailingSlash(str) {
+    return str.endsWith('/') ? str.substring(0, str.length - 1) : str;
+}
+export function removeLeadingSlash(str) {
+    return str.startsWith('/') ? str.substring(1) : str;
+}
+//# sourceMappingURL=strings.js.map

package/bin/utils/strings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"strings.js","sourceRoot":"","sources":["../../src/utils/strings.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,CAAC"}

package/bin/utils/text.d.ts

@@ -0,0 +1,2 @@
+import type { Element } from 'hast';
+export declare function getText(element: Element | undefined): string;

package/bin/utils/text.js

@@ -0,0 +1,11 @@
+import { visit } from 'unist-util-visit';
+export function getText(element) {
+    if (!element)
+        return '';
+    let text = '';
+    visit(element, 'text', function (node) {
+        text += node.value;
+    });
+    return text;
+}
+//# sourceMappingURL=text.js.map

package/bin/utils/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/utils/text.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEzC,MAAM,UAAU,OAAO,CAAC,OAA4B;IAClD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,IAAI;QACnC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}

package/bin/utils/title.d.ts

@@ -0,0 +1,10 @@
+import type { Element, ElementContent, Root as HastRoot } from 'hast';
+import type { Root as MdastRoot, BlockContent } from 'mdast';
+export declare function findTitle(node: Element | ElementContent | BlockContent | MdastRoot | HastRoot | undefined, opts?: {
+    delete: boolean;
+    nodeType?: string;
+    tagName?: string;
+}): string;
+export declare function getTitleFromHeading(root: MdastRoot): string;
+export declare function getDescriptionFromRoot(root: MdastRoot): string;
+export declare function getTitleFromLink(url: string): string;

package/bin/utils/title.js

@@ -0,0 +1,58 @@
+import { visit, CONTINUE, EXIT } from 'unist-util-visit';
+export function findTitle(node, opts = {
+    delete: true,
+    nodeType: undefined,
+    tagName: undefined,
+}) {
+    let title = '';
+    if (!node)
+        return title;
+    visit(node, opts.nodeType ? opts.nodeType : node, function (subNode) {
+        if (opts.tagName && subNode.type === 'element' && subNode.tagName !== opts.tagName) {
+            return CONTINUE;
+        }
+        visit(subNode, 'text', function (textNode, index, parent) {
+            title += textNode.value;
+            if (opts.delete && parent && typeof index === 'number') {
+                parent.children.splice(index, 1);
+            }
+        });
+    });
+    return title.trim();
+}
+export function getTitleFromHeading(root) {
+    let headingElement = undefined;
+    visit(root, 'heading', function (subNode, index, parent) {
+        headingElement = subNode;
+        if (parent && typeof index === 'number') {
+            parent.children.splice(index, 1);
+        }
+        return EXIT;
+    });
+    return findTitle(headingElement);
+}
+export function getDescriptionFromRoot(root) {
+    let descriptionElement = undefined;
+    visit(root, 'paragraph', function (subNode, index, parent) {
+        if (typeof index !== 'number' || index !== 0 || !parent || parent.type !== 'root')
+            return EXIT;
+        descriptionElement = subNode;
+        if (typeof index === 'number') {
+            parent.children.splice(index, 1);
+        }
+        return EXIT;
+    });
+    return findTitle(descriptionElement);
+}
+export function getTitleFromLink(url) {
+    if (url.startsWith('http')) {
+        url = new URL(url).pathname;
+    }
+    const lastPathname = url.split('/').at(-1) ?? '';
+    const dashSplitPathname = lastPathname.split('-').flatMap((i) => i.split('_'));
+    dashSplitPathname.forEach((str, index) => {
+        dashSplitPathname[index] = str[0] ? `${str[0].toUpperCase()}${str.substring(1)}` : str;
+    });
+    return dashSplitPathname.join(' ');
+}
+//# sourceMappingURL=title.js.map

package/bin/utils/title.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"title.js","sourceRoot":"","sources":["../../src/utils/title.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEzD,MAAM,UAAU,SAAS,CACvB,IAAgF,EAChF,OAAiE;IAC/D,MAAM,EAAE,IAAI;IACZ,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;CACnB;IAED,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAExB,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,OAAO;QACjE,IAAI,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YACnF,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,QAAQ,EAAE,KAAK,EAAE,MAAM;YACtD,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACnC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAe;IACjD,IAAI,cAAc,GAA6B,SAAS,CAAC;IACzD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QACrD,cAAc,GAAG,OAAO,CAAC;QACzB,IAAI,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACxC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,OAAO,SAAS,CAAC,cAAc,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,IAAe;IACpD,IAAI,kBAAkB,GAA6B,SAAS,CAAC;IAC7D,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,OAAO,EAAE,KAAK,EAAE,MAAM;QACvD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QAE/F,kBAAkB,GAAG,OAAO,CAAC;QAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,OAAO,SAAS,CAAC,kBAAkB,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC9B,CAAC;IACD,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,iBAAiB,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/E,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QACvC,iBAAiB,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrC,CAAC"}

package/bin/utils/url.d.ts

@@ -0,0 +1,3 @@
+export declare function checkUrl({ url }: {
+    url: string;
+}): boolean;

package/bin/utils/url.js

@@ -0,0 +1,10 @@
+export function checkUrl({ url }) {
+    try {
+        new URL(url);
+    }
+    catch {
+        throw Error(`Invalid link: ${url}\nMake sure the link starts with http:// or https://`);
+    }
+    return true;
+}
+//# sourceMappingURL=url.js.map

package/bin/utils/url.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/utils/url.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,QAAQ,CAAC,EAAE,GAAG,EAAmB;IAC/C,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,CAAC,iBAAiB,GAAG,sDAAsD,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "3.0.187",
+  "version": "3.0.189",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -38,28 +38,37 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.164",
+    "@mintlify/common": "1.0.165",
     "@mintlify/openapi-parser": "^0.0.0",
-    "axios": "^1.2.2",
-    "cheerio": "^1.0.0-rc.12",
     "fs-extra": "^11.1.1",
+    "hast": "^1.0.0",
+    "hast-util-to-mdast": "^10.1.0",
     "js-yaml": "^4.1.0",
-    "node-html-markdown": "^1.3.0",
-    "ora": "^6.1.2",
+    "mdast": "^3.0.0",
+    "mdast-util-mdx-jsx": "^3.1.3",
     "puppeteer": "^22.14.0",
-    "yargs": "^17.6.0"
+    "rehype-parse": "^9.0.0",
+    "remark-gfm": "^4.0.0",
+    "remark-mdx": "^3.0.1",
+    "remark-parse": "^11.0.0",
+    "remark-stringify": "^11.0.0",
+    "traverse": "^0.6.10",
+    "unified": "^11.0.5",
+    "unist-util-visit": "^5.0.0",
+    "yargs": "^17.6.0",
+    "zod": "^3.20.6"
   },
   "devDependencies": {
     "@mintlify/eslint-config": "1.0.5",
     "@mintlify/eslint-config-typescript": "1.0.10",
-    "@mintlify/models": "0.0.136",
+    "@mintlify/models": "0.0.137",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.205",
+    "@mintlify/validation": "0.1.206",
     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
     "@tsconfig/recommended": "1.x",
-    "@types/cheerio": "^0.22.31",
     "@types/node": "^18.7.13",
+    "@types/traverse": "^0.6.37",
     "@types/yargs": "^17.0.13",
     "@typescript-eslint/eslint-plugin": "6.x",
     "@typescript-eslint/parser": "6.x",
@@ -72,5 +81,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "0ed4e6fadb34fc030dabe5bcb74f99e3d66f2332"
+  "gitHead": "e69843f13b6c803075baa3325cc70f0ab62062c8"
 }

package/src/assert.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod';
+
+export const StringArraySchema = z.array(z.string());
+
+export function assertIsNumber(val: unknown): asserts val is number {
+  z.number().parse(val);
+}
+
+export function assertIsDefined<T>(val: T): asserts val is NonNullable<T> {
+  if (val === undefined || val === null) throw new Error('Value is nullable.');
+}
+
+export function assertIsStringArray(val: unknown): asserts val is Array<string> {
+  StringArraySchema.parse(val);
+}