@mintlify/scraping 3.0.187 → 3.0.189

package/bin/scraping/site-scrapers/scrapeReadMePage.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC1D,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;IAC9C,CAAC;IAED,+EAA+E;IAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEzC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMeSection.d.ts

@@ -1,2 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-export declare function scrapeReadMeSection(html: string, origin: string, cliDir: string, imageBaseDir: string, overwrite: boolean, version: string | undefined): Promise<NavigationEntry[]>;

package/bin/scraping/site-scrapers/scrapeReadMeSection.js

@@ -1,39 +0,0 @@
-import * as cheerio from 'cheerio';
-import downloadLogoImage from '../downloadLogoImage.js';
-import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
-import getLinksRecursively from './links-per-group/getLinksRecursively.js';
-import { scrapeReadMePage } from './scrapeReadMePage.js';
-export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
-    const $ = cheerio.load(html);
-    // Download the logo
-    const logoSrc = $('.rm-Logo-img').first().attr('src');
-    downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite).catch(console.error);
-    // Get all the navigation sections, but only from the first
-    // sidebar found. There are multiple in the HTML for mobile
-    // responsiveness but they all have the same links.
-    const navigationSections = $('.rm-Sidebar').first().find('.rm-Sidebar-section');
-    const groupsConfig = navigationSections.toArray().map((s) => {
-        const section = $(s);
-        const sectionTitle = section.find('h3').first().text();
-        // Get all links, then use filter to remove duplicates.
-        // There are duplicates because of nested navigation, eg:
-        // subgroupTitle -> /first-page
-        // -- First Page -> /first-page   ** DUPLICATE **
-        // -- Second Page -> /second-page
-        const linkSections = section.find('.rm-Sidebar-list').first().children();
-        const pages = getLinksRecursively(linkSections, $).filter((value, index, array) => array.indexOf(value) === index);
-        // Follows the same structure as mint.json
-        return {
-            group: sectionTitle,
-            pages: pages,
-        };
-    });
-    // Scrape each link in the navigation.
-    return Promise.all(groupsConfig.map(async (navEntry) => {
-        return await scrapeGettingFileNameFromUrl(
-        // ReadMe requires a directory on all sections whereas we use root.
-        // /docs is their default directory so we remove it
-        navEntry, cliDir, origin, overwrite, scrapeReadMePage, false, version, '/docs');
-    }));
-}
-//# sourceMappingURL=scrapeReadMeSection.js.map

package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0CAA0C,CAAC;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAEjF,2DAA2D;IAC3D,2DAA2D;IAC3D,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAe,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACtE,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAEvD,uDAAuD;QACvD,yDAAyD;QACzD,+BAA+B;QAC/B,iDAAiD;QACjD,iCAAiC;QACjC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,MAAM,CACvD,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CACxD,CAAC;QAEF,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,KAAK;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,sCAAsC;IACtC,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,QAAyB,EAAE,EAAE;QACnD,OAAO,MAAM,4BAA4B;QACvC,mEAAmE;QACnE,mDAAmD;QACnD,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,gBAAgB,EAChB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/bin/util.d.ts

@@ -1,29 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-import { Ora as OraType } from 'ora';
-export declare const MintConfig: (name: string, color: string, ctaName: string, ctaUrl: string, filename: string) => {
-    name: string;
-    logo: string;
-    favicon: string;
-    colors: {
-        primary: string;
-    };
-    topbarLinks: never[];
-    topbarCtaButton: {
-        name: string;
-        url: string;
-    };
-    anchors: never[];
-    navigation: {
-        group: string;
-        pages: string[];
-    }[];
-};
-export declare const Page: (title: string, description?: string, markdown?: string) => string;
-export declare function getOrigin(url: string): string;
-export declare function objToReadableString(objs: NavigationEntry[]): string;
-export declare const toFilename: (title: string) => string;
-export declare const addMdx: (fileName: string) => string;
-export declare const createPage: (title: string, description?: string, markdown?: string, overwrite?: boolean, rootDir?: string, fileName?: string) => void;
-export declare const buildLogger: (startText?: string) => OraType;
-export declare const getFileExtension: (filename: string) => string | undefined;
-export declare const fileBelongsInPagesFolder: (filename: string) => boolean | "" | undefined;

package/bin/util.js

@@ -1,97 +0,0 @@
-import { existsSync, mkdirSync, writeFileSync } from 'fs';
-import Ora from 'ora';
-import path from 'path';
-export const MintConfig = (name, color, ctaName, ctaUrl, filename) => {
-    return {
-        name,
-        logo: '',
-        favicon: '',
-        colors: {
-            primary: color,
-        },
-        topbarLinks: [],
-        topbarCtaButton: {
-            name: ctaName,
-            url: ctaUrl,
-        },
-        anchors: [],
-        navigation: [
-            {
-                group: 'Home',
-                pages: [filename],
-            },
-        ],
-        // footerSocials: {}, // support object type for footer tyoes
-    };
-};
-export const Page = (title, description, markdown) => {
-    // If we are an empty String we want to add two quotes,
-    // if we added as we went we would detect the first quote
-    // as the closing quote.
-    const startsWithQuote = title.startsWith('"');
-    const endsWithQuote = title.startsWith('"');
-    if (!startsWithQuote) {
-        title = '"' + title;
-    }
-    if (!endsWithQuote) {
-        title = title + '"';
-    }
-    const optionalDescription = description ? `\ndescription: "${description}"` : '';
-    return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
-};
-export function getOrigin(url) {
-    // eg. https://google.com -> https://google.com
-    // https://google.com/page -> https://google.com
-    return new URL(url).origin;
-}
-export function objToReadableString(objs) {
-    // Two spaces as indentation
-    return objs.map((obj) => JSON.stringify(obj, null, 2)).join(',\n');
-}
-export const toFilename = (title) => {
-    // Gets rid of special characters at the start and end
-    // of the name by converting to spaces then using trim.
-    return title
-        .replace(/[^a-z0-9]/gi, ' ')
-        .trim()
-        .replace(/ /g, '-')
-        .toLowerCase();
-};
-export const addMdx = (fileName) => {
-    if (fileName.endsWith('.mdx')) {
-        return fileName;
-    }
-    return fileName + '.mdx';
-};
-export const createPage = (title, description, markdown, overwrite = false, rootDir = '', fileName) => {
-    const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
-    // Create the folders needed if they're missing
-    mkdirSync(rootDir, { recursive: true });
-    if (!overwrite && existsSync(writePath)) {
-        console.log(`❌ Skipping existing file ${writePath}`);
-        return;
-    }
-    // Write the page to disk
-    try {
-        writeFileSync(writePath, Page(title, description, markdown));
-        console.log('✏️ - ' + writePath);
-    }
-    catch (e) {
-        console.error(e);
-    }
-};
-export const buildLogger = (startText = '') => {
-    const logger = Ora().start(startText);
-    return logger;
-};
-export const getFileExtension = (filename) => {
-    const ext = filename.substring(filename.lastIndexOf('.') + 1, filename.length);
-    if (filename === ext)
-        return undefined;
-    return ext.toLowerCase();
-};
-export const fileBelongsInPagesFolder = (filename) => {
-    const extension = getFileExtension(filename);
-    return extension && (extension === 'mdx' || extension === 'md' || extension === 'tsx');
-};
-//# sourceMappingURL=util.js.map

package/bin/util.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC1D,OAAO,GAAuB,MAAM,KAAK,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,WAAoB,EAAE,QAAiB,EAAE,EAAE;IAC7E,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;IACtB,CAAC;IACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;IACtB,CAAC;IAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,CAAC,CAAC,mBAAmB,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,SAAS,GAAG,KAAK,EACjB,OAAO,GAAG,EAAE,EACZ,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC;QACH,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,GAAG,EAAE,EAAW,EAAE;IACrD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,SAAS,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CAAC;AACzF,CAAC,CAAC"}

package/src/browser.ts

@@ -1,24 +0,0 @@
-import { launch } from 'puppeteer';
-
-export async function startBrowser() {
-  try {
-    return await launch({
-      headless: true,
-      ignoreHTTPSErrors: true,
-    });
-  } catch (err) {
-    console.log('Could not create a browser instance: ', err);
-    process.exit(1);
-  }
-}
-
-export async function getHtmlWithPuppeteer(href: string) {
-  const browser = await startBrowser();
-  const page = await browser.newPage();
-  await page.goto(href, {
-    waitUntil: 'networkidle2',
-  });
-  const html = await page.content();
-  void browser.close();
-  return html;
-}

package/src/checks.ts

@@ -1,32 +0,0 @@
-import { Framework } from './scraping/detectFramework.js';
-
-// This checks the link is written correctly, not that the page exists.
-export function checkUrl({ url }: { url: string }) {
-  try {
-    new URL(url);
-  } catch {
-    throw Error(`Invalid link: ${url}\nMake sure the link starts with http:// or https://`);
-  }
-  return true;
-}
-
-export function checkVersion({
-  tool,
-  docusaurusVersion,
-}: {
-  tool: Framework | undefined;
-  docusaurusVersion: string | undefined;
-}) {
-  if (tool === 'docusaurus') {
-    if (docusaurusVersion === undefined) {
-      throw Error(
-        'When using Docusaurus, you must specify the version (1,2,3) using the --docusaurusVersion flag'
-      );
-    }
-  } else {
-    if (docusaurusVersion !== undefined) {
-      throw Error('The --docusaurusVersion flag is only applicable when using Docusaurus.');
-    }
-  }
-  return true;
-}

package/src/downloadImage.ts

@@ -1,102 +0,0 @@
-import axios from 'axios';
-import { existsSync, mkdirSync, createWriteStream } from 'fs';
-import path from 'path';
-
-import { SUPPORTED_MEDIA_EXTENSIONS } from './constants.js';
-import { getFileExtension } from './util.js';
-
-async function writeImageToFile(imageSrc: string, writePath: string, overwrite: boolean) {
-  // Avoid unnecessary downloads
-  if (existsSync(writePath) && !overwrite) {
-    return Promise.reject({
-      code: 'EEXIST',
-    });
-  }
-
-  // Create the folders needed if they're missing
-  mkdirSync(path.dirname(writePath), { recursive: true });
-
-  const writer = createWriteStream(writePath);
-
-  try {
-    const response = await axios.get(imageSrc, {
-      responseType: 'stream',
-    });
-    // wx prevents overwriting an image with the exact same name
-    // being created in the time we were downloading
-    response.data.pipe(writer, {
-      flag: 'wx',
-    });
-
-    return new Promise((resolve, reject) => {
-      writer.on('finish', resolve);
-      writer.on('error', reject);
-    });
-  } catch (e) {
-    return Promise.reject({
-      code: 'ENOTFOUND',
-    });
-  }
-}
-
-export function isValidImageSrc(src: string) {
-  if (!src) {
-    return false;
-  }
-  // We do not support downloading base64 in-line images.
-  if (src.startsWith('data:')) {
-    return false;
-  }
-
-  const imageHref = removeMetadataFromImageSrc(src);
-  const ext = getFileExtension(imageHref);
-
-  if (ext && !SUPPORTED_MEDIA_EXTENSIONS.includes(ext)) {
-    console.error('🚨 We do not support the file extension: ' + ext);
-    return false;
-  }
-
-  return true;
-}
-
-export function getLengthUntilMetadata(src: string, ext: string) {
-  const lengthUntilMetadata = src.indexOf(`.${ext}`) + `.${ext}`.length;
-  return src.slice(0, lengthUntilMetadata);
-}
-
-export function removeMetadataFromImageSrc(src: string) {
-  // Some frameworks add metadata after the file extension, we need to remove that.
-  if (src.includes('gitbook/image')) {
-    for (const ext of SUPPORTED_MEDIA_EXTENSIONS) {
-      if (src.includes(`.${ext}`)) {
-        return getLengthUntilMetadata(src, ext);
-      }
-    }
-  }
-  return src.split('#')[0]!.split('?')[0]!;
-}
-
-export function cleanImageSrc(src: string, origin: string) {
-  // Add origin if the image tags are using relative sources
-  return src.startsWith('http') ? src : new URL(src, origin).href;
-}
-
-export default async function downloadImage(
-  imageSrc: string,
-  writePath: string,
-  overwrite = false
-) {
-  await writeImageToFile(imageSrc, writePath, overwrite)
-    .then(() => {
-      console.log('🖼️ - ' + writePath);
-    })
-    .catch((e) => {
-      if (e.code === 'EEXIST') {
-        console.log(`❌ Skipping existing image ${writePath}`);
-      } else if (e.code === 'ENOTFOUND') {
-        console.error(`🚨 Cannot download the image, address not found ${imageSrc}`);
-      } else {
-        console.error(e);
-      }
-    });
-}

package/src/scraping/combineNavWithEmptyGroupTitles.ts

@@ -1,21 +0,0 @@
-import { Navigation, NavigationGroup } from '@mintlify/models';
-
-export default function combineNavWithEmptyGroupTitles(navArray: Navigation): Navigation {
-  const newNavArray: Navigation = [];
-
-  navArray.forEach((nav: NavigationGroup) => {
-    // The first run through the loop will always have -1 as the index.
-    // JavaScript returns undefined when we look for an index outside the size of the array.
-    const prev = newNavArray[newNavArray.length - 1];
-    if (prev == null) {
-      newNavArray.push(nav);
-    } else if (!nav.group && !prev.group) {
-      // Joins multiple groups without a title together IF they occur side by side
-      prev.pages = prev.pages.concat(nav.pages);
-    } else {
-      newNavArray.push(nav);
-    }
-  });
-
-  return newNavArray;
-}

package/src/scraping/detectFramework.ts

@@ -1,55 +0,0 @@
-import * as cheerio from 'cheerio';
-
-export const frameworks = ['docusaurus', 'gitbook', 'readme', 'intercom'] as const;
-export type Framework = (typeof frameworks)[number];
-
-export type FrameworkHint =
-  | {
-      framework: 'docusaurus';
-      version: '1' | '2' | '3';
-    }
-  | {
-      framework: 'gitbook' | 'readme' | 'intercom' | undefined;
-    };
-
-export function detectFramework(html: string): FrameworkHint {
-  const $ = cheerio.load(html);
-  const docusaurusMeta = $('meta[name="generator"]');
-
-  if (
-    docusaurusMeta.length > 0 &&
-    docusaurusMeta.attr('content') &&
-    typeof docusaurusMeta.attr('content') === 'string' &&
-    (docusaurusMeta.attr('content') as string).includes('Docusaurus')
-  ) {
-    const metaAttrString = docusaurusMeta.attr('content') as string;
-    if (metaAttrString.includes('v3')) {
-      return { framework: 'docusaurus', version: '3' };
-    }
-    if (metaAttrString.includes('v2')) {
-      return { framework: 'docusaurus', version: '2' };
-    } else if (metaAttrString.includes('v1')) {
-      console.warn(
-        'WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.'
-      );
-      return { framework: 'docusaurus', version: '1' };
-    }
-  }
-
-  const isGitBook = $('head link[rel="preconnect"][href="https://api.gitbook.com"]').length > 0;
-  if (isGitBook) {
-    return { framework: 'gitbook' };
-  }
-
-  const isReadMe = $('meta[name="readme-deploy"]').length > 0;
-  if (isReadMe) {
-    return { framework: 'readme' };
-  }
-
-  const isIntercom = $("meta[name='intercom:trackingEvent']").length > 0;
-  if (isIntercom) {
-    return { framework: 'intercom' };
-  }
-
-  return { framework: undefined };
-}

package/src/scraping/downloadAllImages.ts

@@ -1,61 +0,0 @@
-import { Cheerio, CheerioAPI, Element } from 'cheerio';
-import path from 'path';
-
-import downloadImage, {
-  cleanImageSrc,
-  isValidImageSrc,
-  removeMetadataFromImageSrc,
-} from '../downloadImage.js';
-
-export default async function downloadAllImages(
-  $: CheerioAPI,
-  content: Cheerio<Element>,
-  origin: string,
-  baseDir: string,
-  overwrite: boolean,
-  modifyFileName?: (fileName: string) => string
-) {
-  if (!baseDir) {
-    console.debug('Skipping image downloading');
-    return;
-  }
-
-  // We remove duplicates because some frameworks duplicate img tags
-  // to show the image larger when clicked on.
-  const imageSrcs = [
-    ...new Set(
-      content
-        .find('img[src]')
-        .map((_, image) => $(image).attr('src'))
-        .toArray()
-    ),
-  ];
-
-  // Wait to all images to download before continuing
-  const origToNewArray = await Promise.all(
-    imageSrcs.map(async (imageSrc) => {
-      if (!imageSrc || !isValidImageSrc(imageSrc)) {
-        return {};
-      }
-
-      const imageHref = cleanImageSrc(imageSrc, origin);
-      let imageBasename = imageHref;
-      if (!imageHref.includes('gitbook/image')) {
-        imageBasename = path.basename(imageHref);
-      }
-
-      let fileName = removeMetadataFromImageSrc(imageBasename) || '';
-      if (modifyFileName) {
-        fileName = modifyFileName(fileName) || '';
-      }
-
-      const writePath = path.join(baseDir, fileName);
-
-      await downloadImage(imageHref, writePath, overwrite);
-
-      return { [imageSrc]: writePath };
-    })
-  );
-
-  return origToNewArray.reduce((result, current) => Object.assign(result, current), {});
-}

package/src/scraping/downloadLogoImage.ts

@@ -1,24 +0,0 @@
-import path from 'path';
-
-import downloadImage, {
-  cleanImageSrc,
-  isValidImageSrc,
-  removeMetadataFromImageSrc,
-} from '../downloadImage.js';
-import { getFileExtension } from '../util.js';
-
-export default async function downloadLogoImage(
-  imageSrc: string | undefined,
-  imageBaseDir: string,
-  origin: string,
-  overwrite: boolean
-) {
-  if (!imageSrc || !isValidImageSrc(imageSrc)) return;
-
-  const imageHref = cleanImageSrc(imageSrc, origin);
-
-  const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
-  const imagePath = path.join(imageBaseDir, 'logo', 'logo-light-mode.' + ext);
-
-  await downloadImage(imageHref, imagePath, overwrite);
-}

package/src/scraping/replaceImagePaths.ts

@@ -1,17 +0,0 @@
-export default function replaceImagePaths(
-  origToWritePath: Record<string, string>,
-  cliDir: string,
-  markdown: string
-) {
-  // Change image paths to use the downloaded locations
-  for (const [origHref, writePath] of Object.entries(origToWritePath)) {
-    // Use relative paths within the folder we are in
-    if (writePath.startsWith(cliDir)) {
-      markdown = markdown.replaceAll(origHref, writePath.slice(cliDir.length));
-    } else {
-      markdown = markdown.replaceAll(origHref, writePath);
-    }
-  }
-
-  return markdown;
-}

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -1,84 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-import axios from 'axios';
-import path from 'path';
-
-import { getHtmlWithPuppeteer } from '../browser.js';
-import { createPage } from '../util.js';
-
-export async function scrapeFileGettingFileNameFromUrl(
-  pathname: string,
-  cliDir: string,
-  origin: string,
-  overwrite: boolean,
-  scrapePageFunc: (
-    html: string,
-    origin: string,
-    cliDir: string,
-    imageBaseDir: string,
-    overwrite: boolean,
-    version: string | undefined
-  ) => Promise<{
-    title?: string;
-    description?: string;
-    markdown?: string;
-  }>,
-  puppeteer = false,
-  version: string | undefined,
-  baseToRemove?: string
-): Promise<NavigationEntry> {
-  // Skip scraping external links
-  if (pathname.startsWith('https://') || pathname.startsWith('http://')) {
-    return pathname;
-  }
-
-  // Removes file name from the end
-  const splitSubpath = pathname.split('/');
-  let folders = splitSubpath.slice(0, splitSubpath.length - 1).join('/');
-
-  // Remove base dir if passed in
-  if (baseToRemove && folders.startsWith(baseToRemove)) {
-    folders = folders.replace(baseToRemove, '');
-  }
-
-  // TO DO: Improve this by putting each page's images in a separate
-  // folder named after the title of the page.
-  const imageBaseDir = path.join(cliDir, 'images', folders);
-
-  // Scrape each page separately
-  const href = new URL(pathname, origin).href;
-  let html: string;
-  if (puppeteer) {
-    html = await getHtmlWithPuppeteer(href);
-  } else {
-    const res = await axios.get(href);
-    html = res.data;
-  }
-
-  const { title, description, markdown } = await scrapePageFunc(
-    html,
-    origin,
-    cliDir,
-    imageBaseDir,
-    overwrite,
-    version
-  );
-
-  // Check if page didn't have content
-  if (!title && !markdown) {
-    return {
-      group: '',
-      pages: [],
-    };
-  }
-
-  const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
-
-  // Default to introduction.mdx if we encountered index.html
-  const fileName = splitSubpath[splitSubpath.length - 1] || 'introduction';
-
-  // Will create subfolders as needed
-  createPage(title ?? '', description, markdown, overwrite, newFileLocation, fileName);
-
-  // Removes first slash if we are in a folder, Mintlify doesn't need it
-  return folders ? path.join(folders, fileName).substring(1) : fileName;
-}

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -1,56 +0,0 @@
-import { NavigationEntry } from '@mintlify/models';
-
-import { scrapeFileGettingFileNameFromUrl } from './scrapeFileGettingFileNameFromUrl.js';
-
-export async function scrapeGettingFileNameFromUrl(
-  navEntry: NavigationEntry,
-  cliDir: string,
-  origin: string,
-  overwrite: boolean,
-  scrapePageFunc: (
-    html: string,
-    origin: string,
-    cliDir: string,
-    imageBaseDir: string,
-    overwrite: boolean,
-    version: string | undefined
-  ) => Promise<{
-    title?: string;
-    description?: string;
-    markdown?: string;
-  }>,
-  puppeteer = false,
-  version: string | undefined,
-  baseToRemove?: string
-): Promise<NavigationEntry> {
-  if (typeof navEntry !== 'string') {
-    const newPages: NavigationEntry[] = [];
-    for (const nestedNavEntry of navEntry.pages) {
-      newPages.push(
-        await scrapeGettingFileNameFromUrl(
-          nestedNavEntry,
-          cliDir,
-          origin,
-          overwrite,
-          scrapePageFunc,
-          puppeteer,
-          version,
-          baseToRemove
-        )
-      );
-    }
-    navEntry.pages = newPages;
-    return navEntry;
-  }
-
-  return await scrapeFileGettingFileNameFromUrl(
-    navEntry,
-    cliDir,
-    origin,
-    overwrite,
-    scrapePageFunc,
-    puppeteer,
-    version,
-    baseToRemove
-  );
-}