mintlify 3.0.7 → 3.0.8

package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js

@@ -0,0 +1,30 @@
+export default async function openNestedDocusaurusMenus(page) {
+    let prevEncountered = [];
+    let encounteredHref = ["fake-href-to-make-loop-run-at-least-once"];
+    // Loop until we've encountered every link
+    while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+        prevEncountered = encounteredHref;
+        encounteredHref = await page.evaluate((encounteredHref) => {
+            const collapsible = Array.from(document.querySelectorAll(".menu__link.menu__link--sublist"));
+            const linksFound = [];
+            collapsible.forEach(async (collapsibleItem) => {
+                const href = collapsibleItem?.getAttribute("href");
+                // Should never occur but we keep it as a fail-safe
+                if (href?.startsWith("https://") || href?.startsWith("http://")) {
+                    return;
+                }
+                // Click any links we haven't seen before
+                if (href && !encounteredHref.includes(href)) {
+                    collapsibleItem?.click();
+                }
+                if (href) {
+                    linksFound.push(href);
+                }
+            });
+            return linksFound;
+        }, encounteredHref // Need to pass array into the browser
+        );
+    }
+    return await page.content();
+}
+//# sourceMappingURL=openNestedDocusaurusMenus.js.map

package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openNestedDocusaurusMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedDocusaurusMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAU;IAChE,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,GAAG,CAAC,0CAA0C,CAAC,CAAC;IAEnE,0CAA0C;IAC1C,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE;QACvE,eAAe,GAAG,eAAe,CAAC;QAClC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CACnC,CAAC,eAAe,EAAE,EAAE;YAClB,MAAM,WAAW,GAAkB,KAAK,CAAC,IAAI,CAC3C,QAAQ,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAC7D,CAAC;YAEF,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,eAA4B,EAAE,EAAE;gBACzD,MAAM,IAAI,GAAG,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEnD,mDAAmD;gBACnD,IAAI,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;oBAC/D,OAAO;iBACR;gBAED,yCAAyC;gBACzC,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAC3C,eAAe,EAAE,KAAK,EAAE,CAAC;iBAC1B;gBAED,IAAI,IAAI,EAAE;oBACR,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACvB;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,UAAU,CAAC;QACpB,CAAC,EACD,eAAe,CAAC,sCAAsC;SACvD,CAAC;KACH;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}

package/bin/scraping/site-scrapers/openNestedGitbookMenus.js

@@ -0,0 +1,17 @@
+export default async function openNestedGitbookMenus(page) {
+    let clickedAny = true;
+    // Loop until we've encountered every closed menu
+    while (clickedAny) {
+        clickedAny = await page.evaluate(() => {
+            // Right pointing arrow. Only closed menus have this icon
+            const icons = Array.from(document.querySelectorAll('path[d="M9 18l6-6-6-6"]'));
+            icons.forEach(async (icon) => {
+                const toClick = icon?.parentElement?.parentElement;
+                toClick.click();
+            });
+            return icons.length > 0;
+        });
+    }
+    return await page.content();
+}
+//# sourceMappingURL=openNestedGitbookMenus.js.map

package/bin/scraping/site-scrapers/openNestedGitbookMenus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openNestedGitbookMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedGitbookMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAU;IAC7D,IAAI,UAAU,GAAG,IAAI,CAAC;IAEtB,iDAAiD;IACjD,OAAO,UAAU,EAAE;QACjB,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACpC,yDAAyD;YACzD,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CACrC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CACrD,CAAC;YAEF,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAiB,EAAE,EAAE;gBACxC,MAAM,OAAO,GAAG,IAAI,EAAE,aAAa,EAAE,aAAa,CAAC;gBACnD,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,CAAC,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;KACJ;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}

package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js

@@ -0,0 +1,49 @@
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+export async function scrapeDocusaurusPage(html, origin, cliDir, imageBaseDir, overwrite, version // expects "2", or "3". Have not written support for "1" yet
+) {
+    const $ = cheerio.load(html);
+    const article = version === "3" ? $(".theme-doc-markdown").first() : $("article").first();
+    if (article.length === 0) {
+        // Index pages with no additional text don't have the markdown class
+        return {};
+    }
+    const titleComponent = article.find("h1");
+    const title = titleComponent.text().trim();
+    // Do not include title in the content when we insert it in our metadata
+    titleComponent.remove();
+    const markdownContent = version === "3" ? article : article.find(".markdown").first();
+    const origToWritePath = await downloadAllImages($, markdownContent, origin, imageBaseDir, overwrite);
+    const markdownHtml = markdownContent.html();
+    const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
+    let markdown = nhm.translate(markdownHtml);
+    if (markdown == null) {
+        console.error("We do not support scraping this page. Content will be empty");
+        return { title, description: null, markdown: "" };
+    }
+    // Description only exists in meta tags. The code is commented out because its prone to incorrectly
+    // including a description if the first line of text had markdown annotations like `.
+    // The commented out alternative is to ignore description if it's the first line of text,
+    // this means it was not set in the metadata and Docusaurus defaulted to the text.
+    const description = null;
+    // let description = $('meta[property="og:description"]').attr("content");
+    // if (markdown.startsWith(description)) {
+    //   description = null;
+    // }
+    // Remove Docusaurus links from headers
+    // When we parse their HTML the parser adds things like:
+    // [](#setup "Direct link to heading")
+    // to the end of each header.
+    markdown = markdown.replace(/\[\]\(#.+ ".+"\)\n/g, "\n");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeDocusaurusPage.js.map

package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeDocusaurusPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B,CAAC,4DAA4D;;IAExF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,OAAO,GACX,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IAE5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,oEAAoE;QACpE,OAAO,EAAE,CAAC;KACX;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,wEAAwE;IACxE,cAAc,CAAC,MAAM,EAAE,CAAC;IAExB,MAAM,eAAe,GACnB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,eAAe,EACf,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;IAE5C,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAE3C,IAAI,QAAQ,IAAI,IAAI,EAAE;QACpB,OAAO,CAAC,KAAK,CACX,6DAA6D,CAC9D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;KACnD;IAED,mGAAmG;IACnG,qFAAqF;IACrF,yFAAyF;IACzF,kFAAkF;IAClF,MAAM,WAAW,GAAG,IAAI,CAAC;IACzB,0EAA0E;IAC1E,0CAA0C;IAC1C,wBAAwB;IACxB,IAAI;IAEJ,uCAAuC;IACvC,wDAAwD;IACxD,sCAAsC;IACtC,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAEzD,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js

@@ -0,0 +1,30 @@
+import cheerio from "cheerio";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
+import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
+import { getDocusaurusLinksPerGroup } from "./links-per-group/getDocusaurusLinksPerGroup.js";
+import downloadLogoImage from "../downloadLogoImage.js";
+export async function scrapeDocusaurusSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
+    const $ = cheerio.load(html);
+    // Download the logo
+    const logoSrc = $(".navbar__logo img").attr("src");
+    downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
+    // Get all the navigation sections
+    const navigationSections = $(".theme-doc-sidebar-menu").first().children();
+    // Get all links per group
+    const groupsConfig = getDocusaurusLinksPerGroup(navigationSections, $, version);
+    // Merge groups with empty titles together
+    const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
+    // Scrape each link in the navigation.
+    const groupsConfigCleanPaths = await Promise.all(reducedGroupsConfig.map(async (groupConfig) => {
+        groupConfig.pages = (await Promise.all(groupConfig.pages.map(async (navEntry) => 
+        // Docusaurus requires a directory on all sections wheras we use root.
+        // /docs is their default directory so we remove it
+        scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapeDocusaurusPage, false, version, "/docs"))))
+            // Remove skipped index pages (they return undefined from the above function)
+            .filter(Boolean);
+        return groupConfig;
+    }));
+    return groupsConfigCleanPaths;
+}
+//# sourceMappingURL=scrapeDocusaurusSection.js.map

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iDAAiD,CAAC;AAC7F,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAAe;IAEf,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,0BAA0B,CAC/D,kBAAkB,EAClB,CAAC,EACD,OAAO,CACR,CAAC;IAEF,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QAC5C,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC5D,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookPage.js

@@ -0,0 +1,47 @@
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
+) {
+    const $ = cheerio.load(html);
+    const titleComponent = $('[data-testid="page.title"]').first();
+    const titleAndDescription = titleComponent.parent().parent().parent().text();
+    const description = titleAndDescription
+        .replace(titleComponent.text(), "")
+        .trim();
+    const title = titleComponent.text().trim();
+    const content = $('[data-testid="page.contentEditor"]').first();
+    // Replace code blocks with parseable html
+    const codeBlocks = content.find('[spellcheck="false"] div');
+    codeBlocks.each((i, c) => {
+        const code = $(c);
+        code.find('[contenteditable="false"]').empty();
+        const codeContent = code
+            .children()
+            .toArray()
+            .map((d) => $(d).text())
+            .filter((text) => text !== "")
+            .join("\n");
+        code.replaceWith(`<pre><code>${codeContent}</code></pre>`);
+    });
+    const contentHtml = $.html(content);
+    const modifyFileName = (fileName) => 
+    // Remove GitBook metadata from the start
+    // The first four %2F split metadata fields. Remaining ones are part of the file name.
+    fileName.split("%2F").slice(4).join("%2F");
+    const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite, modifyFileName);
+    const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line
+    markdown = markdown.replace(/# \n\n/g, "# ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeGitBookPage.js.map

package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,0CAA0C;IAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC5D,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC;QAC/C,MAAM,WAAW,GAAG,IAAI;aACrB,QAAQ,EAAE;aACV,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;aAC7B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,cAAc,WAAW,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,EAAE;IAClC,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookSection.js

@@ -0,0 +1,52 @@
+import cheerio from "cheerio";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
+import getLinksRecursivelyGitBook from "./links-per-group/getLinksRecursivelyGitBook.js";
+import alternateGroupTitle from "./alternateGroupTitle.js";
+import downloadLogoImage from "../downloadLogoImage.js";
+export async function scrapeGitBookSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
+    const $ = cheerio.load(html);
+    // Download the logo
+    const logoSrc = $('a[data-testid="public.headerHomeLink"] img')
+        .first()
+        .attr("src");
+    downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
+    // Get all the navigation sections
+    // Some variants of the GitBook UI show the logo and search base in the side navigation bar,
+    // but the navigation sections are always the last value.
+    const navigationSections = $('div[data-testid="page.desktopTableOfContents"] > nav > div:first-child')
+        .children()
+        .eq(-1)
+        .children()
+        .first()
+        .children();
+    // Get all links per group
+    const groupsConfig = navigationSections
+        .toArray()
+        .map((s) => {
+        const section = $(s);
+        const sectionTitle = $(section)
+            .find('div > div[dir="auto"]')
+            .first()
+            .text();
+        // Only present if the nested navigation is not in a group
+        const firstLink = section.children().eq(0);
+        const firstHref = firstLink.attr("href");
+        const linkSections = section.children().eq(1).children();
+        const pages = getLinksRecursivelyGitBook(linkSections, $);
+        return {
+            group: sectionTitle || alternateGroupTitle(firstLink, pages),
+            pages: firstHref ? [firstHref, ...pages] : pages,
+        };
+    })
+        .filter(Boolean);
+    // Merge groups with empty titles together
+    const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
+    // Scrape each link in the navigation.
+    const groupsConfigCleanPaths = await Promise.all(reducedGroupsConfig.map(async (navEntry) => {
+        return await scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapeGitBookPage, true, version);
+    }));
+    return groupsConfigCleanPaths;
+}
+//# sourceMappingURL=scrapeGitBookSection.js.map

package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,4CAA4C,CAAC;SAC5D,KAAK,EAAE;SACP,IAAI,CAAC,KAAK,CAAC,CAAC;IACf,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,kCAAkC;IAClC,4FAA4F;IAC5F,yDAAyD;IACzD,MAAM,kBAAkB,GAAG,CAAC,CAC1B,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,0BAA0B,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC9D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,EACJ,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMePage.js

@@ -0,0 +1,36 @@
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
+) {
+    const $ = cheerio.load(html);
+    const titleComponent = $("h1").first();
+    const title = titleComponent.text().trim();
+    let description = $(".markdown-body", titleComponent.parent()).text().trim();
+    if (!description) {
+        description = $(".rm-Article > header p").text().trim();
+    }
+    let content = $(".content-body .markdown-body").first();
+    if (content.length === 0) {
+        content = $(".rm-Article > .markdown-body");
+    }
+    // API Pages don't have a markdown body in the same position so there's no HTML
+    let contentHtml = content.html() || "";
+    const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite);
+    const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line and increase their depth by one
+    markdown = markdown.replace(/# \n\n/g, "## ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Remove ReadMe anchor links
+    markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeReadMePage.js.map

package/bin/scraping/site-scrapers/scrapeReadMePage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IAED,+EAA+E;IAC/E,IAAI,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEvC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMeSection.js

@@ -0,0 +1,44 @@
+import cheerio from "cheerio";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import getLinksRecursively from "./links-per-group/getLinksRecursively.js";
+import downloadLogoImage from "../downloadLogoImage.js";
+export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
+    const $ = cheerio.load(html);
+    // Download the logo
+    const logoSrc = $(".rm-Logo-img").first().attr("src");
+    downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
+    // Get all the navigation sections, but only from the first
+    // sidebar found. There are multiple in the HTML for mobile
+    // responsiveness but they all have the same links.
+    const navigationSections = $(".rm-Sidebar")
+        .first()
+        .find(".rm-Sidebar-section");
+    const groupsConfig = navigationSections
+        .toArray()
+        .map((s) => {
+        const section = $(s);
+        const sectionTitle = section.find("h3").first().text();
+        // Get all links, then use filter to remove duplicates.
+        // There are duplicates because of nested navigation, eg:
+        // subgroupTitle -> /first-page
+        // -- First Page -> /first-page   ** DUPLICATE **
+        // -- Second Page -> /second-page
+        const linkSections = section.find(".rm-Sidebar-list").first().children();
+        const pages = getLinksRecursively(linkSections, $).filter((value, index, self) => self.indexOf(value) === index);
+        // Follows the same structure as mint.json
+        return {
+            group: sectionTitle,
+            pages: pages,
+        };
+    });
+    // Scrape each link in the navigation.
+    const groupsConfigCleanPaths = await Promise.all(groupsConfig.map(async (navEntry) => {
+        return await scrapeGettingFileNameFromUrl(
+        // ReadMe requires a directory on all sections whereas we use root.
+        // /docs is their default directory so we remove it
+        navEntry, cliDir, origin, overwrite, scrapeReadMePage, false, version, "/docs");
+    }));
+    return groupsConfigCleanPaths;
+}
+//# sourceMappingURL=scrapeReadMeSection.js.map

package/bin/scraping/site-scrapers/scrapeReadMeSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeReadMeSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMeSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0CAA0C,CAAC;AAC3E,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,2DAA2D;IAC3D,2DAA2D;IAC3D,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC;SACxC,KAAK,EAAE;SACP,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAE/B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAEvD,uDAAuD;QACvD,yDAAyD;QACzD,+BAA+B;QAC/B,iDAAiD;QACjD,iCAAiC;QACjC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,MAAM,CACvD,CAAC,KAAa,EAAE,KAAa,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CACtE,CAAC;QAEF,0CAA0C;QAC1C,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,KAAK;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QACvD,OAAO,MAAM,4BAA4B;QACvC,mEAAmE;QACnE,mDAAmD;QACnD,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,gBAAgB,EAChB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/bin/util.js

@@ -0,0 +1,129 @@
+import { mkdirSync, writeFileSync } from "fs";
+import Ora from "ora";
+import path from "path";
+import shell from "shelljs";
+import stopIfInvalidLink from "./validation/stopIfInvalidLink.js";
+export const MintConfig = (name, color, ctaName, ctaUrl, filename) => {
+    return {
+        name,
+        logo: "",
+        favicon: "",
+        colors: {
+            primary: color,
+        },
+        topbarLinks: [],
+        topbarCtaButton: {
+            name: ctaName,
+            url: ctaUrl,
+        },
+        anchors: [],
+        navigation: [
+            {
+                group: "Home",
+                pages: [filename],
+            },
+        ],
+        // footerSocials: {}, // support object type for footer tyoes
+    };
+};
+export const Page = (title, description, markdown) => {
+    // If we are an empty String we want to add two quotes,
+    // if we added as we went we would detect the first quote
+    // as the closing quote.
+    const startsWithQuote = title.startsWith('"');
+    const endsWithQuote = title.startsWith('"');
+    if (!startsWithQuote) {
+        title = '"' + title;
+    }
+    if (!endsWithQuote) {
+        title = title + '"';
+    }
+    const optionalDescription = description
+        ? `\ndescription: "${description}"`
+        : "";
+    return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
+};
+export function getOrigin(url) {
+    // eg. https://google.com -> https://google.com
+    // https://google.com/page -> https://google.com
+    return new URL(url).origin;
+}
+export function objToReadableString(objs) {
+    // Two spaces as indentation
+    return objs.map((obj) => JSON.stringify(obj, null, 2)).join(",\n");
+}
+export const toFilename = (title) => {
+    // Gets rid of special characters at the start and end
+    // of the name by converting to spaces then using trim.
+    return title
+        .replace(/[^a-z0-9]/gi, " ")
+        .trim()
+        .replace(/ /g, "-")
+        .toLowerCase();
+};
+export const addMdx = (fileName) => {
+    if (fileName.endsWith(".mdx")) {
+        return fileName;
+    }
+    return fileName + ".mdx";
+};
+export const createPage = (title, description, markdown, overwrite = false, rootDir = "", fileName) => {
+    const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
+    // Create the folders needed if they're missing
+    mkdirSync(rootDir, { recursive: true });
+    // Write the page to memory
+    if (overwrite) {
+        writeFileSync(writePath, Page(title, description, markdown));
+        console.log("✏️ - " + writePath);
+    }
+    else {
+        try {
+            writeFileSync(writePath, Page(title, description, markdown), {
+                flag: "wx",
+            });
+            console.log("✏️ - " + writePath);
+        }
+        catch (e) {
+            // We do a try-catch instead of an if-statement to avoid a race condition
+            // of the file being created after we started writing.
+            if (e.code === "EEXIST") {
+                console.log(`❌ Skipping existing file ${writePath}`);
+            }
+            else {
+                console.error(e);
+            }
+        }
+    }
+};
+export function getHrefFromArgs(argv) {
+    const href = argv.url;
+    stopIfInvalidLink(href);
+    return href;
+}
+export const buildLogger = (startText = "") => {
+    const logger = Ora().start(startText);
+    return logger;
+};
+export const getFileExtension = (filename) => {
+    const ext = filename.substring(filename.lastIndexOf(".") + 1, filename.length);
+    if (filename === ext)
+        return undefined;
+    return ext;
+};
+export const fileBelongsInPagesFolder = (filename) => {
+    const extension = getFileExtension(filename);
+    return (extension &&
+        (extension === "mdx" || extension === "md" || extension === "tsx"));
+};
+export const ensureYarn = (logger) => {
+    const yarnInstalled = shell.which("yarn");
+    if (!yarnInstalled) {
+        logger.fail(`yarn must be installed, run
+
+    npm install --global yarn
+
+    `);
+        process.exit(1);
+    }
+};
+//# sourceMappingURL=util.js.map

package/bin/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAClB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,EAAE;IACF,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE;QACpB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;KACrB;IACD,IAAI,CAAC,aAAa,EAAE;QAClB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;KACrB;IAED,MAAM,mBAAmB,GAAG,WAAW;QACrC,CAAC,CAAC,mBAAmB,WAAW,GAAG;QACnC,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAA2B;IAC7D,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,YAAqB,KAAK,EAC1B,UAAkB,EAAE,EACpB,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE;QACb,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;KAClC;SAAM;QACL,IAAI;YACF,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;SAClC;QAAC,OAAO,CAAC,EAAE;YACV,yEAAyE;YACzE,sDAAsD;YACtD,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE;gBACvB,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;aACtD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;SACF;KACF;AACH,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,IAAS;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;IACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,YAAoB,EAAE,EAAE,EAAE;IACpD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAC5B,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAC7B,QAAQ,CAAC,MAAM,CAChB,CAAC;IACF,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,CACL,SAAS;QACT,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CACnE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAW,EAAE,EAAE;IACxC,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,CAAC,IAAI,CAAC;;;;KAIX,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC,CAAC"}

package/bin/validation/isValidLink.js

@@ -0,0 +1,11 @@
+export default function isValidLink(href) {
+    // This checks the link is written correctly, not that the page exists.
+    try {
+        new URL(href);
+        return true;
+    }
+    catch (_) {
+        return false;
+    }
+}
+//# sourceMappingURL=isValidLink.js.map

package/bin/validation/isValidLink.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"isValidLink.js","sourceRoot":"","sources":["../../src/validation/isValidLink.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,WAAW,CAAC,IAAY;IAC9C,uEAAuE;IACvE,IAAI;QACF,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;KACb;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,KAAK,CAAC;KACd;AACH,CAAC"}

package/bin/validation/stopIfInvalidLink.js

@@ -0,0 +1,9 @@
+import isValidLink from "./isValidLink.js";
+export default function stopIfInvalidLink(href) {
+    if (!isValidLink(href)) {
+        console.log("Invalid link: " + href);
+        console.log("Make sure the link starts with http:// or https://");
+        process.exit(1);
+    }
+}
+//# sourceMappingURL=stopIfInvalidLink.js.map

package/bin/validation/stopIfInvalidLink.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopIfInvalidLink.js","sourceRoot":"","sources":["../../src/validation/stopIfInvalidLink.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C,MAAM,CAAC,OAAO,UAAU,iBAAiB,CAAC,IAAY;IACpD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mintlify",
-  "version": "3.0.7",
+  "version": "3.0.8",
   "description": "Alias for @mintlify/cli",
   "engines": {
     "node": ">=18.0.0"
@@ -30,6 +30,6 @@
     "mintlify": "index.js"
   },
   "dependencies": {
-    "@mintlify/cli": "3.0.7"
+    "@mintlify/cli": "3.0.8"
   }
 }