mintlify 3.0.7 → 3.0.8

package/bin/scraping/detectFramework.js

@@ -0,0 +1,39 @@
+import cheerio from "cheerio";
+export var Frameworks;
+(function (Frameworks) {
+    Frameworks["DOCUSAURUS"] = "DOCUSAURUS";
+    Frameworks["GITBOOK"] = "GITBOOK";
+    Frameworks["README"] = "README";
+    Frameworks["INTERCOM"] = "INTERCOM";
+})(Frameworks || (Frameworks = {}));
+export function detectFramework(html) {
+    const $ = cheerio.load(html);
+    const docusaurusMeta = $('meta[name="generator"]');
+    if (docusaurusMeta.length > 0 &&
+        docusaurusMeta.attr("content").includes("Docusaurus")) {
+        if (docusaurusMeta.attr("content").includes("v3")) {
+            return { framework: Frameworks.DOCUSAURUS, version: "3" };
+        }
+        if (docusaurusMeta.attr("content").includes("v2")) {
+            return { framework: Frameworks.DOCUSAURUS, version: "2" };
+        }
+        else if (docusaurusMeta.attr("content").includes("v1")) {
+            console.warn("WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3.");
+            return { framework: Frameworks.DOCUSAURUS, version: "1" };
+        }
+    }
+    const isGitBook = $(".gitbook-root").length > 0;
+    if (isGitBook) {
+        return { framework: Frameworks.GITBOOK };
+    }
+    const isReadMe = $('meta[name="readme-deploy"]').length > 0;
+    if (isReadMe) {
+        return { framework: Frameworks.README };
+    }
+    const isIntercom = $("meta[name='intercom:trackingEvent']").length > 0;
+    if (isIntercom) {
+        return { framework: Frameworks.INTERCOM };
+    }
+    return undefined;
+}
+//# sourceMappingURL=detectFramework.js.map

package/bin/scraping/detectFramework.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detectFramework.js","sourceRoot":"","sources":["../../src/scraping/detectFramework.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,CAAN,IAAY,UAKX;AALD,WAAY,UAAU;IACpB,uCAAyB,CAAA;IACzB,iCAAmB,CAAA;IACnB,+BAAiB,CAAA;IACjB,mCAAqB,CAAA;AACvB,CAAC,EALW,UAAU,KAAV,UAAU,QAKrB;AAED,MAAM,UAAU,eAAe,CAAC,IAAI;IAClC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,cAAc,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAEnD,IACE,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EACrD;QACA,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjD,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;QACD,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACjD,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;aAAM,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACxD,OAAO,CAAC,IAAI,CACV,0FAA0F,CAC3F,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;SAC3D;KACF;IAED,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE;QACb,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;KAC1C;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5D,IAAI,QAAQ,EAAE;QACZ,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;KACzC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,IAAI,UAAU,EAAE;QACd,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;KAC3C;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/bin/scraping/downloadAllImages.js

@@ -0,0 +1,33 @@
+import path from "path";
+import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from "../downloadImage.js";
+// To Do: Use CheerioElement instead of any when we bump the cheerio version
+export default async function downloadAllImages($, content, origin, baseDir, overwrite, modifyFileName, skipValidateImageExtension) {
+    if (!baseDir) {
+        console.debug("Skipping image downloading");
+        return;
+    }
+    // We remove duplicates because some frameworks duplicate img tags
+    // to show the image larger when clicked on.
+    const imageSrcs = [
+        ...new Set(content
+            .find("img[src]")
+            .map((i, image) => $(image).attr("src"))
+            .toArray()),
+    ];
+    // Wait to all images to download before continuing
+    const origToNewArray = await Promise.all(imageSrcs.map(async (imageSrc) => {
+        if (!isValidImageSrc(imageSrc, skipValidateImageExtension)) {
+            return;
+        }
+        const imageHref = cleanImageSrc(imageSrc, origin);
+        let fileName = removeMetadataFromImageSrc(path.basename(imageHref));
+        if (modifyFileName) {
+            fileName = modifyFileName(fileName);
+        }
+        const writePath = path.join(baseDir, fileName);
+        await downloadImage(imageHref, writePath, overwrite);
+        return { [imageSrc]: writePath };
+    }));
+    return origToNewArray.reduce((result, current) => Object.assign(result, current), {});
+}
+//# sourceMappingURL=downloadAllImages.js.map

package/bin/scraping/downloadAllImages.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"downloadAllImages.js","sourceRoot":"","sources":["../../src/scraping/downloadAllImages.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,CAAM,EACN,OAAY,EACZ,MAAc,EACd,OAAe,EACf,SAAkB,EAClB,cAAoB,EACpB,0BAAoC;IAEpC,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO;KACR;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,SAAS,GAAG;QAChB,GAAG,IAAI,GAAG,CACR,OAAO;aACJ,IAAI,CAAC,UAAU,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACvC,OAAO,EAAE,CACb;KACF,CAAC;IAEF,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACvC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,0BAA0B,CAAC,EAAE;YAC1D,OAAO;SACR;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE;YAClB,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;SACrC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE/C,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAErD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,cAAc,CAAC,MAAM,CAC1B,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnD,EAAE,CACH,CAAC;AACJ,CAAC"}

package/bin/scraping/downloadLogoImage.js

@@ -0,0 +1,13 @@
+import path from "path";
+import downloadImage, { cleanImageSrc, isValidImageSrc, removeMetadataFromImageSrc, } from "../downloadImage.js";
+import { getFileExtension } from "../util.js";
+// To Do: Use CheerioElement instead of any when we bump the cheerio version
+export default async function downloadLogoImage(imageSrc, imageBaseDir, origin, overwrite, skipValidateImageExtension) {
+    if (!isValidImageSrc(imageSrc, skipValidateImageExtension))
+        return;
+    const imageHref = cleanImageSrc(imageSrc, origin);
+    const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
+    const imagePath = path.join(imageBaseDir, "logo", "logo-light-mode." + ext);
+    await downloadImage(imageHref, imagePath, overwrite);
+}
+//# sourceMappingURL=downloadLogoImage.js.map

package/bin/scraping/downloadLogoImage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"downloadLogoImage.js","sourceRoot":"","sources":["../../src/scraping/downloadLogoImage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,aAAa,EAAE,EACpB,aAAa,EACb,eAAe,EACf,0BAA0B,GAC3B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,iBAAiB,CAC7C,QAAgB,EAChB,YAAoB,EACpB,MAAc,EACd,SAAkB,EAClB,0BAAoC;IAEpC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,0BAA0B,CAAC;QAAE,OAAO;IAEnE,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAElD,MAAM,GAAG,GAAG,gBAAgB,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAE5E,MAAM,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}

package/bin/scraping/getSitemapLinks.js

@@ -0,0 +1,18 @@
+import axios from "axios";
+// Not in use.
+// Gets all links in a sitemap.
+export const getSitemapLinks = async (url) => {
+    const hostname = url.hostname.replace(".", "\\.");
+    const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");
+    try {
+        const indexData = (await axios.get(url.href)).data;
+        const array = indexData.match(regex);
+        return array || [];
+    }
+    catch (err) {
+        console.error(err);
+        console.log("Skipping sitemap links because we encountered an error.");
+        return [];
+    }
+};
+//# sourceMappingURL=getSitemapLinks.js.map

package/bin/scraping/getSitemapLinks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getSitemapLinks.js","sourceRoot":"","sources":["../../src/scraping/getSitemapLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,cAAc;AACd,+BAA+B;AAC/B,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,GAAQ,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,cAAc,QAAQ,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAExE,IAAI;QACF,MAAM,SAAS,GAAG,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAc,CAAC;QAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAoB,CAAC;QACxD,OAAO,KAAK,IAAI,EAAE,CAAC;KACpB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;QACvE,OAAO,EAAE,CAAC;KACX;AACH,CAAC,CAAC"}

package/bin/scraping/replaceImagePaths.js

@@ -0,0 +1,17 @@
+export default function replaceImagePaths(origToWritePath, cliDir, markdown) {
+    if (origToWritePath == null) {
+        return markdown;
+    }
+    // Change image paths to use the downloaded locations
+    for (const [origHref, writePath] of Object.entries(origToWritePath)) {
+        // Use relative paths within the folder we are in
+        if (writePath.startsWith(cliDir)) {
+            markdown = markdown.replaceAll(origHref, writePath.slice(cliDir.length));
+        }
+        else {
+            markdown = markdown.replaceAll(origHref, writePath);
+        }
+    }
+    return markdown;
+}
+//# sourceMappingURL=replaceImagePaths.js.map

package/bin/scraping/replaceImagePaths.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replaceImagePaths.js","sourceRoot":"","sources":["../../src/scraping/replaceImagePaths.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,eAAuB,EACvB,MAAc,EACd,QAAgB;IAEhB,IAAI,eAAe,IAAI,IAAI,EAAE;QAC3B,OAAO,QAAQ,CAAC;KACjB;IAED,qDAAqD;IACrD,KAAK,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;QACnE,iDAAiD;QACjD,IAAI,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;YAChC,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;SAC1E;aAAM;YACL,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;SACrD;KACF;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/bin/scraping/scrapeFileGettingFileNameFromUrl.js

@@ -0,0 +1,43 @@
+import path from "path";
+import axios from "axios";
+import { getHtmlWithPuppeteer } from "../browser.js";
+import { createPage } from "../util.js";
+export async function scrapeFileGettingFileNameFromUrl(pathname, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, version, baseToRemove) {
+    // Skip scraping external links
+    if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
+        return pathname;
+    }
+    // Removes file name from the end
+    const splitSubpath = pathname.split("/");
+    let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
+    // Remove base dir if passed in
+    if (baseToRemove && folders.startsWith(baseToRemove)) {
+        folders = folders.replace(baseToRemove, "");
+    }
+    // TO DO: Improve this by putting each page's images in a separate
+    // folder named after the title of the page.
+    const imageBaseDir = path.join(cliDir, "images", folders);
+    // Scrape each page separately
+    const href = new URL(pathname, origin).href;
+    let html;
+    if (puppeteer) {
+        html = await getHtmlWithPuppeteer(href);
+    }
+    else {
+        const res = await axios.get(href);
+        html = res.data;
+    }
+    const { title, description, markdown } = await scrapePageFunc(html, origin, cliDir, imageBaseDir, overwrite, version);
+    // Check if page didn't have content
+    if (!title && !markdown) {
+        return undefined;
+    }
+    const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
+    // Default to introduction.mdx if we encountered index.html
+    const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
+    // Will create subfolders as needed
+    createPage(title, description, markdown, overwrite, newFileLocation, fileName);
+    // Removes first slash if we are in a folder, Mintlify doesn't need it
+    return folders ? path.join(folders, fileName).substring(1) : fileName;
+}
+//# sourceMappingURL=scrapeFileGettingFileNameFromUrl.js.map

package/bin/scraping/scrapeFileGettingFileNameFromUrl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeFileGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeFileGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,QAAgB,EAChB,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,+BAA+B;IAC/B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO,QAAQ,CAAC;KACjB;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvE,+BAA+B;IAC/B,IAAI,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QACpD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KAC7C;IAED,kEAAkE;IAClE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC5C,IAAI,IAAY,CAAC;IACjB,IAAI,SAAS,EAAE;QACb,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAC3D,IAAI,EACJ,MAAM,EACN,MAAM,EACN,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IAEF,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;QACvB,OAAO,SAAS,CAAC;KAClB;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEtE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,cAAc,CAAC;IAEzE,mCAAmC;IACnC,UAAU,CACR,KAAK,EACL,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,QAAQ,CACT,CAAC;IAEF,sEAAsE;IACtE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxE,CAAC"}

package/bin/scraping/scrapeGettingFileNameFromUrl.js

@@ -0,0 +1,13 @@
+import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
+export async function scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer = false, version, baseToRemove) {
+    if (typeof navEntry !== "string") {
+        const newPages = [];
+        for (const nestedNavEntry of navEntry.pages) {
+            newPages.push(await scrapeGettingFileNameFromUrl(nestedNavEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer, version, baseToRemove));
+        }
+        navEntry.pages = newPages;
+        return navEntry;
+    }
+    return await scrapeFileGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapePageFunc, puppeteer, version, baseToRemove);
+}
+//# sourceMappingURL=scrapeGettingFileNameFromUrl.js.map

package/bin/scraping/scrapeGettingFileNameFromUrl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeGettingFileNameFromUrl.js","sourceRoot":"","sources":["../../src/scraping/scrapeGettingFileNameFromUrl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AAEzF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,QAA6B,EAC7B,MAAc,EACd,MAAc,EACd,SAAkB,EAClB,cAWE,EACF,SAAS,GAAG,KAAK,EACjB,OAA2B,EAC3B,YAAqB;IAErB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,MAAM,QAAQ,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,cAAc,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC3C,QAAQ,CAAC,IAAI,CACX,MAAM,4BAA4B,CAChC,cAAc,EACd,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CACF,CAAC;SACH;QACD,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;QAC1B,OAAO,QAAQ,CAAC;KACjB;IAED,OAAO,MAAM,gCAAgC,CAC3C,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,cAAc,EACd,SAAS,EACT,OAAO,EACP,YAAY,CACb,CAAC;AACJ,CAAC"}

package/bin/scraping/scrapePage.js

@@ -0,0 +1,10 @@
+import path from "path";
+import { createPage, getOrigin } from "../util.js";
+export async function scrapePage(scrapeFunc, href, html, overwrite, version) {
+    const origin = getOrigin(href);
+    const cwd = process.cwd();
+    const imageBaseDir = path.join(cwd, "images");
+    const { title, description, markdown } = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
+    createPage(title, description, markdown, overwrite, process.cwd());
+}
+//# sourceMappingURL=scrapePage.js.map

package/bin/scraping/scrapePage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapePage.js","sourceRoot":"","sources":["../../src/scraping/scrapePage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAwB,EACxB,IAAY,EACZ,IAAY,EACZ,SAAkB,EAClB,OAA2B;IAE3B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CACvD,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AACrE,CAAC"}

package/bin/scraping/scrapePageCommands.js

@@ -0,0 +1,55 @@
+import axios from "axios";
+import { scrapePage } from "./scrapePage.js";
+import { scrapeDocusaurusPage } from "./site-scrapers/scrapeDocusaurusPage.js";
+import { scrapeGitBookPage } from "./site-scrapers/scrapeGitBookPage.js";
+import { scrapeReadMePage } from "./site-scrapers/scrapeReadMePage.js";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs } from "../util.js";
+import { getHtmlWithPuppeteer } from "../browser.js";
+import { scrapeIntercomPage } from "./site-scrapers/Intercom/scrapeIntercomPage.js";
+function validateFramework(framework) {
+    if (!framework) {
+        console.log("Could not detect the framework automatically. Please use one of:");
+        console.log("scrape-page-docusaurus");
+        console.log("scrape-page-gitbook");
+        console.log("scrape-page-readme");
+        console.log("scrape-page-intercom");
+        return process.exit(1);
+    }
+}
+export async function scrapePageWrapper(argv, scrapeFunc, options) {
+    const href = getHrefFromArgs(argv);
+    let html;
+    if (options?.puppeteer) {
+        html = await getHtmlWithPuppeteer(href);
+    }
+    else {
+        const res = await axios.get(href);
+        html = res.data;
+    }
+    await scrapePage(scrapeFunc, href, html, !!argv.overwrite, options?.version);
+    process.exit(0);
+}
+export async function scrapePageAutomatically(argv) {
+    const href = getHrefFromArgs(argv);
+    const res = await axios.get(href);
+    const html = res.data;
+    const { framework, version } = detectFramework(html);
+    validateFramework(framework);
+    console.log("Detected framework: " + framework);
+    switch (framework) {
+        case Frameworks.DOCUSAURUS:
+            await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
+            break;
+        case Frameworks.GITBOOK:
+            await scrapePageWrapper(argv, scrapeGitBookPage, { puppeteer: true });
+            break;
+        case Frameworks.README:
+            await scrapePageWrapper(argv, scrapeReadMePage);
+            break;
+        case Frameworks.INTERCOM:
+            await scrapePageWrapper(argv, scrapeIntercomPage);
+            break;
+    }
+}
+//# sourceMappingURL=scrapePageCommands.js.map

package/bin/scraping/scrapePageCommands.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapePageCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapePageCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AAEpF,SAAS,iBAAiB,CAAC,SAAS;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAwB,EACxB,UAAwB,EACxB,OAAmD;IAEnD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,IAAY,CAAC;IACjB,IAAI,OAAO,EAAE,SAAS,EAAE;QACtB,IAAI,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;KACzC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;KACjB;IACD,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAS;IACrD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjE,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAChD,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAClD,MAAM;KACT;AACH,CAAC"}

package/bin/scraping/scrapeSection.js

@@ -0,0 +1,12 @@
+import path from "path";
+import { objToReadableString } from "../util.js";
+export async function scrapeSection(scrapeFunc, html, origin, overwrite, version) {
+    console.log(`Started scraping${overwrite ? ", overwrite mode is on" : ""}...`);
+    const cwd = process.cwd();
+    const imageBaseDir = path.join(cwd, "images");
+    const groupsConfig = await scrapeFunc(html, origin, cwd, imageBaseDir, overwrite, version);
+    console.log("Finished scraping.");
+    console.log("Add the following to your navigation in mint.json:");
+    console.log(objToReadableString(groupsConfig));
+}
+//# sourceMappingURL=scrapeSection.js.map

package/bin/scraping/scrapeSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeSection.js","sourceRoot":"","sources":["../../src/scraping/scrapeSection.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA2B,EAC3B,IAAY,EACZ,MAAc,EACd,SAAkB,EAClB,OAA2B;IAE3B,OAAO,CAAC,GAAG,CACT,mBAAmB,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,EAAE,KAAK,CAClE,CAAC;IACF,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,IAAI,EACJ,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,EACT,OAAO,CACR,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC;AACjD,CAAC"}

package/bin/scraping/scrapeSectionCommands.js

@@ -0,0 +1,66 @@
+import axios from "axios";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs, getOrigin } from "../util.js";
+import { scrapeSection } from "./scrapeSection.js";
+import { scrapeDocusaurusSection } from "./site-scrapers/scrapeDocusaurusSection.js";
+import openNestedDocusaurusMenus from "./site-scrapers/openNestedDocusaurusMenus.js";
+import { scrapeGitBookSection } from "./site-scrapers/scrapeGitBookSection.js";
+import openNestedGitbookMenus from "./site-scrapers/openNestedGitbookMenus.js";
+import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
+import { startBrowser } from "../browser.js";
+import { scrapeIntercomSection } from "./site-scrapers/Intercom/scrapeIntercomSection.js";
+export async function scrapeSectionAxiosWrapper(argv, scrapeFunc) {
+    const href = getHrefFromArgs(argv);
+    const res = await axios.get(href);
+    const html = res.data;
+    await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, undefined);
+    process.exit(0);
+}
+export async function scrapeDocusaurusSectionCommand(argv, version // "1" | "2" | "3"
+) {
+    await scrapeSectionOpeningAllNested(argv, openNestedDocusaurusMenus, scrapeDocusaurusSection, version);
+}
+export async function scrapeGitbookSectionCommand(argv) {
+    await scrapeSectionOpeningAllNested(argv, openNestedGitbookMenus, scrapeGitBookSection);
+}
+async function scrapeSectionOpeningAllNested(argv, openLinks, scrapeFunc, version) {
+    const href = getHrefFromArgs(argv);
+    const browser = await startBrowser();
+    const page = await browser.newPage();
+    await page.goto(href, {
+        waitUntil: "networkidle2",
+    });
+    const html = await openLinks(page);
+    browser.close();
+    await scrapeSection(scrapeFunc, html, getOrigin(href), !!argv.overwrite, version);
+    process.exit(0);
+}
+export async function scrapeSectionAutomatically(argv) {
+    const href = getHrefFromArgs(argv);
+    const res = await axios.get(href);
+    const html = res.data;
+    const { framework, version } = detectFramework(html);
+    validateFramework(framework);
+    console.log("Detected framework: " + framework);
+    switch (framework) {
+        case Frameworks.DOCUSAURUS:
+            await scrapeDocusaurusSectionCommand(argv, version);
+            break;
+        case Frameworks.GITBOOK:
+            await scrapeGitbookSectionCommand(argv);
+            break;
+        case Frameworks.README:
+            await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
+            break;
+        case Frameworks.INTERCOM:
+            await scrapeSectionAxiosWrapper(argv, scrapeIntercomSection);
+            break;
+    }
+}
+function validateFramework(framework) {
+    if (!framework) {
+        console.log("Could not detect the framework automatically. We only support Docusaurus (V2 and V3), GitBook, and ReadMe.");
+        process.exit();
+    }
+}
+//# sourceMappingURL=scrapeSectionCommands.js.map

package/bin/scraping/scrapeSectionCommands.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeSectionCommands.js","sourceRoot":"","sources":["../../src/scraping/scrapeSectionCommands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,yBAAyB,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,yCAAyC,CAAC;AAC/E,OAAO,sBAAsB,MAAM,2CAA2C,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,mDAAmD,CAAC;AAE1F,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAwB,EACxB,UAA2B;IAE3B,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,CAAC,CAAC,IAAI,CAAC,SAAS,EAChB,SAAS,CACV,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAS,EACT,OAAe,CAAC,kBAAkB;;IAElC,MAAM,6BAA6B,CACjC,IAAI,EACJ,yBAAyB,EACzB,uBAAuB,EACvB,OAAO,CACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,MAAM,6BAA6B,CACjC,IAAI,EACJ,sBAAsB,EACtB,oBAAoB,CACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,6BAA6B,CAC1C,IAAS,EACT,SAAc,EACd,UAA2B,EAC3B,OAAgB;IAEhB,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACpB,SAAS,EAAE,cAAc;KAC1B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,MAAM,aAAa,CACjB,UAAU,EACV,IAAI,EACJ,SAAS,CAAC,IAAI,CAAC,EACf,CAAC,CAAC,IAAI,CAAC,SAAS,EAChB,OAAO,CACR,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAS;IACxD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAEhD,QAAQ,SAAS,EAAE;QACjB,KAAK,UAAU,CAAC,UAAU;YACxB,MAAM,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM;QACR,KAAK,UAAU,CAAC,OAAO;YACrB,MAAM,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM;QACR,KAAK,UAAU,CAAC,MAAM;YACpB,MAAM,yBAAyB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAC3D,MAAM;QACR,KAAK,UAAU,CAAC,QAAQ;YACtB,MAAM,yBAAyB,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;YAC7D,MAAM;KACT;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiC;IAC1D,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,CAAC,GAAG,CACT,4GAA4G,CAC7G,CAAC;QACF,OAAO,CAAC,IAAI,EAAE,CAAC;KAChB;AACH,CAAC"}

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js

@@ -0,0 +1,27 @@
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../../downloadAllImages.js";
+import replaceImagePaths from "../../replaceImagePaths.js";
+export async function scrapeIntercomPage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
+) {
+    const $ = cheerio.load(html);
+    const titleComponent = $(".t__h1").first();
+    const title = titleComponent.text().trim();
+    let description = $(".article__desc", titleComponent.parent()).text().trim();
+    let content = $("article").first();
+    const contentHtml = $.html(content);
+    const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite, undefined, true);
+    const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
+    let markdown = nhm.translate(contentHtml);
+    // Keep headers on one line
+    markdown = markdown.replace(/# \n\n/g, "# ");
+    // Remove unnecessary nonwidth blank space characters
+    markdown = markdown.replace(/\u200b/g, "");
+    // Reduce unnecessary blank lines
+    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    // Mintlify doesn't support bolded headers, remove the asterisks
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+    return { title, description, markdown };
+}
+//# sourceMappingURL=scrapeIntercomPage.js.map

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeIntercomPage.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,IAAI,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACnC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,SAAS,EACT,IAAI,CACL,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IAEjE,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEhE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js

@@ -0,0 +1,32 @@
+import cheerio from "cheerio";
+import { scrapeIntercomPage } from "./scrapeIntercomPage.js";
+import { scrapeGettingFileNameFromUrl } from "../../scrapeGettingFileNameFromUrl.js";
+import downloadLogoImage from "../../downloadLogoImage.js";
+import axios from "axios";
+export async function scrapeIntercomSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
+    let $ = cheerio.load(html);
+    const logoSrc = $(".header__logo img").first().attr("src");
+    downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
+    const collectionsLink = $(".section .g__space a");
+    const collectionsMap = collectionsLink
+        .toArray()
+        .map(async (s) => {
+        const href = $(s).attr("href");
+        const res = await axios.get(`${origin}${href}`);
+        const html = res.data;
+        $ = cheerio.load(html);
+        const sectionTitle = $(".collection h1").first().text().trim();
+        const sectionPages = $(".section .g__space a")
+            .toArray()
+            .map((s) => $(s).attr("href"));
+        return {
+            group: sectionTitle,
+            pages: sectionPages,
+        };
+    });
+    const collections = await Promise.all(collectionsMap);
+    return await Promise.all(collections.map(async (entry) => {
+        return await scrapeGettingFileNameFromUrl(entry, cliDir, origin, overwrite, scrapeIntercomPage, false, version);
+    }));
+}
+//# sourceMappingURL=scrapeIntercomSection.js.map

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeIntercomSection.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,uCAAuC,CAAC;AACrF,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3B,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3D,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,eAAe,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,eAAe;SACnC,OAAO,EAAE;SACT,GAAG,CAAC,KAAK,EAAE,CAAkB,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,YAAY,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,YAAY,GAAG,CAAC,CAAC,sBAAsB,CAAC;aAC3C,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAClD,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,MAAM,WAAW,GAAqB,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAExE,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,KAA0B,EAAE,EAAE;QACnD,OAAO,MAAM,4BAA4B,CACvC,KAAK,EACL,MAAM,EACN,MAAM,EACN,SAAS,EACT,kBAAkB,EAClB,KAAK,EACL,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/bin/scraping/site-scrapers/alternateGroupTitle.js

@@ -0,0 +1,9 @@
+export default function alternateGroupTitle(firstLink, pages) {
+    // Only assign titles to nested navigation menus outside a section.
+    // Others should not have a title so we can merge them into one section.
+    if (pages.length > 0) {
+        return firstLink?.text();
+    }
+    return "";
+}
+//# sourceMappingURL=alternateGroupTitle.js.map

package/bin/scraping/site-scrapers/alternateGroupTitle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"alternateGroupTitle.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/alternateGroupTitle.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,SAAS,EAAE,KAAK;IAC1D,mEAAmE;IACnE,wEAAwE;IACxE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;QACpB,OAAO,SAAS,EAAE,IAAI,EAAE,CAAC;KAC1B;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js

@@ -0,0 +1,34 @@
+import alternateGroupTitle from "../alternateGroupTitle.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+export function getDocusaurusLinksPerGroup(navigationSections, $, version) {
+    if (version === "3" || version === "2") {
+        return getDocusaurusLinksPerGroupLoop(navigationSections, $);
+    }
+    return [];
+}
+function getDocusaurusLinksPerGroupLoop(navigationSections, $) {
+    return navigationSections.toArray().map((s) => {
+        const section = $(s);
+        // Links without a group
+        if (section.hasClass("theme-doc-sidebar-item-link")) {
+            const linkHref = section.find("a[href]").first().attr("href");
+            return {
+                group: "",
+                pages: [linkHref],
+            };
+        }
+        const firstLink = section
+            .find(".menu__list-item-collapsible")
+            .first()
+            .find("a[href]");
+        const sectionTitle = firstLink.text();
+        const firstHref = firstLink.attr("href");
+        const linkSections = section.children().eq(1).children();
+        const pages = getLinksRecursively(linkSections, $);
+        return {
+            group: sectionTitle || alternateGroupTitle(firstLink, pages),
+            pages: firstHref ? [firstHref, ...pages] : pages,
+        };
+    });
+}
+//# sourceMappingURL=getDocusaurusLinksPerGroup.js.map

package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getDocusaurusLinksPerGroup.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,MAAM,2BAA2B,CAAC;AAC5D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,UAAU,0BAA0B,CACxC,kBAAuB,EACvB,CAAM,EACN,OAA2B;IAE3B,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,GAAG,EAAE;QACtC,OAAO,8BAA8B,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;KAC9D;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,8BAA8B,CAAC,kBAAuB,EAAE,CAAM;IACrE,OAAO,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE;QACpD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,wBAAwB;QACxB,IAAI,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,SAAS,GAAG,OAAO;aACtB,IAAI,CAAC,8BAA8B,CAAC;aACpC,KAAK,EAAE;aACP,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnB,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js

@@ -0,0 +1,38 @@
+// Used by Docusaurus and ReadMe section scrapers
+export default function getLinksRecursively(linkSections, $) {
+    if (linkSections == null || linkSections.length === 0) {
+        return [];
+    }
+    return linkSections
+        .map((i, s) => {
+        const subsection = $(s);
+        let link = subsection.children().first();
+        if (!link.attr("href")) {
+            // Docusaurus nests the <a> inside a <div>
+            link = link.find("a[href]").first();
+        }
+        const linkHref = link.attr("href");
+        // Skip missing links. For example, GitBook uses
+        // empty divs are used for styling a line beside the nav.
+        // Skip external links until Mintlify supports them
+        if (!linkHref ||
+            linkHref === "#" ||
+            linkHref.startsWith("https://") ||
+            linkHref.startsWith("http://")) {
+            return undefined;
+        }
+        const childLinks = subsection.children().eq(1).children();
+        if (childLinks.length > 0) {
+            // Put the section link in the list of pages.
+            // When we support the section itself being a link we should update this
+            return {
+                group: link.text(),
+                pages: [linkHref, ...getLinksRecursively(childLinks, $)],
+            };
+        }
+        return linkHref;
+    })
+        .toArray()
+        .filter(Boolean);
+}
+//# sourceMappingURL=getLinksRecursively.js.map

package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getLinksRecursively.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursively.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,YAAiB,EAAE,CAAM;IACnE,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QACrD,OAAO,EAAE,CAAC;KACX;IAED,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAEzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YACtB,0CAA0C;YAC1C,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;SACrC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,KAAK,GAAG;YAChB,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;YACA,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aACzD,CAAC;SACH;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}

package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js

@@ -0,0 +1,38 @@
+// Used by GitBook section scraper
+export default function getLinksRecursivelyGitBook(linkSections, $) {
+    if (linkSections == null || linkSections.length === 0) {
+        return [];
+    }
+    return linkSections
+        .map((i, s) => {
+        let subsection = $(s);
+        // GitBook has an extra div when more than one layer deep
+        if (subsection.children().length === 1) {
+            subsection = subsection.children().first();
+        }
+        const link = subsection.children().first();
+        const linkHref = link.attr("href");
+        // Skip missing links. For example, GitBook uses
+        // empty divs are used for styling a line beside the nav.
+        // Skip external links until Mintlify supports them
+        if (!linkHref ||
+            linkHref === "#" ||
+            linkHref.startsWith("https://") ||
+            linkHref.startsWith("http://")) {
+            return undefined;
+        }
+        const childLinks = subsection.children().eq(1).children();
+        if (childLinks.length > 0) {
+            // Put the section link in the list of pages.
+            // When we support the section itself being a link we should update this
+            return {
+                group: link.text(),
+                pages: [linkHref, ...getLinksRecursivelyGitBook(childLinks, $)],
+            };
+        }
+        return linkHref;
+    })
+        .toArray()
+        .filter(Boolean);
+}
+//# sourceMappingURL=getLinksRecursivelyGitBook.js.map

package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getLinksRecursivelyGitBook.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,MAAM,CAAC,OAAO,UAAU,0BAA0B,CAAC,YAAiB,EAAE,CAAM;IAC1E,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QACrD,OAAO,EAAE,CAAC;KACX;IAED,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtB,yDAAyD;QACzD,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE;YACtC,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;SAC5C;QAED,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnC,gDAAgD;QAChD,yDAAyD;QACzD,mDAAmD;QACnD,IACE,CAAC,QAAQ;YACT,QAAQ,KAAK,GAAG;YAChB,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAC9B;YACA,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,6CAA6C;YAC7C,wEAAwE;YACxE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,0BAA0B,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;aAChE,CAAC;SACH;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;SACD,OAAO,EAAE;SACT,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}