@mintlify/scraping 3.0.14 → 3.0.16

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js

@@ -1,26 +1,26 @@
-import cheerio from "cheerio";
-import { NodeHtmlMarkdown } from "node-html-markdown";
-import downloadAllImages from "../../downloadAllImages.js";
-import replaceImagePaths from "../../replaceImagePaths.js";
+import cheerio from 'cheerio';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+import downloadAllImages from '../../downloadAllImages.js';
+import replaceImagePaths from '../../replaceImagePaths.js';
 export async function scrapeIntercomPage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
 ) {
     const $ = cheerio.load(html);
-    const titleComponent = $(".t__h1").first();
+    const titleComponent = $('.t__h1').first();
     const title = titleComponent.text().trim();
-    const description = $(".article__desc", titleComponent.parent()).text().trim();
-    const content = $("article").first();
+    const description = $('.article__desc', titleComponent.parent()).text().trim();
+    const content = $('article').first();
     const contentHtml = $.html(content);
     const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite, undefined);
     const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
     let markdown = nhm.translate(contentHtml);
     // Keep headers on one line
-    markdown = markdown.replace(/# \n\n/g, "# ");
+    markdown = markdown.replace(/# \n\n/g, '# ');
     // Remove unnecessary nonwidth blank space characters
-    markdown = markdown.replace(/\u200b/g, "");
+    markdown = markdown.replace(/\u200b/g, '');
     // Reduce unnecessary blank lines
-    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    markdown = markdown.replace(/\n\n\n/g, '\n\n');
     // Mintlify doesn't support bolded headers, remove the asterisks
-    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
     if (origToWritePath) {
         markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
     }

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomPage.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeIntercomPage.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE/E,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"scrapeIntercomPage.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE/E,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js

@@ -1,24 +1,22 @@
-import cheerio from "cheerio";
-import { scrapeIntercomPage } from "./scrapeIntercomPage.js";
-import { scrapeGettingFileNameFromUrl } from "../../scrapeGettingFileNameFromUrl.js";
-import downloadLogoImage from "../../downloadLogoImage.js";
-import axios from "axios";
+import axios from 'axios';
+import cheerio from 'cheerio';
+import downloadLogoImage from '../../downloadLogoImage.js';
+import { scrapeGettingFileNameFromUrl } from '../../scrapeGettingFileNameFromUrl.js';
+import { scrapeIntercomPage } from './scrapeIntercomPage.js';
 export async function scrapeIntercomSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
     let $ = cheerio.load(html);
-    const logoSrc = $(".header__logo img").first().attr("src");
+    const logoSrc = $('.header__logo img').first().attr('src');
     downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
-    const collectionsLink = $(".section .g__space a");
-    const collectionsMap = collectionsLink
-        .toArray()
-        .map(async (s) => {
-        const href = $(s).attr("href");
+    const collectionsLink = $('.section .g__space a');
+    const collectionsMap = collectionsLink.toArray().map(async (s) => {
+        const href = $(s).attr('href');
         const res = await axios.get(`${origin}${href}`);
         const html = res.data;
         $ = cheerio.load(html);
-        const sectionTitle = $(".collection h1").first().text().trim();
-        const sectionPages = $(".section .g__space a")
+        const sectionTitle = $('.collection h1').first().text().trim();
+        const sectionPages = $('.section .g__space a')
             .toArray()
-            .map((s) => $(s).attr("href"))
+            .map((s) => $(s).attr('href'))
             .filter((page) => page !== undefined);
         return {
             group: sectionTitle,

package/bin/scraping/site-scrapers/Intercom/scrapeIntercomSection.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeIntercomSection.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,uCAAuC,CAAC;AACrF,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3B,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3D,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,eAAe,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,eAAe;SACnC,OAAO,EAAE;SACT,GAAG,CAAC,KAAK,EAAE,CAAkB,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,YAAY,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,YAAY,GAAG,CAAC,CAAC,sBAAsB,CAAC;aAC3C,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAC9C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAa,CAAC;QACpD,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,MAAM,WAAW,GAAqB,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAExE,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,KAA0B,EAAE,EAAE;QACnD,OAAO,MAAM,4BAA4B,CACvC,KAAK,EACL,MAAM,EACN,MAAM,EACN,SAAS,EACT,kBAAkB,EAClB,KAAK,EACL,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"scrapeIntercomSection.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,iBAAiB,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,uCAAuC,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3B,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3D,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,eAAe,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAkB,EAAE,EAAE;QAChF,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,YAAY,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,YAAY,GAAG,CAAC,CAAC,sBAAsB,CAAC;aAC3C,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAC9C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAa,CAAC;QACpD,OAAO;YACL,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAqB,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAExE,OAAO,MAAM,OAAO,CAAC,GAAG,CACtB,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,KAA0B,EAAE,EAAE;QACnD,OAAO,MAAM,4BAA4B,CACvC,KAAK,EACL,MAAM,EACN,MAAM,EACN,SAAS,EACT,kBAAkB,EAClB,KAAK,EACL,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/bin/scraping/site-scrapers/alternateGroupTitle.js

@@ -4,6 +4,6 @@ export default function alternateGroupTitle(firstLink, pages) {
     if (pages.length > 0) {
         return firstLink?.text();
     }
-    return "";
+    return '';
 }
 //# sourceMappingURL=alternateGroupTitle.js.map

package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js

@@ -1,7 +1,7 @@
-import alternateGroupTitle from "../alternateGroupTitle.js";
-import getLinksRecursively from "./getLinksRecursively.js";
+import alternateGroupTitle from '../alternateGroupTitle.js';
+import getLinksRecursively from './getLinksRecursively.js';
 export function getDocusaurusLinksPerGroup(navigationSections, $, version) {
-    if (version === "3" || version === "2") {
+    if (version === '3' || version === '2') {
         return getDocusaurusLinksPerGroupLoop(navigationSections, $);
     }
     return [];
@@ -10,19 +10,16 @@ function getDocusaurusLinksPerGroupLoop(navigationSections, $) {
     return navigationSections.toArray().map((s) => {
         const section = $(s);
         // Links without a group
-        if (section.hasClass("theme-doc-sidebar-item-link")) {
-            const linkHref = section.find("a[href]").first().attr("href");
+        if (section.hasClass('theme-doc-sidebar-item-link')) {
+            const linkHref = section.find('a[href]').first().attr('href');
             return {
-                group: "",
+                group: '',
                 pages: [linkHref],
             };
         }
-        const firstLink = section
-            .find(".menu__list-item-collapsible")
-            .first()
-            .find("a[href]");
+        const firstLink = section.find('.menu__list-item-collapsible').first().find('a[href]');
         const sectionTitle = firstLink.text();
-        const firstHref = firstLink.attr("href");
+        const firstHref = firstLink.attr('href');
         const linkSections = section.children().eq(1).children();
         const pages = getLinksRecursively(linkSections, $);
         return {

package/bin/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.js.map

@@ -1 +1 @@
-{"version":3,"file":"getDocusaurusLinksPerGroup.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,MAAM,2BAA2B,CAAC;AAC5D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,UAAU,0BAA0B,CACxC,kBAAuB,EACvB,CAAM,EACN,OAA2B;IAE3B,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,GAAG,EAAE;QACtC,OAAO,8BAA8B,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;KAC9D;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,8BAA8B,CAAC,kBAAuB,EAAE,CAAM;IACrE,OAAO,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE;QACpD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,wBAAwB;QACxB,IAAI,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,SAAS,GAAG,OAAO;aACtB,IAAI,CAAC,8BAA8B,CAAC;aACpC,KAAK,EAAE;aACP,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnB,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"getDocusaurusLinksPerGroup.js","sourceRoot":"","sources":["../../../../src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,MAAM,2BAA2B,CAAC;AAC5D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAE3D,MAAM,UAAU,0BAA0B,CACxC,kBAAuB,EACvB,CAAM,EACN,OAA2B;IAE3B,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,GAAG,EAAE;QACtC,OAAO,8BAA8B,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;KAC9D;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,8BAA8B,CAAC,kBAAuB,EAAE,CAAM;IACrE,OAAO,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE;QACpD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,wBAAwB;QACxB,IAAI,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE;YACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB,CAAC;SACH;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEvF,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAEzD,MAAM,KAAK,GAAG,mBAAmB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAEnD,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/bin/scraping/site-scrapers/links-per-group/getLinksRecursively.js

@@ -7,18 +7,18 @@ export default function getLinksRecursively(linkSections, $) {
         .map((i, s) => {
         const subsection = $(s);
         let link = subsection.children().first();
-        if (!link.attr("href")) {
+        if (!link.attr('href')) {
             // Docusaurus nests the <a> inside a <div>
-            link = link.find("a[href]").first();
+            link = link.find('a[href]').first();
         }
-        const linkHref = link.attr("href");
+        const linkHref = link.attr('href');
         // Skip missing links. For example, GitBook uses
         // empty divs are used for styling a line beside the nav.
         // Skip external links until Mintlify supports them
         if (!linkHref ||
-            linkHref === "#" ||
-            linkHref.startsWith("https://") ||
-            linkHref.startsWith("http://")) {
+            linkHref === '#' ||
+            linkHref.startsWith('https://') ||
+            linkHref.startsWith('http://')) {
             return undefined;
         }
         const childLinks = subsection.children().eq(1).children();

package/bin/scraping/site-scrapers/links-per-group/getLinksRecursivelyGitBook.js

@@ -11,14 +11,14 @@ export default function getLinksRecursivelyGitBook(linkSections, $) {
             subsection = subsection.children().first();
         }
         const link = subsection.children().first();
-        const linkHref = link.attr("href");
+        const linkHref = link.attr('href');
         // Skip missing links. For example, GitBook uses
         // empty divs are used for styling a line beside the nav.
         // Skip external links until Mintlify supports them
         if (!linkHref ||
-            linkHref === "#" ||
-            linkHref.startsWith("https://") ||
-            linkHref.startsWith("http://")) {
+            linkHref === '#' ||
+            linkHref.startsWith('https://') ||
+            linkHref.startsWith('http://')) {
             return undefined;
         }
         const childLinks = subsection.children().eq(1).children();

package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.d.ts

@@ -1,2 +1,2 @@
-import { Page } from "puppeteer";
+import { Page } from 'puppeteer';
 export default function openNestedDocusaurusMenus(page: Page): Promise<string>;

package/bin/scraping/site-scrapers/openNestedDocusaurusMenus.js

@@ -1,16 +1,16 @@
 export default async function openNestedDocusaurusMenus(page) {
     let prevEncountered = [];
-    let encounteredHref = ["fake-href-to-make-loop-run-at-least-once"];
+    let encounteredHref = ['fake-href-to-make-loop-run-at-least-once'];
     // Loop until we've encountered every link
     while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
         prevEncountered = encounteredHref;
         encounteredHref = await page.evaluate((encounteredHref) => {
-            const collapsible = Array.from(document.querySelectorAll(".menu__link.menu__link--sublist"));
+            const collapsible = Array.from(document.querySelectorAll('.menu__link.menu__link--sublist'));
             const linksFound = [];
             collapsible.forEach(async (collapsibleItem) => {
-                const href = collapsibleItem?.getAttribute("href");
+                const href = collapsibleItem?.getAttribute('href');
                 // Should never occur but we keep it as a fail-safe
-                if (href?.startsWith("https://") || href?.startsWith("http://")) {
+                if (href?.startsWith('https://') || href?.startsWith('http://')) {
                     return;
                 }
                 // Click any links we haven't seen before

package/bin/scraping/site-scrapers/openNestedGitbookMenus.d.ts

@@ -1,2 +1,2 @@
-import { Page } from "puppeteer";
+import { Page } from 'puppeteer';
 export default function openNestedGitbookMenus(page: Page): Promise<string>;

package/bin/scraping/site-scrapers/openNestedGitbookMenus.js.map

@@ -1 +1 @@
-{"version":3,"file":"openNestedGitbookMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedGitbookMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAU;IAC7D,IAAI,UAAU,GAAG,IAAI,CAAC;IAEtB,iDAAiD;IACjD,OAAO,UAAU,EAAE;QACjB,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACpC,yDAAyD;YACzD,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CACrC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CACrD,CAAC;YAEF,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAiB,EAAE,EAAE;gBACxC,MAAM,OAAO,GAAG,IAAI,EAAE,aAAa,EAAE,aAAa,CAAC;gBACnD,IAAI,OAAO,EAAE;oBACX,OAAO,CAAC,KAAK,EAAE,CAAC;iBACjB;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;KACJ;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}
+{"version":3,"file":"openNestedGitbookMenus.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/openNestedGitbookMenus.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAU;IAC7D,IAAI,UAAU,GAAG,IAAI,CAAC;IAEtB,iDAAiD;IACjD,OAAO,UAAU,EAAE;QACjB,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACpC,yDAAyD;YACzD,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,CAAC;YAE9F,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAiB,EAAE,EAAE;gBACxC,MAAM,OAAO,GAAG,IAAI,EAAE,aAAa,EAAE,aAAa,CAAC;gBACnD,IAAI,OAAO,EAAE;oBACX,OAAO,CAAC,KAAK,EAAE,CAAC;iBACjB;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;KACJ;IAED,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AAC9B,CAAC"}

package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js

@@ -1,29 +1,29 @@
-import cheerio from "cheerio";
-import { NodeHtmlMarkdown } from "node-html-markdown";
-import downloadAllImages from "../downloadAllImages.js";
-import replaceImagePaths from "../replaceImagePaths.js";
+import cheerio from 'cheerio';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+import downloadAllImages from '../downloadAllImages.js';
+import replaceImagePaths from '../replaceImagePaths.js';
 export async function scrapeDocusaurusPage(html, origin, cliDir, imageBaseDir, overwrite, version // expects "2", or "3". Have not written support for "1" yet
 ) {
     const $ = cheerio.load(html);
-    const article = version === "3" ? $(".theme-doc-markdown").first() : $("article").first();
+    const article = version === '3' ? $('.theme-doc-markdown').first() : $('article').first();
     if (article.length === 0) {
         // Index pages with no additional text don't have the markdown class
         return {
-            title: ''
+            title: '',
         };
     }
-    const titleComponent = article.find("h1");
+    const titleComponent = article.find('h1');
     const title = titleComponent.text().trim();
     // Do not include title in the content when we insert it in our metadata
     titleComponent.remove();
-    const markdownContent = version === "3" ? article : article.find(".markdown").first();
+    const markdownContent = version === '3' ? article : article.find('.markdown').first();
     const origToWritePath = await downloadAllImages($, markdownContent, origin, imageBaseDir, overwrite);
     const markdownHtml = markdownContent.html();
     const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
     let markdown = markdownHtml ? nhm.translate(markdownHtml) : null;
     if (markdown == null) {
-        console.error("We do not support scraping this page. Content will be empty");
-        return { title, description: undefined, markdown: "" };
+        console.error('We do not support scraping this page. Content will be empty');
+        return { title, description: undefined, markdown: '' };
     }
     // Description only exists in meta tags. The code is commented out because its prone to incorrectly
     // including a description if the first line of text had markdown annotations like `.
@@ -38,13 +38,13 @@ export async function scrapeDocusaurusPage(html, origin, cliDir, imageBaseDir, o
     // When we parse their HTML the parser adds things like:
     // [](#setup "Direct link to heading")
     // to the end of each header.
-    markdown = markdown.replace(/\[\]\(#.+ ".+"\)\n/g, "\n");
+    markdown = markdown.replace(/\[\]\(#.+ ".+"\)\n/g, '\n');
     // Remove unnecessary nonwidth blank space characters
-    markdown = markdown.replace(/\u200b/g, "");
+    markdown = markdown.replace(/\u200b/g, '');
     // Reduce unnecessary blank lines
-    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    markdown = markdown.replace(/\n\n\n/g, '\n\n');
     // Mintlify doesn't support bolded headers, remove the asterisks
-    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
     if (origToWritePath) {
         markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
     }

package/bin/scraping/site-scrapers/scrapeDocusaurusPage.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeDocusaurusPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B,CAAC,4DAA4D;;IAMxF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,OAAO,GACX,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IAE5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,oEAAoE;QACpE,OAAO;YACL,KAAK,EAAE,EAAE;SACV,CAAC;KACH;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,wEAAwE;IACxE,cAAc,CAAC,MAAM,EAAE,CAAC;IAExB,MAAM,eAAe,GACnB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,eAAe,EACf,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;IAE5C,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEjE,IAAI,QAAQ,IAAI,IAAI,EAAE;QACpB,OAAO,CAAC,KAAK,CACX,6DAA6D,CAC9D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;KACxD;IAED,mGAAmG;IACnG,qFAAqF;IACrF,yFAAyF;IACzF,kFAAkF;IAClF,MAAM,WAAW,GAAG,SAAS,CAAC;IAC9B,0EAA0E;IAC1E,0CAA0C;IAC1C,wBAAwB;IACxB,IAAI;IAEJ,uCAAuC;IACvC,wDAAwD;IACxD,sCAAsC;IACtC,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAEzD,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"scrapeDocusaurusPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B,CAAC,4DAA4D;;IAMxF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,OAAO,GAAG,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IAE1F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,oEAAoE;QACpE,OAAO;YACL,KAAK,EAAE,EAAE;SACV,CAAC;KACH;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,wEAAwE;IACxE,cAAc,CAAC,MAAM,EAAE,CAAC;IAExB,MAAM,eAAe,GAAG,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;IAEtF,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,eAAe,EACf,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;IAE5C,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEjE,IAAI,QAAQ,IAAI,IAAI,EAAE;QACpB,OAAO,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC7E,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;KACxD;IAED,mGAAmG;IACnG,qFAAqF;IACrF,yFAAyF;IACzF,kFAAkF;IAClF,MAAM,WAAW,GAAG,SAAS,CAAC;IAC9B,0EAA0E;IAC1E,0CAA0C;IAC1C,wBAAwB;IACxB,IAAI;IAEJ,uCAAuC;IACvC,wDAAwD;IACxD,sCAAsC;IACtC,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAEzD,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js

@@ -1,16 +1,16 @@
-import cheerio from "cheerio";
-import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
-import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
-import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
-import { getDocusaurusLinksPerGroup } from "./links-per-group/getDocusaurusLinksPerGroup.js";
-import downloadLogoImage from "../downloadLogoImage.js";
+import cheerio from 'cheerio';
+import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
+import downloadLogoImage from '../downloadLogoImage.js';
+import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
+import { getDocusaurusLinksPerGroup } from './links-per-group/getDocusaurusLinksPerGroup.js';
+import { scrapeDocusaurusPage } from './scrapeDocusaurusPage.js';
 export async function scrapeDocusaurusSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
     const $ = cheerio.load(html);
     // Download the logo
-    const logoSrc = $(".navbar__logo img").attr("src");
+    const logoSrc = $('.navbar__logo img').attr('src');
     downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite);
     // Get all the navigation sections
-    const navigationSections = $(".theme-doc-sidebar-menu").first().children();
+    const navigationSections = $('.theme-doc-sidebar-menu').first().children();
     // Get all links per group
     const groupsConfig = getDocusaurusLinksPerGroup(navigationSections, $, version);
     // Merge groups with empty titles together
@@ -20,7 +20,7 @@ export async function scrapeDocusaurusSection(html, origin, cliDir, imageBaseDir
         groupConfig.pages = (await Promise.all(groupConfig.pages.map(async (navEntry) => 
         // Docusaurus requires a directory on all sections wheras we use root.
         // /docs is their default directory so we remove it
-        scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapeDocusaurusPage, false, version, "/docs"))))
+        scrapeGettingFileNameFromUrl(navEntry, cliDir, origin, overwrite, scrapeDocusaurusPage, false, version, '/docs'))))
             // Remove skipped index pages (they return undefined from the above function)
             .filter(Boolean);
         return groupConfig;

package/bin/scraping/site-scrapers/scrapeDocusaurusSection.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iDAAiD,CAAC;AAC7F,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAAgB;IAEhB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,0BAA0B,CAC/D,kBAAkB,EAClB,CAAC,EACD,OAAO,CACR,CAAC;IAEF,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QAC5C,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC5D,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}
+{"version":3,"file":"scrapeDocusaurusSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeDocusaurusSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,0BAA0B,EAAE,MAAM,iDAAiD,CAAC;AAC7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAAgB;IAEhB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE5D,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE3E,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,0BAA0B,CAAC,kBAAkB,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAElG,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,MAAM,sBAAsB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9C,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QAC5C,WAAW,CAAC,KAAK,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC5D,sEAAsE;QACtE,mDAAmD;QACnD,4BAA4B,CAC1B,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,KAAK,EACL,OAAO,EACP,OAAO,CACR,CACF,CACF,CACF;YACC,6EAA6E;aAC5E,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,sBAAsB,CAAC;AAChC,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookPage.js

@@ -1,15 +1,13 @@
-import cheerio from "cheerio";
-import { NodeHtmlMarkdown } from "node-html-markdown";
-import downloadAllImages from "../downloadAllImages.js";
-import replaceImagePaths from "../replaceImagePaths.js";
+import cheerio from 'cheerio';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+import downloadAllImages from '../downloadAllImages.js';
+import replaceImagePaths from '../replaceImagePaths.js';
 export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
 ) {
     const $ = cheerio.load(html);
     const titleComponent = $('[data-testid="page.title"]').first();
     const titleAndDescription = titleComponent.parent().parent().parent().text();
-    const description = titleAndDescription
-        .replace(titleComponent.text(), "")
-        .trim();
+    const description = titleAndDescription.replace(titleComponent.text(), '').trim();
     const title = titleComponent.text().trim();
     const content = $('[data-testid="page.contentEditor"]').first();
     // Replace code blocks with parseable html
@@ -21,26 +19,26 @@ export async function scrapeGitBookPage(html, origin, cliDir, imageBaseDir, over
             .children()
             .toArray()
             .map((d) => $(d).text())
-            .filter((text) => text !== "")
-            .join("\n");
+            .filter((text) => text !== '')
+            .join('\n');
         code.replaceWith(`<pre><code>${codeContent}</code></pre>`);
     });
     const contentHtml = $.html(content);
     const modifyFileName = (fileName) => 
     // Remove GitBook metadata from the start
     // The first four %2F split metadata fields. Remaining ones are part of the file name.
-    fileName.split("%2F").slice(4).join("%2F");
+    fileName.split('%2F').slice(4).join('%2F');
     const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite, modifyFileName);
     const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
     let markdown = nhm.translate(contentHtml);
     // Keep headers on one line
-    markdown = markdown.replace(/# \n\n/g, "# ");
+    markdown = markdown.replace(/# \n\n/g, '# ');
     // Remove unnecessary nonwidth blank space characters
-    markdown = markdown.replace(/\u200b/g, "");
+    markdown = markdown.replace(/\u200b/g, '');
     // Reduce unnecessary blank lines
-    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    markdown = markdown.replace(/\n\n\n/g, '\n\n');
     // Mintlify doesn't support bolded headers, remove the asterisks
-    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
     if (origToWritePath) {
         markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
     }

package/bin/scraping/site-scrapers/scrapeGitBookPage.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB;SACpC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;SAClC,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,0CAA0C;IAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC5D,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC;QAC/C,MAAM,WAAW,GAAG,IAAI;aACrB,QAAQ,EAAE;aACV,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;aAC7B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,cAAc,WAAW,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC1C,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"scrapeGitBookPage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookPage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,4BAA4B,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;IAE7E,MAAM,WAAW,GAAG,mBAAmB,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3C,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;IAEhE,0CAA0C;IAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC5D,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC;QAC/C,MAAM,WAAW,GAAG,IAAI;aACrB,QAAQ,EAAE;aACV,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;aAC7B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,cAAc,WAAW,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpC,MAAM,cAAc,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC1C,yCAAyC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE7C,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,EACT,cAAc,CACf,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2BAA2B;IAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeGitBookSection.js

@@ -1,16 +1,14 @@
-import cheerio from "cheerio";
-import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
-import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
-import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
-import getLinksRecursivelyGitBook from "./links-per-group/getLinksRecursivelyGitBook.js";
-import alternateGroupTitle from "./alternateGroupTitle.js";
-import downloadLogoImage from "../downloadLogoImage.js";
+import cheerio from 'cheerio';
+import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
+import downloadLogoImage from '../downloadLogoImage.js';
+import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
+import alternateGroupTitle from './alternateGroupTitle.js';
+import getLinksRecursivelyGitBook from './links-per-group/getLinksRecursivelyGitBook.js';
+import { scrapeGitBookPage } from './scrapeGitBookPage.js';
 export async function scrapeGitBookSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
     const $ = cheerio.load(html);
     // Download the logo
-    const logoSrc = $('a[data-testid="public.headerHomeLink"] img')
-        .first()
-        .attr("src");
+    const logoSrc = $('a[data-testid="public.headerHomeLink"] img').first().attr('src');
     downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite).catch(console.error);
     // Get all the navigation sections
     // Some variants of the GitBook UI show the logo and search base in the side navigation bar,
@@ -26,13 +24,10 @@ export async function scrapeGitBookSection(html, origin, cliDir, imageBaseDir, o
         .toArray()
         .map((s) => {
         const section = $(s);
-        const sectionTitle = $(section)
-            .find('div > div[dir="auto"]')
-            .first()
-            .text();
+        const sectionTitle = $(section).find('div > div[dir="auto"]').first().text();
         // Only present if the nested navigation is not in a group
         const firstLink = section.children().eq(0);
-        const firstHref = firstLink.attr("href");
+        const firstHref = firstLink.attr('href');
         const linkSections = section.children().eq(1).children();
         const pages = getLinksRecursivelyGitBook(linkSections, $);
         return {

package/bin/scraping/site-scrapers/scrapeGitBookSection.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,4CAA4C,CAAC;SAC5D,KAAK,EAAE;SACP,IAAI,CAAC,KAAK,CAAC,CAAC;IACf,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAEjF,kCAAkC;IAClC,4FAA4F;IAC5F,yDAAyD;IACzD,MAAM,kBAAkB,GAAoB,CAAC,CAC3C,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,uBAAuB,CAAC;aAC7B,KAAK,EAAE;aACP,IAAI,EAAE,CAAC;QAEV,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAoB,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1E,MAAM,KAAK,GAAG,0BAA0B,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,OAAO,OAAO,CAAC,GAAG,CAChB,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC9D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,EACJ,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"scrapeGitBookSection.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeGitBookSection.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,8BAA8B,MAAM,sCAAsC,CAAC;AAClF,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,0BAA0B,MAAM,iDAAiD,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,OAA2B;IAE3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,oBAAoB;IACpB,MAAM,OAAO,GAAG,CAAC,CAAC,4CAA4C,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpF,iBAAiB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAEjF,kCAAkC;IAClC,4FAA4F;IAC5F,yDAAyD;IACzD,MAAM,kBAAkB,GAAoB,CAAC,CAC3C,wEAAwE,CACzE;SACE,QAAQ,EAAE;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,KAAK,EAAE;SACP,QAAQ,EAAE,CAAC;IAEd,0BAA0B;IAC1B,MAAM,YAAY,GAAqB,kBAAkB;SACtD,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,CAAkB,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QAE7E,0DAA0D;QAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAoB,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1E,MAAM,KAAK,GAAG,0BAA0B,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO;YACL,KAAK,EAAE,YAAY,IAAI,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC5D,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC;IAEzE,sCAAsC;IACtC,OAAO,OAAO,CAAC,GAAG,CAChB,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAA6B,EAAE,EAAE;QAC9D,OAAO,MAAM,4BAA4B,CACvC,QAAQ,EACR,MAAM,EACN,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,IAAI,EACJ,OAAO,CACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMePage.js

@@ -1,35 +1,35 @@
-import cheerio from "cheerio";
-import { NodeHtmlMarkdown } from "node-html-markdown";
-import downloadAllImages from "../downloadAllImages.js";
-import replaceImagePaths from "../replaceImagePaths.js";
+import cheerio from 'cheerio';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+import downloadAllImages from '../downloadAllImages.js';
+import replaceImagePaths from '../replaceImagePaths.js';
 export async function scrapeReadMePage(html, origin, cliDir, imageBaseDir, overwrite, _ // version
 ) {
     const $ = cheerio.load(html);
-    const titleComponent = $("h1").first();
+    const titleComponent = $('h1').first();
     const title = titleComponent.text().trim();
-    let description = $(".markdown-body", titleComponent.parent()).text().trim();
+    let description = $('.markdown-body', titleComponent.parent()).text().trim();
     if (!description) {
-        description = $(".rm-Article > header p").text().trim();
+        description = $('.rm-Article > header p').text().trim();
     }
-    let content = $(".content-body .markdown-body").first();
+    let content = $('.content-body .markdown-body').first();
     if (content.length === 0) {
-        content = $(".rm-Article > .markdown-body");
+        content = $('.rm-Article > .markdown-body');
     }
     // API Pages don't have a markdown body in the same position so there's no HTML
-    const contentHtml = content.html() || "";
+    const contentHtml = content.html() || '';
     const origToWritePath = await downloadAllImages($, content, origin, imageBaseDir, overwrite);
     const nhm = new NodeHtmlMarkdown({ useInlineLinks: false });
     let markdown = nhm.translate(contentHtml);
     // Keep headers on one line and increase their depth by one
-    markdown = markdown.replace(/# \n\n/g, "## ");
+    markdown = markdown.replace(/# \n\n/g, '## ');
     // Remove unnecessary nonwidth blank space characters
-    markdown = markdown.replace(/\u200b/g, "");
+    markdown = markdown.replace(/\u200b/g, '');
     // Remove ReadMe anchor links
-    markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+    markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, '\n');
     // Reduce unnecessary blank lines
-    markdown = markdown.replace(/\n\n\n/g, "\n\n");
+    markdown = markdown.replace(/\n\n\n/g, '\n\n');
     // Mintlify doesn't support bolded headers, remove the asterisks
-    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, '$1 $2\n');
     if (origToWritePath) {
         markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
     }

package/bin/scraping/site-scrapers/scrapeReadMePage.js.map

@@ -1 +1 @@
-{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IAED,+EAA+E;IAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEzC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAC7C,CAAC,EACD,OAAO,EACP,MAAM,EACN,YAAY,EACZ,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"scrapeReadMePage.js","sourceRoot":"","sources":["../../../src/scraping/site-scrapers/scrapeReadMePage.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,SAAkB,EAClB,CAAqB,CAAC,UAAU;;IAEhC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,WAAW,EAAE;QAChB,WAAW,GAAG,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;KACzD;IAED,IAAI,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;QACxB,OAAO,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC;KAC7C;IAED,+EAA+E;IAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAEzC,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAE9C,qDAAqD;IACrD,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAEtD,iCAAiC;IACjC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE/C,gEAAgE;IAChE,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;IACjE,IAAI,eAAe,EAAE;QACnB,QAAQ,GAAG,iBAAiB,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;KACjE;IAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/bin/scraping/site-scrapers/scrapeReadMeSection.js

@@ -1,30 +1,26 @@
-import cheerio from "cheerio";
-import { scrapeReadMePage } from "./scrapeReadMePage.js";
-import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
-import getLinksRecursively from "./links-per-group/getLinksRecursively.js";
-import downloadLogoImage from "../downloadLogoImage.js";
+import cheerio from 'cheerio';
+import downloadLogoImage from '../downloadLogoImage.js';
+import { scrapeGettingFileNameFromUrl } from '../scrapeGettingFileNameFromUrl.js';
+import getLinksRecursively from './links-per-group/getLinksRecursively.js';
+import { scrapeReadMePage } from './scrapeReadMePage.js';
 export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, overwrite, version) {
     const $ = cheerio.load(html);
     // Download the logo
-    const logoSrc = $(".rm-Logo-img").first().attr("src");
+    const logoSrc = $('.rm-Logo-img').first().attr('src');
     downloadLogoImage(logoSrc, imageBaseDir, origin, overwrite).catch(console.error);
     // Get all the navigation sections, but only from the first
     // sidebar found. There are multiple in the HTML for mobile
     // responsiveness but they all have the same links.
-    const navigationSections = $(".rm-Sidebar")
-        .first()
-        .find(".rm-Sidebar-section");
-    const groupsConfig = navigationSections
-        .toArray()
-        .map((s) => {
+    const navigationSections = $('.rm-Sidebar').first().find('.rm-Sidebar-section');
+    const groupsConfig = navigationSections.toArray().map((s) => {
         const section = $(s);
-        const sectionTitle = section.find("h3").first().text();
+        const sectionTitle = section.find('h3').first().text();
         // Get all links, then use filter to remove duplicates.
         // There are duplicates because of nested navigation, eg:
         // subgroupTitle -> /first-page
         // -- First Page -> /first-page   ** DUPLICATE **
         // -- Second Page -> /second-page
-        const linkSections = section.find(".rm-Sidebar-list").first().children();
+        const linkSections = section.find('.rm-Sidebar-list').first().children();
         const pages = getLinksRecursively(linkSections, $).filter((value, index, self) => self.indexOf(value) === index);
         // Follows the same structure as mint.json
         return {
@@ -37,7 +33,7 @@ export async function scrapeReadMeSection(html, origin, cliDir, imageBaseDir, ov
         return await scrapeGettingFileNameFromUrl(
         // ReadMe requires a directory on all sections whereas we use root.
         // /docs is their default directory so we remove it
-        navEntry, cliDir, origin, overwrite, scrapeReadMePage, false, version, "/docs");
+        navEntry, cliDir, origin, overwrite, scrapeReadMePage, false, version, '/docs');
     }));
 }
 //# sourceMappingURL=scrapeReadMeSection.js.map