mintlify 1.1.6 → 1.1.8

package/src/scraping/detectFramework.ts

@@ -14,17 +14,27 @@ export function detectFramework(html) {
     docusaurusMeta.length > 0 &&
     docusaurusMeta.attr("content").includes("Docusaurus")
   ) {
-    return Frameworks.DOCUSAURUS;
+    if (docusaurusMeta.attr("content").includes("v3")) {
+      return { framework: Frameworks.DOCUSAURUS, version: "3" };
+    }
+    if (docusaurusMeta.attr("content").includes("v2")) {
+      return { framework: Frameworks.DOCUSAURUS, version: "2" };
+    } else if (docusaurusMeta.attr("content").includes("v1")) {
+      console.warn(
+        "WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3."
+      );
+      return { framework: Frameworks.DOCUSAURUS, version: "1" };
+    }
   }
 
   const isGitBook = $(".gitbook-root").length > 0;
   if (isGitBook) {
-    return Frameworks.GITBOOK;
+    return { framework: Frameworks.GITBOOK };
   }
 
   const isReadMe = $('meta[name="readme-deploy"]').length > 0;
   if (isReadMe) {
-    return Frameworks.README;
+    return { framework: Frameworks.README };
   }
 
   return undefined;

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -12,13 +12,15 @@ export async function scrapeFileGettingFileNameFromUrl(
     html: string,
     origin: string,
     cliDir: string,
-    imageBaseDir: string
+    imageBaseDir: string,
+    version: string | undefined
   ) => Promise<{
     title?: string;
     description?: string;
     markdown?: string;
   }>,
   puppeteer = false,
+  version: string | undefined,
   baseToRemove?: string
 ) {
   // Skip scraping external links
@@ -53,7 +55,8 @@ export async function scrapeFileGettingFileNameFromUrl(
     html,
     origin,
     cliDir,
-    imageBaseDir
+    imageBaseDir,
+    version
   );
 
   // Check if page didn't have content

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -10,13 +10,15 @@ export async function scrapeGettingFileNameFromUrl(
     html: string,
     origin: string,
     cliDir: string,
-    imageBaseDir: string
+    imageBaseDir: string,
+    version: string | undefined
   ) => Promise<{
     title?: string;
     description?: string;
     markdown?: string;
   }>,
   puppeteer = false,
+  version: string | undefined,
   baseToRemove?: string
 ): Promise<NavigationEntry> {
   if (isNavigation(navEntry)) {
@@ -30,6 +32,7 @@ export async function scrapeGettingFileNameFromUrl(
           overwrite,
           scrapePageFunc,
           puppeteer,
+          version,
           baseToRemove
         )
       );
@@ -45,6 +48,7 @@ export async function scrapeGettingFileNameFromUrl(
     overwrite,
     scrapePageFunc,
     puppeteer,
+    version,
     baseToRemove
   );
 }

package/src/scraping/scrapePage.ts

@@ -6,11 +6,13 @@ export async function scrapePage(
     html: string,
     origin: string,
     cliDir: string,
-    imageBaseDir: string
+    imageBaseDir: string,
+    version: string | undefined
   ) => Promise<any>,
   href: string,
   html: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string | undefined
 ) {
   const origin = getOrigin(href);
   const imageBaseDir = path.join(process.cwd(), "images");
@@ -18,7 +20,8 @@ export async function scrapePage(
     html,
     origin,
     process.cwd(),
-    imageBaseDir
+    imageBaseDir,
+    version
   );
   createPage(title, description, markdown, overwrite, process.cwd());
 }

package/src/scraping/scrapePageCommands.ts

@@ -19,16 +19,20 @@ function validateFramework(framework) {
   }
 }
 
-export async function scrapePageWrapper(argv, scrapeFunc, puppeteer = false) {
+export async function scrapePageWrapper(
+  argv: any,
+  scrapeFunc: any,
+  options?: { version?: string; puppeteer?: boolean }
+) {
   const href = getHrefFromArgs(argv);
   let html: string;
-  if (puppeteer) {
+  if (options.puppeteer) {
     html = await getHtmlWithPuppeteer(href);
   } else {
     const res = await axios.default.get(href);
     html = res.data;
   }
-  await scrapePage(scrapeFunc, href, html, argv.overwrite);
+  await scrapePage(scrapeFunc, href, html, argv.overwrite, options.version);
   process.exit(0);
 }
 
@@ -36,16 +40,16 @@ export async function scrapePageAutomatically(argv: any) {
   const href = getHrefFromArgs(argv);
   const res = await axios.default.get(href);
   const html = res.data;
-  const framework = detectFramework(html);
+  const { framework, version } = detectFramework(html);
 
   validateFramework(framework);
 
   console.log("Detected framework: " + framework);
 
   if (framework === Frameworks.DOCUSAURUS) {
-    await scrapePageWrapper(argv, scrapeDocusaurusPage);
+    await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
   } else if (framework === Frameworks.GITBOOK) {
-    await scrapePageWrapper(argv, scrapeGitBookPage, true);
+    await scrapePageWrapper(argv, scrapeGitBookPage, { puppeteer: true });
   } else if (framework === Frameworks.README) {
     await scrapePageWrapper(argv, scrapeReadMePage);
   }

package/src/scraping/scrapeSection.ts

@@ -4,12 +4,19 @@ export async function scrapeSection(
   scrapeFunc: any,
   html: string,
   origin: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string | undefined
 ) {
   console.log(
     `Started scraping${overwrite ? ", overwrite mode is on" : ""}...`
   );
-  const groupsConfig = await scrapeFunc(html, origin, process.cwd(), overwrite);
+  const groupsConfig = await scrapeFunc(
+    html,
+    origin,
+    process.cwd(),
+    overwrite,
+    version
+  );
   console.log("Finished scraping.");
   console.log("Add the following to your navigation in mint.json:");
   console.log(objToReadableString(groupsConfig));

package/src/scraping/scrapeSectionCommands.ts

@@ -13,15 +13,25 @@ export async function scrapeSectionAxiosWrapper(argv: any, scrapeFunc: any) {
   const href = getHrefFromArgs(argv);
   const res = await axios.default.get(href);
   const html = res.data;
-  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  await scrapeSection(
+    scrapeFunc,
+    html,
+    getOrigin(href),
+    argv.overwrite,
+    undefined
+  );
   process.exit(0);
 }
 
-export async function scrapeDocusaurusSectionCommand(argv: any) {
+export async function scrapeDocusaurusSectionCommand(
+  argv: any,
+  version: string // "1" | "2" | "3"
+) {
   await scrapeSectionOpeningAllNested(
     argv,
     openNestedDocusaurusMenus,
-    scrapeDocusaurusSection
+    scrapeDocusaurusSection,
+    version
   );
 }
 
@@ -36,7 +46,8 @@ export async function scrapeGitbookSectionCommand(argv: any) {
 async function scrapeSectionOpeningAllNested(
   argv: any,
   openLinks: any,
-  scrapeFunc: any
+  scrapeFunc: any,
+  version?: string
 ) {
   const href = getHrefFromArgs(argv);
 
@@ -48,7 +59,13 @@ async function scrapeSectionOpeningAllNested(
 
   const html = await openLinks(page);
   browser.close();
-  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  await scrapeSection(
+    scrapeFunc,
+    html,
+    getOrigin(href),
+    argv.overwrite,
+    version
+  );
   process.exit(0);
 }
 
@@ -56,14 +73,14 @@ export async function scrapeSectionAutomatically(argv: any) {
   const href = getHrefFromArgs(argv);
   const res = await axios.default.get(href);
   const html = res.data;
-  const framework = detectFramework(html);
+  const { framework, version } = detectFramework(html);
 
   validateFramework(framework);
 
   console.log("Detected framework: " + framework);
 
   if (framework === Frameworks.DOCUSAURUS) {
-    await scrapeDocusaurusSectionCommand(argv);
+    await scrapeDocusaurusSectionCommand(argv, version);
   } else if (framework === Frameworks.GITBOOK) {
     await scrapeGitbookSectionCommand(argv);
   } else if (framework === Frameworks.README) {

package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts

@@ -0,0 +1,46 @@
+import alternateGroupTitle from "../alternateGroupTitle.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+
+export function getDocusaurusLinksPerGroup(
+  navigationSections: any,
+  $: any,
+  version: string | undefined
+) {
+  if (version === "3" || version === "2") {
+    return getDocusaurusLinksPerGroupLoop(navigationSections, $);
+  }
+  return [];
+}
+
+function getDocusaurusLinksPerGroupLoop(navigationSections: any, $: any) {
+  return navigationSections
+    .map((i, s) => {
+      const section = $(s);
+
+      // Links without a group
+      if (section.hasClass("theme-doc-sidebar-item-link")) {
+        const linkHref = section.find("a[href]").first().attr("href");
+        return {
+          group: "",
+          pages: [linkHref],
+        };
+      }
+
+      const firstLink = section
+        .find(".menu__list-item-collapsible")
+        .first()
+        .find("a[href]");
+
+      const sectionTitle = firstLink.text();
+      const firstHref = firstLink.attr("href");
+      const linkSections = section.children().eq(1).children();
+
+      const pages = getLinksRecursively(linkSections, $);
+
+      return {
+        group: sectionTitle || alternateGroupTitle(firstLink, pages),
+        pages: firstHref ? [firstHref, ...pages] : pages,
+      };
+    })
+    .toArray();
+}

package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts

@@ -7,34 +7,46 @@ export async function scrapeDocusaurusPage(
   html: string,
   origin: string,
   cliDir: string,
-  imageBaseDir: string
+  imageBaseDir: string,
+  version: string | undefined // expects "2", or "3". Have not written support for "1" yet
 ) {
   const $ = cheerio.load(html);
 
-  const content = $(".theme-doc-markdown").first();
+  const article =
+    version === "3" ? $(".theme-doc-markdown").first() : $("article").first();
 
-  // Index pages with no additional text don't have the markdown class
-  if (content.length === 0) {
+  if (article.length === 0) {
+    // Index pages with no additional text don't have the markdown class
     return {};
   }
 
-  const titleComponent = content.find("h1");
+  const titleComponent = article.find("h1");
   const title = titleComponent.text().trim();
 
   // Do not include title in the content when we insert it in our metadata
   titleComponent.remove();
 
+  const markdownContent =
+    version === "3" ? article : article.find(".markdown").first();
+
   const origToWritePath = await downloadAllImages(
     $,
-    content,
+    markdownContent,
     origin,
     imageBaseDir
   );
 
-  const contentHtml = content.html();
+  const markdownHtml = markdownContent.html();
 
   const nhm = new NodeHtmlMarkdown();
-  let markdown = nhm.translate(contentHtml);
+  let markdown = nhm.translate(markdownHtml);
+
+  if (markdown == null) {
+    console.error(
+      "We do not support scraping this page. Content will be empty"
+    );
+    return { title, description: null, markdown: "" };
+  }
 
   // Description only exists in meta tags. The code is commented out because its prone to incorrectly
   // including a description if the first line of text had markdown annotations like `.

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -3,14 +3,14 @@ import { NavigationEntry } from "../..//navigation.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
 import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
 import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
-import getLinksRecursively from "./getLinksRecursively.js";
-import alternateGroupTitle from "./alternateGroupTitle.js";
+import { getDocusaurusLinksPerGroup } from "./links-per-group/getDocusaurusLinksPerGroup.js";
 
 export async function scrapeDocusaurusSection(
   html: string,
   origin: string,
   cliDir: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string
 ) {
   const $ = cheerio.load(html);
 
@@ -18,36 +18,11 @@ export async function scrapeDocusaurusSection(
   const navigationSections = $(".theme-doc-sidebar-menu").first().children();
 
   // Get all links per group
-  const groupsConfig = navigationSections
-    .map((i, s) => {
-      const section = $(s);
-
-      // Links without a group
-      if (section.hasClass("theme-doc-sidebar-item-link")) {
-        const linkHref = section.find("a[href]").first().attr("href");
-        return {
-          group: "",
-          pages: [linkHref],
-        };
-      }
-
-      const firstLink = section
-        .find(".menu__list-item-collapsible")
-        .first()
-        .find("a[href]");
-
-      const sectionTitle = firstLink.text();
-      const firstHref = firstLink.attr("href");
-      const linkSections = section.children().eq(1).children();
-
-      const pages = getLinksRecursively(linkSections, $);
-
-      return {
-        group: sectionTitle || alternateGroupTitle(firstLink, pages),
-        pages: firstHref ? [firstHref, ...pages] : pages,
-      };
-    })
-    .toArray();
+  const groupsConfig = getDocusaurusLinksPerGroup(
+    navigationSections,
+    $,
+    version
+  );
 
   // Merge groups with empty titles together
   const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
@@ -67,6 +42,7 @@ export async function scrapeDocusaurusSection(
               overwrite,
               scrapeDocusaurusPage,
               false,
+              version,
               "/docs"
             )
           )

package/src/scraping/site-scrapers/scrapeGitBookPage.ts

@@ -7,7 +7,8 @@ export async function scrapeGitBookPage(
   html: string,
   origin: string,
   cliDir: string,
-  imageBaseDir: string
+  imageBaseDir: string,
+  _: string | undefined // version
 ) {
   const $ = cheerio.load(html);
 

package/src/scraping/site-scrapers/scrapeGitBookSection.ts

@@ -3,23 +3,26 @@ import { NavigationEntry } from "../../navigation.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
 import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
 import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
-import getLinksRecursively from "./getLinksRecursively.js";
+import getLinksRecursively from "./links-per-group/getLinksRecursively.js";
 import alternateGroupTitle from "./alternateGroupTitle.js";
 
 export async function scrapeGitBookSection(
   html: string,
   origin: string,
   cliDir: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string | undefined
 ) {
   const $ = cheerio.load(html);
 
   // Get all the navigation sections
+  // Some variants of the GitBook UI show the logo and search base in the side navigation bar,
+  // but the navigation sections are always the last value.
   const navigationSections = $(
-    'div[data-testid="page.desktopTableOfContents"] > div > div:first-child'
+    'div[data-testid="page.desktopTableOfContents"] > nav > div:first-child'
   )
     .children()
-    .first()
+    .eq(-1)
     .children()
     .first()
     .children();
@@ -60,7 +63,8 @@ export async function scrapeGitBookSection(
         origin,
         overwrite,
         scrapeGitBookPage,
-        true
+        true,
+        version
       );
     })
   );

package/src/scraping/site-scrapers/scrapeReadMePage.ts

@@ -7,7 +7,8 @@ export async function scrapeReadMePage(
   html: string,
   origin: string,
   cliDir: string,
-  imageBaseDir: string
+  imageBaseDir: string,
+  _: string | undefined // version
 ) {
   const $ = cheerio.load(html);
 

package/src/scraping/site-scrapers/scrapeReadMeSection.ts

@@ -1,14 +1,15 @@
 import cheerio from "cheerio";
 import { scrapeReadMePage } from "./scrapeReadMePage.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
-import getLinksRecursively from "./getLinksRecursively.js";
+import getLinksRecursively from "./links-per-group/getLinksRecursively.js";
 import { NavigationEntry } from "../../navigation.js";
 
 export async function scrapeReadMeSection(
   html: string,
   origin: string,
   cliDir: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string | undefined
 ) {
   const $ = cheerio.load(html);
 
@@ -53,6 +54,7 @@ export async function scrapeReadMeSection(
         overwrite,
         scrapeReadMePage,
         false,
+        version,
         "/docs"
       );
     })

package/bin/local-preview/helper-commands/cleanCommand.js

@@ -1,8 +0,0 @@
-import shell from "shelljs";
-import { CLIENT_PATH } from "../../constants.js";
-const cleanCommand = () => {
-    shell.cd(CLIENT_PATH);
-    shell.exec("git clean -d -x -e node_modules -n");
-};
-export default cleanCommand;
-//# sourceMappingURL=cleanCommand.js.map

package/bin/local-preview/helper-commands/cleanCommand.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cleanCommand.js","sourceRoot":"","sources":["../../../src/local-preview/helper-commands/cleanCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,YAAY,GAAG,GAAG,EAAE;IACxB,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;AACnD,CAAC,CAAC;AAEF,eAAe,YAAY,CAAC"}