mintlify 1.1.1 → 1.1.2

package/src/scraping/scrapeSectionCommands.ts

@@ -3,22 +3,12 @@ import { detectFramework, Frameworks } from "./detectFramework.js";
 import { getHrefFromArgs, getOrigin } from "../util.js";
 import { scrapeSection } from "./scrapeSection.js";
 import { scrapeDocusaurusSection } from "./site-scrapers/scrapeDocusaurusSection.js";
+import openNestedDocusaurusMenus from "./site-scrapers/openNestedDocusaurusMenus.js";
 import { scrapeGitBookSection } from "./site-scrapers/scrapeGitBookSection.js";
+import openNestedGitbookMenus from "./site-scrapers/openNestedGitbookMenus.js";
 import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
 import { startBrowser } from "../browser.js";
 
-function validateFramework(framework: Frameworks | undefined) {
-  if (!framework) {
-    console.log(
-      "Could not detect the framework automatically. Please use one of:"
-    );
-    console.log("scrape-page-docusaurus");
-    console.log("scrape-page-gitbook");
-    console.log("scrape-page-readme");
-    return process.exit(1);
-  }
-}
-
 export async function scrapeSectionAxiosWrapper(argv: any, scrapeFunc: any) {
   const href = getHrefFromArgs(argv);
   const res = await axios.default.get(href);
@@ -27,11 +17,27 @@ export async function scrapeSectionAxiosWrapper(argv: any, scrapeFunc: any) {
   process.exit(0);
 }
 
+export async function scrapeDocusaurusSectionCommand(argv: any) {
+  await scrapeSectionOpeningAllNested(
+    argv,
+    openNestedDocusaurusMenus,
+    scrapeDocusaurusSection
+  );
+}
+
 export async function scrapeGitbookSectionCommand(argv: any) {
-  await scrapeSectionGitBookWrapper(argv, scrapeGitBookSection);
+  await scrapeSectionOpeningAllNested(
+    argv,
+    openNestedGitbookMenus,
+    scrapeGitBookSection
+  );
 }
 
-async function scrapeSectionGitBookWrapper(argv: any, scrapeFunc: any) {
+async function scrapeSectionOpeningAllNested(
+  argv: any,
+  openLinks: any,
+  scrapeFunc: any
+) {
   const href = getHrefFromArgs(argv);
 
   const browser = await startBrowser();
@@ -40,51 +46,7 @@ async function scrapeSectionGitBookWrapper(argv: any, scrapeFunc: any) {
     waitUntil: "networkidle2",
   });
 
-  let prevEncountered: string[] = [];
-  let encounteredHref = ["fake"];
-
-  // Loop until we've encountered every link
-  while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
-    prevEncountered = encounteredHref;
-    encounteredHref = await page.evaluate(
-      (encounteredHref) => {
-        const icons: HTMLElement[] = Array.from(
-          document.querySelectorAll('path[d="M9 18l6-6-6-6"]')
-        );
-
-        const linksFound: string[] = [];
-        icons.forEach(async (icon: HTMLElement) => {
-          const toClick = icon?.parentElement?.parentElement;
-          const link = toClick?.parentElement?.parentElement;
-
-          // Skip icons not in the side navigation
-          if (!link?.hasAttribute("href")) {
-            return;
-          }
-
-          const href = link.getAttribute("href");
-
-          // Should never occur but we keep it as a fail-safe
-          if (href?.startsWith("https://") || href?.startsWith("http://")) {
-            return;
-          }
-
-          // Click any links we haven't seen before
-          if (href && !encounteredHref.includes(href)) {
-            toClick?.click();
-          }
-          if (href) {
-            linksFound.push(href);
-          }
-        });
-
-        return linksFound;
-      },
-      encounteredHref // Need to pass array into the browser
-    );
-  }
-
-  const html = await page.content();
+  const html = await openLinks(page);
   browser.close();
   await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
   process.exit(0);
@@ -101,10 +63,22 @@ export async function scrapeSectionAutomatically(argv: any) {
   console.log("Detected framework: " + framework);
 
   if (framework === Frameworks.DOCUSAURUS) {
-    await scrapeSectionAxiosWrapper(argv, scrapeDocusaurusSection);
+    await scrapeDocusaurusSectionCommand(argv);
   } else if (framework === Frameworks.GITBOOK) {
     await scrapeGitbookSectionCommand(argv);
   } else if (framework === Frameworks.README) {
     await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
   }
 }
+
+function validateFramework(framework: Frameworks | undefined) {
+  if (!framework) {
+    console.log(
+      "Could not detect the framework automatically. Please use one of:"
+    );
+    console.log("scrape-page-docusaurus");
+    console.log("scrape-page-gitbook");
+    console.log("scrape-page-readme");
+    return process.exit(1);
+  }
+}

package/src/scraping/site-scrapers/alternateGroupTitle.ts

@@ -0,0 +1,8 @@
+export default function alternateGroupTitle(firstLink, pages) {
+  // Only assign titles to nested navigation menus outside a section.
+  // Others should not have a title so we can merge them into one section.
+  if (pages.length > 0) {
+    return firstLink?.text();
+  }
+  return "";
+}

package/src/scraping/site-scrapers/getLinksRecursively.ts

@@ -1,4 +1,4 @@
-// Used by GitBook and ReadMe section scrapers
+// Used by Docusaurus, GitBook, and ReadMe section scrapers
 export default function getLinksRecursively(linkSections: any, $: any) {
   if (linkSections == null || linkSections.length === 0) {
     return [];
@@ -7,8 +7,12 @@ export default function getLinksRecursively(linkSections: any, $: any) {
   return linkSections
     .map((i, s) => {
       const subsection = $(s);
-      const link = subsection.children().first();
+      let link = subsection.children().first();
 
+      if (!link.attr("href")) {
+        // Docusaurus nests the <a> inside a <div>
+        link = link.find("a[href]").first();
+      }
       const linkHref = link.attr("href");
 
       // Skip missing links. For example, GitBook uses
@@ -16,6 +20,7 @@ export default function getLinksRecursively(linkSections: any, $: any) {
       // Skip external links until Mintlify supports them
       if (
         !linkHref ||
+        linkHref === "#" ||
         linkHref.startsWith("https://") ||
         linkHref.startsWith("http://")
       ) {

package/src/scraping/site-scrapers/openNestedDocusaurusMenus.ts

@@ -0,0 +1,42 @@
+import { Page } from "puppeteer";
+
+export default async function openNestedDocusaurusMenus(page: Page) {
+  let prevEncountered: string[] = [];
+  let encounteredHref = ["fake-href-to-make-loop-run-at-least-once"];
+
+  // Loop until we've encountered every link
+  while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+    prevEncountered = encounteredHref;
+    encounteredHref = await page.evaluate(
+      (encounteredHref) => {
+        const collapsible: HTMLElement[] = Array.from(
+          document.querySelectorAll(".menu__link.menu__link--sublist")
+        );
+
+        const linksFound: string[] = [];
+        collapsible.forEach(async (collapsibleItem: HTMLElement) => {
+          const href = collapsibleItem?.getAttribute("href");
+
+          // Should never occur but we keep it as a fail-safe
+          if (href?.startsWith("https://") || href?.startsWith("http://")) {
+            return;
+          }
+
+          // Click any links we haven't seen before
+          if (href && !encounteredHref.includes(href)) {
+            collapsibleItem?.click();
+          }
+
+          if (href) {
+            linksFound.push(href);
+          }
+        });
+
+        return linksFound;
+      },
+      encounteredHref // Need to pass array into the browser
+    );
+  }
+
+  return await page.content();
+}

package/src/scraping/site-scrapers/openNestedGitbookMenus.ts

@@ -0,0 +1,49 @@
+import { Page } from "puppeteer";
+
+export default async function openNestedGitbookMenus(page: Page) {
+  let prevEncountered: string[] = [];
+  let encounteredHref = ["fake-href-to-make-loop-run-at-least-once"];
+
+  // Loop until we've encountered every link
+  while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+    prevEncountered = encounteredHref;
+    encounteredHref = await page.evaluate(
+      (encounteredHref) => {
+        const icons: HTMLElement[] = Array.from(
+          document.querySelectorAll('path[d="M9 18l6-6-6-6"]')
+        );
+
+        const linksFound: string[] = [];
+        icons.forEach(async (icon: HTMLElement) => {
+          const toClick = icon?.parentElement?.parentElement;
+          const link = toClick?.parentElement?.parentElement;
+
+          // Skip icons not in the side navigation
+          if (!link?.hasAttribute("href")) {
+            return;
+          }
+
+          const href = link.getAttribute("href");
+
+          // Should never occur but we keep it as a fail-safe
+          if (href?.startsWith("https://") || href?.startsWith("http://")) {
+            return;
+          }
+
+          // Click any links we haven't seen before
+          if (href && !encounteredHref.includes(href)) {
+            toClick?.click();
+          }
+          if (href) {
+            linksFound.push(href);
+          }
+        });
+
+        return linksFound;
+      },
+      encounteredHref // Need to pass array into the browser
+    );
+  }
+
+  return await page.content();
+}

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -1,6 +1,10 @@
 import cheerio from "cheerio";
+import { NavigationEntry } from "../..//navigation.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
 import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+import alternateGroupTitle from "./alternateGroupTitle.js";
 
 export async function scrapeDocusaurusSection(
   html: string,
@@ -15,51 +19,49 @@ export async function scrapeDocusaurusSection(
 
   // Get all links per group
   const groupsConfig = navigationSections
-    .map((i, section) => {
-      const sectionComponent = $(section);
+    .map((i, s) => {
+      const section = $(s);
 
       // Links without a group
-      if (sectionComponent.hasClass("theme-doc-sidebar-item-link")) {
-        const linkHref = sectionComponent.find("a[href]").first().attr("href");
+      if (section.hasClass("theme-doc-sidebar-item-link")) {
+        const linkHref = section.find("a[href]").first().attr("href");
         return {
           group: "",
           pages: [linkHref],
         };
       }
 
-      const sectionTitle = sectionComponent
+      const firstLink = section
         .find(".menu__list-item-collapsible")
         .first()
-        .text();
+        .find("a[href]");
 
-      // The category title can be a page too so we find from the
-      // section component instead of the more specific menu__list child
-      const linkPaths = sectionComponent
-        .find("a[href]")
-        .map((i, link) => {
-          return $(link).attr("href");
-        })
-        .filter((i, link) => link !== "#")
-        .toArray();
+      const sectionTitle = firstLink.text();
+      const firstHref = firstLink.attr("href");
+      const linkSections = section.children().eq(1).children();
+
+      const pages = getLinksRecursively(linkSections, $);
 
-      // Follows the same structure as mint.json
       return {
-        group: sectionTitle,
-        pages: linkPaths,
+        group: sectionTitle || alternateGroupTitle(firstLink, pages),
+        pages: firstHref ? [firstHref, ...pages] : pages,
       };
     })
     .toArray();
 
+  // Merge groups with empty titles together
+  const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
+
   // Scrape each link in the navigation.
   const groupsConfigCleanPaths = await Promise.all(
-    groupsConfig.map(async (groupConfig) => {
+    reducedGroupsConfig.map(async (groupConfig) => {
       groupConfig.pages = (
         await Promise.all(
-          groupConfig.pages.map(async (pathname: string) =>
+          groupConfig.pages.map(async (navEntry: NavigationEntry) =>
             // Docusaurus requires a directory on all sections wheras we use root.
             // /docs is their default directory so we remove it
             scrapeGettingFileNameFromUrl(
-              pathname,
+              navEntry,
               cliDir,
               origin,
               overwrite,

package/src/scraping/site-scrapers/scrapeGitBookSection.ts

@@ -4,6 +4,7 @@ import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js
 import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
 import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
 import getLinksRecursively from "./getLinksRecursively.js";
+import alternateGroupTitle from "./alternateGroupTitle.js";
 
 export async function scrapeGitBookSection(
   html: string,
@@ -40,7 +41,7 @@ export async function scrapeGitBookSection(
       const pages = getLinksRecursively(linkSections, $);
 
       return {
-        group: sectionTitle || alternateTitle(firstLink, pages),
+        group: sectionTitle || alternateGroupTitle(firstLink, pages),
         pages: firstHref ? [firstHref, ...pages] : pages,
       };
     })
@@ -66,12 +67,3 @@ export async function scrapeGitBookSection(
 
   return groupsConfigCleanPaths;
 }
-
-function alternateTitle(firstLink, pages) {
-  // Only assign titles to nested navigation menus outside a section.
-  // Others should not have a title so we can merge them into one section.
-  if (pages.length > 0) {
-    return firstLink?.text();
-  }
-  return "";
-}

package/bin/local-preview/helper-commands/cleanCommand.js

@@ -1,8 +0,0 @@
-import shell from "shelljs";
-import { CLIENT_PATH } from "../../constants.js";
-const cleanCommand = () => {
-    shell.cd(CLIENT_PATH);
-    shell.exec("git clean -d -x -e node_modules -n");
-};
-export default cleanCommand;
-//# sourceMappingURL=cleanCommand.js.map

package/bin/local-preview/helper-commands/cleanCommand.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cleanCommand.js","sourceRoot":"","sources":["../../../src/local-preview/helper-commands/cleanCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,YAAY,GAAG,GAAG,EAAE;IACxB,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;AACnD,CAAC,CAAC;AAEF,eAAe,YAAY,CAAC"}