mintlify 1.0.7 → 1.0.8

package/src/scraping/combineNavWithEmptyGroupTitles.ts

@@ -0,0 +1,21 @@
+import { Navigation } from "../navigation.js";
+
+export default function combineNavWithEmptyGroupTitles(navArray: Navigation[]) {
+  let newNavArray = [];
+
+  navArray.forEach((nav: Navigation) => {
+    // The first run through the loop will always have -1 as the index.
+    // JavaScript returns undefined when we look for an index outside the size of the array.
+    const prev = newNavArray[newNavArray.length - 1];
+    if (prev == null) {
+      newNavArray.push(nav);
+    } else if (!nav.group && !prev.group) {
+      // Joins multiple groups without a title together IF they occur side by side
+      prev.pages = prev.pages.concat(nav.pages);
+    } else {
+      newNavArray.push(nav);
+    }
+  });
+
+  return newNavArray;
+}

package/src/scraping/getSitemapLinks.ts

@@ -1,5 +1,7 @@
 import axios from "axios";
 
+// Not in use.
+// Gets all links in a sitemap.
 export const getSitemapLinks = async (url: URL) => {
   const hostname = url.hostname.replace(".", "\\.");
   const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -0,0 +1,81 @@
+import path from "path";
+import axios from "axios";
+import { getHtmlWithPuppeteer } from "../browser.js";
+import { createPage } from "../util.js";
+
+export async function scrapeFileGettingFileNameFromUrl(
+  pathname: string,
+  cliDir: string,
+  origin: string,
+  overwrite: boolean,
+  scrapePageFunc: (
+    html: string,
+    origin: string,
+    cliDir: string,
+    imageBaseDir: string
+  ) => Promise<{
+    title?: string;
+    description?: string;
+    markdown?: string;
+  }>,
+  puppeteer = false,
+  baseToRemove?: string
+) {
+  // Skip scraping external links
+  if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
+    return pathname;
+  }
+
+  // Removes file name from the end
+  const splitSubpath = pathname.split("/");
+  let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
+
+  // Remove base dir if passed in
+  if (baseToRemove && folders.startsWith(baseToRemove)) {
+    folders = folders.replace(baseToRemove, "");
+  }
+
+  // TO DO: Improve this by putting each page's images in a separate
+  // folder named after the title of the page.
+  const imageBaseDir = path.join(cliDir, "images", folders);
+
+  // Scrape each page separately
+  const href = new URL(pathname, origin).href;
+  let html: string;
+  if (puppeteer) {
+    html = await getHtmlWithPuppeteer(href);
+  } else {
+    const res = await axios.default.get(href);
+    html = res.data;
+  }
+
+  const { title, description, markdown } = await scrapePageFunc(
+    html,
+    origin,
+    cliDir,
+    imageBaseDir
+  );
+
+  // Check if page didn't have content
+  if (!title && !markdown) {
+    return undefined;
+  }
+
+  const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
+
+  // Default to introduction.mdx if we encountered index.html
+  const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
+
+  // Will create subfolders as needed
+  createPage(
+    title,
+    description,
+    markdown,
+    overwrite,
+    newFileLocation,
+    fileName
+  );
+
+  // Removes first slash if we are in a folder, Mintlify doesn't need it
+  return folders ? path.join(folders, fileName).substring(1) : fileName;
+}

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -1,12 +1,10 @@
-import path from "path";
-import axios from "axios";
-import { getHtmlWithPuppeteer } from "../browser.js";
-import { createPage } from "../util.js";
+import { NavigationEntry, isNavigation } from "../navigation.js";
+import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
 
 export async function scrapeGettingFileNameFromUrl(
+  navEntry: NavigationEntry,
   cliDir: string,
   origin: string,
-  pathname: string,
   overwrite: boolean,
   scrapePageFunc: (
     html: string,
@@ -20,62 +18,33 @@ export async function scrapeGettingFileNameFromUrl(
   }>,
   puppeteer = false,
   baseToRemove?: string
-) {
-  // Skip scraping external links
-  if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
-    return pathname;
+): Promise<NavigationEntry> {
+  if (isNavigation(navEntry)) {
+    const newPages = [];
+    for (const nestedNavEntry of navEntry.pages) {
+      newPages.push(
+        await scrapeGettingFileNameFromUrl(
+          nestedNavEntry,
+          cliDir,
+          origin,
+          overwrite,
+          scrapePageFunc,
+          puppeteer,
+          baseToRemove
+        )
+      );
+    }
+    navEntry.pages = newPages;
+    return navEntry;
   }
 
-  // Removes file name from the end
-  const splitSubpath = pathname.split("/");
-  let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
-
-  // Remove base dir if passed in
-  if (baseToRemove && folders.startsWith(baseToRemove)) {
-    folders = folders.replace(baseToRemove, "");
-  }
-
-  // TO DO: Improve this by putting each page's images in a separate
-  // folder named after the title of the page.
-  const imageBaseDir = path.join(cliDir, "images", folders);
-
-  // Scrape each page separately
-  const href = new URL(pathname, origin).href;
-  let html: string;
-  if (puppeteer) {
-    html = await getHtmlWithPuppeteer(href);
-  } else {
-    const res = await axios.default.get(href);
-    html = res.data;
-  }
-
-  const { title, description, markdown } = await scrapePageFunc(
-    html,
-    origin,
+  return await scrapeFileGettingFileNameFromUrl(
+    navEntry,
     cliDir,
-    imageBaseDir
-  );
-
-  // Check if page didn't have content
-  if (!title && !markdown) {
-    return undefined;
-  }
-
-  const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
-
-  // Default to introduction.mdx if we encountered index.html
-  const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
-
-  // Will create subfolders as needed
-  createPage(
-    title,
-    description,
-    markdown,
+    origin,
     overwrite,
-    newFileLocation,
-    fileName
+    scrapePageFunc,
+    puppeteer,
+    baseToRemove
   );
-
-  // Removes first slash if we are in a folder, Mintlify doesn't need it
-  return folders ? path.join(folders, fileName).substring(1) : fileName;
 }

package/src/scraping/scrapePageCommands.ts

@@ -0,0 +1,52 @@
+import axios from "axios";
+import { scrapePage } from "./scrapePage.js";
+import { scrapeDocusaurusPage } from "./site-scrapers/scrapeDocusaurusPage.js";
+import { scrapeGitBookPage } from "./site-scrapers/scrapeGitBookPage.js";
+import { scrapeReadMePage } from "./site-scrapers/scrapeReadMePage.js";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs } from "../util.js";
+import { getHtmlWithPuppeteer } from "../browser.js";
+
+function validateFramework(framework) {
+  if (!framework) {
+    console.log(
+      "Could not detect the framework automatically. Please use one of:"
+    );
+    console.log("scrape-page-docusaurus");
+    console.log("scrape-page-gitbook");
+    console.log("scrape-page-readme");
+    return process.exit(1);
+  }
+}
+
+export async function scrapePageWrapper(argv, scrapeFunc, puppeteer = false) {
+  const href = getHrefFromArgs(argv);
+  let html: string;
+  if (puppeteer) {
+    html = await getHtmlWithPuppeteer(href);
+  } else {
+    const res = await axios.default.get(href);
+    html = res.data;
+  }
+  await scrapePage(scrapeFunc, href, html, argv.overwrite);
+  process.exit(0);
+}
+
+export async function scrapePageAutomatically(argv: any) {
+  const href = getHrefFromArgs(argv);
+  const res = await axios.default.get(href);
+  const html = res.data;
+  const framework = detectFramework(html);
+
+  validateFramework(framework);
+
+  console.log("Detected framework: " + framework);
+
+  if (framework === Frameworks.DOCUSAURUS) {
+    await scrapePageWrapper(argv, scrapeDocusaurusPage);
+  } else if (framework === Frameworks.GITBOOK) {
+    await scrapePageWrapper(argv, scrapeGitBookPage, true);
+  } else if (framework === Frameworks.README) {
+    await scrapePageWrapper(argv, scrapeReadMePage);
+  }
+}

package/src/scraping/scrapeSectionCommands.ts

@@ -0,0 +1,110 @@
+import axios from "axios";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs, getOrigin } from "../util.js";
+import { scrapeSection } from "./scrapeSection.js";
+import { scrapeDocusaurusSection } from "./site-scrapers/scrapeDocusaurusSection.js";
+import { scrapeGitBookSection } from "./site-scrapers/scrapeGitBookSection.js";
+import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
+import { startBrowser } from "../browser.js";
+
+function validateFramework(framework: Frameworks | undefined) {
+  if (!framework) {
+    console.log(
+      "Could not detect the framework automatically. Please use one of:"
+    );
+    console.log("scrape-page-docusaurus");
+    console.log("scrape-page-gitbook");
+    console.log("scrape-page-readme");
+    return process.exit(1);
+  }
+}
+
+export async function scrapeSectionAxiosWrapper(argv: any, scrapeFunc: any) {
+  const href = getHrefFromArgs(argv);
+  const res = await axios.default.get(href);
+  const html = res.data;
+  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  process.exit(0);
+}
+
+export async function scrapeGitbookSectionCommand(argv: any) {
+  await scrapeSectionGitBookWrapper(argv, scrapeGitBookSection);
+}
+
+async function scrapeSectionGitBookWrapper(argv: any, scrapeFunc: any) {
+  const href = getHrefFromArgs(argv);
+
+  const browser = await startBrowser();
+  const page = await browser.newPage();
+  await page.goto(href, {
+    waitUntil: "networkidle2",
+  });
+
+  let prevEncountered: string[] = [];
+  let encounteredHref = ["fake"];
+
+  // Loop until we've encountered every link
+  while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+    prevEncountered = encounteredHref;
+    encounteredHref = await page.evaluate(
+      (encounteredHref) => {
+        const icons: HTMLElement[] = Array.from(
+          document.querySelectorAll('path[d="M9 18l6-6-6-6"]')
+        );
+
+        const linksFound: string[] = [];
+        icons.forEach(async (icon: HTMLElement) => {
+          const toClick = icon?.parentElement?.parentElement;
+          const link = toClick?.parentElement?.parentElement;
+
+          // Skip icons not in the side navigation
+          if (!link?.hasAttribute("href")) {
+            return;
+          }
+
+          const href = link.getAttribute("href");
+
+          // Should never occur but we keep it as a fail-safe
+          if (href?.startsWith("https://") || href?.startsWith("http://")) {
+            return;
+          }
+
+          // Click any links we haven't seen before
+          if (href && !encounteredHref.includes(href)) {
+            toClick?.click();
+          }
+          if (href) {
+            linksFound.push(href);
+          }
+        });
+
+        return linksFound;
+      },
+      encounteredHref // Need to pass array into the browser
+    );
+  }
+
+  const html = await page.content();
+  browser.close();
+  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  process.exit(0);
+}
+
+export async function scrapeSectionAutomatically(argv: any) {
+  const href = getHrefFromArgs(argv);
+  const res = await axios.default.get(href);
+  const html = res.data;
+  const framework = detectFramework(html);
+
+  validateFramework(framework);
+
+  console.log("Detected framework: " + framework);
+
+  if (framework === Frameworks.DOCUSAURUS) {
+    await scrapeSectionAxiosWrapper(argv, scrapeDocusaurusSection);
+  } else if (framework === Frameworks.GITBOOK) {
+    await scrapeGitbookSectionCommand(argv);
+  } else if (framework === Frameworks.README) {
+    await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
+  }
+}

package/src/scraping/site-scrapers/getLinksRecursively.ts

@@ -0,0 +1,40 @@
+// Used by GitBook and ReadMe section scrapers
+export default function getLinksRecursively(linkSections: any, $: any) {
+  if (linkSections == null || linkSections.length === 0) {
+    return [];
+  }
+
+  return linkSections
+    .map((i, s) => {
+      const subsection = $(s);
+      const link = subsection.children().first();
+
+      const linkHref = link.attr("href");
+
+      // Skip missing links. For example, GitBook uses
+      // empty divs are used for styling a line beside the nav.
+      // Skip external links until Mintlify supports them
+      if (
+        !linkHref ||
+        linkHref.startsWith("https://") ||
+        linkHref.startsWith("http://")
+      ) {
+        return undefined;
+      }
+
+      const childLinks = subsection.children().eq(1).children();
+
+      if (childLinks.length > 0) {
+        // Put the section link in the list of pages.
+        // When we support the section itself being a link we should update this
+        return {
+          group: link.text(),
+          pages: [linkHref, ...getLinksRecursively(childLinks, $)],
+        };
+      }
+
+      return linkHref;
+    })
+    .toArray()
+    .filter(Boolean);
+}

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -59,9 +59,9 @@ export async function scrapeDocusaurusSection(
             // Docusaurus requires a directory on all sections wheras we use root.
             // /docs is their default directory so we remove it
             scrapeGettingFileNameFromUrl(
+              pathname,
               cliDir,
               origin,
-              pathname,
               overwrite,
               scrapeDocusaurusPage,
               false,

package/src/scraping/site-scrapers/scrapeGitBookSection.ts

@@ -1,7 +1,9 @@
 import cheerio from "cheerio";
+import { NavigationEntry } from "../../navigation.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
-import { getSitemapLinks } from "../getSitemapLinks.js";
 import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
+import getLinksRecursively from "./getLinksRecursively.js";
 
 export async function scrapeGitBookSection(
   html: string,
@@ -22,95 +24,54 @@ export async function scrapeGitBookSection(
     .children();
 
   // Get all links per group
-  let allNavPathnames = [];
   const groupsConfig = navigationSections
-    .map((i, section) => {
+    .map((i, s) => {
+      const section = $(s);
       const sectionTitle = $(section)
         .find('div > div[dir="auto"]')
         .first()
         .text();
 
-      const linkPaths = $(section)
-        .find("a[href]")
-        .map((i, link) => {
-          const linkHref = $(link).attr("href");
+      // Only present if the nested navigation is not in a group
+      const firstLink = section.children().eq(0);
+      const firstHref = firstLink.attr("href");
 
-          // Skip external links until Mintlify supports them
-          if (
-            linkHref.startsWith("https://") ||
-            linkHref.startsWith("http://")
-          ) {
-            return undefined;
-          }
+      const linkSections = section.children().eq(1).children();
+      const pages = getLinksRecursively(linkSections, $);
 
-          return linkHref;
-        })
-        .toArray();
-
-      allNavPathnames = allNavPathnames.concat(linkPaths);
-
-      // Follows the same structure as mint.json
       return {
-        group: sectionTitle,
-        pages: linkPaths,
+        group: sectionTitle || alternateTitle(firstLink, pages),
+        pages: firstHref ? [firstHref, ...pages] : pages,
       };
     })
-    .toArray();
+    .toArray()
+    .filter(Boolean);
 
-  // Scrape every link not in the navigation. Nested docs
-  // don't show up in navigation without clicking buttons,
-  // so this lets us download the files for the user to add
-  // manually to mint.json.
-  const sitemapPaths = (await getSitemapLinks(new URL("sitemap.xml", origin)))
-    .map((sitemapLinks: string) => {
-      return new URL(sitemapLinks).pathname;
-    })
-    .filter((pathname: string) => !allNavPathnames.includes(pathname));
+  // Merge groups with empty titles together
+  const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
 
-  const sitemapPathnamesForConfig = [];
-  for (const pathname of sitemapPaths) {
-    sitemapPathnamesForConfig.push(
-      await scrapeGettingFileNameFromUrl(
+  // Scrape each link in the navigation.
+  const groupsConfigCleanPaths = await Promise.all(
+    reducedGroupsConfig.map(async (navEntry: NavigationEntry) => {
+      return await scrapeGettingFileNameFromUrl(
+        navEntry,
         cliDir,
         origin,
-        pathname,
         overwrite,
         scrapeGitBookPage,
         true
-      )
-    );
-  }
-
-  // Scrape each link in the navigation.
-  const groupsConfigCleanPaths = await Promise.all(
-    groupsConfig.map(async (groupConfig) => {
-      const newPages = [];
-      for (const pathname of groupConfig.pages) {
-        newPages.push(
-          await scrapeGettingFileNameFromUrl(
-            cliDir,
-            origin,
-            pathname,
-            overwrite,
-            scrapeGitBookPage,
-            true
-          )
-        );
-      }
-      groupConfig.pages = newPages;
-      return groupConfig;
+      );
     })
   );
 
-  if (sitemapPathnamesForConfig.length > 0) {
-    return groupsConfigCleanPaths.concat([
-      {
-        group:
-          "ATTENTION! WE CANNOT DETECT GROUPS FOR NESTED DOCS. PLEASE MOVE THEM INTO THEIR ORIGINAL GROUPS.",
-        pages: sitemapPathnamesForConfig,
-      },
-    ]);
-  }
-
   return groupsConfigCleanPaths;
 }
+
+function alternateTitle(firstLink, pages) {
+  // Only assign titles to nested navigation menus outside a section.
+  // Others should not have a title so we can merge them into one section.
+  if (pages.length > 0) {
+    return firstLink?.text();
+  }
+  return "";
+}

package/src/scraping/site-scrapers/scrapeReadMePage.ts

@@ -22,7 +22,9 @@ export async function scrapeReadMePage(
   if (content.length === 0) {
     content = $(".rm-Article > .markdown-body");
   }
-  const contentHtml = content.html();
+
+  // API Pages don't have a markdown body in the same position so there's no HTML
+  let contentHtml = content.html() || "";
 
   const origToWritePath = await downloadAllImages(
     $,

package/src/scraping/site-scrapers/scrapeReadMeSection.ts

@@ -1,6 +1,8 @@
 import cheerio from "cheerio";
 import { scrapeReadMePage } from "./scrapeReadMePage.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+import { NavigationEntry } from "../../navigation.js";
 
 export async function scrapeReadMeSection(
   html: string,
@@ -18,60 +20,41 @@ export async function scrapeReadMeSection(
     .find(".rm-Sidebar-section");
 
   const groupsConfig = navigationSections
-    .map((i, section) => {
-      const sectionTitle = $(section).find("h3").first().text();
+    .map((i, s) => {
+      const section = $(s);
+      const sectionTitle = section.find("h3").first().text();
 
       // Get all links, then use filter to remove duplicates.
       // There are duplicates because of nested navigation, eg:
       // subgroupTitle -> /first-page
       // -- First Page -> /first-page   ** DUPLICATE **
       // -- Second Page -> /second-page
-      const linkPaths = $(section)
-        .find("a[href]")
-        .map((i, link) => {
-          const linkHref = $(link).attr("href");
-
-          // Skip external links until Mintlify supports them
-          if (
-            linkHref.startsWith("https://") ||
-            linkHref.startsWith("http://")
-          ) {
-            return undefined;
-          }
-
-          return linkHref;
-        })
-        .toArray()
-        .filter(
-          (value: string, index: number, self) => self.indexOf(value) === index
-        );
+      const linkSections = section.find(".rm-Sidebar-list").first().children();
+      const pages = getLinksRecursively(linkSections, $).filter(
+        (value: string, index: number, self) => self.indexOf(value) === index
+      );
 
       // Follows the same structure as mint.json
       return {
         group: sectionTitle,
-        pages: linkPaths,
+        pages: pages,
       };
     })
     .toArray();
 
   return await Promise.all(
-    groupsConfig.map(async (groupConfig) => {
-      groupConfig.pages = await Promise.all(
-        groupConfig.pages.map(async (pathname: string) =>
-          // ReadMe requires a directory on all sections wheras we use root.
-          // /docs is their default directory so we remove it
-          scrapeGettingFileNameFromUrl(
-            cliDir,
-            origin,
-            pathname,
-            overwrite,
-            scrapeReadMePage,
-            false,
-            "/docs"
-          )
-        )
+    groupsConfig.map(async (navEntry: NavigationEntry) => {
+      return await scrapeGettingFileNameFromUrl(
+        // ReadMe requires a directory on all sections wheras we use root.
+        // /docs is their default directory so we remove it
+        navEntry,
+        cliDir,
+        origin,
+        overwrite,
+        scrapeReadMePage,
+        false,
+        "/docs"
       );
-      return groupConfig;
     })
   );
 }