mintlify 1.0.6 → 1.0.8

package/src/scraping/scrapeSectionCommands.ts

@@ -0,0 +1,110 @@
+import axios from "axios";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs, getOrigin } from "../util.js";
+import { scrapeSection } from "./scrapeSection.js";
+import { scrapeDocusaurusSection } from "./site-scrapers/scrapeDocusaurusSection.js";
+import { scrapeGitBookSection } from "./site-scrapers/scrapeGitBookSection.js";
+import { scrapeReadMeSection } from "./site-scrapers/scrapeReadMeSection.js";
+import { startBrowser } from "../browser.js";
+
+function validateFramework(framework: Frameworks | undefined) {
+  if (!framework) {
+    console.log(
+      "Could not detect the framework automatically. Please use one of:"
+    );
+    console.log("scrape-page-docusaurus");
+    console.log("scrape-page-gitbook");
+    console.log("scrape-page-readme");
+    return process.exit(1);
+  }
+}
+
+export async function scrapeSectionAxiosWrapper(argv: any, scrapeFunc: any) {
+  const href = getHrefFromArgs(argv);
+  const res = await axios.default.get(href);
+  const html = res.data;
+  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  process.exit(0);
+}
+
+export async function scrapeGitbookSectionCommand(argv: any) {
+  await scrapeSectionGitBookWrapper(argv, scrapeGitBookSection);
+}
+
+async function scrapeSectionGitBookWrapper(argv: any, scrapeFunc: any) {
+  const href = getHrefFromArgs(argv);
+
+  const browser = await startBrowser();
+  const page = await browser.newPage();
+  await page.goto(href, {
+    waitUntil: "networkidle2",
+  });
+
+  let prevEncountered: string[] = [];
+  let encounteredHref = ["fake"];
+
+  // Loop until we've encountered every link
+  while (!encounteredHref.every((href) => prevEncountered.includes(href))) {
+    prevEncountered = encounteredHref;
+    encounteredHref = await page.evaluate(
+      (encounteredHref) => {
+        const icons: HTMLElement[] = Array.from(
+          document.querySelectorAll('path[d="M9 18l6-6-6-6"]')
+        );
+
+        const linksFound: string[] = [];
+        icons.forEach(async (icon: HTMLElement) => {
+          const toClick = icon?.parentElement?.parentElement;
+          const link = toClick?.parentElement?.parentElement;
+
+          // Skip icons not in the side navigation
+          if (!link?.hasAttribute("href")) {
+            return;
+          }
+
+          const href = link.getAttribute("href");
+
+          // Should never occur but we keep it as a fail-safe
+          if (href?.startsWith("https://") || href?.startsWith("http://")) {
+            return;
+          }
+
+          // Click any links we haven't seen before
+          if (href && !encounteredHref.includes(href)) {
+            toClick?.click();
+          }
+          if (href) {
+            linksFound.push(href);
+          }
+        });
+
+        return linksFound;
+      },
+      encounteredHref // Need to pass array into the browser
+    );
+  }
+
+  const html = await page.content();
+  browser.close();
+  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  process.exit(0);
+}
+
+export async function scrapeSectionAutomatically(argv: any) {
+  const href = getHrefFromArgs(argv);
+  const res = await axios.default.get(href);
+  const html = res.data;
+  const framework = detectFramework(html);
+
+  validateFramework(framework);
+
+  console.log("Detected framework: " + framework);
+
+  if (framework === Frameworks.DOCUSAURUS) {
+    await scrapeSectionAxiosWrapper(argv, scrapeDocusaurusSection);
+  } else if (framework === Frameworks.GITBOOK) {
+    await scrapeGitbookSectionCommand(argv);
+  } else if (framework === Frameworks.README) {
+    await scrapeSectionAxiosWrapper(argv, scrapeReadMeSection);
+  }
+}

package/src/scraping/site-scrapers/getLinksRecursively.ts

@@ -0,0 +1,40 @@
+// Used by GitBook and ReadMe section scrapers
+export default function getLinksRecursively(linkSections: any, $: any) {
+  if (linkSections == null || linkSections.length === 0) {
+    return [];
+  }
+
+  return linkSections
+    .map((i, s) => {
+      const subsection = $(s);
+      const link = subsection.children().first();
+
+      const linkHref = link.attr("href");
+
+      // Skip missing links. For example, GitBook uses
+      // empty divs are used for styling a line beside the nav.
+      // Skip external links until Mintlify supports them
+      if (
+        !linkHref ||
+        linkHref.startsWith("https://") ||
+        linkHref.startsWith("http://")
+      ) {
+        return undefined;
+      }
+
+      const childLinks = subsection.children().eq(1).children();
+
+      if (childLinks.length > 0) {
+        // Put the section link in the list of pages.
+        // When we support the section itself being a link we should update this
+        return {
+          group: link.text(),
+          pages: [linkHref, ...getLinksRecursively(childLinks, $)],
+        };
+      }
+
+      return linkHref;
+    })
+    .toArray()
+    .filter(Boolean);
+}

package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts

@@ -0,0 +1,67 @@
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+
+export async function scrapeDocusaurusPage(
+  html: string,
+  origin: string,
+  cliDir: string,
+  imageBaseDir: string
+) {
+  const $ = cheerio.load(html);
+
+  const content = $(".theme-doc-markdown").first();
+
+  // Index pages with no additional text don't have the markdown class
+  if (content.length === 0) {
+    return {};
+  }
+
+  const titleComponent = content.find("h1");
+  const title = titleComponent.text().trim();
+
+  // Do not include title in the content when we insert it in our metadata
+  titleComponent.remove();
+
+  const origToWritePath = await downloadAllImages(
+    $,
+    content,
+    origin,
+    imageBaseDir
+  );
+
+  const contentHtml = content.html();
+
+  const nhm = new NodeHtmlMarkdown();
+  let markdown = nhm.translate(contentHtml);
+
+  // Description only exists in meta tags. The code is commented out because its prone to incorrectly
+  // including a description if the first line of text had markdown annotations like `.
+  // The commented out alternative is to ignore description if it's the first line of text,
+  // this means it was not set in the metadata and Docusaurus defaulted to the text.
+  const description = null;
+  // let description = $('meta[property="og:description"]').attr("content");
+  // if (markdown.startsWith(description)) {
+  //   description = null;
+  // }
+
+  // Remove Docusaurus links from headers
+  // When we parse their HTML the parser adds things like:
+  // [](#setup "Direct link to heading")
+  // to the end of each header.
+  markdown = markdown.replace(/\[\]\(#.+ ".+"\)\n/g, "\n");
+
+  // Remove unnecessary nonwidth blank space characters
+  markdown = markdown.replace(/\u200b/g, "");
+
+  // Reduce unnecessary blank lines
+  markdown = markdown.replace(/\n\n\n/g, "\n\n");
+
+  // Mintlify doesn't support bolded headers, remove the asterisks
+  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+
+  markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+
+  return { title, description, markdown };
+}

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -0,0 +1,80 @@
+import cheerio from "cheerio";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
+
+export async function scrapeDocusaurusSection(
+  html: string,
+  origin: string,
+  cliDir: string,
+  overwrite: boolean
+) {
+  const $ = cheerio.load(html);
+
+  // Get all the navigation sections
+  const navigationSections = $(".theme-doc-sidebar-menu").first().children();
+
+  // Get all links per group
+  const groupsConfig = navigationSections
+    .map((i, section) => {
+      const sectionComponent = $(section);
+
+      // Links without a group
+      if (sectionComponent.hasClass("theme-doc-sidebar-item-link")) {
+        const linkHref = sectionComponent.find("a[href]").first().attr("href");
+        return {
+          group: "",
+          pages: [linkHref],
+        };
+      }
+
+      const sectionTitle = sectionComponent
+        .find(".menu__list-item-collapsible")
+        .first()
+        .text();
+
+      // The category title can be a page too so we find from the
+      // section component instead of the more specific menu__list child
+      const linkPaths = sectionComponent
+        .find("a[href]")
+        .map((i, link) => {
+          return $(link).attr("href");
+        })
+        .filter((i, link) => link !== "#")
+        .toArray();
+
+      // Follows the same structure as mint.json
+      return {
+        group: sectionTitle,
+        pages: linkPaths,
+      };
+    })
+    .toArray();
+
+  // Scrape each link in the navigation.
+  const groupsConfigCleanPaths = await Promise.all(
+    groupsConfig.map(async (groupConfig) => {
+      groupConfig.pages = (
+        await Promise.all(
+          groupConfig.pages.map(async (pathname: string) =>
+            // Docusaurus requires a directory on all sections wheras we use root.
+            // /docs is their default directory so we remove it
+            scrapeGettingFileNameFromUrl(
+              pathname,
+              cliDir,
+              origin,
+              overwrite,
+              scrapeDocusaurusPage,
+              false,
+              "/docs"
+            )
+          )
+        )
+      )
+        // Remove skipped index pages (they return undefined from the above function)
+        .filter(Boolean);
+      return groupConfig;
+    })
+  );
+
+  return groupsConfigCleanPaths;
+}

package/src/{scrapeGitBookPage.ts → scraping/site-scrapers/scrapeGitBookPage.ts}

@@ -1,14 +1,19 @@
-import axios from "axios";
 import cheerio from "cheerio";
 import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
 
-export async function scrapeGitBookPage(url: string) {
-  const res = await axios.default.get(url);
-  const $ = cheerio.load(res.data);
+export async function scrapeGitBookPage(
+  html: string,
+  origin: string,
+  cliDir: string,
+  imageBaseDir: string
+) {
+  const $ = cheerio.load(html);
 
   const titleComponent = $('[data-testid="page.title"]').first();
   const titleAndDescription = titleComponent.parent().parent().parent().text();
-  console.log(titleAndDescription);
+
   const description = titleAndDescription
     .replace(titleComponent.text(), "")
     .trim();
@@ -17,6 +22,19 @@ export async function scrapeGitBookPage(url: string) {
   const content = $('[data-testid="page.contentEditor"]').first();
   const contentHtml = $.html(content);
 
+  const modifyFileName = (fileName) =>
+    // Remove GitBook metadata from the start
+    // The first four %2F split metadata fields. Remaining ones are part of the file name.
+    fileName.split("%2F").slice(4).join("%2F");
+
+  const origToWritePath = await downloadAllImages(
+    $,
+    content,
+    origin,
+    imageBaseDir,
+    modifyFileName
+  );
+
   const nhm = new NodeHtmlMarkdown();
   let markdown = nhm.translate(contentHtml);
 
@@ -32,5 +50,7 @@ export async function scrapeGitBookPage(url: string) {
   // Mintlify doesn't support bolded headers, remove the asterisks
   markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
 
+  markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+
   return { title, description, markdown };
 }

package/src/scraping/site-scrapers/scrapeGitBookSection.ts

@@ -0,0 +1,77 @@
+import cheerio from "cheerio";
+import { NavigationEntry } from "../../navigation.js";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import { scrapeGitBookPage } from "./scrapeGitBookPage.js";
+import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+
+export async function scrapeGitBookSection(
+  html: string,
+  origin: string,
+  cliDir: string,
+  overwrite: boolean
+) {
+  const $ = cheerio.load(html);
+
+  // Get all the navigation sections
+  const navigationSections = $(
+    'div[data-testid="page.desktopTableOfContents"] > div > div:first-child'
+  )
+    .children()
+    .first()
+    .children()
+    .first()
+    .children();
+
+  // Get all links per group
+  const groupsConfig = navigationSections
+    .map((i, s) => {
+      const section = $(s);
+      const sectionTitle = $(section)
+        .find('div > div[dir="auto"]')
+        .first()
+        .text();
+
+      // Only present if the nested navigation is not in a group
+      const firstLink = section.children().eq(0);
+      const firstHref = firstLink.attr("href");
+
+      const linkSections = section.children().eq(1).children();
+      const pages = getLinksRecursively(linkSections, $);
+
+      return {
+        group: sectionTitle || alternateTitle(firstLink, pages),
+        pages: firstHref ? [firstHref, ...pages] : pages,
+      };
+    })
+    .toArray()
+    .filter(Boolean);
+
+  // Merge groups with empty titles together
+  const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
+
+  // Scrape each link in the navigation.
+  const groupsConfigCleanPaths = await Promise.all(
+    reducedGroupsConfig.map(async (navEntry: NavigationEntry) => {
+      return await scrapeGettingFileNameFromUrl(
+        navEntry,
+        cliDir,
+        origin,
+        overwrite,
+        scrapeGitBookPage,
+        true
+      );
+    })
+  );
+
+  return groupsConfigCleanPaths;
+}
+
+function alternateTitle(firstLink, pages) {
+  // Only assign titles to nested navigation menus outside a section.
+  // Others should not have a title so we can merge them into one section.
+  if (pages.length > 0) {
+    return firstLink?.text();
+  }
+  return "";
+}

package/src/scraping/site-scrapers/scrapeReadMePage.ts

@@ -0,0 +1,57 @@
+import cheerio from "cheerio";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import downloadAllImages from "../downloadAllImages.js";
+import replaceImagePaths from "../replaceImagePaths.js";
+
+export async function scrapeReadMePage(
+  html: string,
+  origin: string,
+  cliDir: string,
+  imageBaseDir: string
+) {
+  const $ = cheerio.load(html);
+
+  const titleComponent = $("h1").first();
+  const title = titleComponent.text().trim();
+  let description = $(".markdown-body", titleComponent.parent()).text().trim();
+  if (!description) {
+    description = $(".rm-Article > header p").text().trim();
+  }
+
+  let content = $(".content-body .markdown-body").first();
+  if (content.length === 0) {
+    content = $(".rm-Article > .markdown-body");
+  }
+
+  // API Pages don't have a markdown body in the same position so there's no HTML
+  let contentHtml = content.html() || "";
+
+  const origToWritePath = await downloadAllImages(
+    $,
+    content,
+    origin,
+    imageBaseDir
+  );
+
+  const nhm = new NodeHtmlMarkdown();
+  let markdown = nhm.translate(contentHtml);
+
+  // Keep headers on one line and increase their depth by one
+  markdown = markdown.replace(/# \n\n/g, "## ");
+
+  // Remove unnecessary nonwidth blank space characters
+  markdown = markdown.replace(/\u200b/g, "");
+
+  // Remove ReadMe anchor links
+  markdown = markdown.replace(/\n\[\]\(#.+\)\n/g, "\n");
+
+  // Reduce unnecessary blank lines
+  markdown = markdown.replace(/\n\n\n/g, "\n\n");
+
+  // Mintlify doesn't support bolded headers, remove the asterisks
+  markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
+
+  markdown = replaceImagePaths(origToWritePath, cliDir, markdown);
+
+  return { title, description, markdown };
+}

package/src/scraping/site-scrapers/scrapeReadMeSection.ts

@@ -0,0 +1,60 @@
+import cheerio from "cheerio";
+import { scrapeReadMePage } from "./scrapeReadMePage.js";
+import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+import { NavigationEntry } from "../../navigation.js";
+
+export async function scrapeReadMeSection(
+  html: string,
+  origin: string,
+  cliDir: string,
+  overwrite: boolean
+) {
+  const $ = cheerio.load(html);
+
+  // Get all the navigation sections, but only from the first
+  // sidebar found. There are multiple in the HTML for mobile
+  // responsiveness but they all have the same links.
+  const navigationSections = $(".rm-Sidebar")
+    .first()
+    .find(".rm-Sidebar-section");
+
+  const groupsConfig = navigationSections
+    .map((i, s) => {
+      const section = $(s);
+      const sectionTitle = section.find("h3").first().text();
+
+      // Get all links, then use filter to remove duplicates.
+      // There are duplicates because of nested navigation, eg:
+      // subgroupTitle -> /first-page
+      // -- First Page -> /first-page   ** DUPLICATE **
+      // -- Second Page -> /second-page
+      const linkSections = section.find(".rm-Sidebar-list").first().children();
+      const pages = getLinksRecursively(linkSections, $).filter(
+        (value: string, index: number, self) => self.indexOf(value) === index
+      );
+
+      // Follows the same structure as mint.json
+      return {
+        group: sectionTitle,
+        pages: pages,
+      };
+    })
+    .toArray();
+
+  return await Promise.all(
+    groupsConfig.map(async (navEntry: NavigationEntry) => {
+      return await scrapeGettingFileNameFromUrl(
+        // ReadMe requires a directory on all sections wheras we use root.
+        // /docs is their default directory so we remove it
+        navEntry,
+        cliDir,
+        origin,
+        overwrite,
+        scrapeReadMePage,
+        false,
+        "/docs"
+      );
+    })
+  );
+}

package/src/util.ts

@@ -1,12 +1,13 @@
 import { mkdirSync, writeFileSync } from "fs";
-import { Page } from "./templates.js";
+import Ora from "ora";
 import path from "path";
+import { Page } from "./init-command/templates.js";
+import stopIfInvalidLink from "./validation/stopIfInvalidLink.js";
 
-export function getWebsite(url: string) {
-  // Gets the website
+export function getOrigin(url: string) {
   // eg. https://google.com -> https://google.com
   // https://google.com/page -> https://google.com
-  return url.split("/").slice(0, 3).join("/");
+  return new URL(url).origin;
 }
 
 export function objToReadableString(objs: Object[]) {
@@ -35,14 +36,59 @@ export const createPage = (
   title: string,
   description?: string,
   markdown?: string,
+  overwrite: boolean = false,
   rootDir: string = "",
   fileName?: string
 ) => {
+  const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
+
   // Create the folders needed if they're missing
   mkdirSync(rootDir, { recursive: true });
 
   // Write the page to memory
-  const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
-  writeFileSync(writePath, Page(title, description, markdown));
-  console.log("✏️ - " + writePath);
+  if (overwrite) {
+    writeFileSync(writePath, Page(title, description, markdown));
+    console.log("✏️ - " + writePath);
+  } else {
+    try {
+      writeFileSync(writePath, Page(title, description, markdown), {
+        flag: "wx",
+      });
+      console.log("✏️ - " + writePath);
+    } catch (e) {
+      // We do a try-catch instead of an if-statement to avoid a race condition
+      // of the file being created after we started writing.
+      if (e.code === "EEXIST") {
+        console.log(`❌ Skipping existing file ${writePath}`);
+      } else {
+        console.error(e);
+      }
+    }
+  }
+};
+
+export function getHrefFromArgs(argv: any) {
+  const href = argv.url;
+  stopIfInvalidLink(href);
+  return href;
+}
+
+export const buildLogger = (startText: string = "") => {
+  const logger = Ora().start(startText);
+  return logger;
+};
+
+export const getFileExtension = (filename: string) => {
+  return (
+    filename.substring(filename.lastIndexOf(".") + 1, filename.length) ||
+    filename
+  );
+};
+
+export const fileBelongsInPagesFolder = (filename: string) => {
+  const extension = getFileExtension(filename);
+  return (
+    extension &&
+    (extension === "mdx" || extension === "md" || extension === "tsx")
+  );
 };

package/src/validation/isValidLink.ts

@@ -0,0 +1,9 @@
+export default function isValidLink(href: string) {
+  // This checks the link is written correctly, not that the page exists.
+  try {
+    new URL(href);
+    return true;
+  } catch (_) {
+    return false;
+  }
+}

package/src/validation/stopIfInvalidLink.ts

@@ -0,0 +1,9 @@
+import isValidLink from "./isValidLink.js";
+
+export default function stopIfInvalidLink(href: string) {
+  if (!isValidLink(href)) {
+    console.log("Invalid link: " + href);
+    console.log("Make sure the link starts with http:// or https://");
+    process.exit(1);
+  }
+}

package/tsconfig.json

@@ -3,7 +3,7 @@
     "module": "node16",
     "moduleResolution": "node16",
     "target": "es2017",
-    "lib": ["es2016", "dom"],
+    "lib": ["es2016", "dom", "ES2021.String"],
     "sourceMap": true,
     "outDir": "bin",
     "baseUrl": ".",

package/bin/scrapeGitBook.js

@@ -1,28 +0,0 @@
-import axios from "axios";
-import cheerio from "cheerio";
-import { NodeHtmlMarkdown } from "node-html-markdown";
-export async function scrapeGitBook(url) {
-    const res = await axios.default.get(url);
-    const $ = cheerio.load(res.data);
-    const titleComponent = $('[data-testid="page.title"]').first();
-    const titleAndDescription = titleComponent.parent().parent().parent().text();
-    console.log(titleAndDescription);
-    const description = titleAndDescription
-        .replace(titleComponent.text(), "")
-        .trim();
-    const title = titleComponent.text().trim();
-    const content = $('[data-testid="page.contentEditor"]').first();
-    const contentHtml = $.html(content);
-    const nhm = new NodeHtmlMarkdown();
-    let markdown = nhm.translate(contentHtml);
-    // Keep headers on one line and increase their depth by one
-    markdown = markdown.replace(/# \n\n/g, "## ");
-    // Remove unnecessary nonwidth blank space characters
-    markdown = markdown.replace(/\u200b/g, "");
-    // Reduce unnecessary blank lines
-    markdown = markdown.replace(/\n\n\n/g, "\n\n");
-    // Mintlify doesn't support bolded headers, remove the asterisks
-    markdown = markdown.replace(/(\n#+) \*\*(.*)\*\*\n/g, "$1 $2\n");
-    return { title, description, markdown };
-}
-//# sourceMappingURL=scrapeGitBook.js.map