mintlify 1.0.6 → 1.0.8

package/src/local-preview/metadata.ts

@@ -0,0 +1,154 @@
+// TODO - add types
+import fse from "fs-extra";
+import { promises as _promises } from "fs";
+import matter from "gray-matter";
+import path from "path";
+
+import categorizeFiles from "./categorizeFiles.js";
+import { getConfigObj } from "./mintConfigFile.js";
+import { getOpenApiTitleAndDescription } from "./getOpenApiContext.js";
+import { slugToTitle } from "./slugToTitle.js";
+import { CLIENT_PATH, CMD_EXEC_PATH } from "../constants.js";
+
+const { readFile } = _promises;
+
+// End matter is front matter, but at the end
+const getIndexOfEndMatter = (fileContents: string) => {
+  const frontMatters = fileContents.match(
+    /---\n(title:.+\n|description:.+\n|sidebarTitle:.+\n|api:.+\n|openapi:.+\n)+---$/m
+  );
+  if (frontMatters) {
+    return fileContents.indexOf(frontMatters[0]);
+  }
+
+  return -1;
+};
+
+export const potentiallyRemoveEndMatter = (fileContents: string) => {
+  const endMatterIndex = getIndexOfEndMatter(fileContents);
+
+  if (endMatterIndex === -1) {
+    return fileContents;
+  }
+
+  return fileContents.substring(0, endMatterIndex);
+};
+
+const getMetadata = (fileContents: string) => {
+  const { data } = matter(fileContents);
+
+  if (Object.keys(data).length > 0) {
+    return data;
+  }
+
+  const startIndex = getIndexOfEndMatter(fileContents);
+  if (startIndex === -1) {
+    return {};
+  }
+
+  const fileContentFromFrontMatter = fileContents.substring(startIndex);
+  const { data: nonTopFrontMatter } = matter(fileContentFromFrontMatter);
+  return nonTopFrontMatter;
+};
+
+export const createPage = (
+  path: string,
+  content: string,
+  openApiObj: object | null
+) => {
+  const slug = path.replace(/\.mdx?$/, "").substring(1);
+  let defaultTitle = slugToTitle(slug);
+  const metadata = getMetadata(content);
+  // Append data from OpenAPI if it exists
+  const { title, description } = getOpenApiTitleAndDescription(
+    openApiObj,
+    metadata?.openapi
+  );
+  if (title) {
+    defaultTitle = title;
+  }
+  return {
+    [slug]: { title: defaultTitle, description, ...metadata, href: `/${slug}` },
+  };
+};
+
+export const createMetadataFileFromPages = (pages: any, configObj: any) => {
+  const targetPath = path.join(CLIENT_PATH, "src", "metadata.json");
+  const createNav = (nav) => {
+    return {
+      group: nav.group,
+      pages: nav.pages.map((page) => {
+        if (typeof page === "string") {
+          return pages[page];
+        }
+
+        return createNav(page);
+      }),
+    };
+  };
+
+  if (configObj?.navigation == null) {
+    return;
+  }
+
+  let navFile = configObj.navigation.map((nav) => createNav(nav));
+  const filterOutNullInPages = (pages) => {
+    const newPages = [];
+    pages.forEach((page) => {
+      if (page == null) {
+        return;
+      }
+      if (page?.pages) {
+        const newGroup = filterOutNullInGroup(page);
+        newPages.push(newGroup);
+      } else {
+        newPages.push(page);
+      }
+    });
+
+    return newPages;
+  };
+  const filterOutNullInGroup = (group) => {
+    const newPages = filterOutNullInPages(group.pages);
+    const newGroup = {
+      ...group,
+      pages: newPages,
+    };
+    return newGroup;
+  };
+  const newNavFile = navFile.map((group) => {
+    return filterOutNullInGroup(group);
+  });
+  fse.outputFileSync(targetPath, JSON.stringify(newNavFile, null, 2), {
+    flag: "w",
+  });
+};
+
+export const createMetadataFile = async () => {
+  // create pages
+  const { markdownFiles, openApiBuffer } = await categorizeFiles();
+  let openApiObj = null;
+  if (openApiBuffer) {
+    openApiObj = JSON.parse(openApiBuffer.toString());
+  }
+  // create config object
+  const configObj = await getConfigObj();
+  let pages = {};
+  const mdPromises = [];
+  markdownFiles.forEach((filename) => {
+    mdPromises.push(
+      (async () => {
+        const sourcePath = path.join(CMD_EXEC_PATH, filename);
+        const fileContent = await readFile(sourcePath);
+        const contentStr = fileContent.toString();
+        const page = createPage(filename, contentStr, openApiObj);
+        pages = {
+          ...pages,
+          ...page,
+        };
+      })()
+    );
+  });
+  await Promise.all(mdPromises);
+  createMetadataFileFromPages(pages, configObj);
+};

package/src/local-preview/mintConfigFile.ts

@@ -0,0 +1,48 @@
+import { promises as _promises } from "fs";
+import fse, { pathExists } from "fs-extra";
+import path from "path";
+
+import { CLIENT_PATH, CMD_EXEC_PATH } from "../constants.js";
+
+const { readFile } = _promises;
+
+const getConfigPath = async (): Promise<string | null> => {
+  let configPath = null;
+  if (await pathExists(path.join(CMD_EXEC_PATH, "mint.config.json"))) {
+    configPath = path.join(CMD_EXEC_PATH, "mint.config.json");
+  }
+
+  if (await pathExists(path.join(CMD_EXEC_PATH, "mint.json"))) {
+    configPath = path.join(CMD_EXEC_PATH, "mint.json");
+  }
+  return configPath;
+};
+
+export const getConfigObj = async (): Promise<object | null> => {
+  const configPath = await getConfigPath();
+  let configObj = null;
+  if (configPath) {
+    const configContents = await readFile(configPath);
+    configObj = JSON.parse(configContents.toString());
+  }
+  return configObj;
+};
+
+export const updateConfigFile = async (logger: any) => {
+  const configTargetPath = path.join(CLIENT_PATH, "src", "mint.json");
+  await fse.remove(configTargetPath);
+  let configObj = null;
+  const configPath = await getConfigPath();
+
+  if (configPath) {
+    await fse.remove(configTargetPath);
+    await fse.copy(configPath, configTargetPath);
+    logger.succeed("mint config file synced");
+    const configContents = await readFile(configPath);
+    configObj = JSON.parse(configContents.toString());
+  } else {
+    logger.fail("Must be ran in a directory where a mint.json file exists.");
+    process.exit(1);
+  }
+  return configObj;
+};

package/src/local-preview/openApiCheck.ts

@@ -0,0 +1,19 @@
+import SwaggerParser from "@apidevtools/swagger-parser";
+import { promises as _promises } from "fs";
+
+const openApiCheck = async (
+  path: string
+): Promise<{ buffer: Buffer | undefined; isOpenApi: boolean }> => {
+  let buffer = undefined;
+  let isOpenApi = false;
+  try {
+    const api = await SwaggerParser.validate(path);
+    buffer = Buffer.from(JSON.stringify(api, null, 2), "utf-8");
+    isOpenApi = true;
+  } catch {
+    // not valid openApi
+  }
+  return { buffer, isOpenApi };
+};
+
+export default openApiCheck;

package/src/local-preview/slugToTitle.ts

@@ -0,0 +1,7 @@
+export const slugToTitle = (slug) => {
+  const slugArr = slug.split("/");
+  let defaultTitle = slugArr[slugArr.length - 1].split("-").join(" "); //replace all dashes
+  defaultTitle = defaultTitle.split("_").join(" "); //replace all underscores
+  defaultTitle = defaultTitle.charAt(0).toUpperCase() + defaultTitle.slice(1); //capitalize first letter
+  return defaultTitle;
+};

package/src/navigation.ts

@@ -0,0 +1,12 @@
+export type NavigationEntry = string | Navigation;
+
+export type Navigation = {
+  group: string;
+  pages: NavigationEntry[];
+};
+
+export function isNavigation(
+  navEntry: NavigationEntry
+): navEntry is Navigation {
+  return typeof navEntry !== "string";
+}

package/src/pageTemplate.ts

@@ -0,0 +1,32 @@
+import inquirer from "inquirer";
+import { createPage } from "./util.js";
+
+const generatePageTemplate = () => {
+  inquirer
+    .prompt([
+      {
+        type: "input",
+        name: "title",
+        message: "What is the title of the new page?",
+      },
+      {
+        type: "input",
+        name: "description",
+        message: "What is the description?",
+        default: "",
+      },
+    ])
+    .then((answers) => {
+      const { title, description } = answers;
+
+      createPage(title, description);
+      console.log("🌱 Created initial files for Mintlify docs");
+      process.exit(0);
+    })
+    .catch((error) => {
+      console.error(error);
+      process.exit(1);
+    });
+};
+
+export default generatePageTemplate;

package/src/scraping/combineNavWithEmptyGroupTitles.ts

@@ -0,0 +1,21 @@
+import { Navigation } from "../navigation.js";
+
+export default function combineNavWithEmptyGroupTitles(navArray: Navigation[]) {
+  let newNavArray = [];
+
+  navArray.forEach((nav: Navigation) => {
+    // The first run through the loop will always have -1 as the index.
+    // JavaScript returns undefined when we look for an index outside the size of the array.
+    const prev = newNavArray[newNavArray.length - 1];
+    if (prev == null) {
+      newNavArray.push(nav);
+    } else if (!nav.group && !prev.group) {
+      // Joins multiple groups without a title together IF they occur side by side
+      prev.pages = prev.pages.concat(nav.pages);
+    } else {
+      newNavArray.push(nav);
+    }
+  });
+
+  return newNavArray;
+}

package/src/scraping/detectFramework.ts

@@ -0,0 +1,31 @@
+import cheerio from "cheerio";
+
+export enum Frameworks {
+  DOCUSAURUS = "DOCUSAURUS",
+  GITBOOK = "GITBOOK",
+  README = "README",
+}
+
+export function detectFramework(html) {
+  const $ = cheerio.load(html);
+  const docusaurusMeta = $('meta[name="generator"]');
+
+  if (
+    docusaurusMeta.length > 0 &&
+    docusaurusMeta.attr("content").includes("Docusaurus")
+  ) {
+    return Frameworks.DOCUSAURUS;
+  }
+
+  const isGitBook = $(".gitbook-root").length > 0;
+  if (isGitBook) {
+    return Frameworks.GITBOOK;
+  }
+
+  const isReadMe = $('meta[name="readme-deploy"]').length > 0;
+  if (isReadMe) {
+    return Frameworks.README;
+  }
+
+  return undefined;
+}

package/src/scraping/downloadAllImages.ts

@@ -0,0 +1,79 @@
+import path from "path";
+import downloadImage from "../downloadImage.js";
+
+// To Do: Use CheerioElement instead of any when we bump the cheerio version
+export default async function downloadAllImages(
+  $: any,
+  content: any,
+  origin: string,
+  baseDir: string,
+  modifyFileName?: any
+) {
+  if (!baseDir) {
+    console.debug("Skipping image downloading");
+    return;
+  }
+
+  // We remove duplicates because some frameworks duplicate img tags
+  // to show the image larger when clicked on.
+  const imageSrcs = [
+    ...new Set(
+      content
+        .find("img[src]")
+        .map((i, image) => $(image).attr("src"))
+        .toArray()
+    ),
+  ];
+
+  // Wait to all images to download before continuing
+  const origToNewArray = await Promise.all(
+    imageSrcs.map(async (origImageSrc: string) => {
+      // Add origin if the image tags are using relative sources
+      const imageHref = origImageSrc.startsWith("http")
+        ? origImageSrc
+        : new URL(origImageSrc, origin).href;
+
+      let fileName = removeMetadataFromExtension(path.basename(imageHref));
+      if (modifyFileName) {
+        fileName = modifyFileName(fileName);
+      }
+
+      if (!fileName) {
+        console.error("Invalid image path " + imageHref);
+        return;
+      }
+
+      const writePath = path.join(baseDir, fileName);
+
+      await downloadImage(imageHref, writePath)
+        .then(() => {
+          console.log("🖼️ - " + writePath);
+        })
+        .catch((e) => {
+          if (e.code === "EEXIST") {
+            console.log(`❌ Skipping existing image ${writePath}`);
+          } else {
+            console.error(e);
+          }
+        });
+
+      return { [origImageSrc]: writePath };
+    })
+  );
+
+  return origToNewArray.reduce(
+    (result, current) => Object.assign(result, current),
+    {}
+  );
+}
+
+function removeMetadataFromExtension(src: string) {
+  // Part of the URL standard
+  const metadataSymbols = ["?", "#"];
+
+  metadataSymbols.forEach((dividerSymbol) => {
+    // Some frameworks add metadata after the file extension, we need to remove that.
+    src = src.split(dividerSymbol)[0];
+  });
+  return src;
+}

package/src/scraping/getSitemapLinks.ts

@@ -0,0 +1,18 @@
+import axios from "axios";
+
+// Not in use.
+// Gets all links in a sitemap.
+export const getSitemapLinks = async (url: URL) => {
+  const hostname = url.hostname.replace(".", "\\.");
+  const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");
+
+  try {
+    const indexData = (await axios.default.get(url.href)).data as string;
+    const array = indexData.match(regex) as string[] | null;
+    return array || [];
+  } catch (err) {
+    console.error(err);
+    console.log("Skipping sitemap links because we encountered an error.");
+    return [];
+  }
+};

package/src/scraping/replaceImagePaths.ts

@@ -0,0 +1,21 @@
+export default function replaceImagePaths(
+  origToWritePath: object,
+  cliDir: string,
+  markdown: string
+) {
+  if (origToWritePath == null) {
+    return markdown;
+  }
+
+  // Change image paths to use the downloaded locations
+  for (const [origHref, writePath] of Object.entries(origToWritePath)) {
+    // Use relative paths within the folder we are in
+    if (writePath.startsWith(cliDir)) {
+      markdown = markdown.replaceAll(origHref, writePath.slice(cliDir.length));
+    } else {
+      markdown = markdown.replaceAll(origHref, writePath);
+    }
+  }
+
+  return markdown;
+}

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -0,0 +1,81 @@
+import path from "path";
+import axios from "axios";
+import { getHtmlWithPuppeteer } from "../browser.js";
+import { createPage } from "../util.js";
+
+export async function scrapeFileGettingFileNameFromUrl(
+  pathname: string,
+  cliDir: string,
+  origin: string,
+  overwrite: boolean,
+  scrapePageFunc: (
+    html: string,
+    origin: string,
+    cliDir: string,
+    imageBaseDir: string
+  ) => Promise<{
+    title?: string;
+    description?: string;
+    markdown?: string;
+  }>,
+  puppeteer = false,
+  baseToRemove?: string
+) {
+  // Skip scraping external links
+  if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
+    return pathname;
+  }
+
+  // Removes file name from the end
+  const splitSubpath = pathname.split("/");
+  let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
+
+  // Remove base dir if passed in
+  if (baseToRemove && folders.startsWith(baseToRemove)) {
+    folders = folders.replace(baseToRemove, "");
+  }
+
+  // TO DO: Improve this by putting each page's images in a separate
+  // folder named after the title of the page.
+  const imageBaseDir = path.join(cliDir, "images", folders);
+
+  // Scrape each page separately
+  const href = new URL(pathname, origin).href;
+  let html: string;
+  if (puppeteer) {
+    html = await getHtmlWithPuppeteer(href);
+  } else {
+    const res = await axios.default.get(href);
+    html = res.data;
+  }
+
+  const { title, description, markdown } = await scrapePageFunc(
+    html,
+    origin,
+    cliDir,
+    imageBaseDir
+  );
+
+  // Check if page didn't have content
+  if (!title && !markdown) {
+    return undefined;
+  }
+
+  const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
+
+  // Default to introduction.mdx if we encountered index.html
+  const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
+
+  // Will create subfolders as needed
+  createPage(
+    title,
+    description,
+    markdown,
+    overwrite,
+    newFileLocation,
+    fileName
+  );
+
+  // Removes first slash if we are in a folder, Mintlify doesn't need it
+  return folders ? path.join(folders, fileName).substring(1) : fileName;
+}

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -0,0 +1,50 @@
+import { NavigationEntry, isNavigation } from "../navigation.js";
+import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
+
+export async function scrapeGettingFileNameFromUrl(
+  navEntry: NavigationEntry,
+  cliDir: string,
+  origin: string,
+  overwrite: boolean,
+  scrapePageFunc: (
+    html: string,
+    origin: string,
+    cliDir: string,
+    imageBaseDir: string
+  ) => Promise<{
+    title?: string;
+    description?: string;
+    markdown?: string;
+  }>,
+  puppeteer = false,
+  baseToRemove?: string
+): Promise<NavigationEntry> {
+  if (isNavigation(navEntry)) {
+    const newPages = [];
+    for (const nestedNavEntry of navEntry.pages) {
+      newPages.push(
+        await scrapeGettingFileNameFromUrl(
+          nestedNavEntry,
+          cliDir,
+          origin,
+          overwrite,
+          scrapePageFunc,
+          puppeteer,
+          baseToRemove
+        )
+      );
+    }
+    navEntry.pages = newPages;
+    return navEntry;
+  }
+
+  return await scrapeFileGettingFileNameFromUrl(
+    navEntry,
+    cliDir,
+    origin,
+    overwrite,
+    scrapePageFunc,
+    puppeteer,
+    baseToRemove
+  );
+}

package/src/scraping/scrapePage.ts

@@ -0,0 +1,24 @@
+import path from "path";
+import { createPage, getOrigin } from "../util.js";
+
+export async function scrapePage(
+  scrapeFunc: (
+    html: string,
+    origin: string,
+    cliDir: string,
+    imageBaseDir: string
+  ) => Promise<any>,
+  href: string,
+  html: string,
+  overwrite: boolean
+) {
+  const origin = getOrigin(href);
+  const imageBaseDir = path.join(process.cwd(), "images");
+  const { title, description, markdown } = await scrapeFunc(
+    html,
+    origin,
+    process.cwd(),
+    imageBaseDir
+  );
+  createPage(title, description, markdown, overwrite, process.cwd());
+}

package/src/scraping/scrapePageCommands.ts

@@ -0,0 +1,52 @@
+import axios from "axios";
+import { scrapePage } from "./scrapePage.js";
+import { scrapeDocusaurusPage } from "./site-scrapers/scrapeDocusaurusPage.js";
+import { scrapeGitBookPage } from "./site-scrapers/scrapeGitBookPage.js";
+import { scrapeReadMePage } from "./site-scrapers/scrapeReadMePage.js";
+import { detectFramework, Frameworks } from "./detectFramework.js";
+import { getHrefFromArgs } from "../util.js";
+import { getHtmlWithPuppeteer } from "../browser.js";
+
+function validateFramework(framework) {
+  if (!framework) {
+    console.log(
+      "Could not detect the framework automatically. Please use one of:"
+    );
+    console.log("scrape-page-docusaurus");
+    console.log("scrape-page-gitbook");
+    console.log("scrape-page-readme");
+    return process.exit(1);
+  }
+}
+
+export async function scrapePageWrapper(argv, scrapeFunc, puppeteer = false) {
+  const href = getHrefFromArgs(argv);
+  let html: string;
+  if (puppeteer) {
+    html = await getHtmlWithPuppeteer(href);
+  } else {
+    const res = await axios.default.get(href);
+    html = res.data;
+  }
+  await scrapePage(scrapeFunc, href, html, argv.overwrite);
+  process.exit(0);
+}
+
+export async function scrapePageAutomatically(argv: any) {
+  const href = getHrefFromArgs(argv);
+  const res = await axios.default.get(href);
+  const html = res.data;
+  const framework = detectFramework(html);
+
+  validateFramework(framework);
+
+  console.log("Detected framework: " + framework);
+
+  if (framework === Frameworks.DOCUSAURUS) {
+    await scrapePageWrapper(argv, scrapeDocusaurusPage);
+  } else if (framework === Frameworks.GITBOOK) {
+    await scrapePageWrapper(argv, scrapeGitBookPage, true);
+  } else if (framework === Frameworks.README) {
+    await scrapePageWrapper(argv, scrapeReadMePage);
+  }
+}

package/src/scraping/scrapeSection.ts

@@ -0,0 +1,16 @@
+import { objToReadableString } from "../util.js";
+
+export async function scrapeSection(
+  scrapeFunc: any,
+  html: string,
+  origin: string,
+  overwrite: boolean
+) {
+  console.log(
+    `Started scraping${overwrite ? ", overwrite mode is on" : ""}...`
+  );
+  const groupsConfig = await scrapeFunc(html, origin, process.cwd(), overwrite);
+  console.log("Finished scraping.");
+  console.log("Add the following to your navigation in mint.json:");
+  console.log(objToReadableString(groupsConfig));
+}