mintlify 2.0.4 → 2.0.6

package/src/local-preview/index.ts

@@ -120,7 +120,8 @@ const dev = async () => {
     `);
     process.exit(1);
   }
-  shellExec(`yarn preconfigure ../../../../..${CMD_EXEC_PATH}`);
+  const relativePath = path.relative(CLIENT_PATH, CMD_EXEC_PATH);
+  shellExec(`yarn preconfigure ${relativePath}`);
   logger.succeed("Local Mintlify instance initialized");
   run();
 };

package/src/local-preview/listener/{categorizeFiles.ts → categorize.ts}

@@ -2,8 +2,9 @@
 import path from "path";
 
 import { getFileExtension, openApiCheck, getFileList } from "./utils.js";
+import { PotentialFileCategory } from "./utils/types.js";
 
-const categorizeFiles = async (contentDirectoryPath: string) => {
+export const categorizeFiles = async (contentDirectoryPath: string) => {
   const allFilesInCmdExecutionPath = await getFileList(contentDirectoryPath);
   const contentFilenames = [];
   const staticFilenames = [];
@@ -52,4 +53,57 @@ const categorizeFiles = async (contentDirectoryPath: string) => {
   return { contentFilenames, staticFilenames, openApiFiles, snippets };
 };
 
-export default categorizeFiles;
+const excludedMdFiles = ["readme", "license", "contributing", "contribute"];
+
+const supportedStaticFileExtensions = [
+  ".jpeg",
+  ".jpg",
+  ".jfif",
+  ".pjpeg",
+  ".pjp",
+  ".png",
+  ".svg",
+  ".svgz",
+  ".ico",
+  ".webp",
+  ".gif",
+  ".apng",
+  ".avif",
+  ".bmp",
+  ".mp4",
+];
+
+export const getCategory = (filePath: string): PotentialFileCategory => {
+  filePath = filePath.toLowerCase();
+
+  const parsed = path.parse(filePath);
+
+  if (parsed.base === "mint.json") {
+    return "mintConfig";
+  }
+
+  const fileName = parsed.name;
+  const extension = parsed.ext;
+  if (
+    filePath.startsWith("_snippets") &&
+    (extension === ".mdx" || extension === ".md")
+  ) {
+    return "snippet";
+  } else if (extension === ".mdx") {
+    return "page";
+  } else if (extension === ".md") {
+    // Exclude common markdown files people don't want on their docs website
+    if (excludedMdFiles.includes(fileName)) {
+      throw new Error("Excluded Md File");
+    }
+    return "page";
+  } else if (extension === ".yaml" || extension === ".yml") {
+    return "potentialYamlOpenApiSpec";
+  } else if (extension === ".json") {
+    return "potentialJsonOpenApiSpec";
+  } else if (supportedStaticFileExtensions.includes(extension)) {
+    return "staticFile";
+  }
+
+  throw new Error("Unsupported File Type, No change enacted");
+};

package/src/local-preview/listener/generate.ts

@@ -4,7 +4,7 @@ import { outputFile } from "fs-extra";
 import path from "path";
 import createPage from "./utils/createPage.js";
 import { OpenApiFile } from "./utils/types.js";
-import categorizeFiles from "./categorizeFiles.js";
+import { categorizeFiles } from "./categorize.js";
 import { CMD_EXEC_PATH } from "../../constants.js";
 import { getConfigObj } from "./utils/mintConfigFile.js";
 const { readFile } = _promises;

package/src/local-preview/listener/index.ts

@@ -1,12 +1,13 @@
 import chokidar from "chokidar";
 import fse from "fs-extra";
 import pathUtil from "path";
-import { fileIsMdxOrMd } from "./utils/fileIsMdxOrMd.js";
 import { openApiCheck } from "./utils.js";
 import { updateGeneratedNav, updateOpenApiFiles } from "./update.js";
 import { CLIENT_PATH, CMD_EXEC_PATH } from "../../constants.js";
 import { promises as _promises } from "fs";
 import createPage from "./utils/createPage.js";
+import { getCategory } from "./categorize.js";
+import { PotentialFileCategory, FileCategory } from "./utils/types.js";
 
 const { readFile } = _promises;
 
@@ -14,156 +15,175 @@ const listener = () => {
   chokidar
     .watch(CMD_EXEC_PATH, {
       ignoreInitial: true,
-      ignored: ["node_modules", ".git"],
+      ignored: ["node_modules", ".git", ".idea"],
       cwd: CMD_EXEC_PATH,
     })
-    .on("all", async (event, filename) => {
-      if (event === "unlink" || event === "unlinkDir") {
-        if (fileIsMdxOrMd(filename)) {
-          const targetPath = pathUtil.join(
-            CLIENT_PATH,
-            "src",
-            "_props",
-            filename
-          );
-          await fse.remove(targetPath);
-          console.log(
-            `${
-              filename.startsWith("_snippets") ? "Snippet" : "Page"
-            } deleted: `,
-            filename
-          );
-        } else if (filename === "mint.json") {
-          const targetPath = pathUtil.join(
-            CLIENT_PATH,
-            "src",
-            "_props",
-            "mint.json"
-          );
-          await fse.remove(targetPath);
-          console.log(
-            "⚠️ mint.json deleted. Please create a new mint.json file as it is mandatory."
-          );
-          process.exit(1);
-        } else {
-          const extension = pathUtil.parse(filename).ext.slice(1);
-          if (
-            extension &&
-            (extension === "json" ||
-              extension === "yaml" ||
-              extension === "yml")
-          ) {
-            await updateOpenApiFiles();
-            await updateGeneratedNav();
-            return;
-          }
-          // all other files
-          const targetPath = pathUtil.join(CLIENT_PATH, "public", filename);
+    .on("add", async (filename: string) => {
+      try {
+        const category = await onUpdateEvent(filename);
+        switch (category) {
+          case "page":
+            console.log("New page detected: ", filename);
+            break;
+          case "snippet":
+            console.log("New snippet detected: ", filename);
+            break;
+          case "mintConfig":
+            console.log("Config added");
+            break;
+          case "openApi":
+            console.log("OpenApi spec added: ", filename);
+            break;
+          case "staticFile":
+            console.log("Static file added: ", filename);
+            break;
+        }
+      } catch (error) {
+        console.error(error.message);
+      }
+    })
+    .on("change", async (filename: string) => {
+      try {
+        const category = await onUpdateEvent(filename);
+        switch (category) {
+          case "page":
+            console.log("Page edited: ", filename);
+            break;
+          case "snippet":
+            console.log("Snippet edited: ", filename);
+            break;
+          case "mintConfig":
+            console.log("Config edited");
+            break;
+          case "openApi":
+            console.log("OpenApi spec edited: ", filename);
+            break;
+          case "staticFile":
+            console.log("Static file edited: ", filename);
+            break;
+        }
+      } catch (error) {
+        console.error(error.message);
+      }
+    })
+    .on("unlink", async (filename: string) => {
+      try {
+        const potentialCategory = getCategory(filename);
+        const targetPath = getTargetPath(potentialCategory, filename);
+        if (
+          potentialCategory === "page" ||
+          potentialCategory === "snippet" ||
+          potentialCategory === "mintConfig" ||
+          potentialCategory === "staticFile"
+        ) {
           await fse.remove(targetPath);
-          console.log("Static file deleted: ", filename);
         }
-      } else {
-        const filePath = pathUtil.join(CMD_EXEC_PATH, filename);
-        let regenerateNav = false;
-        if (fileIsMdxOrMd(filename)) {
-          const targetPath = pathUtil.join(
-            CLIENT_PATH,
-            "src",
-            "_props",
-            filename
-          );
-          if (filename.startsWith("_snippets")) {
-            const sourcePath = pathUtil.join(CMD_EXEC_PATH, filename);
-            await fse.remove(targetPath);
-            await fse.copy(sourcePath, targetPath);
-            return;
-          }
-          regenerateNav = true;
-
-          const contentStr = (await readFile(filePath)).toString();
-          const { pageContent } = await createPage(
-            filename,
-            contentStr,
-            CMD_EXEC_PATH,
-            []
-          );
-          await fse.outputFile(targetPath, pageContent, {
-            flag: "w",
-          });
-          switch (event) {
-            case "add":
-            case "addDir":
-              console.log("New page detected: ", filename);
-              break;
-            default:
-              console.log("Page edited: ", filename);
-              break;
-          }
-        } else if (filename === "mint.json") {
-          regenerateNav = true;
-          const targetPath = pathUtil.join(
-            CLIENT_PATH,
-            "src",
-            "_props",
-            "mint.json"
-          );
-          await fse.copy(filePath, targetPath);
-          switch (event) {
-            case "add":
-            case "addDir":
-              console.log("Config added");
-              break;
-            default:
-              console.log("Config edited");
-              break;
-          }
-        } else {
-          const extension = pathUtil.parse(filename).ext.slice(1);
-          let isOpenApi = false;
-          if (
-            extension &&
-            (extension === "json" ||
-              extension === "yaml" ||
-              extension === "yml")
-          ) {
-            const openApiInfo = await openApiCheck(
-              pathUtil.join(CMD_EXEC_PATH, filename)
+        switch (potentialCategory) {
+          case "page":
+            console.log(`Page deleted: ${filename}`);
+            break;
+          case "snippet":
+            console.log(`Snippet deleted: ${filename}`);
+            break;
+          case "mintConfig":
+            console.log(
+              "⚠️ mint.json deleted. Please create a new mint.json file as it is mandatory."
             );
-            isOpenApi = openApiInfo.isOpenApi;
-            if (isOpenApi) {
-              await updateOpenApiFiles();
-              regenerateNav = true;
-            }
-          }
-          if (!isOpenApi) {
-            // all other files
-            const targetPath = pathUtil.join(CLIENT_PATH, "public", filename);
-            await fse.copy(filePath, targetPath);
-          }
-          switch (event) {
-            case "add":
-            case "addDir":
-              if (isOpenApi) {
-                console.log("OpenApi file added: ", filename);
-              } else {
-                console.log("Static file added: ", filename);
-              }
-              break;
-            default:
-              if (isOpenApi) {
-                console.log("OpenApi file edited: ", filename);
-              } else {
-                console.log("Static file edited: ", filename);
-              }
-              break;
-          }
-        }
-        if (regenerateNav) {
-          // TODO: Instead of re-generating the entire nav, optimize by just updating the specific page that changed.
-          await updateGeneratedNav();
+            process.exit(1);
+          case "potentialJsonOpenApiSpec":
+          case "potentialYamlOpenApiSpec":
+            await updateOpenApiFiles();
+            await updateGeneratedNav();
+            break;
+          case "staticFile":
+            console.log("Static file deleted: ", filename);
+            break;
         }
+      } catch (error) {
+        console.error(error.message);
       }
     });
 };
 
+const getTargetPath = (
+  potentialCategory: PotentialFileCategory,
+  filePath: string
+): string => {
+  switch (potentialCategory) {
+    case "page":
+    case "snippet":
+      return pathUtil.join(CLIENT_PATH, "src", "_props", filePath);
+    case "mintConfig":
+      return pathUtil.join(CLIENT_PATH, "src", "_props", "mint.json");
+    case "potentialYamlOpenApiSpec":
+    case "potentialJsonOpenApiSpec":
+      return pathUtil.join(CLIENT_PATH, "src", "_props", "openApiFiles.json");
+    case "staticFile":
+      return pathUtil.join(CLIENT_PATH, "public", filePath);
+    default:
+      throw new Error("Invalid category");
+  }
+};
+
+/**
+ * This function is called when a file is added or changed
+ * @param filename
+ * @returns FileCategory
+ */
+const onUpdateEvent = async (filename: string): Promise<FileCategory> => {
+  const filePath = pathUtil.join(CMD_EXEC_PATH, filename);
+  const potentialCategory = getCategory(filename);
+  const targetPath = getTargetPath(potentialCategory, filename);
+  let regenerateNav = false;
+  let category: FileCategory =
+    potentialCategory === "potentialYamlOpenApiSpec" ||
+    potentialCategory === "potentialJsonOpenApiSpec"
+      ? "staticFile"
+      : potentialCategory;
+  switch (potentialCategory) {
+    case "page":
+      regenerateNav = true;
+      const contentStr = (await readFile(filePath)).toString();
+      const { pageContent } = await createPage(
+        filename,
+        contentStr,
+        CMD_EXEC_PATH,
+        []
+      );
+      await fse.outputFile(targetPath, pageContent, {
+        flag: "w",
+      });
+      break;
+    case "snippet":
+      await fse.copy(filePath, targetPath);
+      break;
+    case "mintConfig":
+      regenerateNav = true;
+      await fse.copy(filePath, targetPath);
+      break;
+    case "potentialYamlOpenApiSpec":
+    case "potentialJsonOpenApiSpec":
+      let isOpenApi = false;
+      const openApiInfo = await openApiCheck(
+        pathUtil.join(CMD_EXEC_PATH, filename)
+      );
+      isOpenApi = openApiInfo.isOpenApi;
+      if (isOpenApi) {
+        // TODO: Instead of re-generating all openApi files, optimize by just updating the specific file that changed.
+        await updateOpenApiFiles();
+        regenerateNav = true;
+        category = "openApi";
+      }
+      break;
+    case "staticFile":
+      await fse.copy(filePath, targetPath);
+      break;
+  }
+  if (regenerateNav) {
+    // TODO: Instead of re-generating the entire nav, optimize by just updating the specific page that changed.
+    await updateGeneratedNav();
+  }
+  return category;
+};
+
 export default listener;

package/src/local-preview/listener/update.ts

@@ -5,7 +5,7 @@ import { getConfigPath } from "./utils/mintConfigFile.js";
 import { CLIENT_PATH, CMD_EXEC_PATH } from "../../constants.js";
 import { join } from "path";
 import { generateNav } from "./generate.js";
-import categorizeFiles from "./categorizeFiles.js";
+import { categorizeFiles } from "./categorize.js";
 
 const { readFile } = _promises;
 

package/src/local-preview/listener/utils/types.ts

@@ -1 +1,16 @@
 export type OpenApiFile = { name: string; spec: any };
+
+export type PotentialFileCategory =
+  | "page"
+  | "snippet"
+  | "mintConfig"
+  | "potentialYamlOpenApiSpec"
+  | "potentialJsonOpenApiSpec"
+  | "staticFile";
+
+export type FileCategory =
+  | "page"
+  | "snippet"
+  | "mintConfig"
+  | "openApi"
+  | "staticFile";

package/src/scraping/combineNavWithEmptyGroupTitles.ts

@@ -1,9 +1,9 @@
-import { Navigation } from "../navigation.js";
-
-export default function combineNavWithEmptyGroupTitles(navArray: Navigation[]) {
+export default function combineNavWithEmptyGroupTitles(
+  navArray: MintNavigation[]
+): MintNavigation[] {
   let newNavArray = [];
 
-  navArray.forEach((nav: Navigation) => {
+  navArray.forEach((nav: MintNavigation) => {
     // The first run through the loop will always have -1 as the index.
     // JavaScript returns undefined when we look for an index outside the size of the array.
     const prev = newNavArray[newNavArray.length - 1];

package/src/scraping/downloadAllImages.ts

@@ -1,5 +1,9 @@
 import path from "path";
-import downloadImage from "../downloadImage.js";
+import downloadImage, {
+  cleanImageSrc,
+  isValidImageSrc,
+  removeMetadataFromImageSrc,
+} from "../downloadImage.js";
 
 // To Do: Use CheerioElement instead of any when we bump the cheerio version
 export default async function downloadAllImages(
@@ -7,6 +11,7 @@ export default async function downloadAllImages(
   content: any,
   origin: string,
   baseDir: string,
+  overwrite: boolean,
   modifyFileName?: any
 ) {
   if (!baseDir) {
@@ -27,42 +32,21 @@ export default async function downloadAllImages(
 
   // Wait to all images to download before continuing
   const origToNewArray = await Promise.all(
-    imageSrcs.map(async (origImageSrc: string) => {
-      // We do not support downloading base64 in-line images.
-      if (origImageSrc.startsWith("data:")) {
-        return undefined;
-      }
+    imageSrcs.map(async (imageSrc: string) => {
+      if (!isValidImageSrc(imageSrc)) return;
 
-      // Add origin if the image tags are using relative sources
-      const imageHref = origImageSrc.startsWith("http")
-        ? origImageSrc
-        : new URL(origImageSrc, origin).href;
+      const imageHref = cleanImageSrc(imageSrc, origin);
 
-      let fileName = removeMetadataFromExtension(path.basename(imageHref));
+      let fileName = removeMetadataFromImageSrc(path.basename(imageHref));
       if (modifyFileName) {
         fileName = modifyFileName(fileName);
       }
 
-      if (!fileName) {
-        console.error("Invalid image path " + imageHref);
-        return;
-      }
-
       const writePath = path.join(baseDir, fileName);
 
-      await downloadImage(imageHref, writePath)
-        .then(() => {
-          console.log("🖼️ - " + writePath);
-        })
-        .catch((e) => {
-          if (e.code === "EEXIST") {
-            console.log(`❌ Skipping existing image ${writePath}`);
-          } else {
-            console.error(e);
-          }
-        });
+      await downloadImage(imageHref, writePath, overwrite);
 
-      return { [origImageSrc]: writePath };
+      return { [imageSrc]: writePath };
     })
   );
 
@@ -71,14 +55,3 @@ export default async function downloadAllImages(
     {}
   );
 }
-
-function removeMetadataFromExtension(src: string) {
-  // Part of the URL standard
-  const metadataSymbols = ["?", "#"];
-
-  metadataSymbols.forEach((dividerSymbol) => {
-    // Some frameworks add metadata after the file extension, we need to remove that.
-    src = src.split(dividerSymbol)[0];
-  });
-  return src;
-}

package/src/scraping/downloadLogoImage.ts

@@ -0,0 +1,24 @@
+import path from "path";
+import downloadImage, {
+  cleanImageSrc,
+  isValidImageSrc,
+  removeMetadataFromImageSrc,
+} from "../downloadImage.js";
+import { getFileExtension } from "../util.js";
+
+// To Do: Use CheerioElement instead of any when we bump the cheerio version
+export default async function downloadLogoImage(
+  imageSrc: string,
+  imageBaseDir: string,
+  origin: string,
+  overwrite: boolean
+) {
+  if (!isValidImageSrc(imageSrc)) return;
+
+  const imageHref = cleanImageSrc(imageSrc, origin);
+
+  const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
+  const imagePath = path.join(imageBaseDir, "logo", "logo-light-mode." + ext);
+
+  await downloadImage(imageHref, imagePath, overwrite);
+}

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -13,6 +13,7 @@ export async function scrapeFileGettingFileNameFromUrl(
     origin: string,
     cliDir: string,
     imageBaseDir: string,
+    overwrite: boolean,
     version: string | undefined
   ) => Promise<{
     title?: string;
@@ -22,7 +23,7 @@ export async function scrapeFileGettingFileNameFromUrl(
   puppeteer = false,
   version: string | undefined,
   baseToRemove?: string
-) {
+): Promise<MintNavigationEntry> {
   // Skip scraping external links
   if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
     return pathname;
@@ -56,6 +57,7 @@ export async function scrapeFileGettingFileNameFromUrl(
     origin,
     cliDir,
     imageBaseDir,
+    overwrite,
     version
   );
 

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -1,8 +1,7 @@
-import { NavigationEntry, isNavigation } from "../navigation.js";
 import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
 
 export async function scrapeGettingFileNameFromUrl(
-  navEntry: NavigationEntry,
+  navEntry: MintNavigationEntry,
   cliDir: string,
   origin: string,
   overwrite: boolean,
@@ -11,6 +10,7 @@ export async function scrapeGettingFileNameFromUrl(
     origin: string,
     cliDir: string,
     imageBaseDir: string,
+    overwrite: boolean,
     version: string | undefined
   ) => Promise<{
     title?: string;
@@ -20,8 +20,8 @@ export async function scrapeGettingFileNameFromUrl(
   puppeteer = false,
   version: string | undefined,
   baseToRemove?: string
-): Promise<NavigationEntry> {
-  if (isNavigation(navEntry)) {
+): Promise<MintNavigationEntry> {
+  if (typeof navEntry !== "string") {
     const newPages = [];
     for (const nestedNavEntry of navEntry.pages) {
       newPages.push(

package/src/scraping/scrapePage.ts

@@ -2,25 +2,22 @@ import path from "path";
 import { createPage, getOrigin } from "../util.js";
 
 export async function scrapePage(
-  scrapeFunc: (
-    html: string,
-    origin: string,
-    cliDir: string,
-    imageBaseDir: string,
-    version: string | undefined
-  ) => Promise<any>,
+  scrapeFunc: ScrapePageFn,
   href: string,
   html: string,
   overwrite: boolean,
   version: string | undefined
 ) {
   const origin = getOrigin(href);
-  const imageBaseDir = path.join(process.cwd(), "images");
+  const cwd = process.cwd();
+  const imageBaseDir = path.join(cwd, "images");
+
   const { title, description, markdown } = await scrapeFunc(
     html,
     origin,
-    process.cwd(),
+    cwd,
     imageBaseDir,
+    overwrite,
     version
   );
   createPage(title, description, markdown, overwrite, process.cwd());

package/src/scraping/scrapePageCommands.ts

@@ -21,7 +21,7 @@ function validateFramework(framework) {
 
 export async function scrapePageWrapper(
   argv: any,
-  scrapeFunc: any,
+  scrapeFunc: ScrapePageFn,
   options?: { version?: string; puppeteer?: boolean }
 ) {
   const href = getHrefFromArgs(argv);
@@ -32,7 +32,7 @@ export async function scrapePageWrapper(
     const res = await axios.get(href);
     html = res.data;
   }
-  await scrapePage(scrapeFunc, href, html, argv.overwrite, options?.version);
+  await scrapePage(scrapeFunc, href, html, !!argv.overwrite, options?.version);
   process.exit(0);
 }