@mintlify/previewing 3.0.0

package/src/local-preview/listener/utils/createPage.ts

@@ -0,0 +1,211 @@
+import matter from "gray-matter";
+import isAbsoluteUrl from "is-absolute-url";
+import { remark } from "remark";
+import remarkFrontmatter from "remark-frontmatter";
+import remarkGfm from "remark-gfm";
+import remarkMdx from "remark-mdx";
+import { visit } from "unist-util-visit";
+
+const createPage = async (
+  pagePath: string,
+  pageContent: string,
+  contentDirectoryPath: string,
+  openApiFiles: OpenApiFile[]
+) => {
+  const { data: metadata } = matter(pageContent);
+  try {
+    const parsedContent = await preParseMdx(pageContent, contentDirectoryPath);
+    pageContent = parsedContent;
+  } catch (error) {
+    pageContent = `🚧 A parsing error occured. Please contact the owner of this website.`;
+  }
+
+  // Replace .mdx so we can pass file paths into this function
+  const slug = pagePath.replace(/\.mdx?$/, "");
+  let defaultTitle = slugToTitle(slug);
+  // Append data from OpenAPI if it exists
+  const { title, description } = getOpenApiTitleAndDescription(
+    openApiFiles,
+    metadata?.openapi
+  );
+
+  if (title) {
+    defaultTitle = title;
+  }
+
+  const pageMetadata = {
+    title: defaultTitle,
+    description,
+    ...metadata,
+    href: optionallyAddLeadingSlash(slug),
+  };
+
+  return {
+    pageMetadata,
+    pageContent,
+    slug: removeLeadingSlash(slug),
+  };
+};
+
+const preParseMdx = async (
+  fileContent: string,
+  contentDirectoryPath: string
+) => {
+  const removeContentDirectoryPath = (filePath: string) => {
+    const pathArr = createPathArr(filePath);
+    const contentDirectoryPathArr = createPathArr(contentDirectoryPath);
+    contentDirectoryPathArr.reverse().forEach((dir: string, index: number) => {
+      if (pathArr[index] === dir) {
+        pathArr.pop();
+      }
+    });
+    return "/" + pathArr.join("/");
+  };
+
+  const removeContentDirectoryPaths = () => {
+    return (tree) => {
+      visit(tree, (node) => {
+        if (node == null) {
+          return;
+        }
+        if (node.name === "img" || node.name === "source") {
+          const srcAttrIndex = node.attributes.findIndex(
+            (attr) => attr?.name === "src"
+          );
+          const nodeUrl = node.attributes[srcAttrIndex].value;
+          if (
+            // <img/> component
+            srcAttrIndex !== -1 &&
+            !isAbsoluteUrl(nodeUrl) &&
+            !isDataString(nodeUrl)
+          ) {
+            node.attributes[srcAttrIndex].value =
+              removeContentDirectoryPath(nodeUrl);
+          }
+        } else if (
+          // ![]() format
+          node.type === "image" &&
+          node.url &&
+          !isAbsoluteUrl(node.url) &&
+          !isDataString(node.url)
+        ) {
+          node.url = removeContentDirectoryPath(node.url);
+        }
+      });
+      return tree;
+    };
+  };
+
+  const file = await remark()
+    .use(remarkMdx)
+    .use(remarkGfm)
+    .use(remarkFrontmatter, ["yaml", "toml"])
+    .use(removeContentDirectoryPaths)
+    .process(fileContent);
+  return String(file);
+};
+
+const removeLeadingSlash = (str: string) => {
+  const path = createPathArr(str);
+  return path.join("/");
+};
+
+const createPathArr = (path: string) => {
+  return path.split("/").filter((dir) => dir !== "");
+};
+
+const isDataString = (str: string) => str.startsWith("data:");
+
+const getOpenApiTitleAndDescription = (
+  openApiFiles: OpenApiFile[],
+  openApiMetaField: string
+) => {
+  const { operation } = getOpenApiOperationMethodAndEndpoint(
+    openApiFiles,
+    openApiMetaField
+  );
+
+  if (operation == null) {
+    return {};
+  }
+
+  return {
+    title: operation.summary,
+    description: operation.description,
+  };
+};
+
+const getOpenApiOperationMethodAndEndpoint = (
+  openApiFiles: OpenApiFile[],
+  openApiMetaField: string
+) => {
+  const { endpoint, method, filename } =
+    extractMethodAndEndpoint(openApiMetaField);
+
+  let path;
+
+  openApiFiles?.forEach((file) => {
+    const openApiFile = file.spec;
+    const openApiPath = openApiFile.paths && openApiFile.paths[endpoint];
+    const isFilenameOrNone = !filename || filename === file.filename;
+    if (openApiPath && isFilenameOrNone) {
+      path = openApiPath;
+    }
+  });
+
+  if (path == null) {
+    return {};
+  }
+
+  let operation;
+  if (method) {
+    operation = path[method.toLowerCase()];
+  } else {
+    const firstOperationKey = Object.keys(path)[0];
+    operation = path[firstOperationKey];
+  }
+
+  return {
+    operation,
+    method,
+    endpoint,
+  };
+};
+
+const extractMethodAndEndpoint = (openApiMetaField: string) => {
+  const methodRegex = /(get|post|put|delete|patch)\s/i;
+  const trimmed = openApiMetaField.trim();
+  const foundMethod = trimmed.match(methodRegex);
+
+  const startIndexOfMethod = foundMethod
+    ? openApiMetaField.indexOf(foundMethod[0])
+    : 0;
+  const endIndexOfMethod = foundMethod
+    ? startIndexOfMethod + foundMethod[0].length - 1
+    : 0;
+
+  const filename = openApiMetaField.substring(0, startIndexOfMethod).trim();
+
+  return {
+    method: foundMethod ? foundMethod[0].slice(0, -1).toUpperCase() : undefined,
+    endpoint: openApiMetaField.substring(endIndexOfMethod).trim(),
+    filename: filename ? filename : undefined,
+  };
+};
+
+function optionallyAddLeadingSlash(path: string) {
+  if (path.startsWith("/")) {
+    return path;
+  }
+  return "/" + path;
+}
+
+export const slugToTitle = (slug: string) => {
+  const slugArr = slug.split("/");
+  let defaultTitle = slugArr[slugArr.length - 1].split("-").join(" "); //replace all dashes
+  defaultTitle = defaultTitle.split("_").join(" "); //replace all underscores
+  defaultTitle = defaultTitle.charAt(0).toUpperCase() + defaultTitle.slice(1); //capitalize first letter
+  return defaultTitle;
+};
+
+export default createPage;

package/src/local-preview/listener/utils/getOpenApiContext.ts

@@ -0,0 +1,77 @@
+export const extractMethodAndEndpoint = (
+  api: string
+): { method?: string; endpoint: string; filename?: string } => {
+  const methodRegex = /(get|post|put|delete|patch)\s/i;
+  const trimmed = api.trim();
+  const foundMethod = trimmed.match(methodRegex);
+
+  const startIndexOfMethod = foundMethod ? api.indexOf(foundMethod[0]) : 0;
+  const endIndexOfMethod = foundMethod
+    ? startIndexOfMethod + foundMethod[0].length - 1
+    : 0;
+
+  const filename = api.substring(0, startIndexOfMethod).trim();
+
+  return {
+    method: foundMethod ? foundMethod[0].slice(0, -1).toUpperCase() : undefined,
+    endpoint: api.substring(endIndexOfMethod).trim(),
+    filename: filename ? filename : undefined,
+  };
+};
+
+export const getOpenApiOperationMethodAndEndpoint = (
+  openApi: any,
+  openApiMetaField: string
+) => {
+  const { endpoint, method, filename } =
+    extractMethodAndEndpoint(openApiMetaField);
+
+  let path: any;
+
+  openApi.files?.forEach((file: any) => {
+    const openApiFile = file.openapi;
+    const openApiPath = openApiFile.paths && openApiFile.paths[endpoint];
+    const isFilenameOrNone = !filename || filename === file.name;
+    if (openApiPath && isFilenameOrNone) {
+      path = openApiPath;
+    }
+  });
+
+  if (path == null) {
+    return {};
+  }
+
+  let operation;
+  if (method) {
+    operation = path[method.toLowerCase()];
+  } else {
+    const firstOperationKey = Object.keys(path)[0];
+    operation = path[firstOperationKey];
+  }
+
+  return {
+    operation,
+    method,
+    endpoint,
+  };
+};
+
+export const getOpenApiTitleAndDescription = (openApi, openApiMetaField) => {
+  if (openApi == null || !openApiMetaField || openApiMetaField == null) {
+    return {};
+  }
+
+  const { operation } = getOpenApiOperationMethodAndEndpoint(
+    openApi,
+    openApiMetaField
+  );
+
+  if (operation == null) {
+    return {};
+  }
+
+  return {
+    title: operation.summary,
+    description: operation.description,
+  };
+};

package/src/local-preview/listener/utils/mintConfigFile.ts

@@ -0,0 +1,28 @@
+import { promises as _promises } from "fs";
+import { pathExists } from "fs-extra";
+import pathUtil from "path";
+
+const { readFile } = _promises;
+
+// TODO: Put in prebuild package
+export const getConfigPath = async (
+  contentDirectoryPath: string
+): Promise<string | null> => {
+  if (await pathExists(pathUtil.join(contentDirectoryPath, "mint.json"))) {
+    return pathUtil.join(contentDirectoryPath, "mint.json");
+  }
+  return null;
+};
+
+// TODO: Put in prebuild package
+export const getConfigObj = async (
+  contentDirectoryPath: string
+): Promise<any> => {
+  const configPath = await getConfigPath(contentDirectoryPath);
+  let configObj = null;
+  if (configPath) {
+    const configContents = await readFile(configPath);
+    configObj = await JSON.parse(configContents.toString());
+  }
+  return configObj;
+};

package/src/local-preview/listener/utils/toTitleCase.ts

@@ -0,0 +1,40 @@
+export function toTitleCase(text: string) {
+  const smallWords = /^(a|an|and|as|at|but|by|en|for|if|in|nor|of|on|or|per|the|to|v.?|vs.?|via)$/i;
+  const alphanumericPattern = /([A-Za-z0-9\u00C0-\u00FF])/;
+  const wordSeparators = /([ :–—-])/;
+
+  return text
+    .split(wordSeparators)
+    .map(function (current, index, array) {
+      if (
+        // Check for small words
+        current.search(smallWords) > -1 &&
+        // Skip first and last word
+        index !== 0 &&
+        index !== array.length - 1 &&
+        // Ignore title end and subtitle start
+        array[index - 3] !== ':' &&
+        array[index + 1] !== ':' &&
+        // Ignore small words that start a hyphenated phrase
+        (array[index + 1] !== '-' || (array[index - 1] === '-' && array[index + 1] === '-'))
+      ) {
+        return current.toLowerCase();
+      }
+
+      // Ignore intentional capitalization
+      if (current.substring(1).search(/[A-Z]|\../) > -1) {
+        return current;
+      }
+
+      // Ignore URLs
+      if (array[index + 1] === ':' && array[index + 2] !== '') {
+        return current;
+      }
+
+      // Capitalize the first letter
+      return current.replace(alphanumericPattern, function (match: string) {
+        return match.toUpperCase();
+      });
+    })
+    .join('');
+}

package/src/local-preview/listener/utils/types.ts

@@ -0,0 +1,14 @@
+export type PotentialFileCategory =
+  | "page"
+  | "snippet"
+  | "mintConfig"
+  | "potentialYamlOpenApiSpec"
+  | "potentialJsonOpenApiSpec"
+  | "staticFile";
+
+export type FileCategory =
+  | "page"
+  | "snippet"
+  | "mintConfig"
+  | "openApi"
+  | "staticFile";

package/src/local-preview/listener/utils.ts

@@ -0,0 +1,87 @@
+import SwaggerParser from "@apidevtools/swagger-parser";
+import { promises as _promises } from "fs";
+
+const { readdir, stat } = _promises;
+
+export const getFileExtension = (filename: string) => {
+  return (
+    filename.substring(filename.lastIndexOf(".") + 1, filename.length) ||
+    filename
+  );
+};
+
+export type OpenApiCheckResult = {
+  spec: any;
+  isOpenApi: boolean;
+};
+
+export const openApiCheck = async (
+  path: string
+): Promise<OpenApiCheckResult> => {
+  let spec;
+  let isOpenApi = false;
+  try {
+    spec = await SwaggerParser.validate(path);
+    isOpenApi = true;
+  } catch {
+    // not valid openApi
+  }
+  return { spec, isOpenApi };
+};
+
+export const filterOutNullInGroup = (group: MintNavigation) => {
+  const newPages = filterOutNullInPages(group.pages);
+  const newGroup = {
+    ...group,
+    pages: newPages,
+  };
+  return newGroup;
+};
+
+const filterOutNullInPages = (pages: (MintNavigationEntry | null)[]) => {
+  const newPages: MintNavigationEntry[] = [];
+  pages.forEach((page) => {
+    if (page == null) {
+      return;
+    }
+    if (page.hasOwnProperty("pages")) {
+      const group = page as MintNavigation;
+      const newGroup = filterOutNullInGroup(group);
+      newPages.push(newGroup);
+    } else {
+      newPages.push(page);
+    }
+  });
+
+  return newPages;
+};
+
+export const getFileList = async (dirName: string, og = dirName) => {
+  let files: string[] = [];
+  const items = await readdir(dirName, { withFileTypes: true });
+
+  for (const item of items) {
+    if (item.isDirectory()) {
+      files = [...files, ...(await getFileList(`${dirName}/${item.name}`, og))];
+    } else {
+      const path = `${dirName}/${item.name}`;
+      const name = path.replace(og, "");
+      files.push(name);
+    }
+  }
+
+  return files;
+};
+
+export const isFileSizeValid = async (
+  path: string,
+  maxFileSizeInMb: number
+): Promise<boolean> => {
+  const maxFileSizeBytes = maxFileSizeInMb * 1000000;
+  const stats = await stat(path);
+  return stats.size <= maxFileSizeBytes;
+};
+
+export function isError(obj: unknown) {
+  return Object.prototype.toString.call(obj) === "[object Error]";
+}

package/src/types.d.ts

@@ -0,0 +1,35 @@
+type MintPage = {
+  title?: string;
+  description?: string;
+  markdown?: string;
+};
+
+type MintNavigation = {
+  group: string;
+  version?: string;
+  pages: MintNavigationEntry[];
+};
+type MintNavigationEntry = string | MintNavigation;
+
+type ScrapePageFn = (
+  html: string,
+  origin: string,
+  cliDir: string,
+  imageBaseDir: string,
+  overwrite: boolean,
+  version: string | undefined
+) => Promise<MintPage>;
+
+type ScrapeSectionFn = (
+  html: string,
+  origin: string,
+  cliDir: string,
+  imageBaseDir: string,
+  overwrite: boolean,
+  version: string | undefined
+) => Promise<MintNavigationEntry[]>;
+
+type OpenApiFile = {
+  filename: string;
+  spec: any;
+};

package/src/util.ts

@@ -0,0 +1,154 @@
+import { mkdirSync, writeFileSync } from "fs";
+import Ora, { Ora as OraType } from "ora";
+import path from "path";
+import shell from "shelljs";
+
+export const MintConfig = (
+  name: string,
+  color: string,
+  ctaName: string,
+  ctaUrl: string,
+  filename: string
+) => {
+  return {
+    name,
+    logo: "",
+    favicon: "",
+    colors: {
+      primary: color,
+    },
+    topbarLinks: [],
+    topbarCtaButton: {
+      name: ctaName,
+      url: ctaUrl,
+    },
+    anchors: [],
+    navigation: [
+      {
+        group: "Home",
+        pages: [filename],
+      },
+    ],
+    // footerSocials: {}, // support object type for footer tyoes
+  };
+};
+
+export const Page = (
+  title: string,
+  description?: string,
+  markdown?: string
+) => {
+  // If we are an empty String we want to add two quotes,
+  // if we added as we went we would detect the first quote
+  // as the closing quote.
+  const startsWithQuote = title.startsWith('"');
+  const endsWithQuote = title.startsWith('"');
+  if (!startsWithQuote) {
+    title = '"' + title;
+  }
+  if (!endsWithQuote) {
+    title = title + '"';
+  }
+
+  const optionalDescription = description
+    ? `\ndescription: "${description}"`
+    : "";
+  return `---\ntitle: ${title}${optionalDescription}\n---\n\n${markdown}`;
+};
+
+export function getOrigin(url: string) {
+  // eg. https://google.com -> https://google.com
+  // https://google.com/page -> https://google.com
+  return new URL(url).origin;
+}
+
+export function objToReadableString(objs: MintNavigationEntry[]) {
+  // Two spaces as indentation
+  return objs.map((obj) => JSON.stringify(obj, null, 2)).join(",\n");
+}
+
+export const toFilename = (title: string) => {
+  // Gets rid of special characters at the start and end
+  // of the name by converting to spaces then using trim.
+  return title
+    .replace(/[^a-z0-9]/gi, " ")
+    .trim()
+    .replace(/ /g, "-")
+    .toLowerCase();
+};
+
+export const addMdx = (fileName: string) => {
+  if (fileName.endsWith(".mdx")) {
+    return fileName;
+  }
+  return fileName + ".mdx";
+};
+
+export const createPage = (
+  title: string,
+  description?: string,
+  markdown?: string,
+  overwrite = false,
+  rootDir = "",
+  fileName?: string
+) => {
+  const writePath = path.join(rootDir, addMdx(fileName || toFilename(title)));
+
+  // Create the folders needed if they're missing
+  mkdirSync(rootDir, { recursive: true });
+
+  // Write the page to memory
+  if (overwrite) {
+    writeFileSync(writePath, Page(title, description, markdown));
+    console.log("✏️ - " + writePath);
+  } else {
+    try {
+      writeFileSync(writePath, Page(title, description, markdown), {
+        flag: "wx",
+      });
+      console.log("✏️ - " + writePath);
+    } catch (e) {
+      // We do a try-catch instead of an if-statement to avoid a race condition
+      // of the file being created after we started writing.
+      if (e.code === "EEXIST") {
+        console.log(`❌ Skipping existing file ${writePath}`);
+      } else {
+        console.error(e);
+      }
+    }
+  }
+};
+
+export const buildLogger = (startText = ""): OraType => {
+  const logger = Ora().start(startText);
+  return logger;
+};
+
+export const getFileExtension = (filename: string) => {
+  const ext = filename.substring(
+    filename.lastIndexOf(".") + 1,
+    filename.length
+  );
+  if (filename === ext) return undefined;
+  return ext;
+};
+
+export const fileBelongsInPagesFolder = (filename: string) => {
+  const extension = getFileExtension(filename);
+  return (
+    extension &&
+    (extension === "mdx" || extension === "md" || extension === "tsx")
+  );
+};
+
+export const ensureYarn = (logger: OraType) => {
+  const yarnInstalled = shell.which("yarn");
+  if (!yarnInstalled) {
+    logger.fail(`yarn must be installed, run
+
+    npm install --global yarn
+
+    `);
+    process.exit(1);
+  }
+};

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  // "extends": "@mintlify/ts-config",
+  "compilerOptions": {
+    "module": "ES2020",
+    "moduleResolution": "node",
+    "target": "es2020",
+    "lib": ["es2020", "dom", "ES2021.String"],
+    "sourceMap": true,
+    "outDir": "bin",
+    "baseUrl": ".",
+    "paths": {
+      "*": ["node_modules/*", "src/types/*"]
+    },
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "esModuleInterop": true
+  },
+  "include": ["src/**/*"]
+}