mintlify 2.1.0 → 3.0.5

package/src/local-preview/listener/utils/getOpenApiContext.ts

@@ -1,77 +0,0 @@
-export const extractMethodAndEndpoint = (
-  api: string
-): { method?: string; endpoint: string; filename?: string } => {
-  const methodRegex = /(get|post|put|delete|patch)\s/i;
-  const trimmed = api.trim();
-  const foundMethod = trimmed.match(methodRegex);
-
-  const startIndexOfMethod = foundMethod ? api.indexOf(foundMethod[0]) : 0;
-  const endIndexOfMethod = foundMethod
-    ? startIndexOfMethod + foundMethod[0].length - 1
-    : 0;
-
-  const filename = api.substring(0, startIndexOfMethod).trim();
-
-  return {
-    method: foundMethod ? foundMethod[0].slice(0, -1).toUpperCase() : undefined,
-    endpoint: api.substring(endIndexOfMethod).trim(),
-    filename: filename ? filename : undefined,
-  };
-};
-
-export const getOpenApiOperationMethodAndEndpoint = (
-  openApi: any,
-  openApiMetaField: string
-) => {
-  const { endpoint, method, filename } =
-    extractMethodAndEndpoint(openApiMetaField);
-
-  let path: any;
-
-  openApi.files?.forEach((file: any) => {
-    const openApiFile = file.openapi;
-    const openApiPath = openApiFile.paths && openApiFile.paths[endpoint];
-    const isFilenameOrNone = !filename || filename === file.name;
-    if (openApiPath && isFilenameOrNone) {
-      path = openApiPath;
-    }
-  });
-
-  if (path == null) {
-    return {};
-  }
-
-  let operation;
-  if (method) {
-    operation = path[method.toLowerCase()];
-  } else {
-    const firstOperationKey = Object.keys(path)[0];
-    operation = path[firstOperationKey];
-  }
-
-  return {
-    operation,
-    method,
-    endpoint,
-  };
-};
-
-export const getOpenApiTitleAndDescription = (openApi, openApiMetaField) => {
-  if (openApi == null || !openApiMetaField || openApiMetaField == null) {
-    return {};
-  }
-
-  const { operation } = getOpenApiOperationMethodAndEndpoint(
-    openApi,
-    openApiMetaField
-  );
-
-  if (operation == null) {
-    return {};
-  }
-
-  return {
-    title: operation.summary,
-    description: operation.description,
-  };
-};

package/src/local-preview/listener/utils/mintConfigFile.ts

@@ -1,28 +0,0 @@
-import { promises as _promises } from "fs";
-import { pathExists } from "fs-extra";
-import pathUtil from "path";
-
-const { readFile } = _promises;
-
-// TODO: Put in prebuild package
-export const getConfigPath = async (
-  contentDirectoryPath: string
-): Promise<string | null> => {
-  if (await pathExists(pathUtil.join(contentDirectoryPath, "mint.json"))) {
-    return pathUtil.join(contentDirectoryPath, "mint.json");
-  }
-  return null;
-};
-
-// TODO: Put in prebuild package
-export const getConfigObj = async (
-  contentDirectoryPath: string
-): Promise<any> => {
-  const configPath = await getConfigPath(contentDirectoryPath);
-  let configObj = null;
-  if (configPath) {
-    const configContents = await readFile(configPath);
-    configObj = await JSON.parse(configContents.toString());
-  }
-  return configObj;
-};

package/src/local-preview/listener/utils/toTitleCase.ts

@@ -1,40 +0,0 @@
-export function toTitleCase(text: string) {
-  const smallWords = /^(a|an|and|as|at|but|by|en|for|if|in|nor|of|on|or|per|the|to|v.?|vs.?|via)$/i;
-  const alphanumericPattern = /([A-Za-z0-9\u00C0-\u00FF])/;
-  const wordSeparators = /([ :–—-])/;
-
-  return text
-    .split(wordSeparators)
-    .map(function (current, index, array) {
-      if (
-        // Check for small words
-        current.search(smallWords) > -1 &&
-        // Skip first and last word
-        index !== 0 &&
-        index !== array.length - 1 &&
-        // Ignore title end and subtitle start
-        array[index - 3] !== ':' &&
-        array[index + 1] !== ':' &&
-        // Ignore small words that start a hyphenated phrase
-        (array[index + 1] !== '-' || (array[index - 1] === '-' && array[index + 1] === '-'))
-      ) {
-        return current.toLowerCase();
-      }
-
-      // Ignore intentional capitalization
-      if (current.substring(1).search(/[A-Z]|\../) > -1) {
-        return current;
-      }
-
-      // Ignore URLs
-      if (array[index + 1] === ':' && array[index + 2] !== '') {
-        return current;
-      }
-
-      // Capitalize the first letter
-      return current.replace(alphanumericPattern, function (match: string) {
-        return match.toUpperCase();
-      });
-    })
-    .join('');
-}

package/src/local-preview/listener/utils/types.ts

@@ -1,16 +0,0 @@
-export type OpenApiFile = { name: string; spec: any };
-
-export type PotentialFileCategory =
-  | "page"
-  | "snippet"
-  | "mintConfig"
-  | "potentialYamlOpenApiSpec"
-  | "potentialJsonOpenApiSpec"
-  | "staticFile";
-
-export type FileCategory =
-  | "page"
-  | "snippet"
-  | "mintConfig"
-  | "openApi"
-  | "staticFile";

package/src/local-preview/listener/utils.ts

@@ -1,78 +0,0 @@
-import SwaggerParser from "@apidevtools/swagger-parser";
-import { promises as _promises } from "fs";
-
-const { readdir, stat } = _promises;
-
-export const getFileExtension = (filename) => {
-  return (
-    filename.substring(filename.lastIndexOf(".") + 1, filename.length) ||
-    filename
-  );
-};
-
-export const openApiCheck = async (path) => {
-  let spec;
-  let isOpenApi = false;
-  try {
-    spec = await SwaggerParser.validate(path);
-    isOpenApi = true;
-  } catch {
-    // not valid openApi
-  }
-  return { spec, isOpenApi };
-};
-
-export const filterOutNullInGroup = (group) => {
-  const newPages = filterOutNullInPages(group.pages);
-  const newGroup = {
-    ...group,
-    pages: newPages,
-  };
-  return newGroup;
-};
-
-const filterOutNullInPages = (pages) => {
-  if (!Array.isArray(pages)) {
-    return [];
-  }
-  const newPages = [];
-  pages.forEach((page) => {
-    if (page == null) {
-      return;
-    }
-    if (page.hasOwnProperty("pages")) {
-      const newGroup = filterOutNullInGroup(page);
-      newPages.push(newGroup);
-    } else {
-      newPages.push(page);
-    }
-  });
-
-  return newPages;
-};
-
-export const getFileList = async (dirName: string, og = dirName) => {
-  let files = [];
-  const items = await readdir(dirName, { withFileTypes: true });
-
-  for (const item of items) {
-    if (item.isDirectory()) {
-      files = [...files, ...(await getFileList(`${dirName}/${item.name}`, og))];
-    } else {
-      const path = `${dirName}/${item.name}`;
-      const name = path.replace(og, "");
-      files.push(name);
-    }
-  }
-
-  return files;
-};
-
-export const isFileSizeValid = async (
-  path: string,
-  maxFileSizeInMb: number
-): Promise<boolean> => {
-  const maxFileSizeBytes = maxFileSizeInMb * 1000000;
-  const stats = await stat(path);
-  return stats.size <= maxFileSizeBytes;
-};

package/src/scraping/combineNavWithEmptyGroupTitles.ts

@@ -1,21 +0,0 @@
-export default function combineNavWithEmptyGroupTitles(
-  navArray: MintNavigation[]
-): MintNavigation[] {
-  let newNavArray = [];
-
-  navArray.forEach((nav: MintNavigation) => {
-    // The first run through the loop will always have -1 as the index.
-    // JavaScript returns undefined when we look for an index outside the size of the array.
-    const prev = newNavArray[newNavArray.length - 1];
-    if (prev == null) {
-      newNavArray.push(nav);
-    } else if (!nav.group && !prev.group) {
-      // Joins multiple groups without a title together IF they occur side by side
-      prev.pages = prev.pages.concat(nav.pages);
-    } else {
-      newNavArray.push(nav);
-    }
-  });
-
-  return newNavArray;
-}

package/src/scraping/detectFramework.ts

@@ -1,47 +0,0 @@
-import cheerio from "cheerio";
-
-export enum Frameworks {
-  DOCUSAURUS = "DOCUSAURUS",
-  GITBOOK = "GITBOOK",
-  README = "README",
-  INTERCOM = "INTERCOM",
-}
-
-export function detectFramework(html) {
-  const $ = cheerio.load(html);
-  const docusaurusMeta = $('meta[name="generator"]');
-
-  if (
-    docusaurusMeta.length > 0 &&
-    docusaurusMeta.attr("content").includes("Docusaurus")
-  ) {
-    if (docusaurusMeta.attr("content").includes("v3")) {
-      return { framework: Frameworks.DOCUSAURUS, version: "3" };
-    }
-    if (docusaurusMeta.attr("content").includes("v2")) {
-      return { framework: Frameworks.DOCUSAURUS, version: "2" };
-    } else if (docusaurusMeta.attr("content").includes("v1")) {
-      console.warn(
-        "WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3."
-      );
-      return { framework: Frameworks.DOCUSAURUS, version: "1" };
-    }
-  }
-
-  const isGitBook = $(".gitbook-root").length > 0;
-  if (isGitBook) {
-    return { framework: Frameworks.GITBOOK };
-  }
-
-  const isReadMe = $('meta[name="readme-deploy"]').length > 0;
-  if (isReadMe) {
-    return { framework: Frameworks.README };
-  }
-
-  const isIntercom = $("meta[name='intercom:trackingEvent']").length > 0;
-  if (isIntercom) {
-    return { framework: Frameworks.INTERCOM };
-  }
-
-  return undefined;
-}

package/src/scraping/downloadAllImages.ts

@@ -1,60 +0,0 @@
-import path from "path";
-import downloadImage, {
-  cleanImageSrc,
-  isValidImageSrc,
-  removeMetadataFromImageSrc,
-} from "../downloadImage.js";
-
-// To Do: Use CheerioElement instead of any when we bump the cheerio version
-export default async function downloadAllImages(
-  $: any,
-  content: any,
-  origin: string,
-  baseDir: string,
-  overwrite: boolean,
-  modifyFileName?: any,
-  skipValidateImageExtension?: boolean
-) {
-  if (!baseDir) {
-    console.debug("Skipping image downloading");
-    return;
-  }
-
-  // We remove duplicates because some frameworks duplicate img tags
-  // to show the image larger when clicked on.
-  const imageSrcs = [
-    ...new Set(
-      content
-        .find("img[src]")
-        .map((i, image) => $(image).attr("src"))
-        .toArray()
-    ),
-  ];
-
-  // Wait to all images to download before continuing
-  const origToNewArray = await Promise.all(
-    imageSrcs.map(async (imageSrc: string) => {
-      if (!isValidImageSrc(imageSrc, skipValidateImageExtension)) {
-        return;
-      }
-
-      const imageHref = cleanImageSrc(imageSrc, origin);
-
-      let fileName = removeMetadataFromImageSrc(path.basename(imageHref));
-      if (modifyFileName) {
-        fileName = modifyFileName(fileName);
-      }
-
-      const writePath = path.join(baseDir, fileName);
-
-      await downloadImage(imageHref, writePath, overwrite);
-
-      return { [imageSrc]: writePath };
-    })
-  );
-
-  return origToNewArray.reduce(
-    (result, current) => Object.assign(result, current),
-    {}
-  );
-}

package/src/scraping/downloadLogoImage.ts

@@ -1,25 +0,0 @@
-import path from "path";
-import downloadImage, {
-  cleanImageSrc,
-  isValidImageSrc,
-  removeMetadataFromImageSrc,
-} from "../downloadImage.js";
-import { getFileExtension } from "../util.js";
-
-// To Do: Use CheerioElement instead of any when we bump the cheerio version
-export default async function downloadLogoImage(
-  imageSrc: string,
-  imageBaseDir: string,
-  origin: string,
-  overwrite: boolean,
-  skipValidateImageExtension?: boolean
-) {
-  if (!isValidImageSrc(imageSrc, skipValidateImageExtension)) return;
-
-  const imageHref = cleanImageSrc(imageSrc, origin);
-
-  const ext = getFileExtension(removeMetadataFromImageSrc(imageSrc));
-  const imagePath = path.join(imageBaseDir, "logo", "logo-light-mode." + ext);
-
-  await downloadImage(imageHref, imagePath, overwrite);
-}

package/src/scraping/getSitemapLinks.ts

@@ -1,18 +0,0 @@
-import axios from "axios";
-
-// Not in use.
-// Gets all links in a sitemap.
-export const getSitemapLinks = async (url: URL) => {
-  const hostname = url.hostname.replace(".", "\\.");
-  const regex = new RegExp(`https?:\/\/${hostname}.+?(?=<\/loc>)`, "gmi");
-
-  try {
-    const indexData = (await axios.get(url.href)).data as string;
-    const array = indexData.match(regex) as string[] | null;
-    return array || [];
-  } catch (err) {
-    console.error(err);
-    console.log("Skipping sitemap links because we encountered an error.");
-    return [];
-  }
-};

package/src/scraping/replaceImagePaths.ts

@@ -1,21 +0,0 @@
-export default function replaceImagePaths(
-  origToWritePath: object,
-  cliDir: string,
-  markdown: string
-) {
-  if (origToWritePath == null) {
-    return markdown;
-  }
-
-  // Change image paths to use the downloaded locations
-  for (const [origHref, writePath] of Object.entries(origToWritePath)) {
-    // Use relative paths within the folder we are in
-    if (writePath.startsWith(cliDir)) {
-      markdown = markdown.replaceAll(origHref, writePath.slice(cliDir.length));
-    } else {
-      markdown = markdown.replaceAll(origHref, writePath);
-    }
-  }
-
-  return markdown;
-}

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -1,86 +0,0 @@
-import path from "path";
-import axios from "axios";
-import { getHtmlWithPuppeteer } from "../browser.js";
-import { createPage } from "../util.js";
-
-export async function scrapeFileGettingFileNameFromUrl(
-  pathname: string,
-  cliDir: string,
-  origin: string,
-  overwrite: boolean,
-  scrapePageFunc: (
-    html: string,
-    origin: string,
-    cliDir: string,
-    imageBaseDir: string,
-    overwrite: boolean,
-    version: string | undefined
-  ) => Promise<{
-    title?: string;
-    description?: string;
-    markdown?: string;
-  }>,
-  puppeteer = false,
-  version: string | undefined,
-  baseToRemove?: string
-): Promise<MintNavigationEntry> {
-  // Skip scraping external links
-  if (pathname.startsWith("https://") || pathname.startsWith("http://")) {
-    return pathname;
-  }
-
-  // Removes file name from the end
-  const splitSubpath = pathname.split("/");
-  let folders = splitSubpath.slice(0, splitSubpath.length - 1).join("/");
-
-  // Remove base dir if passed in
-  if (baseToRemove && folders.startsWith(baseToRemove)) {
-    folders = folders.replace(baseToRemove, "");
-  }
-
-  // TO DO: Improve this by putting each page's images in a separate
-  // folder named after the title of the page.
-  const imageBaseDir = path.join(cliDir, "images", folders);
-
-  // Scrape each page separately
-  const href = new URL(pathname, origin).href;
-  let html: string;
-  if (puppeteer) {
-    html = await getHtmlWithPuppeteer(href);
-  } else {
-    const res = await axios.get(href);
-    html = res.data;
-  }
-
-  const { title, description, markdown } = await scrapePageFunc(
-    html,
-    origin,
-    cliDir,
-    imageBaseDir,
-    overwrite,
-    version
-  );
-
-  // Check if page didn't have content
-  if (!title && !markdown) {
-    return undefined;
-  }
-
-  const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;
-
-  // Default to introduction.mdx if we encountered index.html
-  const fileName = splitSubpath[splitSubpath.length - 1] || "introduction";
-
-  // Will create subfolders as needed
-  createPage(
-    title,
-    description,
-    markdown,
-    overwrite,
-    newFileLocation,
-    fileName
-  );
-
-  // Removes first slash if we are in a folder, Mintlify doesn't need it
-  return folders ? path.join(folders, fileName).substring(1) : fileName;
-}

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -1,54 +0,0 @@
-import { scrapeFileGettingFileNameFromUrl } from "./scrapeFileGettingFileNameFromUrl.js";
-
-export async function scrapeGettingFileNameFromUrl(
-  navEntry: MintNavigationEntry,
-  cliDir: string,
-  origin: string,
-  overwrite: boolean,
-  scrapePageFunc: (
-    html: string,
-    origin: string,
-    cliDir: string,
-    imageBaseDir: string,
-    overwrite: boolean,
-    version: string | undefined
-  ) => Promise<{
-    title?: string;
-    description?: string;
-    markdown?: string;
-  }>,
-  puppeteer = false,
-  version: string | undefined,
-  baseToRemove?: string
-): Promise<MintNavigationEntry> {
-  if (typeof navEntry !== "string") {
-    const newPages = [];
-    for (const nestedNavEntry of navEntry.pages) {
-      newPages.push(
-        await scrapeGettingFileNameFromUrl(
-          nestedNavEntry,
-          cliDir,
-          origin,
-          overwrite,
-          scrapePageFunc,
-          puppeteer,
-          version,
-          baseToRemove
-        )
-      );
-    }
-    navEntry.pages = newPages;
-    return navEntry;
-  }
-
-  return await scrapeFileGettingFileNameFromUrl(
-    navEntry,
-    cliDir,
-    origin,
-    overwrite,
-    scrapePageFunc,
-    puppeteer,
-    version,
-    baseToRemove
-  );
-}

package/src/scraping/scrapePage.ts

@@ -1,24 +0,0 @@
-import path from "path";
-import { createPage, getOrigin } from "../util.js";
-
-export async function scrapePage(
-  scrapeFunc: ScrapePageFn,
-  href: string,
-  html: string,
-  overwrite: boolean,
-  version: string | undefined
-) {
-  const origin = getOrigin(href);
-  const cwd = process.cwd();
-  const imageBaseDir = path.join(cwd, "images");
-
-  const { title, description, markdown } = await scrapeFunc(
-    html,
-    origin,
-    cwd,
-    imageBaseDir,
-    overwrite,
-    version
-  );
-  createPage(title, description, markdown, overwrite, process.cwd());
-}

package/src/scraping/scrapePageCommands.ts

@@ -1,66 +0,0 @@
-import axios from "axios";
-import { scrapePage } from "./scrapePage.js";
-import { scrapeDocusaurusPage } from "./site-scrapers/scrapeDocusaurusPage.js";
-import { scrapeGitBookPage } from "./site-scrapers/scrapeGitBookPage.js";
-import { scrapeReadMePage } from "./site-scrapers/scrapeReadMePage.js";
-import { detectFramework, Frameworks } from "./detectFramework.js";
-import { getHrefFromArgs } from "../util.js";
-import { getHtmlWithPuppeteer } from "../browser.js";
-import { ArgumentsCamelCase } from "yargs";
-import { scrapeIntercomPage } from "./site-scrapers/Intercom/scrapeIntercomPage.js";
-
-function validateFramework(framework) {
-  if (!framework) {
-    console.log(
-      "Could not detect the framework automatically. Please use one of:"
-    );
-    console.log("scrape-page-docusaurus");
-    console.log("scrape-page-gitbook");
-    console.log("scrape-page-readme");
-    console.log("scrape-page-intercom");
-    return process.exit(1);
-  }
-}
-
-export async function scrapePageWrapper(
-  argv: ArgumentsCamelCase,
-  scrapeFunc: ScrapePageFn,
-  options?: { version?: string; puppeteer?: boolean }
-) {
-  const href = getHrefFromArgs(argv);
-  let html: string;
-  if (options?.puppeteer) {
-    html = await getHtmlWithPuppeteer(href);
-  } else {
-    const res = await axios.get(href);
-    html = res.data;
-  }
-  await scrapePage(scrapeFunc, href, html, !!argv.overwrite, options?.version);
-  process.exit(0);
-}
-
-export async function scrapePageAutomatically(argv: any) {
-  const href = getHrefFromArgs(argv);
-  const res = await axios.get(href);
-  const html = res.data;
-  const { framework, version } = detectFramework(html);
-
-  validateFramework(framework);
-
-  console.log("Detected framework: " + framework);
-
-  switch (framework) {
-    case Frameworks.DOCUSAURUS:
-      await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
-      break;
-    case Frameworks.GITBOOK:
-      await scrapePageWrapper(argv, scrapeGitBookPage, { puppeteer: true });
-      break;
-    case Frameworks.README:
-      await scrapePageWrapper(argv, scrapeReadMePage);
-      break;
-    case Frameworks.INTERCOM:
-      await scrapePageWrapper(argv, scrapeIntercomPage);
-      break;
-  }
-}