mintlify 1.1.5 → 1.1.7

package/src/local-preview/index.ts

@@ -44,18 +44,16 @@ const { readFile } = _promises;
 const copyFiles = async (logger: any) => {
   logger.start("Syncing doc files...");
   shell.cd(CMD_EXEC_PATH);
-  const { markdownFiles, staticFiles, openApiBuffer } = await categorizeFiles();
+  const { markdownFiles, staticFiles, openApi } = await categorizeFiles();
 
   const configObj = await updateConfigFile(logger);
 
   const openApiTargetPath = path.join(CLIENT_PATH, "src", "openapi.json");
-  let openApiObj = null;
-  if (openApiBuffer) {
+  if (openApi) {
     logger.succeed("OpenApi file synced");
-    await fse.outputFile(openApiTargetPath, Buffer.from(openApiBuffer), {
+    await fse.outputFile(openApiTargetPath, JSON.stringify(openApi), {
       flag: "w",
     });
-    openApiObj = JSON.parse(openApiBuffer.toString());
   } else {
     await fse.outputFile(openApiTargetPath, "{}", { flag: "w" });
   }
@@ -73,7 +71,7 @@ const copyFiles = async (logger: any) => {
 
         const fileContent = await readFile(sourcePath);
         const contentStr = fileContent.toString();
-        const page = createPage(filename, contentStr, openApiObj);
+        const page = createPage(filename, contentStr, openApi);
         pages = {
           ...pages,
           ...page,

package/src/local-preview/utils/categorizeFiles.ts

@@ -27,13 +27,13 @@ const getFileList = async (dirName: string, og = dirName) => {
 const categorizeFiles = async (): Promise<{
   markdownFiles: string[];
   staticFiles: string[];
-  openApiBuffer: Buffer | undefined;
+  openApi: object | undefined;
 }> => {
   const allFilesInCmdExecutionPath = await getFileList(CMD_EXEC_PATH);
   const markdownFiles = [];
   const staticFiles = [];
   const promises = [];
-  let openApiBuffer = undefined;
+  const openApiFiles = [];
   allFilesInCmdExecutionPath.forEach((file) => {
     promises.push(
       (async () => {
@@ -53,7 +53,11 @@ const categorizeFiles = async (): Promise<{
           );
           isOpenApi = openApiInfo.isOpenApi;
           if (isOpenApi) {
-            openApiBuffer = openApiInfo.buffer;
+            const fileName = path.parse(file.name).base;
+            openApiFiles.push({
+              name: fileName.substring(0, fileName.lastIndexOf(".")),
+              openapi: openApiInfo.openapi,
+            });
           }
         } else if (
           (!file.name.endsWith("mint.config.json") ||
@@ -68,7 +72,11 @@ const categorizeFiles = async (): Promise<{
   });
   await Promise.all(promises);
 
-  return { markdownFiles, staticFiles, openApiBuffer };
+  const openApi = {
+    files: openApiFiles,
+  };
+
+  return { markdownFiles, staticFiles, openApi };
 };
 
 export default categorizeFiles;

package/src/local-preview/utils/getOpenApiContext.ts

@@ -1,25 +1,41 @@
-export const extractMethodAndEndpoint = (api) => {
-  const methodRegex = /^get|post|put|delete|patch/i;
+export const extractMethodAndEndpoint = (
+  api: string
+): { method?: string; endpoint: string; filename?: string } => {
+  const methodRegex = /(get|post|put|delete|patch)\s/i;
   const trimmed = api.trim();
   const foundMethod = trimmed.match(methodRegex);
 
+  const startIndexOfMethod = foundMethod ? api.indexOf(foundMethod[0]) : 0;
   const endIndexOfMethod = foundMethod
-    ? api.indexOf(foundMethod[0]) + foundMethod[0].length
+    ? startIndexOfMethod + foundMethod[0].length - 1
     : 0;
 
+  const filename = api.substring(0, startIndexOfMethod).trim();
+
   return {
-    method: foundMethod ? foundMethod[0].toUpperCase() : undefined,
+    method: foundMethod ? foundMethod[0].slice(0, -1).toUpperCase() : undefined,
     endpoint: api.substring(endIndexOfMethod).trim(),
+    filename: filename ? filename : undefined,
   };
 };
 
 export const getOpenApiOperationMethodAndEndpoint = (
-  openApiObj,
-  openApiMetaField
+  openApi: any,
+  openApiMetaField: string
 ) => {
-  const { endpoint, method } = extractMethodAndEndpoint(openApiMetaField);
+  const { endpoint, method, filename } =
+    extractMethodAndEndpoint(openApiMetaField);
+
+  let path: any;
 
-  const path = openApiObj?.paths && openApiObj.paths[endpoint];
+  openApi.files?.forEach((file: any) => {
+    const openApiFile = file.openapi;
+    const openApiPath = openApiFile.paths && openApiFile.paths[endpoint];
+    const isFilenameOrNone = !filename || filename === file.name;
+    if (openApiPath && isFilenameOrNone) {
+      path = openApiPath;
+    }
+  });
 
   if (path == null) {
     return {};
@@ -40,13 +56,13 @@ export const getOpenApiOperationMethodAndEndpoint = (
   };
 };
 
-export const getOpenApiTitleAndDescription = (openApiObj, openApiMetaField) => {
-  if (openApiObj == null || !openApiMetaField || openApiMetaField == null) {
+export const getOpenApiTitleAndDescription = (openApi, openApiMetaField) => {
+  if (openApi == null || !openApiMetaField || openApiMetaField == null) {
     return {};
   }
 
   const { operation } = getOpenApiOperationMethodAndEndpoint(
-    openApiObj,
+    openApi,
     openApiMetaField
   );
 

package/src/local-preview/utils/listener.ts

@@ -81,13 +81,18 @@ const listener = () => {
             );
             isOpenApi = openApiInfo.isOpenApi;
             if (isOpenApi) {
-              await fse.outputFile(path.join(CLIENT_PATH, "src", "openapi.json"), Buffer.from(openApiInfo.buffer), {
-                flag: "w",
-              });
+              await fse.outputFile(
+                path.join(CLIENT_PATH, "src", "openapi.json"),
+                JSON.stringify(openApiInfo.openapi),
+                {
+                  flag: "w",
+                }
+              );
               updateMetadata = true;
             }
           }
-          if (!isOpenApi) { // all other files
+          if (!isOpenApi) {
+            // all other files
             const targetPath = path.join(CLIENT_PATH, "public", filename);
             await fse.copy(filePath, targetPath);
           }

package/src/local-preview/utils/metadata.ts

@@ -54,14 +54,14 @@ const getMetadata = (fileContents: string) => {
 export const createPage = (
   path: string,
   content: string,
-  openApiObj: object | null
+  openApi: object | null
 ) => {
   const slug = path.replace(/\.mdx?$/, "").substring(1);
   let defaultTitle = slugToTitle(slug);
   const metadata = getMetadata(content);
   // Append data from OpenAPI if it exists
   const { title, description } = getOpenApiTitleAndDescription(
-    openApiObj,
+    openApi,
     metadata?.openapi
   );
   if (title) {
@@ -127,11 +127,7 @@ export const createMetadataFileFromPages = (pages: any, configObj: any) => {
 
 export const createMetadataFile = async () => {
   // create pages
-  const { markdownFiles, openApiBuffer } = await categorizeFiles();
-  let openApiObj = null;
-  if (openApiBuffer) {
-    openApiObj = JSON.parse(openApiBuffer.toString());
-  }
+  const { markdownFiles, openApi } = await categorizeFiles();
   // create config object
   const configObj = await getConfigObj();
   let pages = {};
@@ -142,7 +138,7 @@ export const createMetadataFile = async () => {
         const sourcePath = path.join(CMD_EXEC_PATH, filename);
         const fileContent = await readFile(sourcePath);
         const contentStr = fileContent.toString();
-        const page = createPage(filename, contentStr, openApiObj);
+        const page = createPage(filename, contentStr, openApi);
         pages = {
           ...pages,
           ...page,

package/src/local-preview/utils/openApiCheck.ts

@@ -3,17 +3,16 @@ import { promises as _promises } from "fs";
 
 const openApiCheck = async (
   path: string
-): Promise<{ buffer: Buffer | undefined; isOpenApi: boolean }> => {
-  let buffer = undefined;
+): Promise<{ openapi: any; isOpenApi: boolean }> => {
+  let openapi;
   let isOpenApi = false;
   try {
-    const api = await SwaggerParser.validate(path);
-    buffer = Buffer.from(JSON.stringify(api, null, 2), "utf-8");
+    openapi = await SwaggerParser.validate(path);
     isOpenApi = true;
   } catch {
     // not valid openApi
   }
-  return { buffer, isOpenApi };
+  return { openapi, isOpenApi };
 };
 
 export default openApiCheck;

package/src/scraping/detectFramework.ts

@@ -14,17 +14,27 @@ export function detectFramework(html) {
     docusaurusMeta.length > 0 &&
     docusaurusMeta.attr("content").includes("Docusaurus")
   ) {
-    return Frameworks.DOCUSAURUS;
+    if (docusaurusMeta.attr("content").includes("v3")) {
+      return { framework: Frameworks.DOCUSAURUS, version: "3" };
+    }
+    if (docusaurusMeta.attr("content").includes("v2")) {
+      return { framework: Frameworks.DOCUSAURUS, version: "2" };
+    } else if (docusaurusMeta.attr("content").includes("v1")) {
+      console.warn(
+        "WARNING: We detected Docusaurus version 1 but we only support scraping versions 2 and 3."
+      );
+      return { framework: Frameworks.DOCUSAURUS, version: "1" };
+    }
   }
 
   const isGitBook = $(".gitbook-root").length > 0;
   if (isGitBook) {
-    return Frameworks.GITBOOK;
+    return { framework: Frameworks.GITBOOK };
   }
 
   const isReadMe = $('meta[name="readme-deploy"]').length > 0;
   if (isReadMe) {
-    return Frameworks.README;
+    return { framework: Frameworks.README };
   }
 
   return undefined;

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -12,13 +12,15 @@ export async function scrapeFileGettingFileNameFromUrl(
     html: string,
     origin: string,
     cliDir: string,
-    imageBaseDir: string
+    imageBaseDir: string,
+    version: string | undefined
   ) => Promise<{
     title?: string;
     description?: string;
     markdown?: string;
   }>,
   puppeteer = false,
+  version: string | undefined,
   baseToRemove?: string
 ) {
   // Skip scraping external links
@@ -53,7 +55,8 @@ export async function scrapeFileGettingFileNameFromUrl(
     html,
     origin,
     cliDir,
-    imageBaseDir
+    imageBaseDir,
+    version
   );
 
   // Check if page didn't have content

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -10,13 +10,15 @@ export async function scrapeGettingFileNameFromUrl(
     html: string,
     origin: string,
     cliDir: string,
-    imageBaseDir: string
+    imageBaseDir: string,
+    version: string | undefined
   ) => Promise<{
     title?: string;
     description?: string;
     markdown?: string;
   }>,
   puppeteer = false,
+  version: string | undefined,
   baseToRemove?: string
 ): Promise<NavigationEntry> {
   if (isNavigation(navEntry)) {
@@ -30,6 +32,7 @@ export async function scrapeGettingFileNameFromUrl(
           overwrite,
           scrapePageFunc,
           puppeteer,
+          version,
           baseToRemove
         )
       );
@@ -45,6 +48,7 @@ export async function scrapeGettingFileNameFromUrl(
     overwrite,
     scrapePageFunc,
     puppeteer,
+    version,
     baseToRemove
   );
 }

package/src/scraping/scrapePage.ts

@@ -6,11 +6,13 @@ export async function scrapePage(
     html: string,
     origin: string,
     cliDir: string,
-    imageBaseDir: string
+    imageBaseDir: string,
+    version: string | undefined
   ) => Promise<any>,
   href: string,
   html: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string | undefined
 ) {
   const origin = getOrigin(href);
   const imageBaseDir = path.join(process.cwd(), "images");
@@ -18,7 +20,8 @@ export async function scrapePage(
     html,
     origin,
     process.cwd(),
-    imageBaseDir
+    imageBaseDir,
+    version
   );
   createPage(title, description, markdown, overwrite, process.cwd());
 }

package/src/scraping/scrapePageCommands.ts

@@ -19,16 +19,20 @@ function validateFramework(framework) {
   }
 }
 
-export async function scrapePageWrapper(argv, scrapeFunc, puppeteer = false) {
+export async function scrapePageWrapper(
+  argv: any,
+  scrapeFunc: any,
+  options?: { version?: string; puppeteer?: boolean }
+) {
   const href = getHrefFromArgs(argv);
   let html: string;
-  if (puppeteer) {
+  if (options.puppeteer) {
     html = await getHtmlWithPuppeteer(href);
   } else {
     const res = await axios.default.get(href);
     html = res.data;
   }
-  await scrapePage(scrapeFunc, href, html, argv.overwrite);
+  await scrapePage(scrapeFunc, href, html, argv.overwrite, options.version);
   process.exit(0);
 }
 
@@ -36,16 +40,16 @@ export async function scrapePageAutomatically(argv: any) {
   const href = getHrefFromArgs(argv);
   const res = await axios.default.get(href);
   const html = res.data;
-  const framework = detectFramework(html);
+  const { framework, version } = detectFramework(html);
 
   validateFramework(framework);
 
   console.log("Detected framework: " + framework);
 
   if (framework === Frameworks.DOCUSAURUS) {
-    await scrapePageWrapper(argv, scrapeDocusaurusPage);
+    await scrapePageWrapper(argv, scrapeDocusaurusPage, { version });
   } else if (framework === Frameworks.GITBOOK) {
-    await scrapePageWrapper(argv, scrapeGitBookPage, true);
+    await scrapePageWrapper(argv, scrapeGitBookPage, { puppeteer: true });
   } else if (framework === Frameworks.README) {
     await scrapePageWrapper(argv, scrapeReadMePage);
   }

package/src/scraping/scrapeSection.ts

@@ -4,12 +4,19 @@ export async function scrapeSection(
   scrapeFunc: any,
   html: string,
   origin: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string | undefined
 ) {
   console.log(
     `Started scraping${overwrite ? ", overwrite mode is on" : ""}...`
   );
-  const groupsConfig = await scrapeFunc(html, origin, process.cwd(), overwrite);
+  const groupsConfig = await scrapeFunc(
+    html,
+    origin,
+    process.cwd(),
+    overwrite,
+    version
+  );
   console.log("Finished scraping.");
   console.log("Add the following to your navigation in mint.json:");
   console.log(objToReadableString(groupsConfig));

package/src/scraping/scrapeSectionCommands.ts

@@ -13,15 +13,25 @@ export async function scrapeSectionAxiosWrapper(argv: any, scrapeFunc: any) {
   const href = getHrefFromArgs(argv);
   const res = await axios.default.get(href);
   const html = res.data;
-  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  await scrapeSection(
+    scrapeFunc,
+    html,
+    getOrigin(href),
+    argv.overwrite,
+    undefined
+  );
   process.exit(0);
 }
 
-export async function scrapeDocusaurusSectionCommand(argv: any) {
+export async function scrapeDocusaurusSectionCommand(
+  argv: any,
+  version: string // "1" | "2" | "3"
+) {
   await scrapeSectionOpeningAllNested(
     argv,
     openNestedDocusaurusMenus,
-    scrapeDocusaurusSection
+    scrapeDocusaurusSection,
+    version
   );
 }
 
@@ -36,7 +46,8 @@ export async function scrapeGitbookSectionCommand(argv: any) {
 async function scrapeSectionOpeningAllNested(
   argv: any,
   openLinks: any,
-  scrapeFunc: any
+  scrapeFunc: any,
+  version?: string
 ) {
   const href = getHrefFromArgs(argv);
 
@@ -48,7 +59,13 @@ async function scrapeSectionOpeningAllNested(
 
   const html = await openLinks(page);
   browser.close();
-  await scrapeSection(scrapeFunc, html, getOrigin(href), argv.overwrite);
+  await scrapeSection(
+    scrapeFunc,
+    html,
+    getOrigin(href),
+    argv.overwrite,
+    version
+  );
   process.exit(0);
 }
 
@@ -56,14 +73,14 @@ export async function scrapeSectionAutomatically(argv: any) {
   const href = getHrefFromArgs(argv);
   const res = await axios.default.get(href);
   const html = res.data;
-  const framework = detectFramework(html);
+  const { framework, version } = detectFramework(html);
 
   validateFramework(framework);
 
   console.log("Detected framework: " + framework);
 
   if (framework === Frameworks.DOCUSAURUS) {
-    await scrapeDocusaurusSectionCommand(argv);
+    await scrapeDocusaurusSectionCommand(argv, version);
   } else if (framework === Frameworks.GITBOOK) {
     await scrapeGitbookSectionCommand(argv);
   } else if (framework === Frameworks.README) {

package/src/scraping/site-scrapers/links-per-group/getDocusaurusLinksPerGroup.ts

@@ -0,0 +1,46 @@
+import alternateGroupTitle from "../alternateGroupTitle.js";
+import getLinksRecursively from "./getLinksRecursively.js";
+
+export function getDocusaurusLinksPerGroup(
+  navigationSections: any,
+  $: any,
+  version: string | undefined
+) {
+  if (version === "3" || version === "2") {
+    return getDocusaurusLinksPerGroupLoop(navigationSections, $);
+  }
+  return [];
+}
+
+function getDocusaurusLinksPerGroupLoop(navigationSections: any, $: any) {
+  return navigationSections
+    .map((i, s) => {
+      const section = $(s);
+
+      // Links without a group
+      if (section.hasClass("theme-doc-sidebar-item-link")) {
+        const linkHref = section.find("a[href]").first().attr("href");
+        return {
+          group: "",
+          pages: [linkHref],
+        };
+      }
+
+      const firstLink = section
+        .find(".menu__list-item-collapsible")
+        .first()
+        .find("a[href]");
+
+      const sectionTitle = firstLink.text();
+      const firstHref = firstLink.attr("href");
+      const linkSections = section.children().eq(1).children();
+
+      const pages = getLinksRecursively(linkSections, $);
+
+      return {
+        group: sectionTitle || alternateGroupTitle(firstLink, pages),
+        pages: firstHref ? [firstHref, ...pages] : pages,
+      };
+    })
+    .toArray();
+}

package/src/scraping/site-scrapers/scrapeDocusaurusPage.ts

@@ -7,34 +7,46 @@ export async function scrapeDocusaurusPage(
   html: string,
   origin: string,
   cliDir: string,
-  imageBaseDir: string
+  imageBaseDir: string,
+  version: string | undefined // expects "2", or "3". Have not written support for "1" yet
 ) {
   const $ = cheerio.load(html);
 
-  const content = $(".theme-doc-markdown").first();
+  const article =
+    version === "3" ? $(".theme-doc-markdown").first() : $("article").first();
 
-  // Index pages with no additional text don't have the markdown class
-  if (content.length === 0) {
+  if (article.length === 0) {
+    // Index pages with no additional text don't have the markdown class
     return {};
   }
 
-  const titleComponent = content.find("h1");
+  const titleComponent = article.find("h1");
   const title = titleComponent.text().trim();
 
   // Do not include title in the content when we insert it in our metadata
   titleComponent.remove();
 
+  const markdownContent =
+    version === "3" ? article : article.find(".markdown").first();
+
   const origToWritePath = await downloadAllImages(
     $,
-    content,
+    markdownContent,
     origin,
     imageBaseDir
   );
 
-  const contentHtml = content.html();
+  const markdownHtml = markdownContent.html();
 
   const nhm = new NodeHtmlMarkdown();
-  let markdown = nhm.translate(contentHtml);
+  let markdown = nhm.translate(markdownHtml);
+
+  if (markdown == null) {
+    console.error(
+      "We do not support scraping this page. Content will be empty"
+    );
+    return { title, description: null, markdown: "" };
+  }
 
   // Description only exists in meta tags. The code is commented out because its prone to incorrectly
   // including a description if the first line of text had markdown annotations like `.

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -3,14 +3,14 @@ import { NavigationEntry } from "../..//navigation.js";
 import { scrapeGettingFileNameFromUrl } from "../scrapeGettingFileNameFromUrl.js";
 import combineNavWithEmptyGroupTitles from "../combineNavWithEmptyGroupTitles.js";
 import { scrapeDocusaurusPage } from "./scrapeDocusaurusPage.js";
-import getLinksRecursively from "./getLinksRecursively.js";
-import alternateGroupTitle from "./alternateGroupTitle.js";
+import { getDocusaurusLinksPerGroup } from "./links-per-group/getDocusaurusLinksPerGroup.js";
 
 export async function scrapeDocusaurusSection(
   html: string,
   origin: string,
   cliDir: string,
-  overwrite: boolean
+  overwrite: boolean,
+  version: string
 ) {
   const $ = cheerio.load(html);
 
@@ -18,36 +18,11 @@ export async function scrapeDocusaurusSection(
   const navigationSections = $(".theme-doc-sidebar-menu").first().children();
 
   // Get all links per group
-  const groupsConfig = navigationSections
-    .map((i, s) => {
-      const section = $(s);
-
-      // Links without a group
-      if (section.hasClass("theme-doc-sidebar-item-link")) {
-        const linkHref = section.find("a[href]").first().attr("href");
-        return {
-          group: "",
-          pages: [linkHref],
-        };
-      }
-
-      const firstLink = section
-        .find(".menu__list-item-collapsible")
-        .first()
-        .find("a[href]");
-
-      const sectionTitle = firstLink.text();
-      const firstHref = firstLink.attr("href");
-      const linkSections = section.children().eq(1).children();
-
-      const pages = getLinksRecursively(linkSections, $);
-
-      return {
-        group: sectionTitle || alternateGroupTitle(firstLink, pages),
-        pages: firstHref ? [firstHref, ...pages] : pages,
-      };
-    })
-    .toArray();
+  const groupsConfig = getDocusaurusLinksPerGroup(
+    navigationSections,
+    $,
+    version
+  );
 
   // Merge groups with empty titles together
   const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
@@ -67,6 +42,7 @@ export async function scrapeDocusaurusSection(
               overwrite,
               scrapeDocusaurusPage,
               false,
+              version,
               "/docs"
             )
           )

package/src/scraping/site-scrapers/scrapeGitBookPage.ts

@@ -7,7 +7,8 @@ export async function scrapeGitBookPage(
   html: string,
   origin: string,
   cliDir: string,
-  imageBaseDir: string
+  imageBaseDir: string,
+  _: string | undefined // version
 ) {
   const $ = cheerio.load(html);