@mintlify/scraping 4.0.1071 → 4.0.1073

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.1071",
+  "version": "4.0.1073",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -43,7 +43,7 @@
     "format:check": "oxfmt --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.1201",
+    "@mintlify/common": "1.0.1203",
     "fast-xml-parser": "5.7.3",
     "fs-extra": "11.1.1",
     "graphql": "16.11.0",
@@ -64,7 +64,7 @@
   "devDependencies": {
     "@mintlify/models": "0.0.364",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.887",
+    "@mintlify/validation": "0.1.889",
     "@tsconfig/recommended": "1.0.2",
     "@types/hast": "3.0.4",
     "@types/mdast": "4.0.4",
@@ -87,5 +87,5 @@
       "optional": true
     }
   },
-  "gitHead": "27059a381f1ab3e02e10ece59dfcfaa048a9bd8e"
+  "gitHead": "ae113e1c66224519c407cd9bbfb7c0c729a86628"
 }

package/src/openapi/common.ts

@@ -10,12 +10,15 @@ import {
   registerXMintContent,
   prepareStringToBeValidFilename,
   generateUniqueFilenameWithoutExtension,
-  getTagDisplayName,
+  createOpenApiBadgeResolver,
+  createOpenApiTagResolver,
+  findOpenApiNavGroup,
+  isOpenApi32,
   mergePageGroups,
   type LoadedOverlay,
 } from '@mintlify/common';
 import type { DecoratedNavigationPage, PageMetaTags } from '@mintlify/models';
-import { XMint } from '@mintlify/validation';
+import { getPathItemOperations, XMint } from '@mintlify/validation';
 import { outputFile } from 'fs-extra';
 import fse from 'fs-extra';
 import fs from 'fs/promises';
@@ -104,7 +107,9 @@ export const createOpenApiFrontmatterContent = ({
     Object.entries(metadata)
       .filter(([k]) => !reserved.has(k))
       .forEach(([key, value]) => {
-        frontmatter += `\n${key}: ${value}`;
+        const serialized =
+          typeof value === 'string' ? yaml.dump(value, { lineWidth: -1 }).trimEnd() : value;
+        frontmatter += `\n${key}: ${serialized}`;
       });
   }
 
@@ -135,6 +140,8 @@ export type GenerateOpenApiPagesOptions = {
   mountPath?: string;
   overwrite?: boolean;
   localSchema?: boolean;
+  resolveTags?: ReturnType<typeof createOpenApiTagResolver>;
+  resolveBadge?: ReturnType<typeof createOpenApiBadgeResolver>;
   writeFile?: (filename: string, content: string) => void | Promise<void>;
   overlays?: readonly LoadedOverlay[];
 };
@@ -170,121 +177,130 @@ export function processOpenApiPath<N, DN>(
   pagesAcc: Record<string, DecoratedNavigationPage>,
   options: GenerateOpenApiPagesOptions,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  findNavGroup: (nav: any, groupName?: string) => any
+  findNavGroup: (nav: any, groupName?: string, tagKey?: string) => any
 ): string[] {
   const openApiFilePathFromRoot = options.openApiFilePath
     ? optionallyAddLeadingSlash(options.openApiFilePath)
     : undefined;
 
   const generatedSlugs: string[] = [];
+  const resolveTags = options.resolveTags ?? createOpenApiTagResolver(schema);
+  const resolveBadge = options.resolveBadge ?? createOpenApiBadgeResolver(schema);
+  const nestedTags = isOpenApi32(schema);
 
-  Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
-    if (method in pathItemObject) {
-      const operation = pathItemObject[method];
-      if (isExcludedOperation(operation as MaybeOperationObjectWithExtensions)) {
-        return;
-      }
-      const xMint = operation?.['x-mint'];
-      const xMintGroups = getXMintGroups({
-        pathObject: pathItemObject,
-        operationObject: operation,
-      });
-
-      if (xMint?.href) {
-        xMint.href = optionallyAddLeadingSlash(xMint.href);
-      }
-
-      const tagName = operation?.tags?.[0];
-      const groupName = getTagDisplayName(tagName, schema);
-      let title =
-        prepareStringToBeValidFilename(operation?.summary) ??
-        `${method}-${prepareStringToBeValidFilename(path)}`;
-
-      let folder = prepareStringToBeValidFilename(tagName) ?? '';
-      let base = posix.join(options.outDir ?? '', folder, title);
-
-      if (xMint?.href) {
-        const slug = optionallyRemoveLeadingSlash(xMint.href);
-        title = posix.parse(slug).name;
-        folder = posix.parse(slug).dir;
-        base = posix.join(options.mountPath ?? '', folder, title);
-      }
+  getPathItemOperations(pathItemObject).forEach(([method, operation]) => {
+    if (isExcludedOperation(operation as MaybeOperationObjectWithExtensions)) {
+      return;
+    }
+    const xMint = operation?.['x-mint'];
+    const xMintGroups = getXMintGroups({
+      pathObject: pathItemObject,
+      operationObject: operation,
+    });
+
+    if (xMint?.href) {
+      xMint.href = optionallyAddLeadingSlash(xMint.href);
+    }
 
-      const navGroup = findNavGroup(nav, groupName);
-      const decoratedNavGroup = findNavGroup(decoratedNav, groupName);
+    const tagPath = resolveTags(operation?.tags);
+    const tagName = tagPath.at(-1)?.name;
+    let title =
+      prepareStringToBeValidFilename(operation?.summary) ??
+      `${method.toLowerCase()}-${prepareStringToBeValidFilename(path)}`;
 
-      const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
+    let folder = prepareStringToBeValidFilename(tagName) ?? '';
+    let base = posix.join(options.outDir ?? '', folder, title);
 
-      const openapiMetaTag = buildOpenApiMetaTag({
-        filePath: openApiFilePathFromRoot,
-        method,
-        path,
-      });
-      const { title: titleTag, description } = getOpenApiTitleAndDescription(
-        [
-          {
-            filename: options.openApiFilePath
-              ? parse(options.openApiFilePath).name
-              : 'filler-filename',
-            spec: schema,
-            originalFileLocation: openApiFilePathFromRoot,
-          },
-        ],
-        openapiMetaTag
-      );
-
-      let xMintMetadata = xMint?.metadata;
-      if (xMintGroups.length > 0) {
-        xMintMetadata = {
-          ...xMintMetadata,
-          groups: mergePageGroups(xMintMetadata?.groups, xMintGroups),
-        };
-      }
-
-      const slugTitle = slugToTitle(filenameWithoutExtension);
+    if (xMint?.href) {
+      const slug = optionallyRemoveLeadingSlash(xMint.href);
+      title = posix.parse(slug).name;
+      folder = posix.parse(slug).dir;
+      base = posix.join(options.mountPath ?? '', folder, title);
+    }
 
-      const page: DecoratedNavigationPage = {
-        title: titleTag ?? slugTitle,
-        description,
-        deprecated: operation?.deprecated,
-        version: options.version,
-        // When a file-path spec has x-mint.href overriding the path, use the slug-derived
-        // title for the sidebar so the nav shows the user-chosen name rather than the
-        // OpenAPI summary. Only for openApiFilePath specs to avoid changing URL-based behavior.
-        ...(xMint?.href && titleTag && options.openApiFilePath ? { sidebarTitle: slugTitle } : {}),
+    const hidden = isHiddenOperation(operation as MaybeOperationObjectWithExtensions);
+    const navGroup = hidden && nestedTags ? [] : findOpenApiNavGroup(nav, tagPath, findNavGroup);
+    const decoratedNavGroup =
+      hidden && nestedTags ? [] : findOpenApiNavGroup(decoratedNav, tagPath, findNavGroup);
+
+    const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(
+      nestedTags ? pagesAcc : navGroup,
+      base
+    );
+
+    const openapiMetaTag = buildOpenApiMetaTag({
+      filePath: openApiFilePathFromRoot,
+      method,
+      path,
+    });
+    const { title: titleTag, description } = getOpenApiTitleAndDescription(
+      [
+        {
+          filename: options.openApiFilePath
+            ? parse(options.openApiFilePath).name
+            : 'filler-filename',
+          spec: schema,
+          originalFileLocation: openApiFilePathFromRoot,
+        },
+      ],
+      openapiMetaTag
+    );
+
+    let xMintMetadata = xMint?.metadata;
+    if (xMintGroups.length > 0) {
+      xMintMetadata = {
         ...xMintMetadata,
-        openapi: openapiMetaTag,
-        href: posix.resolve('/', filenameWithoutExtension),
+        groups: mergePageGroups(xMintMetadata?.groups, xMintGroups),
       };
+    }
+    const badge = resolveBadge(operation?.tags);
+    if (badge !== undefined && xMintMetadata?.tag === undefined) {
+      xMintMetadata = { ...xMintMetadata, tag: badge };
+    }
 
-      if (!isHiddenOperation(operation as MaybeOperationObjectWithExtensions)) {
-        navGroup.push(filenameWithoutExtension);
-        decoratedNavGroup.push(page);
-      }
-      pagesAcc[filenameWithoutExtension] = page;
-      generatedSlugs.push(filenameWithoutExtension);
+    const slugTitle = slugToTitle(filenameWithoutExtension);
+
+    const page: DecoratedNavigationPage = {
+      title: titleTag ?? slugTitle,
+      description,
+      deprecated: operation?.deprecated,
+      version: options.version,
+      // When a file-path spec has x-mint.href overriding the path, use the slug-derived
+      // title for the sidebar so the nav shows the user-chosen name rather than the
+      // OpenAPI summary. Only for openApiFilePath specs to avoid changing URL-based behavior.
+      ...(xMint?.href && titleTag && options.openApiFilePath ? { sidebarTitle: slugTitle } : {}),
+      ...xMintMetadata,
+      openapi: openapiMetaTag,
+      href: posix.resolve('/', filenameWithoutExtension),
+    };
+
+    if (!isHiddenOperation(operation as MaybeOperationObjectWithExtensions)) {
+      navGroup.push(filenameWithoutExtension);
+      decoratedNavGroup.push(page);
+    }
+    pagesAcc[filenameWithoutExtension] = page;
+    generatedSlugs.push(filenameWithoutExtension);
 
-      if (!options.writeFiles) {
-        registerXMintContent(filenameWithoutExtension, xMint?.content);
-      }
+    if (!options.writeFiles) {
+      registerXMintContent(filenameWithoutExtension, xMint?.content);
+    }
 
-      const targetPath = options.outDirBasePath
-        ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
-        : `${filenameWithoutExtension}.mdx`;
-      if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
-        const input = {
-          openApiMetaTag: openapiMetaTag,
-          version: options.version,
-          deprecated: operation?.deprecated,
-          metadata: xMintMetadata,
-          extraContent: xMint?.content,
-        };
-        writePromises.push(
-          options.writeFile
-            ? Promise.resolve(options.writeFile(targetPath, createOpenApiFrontmatterContent(input)))
-            : createOpenApiFrontmatter({ filename: targetPath, ...input })
-        );
-      }
+    const targetPath = options.outDirBasePath
+      ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
+      : `${filenameWithoutExtension}.mdx`;
+    if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
+      const input = {
+        openApiMetaTag: openapiMetaTag,
+        version: options.version,
+        deprecated: operation?.deprecated,
+        metadata: xMintMetadata,
+        extraContent: xMint?.content,
+      };
+      writePromises.push(
+        options.writeFile
+          ? Promise.resolve(options.writeFile(targetPath, createOpenApiFrontmatterContent(input)))
+          : createOpenApiFrontmatter({ filename: targetPath, ...input })
+      );
     }
   });
 
@@ -301,90 +317,101 @@ export function processOpenApiWebhook<N, DN>(
   pagesAcc: Record<string, DecoratedNavigationPage>,
   options: GenerateOpenApiPagesOptions,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  findNavGroup: (nav: any, groupName?: string) => any
+  findNavGroup: (nav: any, groupName?: string, tagKey?: string) => any
 ): string[] {
   const openApiFilePathFromRoot = options.openApiFilePath
     ? optionallyAddLeadingSlash(options.openApiFilePath)
     : undefined;
 
   const generatedSlugs: string[] = [];
+  const resolveTags = options.resolveTags ?? createOpenApiTagResolver(schema);
+  const resolveBadge = options.resolveBadge ?? createOpenApiBadgeResolver(schema);
+  const nestedTags = isOpenApi32(schema);
 
-  Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
-    if (method in webhookObject) {
-      const operation = webhookObject[method];
-      if (isExcludedOperation(operation as MaybeOperationObjectWithExtensions)) {
-        return;
-      }
-      const xMint = operation?.['x-mint'];
-
-      if (xMint?.href) {
-        xMint.href = optionallyAddLeadingSlash(xMint.href);
-      }
+  getPathItemOperations(webhookObject).forEach(([, operation]) => {
+    if (isExcludedOperation(operation as MaybeOperationObjectWithExtensions)) {
+      return;
+    }
+    const xMint = operation?.['x-mint'];
 
-      const tagName = operation?.tags?.[0];
-      const groupName = getTagDisplayName(tagName, schema);
-      let title =
-        prepareStringToBeValidFilename(operation?.summary) ??
-        `${prepareStringToBeValidFilename(webhook)}`;
-      let folder = prepareStringToBeValidFilename(tagName) ?? '';
-      let base = posix.join(options.outDir ?? '', folder, title);
-
-      if (xMint?.href) {
-        const slug = optionallyRemoveLeadingSlash(xMint.href);
-        title = posix.parse(slug).name;
-        folder = posix.parse(slug).dir;
-        base = posix.join(options.mountPath ?? '', folder, title);
-      }
+    if (xMint?.href) {
+      xMint.href = optionallyAddLeadingSlash(xMint.href);
+    }
 
-      const navGroup = findNavGroup(nav, groupName);
-      const decoratedNavGroup = findNavGroup(decoratedNav, groupName);
+    const tagPath = resolveTags(operation?.tags);
+    const tagName = tagPath.at(-1)?.name;
+    let title =
+      prepareStringToBeValidFilename(operation?.summary) ??
+      `${prepareStringToBeValidFilename(webhook)}`;
+    let folder = prepareStringToBeValidFilename(tagName) ?? '';
+    let base = posix.join(options.outDir ?? '', folder, title);
+
+    if (xMint?.href) {
+      const slug = optionallyRemoveLeadingSlash(xMint.href);
+      title = posix.parse(slug).name;
+      folder = posix.parse(slug).dir;
+      base = posix.join(options.mountPath ?? '', folder, title);
+    }
 
-      const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
+    const hidden = isHiddenOperation(operation as MaybeOperationObjectWithExtensions);
+    const navGroup = hidden && nestedTags ? [] : findOpenApiNavGroup(nav, tagPath, findNavGroup);
+    const decoratedNavGroup =
+      hidden && nestedTags ? [] : findOpenApiNavGroup(decoratedNav, tagPath, findNavGroup);
+
+    const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(
+      nestedTags ? pagesAcc : navGroup,
+      base
+    );
+
+    const openapiMetaTag = buildOpenApiMetaTag({
+      filePath: openApiFilePathFromRoot,
+      method: 'webhook',
+      path: webhook,
+    });
+
+    const badge = resolveBadge(operation?.tags);
+    const xMintMetadata =
+      badge !== undefined && xMint?.metadata?.tag === undefined
+        ? { ...xMint?.metadata, tag: badge }
+        : xMint?.metadata;
+
+    const page: DecoratedNavigationPage = {
+      title: slugToTitle(filenameWithoutExtension),
+      description: operation?.description,
+      version: options.version,
+      deprecated: operation?.deprecated,
+      ...xMintMetadata,
+      openapi: openapiMetaTag,
+      href: posix.resolve('/', filenameWithoutExtension),
+    };
+
+    if (!isHiddenOperation(operation as MaybeOperationObjectWithExtensions)) {
+      navGroup.push(filenameWithoutExtension);
+      decoratedNavGroup.push(page);
+    }
+    pagesAcc[filenameWithoutExtension] = page;
+    generatedSlugs.push(filenameWithoutExtension);
 
-      const openapiMetaTag = buildOpenApiMetaTag({
-        filePath: openApiFilePathFromRoot,
-        method: 'webhook',
-        path: webhook,
-      });
+    if (!options.writeFiles) {
+      registerXMintContent(filenameWithoutExtension, xMint?.content);
+    }
 
-      const page: DecoratedNavigationPage = {
-        title: slugToTitle(filenameWithoutExtension),
-        description: operation?.description,
+    const targetPath = options.outDirBasePath
+      ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
+      : `${filenameWithoutExtension}.mdx`;
+    if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
+      const input = {
+        openApiMetaTag: openapiMetaTag,
         version: options.version,
         deprecated: operation?.deprecated,
-        ...xMint?.metadata,
-        openapi: openapiMetaTag,
-        href: posix.resolve('/', filenameWithoutExtension),
+        metadata: xMintMetadata,
+        extraContent: xMint?.content,
       };
-
-      if (!isHiddenOperation(operation as MaybeOperationObjectWithExtensions)) {
-        navGroup.push(filenameWithoutExtension);
-        decoratedNavGroup.push(page);
-      }
-      pagesAcc[filenameWithoutExtension] = page;
-      generatedSlugs.push(filenameWithoutExtension);
-
-      if (!options.writeFiles) {
-        registerXMintContent(filenameWithoutExtension, xMint?.content);
-      }
-
-      const targetPath = options.outDirBasePath
-        ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
-        : `${filenameWithoutExtension}.mdx`;
-      if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
-        const input = {
-          openApiMetaTag: openapiMetaTag,
-          version: options.version,
-          deprecated: operation?.deprecated,
-          metadata: xMint?.metadata,
-          extraContent: xMint?.content,
-        };
-        writePromises.push(
-          options.writeFile
-            ? Promise.resolve(options.writeFile(targetPath, createOpenApiFrontmatterContent(input)))
-            : createOpenApiFrontmatter({ filename: targetPath, ...input })
-        );
-      }
+      writePromises.push(
+        options.writeFile
+          ? Promise.resolve(options.writeFile(targetPath, createOpenApiFrontmatterContent(input)))
+          : createOpenApiFrontmatter({ filename: targetPath, ...input })
+      );
     }
   });
 

package/src/openapi/generateOpenApiPages.ts

@@ -1,4 +1,9 @@
-import { validate, DEFAULT_API_GROUP_NAME } from '@mintlify/common';
+import {
+  createOpenApiBadgeResolver,
+  createOpenApiTagResolver,
+  validate,
+  DEFAULT_API_GROUP_NAME,
+} from '@mintlify/common';
 import type {
   DecoratedNavigation,
   DecoratedNavigationGroup,
@@ -40,6 +45,12 @@ export async function generateOpenApiPages(
     throw new Error('No paths or webhooks defined.');
   }
 
+  const generationOptions = {
+    ...opts,
+    resolveTags: createOpenApiTagResolver(schema as OpenAPI.Document),
+    resolveBadge: createOpenApiBadgeResolver(schema as OpenAPI.Document),
+  };
+
   const nav: Navigation = [];
   const decoratedNav: DecoratedNavigation = [];
   const writePromises: Promise<void>[] = [];
@@ -58,7 +69,7 @@ export async function generateOpenApiPages(
         decoratedNav,
         writePromises,
         pagesAcc,
-        opts ?? {},
+        generationOptions,
         findNavGroup
       );
     });
@@ -77,7 +88,7 @@ export async function generateOpenApiPages(
         decoratedNav,
         writePromises,
         pagesAcc,
-        opts ?? {},
+        generationOptions,
         findNavGroup
       );
     });
@@ -94,19 +105,26 @@ export async function generateOpenApiPages(
   };
 }
 
+const tagKeys = new WeakMap<object, string>();
+
 const findNavGroup = <T extends NavigationGroup | DecoratedNavigationGroup>(
   nav: T['pages'][number][],
-  groupName: string = DEFAULT_API_GROUP_NAME
+  groupName: string = DEFAULT_API_GROUP_NAME,
+  tagKey?: string
 ): T['pages'][number][] => {
   const group = nav.find(
     (fileOrGroup) =>
-      typeof fileOrGroup === 'object' && 'group' in fileOrGroup && fileOrGroup.group === groupName
+      typeof fileOrGroup === 'object' &&
+      'group' in fileOrGroup &&
+      fileOrGroup.group === groupName &&
+      (tagKey === undefined || tagKeys.get(fileOrGroup) === tagKey)
   ) as T | undefined;
   if (group === undefined) {
     const newGroup = {
       group: groupName,
       pages: [],
     };
+    if (tagKey !== undefined) tagKeys.set(newGroup, tagKey);
     nav.push(newGroup);
     return newGroup.pages;
   } else {

package/src/openapi/generateOpenApiPagesForDocsConfig.ts

@@ -1,4 +1,9 @@
-import { validate, findNavGroup } from '@mintlify/common';
+import {
+  createOpenApiBadgeResolver,
+  createOpenApiTagResolver,
+  validate,
+  findNavGroup,
+} from '@mintlify/common';
 import { DecoratedNavigationPage } from '@mintlify/models';
 import { DecoratedGroupsConfig, GroupsConfig } from '@mintlify/validation';
 import { OpenAPI } from 'openapi-types';
@@ -35,6 +40,12 @@ export async function generateOpenApiPagesForDocsConfig(
     throw new Error('No paths or webhooks defined.');
   }
 
+  const generationOptions = {
+    ...opts,
+    resolveTags: createOpenApiTagResolver(schema as OpenAPI.Document),
+    resolveBadge: createOpenApiBadgeResolver(schema as OpenAPI.Document),
+  };
+
   const nav: GroupsConfig = [];
   const decoratedNav: DecoratedGroupsConfig = [];
   const writePromises: Promise<void>[] = [];
@@ -53,7 +64,7 @@ export async function generateOpenApiPagesForDocsConfig(
         decoratedNav,
         writePromises,
         pagesAcc,
-        opts ?? {},
+        generationOptions,
         findNavGroup
       );
     });
@@ -72,7 +83,7 @@ export async function generateOpenApiPagesForDocsConfig(
         decoratedNav,
         writePromises,
         pagesAcc,
-        opts ?? {},
+        generationOptions,
         findNavGroup
       );
     });