@mintlify/scraping 4.0.321 → 4.0.323

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.321",
+  "version": "4.0.323",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -38,7 +38,7 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.465",
+    "@mintlify/common": "1.0.467",
     "@mintlify/openapi-parser": "^0.0.7",
     "fs-extra": "^11.1.1",
     "hast-util-to-mdast": "^10.1.0",
@@ -61,7 +61,7 @@
     "@mintlify/models": "0.0.214",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.426",
+    "@mintlify/validation": "0.1.427",
     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
     "@tsconfig/recommended": "1.x",
     "@types/hast": "^3.0.4",
@@ -77,5 +77,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "f9f611507cf6d1bc39a70c9542e7d5276e6432b8"
+  "gitHead": "72ab4184ddbc325a2afeaedb9e34d4afbdf0826d"
 }

package/src/openapi/common.ts

@@ -2,11 +2,13 @@ import {
   getOpenApiTitleAndDescription,
   OperationObject,
   optionallyAddLeadingSlash,
+  optionallyRemoveLeadingSlash,
   slugToTitle,
   isAllowedLocalSchemaUrl,
   buildOpenApiMetaTag,
 } from '@mintlify/common';
-import type { DecoratedNavigationPage } from '@mintlify/models';
+import type { DecoratedNavigationPage, PageMetaTags } from '@mintlify/models';
+import { XMint } from '@mintlify/validation';
 import { outputFile } from 'fs-extra';
 import fse from 'fs-extra';
 import fs from 'fs/promises';
@@ -20,6 +22,12 @@ import {
 } from '../apiPages/common.js';
 import { fetchOpenApi } from '../utils/network.js';
 
+export type OpenApiExtensions = {
+  'x-mint'?: XMint;
+  'x-excluded'?: boolean;
+  'x-hidden'?: boolean;
+};
+
 export const getOpenApiDefinition = async (
   pathOrDocumentOrUrl: string | OpenAPI.Document | URL,
   localSchema?: boolean
@@ -59,17 +67,40 @@ export const createOpenApiFrontmatter = async ({
   openApiMetaTag,
   version,
   deprecated,
+  metadata,
+  extraContent,
 }: {
   filename: string;
   openApiMetaTag: string;
   version?: string;
   deprecated?: boolean;
+  metadata?: PageMetaTags;
+  extraContent?: string;
 }) => {
-  const data = `---
-openapi: ${openApiMetaTag}${version ? `\nversion: ${version}` : ''}${
-    deprecated ? `\ndeprecated: ${deprecated}` : ''
+  let frontmatter = `---\nopenapi: ${openApiMetaTag}`;
+  if (metadata && 'version' in metadata) {
+    frontmatter += `\nversion: ${metadata.version}`;
+  } else if (version) {
+    frontmatter += `\nversion: ${version}`;
+  }
+  if (metadata && 'deprecated' in metadata) {
+    frontmatter += `\ndeprecated: ${metadata.deprecated}`;
+  } else if (deprecated) {
+    frontmatter += `\ndeprecated: ${deprecated}`;
+  }
+
+  if (metadata) {
+    const reserved = new Set(['openapi', 'version', 'deprecated']);
+    Object.entries(metadata)
+      .filter(([k]) => !reserved.has(k))
+      .forEach(([key, value]) => {
+        frontmatter += `\n${key}: ${value}`;
+      });
   }
----`;
+
+  frontmatter += `\n---`;
+
+  const data = extraContent ? `${frontmatter}\n\n${extraContent}` : frontmatter;
 
   await outputFile(filename, data);
 };
@@ -107,7 +138,7 @@ const isExcludedOperation = (operation: MaybeOperationObjectWithExtensions) => {
 
 export function processOpenApiPath<N, DN>(
   path: string,
-  pathItemObject: OpenAPIV3.PathItemObject,
+  pathItemObject: OpenAPIV3.PathItemObject<OpenApiExtensions>,
   schema: OpenAPI.Document,
   nav: N,
   decoratedNav: DN,
@@ -127,17 +158,32 @@ export function processOpenApiPath<N, DN>(
       if (isExcludedOperation(operation as MaybeOperationObjectWithExtensions)) {
         return;
       }
+      const xMint = operation?.['x-mint'];
+
+      if (xMint?.href) {
+        xMint.href = optionallyAddLeadingSlash(xMint.href);
+      }
+
       const groupName = operation?.tags?.[0];
-      const title =
+      let title =
         prepareStringToBeValidFilename(operation?.summary) ??
         `${method}-${prepareStringToBeValidFilename(path)}`;
-      const folder = prepareStringToBeValidFilename(groupName) ?? '';
-      const base = join(options.outDir ?? '', folder, title);
+
+      let folder = prepareStringToBeValidFilename(groupName) ?? '';
+      let base = join(options.outDir ?? '', folder, title);
+
+      if (xMint?.href) {
+        const slug = optionallyRemoveLeadingSlash(xMint.href);
+        title = parse(slug).name;
+        folder = parse(slug).dir;
+        base = join(folder, title);
+      }
 
       const navGroup = findNavGroup(nav, groupName);
       const decoratedNavGroup = findNavGroup(decoratedNav, groupName);
 
       const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
+
       const openapiMetaTag = buildOpenApiMetaTag({
         filePath: openApiFilePathFromRoot,
         method,
@@ -158,8 +204,8 @@ export function processOpenApiPath<N, DN>(
       const page: DecoratedNavigationPage = {
         openapi: openapiMetaTag,
         href: resolve('/', filenameWithoutExtension),
-        title: titleTag ?? slugToTitle(filenameWithoutExtension),
-        description,
+        title: xMint?.metadata?.title ?? titleTag ?? slugToTitle(filenameWithoutExtension),
+        description: xMint?.metadata?.description ?? description,
         deprecated: operation?.deprecated,
         version: options.version,
       };
@@ -169,6 +215,7 @@ export function processOpenApiPath<N, DN>(
         decoratedNavGroup.push(page);
       }
       pagesAcc[filenameWithoutExtension] = page;
+
       const targetPath = options.outDirBasePath
         ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
         : `${filenameWithoutExtension}.mdx`;
@@ -179,6 +226,8 @@ export function processOpenApiPath<N, DN>(
             openApiMetaTag: openapiMetaTag,
             version: options.version,
             deprecated: operation?.deprecated,
+            metadata: xMint?.metadata,
+            extraContent: xMint?.content,
           })
         );
       }
@@ -188,7 +237,7 @@ export function processOpenApiPath<N, DN>(
 
 export function processOpenApiWebhook<N, DN>(
   webhook: string,
-  webhookObject: OpenAPIV3.PathItemObject,
+  webhookObject: OpenAPIV3.PathItemObject<OpenApiExtensions>,
   _schema: OpenAPI.Document,
   nav: N,
   decoratedNav: DN,
@@ -208,12 +257,25 @@ export function processOpenApiWebhook<N, DN>(
       if (isExcludedOperation(operation as MaybeOperationObjectWithExtensions)) {
         return;
       }
+      const xMint = operation?.['x-mint'];
+
+      if (xMint?.href) {
+        xMint.href = optionallyAddLeadingSlash(xMint.href);
+      }
+
       const groupName = operation?.tags?.[0];
-      const title =
+      let title =
         prepareStringToBeValidFilename(operation?.summary) ??
         `${prepareStringToBeValidFilename(webhook)}`;
-      const folder = prepareStringToBeValidFilename(groupName) ?? '';
-      const base = join(options.outDir ?? '', folder, title);
+      let folder = prepareStringToBeValidFilename(groupName) ?? '';
+      let base = join(options.outDir ?? '', folder, title);
+
+      if (xMint?.href) {
+        const slug = optionallyRemoveLeadingSlash(xMint.href);
+        title = parse(slug).name;
+        folder = parse(slug).dir;
+        base = join(folder, title);
+      }
 
       const navGroup = findNavGroup(nav, groupName);
       const decoratedNavGroup = findNavGroup(decoratedNav, groupName);
@@ -226,12 +288,12 @@ export function processOpenApiWebhook<N, DN>(
         path: webhook,
       });
 
-      const description = operation?.description;
+      const description = xMint?.metadata?.description ?? operation?.description;
 
       const page: DecoratedNavigationPage = {
         openapi: openapiMetaTag,
         href: resolve('/', filenameWithoutExtension),
-        title: slugToTitle(filenameWithoutExtension),
+        title: xMint?.metadata?.title ?? slugToTitle(filenameWithoutExtension),
         description,
         version: options.version,
         deprecated: operation?.deprecated,
@@ -252,6 +314,8 @@ export function processOpenApiWebhook<N, DN>(
             openApiMetaTag: openapiMetaTag,
             version: options.version,
             deprecated: operation?.deprecated,
+            metadata: xMint?.metadata,
+            extraContent: xMint?.content,
           })
         );
       }