@mintlify/scraping 4.0.70 → 4.0.72

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.70",
+  "version": "4.0.72",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -38,7 +38,7 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.225",
+    "@mintlify/common": "1.0.227",
     "@mintlify/openapi-parser": "^0.0.7",
     "fs-extra": "^11.1.1",
     "hast-util-to-mdast": "^10.1.0",
@@ -61,7 +61,7 @@
     "@mintlify/models": "0.0.160",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.251",
+    "@mintlify/validation": "0.1.252",
     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
     "@tsconfig/recommended": "1.x",
     "@types/hast": "^3.0.4",
@@ -78,5 +78,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "c7f9ee676d859e5b3077b78d25f23303064ccf42"
+  "gitHead": "c3fd918eaf9d5311d70091042a1c2b208766281b"
 }

package/src/index.ts

@@ -1 +1,2 @@
 export { generateOpenApiPages } from './openapi/generateOpenApiPages.js';
+export { generateOpenApiPagesForDocsConfig } from './openapi/generateOpenApiPagesForDocsConfig.js';

package/src/openapi/common.ts

@@ -0,0 +1,168 @@
+import {
+  getOpenApiTitleAndDescription,
+  optionallyAddLeadingSlash,
+  slugToTitle,
+} from '@mintlify/common';
+import type { DecoratedNavigationPage, NavigationEntry } from '@mintlify/models';
+import { outputFile } from 'fs-extra';
+import fse from 'fs-extra';
+import fs from 'fs/promises';
+import yaml from 'js-yaml';
+import { OpenAPI, OpenAPIV3 } from 'openapi-types';
+import path, { join, parse, resolve } from 'path';
+
+import { fetchOpenApi } from '../utils/network.js';
+
+export const getOpenApiDefinition = async (
+  pathOrDocumentOrUrl: string | OpenAPI.Document | URL
+): Promise<{ document: OpenAPI.Document; isUrl: boolean }> => {
+  if (typeof pathOrDocumentOrUrl === 'string') {
+    if (pathOrDocumentOrUrl.startsWith('http://')) {
+      // This is an invalid location either for a file or a URL
+      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
+    } else {
+      try {
+        const url = new URL(pathOrDocumentOrUrl);
+        pathOrDocumentOrUrl = url;
+      } catch {
+        const pathname = path.join(process.cwd(), pathOrDocumentOrUrl.toString());
+        const file = await fs.readFile(pathname, 'utf-8');
+        pathOrDocumentOrUrl = yaml.load(file) as OpenAPI.Document;
+      }
+    }
+  }
+  const isUrl = pathOrDocumentOrUrl instanceof URL;
+  if (pathOrDocumentOrUrl instanceof URL) {
+    if (pathOrDocumentOrUrl.protocol !== 'https:') {
+      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
+    }
+    pathOrDocumentOrUrl = await fetchOpenApi(pathOrDocumentOrUrl);
+  }
+
+  return { document: pathOrDocumentOrUrl, isUrl };
+};
+
+// returns a filename that is unique within the given array of pages
+export const generateUniqueFilenameWithoutExtension = (
+  pages: NavigationEntry[],
+  base: string
+): string => {
+  let filename = base;
+  if (pages.includes(filename)) {
+    let extension = 1;
+    filename = `${base}-${extension}`;
+    while (pages.includes(filename)) {
+      extension += 1;
+      filename = `${base}-${extension}`;
+    }
+  }
+  return filename;
+};
+
+export const createOpenApiFrontmatter = async (
+  filename: string,
+  openApiMetaTag: string,
+  version?: string
+) => {
+  const data = `---
+openapi: ${openApiMetaTag}${version ? `\nversion: ${version}` : ''}
+---`;
+
+  await outputFile(filename, data);
+};
+
+export const prepareStringToBeValidFilename = (str?: string) =>
+  str
+    ? str
+        .replaceAll(' ', '-')
+        .replace(/\{.*?\}/g, '-') // remove path parameters
+        .replace(/^-/, '')
+        .replace(/-$/, '')
+        .replace(/[{}(),.'\n\/]/g, '') // remove special characters
+        .replaceAll(/--/g, '-') // replace double hyphens
+        .toLowerCase()
+    : undefined;
+
+export type GenerateOpenApiPagesOptions = {
+  openApiFilePath?: string;
+  version?: string;
+  writeFiles?: boolean;
+  outDir?: string;
+  outDirBasePath?: string;
+  overwrite?: boolean;
+};
+
+export type OpenApiPageGenerationResult<N, DN> = {
+  nav: N;
+  decoratedNav: DN;
+  spec: OpenAPI.Document;
+  pagesAcc: Record<string, DecoratedNavigationPage>;
+  isUrl: boolean;
+};
+
+export function processOpenApiPath<N, DN>(
+  path: string,
+  pathItemObject: OpenAPIV3.PathItemObject,
+  schema: OpenAPI.Document,
+  nav: N,
+  decoratedNav: DN,
+  writePromises: Promise<void>[],
+  pagesAcc: Record<string, DecoratedNavigationPage>,
+  options: GenerateOpenApiPagesOptions,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  findNavGroup: (nav: any, groupName?: string) => any
+) {
+  const openApiFilePathFromRoot = options.openApiFilePath
+    ? optionallyAddLeadingSlash(options.openApiFilePath)
+    : undefined;
+
+  Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
+    if (method in pathItemObject) {
+      const operation = pathItemObject[method];
+      const groupName = operation?.tags?.[0];
+      const title =
+        prepareStringToBeValidFilename(operation?.summary) ??
+        `${method}-${prepareStringToBeValidFilename(path)}`;
+      const folder = prepareStringToBeValidFilename(groupName) ?? '';
+      const base = join(options.outDir ?? '', folder, title);
+
+      const navGroup = findNavGroup(nav, groupName);
+      const decoratedNavGroup = findNavGroup(decoratedNav, groupName);
+
+      const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
+      const openapiMetaTag = `${
+        openApiFilePathFromRoot ? `${openApiFilePathFromRoot} ` : ''
+      }${method} ${path}`;
+      const { title: titleTag, description } = getOpenApiTitleAndDescription(
+        [
+          {
+            filename: options.openApiFilePath
+              ? parse(options.openApiFilePath).name
+              : 'filler-filename',
+            spec: schema,
+            originalFileLocation: options.openApiFilePath,
+          },
+        ],
+        openapiMetaTag
+      );
+      navGroup.push(filenameWithoutExtension);
+      const page: DecoratedNavigationPage = {
+        openapi: openapiMetaTag,
+        href: resolve('/', filenameWithoutExtension),
+        title: titleTag ?? slugToTitle(filenameWithoutExtension),
+        description,
+        version: options.version,
+      };
+      decoratedNavGroup.push(page);
+      pagesAcc[filenameWithoutExtension] = page;
+      const targetPath = options.outDirBasePath
+        ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
+        : `${filenameWithoutExtension}.mdx`;
+      if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
+        writePromises.push(createOpenApiFrontmatter(targetPath, openapiMetaTag, options.version));
+      }
+    }
+  });
+}
+
+export const DEFAULT_API_GROUP_NAME = 'API Reference';

package/src/openapi/generateOpenApiPages.ts

@@ -1,80 +1,26 @@
-import {
-  getOpenApiTitleAndDescription,
-  optionallyAddLeadingSlash,
-  slugToTitle,
-  validate,
-} from '@mintlify/common';
+import { validate } from '@mintlify/common';
 import type {
   DecoratedNavigation,
   DecoratedNavigationGroup,
   Navigation,
-  NavigationEntry,
   NavigationGroup,
   DecoratedNavigationPage,
 } from '@mintlify/models';
-import fse, { outputFile } from 'fs-extra';
-import fs from 'fs/promises';
-import yaml from 'js-yaml';
-import { OpenAPI, OpenAPIV3 } from 'openapi-types';
-import { join, resolve, parse } from 'path';
-import path from 'path';
-
-import { fetchOpenApi } from '../utils/network.js';
-
-export const getOpenApiDefinition = async (
-  pathOrDocumentOrUrl: string | OpenAPI.Document | URL
-): Promise<{ document: OpenAPI.Document; isUrl: boolean }> => {
-  if (typeof pathOrDocumentOrUrl === 'string') {
-    if (pathOrDocumentOrUrl.startsWith('http://')) {
-      // This is an invalid location either for a file or a URL
-      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
-    } else {
-      try {
-        const url = new URL(pathOrDocumentOrUrl);
-        pathOrDocumentOrUrl = url;
-      } catch {
-        const pathname = path.join(process.cwd(), pathOrDocumentOrUrl.toString());
-        const file = await fs.readFile(pathname, 'utf-8');
-        pathOrDocumentOrUrl = yaml.load(file) as OpenAPI.Document;
-      }
-    }
-  }
-
-  const isUrl = pathOrDocumentOrUrl instanceof URL;
-  if (pathOrDocumentOrUrl instanceof URL) {
-    if (pathOrDocumentOrUrl.protocol !== 'https:') {
-      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
-    }
-    pathOrDocumentOrUrl = await fetchOpenApi(pathOrDocumentOrUrl);
-  }
-
-  return { document: pathOrDocumentOrUrl as OpenAPI.Document, isUrl };
-};
+import { OpenAPI } from 'openapi-types';
 
-export const generateOpenApiPages = async (
+import {
+  getOpenApiDefinition,
+  GenerateOpenApiPagesOptions,
+  OpenApiPageGenerationResult,
+  processOpenApiPath,
+  DEFAULT_API_GROUP_NAME,
+} from './common.js';
+
+export async function generateOpenApiPages(
   pathOrDocumentOrUrl: string | OpenAPI.Document | URL,
-  opts?: {
-    openApiFilePath?: string | undefined;
-    version?: string | undefined;
-    writeFiles?: boolean;
-    outDir?: string;
-    outDirBasePath?: string;
-    overwrite?: boolean;
-  }
-): Promise<{
-  nav: Navigation;
-  decoratedNav: DecoratedNavigation;
-  spec: OpenAPI.Document;
-  pagesAcc: Record<string, DecoratedNavigationPage>;
-  isUrl: boolean;
-}> => {
-  const { openApiFilePath, version, writeFiles, outDir, outDirBasePath } = opts ?? {};
-  const openApiFilePathFromRoot = openApiFilePath
-    ? optionallyAddLeadingSlash(openApiFilePath)
-    : undefined;
-
+  opts?: GenerateOpenApiPagesOptions
+): Promise<OpenApiPageGenerationResult<Navigation, DecoratedNavigation>> {
   const { document, isUrl } = await getOpenApiDefinition(pathOrDocumentOrUrl);
-
   const { schema } = await validate(document);
 
   if (schema?.paths === undefined || Object.keys(schema.paths).length === 0) {
@@ -85,56 +31,22 @@ export const generateOpenApiPages = async (
   const decoratedNav: DecoratedNavigation = [];
   const writePromises: Promise<void>[] = [];
   const pagesAcc: Record<string, DecoratedNavigationPage> = {};
+
   Object.entries(schema.paths).forEach(([path, pathItemObject]) => {
     if (!pathItemObject || typeof pathItemObject !== 'object') {
       return;
     }
-    const typedPathItemObject = pathItemObject as OpenAPIV3.PathItemObject;
-    Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
-      if (method in typedPathItemObject) {
-        const operation = typedPathItemObject[method];
-        const groupName = operation?.tags?.[0];
-        const title =
-          prepareStringToBeValidFilename(operation?.summary) ??
-          `${method}-${prepareStringToBeValidFilename(path)}`;
-        const folder = prepareStringToBeValidFilename(groupName) ?? '';
-        const base = join(outDir ?? '', folder, title);
-
-        const navGroup = findNavGroup<NavigationGroup>(nav, groupName);
-        const decoratedNavGroup = findNavGroup<DecoratedNavigationGroup>(decoratedNav, groupName);
-
-        const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
-        const openapiMetaTag = `${
-          openApiFilePathFromRoot ? `${openApiFilePathFromRoot} ` : ''
-        }${method} ${path}`;
-        const { title: titleTag, description } = getOpenApiTitleAndDescription(
-          [
-            {
-              filename: openApiFilePath ? parse(openApiFilePath).name : 'filler-filename',
-              spec: schema as OpenAPI.Document,
-              originalFileLocation: openApiFilePath,
-            },
-          ],
-          openapiMetaTag
-        );
-        navGroup.push(filenameWithoutExtension);
-        const page: DecoratedNavigationPage = {
-          openapi: openapiMetaTag,
-          href: resolve('/', filenameWithoutExtension),
-          title: titleTag ?? slugToTitle(filenameWithoutExtension),
-          description,
-          version,
-        };
-        decoratedNavGroup.push(page);
-        pagesAcc[filenameWithoutExtension] = page;
-        const targetPath = outDirBasePath
-          ? join(outDirBasePath, `${filenameWithoutExtension}.mdx`)
-          : `${filenameWithoutExtension}.mdx`;
-        if (writeFiles && (!fse.pathExistsSync(targetPath) || opts?.overwrite)) {
-          writePromises.push(createOpenApiFrontmatter(targetPath, openapiMetaTag, version));
-        }
-      }
-    });
+    processOpenApiPath<Navigation, DecoratedNavigation>(
+      path,
+      pathItemObject,
+      schema as OpenAPI.Document,
+      nav,
+      decoratedNav,
+      writePromises,
+      pagesAcc,
+      opts ?? {},
+      findNavGroup
+    );
   });
 
   await Promise.all(writePromises);
@@ -146,16 +58,11 @@ export const generateOpenApiPages = async (
     pagesAcc,
     isUrl,
   };
-};
-
-// returns the group with the given group name, or the top-level group if no group name is provided
+}
 const findNavGroup = <T extends NavigationGroup | DecoratedNavigationGroup>(
   nav: T['pages'][number][],
-  groupName?: string
+  groupName: string = DEFAULT_API_GROUP_NAME
 ): T['pages'][number][] => {
-  if (groupName === undefined) {
-    groupName = 'API Reference';
-  }
   const group = nav.find(
     (fileOrGroup) =>
       typeof fileOrGroup === 'object' && 'group' in fileOrGroup && fileOrGroup.group === groupName
@@ -171,41 +78,3 @@ const findNavGroup = <T extends NavigationGroup | DecoratedNavigationGroup>(
     return group.pages;
   }
 };
-
-// returns a filename that is unique within the given array of pages
-const generateUniqueFilenameWithoutExtension = (pages: NavigationEntry[], base: string): string => {
-  let filename = base;
-  if (pages.includes(filename)) {
-    let extension = 1;
-    filename = `${base}-${extension}`;
-    while (pages.includes(filename)) {
-      extension += 1;
-      filename = `${base}-${extension}`;
-    }
-  }
-  return filename;
-};
-
-const createOpenApiFrontmatter = async (
-  filename: string,
-  openApiMetaTag: string,
-  version?: string
-) => {
-  const data = `---
-openapi: ${openApiMetaTag}${version ? `\nversion: ${version}` : ''}
----`;
-
-  await outputFile(filename, data);
-};
-
-export const prepareStringToBeValidFilename = (str?: string) =>
-  str
-    ? str
-        .replaceAll(' ', '-')
-        .replace(/\{.*?\}/g, '-') // remove path parameters
-        .replace(/^-/, '')
-        .replace(/-$/, '')
-        .replace(/[{}(),.'\n\/]/g, '') // remove special characters
-        .replaceAll(/--/g, '-') // replace double hyphens
-        .toLowerCase()
-    : undefined;

package/src/openapi/generateOpenApiPagesForDocsConfig.ts

@@ -0,0 +1,81 @@
+import { validate } from '@mintlify/common';
+import { DecoratedNavigationPage } from '@mintlify/models';
+import {
+  DecoratedGroupsConfig,
+  GroupsConfig,
+  PagesConfig,
+  DecoratedPagesConfig,
+} from '@mintlify/validation';
+import { OpenAPI } from 'openapi-types';
+
+import {
+  getOpenApiDefinition,
+  GenerateOpenApiPagesOptions,
+  OpenApiPageGenerationResult,
+  processOpenApiPath,
+  DEFAULT_API_GROUP_NAME,
+} from './common.js';
+
+export async function generateOpenApiPagesForDocsConfig(
+  pathOrDocumentOrUrl: string | OpenAPI.Document | URL,
+  opts?: GenerateOpenApiPagesOptions
+): Promise<OpenApiPageGenerationResult<GroupsConfig, DecoratedGroupsConfig>> {
+  const { document, isUrl } = await getOpenApiDefinition(pathOrDocumentOrUrl);
+  const { schema } = await validate(document);
+
+  if (schema?.paths === undefined || Object.keys(schema.paths).length === 0) {
+    throw new Error('No paths defined.');
+  }
+
+  const nav: GroupsConfig = [];
+  const decoratedNav: DecoratedGroupsConfig = [];
+  const writePromises: Promise<void>[] = [];
+  const pagesAcc: Record<string, DecoratedNavigationPage> = {};
+
+  Object.entries(schema.paths).forEach(([path, pathItemObject]) => {
+    if (!pathItemObject || typeof pathItemObject !== 'object') {
+      return;
+    }
+    processOpenApiPath<GroupsConfig, DecoratedGroupsConfig>(
+      path,
+      pathItemObject,
+      schema as OpenAPI.Document,
+      nav,
+      decoratedNav,
+      writePromises,
+      pagesAcc,
+      opts ?? {},
+      findNavGroup
+    );
+  });
+
+  await Promise.all(writePromises);
+
+  return {
+    nav,
+    decoratedNav,
+    spec: schema as OpenAPI.Document,
+    pagesAcc,
+    isUrl,
+  };
+}
+
+const findNavGroup = (
+  nav: GroupsConfig | DecoratedGroupsConfig,
+  groupName: string = DEFAULT_API_GROUP_NAME
+): PagesConfig | DecoratedPagesConfig => {
+  const group = nav.find(
+    (fileOrGroup) =>
+      typeof fileOrGroup === 'object' && 'group' in fileOrGroup && fileOrGroup.group === groupName
+  );
+  if (group === undefined || !('pages' in group)) {
+    const newGroup = {
+      group: groupName,
+      pages: [],
+    };
+    nav.push(newGroup);
+    return newGroup.pages;
+  } else {
+    return group.pages;
+  }
+};