@mintlify/cli 4.0.1080 → 4.0.1081

package/src/migrateMdx.tsx

@@ -1,469 +0,0 @@
-import { potentiallyParseOpenApiString, parseFrontmatter } from '@mintlify/common';
-import { getConfigObj, getConfigPath } from '@mintlify/prebuild';
-import { addLog, ErrorLog, SuccessLog } from '@mintlify/previewing';
-import {
-  divisions,
-  DocsConfig,
-  NavigationConfig,
-  validateDocsConfig,
-  XMint,
-  XMcp,
-} from '@mintlify/validation';
-import fs from 'fs';
-import { outputFile } from 'fs-extra';
-import inquirer from 'inquirer';
-import yaml from 'js-yaml';
-import { OpenAPI, OpenAPIV3 } from 'openapi-types';
-import path from 'path';
-
-import { CMD_EXEC_PATH } from './constants.js';
-
-const specCache: Record<string, OpenAPI.Document> = {};
-
-const candidateSpecCache: Record<string, OpenAPI.Document> = {};
-
-const specLocks = new Map<string, Promise<void>>();
-
-async function withSpecLock(specPath: string, task: () => Promise<void>) {
-  const key = path.resolve(specPath);
-  const previous = specLocks.get(key) ?? Promise.resolve();
-  let releaseNext: () => void;
-  const next = new Promise<void>((resolve) => {
-    releaseNext = resolve;
-  });
-  specLocks.set(key, next);
-
-  await previous;
-  try {
-    await task();
-  } finally {
-    releaseNext!();
-  }
-}
-
-let inquirerLockQueue: Promise<void> = Promise.resolve();
-
-async function withInquirerLock<T>(task: () => Promise<T>): Promise<T> {
-  const previous = inquirerLockQueue;
-  let releaseNext: () => void;
-  const next = new Promise<void>((resolve) => {
-    releaseNext = resolve;
-  });
-  inquirerLockQueue = next;
-  await previous;
-  try {
-    return await task();
-  } finally {
-    releaseNext!();
-  }
-}
-
-export async function migrateMdx() {
-  const docsConfigPath = await getConfigPath(CMD_EXEC_PATH, 'docs');
-
-  if (!docsConfigPath) {
-    addLog(<ErrorLog message="docs.json not found in current directory" />);
-    return;
-  }
-
-  const rawConfig = JSON.parse(await fs.promises.readFile(docsConfigPath, 'utf-8')) as DocsConfig;
-
-  const docsConfigObj = await getConfigObj(CMD_EXEC_PATH, 'docs');
-
-  const validationResults = await validateDocsConfig(docsConfigObj as DocsConfig);
-  if (!validationResults.success) {
-    addLog(<ErrorLog message="docs.json is invalid" />);
-    return;
-  }
-
-  const validatedDocsConfig = validationResults.data as DocsConfig;
-
-  await buildCandidateSpecCacheIfNeeded(CMD_EXEC_PATH);
-
-  const updatedNavigation = await processNav(validatedDocsConfig.navigation);
-
-  rawConfig.navigation = updatedNavigation;
-  await outputFile(docsConfigPath, JSON.stringify(rawConfig, null, 2));
-  addLog(<SuccessLog message="docs.json updated" />);
-
-  for (const specPath in specCache) {
-    const specObj = specCache[specPath] as OpenAPI.Document;
-    const ext = path.extname(specPath).toLowerCase();
-    const stringified = ext === '.json' ? JSON.stringify(specObj, null, 2) : yaml.dump(specObj);
-    await outputFile(specPath, stringified);
-    addLog(<SuccessLog message={`updated ${path.relative(CMD_EXEC_PATH, specPath)}`} />);
-  }
-
-  addLog(<SuccessLog message="migration complete" />);
-}
-
-async function processNav(nav: NavigationConfig): Promise<NavigationConfig> {
-  let newNav: NavigationConfig = { ...nav };
-
-  if ('pages' in newNav) {
-    const newPages = await Promise.all(
-      newNav.pages.map(async (page) => {
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-        if (typeof page === 'object' && page !== null && 'group' in page) {
-          return processNav(page as NavigationConfig);
-        }
-        if (typeof page === 'string' && !/\s/.test(page)) {
-          const mdxCandidatePath = path.join(CMD_EXEC_PATH, `${page}.mdx`);
-          if (!fs.existsSync(mdxCandidatePath)) {
-            return page;
-          }
-          const fmParsed = parseFrontmatter<Record<string, string>>(
-            await fs.promises.readFile(mdxCandidatePath, 'utf-8')
-          );
-          const frontmatter = fmParsed.attributes;
-          const content = fmParsed.body;
-          if (!frontmatter.openapi) {
-            return page;
-          }
-          const parsed = potentiallyParseOpenApiString(frontmatter.openapi);
-          if (!parsed) {
-            addLog(
-              <ErrorLog
-                message={`invalid openapi frontmatter in ${mdxCandidatePath}: ${frontmatter.openapi}`}
-              />
-            );
-            return page;
-          }
-
-          const { filename, method, endpoint: endpointPath } = parsed;
-          let specPath = filename;
-
-          if (specPath && URL.canParse(specPath)) {
-            return page;
-          }
-
-          if (!specPath) {
-            const methodLower = method.toLowerCase();
-            const matchingSpecs = await findMatchingOpenApiSpecs(
-              {
-                method: methodLower,
-                endpointPath,
-              },
-              candidateSpecCache
-            );
-
-            if (matchingSpecs.length === 0) {
-              addLog(
-                <ErrorLog
-                  message={`no OpenAPI spec found for ${method.toUpperCase()} ${endpointPath} in repository`}
-                />
-              );
-              return page;
-            }
-
-            if (matchingSpecs.length === 1) {
-              specPath = path.relative(CMD_EXEC_PATH, matchingSpecs[0]!);
-            } else {
-              const answer = await withInquirerLock(() =>
-                inquirer.prompt([
-                  {
-                    type: 'list',
-                    name: 'chosen',
-                    message: `multiple OpenAPI specs found for ${method.toUpperCase()} ${endpointPath}. which one should be used for ${path.relative(
-                      CMD_EXEC_PATH,
-                      mdxCandidatePath
-                    )}?`,
-                    choices: matchingSpecs.map((p) => ({
-                      name: path.relative(CMD_EXEC_PATH, p),
-                      value: path.relative(CMD_EXEC_PATH, p),
-                    })),
-                  },
-                ])
-              );
-              specPath = answer.chosen as string;
-            }
-          }
-
-          const href = `/${page}`;
-          const pageName = specPath ? `${specPath} ${method} ${endpointPath}` : frontmatter.openapi;
-          delete frontmatter.openapi;
-          await withSpecLock(path.resolve(specPath), () =>
-            migrateToXMint({
-              specPath,
-              method,
-              endpointPath,
-              frontmatter,
-              content,
-              href,
-            })
-          );
-
-          try {
-            await fs.promises.unlink(mdxCandidatePath);
-          } catch (err) {
-            addLog(
-              <ErrorLog
-                message={`failed to delete ${mdxCandidatePath}: ${(err as Error).message}`}
-              />
-            );
-          }
-
-          return pageName;
-        }
-        return page;
-      })
-    );
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    newNav.pages = newPages as any;
-  }
-
-  for (const division of ['groups', ...divisions]) {
-    if (division in newNav) {
-      const items = newNav[division as keyof typeof newNav] as NavigationConfig[];
-      newNav = {
-        ...newNav,
-        [division]: await Promise.all(items.map((item) => processNav(item))),
-      };
-    }
-  }
-
-  return newNav;
-}
-
-async function migrateToXMint(args: {
-  specPath: string;
-  method: string;
-  endpointPath: string;
-  frontmatter: Record<string, string>;
-  content: string;
-  href: string;
-}) {
-  const { specPath, method, endpointPath, frontmatter, content, href } = args;
-
-  if (!fs.existsSync(specPath)) {
-    addLog(<ErrorLog message={`spec file not found: ${specPath}`} />);
-    return;
-  }
-
-  let specObj: OpenAPI.Document;
-  if (path.resolve(specPath) in specCache) {
-    specObj = specCache[path.resolve(specPath)] as OpenAPI.Document;
-  } else {
-    const pathname = path.join(CMD_EXEC_PATH, specPath);
-    const file = await fs.promises.readFile(pathname, 'utf-8');
-    const ext = path.extname(specPath).toLowerCase();
-    if (ext === '.json') {
-      specObj = JSON.parse(file) as OpenAPI.Document;
-    } else if (ext === '.yml' || ext === '.yaml') {
-      specObj = yaml.load(file) as OpenAPI.Document;
-    } else {
-      addLog(<ErrorLog message={`unsupported spec file extension: ${specPath}`} />);
-      return;
-    }
-  }
-
-  const methodLower = method.toLowerCase();
-
-  if (
-    !editXMint(specObj, endpointPath, methodLower, {
-      metadata: Object.keys(frontmatter).length > 0 ? frontmatter : undefined,
-      content: content.length > 0 ? content : undefined,
-      href,
-    })
-  ) {
-    addLog(
-      <ErrorLog
-        message={`operation not found in spec: ${method.toUpperCase()} ${endpointPath} in ${specPath}`}
-      />
-    );
-    return;
-  }
-
-  specCache[path.resolve(specPath)] = specObj;
-}
-
-function editXMint(
-  document: OpenAPI.Document,
-  path: string,
-  method: string,
-  newXMint: XMint
-): boolean {
-  if (method === 'webhook') {
-    return editWebhookXMint(document, path, newXMint);
-  }
-
-  if (!document.paths || !document.paths[path]) {
-    return false;
-  }
-
-  const pathItem = document.paths[path] as OpenAPIV3.PathItemObject;
-  const normalizedMethod = method.toLowerCase() as keyof OpenAPIV3.PathItemObject;
-
-  if (!pathItem[normalizedMethod]) {
-    return false;
-  }
-
-  const operation = pathItem[normalizedMethod] as OpenAPI.Operation<{
-    'x-mint'?: XMint;
-    'x-mcp'?: XMcp;
-  }>;
-  operation['x-mint'] = newXMint;
-
-  if ('x-mcp' in operation && !('mcp' in operation['x-mint'])) {
-    operation['x-mint']['mcp'] = operation['x-mcp'];
-    delete operation['x-mcp'];
-  }
-
-  return true;
-}
-
-function editWebhookXMint(document: OpenAPI.Document, path: string, newXMint: XMint): boolean {
-  const webhookObject = (
-    document as OpenAPIV3.Document & {
-      webhooks?: Record<string, OpenAPIV3.PathItemObject>;
-    }
-  ).webhooks?.[path];
-  if (!webhookObject || typeof webhookObject !== 'object') {
-    return false;
-  }
-
-  if (!webhookObject['post']) {
-    return false;
-  }
-
-  const operation = webhookObject['post'] as OpenAPI.Operation<{
-    'x-mint'?: XMint;
-    'x-mcp'?: XMcp;
-  }>;
-  operation['x-mint'] = newXMint;
-
-  if ('x-mcp' in operation && !('mcp' in operation['x-mint'])) {
-    operation['x-mint']['mcp'] = operation['x-mcp'];
-    delete operation['x-mcp'];
-  }
-  return true;
-}
-
-async function findMatchingOpenApiSpecs(
-  args: {
-    method: string;
-    endpointPath: string;
-  },
-  docsByPath?: Record<string, OpenAPI.Document>
-): Promise<string[]> {
-  const { method, endpointPath } = args;
-  const docsEntries: Array<[string, OpenAPI.Document | undefined]> = docsByPath
-    ? Object.entries(docsByPath)
-    : (await collectOpenApiFiles(CMD_EXEC_PATH)).map((absPath) => [absPath, undefined]);
-  const normalizedMethod = method.toLowerCase();
-
-  const endpointVariants = new Set<string>([endpointPath]);
-  if (!endpointPath.startsWith('/')) {
-    endpointVariants.add(`/${endpointPath}`);
-  } else {
-    endpointVariants.add(endpointPath.replace(/^\/+/, ''));
-  }
-
-  const matches: string[] = [];
-
-  for (const [absPath, maybeDoc] of docsEntries) {
-    try {
-      const doc = maybeDoc || (await loadOpenApiDocument(absPath));
-      if (!doc) continue;
-
-      if (normalizedMethod === 'webhook') {
-        const webhooks = (
-          doc as OpenAPIV3.Document & {
-            webhooks?: Record<string, OpenAPIV3.PathItemObject>;
-          }
-        ).webhooks;
-        if (!webhooks) continue;
-        for (const key of Object.keys(webhooks)) {
-          if (endpointVariants.has(key)) {
-            const pathItem = webhooks[key];
-            if (pathItem && typeof pathItem === 'object' && 'post' in pathItem && pathItem.post) {
-              matches.push(absPath);
-              break;
-            }
-          }
-        }
-        continue;
-      }
-
-      if (!doc.paths) continue;
-      for (const variant of endpointVariants) {
-        const pathItem = (doc.paths as Record<string, unknown>)[variant] as
-          | OpenAPIV3.PathItemObject
-          | undefined;
-        if (!pathItem) continue;
-        const hasOperation = !!(pathItem as Record<string, unknown>)[
-          normalizedMethod as keyof OpenAPIV3.PathItemObject
-        ];
-        if (hasOperation) {
-          matches.push(absPath);
-          break;
-        }
-      }
-    } catch {}
-  }
-
-  return matches.map((abs) => path.resolve(abs)).filter((v, i, a) => a.indexOf(v) === i);
-}
-
-async function collectOpenApiFiles(rootDir: string): Promise<string[]> {
-  const results: string[] = [];
-  const excludedDirs = new Set([
-    'node_modules',
-    '.git',
-    'dist',
-    'build',
-    '.next',
-    '.vercel',
-    'out',
-    'coverage',
-    'tmp',
-    'temp',
-  ]);
-
-  async function walk(currentDir: string) {
-    const entries = await fs.promises.readdir(currentDir, { withFileTypes: true });
-    for (const entry of entries) {
-      const abs = path.join(currentDir, entry.name);
-      if (entry.isDirectory()) {
-        if (excludedDirs.has(entry.name)) continue;
-        await walk(abs);
-      } else if (entry.isFile()) {
-        if (/\.(ya?ml|json)$/i.test(entry.name)) {
-          results.push(abs);
-        }
-      }
-    }
-  }
-
-  await walk(rootDir);
-  return results;
-}
-
-async function loadOpenApiDocument(absPath: string): Promise<OpenAPI.Document | undefined> {
-  try {
-    const file = await fs.promises.readFile(absPath, 'utf-8');
-    const ext = path.extname(absPath).toLowerCase();
-    let doc: OpenAPI.Document | undefined;
-    if (ext === '.json') {
-      doc = JSON.parse(file) as OpenAPI.Document;
-    } else if (ext === '.yml' || ext === '.yaml') {
-      doc = yaml.load(file) as OpenAPI.Document;
-    }
-    return doc;
-  } catch {
-    return undefined;
-  }
-}
-
-async function buildCandidateSpecCacheIfNeeded(rootDir: string) {
-  if (Object.keys(candidateSpecCache).length > 0) return;
-  const files = await collectOpenApiFiles(rootDir);
-  await Promise.all(
-    files.map(async (abs) => {
-      const doc = await loadOpenApiDocument(abs);
-      if (doc) {
-        candidateSpecCache[path.resolve(abs)] = doc;
-      }
-    })
-  );
-}

package/src/scrape.tsx

@@ -1,122 +0,0 @@
-import { MintConfigType } from '@mintlify/models';
-import { addLog, ErrorLog, SuccessLog, SpinnerLog, InfoLog } from '@mintlify/previewing';
-import {
-  scrapePageGroup,
-  scrapeAllSiteTabs,
-  htmlToHast,
-  detectFramework,
-  framework,
-  fetchPageHtml,
-  write,
-  getErrorMessage,
-  generateOpenApiPages,
-  FINAL_SUCCESS_MESSAGE,
-} from '@mintlify/scraping';
-import { upgradeToDocsConfig } from '@mintlify/validation';
-
-import { terminate } from './helpers.js';
-
-export async function scrapeSite(url: string, filter?: string) {
-  try {
-    const urlObj = new URL(url);
-    addLog(<SpinnerLog message={`Fetching ${urlObj.toString()}...`} />);
-    const html = await fetchPageHtml(urlObj);
-    addLog(<SuccessLog message={`Successfully retrieved HTML from ${urlObj.toString()}`} />);
-
-    addLog(<SpinnerLog message="Scraping site..." />);
-    const result = await scrapeAllSiteTabs(html, urlObj, { filter });
-    if (result.success) {
-      const mintConfig = result.data as MintConfigType;
-      const docsConfig = upgradeToDocsConfig(mintConfig, {
-        shouldUpgradeTheme: true,
-      });
-      docsConfig.theme = 'aspen';
-      write('docs.json', JSON.stringify(docsConfig, undefined, 2));
-      addLog(<SuccessLog message={FINAL_SUCCESS_MESSAGE} />);
-    } else {
-      addLog(<ErrorLog message={result.message} />);
-      await terminate(1);
-    }
-    await terminate(0);
-  } catch (error) {
-    const errorMessage = getErrorMessage(error);
-    addLog(<ErrorLog message={errorMessage} />);
-    await terminate(1);
-  }
-}
-
-export async function scrapePage(url: string) {
-  try {
-    const urlObj = new URL(url);
-    addLog(<SpinnerLog message={`Fetching ${urlObj.toString()}...`} />);
-    const html = await fetchPageHtml(urlObj);
-    addLog(<SuccessLog message={`Successfully retrieved HTML from ${urlObj.toString()}`} />);
-
-    const hast = htmlToHast(html);
-    detectFramework(hast);
-
-    const needsBrowser = framework.vendor === 'gitbook';
-    addLog(<SpinnerLog message="Scraping page..." />);
-    const results = await scrapePageGroup([urlObj], needsBrowser);
-    const result = results[0] || {
-      success: false,
-      message: `An unknown error occurred when scraping ${url}`,
-    };
-
-    if (result.success) {
-      addLog(
-        <SuccessLog
-          message={`Successfully scraped ${url} ${result.data ? `into ${result.data[1]}` : ''}`}
-        />
-      );
-    } else {
-      addLog(<ErrorLog message={result.message} />);
-      await terminate(1);
-    }
-    await terminate(0);
-  } catch (error) {
-    const errorMessage = getErrorMessage(error);
-    addLog(<ErrorLog message={errorMessage} />);
-    await terminate(1);
-  }
-}
-
-interface ScrapeOpenApiOptions {
-  openapiLocation: string;
-  writeFiles: boolean;
-  outDir?: string;
-  overwrite: boolean;
-}
-
-export async function scrapeOpenApi({
-  openapiLocation,
-  writeFiles,
-  outDir,
-  overwrite,
-}: ScrapeOpenApiOptions) {
-  try {
-    addLog(<SpinnerLog message={`Processing OpenAPI spec from ${openapiLocation}...`} />);
-    const { nav, isUrl } = await generateOpenApiPages(openapiLocation, {
-      openApiFilePath: undefined,
-      version: undefined,
-      writeFiles,
-      outDir,
-      overwrite,
-    });
-    addLog(<SuccessLog message="Successfully generated OpenAPI pages" />);
-    addLog(<InfoLog message="Navigation object suggestion:" />);
-    addLog(<InfoLog message={JSON.stringify(nav, undefined, 2)} />);
-    if (isUrl) {
-      addLog(<InfoLog message="OpenAPI location suggestion:" />);
-      addLog(<InfoLog message={`openapi: ${openapiLocation}`} />);
-    }
-    await terminate(0);
-  } catch (error) {
-    if (error instanceof Error) {
-      addLog(<ErrorLog message={error.message} />);
-    } else {
-      addLog(<ErrorLog message={String(error)} />);
-    }
-    await terminate(1);
-  }
-}