@mintlify/cli 4.0.953 → 4.0.955

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/cli",
-  "version": "4.0.953",
+  "version": "4.0.955",
   "description": "The Mintlify CLI",
   "engines": {
     "node": ">=18.0.0"
@@ -45,12 +45,13 @@
   },
   "dependencies": {
     "@inquirer/prompts": "7.9.0",
-    "@mintlify/common": "1.0.728",
-    "@mintlify/link-rot": "3.0.888",
+    "@mintlify/common": "1.0.729",
+    "@mintlify/link-rot": "3.0.890",
     "@mintlify/models": "0.0.272",
-    "@mintlify/prebuild": "1.0.864",
-    "@mintlify/previewing": "4.0.921",
-    "@mintlify/validation": "0.1.594",
+    "@mintlify/prebuild": "1.0.866",
+    "@mintlify/previewing": "4.0.923",
+    "@mintlify/scraping": "4.0.591",
+    "@mintlify/validation": "0.1.595",
     "adm-zip": "0.5.16",
     "chalk": "5.2.0",
     "color": "4.2.3",
@@ -86,5 +87,5 @@
     "vitest": "2.0.4",
     "vitest-mock-process": "1.0.4"
   },
-  "gitHead": "fdbcfc062b0d175a55530b0021aca2acaa98906b"
+  "gitHead": "c40f23d742edb39a24f4259970f9cd5f0a506f3c"
 }

package/src/cli.tsx

@@ -12,6 +12,7 @@ import {
   BrokenLinksLog,
   WarningLog,
 } from '@mintlify/previewing';
+import { checkUrl } from '@mintlify/scraping';
 import { render, Text } from 'ink';
 import path from 'path';
 import yargs from 'yargs';
@@ -32,6 +33,7 @@ import {
 import { init } from './init.js';
 import { mdxLinter } from './mdxLinter.js';
 import { migrateMdx } from './migrateMdx.js';
+import { scrapeSite, scrapePage, scrapeOpenApi } from './scrape.js';
 import { update } from './update.js';
 
 export const cli = ({ packageName = 'mint' }: { packageName?: string }) => {
@@ -408,6 +410,80 @@ export const cli = ({ packageName = 'mint' }: { packageName?: string }) => {
           }
         }
       )
+      .command('scrape', 'Scrape documentation from external sites', (yargs) =>
+        yargs
+          .command(
+            ['$0 <url>', 'site <url>'],
+            'Scrape an entire documentation site',
+            (yargs) =>
+              yargs
+                .positional('url', {
+                  describe: 'The URL of the documentation site to scrape',
+                  type: 'string',
+                  demandOption: true,
+                })
+                .option('filter', {
+                  describe:
+                    'Only scrape URLs matching this path filter (e.g. /docs will match /docs and /docs/*)',
+                  type: 'string',
+                  alias: 'f',
+                })
+                .check(checkUrl),
+            async ({ url, filter }) => {
+              await scrapeSite(url, filter);
+            }
+          )
+          .command(
+            'page <url>',
+            'Scrape a single documentation page',
+            (yargs) =>
+              yargs
+                .positional('url', {
+                  describe: 'The URL of the documentation page to scrape',
+                  type: 'string',
+                  demandOption: true,
+                })
+                .check(checkUrl),
+            async ({ url }) => {
+              await scrapePage(url);
+            }
+          )
+          .command(
+            'openapi <openapiLocation>',
+            'Generate MDX files from an OpenAPI spec',
+            (yargs) =>
+              yargs
+                .positional('openapiLocation', {
+                  describe: 'The filename or URL location of the OpenAPI spec',
+                  type: 'string',
+                  demandOption: true,
+                })
+                .option('writeFiles', {
+                  describe: 'Whether or not to write the frontmatter files',
+                  default: true,
+                  type: 'boolean',
+                  alias: 'w',
+                })
+                .option('outDir', {
+                  describe: 'The folder in which to write any created frontmatter files',
+                  type: 'string',
+                  alias: 'o',
+                })
+                .option('overwrite', {
+                  describe: 'Whether or not to overwrite existing files',
+                  default: false,
+                  type: 'boolean',
+                }),
+            async (argv) => {
+              await scrapeOpenApi({
+                openapiLocation: argv.openapiLocation,
+                writeFiles: argv.writeFiles,
+                outDir: argv.outDir,
+                overwrite: argv.overwrite,
+              });
+            }
+          )
+      )
       // Print the help menu when the user enters an invalid command.
       .strictCommands()
       .demandCommand(1, 'unknown command. see above for the list of supported commands.')

package/src/scrape.tsx

@@ -0,0 +1,122 @@
+import { MintConfigType } from '@mintlify/models';
+import { addLog, ErrorLog, SuccessLog, SpinnerLog, InfoLog } from '@mintlify/previewing';
+import {
+  scrapePageGroup,
+  scrapeAllSiteTabs,
+  htmlToHast,
+  detectFramework,
+  framework,
+  fetchPageHtml,
+  write,
+  getErrorMessage,
+  generateOpenApiPages,
+  FINAL_SUCCESS_MESSAGE,
+} from '@mintlify/scraping';
+import { upgradeToDocsConfig } from '@mintlify/validation';
+
+import { terminate } from './helpers.js';
+
+export async function scrapeSite(url: string, filter?: string) {
+  try {
+    const urlObj = new URL(url);
+    addLog(<SpinnerLog message={`Fetching ${urlObj.toString()}...`} />);
+    const html = await fetchPageHtml(urlObj);
+    addLog(<SuccessLog message={`Successfully retrieved HTML from ${urlObj.toString()}`} />);
+
+    addLog(<SpinnerLog message="Scraping site..." />);
+    const result = await scrapeAllSiteTabs(html, urlObj, { filter });
+    if (result.success) {
+      const mintConfig = result.data as MintConfigType;
+      const docsConfig = upgradeToDocsConfig(mintConfig, {
+        shouldUpgradeTheme: true,
+      });
+      docsConfig.theme = 'aspen';
+      write('docs.json', JSON.stringify(docsConfig, undefined, 2));
+      addLog(<SuccessLog message={FINAL_SUCCESS_MESSAGE} />);
+    } else {
+      addLog(<ErrorLog message={result.message} />);
+      await terminate(1);
+    }
+    await terminate(0);
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    addLog(<ErrorLog message={errorMessage} />);
+    await terminate(1);
+  }
+}
+
+export async function scrapePage(url: string) {
+  try {
+    const urlObj = new URL(url);
+    addLog(<SpinnerLog message={`Fetching ${urlObj.toString()}...`} />);
+    const html = await fetchPageHtml(urlObj);
+    addLog(<SuccessLog message={`Successfully retrieved HTML from ${urlObj.toString()}`} />);
+
+    const hast = htmlToHast(html);
+    detectFramework(hast);
+
+    const needsBrowser = framework.vendor === 'gitbook';
+    addLog(<SpinnerLog message="Scraping page..." />);
+    const results = await scrapePageGroup([urlObj], needsBrowser);
+    const result = results[0] || {
+      success: false,
+      message: `An unknown error occurred when scraping ${url}`,
+    };
+
+    if (result.success) {
+      addLog(
+        <SuccessLog
+          message={`Successfully scraped ${url} ${result.data ? `into ${result.data[1]}` : ''}`}
+        />
+      );
+    } else {
+      addLog(<ErrorLog message={result.message} />);
+      await terminate(1);
+    }
+    await terminate(0);
+  } catch (error) {
+    const errorMessage = getErrorMessage(error);
+    addLog(<ErrorLog message={errorMessage} />);
+    await terminate(1);
+  }
+}
+
+interface ScrapeOpenApiOptions {
+  openapiLocation: string;
+  writeFiles: boolean;
+  outDir?: string;
+  overwrite: boolean;
+}
+
+export async function scrapeOpenApi({
+  openapiLocation,
+  writeFiles,
+  outDir,
+  overwrite,
+}: ScrapeOpenApiOptions) {
+  try {
+    addLog(<SpinnerLog message={`Processing OpenAPI spec from ${openapiLocation}...`} />);
+    const { nav, isUrl } = await generateOpenApiPages(openapiLocation, {
+      openApiFilePath: undefined,
+      version: undefined,
+      writeFiles,
+      outDir,
+      overwrite,
+    });
+    addLog(<SuccessLog message="Successfully generated OpenAPI pages" />);
+    addLog(<InfoLog message="Navigation object suggestion:" />);
+    addLog(<InfoLog message={JSON.stringify(nav, undefined, 2)} />);
+    if (isUrl) {
+      addLog(<InfoLog message="OpenAPI location suggestion:" />);
+      addLog(<InfoLog message={`openapi: ${openapiLocation}`} />);
+    }
+    await terminate(0);
+  } catch (error) {
+    if (error instanceof Error) {
+      addLog(<ErrorLog message={error.message} />);
+    } else {
+      addLog(<ErrorLog message={String(error)} />);
+    }
+    await terminate(1);
+  }
+}