@mintlify/scraping 3.0.16 → 3.0.18

package/bin/util.d.ts

@@ -1,3 +1,4 @@
+import { NavigationEntry } from '@mintlify/models';
 import { Ora as OraType } from 'ora';
 export declare const MintConfig: (name: string, color: string, ctaName: string, ctaUrl: string, filename: string) => {
     name: string;
@@ -19,7 +20,7 @@ export declare const MintConfig: (name: string, color: string, ctaName: string,
 };
 export declare const Page: (title: string, description?: string, markdown?: string) => string;
 export declare function getOrigin(url: string): string;
-export declare function objToReadableString(objs: MintNavigationEntry[]): string;
+export declare function objToReadableString(objs: NavigationEntry[]): string;
 export declare const toFilename: (title: string) => string;
 export declare const addMdx: (fileName: string) => string;
 export declare const createPage: (title: string, description?: string, markdown?: string, overwrite?: boolean, rootDir?: string, fileName?: string) => void;

package/bin/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,GAAuB,MAAM,KAAK,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAE5B,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,WAAoB,EAAE,QAAiB,EAAE,EAAE;IAC7E,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE;QACpB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;KACrB;IACD,IAAI,CAAC,aAAa,EAAE;QAClB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;KACrB;IAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,CAAC,CAAC,mBAAmB,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAA2B;IAC7D,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,SAAS,GAAG,KAAK,EACjB,OAAO,GAAG,EAAE,EACZ,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE;QACb,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;KAClC;SAAM;QACL,IAAI;YACF,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;SAClC;QAAC,OAAO,CAAC,EAAE;YACV,yEAAyE;YACzE,sDAAsD;YACtD,IAAK,CAAsB,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC9C,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;aACtD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;SACF;KACF;AACH,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,IAAS;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;IACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,GAAG,EAAE,EAAW,EAAE;IACrD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,SAAS,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAe,EAAE,EAAE;IAC5C,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,CAAC,IAAI,CAAC;;;;KAIX,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,GAAuB,MAAM,KAAK,CAAC;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,SAAS,CAAC;AAE5B,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,EAAE;IACF,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,MAAM,EAAE;YACN,OAAO,EAAE,KAAK;SACf;QACD,WAAW,EAAE,EAAE;QACf,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,MAAM;SACZ;QACD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE;YACV;gBACE,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,CAAC,QAAQ,CAAC;aAClB;SACF;QACD,6DAA6D;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,WAAoB,EAAE,QAAiB,EAAE,EAAE;IAC7E,uDAAuD;IACvD,yDAAyD;IACzD,wBAAwB;IACxB,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,eAAe,EAAE;QACpB,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC;KACrB;IACD,IAAI,CAAC,aAAa,EAAE;QAClB,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;KACrB;IAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,CAAC,CAAC,mBAAmB,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO,eAAe,KAAK,GAAG,mBAAmB,YAAY,QAAQ,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,+CAA+C;IAC/C,gDAAgD;IAChD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,4BAA4B;IAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,sDAAsD;IACtD,uDAAuD;IACvD,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC7B,OAAO,QAAQ,CAAC;KACjB;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,KAAa,EACb,WAAoB,EACpB,QAAiB,EACjB,SAAS,GAAG,KAAK,EACjB,OAAO,GAAG,EAAE,EACZ,QAAiB,EACjB,EAAE;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5E,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,2BAA2B;IAC3B,IAAI,SAAS,EAAE;QACb,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;KAClC;SAAM;QACL,IAAI;YACF,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE;gBAC3D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;SAClC;QAAC,OAAO,CAAC,EAAE;YACV,yEAAyE;YACzE,sDAAsD;YACtD,IAAK,CAAsB,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC9C,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;aACtD;iBAAM;gBACL,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;SACF;KACF;AACH,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,IAAS;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;IACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,GAAG,EAAE,EAAW,EAAE;IACrD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC;AAC3B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,EAAE;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,SAAS,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAe,EAAE,EAAE;IAC5C,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,CAAC,IAAI,CAAC;;;;KAIX,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "3.0.16",
+  "version": "3.0.18",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -29,10 +29,16 @@
     "prepare": "npm run build",
     "build": "tsc",
     "watch": "tsc --watch",
+    "test": "jest",
     "lint": "eslint . --cache",
     "format": "prettier \"./src/**/*.ts\" --write",
     "format:check": "prettier \"./src/**/*.ts\" --check"
   },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node",
+    "verbose": true
+  },
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.0",
     "axios": "^1.2.2",
@@ -50,6 +56,7 @@
     "@mintlify/eslint-config-typescript": "1.0.7",
     "@mintlify/prettier-config": "1.0.1",
     "@mintlify/ts-config": "1.0.7",
+    "@mintlify/validation": "0.1.60",
     "@trivago/prettier-plugin-sort-imports": "3.x",
     "@tsconfig/recommended": "1.x",
     "@types/cheerio": "^0.22.31",
@@ -61,10 +68,11 @@
     "eslint": "8.x",
     "eslint-config-prettier": "8.x",
     "eslint-plugin-unused-imports": "2.x",
-    "jest": "^29.3.1",
+    "jest": "^29.6.1",
     "openapi-types": "^12.1.3",
     "prettier": "2.x",
+    "ts-jest": "^29.1.1",
     "typescript": "^4.8.2"
   },
-  "gitHead": "d470561e9abd9ef22eb1a275e121bd4940ffd2ad"
+  "gitHead": "1a17dd9dcd1b9a928ad1cd592a20c426c011bcd1"
 }

package/src/cli.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 /* eslint-disable @typescript-eslint/no-empty-function */
+import { existsSync } from 'fs';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
@@ -104,18 +105,29 @@ yargs(hideBin(process.argv))
         });
     },
     async (argv) => {
-      if (argv.openapiFilename === undefined) {
-        throw new Error(
-          'Provide the path to your OpenAPI file (e.g. mintlify-scrape openapi-files ./openapi.json)'
+      try {
+        if (argv.openapiFilename === undefined) {
+          throw new Error(
+            'Provide the path to your OpenAPI file (e.g. mintlify-scrape openapi-files ./openapi.json)'
+          );
+        }
+        if (!existsSync(argv.openapiFilename)) {
+          throw new Error(`The file "${argv.openapiFilename}" does not exist`);
+        }
+        const { nav } = await generateOpenApiPages(
+          argv.openapiFilename,
+          argv.writeFiles,
+          argv.outDir
         );
+        console.log('navigation object suggestion:');
+        console.log(JSON.stringify(nav, undefined, 2));
+      } catch (error) {
+        if (error instanceof Error) {
+          console.error(error.message);
+        } else {
+          console.error(error);
+        }
       }
-      const { nav } = await generateOpenApiPages(
-        argv.openapiFilename,
-        argv.writeFiles,
-        argv.outDir
-      );
-      console.log('navigation object suggestion:');
-      console.log(JSON.stringify(nav, undefined, 2));
     }
   )
   // Print the help menu when the user enters an invalid command.

package/src/openapi/index.ts

@@ -1,4 +1,11 @@
 import SwaggerParser from '@apidevtools/swagger-parser';
+import {
+  DecoratedNavigation,
+  DecoratedNavigationGroup,
+  Navigation,
+  NavigationEntry,
+  NavigationGroup,
+} from '@mintlify/models';
 import { outputFile } from 'fs-extra';
 import { join } from 'node:path';
 import { OpenAPI, OpenAPIV3 } from 'openapi-types';
@@ -8,8 +15,8 @@ export const generateOpenApiPages = async (
   writeFiles?: boolean,
   outDir?: string
 ): Promise<{
-  nav: MintNavigationEntry[];
-  decoratedNav: DecoratedMintNavigation[];
+  nav: Navigation;
+  decoratedNav: DecoratedNavigation;
   spec: OpenAPI.Document;
 }> => {
   let spec: OpenAPI.Document | undefined = undefined;
@@ -23,8 +30,8 @@ export const generateOpenApiPages = async (
     throw new Error('No paths defined.');
   }
 
-  const nav: MintNavigationEntry[] = [];
-  const decoratedNav: DecoratedMintNavigation[] = [];
+  const nav: Navigation = [];
+  const decoratedNav: DecoratedNavigation = [];
 
   Object.entries(spec.paths).forEach(([path, pathItemObject]) => {
     if (pathItemObject === undefined) {
@@ -35,12 +42,13 @@ export const generateOpenApiPages = async (
         const operation = pathItemObject[method];
         const groupName = operation?.tags?.[0];
         const title =
-          operation?.summary?.replaceAll(' ', '-') ??
-          `${method}-${path.replaceAll('/', '-').replace(/^-/, '').replace(/-$/, '')}`;
-        const base = join(outDir ?? '', groupName ?? '', title);
+          prepareStringToBeValidFilename(operation?.summary) ??
+          `${method}-${prepareStringToBeValidFilename(path)}`;
+        const folder = prepareStringToBeValidFilename(groupName) ?? '';
+        const base = join(outDir ?? '', folder, title);
 
-        const navGroup = findNavGroup<MintNavigation>(nav, groupName);
-        const decoratedNavGroup = findNavGroup<DecoratedMintNavigation>(decoratedNav, groupName);
+        const navGroup = findNavGroup<NavigationGroup>(nav, groupName);
+        const decoratedNavGroup = findNavGroup<DecoratedNavigationGroup>(decoratedNav, groupName);
 
         const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
         navGroup.push(filenameWithoutExtension);
@@ -62,7 +70,7 @@ export const generateOpenApiPages = async (
 };
 
 // returns the group with the given group name, or the top-level group if no group name is provided
-const findNavGroup = <T extends MintNavigation | DecoratedMintNavigation>(
+const findNavGroup = <T extends NavigationGroup | DecoratedNavigationGroup>(
   nav: T['pages'][number][],
   groupName?: string
 ): T['pages'][number][] => {
@@ -87,10 +95,7 @@ const findNavGroup = <T extends MintNavigation | DecoratedMintNavigation>(
 };
 
 // returns a filename that is unique within the given array of pages
-const generateUniqueFilenameWithoutExtension = (
-  pages: MintNavigationEntry[],
-  base: string
-): string => {
+const generateUniqueFilenameWithoutExtension = (pages: NavigationEntry[], base: string): string => {
   let filename = base;
   if (pages.includes(filename)) {
     let extension = 1;
@@ -114,3 +119,15 @@ openapi: ${method} ${path}
 
   outputFile(filename, data);
 };
+
+export const prepareStringToBeValidFilename = (str?: string) =>
+  str
+    ? str
+        .replaceAll(' ', '-')
+        .replace(/\{.*?\}/g, '-') // remove path parameters
+        .replace(/^-/, '')
+        .replace(/-$/, '')
+        .replace(/[{}(),.'\n\/]/g, '') // remove special characters
+        .replaceAll(/--/g, '-') // replace double hyphens
+        .toLowerCase()
+    : undefined;

package/src/scraping/combineNavWithEmptyGroupTitles.ts

@@ -1,9 +1,9 @@
-export default function combineNavWithEmptyGroupTitles(
-  navArray: MintNavigation[]
-): MintNavigation[] {
-  const newNavArray: MintNavigation[] = [];
+import { Navigation, NavigationGroup } from '@mintlify/models';
 
-  navArray.forEach((nav: MintNavigation) => {
+export default function combineNavWithEmptyGroupTitles(navArray: Navigation): Navigation {
+  const newNavArray: Navigation = [];
+
+  navArray.forEach((nav: NavigationGroup) => {
     // The first run through the loop will always have -1 as the index.
     // JavaScript returns undefined when we look for an index outside the size of the array.
     const prev = newNavArray[newNavArray.length - 1];

package/src/scraping/scrapeFileGettingFileNameFromUrl.ts

@@ -1,3 +1,4 @@
+import { NavigationEntry } from '@mintlify/models';
 import axios from 'axios';
 import path from 'path';
 
@@ -24,7 +25,7 @@ export async function scrapeFileGettingFileNameFromUrl(
   puppeteer = false,
   version: string | undefined,
   baseToRemove?: string
-): Promise<MintNavigationEntry> {
+): Promise<NavigationEntry> {
   // Skip scraping external links
   if (pathname.startsWith('https://') || pathname.startsWith('http://')) {
     return pathname;
@@ -64,7 +65,10 @@ export async function scrapeFileGettingFileNameFromUrl(
 
   // Check if page didn't have content
   if (!title && !markdown) {
-    return { pages: [] };
+    return {
+      group: '',
+      pages: [],
+    };
   }
 
   const newFileLocation = folders ? path.join(cliDir, folders) : cliDir;

package/src/scraping/scrapeGettingFileNameFromUrl.ts

@@ -1,7 +1,9 @@
+import { NavigationEntry } from '@mintlify/models';
+
 import { scrapeFileGettingFileNameFromUrl } from './scrapeFileGettingFileNameFromUrl.js';
 
 export async function scrapeGettingFileNameFromUrl(
-  navEntry: MintNavigationEntry,
+  navEntry: NavigationEntry,
   cliDir: string,
   origin: string,
   overwrite: boolean,
@@ -20,9 +22,9 @@ export async function scrapeGettingFileNameFromUrl(
   puppeteer = false,
   version: string | undefined,
   baseToRemove?: string
-): Promise<MintNavigationEntry> {
+): Promise<NavigationEntry> {
   if (typeof navEntry !== 'string') {
-    const newPages: MintNavigationEntry[] = [];
+    const newPages: NavigationEntry[] = [];
     for (const nestedNavEntry of navEntry.pages) {
       newPages.push(
         await scrapeGettingFileNameFromUrl(

package/src/scraping/scrapePage.ts

@@ -2,6 +2,21 @@ import path from 'path';
 
 import { createPage, getOrigin } from '../util.js';
 
+type ScrapePageResult = {
+  title: string;
+  description?: string;
+  markdown?: string;
+};
+
+export type ScrapePageFn = (
+  html: string,
+  origin: string,
+  cliDir: string,
+  imageBaseDir: string,
+  overwrite: boolean,
+  version: string | undefined
+) => Promise<ScrapePageResult>;
+
 export async function scrapePage(
   scrapeFunc: ScrapePageFn,
   href: string,

package/src/scraping/scrapePageCommands.ts

@@ -4,7 +4,7 @@ import { ArgumentsCamelCase } from 'yargs';
 import { getHtmlWithPuppeteer } from '../browser.js';
 import { getHrefFromArgs } from '../util.js';
 import { detectFramework, Frameworks } from './detectFramework.js';
-import { scrapePage } from './scrapePage.js';
+import { scrapePage, ScrapePageFn } from './scrapePage.js';
 import { scrapeIntercomPage } from './site-scrapers/Intercom/scrapeIntercomPage.js';
 import { scrapeDocusaurusPage } from './site-scrapers/scrapeDocusaurusPage.js';
 import { scrapeGitBookPage } from './site-scrapers/scrapeGitBookPage.js';

package/src/scraping/scrapeSection.ts

@@ -1,7 +1,17 @@
+import { NavigationEntry } from '@mintlify/models';
 import path from 'path';
 
 import { objToReadableString } from '../util.js';
 
+export type ScrapeSectionFn = (
+  html: string,
+  origin: string,
+  cliDir: string,
+  imageBaseDir: string,
+  overwrite: boolean,
+  version: string | undefined
+) => Promise<NavigationEntry[]>;
+
 export async function scrapeSection(
   scrapeFunc: ScrapeSectionFn,
   html: string,

package/src/scraping/scrapeSectionCommands.ts

@@ -4,7 +4,7 @@ import { ArgumentsCamelCase } from 'yargs';
 import { startBrowser } from '../browser.js';
 import { getHrefFromArgs, getOrigin } from '../util.js';
 import { detectFramework, Frameworks } from './detectFramework.js';
-import { scrapeSection } from './scrapeSection.js';
+import { ScrapeSectionFn, scrapeSection } from './scrapeSection.js';
 import { scrapeIntercomSection } from './site-scrapers/Intercom/scrapeIntercomSection.js';
 import openNestedDocusaurusMenus from './site-scrapers/openNestedDocusaurusMenus.js';
 import openNestedGitbookMenus from './site-scrapers/openNestedGitbookMenus.js';

package/src/scraping/site-scrapers/Intercom/scrapeIntercomSection.ts

@@ -1,3 +1,4 @@
+import { Navigation, NavigationEntry } from '@mintlify/models';
 import axios from 'axios';
 import cheerio from 'cheerio';
 
@@ -12,7 +13,7 @@ export async function scrapeIntercomSection(
   imageBaseDir: string,
   overwrite: boolean,
   version: string | undefined
-): Promise<MintNavigationEntry[]> {
+): Promise<NavigationEntry[]> {
   let $ = cheerio.load(html);
 
   const logoSrc = $('.header__logo img').first().attr('src');
@@ -35,10 +36,10 @@ export async function scrapeIntercomSection(
     };
   });
 
-  const collections: MintNavigation[] = await Promise.all(collectionsMap);
+  const collections: Navigation = await Promise.all(collectionsMap);
 
   return await Promise.all(
-    collections.map(async (entry: MintNavigationEntry) => {
+    collections.map(async (entry: NavigationEntry) => {
       return await scrapeGettingFileNameFromUrl(
         entry,
         cliDir,

package/src/scraping/site-scrapers/scrapeDocusaurusSection.ts

@@ -1,3 +1,4 @@
+import { Navigation, NavigationEntry } from '@mintlify/models';
 import cheerio from 'cheerio';
 
 import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
@@ -13,7 +14,7 @@ export async function scrapeDocusaurusSection(
   imageBaseDir: string,
   overwrite: boolean,
   version?: string
-): Promise<MintNavigation[]> {
+): Promise<Navigation> {
   const $ = cheerio.load(html);
 
   // Download the logo
@@ -24,7 +25,7 @@ export async function scrapeDocusaurusSection(
   const navigationSections = $('.theme-doc-sidebar-menu').first().children();
 
   // Get all links per group
-  const groupsConfig: MintNavigation[] = getDocusaurusLinksPerGroup(navigationSections, $, version);
+  const groupsConfig: Navigation = getDocusaurusLinksPerGroup(navigationSections, $, version);
 
   // Merge groups with empty titles together
   const reducedGroupsConfig = combineNavWithEmptyGroupTitles(groupsConfig);
@@ -34,7 +35,7 @@ export async function scrapeDocusaurusSection(
     reducedGroupsConfig.map(async (groupConfig) => {
       groupConfig.pages = (
         await Promise.all(
-          groupConfig.pages.map(async (navEntry: MintNavigationEntry) =>
+          groupConfig.pages.map(async (navEntry: NavigationEntry) =>
             // Docusaurus requires a directory on all sections wheras we use root.
             // /docs is their default directory so we remove it
             scrapeGettingFileNameFromUrl(

package/src/scraping/site-scrapers/scrapeGitBookSection.ts

@@ -1,3 +1,4 @@
+import { Navigation, NavigationEntry } from '@mintlify/models';
 import cheerio from 'cheerio';
 
 import combineNavWithEmptyGroupTitles from '../combineNavWithEmptyGroupTitles.js';
@@ -14,7 +15,7 @@ export async function scrapeGitBookSection(
   imageBaseDir: string,
   overwrite: boolean,
   version: string | undefined
-): Promise<MintNavigationEntry[]> {
+): Promise<NavigationEntry[]> {
   const $ = cheerio.load(html);
 
   // Download the logo
@@ -34,7 +35,7 @@ export async function scrapeGitBookSection(
     .children();
 
   // Get all links per group
-  const groupsConfig: MintNavigation[] = navigationSections
+  const groupsConfig: Navigation = navigationSections
     .toArray()
     .map((s: cheerio.Element) => {
       const section = $(s);
@@ -59,7 +60,7 @@ export async function scrapeGitBookSection(
 
   // Scrape each link in the navigation.
   return Promise.all(
-    reducedGroupsConfig.map(async (navEntry: MintNavigationEntry) => {
+    reducedGroupsConfig.map(async (navEntry: NavigationEntry) => {
       return await scrapeGettingFileNameFromUrl(
         navEntry,
         cliDir,

package/src/scraping/site-scrapers/scrapeReadMeSection.ts

@@ -1,3 +1,4 @@
+import { Navigation, NavigationEntry } from '@mintlify/models';
 import cheerio from 'cheerio';
 
 import downloadLogoImage from '../downloadLogoImage.js';
@@ -12,7 +13,7 @@ export async function scrapeReadMeSection(
   imageBaseDir: string,
   overwrite: boolean,
   version: string | undefined
-): Promise<MintNavigationEntry[]> {
+): Promise<NavigationEntry[]> {
   const $ = cheerio.load(html);
 
   // Download the logo
@@ -24,7 +25,7 @@ export async function scrapeReadMeSection(
   // responsiveness but they all have the same links.
   const navigationSections = $('.rm-Sidebar').first().find('.rm-Sidebar-section');
 
-  const groupsConfig: MintNavigation[] = navigationSections.toArray().map((s: cheerio.Element) => {
+  const groupsConfig: Navigation = navigationSections.toArray().map((s: cheerio.Element) => {
     const section = $(s);
     const sectionTitle = section.find('h3').first().text();
 
@@ -47,7 +48,7 @@ export async function scrapeReadMeSection(
 
   // Scrape each link in the navigation.
   return Promise.all(
-    groupsConfig.map(async (navEntry: MintNavigationEntry) => {
+    groupsConfig.map(async (navEntry: NavigationEntry) => {
       return await scrapeGettingFileNameFromUrl(
         // ReadMe requires a directory on all sections whereas we use root.
         // /docs is their default directory so we remove it

package/src/util.ts

@@ -1,3 +1,4 @@
+import { NavigationEntry } from '@mintlify/models';
 import { mkdirSync, writeFileSync } from 'fs';
 import Ora, { Ora as OraType } from 'ora';
 import path from 'path';
@@ -58,7 +59,7 @@ export function getOrigin(url: string) {
   return new URL(url).origin;
 }
 
-export function objToReadableString(objs: MintNavigationEntry[]) {
+export function objToReadableString(objs: NavigationEntry[]) {
   // Two spaces as indentation
   return objs.map((obj) => JSON.stringify(obj, null, 2)).join(',\n');
 }

package/src/types.d.ts

@@ -1,65 +0,0 @@
-type MintPage = {
-  title: string;
-  description?: string;
-  markdown?: string;
-};
-
-type MintNavigation = {
-  group?: string;
-  version?: string;
-  pages: MintNavigationEntry[];
-};
-type MintNavigationEntry = string | MintNavigation;
-
-type DecoratedMintNavigation = {
-  group: string;
-  version?: string;
-  pages: DecoratedMintNavigationEntry[];
-};
-
-type DecoratedMintNavigationEntry = DecoratedMintNavigation | PageMetadata;
-
-type PageMetadata = Partial<Record<PageMetadataKeys, string>>;
-
-const pageMetadataKeys = [
-  'title',
-  'description',
-  'sidebarTitle',
-  'href',
-  'api',
-  'openapi',
-  'contentType',
-  'auth',
-  'version',
-  'mode',
-  'hideFooterPagination',
-  'authors',
-  'lastUpdatedDate',
-  'createdDate',
-  'size',
-] as const;
-
-type PageMetadataKeys = (typeof pageMetadataKeys)[number];
-
-type ScrapePageFn = (
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined
-) => Promise<MintPage>;
-
-type ScrapeSectionFn = (
-  html: string,
-  origin: string,
-  cliDir: string,
-  imageBaseDir: string,
-  overwrite: boolean,
-  version: string | undefined
-) => Promise<MintNavigationEntry[]>;
-
-type OpenApiFile = {
-  filename: string;
-  spec: any;
-};