@mintlify/scraping 4.0.231 → 4.0.233

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/scraping",
-  "version": "4.0.231",
+  "version": "4.0.233",
   "description": "Scrape documentation frameworks to Mintlify docs",
   "engines": {
     "node": ">=18.0.0"
@@ -38,7 +38,7 @@
     "format:check": "prettier . --check"
   },
   "dependencies": {
-    "@mintlify/common": "1.0.375",
+    "@mintlify/common": "1.0.377",
     "@mintlify/openapi-parser": "^0.0.7",
     "fs-extra": "^11.1.1",
     "hast-util-to-mdast": "^10.1.0",
@@ -58,10 +58,10 @@
   },
   "devDependencies": {
     "@mintlify/eslint-config-typescript": "1.0.13",
-    "@mintlify/models": "0.0.189",
+    "@mintlify/models": "0.0.190",
     "@mintlify/prettier-config": "1.0.4",
     "@mintlify/ts-config": "2.0.2",
-    "@mintlify/validation": "0.1.362",
+    "@mintlify/validation": "0.1.364",
     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
     "@tsconfig/recommended": "1.x",
     "@types/hast": "^3.0.4",
@@ -77,5 +77,5 @@
     "typescript": "^5.5.3",
     "vitest": "^2.0.4"
   },
-  "gitHead": "ac5006c64de5ace7b7516691d1366462056e6ea3"
+  "gitHead": "0ccb32b16c5a402d2557209b60cfd218f93bc5c5"
 }

package/src/asyncapi/generateAsyncApiPagesForDocsConfig.ts

@@ -13,6 +13,7 @@ type GenerateAsyncApiPagesOptions = {
   outDir?: string;
   outDirBasePath?: string;
   overwrite?: boolean;
+  localSchema?: boolean;
 };
 
 type AsyncApiPageGenerationResult = {
@@ -30,8 +31,10 @@ export async function generateAsyncApiPagesForDocsConfig(
   let document: AsyncAPIDocumentInterface | undefined = undefined;
   let isUrl: boolean;
   if (typeof spec === 'string' || spec instanceof URL) {
-    const { document: asyncApiDocument, isUrl: isUrlFromDefinition } =
-      await getAsyncApiDefinition(spec);
+    const { document: asyncApiDocument, isUrl: isUrlFromDefinition } = await getAsyncApiDefinition(
+      spec,
+      opts?.localSchema
+    );
     if (asyncApiDocument) {
       document = asyncApiDocument;
     }

package/src/asyncapi/getAsyncApiDefinition.ts

@@ -1,17 +1,24 @@
-import { AsyncAPIDocumentInterface, validateAsyncApi } from '@mintlify/common';
+import {
+  AsyncAPIDocumentInterface,
+  validateAsyncApi,
+  isAllowedLocalSchemaUrl,
+} from '@mintlify/common';
 import * as fs from 'fs/promises';
 import * as path from 'path';
 
 import { fetchAsyncApi } from '../utils/network.js';
 
 export const getAsyncApiDefinition = async (
-  pathOrDocumentOrUrl: string | URL
+  pathOrDocumentOrUrl: string | URL,
+  localSchema?: boolean
 ): Promise<{ document: AsyncAPIDocumentInterface | undefined; isUrl: boolean }> => {
   let document: AsyncAPIDocumentInterface | undefined = undefined;
   if (typeof pathOrDocumentOrUrl === 'string') {
-    if (pathOrDocumentOrUrl.startsWith('http://')) {
+    if (pathOrDocumentOrUrl.startsWith('http:') && !localSchema) {
       // This is an invalid location either for a file or a URL
-      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
+      throw new Error(
+        'Only HTTPS URLs are supported. HTTP URLs are only supported with the cli option --local-schema.'
+      );
     } else {
       try {
         const url = new URL(pathOrDocumentOrUrl);
@@ -35,8 +42,10 @@ export const getAsyncApiDefinition = async (
   }
   const isUrl = pathOrDocumentOrUrl instanceof URL;
   if (pathOrDocumentOrUrl instanceof URL) {
-    if (pathOrDocumentOrUrl.protocol !== 'https:') {
-      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
+    if (!isAllowedLocalSchemaUrl(pathOrDocumentOrUrl.toString(), localSchema)) {
+      throw new Error(
+        'Only HTTPS URLs are supported. HTTP URLs are only supported with the cli option --local-schema.'
+      );
     }
     document = await fetchAsyncApi(pathOrDocumentOrUrl);
   }

package/src/asyncapi/processAsyncApiChannel.ts

@@ -49,9 +49,7 @@ export const processAsyncApiChannel = ({
   findNavGroup,
 }: ProcessAsyncApiChannelArgs) => {
   const asyncApiFilePathFromRoot = opts?.asyncApiFilePath
-    ? opts.asyncApiFilePath.startsWith('https://')
-      ? opts.asyncApiFilePath
-      : optionallyAddLeadingSlash(opts.asyncApiFilePath)
+    ? optionallyAddLeadingSlash(opts.asyncApiFilePath)
     : undefined;
 
   const tags = channel.tags().all();

package/src/openapi/common.ts

@@ -3,6 +3,7 @@ import {
   OperationObject,
   optionallyAddLeadingSlash,
   slugToTitle,
+  isAllowedLocalSchemaUrl,
   buildOpenApiMetaTag,
 } from '@mintlify/common';
 import type { DecoratedNavigationPage } from '@mintlify/models';
@@ -20,12 +21,15 @@ import {
 import { fetchOpenApi } from '../utils/network.js';
 
 export const getOpenApiDefinition = async (
-  pathOrDocumentOrUrl: string | OpenAPI.Document | URL
+  pathOrDocumentOrUrl: string | OpenAPI.Document | URL,
+  localSchema?: boolean
 ): Promise<{ document: OpenAPI.Document; isUrl: boolean }> => {
   if (typeof pathOrDocumentOrUrl === 'string') {
-    if (pathOrDocumentOrUrl.startsWith('http://')) {
+    if (pathOrDocumentOrUrl.startsWith('http:') && !localSchema) {
       // This is an invalid location either for a file or a URL
-      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
+      throw new Error(
+        'Only HTTPS URLs are supported. HTTP URLs are only supported with the cli option --local-schema.'
+      );
     } else {
       try {
         const url = new URL(pathOrDocumentOrUrl);
@@ -39,8 +43,10 @@ export const getOpenApiDefinition = async (
   }
   const isUrl = pathOrDocumentOrUrl instanceof URL;
   if (pathOrDocumentOrUrl instanceof URL) {
-    if (pathOrDocumentOrUrl.protocol !== 'https:') {
-      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
+    if (!isAllowedLocalSchemaUrl(pathOrDocumentOrUrl.toString(), localSchema)) {
+      throw new Error(
+        'Only HTTPS URLs are supported. HTTP URLs are only supported with the cli option --local-schema.'
+      );
     }
     pathOrDocumentOrUrl = await fetchOpenApi(pathOrDocumentOrUrl);
   }
@@ -75,6 +81,7 @@ export type GenerateOpenApiPagesOptions = {
   outDir?: string;
   outDirBasePath?: string;
   overwrite?: boolean;
+  localSchema?: boolean;
 };
 
 export type OpenApiPageGenerationResult<N, DN> = {

package/src/openapi/generateOpenApiPages.ts

@@ -21,7 +21,7 @@ export async function generateOpenApiPages(
   pathOrDocumentOrUrl: string | OpenAPI.Document | URL,
   opts?: GenerateOpenApiPagesOptions
 ): Promise<OpenApiPageGenerationResult<Navigation, DecoratedNavigation>> {
-  const { document, isUrl } = await getOpenApiDefinition(pathOrDocumentOrUrl);
+  const { document, isUrl } = await getOpenApiDefinition(pathOrDocumentOrUrl, opts?.localSchema);
   const { schema } = await validate(document);
 
   if (

package/src/openapi/generateOpenApiPagesForDocsConfig.ts

@@ -16,7 +16,7 @@ export async function generateOpenApiPagesForDocsConfig(
   pathOrDocumentOrUrl: string | OpenAPI.Document | URL,
   opts?: GenerateOpenApiPagesOptions
 ): Promise<OpenApiPageGenerationResult<GroupsConfig, DecoratedGroupsConfig>> {
-  const { document, isUrl } = await getOpenApiDefinition(pathOrDocumentOrUrl);
+  const { document, isUrl } = await getOpenApiDefinition(pathOrDocumentOrUrl, opts?.localSchema);
   const { schema } = await validate(document);
 
   if (