@mintlify/prebuild 1.0.1156 → 1.0.1158

package/dist/prebuild/update/read/readContent.d.ts

@@ -6,6 +6,12 @@ export type FileWithImportsAndMetadata = FileWithImports & {
     metadata?: Record<string, unknown>;
     contentDirectoryPath?: string;
 };
+export type RssCandidatePage = {
+    targetPath: string;
+    sourcePath: string;
+    tree: Root;
+    metadata: DecoratedNavigationPage;
+};
 type ReadPageContentsArgs = {
     contentDirectoryPath: string;
     openApiFiles: OpenApiFile[];
@@ -14,12 +20,7 @@ type ReadPageContentsArgs = {
     pagesAcc: Record<string, DecoratedNavigationPage>;
 };
 export declare const readPageContents: ({ contentDirectoryPath, openApiFiles, asyncApiFiles, contentFilenames, pagesAcc, }: ReadPageContentsArgs) => Promise<{
-    mdxFilesWithNoImports: {
-        targetPath: string;
-        sourcePath: string;
-        tree: Root;
-        metadata: DecoratedNavigationPage;
-    }[];
+    mdxFilesWithNoImports: RssCandidatePage[];
     pagesAcc: Record<string, DecoratedNavigationPage>;
     filesWithImports: FileWithImportsAndMetadata[];
 }>;

package/dist/prebuild/update/read/readContent.js

@@ -1,14 +1,19 @@
-import { hasImports, findAndRemoveImports, getDecoratedNavPageAndSlug, parseFrontmatter, replaceVariables, } from '@mintlify/common';
+import { containsUpdates, hasImports, findAndRemoveImports, getDecoratedNavPageAndSlug, parseFrontmatter, replaceVariables, stringifyTree, } from '@mintlify/common';
 import { promises as _promises } from 'fs';
+import { outputFile } from 'fs-extra';
 import { join } from 'path';
 import { preparseMdxTree } from '../../../createPage/preparseMdx/index.js';
 import { preserveAutoGeneratedMetadata } from '../preserveAutoGeneratedMetadata.js';
 const { readFile } = _promises;
+const PAGE_PARSE_CONCURRENCY = 8;
 export const readPageContents = async ({ contentDirectoryPath, openApiFiles, asyncApiFiles, contentFilenames, pagesAcc, }) => {
     const filesWithImports = [];
-    const pagePromises = contentFilenames.map(async (filename) => {
+    const mdxFilesWithNoImports = [];
+    const remainingFilenames = [...contentFilenames];
+    const processFile = async (filename) => {
         const sourcePath = join(contentDirectoryPath, filename);
         const targetPath = join('src', '_props', filename);
+        let treeToWrite;
         try {
             const contentStr = (await readFile(sourcePath)).toString();
             const tree = await preparseMdxTree(contentStr, contentDirectoryPath, sourcePath);
@@ -29,15 +34,33 @@ export const readPageContents = async ({ contentDirectoryPath, openApiFiles, asy
             }
             const { slug, pageMetadata } = getDecoratedNavPageAndSlug(filename, contentStr, openApiFiles, asyncApiFiles);
             preserveAutoGeneratedMetadata(contentStr, slug, pageMetadata, pagesAcc);
-            return { targetPath, sourcePath, tree, metadata: pageMetadata };
+            // Only RSS candidates are needed by later phases (writeRssFiles). All
+            // other import-free pages are written and dropped immediately so peak
+            // memory does not grow with the number of pages.
+            if (pageMetadata.rss === true || containsUpdates(tree)) {
+                mdxFilesWithNoImports.push({ targetPath, sourcePath, tree, metadata: pageMetadata });
+                return;
+            }
+            treeToWrite = tree;
         }
         catch (error) {
             const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred reading and parsing file.';
             console.error(`${sourcePath}: ${errorMessage}`);
             throw new Error('Failed to parse MDX files, please correct errors before continuing');
         }
-    });
-    const mdxFilesWithNoImports = (await Promise.all(pagePromises)).filter((item) => item !== undefined);
+        await outputFile(targetPath, stringifyTree(treeToWrite), {
+            flag: 'w',
+        });
+    };
+    const processFilesUntilDone = async () => {
+        let filename = remainingFilenames.shift();
+        while (filename !== undefined) {
+            await processFile(filename);
+            filename = remainingFilenames.shift();
+        }
+    };
+    const workerCount = Math.min(PAGE_PARSE_CONCURRENCY, remainingFilenames.length);
+    await Promise.all(Array.from({ length: workerCount }, processFilesUntilDone));
     return { mdxFilesWithNoImports, pagesAcc, filesWithImports };
 };
 export const readSnippetsV2Contents = (contentDirectoryPath, snippetV2Filenames, variables) => {