@mintlify/prebuild 1.0.1129 → 1.0.1131

package/dist/prebuild/categorizeFilePaths.d.ts

@@ -1,7 +1,11 @@
 import { AsyncAPIFile } from '@mintlify/common';
 import { OpenApiFile } from '@mintlify/models';
 import type { InvalidSpecFile } from './invalidSpecFiles.js';
-export declare const categorizeFilePaths: (contentDirectoryPath: string, mintIgnore?: string[], disableOpenApi?: boolean) => Promise<{
+export type PageScan = {
+    frontmatter: string;
+    rssCandidate: boolean;
+};
+export declare const categorizeFilePaths: (contentDirectoryPath: string, mintIgnore?: string[], disableOpenApi?: boolean, lazyPages?: boolean) => Promise<{
     contentFilenames: string[];
     staticFilenames: string[];
     openApiFiles: OpenApiFile[];
@@ -10,4 +14,5 @@ export declare const categorizeFilePaths: (contentDirectoryPath: string, mintIgn
     snippets: string[];
     snippetsV2: string[];
     fileImportsMap: Map<string, Set<string>>;
+    pageScans: Map<string, PageScan>;
 }>;

package/dist/prebuild/categorizeFilePaths.js

@@ -1,46 +1,135 @@
-import { validate, validateAsyncApi, buildImportMap, getFileCategory, isSnippetExtension, getAST, } from '@mintlify/common';
+import { validate, validateAsyncApi, extractImportSources, getFileCategory, isSnippetExtension, getAST, resolveImportPath, } from '@mintlify/common';
 import { readFile } from 'fs/promises';
 import yaml from 'js-yaml';
 import * as path from 'path';
 import { formatError } from '../errorMessages/formatError.js';
 import { getFileList } from '../fs/index.js';
 import { getFileExtension } from '../utils.js';
-export const categorizeFilePaths = async (contentDirectoryPath, mintIgnore = [], disableOpenApi) => {
-    const allFiles = getFileList(contentDirectoryPath, contentDirectoryPath, mintIgnore);
-    const mdxFiles = [];
+const MDX_IMPORT_PATTERN = /^import\s/m;
+const IMPORT_SOURCE_PATTERN = /import\s+(?:[\w$*{},\s]+?\s+from\s+)?['"]([^'"]+)['"]/g;
+const FRONTMATTER_PATTERN = /^\uFEFF?---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/;
+const MDX_SCAN_CONCURRENCY = 16;
+const FENCE_PATTERN = /^(`{3,}|~{3,})/;
+const extractImportSourcesFromContent = (content) => {
+    const sources = [];
+    let needsAst = false;
+    let fence;
+    for (const line of content.split('\n')) {
+        const trimmed = line.trimStart();
+        const fenceRun = FENCE_PATTERN.exec(trimmed)?.[1];
+        if (fence !== undefined) {
+            if (fenceRun !== undefined &&
+                fenceRun.charAt(0) === fence.char &&
+                fenceRun.length >= fence.length) {
+                fence = undefined;
+            }
+            continue;
+        }
+        if (fenceRun !== undefined) {
+            fence = { char: fenceRun.charAt(0), length: fenceRun.length };
+            continue;
+        }
+        if (!line.startsWith('import ') && !line.startsWith('import\t')) {
+            continue;
+        }
+        const matches = Array.from(line.matchAll(IMPORT_SOURCE_PATTERN));
+        if (matches.length === 0) {
+            needsAst = true;
+            continue;
+        }
+        for (const match of matches) {
+            const source = match[1];
+            if (source) {
+                sources.push(source);
+            }
+        }
+    }
+    return { sources, needsAst };
+};
+const getImportSources = (content, filePath, lazyPages) => {
+    if (lazyPages) {
+        if (!MDX_IMPORT_PATTERN.test(content)) {
+            return [];
+        }
+        const { sources, needsAst } = extractImportSourcesFromContent(content);
+        if (!needsAst) {
+            return sources;
+        }
+    }
+    return extractImportSources(getAST(content, filePath));
+};
+export const categorizeFilePaths = async (contentDirectoryPath, mintIgnore = [], disableOpenApi, lazyPages) => {
+    const allFilenames = [];
+    for await (const filename of getFileList(contentDirectoryPath, contentDirectoryPath, mintIgnore)) {
+        allFilenames.push(filename);
+    }
+    const mdxCandidates = [];
     const nonMdxFiles = [];
-    for await (const filename of allFiles) {
+    for (const filename of allFilenames) {
         const extension = getFileExtension(filename);
         if (isSnippetExtension(extension)) {
+            mdxCandidates.push(filename);
+        }
+        else {
+            nonMdxFiles.push({ filename, extension });
+        }
+    }
+    const importedFiles = new Set();
+    const fileImportsMap = new Map();
+    const pageScans = new Map();
+    const mdxResults = new Array(mdxCandidates.length);
+    const queue = mdxCandidates.map((filename, index) => ({ filename, index }));
+    await Promise.all(Array.from({ length: Math.min(MDX_SCAN_CONCURRENCY, queue.length) }, async () => {
+        let entry = queue.shift();
+        while (entry) {
+            const { filename, index } = entry;
             const filePath = path.join(contentDirectoryPath, filename);
-            const content = await readFile(filePath, 'utf8');
             try {
-                const tree = getAST(content, filePath);
-                mdxFiles.push({ path: filename, tree });
+                const content = await readFile(filePath, 'utf8');
+                const resolvedImports = new Set();
+                for (const source of getImportSources(content, filePath, lazyPages)) {
+                    const resolved = resolveImportPath(source, filename);
+                    if (resolved) {
+                        const normalized = resolved.toLowerCase();
+                        importedFiles.add(normalized);
+                        resolvedImports.add(normalized);
+                    }
+                }
+                fileImportsMap.set(filename.toLowerCase(), resolvedImports);
+                if (lazyPages) {
+                    pageScans.set(filename, {
+                        frontmatter: FRONTMATTER_PATTERN.exec(content)?.[0] ?? '',
+                        rssCandidate: content.includes('<Update'),
+                    });
+                }
+                mdxResults[index] = filename;
             }
             catch (error) {
                 console.error(formatError(error, filePath, contentDirectoryPath));
             }
+            entry = queue.shift();
         }
-        else {
-            nonMdxFiles.push({ filename, extension });
+    }));
+    const mdxFilenames = [];
+    for (const filename of mdxResults) {
+        if (filename !== undefined) {
+            mdxFilenames.push(filename);
         }
     }
-    const { importedFiles, fileImportsMap } = buildImportMap(mdxFiles);
     const contentFilenames = [];
     const snippets = [];
     const snippetsV2 = [];
-    for (const file of mdxFiles) {
-        const category = getFileCategory(file.path, { importedFiles });
+    for (const filename of mdxFilenames) {
+        const category = getFileCategory(filename, { importedFiles });
         switch (category) {
             case 'snippet':
-                snippets.push(file.path);
+                snippets.push(filename);
                 break;
             case 'snippet-v2':
-                snippetsV2.push(file.path);
+                snippetsV2.push(filename);
                 break;
             case 'page':
-                contentFilenames.push(file.path);
+                contentFilenames.push(filename);
                 break;
         }
     }
@@ -124,5 +213,6 @@ export const categorizeFilePaths = async (contentDirectoryPath, mintIgnore = [],
         snippets,
         snippetsV2,
         fileImportsMap,
+        pageScans,
     };
 };

package/dist/prebuild/index.d.ts

@@ -1,12 +1,13 @@
 export interface PrebuildResult {
     fileImportsMap: Map<string, Set<string>>;
 }
-export declare const prebuild: (contentDirectoryPath: string, { localSchema, groups, disableOpenApi, strict, allowSourceRefs, }?: {
+export declare const prebuild: (contentDirectoryPath: string, { localSchema, groups, disableOpenApi, strict, allowSourceRefs, lazyPages, }?: {
     localSchema?: boolean;
     groups?: string[];
     disableOpenApi?: boolean;
     strict?: boolean;
     allowSourceRefs?: boolean;
+    lazyPages?: boolean;
 }) => Promise<PrebuildResult | undefined>;
 export * from './categorizeFilePaths.js';
 export * from './getOpenApiFiles.js';

package/dist/prebuild/index.js

@@ -3,7 +3,7 @@ import { categorizeFilePaths } from './categorizeFilePaths.js';
 import { warnInvalidSpecFiles } from './invalidSpecFiles.js';
 import { update } from './update/index.js';
 import { clearWarnings, checkStrictMode } from './warnings.js';
-export const prebuild = async (contentDirectoryPath, { localSchema, groups, disableOpenApi, strict, allowSourceRefs, } = {}) => {
+export const prebuild = async (contentDirectoryPath, { localSchema, groups, disableOpenApi, strict, allowSourceRefs, lazyPages, } = {}) => {
     if (process.env.IS_MULTI_TENANT === 'true') {
         console.log('Skipping prebuild in multi-tenant mode.');
         return;
@@ -16,7 +16,7 @@ export const prebuild = async (contentDirectoryPath, { localSchema, groups, disa
         throw Error('must be run in a directory where a docs.json file exists.');
     }
     const mintIgnore = await getMintIgnore(contentDirectoryPath);
-    const { contentFilenames, staticFilenames, openApiFiles, asyncApiFiles, invalidSpecFiles, snippets, snippetsV2, fileImportsMap, } = await categorizeFilePaths(contentDirectoryPath, mintIgnore, disableOpenApi);
+    const { contentFilenames, staticFilenames, openApiFiles, asyncApiFiles, invalidSpecFiles, snippets, snippetsV2, fileImportsMap, pageScans, } = await categorizeFilePaths(contentDirectoryPath, mintIgnore, disableOpenApi, lazyPages);
     await update({
         contentDirectoryPath,
         staticFilenames,
@@ -33,6 +33,9 @@ export const prebuild = async (contentDirectoryPath, { localSchema, groups, disa
         strict,
         invalidSpecFiles,
         allowSourceRefs,
+        lazyPages,
+        pageScans,
+        fileImportsMap,
     });
     // Deferred until after update so files that fail the build don't also warn
     warnInvalidSpecFiles(invalidSpecFiles);

package/dist/prebuild/update/index.d.ts

@@ -1,6 +1,7 @@
 import { AsyncAPIFile } from '@mintlify/common';
 import type { OpenApiFile } from '@mintlify/models';
 import type { Root } from 'mdast';
+import type { PageScan } from '../categorizeFilePaths.js';
 import type { InvalidSpecFile } from '../invalidSpecFiles.js';
 type UpdateArgs = {
     contentDirectoryPath: string;
@@ -18,8 +19,11 @@ type UpdateArgs = {
     strict?: boolean;
     invalidSpecFiles?: InvalidSpecFile[];
     allowSourceRefs?: boolean;
+    lazyPages?: boolean;
+    pageScans?: Map<string, PageScan>;
+    fileImportsMap?: Map<string, Set<string>>;
 };
-export declare const update: ({ contentDirectoryPath, staticFilenames, openApiFiles, asyncApiFiles, contentFilenames, snippets, snippetV2Filenames, docsConfigPath, localSchema, groups, mintIgnore, disableOpenApi, strict, invalidSpecFiles, allowSourceRefs, }: UpdateArgs) => Promise<{
+export declare const update: ({ contentDirectoryPath, staticFilenames, openApiFiles, asyncApiFiles, contentFilenames, snippets, snippetV2Filenames, docsConfigPath, localSchema, groups, mintIgnore, disableOpenApi, strict, invalidSpecFiles, allowSourceRefs, lazyPages, pageScans, fileImportsMap, }: UpdateArgs) => Promise<{
     name: string;
     $schema: string;
     theme: "mint";

package/dist/prebuild/update/index.js

@@ -4,6 +4,7 @@ import { outputFile } from 'fs-extra';
 import { join } from 'path';
 import { updateDocsConfigFile } from './docsConfig/index.js';
 import { updateMintConfigFile } from './mintConfig/index.js';
+import { scanPageMetadata } from './read/read-page-metadata.js';
 import { readPageContents, readSnippetsV2Contents } from './read/readContent.js';
 import { resolveImportsAndWriteFiles } from './resolveImportsAndWriteFiles.js';
 import { updateFavicons } from './updateFavicons.js';
@@ -12,7 +13,52 @@ import { writeAsyncApiFiles } from './write/writeAsyncApiFiles.js';
 import { writeFiles, writeFile } from './write/writeFiles.js';
 import { writeOpenApiData } from './write/writeOpenApiData.js';
 import { writeRssFiles } from './write/writeRssFiles.js';
-export const update = async ({ contentDirectoryPath, staticFilenames, openApiFiles, asyncApiFiles, contentFilenames, snippets, snippetV2Filenames, docsConfigPath, localSchema, groups, mintIgnore, disableOpenApi, strict, invalidSpecFiles, allowSourceRefs, }) => {
+const expandRssCandidates = (rssCandidateFilenames, contentFilenames, fileImportsMap, pageScans) => {
+    const carriers = new Set();
+    for (const [filename, scan] of pageScans) {
+        if (scan.rssCandidate) {
+            carriers.add(filename.toLowerCase());
+        }
+    }
+    let changed = carriers.size > 0;
+    while (changed) {
+        changed = false;
+        for (const [filename, imports] of fileImportsMap) {
+            if (carriers.has(filename))
+                continue;
+            for (const imported of imports) {
+                if (carriers.has(imported)) {
+                    carriers.add(filename);
+                    changed = true;
+                    break;
+                }
+            }
+        }
+    }
+    const candidates = new Set(rssCandidateFilenames);
+    for (const filename of contentFilenames) {
+        if (carriers.has(filename.toLowerCase())) {
+            candidates.add(filename);
+        }
+    }
+    return [...candidates];
+};
+const filterToImportClosure = (snippetV2Filenames, rootFilenames, fileImportsMap) => {
+    const needed = new Set();
+    const stack = rootFilenames.map((filename) => filename.toLowerCase());
+    let current = stack.pop();
+    while (current !== undefined) {
+        for (const imported of fileImportsMap.get(current) ?? []) {
+            if (!needed.has(imported)) {
+                needed.add(imported);
+                stack.push(imported);
+            }
+        }
+        current = stack.pop();
+    }
+    return snippetV2Filenames.filter((filename) => needed.has(filename.toLowerCase()));
+};
+export const update = async ({ contentDirectoryPath, staticFilenames, openApiFiles, asyncApiFiles, contentFilenames, snippets, snippetV2Filenames, docsConfigPath, localSchema, groups, mintIgnore, disableOpenApi, strict, invalidSpecFiles, allowSourceRefs, lazyPages, pageScans, fileImportsMap, }) => {
     const mintConfigResult = await updateMintConfigFile(contentDirectoryPath, openApiFiles, localSchema, strict, invalidSpecFiles);
     // we used the original mint config without openapi pages injected
     // because we will do it in `updateDocsConfigFile`, this will avoid duplicated openapi pages
@@ -28,14 +74,32 @@ export const update = async ({ contentDirectoryPath, staticFilenames, openApiFil
         invalidSpecFiles,
         allowSourceRefs,
     });
+    let contentFilenamesToParse = contentFilenames;
+    if (lazyPages) {
+        const { rssCandidateFilenames } = await scanPageMetadata({
+            contentDirectoryPath,
+            openApiFiles: newOpenApiFiles,
+            asyncApiFiles: newAsyncApiFiles,
+            contentFilenames,
+            pagesAcc,
+            pageScans,
+        });
+        contentFilenamesToParse =
+            fileImportsMap && pageScans
+                ? expandRssCandidates(rssCandidateFilenames, contentFilenames, fileImportsMap, pageScans)
+                : rssCandidateFilenames;
+    }
     const pagePromises = readPageContents({
         contentDirectoryPath,
         openApiFiles: newOpenApiFiles,
         asyncApiFiles: newAsyncApiFiles,
-        contentFilenames,
+        contentFilenames: contentFilenamesToParse,
         pagesAcc,
     });
-    const snippetV2Promises = readSnippetsV2Contents(contentDirectoryPath, snippetV2Filenames, newDocsConfig.variables ?? undefined);
+    const snippetV2FilenamesToParse = lazyPages && fileImportsMap
+        ? filterToImportClosure(snippetV2Filenames, contentFilenamesToParse, fileImportsMap)
+        : snippetV2Filenames;
+    const snippetV2Promises = readSnippetsV2Contents(contentDirectoryPath, snippetV2FilenamesToParse, newDocsConfig.variables ?? undefined);
     const [snippetV2Contents, { mdxFilesWithNoImports, filesWithImports }] = await Promise.all([
         snippetV2Promises,
         pagePromises,

package/dist/prebuild/update/read/read-page-metadata.d.ts

@@ -0,0 +1,15 @@
+import { AsyncAPIFile } from '@mintlify/common';
+import type { DecoratedNavigationPage, OpenApiFile } from '@mintlify/models';
+import type { PageScan } from '../../categorizeFilePaths.js';
+type ScanPageMetadataArgs = {
+    contentDirectoryPath: string;
+    openApiFiles: OpenApiFile[];
+    asyncApiFiles: AsyncAPIFile[];
+    contentFilenames: string[];
+    pagesAcc: Record<string, DecoratedNavigationPage>;
+    pageScans?: Map<string, PageScan>;
+};
+export declare const scanPageMetadata: ({ contentDirectoryPath, openApiFiles, asyncApiFiles, contentFilenames, pagesAcc, pageScans, }: ScanPageMetadataArgs) => Promise<{
+    rssCandidateFilenames: string[];
+}>;
+export {};

package/dist/prebuild/update/read/read-page-metadata.js

@@ -0,0 +1,46 @@
+import { getDecoratedNavPageAndSlug, parseFrontmatter } from '@mintlify/common';
+import { promises as _promises } from 'fs';
+import { join } from 'path';
+import { preserveAutoGeneratedMetadata } from '../preserveAutoGeneratedMetadata.js';
+const { readFile } = _promises;
+const SCAN_CONCURRENCY = 16;
+const hasRssFrontmatter = (contentStr) => {
+    try {
+        return parseFrontmatter(contentStr).attributes.rss === true;
+    }
+    catch {
+        return false;
+    }
+};
+export const scanPageMetadata = async ({ contentDirectoryPath, openApiFiles, asyncApiFiles, contentFilenames, pagesAcc, pageScans, }) => {
+    const rssCandidateFilenames = [];
+    const queue = [...contentFilenames];
+    const scanFile = async (filename) => {
+        const scan = pageScans?.get(filename);
+        const contentStr = scan?.frontmatter ?? (await readFile(join(contentDirectoryPath, filename), 'utf8'));
+        const { slug, pageMetadata } = getDecoratedNavPageAndSlug(filename, contentStr, openApiFiles, asyncApiFiles);
+        preserveAutoGeneratedMetadata(contentStr, slug, pageMetadata, pagesAcc);
+        const rssCandidate = scan
+            ? scan.rssCandidate || hasRssFrontmatter(contentStr)
+            : contentStr.includes('<Update') || hasRssFrontmatter(contentStr);
+        if (rssCandidate) {
+            rssCandidateFilenames.push(filename);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(SCAN_CONCURRENCY, queue.length) }, async () => {
+        let filename = queue.shift();
+        while (filename !== undefined) {
+            try {
+                await scanFile(filename);
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error
+                    ? error.message
+                    : 'Unknown error occurred reading page metadata.';
+                console.error(`${join(contentDirectoryPath, filename)}: ${errorMessage}`);
+            }
+            filename = queue.shift();
+        }
+    }));
+    return { rssCandidateFilenames };
+};

package/dist/prebuild/update/write/writeFiles.js

@@ -1,13 +1,31 @@
 import fse from 'fs-extra';
 import path from 'path';
+const isUnchangedFile = async (sourcePath, targetPath) => {
+    try {
+        const [sourceStat, targetStat] = await Promise.all([
+            fse.stat(sourcePath),
+            fse.stat(targetPath),
+        ]);
+        return (sourceStat.isFile() &&
+            targetStat.isFile() &&
+            sourceStat.size === targetStat.size &&
+            Math.abs(sourceStat.mtimeMs - targetStat.mtimeMs) < 2);
+    }
+    catch {
+        return false;
+    }
+};
 export const writeFiles = (contentDirectoryPath, targetDirectoryPath, filenames) => {
     const filePromises = [];
     filenames.forEach((filename) => {
         filePromises.push((async () => {
             const sourcePath = path.join(contentDirectoryPath, filename);
             const targetPath = path.join(targetDirectoryPath, filename);
+            if (await isUnchangedFile(sourcePath, targetPath)) {
+                return;
+            }
             await fse.remove(targetPath);
-            await fse.copy(sourcePath, targetPath);
+            await fse.copy(sourcePath, targetPath, { preserveTimestamps: true });
         })());
     });
     return filePromises;