@mintlify/previewing 4.0.1504 → 4.0.1506

package/dist/__tests__/export-assets.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/export-assets.test.js

@@ -0,0 +1,144 @@
+import { toMintExportAssetPath } from '@mintlify/common';
+import fse from 'fs-extra';
+import os from 'os';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { collectAssetPaths, localizeAssets, rewriteCssUrls, } from '../local-preview/export-assets.js';
+import { fetchPublicAsset } from '../local-preview/export-fetch.js';
+import { generateExportLlmsTxt } from '../local-preview/export-llms.js';
+const ICON_URL = 'https://d3gk2c5xim1je2.cloudfront.net/fontawesome/v7.2.0/regular/house.svg';
+const GOOGLE_CSS_URL = 'https://fonts.googleapis.com/css2?family=Open+Sans:wght@400&display=swap';
+const asset = (body, contentType) => ({
+    contentType,
+    body: Buffer.from(body),
+});
+describe('export assets', () => {
+    let exportDir;
+    beforeEach(async () => {
+        exportDir = await fse.mkdtemp(path.join(os.tmpdir(), 'mint-export-assets-test-'));
+    });
+    afterEach(async () => {
+        await fse.remove(exportDir);
+    });
+    it('collects asset paths from html, including escaped style attributes', async () => {
+        const iconPath = toMintExportAssetPath(ICON_URL);
+        await fse.outputFile(path.join(exportDir, 'guides', 'index.html'), `<svg style="mask-image:url(&quot;${iconPath}&quot;)"></svg><script>self.__next_f.push([1,"${iconPath}"])</script>`);
+        await fse.outputFile(path.join(exportDir, 'app.js'), toMintExportAssetPath(GOOGLE_CSS_URL));
+        expect([...(await collectAssetPaths(exportDir))]).toEqual([iconPath]);
+    });
+    it('collects icons for every language in an exported dropdown code group', async () => {
+        const codeGroup = await fse.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), 'fixtures', 'code-group-dropdown.html'), 'utf8');
+        await fse.outputFile(path.join(exportDir, 'index.html'), codeGroup);
+        expect([...(await collectAssetPaths(exportDir))].sort()).toEqual([
+            '/_mintlify/export-assets/d3gk2c5xim1je2.cloudfront.net/devicon/java.svg',
+            '/_mintlify/export-assets/d3gk2c5xim1je2.cloudfront.net/devicon/ruby.svg',
+            '/_mintlify/export-assets/d3gk2c5xim1je2.cloudfront.net/devicon/rust.svg',
+        ]);
+    });
+    it('rewrites css urls to relative local paths', () => {
+        const cssAssetPath = toMintExportAssetPath(GOOGLE_CSS_URL, 'css');
+        const found = [];
+        const css = rewriteCssUrls("@font-face{src:url(https://fonts.gstatic.com/s/opensans/v40/a.woff2) format('woff2')}.x{background:url(data:image/png;base64,AA)}", GOOGLE_CSS_URL, cssAssetPath, (assetPath) => found.push(assetPath));
+        expect(found).toEqual(['/_mintlify/export-assets/fonts.gstatic.com/s/opensans/v40/a.woff2']);
+        expect(css).toBe("@font-face{src:url(../../../fonts.gstatic.com/s/opensans/v40/a.woff2) format('woff2')}.x{background:url(data:image/png;base64,AA)}");
+        expect(path.posix.resolve(path.posix.dirname(cssAssetPath), '../../../fonts.gstatic.com')).toBe('/_mintlify/export-assets/fonts.gstatic.com');
+    });
+    it('downloads referenced assets and follows css urls', async () => {
+        const iconPath = toMintExportAssetPath(ICON_URL);
+        const cssPath = toMintExportAssetPath(GOOGLE_CSS_URL, 'css');
+        const missingPath = toMintExportAssetPath('https://d3gk2c5xim1je2.cloudfront.net/missing.svg');
+        await fse.outputFile(path.join(exportDir, 'index.html'), `<link href="${cssPath}"><i style="mask:url(${iconPath})"></i><i style="mask:url(${missingPath})"></i>`);
+        const fetchAsset = vi.fn(async (url) => {
+            if (url === ICON_URL)
+                return asset('<svg/>', 'image/svg+xml');
+            if (url === GOOGLE_CSS_URL) {
+                return asset('@font-face{src:url(https://fonts.gstatic.com/s/a.woff2)}', 'text/css; charset=utf-8');
+            }
+            if (url === 'https://fonts.gstatic.com/s/a.woff2')
+                return asset(new Uint8Array([1, 2, 3]));
+            return undefined;
+        });
+        const result = await localizeAssets(exportDir, fetchAsset);
+        expect(result.failed).toEqual(['https://d3gk2c5xim1je2.cloudfront.net/missing.svg']);
+        await expect(fse.readFile(path.join(exportDir, iconPath), 'utf8')).resolves.toBe('<svg/>');
+        await expect(fse.readFile(path.join(exportDir, cssPath), 'utf8')).resolves.toBe('@font-face{src:url(../../../fonts.gstatic.com/s/a.woff2)}');
+        expect([
+            ...(await fse.readFile(path.join(exportDir, '_mintlify/export-assets/fonts.gstatic.com/s/a.woff2'))),
+        ]).toEqual([1, 2, 3]);
+    });
+});
+describe('localizeAssets path safety', () => {
+    let exportDir;
+    beforeEach(async () => {
+        exportDir = await fse.mkdtemp(path.join(os.tmpdir(), 'mint-export-safety-test-'));
+    });
+    afterEach(async () => {
+        await fse.remove(exportDir);
+    });
+    it('never writes outside the asset directory, even for encoded traversal', async () => {
+        const cssUrl = 'https://fonts.example.com/style.css';
+        await fse.outputFile(path.join(exportDir, 'serve.js'), 'original launcher');
+        await fse.outputFile(path.join(exportDir, 'index.html'), `<link href="${toMintExportAssetPath(cssUrl)}">`);
+        const fetchAsset = vi.fn(async (url) => url === cssUrl
+            ? asset('@font-face{src:url(https://evil.example/%2e%2e%2f%2e%2e%2f%2e%2e%2fserve.js)}', 'text/css')
+            : asset('malicious launcher'));
+        const result = await localizeAssets(exportDir, fetchAsset);
+        await expect(fse.readFile(path.join(exportDir, 'serve.js'), 'utf8')).resolves.toBe('original launcher');
+        expect(result.failed).toContain('/_mintlify/export-assets/evil.example/%2e%2e%2f%2e%2e%2f%2e%2e%2fserve.js');
+    });
+});
+describe('localizeAssets private destinations', () => {
+    let exportDir;
+    beforeEach(async () => {
+        exportDir = await fse.mkdtemp(path.join(os.tmpdir(), 'mint-export-private-test-'));
+    });
+    afterEach(async () => {
+        await fse.remove(exportDir);
+    });
+    it('does not download private addresses referenced by a remote stylesheet', async () => {
+        const cssUrl = 'https://fonts.example.com/style.css';
+        await fse.outputFile(path.join(exportDir, 'index.html'), `<link href="${toMintExportAssetPath(cssUrl)}">`);
+        const fetchAsset = vi.fn(async (url) => url === cssUrl
+            ? asset('.a{background:url(http://127.0.0.1:9000/secret.png)}', 'text/css')
+            : fetchPublicAsset(url));
+        const result = await localizeAssets(exportDir, fetchAsset);
+        expect(result.failed).toEqual(['http://127.0.0.1:9000/secret.png']);
+        await expect(fse.pathExists(path.join(exportDir, '_mintlify/export-assets/_http/127.0.0.1~9000/secret.png'))).resolves.toBe(false);
+    });
+});
+describe('localizeAssets stylesheets', () => {
+    let exportDir;
+    beforeEach(async () => {
+        exportDir = await fse.mkdtemp(path.join(os.tmpdir(), 'mint-export-css-test-'));
+    });
+    afterEach(async () => {
+        await fse.remove(exportDir);
+    });
+    it('bundles absolute font urls from the client css and leaves relative urls alone', async () => {
+        const cssFile = path.join(exportDir, '_next', 'static', 'chunks', 'app.css');
+        await fse.outputFile(cssFile, '@font-face{src:url(https://cdn.jsdelivr.net/gh/x/latex-css/fonts/LM-regular.woff2)}@font-face{src:url(//fonts.cdnfonts.com/s/1/cmuntt.woff)}@font-face{src:url(../media/inter.woff2)}');
+        const fetchAsset = vi.fn(async () => asset(new Uint8Array([7])));
+        const result = await localizeAssets(exportDir, fetchAsset);
+        expect(result.failed).toEqual([]);
+        await expect(fse.readFile(cssFile, 'utf8')).resolves.toBe('@font-face{src:url(../../../_mintlify/export-assets/cdn.jsdelivr.net/gh/x/latex-css/fonts/LM-regular.woff2)}@font-face{src:url(../../../_mintlify/export-assets/fonts.cdnfonts.com/s/1/cmuntt.woff)}@font-face{src:url(../media/inter.woff2)}');
+        await expect(fse.pathExists(path.join(exportDir, '_mintlify/export-assets/fonts.cdnfonts.com/s/1/cmuntt.woff'))).resolves.toBe(true);
+    });
+});
+describe('generateExportLlmsTxt', () => {
+    it('lists exported pages with titles and descriptions', () => {
+        const llmsTxt = generateExportLlmsTxt({ name: 'Conduit', description: 'Self-hosted docs' }, {
+            groups: [
+                {
+                    group: 'Getting started',
+                    pages: [
+                        { href: '/index', title: 'Introduction', description: 'Welcome' },
+                        { href: '/secret', title: 'Secret' },
+                        { group: 'Nested', pages: [{ href: '/guides/math', title: 'Math [LaTeX]' }] },
+                        { href: 'https://example.com', title: 'External' },
+                    ],
+                },
+            ],
+        }, ['/', '/index', '/guides/math']);
+        expect(llmsTxt).toBe('# Conduit\n\n> Self-hosted docs\n\n## Docs\n\n- [Introduction](/index): Welcome\n- [Math \\[LaTeX\\]](/guides/math)\n');
+    });
+});

package/dist/__tests__/export-fetch.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/export-fetch.test.js

@@ -0,0 +1,81 @@
+import { PrivateIPError } from '@mintlify/common/ssrf';
+import { createServer } from 'http';
+import { fetchPublicAsset } from '../local-preview/export-fetch.js';
+const loopbackLookup = (_hostname, options, callback) => options.all
+    ? callback(null, [{ address: '127.0.0.1', family: 4 }])
+    : callback(null, '127.0.0.1', 4);
+describe('fetchPublicAsset', () => {
+    let server;
+    let origin;
+    beforeAll(async () => {
+        server = createServer((req, res) => {
+            switch (req.url) {
+                case '/icon.svg':
+                    res.writeHead(200, { 'content-type': 'image/svg+xml' }).end('<svg/>');
+                    return;
+                case '/moved':
+                    res.writeHead(302, { location: '/icon.svg' }).end();
+                    return;
+                case '/to-literal':
+                    res.writeHead(302, { location: 'http://127.0.0.1/secret' }).end();
+                    return;
+                case '/to-metadata':
+                    res.writeHead(302, { location: 'http://169.254.169.254/latest' }).end();
+                    return;
+                case '/loop':
+                    res.writeHead(302, { location: '/loop' }).end();
+                    return;
+                case '/large':
+                    res.writeHead(200).end(Buffer.alloc(2048));
+                    return;
+                case '/hang':
+                    return;
+                default:
+                    res.writeHead(404).end('not found');
+            }
+        });
+        await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+        const address = server.address();
+        if (!address || typeof address === 'string')
+            throw new Error('server did not bind a port');
+        origin = `http://assets.test:${address.port}`;
+    });
+    afterAll(async () => {
+        server.closeAllConnections();
+        await new Promise((resolve) => server.close(resolve));
+    });
+    it('downloads the body and content type', async () => {
+        const asset = await fetchPublicAsset(`${origin}/icon.svg`, { lookup: loopbackLookup });
+        expect(asset?.contentType).toBe('image/svg+xml');
+        expect(asset?.body.toString()).toBe('<svg/>');
+    });
+    it('follows redirects', async () => {
+        const asset = await fetchPublicAsset(`${origin}/moved`, { lookup: loopbackLookup });
+        expect(asset?.body.toString()).toBe('<svg/>');
+    });
+    it('refuses redirects to private ip literals', async () => {
+        await expect(fetchPublicAsset(`${origin}/to-literal`, { lookup: loopbackLookup })).resolves.toBeUndefined();
+        await expect(fetchPublicAsset(`${origin}/to-metadata`, { lookup: loopbackLookup })).resolves.toBeUndefined();
+    });
+    it.each([
+        'http://127.0.0.1/a',
+        'http://[::1]/a',
+        'http://[::ffff:7f00:1]/a',
+        'file:///etc/passwd',
+    ])('refuses %s without connecting', async (url) => {
+        await expect(fetchPublicAsset(url)).resolves.toBeUndefined();
+    });
+    it('refuses hosts that resolve to private addresses', async () => {
+        await expect(fetchPublicAsset('http://localhost/a')).rejects.toBeInstanceOf(PrivateIPError);
+    });
+    it('returns undefined for errors and redirect loops', async () => {
+        await expect(fetchPublicAsset(`${origin}/missing`, { lookup: loopbackLookup })).resolves.toBeUndefined();
+        await expect(fetchPublicAsset(`${origin}/loop`, { lookup: loopbackLookup })).resolves.toBeUndefined();
+    });
+    it('caps the response size', async () => {
+        await expect(fetchPublicAsset(`${origin}/large`, { lookup: loopbackLookup, maxBytes: 1024 })).rejects.toThrow('asset exceeds 1024 bytes');
+    });
+    it('times out slow responses', async () => {
+        await expect(fetchPublicAsset(`${origin}/hang`, { lookup: loopbackLookup, timeoutMs: 100 })).rejects.toThrow();
+    });
+});

package/dist/__tests__/export.test.js

@@ -30,14 +30,32 @@ describe('generatePages', () => {
         vi.unstubAllGlobals();
         await fse.remove(outputDir);
     });
+    const pageFetch = () => vi.fn(async (_input, init) => new Headers(init?.headers).get('RSC') === '1'
+        ? new Response('0:{"b":"build"}\n', { headers: { 'content-type': 'text/x-component' } })
+        : new Response('<html>Introduction</html>', { headers: { 'content-type': 'text/html' } }));
     it('uses the first accessible route for the root page', async () => {
-        const fetchMock = vi.fn().mockResolvedValue(new Response('<html>Introduction</html>'));
+        const fetchMock = pageFetch();
         vi.stubGlobal('fetch', fetchMock);
         const failedPageCount = await generatePages('http://localhost:3000', ['/'], outputDir, '/introduction');
         expect(failedPageCount).toBe(0);
         expect(fetchMock).toHaveBeenCalledWith('http://localhost:3000/introduction');
         await expect(fse.readFile(path.join(outputDir, 'index.html'), 'utf8')).resolves.toBe('<html>Introduction</html>');
     });
+    it('saves the rsc payload next to each page for soft navigation', async () => {
+        const fetchMock = pageFetch();
+        vi.stubGlobal('fetch', fetchMock);
+        const failedPageCount = await generatePages('http://localhost:3000', ['/', '/guides/math'], outputDir);
+        expect(failedPageCount).toBe(0);
+        expect(fetchMock).toHaveBeenCalledWith('http://localhost:3000/guides/math', {
+            headers: { RSC: '1' },
+        });
+        await expect(fse.readFile(path.join(outputDir, 'index.txt'), 'utf8')).resolves.toBe('0:{"b":"build"}\n');
+        await expect(fse.readFile(path.join(outputDir, 'guides', 'math', 'index.txt'), 'utf8')).resolves.toBe('0:{"b":"build"}\n');
+    });
+    it('fails the page when the rsc request does not return a payload', async () => {
+        vi.stubGlobal('fetch', vi.fn(async () => new Response('<html></html>', { headers: { 'content-type': 'text/html' } })));
+        await expect(generatePages('http://localhost:3000', ['/quickstart'], outputDir)).resolves.toBe(1);
+    });
 });
 describe('writeStaticUserInfo', () => {
     let outputDir;

package/dist/local-preview/export-assets.d.ts

@@ -0,0 +1,8 @@
+import { type FetchAsset } from './export-fetch.js';
+export type LocalizeAssetsResult = {
+    downloaded: string[];
+    failed: string[];
+};
+export declare function collectAssetPaths(exportDir: string): Promise<Set<string>>;
+export declare function rewriteCssUrls(css: string, cssUrl: string | undefined, cssAssetPath: string, onAsset: (assetPath: string) => void): string;
+export declare function localizeAssets(exportDir: string, fetchAsset?: FetchAsset): Promise<LocalizeAssetsResult>;

package/dist/local-preview/export-assets.js

@@ -0,0 +1,135 @@
+import { MINT_EXPORT_ASSETS_PREFIX, fromMintExportAssetPath, getLanguageIconUrl, toMintExportAssetPath, } from '@mintlify/common';
+import fse from 'fs-extra';
+import path from 'path';
+import { fetchPublicAsset } from './export-fetch.js';
+const ASSET_PATH_PATTERN = new RegExp(`${MINT_EXPORT_ASSETS_PREFIX.replace(/\//g, '\\/')}\\/[A-Za-z0-9._~\\-/%+@]+`, 'g');
+const CSS_URL_PATTERN = /url\(\s*(['"]?)([^'")]+)\1\s*\)/g;
+const CODE_LANGUAGE_PATTERN = /\slanguage="([^"]+)"/g;
+const CONCURRENCY = 10;
+async function* walkFiles(dir) {
+    for (const entry of await fse.readdir(dir, { withFileTypes: true })) {
+        const entryPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            yield* walkFiles(entryPath);
+        }
+        else if (entry.isFile()) {
+            yield entryPath;
+        }
+    }
+}
+export async function collectAssetPaths(exportDir) {
+    const assetPaths = new Set();
+    for await (const file of walkFiles(exportDir)) {
+        if (!file.endsWith('.html'))
+            continue;
+        const html = await fse.readFile(file, 'utf8');
+        for (const match of html.matchAll(ASSET_PATH_PATTERN)) {
+            assetPaths.add(match[0]);
+        }
+        for (const [, language = ''] of html.matchAll(CODE_LANGUAGE_PATTERN)) {
+            const iconUrl = getLanguageIconUrl(language);
+            if (iconUrl)
+                assetPaths.add(toMintExportAssetPath(iconUrl));
+        }
+    }
+    return assetPaths;
+}
+const resolveOutputPath = (exportDir, assetPath) => {
+    let decoded;
+    try {
+        decoded = decodeURIComponent(assetPath);
+    }
+    catch {
+        return undefined;
+    }
+    const segments = decoded.split(/[\\/]/);
+    if (segments.some((segment) => segment === '..' || segment === '.'))
+        return undefined;
+    const assetsDir = path.resolve(exportDir, `.${MINT_EXPORT_ASSETS_PREFIX}`);
+    const outputPath = path.resolve(exportDir, `.${decoded}`);
+    return outputPath.startsWith(assetsDir + path.sep) ? outputPath : undefined;
+};
+const isCss = (url, contentType) => !!contentType?.includes('text/css') || new URL(url).pathname.endsWith('.css');
+export function rewriteCssUrls(css, cssUrl, cssAssetPath, onAsset) {
+    return css.replace(CSS_URL_PATTERN, (match, quote, rawUrl) => {
+        if (rawUrl.startsWith('data:') || rawUrl.startsWith('#'))
+            return match;
+        let absoluteUrl;
+        try {
+            absoluteUrl = new URL(rawUrl.startsWith('//') ? `https:${rawUrl}` : rawUrl, cssUrl).toString();
+        }
+        catch {
+            return match;
+        }
+        const assetPath = toMintExportAssetPath(absoluteUrl);
+        if (assetPath === absoluteUrl)
+            return match;
+        onAsset(assetPath);
+        const relativePath = path.posix.relative(path.posix.dirname(cssAssetPath), assetPath);
+        return `url(${quote}${relativePath}${quote})`;
+    });
+}
+async function localizeStylesheets(exportDir, onAsset) {
+    const nextStaticDir = path.join(exportDir, '_next', 'static');
+    if (!(await fse.pathExists(nextStaticDir)))
+        return;
+    for await (const file of walkFiles(nextStaticDir)) {
+        if (!file.endsWith('.css'))
+            continue;
+        const css = await fse.readFile(file, 'utf8');
+        const cssAssetPath = `/${path.relative(exportDir, file).split(path.sep).join('/')}`;
+        const rewritten = rewriteCssUrls(css, undefined, cssAssetPath, onAsset);
+        if (rewritten !== css)
+            await fse.writeFile(file, rewritten, 'utf8');
+    }
+}
+export async function localizeAssets(exportDir, fetchAsset = fetchPublicAsset) {
+    const queue = [...(await collectAssetPaths(exportDir))];
+    const seen = new Set(queue);
+    const downloaded = [];
+    const failed = [];
+    const enqueue = (assetPath) => {
+        if (seen.has(assetPath))
+            return;
+        seen.add(assetPath);
+        queue.push(assetPath);
+    };
+    await localizeStylesheets(exportDir, enqueue);
+    const localize = async (assetPath) => {
+        const url = fromMintExportAssetPath(assetPath);
+        const outputPath = resolveOutputPath(exportDir, assetPath);
+        if (!url || !outputPath) {
+            failed.push(assetPath);
+            return;
+        }
+        const asset = await fetchAsset(url);
+        if (!asset) {
+            failed.push(url);
+            return;
+        }
+        await fse.ensureDir(path.dirname(outputPath));
+        if (isCss(url, asset.contentType)) {
+            const css = rewriteCssUrls(asset.body.toString('utf8'), url, assetPath, enqueue);
+            await fse.writeFile(outputPath, css, 'utf8');
+        }
+        else {
+            await fse.writeFile(outputPath, asset.body);
+        }
+        downloaded.push(assetPath);
+    };
+    const worker = async () => {
+        while (queue.length > 0) {
+            const assetPath = queue.shift();
+            try {
+                await localize(assetPath);
+            }
+            catch {
+                failed.push(fromMintExportAssetPath(assetPath) ?? assetPath);
+            }
+        }
+    };
+    while (queue.length > 0) {
+        await Promise.all(Array.from({ length: Math.min(CONCURRENCY, queue.length) }, worker));
+    }
+    return { downloaded, failed };
+}

package/dist/local-preview/export-fetch.d.ts

@@ -0,0 +1,12 @@
+import { type LookupFunction } from 'net';
+export type PublicAsset = {
+    contentType: string | undefined;
+    body: Buffer;
+};
+export type FetchAsset = (url: string) => Promise<PublicAsset | undefined>;
+export type FetchPublicAssetOptions = {
+    lookup?: LookupFunction;
+    timeoutMs?: number;
+    maxBytes?: number;
+};
+export declare function fetchPublicAsset(url: string, { lookup, timeoutMs, maxBytes, }?: FetchPublicAssetOptions): Promise<PublicAsset | undefined>;

package/dist/local-preview/export-fetch.js

@@ -0,0 +1,55 @@
+import { isPrivateIP, safeLookup } from '@mintlify/common/ssrf';
+import http from 'http';
+import https from 'https';
+import { isIP } from 'net';
+const MAX_REDIRECTS = 5;
+const TIMEOUT_MS = 30_000;
+const MAX_ASSET_BYTES = 25 * 1024 * 1024;
+const BROWSER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36';
+const requestAsset = (url, { lookup, timeoutMs, maxBytes }) => new Promise((resolve, reject) => {
+    const client = url.protocol === 'https:' ? https : http;
+    const req = client.get(url, {
+        headers: { 'user-agent': BROWSER_USER_AGENT },
+        lookup,
+        signal: AbortSignal.timeout(timeoutMs),
+    }, (res) => {
+        const chunks = [];
+        let size = 0;
+        res.on('data', (chunk) => {
+            size += chunk.byteLength;
+            if (size > maxBytes) {
+                res.destroy();
+                reject(new Error(`asset exceeds ${maxBytes} bytes: ${url}`));
+                return;
+            }
+            chunks.push(chunk);
+        });
+        res.on('end', () => resolve({
+            status: res.statusCode ?? 0,
+            location: res.headers.location,
+            contentType: res.headers['content-type'],
+            body: Buffer.concat(chunks),
+        }));
+        res.on('error', reject);
+    });
+    req.on('error', reject);
+});
+export async function fetchPublicAsset(url, { lookup = safeLookup, timeoutMs = TIMEOUT_MS, maxBytes = MAX_ASSET_BYTES, } = {}) {
+    let current = new URL(url);
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+        if (current.protocol !== 'https:' && current.protocol !== 'http:')
+            return undefined;
+        const host = current.hostname.replace(/^\[|\]$/g, '');
+        if (isIP(host) && isPrivateIP(host))
+            return undefined;
+        const res = await requestAsset(current, { lookup, timeoutMs, maxBytes });
+        if (res.status >= 300 && res.status < 400 && res.location) {
+            current = new URL(res.location, current);
+            continue;
+        }
+        return res.status >= 200 && res.status < 300
+            ? { contentType: res.contentType, body: res.body }
+            : undefined;
+    }
+    return undefined;
+}

package/dist/local-preview/export-llms.d.ts

@@ -0,0 +1 @@
+export declare function generateExportLlmsTxt(docsConfig: unknown, generatedNav: unknown, routes: string[]): string;

package/dist/local-preview/export-llms.js

@@ -0,0 +1,39 @@
+import { divisions } from '@mintlify/validation';
+const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
+const getString = (record, key) => {
+    const value = record[key];
+    return typeof value === 'string' && value.trim() ? value.trim() : undefined;
+};
+function collectPages(entry, pages) {
+    if (Array.isArray(entry)) {
+        entry.forEach((item) => collectPages(item, pages));
+        return;
+    }
+    if (!isRecord(entry))
+        return;
+    const href = getString(entry, 'href');
+    const title = getString(entry, 'sidebarTitle') ?? getString(entry, 'title');
+    if (href && title && 'title' in entry) {
+        if (href.startsWith('/') && !href.startsWith('//') && !pages.has(href)) {
+            pages.set(href, { title, href, description: getString(entry, 'description') });
+        }
+        return;
+    }
+    for (const key of ['root', 'pages', 'groups', ...divisions]) {
+        collectPages(entry[key], pages);
+    }
+}
+const escapeLinkText = (text) => text.replace(/([[\]])/g, '\\$1');
+export function generateExportLlmsTxt(docsConfig, generatedNav, routes) {
+    const config = isRecord(docsConfig) ? docsConfig : {};
+    const name = getString(config, 'name') ?? 'Documentation';
+    const description = getString(config, 'description');
+    const pages = new Map();
+    collectPages(generatedNav, pages);
+    const header = `# ${name}\n${description ? `\n> ${description}\n` : ''}\n## Docs\n\n`;
+    const exportedRoutes = new Set(routes);
+    const lines = [...pages.values()]
+        .filter((page) => exportedRoutes.has(page.href))
+        .map((page) => `- [${escapeLinkText(page.title)}](${page.href})${page.description ? `: ${page.description}` : ''}\n`);
+    return header + lines.join('');
+}

package/dist/local-preview/export.d.ts

@@ -1,4 +1,5 @@
 import { ArgumentsCamelCase } from 'yargs';
 export declare function generatePages(baseUrl: string, routes: string[], outputDir: string, rootRoute?: string): Promise<number>;
 export declare function writeStaticUserInfo(tmpDir: string, groups: string[] | undefined): Promise<void>;
+export declare function writeLlmsTxt(tmpDir: string, routes: string[]): Promise<void>;
 export declare const exportSite: (argv: ArgumentsCamelCase) => Promise<void>;

package/dist/local-preview/export.js

@@ -11,11 +11,15 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import { CLIENT_PATH, CMD_EXEC_PATH, VERSION_PATH, NEXT_PUBLIC_PATH, NEXT_PROPS_PATH, PREVIEW_PATH, configureSharedPreviewPath, } from '../constants.js';
 import { addLog, clearLogs } from '../logging-state.js';
-import { ErrorLog, SpinnerLog, SuccessLog } from '../logs.js';
+import { ErrorLog, SpinnerLog, SuccessLog, WarningLog } from '../logs.js';
+import { localizeAssets } from './export-assets.js';
+import { generateExportLlmsTxt } from './export-llms.js';
 import { getGroupFilteredRoutes } from './getGroupFilteredRoutes.js';
 import { setupNext } from './setupNext.js';
 import { silentUpdateClient } from './update.js';
 const CONCURRENCY = 10;
+const RSC_CONTENT_TYPE = 'text/x-component';
+const RSC_PAYLOAD_FILE = 'index.txt';
 const EXPORT_SCRIPTS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'export-scripts');
 function resolveRequestRoute(route, rootRoute) {
     if (route === '/' && rootRoute)
@@ -92,17 +96,22 @@ export async function generatePages(baseUrl, routes, outputDir, rootRoute) {
             const route = queue.shift();
             try {
                 const requestRoute = resolveRequestRoute(route, rootRoute);
-                const res = await fetch(`${baseUrl}${requestRoute}`);
-                if (!res.ok) {
+                const [res, rscRes] = await Promise.all([
+                    fetch(`${baseUrl}${requestRoute}`),
+                    fetch(`${baseUrl}${requestRoute}`, { headers: { RSC: '1' } }),
+                ]);
+                const isRscPayload = rscRes.headers.get('content-type')?.startsWith(RSC_CONTENT_TYPE);
+                if (!res.ok || !rscRes.ok || !isRscPayload) {
                     failedPageCount += 1;
-                    addLog(_jsx(ErrorLog, { message: `failed to generate ${route} (status ${res.status})` }));
+                    addLog(_jsx(ErrorLog, { message: `failed to generate ${route} (status ${res.ok ? rscRes.status : res.status})` }));
                     continue;
                 }
-                const filePath = route === '/'
-                    ? path.join(outputDir, 'index.html')
-                    : path.join(outputDir, route, 'index.html');
-                await fse.ensureDir(path.dirname(filePath));
-                await fse.writeFile(filePath, await res.text(), 'utf8');
+                const pageDir = route === '/' ? outputDir : path.join(outputDir, route);
+                await fse.ensureDir(pageDir);
+                await Promise.all([
+                    fse.writeFile(path.join(pageDir, 'index.html'), await res.text(), 'utf8'),
+                    fse.writeFile(path.join(pageDir, RSC_PAYLOAD_FILE), await rscRes.text(), 'utf8'),
+                ]);
             }
             catch (err) {
                 failedPageCount += 1;
@@ -154,9 +163,20 @@ async function bundleStaticAssets(tmpDir, groups) {
     }
     await writeStaticUserInfo(tmpDir, groups);
     await fse.copy(path.join(EXPORT_SCRIPTS_DIR, 'serve.js'), path.join(tmpDir, 'serve.js'));
+    await fse.copy(path.join(EXPORT_SCRIPTS_DIR, 'README.md'), path.join(tmpDir, 'README.md'));
     await fse.copy(path.join(EXPORT_SCRIPTS_DIR, 'start-docs.sh'), path.join(tmpDir, 'Start Docs.command'));
     await fse.copy(path.join(EXPORT_SCRIPTS_DIR, 'start-docs.bat'), path.join(tmpDir, 'Start Docs.bat'));
 }
+export async function writeLlmsTxt(tmpDir, routes) {
+    const llmsTxtPath = path.join(tmpDir, 'llms.txt');
+    if (await pathExists(llmsTxtPath))
+        return;
+    const [docsConfig, generatedNav] = await Promise.all([
+        fse.readJson(path.join(NEXT_PROPS_PATH, 'docs.json')).catch(() => undefined),
+        fse.readJson(path.join(NEXT_PROPS_PATH, 'generatedDocsNav.json')).catch(() => undefined),
+    ]);
+    await fse.writeFile(llmsTxtPath, generateExportLlmsTxt(docsConfig, generatedNav, routes), 'utf8');
+}
 async function closeServer(server) {
     await new Promise((resolve) => {
         server.close(() => resolve());
@@ -179,11 +199,17 @@ async function generateExportArchive(routes, outputPath, groups) {
         }
         addLog(_jsx(SpinnerLog, { message: "collecting static assets..." }));
         await bundleStaticAssets(exportDir, groups);
+        await writeLlmsTxt(exportDir, routes);
+        addLog(_jsx(SpinnerLog, { message: "bundling external assets..." }));
+        const { failed } = await localizeAssets(exportDir);
         addLog(_jsx(SpinnerLog, { message: "creating zip archive..." }));
         const zip = new AdmZip();
         zip.addLocalFolder(exportDir);
         zip.writeZip(outputPath);
         clearLogs();
+        for (const url of failed) {
+            addLog(_jsx(WarningLog, { message: `could not bundle ${url}` }));
+        }
         addLog(_jsx(SuccessLog, { message: `exported to ${outputPath}` }));
     }
     finally {
@@ -197,6 +223,7 @@ export const exportSite = async (argv) => {
     configureSharedPreviewPath();
     const hasInternet = await isOnline();
     const clientVersion = argv['client-version'];
+    const localClientVersion = argv['local-client-version'];
     const groups = argv.groups;
     const cliVersion = argv.cliVersion;
     const disableOpenApi = argv.disableOpenapi;
@@ -205,7 +232,7 @@ export const exportSite = async (argv) => {
     const versionString = (await pathExists(VERSION_PATH))
         ? fse.readFileSync(VERSION_PATH, 'utf8')
         : null;
-    if (!versionString && !hasInternet) {
+    if (!versionString && !localClientVersion && !hasInternet) {
         await fatal('export requires an internet connection on first run.');
     }
     addLog(_jsx(SpinnerLog, { message: "preparing export..." }));
@@ -213,7 +240,7 @@ export const exportSite = async (argv) => {
         versionString,
         clientVersion,
         cliVersion,
-        localClientVersion: undefined,
+        localClientVersion,
     });
     if (error)
         await fatal(error);
@@ -242,6 +269,7 @@ export const exportSite = async (argv) => {
         await fatal(err instanceof Error && err.message ? err.message : 'failed to collect routes');
     }
     addLog(_jsx(SpinnerLog, { message: "starting local server..." }));
+    process.env.MINT_EXPORT = JSON.stringify({ groups });
     try {
         await generateExportArchive(routes, outputPath, groups);
     }