@mintlify/previewing 4.0.1533 → 4.0.1535

package/dist/__tests__/export-search.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/__tests__/export-search.test.js

@@ -0,0 +1,209 @@
+import { MINT_EXPORT_SEARCH_INDEX_PATH } from '@mintlify/common';
+import fse from 'fs-extra';
+import os from 'os';
+import path from 'path';
+import { extractSearchDocuments, getSearchablePages, writeSearchIndex, } from '../local-preview/export-search.js';
+const page = (title, body, description = '') => `<!DOCTYPE html><html><head>
+<meta name="description" content="${description}"/></head><body>
+<nav><a href="/other">Sidebar link</a></nav>
+<h1 id="page-title">${title}</h1>
+<div id="content" data-page-title="${title}">${body}</div>
+<footer>Footer text</footer>
+</body></html>`;
+const nav = (value) => value;
+const config = (value) => value;
+describe('extractSearchDocuments', () => {
+    it('splits content into sections at headings with ids', () => {
+        const documents = extractSearchDocuments(page('Configuration', '<p>Intro text.</p><h2 id="install"><a href="#install">​</a>Install</h2><p>Run <code>npm i</code>.</p>' +
+            '<h2 id="_R_abc_">Card title</h2><p>Card body.</p><h3 id="flags">Flags</h3><ul><li>--port</li><li>--host</li></ul>' +
+            '<button>Copy</button><script>ignored()</script><span aria-hidden="true">hidden</span>', 'Configure Conduit'), '/guides/configuration', { breadcrumbs: ['Guides'], version: 'v2' });
+        expect(documents).toEqual([
+            {
+                id: '/guides/configuration#0.0',
+                page: '/guides/configuration',
+                title: 'Configuration',
+                header: 'Configuration',
+                content: 'Configure Conduit Intro text.',
+                breadcrumbs: ['Guides'],
+                version: 'v2',
+            },
+            {
+                id: '/guides/configuration#1.0',
+                page: '/guides/configuration',
+                title: 'Configuration',
+                header: 'Install',
+                hash: 'install',
+                content: 'Run npm i. Card title Card body.',
+                breadcrumbs: ['Guides'],
+                version: 'v2',
+            },
+            {
+                id: '/guides/configuration#2.0',
+                page: '/guides/configuration',
+                title: 'Configuration',
+                header: 'Flags',
+                hash: 'flags',
+                content: '--port --host',
+                breadcrumbs: ['Guides'],
+                version: 'v2',
+            },
+        ]);
+    });
+    it('splits long sections into uniquely identified, overlapping chunks so their full text stays searchable', () => {
+        const words = Array.from({ length: 900 }, (_, i) => `w${i}`).join(' ');
+        const body = `<h2 id="install">Install</h2><p>${words} lastword</p>` +
+            '<h2 id="install-1">Install</h2><p>Again.</p>';
+        const documents = extractSearchDocuments(page('Long', body), '/long').slice(1);
+        expect(documents.map((document) => [document.id, document.hash])).toEqual([
+            ['/long#1.0', 'install'],
+            ['/long#1.1', 'install'],
+            ['/long#1.2', 'install'],
+            ['/long#2.0', 'install-1'],
+        ]);
+        expect(documents.every((document) => document.content.length <= 2000)).toBe(true);
+        expect(documents[2]?.content.endsWith('lastword')).toBe(true);
+        expect(documents[1]?.content.split(' ')[0]).toBe(documents[0]?.content.split(' ').at(-1));
+        expect(documents[2]?.content.split(' ')[0]).toBe(documents[1]?.content.split(' ').at(-1));
+    });
+    it('falls back to the content area on API pages and keeps table cells apart', () => {
+        const html = `<html><head><meta name="description" content="List users"/></head><body>
+<div id="content-area"><h1 id="page-title">Get users</h1>
+<h2 id="params">Params</h2><table><tr><th>Name</th><th>Type</th></tr><tr><td>limit</td><td>integer</td></tr></table>
+</div></body></html>`;
+        expect(extractSearchDocuments(html, '/api/users')).toEqual([
+            expect.objectContaining({ header: 'Get users', content: 'List users Get users' }),
+            expect.objectContaining({ hash: 'params', content: 'Name Type limit integer' }),
+        ]);
+    });
+    it('returns nothing for pages without a content region', () => {
+        expect(extractSearchDocuments('<html><body><p>404</p></body></html>', '/missing')).toEqual([]);
+    });
+});
+describe('getSearchablePages', () => {
+    it('tags pages with breadcrumbs, language and version', () => {
+        const pages = getSearchablePages(nav({
+            languages: [
+                {
+                    language: 'en',
+                    versions: [
+                        {
+                            version: 'v2',
+                            groups: [{ group: 'Start', pages: [{ href: '/quickstart', title: 'Quickstart' }] }],
+                        },
+                    ],
+                },
+                {
+                    language: 'fr',
+                    groups: [{ group: 'Début', pages: [{ href: '/fr/quickstart', title: 'Démarrage' }] }],
+                },
+            ],
+        }), config({
+            navigation: {
+                languages: [
+                    {
+                        language: 'en',
+                        versions: [
+                            {
+                                version: 'v2',
+                                tabs: [{ tab: 'Docs', groups: [{ group: 'Start', pages: ['quickstart'] }] }],
+                            },
+                        ],
+                    },
+                    { language: 'fr', groups: [{ group: 'Début', pages: ['fr/quickstart'] }] },
+                ],
+            },
+        }), undefined);
+        expect(pages.get('/quickstart')).toEqual({
+            breadcrumbs: ['Docs', 'Start'],
+            locale: 'en',
+            version: 'v2',
+        });
+        expect(pages.get('/fr/quickstart')).toEqual({ breadcrumbs: ['Début'], locale: 'fr' });
+    });
+    it('leaves out pages that hosted search leaves out', () => {
+        const generatedNav = nav({
+            groups: [
+                {
+                    group: 'Guides',
+                    pages: [
+                        { href: '/visible', title: 'Visible' },
+                        { href: '/hidden', title: 'Hidden', hidden: true },
+                        { href: '/noindex', title: 'Noindex', noindex: true },
+                        { href: '/unsearchable', title: 'Unsearchable', searchable: false },
+                        { href: '/admin', title: 'Admin', groups: ['admin'] },
+                    ],
+                },
+            ],
+        });
+        expect([...getSearchablePages(generatedNav, undefined, undefined).keys()]).toEqual([
+            '/visible',
+        ]);
+        expect([...getSearchablePages(generatedNav, undefined, ['admin']).keys()]).toEqual([
+            '/visible',
+            '/admin',
+        ]);
+        expect([
+            ...getSearchablePages(generatedNav, config({ seo: { indexing: 'all' } }), undefined).keys(),
+        ]).toEqual(['/visible', '/hidden']);
+    });
+});
+describe('writeSearchIndex', () => {
+    let exportDir;
+    beforeEach(async () => {
+        exportDir = await fse.mkdtemp(path.join(os.tmpdir(), 'mint-export-search-test-'));
+    });
+    afterEach(async () => {
+        await fse.remove(exportDir);
+    });
+    it('indexes each searchable exported page once and links /index pages to their parent', async () => {
+        await fse.outputFile(path.join(exportDir, 'index', 'index.html'), page('Introduction', '<p>Welcome.</p>'));
+        await fse.outputFile(path.join(exportDir, 'quickstart', 'index.html'), page('Quickstart', '<p>Install.</p>'));
+        await fse.outputFile(path.join(exportDir, 'secret', 'index.html'), page('Secret', '<p>Hidden.</p>'));
+        const count = await writeSearchIndex(exportDir, {
+            generatedNav: nav({
+                groups: [
+                    {
+                        group: 'Getting started',
+                        pages: [
+                            { href: '/index', title: 'Introduction' },
+                            { href: '/quickstart', title: 'Quickstart' },
+                            { href: '/missing', title: 'Missing' },
+                            { href: '/secret', title: 'Secret', noindex: true },
+                        ],
+                    },
+                ],
+            }),
+            docsConfig: config({
+                navigation: {
+                    groups: [
+                        { group: 'Getting started', pages: ['index', 'quickstart', 'missing', 'secret'] },
+                    ],
+                },
+            }),
+            groups: undefined,
+        });
+        const documents = await fse.readJson(path.join(exportDir, MINT_EXPORT_SEARCH_INDEX_PATH));
+        expect(count).toBe(2);
+        expect(documents).toEqual([
+            expect.objectContaining({
+                page: '/',
+                title: 'Introduction',
+                breadcrumbs: ['Getting started'],
+            }),
+            expect.objectContaining({
+                page: '/quickstart',
+                title: 'Quickstart',
+                breadcrumbs: ['Getting started'],
+            }),
+        ]);
+    });
+    it('writes an empty index when there is no generated navigation', async () => {
+        const count = await writeSearchIndex(exportDir, {
+            generatedNav: undefined,
+            docsConfig: undefined,
+            groups: undefined,
+        });
+        expect(count).toBe(0);
+        expect(await fse.readJson(path.join(exportDir, MINT_EXPORT_SEARCH_INDEX_PATH))).toEqual([]);
+    });
+});

package/dist/local-preview/export-search.d.ts

@@ -0,0 +1,22 @@
+import type { DecoratedNavigationConfig, DocsConfig } from '@mintlify/validation';
+type PageSearchInfo = {
+    breadcrumbs?: string[];
+    locale?: string;
+    version?: string;
+};
+export type SearchDocument = PageSearchInfo & {
+    id: string;
+    page: string;
+    title: string;
+    header: string;
+    hash?: string;
+    content: string;
+};
+export declare function extractSearchDocuments(html: string, page: string, { breadcrumbs, locale, version }?: PageSearchInfo): SearchDocument[];
+export declare function getSearchablePages(generatedNav: DecoratedNavigationConfig, docsConfig: Pick<DocsConfig, 'navigation' | 'seo'> | undefined, groups: string[] | undefined): Map<string, PageSearchInfo>;
+export declare function writeSearchIndex(exportDir: string, { generatedNav, docsConfig, groups, }: {
+    generatedNav: DecoratedNavigationConfig | undefined;
+    docsConfig: Pick<DocsConfig, 'navigation' | 'seo'> | undefined;
+    groups: string[] | undefined;
+}): Promise<number>;
+export {};

package/dist/local-preview/export-search.js

@@ -0,0 +1,160 @@
+import { WILDCARD_GROUP, checkNavAccess, decoratedNavChildren, decoratedNavDivisions, generatePathToBreadcrumbsMapForDocsConfig, generatePathToHiddenFromSearchDict, generatePathToLanguageDict, generatePathToVersionDict, isPage, MINT_EXPORT_SEARCH_INDEX_PATH, optionallyRemoveLeadingSlash, } from '@mintlify/common';
+import fse from 'fs-extra';
+import { fromHtml } from 'hast-util-from-html';
+import { toText } from 'hast-util-to-text';
+import path from 'path';
+const MAX_CONTENT_LENGTH = 2000;
+const SKIPPED_TAGS = new Set(['script', 'style', 'svg', 'button', 'noscript', 'template']);
+const SECTION_HEADINGS = new Set(['h2', 'h3']);
+const isElement = (node) => node.type === 'element';
+const getProperty = (element, name) => {
+    const value = element.properties[name];
+    return typeof value === 'string' ? value : undefined;
+};
+const isHidden = (element) => SKIPPED_TAGS.has(element.tagName) || element.properties.ariaHidden === 'true';
+const REACT_GENERATED_ID = /^(_R_|«|:)/;
+const normalizeWhitespace = (text) => text
+    .replace(/[\u200b\u200c\u200d\ufeff]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+function splitContent(text) {
+    const chunks = [];
+    let current = '';
+    for (const word of text.split(' ')) {
+        if (current && current.length + word.length + 1 > MAX_CONTENT_LENGTH) {
+            chunks.push(current);
+            current = `${current.slice(current.lastIndexOf(' ') + 1)} ${word}`;
+        }
+        else {
+            current = current ? `${current} ${word}` : word;
+        }
+    }
+    return [...chunks, current];
+}
+function findElement(node, predicate) {
+    for (const child of node.children) {
+        if (!isElement(child))
+            continue;
+        if (predicate(child))
+            return child;
+        const found = findElement(child, predicate);
+        if (found)
+            return found;
+    }
+    return undefined;
+}
+function collectSections(content, pageTitle) {
+    const sections = [{ header: pageTitle, text: [] }];
+    const visit = (node) => {
+        if (node.type === 'text') {
+            sections[sections.length - 1]?.text.push(node.value);
+            return;
+        }
+        if (!isElement(node) || isHidden(node))
+            return;
+        const hash = getProperty(node, 'id');
+        if (SECTION_HEADINGS.has(node.tagName) && hash && !REACT_GENERATED_ID.test(hash)) {
+            sections.push({ header: normalizeWhitespace(toText(node)), hash, text: [] });
+            return;
+        }
+        const isBlock = /^(p|div|li|tr|td|th|pre|h[1-6]|blockquote|section|table|ul|ol)$/.test(node.tagName);
+        if (isBlock)
+            sections[sections.length - 1]?.text.push(' ');
+        node.children.forEach(visit);
+        if (isBlock)
+            sections[sections.length - 1]?.text.push(' ');
+    };
+    content.children.forEach(visit);
+    return sections;
+}
+const findById = (tree, id) => findElement(tree, (element) => getProperty(element, 'id') === id);
+function getPageTitle(tree, content, page) {
+    const frontmatterTitle = getProperty(content, 'dataPageTitle');
+    if (frontmatterTitle)
+        return frontmatterTitle;
+    const heading = findById(tree, 'page-title');
+    return heading ? normalizeWhitespace(toText(heading)) : page;
+}
+function getPageDescription(tree) {
+    const meta = findElement(tree, (element) => element.tagName === 'meta' && getProperty(element, 'name') === 'description');
+    return meta && getProperty(meta, 'content');
+}
+export function extractSearchDocuments(html, page, { breadcrumbs, locale, version } = {}) {
+    const tree = fromHtml(html);
+    const content = findById(tree, 'content') ?? findById(tree, 'content-area');
+    if (!content)
+        return [];
+    const title = getPageTitle(tree, content, page);
+    const description = getPageDescription(tree);
+    return collectSections(content, title).flatMap((section, index) => {
+        const text = section.text.join('');
+        const searchableText = index === 0 && description ? `${description} ${text}` : text;
+        return splitContent(normalizeWhitespace(searchableText)).map((chunk, part) => ({
+            id: `${page}#${index}.${part}`,
+            page,
+            title,
+            header: section.header,
+            ...(section.hash && { hash: section.hash }),
+            content: chunk,
+            ...(breadcrumbs?.length && { breadcrumbs }),
+            ...(locale && { locale }),
+            ...(version && { version }),
+        }));
+    });
+}
+function collectNavPages(node, pages = []) {
+    if (isPage(node))
+        pages.push(node);
+    for (const child of [...decoratedNavChildren(node), ...decoratedNavDivisions(node)]) {
+        collectNavPages(child, pages);
+    }
+    return pages;
+}
+export function getSearchablePages(generatedNav, docsConfig, groups) {
+    const hiddenFromSearch = docsConfig?.seo?.indexing === 'all'
+        ? new Map()
+        : generatePathToHiddenFromSearchDict(generatedNav);
+    const breadcrumbs = docsConfig?.navigation
+        ? generatePathToBreadcrumbsMapForDocsConfig(docsConfig.navigation)
+        : new Map();
+    const languages = generatePathToLanguageDict(generatedNav);
+    const versions = generatePathToVersionDict(generatedNav);
+    const userGroups = new Set([...(groups ?? []), WILDCARD_GROUP]);
+    const pages = new Map();
+    for (const navPage of collectNavPages(generatedNav)) {
+        const slug = optionallyRemoveLeadingSlash(navPage.href);
+        if (pages.has(navPage.href) ||
+            hiddenFromSearch.get(slug) ||
+            navPage.noindex ||
+            navPage.searchable === false ||
+            !checkNavAccess(navPage, userGroups, false, false)) {
+            continue;
+        }
+        const locale = languages.get(slug);
+        const version = versions.get(slug);
+        pages.set(navPage.href, {
+            breadcrumbs: breadcrumbs.get(slug),
+            ...(locale && { locale }),
+            ...(version && { version }),
+        });
+    }
+    return pages;
+}
+const toPagePath = (route) => route.replace(/\/index$/, '') || '/';
+export async function writeSearchIndex(exportDir, { generatedNav, docsConfig, groups, }) {
+    const searchablePages = generatedNav
+        ? getSearchablePages(generatedNav, docsConfig, groups)
+        : new Map();
+    const indexedPages = new Set();
+    const documents = [];
+    for (const [route, info] of searchablePages) {
+        const page = toPagePath(route);
+        const htmlPath = path.join(exportDir, route, 'index.html');
+        if (indexedPages.has(page) || !(await fse.pathExists(htmlPath)))
+            continue;
+        indexedPages.add(page);
+        documents.push(...extractSearchDocuments(await fse.readFile(htmlPath, 'utf8'), page, info));
+    }
+    await fse.outputJson(path.join(exportDir, MINT_EXPORT_SEARCH_INDEX_PATH), documents);
+    return documents.length;
+}

package/dist/local-preview/export.js

@@ -15,6 +15,7 @@ import { ErrorLog, SpinnerLog, SuccessLog, WarningLog } from '../logs.js';
 import { localizeAssets } from './export-assets.js';
 import { applyBasePathToExport, findBasePathConflict, normalizeExportBasePath, } from './export-base-path.js';
 import { generateExportLlmsTxt } from './export-llms.js';
+import { writeSearchIndex } from './export-search.js';
 import { getGroupFilteredRoutes } from './getGroupFilteredRoutes.js';
 import { setupNext } from './setupNext.js';
 import { silentUpdateClient } from './update.js';
@@ -208,6 +209,12 @@ async function generateExportArchive(routes, outputPath, groups, basePath) {
         await bundleStaticAssets(exportDir, groups, basePath);
         await applyBasePathToExport(exportDir, basePath);
         await writeLlmsTxt(exportDir, routes, basePath);
+        addLog(_jsx(SpinnerLog, { message: "building search index..." }));
+        const [docsConfig, generatedNav] = await Promise.all([
+            fse.readJson(path.join(NEXT_PROPS_PATH, 'docs.json')).catch(() => undefined),
+            fse.readJson(path.join(NEXT_PROPS_PATH, 'generatedDocsNav.json')).catch(() => undefined),
+        ]);
+        await writeSearchIndex(exportDir, { generatedNav, docsConfig, groups });
         addLog(_jsx(SpinnerLog, { message: "bundling external assets..." }));
         const { failed } = await localizeAssets(exportDir);
         addLog(_jsx(SpinnerLog, { message: "creating zip archive..." }));