@rspress/plugin-llms 2.0.0-alpha.8 → 2.0.0-beta.1

package/dist/index.d.ts

@@ -228,6 +228,15 @@ declare interface Hero {
     }[];
 }
 
+export declare interface LlmsTxt {
+    onTitleGenerate?: (context: {
+        title: string | undefined;
+        description: string | undefined;
+    }) => string;
+    onLineGenerate?: (page: PageIndexInfo) => string;
+    onAfterLlmsTxtGenerate?: (llmsTxtContent: string) => string;
+}
+
 declare interface Locale {
     lang: string;
     label: string;
@@ -302,21 +311,6 @@ declare interface MarkdownOptions {
      * Register prism languages
      */
     highlightLanguages?: (string | [string, string])[];
-    /**
-     * Whether to enable mdx-rs, default is true
-     */
-    mdxRs?: boolean | MdxRsOptions;
-    /**
-     * @deprecated, use `mdxRs` instead
-     */
-    experimentalMdxRs?: boolean;
-}
-
-declare interface MdxRsOptions {
-    /**
-     * Determine whether the file use mdxRs compiler
-     */
-    include?: (filepath: string) => boolean;
 }
 
 declare type Nav = NavItem[] | {
@@ -349,12 +343,29 @@ declare interface NavItemWithLinkAndChildren {
     position?: 'left' | 'right';
 }
 
-declare interface Options {
-    output: {
-        llmsTxt?: boolean;
-        mdFiles?: boolean;
-        llmsFullTxt?: boolean;
-    };
+export declare interface Options {
+    /**
+     * Whether to generate llms.txt.
+     * @default true
+     */
+    llmsTxt?: boolean | LlmsTxt;
+    /**
+     * Whether to generate llms.txt related md files for each route.
+     * @default true
+     */
+    mdFiles?: boolean;
+    /**
+     * Whether to generate llms.full.txt.
+     * @default true
+     */
+    llmsFullTxt?: boolean;
+    /**
+     * Whether to exclude some routes from llms.txt.
+     * @default undefined
+     */
+    exclude?: (context: {
+        page: PageIndexInfo;
+    }) => boolean;
 }
 
 /**
@@ -368,7 +379,7 @@ declare interface PageIndexInfo {
     routePath: string;
     toc: Header[];
     content: string;
-    flattenContent?: string;
+    _flattenContent?: string;
     _html: string;
     frontmatter: FrontMatterMeta;
     lang: string;

package/dist/index.js

@@ -1,4 +1,5 @@
 import * as __WEBPACK_EXTERNAL_MODULE_node_path_c5b9b54f__ from "node:path";
+import * as __WEBPACK_EXTERNAL_MODULE__rspress_runtime_server_673c713a__ from "@rspress/runtime/server";
 import * as __WEBPACK_EXTERNAL_MODULE__rspress_core_1b2e46ce__ from "@rspress/core";
 import * as __WEBPACK_EXTERNAL_MODULE_unist_util_visit_555e002a__ from "unist-util-visit";
 import * as __WEBPACK_EXTERNAL_MODULE_unified__ from "unified";
@@ -5831,11 +5832,33 @@ function normalizeSlash(url) {
 function isExternalUrl(url = '') {
     return url.startsWith('http://') || url.startsWith('https://') || url.startsWith('mailto:') || url.startsWith('tel:');
 }
+const parseUrl = (url)=>{
+    const [withoutHash, hash = ''] = url.split('#');
+    return {
+        url: withoutHash,
+        hash
+    };
+};
+function normalizeHref(url, cleanUrls = false) {
+    if (!url) return '/';
+    if (isExternalUrl(url)) return url;
+    if (url.startsWith('#')) return url;
+    let { url: cleanUrl, hash } = parseUrl(decodeURIComponent(url));
+    if (cleanUrls) {
+        if (cleanUrl.endsWith('.html')) cleanUrl = cleanUrl.replace(/\.html$/, '');
+        if (cleanUrls && cleanUrl.endsWith('/index')) cleanUrl = cleanUrl.replace(/\/index$/, '/');
+    } else if (!cleanUrl.endsWith('.html')) if (cleanUrl.endsWith('/')) cleanUrl += 'index.html';
+    else cleanUrl += '.html';
+    return addLeadingSlash(hash ? `${cleanUrl}#${hash}` : cleanUrl);
+}
 function withBase(url, base) {
     const normalizedUrl = addLeadingSlash(url);
     const normalizedBase = normalizeSlash(base);
     return normalizedUrl.startsWith(normalizedBase) ? normalizedUrl : `${normalizedBase}${normalizedUrl}`;
 }
+function removeBase(url, base) {
+    return addLeadingSlash(url).replace(new RegExp(`^${normalizeSlash(base)}`), '');
+}
 const matchSidebar = (pattern, currentPathname, base)=>{
     const prefix = withBase(pattern, base);
     if (prefix === currentPathname) return true;
@@ -5844,23 +5867,56 @@ const matchSidebar = (pattern, currentPathname, base)=>{
     const prefixWithDot = `${prefix}.`;
     return currentPathname.startsWith(prefixWithDot);
 };
-function generateLlmsTxt(pageDataArray, navList) {
+const getSidebarDataGroup = (sidebar, currentPathname, base)=>{
+    const navRoutes = Object.keys(sidebar).sort((a, b)=>b.length - a.length);
+    for (const name of navRoutes)if (matchSidebar(name, currentPathname, base)) {
+        const sidebarGroup = sidebar[name];
+        return sidebarGroup;
+    }
+    return [];
+};
+function routePathToMdPath(routePath) {
+    let url = routePath;
+    url = normalizeHref(url, false);
+    url = url.replace(/\.html$/, '.md');
+    return url;
+}
+function generateLlmsTxt(pageDataArray, navList, others, llmsTxtOptions, title, description) {
     const lines = [];
+    const { onAfterLlmsTxtGenerate, onLineGenerate, onTitleGenerate } = 'boolean' == typeof llmsTxtOptions ? {} : llmsTxtOptions;
+    const summary = onTitleGenerate ? onTitleGenerate({
+        title,
+        description
+    }) : `# ${title}${description ? `\n\n> ${description}` : ''}`;
     for(let i = 0; i < navList.length; i++){
         const nav = navList[i];
         const pages = pageDataArray[i];
         const { text } = nav;
         if (0 === pages.length) continue;
         const title = text;
-        lines.push(`# ${title}\n`);
+        lines.push(`\n## ${title}\n`);
         for (const page of pages){
-            lines.push(`[${page.title}](${page.routePath.replace(/\.html$/, '.md')})`);
-            lines.push('');
-        }
-    }
-    return lines.join('\n');
-}
-function generateLlmsFullTxt(pageDataArray, navList) {
+            const { routePath, lang, title, frontmatter } = page;
+            if ('/' === routePath || routePath === `/${lang}/`) continue;
+            const line = onLineGenerate ? onLineGenerate(page) : `- [${title}](${routePathToMdPath(routePath)})${frontmatter.description ? `: ${frontmatter.description}` : ''}`;
+            lines.push(line);
+        }
+    }
+    let hasOthers = false;
+    const otherLines = [];
+    otherLines.push('\n## Others\n');
+    for (const page of others){
+        const { routePath, lang, title, frontmatter } = page;
+        if ('/' === routePath || routePath === `/${lang}/`) continue;
+        const line = onLineGenerate ? onLineGenerate(page) : `- [${title}](${routePathToMdPath(routePath)})${frontmatter.description ? `: ${frontmatter.description}` : ''}`;
+        otherLines.push(line);
+        hasOthers = true;
+    }
+    if (hasOthers) lines.push(...otherLines);
+    const llmsTxt = `${summary}\n${lines.join('\n')}`;
+    return onAfterLlmsTxtGenerate ? onAfterLlmsTxtGenerate(llmsTxt) : llmsTxt;
+}
+function generateLlmsFullTxt(pageDataArray, navList, others) {
     const lines = [];
     for(let i = 0; i < navList.length; i++){
         const nav = navList[i];
@@ -5869,10 +5925,15 @@ function generateLlmsFullTxt(pageDataArray, navList) {
         const title = nav.text;
         lines.push(`# ${title}\n`);
         for (const page of pages){
-            lines.push(page.mdContent ?? page.flattenContent ?? page.content);
+            lines.push(page.mdContent ?? page._flattenContent ?? page.content);
             lines.push('\n');
         }
     }
+    lines.push('# Others\n');
+    for (const page of others){
+        lines.push(page.mdContent ?? page._flattenContent ?? page.content);
+        lines.push('\n');
+    }
     return lines.join('\n');
 }
 function default_ok() {}
@@ -20842,25 +20903,36 @@ function mdxToMd(content, filepath, docDirectory, routeService) {
         path: filepath
     });
 }
-const rsbuildPluginLlms = ({ pageDataList, routes, baseRef, docDirectoryRef, routeServiceRef, nav, output })=>({
+const rsbuildPluginLlms = ({ pageDataList, routes, titleRef, descriptionRef, sidebar, baseRef, docDirectoryRef, routeServiceRef, nav, rspressPluginOptions })=>({
         name: 'rsbuild-plugin-llms',
         async setup (api) {
-            const { llmsTxt, mdFiles, llmsFullTxt } = output;
+            const { llmsTxt = true, mdFiles = true, llmsFullTxt = true, exclude } = rspressPluginOptions;
             api.onBeforeBuild(async ()=>{
                 const base = baseRef.current;
                 const docDirectory = docDirectoryRef.current;
-                const newPageDataList = mergeRouteMetaWithPageData(routes, pageDataList);
-                const navList = Array.isArray(nav) ? nav.map((i)=>i.default).flat().filter(Boolean) : [];
-                const pageArray = new Array(navList.length).fill([]);
+                const newPageDataList = mergeRouteMetaWithPageData(routes, pageDataList, exclude);
+                const navList = Array.isArray(nav) ? nav.map((i)=>{
+                    const nav = i.nav.default;
+                    const lang = i.lang;
+                    return nav.map((i)=>({
+                            ...i,
+                            lang
+                        }));
+                }).flat().filter((i)=>i.activeMatch || i.link) : [];
+                const others = [];
+                const pageArray = new Array(navList.length).fill(0).map(()=>[]);
                 newPageDataList.forEach((pageData)=>{
-                    const { routePath } = pageData;
-                    pageArray.forEach((pageArrayItem, index)=>{
-                        const navItem = navList[index];
-                        if (matchSidebar(navItem.activeMatch ?? navItem.link, routePath, base)) pageArrayItem.push(pageData);
-                    });
+                    const { routePath, lang } = pageData;
+                    for(let i = 0; i < pageArray.length; i++){
+                        const pageArrayItem = pageArray[i];
+                        const navItem = navList[i];
+                        if (lang === navItem.lang && new RegExp(navItem.activeMatch ?? navItem.link).test(removeBase(routePath, base))) return void pageArrayItem.push(pageData);
+                    }
+                    others.push(pageData);
                 });
+                for (const array of pageArray)organizeBySidebar(sidebar, array, base);
                 if (llmsTxt) {
-                    const llmsTxtContent = generateLlmsTxt(pageArray, navList);
+                    const llmsTxtContent = generateLlmsTxt(pageArray, navList, others, rspressPluginOptions.llmsTxt ?? {}, titleRef.current, descriptionRef.current);
                     api.processAssets({
                         targets: [
                             'node'
@@ -20875,7 +20947,7 @@ const rsbuildPluginLlms = ({ pageDataList, routes, baseRef, docDirectoryRef, rou
                 await Promise.all([
                     ...newPageDataList.values()
                 ].map(async (pageData)=>{
-                    const content = pageData.flattenContent ?? pageData.content;
+                    const content = pageData._flattenContent ?? pageData.content;
                     const filepath = pageData._filepath;
                     const isMD = 'md' === __WEBPACK_EXTERNAL_MODULE_node_path_c5b9b54f__["default"].extname(filepath).slice(1);
                     let mdContent;
@@ -20903,7 +20975,7 @@ const rsbuildPluginLlms = ({ pageDataList, routes, baseRef, docDirectoryRef, rou
                     });
                 });
                 if (llmsFullTxt) {
-                    const llmsFullTxtContent = generateLlmsFullTxt(pageArray, navList);
+                    const llmsFullTxtContent = generateLlmsFullTxt(pageArray, navList, others);
                     api.processAssets({
                         targets: [
                             'node'
@@ -20917,8 +20989,13 @@ const rsbuildPluginLlms = ({ pageDataList, routes, baseRef, docDirectoryRef, rou
             });
         }
     });
-function mergeRouteMetaWithPageData(routeMetaList, pageDataList) {
-    const m = new Map(pageDataList.map((pageData)=>[
+function mergeRouteMetaWithPageData(routeMetaList, pageDataList, exclude) {
+    const m = new Map(pageDataList.filter((pageData)=>{
+        if (exclude) return !exclude?.({
+            page: pageData
+        });
+        return true;
+    }).map((pageData)=>[
             pageData.routePath,
             pageData
         ]));
@@ -20929,16 +21006,42 @@ function mergeRouteMetaWithPageData(routeMetaList, pageDataList) {
     });
     return mergedPageDataList;
 }
-function pluginLlms(options = {
-    output: {}
-}) {
-    const { llmsTxt = true, llmsFullTxt = true, mdFiles = true } = options.output;
+function flatSidebar(sidebar) {
+    if (!sidebar) return [];
+    return sidebar.flatMap((i)=>{
+        if ('string' == typeof i) return i;
+        if ('link' in i && 'string' == typeof i.link) return [
+            i.link,
+            ...flatSidebar(i?.items ?? [])
+        ];
+        if ('items' in i && Array.isArray(i.items)) return flatSidebar(i.items);
+    }).filter(Boolean);
+}
+function organizeBySidebar(sidebar, pages, base) {
+    if (0 === pages.length) return;
+    const pageItem = pages[0];
+    const currSidebar = getSidebarDataGroup(sidebar, pageItem.routePath, base);
+    if (0 === currSidebar.length) return;
+    const orderList = flatSidebar(currSidebar);
+    pages.sort((a, b)=>{
+        const aIndex = orderList.findIndex((order)=>(0, __WEBPACK_EXTERNAL_MODULE__rspress_runtime_server_673c713a__.matchPath)(order, a.routePath));
+        const bIndex = orderList.findIndex((order)=>(0, __WEBPACK_EXTERNAL_MODULE__rspress_runtime_server_673c713a__.matchPath)(order, b.routePath));
+        return aIndex - bIndex;
+    });
+}
+function pluginLlms(options = {}) {
     const baseRef = {
         current: ''
     };
     const docDirectoryRef = {
         current: ''
     };
+    const titleRef = {
+        current: ''
+    };
+    const descriptionRef = {
+        current: ''
+    };
     const pageDataList = [];
     const routes = [];
     const sidebar = {};
@@ -20958,9 +21061,18 @@ function pluginLlms(options = {
             if (isProd) routes.push(..._routes);
         },
         beforeBuild (config) {
-            Object.assign(sidebar, config.themeConfig?.sidebar ?? {});
-            const configNav = config.themeConfig?.locales?.map((i)=>i.nav).filter(Boolean);
+            const configSidebar = config?.themeConfig?.locales?.map((i)=>i.sidebar).reduce((prev, curr)=>{
+                Object.assign(prev, curr);
+                return prev;
+            }, {});
+            Object.assign(sidebar, configSidebar);
+            const configNav = config.themeConfig?.locales?.filter((i)=>Boolean(i.nav))?.map((i)=>({
+                    nav: i.nav,
+                    lang: i.lang
+                }));
             nav.push(...configNav);
+            titleRef.current = config.title;
+            descriptionRef.current = config.description;
             baseRef.current = config.base ?? '/';
             docDirectoryRef.current = config.root ?? 'docs';
         },
@@ -20970,16 +21082,14 @@ function pluginLlms(options = {
                     ...options,
                     pageDataList,
                     routes,
+                    titleRef,
+                    descriptionRef,
                     sidebar,
                     docDirectoryRef,
                     routeServiceRef,
                     nav,
                     baseRef,
-                    output: {
-                        llmsFullTxt,
-                        llmsTxt,
-                        mdFiles
-                    }
+                    rspressPluginOptions: options
                 })
             ]
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rspress/plugin-llms",
-  "version": "2.0.0-alpha.8",
+  "version": "2.0.0-beta.1",
   "description": "A plugin for rspress to generate llms.txt, llms-full.txt, md files to let llm understand your website.",
   "bugs": "https://github.com/web-infra-dev/rspress/issues",
   "repository": {
@@ -23,9 +23,9 @@
     "unist-util-visit-children": "^3.0.0"
   },
   "devDependencies": {
-    "@microsoft/api-extractor": "^7.52.3",
+    "@microsoft/api-extractor": "^7.52.4",
     "@rsbuild/core": "1.3.0",
-    "@rslib/core": "0.6.3",
+    "@rslib/core": "0.6.5",
     "@types/hast": "^3.0.4",
     "@types/node": "^18.11.17",
     "remark-mdx": "^3.1.0",
@@ -34,14 +34,14 @@
     "typescript": "^5.8.2",
     "vfile": "^6.0.3",
     "@rspress/config": "1.0.0",
-    "@rspress/shared": "2.0.0-alpha-canary-202504162001"
+    "@rspress/shared": "2.0.0-beta.1"
   },
   "peerDependencies": {
-    "@rspress/core": "^2.0.0-alpha.8",
-    "@rspress/runtime": "^2.0.0-alpha.9"
+    "@rspress/core": "^2.0.0-beta.1",
+    "@rspress/runtime": "^2.0.0-beta.1"
   },
   "engines": {
-    "node": ">=14.17.6"
+    "node": ">=18.0.0"
   },
   "publishConfig": {
     "access": "public",