@module-federation/rspress-plugin 0.0.0-next-20250723062542 → 0.0.0-next-20250724024905

package/dist/esm/index.js

@@ -8,6 +8,7 @@ import external_node_path_default from "node:path";
 import { load } from "cheerio";
 import { htmlToText } from "html-to-text";
 import { groupBy } from "lodash-es";
+import external_turndown_default from "turndown";
 import external_fs_default from "fs";
 const logger = createLogger('[ Module Federation Rspress Plugin ]');
 const src_logger = logger;
@@ -19,6 +20,10 @@ function findSearchIndexPaths(outputDir) {
     const searchIndexFiles = files.filter((file)=>file.startsWith(SEARCH_INDEX_NAME) && file.endsWith('.json') && external_fs_default.statSync(external_path_default.join(staticDir, file)).isFile());
     if (searchIndexFiles) return searchIndexFiles.map((searchIndexFile)=>external_path_default.join(staticDir, searchIndexFile));
 }
+function findMarkdownFilePath(outputDir, defaultLang, route) {
+    const filepath = external_path_default.join(outputDir, route.relativePath.replace(new RegExp(`^${defaultLang}/`), ''));
+    return filepath.replace(external_path_default.extname(filepath), '.md');
+}
 function generateTocFromHtml(html) {
     const $ = load(html);
     const headings = $('h1, h2, h3, h4, h5, h6');
@@ -42,8 +47,22 @@ function generateTocFromHtml(html) {
     };
 }
 const replaceHtmlExt = (filepath)=>filepath.replace(external_node_path_default.extname(filepath), '.html');
-async function extractPageDataFromHtml(routes, options) {
-    return Promise.all(routes.map(async (route)=>{
+const getRouteId = (route)=>route.absolutePath;
+async function getRouteHtmlMap(routesMap, options) {
+    const routeHtmlMap = {};
+    await Promise.all(Object.entries(routesMap).map(async ([routeId, route])=>{
+        const { outputDir, defaultLang } = options;
+        const htmlPath = replaceHtmlExt(external_node_path_default.join(outputDir, route.lang === defaultLang ? route.relativePath.replace(route.lang, '') : route.relativePath));
+        const content = await promises_default.readFile(htmlPath, 'utf-8');
+        routeHtmlMap[routeId] = {
+            filepath: htmlPath,
+            content
+        };
+    }));
+    return routeHtmlMap;
+}
+async function extractPageDataFromHtml(routesMap, routeHtmlMap, options) {
+    return Promise.all(Object.entries(routesMap).map(async ([routeId, route])=>{
         const { domain, searchCodeBlocks, outputDir, defaultLang } = options;
         const defaultIndexInfo = {
             title: '',
@@ -59,8 +78,8 @@ async function extractPageDataFromHtml(routes, options) {
             _filepath: route.absolutePath,
             _relativePath: ''
         };
-        const htmlPath = replaceHtmlExt(external_node_path_default.join(outputDir, route.lang === defaultLang ? route.relativePath.replace(route.lang, '') : route.relativePath));
-        const html = await promises_default.readFile(htmlPath, 'utf-8');
+        const html = routeHtmlMap[routeId].content;
+        if (!html) throw new Error(`html not found for route "${routeId}"`);
         let { toc: rawToc, title } = generateTocFromHtml(html);
         let content = html;
         content = htmlToText(html, {
@@ -129,14 +148,14 @@ function deletePrivateField(obj) {
     for(const key in newObj)if (key.startsWith('_')) delete newObj[key];
     return newObj;
 }
-async function rebuildSearchIndexByHtml(routes, options) {
+async function rebuildSearchIndexByHtml(routesMap, routeHtmlMap, options) {
     const { versioned, outputDir } = options;
     const searchFilePaths = findSearchIndexPaths(outputDir);
     if (!searchFilePaths) {
         src_logger.error('Cannot find search index files!');
         process.exit(1);
     }
-    const pages = await extractPageDataFromHtml(routes, options);
+    const pages = await extractPageDataFromHtml(routesMap, routeHtmlMap, options);
     const groupedPages = groupBy(pages, (page)=>{
         if (page.frontmatter?.pageType === 'home') return 'noindex';
         const version = versioned ? page.version : '';
@@ -157,6 +176,20 @@ async function rebuildSearchIndexByHtml(routes, options) {
         await promises_default.writeFile(searchFilePath, stringifiedIndex);
     }));
 }
+async function rebuildLLMsMDFilesByHtml(routesMap, routeHtmlMap, options) {
+    const { outputDir, defaultLang } = options;
+    const turndownService = new external_turndown_default();
+    await Promise.all(Object.entries(routeHtmlMap).map(async ([routeId, routeHtml])=>{
+        try {
+            const { content } = routeHtml;
+            const markdown = turndownService.turndown(content);
+            const mdFilePath = findMarkdownFilePath(outputDir, defaultLang, routesMap[routeId]);
+            await promises_default.writeFile(mdFilePath, markdown);
+        } catch (e) {
+            src_logger.error(`rebuildLLMsMDFilesByHtml ${routeId} error:${e}`);
+        }
+    }));
+}
 const isDev = ()=>'development' === process.env.NODE_ENV;
 function replaceEntryWithBootstrapEntry(bundlerConfig) {
     const { entry } = bundlerConfig;
@@ -201,7 +234,7 @@ function replaceEntryWithBootstrapEntry(bundlerConfig) {
     bundlerConfig.entry = replaceWithAsyncEntry(entry, bundlerConfig.name || 'index');
 }
 function plugin_pluginModuleFederation(mfConfig, rspressOptions) {
-    const { autoShared = true, rebuildSearchIndex = true } = rspressOptions || {};
+    const { autoShared = true, rebuildSearchIndex = true, rebuildLLMsMDFiles = true } = rspressOptions || {};
     if (autoShared) mfConfig.shared = {
         react: {
             singleton: true,
@@ -231,7 +264,7 @@ function plugin_pluginModuleFederation(mfConfig, rspressOptions) {
     };
     let enableSSG = false;
     let outputDir = '';
-    let routes = [];
+    let routesMap = {};
     return {
         name: 'plugin-module-federation',
         async config (config) {
@@ -268,12 +301,15 @@ function plugin_pluginModuleFederation(mfConfig, rspressOptions) {
             }
         },
         routeGenerated (routeMetaArr) {
-            routes = routeMetaArr;
+            routesMap = routeMetaArr.reduce((prev, cur)=>{
+                prev[getRouteId(cur)] = cur;
+                return prev;
+            }, {});
         },
         async afterBuild (config) {
-            if (!mfConfig.remotes || isDev() || !rebuildSearchIndex) return;
+            if (!mfConfig.remotes || isDev() || !rebuildSearchIndex || !rebuildLLMsMDFiles) return;
             if (!enableSSG) {
-                src_logger.error('rebuildSearchIndex is only supported for ssg');
+                src_logger.error('rebuildSearchIndex and rebuildLLMsMDFiles are only supported for ssg');
                 process.exit(1);
             }
             const searchConfig = config?.search || {};
@@ -281,7 +317,7 @@ function plugin_pluginModuleFederation(mfConfig, rspressOptions) {
             const domain = searchConfig?.mode === 'remote' ? searchConfig.domain ?? '' : '';
             const versioned = searchConfig && 'remote' !== searchConfig.mode && searchConfig.versioned;
             const searchCodeBlocks = 'codeBlocks' in searchConfig ? Boolean(searchConfig.codeBlocks) : true;
-            await rebuildSearchIndexByHtml(routes, {
+            const routeHtmlMap = await getRouteHtmlMap(routesMap, {
                 outputDir,
                 versioned,
                 replaceRules,
@@ -289,7 +325,22 @@ function plugin_pluginModuleFederation(mfConfig, rspressOptions) {
                 searchCodeBlocks,
                 defaultLang: config.lang || 'en'
             });
-            src_logger.info('rebuildSearchIndex success!');
+            const rebuildOptions = {
+                outputDir,
+                versioned,
+                replaceRules,
+                domain,
+                searchCodeBlocks,
+                defaultLang: config.lang || 'en'
+            };
+            if (rebuildSearchIndex) {
+                await rebuildSearchIndexByHtml(routesMap, routeHtmlMap, rebuildOptions);
+                src_logger.info('rebuildSearchIndex success!');
+            }
+            if (rebuildLLMsMDFiles && config.plugins?.find((p)=>'@rspress/plugin-llms' === p.name)) {
+                await rebuildLLMsMDFilesByHtml(routesMap, routeHtmlMap, rebuildOptions);
+                src_logger.info('rebuildLLMsMDFiles success!');
+            }
         }
     };
 }

package/dist/esm/packages/rspress-plugin/src/findPath.d.ts

@@ -0,0 +1,3 @@
+import type { RouteMeta } from '@rspress/shared';
+export declare function findSearchIndexPaths(outputDir: string): string[] | undefined;
+export declare function findMarkdownFilePath(outputDir: string, defaultLang: string, route: RouteMeta): string;

package/dist/esm/packages/rspress-plugin/src/plugin.d.ts

@@ -3,6 +3,7 @@ import type { RspressPlugin } from '@rspress/shared';
 type RspressPluginOptions = {
     autoShared?: boolean;
     rebuildSearchIndex?: boolean;
+    rebuildLLMsMDFiles?: boolean;
 };
 export declare function pluginModuleFederation(mfConfig: moduleFederationPlugin.ModuleFederationPluginOptions, rspressOptions?: RspressPluginOptions): RspressPlugin;
 export { createModuleFederationConfig } from '@module-federation/sdk';

package/dist/esm/packages/rspress-plugin/src/rebuild.d.ts

@@ -0,0 +1,20 @@
+import type { RouteMeta, ReplaceRule } from '@rspress/shared';
+interface RouteHtmlMap {
+    [routeId: string]: {
+        filepath: string;
+        content: string;
+    };
+}
+export type RebuildOptions = {
+    domain: string;
+    searchCodeBlocks: boolean;
+    replaceRules: ReplaceRule[];
+    versioned?: boolean;
+    outputDir: string;
+    defaultLang: string;
+};
+export declare const getRouteId: (route: RouteMeta) => string;
+export declare function getRouteHtmlMap(routesMap: Record<string, RouteMeta>, options: RebuildOptions): Promise<RouteHtmlMap>;
+export declare function rebuildSearchIndexByHtml(routesMap: Record<string, RouteMeta>, routeHtmlMap: RouteHtmlMap, options: RebuildOptions): Promise<void>;
+export declare function rebuildLLMsMDFilesByHtml(routesMap: Record<string, RouteMeta>, routeHtmlMap: RouteHtmlMap, options: RebuildOptions): Promise<void>;
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@module-federation/rspress-plugin",
-  "version": "0.0.0-next-20250723062542",
+  "version": "0.0.0-next-20250724024905",
   "type": "module",
   "description": "Module Federation plugin for Rspress",
   "keywords": [
@@ -34,17 +34,19 @@
     "@rspress/shared": "2.0.0-beta.16",
     "@types/html-to-text": "^9.0.4",
     "@types/lodash-es": "^4.17.12",
-    "@types/react": "^18.3.11"
+    "@types/react": "^18.3.11",
+    "@types/turndown": "5.0.5 "
   },
   "dependencies": {
     "cheerio": "1.0.0-rc.12",
     "fs-extra": "11.3.0",
     "html-to-text": "^9.0.5",
     "lodash-es": "^4.17.21",
-    "@module-federation/enhanced": "0.0.0-next-20250723062542",
-    "@module-federation/sdk": "0.0.0-next-20250723062542",
-    "@module-federation/rsbuild-plugin": "0.0.0-next-20250723062542",
-    "@module-federation/error-codes": "0.0.0-next-20250723062542"
+    "turndown": "7.2.0",
+    "@module-federation/sdk": "0.0.0-next-20250724024905",
+    "@module-federation/enhanced": "0.0.0-next-20250724024905",
+    "@module-federation/rsbuild-plugin": "0.0.0-next-20250724024905",
+    "@module-federation/error-codes": "0.0.0-next-20250724024905"
   },
   "scripts": {
     "build": "rslib build",

package/dist/esm/packages/rspress-plugin/src/findSearchIndexPath.d.ts

@@ -1 +0,0 @@
-export declare function findSearchIndexPaths(outputDir: string): string[] | undefined;

package/dist/esm/packages/rspress-plugin/src/rebuildSearchIndexByHtml.d.ts

@@ -1,10 +0,0 @@
-import type { RouteMeta, ReplaceRule } from '@rspress/shared';
-export type RebuildSearchIndexByHtmlOptions = {
-    domain: string;
-    searchCodeBlocks: boolean;
-    replaceRules: ReplaceRule[];
-    versioned?: boolean;
-    outputDir: string;
-    defaultLang: string;
-};
-export declare function rebuildSearchIndexByHtml(routes: RouteMeta[], options: RebuildSearchIndexByHtmlOptions): Promise<void>;