@module-federation/treeshake-server 0.0.1 → 2.0.1

package/dist/server.mjs

@@ -4,15 +4,18 @@ import node_path from "node:path";
 import pino from "pino";
 import { Hono } from "hono";
 import { cors } from "hono/cors";
+import node_os from "node:os";
 import { zValidator } from "@hono/zod-validator";
 import { nanoid } from "nanoid";
+import p_limit from "p-limit";
 import { z } from "zod";
 import { createHash } from "node:crypto";
-import node_os from "node:os";
+import promises from "node:fs/promises";
 import { spawn } from "node:child_process";
 import json_stable_stringify from "json-stable-stringify";
+import { timeout } from "hono/timeout";
 import { serve } from "@hono/node-server";
-const SERVER_VERSION = 'v0-011501';
+const SERVER_VERSION = 'v0-0205';
 const UPLOADED_DIR = '_shared-tree-shaking';
 const createLogger = (opts)=>pino({
         level: (null == opts ? void 0 : opts.level) ?? 'info',
@@ -71,7 +74,8 @@ class LocalObjectStore {
         _define_property(this, "rootDir", void 0);
         _define_property(this, "publicBaseUrl", void 0);
         this.rootDir = (null == opts ? void 0 : opts.rootDir) ?? node_path.join(process.cwd(), 'log', 'static');
-        const base = (null == opts ? void 0 : opts.publicBaseUrl) ?? '/';
+        const port = process.env.PORT || 3000;
+        const base = (null == opts ? void 0 : opts.publicBaseUrl) === '/' ? `http://localhost:${port}/` : (null == opts ? void 0 : opts.publicBaseUrl) ?? '/';
         this.publicBaseUrl = base.endsWith('/') ? base : `${base}/`;
     }
 }
@@ -238,14 +242,15 @@ function normalizeConfig(config) {
     return normalizedConfig;
 }
 function extractBuildConfig(config, type) {
-    const { shared, plugins, target, libraryType, usedExports } = config;
+    const { shared, plugins, target, libraryType, usedExports, hostName } = config;
     return {
         shared,
         plugins,
         target,
         libraryType,
         usedExports,
-        type
+        type,
+        hostName
     };
 }
 const UploadOptionsSchema = z.object({
@@ -374,11 +379,11 @@ const startPeriodicPrune = (intervalMs = DEFAULT_INTERVAL)=>{
         maybePrune();
     }, intervalMs);
 };
-const createUniqueTempDirByKey = (key)=>{
+const createUniqueTempDirByKey = async (key)=>{
     const base = node_path.join(node_os.tmpdir(), `re-shake-share-${key}`);
     let candidate = base;
     for(;;)try {
-        node_fs.mkdirSync(candidate, {
+        await promises.mkdir(candidate, {
             recursive: false
         });
         return candidate;
@@ -387,15 +392,22 @@ const createUniqueTempDirByKey = (key)=>{
         candidate = `${base}-${rand}`;
     }
 };
-const prepareProject = (config, excludeShared)=>{
+const prepareProject = async (config, excludeShared)=>{
     const key = createHash('sha256').update(JSON.stringify(config)).digest('hex');
-    const dir = createUniqueTempDirByKey(key);
-    const templateDir = node_path.join(__dirname, '..', 'template', 're-shake-share');
-    node_fs.cpSync(templateDir, dir, {
+    const dir = await createUniqueTempDirByKey(key);
+    const templateDir = node_path.join(__dirname, '.', 'template', 're-shake-share');
+    await promises.cp(templateDir, dir, {
         recursive: true
     });
     const pkgPath = node_path.join(dir, 'package.json');
-    const pkg = JSON.parse(node_fs.readFileSync(pkgPath, 'utf-8'));
+    const indexPath = node_path.join(dir, 'index.ts');
+    const rspackConfigPath = node_path.join(dir, 'rspack.config.ts');
+    const [pkgContent, indexContent, rspackConfigContent] = await Promise.all([
+        promises.readFile(pkgPath, 'utf-8'),
+        promises.readFile(indexPath, 'utf-8'),
+        promises.readFile(rspackConfigPath, 'utf-8')
+    ]);
+    const pkg = JSON.parse(pkgContent);
     const deps = {
         ...pkg.dependencies || {}
     };
@@ -436,28 +448,29 @@ const prepareProject = (config, excludeShared)=>{
     });
     pluginOptionStr += '\n]';
     pkg.dependencies = deps;
-    node_fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2));
+    const newPkgContent = JSON.stringify(pkg, null, 2);
     const sharedImportPlaceholder = "${SHARED_IMPORT}";
+    const newIndexContent = indexContent.replace(sharedImportPlaceholder, sharedImport);
     const pluginsPlaceholder = "${ PLUGINS }";
     const mfConfigPlaceholder = "${ MF_CONFIG }";
-    const indexPath = node_path.join(dir, 'index.ts');
-    const indexContent = node_fs.readFileSync(indexPath, 'utf-8');
-    node_fs.writeFileSync(indexPath, indexContent.replace(sharedImportPlaceholder, sharedImport));
-    const rspackConfigPath = node_path.join(dir, 'rspack.config.ts');
-    let cfg = node_fs.readFileSync(rspackConfigPath, 'utf-8');
+    let cfg = rspackConfigContent;
     cfg += pluginImportStr;
     cfg = cfg.replace(pluginsPlaceholder, pluginOptionStr);
     cfg = cfg.replace(mfConfigPlaceholder, JSON.stringify(mfConfig, null, 2));
-    node_fs.writeFileSync(rspackConfigPath, cfg);
+    await Promise.all([
+        promises.writeFile(pkgPath, newPkgContent),
+        promises.writeFile(indexPath, newIndexContent),
+        promises.writeFile(rspackConfigPath, cfg)
+    ]);
     return dir;
 };
 const installDependencies = async (cwd)=>{
     markInstallStart();
     try {
-        await runCommand('pnpm i', {
+        await runCommand("pnpm i --ignore-scripts --prefer-offline --reporter=silent ", {
             cwd,
             env: {
-                npm_config_registry: 'https://registry.npmjs.org/'
+                npm_config_registry: process.env.MF_NPM_REGISTRY || 'https://registry.npmjs.org/'
             }
         });
     } finally{
@@ -472,12 +485,13 @@ const buildProject = async (cwd, type)=>{
             cwd
         })));
 };
-const retrieveSharedFilepaths = (projectDir, type)=>{
+const retrieveSharedFilepaths = async (projectDir, type)=>{
     const sharedFilepaths = [];
-    const collectSharedFilepaths = (t)=>{
+    const collectSharedFilepaths = async (t)=>{
         const dir = 'full' === t ? 'full-shared' : 'dist';
         const distDir = node_path.join(projectDir, dir);
-        const stats = JSON.parse(node_fs.readFileSync(node_path.join(distDir, STATS_NAME), 'utf-8'));
+        const statsContent = await promises.readFile(node_path.join(distDir, STATS_NAME), 'utf-8');
+        const stats = JSON.parse(statsContent);
         stats.shared.forEach((s)=>{
             const { name, version, fallback, fallbackName } = s;
             if (fallback && fallbackName) {
@@ -495,22 +509,31 @@ const retrieveSharedFilepaths = (projectDir, type)=>{
             }
         });
     };
-    collectSharedFilepaths(type);
+    await collectSharedFilepaths(type);
     return sharedFilepaths;
 };
 const runBuild = async (normalizedConfig, excludeShared, type)=>{
-    const tmpDir = prepareProject(normalizedConfig, excludeShared);
+    const tStart = Date.now();
+    const tmpDir = await prepareProject(normalizedConfig, excludeShared);
+    const tPrepare = Date.now();
+    logger_logger.info(`prepareProject took ${tPrepare - tStart}ms`);
     await installDependencies(tmpDir);
+    const tInstall = Date.now();
+    logger_logger.info(`installDependencies took ${tInstall - tPrepare}ms`);
     await buildProject(tmpDir, type);
-    const sharedFilePaths = retrieveSharedFilepaths(tmpDir, type);
+    const tBuild = Date.now();
+    logger_logger.info(`buildProject took ${tBuild - tInstall}ms`);
+    const sharedFilePaths = await retrieveSharedFilepaths(tmpDir, type);
+    const tRetrieve = Date.now();
+    logger_logger.info(`retrieveSharedFilepaths took ${tRetrieve - tBuild}ms`);
     return {
         sharedFilePaths,
         dir: tmpDir
     };
 };
-function cleanUp(tmpDir) {
+async function cleanUp(tmpDir) {
     if (!tmpDir) return;
-    node_fs.rmSync(tmpDir, {
+    await promises.rm(tmpDir, {
         recursive: true,
         force: true
     });
@@ -608,7 +631,7 @@ async function uploadToCacheStore(sharedFilePaths, normalizedConfig, store) {
                 const jsonFilePath = filepath.replace(/\.js$/, '.json');
                 const jsonFile = JSON.stringify(res);
                 const jsonCdnUrl = cdnPath.replace(/\.js$/, '.json');
-                node_fs.writeFileSync(jsonFilePath, jsonFile);
+                await promises.writeFile(jsonFilePath, jsonFile);
                 await store.uploadFile(jsonFilePath, jsonCdnUrl);
             } catch (error) {
                 logger_logger.error(`Failed to upload ${name}@${version} json file: ${error}`);
@@ -626,25 +649,26 @@ const downloadToFile = async (url, destPath)=>{
     const res = await fetch(url);
     if (!res.ok) throw new Error(`Download failed: ${res.status} ${res.statusText} - ${url}`);
     const buf = Buffer.from(await res.arrayBuffer());
-    await node_fs.promises.mkdir(node_path.dirname(destPath), {
+    await promises.mkdir(node_path.dirname(destPath), {
         recursive: true
     });
-    await node_fs.promises.writeFile(destPath, buf);
+    await promises.writeFile(destPath, buf);
     return destPath;
 };
-async function uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher) {
-    const tmpDir = createUniqueTempDirByKey(`upload-project${Date.now().toString()}`);
+async function uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher, store) {
+    const tmpDir = await createUniqueTempDirByKey(`upload-project${Date.now().toString()}`);
     const uploaded = [];
     try {
         for (const item of uploadResults)try {
-            const config = normalizedConfig[normalizedKey(item.name, item.version)];
+            const sharedKey = normalizedKey(item.name, item.version);
+            const config = normalizedConfig[sharedKey];
             if (!config) {
                 logger_logger.error(`No config found for ${item.name}`);
                 continue;
             }
             const { uploadOptions } = config;
             if (!uploadOptions) throw new Error(`No uploadOptions found for ${item.name}`);
-            const filename = node_path.basename(new URL(item.cdnUrl).pathname);
+            const filename = node_path.basename(new URL(item.cdnUrl, 'http://dummy.com').pathname);
             const localPath = node_path.join(tmpDir, filename);
             const hash = createCacheHash({
                 ...config,
@@ -706,16 +730,20 @@ async function uploadProject(uploadResults, sharedFilePaths, normalizedConfig, p
                 version: s.version,
                 globalName: s.globalName,
                 cdnUrl,
-                type: s.type
+                type: s.type,
+                modules: s.modules,
+                canTreeShaking: s.canTreeShaking
             });
         } catch (error) {
             logger_logger.error(`Failed to upload ${s.name}@${s.version}: ${error}`);
         }
         return uploaded;
     } finally{
-        node_fs.rmSync(tmpDir, {
+        promises.rm(tmpDir, {
             recursive: true,
             force: true
+        }).catch((err)=>{
+            logger_logger.error(`Failed to cleanup dir ${tmpDir}: ${err}`);
         });
     }
 }
@@ -724,14 +752,14 @@ async function upload(sharedFilePaths, uploadResults, normalizedConfig, uploadOp
     if (!uploadOptions) {
         const hydrated = await Promise.all(uploadResults.map(async (item)=>{
             if ('full' !== item.type) return item;
-            const tmpDir = createUniqueTempDirByKey(`download-full-json${Date.now().toString()}`);
+            const tmpDir = await createUniqueTempDirByKey(`download-full-json${Date.now().toString()}`);
             const jsonPath = node_path.join(tmpDir, `${item.name}-${item.version}.json`);
             try {
                 const tJson0 = Date.now();
                 await downloadToFile(item.cdnUrl.replace('.js', '.json'), jsonPath);
                 const tJson = Date.now() - tJson0;
                 logger_logger.info(`Downloaded ${item.name}@${item.version} json in ${tJson}ms`);
-                const jsonContent = JSON.parse(node_fs.readFileSync(jsonPath, 'utf8'));
+                const jsonContent = JSON.parse(await promises.readFile(jsonPath, 'utf8'));
                 return {
                     ...item,
                     canTreeShaking: jsonContent.canTreeShaking,
@@ -744,7 +772,7 @@ async function upload(sharedFilePaths, uploadResults, normalizedConfig, uploadOp
                     canTreeShaking: item.canTreeShaking ?? true
                 };
             } finally{
-                node_fs.rmSync(tmpDir, {
+                await promises.rm(tmpDir, {
                     recursive: true,
                     force: true
                 });
@@ -756,9 +784,10 @@ async function upload(sharedFilePaths, uploadResults, normalizedConfig, uploadOp
         ];
     }
     if (!publisher) throw new Error('uploadOptions provided but no projectPublisher configured (configure the selected adapter to enable it or omit uploadOptions)');
-    const projectUploadResults = await uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher);
+    const projectUploadResults = await uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher, store);
     return projectUploadResults;
 }
+const buildLimit = p_limit(Math.max(1, node_os.cpus().length));
 const buildRoute = new Hono();
 buildRoute.post('/', zValidator('json', ConfigSchema), async (c)=>{
     const logger = c.get('logger');
@@ -771,16 +800,27 @@ buildRoute.post('/', zValidator('json', ConfigSchema), async (c)=>{
     const store = c.get('objectStore');
     const publisher = c.get('projectPublisher');
     try {
+        const t0 = Date.now();
         const { cacheItems, excludeShared, restConfig } = await retrieveCacheItems(normalizedConfig, 're-shake', store);
+        const tRetrieveCache = Date.now();
+        logger.info(`retrieveCacheItems took ${tRetrieveCache - t0}ms`);
         let sharedFilePaths = [];
         let dir;
         if (Object.keys(restConfig).length > 0) {
-            const buildResult = await runBuild(normalizedConfig, excludeShared, 're-shake');
+            const buildResult = await buildLimit(()=>runBuild(normalizedConfig, excludeShared, 're-shake'));
             sharedFilePaths = buildResult.sharedFilePaths;
             dir = buildResult.dir;
         }
+        const tBuild = Date.now();
+        logger.info(`runBuild took ${tBuild - tRetrieveCache}ms`);
         const uploadResults = await upload(sharedFilePaths, cacheItems, normalizedConfig, body.uploadOptions, store, publisher);
-        cleanUp(dir);
+        const tUpload = Date.now();
+        logger.info(`upload took ${tUpload - tBuild}ms`);
+        cleanUp(dir).catch((err)=>{
+            logger.error(`Failed to cleanup dir ${dir}: ${err}`);
+        });
+        const tCleanUp = Date.now();
+        logger.info(`cleanUp scheduled (non-blocking) took ${tCleanUp - tUpload}ms`);
         return c.json({
             jobId,
             status: 'success',
@@ -818,16 +858,27 @@ async function handleCheckTreeshake(c, body) {
     const store = c.get('objectStore');
     const publisher = c.get('projectPublisher');
     try {
+        const t0 = Date.now();
         const { cacheItems, excludeShared, restConfig } = await retrieveCacheItems(normalizedConfig, 'full', store);
+        const tRetrieveCache = Date.now();
+        logger.info(`retrieveCacheItems took ${tRetrieveCache - t0}ms`);
         let sharedFilePaths = [];
         let dir;
         if (Object.keys(restConfig).length > 0) {
-            const buildResult = await runBuild(normalizedConfig, excludeShared, 'full');
+            const buildResult = await buildLimit(()=>runBuild(normalizedConfig, excludeShared, 'full'));
             sharedFilePaths = buildResult.sharedFilePaths;
             dir = buildResult.dir;
         }
+        const tBuild = Date.now();
+        logger.info(`runBuild took ${tBuild - tRetrieveCache}ms`);
         const uploadResults = await upload(sharedFilePaths, cacheItems, normalizedConfig, body.uploadOptions, store, publisher);
-        cleanUp(dir);
+        const tUpload = Date.now();
+        logger.info(`upload took ${tUpload - tBuild}ms`);
+        cleanUp(dir).catch((err)=>{
+            logger.error(`Failed to cleanup dir ${dir}: ${err}`);
+        });
+        const tCleanUp = Date.now();
+        logger.info(`cleanUp scheduled (non-blocking) took ${tCleanUp - tUpload}ms`);
         return c.json({
             jobId,
             status: 'success',
@@ -893,6 +944,7 @@ function createApp(deps, opts) {
     }));
     app.use('*', loggerMiddleware);
     app.use('*', createDiMiddleware(deps));
+    app.use('*', timeout(60000));
     if (null == opts ? void 0 : null == (_opts_appExtensions = opts.appExtensions) ? void 0 : _opts_appExtensions.length) for (const extend of opts.appExtensions)extend(app);
     app.get('/tree-shaking-shared/healthz', (c)=>c.json({
             status: 'ok',
@@ -985,7 +1037,8 @@ function createServer(opts) {
     return serve({
         fetch: opts.app.fetch,
         port,
-        hostname
+        hostname,
+        overrideGlobalObjects: false
     });
 }
 const hasIndexHtml = (dir)=>Boolean(dir && node_fs.existsSync(node_path.join(dir, 'index.html')));
@@ -1046,11 +1099,8 @@ async function main() {
         port: resolved.port,
         hostname: resolved.hostname
     });
-    console.log(`Build service listening on http://${resolved.hostname}:${resolved.port}`);
-    if (frontendAdapters.length) {
-        const basePath = process.env.TREESHAKE_FRONTEND_BASE_PATH ?? '/tree-shaking';
-        console.log(`Treeshake UI available at http://${resolved.hostname}:${resolved.port}${basePath}`);
-    }
+    console.log('Build service listening.');
+    if (frontendAdapters.length) console.log('Treeshake UI available.');
 }
 main().catch((err)=>{
     console.error(err);

package/dist/services/buildService.d.ts

@@ -1,8 +1,8 @@
 import { type BuildType, type NormalizedConfig } from '../domain/build/normalize-config';
 import type { SharedFilePath } from './uploadService';
-export declare const createUniqueTempDirByKey: (key: string) => string;
+export declare const createUniqueTempDirByKey: (key: string) => Promise<string>;
 export declare const runBuild: (normalizedConfig: NormalizedConfig, excludeShared: Array<[string, string]>, type: BuildType) => Promise<{
     sharedFilePaths: SharedFilePath[];
     dir: string;
 }>;
-export declare function cleanUp(tmpDir?: string): void;
+export declare function cleanUp(tmpDir?: string): Promise<void>;

package/dist/services/uploadService.d.ts

@@ -32,5 +32,5 @@ export interface UploadOpts {
     cdnRegion: string;
     publicRoot: string;
 }
-export declare function uploadProject(uploadResults: UploadResult[], sharedFilePaths: SharedFilePath[], normalizedConfig: NormalizedConfig, publisher: ProjectPublisher): Promise<UploadResult[]>;
+export declare function uploadProject(uploadResults: UploadResult[], sharedFilePaths: SharedFilePath[], normalizedConfig: NormalizedConfig, publisher: ProjectPublisher, store: ObjectStore): Promise<UploadResult[]>;
 export declare function upload(sharedFilePaths: SharedFilePath[], uploadResults: UploadResult[], normalizedConfig: NormalizedConfig, uploadOptions: Config['uploadOptions'], store: ObjectStore, publisher?: ProjectPublisher): Promise<UploadResult[]>;

package/dist/template/re-shake-share/package.json

@@ -7,8 +7,8 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@rspack/cli": "^1.5.8",
-    "@rspack/core": "^1.5.8"
+    "@rspack/cli": "1.5.8",
+    "@rspack/core": "1.5.8"
   },
   "pnpm": {
     "overrides": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@module-federation/treeshake-server",
-  "version": "0.0.1",
+  "version": "2.0.1",
   "description": "Build service powered by Hono that installs dependencies, builds with Rspack, and uploads artifacts.",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -13,17 +13,10 @@
     }
   },
   "bin": {
-    "treeshake-server": "dist/server.js"
-  },
-  "scripts": {
-    "build": "pnpm -C ../treeshake-frontend build && rslib build && node scripts/copy-frontend.js",
-    "lint": "biome lint src scripts",
-    "prestart": "npm run build",
-    "dev": "node scripts/dev.js",
-    "start": "node dist/server.js",
-    "test": "rstest"
+    "treeshake-server": "bin/treeshake-server.js"
   },
   "files": [
+    "bin/",
     "dist/",
     "README.md"
   ],
@@ -34,11 +27,12 @@
     "@hono/node-server": "1.19.5",
     "@hono/zod-validator": "0.7.4",
     "dotenv": "16.4.5",
-    "hono": "4.10.2",
+    "hono": "4.11.7",
     "json-stable-stringify": "1.3.0",
     "nanoid": "5.1.6",
+    "p-limit": "^7.2.0",
     "pino": "10.1.0",
-    "undici": "5.29.0",
+    "undici": "6.23.0",
     "zod": "4.1.12"
   },
   "devDependencies": {
@@ -57,5 +51,18 @@
   },
   "publishConfig": {
     "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/module-federation/core.git",
+    "directory": "packages/treeshake-server"
+  },
+  "scripts": {
+    "build": "pnpm -C ../treeshake-frontend build && rslib build && node scripts/copy-frontend.js",
+    "lint": "biome lint src scripts",
+    "prestart": "npm run build",
+    "dev": "node scripts/dev.js",
+    "start": "node dist/server.js",
+    "test": "node ../../scripts/ensure-playwright.js && rstest"
   }
-}
+}