@module-federation/treeshake-server 0.0.1 → 2.0.1

package/dist/index.mjs

@@ -1,15 +1,18 @@
 import { Hono } from "hono";
 import { cors } from "hono/cors";
 import pino from "pino";
+import node_os from "node:os";
 import { zValidator } from "@hono/zod-validator";
 import { nanoid } from "nanoid";
+import p_limit from "p-limit";
 import { z } from "zod";
 import { createHash } from "node:crypto";
-import node_fs from "node:fs";
-import node_os from "node:os";
+import promises from "node:fs/promises";
 import node_path from "node:path";
 import { spawn } from "node:child_process";
 import json_stable_stringify from "json-stable-stringify";
+import node_fs from "node:fs";
+import { timeout } from "hono/timeout";
 import { serve } from "@hono/node-server";
 function createDiMiddleware(deps) {
     return async (c, next)=>{
@@ -75,14 +78,15 @@ function normalizeConfig(config) {
     return normalizedConfig;
 }
 function extractBuildConfig(config, type) {
-    const { shared, plugins, target, libraryType, usedExports } = config;
+    const { shared, plugins, target, libraryType, usedExports, hostName } = config;
     return {
         shared,
         plugins,
         target,
         libraryType,
         usedExports,
-        type
+        type,
+        hostName
     };
 }
 const UploadOptionsSchema = z.object({
@@ -211,11 +215,11 @@ const startPeriodicPrune = (intervalMs = DEFAULT_INTERVAL)=>{
         maybePrune();
     }, intervalMs);
 };
-const createUniqueTempDirByKey = (key)=>{
+const createUniqueTempDirByKey = async (key)=>{
     const base = node_path.join(node_os.tmpdir(), `re-shake-share-${key}`);
     let candidate = base;
     for(;;)try {
-        node_fs.mkdirSync(candidate, {
+        await promises.mkdir(candidate, {
             recursive: false
         });
         return candidate;
@@ -224,15 +228,22 @@ const createUniqueTempDirByKey = (key)=>{
         candidate = `${base}-${rand}`;
     }
 };
-const prepareProject = (config, excludeShared)=>{
+const prepareProject = async (config, excludeShared)=>{
     const key = createHash('sha256').update(JSON.stringify(config)).digest('hex');
-    const dir = createUniqueTempDirByKey(key);
-    const templateDir = node_path.join(__dirname, '..', 'template', 're-shake-share');
-    node_fs.cpSync(templateDir, dir, {
+    const dir = await createUniqueTempDirByKey(key);
+    const templateDir = node_path.join(__dirname, '.', 'template', 're-shake-share');
+    await promises.cp(templateDir, dir, {
         recursive: true
     });
     const pkgPath = node_path.join(dir, 'package.json');
-    const pkg = JSON.parse(node_fs.readFileSync(pkgPath, 'utf-8'));
+    const indexPath = node_path.join(dir, 'index.ts');
+    const rspackConfigPath = node_path.join(dir, 'rspack.config.ts');
+    const [pkgContent, indexContent, rspackConfigContent] = await Promise.all([
+        promises.readFile(pkgPath, 'utf-8'),
+        promises.readFile(indexPath, 'utf-8'),
+        promises.readFile(rspackConfigPath, 'utf-8')
+    ]);
+    const pkg = JSON.parse(pkgContent);
     const deps = {
         ...pkg.dependencies || {}
     };
@@ -273,28 +284,29 @@ const prepareProject = (config, excludeShared)=>{
     });
     pluginOptionStr += '\n]';
     pkg.dependencies = deps;
-    node_fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2));
+    const newPkgContent = JSON.stringify(pkg, null, 2);
     const sharedImportPlaceholder = "${SHARED_IMPORT}";
+    const newIndexContent = indexContent.replace(sharedImportPlaceholder, sharedImport);
     const pluginsPlaceholder = "${ PLUGINS }";
     const mfConfigPlaceholder = "${ MF_CONFIG }";
-    const indexPath = node_path.join(dir, 'index.ts');
-    const indexContent = node_fs.readFileSync(indexPath, 'utf-8');
-    node_fs.writeFileSync(indexPath, indexContent.replace(sharedImportPlaceholder, sharedImport));
-    const rspackConfigPath = node_path.join(dir, 'rspack.config.ts');
-    let cfg = node_fs.readFileSync(rspackConfigPath, 'utf-8');
+    let cfg = rspackConfigContent;
     cfg += pluginImportStr;
     cfg = cfg.replace(pluginsPlaceholder, pluginOptionStr);
     cfg = cfg.replace(mfConfigPlaceholder, JSON.stringify(mfConfig, null, 2));
-    node_fs.writeFileSync(rspackConfigPath, cfg);
+    await Promise.all([
+        promises.writeFile(pkgPath, newPkgContent),
+        promises.writeFile(indexPath, newIndexContent),
+        promises.writeFile(rspackConfigPath, cfg)
+    ]);
     return dir;
 };
 const installDependencies = async (cwd)=>{
     markInstallStart();
     try {
-        await runCommand('pnpm i', {
+        await runCommand("pnpm i --ignore-scripts --prefer-offline --reporter=silent ", {
             cwd,
             env: {
-                npm_config_registry: 'https://registry.npmjs.org/'
+                npm_config_registry: process.env.MF_NPM_REGISTRY || 'https://registry.npmjs.org/'
             }
         });
     } finally{
@@ -309,12 +321,13 @@ const buildProject = async (cwd, type)=>{
             cwd
         })));
 };
-const retrieveSharedFilepaths = (projectDir, type)=>{
+const retrieveSharedFilepaths = async (projectDir, type)=>{
     const sharedFilepaths = [];
-    const collectSharedFilepaths = (t)=>{
+    const collectSharedFilepaths = async (t)=>{
         const dir = 'full' === t ? 'full-shared' : 'dist';
         const distDir = node_path.join(projectDir, dir);
-        const stats = JSON.parse(node_fs.readFileSync(node_path.join(distDir, STATS_NAME), 'utf-8'));
+        const statsContent = await promises.readFile(node_path.join(distDir, STATS_NAME), 'utf-8');
+        const stats = JSON.parse(statsContent);
         stats.shared.forEach((s)=>{
             const { name, version, fallback, fallbackName } = s;
             if (fallback && fallbackName) {
@@ -332,22 +345,31 @@ const retrieveSharedFilepaths = (projectDir, type)=>{
             }
         });
     };
-    collectSharedFilepaths(type);
+    await collectSharedFilepaths(type);
     return sharedFilepaths;
 };
 const runBuild = async (normalizedConfig, excludeShared, type)=>{
-    const tmpDir = prepareProject(normalizedConfig, excludeShared);
+    const tStart = Date.now();
+    const tmpDir = await prepareProject(normalizedConfig, excludeShared);
+    const tPrepare = Date.now();
+    logger_logger.info(`prepareProject took ${tPrepare - tStart}ms`);
     await installDependencies(tmpDir);
+    const tInstall = Date.now();
+    logger_logger.info(`installDependencies took ${tInstall - tPrepare}ms`);
     await buildProject(tmpDir, type);
-    const sharedFilePaths = retrieveSharedFilepaths(tmpDir, type);
+    const tBuild = Date.now();
+    logger_logger.info(`buildProject took ${tBuild - tInstall}ms`);
+    const sharedFilePaths = await retrieveSharedFilepaths(tmpDir, type);
+    const tRetrieve = Date.now();
+    logger_logger.info(`retrieveSharedFilepaths took ${tRetrieve - tBuild}ms`);
     return {
         sharedFilePaths,
         dir: tmpDir
     };
 };
-function cleanUp(tmpDir) {
+async function cleanUp(tmpDir) {
     if (!tmpDir) return;
-    node_fs.rmSync(tmpDir, {
+    await promises.rm(tmpDir, {
         recursive: true,
         force: true
     });
@@ -359,7 +381,7 @@ const encodeName = function(name, prefix = '', withExt = false) {
 function retrieveGlobalName(mfName, sharedName, version) {
     return encodeName(`${mfName}_${sharedName}_${version}`);
 }
-const SERVER_VERSION = 'v0-011501';
+const SERVER_VERSION = 'v0-0205';
 const UPLOADED_DIR = '_shared-tree-shaking';
 function createCacheHash(config, type) {
     const relevant = extractBuildConfig({
@@ -447,7 +469,7 @@ async function uploadToCacheStore(sharedFilePaths, normalizedConfig, store) {
                 const jsonFilePath = filepath.replace(/\.js$/, '.json');
                 const jsonFile = JSON.stringify(res);
                 const jsonCdnUrl = cdnPath.replace(/\.js$/, '.json');
-                node_fs.writeFileSync(jsonFilePath, jsonFile);
+                await promises.writeFile(jsonFilePath, jsonFile);
                 await store.uploadFile(jsonFilePath, jsonCdnUrl);
             } catch (error) {
                 logger_logger.error(`Failed to upload ${name}@${version} json file: ${error}`);
@@ -465,25 +487,26 @@ const downloadToFile = async (url, destPath)=>{
     const res = await fetch(url);
     if (!res.ok) throw new Error(`Download failed: ${res.status} ${res.statusText} - ${url}`);
     const buf = Buffer.from(await res.arrayBuffer());
-    await node_fs.promises.mkdir(node_path.dirname(destPath), {
+    await promises.mkdir(node_path.dirname(destPath), {
         recursive: true
     });
-    await node_fs.promises.writeFile(destPath, buf);
+    await promises.writeFile(destPath, buf);
     return destPath;
 };
-async function uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher) {
-    const tmpDir = createUniqueTempDirByKey(`upload-project${Date.now().toString()}`);
+async function uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher, store) {
+    const tmpDir = await createUniqueTempDirByKey(`upload-project${Date.now().toString()}`);
     const uploaded = [];
     try {
         for (const item of uploadResults)try {
-            const config = normalizedConfig[normalizedKey(item.name, item.version)];
+            const sharedKey = normalizedKey(item.name, item.version);
+            const config = normalizedConfig[sharedKey];
             if (!config) {
                 logger_logger.error(`No config found for ${item.name}`);
                 continue;
             }
             const { uploadOptions } = config;
             if (!uploadOptions) throw new Error(`No uploadOptions found for ${item.name}`);
-            const filename = node_path.basename(new URL(item.cdnUrl).pathname);
+            const filename = node_path.basename(new URL(item.cdnUrl, 'http://dummy.com').pathname);
             const localPath = node_path.join(tmpDir, filename);
             const hash = createCacheHash({
                 ...config,
@@ -545,16 +568,20 @@ async function uploadProject(uploadResults, sharedFilePaths, normalizedConfig, p
                 version: s.version,
                 globalName: s.globalName,
                 cdnUrl,
-                type: s.type
+                type: s.type,
+                modules: s.modules,
+                canTreeShaking: s.canTreeShaking
             });
         } catch (error) {
             logger_logger.error(`Failed to upload ${s.name}@${s.version}: ${error}`);
         }
         return uploaded;
     } finally{
-        node_fs.rmSync(tmpDir, {
+        promises.rm(tmpDir, {
             recursive: true,
             force: true
+        }).catch((err)=>{
+            logger_logger.error(`Failed to cleanup dir ${tmpDir}: ${err}`);
         });
     }
 }
@@ -563,14 +590,14 @@ async function upload(sharedFilePaths, uploadResults, normalizedConfig, uploadOp
     if (!uploadOptions) {
         const hydrated = await Promise.all(uploadResults.map(async (item)=>{
             if ('full' !== item.type) return item;
-            const tmpDir = createUniqueTempDirByKey(`download-full-json${Date.now().toString()}`);
+            const tmpDir = await createUniqueTempDirByKey(`download-full-json${Date.now().toString()}`);
             const jsonPath = node_path.join(tmpDir, `${item.name}-${item.version}.json`);
             try {
                 const tJson0 = Date.now();
                 await downloadToFile(item.cdnUrl.replace('.js', '.json'), jsonPath);
                 const tJson = Date.now() - tJson0;
                 logger_logger.info(`Downloaded ${item.name}@${item.version} json in ${tJson}ms`);
-                const jsonContent = JSON.parse(node_fs.readFileSync(jsonPath, 'utf8'));
+                const jsonContent = JSON.parse(await promises.readFile(jsonPath, 'utf8'));
                 return {
                     ...item,
                     canTreeShaking: jsonContent.canTreeShaking,
@@ -583,7 +610,7 @@ async function upload(sharedFilePaths, uploadResults, normalizedConfig, uploadOp
                     canTreeShaking: item.canTreeShaking ?? true
                 };
             } finally{
-                node_fs.rmSync(tmpDir, {
+                await promises.rm(tmpDir, {
                     recursive: true,
                     force: true
                 });
@@ -595,9 +622,10 @@ async function upload(sharedFilePaths, uploadResults, normalizedConfig, uploadOp
         ];
     }
     if (!publisher) throw new Error('uploadOptions provided but no projectPublisher configured (configure the selected adapter to enable it or omit uploadOptions)');
-    const projectUploadResults = await uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher);
+    const projectUploadResults = await uploadProject(uploadResults, sharedFilePaths, normalizedConfig, publisher, store);
     return projectUploadResults;
 }
+const buildLimit = p_limit(Math.max(1, node_os.cpus().length));
 const buildRoute = new Hono();
 buildRoute.post('/', zValidator('json', ConfigSchema), async (c)=>{
     const logger = c.get('logger');
@@ -610,16 +638,27 @@ buildRoute.post('/', zValidator('json', ConfigSchema), async (c)=>{
     const store = c.get('objectStore');
     const publisher = c.get('projectPublisher');
     try {
+        const t0 = Date.now();
         const { cacheItems, excludeShared, restConfig } = await retrieveCacheItems(normalizedConfig, 're-shake', store);
+        const tRetrieveCache = Date.now();
+        logger.info(`retrieveCacheItems took ${tRetrieveCache - t0}ms`);
         let sharedFilePaths = [];
         let dir;
         if (Object.keys(restConfig).length > 0) {
-            const buildResult = await runBuild(normalizedConfig, excludeShared, 're-shake');
+            const buildResult = await buildLimit(()=>runBuild(normalizedConfig, excludeShared, 're-shake'));
             sharedFilePaths = buildResult.sharedFilePaths;
             dir = buildResult.dir;
         }
+        const tBuild = Date.now();
+        logger.info(`runBuild took ${tBuild - tRetrieveCache}ms`);
         const uploadResults = await upload(sharedFilePaths, cacheItems, normalizedConfig, body.uploadOptions, store, publisher);
-        cleanUp(dir);
+        const tUpload = Date.now();
+        logger.info(`upload took ${tUpload - tBuild}ms`);
+        cleanUp(dir).catch((err)=>{
+            logger.error(`Failed to cleanup dir ${dir}: ${err}`);
+        });
+        const tCleanUp = Date.now();
+        logger.info(`cleanUp scheduled (non-blocking) took ${tCleanUp - tUpload}ms`);
         return c.json({
             jobId,
             status: 'success',
@@ -657,16 +696,27 @@ async function handleCheckTreeshake(c, body) {
     const store = c.get('objectStore');
     const publisher = c.get('projectPublisher');
     try {
+        const t0 = Date.now();
         const { cacheItems, excludeShared, restConfig } = await retrieveCacheItems(normalizedConfig, 'full', store);
+        const tRetrieveCache = Date.now();
+        logger.info(`retrieveCacheItems took ${tRetrieveCache - t0}ms`);
         let sharedFilePaths = [];
         let dir;
         if (Object.keys(restConfig).length > 0) {
-            const buildResult = await runBuild(normalizedConfig, excludeShared, 'full');
+            const buildResult = await buildLimit(()=>runBuild(normalizedConfig, excludeShared, 'full'));
             sharedFilePaths = buildResult.sharedFilePaths;
             dir = buildResult.dir;
         }
+        const tBuild = Date.now();
+        logger.info(`runBuild took ${tBuild - tRetrieveCache}ms`);
         const uploadResults = await upload(sharedFilePaths, cacheItems, normalizedConfig, body.uploadOptions, store, publisher);
-        cleanUp(dir);
+        const tUpload = Date.now();
+        logger.info(`upload took ${tUpload - tBuild}ms`);
+        cleanUp(dir).catch((err)=>{
+            logger.error(`Failed to cleanup dir ${dir}: ${err}`);
+        });
+        const tCleanUp = Date.now();
+        logger.info(`cleanUp scheduled (non-blocking) took ${tCleanUp - tUpload}ms`);
         return c.json({
             jobId,
             status: 'success',
@@ -767,6 +817,7 @@ function createApp(deps, opts) {
     }));
     app.use('*', loggerMiddleware);
     app.use('*', createDiMiddleware(deps));
+    app.use('*', timeout(60000));
     if (null == opts ? void 0 : null == (_opts_appExtensions = opts.appExtensions) ? void 0 : _opts_appExtensions.length) for (const extend of opts.appExtensions)extend(app);
     app.get('/tree-shaking-shared/healthz', (c)=>c.json({
             status: 'ok',
@@ -786,7 +837,8 @@ function createServer(opts) {
     return serve({
         fetch: opts.app.fetch,
         port,
-        hostname
+        hostname,
+        overrideGlobalObjects: false
     });
 }
 async function createAdapterDeps(params) {
@@ -832,7 +884,8 @@ class LocalObjectStore {
         localObjectStore_define_property(this, "rootDir", void 0);
         localObjectStore_define_property(this, "publicBaseUrl", void 0);
         this.rootDir = (null == opts ? void 0 : opts.rootDir) ?? node_path.join(process.cwd(), 'log', 'static');
-        const base = (null == opts ? void 0 : opts.publicBaseUrl) ?? '/';
+        const port = process.env.PORT || 3000;
+        const base = (null == opts ? void 0 : opts.publicBaseUrl) === '/' ? `http://localhost:${port}/` : (null == opts ? void 0 : opts.publicBaseUrl) ?? '/';
         this.publicBaseUrl = base.endsWith('/') ? base : `${base}/`;
     }
 }