@epublishing/grunt-epublishing 1.1.4 → 1.2.0

package/lib/cli.js

@@ -1,24 +1,23 @@
 /**
  * CLI Implementation
- * 
+ *
  * Main entry point for the build tools.
  * Orchestrates webpack, sass, concat/minify tasks.
+ *
+ * Heavy modules (webpack, sass-embedded, terser) are lazy-loaded inside
+ * their task functions so that subprocess workers only pay for what they use.
  */
 
 'use strict';
 
+const os = require('os');
 const fs = require('fs');
 const path = require('path');
 const { spawn } = require('child_process');
 const { program } = require('commander');
 const chalk = require('chalk');
 const ora = require('ora');
-const Resolver = require('@epublishing/jade-resolver');
 const { loadConfig } = require('./config-loader');
-const { configureWebpack, runWebpack, watchWebpack, splitEntries } = require('./webpack.config');
-const { compileSass } = require('./sass-compiler');
-const { runConcatMinify } = require('./concat-minify');
-const { writeTsConfigs, cleanTsConfigs } = require('./tsconfig-gen');
 
 // Package info for banner
 const pkg = require('../package.json');
@@ -28,7 +27,7 @@ const pkg = require('../package.json');
  */
 function printBanner(options) {
   if (options.noBanner) return;
-  
+
   console.log(chalk.cyan(`
 ╔═══════════════════════════════════════════╗
 ║                                           ║
@@ -43,7 +42,7 @@ function printBanner(options) {
  */
 function printOptions(options) {
   if (options.noBanner) return;
-  
+
   const flags = [];
   if (options.watch) flags.push('watch');
   if (options.noMinify) flags.push('no-minify');
@@ -51,7 +50,8 @@ function printOptions(options) {
   if (options.lint) flags.push('lint');
   if (options.verbose) flags.push('verbose');
   if (options.parallel) flags.push('parallel');
-  
+  if (options.isolateTasks) flags.push('isolate-tasks');
+
   if (flags.length > 0) {
     console.log(chalk.gray(`  Options: ${flags.join(', ')}\n`));
   }
@@ -64,6 +64,7 @@ function printOptions(options) {
  * @param {Object} options - Build options
  */
 async function runNpmInstall(config, options) {
+  const Resolver = require('@epublishing/jade-resolver');
   const paths = config.paths || {};
   // Build paths object with ONLY jade and jade child gem directories (absolute paths)
   const gemPaths = {};
@@ -125,35 +126,36 @@ async function runNpmInstall(config, options) {
  * @param {Object} options - Build options
  */
 async function runWebpackBuild(config, options) {
+  const { configureWebpack, runWebpack, splitEntries } = require('./webpack.config');
   const spinner = ora('Building webpack bundles...').start();
-  
+
   try {
     const webpackConfigs = configureWebpack(config, options);
     const targets = Object.keys(webpackConfigs);
-    
+
     if (targets.length === 0) {
       spinner.info('No webpack targets configured');
       return;
     }
-    
+
     for (const targetName of targets) {
       const targetConfig = webpackConfigs[targetName];
-      
+
       // Split entries if there are many (memory optimization)
       if (targetConfig.entry && Object.keys(targetConfig.entry).length > 10 && !options.parallel) {
         const batches = splitEntries(targetConfig.entry, 10);
         spinner.text = `Building ${targetName} (${batches.length} batches)...`;
-        
+
         for (let i = 0; i < batches.length; i++) {
           const batchConfig = { ...targetConfig, entry: batches[i] };
           spinner.text = `Building ${targetName} batch ${i + 1}/${batches.length}...`;
-          
+
           const stats = await runWebpack(batchConfig);
-          
+
           if (options.verbose) {
             console.log(stats.toString({ colors: true }));
           }
-          
+
           // Allow GC between batches
           if (global.gc) {
             global.gc();
@@ -162,13 +164,13 @@ async function runWebpackBuild(config, options) {
       } else {
         spinner.text = `Building ${targetName}...`;
         const stats = await runWebpack(targetConfig);
-        
+
         if (options.verbose) {
           console.log(stats.toString({ colors: true }));
         }
       }
     }
-    
+
     spinner.succeed(`Webpack: ${targets.length} target(s) built`);
   } catch (error) {
     spinner.fail('Webpack build failed');
@@ -182,56 +184,57 @@ async function runWebpackBuild(config, options) {
  * @param {Object} options - Build options
  */
 async function runWebpackWatch(config, options) {
+  const { configureWebpack, watchWebpack } = require('./webpack.config');
   const webpackConfigs = configureWebpack(config, { ...options, watch: true });
   const targets = Object.keys(webpackConfigs);
-  
+
   if (targets.length === 0) {
     console.log(chalk.yellow('No webpack targets configured'));
     return;
   }
-  
+
   console.log(chalk.cyan('Starting webpack watch mode...'));
-  
+
   const watchers = [];
-  
+
   for (const targetName of targets) {
     const targetConfig = webpackConfigs[targetName];
-    
+
     const watcher = watchWebpack(targetConfig, (err, stats) => {
       if (err) {
         console.error(chalk.red(`[${targetName}] Error:`, err.message));
         return;
       }
-      
+
       const info = stats.toJson();
-      
+
       if (stats.hasErrors()) {
         console.error(chalk.red(`[${targetName}] Errors:`));
         info.errors.forEach(e => console.error(e.message));
         return;
       }
-      
+
       if (stats.hasWarnings() && options.verbose) {
         console.warn(chalk.yellow(`[${targetName}] Warnings:`));
         info.warnings.forEach(w => console.warn(w.message));
       }
-      
+
       console.log(chalk.green(`[${targetName}] Rebuilt in ${info.time}ms`));
     });
-    
+
     watchers.push(watcher);
   }
-  
+
   // Handle process termination
   const cleanup = () => {
     console.log(chalk.yellow('\nStopping watchers...'));
     watchers.forEach(w => w.close());
     process.exit(0);
   };
-  
+
   process.on('SIGINT', cleanup);
   process.on('SIGTERM', cleanup);
-  
+
   // Keep process alive
   await new Promise(() => {});
 }
@@ -242,17 +245,18 @@ async function runWebpackWatch(config, options) {
  * @param {Object} options - Build options
  */
 async function runSassBuild(config, options) {
+  const { compileSass } = require('./sass-compiler');
   const spinner = ora('Compiling Sass...').start();
-  
+
   try {
     const results = await compileSass(config, {
       verbose: options.verbose,
       maxConcurrency: options.parallel ? 4 : 2,
     });
-    
+
     const successful = results.filter(r => r.success).length;
     const failed = results.filter(r => !r.success).length;
-    
+
     if (failed > 0) {
       spinner.warn(`Sass: ${successful} compiled, ${failed} failed`);
       results.filter(r => !r.success).forEach(r => {
@@ -275,29 +279,30 @@ async function runSassBuild(config, options) {
  * @param {Object} options - Build options
  */
 async function runConcatMinifyBuild(config, options) {
+  const { runConcatMinify } = require('./concat-minify');
   const spinner = ora('Concatenating and minifying...').start();
-  
+
   try {
     const results = await runConcatMinify(config, {
       verbose: options.verbose,
       noMinify: options.noMinify,
     });
-    
+
     const concatCount = results.concat.filter(r => r.success).length;
     const minifyCount = results.minify.filter(r => r.success).length;
-    
+
     if (concatCount > 0 || minifyCount > 0) {
       spinner.succeed(`Concat: ${concatCount} bundles, Minify: ${minifyCount} files`);
     } else {
       spinner.info('No concat/minify targets configured');
     }
-    
+
     // Report any failures
     const failures = [
       ...results.concat.filter(r => !r.success),
       ...results.minify.filter(r => !r.success),
     ];
-    
+
     if (failures.length > 0) {
       console.warn(chalk.yellow('  Some targets failed:'));
       failures.forEach(f => {
@@ -317,30 +322,30 @@ async function runConcatMinifyBuild(config, options) {
  */
 async function runClean(config, options) {
   const spinner = ora('Cleaning...').start();
-  
+
   try {
     const cleanConfig = config.clean || {};
     let cleaned = 0;
-    
+
     for (const [targetName, targetConfig] of Object.entries(cleanConfig)) {
       const src = Array.isArray(targetConfig.src) ? targetConfig.src : [targetConfig.src];
-      
+
       for (const pattern of src) {
         const resolvedPath = path.resolve(config.paths?.site || process.cwd(), pattern);
-        
+
         // Simple glob handling for clean
         if (pattern.includes('*')) {
           // Skip complex globs for now
           continue;
         }
-        
+
         if (fs.existsSync(resolvedPath)) {
           await fs.promises.rm(resolvedPath, { recursive: true, force: true });
           cleaned++;
         }
       }
     }
-    
+
     spinner.succeed(`Cleaned ${cleaned} path(s)`);
   } catch (error) {
     spinner.fail('Clean failed');
@@ -348,6 +353,60 @@ async function runClean(config, options) {
   }
 }
 
+/**
+ * Run a single build task in an isolated subprocess with a controlled heap.
+ * When the subprocess exits, the OS fully reclaims all its memory.
+ *
+ * @param {string} taskName - Task to run (webpack, sass, concat)
+ * @param {Object} configPaths - config.paths object (all strings, serializable)
+ * @param {Object} options - Serializable build options
+ * @param {number} heapSize - Max heap size in MB for the subprocess
+ * @returns {Promise<void>}
+ */
+function runTaskInSubprocess(taskName, configPaths, options, heapSize) {
+  return new Promise((resolve, reject) => {
+    const tmpFile = path.join(os.tmpdir(), `epb-paths-${process.pid}-${taskName}.json`);
+    fs.writeFileSync(tmpFile, JSON.stringify(configPaths));
+
+    const workerPath = path.join(__dirname, 'task-worker.js');
+    const serializableOptions = {
+      verbose: options.verbose || false,
+      noMinify: options.noMinify || false,
+      analyze: options.analyze || false,
+      lint: options.lint || false,
+      parallel: false, // never parallel inside a subprocess
+    };
+
+    const nodeArgs = [
+      `--max-old-space-size=${heapSize}`,
+      workerPath,
+      taskName,
+      tmpFile,
+      JSON.stringify(serializableOptions),
+    ];
+
+    const proc = spawn(process.execPath, nodeArgs, {
+      stdio: 'inherit',
+      // Clear NODE_OPTIONS so our --max-old-space-size takes effect
+      env: { ...process.env, NODE_OPTIONS: '' },
+    });
+
+    proc.on('close', (code) => {
+      try { fs.unlinkSync(tmpFile); } catch {}
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`${taskName} subprocess exited with code ${code}`));
+      }
+    });
+
+    proc.on('error', (err) => {
+      try { fs.unlinkSync(tmpFile); } catch {}
+      reject(err);
+    });
+  });
+}
+
 /**
  * Run full build
  * @param {Object} config - Build configuration
@@ -355,9 +414,10 @@ async function runClean(config, options) {
  */
 async function runFullBuild(config, options) {
   const startTime = Date.now();
-  
+
   // Generate tsconfig files first
   try {
+    const { writeTsConfigs } = require('./tsconfig-gen');
     const written = await writeTsConfigs(process.cwd());
     if (options.verbose && written.length > 0) {
       console.log(chalk.gray(`  Generated ${written.length} tsconfig.json file(s)`));
@@ -365,9 +425,19 @@ async function runFullBuild(config, options) {
   } catch {
     // TSConfig generation is optional, continue on error
   }
-  
-  // Run tasks sequentially for memory optimization (unless --parallel)
-  if (options.parallel) {
+
+  if (options.isolateTasks) {
+    // Subprocess isolation: each task runs in its own Node process.
+    // When the subprocess exits, the OS fully reclaims all memory.
+    const heapSize = options.heapSize || 512;
+
+    console.log(chalk.gray(`  Running tasks in isolated subprocesses (heap: ${heapSize}MB)\n`));
+
+    await runTaskInSubprocess('webpack', config.paths, options, heapSize);
+    await runTaskInSubprocess('sass', config.paths, options, heapSize);
+    await runTaskInSubprocess('concat', config.paths, options, heapSize);
+  } else if (options.parallel) {
+    // Run tasks in parallel (uses more memory)
     await Promise.all([
       runWebpackBuild(config, options),
       runSassBuild(config, options),
@@ -377,20 +447,21 @@ async function runFullBuild(config, options) {
     // Sequential execution with GC hints between tasks
     await runWebpackBuild(config, options);
     if (global.gc) global.gc();
-    
+
     await runSassBuild(config, options);
     if (global.gc) global.gc();
-    
+
     await runConcatMinifyBuild(config, options);
   }
-  
+
   // Clean tsconfig files after build
   try {
+    const { cleanTsConfigs } = require('./tsconfig-gen');
     await cleanTsConfigs(process.cwd());
   } catch {
     // Cleanup is optional
   }
-  
+
   const duration = ((Date.now() - startTime) / 1000).toFixed(2);
   console.log(chalk.green(`\n✓ Build completed in ${duration}s\n`));
 }
@@ -410,6 +481,8 @@ async function run(argv) {
     .option('--lint', 'Run ESLint')
     .option('-v, --verbose', 'Verbose output')
     .option('--parallel', 'Run tasks in parallel (uses more memory)')
+    .option('--isolate-tasks', 'Run each task in an isolated subprocess for lower memory usage')
+    .option('--heap-size <mb>', 'Max heap size in MB per subprocess (default: 512)', parseInt)
     .option('--no-banner', 'Hide banner')
     .option('--env <env>', 'Set NODE_ENV')
     .option('--gc-between-tasks', 'Force garbage collection between tasks')
@@ -437,7 +510,7 @@ async function run(argv) {
 
   // Load configuration
   const spinner = ora('Loading configuration...').start();
-  
+
   let config;
   try {
     config = await loadConfig(process.cwd(), { verbose: options.verbose });
@@ -487,7 +560,8 @@ async function run(argv) {
       await runClean(config, options);
       break;
 
-    case 'tsconfig':
+    case 'tsconfig': {
+      const { writeTsConfigs } = require('./tsconfig-gen');
       const tsconfigSpinner = ora('Generating tsconfig.json files...').start();
       try {
         const written = await writeTsConfigs(process.cwd());
@@ -497,8 +571,10 @@ async function run(argv) {
         throw error;
       }
       break;
+    }
 
-    case 'clean-tsconfig':
+    case 'clean-tsconfig': {
+      const { cleanTsConfigs } = require('./tsconfig-gen');
       const cleanTsconfigSpinner = ora('Cleaning tsconfig.json files...').start();
       try {
         const removed = await cleanTsConfigs(process.cwd());
@@ -508,6 +584,7 @@ async function run(argv) {
         throw error;
       }
       break;
+    }
 
     case 'all':
     default:
@@ -532,4 +609,3 @@ module.exports = {
   runClean,
   runFullBuild,
 };
-

package/lib/task-worker.js

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+/**
+ * Task Worker - Isolated subprocess for a single build task.
+ *
+ * Invoked by the main CLI in --isolate-tasks mode. Each task runs in
+ * its own Node process with a controlled heap size (via --max-old-space-size).
+ * When the subprocess exits, the OS fully reclaims all its memory.
+ *
+ * Usage: node --max-old-space-size=512 task-worker.js <task> <paths-file> [options-json]
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const args = process.argv.slice(2);
+const taskName = args[0];
+const pathsFile = args[1];
+const optionsJson = args[2] || '{}';
+
+if (!taskName || !pathsFile) {
+  console.error('Usage: task-worker.js <task> <paths-file> [options-json]');
+  process.exit(1);
+}
+
+// Read paths (written by the main process after bundle exec discovery)
+const paths = JSON.parse(fs.readFileSync(pathsFile, 'utf8'));
+const options = JSON.parse(optionsJson);
+
+// Rebuild config from paths — re-reads grunt-config.js files but skips
+// the expensive `bundle exec` gem-path discovery (already done by parent).
+const { baseConfig, mergeConfig } = require('./config-loader');
+const { resolveTemplates } = require('./template-resolver');
+
+let config = JSON.parse(JSON.stringify(baseConfig));
+config.paths = paths;
+
+if (paths.jade) config = mergeConfig(config, paths.jade);
+for (const [key, val] of Object.entries(paths)) {
+  if (key.startsWith('jade_') && val) config = mergeConfig(config, val);
+}
+if (paths.site) config = mergeConfig(config, paths.site);
+config = resolveTemplates(config, config.paths);
+
+// Import only the task function we need (lazy loading keeps memory low)
+const {
+  runWebpackBuild,
+  runSassBuild,
+  runConcatMinifyBuild,
+} = require('./cli');
+
+async function main() {
+  switch (taskName) {
+    case 'webpack':
+      await runWebpackBuild(config, options);
+      break;
+    case 'sass':
+      await runSassBuild(config, options);
+      break;
+    case 'concat':
+      await runConcatMinifyBuild(config, options);
+      break;
+    default:
+      throw new Error(`Unknown task: ${taskName}`);
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(`[task-worker] ${taskName} failed:`, err.message);
+    if (process.env.DEBUG) console.error(err.stack);
+    process.exit(1);
+  });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@epublishing/grunt-epublishing",
   "description": "Modern front-end build tools for ePublishing Jade and client sites.",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "homepage": "https://www.epublishing.com",
   "contributors": [
     {