@epublishing/grunt-epublishing 1.1.3 → 1.2.0

package/lib/cli.js

@@ -1,24 +1,23 @@
 /**
  * CLI Implementation
- * 
+ *
  * Main entry point for the build tools.
  * Orchestrates webpack, sass, concat/minify tasks.
+ *
+ * Heavy modules (webpack, sass-embedded, terser) are lazy-loaded inside
+ * their task functions so that subprocess workers only pay for what they use.
  */
 
 'use strict';
 
+const os = require('os');
 const fs = require('fs');
 const path = require('path');
 const { spawn } = require('child_process');
 const { program } = require('commander');
 const chalk = require('chalk');
 const ora = require('ora');
-const Resolver = require('@epublishing/jade-resolver');
 const { loadConfig } = require('./config-loader');
-const { configureWebpack, runWebpack, watchWebpack, splitEntries } = require('./webpack.config');
-const { compileSass } = require('./sass-compiler');
-const { runConcatMinify } = require('./concat-minify');
-const { writeTsConfigs, cleanTsConfigs } = require('./tsconfig-gen');
 
 // Package info for banner
 const pkg = require('../package.json');
@@ -28,7 +27,7 @@ const pkg = require('../package.json');
  */
 function printBanner(options) {
   if (options.noBanner) return;
-  
+
   console.log(chalk.cyan(`
 ╔═══════════════════════════════════════════╗
 ║                                           ║
@@ -43,7 +42,7 @@ function printBanner(options) {
  */
 function printOptions(options) {
   if (options.noBanner) return;
-  
+
   const flags = [];
   if (options.watch) flags.push('watch');
   if (options.noMinify) flags.push('no-minify');
@@ -51,7 +50,8 @@ function printOptions(options) {
   if (options.lint) flags.push('lint');
   if (options.verbose) flags.push('verbose');
   if (options.parallel) flags.push('parallel');
-  
+  if (options.isolateTasks) flags.push('isolate-tasks');
+
   if (flags.length > 0) {
     console.log(chalk.gray(`  Options: ${flags.join(', ')}\n`));
   }
@@ -64,16 +64,27 @@ function printOptions(options) {
  * @param {Object} options - Build options
  */
 async function runNpmInstall(config, options) {
+  const Resolver = require('@epublishing/jade-resolver');
   const paths = config.paths || {};
-  const resolver = new Resolver({ ...paths }, { includeSite: false });
-  resolver.removePath('site');
-
+  // Build paths object with ONLY jade and jade child gem directories (absolute paths)
+  const gemPaths = {};
+  if (paths.jade && path.isAbsolute(paths.jade)) {
+    gemPaths.jade = paths.jade;
+  }
+  if (paths.jadechild && path.isAbsolute(paths.jadechild)) {
+    gemPaths.jadechild = paths.jadechild;
+  }
+  for (const [key, value] of Object.entries(paths)) {
+    if (key.startsWith('jade_') && value && path.isAbsolute(value)) {
+      gemPaths[key] = value;
+    }
+  }
+  const resolver = new Resolver(gemPaths, { includeSite: false });
   const packages = resolver.find('package.json');
   if (!packages || packages.length === 0) {
     return;
   }
 
-  const origCwd = process.cwd();
   const dirs = [...new Set(packages.map((p) => path.dirname(p)))];
 
   for (const dir of dirs) {
@@ -84,13 +95,12 @@ async function runNpmInstall(config, options) {
       const hasLock = fs.existsSync(path.join(dir, 'package-lock.json'));
       const cmd = hasLock ? 'ci' : 'install';
       await new Promise((resolve, reject) => {
-        process.chdir(dir);
         const proc = spawn('npm', [cmd], {
+          cwd: dir,
           stdio: options.verbose ? 'inherit' : 'pipe',
           shell: true,
         });
         proc.on('close', (code) => {
-          process.chdir(origCwd);
           if (code === 0) {
             spinner.succeed(`npm ${cmd} completed in ${basename}`);
             resolve();
@@ -100,13 +110,11 @@ async function runNpmInstall(config, options) {
           }
         });
         proc.on('error', (err) => {
-          process.chdir(origCwd);
           spinner.fail(`npm ${cmd} failed in ${basename}`);
           reject(err);
         });
       });
     } catch (err) {
-      process.chdir(origCwd);
       throw err;
     }
   }
@@ -118,35 +126,36 @@ async function runNpmInstall(config, options) {
  * @param {Object} options - Build options
  */
 async function runWebpackBuild(config, options) {
+  const { configureWebpack, runWebpack, splitEntries } = require('./webpack.config');
   const spinner = ora('Building webpack bundles...').start();
-  
+
   try {
     const webpackConfigs = configureWebpack(config, options);
     const targets = Object.keys(webpackConfigs);
-    
+
     if (targets.length === 0) {
       spinner.info('No webpack targets configured');
       return;
     }
-    
+
     for (const targetName of targets) {
       const targetConfig = webpackConfigs[targetName];
-      
+
       // Split entries if there are many (memory optimization)
       if (targetConfig.entry && Object.keys(targetConfig.entry).length > 10 && !options.parallel) {
         const batches = splitEntries(targetConfig.entry, 10);
         spinner.text = `Building ${targetName} (${batches.length} batches)...`;
-        
+
         for (let i = 0; i < batches.length; i++) {
           const batchConfig = { ...targetConfig, entry: batches[i] };
           spinner.text = `Building ${targetName} batch ${i + 1}/${batches.length}...`;
-          
+
           const stats = await runWebpack(batchConfig);
-          
+
           if (options.verbose) {
             console.log(stats.toString({ colors: true }));
           }
-          
+
           // Allow GC between batches
           if (global.gc) {
             global.gc();
@@ -155,13 +164,13 @@ async function runWebpackBuild(config, options) {
       } else {
         spinner.text = `Building ${targetName}...`;
         const stats = await runWebpack(targetConfig);
-        
+
         if (options.verbose) {
           console.log(stats.toString({ colors: true }));
         }
       }
     }
-    
+
     spinner.succeed(`Webpack: ${targets.length} target(s) built`);
   } catch (error) {
     spinner.fail('Webpack build failed');
@@ -175,56 +184,57 @@ async function runWebpackBuild(config, options) {
  * @param {Object} options - Build options
  */
 async function runWebpackWatch(config, options) {
+  const { configureWebpack, watchWebpack } = require('./webpack.config');
   const webpackConfigs = configureWebpack(config, { ...options, watch: true });
   const targets = Object.keys(webpackConfigs);
-  
+
   if (targets.length === 0) {
     console.log(chalk.yellow('No webpack targets configured'));
     return;
   }
-  
+
   console.log(chalk.cyan('Starting webpack watch mode...'));
-  
+
   const watchers = [];
-  
+
   for (const targetName of targets) {
     const targetConfig = webpackConfigs[targetName];
-    
+
     const watcher = watchWebpack(targetConfig, (err, stats) => {
       if (err) {
         console.error(chalk.red(`[${targetName}] Error:`, err.message));
         return;
       }
-      
+
       const info = stats.toJson();
-      
+
       if (stats.hasErrors()) {
         console.error(chalk.red(`[${targetName}] Errors:`));
         info.errors.forEach(e => console.error(e.message));
         return;
       }
-      
+
       if (stats.hasWarnings() && options.verbose) {
         console.warn(chalk.yellow(`[${targetName}] Warnings:`));
         info.warnings.forEach(w => console.warn(w.message));
       }
-      
+
       console.log(chalk.green(`[${targetName}] Rebuilt in ${info.time}ms`));
     });
-    
+
     watchers.push(watcher);
   }
-  
+
   // Handle process termination
   const cleanup = () => {
     console.log(chalk.yellow('\nStopping watchers...'));
     watchers.forEach(w => w.close());
     process.exit(0);
   };
-  
+
   process.on('SIGINT', cleanup);
   process.on('SIGTERM', cleanup);
-  
+
   // Keep process alive
   await new Promise(() => {});
 }
@@ -235,17 +245,18 @@ async function runWebpackWatch(config, options) {
  * @param {Object} options - Build options
  */
 async function runSassBuild(config, options) {
+  const { compileSass } = require('./sass-compiler');
   const spinner = ora('Compiling Sass...').start();
-  
+
   try {
     const results = await compileSass(config, {
       verbose: options.verbose,
       maxConcurrency: options.parallel ? 4 : 2,
     });
-    
+
     const successful = results.filter(r => r.success).length;
     const failed = results.filter(r => !r.success).length;
-    
+
     if (failed > 0) {
       spinner.warn(`Sass: ${successful} compiled, ${failed} failed`);
       results.filter(r => !r.success).forEach(r => {
@@ -268,29 +279,30 @@ async function runSassBuild(config, options) {
  * @param {Object} options - Build options
  */
 async function runConcatMinifyBuild(config, options) {
+  const { runConcatMinify } = require('./concat-minify');
   const spinner = ora('Concatenating and minifying...').start();
-  
+
   try {
     const results = await runConcatMinify(config, {
       verbose: options.verbose,
       noMinify: options.noMinify,
     });
-    
+
     const concatCount = results.concat.filter(r => r.success).length;
     const minifyCount = results.minify.filter(r => r.success).length;
-    
+
     if (concatCount > 0 || minifyCount > 0) {
       spinner.succeed(`Concat: ${concatCount} bundles, Minify: ${minifyCount} files`);
     } else {
       spinner.info('No concat/minify targets configured');
     }
-    
+
     // Report any failures
     const failures = [
       ...results.concat.filter(r => !r.success),
       ...results.minify.filter(r => !r.success),
     ];
-    
+
     if (failures.length > 0) {
       console.warn(chalk.yellow('  Some targets failed:'));
       failures.forEach(f => {
@@ -310,30 +322,30 @@ async function runConcatMinifyBuild(config, options) {
  */
 async function runClean(config, options) {
   const spinner = ora('Cleaning...').start();
-  
+
   try {
     const cleanConfig = config.clean || {};
     let cleaned = 0;
-    
+
     for (const [targetName, targetConfig] of Object.entries(cleanConfig)) {
       const src = Array.isArray(targetConfig.src) ? targetConfig.src : [targetConfig.src];
-      
+
       for (const pattern of src) {
         const resolvedPath = path.resolve(config.paths?.site || process.cwd(), pattern);
-        
+
         // Simple glob handling for clean
         if (pattern.includes('*')) {
           // Skip complex globs for now
           continue;
         }
-        
+
         if (fs.existsSync(resolvedPath)) {
           await fs.promises.rm(resolvedPath, { recursive: true, force: true });
           cleaned++;
         }
       }
     }
-    
+
     spinner.succeed(`Cleaned ${cleaned} path(s)`);
   } catch (error) {
     spinner.fail('Clean failed');
@@ -341,6 +353,60 @@ async function runClean(config, options) {
   }
 }
 
+/**
+ * Run a single build task in an isolated subprocess with a controlled heap.
+ * When the subprocess exits, the OS fully reclaims all its memory.
+ *
+ * @param {string} taskName - Task to run (webpack, sass, concat)
+ * @param {Object} configPaths - config.paths object (all strings, serializable)
+ * @param {Object} options - Serializable build options
+ * @param {number} heapSize - Max heap size in MB for the subprocess
+ * @returns {Promise<void>}
+ */
+function runTaskInSubprocess(taskName, configPaths, options, heapSize) {
+  return new Promise((resolve, reject) => {
+    const tmpFile = path.join(os.tmpdir(), `epb-paths-${process.pid}-${taskName}.json`);
+    fs.writeFileSync(tmpFile, JSON.stringify(configPaths));
+
+    const workerPath = path.join(__dirname, 'task-worker.js');
+    const serializableOptions = {
+      verbose: options.verbose || false,
+      noMinify: options.noMinify || false,
+      analyze: options.analyze || false,
+      lint: options.lint || false,
+      parallel: false, // never parallel inside a subprocess
+    };
+
+    const nodeArgs = [
+      `--max-old-space-size=${heapSize}`,
+      workerPath,
+      taskName,
+      tmpFile,
+      JSON.stringify(serializableOptions),
+    ];
+
+    const proc = spawn(process.execPath, nodeArgs, {
+      stdio: 'inherit',
+      // Clear NODE_OPTIONS so our --max-old-space-size takes effect
+      env: { ...process.env, NODE_OPTIONS: '' },
+    });
+
+    proc.on('close', (code) => {
+      try { fs.unlinkSync(tmpFile); } catch {}
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`${taskName} subprocess exited with code ${code}`));
+      }
+    });
+
+    proc.on('error', (err) => {
+      try { fs.unlinkSync(tmpFile); } catch {}
+      reject(err);
+    });
+  });
+}
+
 /**
  * Run full build
  * @param {Object} config - Build configuration
@@ -348,9 +414,10 @@ async function runClean(config, options) {
  */
 async function runFullBuild(config, options) {
   const startTime = Date.now();
-  
+
   // Generate tsconfig files first
   try {
+    const { writeTsConfigs } = require('./tsconfig-gen');
     const written = await writeTsConfigs(process.cwd());
     if (options.verbose && written.length > 0) {
       console.log(chalk.gray(`  Generated ${written.length} tsconfig.json file(s)`));
@@ -358,9 +425,19 @@ async function runFullBuild(config, options) {
   } catch {
     // TSConfig generation is optional, continue on error
   }
-  
-  // Run tasks sequentially for memory optimization (unless --parallel)
-  if (options.parallel) {
+
+  if (options.isolateTasks) {
+    // Subprocess isolation: each task runs in its own Node process.
+    // When the subprocess exits, the OS fully reclaims all memory.
+    const heapSize = options.heapSize || 512;
+
+    console.log(chalk.gray(`  Running tasks in isolated subprocesses (heap: ${heapSize}MB)\n`));
+
+    await runTaskInSubprocess('webpack', config.paths, options, heapSize);
+    await runTaskInSubprocess('sass', config.paths, options, heapSize);
+    await runTaskInSubprocess('concat', config.paths, options, heapSize);
+  } else if (options.parallel) {
+    // Run tasks in parallel (uses more memory)
     await Promise.all([
       runWebpackBuild(config, options),
       runSassBuild(config, options),
@@ -370,20 +447,21 @@ async function runFullBuild(config, options) {
     // Sequential execution with GC hints between tasks
     await runWebpackBuild(config, options);
     if (global.gc) global.gc();
-    
+
     await runSassBuild(config, options);
     if (global.gc) global.gc();
-    
+
     await runConcatMinifyBuild(config, options);
   }
-  
+
   // Clean tsconfig files after build
   try {
+    const { cleanTsConfigs } = require('./tsconfig-gen');
     await cleanTsConfigs(process.cwd());
   } catch {
     // Cleanup is optional
   }
-  
+
   const duration = ((Date.now() - startTime) / 1000).toFixed(2);
   console.log(chalk.green(`\n✓ Build completed in ${duration}s\n`));
 }
@@ -403,6 +481,8 @@ async function run(argv) {
     .option('--lint', 'Run ESLint')
     .option('-v, --verbose', 'Verbose output')
     .option('--parallel', 'Run tasks in parallel (uses more memory)')
+    .option('--isolate-tasks', 'Run each task in an isolated subprocess for lower memory usage')
+    .option('--heap-size <mb>', 'Max heap size in MB per subprocess (default: 512)', parseInt)
     .option('--no-banner', 'Hide banner')
     .option('--env <env>', 'Set NODE_ENV')
     .option('--gc-between-tasks', 'Force garbage collection between tasks')
@@ -412,6 +492,9 @@ async function run(argv) {
   const options = program.opts();
   const task = program.args[0] || 'all';
 
+  // Debug: print paths and webpack alias (with -v or DEBUG=1)
+  const debug = options.verbose || process.env.DEBUG;
+
   // Set NODE_ENV if specified
   if (options.env) {
     process.env.NODE_ENV = options.env;
@@ -427,7 +510,7 @@ async function run(argv) {
 
   // Load configuration
   const spinner = ora('Loading configuration...').start();
-  
+
   let config;
   try {
     config = await loadConfig(process.cwd(), { verbose: options.verbose });
@@ -437,6 +520,15 @@ async function run(argv) {
     throw error;
   }
 
+  if (debug && config.paths) {
+    console.log(chalk.gray('  Paths: jade=') + (config.paths.jade || '(none)'));
+    console.log(chalk.gray('  Paths: jadechild=') + (config.paths.jadechild || '(none)'));
+    const jadeChildKeys = Object.keys(config.paths).filter((k) => k.startsWith('jade_'));
+    jadeChildKeys.forEach((k) => {
+      console.log(chalk.gray(`  Paths: ${k}=`) + config.paths[k]);
+    });
+  }
+
   // Install npm packages from jade hierarchy (jade, jadechild) before build
   if (task === 'webpack' || task === 'all') {
     try {
@@ -468,7 +560,8 @@ async function run(argv) {
       await runClean(config, options);
       break;
 
-    case 'tsconfig':
+    case 'tsconfig': {
+      const { writeTsConfigs } = require('./tsconfig-gen');
       const tsconfigSpinner = ora('Generating tsconfig.json files...').start();
       try {
         const written = await writeTsConfigs(process.cwd());
@@ -478,8 +571,10 @@ async function run(argv) {
         throw error;
       }
       break;
+    }
 
-    case 'clean-tsconfig':
+    case 'clean-tsconfig': {
+      const { cleanTsConfigs } = require('./tsconfig-gen');
       const cleanTsconfigSpinner = ora('Cleaning tsconfig.json files...').start();
       try {
         const removed = await cleanTsConfigs(process.cwd());
@@ -489,6 +584,7 @@ async function run(argv) {
         throw error;
       }
       break;
+    }
 
     case 'all':
     default:
@@ -513,4 +609,3 @@ module.exports = {
   runClean,
   runFullBuild,
 };
-

package/lib/config-loader.js

@@ -11,8 +11,8 @@
 
 const fs = require('fs');
 const path = require('path');
+const { spawn } = require('child_process');
 const _ = require('lodash');
-const getGemPaths = require('@epublishing/get-gem-paths');
 const { resolveConfig, resolveTemplates } = require('./template-resolver');
 
 /**
@@ -41,6 +41,43 @@ function getFilename(fullPath) {
   return path.basename(fullPath);
 }
 
+/**
+ * Get gem paths by running the get-gem-paths Ruby script via bundle exec.
+ * This ensures Bundler has loaded the site's Gemfile and gem specs.
+ * @param {string} sitePath - Path to the site (with Gemfile)
+ * @param {string} pattern - Gem name pattern (e.g. 'jade')
+ * @returns {Promise<Array<{name: string, path: string, version: string}>>}
+ */
+function getGemPathsFromBundler(sitePath, pattern = 'jade') {
+  return new Promise((resolve, reject) => {
+    const rubyScript = path.join(
+      path.dirname(require.resolve('@epublishing/get-gem-paths')),
+      'scripts',
+      'get-gem-paths.rb'
+    );
+    const proc = spawn('bundle', ['exec', 'ruby', rubyScript, pattern], {
+      cwd: sitePath || process.cwd(),
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    proc.stdout.on('data', (d) => { stdout += d; });
+    proc.stderr.on('data', (d) => { stderr += d; });
+    proc.on('close', (code) => {
+      if (code !== 0) {
+        reject(new Error(`get-gem-paths failed (${code}): ${stderr || stdout}`));
+        return;
+      }
+      try {
+        resolve(JSON.parse(stdout));
+      } catch (e) {
+        reject(new Error(`get-gem-paths invalid JSON: ${stdout}`));
+      }
+    });
+    proc.on('error', reject);
+  });
+}
+
 /**
  * Custom merger for lodash merge
  * Handles array merging with JS file override logic
@@ -127,8 +164,8 @@ async function loadConfig(sitePath = process.cwd(), options = {}) {
   const jadeChildPaths = {};
   
   try {
-    const gems = await getGemPaths('jade_?', sitePath);
-    
+    const gems = await getGemPathsFromBundler(sitePath, 'jade');
+
     for (const gem of gems) {
       if (gem.name === 'jade') {
         jadePath = gem.path;

package/lib/task-worker.js

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+/**
+ * Task Worker - Isolated subprocess for a single build task.
+ *
+ * Invoked by the main CLI in --isolate-tasks mode. Each task runs in
+ * its own Node process with a controlled heap size (via --max-old-space-size).
+ * When the subprocess exits, the OS fully reclaims all its memory.
+ *
+ * Usage: node --max-old-space-size=512 task-worker.js <task> <paths-file> [options-json]
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const args = process.argv.slice(2);
+const taskName = args[0];
+const pathsFile = args[1];
+const optionsJson = args[2] || '{}';
+
+if (!taskName || !pathsFile) {
+  console.error('Usage: task-worker.js <task> <paths-file> [options-json]');
+  process.exit(1);
+}
+
+// Read paths (written by the main process after bundle exec discovery)
+const paths = JSON.parse(fs.readFileSync(pathsFile, 'utf8'));
+const options = JSON.parse(optionsJson);
+
+// Rebuild config from paths — re-reads grunt-config.js files but skips
+// the expensive `bundle exec` gem-path discovery (already done by parent).
+const { baseConfig, mergeConfig } = require('./config-loader');
+const { resolveTemplates } = require('./template-resolver');
+
+let config = JSON.parse(JSON.stringify(baseConfig));
+config.paths = paths;
+
+if (paths.jade) config = mergeConfig(config, paths.jade);
+for (const [key, val] of Object.entries(paths)) {
+  if (key.startsWith('jade_') && val) config = mergeConfig(config, val);
+}
+if (paths.site) config = mergeConfig(config, paths.site);
+config = resolveTemplates(config, config.paths);
+
+// Import only the task function we need (lazy loading keeps memory low)
+const {
+  runWebpackBuild,
+  runSassBuild,
+  runConcatMinifyBuild,
+} = require('./cli');
+
+async function main() {
+  switch (taskName) {
+    case 'webpack':
+      await runWebpackBuild(config, options);
+      break;
+    case 'sass':
+      await runSassBuild(config, options);
+      break;
+    case 'concat':
+      await runConcatMinifyBuild(config, options);
+      break;
+    default:
+      throw new Error(`Unknown task: ${taskName}`);
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(`[task-worker] ${taskName} failed:`, err.message);
+    if (process.env.DEBUG) console.error(err.stack);
+    process.exit(1);
+  });

package/lib/webpack.config.js

@@ -90,11 +90,16 @@ function createWebpackConfig(config, options = {}) {
   const resolveAlias = {
     lodash: lodashDir,
   };
-  if (paths.jadechild) {
-    resolveAlias['jade-child'] = path.join(paths.jadechild, paths.js_src || 'app/js', 'jade-child.js');
+  const jadeChildPath = paths.jadechild || (() => {
+    const jadeChildKey = Object.keys(paths).find((k) => k.startsWith('jade_') && k !== 'jadechild');
+    return jadeChildKey ? paths[jadeChildKey] : null;
+  })();
+  if (jadeChildPath && typeof jadeChildPath === 'string') {
+    const jadeChildJs = path.resolve(jadeChildPath, paths.js_src || 'app/js', 'jade-child.js');
+    resolveAlias['jade-child'] = jadeChildJs;
   }
-  if (paths.jade) {
-    resolveAlias['jade-engine'] = path.join(paths.jade, paths.js_src || 'app/js', 'jade-engine.js');
+  if (paths.jade && typeof paths.jade === 'string') {
+    resolveAlias['jade-engine'] = path.resolve(paths.jade, paths.js_src || 'app/js', 'jade-engine.js');
   }
 
   // Build plugins array

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@epublishing/grunt-epublishing",
   "description": "Modern front-end build tools for ePublishing Jade and client sites.",
-  "version": "1.1.3",
+  "version": "1.2.0",
   "homepage": "https://www.epublishing.com",
   "contributors": [
     {