@atlaspack/bundler-default 2.14.1-dev.145 → 2.14.1-dev.146

package/CHANGELOG.md

@@ -1,5 +1,135 @@
 # @atlaspack/bundler-default
 
+## 3.0.1
+
+### Patch Changes
+
+- [#608](https://github.com/atlassian-labs/atlaspack/pull/608) [`471b99e`](https://github.com/atlassian-labs/atlaspack/commit/471b99e41b4d97328c88f65e90bea284372cb1b0) Thanks [@mattcompiles](https://github.com/mattcompiles)! - Fix require of ES Module error
+
+## 3.0.0
+
+### Major Changes
+
+- [#600](https://github.com/atlassian-labs/atlaspack/pull/600) [`1b52b99`](https://github.com/atlassian-labs/atlaspack/commit/1b52b99db4298b04c1a6eb0f97994d75a2d436f9) Thanks [@mattcompiles](https://github.com/mattcompiles)! - ### Breaking change
+
+  This new config replaces the previously released `sharedBundleMergeThreshold`.
+
+  The following options are available for each merge group.
+
+  ### Options
+
+  #### overlapThreshold
+
+  > The same as `sharedBundleMergeThreshold` from #535
+
+  Merge bundles share a percentage of source bundles
+
+  ```json
+  "@atlaspack/bundler-default": {
+    "sharedBundleMerge": [{
+      "overlapThreshold": 0.75
+    }]
+  }
+  ```
+
+  #### maxBundleSize
+
+  Merge bundles that are smaller than a configured amount of bytes.
+
+  > Keep in mind these bytes are pre-optimisation
+
+  ```json
+  "@atlaspack/bundler-default": {
+    "sharedBundleMerge": [{
+      "maxBundleSize": 20000
+    }]
+  }
+  ```
+
+  #### sourceBundles
+
+  Merge bundles that share a set of source bundles. The matching is relative to the project root, like how manual shared bundle roots work.
+
+  ```json
+  "@atlaspack/bundler-default": {
+    "sharedBundleMerge": [{
+      "sourceBundles": ["src/important-route", "src/important-route-2"]
+    }]
+  }
+  ```
+
+  #### minBundlesInGroup
+
+  Merge bundles that belong to a bundle group that's larger than a set amount. This is useful for targetting bundles that would be deleted by the `maxParallelRequests` option.
+
+  ```json
+  "@atlaspack/bundler-default": {
+    "maxParallelRequests": 30,
+    "sharedBundleMerge": [{
+      "minBundlesInGroup": 30
+    }]
+  }
+  ```
+
+  ## Combining options
+
+  When multiple options are provided, all must be true for a merge to be relevant.
+
+  For example, merge bundles that are smaller than 20kb and share at least 50% of the same source bundles.
+
+  ```json
+  "@atlaspack/bundler-default": {
+    "sharedBundleMerge": [{
+      "overlapThreshold": 0.5,
+      "maxBundleSize": 20000
+    }]
+  }
+  ```
+
+  ## Multiple merges
+
+  You can also have multiple merge configs.
+
+  ```json
+  "@atlaspack/bundler-default": {
+    "sharedBundleMerge": [
+       {
+          "overlapThreshold": 0.75,
+          "maxBundleSize": 20000
+       },
+       {
+          "minBundlesInGroup": 30
+          "sourceBundles": ["src/important-route", "src/important-route-2"]
+       }
+    ]
+  }
+  ```
+
+### Patch Changes
+
+- Updated dependencies [[`1b52b99`](https://github.com/atlassian-labs/atlaspack/commit/1b52b99db4298b04c1a6eb0f97994d75a2d436f9)]:
+  - @atlaspack/graph@3.5.0
+
+## 2.16.3
+
+### Patch Changes
+
+- Updated dependencies [[`35fdd4b`](https://github.com/atlassian-labs/atlaspack/commit/35fdd4b52da0af20f74667f7b8adfb2f90279b7c), [`6dd4ccb`](https://github.com/atlassian-labs/atlaspack/commit/6dd4ccb753541de32322d881f973d571dd57e4ca)]:
+  - @atlaspack/rust@3.3.5
+  - @atlaspack/plugin@2.14.10
+  - @atlaspack/utils@2.14.10
+
+## 2.16.2
+
+### Patch Changes
+
+- Updated dependencies [[`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa), [`e052521`](https://github.com/atlassian-labs/atlaspack/commit/e0525210850ed1606146eb86991049cf567c5dec), [`15c6d70`](https://github.com/atlassian-labs/atlaspack/commit/15c6d7000bd89da876bc590aa75b17a619a41896), [`e4d966c`](https://github.com/atlassian-labs/atlaspack/commit/e4d966c3c9c4292c5013372ae65b10d19d4bacc6), [`209692f`](https://github.com/atlassian-labs/atlaspack/commit/209692ffb11eae103a0d65c5e1118a5aa1625818), [`42a775d`](https://github.com/atlassian-labs/atlaspack/commit/42a775de8eec638ad188f3271964170d8c04d84b), [`29c2f10`](https://github.com/atlassian-labs/atlaspack/commit/29c2f106de9679adfb5afa04e1910471dc65a427), [`f4da1e1`](https://github.com/atlassian-labs/atlaspack/commit/f4da1e120e73eeb5e8b8927f05e88f04d6148c7b), [`1ef91fc`](https://github.com/atlassian-labs/atlaspack/commit/1ef91fcc863fdd2831511937083dbbc1263b3d9d)]:
+  - @atlaspack/rust@3.3.4
+  - @atlaspack/feature-flags@2.16.0
+  - @atlaspack/utils@2.14.9
+  - @atlaspack/graph@3.4.7
+  - @atlaspack/plugin@2.14.9
+
 ## 2.16.1
 
 ### Patch Changes

package/lib/bundleMerge.js

@@ -25,67 +25,134 @@ function _graph() {
   };
   return data;
 }
+var _memoize = require("./memoize");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function getBundlesForBundleGroup(bundleGraph, bundleGroupId) {
+  let count = 0;
+  bundleGraph.traverse(nodeId => {
+    var _bundleGraph$getNode;
+    if (((_bundleGraph$getNode = bundleGraph.getNode(nodeId)) === null || _bundleGraph$getNode === void 0 ? void 0 : _bundleGraph$getNode.bundleBehavior) !== 'inline') {
+      count++;
+    }
+  }, bundleGroupId);
+  return count;
+}
+let getBundleOverlapBitSet = (sourceBundlesA, sourceBundlesB) => {
+  let allSourceBundles = _graph().BitSet.union(sourceBundlesA, sourceBundlesB);
+  let sharedSourceBundles = _graph().BitSet.intersect(sourceBundlesA, sourceBundlesB);
+  return sharedSourceBundles.size() / allSourceBundles.size();
+};
+
 // Returns a decimal showing the proportion source bundles are common to
 // both bundles versus the total number of source bundles.
-function scoreBundleMerge(bundleA, bundleB) {
-  let sharedSourceBundles = 0;
-  let allSourceBundles = new Set([...bundleA.sourceBundles, ...bundleB.sourceBundles]);
-  for (let bundle of bundleB.sourceBundles) {
-    if (bundleA.sourceBundles.has(bundle)) {
-      sharedSourceBundles++;
+function checkBundleThreshold(bundleA, bundleB, threshold) {
+  return getBundleOverlapBitSet(bundleA.sourceBundleBitSet, bundleB.sourceBundleBitSet) >= threshold;
+}
+let checkSharedSourceBundles = (0, _memoize.memoize)((bundle, importantAncestorBundles) => {
+  return importantAncestorBundles.every(ancestorId => bundle.sourceBundles.has(ancestorId));
+});
+let hasSuitableBundleGroup = (0, _memoize.memoize)((bundleGraph, bundle, minBundlesInGroup) => {
+  for (let sourceBundle of bundle.sourceBundles) {
+    let bundlesInGroup = getBundlesForBundleGroup(bundleGraph, sourceBundle);
+    if (bundlesInGroup >= minBundlesInGroup) {
+      return true;
+    }
+  }
+  return false;
+});
+function validMerge(bundleGraph, config, bundleA, bundleB) {
+  if (config.maxBundleSize != null) {
+    if (bundleA.bundle.size > config.maxBundleSize || bundleB.bundle.size > config.maxBundleSize) {
+      return false;
+    }
+  }
+  if (config.overlapThreshold != null) {
+    if (!checkBundleThreshold(bundleA, bundleB, config.overlapThreshold)) {
+      return false;
+    }
+  }
+  if (config.sourceBundles != null) {
+    if (!checkSharedSourceBundles(bundleA.bundle, config.sourceBundles) || !checkSharedSourceBundles(bundleB.bundle, config.sourceBundles)) {
+      return false;
+    }
+  }
+  if (config.minBundlesInGroup != null) {
+    if (!hasSuitableBundleGroup(bundleGraph, bundleA.bundle, config.minBundlesInGroup) || !hasSuitableBundleGroup(bundleGraph, bundleB.bundle, config.minBundlesInGroup)) {
+      return false;
     }
   }
-  return sharedSourceBundles / allSourceBundles.size;
+  return true;
 }
 function getMergeClusters(graph, candidates) {
   let clusters = [];
-  for (let candidate of candidates) {
+  for (let [candidate, edgeType] of candidates.entries()) {
     let cluster = [];
     graph.traverse(nodeId => {
       cluster.push((0, _nullthrows().default)(graph.getNode(nodeId)));
       // Remove node from candidates as it has already been processed
       candidates.delete(nodeId);
-    }, candidate);
+    }, candidate, edgeType);
     clusters.push(cluster);
   }
   return clusters;
 }
-function findMergeCandidates(bundleGraph, bundles, threshold) {
-  let graph = new (_graph().ContentGraph)();
-  let seen = new Set();
-  let candidates = new Set();
-
-  // Build graph of clustered merge candidates
-  for (let bundleId of bundles) {
+function getPossibleMergeCandidates(bundleGraph, bundles) {
+  let mergeCandidates = bundles.map(bundleId => {
     let bundle = bundleGraph.getNode(bundleId);
-    (0, _assert().default)(bundle && bundle !== 'root');
-    if (bundle.type !== 'js') {
-      continue;
+    (0, _assert().default)(bundle && bundle !== 'root', 'Bundle should exist');
+    let sourceBundleBitSet = new (_graph().BitSet)(bundleGraph.nodes.length);
+    for (let sourceBundle of bundle.sourceBundles) {
+      sourceBundleBitSet.add(sourceBundle);
     }
-    for (let otherBundleId of bundles) {
-      if (bundleId === otherBundleId) {
-        continue;
+    return {
+      id: bundleId,
+      bundle,
+      sourceBundleBitSet,
+      contentKey: bundleId.toString()
+    };
+  });
+  const uniquePairs = [];
+  for (let i = 0; i < mergeCandidates.length; i++) {
+    for (let j = i + 1; j < mergeCandidates.length; j++) {
+      let a = mergeCandidates[i];
+      let b = mergeCandidates[j];
+      if (
+      // $FlowFixMe both bundles will always have internalizedAssets
+      a.bundle.internalizedAssets.equals(b.bundle.internalizedAssets)) {
+        uniquePairs.push([a, b]);
       }
-      let key = [bundleId, otherBundleId].sort().join(':');
-      if (seen.has(key)) {
+    }
+  }
+  return uniquePairs;
+}
+function findMergeCandidates(bundleGraph, bundles, config) {
+  let graph = new (_graph().ContentGraph)();
+  let candidates = new Map();
+  let allPossibleMergeCandidates = getPossibleMergeCandidates(bundleGraph, bundles);
+
+  // Build graph of clustered merge candidates
+  for (let i = 0; i < config.length; i++) {
+    // Ensure edge type coresponds to config index
+    let edgeType = i + 1;
+    for (let group of allPossibleMergeCandidates) {
+      let candidateA = group[0];
+      let candidateB = group[1];
+      if (!validMerge(bundleGraph, config[i], candidateA, candidateB)) {
         continue;
       }
-      seen.add(key);
-      let otherBundle = bundleGraph.getNode(otherBundleId);
-      (0, _assert().default)(otherBundle && otherBundle !== 'root');
-      let score = scoreBundleMerge(bundle, otherBundle);
-      if (score >= threshold) {
-        let bundleNode = graph.addNodeByContentKeyIfNeeded(bundleId.toString(), bundleId);
-        let otherBundleNode = graph.addNodeByContentKeyIfNeeded(otherBundleId.toString(), otherBundleId);
+      let bundleNode = graph.addNodeByContentKeyIfNeeded(candidateA.contentKey, candidateA.id);
+      let otherBundleNode = graph.addNodeByContentKeyIfNeeded(candidateB.contentKey, candidateB.id);
 
-        // Add edge in both directions
-        graph.addEdge(bundleNode, otherBundleNode);
-        graph.addEdge(otherBundleNode, bundleNode);
-        candidates.add(bundleNode);
-        candidates.add(otherBundleNode);
-      }
+      // Add edge in both directions
+      graph.addEdge(bundleNode, otherBundleNode, edgeType);
+      graph.addEdge(otherBundleNode, bundleNode, edgeType);
+      candidates.set(bundleNode, edgeType);
+      candidates.set(otherBundleNode, edgeType);
     }
+
+    // Remove bundles that have been allocated to a higher priority merge
+    allPossibleMergeCandidates = allPossibleMergeCandidates.filter(group => !graph.hasContentKey(group[0].contentKey) && !graph.hasContentKey(group[1].contentKey));
   }
+  (0, _memoize.clearCaches)();
   return getMergeClusters(graph, candidates);
 }

package/lib/bundlerConfig.js

@@ -61,7 +61,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 30000,
     maxParallelRequests: 6,
     disableSharedBundles: false,
-    sharedBundleMergeThreshold: 1
+    sharedBundleMerge: []
   },
   '2': {
     minBundles: 1,
@@ -69,7 +69,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 20000,
     maxParallelRequests: 25,
     disableSharedBundles: false,
-    sharedBundleMergeThreshold: 1
+    sharedBundleMerge: []
   }
 };
 const CONFIG_SCHEMA = {
@@ -110,6 +110,30 @@ const CONFIG_SCHEMA = {
         additionalProperties: false
       }
     },
+    sharedBundleMerge: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          overlapThreshold: {
+            type: 'number'
+          },
+          maxBundleSize: {
+            type: 'number'
+          },
+          sourceBundles: {
+            type: 'array',
+            items: {
+              type: 'string'
+            }
+          },
+          minBundlesInGroup: {
+            type: 'number'
+          }
+        },
+        additionalProperties: false
+      }
+    },
     minBundles: {
       type: 'number'
     },
@@ -192,7 +216,7 @@ async function loadBundlerConfig(config, options, logger) {
   return {
     minBundles: modeConfig.minBundles ?? defaults.minBundles,
     minBundleSize: modeConfig.minBundleSize ?? defaults.minBundleSize,
-    sharedBundleMergeThreshold: modeConfig.sharedBundleMergeThreshold ?? defaults.sharedBundleMergeThreshold,
+    sharedBundleMerge: modeConfig.sharedBundleMerge ?? defaults.sharedBundleMerge,
     maxParallelRequests: modeConfig.maxParallelRequests ?? defaults.maxParallelRequests,
     projectRoot: options.projectRoot,
     disableSharedBundles: modeConfig.disableSharedBundles ?? defaults.disableSharedBundles,

package/lib/idealGraph.js

@@ -66,6 +66,7 @@ const idealBundleGraphEdges = exports.idealBundleGraphEdges = Object.freeze({
 // expect from default bundler
 
 function createIdealGraph(assetGraph, config, entries, logger) {
+  var _config$sharedBundleM;
   // Asset to the bundle and group it's an entry of
   let bundleRoots = new Map();
   let bundles = new Map();
@@ -160,17 +161,15 @@ function createIdealGraph(assetGraph, config, entries, logger) {
       assetGraph.traverse((node, _, actions) => {
         if (node.type === 'asset' && (!Array.isArray(c.types) || c.types.includes(node.value.type))) {
           let projectRelativePath = _path().default.relative(config.projectRoot, node.value.filePath);
-          if (!assetRegexes.some(regex => regex.test(projectRelativePath))) {
-            return;
-          }
 
           // We track all matching MSB's for constant modules as they are never duplicated
           // and need to be assigned to all matching bundles
           if (node.value.meta.isConstantModule === true) {
             constantModuleToMSB.get(node.value).push(c);
           }
-          manualAssetToConfig.set(node.value, c);
-          return;
+          if (assetRegexes.some(regex => regex.test(projectRelativePath))) {
+            manualAssetToConfig.set(node.value, c);
+          }
         }
         if (node.type === 'dependency' && (node.value.priority === 'lazy' || (0, _featureFlags().getFeatureFlag)('conditionalBundlingApi') && node.value.priority === 'conditional') && parentAsset) {
           // Don't walk past the bundle group assets
@@ -184,6 +183,11 @@ function createIdealGraph(assetGraph, config, entries, logger) {
     };
   }();
   let manualBundleToInternalizedAsset = new (_utils().DefaultMap)(() => []);
+  let mergeSourceBundleLookup = new Map();
+  let mergeSourceBundleAssets = new Set((_config$sharedBundleM = config.sharedBundleMerge) === null || _config$sharedBundleM === void 0 ? void 0 : _config$sharedBundleM.flatMap(c => {
+    var _c$sourceBundles;
+    return ((_c$sourceBundles = c.sourceBundles) === null || _c$sourceBundles === void 0 ? void 0 : _c$sourceBundles.map(assetMatch => _path().default.join(config.projectRoot, assetMatch))) ?? [];
+  }));
 
   /**
    * Step Create Bundles: Traverse the assetGraph (aka MutableBundleGraph) and create bundles
@@ -250,6 +254,11 @@ function createIdealGraph(assetGraph, config, entries, logger) {
               bundleRoots.set(childAsset, [bundleId, bundleId]);
               bundleGroupBundleIds.add(bundleId);
               bundleGraph.addEdge(bundleGraphRootNodeId, bundleId);
+              // If this asset is relevant for merging then track it's source
+              // bundle id for later
+              if (mergeSourceBundleAssets.has(childAsset.filePath)) {
+                mergeSourceBundleLookup.set(_path().default.relative(config.projectRoot, childAsset.filePath), bundleId);
+              }
               if (manualSharedObject) {
                 // MSB Step 4: If this was the first instance of a match, mark mainAsset for internalization
                 // since MSBs should not have main entry assets
@@ -851,8 +860,8 @@ function createIdealGraph(assetGraph, config, entries, logger) {
 
   // Step merge shared bundles that meet the overlap threshold
   // This step is skipped by default as the threshold defaults to 1
-  if (config.sharedBundleMergeThreshold < 1) {
-    mergeOverlapBundles();
+  if (config.sharedBundleMerge && config.sharedBundleMerge.length > 0) {
+    mergeOverlapBundles(config.sharedBundleMerge);
   }
 
   // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into
@@ -961,6 +970,10 @@ function createIdealGraph(assetGraph, config, entries, logger) {
       let newAssetReference = assetReference.get(asset).map(([dep, bundle]) => bundle === bundleToRemove ? [dep, bundleToKeep] : [dep, bundle]);
       assetReference.set(asset, newAssetReference);
     }
+
+    // Merge any internalized assets
+    (0, _assert().default)(bundleToKeep.internalizedAssets && bundleToRemove.internalizedAssets, 'All shared bundles should have internalized assets');
+    bundleToKeep.internalizedAssets.union(bundleToRemove.internalizedAssets);
     for (let sourceBundleId of bundleToRemove.sourceBundles) {
       if (bundleToKeep.sourceBundles.has(sourceBundleId)) {
         continue;
@@ -968,18 +981,9 @@ function createIdealGraph(assetGraph, config, entries, logger) {
       bundleToKeep.sourceBundles.add(sourceBundleId);
       bundleGraph.addEdge(sourceBundleId, bundleToKeepId);
     }
-
-    // Merge any internalized assets
-    if (bundleToRemove.internalizedAssets) {
-      if (bundleToKeep.internalizedAssets) {
-        bundleToKeep.internalizedAssets.union(bundleToRemove.internalizedAssets);
-      } else {
-        bundleToKeep.internalizedAssets = bundleToRemove.internalizedAssets;
-      }
-    }
     bundleGraph.removeNode(bundleToRemoveId);
   }
-  function mergeOverlapBundles() {
+  function mergeOverlapBundles(mergeConfig) {
     // Find all shared bundles
     let sharedBundles = new Set();
     bundleGraph.traverse(nodeId => {
@@ -998,7 +1002,19 @@ function createIdealGraph(assetGraph, config, entries, logger) {
         sharedBundles.add(nodeId);
       }
     });
-    let clusters = (0, _bundleMerge.findMergeCandidates)(bundleGraph, Array.from(sharedBundles), config.sharedBundleMergeThreshold);
+    let clusters = (0, _bundleMerge.findMergeCandidates)(bundleGraph, Array.from(sharedBundles), mergeConfig.map(config => {
+      var _config$sourceBundles;
+      return {
+        ...config,
+        sourceBundles: (_config$sourceBundles = config.sourceBundles) === null || _config$sourceBundles === void 0 ? void 0 : _config$sourceBundles.map(assetMatch => {
+          let sourceBundleNodeId = mergeSourceBundleLookup.get(assetMatch);
+          if (sourceBundleNodeId == null) {
+            throw new Error(`Source bundle ${assetMatch} not found in merge source bundle lookup`);
+          }
+          return sourceBundleNodeId;
+        })
+      };
+    }));
     for (let cluster of clusters) {
       let [mergeTarget, ...rest] = cluster;
       for (let bundleIdToMerge of rest) {

package/lib/memoize.js

@@ -0,0 +1,40 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.clearCaches = clearCaches;
+exports.memoize = memoize;
+function _manyKeysMap() {
+  const data = _interopRequireDefault(require("many-keys-map"));
+  _manyKeysMap = function () {
+    return data;
+  };
+  return data;
+}
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+// $FlowFixMe
+let caches = [];
+function clearCaches() {
+  for (let cache of caches) {
+    cache.clear();
+  }
+}
+function memoize(fn) {
+  let cache = new (_manyKeysMap().default)();
+  caches.push(cache);
+  return function (...args) {
+    // Navigate through the cache hierarchy
+    let cached = cache.get(args);
+    if (cached !== undefined) {
+      // If the result is cached, return it
+      return cached;
+    }
+
+    // Calculate the result and cache it
+    const result = fn.apply(this, args);
+    // $FlowFixMe
+    cache.set(args, result);
+    return result;
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atlaspack/bundler-default",
-  "version": "2.14.1-dev.145+fa4d24840",
+  "version": "2.14.1-dev.146+33cba81e9",
   "license": "(MIT OR Apache-2.0)",
   "type": "commonjs",
   "publishConfig": {
@@ -16,13 +16,14 @@
     "node": ">= 16.0.0"
   },
   "dependencies": {
-    "@atlaspack/diagnostic": "2.14.1-dev.145+fa4d24840",
-    "@atlaspack/feature-flags": "2.14.1-dev.145+fa4d24840",
-    "@atlaspack/graph": "3.4.1-dev.145+fa4d24840",
-    "@atlaspack/plugin": "2.14.1-dev.145+fa4d24840",
-    "@atlaspack/rust": "3.0.1-dev.145+fa4d24840",
-    "@atlaspack/utils": "2.14.1-dev.145+fa4d24840",
+    "@atlaspack/diagnostic": "2.14.1-dev.146+33cba81e9",
+    "@atlaspack/feature-flags": "2.14.1-dev.146+33cba81e9",
+    "@atlaspack/graph": "3.4.1-dev.146+33cba81e9",
+    "@atlaspack/plugin": "2.14.1-dev.146+33cba81e9",
+    "@atlaspack/rust": "3.0.1-dev.146+33cba81e9",
+    "@atlaspack/utils": "2.14.1-dev.146+33cba81e9",
+    "many-keys-map": "^1.0.3",
     "nullthrows": "^1.1.1"
   },
-  "gitHead": "fa4d24840815e7048a21d74caf01e2df633fabe3"
+  "gitHead": "33cba81e98b6eb098014f5a19d7932821b95f075"
 }

package/src/bundleMerge.js

@@ -2,102 +2,253 @@
 
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
+import {ContentGraph, BitSet} from '@atlaspack/graph';
 import type {NodeId} from '@atlaspack/graph';
 import type {Bundle, IdealBundleGraph} from './idealGraph';
-import {ContentGraph} from '@atlaspack/graph';
+import {memoize, clearCaches} from './memoize';
+
+function getBundlesForBundleGroup(
+  bundleGraph: IdealBundleGraph,
+  bundleGroupId: NodeId,
+): number {
+  let count = 0;
+  bundleGraph.traverse((nodeId) => {
+    if (bundleGraph.getNode(nodeId)?.bundleBehavior !== 'inline') {
+      count++;
+    }
+  }, bundleGroupId);
+  return count;
+}
+
+let getBundleOverlapBitSet = (
+  sourceBundlesA: BitSet,
+  sourceBundlesB: BitSet,
+): number => {
+  let allSourceBundles = BitSet.union(sourceBundlesA, sourceBundlesB);
+  let sharedSourceBundles = BitSet.intersect(sourceBundlesA, sourceBundlesB);
+
+  return sharedSourceBundles.size() / allSourceBundles.size();
+};
 
 // Returns a decimal showing the proportion source bundles are common to
 // both bundles versus the total number of source bundles.
-function scoreBundleMerge(bundleA: Bundle, bundleB: Bundle): number {
-  let sharedSourceBundles = 0;
-  let allSourceBundles = new Set([
-    ...bundleA.sourceBundles,
-    ...bundleB.sourceBundles,
-  ]);
-
-  for (let bundle of bundleB.sourceBundles) {
-    if (bundleA.sourceBundles.has(bundle)) {
-      sharedSourceBundles++;
+function checkBundleThreshold(
+  bundleA: MergeCandidate,
+  bundleB: MergeCandidate,
+  threshold: number,
+): boolean {
+  return (
+    getBundleOverlapBitSet(
+      bundleA.sourceBundleBitSet,
+      bundleB.sourceBundleBitSet,
+    ) >= threshold
+  );
+}
+
+let checkSharedSourceBundles = memoize(
+  (bundle: Bundle, importantAncestorBundles: Array<NodeId>): boolean => {
+    return importantAncestorBundles.every((ancestorId) =>
+      bundle.sourceBundles.has(ancestorId),
+    );
+  },
+);
+
+let hasSuitableBundleGroup = memoize(
+  (
+    bundleGraph: IdealBundleGraph,
+    bundle: Bundle,
+    minBundlesInGroup: number,
+  ): boolean => {
+    for (let sourceBundle of bundle.sourceBundles) {
+      let bundlesInGroup = getBundlesForBundleGroup(bundleGraph, sourceBundle);
+
+      if (bundlesInGroup >= minBundlesInGroup) {
+        return true;
+      }
+    }
+    return false;
+  },
+);
+
+function validMerge(
+  bundleGraph: IdealBundleGraph,
+  config: MergeGroup,
+  bundleA: MergeCandidate,
+  bundleB: MergeCandidate,
+): boolean {
+  if (config.maxBundleSize != null) {
+    if (
+      bundleA.bundle.size > config.maxBundleSize ||
+      bundleB.bundle.size > config.maxBundleSize
+    ) {
+      return false;
+    }
+  }
+
+  if (config.overlapThreshold != null) {
+    if (!checkBundleThreshold(bundleA, bundleB, config.overlapThreshold)) {
+      return false;
     }
   }
 
-  return sharedSourceBundles / allSourceBundles.size;
+  if (config.sourceBundles != null) {
+    if (
+      !checkSharedSourceBundles(bundleA.bundle, config.sourceBundles) ||
+      !checkSharedSourceBundles(bundleB.bundle, config.sourceBundles)
+    ) {
+      return false;
+    }
+  }
+
+  if (config.minBundlesInGroup != null) {
+    if (
+      !hasSuitableBundleGroup(
+        bundleGraph,
+        bundleA.bundle,
+        config.minBundlesInGroup,
+      ) ||
+      !hasSuitableBundleGroup(
+        bundleGraph,
+        bundleB.bundle,
+        config.minBundlesInGroup,
+      )
+    ) {
+      return false;
+    }
+  }
+
+  return true;
 }
 
 function getMergeClusters(
-  graph: ContentGraph<NodeId>,
-  candidates: Set<NodeId>,
+  graph: ContentGraph<NodeId, EdgeType>,
+  candidates: Map<NodeId, EdgeType>,
 ): Array<Array<NodeId>> {
   let clusters = [];
 
-  for (let candidate of candidates) {
+  for (let [candidate, edgeType] of candidates.entries()) {
     let cluster: Array<NodeId> = [];
 
-    graph.traverse((nodeId) => {
-      cluster.push(nullthrows(graph.getNode(nodeId)));
-      // Remove node from candidates as it has already been processed
-      candidates.delete(nodeId);
-    }, candidate);
-
+    graph.traverse(
+      (nodeId) => {
+        cluster.push(nullthrows(graph.getNode(nodeId)));
+        // Remove node from candidates as it has already been processed
+        candidates.delete(nodeId);
+      },
+      candidate,
+      edgeType,
+    );
     clusters.push(cluster);
   }
 
   return clusters;
 }
 
-export function findMergeCandidates(
+type MergeCandidate = {|
+  bundle: Bundle,
+  id: NodeId,
+  sourceBundleBitSet: BitSet,
+  contentKey: string,
+|};
+function getPossibleMergeCandidates(
   bundleGraph: IdealBundleGraph,
   bundles: Array<NodeId>,
-  threshold: number,
-): Array<Array<NodeId>> {
-  let graph = new ContentGraph<NodeId>();
-  let seen = new Set<string>();
-  let candidates = new Set<NodeId>();
-
-  // Build graph of clustered merge candidates
-  for (let bundleId of bundles) {
+): Array<[MergeCandidate, MergeCandidate]> {
+  let mergeCandidates = bundles.map((bundleId) => {
     let bundle = bundleGraph.getNode(bundleId);
-    invariant(bundle && bundle !== 'root');
-    if (bundle.type !== 'js') {
-      continue;
+    invariant(bundle && bundle !== 'root', 'Bundle should exist');
+
+    let sourceBundleBitSet = new BitSet(bundleGraph.nodes.length);
+    for (let sourceBundle of bundle.sourceBundles) {
+      sourceBundleBitSet.add(sourceBundle);
     }
 
-    for (let otherBundleId of bundles) {
-      if (bundleId === otherBundleId) {
-        continue;
+    return {
+      id: bundleId,
+      bundle,
+      sourceBundleBitSet,
+      contentKey: bundleId.toString(),
+    };
+  });
+
+  const uniquePairs = [];
+
+  for (let i = 0; i < mergeCandidates.length; i++) {
+    for (let j = i + 1; j < mergeCandidates.length; j++) {
+      let a = mergeCandidates[i];
+      let b = mergeCandidates[j];
+
+      if (
+        // $FlowFixMe both bundles will always have internalizedAssets
+        a.bundle.internalizedAssets.equals(b.bundle.internalizedAssets)
+      ) {
+        uniquePairs.push([a, b]);
       }
+    }
+  }
+  return uniquePairs;
+}
 
-      let key = [bundleId, otherBundleId].sort().join(':');
-
-      if (seen.has(key)) {
-        continue;
-      }
-      seen.add(key);
+export type MergeGroup = {|
+  overlapThreshold?: number,
+  maxBundleSize?: number,
+  sourceBundles?: Array<NodeId>,
+  minBundlesInGroup?: number,
+|};
+type EdgeType = number;
 
-      let otherBundle = bundleGraph.getNode(otherBundleId);
-      invariant(otherBundle && otherBundle !== 'root');
+export function findMergeCandidates(
+  bundleGraph: IdealBundleGraph,
+  bundles: Array<NodeId>,
+  config: Array<MergeGroup>,
+): Array<Array<NodeId>> {
+  let graph = new ContentGraph<NodeId, EdgeType>();
+  let candidates = new Map<NodeId, EdgeType>();
 
-      let score = scoreBundleMerge(bundle, otherBundle);
+  let allPossibleMergeCandidates = getPossibleMergeCandidates(
+    bundleGraph,
+    bundles,
+  );
 
-      if (score >= threshold) {
-        let bundleNode = graph.addNodeByContentKeyIfNeeded(
-          bundleId.toString(),
-          bundleId,
-        );
-        let otherBundleNode = graph.addNodeByContentKeyIfNeeded(
-          otherBundleId.toString(),
-          otherBundleId,
-        );
+  // Build graph of clustered merge candidates
+  for (let i = 0; i < config.length; i++) {
+    // Ensure edge type coresponds to config index
+    let edgeType = i + 1;
 
-        // Add edge in both directions
-        graph.addEdge(bundleNode, otherBundleNode);
-        graph.addEdge(otherBundleNode, bundleNode);
+    for (let group of allPossibleMergeCandidates) {
+      let candidateA = group[0];
+      let candidateB = group[1];
 
-        candidates.add(bundleNode);
-        candidates.add(otherBundleNode);
+      if (!validMerge(bundleGraph, config[i], candidateA, candidateB)) {
+        continue;
       }
+
+      let bundleNode = graph.addNodeByContentKeyIfNeeded(
+        candidateA.contentKey,
+        candidateA.id,
+      );
+      let otherBundleNode = graph.addNodeByContentKeyIfNeeded(
+        candidateB.contentKey,
+        candidateB.id,
+      );
+
+      // Add edge in both directions
+      graph.addEdge(bundleNode, otherBundleNode, edgeType);
+      graph.addEdge(otherBundleNode, bundleNode, edgeType);
+
+      candidates.set(bundleNode, edgeType);
+      candidates.set(otherBundleNode, edgeType);
     }
+
+    // Remove bundles that have been allocated to a higher priority merge
+    allPossibleMergeCandidates = allPossibleMergeCandidates.filter(
+      (group) =>
+        !graph.hasContentKey(group[0].contentKey) &&
+        !graph.hasContentKey(group[1].contentKey),
+    );
   }
 
+  clearCaches();
+
   return getMergeClusters(graph, candidates);
 }

package/src/bundlerConfig.js

@@ -21,6 +21,13 @@ type ManualSharedBundles = Array<{|
   split?: number,
 |}>;
 
+export type MergeCandidates = Array<{|
+  overlapThreshold?: number,
+  maxBundleSize?: number,
+  sourceBundles?: Array<string>,
+  minBundlesInGroup?: number,
+|}>;
+
 type BaseBundlerConfig = {|
   http?: number,
   minBundles?: number,
@@ -29,7 +36,7 @@ type BaseBundlerConfig = {|
   disableSharedBundles?: boolean,
   manualSharedBundles?: ManualSharedBundles,
   loadConditionalBundlesInParallel?: boolean,
-  sharedBundleMergeThreshold?: number,
+  sharedBundleMerge?: MergeCandidates,
 |};
 
 type BundlerConfig = {|
@@ -44,7 +51,7 @@ export type ResolvedBundlerConfig = {|
   disableSharedBundles: boolean,
   manualSharedBundles: ManualSharedBundles,
   loadConditionalBundlesInParallel?: boolean,
-  sharedBundleMergeThreshold: number,
+  sharedBundleMerge?: MergeCandidates,
 |};
 
 function resolveModeConfig(
@@ -79,7 +86,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 30000,
     maxParallelRequests: 6,
     disableSharedBundles: false,
-    sharedBundleMergeThreshold: 1,
+    sharedBundleMerge: [],
   },
   '2': {
     minBundles: 1,
@@ -87,7 +94,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 20000,
     maxParallelRequests: 25,
     disableSharedBundles: false,
-    sharedBundleMergeThreshold: 1,
+    sharedBundleMerge: [],
   },
 };
 
@@ -129,6 +136,30 @@ const CONFIG_SCHEMA: SchemaEntity = {
         additionalProperties: false,
       },
     },
+    sharedBundleMerge: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          overlapThreshold: {
+            type: 'number',
+          },
+          maxBundleSize: {
+            type: 'number',
+          },
+          sourceBundles: {
+            type: 'array',
+            items: {
+              type: 'string',
+            },
+          },
+          minBundlesInGroup: {
+            type: 'number',
+          },
+        },
+        additionalProperties: false,
+      },
+    },
     minBundles: {
       type: 'number',
     },
@@ -240,9 +271,8 @@ export async function loadBundlerConfig(
   return {
     minBundles: modeConfig.minBundles ?? defaults.minBundles,
     minBundleSize: modeConfig.minBundleSize ?? defaults.minBundleSize,
-    sharedBundleMergeThreshold:
-      modeConfig.sharedBundleMergeThreshold ??
-      defaults.sharedBundleMergeThreshold,
+    sharedBundleMerge:
+      modeConfig.sharedBundleMerge ?? defaults.sharedBundleMerge,
     maxParallelRequests:
       modeConfig.maxParallelRequests ?? defaults.maxParallelRequests,
     projectRoot: options.projectRoot,

package/src/idealGraph.js

@@ -23,8 +23,8 @@ import {DefaultMap, globToRegex} from '@atlaspack/utils';
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 
-import {findMergeCandidates} from './bundleMerge';
-import type {ResolvedBundlerConfig} from './bundlerConfig';
+import {findMergeCandidates, type MergeGroup} from './bundleMerge';
+import type {ResolvedBundlerConfig, MergeCandidates} from './bundlerConfig';
 
 /* BundleRoot - An asset that is the main entry of a Bundle. */
 type BundleRoot = Asset;
@@ -204,17 +204,16 @@ export function createIdealGraph(
             config.projectRoot,
             node.value.filePath,
           );
-          if (!assetRegexes.some((regex) => regex.test(projectRelativePath))) {
-            return;
-          }
 
           // We track all matching MSB's for constant modules as they are never duplicated
           // and need to be assigned to all matching bundles
           if (node.value.meta.isConstantModule === true) {
             constantModuleToMSB.get(node.value).push(c);
           }
-          manualAssetToConfig.set(node.value, c);
-          return;
+
+          if (assetRegexes.some((regex) => regex.test(projectRelativePath))) {
+            manualAssetToConfig.set(node.value, c);
+          }
         }
 
         if (
@@ -245,6 +244,16 @@ export function createIdealGraph(
     Array<Asset>,
   > = new DefaultMap(() => []);
 
+  let mergeSourceBundleLookup = new Map<string, NodeId>();
+  let mergeSourceBundleAssets = new Set(
+    config.sharedBundleMerge?.flatMap(
+      (c) =>
+        c.sourceBundles?.map((assetMatch) =>
+          path.join(config.projectRoot, assetMatch),
+        ) ?? [],
+    ),
+  );
+
   /**
    * Step Create Bundles: Traverse the assetGraph (aka MutableBundleGraph) and create bundles
    * for asset type changes, parallel, inline, and async or lazy dependencies,
@@ -335,6 +344,14 @@ export function createIdealGraph(
                 bundleRoots.set(childAsset, [bundleId, bundleId]);
                 bundleGroupBundleIds.add(bundleId);
                 bundleGraph.addEdge(bundleGraphRootNodeId, bundleId);
+                // If this asset is relevant for merging then track it's source
+                // bundle id for later
+                if (mergeSourceBundleAssets.has(childAsset.filePath)) {
+                  mergeSourceBundleLookup.set(
+                    path.relative(config.projectRoot, childAsset.filePath),
+                    bundleId,
+                  );
+                }
                 if (manualSharedObject) {
                   // MSB Step 4: If this was the first instance of a match, mark mainAsset for internalization
                   // since MSBs should not have main entry assets
@@ -1131,8 +1148,8 @@ export function createIdealGraph(
 
   // Step merge shared bundles that meet the overlap threshold
   // This step is skipped by default as the threshold defaults to 1
-  if (config.sharedBundleMergeThreshold < 1) {
-    mergeOverlapBundles();
+  if (config.sharedBundleMerge && config.sharedBundleMerge.length > 0) {
+    mergeOverlapBundles(config.sharedBundleMerge);
   }
 
   // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into
@@ -1278,6 +1295,13 @@ export function createIdealGraph(
       assetReference.set(asset, newAssetReference);
     }
 
+    // Merge any internalized assets
+    invariant(
+      bundleToKeep.internalizedAssets && bundleToRemove.internalizedAssets,
+      'All shared bundles should have internalized assets',
+    );
+    bundleToKeep.internalizedAssets.union(bundleToRemove.internalizedAssets);
+
     for (let sourceBundleId of bundleToRemove.sourceBundles) {
       if (bundleToKeep.sourceBundles.has(sourceBundleId)) {
         continue;
@@ -1287,21 +1311,10 @@ export function createIdealGraph(
       bundleGraph.addEdge(sourceBundleId, bundleToKeepId);
     }
 
-    // Merge any internalized assets
-    if (bundleToRemove.internalizedAssets) {
-      if (bundleToKeep.internalizedAssets) {
-        bundleToKeep.internalizedAssets.union(
-          bundleToRemove.internalizedAssets,
-        );
-      } else {
-        bundleToKeep.internalizedAssets = bundleToRemove.internalizedAssets;
-      }
-    }
-
     bundleGraph.removeNode(bundleToRemoveId);
   }
 
-  function mergeOverlapBundles() {
+  function mergeOverlapBundles(mergeConfig: MergeCandidates) {
     // Find all shared bundles
     let sharedBundles = new Set<NodeId>();
     bundleGraph.traverse((nodeId) => {
@@ -1331,7 +1344,20 @@ export function createIdealGraph(
     let clusters = findMergeCandidates(
       bundleGraph,
       Array.from(sharedBundles),
-      config.sharedBundleMergeThreshold,
+      mergeConfig.map((config): MergeGroup => ({
+        ...config,
+        sourceBundles: config.sourceBundles?.map((assetMatch) => {
+          let sourceBundleNodeId = mergeSourceBundleLookup.get(assetMatch);
+
+          if (sourceBundleNodeId == null) {
+            throw new Error(
+              `Source bundle ${assetMatch} not found in merge source bundle lookup`,
+            );
+          }
+
+          return sourceBundleNodeId;
+        }),
+      })),
     );
 
     for (let cluster of clusters) {

package/src/memoize.js

@@ -0,0 +1,35 @@
+// @flow strict
+
+// $FlowFixMe
+import ManyKeysMap from 'many-keys-map';
+
+let caches = [];
+
+export function clearCaches() {
+  for (let cache of caches) {
+    cache.clear();
+  }
+}
+
+export function memoize<Args: Array<mixed>, Return>(
+  fn: (...args: Args) => Return,
+): (...args: Args) => Return {
+  let cache = new ManyKeysMap();
+  caches.push(cache);
+
+  return function (...args: Args): Return {
+    // Navigate through the cache hierarchy
+    let cached = cache.get(args);
+    if (cached !== undefined) {
+      // If the result is cached, return it
+      return cached;
+    }
+
+    // Calculate the result and cache it
+    const result = fn.apply(this, args);
+    // $FlowFixMe
+    cache.set(args, result);
+
+    return result;
+  };
+}