@atlaspack/bundler-default 2.14.5-canary.6 → 2.14.5-canary.61

package/CHANGELOG.md

@@ -1,5 +1,93 @@
 # @atlaspack/bundler-default
 
+## 2.16.3
+
+### Patch Changes
+
+- Updated dependencies [[`35fdd4b`](https://github.com/atlassian-labs/atlaspack/commit/35fdd4b52da0af20f74667f7b8adfb2f90279b7c), [`6dd4ccb`](https://github.com/atlassian-labs/atlaspack/commit/6dd4ccb753541de32322d881f973d571dd57e4ca)]:
+  - @atlaspack/rust@3.3.5
+  - @atlaspack/plugin@2.14.10
+  - @atlaspack/utils@2.14.10
+
+## 2.16.2
+
+### Patch Changes
+
+- Updated dependencies [[`124b7ff`](https://github.com/atlassian-labs/atlaspack/commit/124b7fff44f71aac9fbad289a9a9509b3dfc9aaa), [`e052521`](https://github.com/atlassian-labs/atlaspack/commit/e0525210850ed1606146eb86991049cf567c5dec), [`15c6d70`](https://github.com/atlassian-labs/atlaspack/commit/15c6d7000bd89da876bc590aa75b17a619a41896), [`e4d966c`](https://github.com/atlassian-labs/atlaspack/commit/e4d966c3c9c4292c5013372ae65b10d19d4bacc6), [`209692f`](https://github.com/atlassian-labs/atlaspack/commit/209692ffb11eae103a0d65c5e1118a5aa1625818), [`42a775d`](https://github.com/atlassian-labs/atlaspack/commit/42a775de8eec638ad188f3271964170d8c04d84b), [`29c2f10`](https://github.com/atlassian-labs/atlaspack/commit/29c2f106de9679adfb5afa04e1910471dc65a427), [`f4da1e1`](https://github.com/atlassian-labs/atlaspack/commit/f4da1e120e73eeb5e8b8927f05e88f04d6148c7b), [`1ef91fc`](https://github.com/atlassian-labs/atlaspack/commit/1ef91fcc863fdd2831511937083dbbc1263b3d9d)]:
+  - @atlaspack/rust@3.3.4
+  - @atlaspack/feature-flags@2.16.0
+  - @atlaspack/utils@2.14.9
+  - @atlaspack/graph@3.4.7
+  - @atlaspack/plugin@2.14.9
+
+## 2.16.1
+
+### Patch Changes
+
+- Updated dependencies [[`30f6017`](https://github.com/atlassian-labs/atlaspack/commit/30f60175ba4d272c5fc193973c63bc298584775b), [`1ab0a27`](https://github.com/atlassian-labs/atlaspack/commit/1ab0a275aeca40350415e2b03e7440d1dddc6228), [`b8a4ae8`](https://github.com/atlassian-labs/atlaspack/commit/b8a4ae8f83dc0a83d8b145c5f729936ce52080a3)]:
+  - @atlaspack/feature-flags@2.15.1
+  - @atlaspack/rust@3.3.3
+  - @atlaspack/graph@3.4.6
+  - @atlaspack/utils@2.14.8
+  - @atlaspack/plugin@2.14.8
+
+## 2.16.0
+
+### Minor Changes
+
+- [#547](https://github.com/atlassian-labs/atlaspack/pull/547) [`a1773d2`](https://github.com/atlassian-labs/atlaspack/commit/a1773d2a62d0ef7805ac7524621dcabcc1afe929) Thanks [@benjervis](https://github.com/benjervis)! - Add a feature flag for resolving the configuration for `@atlaspack/bundler-default` from CWD, rather than exclusively from the project root.
+
+### Patch Changes
+
+- Updated dependencies [[`a1773d2`](https://github.com/atlassian-labs/atlaspack/commit/a1773d2a62d0ef7805ac7524621dcabcc1afe929), [`556d6ab`](https://github.com/atlassian-labs/atlaspack/commit/556d6ab8ede759fa7f37fcd3f4da336ef1c55e8f)]:
+  - @atlaspack/feature-flags@2.15.0
+  - @atlaspack/rust@3.3.2
+  - @atlaspack/graph@3.4.5
+  - @atlaspack/utils@2.14.7
+  - @atlaspack/plugin@2.14.7
+
+## 2.15.1
+
+### Patch Changes
+
+- Updated dependencies [[`e0f5337`](https://github.com/atlassian-labs/atlaspack/commit/e0f533757bd1019dbd108a04952c87da15286e09)]:
+  - @atlaspack/feature-flags@2.14.4
+  - @atlaspack/rust@3.3.1
+  - @atlaspack/graph@3.4.4
+  - @atlaspack/utils@2.14.6
+  - @atlaspack/plugin@2.14.6
+
+## 2.15.0
+
+### Minor Changes
+
+- [#535](https://github.com/atlassian-labs/atlaspack/pull/535) [`a4bc259`](https://github.com/atlassian-labs/atlaspack/commit/a4bc2590196b6c1e743e4edcb0337e8c4c240ab4) Thanks [@mattcompiles](https://github.com/mattcompiles)! - Add `sharedBundleMergeThreshold` config option
+
+  In apps with lots of dynamic imports, many shared bundles are often removed
+  from the output to prevent an overload in network requests according to the
+  `maxParallelRequests` config. In these cases, setting `sharedBundleMergeThreshold` can
+  merge shared bundles with a high overlap in their source bundles (bundles that share the bundle).
+  This config trades-off potential overfetching to reduce asset duplication.
+
+  The following config would merge shared bundles that have a 75% or higher overlap in source bundles.
+
+  ```json
+  {
+    "@atlaspack/bundler-default": {
+      "sharedBundleMergeThreshold": 0.75
+    }
+  }
+  ```
+
+### Patch Changes
+
+- Updated dependencies [[`11d6f16`](https://github.com/atlassian-labs/atlaspack/commit/11d6f16b6397dee2f217167e5c98b39edb63f7a7), [`e2ba0f6`](https://github.com/atlassian-labs/atlaspack/commit/e2ba0f69702656f3d1ce95ab1454e35062b13b39), [`d2c50c2`](https://github.com/atlassian-labs/atlaspack/commit/d2c50c2c020888b33bb25b8690d9320c2b69e2a6), [`46a90dc`](https://github.com/atlassian-labs/atlaspack/commit/46a90dccd019a26b222c878a92d23acc75dc67c5)]:
+  - @atlaspack/feature-flags@2.14.3
+  - @atlaspack/rust@3.3.0
+  - @atlaspack/graph@3.4.3
+  - @atlaspack/utils@2.14.5
+  - @atlaspack/plugin@2.14.5
+
 ## 2.14.4
 
 ### Patch Changes

package/lib/bundleMerge.js

@@ -0,0 +1,158 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.findMergeCandidates = findMergeCandidates;
+function _assert() {
+  const data = _interopRequireDefault(require("assert"));
+  _assert = function () {
+    return data;
+  };
+  return data;
+}
+function _nullthrows() {
+  const data = _interopRequireDefault(require("nullthrows"));
+  _nullthrows = function () {
+    return data;
+  };
+  return data;
+}
+function _graph() {
+  const data = require("@atlaspack/graph");
+  _graph = function () {
+    return data;
+  };
+  return data;
+}
+var _memoize = require("./memoize");
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function getBundlesForBundleGroup(bundleGraph, bundleGroupId) {
+  let count = 0;
+  bundleGraph.traverse(nodeId => {
+    var _bundleGraph$getNode;
+    if (((_bundleGraph$getNode = bundleGraph.getNode(nodeId)) === null || _bundleGraph$getNode === void 0 ? void 0 : _bundleGraph$getNode.bundleBehavior) !== 'inline') {
+      count++;
+    }
+  }, bundleGroupId);
+  return count;
+}
+let getBundleOverlapBitSet = (sourceBundlesA, sourceBundlesB) => {
+  let allSourceBundles = _graph().BitSet.union(sourceBundlesA, sourceBundlesB);
+  let sharedSourceBundles = _graph().BitSet.intersect(sourceBundlesA, sourceBundlesB);
+  return sharedSourceBundles.size() / allSourceBundles.size();
+};
+
+// Returns a decimal showing the proportion source bundles are common to
+// both bundles versus the total number of source bundles.
+function checkBundleThreshold(bundleA, bundleB, threshold) {
+  return getBundleOverlapBitSet(bundleA.sourceBundleBitSet, bundleB.sourceBundleBitSet) >= threshold;
+}
+let checkSharedSourceBundles = (0, _memoize.memoize)((bundle, importantAncestorBundles) => {
+  return importantAncestorBundles.every(ancestorId => bundle.sourceBundles.has(ancestorId));
+});
+let hasSuitableBundleGroup = (0, _memoize.memoize)((bundleGraph, bundle, minBundlesInGroup) => {
+  for (let sourceBundle of bundle.sourceBundles) {
+    let bundlesInGroup = getBundlesForBundleGroup(bundleGraph, sourceBundle);
+    if (bundlesInGroup >= minBundlesInGroup) {
+      return true;
+    }
+  }
+  return false;
+});
+function validMerge(bundleGraph, config, bundleA, bundleB) {
+  if (config.maxBundleSize != null) {
+    if (bundleA.bundle.size > config.maxBundleSize || bundleB.bundle.size > config.maxBundleSize) {
+      return false;
+    }
+  }
+  if (config.overlapThreshold != null) {
+    if (!checkBundleThreshold(bundleA, bundleB, config.overlapThreshold)) {
+      return false;
+    }
+  }
+  if (config.sourceBundles != null) {
+    if (!checkSharedSourceBundles(bundleA.bundle, config.sourceBundles) || !checkSharedSourceBundles(bundleB.bundle, config.sourceBundles)) {
+      return false;
+    }
+  }
+  if (config.minBundlesInGroup != null) {
+    if (!hasSuitableBundleGroup(bundleGraph, bundleA.bundle, config.minBundlesInGroup) || !hasSuitableBundleGroup(bundleGraph, bundleB.bundle, config.minBundlesInGroup)) {
+      return false;
+    }
+  }
+  return true;
+}
+function getMergeClusters(graph, candidates) {
+  let clusters = [];
+  for (let [candidate, edgeType] of candidates.entries()) {
+    let cluster = [];
+    graph.traverse(nodeId => {
+      cluster.push((0, _nullthrows().default)(graph.getNode(nodeId)));
+      // Remove node from candidates as it has already been processed
+      candidates.delete(nodeId);
+    }, candidate, edgeType);
+    clusters.push(cluster);
+  }
+  return clusters;
+}
+function getPossibleMergeCandidates(bundleGraph, bundles) {
+  let mergeCandidates = bundles.map(bundleId => {
+    let bundle = bundleGraph.getNode(bundleId);
+    (0, _assert().default)(bundle && bundle !== 'root', 'Bundle should exist');
+    let sourceBundleBitSet = new (_graph().BitSet)(bundleGraph.nodes.length);
+    for (let sourceBundle of bundle.sourceBundles) {
+      sourceBundleBitSet.add(sourceBundle);
+    }
+    return {
+      id: bundleId,
+      bundle,
+      sourceBundleBitSet,
+      contentKey: bundleId.toString()
+    };
+  });
+  const uniquePairs = [];
+  for (let i = 0; i < mergeCandidates.length; i++) {
+    for (let j = i + 1; j < mergeCandidates.length; j++) {
+      let a = mergeCandidates[i];
+      let b = mergeCandidates[j];
+      if (
+      // $FlowFixMe both bundles will always have internalizedAssets
+      a.bundle.internalizedAssets.equals(b.bundle.internalizedAssets)) {
+        uniquePairs.push([a, b]);
+      }
+    }
+  }
+  return uniquePairs;
+}
+function findMergeCandidates(bundleGraph, bundles, config) {
+  let graph = new (_graph().ContentGraph)();
+  let candidates = new Map();
+  let allPossibleMergeCandidates = getPossibleMergeCandidates(bundleGraph, bundles);
+
+  // Build graph of clustered merge candidates
+  for (let i = 0; i < config.length; i++) {
+    // Ensure edge type coresponds to config index
+    let edgeType = i + 1;
+    for (let group of allPossibleMergeCandidates) {
+      let candidateA = group[0];
+      let candidateB = group[1];
+      if (!validMerge(bundleGraph, config[i], candidateA, candidateB)) {
+        continue;
+      }
+      let bundleNode = graph.addNodeByContentKeyIfNeeded(candidateA.contentKey, candidateA.id);
+      let otherBundleNode = graph.addNodeByContentKeyIfNeeded(candidateB.contentKey, candidateB.id);
+
+      // Add edge in both directions
+      graph.addEdge(bundleNode, otherBundleNode, edgeType);
+      graph.addEdge(otherBundleNode, bundleNode, edgeType);
+      candidates.set(bundleNode, edgeType);
+      candidates.set(otherBundleNode, edgeType);
+    }
+
+    // Remove bundles that have been allocated to a higher priority merge
+    allPossibleMergeCandidates = allPossibleMergeCandidates.filter(group => !graph.hasContentKey(group[0].contentKey) && !graph.hasContentKey(group[1].contentKey));
+  }
+  (0, _memoize.clearCaches)();
+  return getMergeClusters(graph, candidates);
+}

package/lib/bundlerConfig.js

@@ -11,6 +11,13 @@ function _diagnostic() {
   };
   return data;
 }
+function _featureFlags() {
+  const data = require("@atlaspack/feature-flags");
+  _featureFlags = function () {
+    return data;
+  };
+  return data;
+}
 function _utils() {
   const data = require("@atlaspack/utils");
   _utils = function () {
@@ -53,14 +60,16 @@ const HTTP_OPTIONS = {
     manualSharedBundles: [],
     minBundleSize: 30000,
     maxParallelRequests: 6,
-    disableSharedBundles: false
+    disableSharedBundles: false,
+    sharedBundleMerge: []
   },
   '2': {
     minBundles: 1,
     manualSharedBundles: [],
     minBundleSize: 20000,
     maxParallelRequests: 25,
-    disableSharedBundles: false
+    disableSharedBundles: false,
+    sharedBundleMerge: []
   }
 };
 const CONFIG_SCHEMA = {
@@ -101,6 +110,30 @@ const CONFIG_SCHEMA = {
         additionalProperties: false
       }
     },
+    sharedBundleMerge: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          overlapThreshold: {
+            type: 'number'
+          },
+          maxBundleSize: {
+            type: 'number'
+          },
+          sourceBundles: {
+            type: 'array',
+            items: {
+              type: 'string'
+            }
+          },
+          minBundlesInGroup: {
+            type: 'number'
+          }
+        },
+        additionalProperties: false
+      }
+    },
     minBundles: {
       type: 'number'
     },
@@ -115,14 +148,25 @@ const CONFIG_SCHEMA = {
     },
     loadConditionalBundlesInParallel: {
       type: 'boolean'
+    },
+    sharedBundleMergeThreshold: {
+      type: 'number'
     }
   },
   additionalProperties: false
 };
 async function loadBundlerConfig(config, options, logger) {
-  let conf = await config.getConfig([], {
-    packageKey: '@atlaspack/bundler-default'
-  });
+  var _conf;
+  let conf;
+  if ((0, _featureFlags().getFeatureFlag)('resolveBundlerConfigFromCwd')) {
+    conf = await config.getConfigFrom(`${process.cwd()}/index`, [], {
+      packageKey: '@atlaspack/bundler-default'
+    });
+  } else {
+    conf = await config.getConfig([], {
+      packageKey: '@atlaspack/bundler-default'
+    });
+  }
   if (!conf) {
     const modDefault = {
       ...HTTP_OPTIONS['2'],
@@ -130,7 +174,7 @@ async function loadBundlerConfig(config, options, logger) {
     };
     return modDefault;
   }
-  (0, _assert().default)((conf === null || conf === void 0 ? void 0 : conf.contents) != null);
+  (0, _assert().default)(((_conf = conf) === null || _conf === void 0 ? void 0 : _conf.contents) != null);
   let modeConfig = resolveModeConfig(conf.contents, options.mode);
 
   // minBundles will be ignored if shared bundles are disabled
@@ -172,6 +216,7 @@ async function loadBundlerConfig(config, options, logger) {
   return {
     minBundles: modeConfig.minBundles ?? defaults.minBundles,
     minBundleSize: modeConfig.minBundleSize ?? defaults.minBundleSize,
+    sharedBundleMerge: modeConfig.sharedBundleMerge ?? defaults.sharedBundleMerge,
     maxParallelRequests: modeConfig.maxParallelRequests ?? defaults.maxParallelRequests,
     projectRoot: options.projectRoot,
     disableSharedBundles: modeConfig.disableSharedBundles ?? defaults.disableSharedBundles,

package/lib/idealGraph.js

@@ -47,6 +47,7 @@ function _nullthrows() {
   };
   return data;
 }
+var _bundleMerge = require("./bundleMerge");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 /* BundleRoot - An asset that is the main entry of a Bundle. */
 const dependencyPriorityEdges = {
@@ -65,6 +66,7 @@ const idealBundleGraphEdges = exports.idealBundleGraphEdges = Object.freeze({
 // expect from default bundler
 
 function createIdealGraph(assetGraph, config, entries, logger) {
+  var _config$sharedBundleM;
   // Asset to the bundle and group it's an entry of
   let bundleRoots = new Map();
   let bundles = new Map();
@@ -183,6 +185,11 @@ function createIdealGraph(assetGraph, config, entries, logger) {
     };
   }();
   let manualBundleToInternalizedAsset = new (_utils().DefaultMap)(() => []);
+  let mergeSourceBundleLookup = new Map();
+  let mergeSourceBundleAssets = new Set((_config$sharedBundleM = config.sharedBundleMerge) === null || _config$sharedBundleM === void 0 ? void 0 : _config$sharedBundleM.flatMap(c => {
+    var _c$sourceBundles;
+    return ((_c$sourceBundles = c.sourceBundles) === null || _c$sourceBundles === void 0 ? void 0 : _c$sourceBundles.map(assetMatch => _path().default.join(config.projectRoot, assetMatch))) ?? [];
+  }));
 
   /**
    * Step Create Bundles: Traverse the assetGraph (aka MutableBundleGraph) and create bundles
@@ -249,6 +256,11 @@ function createIdealGraph(assetGraph, config, entries, logger) {
               bundleRoots.set(childAsset, [bundleId, bundleId]);
               bundleGroupBundleIds.add(bundleId);
               bundleGraph.addEdge(bundleGraphRootNodeId, bundleId);
+              // If this asset is relevant for merging then track it's source
+              // bundle id for later
+              if (mergeSourceBundleAssets.has(childAsset.filePath)) {
+                mergeSourceBundleLookup.set(_path().default.relative(config.projectRoot, childAsset.filePath), bundleId);
+              }
               if (manualSharedObject) {
                 // MSB Step 4: If this was the first instance of a match, mark mainAsset for internalization
                 // since MSBs should not have main entry assets
@@ -848,6 +860,12 @@ function createIdealGraph(assetGraph, config, entries, logger) {
     }
   }
 
+  // Step merge shared bundles that meet the overlap threshold
+  // This step is skipped by default as the threshold defaults to 1
+  if (config.sharedBundleMerge && config.sharedBundleMerge.length > 0) {
+    mergeOverlapBundles(config.sharedBundleMerge);
+  }
+
   // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into
   // their source bundles, and remove the bundle.
   // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained
@@ -857,9 +875,9 @@ function createIdealGraph(assetGraph, config, entries, logger) {
       removeBundle(bundleGraph, bundleNodeId, assetReference);
     }
   }
-  let modifiedSourceBundles = new Set();
 
   // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit.
+  let modifiedSourceBundles = new Set();
   if (config.disableSharedBundles === false) {
     for (let bundleGroupId of bundleGraph.getNodeIdsConnectedFrom(rootNodeId)) {
       // Find shared bundles in this bundle group.
@@ -944,6 +962,68 @@ function createIdealGraph(assetGraph, config, entries, logger) {
       }
     }
   }
+  function mergeBundles(bundleGraph, bundleToKeepId, bundleToRemoveId, assetReference) {
+    let bundleToKeep = (0, _nullthrows().default)(bundleGraph.getNode(bundleToKeepId));
+    let bundleToRemove = (0, _nullthrows().default)(bundleGraph.getNode(bundleToRemoveId));
+    (0, _assert().default)(bundleToKeep !== 'root' && bundleToRemove !== 'root');
+    for (let asset of bundleToRemove.assets) {
+      bundleToKeep.assets.add(asset);
+      bundleToKeep.size += asset.stats.size;
+      let newAssetReference = assetReference.get(asset).map(([dep, bundle]) => bundle === bundleToRemove ? [dep, bundleToKeep] : [dep, bundle]);
+      assetReference.set(asset, newAssetReference);
+    }
+
+    // Merge any internalized assets
+    (0, _assert().default)(bundleToKeep.internalizedAssets && bundleToRemove.internalizedAssets, 'All shared bundles should have internalized assets');
+    bundleToKeep.internalizedAssets.union(bundleToRemove.internalizedAssets);
+    for (let sourceBundleId of bundleToRemove.sourceBundles) {
+      if (bundleToKeep.sourceBundles.has(sourceBundleId)) {
+        continue;
+      }
+      bundleToKeep.sourceBundles.add(sourceBundleId);
+      bundleGraph.addEdge(sourceBundleId, bundleToKeepId);
+    }
+    bundleGraph.removeNode(bundleToRemoveId);
+  }
+  function mergeOverlapBundles(mergeConfig) {
+    // Find all shared bundles
+    let sharedBundles = new Set();
+    bundleGraph.traverse(nodeId => {
+      let bundle = bundleGraph.getNode(nodeId);
+      if (!bundle) {
+        throw new Error(`Unable to find bundle ${nodeId} in bundle graph`);
+      }
+      if (bundle === 'root') {
+        return;
+      }
+
+      // Only consider JS shared bundles and non-reused bundles.
+      // These count potentially be considered for merging in future but they're
+      // more complicated to merge
+      if (bundle.sourceBundles.size > 0 && bundle.manualSharedBundle == null && !bundle.mainEntryAsset && bundle.type === 'js') {
+        sharedBundles.add(nodeId);
+      }
+    });
+    let clusters = (0, _bundleMerge.findMergeCandidates)(bundleGraph, Array.from(sharedBundles), mergeConfig.map(config => {
+      var _config$sourceBundles;
+      return {
+        ...config,
+        sourceBundles: (_config$sourceBundles = config.sourceBundles) === null || _config$sourceBundles === void 0 ? void 0 : _config$sourceBundles.map(assetMatch => {
+          let sourceBundleNodeId = mergeSourceBundleLookup.get(assetMatch);
+          if (sourceBundleNodeId == null) {
+            throw new Error(`Source bundle ${assetMatch} not found in merge source bundle lookup`);
+          }
+          return sourceBundleNodeId;
+        })
+      };
+    }));
+    for (let cluster of clusters) {
+      let [mergeTarget, ...rest] = cluster;
+      for (let bundleIdToMerge of rest) {
+        mergeBundles(bundleGraph, mergeTarget, bundleIdToMerge, assetReference);
+      }
+    }
+  }
   function getBigIntFromContentKey(contentKey) {
     let b = Buffer.alloc(64);
     b.write(contentKey);

package/lib/memoize.js

@@ -0,0 +1,40 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.clearCaches = clearCaches;
+exports.memoize = memoize;
+function _manyKeysMap() {
+  const data = _interopRequireDefault(require("many-keys-map"));
+  _manyKeysMap = function () {
+    return data;
+  };
+  return data;
+}
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+// $FlowFixMe
+let caches = [];
+function clearCaches() {
+  for (let cache of caches) {
+    cache.clear();
+  }
+}
+function memoize(fn) {
+  let cache = new (_manyKeysMap().default)();
+  caches.push(cache);
+  return function (...args) {
+    // Navigate through the cache hierarchy
+    let cached = cache.get(args);
+    if (cached !== undefined) {
+      // If the result is cached, return it
+      return cached;
+    }
+
+    // Calculate the result and cache it
+    const result = fn.apply(this, args);
+    // $FlowFixMe
+    cache.set(args, result);
+    return result;
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atlaspack/bundler-default",
-  "version": "2.14.5-canary.6+b5da6b749",
+  "version": "2.14.5-canary.61+1b52b99db",
   "license": "(MIT OR Apache-2.0)",
   "type": "commonjs",
   "publishConfig": {
@@ -16,13 +16,14 @@
     "node": ">= 16.0.0"
   },
   "dependencies": {
-    "@atlaspack/diagnostic": "2.14.1-canary.74+b5da6b749",
-    "@atlaspack/feature-flags": "2.14.1-canary.74+b5da6b749",
-    "@atlaspack/graph": "3.4.1-canary.74+b5da6b749",
-    "@atlaspack/plugin": "2.14.5-canary.6+b5da6b749",
-    "@atlaspack/rust": "3.2.1-canary.6+b5da6b749",
-    "@atlaspack/utils": "2.14.5-canary.6+b5da6b749",
+    "@atlaspack/diagnostic": "2.14.1-canary.129+1b52b99db",
+    "@atlaspack/feature-flags": "2.14.1-canary.129+1b52b99db",
+    "@atlaspack/graph": "3.4.1-canary.129+1b52b99db",
+    "@atlaspack/plugin": "2.14.5-canary.61+1b52b99db",
+    "@atlaspack/rust": "3.2.1-canary.61+1b52b99db",
+    "@atlaspack/utils": "2.14.5-canary.61+1b52b99db",
+    "many-keys-map": "^2.0.1",
     "nullthrows": "^1.1.1"
   },
-  "gitHead": "b5da6b749ddb23cfc212a640df1f07850da8307f"
+  "gitHead": "1b52b99db4298b04c1a6eb0f97994d75a2d436f9"
 }

package/src/bundleMerge.js

@@ -0,0 +1,254 @@
+// @flow strict-local
+
+import invariant from 'assert';
+import nullthrows from 'nullthrows';
+import {ContentGraph, BitSet} from '@atlaspack/graph';
+import type {NodeId} from '@atlaspack/graph';
+import type {Bundle, IdealBundleGraph} from './idealGraph';
+import {memoize, clearCaches} from './memoize';
+
+function getBundlesForBundleGroup(
+  bundleGraph: IdealBundleGraph,
+  bundleGroupId: NodeId,
+): number {
+  let count = 0;
+  bundleGraph.traverse((nodeId) => {
+    if (bundleGraph.getNode(nodeId)?.bundleBehavior !== 'inline') {
+      count++;
+    }
+  }, bundleGroupId);
+  return count;
+}
+
+let getBundleOverlapBitSet = (
+  sourceBundlesA: BitSet,
+  sourceBundlesB: BitSet,
+): number => {
+  let allSourceBundles = BitSet.union(sourceBundlesA, sourceBundlesB);
+  let sharedSourceBundles = BitSet.intersect(sourceBundlesA, sourceBundlesB);
+
+  return sharedSourceBundles.size() / allSourceBundles.size();
+};
+
+// Returns a decimal showing the proportion source bundles are common to
+// both bundles versus the total number of source bundles.
+function checkBundleThreshold(
+  bundleA: MergeCandidate,
+  bundleB: MergeCandidate,
+  threshold: number,
+): boolean {
+  return (
+    getBundleOverlapBitSet(
+      bundleA.sourceBundleBitSet,
+      bundleB.sourceBundleBitSet,
+    ) >= threshold
+  );
+}
+
+let checkSharedSourceBundles = memoize(
+  (bundle: Bundle, importantAncestorBundles: Array<NodeId>): boolean => {
+    return importantAncestorBundles.every((ancestorId) =>
+      bundle.sourceBundles.has(ancestorId),
+    );
+  },
+);
+
+let hasSuitableBundleGroup = memoize(
+  (
+    bundleGraph: IdealBundleGraph,
+    bundle: Bundle,
+    minBundlesInGroup: number,
+  ): boolean => {
+    for (let sourceBundle of bundle.sourceBundles) {
+      let bundlesInGroup = getBundlesForBundleGroup(bundleGraph, sourceBundle);
+
+      if (bundlesInGroup >= minBundlesInGroup) {
+        return true;
+      }
+    }
+    return false;
+  },
+);
+
+function validMerge(
+  bundleGraph: IdealBundleGraph,
+  config: MergeGroup,
+  bundleA: MergeCandidate,
+  bundleB: MergeCandidate,
+): boolean {
+  if (config.maxBundleSize != null) {
+    if (
+      bundleA.bundle.size > config.maxBundleSize ||
+      bundleB.bundle.size > config.maxBundleSize
+    ) {
+      return false;
+    }
+  }
+
+  if (config.overlapThreshold != null) {
+    if (!checkBundleThreshold(bundleA, bundleB, config.overlapThreshold)) {
+      return false;
+    }
+  }
+
+  if (config.sourceBundles != null) {
+    if (
+      !checkSharedSourceBundles(bundleA.bundle, config.sourceBundles) ||
+      !checkSharedSourceBundles(bundleB.bundle, config.sourceBundles)
+    ) {
+      return false;
+    }
+  }
+
+  if (config.minBundlesInGroup != null) {
+    if (
+      !hasSuitableBundleGroup(
+        bundleGraph,
+        bundleA.bundle,
+        config.minBundlesInGroup,
+      ) ||
+      !hasSuitableBundleGroup(
+        bundleGraph,
+        bundleB.bundle,
+        config.minBundlesInGroup,
+      )
+    ) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function getMergeClusters(
+  graph: ContentGraph<NodeId, EdgeType>,
+  candidates: Map<NodeId, EdgeType>,
+): Array<Array<NodeId>> {
+  let clusters = [];
+
+  for (let [candidate, edgeType] of candidates.entries()) {
+    let cluster: Array<NodeId> = [];
+
+    graph.traverse(
+      (nodeId) => {
+        cluster.push(nullthrows(graph.getNode(nodeId)));
+        // Remove node from candidates as it has already been processed
+        candidates.delete(nodeId);
+      },
+      candidate,
+      edgeType,
+    );
+    clusters.push(cluster);
+  }
+
+  return clusters;
+}
+
+type MergeCandidate = {|
+  bundle: Bundle,
+  id: NodeId,
+  sourceBundleBitSet: BitSet,
+  contentKey: string,
+|};
+function getPossibleMergeCandidates(
+  bundleGraph: IdealBundleGraph,
+  bundles: Array<NodeId>,
+): Array<[MergeCandidate, MergeCandidate]> {
+  let mergeCandidates = bundles.map((bundleId) => {
+    let bundle = bundleGraph.getNode(bundleId);
+    invariant(bundle && bundle !== 'root', 'Bundle should exist');
+
+    let sourceBundleBitSet = new BitSet(bundleGraph.nodes.length);
+    for (let sourceBundle of bundle.sourceBundles) {
+      sourceBundleBitSet.add(sourceBundle);
+    }
+
+    return {
+      id: bundleId,
+      bundle,
+      sourceBundleBitSet,
+      contentKey: bundleId.toString(),
+    };
+  });
+
+  const uniquePairs = [];
+
+  for (let i = 0; i < mergeCandidates.length; i++) {
+    for (let j = i + 1; j < mergeCandidates.length; j++) {
+      let a = mergeCandidates[i];
+      let b = mergeCandidates[j];
+
+      if (
+        // $FlowFixMe both bundles will always have internalizedAssets
+        a.bundle.internalizedAssets.equals(b.bundle.internalizedAssets)
+      ) {
+        uniquePairs.push([a, b]);
+      }
+    }
+  }
+  return uniquePairs;
+}
+
+export type MergeGroup = {|
+  overlapThreshold?: number,
+  maxBundleSize?: number,
+  sourceBundles?: Array<NodeId>,
+  minBundlesInGroup?: number,
+|};
+type EdgeType = number;
+
+export function findMergeCandidates(
+  bundleGraph: IdealBundleGraph,
+  bundles: Array<NodeId>,
+  config: Array<MergeGroup>,
+): Array<Array<NodeId>> {
+  let graph = new ContentGraph<NodeId, EdgeType>();
+  let candidates = new Map<NodeId, EdgeType>();
+
+  let allPossibleMergeCandidates = getPossibleMergeCandidates(
+    bundleGraph,
+    bundles,
+  );
+
+  // Build graph of clustered merge candidates
+  for (let i = 0; i < config.length; i++) {
+    // Ensure edge type coresponds to config index
+    let edgeType = i + 1;
+
+    for (let group of allPossibleMergeCandidates) {
+      let candidateA = group[0];
+      let candidateB = group[1];
+
+      if (!validMerge(bundleGraph, config[i], candidateA, candidateB)) {
+        continue;
+      }
+
+      let bundleNode = graph.addNodeByContentKeyIfNeeded(
+        candidateA.contentKey,
+        candidateA.id,
+      );
+      let otherBundleNode = graph.addNodeByContentKeyIfNeeded(
+        candidateB.contentKey,
+        candidateB.id,
+      );
+
+      // Add edge in both directions
+      graph.addEdge(bundleNode, otherBundleNode, edgeType);
+      graph.addEdge(otherBundleNode, bundleNode, edgeType);
+
+      candidates.set(bundleNode, edgeType);
+      candidates.set(otherBundleNode, edgeType);
+    }
+
+    // Remove bundles that have been allocated to a higher priority merge
+    allPossibleMergeCandidates = allPossibleMergeCandidates.filter(
+      (group) =>
+        !graph.hasContentKey(group[0].contentKey) &&
+        !graph.hasContentKey(group[1].contentKey),
+    );
+  }
+
+  clearCaches();
+
+  return getMergeClusters(graph, candidates);
+}

package/src/bundlerConfig.js

@@ -7,6 +7,7 @@ import type {
   BuildMode,
   PluginLogger,
 } from '@atlaspack/types';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {type SchemaEntity, validateSchema} from '@atlaspack/utils';
 import invariant from 'assert';
 
@@ -20,6 +21,13 @@ type ManualSharedBundles = Array<{|
   split?: number,
 |}>;
 
+export type MergeCandidates = Array<{|
+  overlapThreshold?: number,
+  maxBundleSize?: number,
+  sourceBundles?: Array<string>,
+  minBundlesInGroup?: number,
+|}>;
+
 type BaseBundlerConfig = {|
   http?: number,
   minBundles?: number,
@@ -28,6 +36,7 @@ type BaseBundlerConfig = {|
   disableSharedBundles?: boolean,
   manualSharedBundles?: ManualSharedBundles,
   loadConditionalBundlesInParallel?: boolean,
+  sharedBundleMerge?: MergeCandidates,
 |};
 
 type BundlerConfig = {|
@@ -42,6 +51,7 @@ export type ResolvedBundlerConfig = {|
   disableSharedBundles: boolean,
   manualSharedBundles: ManualSharedBundles,
   loadConditionalBundlesInParallel?: boolean,
+  sharedBundleMerge?: MergeCandidates,
 |};
 
 function resolveModeConfig(
@@ -76,6 +86,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 30000,
     maxParallelRequests: 6,
     disableSharedBundles: false,
+    sharedBundleMerge: [],
   },
   '2': {
     minBundles: 1,
@@ -83,6 +94,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 20000,
     maxParallelRequests: 25,
     disableSharedBundles: false,
+    sharedBundleMerge: [],
   },
 };
 
@@ -124,6 +136,30 @@ const CONFIG_SCHEMA: SchemaEntity = {
         additionalProperties: false,
       },
     },
+    sharedBundleMerge: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          overlapThreshold: {
+            type: 'number',
+          },
+          maxBundleSize: {
+            type: 'number',
+          },
+          sourceBundles: {
+            type: 'array',
+            items: {
+              type: 'string',
+            },
+          },
+          minBundlesInGroup: {
+            type: 'number',
+          },
+        },
+        additionalProperties: false,
+      },
+    },
     minBundles: {
       type: 'number',
     },
@@ -139,6 +175,9 @@ const CONFIG_SCHEMA: SchemaEntity = {
     loadConditionalBundlesInParallel: {
       type: 'boolean',
     },
+    sharedBundleMergeThreshold: {
+      type: 'number',
+    },
   },
   additionalProperties: false,
 };
@@ -148,9 +187,17 @@ export async function loadBundlerConfig(
   options: PluginOptions,
   logger: PluginLogger,
 ): Promise<ResolvedBundlerConfig> {
-  let conf = await config.getConfig<BundlerConfig>([], {
-    packageKey: '@atlaspack/bundler-default',
-  });
+  let conf;
+
+  if (getFeatureFlag('resolveBundlerConfigFromCwd')) {
+    conf = await config.getConfigFrom(`${process.cwd()}/index`, [], {
+      packageKey: '@atlaspack/bundler-default',
+    });
+  } else {
+    conf = await config.getConfig<BundlerConfig>([], {
+      packageKey: '@atlaspack/bundler-default',
+    });
+  }
 
   if (!conf) {
     const modDefault = {
@@ -224,6 +271,8 @@ export async function loadBundlerConfig(
   return {
     minBundles: modeConfig.minBundles ?? defaults.minBundles,
     minBundleSize: modeConfig.minBundleSize ?? defaults.minBundleSize,
+    sharedBundleMerge:
+      modeConfig.sharedBundleMerge ?? defaults.sharedBundleMerge,
     maxParallelRequests:
       modeConfig.maxParallelRequests ?? defaults.maxParallelRequests,
     projectRoot: options.projectRoot,

package/src/idealGraph.js

@@ -23,7 +23,8 @@ import {DefaultMap, globToRegex} from '@atlaspack/utils';
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 
-import type {ResolvedBundlerConfig} from './bundlerConfig';
+import {findMergeCandidates, type MergeGroup} from './bundleMerge';
+import type {ResolvedBundlerConfig, MergeCandidates} from './bundlerConfig';
 
 /* BundleRoot - An asset that is the main entry of a Bundle. */
 type BundleRoot = Asset;
@@ -67,7 +68,7 @@ export const idealBundleGraphEdges = Object.freeze({
   conditional: 2,
 });
 
-type IdealBundleGraph = Graph<
+export type IdealBundleGraph = Graph<
   Bundle | 'root',
   $Values<typeof idealBundleGraphEdges>,
 >;
@@ -244,6 +245,16 @@ export function createIdealGraph(
     Array<Asset>,
   > = new DefaultMap(() => []);
 
+  let mergeSourceBundleLookup = new Map<string, NodeId>();
+  let mergeSourceBundleAssets = new Set(
+    config.sharedBundleMerge?.flatMap(
+      (c) =>
+        c.sourceBundles?.map((assetMatch) =>
+          path.join(config.projectRoot, assetMatch),
+        ) ?? [],
+    ),
+  );
+
   /**
    * Step Create Bundles: Traverse the assetGraph (aka MutableBundleGraph) and create bundles
    * for asset type changes, parallel, inline, and async or lazy dependencies,
@@ -334,6 +345,14 @@ export function createIdealGraph(
                 bundleRoots.set(childAsset, [bundleId, bundleId]);
                 bundleGroupBundleIds.add(bundleId);
                 bundleGraph.addEdge(bundleGraphRootNodeId, bundleId);
+                // If this asset is relevant for merging then track it's source
+                // bundle id for later
+                if (mergeSourceBundleAssets.has(childAsset.filePath)) {
+                  mergeSourceBundleLookup.set(
+                    path.relative(config.projectRoot, childAsset.filePath),
+                    bundleId,
+                  );
+                }
                 if (manualSharedObject) {
                   // MSB Step 4: If this was the first instance of a match, mark mainAsset for internalization
                   // since MSBs should not have main entry assets
@@ -1128,6 +1147,12 @@ export function createIdealGraph(
     }
   }
 
+  // Step merge shared bundles that meet the overlap threshold
+  // This step is skipped by default as the threshold defaults to 1
+  if (config.sharedBundleMerge && config.sharedBundleMerge.length > 0) {
+    mergeOverlapBundles(config.sharedBundleMerge);
+  }
+
   // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into
   // their source bundles, and remove the bundle.
   // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained
@@ -1143,9 +1168,9 @@ export function createIdealGraph(
     }
   }
 
+  // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit.
   let modifiedSourceBundles = new Set();
 
-  // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit.
   if (config.disableSharedBundles === false) {
     for (let bundleGroupId of bundleGraph.getNodeIdsConnectedFrom(rootNodeId)) {
       // Find shared bundles in this bundle group.
@@ -1249,6 +1274,102 @@ export function createIdealGraph(
     }
   }
 
+  function mergeBundles(
+    bundleGraph: IdealBundleGraph,
+    bundleToKeepId: NodeId,
+    bundleToRemoveId: NodeId,
+    assetReference: DefaultMap<Asset, Array<[Dependency, Bundle]>>,
+  ) {
+    let bundleToKeep = nullthrows(bundleGraph.getNode(bundleToKeepId));
+    let bundleToRemove = nullthrows(bundleGraph.getNode(bundleToRemoveId));
+    invariant(bundleToKeep !== 'root' && bundleToRemove !== 'root');
+    for (let asset of bundleToRemove.assets) {
+      bundleToKeep.assets.add(asset);
+      bundleToKeep.size += asset.stats.size;
+
+      let newAssetReference = assetReference
+        .get(asset)
+        .map(([dep, bundle]) =>
+          bundle === bundleToRemove ? [dep, bundleToKeep] : [dep, bundle],
+        );
+
+      assetReference.set(asset, newAssetReference);
+    }
+
+    // Merge any internalized assets
+    invariant(
+      bundleToKeep.internalizedAssets && bundleToRemove.internalizedAssets,
+      'All shared bundles should have internalized assets',
+    );
+    bundleToKeep.internalizedAssets.union(bundleToRemove.internalizedAssets);
+
+    for (let sourceBundleId of bundleToRemove.sourceBundles) {
+      if (bundleToKeep.sourceBundles.has(sourceBundleId)) {
+        continue;
+      }
+
+      bundleToKeep.sourceBundles.add(sourceBundleId);
+      bundleGraph.addEdge(sourceBundleId, bundleToKeepId);
+    }
+
+    bundleGraph.removeNode(bundleToRemoveId);
+  }
+
+  function mergeOverlapBundles(mergeConfig: MergeCandidates) {
+    // Find all shared bundles
+    let sharedBundles = new Set<NodeId>();
+    bundleGraph.traverse((nodeId) => {
+      let bundle = bundleGraph.getNode(nodeId);
+
+      if (!bundle) {
+        throw new Error(`Unable to find bundle ${nodeId} in bundle graph`);
+      }
+
+      if (bundle === 'root') {
+        return;
+      }
+
+      // Only consider JS shared bundles and non-reused bundles.
+      // These count potentially be considered for merging in future but they're
+      // more complicated to merge
+      if (
+        bundle.sourceBundles.size > 0 &&
+        bundle.manualSharedBundle == null &&
+        !bundle.mainEntryAsset &&
+        bundle.type === 'js'
+      ) {
+        sharedBundles.add(nodeId);
+      }
+    });
+
+    let clusters = findMergeCandidates(
+      bundleGraph,
+      Array.from(sharedBundles),
+      mergeConfig.map((config): MergeGroup => ({
+        ...config,
+        sourceBundles: config.sourceBundles?.map((assetMatch) => {
+          let sourceBundleNodeId = mergeSourceBundleLookup.get(assetMatch);
+
+          if (sourceBundleNodeId == null) {
+            throw new Error(
+              `Source bundle ${assetMatch} not found in merge source bundle lookup`,
+            );
+          }
+
+          return sourceBundleNodeId;
+        }),
+      })),
+    );
+
+    for (let cluster of clusters) {
+      let [mergeTarget, ...rest] = cluster;
+
+      for (let bundleIdToMerge of rest) {
+        mergeBundles(bundleGraph, mergeTarget, bundleIdToMerge, assetReference);
+      }
+    }
+  }
+
   function getBigIntFromContentKey(contentKey) {
     let b = Buffer.alloc(64);
     b.write(contentKey);

package/src/memoize.js

@@ -0,0 +1,35 @@
+// @flow strict
+
+// $FlowFixMe
+import ManyKeysMap from 'many-keys-map';
+
+let caches = [];
+
+export function clearCaches() {
+  for (let cache of caches) {
+    cache.clear();
+  }
+}
+
+export function memoize<Args: Array<mixed>, Return>(
+  fn: (...args: Args) => Return,
+): (...args: Args) => Return {
+  let cache = new ManyKeysMap();
+  caches.push(cache);
+
+  return function (...args: Args): Return {
+    // Navigate through the cache hierarchy
+    let cached = cache.get(args);
+    if (cached !== undefined) {
+      // If the result is cached, return it
+      return cached;
+    }
+
+    // Calculate the result and cache it
+    const result = fn.apply(this, args);
+    // $FlowFixMe
+    cache.set(args, result);
+
+    return result;
+  };
+}