@atlaspack/bundler-default 2.14.5-canary.6 → 2.14.5-canary.8

package/lib/bundleMerge.js

@@ -0,0 +1,91 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.findMergeCandidates = findMergeCandidates;
+function _assert() {
+  const data = _interopRequireDefault(require("assert"));
+  _assert = function () {
+    return data;
+  };
+  return data;
+}
+function _nullthrows() {
+  const data = _interopRequireDefault(require("nullthrows"));
+  _nullthrows = function () {
+    return data;
+  };
+  return data;
+}
+function _graph() {
+  const data = require("@atlaspack/graph");
+  _graph = function () {
+    return data;
+  };
+  return data;
+}
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+// Returns a decimal showing the proportion source bundles are common to
+// both bundles versus the total number of source bundles.
+function scoreBundleMerge(bundleA, bundleB) {
+  let sharedSourceBundles = 0;
+  let allSourceBundles = new Set([...bundleA.sourceBundles, ...bundleB.sourceBundles]);
+  for (let bundle of bundleB.sourceBundles) {
+    if (bundleA.sourceBundles.has(bundle)) {
+      sharedSourceBundles++;
+    }
+  }
+  return sharedSourceBundles / allSourceBundles.size;
+}
+function getMergeClusters(graph, candidates) {
+  let clusters = [];
+  for (let candidate of candidates) {
+    let cluster = [];
+    graph.traverse(nodeId => {
+      cluster.push((0, _nullthrows().default)(graph.getNode(nodeId)));
+      // Remove node from candidates as it has already been processed
+      candidates.delete(nodeId);
+    }, candidate);
+    clusters.push(cluster);
+  }
+  return clusters;
+}
+function findMergeCandidates(bundleGraph, bundles, threshold) {
+  let graph = new (_graph().ContentGraph)();
+  let seen = new Set();
+  let candidates = new Set();
+
+  // Build graph of clustered merge candidates
+  for (let bundleId of bundles) {
+    let bundle = bundleGraph.getNode(bundleId);
+    (0, _assert().default)(bundle && bundle !== 'root');
+    if (bundle.type !== 'js') {
+      continue;
+    }
+    for (let otherBundleId of bundles) {
+      if (bundleId === otherBundleId) {
+        continue;
+      }
+      let key = [bundleId, otherBundleId].sort().join(':');
+      if (seen.has(key)) {
+        continue;
+      }
+      seen.add(key);
+      let otherBundle = bundleGraph.getNode(otherBundleId);
+      (0, _assert().default)(otherBundle && otherBundle !== 'root');
+      let score = scoreBundleMerge(bundle, otherBundle);
+      if (score >= threshold) {
+        let bundleNode = graph.addNodeByContentKeyIfNeeded(bundleId.toString(), bundleId);
+        let otherBundleNode = graph.addNodeByContentKeyIfNeeded(otherBundleId.toString(), otherBundleId);
+
+        // Add edge in both directions
+        graph.addEdge(bundleNode, otherBundleNode);
+        graph.addEdge(otherBundleNode, bundleNode);
+        candidates.add(bundleNode);
+        candidates.add(otherBundleNode);
+      }
+    }
+  }
+  return getMergeClusters(graph, candidates);
+}

package/lib/bundlerConfig.js

@@ -53,14 +53,16 @@ const HTTP_OPTIONS = {
     manualSharedBundles: [],
     minBundleSize: 30000,
     maxParallelRequests: 6,
-    disableSharedBundles: false
+    disableSharedBundles: false,
+    sharedBundleMergeThreshold: 1
   },
   '2': {
     minBundles: 1,
     manualSharedBundles: [],
     minBundleSize: 20000,
     maxParallelRequests: 25,
-    disableSharedBundles: false
+    disableSharedBundles: false,
+    sharedBundleMergeThreshold: 1
   }
 };
 const CONFIG_SCHEMA = {
@@ -115,6 +117,9 @@ const CONFIG_SCHEMA = {
     },
     loadConditionalBundlesInParallel: {
       type: 'boolean'
+    },
+    sharedBundleMergeThreshold: {
+      type: 'number'
     }
   },
   additionalProperties: false
@@ -172,6 +177,7 @@ async function loadBundlerConfig(config, options, logger) {
   return {
     minBundles: modeConfig.minBundles ?? defaults.minBundles,
     minBundleSize: modeConfig.minBundleSize ?? defaults.minBundleSize,
+    sharedBundleMergeThreshold: modeConfig.sharedBundleMergeThreshold ?? defaults.sharedBundleMergeThreshold,
     maxParallelRequests: modeConfig.maxParallelRequests ?? defaults.maxParallelRequests,
     projectRoot: options.projectRoot,
     disableSharedBundles: modeConfig.disableSharedBundles ?? defaults.disableSharedBundles,

package/lib/idealGraph.js

@@ -47,6 +47,7 @@ function _nullthrows() {
   };
   return data;
 }
+var _bundleMerge = require("./bundleMerge");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 /* BundleRoot - An asset that is the main entry of a Bundle. */
 const dependencyPriorityEdges = {
@@ -848,6 +849,12 @@ function createIdealGraph(assetGraph, config, entries, logger) {
     }
   }
 
+  // Step merge shared bundles that meet the overlap threshold
+  // This step is skipped by default as the threshold defaults to 1
+  if (config.sharedBundleMergeThreshold < 1) {
+    mergeOverlapBundles();
+  }
+
   // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into
   // their source bundles, and remove the bundle.
   // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained
@@ -857,9 +864,9 @@ function createIdealGraph(assetGraph, config, entries, logger) {
       removeBundle(bundleGraph, bundleNodeId, assetReference);
     }
   }
-  let modifiedSourceBundles = new Set();
 
   // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit.
+  let modifiedSourceBundles = new Set();
   if (config.disableSharedBundles === false) {
     for (let bundleGroupId of bundleGraph.getNodeIdsConnectedFrom(rootNodeId)) {
       // Find shared bundles in this bundle group.
@@ -944,6 +951,61 @@ function createIdealGraph(assetGraph, config, entries, logger) {
       }
     }
   }
+  function mergeBundles(bundleGraph, bundleToKeepId, bundleToRemoveId, assetReference) {
+    let bundleToKeep = (0, _nullthrows().default)(bundleGraph.getNode(bundleToKeepId), '18');
+    let bundleToRemove = (0, _nullthrows().default)(bundleGraph.getNode(bundleToRemoveId), '19');
+    (0, _assert().default)(bundleToKeep !== 'root' && bundleToRemove !== 'root');
+    for (let asset of bundleToRemove.assets) {
+      bundleToKeep.assets.add(asset);
+      bundleToKeep.size += asset.stats.size;
+      let newAssetReference = assetReference.get(asset).map(([dep, bundle]) => bundle === bundleToRemove ? [dep, bundleToKeep] : [dep, bundle]);
+      assetReference.set(asset, newAssetReference);
+    }
+    for (let sourceBundleId of bundleToRemove.sourceBundles) {
+      if (bundleToKeep.sourceBundles.has(sourceBundleId)) {
+        continue;
+      }
+      bundleToKeep.sourceBundles.add(sourceBundleId);
+      bundleGraph.addEdge(sourceBundleId, bundleToKeepId);
+    }
+
+    // Merge any internalized assets
+    if (bundleToRemove.internalizedAssets) {
+      if (bundleToKeep.internalizedAssets) {
+        bundleToKeep.internalizedAssets.union(bundleToRemove.internalizedAssets);
+      } else {
+        bundleToKeep.internalizedAssets = bundleToRemove.internalizedAssets;
+      }
+    }
+    bundleGraph.removeNode(bundleToRemoveId);
+  }
+  function mergeOverlapBundles() {
+    // Find all shared bundles
+    let sharedBundles = new Set();
+    bundleGraph.traverse(nodeId => {
+      let bundle = bundleGraph.getNode(nodeId);
+      if (!bundle) {
+        throw new Error(`Unable to find bundle ${nodeId} in bundle graph`);
+      }
+      if (bundle === 'root') {
+        return;
+      }
+
+      // Only consider JS shared bundles and non-reused bundles.
+      // These count potentially be considered for merging in future but they're
+      // more complicated to merge
+      if (bundle.sourceBundles.size > 0 && bundle.manualSharedBundle == null && !bundle.mainEntryAsset && bundle.type === 'js') {
+        sharedBundles.add(nodeId);
+      }
+    });
+    let clusters = (0, _bundleMerge.findMergeCandidates)(bundleGraph, Array.from(sharedBundles), config.sharedBundleMergeThreshold);
+    for (let cluster of clusters) {
+      let [mergeTarget, ...rest] = cluster;
+      for (let bundleIdToMerge of rest) {
+        mergeBundles(bundleGraph, mergeTarget, bundleIdToMerge, assetReference);
+      }
+    }
+  }
   function getBigIntFromContentKey(contentKey) {
     let b = Buffer.alloc(64);
     b.write(contentKey);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atlaspack/bundler-default",
-  "version": "2.14.5-canary.6+b5da6b749",
+  "version": "2.14.5-canary.8+46a90dccd",
   "license": "(MIT OR Apache-2.0)",
   "type": "commonjs",
   "publishConfig": {
@@ -16,13 +16,13 @@
     "node": ">= 16.0.0"
   },
   "dependencies": {
-    "@atlaspack/diagnostic": "2.14.1-canary.74+b5da6b749",
-    "@atlaspack/feature-flags": "2.14.1-canary.74+b5da6b749",
-    "@atlaspack/graph": "3.4.1-canary.74+b5da6b749",
-    "@atlaspack/plugin": "2.14.5-canary.6+b5da6b749",
-    "@atlaspack/rust": "3.2.1-canary.6+b5da6b749",
-    "@atlaspack/utils": "2.14.5-canary.6+b5da6b749",
+    "@atlaspack/diagnostic": "2.14.1-canary.76+46a90dccd",
+    "@atlaspack/feature-flags": "2.14.1-canary.76+46a90dccd",
+    "@atlaspack/graph": "3.4.1-canary.76+46a90dccd",
+    "@atlaspack/plugin": "2.14.5-canary.8+46a90dccd",
+    "@atlaspack/rust": "3.2.1-canary.8+46a90dccd",
+    "@atlaspack/utils": "2.14.5-canary.8+46a90dccd",
     "nullthrows": "^1.1.1"
   },
-  "gitHead": "b5da6b749ddb23cfc212a640df1f07850da8307f"
+  "gitHead": "46a90dccd019a26b222c878a92d23acc75dc67c5"
 }

package/src/bundleMerge.js

@@ -0,0 +1,103 @@
+// @flow strict-local
+
+import invariant from 'assert';
+import nullthrows from 'nullthrows';
+import type {NodeId} from '@atlaspack/graph';
+import type {Bundle, IdealBundleGraph} from './idealGraph';
+import {ContentGraph} from '@atlaspack/graph';
+
+// Returns a decimal showing the proportion source bundles are common to
+// both bundles versus the total number of source bundles.
+function scoreBundleMerge(bundleA: Bundle, bundleB: Bundle): number {
+  let sharedSourceBundles = 0;
+  let allSourceBundles = new Set([
+    ...bundleA.sourceBundles,
+    ...bundleB.sourceBundles,
+  ]);
+
+  for (let bundle of bundleB.sourceBundles) {
+    if (bundleA.sourceBundles.has(bundle)) {
+      sharedSourceBundles++;
+    }
+  }
+
+  return sharedSourceBundles / allSourceBundles.size;
+}
+
+function getMergeClusters(
+  graph: ContentGraph<NodeId>,
+  candidates: Set<NodeId>,
+): Array<Array<NodeId>> {
+  let clusters = [];
+
+  for (let candidate of candidates) {
+    let cluster: Array<NodeId> = [];
+
+    graph.traverse((nodeId) => {
+      cluster.push(nullthrows(graph.getNode(nodeId)));
+      // Remove node from candidates as it has already been processed
+      candidates.delete(nodeId);
+    }, candidate);
+
+    clusters.push(cluster);
+  }
+
+  return clusters;
+}
+
+export function findMergeCandidates(
+  bundleGraph: IdealBundleGraph,
+  bundles: Array<NodeId>,
+  threshold: number,
+): Array<Array<NodeId>> {
+  let graph = new ContentGraph<NodeId>();
+  let seen = new Set<string>();
+  let candidates = new Set<NodeId>();
+
+  // Build graph of clustered merge candidates
+  for (let bundleId of bundles) {
+    let bundle = bundleGraph.getNode(bundleId);
+    invariant(bundle && bundle !== 'root');
+    if (bundle.type !== 'js') {
+      continue;
+    }
+
+    for (let otherBundleId of bundles) {
+      if (bundleId === otherBundleId) {
+        continue;
+      }
+
+      let key = [bundleId, otherBundleId].sort().join(':');
+
+      if (seen.has(key)) {
+        continue;
+      }
+      seen.add(key);
+
+      let otherBundle = bundleGraph.getNode(otherBundleId);
+      invariant(otherBundle && otherBundle !== 'root');
+
+      let score = scoreBundleMerge(bundle, otherBundle);
+
+      if (score >= threshold) {
+        let bundleNode = graph.addNodeByContentKeyIfNeeded(
+          bundleId.toString(),
+          bundleId,
+        );
+        let otherBundleNode = graph.addNodeByContentKeyIfNeeded(
+          otherBundleId.toString(),
+          otherBundleId,
+        );
+
+        // Add edge in both directions
+        graph.addEdge(bundleNode, otherBundleNode);
+        graph.addEdge(otherBundleNode, bundleNode);
+
+        candidates.add(bundleNode);
+        candidates.add(otherBundleNode);
+      }
+    }
+  }
+
+  return getMergeClusters(graph, candidates);
+}

package/src/bundlerConfig.js

@@ -28,6 +28,7 @@ type BaseBundlerConfig = {|
   disableSharedBundles?: boolean,
   manualSharedBundles?: ManualSharedBundles,
   loadConditionalBundlesInParallel?: boolean,
+  sharedBundleMergeThreshold?: number,
 |};
 
 type BundlerConfig = {|
@@ -42,6 +43,7 @@ export type ResolvedBundlerConfig = {|
   disableSharedBundles: boolean,
   manualSharedBundles: ManualSharedBundles,
   loadConditionalBundlesInParallel?: boolean,
+  sharedBundleMergeThreshold: number,
 |};
 
 function resolveModeConfig(
@@ -76,6 +78,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 30000,
     maxParallelRequests: 6,
     disableSharedBundles: false,
+    sharedBundleMergeThreshold: 1,
   },
   '2': {
     minBundles: 1,
@@ -83,6 +86,7 @@ const HTTP_OPTIONS = {
     minBundleSize: 20000,
     maxParallelRequests: 25,
     disableSharedBundles: false,
+    sharedBundleMergeThreshold: 1,
   },
 };
 
@@ -139,6 +143,9 @@ const CONFIG_SCHEMA: SchemaEntity = {
     loadConditionalBundlesInParallel: {
       type: 'boolean',
     },
+    sharedBundleMergeThreshold: {
+      type: 'number',
+    },
   },
   additionalProperties: false,
 };
@@ -224,6 +231,9 @@ export async function loadBundlerConfig(
   return {
     minBundles: modeConfig.minBundles ?? defaults.minBundles,
     minBundleSize: modeConfig.minBundleSize ?? defaults.minBundleSize,
+    sharedBundleMergeThreshold:
+      modeConfig.sharedBundleMergeThreshold ??
+      defaults.sharedBundleMergeThreshold,
     maxParallelRequests:
       modeConfig.maxParallelRequests ?? defaults.maxParallelRequests,
     projectRoot: options.projectRoot,

package/src/idealGraph.js

@@ -23,6 +23,7 @@ import {DefaultMap, globToRegex} from '@atlaspack/utils';
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 
+import {findMergeCandidates} from './bundleMerge';
 import type {ResolvedBundlerConfig} from './bundlerConfig';
 
 /* BundleRoot - An asset that is the main entry of a Bundle. */
@@ -67,7 +68,7 @@ export const idealBundleGraphEdges = Object.freeze({
   conditional: 2,
 });
 
-type IdealBundleGraph = Graph<
+export type IdealBundleGraph = Graph<
   Bundle | 'root',
   $Values<typeof idealBundleGraphEdges>,
 >;
@@ -1128,6 +1129,12 @@ export function createIdealGraph(
     }
   }
 
+  // Step merge shared bundles that meet the overlap threshold
+  // This step is skipped by default as the threshold defaults to 1
+  if (config.sharedBundleMergeThreshold < 1) {
+    mergeOverlapBundles();
+  }
+
   // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into
   // their source bundles, and remove the bundle.
   // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained
@@ -1143,9 +1150,9 @@ export function createIdealGraph(
     }
   }
 
+  // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit.
   let modifiedSourceBundles = new Set();
 
-  // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit.
   if (config.disableSharedBundles === false) {
     for (let bundleGroupId of bundleGraph.getNodeIdsConnectedFrom(rootNodeId)) {
       // Find shared bundles in this bundle group.
@@ -1249,6 +1256,96 @@ export function createIdealGraph(
     }
   }
 
+  function mergeBundles(
+    bundleGraph: IdealBundleGraph,
+    bundleToKeepId: NodeId,
+    bundleToRemoveId: NodeId,
+    assetReference: DefaultMap<Asset, Array<[Dependency, Bundle]>>,
+  ) {
+    let bundleToKeep = nullthrows(bundleGraph.getNode(bundleToKeepId), '18');
+    let bundleToRemove = nullthrows(
+      bundleGraph.getNode(bundleToRemoveId),
+      '19',
+    );
+    invariant(bundleToKeep !== 'root' && bundleToRemove !== 'root');
+    for (let asset of bundleToRemove.assets) {
+      bundleToKeep.assets.add(asset);
+      bundleToKeep.size += asset.stats.size;
+
+      let newAssetReference = assetReference
+        .get(asset)
+        .map(([dep, bundle]) =>
+          bundle === bundleToRemove ? [dep, bundleToKeep] : [dep, bundle],
+        );
+
+      assetReference.set(asset, newAssetReference);
+    }
+
+    for (let sourceBundleId of bundleToRemove.sourceBundles) {
+      if (bundleToKeep.sourceBundles.has(sourceBundleId)) {
+        continue;
+      }
+
+      bundleToKeep.sourceBundles.add(sourceBundleId);
+      bundleGraph.addEdge(sourceBundleId, bundleToKeepId);
+    }
+
+    // Merge any internalized assets
+    if (bundleToRemove.internalizedAssets) {
+      if (bundleToKeep.internalizedAssets) {
+        bundleToKeep.internalizedAssets.union(
+          bundleToRemove.internalizedAssets,
+        );
+      } else {
+        bundleToKeep.internalizedAssets = bundleToRemove.internalizedAssets;
+      }
+    }
+
+    bundleGraph.removeNode(bundleToRemoveId);
+  }
+
+  function mergeOverlapBundles() {
+    // Find all shared bundles
+    let sharedBundles = new Set<NodeId>();
+    bundleGraph.traverse((nodeId) => {
+      let bundle = bundleGraph.getNode(nodeId);
+
+      if (!bundle) {
+        throw new Error(`Unable to find bundle ${nodeId} in bundle graph`);
+      }
+
+      if (bundle === 'root') {
+        return;
+      }
+
+      // Only consider JS shared bundles and non-reused bundles.
+      // These count potentially be considered for merging in future but they're
+      // more complicated to merge
+      if (
+        bundle.sourceBundles.size > 0 &&
+        bundle.manualSharedBundle == null &&
+        !bundle.mainEntryAsset &&
+        bundle.type === 'js'
+      ) {
+        sharedBundles.add(nodeId);
+      }
+    });
+
+    let clusters = findMergeCandidates(
+      bundleGraph,
+      Array.from(sharedBundles),
+      config.sharedBundleMergeThreshold,
+    );
+
+    for (let cluster of clusters) {
+      let [mergeTarget, ...rest] = cluster;
+
+      for (let bundleIdToMerge of rest) {
+        mergeBundles(bundleGraph, mergeTarget, bundleIdToMerge, assetReference);
+      }
+    }
+  }
+
   function getBigIntFromContentKey(contentKey) {
     let b = Buffer.alloc(64);
     b.write(contentKey);