@atlaspack/core 2.17.3 → 2.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/lib/AssetGraph.js +17 -6
  3. package/lib/Atlaspack.js +3 -1
  4. package/lib/BundleGraph.js +6 -5
  5. package/lib/Dependency.js +6 -2
  6. package/lib/Environment.js +4 -3
  7. package/lib/EnvironmentManager.js +137 -0
  8. package/lib/InternalConfig.js +3 -2
  9. package/lib/PackagerRunner.js +52 -15
  10. package/lib/RequestTracker.js +191 -89
  11. package/lib/UncommittedAsset.js +20 -2
  12. package/lib/applyRuntimes.js +2 -1
  13. package/lib/assetUtils.js +2 -1
  14. package/lib/atlaspack-v3/worker/worker.js +8 -0
  15. package/lib/public/Asset.js +3 -2
  16. package/lib/public/Bundle.js +2 -1
  17. package/lib/public/BundleGraph.js +21 -5
  18. package/lib/public/Config.js +98 -3
  19. package/lib/public/Dependency.js +2 -1
  20. package/lib/public/MutableBundleGraph.js +2 -1
  21. package/lib/public/Target.js +2 -1
  22. package/lib/requests/AssetGraphRequest.js +13 -1
  23. package/lib/requests/AssetRequest.js +2 -1
  24. package/lib/requests/BundleGraphRequest.js +13 -1
  25. package/lib/requests/ConfigRequest.js +27 -4
  26. package/lib/requests/TargetRequest.js +18 -16
  27. package/lib/requests/WriteBundleRequest.js +15 -3
  28. package/lib/requests/WriteBundlesRequest.js +1 -0
  29. package/lib/resolveOptions.js +4 -2
  30. package/package.json +13 -13
  31. package/src/AssetGraph.js +12 -6
  32. package/src/Atlaspack.js +5 -4
  33. package/src/BundleGraph.js +13 -8
  34. package/src/Dependency.js +13 -5
  35. package/src/Environment.js +8 -5
  36. package/src/EnvironmentManager.js +145 -0
  37. package/src/InternalConfig.js +6 -5
  38. package/src/PackagerRunner.js +72 -20
  39. package/src/RequestTracker.js +330 -146
  40. package/src/UncommittedAsset.js +23 -3
  41. package/src/applyRuntimes.js +6 -1
  42. package/src/assetUtils.js +4 -3
  43. package/src/atlaspack-v3/worker/compat/plugin-config.js +9 -5
  44. package/src/atlaspack-v3/worker/worker.js +7 -0
  45. package/src/public/Asset.js +9 -2
  46. package/src/public/Bundle.js +2 -1
  47. package/src/public/BundleGraph.js +22 -5
  48. package/src/public/Config.js +129 -14
  49. package/src/public/Dependency.js +2 -1
  50. package/src/public/MutableBundleGraph.js +2 -1
  51. package/src/public/Target.js +2 -1
  52. package/src/requests/AssetGraphRequest.js +13 -3
  53. package/src/requests/AssetRequest.js +2 -1
  54. package/src/requests/BundleGraphRequest.js +13 -3
  55. package/src/requests/ConfigRequest.js +33 -9
  56. package/src/requests/TargetRequest.js +19 -25
  57. package/src/requests/WriteBundleRequest.js +14 -8
  58. package/src/requests/WriteBundlesRequest.js +1 -0
  59. package/src/resolveOptions.js +4 -2
  60. package/src/types.js +9 -7
  61. package/test/Environment.test.js +43 -34
  62. package/test/EnvironmentManager.test.js +192 -0
  63. package/test/PublicEnvironment.test.js +10 -7
  64. package/test/RequestTracker.test.js +115 -3
  65. package/test/public/Config.test.js +108 -0
  66. package/test/requests/ConfigRequest.test.js +187 -3
  67. package/test/test-utils.js +4 -9
@@ -6,9 +6,13 @@ Object.defineProperty(exports, "__esModule", {
6
6
  exports.RequestGraph = void 0;
7
7
  exports.cleanUpOrphans = cleanUpOrphans;
8
8
  exports.default = void 0;
9
+ exports.getBiggestFSEventsInvalidations = getBiggestFSEventsInvalidations;
9
10
  exports.getWatcherOptions = getWatcherOptions;
11
+ exports.invalidateRequestGraph = invalidateRequestGraph;
12
+ exports.invalidateRequestGraphFSEvents = invalidateRequestGraphFSEvents;
10
13
  exports.readAndDeserializeRequestGraph = readAndDeserializeRequestGraph;
11
14
  exports.requestTypes = exports.requestGraphEdgeTypes = void 0;
15
+ exports.runInvalidation = runInvalidation;
12
16
  function _assert() {
13
17
  const data = _interopRequireWildcard(require("assert"));
14
18
  _assert = function () {
@@ -84,6 +88,14 @@ var _projectPath = require("./projectPath");
84
88
  var _ReporterRunner = require("./ReporterRunner");
85
89
  var _ConfigRequest = require("./requests/ConfigRequest");
86
90
  var _utils2 = require("./utils");
91
+ function _perf_hooks() {
92
+ const data = require("perf_hooks");
93
+ _perf_hooks = function () {
94
+ return data;
95
+ };
96
+ return data;
97
+ }
98
+ var _EnvironmentManager = require("./EnvironmentManager");
87
99
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
88
100
  function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
89
101
  function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
@@ -151,7 +163,7 @@ const nodeFromOption = (option, value) => ({
151
163
  hash: (0, _utils2.hashFromOption)(value)
152
164
  });
153
165
  const nodeFromConfigKey = (fileName, configKey, contentHash) => ({
154
- id: `config_key:${(0, _projectPath.fromProjectPathRelative)(fileName)}:${configKey}`,
166
+ id: `config_key:${(0, _projectPath.fromProjectPathRelative)(fileName)}:${JSON.stringify(configKey)}`,
155
167
  type: CONFIG_KEY,
156
168
  configKey,
157
169
  contentHash
@@ -434,8 +446,8 @@ class RequestGraph extends _graph().ContentGraph {
434
446
  this.invalidateOnBuildNodeIds.add(requestNodeId);
435
447
  }
436
448
  invalidateOnEnvChange(requestNodeId, env, value) {
437
- let envNode = nodeFromEnv(env, value);
438
- let envNodeId = this.addNode(envNode);
449
+ const envNode = nodeFromEnv(env, value);
450
+ const envNodeId = this.addNode(envNode);
439
451
  if (!this.hasEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update)) {
440
452
  this.addEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update);
441
453
  }
@@ -582,10 +594,12 @@ class RequestGraph extends _graph().ContentGraph {
582
594
  aboveCache.set(fileNameNodeId, above);
583
595
  return above;
584
596
  };
597
+ const invalidationsByPath = new Map();
585
598
  for (let {
586
599
  path: _path,
587
600
  type
588
601
  } of events) {
602
+ const invalidationsBefore = this.getInvalidNodeCount();
589
603
  if (!enableOptimization && process.env.ATLASPACK_DISABLE_CACHE_TIMEOUT !== 'true' && ++count === 256) {
590
604
  let duration = Date.now() - startTime;
591
605
  predictedTime = duration * (events.length >> 8);
@@ -622,7 +636,10 @@ class RequestGraph extends _graph().ContentGraph {
622
636
  this.invalidNodeIds.add(id);
623
637
  }
624
638
  }
625
- return true;
639
+ return {
640
+ didInvalidate: true,
641
+ invalidationsByPath: new Map()
642
+ };
626
643
  }
627
644
 
628
645
  // sometimes mac os reports update events as create events.
@@ -679,10 +696,18 @@ class RequestGraph extends _graph().ContentGraph {
679
696
  this.removeNode(nodeId, removeOrphans);
680
697
  }
681
698
  let configKeyNodes = this.configKeyNodes.get(_filePath);
682
- if (configKeyNodes && (type === 'delete' || type === 'update')) {
699
+
700
+ // With granular invalidations we will always run this block,
701
+ // so even if we get a create event (for whatever reason), we will still
702
+ // try to limit invalidations from config key changes through hashing.
703
+ //
704
+ // Currently create events can invalidate a large number of nodes due to
705
+ // "create above" invalidations.
706
+ const isConfigKeyChange = (0, _featureFlags().getFeatureFlag)('granularTsConfigInvalidation') || type === 'delete' || type === 'update';
707
+ if (configKeyNodes && isConfigKeyChange) {
683
708
  for (let nodeId of configKeyNodes) {
684
709
  let isInvalid = type === 'delete';
685
- if (type === 'update') {
710
+ if (type !== 'delete') {
686
711
  let node = this.getNode(nodeId);
687
712
  (0, _assert().default)(node && node.type === CONFIG_KEY);
688
713
  let contentHash = await (0, _ConfigRequest.getConfigKeyContentHash)(_filePath, node.configKey, options);
@@ -697,6 +722,8 @@ class RequestGraph extends _graph().ContentGraph {
697
722
  }
698
723
  }
699
724
  }
725
+ const invalidationsAfter = this.getInvalidNodeCount();
726
+ invalidationsByPath.set(_path, (invalidationsByPath.get(_path) ?? 0) + (invalidationsAfter - invalidationsBefore));
700
727
  }
701
728
  if ((0, _featureFlags().getFeatureFlag)('fixQuadraticCacheInvalidation')) {
702
729
  cleanUpOrphans(this);
@@ -714,7 +741,10 @@ class RequestGraph extends _graph().ContentGraph {
714
741
  numberOfInvalidatedNodes: invalidatedNodes.size
715
742
  }
716
743
  });
717
- return didInvalidate && this.invalidNodeIds.size > 0;
744
+ return {
745
+ didInvalidate,
746
+ invalidationsByPath
747
+ };
718
748
  }
719
749
  hasCachedRequestChunk(index) {
720
750
  return this.cachedRequestChunks.has(index);
@@ -725,6 +755,13 @@ class RequestGraph extends _graph().ContentGraph {
725
755
  removeCachedRequestChunkForNode(nodeId) {
726
756
  this.cachedRequestChunks.delete(Math.floor(nodeId / this.nodesPerBlob));
727
757
  }
758
+
759
+ /**
760
+ * Returns the number of invalidated nodes in the graph.
761
+ */
762
+ getInvalidNodeCount() {
763
+ return this.invalidNodeIds.size;
764
+ }
728
765
  }
729
766
  exports.RequestGraph = RequestGraph;
730
767
  class RequestTracker {
@@ -970,84 +1007,88 @@ class RequestTracker {
970
1007
  return result;
971
1008
  }
972
1009
  }
973
- await runCacheImprovements(async cache => {
974
- await cache.getNativeRef().startWriteTransaction();
975
- }, () => Promise.resolve());
976
1010
  let cacheKey = getCacheKey(this.options);
977
- let requestGraphKey = `requestGraph-${cacheKey}`;
1011
+ let requestGraphKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/RequestGraph` : `requestGraph-${cacheKey}`;
1012
+ let snapshotKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/snapshot` : `snapshot-${cacheKey}`;
978
1013
  if (this.options.shouldDisableCache) {
979
1014
  return;
980
1015
  }
981
1016
  let total = 0;
982
- (0, _ReporterRunner.report)({
983
- type: 'cache',
984
- phase: 'start',
985
- total,
986
- size: this.graph.nodes.length
987
- });
988
- let serialisedGraph = this.graph.serialize();
989
-
990
- // Delete an existing request graph cache, to prevent invalid states
991
- await this.options.cache.deleteLargeBlob(requestGraphKey);
992
- const serialiseAndSet = async (key, contents) => {
993
- if (signal !== null && signal !== void 0 && signal.aborted) {
994
- throw new Error('Serialization was aborted');
995
- }
996
- await runCacheImprovements(cache => {
997
- (0, _logger().instrument)(`cache.put(${key})`, () => {
998
- cache.getNativeRef().putNoConfirm(key, (0, _buildCache().serialize)(contents));
999
- });
1000
- return Promise.resolve();
1001
- }, async () => {
1002
- await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
1003
- signal: signal
1004
- } : undefined);
1005
- });
1006
- total += 1;
1017
+ await runCacheImprovements(async cache => {
1018
+ await cache.getNativeRef().startWriteTransaction();
1019
+ }, () => Promise.resolve());
1020
+ try {
1007
1021
  (0, _ReporterRunner.report)({
1008
1022
  type: 'cache',
1009
- phase: 'write',
1023
+ phase: 'start',
1010
1024
  total,
1011
1025
  size: this.graph.nodes.length
1012
1026
  });
1013
- };
1014
- let queue = new (_utils().PromiseQueue)({
1015
- maxConcurrent: 32
1016
- });
1027
+ if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
1028
+ await (0, _EnvironmentManager.writeEnvironmentsToCache)(options.cache);
1029
+ }
1030
+ let serialisedGraph = this.graph.serialize();
1017
1031
 
1018
- // Preallocating a sparse array is faster than pushing when N is high enough
1019
- let cacheableNodes = new Array(serialisedGraph.nodes.length);
1020
- for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
1021
- let node = serialisedGraph.nodes[i];
1022
- let resultCacheKey = node === null || node === void 0 ? void 0 : node.resultCacheKey;
1023
- if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST && resultCacheKey != null && (node === null || node === void 0 ? void 0 : node.result) != null) {
1024
- queue.add(() => serialiseAndSet(resultCacheKey, node.result));
1032
+ // Delete an existing request graph cache, to prevent invalid states
1033
+ await this.options.cache.deleteLargeBlob(requestGraphKey);
1034
+ const serialiseAndSet = async (key, contents) => {
1035
+ if (signal !== null && signal !== void 0 && signal.aborted) {
1036
+ throw new Error('Serialization was aborted');
1037
+ }
1038
+ await runCacheImprovements(cache => {
1039
+ (0, _logger().instrument)(`RequestTracker::writeToCache::cache.put(${key})`, () => {
1040
+ cache.getNativeRef().putNoConfirm(key, (0, _buildCache().serialize)(contents));
1041
+ });
1042
+ return Promise.resolve();
1043
+ }, async () => {
1044
+ await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
1045
+ signal: signal
1046
+ } : undefined);
1047
+ });
1048
+ total += 1;
1049
+ (0, _ReporterRunner.report)({
1050
+ type: 'cache',
1051
+ phase: 'write',
1052
+ total,
1053
+ size: this.graph.nodes.length
1054
+ });
1055
+ };
1056
+ let queue = new (_utils().PromiseQueue)({
1057
+ maxConcurrent: 32
1058
+ });
1025
1059
 
1026
- // eslint-disable-next-line no-unused-vars
1027
- let {
1028
- result: _,
1029
- ...newNode
1030
- } = node;
1031
- cacheableNodes[i] = newNode;
1032
- } else {
1033
- cacheableNodes[i] = node;
1060
+ // Preallocating a sparse array is faster than pushing when N is high enough
1061
+ let cacheableNodes = new Array(serialisedGraph.nodes.length);
1062
+ for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
1063
+ let node = serialisedGraph.nodes[i];
1064
+ let resultCacheKey = node === null || node === void 0 ? void 0 : node.resultCacheKey;
1065
+ if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST && resultCacheKey != null && (node === null || node === void 0 ? void 0 : node.result) != null) {
1066
+ queue.add(() => serialiseAndSet(resultCacheKey, node.result));
1067
+
1068
+ // eslint-disable-next-line no-unused-vars
1069
+ let {
1070
+ result: _,
1071
+ ...newNode
1072
+ } = node;
1073
+ cacheableNodes[i] = newNode;
1074
+ } else {
1075
+ cacheableNodes[i] = node;
1076
+ }
1034
1077
  }
1035
- }
1036
- let nodeCountsPerBlob = [];
1037
- for (let i = 0; i * this.graph.nodesPerBlob < cacheableNodes.length; i += 1) {
1038
- let nodesStartIndex = i * this.graph.nodesPerBlob;
1039
- let nodesEndIndex = Math.min((i + 1) * this.graph.nodesPerBlob, cacheableNodes.length);
1040
- nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
1041
- if (!this.graph.hasCachedRequestChunk(i)) {
1042
- // We assume the request graph nodes are immutable and won't change
1043
- let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
1044
- queue.add(() => serialiseAndSet(getRequestGraphNodeKey(i, cacheKey), nodesToCache).then(() => {
1045
- // Succeeded in writing to disk, save that we have completed this chunk
1046
- this.graph.setCachedRequestChunk(i);
1047
- }));
1078
+ let nodeCountsPerBlob = [];
1079
+ for (let i = 0; i * this.graph.nodesPerBlob < cacheableNodes.length; i += 1) {
1080
+ let nodesStartIndex = i * this.graph.nodesPerBlob;
1081
+ let nodesEndIndex = Math.min((i + 1) * this.graph.nodesPerBlob, cacheableNodes.length);
1082
+ nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
1083
+ if (!this.graph.hasCachedRequestChunk(i)) {
1084
+ // We assume the request graph nodes are immutable and won't change
1085
+ let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
1086
+ queue.add(() => serialiseAndSet(getRequestGraphNodeKey(i, cacheKey), nodesToCache).then(() => {
1087
+ // Succeeded in writing to disk, save that we have completed this chunk
1088
+ this.graph.setCachedRequestChunk(i);
1089
+ }));
1090
+ }
1048
1091
  }
1049
- }
1050
- try {
1051
1092
  await queue.run();
1052
1093
 
1053
1094
  // Set the request graph after the queue is flushed to avoid writing an invalid state
@@ -1056,7 +1097,7 @@ class RequestTracker {
1056
1097
  nodeCountsPerBlob,
1057
1098
  nodes: undefined
1058
1099
  });
1059
- await runCacheImprovements(() => serialiseAndSet(`request_tracker:cache_metadata:${cacheKey}`, {
1100
+ await runCacheImprovements(() => serialiseAndSet(`${cacheKey}/cache_metadata`, {
1060
1101
  version: _constants.ATLASPACK_VERSION,
1061
1102
  entries: this.options.entries,
1062
1103
  mode: this.options.mode,
@@ -1064,15 +1105,16 @@ class RequestTracker {
1064
1105
  watchBackend: this.options.watchBackend
1065
1106
  }), () => Promise.resolve());
1066
1107
  let opts = getWatcherOptions(this.options);
1067
- let snapshotPath = _path2().default.join(this.options.cacheDir, `snapshot-${cacheKey}` + '.txt');
1108
+ let snapshotPath = _path2().default.join(this.options.cacheDir, snapshotKey + '.txt');
1068
1109
  await this.options.outputFS.writeSnapshot(this.options.watchDir, snapshotPath, opts);
1069
1110
  } catch (err) {
1070
1111
  // If we have aborted, ignore the error and continue
1071
1112
  if (!(signal !== null && signal !== void 0 && signal.aborted)) throw err;
1113
+ } finally {
1114
+ await runCacheImprovements(async cache => {
1115
+ await cache.getNativeRef().commitWriteTransaction();
1116
+ }, () => Promise.resolve());
1072
1117
  }
1073
- await runCacheImprovements(async cache => {
1074
- await cache.getNativeRef().commitWriteTransaction();
1075
- }, () => Promise.resolve());
1076
1118
  (0, _ReporterRunner.report)({
1077
1119
  type: 'cache',
1078
1120
  phase: 'end',
@@ -1109,17 +1151,30 @@ function getWatcherOptions({
1109
1151
  };
1110
1152
  }
1111
1153
  function getCacheKey(options) {
1154
+ if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
1155
+ const hash = (0, _rust().hashString)(`${_constants.ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${options.mode}:${options.shouldBuildLazily ? 'lazy' : 'eager'}:${options.watchBackend ?? ''}`);
1156
+ return `RequestTracker/${_constants.ATLASPACK_VERSION}/${hash}`;
1157
+ }
1112
1158
  return (0, _rust().hashString)(`${_constants.ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${options.mode}:${options.shouldBuildLazily ? 'lazy' : 'eager'}:${options.watchBackend ?? ''}`);
1113
1159
  }
1114
1160
  function getRequestGraphNodeKey(index, cacheKey) {
1161
+ if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
1162
+ return `${cacheKey}/RequestGraph/nodes/${index}`;
1163
+ }
1115
1164
  return `requestGraph-nodes-${index}-${cacheKey}`;
1116
1165
  }
1117
1166
  async function readAndDeserializeRequestGraph(cache, requestGraphKey, cacheKey) {
1118
1167
  let bufferLength = 0;
1119
1168
  const getAndDeserialize = async key => {
1120
- let buffer = await cache.getLargeBlob(key);
1121
- bufferLength += Buffer.byteLength(buffer);
1122
- return (0, _buildCache().deserialize)(buffer);
1169
+ if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
1170
+ const buffer = await cache.getBlob(key);
1171
+ bufferLength += Buffer.byteLength(buffer);
1172
+ return (0, _buildCache().deserialize)(buffer);
1173
+ } else {
1174
+ const buffer = await cache.getLargeBlob(key);
1175
+ bufferLength += Buffer.byteLength(buffer);
1176
+ return (0, _buildCache().deserialize)(buffer);
1177
+ }
1123
1178
  };
1124
1179
  let serializedRequestGraph = await getAndDeserialize(requestGraphKey);
1125
1180
  let nodePromises = serializedRequestGraph.nodeCountsPerBlob.map(async (nodesCount, i) => {
@@ -1141,9 +1196,9 @@ async function loadRequestGraph(options) {
1141
1196
  return new RequestGraph();
1142
1197
  }
1143
1198
  let cacheKey = getCacheKey(options);
1144
- let requestGraphKey = `requestGraph-${cacheKey}`;
1199
+ let requestGraphKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/RequestGraph` : `requestGraph-${cacheKey}`;
1145
1200
  let timeout;
1146
- const snapshotKey = `snapshot-${cacheKey}`;
1201
+ const snapshotKey = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? `${cacheKey}/snapshot` : `snapshot-${cacheKey}`;
1147
1202
  const snapshotPath = _path2().default.join(options.cacheDir, snapshotKey + '.txt');
1148
1203
  const commonMeta = {
1149
1204
  cacheKey,
@@ -1163,7 +1218,11 @@ async function loadRequestGraph(options) {
1163
1218
  ...commonMeta
1164
1219
  }
1165
1220
  });
1166
- if (await options.cache.hasLargeBlob(requestGraphKey)) {
1221
+ if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
1222
+ await (0, _EnvironmentManager.loadEnvironmentsFromCache)(options.cache);
1223
+ }
1224
+ const hasRequestGraphInCache = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? await options.cache.has(requestGraphKey) : await options.cache.hasLargeBlob(requestGraphKey);
1225
+ if (hasRequestGraphInCache) {
1167
1226
  try {
1168
1227
  let {
1169
1228
  requestGraph
@@ -1230,6 +1289,20 @@ async function loadRequestGraph(options) {
1230
1289
  * A wrapper around an invalidation type / method
1231
1290
  */
1232
1291
 
1292
+ /**
1293
+ * Details about an invalidation.
1294
+ *
1295
+ * If this is a fs events invalidation, this key will contain statistics about invalidations
1296
+ * by path.
1297
+ *
1298
+ * If this is a env or option invalidation, this key will contain the list of changed environment
1299
+ * variables or options.
1300
+ */
1301
+
1302
+ /**
1303
+ * Number of invalidations for a given file-system event.
1304
+ */
1305
+
1233
1306
  /**
1234
1307
  * Information about a certain cache invalidation type.
1235
1308
  */
@@ -1255,9 +1328,7 @@ async function invalidateRequestGraph(requestGraph, options, events) {
1255
1328
  fn: () => requestGraph.invalidateOptionNodes(options)
1256
1329
  }, {
1257
1330
  key: 'fsEvents',
1258
- fn: async () => {
1259
- await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
1260
- }
1331
+ fn: () => invalidateRequestGraphFSEvents(requestGraph, options, events)
1261
1332
  }];
1262
1333
  const invalidations = [];
1263
1334
  for (const invalidation of invalidationFns) {
@@ -1275,18 +1346,34 @@ async function invalidateRequestGraph(requestGraph, options, events) {
1275
1346
  requestInvalidationRatio: invalidatedCount / requestCount
1276
1347
  };
1277
1348
  }
1278
-
1349
+ /**
1350
+ * Invalidate the request graph based on file-system events.
1351
+ *
1352
+ * Returns statistics about the invalidations.
1353
+ */
1354
+ async function invalidateRequestGraphFSEvents(requestGraph, options, events) {
1355
+ const {
1356
+ invalidationsByPath
1357
+ } = await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
1358
+ const biggestInvalidations = getBiggestFSEventsInvalidations(invalidationsByPath);
1359
+ return {
1360
+ biggestInvalidations
1361
+ };
1362
+ }
1279
1363
  /**
1280
1364
  * Runs an invalidation function and reports metrics.
1281
1365
  */
1282
1366
  async function runInvalidation(requestGraph, invalidationFn) {
1283
- const startInvalidationCount = requestGraph.invalidNodeIds.size;
1367
+ const start = _perf_hooks().performance.now();
1368
+ const startInvalidationCount = requestGraph.getInvalidNodeCount();
1284
1369
  const result = await invalidationFn.fn();
1285
- const count = requestGraph.invalidNodeIds.size - startInvalidationCount;
1370
+ const count = requestGraph.getInvalidNodeCount() - startInvalidationCount;
1371
+ const duration = _perf_hooks().performance.now() - start;
1286
1372
  return {
1287
1373
  key: invalidationFn.key,
1288
1374
  count,
1289
- changes: typeof result === 'object' && Array.isArray(result) ? result : null
1375
+ detail: result ?? null,
1376
+ duration
1290
1377
  };
1291
1378
  }
1292
1379
  function logErrorOnBailout(options, snapshotPath, e) {
@@ -1328,4 +1415,19 @@ function cleanUpOrphans(graph) {
1328
1415
  }
1329
1416
  });
1330
1417
  return removedNodeIds;
1418
+ }
1419
+
1420
+ /**
1421
+ * Returns paths that invalidated the most nodes
1422
+ */
1423
+ function getBiggestFSEventsInvalidations(invalidationsByPath, limit = 10) {
1424
+ const invalidations = [];
1425
+ for (const [path, count] of invalidationsByPath) {
1426
+ invalidations.push({
1427
+ path,
1428
+ count
1429
+ });
1430
+ }
1431
+ invalidations.sort((a, b) => b.count - a.count);
1432
+ return invalidations.slice(0, limit);
1331
1433
  }
@@ -53,6 +53,14 @@ var _assetUtils = require("./assetUtils");
53
53
  var _types = require("./types");
54
54
  var _utils2 = require("./utils");
55
55
  var _projectPath = require("./projectPath");
56
+ function _featureFlags() {
57
+ const data = require("@atlaspack/feature-flags");
58
+ _featureFlags = function () {
59
+ return data;
60
+ };
61
+ return data;
62
+ }
63
+ var _EnvironmentManager = require("./EnvironmentManager");
56
64
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
57
65
  class UncommittedAsset {
58
66
  constructor({
@@ -133,6 +141,8 @@ class UncommittedAsset {
133
141
  hash = (0, _rust().hashBuffer)(content);
134
142
  size = content.length;
135
143
  }
144
+
145
+ // Maybe we should just store this in a file instead of LMDB
136
146
  await this.options.cache.setBlob(contentKey, content);
137
147
  return {
138
148
  size,
@@ -184,6 +194,10 @@ class UncommittedAsset {
184
194
  this.content = buffer;
185
195
  this.clearAST();
186
196
  }
197
+
198
+ /**
199
+ * @deprecated This has been broken on any cache other than FSCache for a long time.
200
+ */
187
201
  setStream(stream) {
188
202
  this.content = stream;
189
203
  this.clearAST();
@@ -248,6 +262,10 @@ class UncommittedAsset {
248
262
  this.value.astGenerator = null;
249
263
  }
250
264
  getCacheKey(key) {
265
+ if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
266
+ const filePath = (0, _projectPath.fromProjectPathRelative)(this.value.filePath);
267
+ return `Asset/${_constants.ATLASPACK_VERSION}/${filePath}/${this.value.id}/${key}`;
268
+ }
251
269
  return (0, _rust().hashString)(_constants.ATLASPACK_VERSION + key + this.value.id);
252
270
  }
253
271
  addDependency(opts) {
@@ -261,7 +279,7 @@ class UncommittedAsset {
261
279
  ...rest,
262
280
  // $FlowFixMe "convert" the $ReadOnlyMaps to the interal mutable one
263
281
  symbols,
264
- env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, this.value.env, env),
282
+ env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, (0, _EnvironmentManager.fromEnvironmentId)(this.value.env), env),
265
283
  sourceAssetId: this.value.id,
266
284
  sourcePath: (0, _projectPath.fromProjectPath)(this.options.projectRoot, this.value.filePath)
267
285
  });
@@ -301,7 +319,7 @@ class UncommittedAsset {
301
319
  bundleBehavior: result.bundleBehavior ?? (this.value.bundleBehavior == null ? null : _types.BundleBehaviorNames[this.value.bundleBehavior]),
302
320
  isBundleSplittable: result.isBundleSplittable ?? this.value.isBundleSplittable,
303
321
  isSource: this.value.isSource,
304
- env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, this.value.env, result.env),
322
+ env: (0, _Environment.mergeEnvironments)(this.options.projectRoot, (0, _EnvironmentManager.fromEnvironmentId)(this.value.env), result.env),
305
323
  dependencies: this.value.type === result.type ? new Map(this.value.dependencies) : new Map(),
306
324
  meta: {
307
325
  ...this.value.meta,
@@ -69,6 +69,7 @@ function _utils() {
69
69
  };
70
70
  return data;
71
71
  }
72
+ var _EnvironmentManager = require("./EnvironmentManager");
72
73
  function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
73
74
  function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
74
75
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
@@ -164,7 +165,7 @@ async function applyRuntimes({
164
165
  let assetGroup = {
165
166
  code,
166
167
  filePath: (0, _projectPath.toProjectPath)(options.projectRoot, sourceName),
167
- env: (0, _Environment.mergeEnvironments)(options.projectRoot, bundle.env, env),
168
+ env: (0, _Environment.mergeEnvironments)(options.projectRoot, (0, _EnvironmentManager.fromEnvironmentId)(bundle.env), env),
168
169
  // Runtime assets should be considered source, as they should be
169
170
  // e.g. compiled to run in the target environment
170
171
  isSource: true
package/lib/assetUtils.js CHANGED
@@ -66,10 +66,11 @@ function _profiler() {
66
66
  return data;
67
67
  }
68
68
  var _IdentifierRegistry = require("./IdentifierRegistry");
69
+ var _EnvironmentManager = require("./EnvironmentManager");
69
70
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
70
71
  function createAssetIdFromOptions(options) {
71
72
  const data = {
72
- environmentId: options.env.id,
73
+ environmentId: (0, _EnvironmentManager.toEnvironmentId)(options.env),
73
74
  filePath: options.filePath,
74
75
  code: options.code,
75
76
  pipeline: options.pipeline,
@@ -149,6 +149,14 @@ class AtlaspackWorker {
149
149
  }
150
150
  };
151
151
  }
152
+ if (result.isExcluded) {
153
+ return {
154
+ invalidations: [],
155
+ resolution: {
156
+ type: 'excluded'
157
+ }
158
+ };
159
+ }
152
160
  return {
153
161
  invalidations: [],
154
162
  resolution: {
@@ -23,6 +23,7 @@ var _Environment2 = require("../Environment");
23
23
  var _projectPath = require("../projectPath");
24
24
  var _types = require("../types");
25
25
  var _utils = require("../utils");
26
+ var _EnvironmentManager = require("../EnvironmentManager");
26
27
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
27
28
  const inspect = Symbol.for('nodejs.util.inspect.custom');
28
29
  const uncommittedAssetValueToAsset = new WeakMap();
@@ -62,7 +63,7 @@ class BaseAsset {
62
63
  return this.#asset.value.type;
63
64
  }
64
65
  get env() {
65
- return new _Environment.default(this.#asset.value.env, this.#asset.options);
66
+ return new _Environment.default((0, _EnvironmentManager.fromEnvironmentId)(this.#asset.value.env), this.#asset.options);
66
67
  }
67
68
  get fs() {
68
69
  return this.#asset.options.inputFS;
@@ -142,7 +143,7 @@ class Asset extends BaseAsset {
142
143
  return this;
143
144
  }
144
145
  get env() {
145
- this.#env ??= new _Environment.default(this.#asset.value.env, this.#asset.options);
146
+ this.#env ??= new _Environment.default((0, _EnvironmentManager.fromEnvironmentId)(this.#asset.value.env), this.#asset.options);
146
147
  return this.#env;
147
148
  }
148
149
  get stats() {
@@ -40,6 +40,7 @@ var _Dependency = require("./Dependency");
40
40
  var _Target = _interopRequireDefault(require("./Target"));
41
41
  var _types = require("../types");
42
42
  var _projectPath = require("../projectPath");
43
+ var _EnvironmentManager = require("../EnvironmentManager");
43
44
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
44
45
  const inspect = Symbol.for('nodejs.util.inspect.custom');
45
46
  const internalBundleToBundle = new (_utils().DefaultWeakMap)(() => new (_utils().DefaultWeakMap)(() => new WeakMap()));
@@ -99,7 +100,7 @@ class Bundle {
99
100
  return this.#bundle.type;
100
101
  }
101
102
  get env() {
102
- return new _Environment.default(this.#bundle.env, this.#options);
103
+ return new _Environment.default((0, _EnvironmentManager.fromEnvironmentId)(this.#bundle.env), this.#options);
103
104
  }
104
105
  get needsStableName() {
105
106
  return this.#bundle.needsStableName;
@@ -32,6 +32,13 @@ var _Dependency = _interopRequireWildcard(require("./Dependency"));
32
32
  var _Target = require("./Target");
33
33
  var _utils = require("../utils");
34
34
  var _BundleGroup = _interopRequireWildcard(require("./BundleGroup"));
35
+ function _featureFlags() {
36
+ const data = require("@atlaspack/feature-flags");
37
+ _featureFlags = function () {
38
+ return data;
39
+ };
40
+ return data;
41
+ }
35
42
  function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
36
43
  function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
37
44
  function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
@@ -263,11 +270,20 @@ class BundleGraph {
263
270
  }
264
271
  for (let bundle of bundles) {
265
272
  const conditions = bundleConditions.get(bundle.id) ?? new Map();
266
- conditions.set(cond.key, {
267
- bundle,
268
- ifTrueBundles,
269
- ifFalseBundles
270
- });
273
+ const currentCondition = conditions.get(cond.key);
274
+ if ((0, _featureFlags().getFeatureFlag)('conditionalBundlingReporterSameConditionFix')) {
275
+ conditions.set(cond.key, {
276
+ bundle,
277
+ ifTrueBundles: [...((currentCondition === null || currentCondition === void 0 ? void 0 : currentCondition.ifTrueBundles) ?? []), ...ifTrueBundles],
278
+ ifFalseBundles: [...((currentCondition === null || currentCondition === void 0 ? void 0 : currentCondition.ifFalseBundles) ?? []), ...ifFalseBundles]
279
+ });
280
+ } else {
281
+ conditions.set(cond.key, {
282
+ bundle,
283
+ ifTrueBundles,
284
+ ifFalseBundles
285
+ });
286
+ }
271
287
  bundleConditions.set(bundle.id, conditions);
272
288
  }
273
289
  }