@atlaspack/core 2.16.2-canary.5 → 2.16.2-canary.500

package/lib/RequestTracker.js

@@ -6,9 +6,13 @@ Object.defineProperty(exports, "__esModule", {
 exports.RequestGraph = void 0;
 exports.cleanUpOrphans = cleanUpOrphans;
 exports.default = void 0;
+exports.getBiggestFSEventsInvalidations = getBiggestFSEventsInvalidations;
 exports.getWatcherOptions = getWatcherOptions;
+exports.invalidateRequestGraph = invalidateRequestGraph;
+exports.invalidateRequestGraphFSEvents = invalidateRequestGraphFSEvents;
 exports.readAndDeserializeRequestGraph = readAndDeserializeRequestGraph;
 exports.requestTypes = exports.requestGraphEdgeTypes = void 0;
+exports.runInvalidation = runInvalidation;
 function _assert() {
   const data = _interopRequireWildcard(require("assert"));
   _assert = function () {
@@ -30,13 +34,6 @@ function _buildCache() {
   };
   return data;
 }
-function _cache() {
-  const data = require("@atlaspack/cache");
-  _cache = function () {
-    return data;
-  };
-  return data;
-}
 function _featureFlags() {
   const data = require("@atlaspack/feature-flags");
   _featureFlags = function () {
@@ -52,7 +49,7 @@ function _graph() {
   return data;
 }
 function _logger() {
-  const data = _interopRequireWildcard(require("@atlaspack/logger"));
+  const data = _interopRequireDefault(require("@atlaspack/logger"));
   _logger = function () {
     return data;
   };
@@ -84,9 +81,16 @@ var _projectPath = require("./projectPath");
 var _ReporterRunner = require("./ReporterRunner");
 var _ConfigRequest = require("./requests/ConfigRequest");
 var _utils2 = require("./utils");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
-function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
+function _perf_hooks() {
+  const data = require("perf_hooks");
+  _perf_hooks = function () {
+    return data;
+  };
+  return data;
+}
+var _EnvironmentManager = require("./EnvironmentManager");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
 const requestGraphEdgeTypes = exports.requestGraphEdgeTypes = {
   subrequest: 2,
   invalidated_by_update: 3,
@@ -151,7 +155,7 @@ const nodeFromOption = (option, value) => ({
   hash: (0, _utils2.hashFromOption)(value)
 });
 const nodeFromConfigKey = (fileName, configKey, contentHash) => ({
-  id: `config_key:${(0, _projectPath.fromProjectPathRelative)(fileName)}:${configKey}`,
+  id: `config_key:${(0, _projectPath.fromProjectPathRelative)(fileName)}:${JSON.stringify(configKey)}`,
   type: CONFIG_KEY,
   configKey,
   contentHash
@@ -162,6 +166,8 @@ const keyFromOptionContentKey = contentKey => contentKey.slice('option:'.length)
 // This constant is chosen by local profiling the time to serialise n nodes and tuning until an average time of ~50 ms per blob.
 // The goal is to free up the event loop periodically to allow interruption by the user.
 const NODES_PER_BLOB = 2 ** 14;
+
+// @ts-expect-error TS2417
 class RequestGraph extends _graph().ContentGraph {
   invalidNodeIds = new Set();
   incompleteNodeIds = new Set();
@@ -176,10 +182,7 @@ class RequestGraph extends _graph().ContentGraph {
   cachedRequestChunks = new Set();
   configKeyNodes = new Map();
   nodesPerBlob = NODES_PER_BLOB;
-
-  // $FlowFixMe[prop-missing]
   static deserialize(opts) {
-    // $FlowFixMe[prop-missing]
     let deserialized = new RequestGraph(opts);
     deserialized.invalidNodeIds = opts.invalidNodeIds;
     deserialized.incompleteNodeIds = opts.incompleteNodeIds;
@@ -192,8 +195,6 @@ class RequestGraph extends _graph().ContentGraph {
     deserialized.configKeyNodes = opts.configKeyNodes;
     return deserialized;
   }
-
-  // $FlowFixMe[prop-missing]
   serialize() {
     return {
       ...super.serialize(),
@@ -279,6 +280,12 @@ class RequestGraph extends _graph().ContentGraph {
     // If the node is invalidated, the cached request chunk on disk needs to be re-written
     this.removeCachedRequestChunkForNode(nodeId);
   }
+
+  /**
+   * Nodes that are invalidated on start-up, such as JavaScript babel configuration files which are
+   * imported when the build kicks-off and might doing arbitrary work such as reading from the file
+   * system.
+   */
   invalidateUnpredictableNodes() {
     for (let nodeId of this.unpredicatableNodeIds) {
       let node = (0, _nullthrows().default)(this.getNode(nodeId));
@@ -286,6 +293,10 @@ class RequestGraph extends _graph().ContentGraph {
       this.invalidateNode(nodeId, _constants.STARTUP);
     }
   }
+
+  /**
+   * Effectively uncacheable nodes.
+   */
   invalidateOnBuildNodes() {
     for (let nodeId of this.invalidateOnBuildNodeIds) {
       let node = (0, _nullthrows().default)(this.getNode(nodeId));
@@ -293,29 +304,47 @@ class RequestGraph extends _graph().ContentGraph {
       this.invalidateNode(nodeId, _constants.STARTUP);
     }
   }
+
+  /**
+   * Nodes invalidated by environment changes, corresponds to `env: ...` inputs.
+   */
   invalidateEnvNodes(env) {
+    const invalidatedKeys = [];
     for (let nodeId of this.envNodeIds) {
       let node = (0, _nullthrows().default)(this.getNode(nodeId));
       (0, _assert().default)(node.type === ENV);
-      if (env[keyFromEnvContentKey(node.id)] !== node.value) {
+      const key = keyFromEnvContentKey(node.id);
+      if (env[key] !== node.value) {
+        invalidatedKeys.push(key);
         let parentNodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update);
         for (let parentNode of parentNodes) {
           this.invalidateNode(parentNode, _constants.ENV_CHANGE);
         }
       }
     }
+    return invalidatedKeys;
   }
+
+  /**
+   * Nodes invalidated by option changes.
+   */
   invalidateOptionNodes(options) {
+    const invalidatedKeys = [];
     for (let nodeId of this.optionNodeIds) {
       let node = (0, _nullthrows().default)(this.getNode(nodeId));
       (0, _assert().default)(node.type === OPTION);
-      if ((0, _utils2.hashFromOption)(options[keyFromOptionContentKey(node.id)]) !== node.hash) {
+      const key = keyFromOptionContentKey(node.id);
+
+      // @ts-expect-error TS7053
+      if ((0, _utils2.hashFromOption)(options[key]) !== node.hash) {
+        invalidatedKeys.push(key);
         let parentNodes = this.getNodeIdsConnectedTo(nodeId, requestGraphEdgeTypes.invalidated_by_update);
         for (let parentNode of parentNodes) {
           this.invalidateNode(parentNode, _constants.OPTION_CHANGE);
         }
       }
     }
+    return invalidatedKeys;
   }
   invalidateOnConfigKeyChange(requestNodeId, filePath, configKey, contentHash) {
     let configKeyNodeId = this.addNode(nodeFromConfigKey(filePath, configKey, contentHash));
@@ -345,9 +374,13 @@ class RequestGraph extends _graph().ContentGraph {
   }
   invalidateOnFileCreate(requestNodeId, input) {
     let node;
+    // @ts-expect-error TS2339
     if (input.glob != null) {
+      // @ts-expect-error TS2339
       node = nodeFromGlob(input.glob);
+      // @ts-expect-error TS2339
     } else if (input.fileName != null && input.aboveFilePath != null) {
+      // @ts-expect-error TS2339
       let aboveFilePath = input.aboveFilePath;
 
       // Create nodes and edges for each part of the filename pattern.
@@ -355,6 +388,7 @@ class RequestGraph extends _graph().ContentGraph {
       // This creates a sort of trie structure within the graph that can be
       // quickly matched by following the edges. This is also memory efficient
       // since common sub-paths (e.g. 'node_modules') are deduplicated.
+      // @ts-expect-error TS2339
       let parts = input.fileName.split('/').reverse();
       let lastNodeId;
       for (let part of parts) {
@@ -389,7 +423,9 @@ class RequestGraph extends _graph().ContentGraph {
       if (!this.hasEdge(lastNodeId, nodeId, requestGraphEdgeTypes.invalidated_by_create_above)) {
         this.addEdge(lastNodeId, nodeId, requestGraphEdgeTypes.invalidated_by_create_above);
       }
+      // @ts-expect-error TS2339
     } else if (input.filePath != null) {
+      // @ts-expect-error TS2339
       node = nodeFromFilePath(input.filePath);
     } else {
       throw new Error('Invalid invalidation');
@@ -408,8 +444,8 @@ class RequestGraph extends _graph().ContentGraph {
     this.invalidateOnBuildNodeIds.add(requestNodeId);
   }
   invalidateOnEnvChange(requestNodeId, env, value) {
-    let envNode = nodeFromEnv(env, value);
-    let envNodeId = this.addNode(envNode);
+    const envNode = nodeFromEnv(env, value);
+    const envNodeId = this.addNode(envNode);
     if (!this.hasEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update)) {
       this.addEdge(requestNodeId, envNodeId, requestGraphEdgeTypes.invalidated_by_update);
     }
@@ -435,6 +471,7 @@ class RequestGraph extends _graph().ContentGraph {
 
     // For now just handling updates. Could add creates/deletes later if needed.
     let invalidations = this.getNodeIdsConnectedFrom(requestNodeId, requestGraphEdgeTypes.invalidated_by_update);
+    // @ts-expect-error TS2322
     return invalidations.map(nodeId => {
       let node = (0, _nullthrows().default)(this.getNode(nodeId));
       switch (node.type) {
@@ -556,10 +593,12 @@ class RequestGraph extends _graph().ContentGraph {
       aboveCache.set(fileNameNodeId, above);
       return above;
     };
+    const invalidationsByPath = new Map();
     for (let {
       path: _path,
       type
     } of events) {
+      const invalidationsBefore = this.getInvalidNodeCount();
       if (!enableOptimization && process.env.ATLASPACK_DISABLE_CACHE_TIMEOUT !== 'true' && ++count === 256) {
         let duration = Date.now() - startTime;
         predictedTime = duration * (events.length >> 8);
@@ -596,7 +635,10 @@ class RequestGraph extends _graph().ContentGraph {
             this.invalidNodeIds.add(id);
           }
         }
-        return true;
+        return {
+          didInvalidate: true,
+          invalidationsByPath: new Map()
+        };
       }
 
       // sometimes mac os reports update events as create events.
@@ -653,10 +695,17 @@ class RequestGraph extends _graph().ContentGraph {
         this.removeNode(nodeId, removeOrphans);
       }
       let configKeyNodes = this.configKeyNodes.get(_filePath);
-      if (configKeyNodes && (type === 'delete' || type === 'update')) {
+
+      // With granular invalidations we will always run this block,
+      // so even if we get a create event (for whatever reason), we will still
+      // try to limit invalidations from config key changes through hashing.
+      //
+      // Currently create events can invalidate a large number of nodes due to
+      // "create above" invalidations.
+      if (configKeyNodes) {
         for (let nodeId of configKeyNodes) {
           let isInvalid = type === 'delete';
-          if (type === 'update') {
+          if (type !== 'delete') {
             let node = this.getNode(nodeId);
             (0, _assert().default)(node && node.type === CONFIG_KEY);
             let contentHash = await (0, _ConfigRequest.getConfigKeyContentHash)(_filePath, node.configKey, options);
@@ -671,6 +720,9 @@ class RequestGraph extends _graph().ContentGraph {
           }
         }
       }
+      const invalidationsAfter = this.getInvalidNodeCount();
+      const invalidationsForEvent = invalidationsAfter - invalidationsBefore;
+      invalidationsByPath.set(_path, (invalidationsByPath.get(_path) ?? 0) + invalidationsForEvent);
     }
     if ((0, _featureFlags().getFeatureFlag)('fixQuadraticCacheInvalidation')) {
       cleanUpOrphans(this);
@@ -688,7 +740,10 @@ class RequestGraph extends _graph().ContentGraph {
         numberOfInvalidatedNodes: invalidatedNodes.size
       }
     });
-    return didInvalidate && this.invalidNodeIds.size > 0;
+    return {
+      didInvalidate,
+      invalidationsByPath
+    };
   }
   hasCachedRequestChunk(index) {
     return this.cachedRequestChunks.has(index);
@@ -699,6 +754,13 @@ class RequestGraph extends _graph().ContentGraph {
   removeCachedRequestChunkForNode(nodeId) {
     this.cachedRequestChunks.delete(Math.floor(nodeId / this.nodesPerBlob));
   }
+
+  /**
+   * Returns the number of invalidated nodes in the graph.
+   */
+  getInvalidNodeCount() {
+    return this.invalidNodeIds.size;
+  }
 }
 exports.RequestGraph = RequestGraph;
 class RequestTracker {
@@ -714,11 +776,6 @@ class RequestTracker {
     this.options = options;
     this.rustAtlaspack = rustAtlaspack;
   }
-
-  // TODO: refactor (abortcontroller should be created by RequestTracker)
-  setSignal(signal) {
-    this.signal = signal;
-  }
   startRequest(request) {
     let didPreviouslyExist = this.graph.hasContentKey(request.id);
     let requestNodeId;
@@ -736,6 +793,7 @@ class RequestTracker {
       promise,
       deferred
     } = (0, _utils().makeDeferredWithPromise)();
+    // @ts-expect-error TS2345
     this.graph.incompleteNodePromises.set(requestNodeId, promise);
     return {
       requestNodeId,
@@ -761,15 +819,12 @@ class RequestTracker {
       return null;
     }
     if (node.result != undefined) {
-      // $FlowFixMe
       let result = node.result;
       return result;
     } else if (node.resultCacheKey != null && ifMatch == null) {
       let key = node.resultCacheKey;
-      if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
-        (0, _assert().default)(this.options.cache.hasLargeBlob(key));
-      }
-      let cachedResult = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? (0, _nullthrows().default)(await this.options.cache.get(key)) : (0, _buildCache().deserialize)(await this.options.cache.getLargeBlob(key));
+      (0, _assert().default)(this.options.cache.hasLargeBlob(key));
+      let cachedResult = (0, _buildCache().deserialize)(await this.options.cache.getLargeBlob(key));
       node.result = cachedResult;
       return cachedResult;
     }
@@ -815,7 +870,7 @@ class RequestTracker {
     let requestId = hasKey ? this.graph.getNodeIdByContentKey(request.id) : undefined;
     let hasValidResult = requestId != null && this.hasValidResult(requestId);
     if (!(opts !== null && opts !== void 0 && opts.force) && hasValidResult) {
-      // $FlowFixMe[incompatible-type]
+      // @ts-expect-error TS2322
       return this.getRequestResult(request.id);
     }
     if (requestId != null) {
@@ -824,7 +879,7 @@ class RequestTracker {
         // There is a another instance of this request already running, wait for its completion and reuse its result
         try {
           if (await incompletePromise) {
-            // $FlowFixMe[incompatible-type]
+            // @ts-expect-error TS2322
             return this.getRequestResult(request.id);
           }
         } catch (e) {
@@ -857,7 +912,6 @@ class RequestTracker {
         options: this.options,
         rustAtlaspack: this.rustAtlaspack
       });
-      (0, _utils2.assertSignalNotAborted)(this.signal);
       this.completeRequest(requestNodeId);
       deferred.resolve(true);
       return result;
@@ -895,133 +949,123 @@ class RequestTracker {
   }
   createAPI(requestId, previousInvalidations) {
     let subRequestContentKeys = new Set();
-    return {
-      api: {
-        invalidateOnFileCreate: input => this.graph.invalidateOnFileCreate(requestId, input),
-        invalidateOnConfigKeyChange: (filePath, configKey, contentHash) => this.graph.invalidateOnConfigKeyChange(requestId, filePath, configKey, contentHash),
-        invalidateOnFileDelete: filePath => this.graph.invalidateOnFileDelete(requestId, filePath),
-        invalidateOnFileUpdate: filePath => this.graph.invalidateOnFileUpdate(requestId, filePath),
-        invalidateOnStartup: () => this.graph.invalidateOnStartup(requestId),
-        invalidateOnBuild: () => this.graph.invalidateOnBuild(requestId),
-        invalidateOnEnvChange: env => this.graph.invalidateOnEnvChange(requestId, env, this.options.env[env]),
-        invalidateOnOptionChange: option => this.graph.invalidateOnOptionChange(requestId, option, this.options[option]),
-        getInvalidations: () => previousInvalidations,
-        storeResult: (result, cacheKey) => {
-          this.storeResult(requestId, result, cacheKey);
-        },
-        getSubRequests: () => this.graph.getSubRequests(requestId),
-        getInvalidSubRequests: () => this.graph.getInvalidSubRequests(requestId),
-        getPreviousResult: ifMatch => {
-          var _this$graph$getNode;
-          let contentKey = (0, _nullthrows().default)((_this$graph$getNode = this.graph.getNode(requestId)) === null || _this$graph$getNode === void 0 ? void 0 : _this$graph$getNode.id);
-          return this.getRequestResult(contentKey, ifMatch);
-        },
-        getRequestResult: id => this.getRequestResult(id),
-        canSkipSubrequest: contentKey => {
-          if (this.graph.hasContentKey(contentKey) && this.hasValidResult(this.graph.getNodeIdByContentKey(contentKey))) {
-            subRequestContentKeys.add(contentKey);
-            return true;
-          }
-          return false;
-        },
-        runRequest: (subRequest, opts) => {
-          subRequestContentKeys.add(subRequest.id);
-          return this.runRequest(subRequest, opts);
+    let api = {
+      invalidateOnFileCreate: input => this.graph.invalidateOnFileCreate(requestId, input),
+      invalidateOnConfigKeyChange: (filePath, configKey, contentHash) => this.graph.invalidateOnConfigKeyChange(requestId, filePath, configKey, contentHash),
+      invalidateOnFileDelete: filePath => this.graph.invalidateOnFileDelete(requestId, filePath),
+      invalidateOnFileUpdate: filePath => this.graph.invalidateOnFileUpdate(requestId, filePath),
+      invalidateOnStartup: () => this.graph.invalidateOnStartup(requestId),
+      invalidateOnBuild: () => this.graph.invalidateOnBuild(requestId),
+      invalidateOnEnvChange: env => this.graph.invalidateOnEnvChange(requestId, env, this.options.env[env]),
+      invalidateOnOptionChange: option => this.graph.invalidateOnOptionChange(requestId, option,
+      // @ts-expect-error TS7053
+      this.options[option]),
+      getInvalidations: () => previousInvalidations,
+      storeResult: (result, cacheKey) => {
+        this.storeResult(requestId, result, cacheKey);
+      },
+      getSubRequests: () => this.graph.getSubRequests(requestId),
+      getInvalidSubRequests: () => this.graph.getInvalidSubRequests(requestId),
+      getPreviousResult: ifMatch => {
+        var _this$graph$getNode;
+        let contentKey = (0, _nullthrows().default)((_this$graph$getNode = this.graph.getNode(requestId)) === null || _this$graph$getNode === void 0 ? void 0 : _this$graph$getNode.id);
+        return this.getRequestResult(contentKey, ifMatch);
+      },
+      getRequestResult: (
+      // @ts-expect-error TS7006
+      id) => this.getRequestResult(id),
+      canSkipSubrequest: contentKey => {
+        if (this.graph.hasContentKey(contentKey) && this.hasValidResult(this.graph.getNodeIdByContentKey(contentKey))) {
+          subRequestContentKeys.add(contentKey);
+          return true;
         }
+        return false;
       },
+      runRequest: (subRequest, opts) => {
+        subRequestContentKeys.add(subRequest.id);
+        return this.runRequest(subRequest, opts);
+      }
+    };
+    return {
+      api,
       subRequestContentKeys
     };
   }
   async writeToCache(signal) {
-    const options = this.options;
-    async function runCacheImprovements(newPath, oldPath) {
-      if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
-        (0, _assert().default)(options.cache instanceof _cache().LMDBLiteCache);
-        const result = await newPath(options.cache);
-        return result;
-      } else {
-        const result = await oldPath();
-        return result;
-      }
-    }
-    await runCacheImprovements(async cache => {
-      await cache.getNativeRef().startWriteTransaction();
-    }, () => Promise.resolve());
     let cacheKey = getCacheKey(this.options);
     let requestGraphKey = `requestGraph-${cacheKey}`;
+    let snapshotKey = `snapshot-${cacheKey}`;
     if (this.options.shouldDisableCache) {
       return;
     }
     let total = 0;
-    (0, _ReporterRunner.report)({
-      type: 'cache',
-      phase: 'start',
-      total,
-      size: this.graph.nodes.length
-    });
-    let serialisedGraph = this.graph.serialize();
-
-    // Delete an existing request graph cache, to prevent invalid states
-    await this.options.cache.deleteLargeBlob(requestGraphKey);
-    const serialiseAndSet = async (key, contents) => {
-      if (signal !== null && signal !== void 0 && signal.aborted) {
-        throw new Error('Serialization was aborted');
-      }
-      await runCacheImprovements(cache => {
-        (0, _logger().instrument)(`cache.put(${key})`, () => {
-          cache.getNativeRef().putNoConfirm(key, (0, _buildCache().serialize)(contents));
-        });
-        return Promise.resolve();
-      }, async () => {
-        await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
-          signal: signal
-        } : undefined);
-      });
-      total += 1;
+    try {
       (0, _ReporterRunner.report)({
         type: 'cache',
-        phase: 'write',
+        phase: 'start',
         total,
         size: this.graph.nodes.length
       });
-    };
-    let queue = new (_utils().PromiseQueue)({
-      maxConcurrent: 32
-    });
+      if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
+        await (0, _EnvironmentManager.writeEnvironmentsToCache)(this.options.cache);
+      }
+      let serialisedGraph = this.graph.serialize();
 
-    // Preallocating a sparse array is faster than pushing when N is high enough
-    let cacheableNodes = new Array(serialisedGraph.nodes.length);
-    for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
-      let node = serialisedGraph.nodes[i];
-      let resultCacheKey = node === null || node === void 0 ? void 0 : node.resultCacheKey;
-      if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST && resultCacheKey != null && (node === null || node === void 0 ? void 0 : node.result) != null) {
-        queue.add(() => serialiseAndSet(resultCacheKey, node.result));
+      // Delete an existing request graph cache, to prevent invalid states
+      await this.options.cache.deleteLargeBlob(requestGraphKey);
+      const serialiseAndSet = async (key, contents) => {
+        if (signal !== null && signal !== void 0 && signal.aborted) {
+          throw new Error('Serialization was aborted');
+        }
+        await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
+          signal: signal
+        } : undefined);
+        total += 1;
+        (0, _ReporterRunner.report)({
+          type: 'cache',
+          phase: 'write',
+          total,
+          size: this.graph.nodes.length
+        });
+      };
+      let queue = new (_utils().PromiseQueue)({
+        maxConcurrent: 32
+      });
 
-        // eslint-disable-next-line no-unused-vars
-        let {
-          result: _,
-          ...newNode
-        } = node;
-        cacheableNodes[i] = newNode;
-      } else {
-        cacheableNodes[i] = node;
+      // Preallocating a sparse array is faster than pushing when N is high enough
+      let cacheableNodes = new Array(serialisedGraph.nodes.length);
+      for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
+        let node = serialisedGraph.nodes[i];
+
+        // @ts-expect-error TS2339
+        let resultCacheKey = node === null || node === void 0 ? void 0 : node.resultCacheKey;
+        if ((node === null || node === void 0 ? void 0 : node.type) === REQUEST && resultCacheKey != null && (node === null || node === void 0 ? void 0 : node.result) != null) {
+          queue.add(() => serialiseAndSet(resultCacheKey, node.result));
+
+          // eslint-disable-next-line no-unused-vars
+          let {
+            result: _,
+            ...newNode
+          } = node;
+          cacheableNodes[i] = newNode;
+        } else {
+          cacheableNodes[i] = node;
+        }
       }
-    }
-    let nodeCountsPerBlob = [];
-    for (let i = 0; i * this.graph.nodesPerBlob < cacheableNodes.length; i += 1) {
-      let nodesStartIndex = i * this.graph.nodesPerBlob;
-      let nodesEndIndex = Math.min((i + 1) * this.graph.nodesPerBlob, cacheableNodes.length);
-      nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
-      if (!this.graph.hasCachedRequestChunk(i)) {
-        // We assume the request graph nodes are immutable and won't change
-        let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
-        queue.add(() => serialiseAndSet(getRequestGraphNodeKey(i, cacheKey), nodesToCache).then(() => {
-          // Succeeded in writing to disk, save that we have completed this chunk
-          this.graph.setCachedRequestChunk(i);
-        }));
+      let nodeCountsPerBlob = [];
+      for (let i = 0; i * this.graph.nodesPerBlob < cacheableNodes.length; i += 1) {
+        let nodesStartIndex = i * this.graph.nodesPerBlob;
+        let nodesEndIndex = Math.min((i + 1) * this.graph.nodesPerBlob, cacheableNodes.length);
+        nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
+        if (!this.graph.hasCachedRequestChunk(i)) {
+          // We assume the request graph nodes are immutable and won't change
+          let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
+          queue.add(() => serialiseAndSet(getRequestGraphNodeKey(i, cacheKey), nodesToCache).then(() => {
+            // Succeeded in writing to disk, save that we have completed this chunk
+            this.graph.setCachedRequestChunk(i);
+          }));
+        }
       }
-    }
-    try {
       await queue.run();
 
       // Set the request graph after the queue is flushed to avoid writing an invalid state
@@ -1030,23 +1074,13 @@ class RequestTracker {
         nodeCountsPerBlob,
         nodes: undefined
       });
-      await runCacheImprovements(() => serialiseAndSet(`request_tracker:cache_metadata:${cacheKey}`, {
-        version: _constants.ATLASPACK_VERSION,
-        entries: this.options.entries,
-        mode: this.options.mode,
-        shouldBuildLazily: this.options.shouldBuildLazily,
-        watchBackend: this.options.watchBackend
-      }), () => Promise.resolve());
       let opts = getWatcherOptions(this.options);
-      let snapshotPath = _path2().default.join(this.options.cacheDir, `snapshot-${cacheKey}` + '.txt');
+      let snapshotPath = _path2().default.join(this.options.cacheDir, snapshotKey + '.txt');
       await this.options.outputFS.writeSnapshot(this.options.watchDir, snapshotPath, opts);
     } catch (err) {
       // If we have aborted, ignore the error and continue
       if (!(signal !== null && signal !== void 0 && signal.aborted)) throw err;
     }
-    await runCacheImprovements(async cache => {
-      await cache.getNativeRef().commitWriteTransaction();
-    }, () => Promise.resolve());
     (0, _ReporterRunner.report)({
       type: 'cache',
       phase: 'end',
@@ -1075,7 +1109,8 @@ function getWatcherOptions({
   watchDir,
   watchBackend
 }) {
-  const uniqueDirs = [...new Set([...watchIgnore, ...['.git', '.hg'], cacheDir])];
+  const vcsDirs = ['.git', '.hg'];
+  const uniqueDirs = [...new Set([...watchIgnore, ...vcsDirs, cacheDir])];
   const ignore = uniqueDirs.map(dir => _path2().default.resolve(watchDir, dir));
   return {
     ignore,
@@ -1091,12 +1126,14 @@ function getRequestGraphNodeKey(index, cacheKey) {
 async function readAndDeserializeRequestGraph(cache, requestGraphKey, cacheKey) {
   let bufferLength = 0;
   const getAndDeserialize = async key => {
-    let buffer = await cache.getLargeBlob(key);
+    const buffer = await cache.getLargeBlob(key);
     bufferLength += Buffer.byteLength(buffer);
     return (0, _buildCache().deserialize)(buffer);
   };
   let serializedRequestGraph = await getAndDeserialize(requestGraphKey);
-  let nodePromises = serializedRequestGraph.nodeCountsPerBlob.map(async (nodesCount, i) => {
+  let nodePromises = serializedRequestGraph.nodeCountsPerBlob.map(
+  // @ts-expect-error TS7006
+  async (nodesCount, i) => {
     let nodes = await getAndDeserialize(getRequestGraphNodeKey(i, cacheKey));
     _assert().default.equal(nodes.length, nodesCount, 'RequestTracker node chunk: invalid node count');
     return nodes;
@@ -1119,15 +1156,29 @@ async function loadRequestGraph(options) {
   let timeout;
   const snapshotKey = `snapshot-${cacheKey}`;
   const snapshotPath = _path2().default.join(options.cacheDir, snapshotKey + '.txt');
+  const commonMeta = {
+    cacheKey,
+    snapshotKey,
+    cacheKeyOptions: {
+      version: _constants.ATLASPACK_VERSION,
+      entries: options.entries,
+      mode: options.mode,
+      shouldBuildLazily: options.shouldBuildLazily,
+      watchBackend: options.watchBackend
+    }
+  };
   _logger().default.verbose({
     origin: '@atlaspack/core',
     message: 'Loading request graph',
     meta: {
-      cacheKey,
-      snapshotKey
+      ...commonMeta
     }
   });
-  if (await options.cache.hasLargeBlob(requestGraphKey)) {
+  if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
+    await (0, _EnvironmentManager.loadEnvironmentsFromCache)(options.cache);
+  }
+  const hasRequestGraphInCache = await options.cache.hasLargeBlob(requestGraphKey);
+  if (hasRequestGraphInCache) {
     try {
       let {
         requestGraph
@@ -1146,16 +1197,22 @@ async function loadRequestGraph(options) {
         origin: '@atlaspack/core',
         message: `File system event count: ${events.length}`,
         meta: {
+          ...commonMeta,
           trackableEvent: 'watcher_events_count',
           watcherEventCount: events.length,
           duration: Date.now() - startTime
         }
       });
-      requestGraph.invalidateUnpredictableNodes();
-      requestGraph.invalidateOnBuildNodes();
-      requestGraph.invalidateEnvNodes(options.env);
-      requestGraph.invalidateOptionNodes(options);
-      await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
+      const invalidationStats = await invalidateRequestGraph(requestGraph, options, events);
+      _logger().default.verbose({
+        origin: '@atlaspack/core',
+        message: 'Request track loaded from cache',
+        meta: {
+          ...commonMeta,
+          trackableEvent: 'request_tracker_cache_key_hit',
+          invalidationStats
+        }
+      });
       return requestGraph;
     } catch (e) {
       // Prevent logging fs events took too long warning
@@ -1170,12 +1227,108 @@ async function loadRequestGraph(options) {
     origin: '@atlaspack/core',
     message: 'Cache entry for request tracker was not found, initializing a clean cache.',
     meta: {
-      cacheKey,
-      snapshotKey
+      ...commonMeta,
+      trackableEvent: 'request_tracker_cache_key_miss'
     }
   });
   return new RequestGraph();
 }
+
+/**
+ * A wrapper around an invalidation type / method
+ */
+
+/**
+ * Details about an invalidation.
+ *
+ * If this is a fs events invalidation, this key will contain statistics about invalidations
+ * by path.
+ *
+ * If this is a env or option invalidation, this key will contain the list of changed environment
+ * variables or options.
+ */
+
+/**
+ * Number of invalidations for a given file-system event.
+ */
+
+/**
+ * Information about a certain cache invalidation type.
+ */
+
+/**
+ * Respond to unpredictable, build, environment changes, option changes and file-system events
+ * invalidating RequestGraph nodes.
+ *
+ * Returns the count of nodes invalidated by each invalidation type.
+ */
+async function invalidateRequestGraph(requestGraph, options, events) {
+  const invalidationFns = [{
+    key: 'unpredictable',
+    // @ts-expect-error TS2322
+    fn: () => requestGraph.invalidateUnpredictableNodes()
+  }, {
+    key: 'onBuild',
+    // @ts-expect-error TS2322
+    fn: () => requestGraph.invalidateOnBuildNodes()
+  }, {
+    key: 'env',
+    fn: () => requestGraph.invalidateEnvNodes(options.env)
+  }, {
+    key: 'option',
+    fn: () => requestGraph.invalidateOptionNodes(options)
+  }, {
+    key: 'fsEvents',
+    fn: () => invalidateRequestGraphFSEvents(requestGraph, options, events)
+  }];
+  const invalidations = [];
+  for (const invalidation of invalidationFns) {
+    invalidations.push(await runInvalidation(requestGraph, invalidation));
+  }
+  const invalidatedCount = invalidations.reduce((acc, invalidation) => acc + invalidation.count, 0);
+  const requestCount = requestGraph.nodes.reduce((acc, node) => acc + ((node === null || node === void 0 ? void 0 : node.type) === REQUEST ? 1 : 0), 0);
+  const nodeCount = requestGraph.nodes.length;
+  const nodeInvalidationRatio = invalidatedCount / nodeCount;
+  const requestInvalidationRatio = invalidatedCount / requestCount;
+  return {
+    invalidations,
+    nodeCount,
+    requestCount,
+    invalidatedCount,
+    nodeInvalidationRatio,
+    requestInvalidationRatio
+  };
+}
+/**
+ * Invalidate the request graph based on file-system events.
+ *
+ * Returns statistics about the invalidations.
+ */
+async function invalidateRequestGraphFSEvents(requestGraph, options, events) {
+  const {
+    invalidationsByPath
+  } = await requestGraph.respondToFSEvents(options.unstableFileInvalidations || events, options, 10000, true);
+  const biggestInvalidations = getBiggestFSEventsInvalidations(invalidationsByPath);
+  return {
+    biggestInvalidations
+  };
+}
+/**
+ * Runs an invalidation function and reports metrics.
+ */
+async function runInvalidation(requestGraph, invalidationFn) {
+  const start = _perf_hooks().performance.now();
+  const startInvalidationCount = requestGraph.getInvalidNodeCount();
+  const result = await invalidationFn.fn();
+  const count = requestGraph.getInvalidNodeCount() - startInvalidationCount;
+  const duration = _perf_hooks().performance.now() - start;
+  return {
+    key: invalidationFn.key,
+    count,
+    detail: result ?? null,
+    duration
+  };
+}
 function logErrorOnBailout(options, snapshotPath, e) {
   if (e.message && e.message.includes('invalid clockspec')) {
     const snapshotContents = options.inputFS.readFileSync(snapshotPath, 'utf-8');
@@ -1215,4 +1368,19 @@ function cleanUpOrphans(graph) {
     }
   });
   return removedNodeIds;
+}
+
+/**
+ * Returns paths that invalidated the most nodes
+ */
+function getBiggestFSEventsInvalidations(invalidationsByPath, limit = 10) {
+  const invalidations = [];
+  for (const [path, count] of invalidationsByPath) {
+    invalidations.push({
+      path,
+      count
+    });
+  }
+  invalidations.sort((a, b) => b.count - a.count);
+  return invalidations.slice(0, limit);
 }