@atlaspack/core 2.17.3 → 2.18.0

package/src/RequestTracker.js

@@ -30,15 +30,15 @@ import nullthrows from 'nullthrows';
 
 import {
   ATLASPACK_VERSION,
-  VALID,
-  INITIAL_BUILD,
   FILE_CREATE,
-  FILE_UPDATE,
   FILE_DELETE,
+  FILE_UPDATE,
   ENV_CHANGE,
+  ERROR,
+  INITIAL_BUILD,
   OPTION_CHANGE,
   STARTUP,
-  ERROR,
+  VALID,
 } from './constants';
 import type {AtlaspackV3} from './atlaspack-v3/AtlaspackV3';
 import {
@@ -69,6 +69,12 @@ import type {
   InternalGlob,
 } from './types';
 import {BuildAbortError, assertSignalNotAborted, hashFromOption} from './utils';
+import {performance} from 'perf_hooks';
+
+import {
+  loadEnvironmentsFromCache,
+  writeEnvironmentsToCache,
+} from './EnvironmentManager';
 
 export const requestGraphEdgeTypes = {
   subrequest: 2,
@@ -143,7 +149,7 @@ type OptionNode = {|
 type ConfigKeyNode = {|
   id: ContentKey,
   +type: typeof CONFIG_KEY,
-  configKey: string,
+  configKey: string[],
   contentHash: string,
 |};
 
@@ -215,7 +221,7 @@ export type RunAPI<TResult: RequestResult> = {|
   invalidateOnFileUpdate: (ProjectPath) => void,
   invalidateOnConfigKeyChange: (
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
     contentHash: string,
   ) => void,
   invalidateOnStartup: () => void,
@@ -282,10 +288,12 @@ const nodeFromOption = (option: string, value: mixed): RequestGraphNode => ({
 
 const nodeFromConfigKey = (
   fileName: ProjectPath,
-  configKey: string,
+  configKey: string[],
   contentHash: string,
 ): RequestGraphNode => ({
-  id: `config_key:${fromProjectPathRelative(fileName)}:${configKey}`,
+  id: `config_key:${fromProjectPathRelative(fileName)}:${JSON.stringify(
+    configKey,
+  )}`,
   type: CONFIG_KEY,
   configKey,
   contentHash,
@@ -526,7 +534,7 @@ export class RequestGraph extends ContentGraph<
   invalidateOnConfigKeyChange(
     requestNodeId: NodeId,
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
     contentHash: string,
   ) {
     let configKeyNodeId = this.addNode(
@@ -713,8 +721,8 @@ export class RequestGraph extends ContentGraph<
     env: string,
     value: string | void,
   ) {
-    let envNode = nodeFromEnv(env, value);
-    let envNodeId = this.addNode(envNode);
+    const envNode = nodeFromEnv(env, value);
+    const envNodeId = this.addNode(envNode);
 
     if (
       !this.hasEdge(
@@ -934,7 +942,10 @@ export class RequestGraph extends ContentGraph<
      * True if this is the start-up (loading phase) invalidation.
      */
     isInitialBuild: boolean = false,
-  ): Async<boolean> {
+  ): Promise<{|
+    didInvalidate: boolean,
+    invalidationsByPath: Map<string, number>,
+  |}> {
     let didInvalidate = false;
     let count = 0;
     let predictedTime = 0;
@@ -972,7 +983,10 @@ export class RequestGraph extends ContentGraph<
       return above;
     };
 
+    const invalidationsByPath = new Map();
     for (let {path: _path, type} of events) {
+      const invalidationsBefore = this.getInvalidNodeCount();
+
       if (
         !enableOptimization &&
         process.env.ATLASPACK_DISABLE_CACHE_TIMEOUT !== 'true' &&
@@ -1018,7 +1032,10 @@ export class RequestGraph extends ContentGraph<
             this.invalidNodeIds.add(id);
           }
         }
-        return true;
+        return {
+          didInvalidate: true,
+          invalidationsByPath: new Map(),
+        };
       }
 
       // sometimes mac os reports update events as create events.
@@ -1099,11 +1116,22 @@ export class RequestGraph extends ContentGraph<
       }
 
       let configKeyNodes = this.configKeyNodes.get(_filePath);
-      if (configKeyNodes && (type === 'delete' || type === 'update')) {
+
+      // With granular invalidations we will always run this block,
+      // so even if we get a create event (for whatever reason), we will still
+      // try to limit invalidations from config key changes through hashing.
+      //
+      // Currently create events can invalidate a large number of nodes due to
+      // "create above" invalidations.
+      const isConfigKeyChange =
+        getFeatureFlag('granularTsConfigInvalidation') ||
+        type === 'delete' ||
+        type === 'update';
+      if (configKeyNodes && isConfigKeyChange) {
         for (let nodeId of configKeyNodes) {
           let isInvalid = type === 'delete';
 
-          if (type === 'update') {
+          if (type !== 'delete') {
             let node = this.getNode(nodeId);
             invariant(node && node.type === CONFIG_KEY);
 
@@ -1131,6 +1159,13 @@ export class RequestGraph extends ContentGraph<
           }
         }
       }
+
+      const invalidationsAfter = this.getInvalidNodeCount();
+      const invalidationsForEvent = invalidationsAfter - invalidationsBefore;
+      invalidationsByPath.set(
+        _path,
+        (invalidationsByPath.get(_path) ?? 0) + invalidationsForEvent,
+      );
     }
 
     if (getFeatureFlag('fixQuadraticCacheInvalidation')) {
@@ -1151,7 +1186,10 @@ export class RequestGraph extends ContentGraph<
       },
     });
 
-    return didInvalidate && this.invalidNodeIds.size > 0;
+    return {
+      didInvalidate,
+      invalidationsByPath,
+    };
   }
 
   hasCachedRequestChunk(index: number): boolean {
@@ -1165,6 +1203,13 @@ export class RequestGraph extends ContentGraph<
   removeCachedRequestChunkForNode(nodeId: number): void {
     this.cachedRequestChunks.delete(Math.floor(nodeId / this.nodesPerBlob));
   }
+
+  /**
+   * Returns the number of invalidated nodes in the graph.
+   */
+  getInvalidNodeCount(): number {
+    return this.invalidNodeIds.size;
+  }
 }
 
 export default class RequestTracker {
@@ -1288,7 +1333,13 @@ export default class RequestTracker {
     }
   }
 
-  respondToFSEvents(events: Array<Event>, threshold: number): Async<boolean> {
+  respondToFSEvents(
+    events: Array<Event>,
+    threshold: number,
+  ): Promise<{|
+    didInvalidate: boolean,
+    invalidationsByPath: Map<string, number>,
+  |}> {
     return this.graph.respondToFSEvents(events, this.options, threshold);
   }
 
@@ -1499,130 +1550,143 @@ export default class RequestTracker {
       }
     }
 
+    let cacheKey = getCacheKey(this.options);
+    let requestGraphKey = getFeatureFlag('cachePerformanceImprovements')
+      ? `${cacheKey}/RequestGraph`
+      : `requestGraph-${cacheKey}`;
+    let snapshotKey = getFeatureFlag('cachePerformanceImprovements')
+      ? `${cacheKey}/snapshot`
+      : `snapshot-${cacheKey}`;
+
+    if (this.options.shouldDisableCache) {
+      return;
+    }
+
+    let total = 0;
     await runCacheImprovements(
       async (cache) => {
         await cache.getNativeRef().startWriteTransaction();
       },
       () => Promise.resolve(),
     );
+    try {
+      report({
+        type: 'cache',
+        phase: 'start',
+        total,
+        size: this.graph.nodes.length,
+      });
 
-    let cacheKey = getCacheKey(this.options);
-    let requestGraphKey = `requestGraph-${cacheKey}`;
-    let snapshotKey = `snapshot-${cacheKey}`;
-
-    if (this.options.shouldDisableCache) {
-      return;
-    }
+      if (getFeatureFlag('environmentDeduplication')) {
+        await writeEnvironmentsToCache(options.cache);
+      }
 
-    let total = 0;
-    report({
-      type: 'cache',
-      phase: 'start',
-      total,
-      size: this.graph.nodes.length,
-    });
+      let serialisedGraph = this.graph.serialize();
 
-    let serialisedGraph = this.graph.serialize();
+      // Delete an existing request graph cache, to prevent invalid states
+      await this.options.cache.deleteLargeBlob(requestGraphKey);
 
-    // Delete an existing request graph cache, to prevent invalid states
-    await this.options.cache.deleteLargeBlob(requestGraphKey);
+      const serialiseAndSet = async (
+        key: string,
+        // $FlowFixMe serialise input is any type
+        contents: any,
+      ): Promise<void> => {
+        if (signal?.aborted) {
+          throw new Error('Serialization was aborted');
+        }
 
-    const serialiseAndSet = async (
-      key: string,
-      // $FlowFixMe serialise input is any type
-      contents: any,
-    ): Promise<void> => {
-      if (signal?.aborted) {
-        throw new Error('Serialization was aborted');
-      }
+        await runCacheImprovements(
+          (cache) => {
+            instrument(
+              `RequestTracker::writeToCache::cache.put(${key})`,
+              () => {
+                cache.getNativeRef().putNoConfirm(key, serialize(contents));
+              },
+            );
+            return Promise.resolve();
+          },
+          async () => {
+            await this.options.cache.setLargeBlob(
+              key,
+              serialize(contents),
+              signal
+                ? {
+                    signal: signal,
+                  }
+                : undefined,
+            );
+          },
+        );
 
-      await runCacheImprovements(
-        (cache) => {
-          instrument(`cache.put(${key})`, () => {
-            cache.getNativeRef().putNoConfirm(key, serialize(contents));
-          });
-          return Promise.resolve();
-        },
-        async () => {
-          await this.options.cache.setLargeBlob(
-            key,
-            serialize(contents),
-            signal
-              ? {
-                  signal: signal,
-                }
-              : undefined,
-          );
-        },
-      );
+        total += 1;
 
-      total += 1;
+        report({
+          type: 'cache',
+          phase: 'write',
+          total,
+          size: this.graph.nodes.length,
+        });
+      };
 
-      report({
-        type: 'cache',
-        phase: 'write',
-        total,
-        size: this.graph.nodes.length,
+      let queue = new PromiseQueue({
+        maxConcurrent: 32,
       });
-    };
 
-    let queue = new PromiseQueue({
-      maxConcurrent: 32,
-    });
-
-    // Preallocating a sparse array is faster than pushing when N is high enough
-    let cacheableNodes = new Array(serialisedGraph.nodes.length);
-    for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
-      let node = serialisedGraph.nodes[i];
+      // Preallocating a sparse array is faster than pushing when N is high enough
+      let cacheableNodes = new Array(serialisedGraph.nodes.length);
+      for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
+        let node = serialisedGraph.nodes[i];
 
-      let resultCacheKey = node?.resultCacheKey;
-      if (
-        node?.type === REQUEST &&
-        resultCacheKey != null &&
-        node?.result != null
-      ) {
-        queue.add(() => serialiseAndSet(resultCacheKey, node.result));
+        let resultCacheKey = node?.resultCacheKey;
+        if (
+          node?.type === REQUEST &&
+          resultCacheKey != null &&
+          node?.result != null
+        ) {
+          queue.add(() => serialiseAndSet(resultCacheKey, node.result));
 
-        // eslint-disable-next-line no-unused-vars
-        let {result: _, ...newNode} = node;
-        cacheableNodes[i] = newNode;
-      } else {
-        cacheableNodes[i] = node;
+          // eslint-disable-next-line no-unused-vars
+          let {result: _, ...newNode} = node;
+          cacheableNodes[i] = newNode;
+        } else {
+          cacheableNodes[i] = node;
+        }
       }
-    }
 
-    let nodeCountsPerBlob = [];
+      let nodeCountsPerBlob = [];
 
-    for (
-      let i = 0;
-      i * this.graph.nodesPerBlob < cacheableNodes.length;
-      i += 1
-    ) {
-      let nodesStartIndex = i * this.graph.nodesPerBlob;
-      let nodesEndIndex = Math.min(
-        (i + 1) * this.graph.nodesPerBlob,
-        cacheableNodes.length,
-      );
+      for (
+        let i = 0;
+        i * this.graph.nodesPerBlob < cacheableNodes.length;
+        i += 1
+      ) {
+        let nodesStartIndex = i * this.graph.nodesPerBlob;
+        let nodesEndIndex = Math.min(
+          (i + 1) * this.graph.nodesPerBlob,
+          cacheableNodes.length,
+        );
 
-      nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
+        nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
 
-      if (!this.graph.hasCachedRequestChunk(i)) {
-        // We assume the request graph nodes are immutable and won't change
-        let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
+        if (!this.graph.hasCachedRequestChunk(i)) {
+          // We assume the request graph nodes are immutable and won't change
+          let nodesToCache = cacheableNodes.slice(
+            nodesStartIndex,
+            nodesEndIndex,
+          );
 
-        queue.add(() =>
-          serialiseAndSet(
-            getRequestGraphNodeKey(i, cacheKey),
-            nodesToCache,
-          ).then(() => {
-            // Succeeded in writing to disk, save that we have completed this chunk
-            this.graph.setCachedRequestChunk(i);
-          }),
-        );
+          queue.add(() =>
+            serialiseAndSet(
+              getRequestGraphNodeKey(i, cacheKey),
+              nodesToCache,
+            ).then(() => {
+              // Succeeded in writing to disk, save that we have completed this chunk
+              this.graph.setCachedRequestChunk(i);
+            }),
+          );
+        }
       }
-    }
 
-    try {
       await queue.run();
 
       // Set the request graph after the queue is flushed to avoid writing an invalid state
@@ -1634,7 +1698,7 @@ export default class RequestTracker {
 
       await runCacheImprovements(
         () =>
-          serialiseAndSet(`request_tracker:cache_metadata:${cacheKey}`, {
+          serialiseAndSet(`${cacheKey}/cache_metadata`, {
             version: ATLASPACK_VERSION,
             entries: this.options.entries,
             mode: this.options.mode,
@@ -1655,15 +1719,15 @@ export default class RequestTracker {
     } catch (err) {
       // If we have aborted, ignore the error and continue
       if (!signal?.aborted) throw err;
+    } finally {
+      await runCacheImprovements(
+        async (cache) => {
+          await cache.getNativeRef().commitWriteTransaction();
+        },
+        () => Promise.resolve(),
+      );
     }
 
-    await runCacheImprovements(
-      async (cache) => {
-        await cache.getNativeRef().commitWriteTransaction();
-      },
-      () => Promise.resolve(),
-    );
-
     report({type: 'cache', phase: 'end', total, size: this.graph.nodes.length});
   }
 
@@ -1695,6 +1759,18 @@ export function getWatcherOptions({
 }
 
 function getCacheKey(options) {
+  if (getFeatureFlag('cachePerformanceImprovements')) {
+    const hash = hashString(
+      `${ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${
+        options.mode
+      }:${options.shouldBuildLazily ? 'lazy' : 'eager'}:${
+        options.watchBackend ?? ''
+      }`,
+    );
+
+    return `RequestTracker/${ATLASPACK_VERSION}/${hash}`;
+  }
+
   return hashString(
     `${ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${options.mode}:${
       options.shouldBuildLazily ? 'lazy' : 'eager'
@@ -1703,6 +1779,10 @@ function getCacheKey(options) {
 }
 
 function getRequestGraphNodeKey(index: number, cacheKey: string) {
+  if (getFeatureFlag('cachePerformanceImprovements')) {
+    return `${cacheKey}/RequestGraph/nodes/${index}`;
+  }
+
   return `requestGraph-nodes-${index}-${cacheKey}`;
 }
 
@@ -1714,9 +1794,15 @@ export async function readAndDeserializeRequestGraph(
   let bufferLength = 0;
 
   const getAndDeserialize = async (key: string) => {
-    let buffer = await cache.getLargeBlob(key);
-    bufferLength += Buffer.byteLength(buffer);
-    return deserialize(buffer);
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      const buffer = await cache.getBlob(key);
+      bufferLength += Buffer.byteLength(buffer);
+      return deserialize(buffer);
+    } else {
+      const buffer = await cache.getLargeBlob(key);
+      bufferLength += Buffer.byteLength(buffer);
+      return deserialize(buffer);
+    }
   };
 
   let serializedRequestGraph = await getAndDeserialize(requestGraphKey);
@@ -1749,9 +1835,14 @@ async function loadRequestGraph(options): Async<RequestGraph> {
   }
 
   let cacheKey = getCacheKey(options);
-  let requestGraphKey = `requestGraph-${cacheKey}`;
+  let requestGraphKey = getFeatureFlag('cachePerformanceImprovements')
+    ? `${cacheKey}/RequestGraph`
+    : `requestGraph-${cacheKey}`;
+
   let timeout;
-  const snapshotKey = `snapshot-${cacheKey}`;
+  const snapshotKey = getFeatureFlag('cachePerformanceImprovements')
+    ? `${cacheKey}/snapshot`
+    : `snapshot-${cacheKey}`;
   const snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt');
 
   const commonMeta = {
@@ -1774,7 +1865,15 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     },
   });
 
-  if (await options.cache.hasLargeBlob(requestGraphKey)) {
+  if (getFeatureFlag('environmentDeduplication')) {
+    await loadEnvironmentsFromCache(options.cache);
+  }
+
+  const hasRequestGraphInCache = getFeatureFlag('cachePerformanceImprovements')
+    ? await options.cache.has(requestGraphKey)
+    : await options.cache.hasLargeBlob(requestGraphKey);
+
+  if (hasRequestGraphInCache) {
     try {
       let {requestGraph} = await readAndDeserializeRequestGraph(
         options.cache,
@@ -1863,7 +1962,11 @@ async function loadRequestGraph(options): Async<RequestGraph> {
  */
 type InvalidationFn = {|
   key: string,
-  fn: () => string[] | void | Promise<void>,
+  fn: () =>
+    | InvalidationDetail
+    | Promise<InvalidationDetail>
+    | void
+    | Promise<void>,
 |};
 
 type InvalidationStats = {|
@@ -1893,6 +1996,33 @@ type InvalidationStats = {|
   invalidations: InvalidationFnStats[],
 |};
 
+/**
+ * Details about an invalidation.
+ *
+ * If this is a fs events invalidation, this key will contain statistics about invalidations
+ * by path.
+ *
+ * If this is a env or option invalidation, this key will contain the list of changed environment
+ * variables or options.
+ */
+type InvalidationDetail = string[] | FSInvalidationStats;
+
+/**
+ * Number of invalidations for a given file-system event.
+ */
+type FSInvalidation = {|
+  path: string,
+  count: number,
+|};
+
+type FSInvalidationStats = {|
+  /**
+   * This list will be sorted by the number of nodes invalidated and only the top 10 will be
+   * included.
+   */
+  biggestInvalidations: FSInvalidation[],
+|};
+
 /**
  * Information about a certain cache invalidation type.
  */
@@ -1913,8 +2043,14 @@ type InvalidationFnStats = {|
   count: number,
   /**
    * If this is a env or option invalidation, this key will contain the list of changed values.
+   *
+   * If this is a fs events invalidation, this key will contain statistics about invalidations
    */
-  changes: null | string[],
+  detail: null | InvalidationDetail,
+  /**
+   * Time in milliseconds it took to run the invalidation.
+   */
+  duration: number,
 |};
 
 /**
@@ -1923,7 +2059,7 @@ type InvalidationFnStats = {|
  *
  * Returns the count of nodes invalidated by each invalidation type.
  */
-async function invalidateRequestGraph(
+export async function invalidateRequestGraph(
   requestGraph: RequestGraph,
   options: AtlaspackOptions,
   events: Event[],
@@ -1947,14 +2083,7 @@ async function invalidateRequestGraph(
     },
     {
       key: 'fsEvents',
-      fn: async () => {
-        await requestGraph.respondToFSEvents(
-          options.unstableFileInvalidations || events,
-          options,
-          10000,
-          true,
-        );
-      },
+      fn: () => invalidateRequestGraphFSEvents(requestGraph, options, events),
     },
   ];
 
@@ -1984,22 +2113,61 @@ async function invalidateRequestGraph(
   };
 }
 
+interface InvalidateRequestGraphFSEventsInput {
+  respondToFSEvents(
+    events: Event[],
+    options: AtlaspackOptions,
+    timeout: number,
+    shouldLog: boolean,
+  ): Promise<{invalidationsByPath: Map<string, number>, ...}>;
+}
+
+/**
+ * Invalidate the request graph based on file-system events.
+ *
+ * Returns statistics about the invalidations.
+ */
+export async function invalidateRequestGraphFSEvents(
+  requestGraph: InvalidateRequestGraphFSEventsInput,
+  options: AtlaspackOptions,
+  events: Event[],
+): Promise<FSInvalidationStats> {
+  const {invalidationsByPath} = await requestGraph.respondToFSEvents(
+    options.unstableFileInvalidations || events,
+    options,
+    10000,
+    true,
+  );
+  const biggestInvalidations =
+    getBiggestFSEventsInvalidations(invalidationsByPath);
+
+  return {
+    biggestInvalidations,
+  };
+}
+
+interface RunInvalidationInput {
+  getInvalidNodeCount(): number;
+}
+
 /**
  * Runs an invalidation function and reports metrics.
  */
-async function runInvalidation(
-  requestGraph: RequestGraph,
+export async function runInvalidation(
+  requestGraph: RunInvalidationInput,
   invalidationFn: InvalidationFn,
 ): Promise<InvalidationFnStats> {
-  const startInvalidationCount = requestGraph.invalidNodeIds.size;
+  const start = performance.now();
+  const startInvalidationCount = requestGraph.getInvalidNodeCount();
   const result = await invalidationFn.fn();
-  const count = requestGraph.invalidNodeIds.size - startInvalidationCount;
+  const count = requestGraph.getInvalidNodeCount() - startInvalidationCount;
+  const duration = performance.now() - start;
 
   return {
     key: invalidationFn.key,
     count,
-    changes:
-      typeof result === 'object' && Array.isArray(result) ? result : null,
+    detail: result ?? null,
+    duration,
   };
 }
 
@@ -2054,3 +2222,19 @@ export function cleanUpOrphans<N, E: number>(graph: Graph<N, E>): NodeId[] {
 
   return removedNodeIds;
 }
+
+/**
+ * Returns paths that invalidated the most nodes
+ */
+export function getBiggestFSEventsInvalidations(
+  invalidationsByPath: Map<string, number>,
+  limit: number = 10,
+): Array<FSInvalidation> {
+  const invalidations = [];
+  for (const [path, count] of invalidationsByPath) {
+    invalidations.push({path, count});
+  }
+  invalidations.sort((a, b) => b.count - a.count);
+
+  return invalidations.slice(0, limit);
+}