@atlaspack/core 2.17.3 → 2.17.4

package/src/RequestTracker.js

@@ -69,6 +69,7 @@ import type {
   InternalGlob,
 } from './types';
 import {BuildAbortError, assertSignalNotAborted, hashFromOption} from './utils';
+import {performance} from 'perf_hooks';
 
 export const requestGraphEdgeTypes = {
   subrequest: 2,
@@ -934,7 +935,10 @@ export class RequestGraph extends ContentGraph<
      * True if this is the start-up (loading phase) invalidation.
      */
     isInitialBuild: boolean = false,
-  ): Async<boolean> {
+  ): Promise<{|
+    didInvalidate: boolean,
+    invalidationsByPath: Map<string, number>,
+  |}> {
     let didInvalidate = false;
     let count = 0;
     let predictedTime = 0;
@@ -972,7 +976,10 @@ export class RequestGraph extends ContentGraph<
       return above;
     };
 
+    const invalidationsByPath = new Map();
     for (let {path: _path, type} of events) {
+      const invalidationsBefore = this.getInvalidNodeCount();
+
       if (
         !enableOptimization &&
         process.env.ATLASPACK_DISABLE_CACHE_TIMEOUT !== 'true' &&
@@ -1018,7 +1025,10 @@ export class RequestGraph extends ContentGraph<
             this.invalidNodeIds.add(id);
           }
         }
-        return true;
+        return {
+          didInvalidate: true,
+          invalidationsByPath: new Map(),
+        };
       }
 
       // sometimes mac os reports update events as create events.
@@ -1131,6 +1141,13 @@ export class RequestGraph extends ContentGraph<
           }
         }
       }
+
+      const invalidationsAfter = this.getInvalidNodeCount();
+      const invalidationsForEvent = invalidationsAfter - invalidationsBefore;
+      invalidationsByPath.set(
+        _path,
+        (invalidationsByPath.get(_path) ?? 0) + invalidationsForEvent,
+      );
     }
 
     if (getFeatureFlag('fixQuadraticCacheInvalidation')) {
@@ -1151,7 +1168,10 @@ export class RequestGraph extends ContentGraph<
       },
     });
 
-    return didInvalidate && this.invalidNodeIds.size > 0;
+    return {
+      didInvalidate,
+      invalidationsByPath,
+    };
   }
 
   hasCachedRequestChunk(index: number): boolean {
@@ -1165,6 +1185,13 @@ export class RequestGraph extends ContentGraph<
   removeCachedRequestChunkForNode(nodeId: number): void {
     this.cachedRequestChunks.delete(Math.floor(nodeId / this.nodesPerBlob));
   }
+
+  /**
+   * Returns the number of invalidated nodes in the graph.
+   */
+  getInvalidNodeCount(): number {
+    return this.invalidNodeIds.size;
+  }
 }
 
 export default class RequestTracker {
@@ -1288,7 +1315,13 @@ export default class RequestTracker {
     }
   }
 
-  respondToFSEvents(events: Array<Event>, threshold: number): Async<boolean> {
+  respondToFSEvents(
+    events: Array<Event>,
+    threshold: number,
+  ): Promise<{|
+    didInvalidate: boolean,
+    invalidationsByPath: Map<string, number>,
+  |}> {
     return this.graph.respondToFSEvents(events, this.options, threshold);
   }
 
@@ -1499,130 +1532,139 @@ export default class RequestTracker {
       }
     }
 
-    await runCacheImprovements(
-      async (cache) => {
-        await cache.getNativeRef().startWriteTransaction();
-      },
-      () => Promise.resolve(),
-    );
-
     let cacheKey = getCacheKey(this.options);
-    let requestGraphKey = `requestGraph-${cacheKey}`;
-    let snapshotKey = `snapshot-${cacheKey}`;
+    let requestGraphKey = getFeatureFlag('cachePerformanceImprovements')
+      ? `${cacheKey}/RequestGraph`
+      : `requestGraph-${cacheKey}`;
+    let snapshotKey = getFeatureFlag('cachePerformanceImprovements')
+      ? `${cacheKey}/snapshot`
+      : `snapshot-${cacheKey}`;
 
     if (this.options.shouldDisableCache) {
       return;
     }
 
     let total = 0;
-    report({
-      type: 'cache',
-      phase: 'start',
-      total,
-      size: this.graph.nodes.length,
-    });
+    await runCacheImprovements(
+      async (cache) => {
+        await cache.getNativeRef().startWriteTransaction();
+      },
+      () => Promise.resolve(),
+    );
+    try {
+      report({
+        type: 'cache',
+        phase: 'start',
+        total,
+        size: this.graph.nodes.length,
+      });
 
-    let serialisedGraph = this.graph.serialize();
+      let serialisedGraph = this.graph.serialize();
 
-    // Delete an existing request graph cache, to prevent invalid states
-    await this.options.cache.deleteLargeBlob(requestGraphKey);
+      // Delete an existing request graph cache, to prevent invalid states
+      await this.options.cache.deleteLargeBlob(requestGraphKey);
 
-    const serialiseAndSet = async (
-      key: string,
-      // $FlowFixMe serialise input is any type
-      contents: any,
-    ): Promise<void> => {
-      if (signal?.aborted) {
-        throw new Error('Serialization was aborted');
-      }
+      const serialiseAndSet = async (
+        key: string,
+        // $FlowFixMe serialise input is any type
+        contents: any,
+      ): Promise<void> => {
+        if (signal?.aborted) {
+          throw new Error('Serialization was aborted');
+        }
 
-      await runCacheImprovements(
-        (cache) => {
-          instrument(`cache.put(${key})`, () => {
-            cache.getNativeRef().putNoConfirm(key, serialize(contents));
-          });
-          return Promise.resolve();
-        },
-        async () => {
-          await this.options.cache.setLargeBlob(
-            key,
-            serialize(contents),
-            signal
-              ? {
-                  signal: signal,
-                }
-              : undefined,
-          );
-        },
-      );
+        await runCacheImprovements(
+          (cache) => {
+            instrument(
+              `RequestTracker::writeToCache::cache.put(${key})`,
+              () => {
+                cache.getNativeRef().putNoConfirm(key, serialize(contents));
+              },
+            );
+            return Promise.resolve();
+          },
+          async () => {
+            await this.options.cache.setLargeBlob(
+              key,
+              serialize(contents),
+              signal
+                ? {
+                    signal: signal,
+                  }
+                : undefined,
+            );
+          },
+        );
 
-      total += 1;
+        total += 1;
 
-      report({
-        type: 'cache',
-        phase: 'write',
-        total,
-        size: this.graph.nodes.length,
-      });
-    };
+        report({
+          type: 'cache',
+          phase: 'write',
+          total,
+          size: this.graph.nodes.length,
+        });
+      };
 
-    let queue = new PromiseQueue({
-      maxConcurrent: 32,
-    });
+      let queue = new PromiseQueue({
+        maxConcurrent: 32,
+      });
 
-    // Preallocating a sparse array is faster than pushing when N is high enough
-    let cacheableNodes = new Array(serialisedGraph.nodes.length);
-    for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
-      let node = serialisedGraph.nodes[i];
+      // Preallocating a sparse array is faster than pushing when N is high enough
+      let cacheableNodes = new Array(serialisedGraph.nodes.length);
+      for (let i = 0; i < serialisedGraph.nodes.length; i += 1) {
+        let node = serialisedGraph.nodes[i];
 
-      let resultCacheKey = node?.resultCacheKey;
-      if (
-        node?.type === REQUEST &&
-        resultCacheKey != null &&
-        node?.result != null
-      ) {
-        queue.add(() => serialiseAndSet(resultCacheKey, node.result));
+        let resultCacheKey = node?.resultCacheKey;
+        if (
+          node?.type === REQUEST &&
+          resultCacheKey != null &&
+          node?.result != null
+        ) {
+          queue.add(() => serialiseAndSet(resultCacheKey, node.result));
 
-        // eslint-disable-next-line no-unused-vars
-        let {result: _, ...newNode} = node;
-        cacheableNodes[i] = newNode;
-      } else {
-        cacheableNodes[i] = node;
+          // eslint-disable-next-line no-unused-vars
+          let {result: _, ...newNode} = node;
+          cacheableNodes[i] = newNode;
+        } else {
+          cacheableNodes[i] = node;
+        }
       }
-    }
 
-    let nodeCountsPerBlob = [];
+      let nodeCountsPerBlob = [];
 
-    for (
-      let i = 0;
-      i * this.graph.nodesPerBlob < cacheableNodes.length;
-      i += 1
-    ) {
-      let nodesStartIndex = i * this.graph.nodesPerBlob;
-      let nodesEndIndex = Math.min(
-        (i + 1) * this.graph.nodesPerBlob,
-        cacheableNodes.length,
-      );
+      for (
+        let i = 0;
+        i * this.graph.nodesPerBlob < cacheableNodes.length;
+        i += 1
+      ) {
+        let nodesStartIndex = i * this.graph.nodesPerBlob;
+        let nodesEndIndex = Math.min(
+          (i + 1) * this.graph.nodesPerBlob,
+          cacheableNodes.length,
+        );
 
-      nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
+        nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex);
 
-      if (!this.graph.hasCachedRequestChunk(i)) {
-        // We assume the request graph nodes are immutable and won't change
-        let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex);
+        if (!this.graph.hasCachedRequestChunk(i)) {
+          // We assume the request graph nodes are immutable and won't change
+          let nodesToCache = cacheableNodes.slice(
+            nodesStartIndex,
+            nodesEndIndex,
+          );
 
-        queue.add(() =>
-          serialiseAndSet(
-            getRequestGraphNodeKey(i, cacheKey),
-            nodesToCache,
-          ).then(() => {
-            // Succeeded in writing to disk, save that we have completed this chunk
-            this.graph.setCachedRequestChunk(i);
-          }),
-        );
+          queue.add(() =>
+            serialiseAndSet(
+              getRequestGraphNodeKey(i, cacheKey),
+              nodesToCache,
+            ).then(() => {
+              // Succeeded in writing to disk, save that we have completed this chunk
+              this.graph.setCachedRequestChunk(i);
+            }),
+          );
+        }
       }
-    }
 
-    try {
       await queue.run();
 
       // Set the request graph after the queue is flushed to avoid writing an invalid state
@@ -1634,7 +1676,7 @@ export default class RequestTracker {
 
       await runCacheImprovements(
         () =>
-          serialiseAndSet(`request_tracker:cache_metadata:${cacheKey}`, {
+          serialiseAndSet(`${cacheKey}/cache_metadata`, {
             version: ATLASPACK_VERSION,
             entries: this.options.entries,
             mode: this.options.mode,
@@ -1655,15 +1697,15 @@ export default class RequestTracker {
     } catch (err) {
       // If we have aborted, ignore the error and continue
       if (!signal?.aborted) throw err;
+    } finally {
+      await runCacheImprovements(
+        async (cache) => {
+          await cache.getNativeRef().commitWriteTransaction();
+        },
+        () => Promise.resolve(),
+      );
     }
 
-    await runCacheImprovements(
-      async (cache) => {
-        await cache.getNativeRef().commitWriteTransaction();
-      },
-      () => Promise.resolve(),
-    );
-
     report({type: 'cache', phase: 'end', total, size: this.graph.nodes.length});
   }
 
@@ -1695,6 +1737,18 @@ export function getWatcherOptions({
 }
 
 function getCacheKey(options) {
+  if (getFeatureFlag('cachePerformanceImprovements')) {
+    const hash = hashString(
+      `${ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${
+        options.mode
+      }:${options.shouldBuildLazily ? 'lazy' : 'eager'}:${
+        options.watchBackend ?? ''
+      }`,
+    );
+
+    return `RequestTracker/${ATLASPACK_VERSION}/${hash}`;
+  }
+
   return hashString(
     `${ATLASPACK_VERSION}:${JSON.stringify(options.entries)}:${options.mode}:${
       options.shouldBuildLazily ? 'lazy' : 'eager'
@@ -1703,6 +1757,10 @@ function getCacheKey(options) {
 }
 
 function getRequestGraphNodeKey(index: number, cacheKey: string) {
+  if (getFeatureFlag('cachePerformanceImprovements')) {
+    return `${cacheKey}/RequestGraph/nodes/${index}`;
+  }
+
   return `requestGraph-nodes-${index}-${cacheKey}`;
 }
 
@@ -1714,9 +1772,15 @@ export async function readAndDeserializeRequestGraph(
   let bufferLength = 0;
 
   const getAndDeserialize = async (key: string) => {
-    let buffer = await cache.getLargeBlob(key);
-    bufferLength += Buffer.byteLength(buffer);
-    return deserialize(buffer);
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      const buffer = await cache.getBlob(key);
+      bufferLength += Buffer.byteLength(buffer);
+      return deserialize(buffer);
+    } else {
+      const buffer = await cache.getLargeBlob(key);
+      bufferLength += Buffer.byteLength(buffer);
+      return deserialize(buffer);
+    }
   };
 
   let serializedRequestGraph = await getAndDeserialize(requestGraphKey);
@@ -1749,9 +1813,14 @@ async function loadRequestGraph(options): Async<RequestGraph> {
   }
 
   let cacheKey = getCacheKey(options);
-  let requestGraphKey = `requestGraph-${cacheKey}`;
+  let requestGraphKey = getFeatureFlag('cachePerformanceImprovements')
+    ? `${cacheKey}/RequestGraph`
+    : `requestGraph-${cacheKey}`;
+
   let timeout;
-  const snapshotKey = `snapshot-${cacheKey}`;
+  const snapshotKey = getFeatureFlag('cachePerformanceImprovements')
+    ? `${cacheKey}/snapshot`
+    : `snapshot-${cacheKey}`;
   const snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt');
 
   const commonMeta = {
@@ -1774,7 +1843,11 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     },
   });
 
-  if (await options.cache.hasLargeBlob(requestGraphKey)) {
+  const hasRequestGraphInCache = getFeatureFlag('cachePerformanceImprovements')
+    ? await options.cache.has(requestGraphKey)
+    : await options.cache.hasLargeBlob(requestGraphKey);
+
+  if (hasRequestGraphInCache) {
     try {
       let {requestGraph} = await readAndDeserializeRequestGraph(
         options.cache,
@@ -1863,7 +1936,11 @@ async function loadRequestGraph(options): Async<RequestGraph> {
  */
 type InvalidationFn = {|
   key: string,
-  fn: () => string[] | void | Promise<void>,
+  fn: () =>
+    | InvalidationDetail
+    | Promise<InvalidationDetail>
+    | void
+    | Promise<void>,
 |};
 
 type InvalidationStats = {|
@@ -1893,6 +1970,33 @@ type InvalidationStats = {|
   invalidations: InvalidationFnStats[],
 |};
 
+/**
+ * Details about an invalidation.
+ *
+ * If this is a fs events invalidation, this key will contain statistics about invalidations
+ * by path.
+ *
+ * If this is a env or option invalidation, this key will contain the list of changed environment
+ * variables or options.
+ */
+type InvalidationDetail = string[] | FSInvalidationStats;
+
+/**
+ * Number of invalidations for a given file-system event.
+ */
+type FSInvalidation = {|
+  path: string,
+  count: number,
+|};
+
+type FSInvalidationStats = {|
+  /**
+   * This list will be sorted by the number of nodes invalidated and only the top 10 will be
+   * included.
+   */
+  biggestInvalidations: FSInvalidation[],
+|};
+
 /**
  * Information about a certain cache invalidation type.
  */
@@ -1913,8 +2017,14 @@ type InvalidationFnStats = {|
   count: number,
   /**
    * If this is a env or option invalidation, this key will contain the list of changed values.
+   *
+   * If this is a fs events invalidation, this key will contain statistics about invalidations
    */
-  changes: null | string[],
+  detail: null | InvalidationDetail,
+  /**
+   * Time in milliseconds it took to run the invalidation.
+   */
+  duration: number,
 |};
 
 /**
@@ -1923,7 +2033,7 @@ type InvalidationFnStats = {|
  *
  * Returns the count of nodes invalidated by each invalidation type.
  */
-async function invalidateRequestGraph(
+export async function invalidateRequestGraph(
   requestGraph: RequestGraph,
   options: AtlaspackOptions,
   events: Event[],
@@ -1947,14 +2057,7 @@ async function invalidateRequestGraph(
     },
     {
       key: 'fsEvents',
-      fn: async () => {
-        await requestGraph.respondToFSEvents(
-          options.unstableFileInvalidations || events,
-          options,
-          10000,
-          true,
-        );
-      },
+      fn: () => invalidateRequestGraphFSEvents(requestGraph, options, events),
     },
   ];
 
@@ -1984,22 +2087,61 @@ async function invalidateRequestGraph(
   };
 }
 
+interface InvalidateRequestGraphFSEventsInput {
+  respondToFSEvents(
+    events: Event[],
+    options: AtlaspackOptions,
+    timeout: number,
+    shouldLog: boolean,
+  ): Promise<{invalidationsByPath: Map<string, number>, ...}>;
+}
+
+/**
+ * Invalidate the request graph based on file-system events.
+ *
+ * Returns statistics about the invalidations.
+ */
+export async function invalidateRequestGraphFSEvents(
+  requestGraph: InvalidateRequestGraphFSEventsInput,
+  options: AtlaspackOptions,
+  events: Event[],
+): Promise<FSInvalidationStats> {
+  const {invalidationsByPath} = await requestGraph.respondToFSEvents(
+    options.unstableFileInvalidations || events,
+    options,
+    10000,
+    true,
+  );
+  const biggestInvalidations =
+    getBiggestFSEventsInvalidations(invalidationsByPath);
+
+  return {
+    biggestInvalidations,
+  };
+}
+
+interface RunInvalidationInput {
+  getInvalidNodeCount(): number;
+}
+
 /**
  * Runs an invalidation function and reports metrics.
  */
-async function runInvalidation(
-  requestGraph: RequestGraph,
+export async function runInvalidation(
+  requestGraph: RunInvalidationInput,
   invalidationFn: InvalidationFn,
 ): Promise<InvalidationFnStats> {
-  const startInvalidationCount = requestGraph.invalidNodeIds.size;
+  const start = performance.now();
+  const startInvalidationCount = requestGraph.getInvalidNodeCount();
   const result = await invalidationFn.fn();
-  const count = requestGraph.invalidNodeIds.size - startInvalidationCount;
+  const count = requestGraph.getInvalidNodeCount() - startInvalidationCount;
+  const duration = performance.now() - start;
 
   return {
     key: invalidationFn.key,
     count,
-    changes:
-      typeof result === 'object' && Array.isArray(result) ? result : null,
+    detail: result ?? null,
+    duration,
   };
 }
 
@@ -2054,3 +2196,19 @@ export function cleanUpOrphans<N, E: number>(graph: Graph<N, E>): NodeId[] {
 
   return removedNodeIds;
 }
+
+/**
+ * Returns paths that invalidated the most nodes
+ */
+export function getBiggestFSEventsInvalidations(
+  invalidationsByPath: Map<string, number>,
+  limit: number = 10,
+): Array<FSInvalidation> {
+  const invalidations = [];
+  for (const [path, count] of invalidationsByPath) {
+    invalidations.push({path, count});
+  }
+  invalidations.sort((a, b) => b.count - a.count);
+
+  return invalidations.slice(0, limit);
+}

package/src/UncommittedAsset.js

@@ -30,7 +30,12 @@ import {ATLASPACK_VERSION} from './constants';
 import {createAsset, createAssetIdFromOptions} from './assetUtils';
 import {BundleBehaviorNames} from './types';
 import {invalidateOnFileCreateToInternal, createInvalidations} from './utils';
-import {type ProjectPath, fromProjectPath} from './projectPath';
+import {
+  type ProjectPath,
+  fromProjectPath,
+  fromProjectPathRelative,
+} from './projectPath';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 
 type UncommittedAssetOptions = {|
   value: Asset,
@@ -153,7 +158,9 @@ export default class UncommittedAsset {
       size = content.length;
     }
 
+    // Maybe we should just store this in a file instead of LMDB
     await this.options.cache.setBlob(contentKey, content);
+
     return {size, hash};
   }
 
@@ -214,6 +221,9 @@ export default class UncommittedAsset {
     this.clearAST();
   }
 
+  /**
+   * @deprecated This has been broken on any cache other than FSCache for a long time.
+   */
   setStream(stream: Readable) {
     this.content = stream;
     this.clearAST();
@@ -296,6 +306,11 @@ export default class UncommittedAsset {
   }
 
   getCacheKey(key: string): string {
+    if (getFeatureFlag('cachePerformanceImprovements')) {
+      const filePath = fromProjectPathRelative(this.value.filePath);
+      return `Asset/${ATLASPACK_VERSION}/${filePath}/${this.value.id}/${key}`;
+    }
+
     return hashString(ATLASPACK_VERSION + key + this.value.id);
   }
 

package/src/atlaspack-v3/worker/compat/plugin-config.js

@@ -100,11 +100,15 @@ export class PluginConfig implements IPluginConfig {
     // eslint-disable-next-line no-unused-vars
     filePaths: Array<FilePath>,
     // eslint-disable-next-line no-unused-vars
-    options?: {|
-      packageKey?: string,
-      parse?: boolean,
-      exclude?: boolean,
-    |},
+    options?:
+      | {|
+          packageKey?: string,
+          parse?: boolean,
+          exclude?: boolean,
+        |}
+      | {|
+          configKey?: string,
+        |},
   ): Promise<?ConfigResultWithFilePath<T>> {
     return this.#inner.getConfigFrom(searchPath, filePaths, options);
   }