@atlaspack/core 2.12.1-dev.3401 → 2.12.1-dev.3443

package/src/PackagerRunner.js

@@ -85,7 +85,6 @@ export type BundleInfo = {|
   +hashReferences: Array<string>,
   +time?: number,
   +cacheKeys: CacheKeyMap,
-  +isLargeBlob: boolean,
 |};
 
 type CacheKeyMap = {|
@@ -340,7 +339,7 @@ export default class PackagerRunner {
     bundleConfigs: Map<string, Config>,
   ): Promise<{|
     type: string,
-    contents: Blob,
+    contents: Buffer | string,
     map: ?string,
   |}> {
     let packaged = await this.package(
@@ -467,7 +466,7 @@ export default class PackagerRunner {
     internalBundle: InternalBundle,
     internalBundleGraph: InternalBundleGraph,
     type: string,
-    contents: Blob,
+    contents: Buffer | string,
     map?: ?SourceMap,
     configs: Map<string, Config>,
     bundleConfigs: Map<string, Config>,
@@ -486,7 +485,7 @@ export default class PackagerRunner {
       bundle.name,
       internalBundle.pipeline,
     );
-    if (!optimizers.length) {
+    if (optimizers.length === 0) {
       return {type: bundle.type, contents, map};
     }
 
@@ -691,64 +690,18 @@ export default class PackagerRunner {
     return devDepHashes;
   }
 
-  async readFromCache(cacheKey: string): Promise<?{|
-    contents: Readable,
-    map: ?Readable,
-  |}> {
-    let contentKey = PackagerRunner.getContentKey(cacheKey);
-    let mapKey = PackagerRunner.getMapKey(cacheKey);
-
-    let isLargeBlob = await this.options.cache.hasLargeBlob(contentKey);
-    let contentExists =
-      isLargeBlob || (await this.options.cache.has(contentKey));
-    if (!contentExists) {
-      return null;
-    }
-
-    let mapExists = await this.options.cache.has(mapKey);
-
-    return {
-      contents: isLargeBlob
-        ? this.options.cache.getStream(contentKey)
-        : blobToStream(await this.options.cache.getBlob(contentKey)),
-      map: mapExists
-        ? blobToStream(await this.options.cache.getBlob(mapKey))
-        : null,
-    };
-  }
-
   async writeToCache(
     cacheKeys: CacheKeyMap,
     type: string,
-    contents: Blob,
+    contents: Buffer | string,
     map: ?string,
   ): Promise<BundleInfo> {
     let size = 0;
     let hash;
     let hashReferences = [];
-    let isLargeBlob = false;
 
     // TODO: don't replace hash references in binary files??
-    if (contents instanceof Readable) {
-      isLargeBlob = true;
-      let boundaryStr = '';
-      let h = new Hash();
-      await this.options.cache.setStream(
-        cacheKeys.content,
-        blobToStream(contents).pipe(
-          new TapStream(buf => {
-            let str = boundaryStr + buf.toString();
-            hashReferences = hashReferences.concat(
-              str.match(HASH_REF_REGEX) ?? [],
-            );
-            size += buf.length;
-            h.writeBuffer(buf);
-            boundaryStr = str.slice(str.length - BOUNDARY_LENGTH);
-          }),
-        ),
-      );
-      hash = h.finish();
-    } else if (typeof contents === 'string') {
+    if (typeof contents === 'string') {
       let buffer = Buffer.from(contents);
       size = buffer.byteLength;
       hash = hashBuffer(buffer);
@@ -770,7 +723,6 @@ export default class PackagerRunner {
       hash,
       hashReferences,
       cacheKeys,
-      isLargeBlob,
     };
     await this.options.cache.set(cacheKeys.info, info);
     return info;

package/src/RequestTracker.js

@@ -4,12 +4,14 @@ import invariant, {AssertionError} from 'assert';
 import path from 'path';
 
 import type {Cache} from '@atlaspack/cache';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {ContentGraph} from '@atlaspack/graph';
 import type {
   ContentGraphOpts,
   ContentKey,
   NodeId,
   SerializedContentGraph,
+  Graph,
 } from '@atlaspack/graph';
 import logger from '@atlaspack/logger';
 import {hashString} from '@atlaspack/rust';
@@ -370,7 +372,7 @@ export class RequestGraph extends ContentGraph<
     return nodeId;
   }
 
-  removeNode(nodeId: NodeId): void {
+  removeNode(nodeId: NodeId, removeOrphans: boolean = true): void {
     this.invalidNodeIds.delete(nodeId);
     this.incompleteNodeIds.delete(nodeId);
     this.incompleteNodePromises.delete(nodeId);
@@ -388,7 +390,7 @@ export class RequestGraph extends ContentGraph<
         configKeyNodes.delete(nodeId);
       }
     }
-    return super.removeNode(nodeId);
+    return super.removeNode(nodeId, removeOrphans);
   }
 
   getRequestNode(nodeId: NodeId): RequestNode {
@@ -816,32 +818,59 @@ export class RequestGraph extends ContentGraph<
     node: FileNameNode,
     filePath: ProjectPath,
     matchNodes: Array<FileNode>,
+    invalidateNode: (NodeId, InvalidateReason) => void,
   ) {
     // If there is an edge between this file_name node and one of the original file nodes pointed to
     // by the original file_name node, and the matched node is inside the current directory, invalidate
     // all connected requests pointed to by the file node.
-    let dirname = path.dirname(fromProjectPathRelative(filePath));
 
     let nodeId = this.getNodeIdByContentKey(node.id);
-    for (let matchNode of matchNodes) {
-      let matchNodeId = this.getNodeIdByContentKey(matchNode.id);
-      if (
-        this.hasEdge(
-          nodeId,
-          matchNodeId,
-          requestGraphEdgeTypes.invalidated_by_create_above,
-        ) &&
-        isDirectoryInside(
-          fromProjectPathRelative(toProjectPathUnsafe(matchNode.id)),
-          dirname,
+    let dirname = path.dirname(fromProjectPathRelative(filePath));
+
+    if (getFeatureFlag('fixQuadraticCacheInvalidation')) {
+      while (dirname !== '/') {
+        if (!this.hasContentKey(dirname)) break;
+        const matchNodeId = this.getNodeIdByContentKey(dirname);
+        if (
+          !this.hasEdge(
+            nodeId,
+            matchNodeId,
+            requestGraphEdgeTypes.invalidated_by_create_above,
+          )
         )
-      ) {
-        let connectedNodes = this.getNodeIdsConnectedTo(
+          break;
+
+        const connectedNodes = this.getNodeIdsConnectedTo(
           matchNodeId,
           requestGraphEdgeTypes.invalidated_by_create,
         );
         for (let connectedNode of connectedNodes) {
-          this.invalidateNode(connectedNode, FILE_CREATE);
+          invalidateNode(connectedNode, FILE_CREATE);
+        }
+
+        dirname = path.dirname(dirname);
+      }
+    } else {
+      for (let matchNode of matchNodes) {
+        let matchNodeId = this.getNodeIdByContentKey(matchNode.id);
+        if (
+          this.hasEdge(
+            nodeId,
+            matchNodeId,
+            requestGraphEdgeTypes.invalidated_by_create_above,
+          ) &&
+          isDirectoryInside(
+            fromProjectPathRelative(toProjectPathUnsafe(matchNode.id)),
+            dirname,
+          )
+        ) {
+          let connectedNodes = this.getNodeIdsConnectedTo(
+            matchNodeId,
+            requestGraphEdgeTypes.invalidated_by_create,
+          );
+          for (let connectedNode of connectedNodes) {
+            this.invalidateNode(connectedNode, FILE_CREATE);
+          }
         }
       }
     }
@@ -864,6 +893,7 @@ export class RequestGraph extends ContentGraph<
           parent,
           toProjectPathUnsafe(dirname),
           matchNodes,
+          invalidateNode,
         );
       }
     }
@@ -878,9 +908,45 @@ export class RequestGraph extends ContentGraph<
     let count = 0;
     let predictedTime = 0;
     let startTime = Date.now();
+    const enableOptimization = getFeatureFlag('fixQuadraticCacheInvalidation');
+    const removeOrphans = !enableOptimization;
+
+    const invalidatedNodes = new Set();
+    const invalidateNode = (nodeId, reason) => {
+      if (enableOptimization && invalidatedNodes.has(nodeId)) {
+        return;
+      }
+      invalidatedNodes.add(nodeId);
+      this.invalidateNode(nodeId, reason);
+    };
+    const aboveCache = new Map();
+    const getAbove = fileNameNodeId => {
+      const cachedResult = aboveCache.get(fileNameNodeId);
+      if (enableOptimization && cachedResult) {
+        return cachedResult;
+      }
+
+      let above = [];
+      const children = this.getNodeIdsConnectedTo(
+        fileNameNodeId,
+        requestGraphEdgeTypes.invalidated_by_create_above,
+      );
+      for (const nodeId of children) {
+        let node = nullthrows(this.getNode(nodeId));
+        if (node.type === FILE) {
+          above.push(node);
+        }
+      }
+      aboveCache.set(fileNameNodeId, above);
+      return above;
+    };
 
     for (let {path: _path, type} of events) {
-      if (++count === 256) {
+      if (
+        !enableOptimization &&
+        process.env.ATLASPACK_DISABLE_CACHE_TIMEOUT !== 'true' &&
+        ++count === 256
+      ) {
         let duration = Date.now() - startTime;
         predictedTime = duration * (events.length >> 8);
         if (predictedTime > threshold) {
@@ -936,7 +1002,7 @@ export class RequestGraph extends ContentGraph<
 
         for (let connectedNode of nodes) {
           didInvalidate = true;
-          this.invalidateNode(connectedNode, FILE_UPDATE);
+          invalidateNode(connectedNode, FILE_UPDATE);
         }
 
         if (type === 'create') {
@@ -946,7 +1012,7 @@ export class RequestGraph extends ContentGraph<
           );
           for (let connectedNode of nodes) {
             didInvalidate = true;
-            this.invalidateNode(connectedNode, FILE_CREATE);
+            invalidateNode(connectedNode, FILE_CREATE);
           }
         }
       } else if (type === 'create') {
@@ -958,21 +1024,15 @@ export class RequestGraph extends ContentGraph<
           );
 
           // Find potential file nodes to be invalidated if this file name pattern matches
-          let above: Array<FileNode> = [];
-          for (const nodeId of this.getNodeIdsConnectedTo(
-            fileNameNodeId,
-            requestGraphEdgeTypes.invalidated_by_create_above,
-          )) {
-            let node = nullthrows(this.getNode(nodeId));
-            // these might also be `glob` nodes which get handled below, we only care about files here.
-            if (node.type === FILE) {
-              above.push(node);
-            }
-          }
-
+          let above: Array<FileNode> = getAbove(fileNameNodeId);
           if (above.length > 0) {
             didInvalidate = true;
-            this.invalidateFileNameNode(fileNameNode, _filePath, above);
+            this.invalidateFileNameNode(
+              fileNameNode,
+              _filePath,
+              above,
+              invalidateNode,
+            );
           }
         }
 
@@ -987,7 +1047,7 @@ export class RequestGraph extends ContentGraph<
             );
             for (let connectedNode of connectedNodes) {
               didInvalidate = true;
-              this.invalidateNode(connectedNode, FILE_CREATE);
+              invalidateNode(connectedNode, FILE_CREATE);
             }
           }
         }
@@ -998,13 +1058,13 @@ export class RequestGraph extends ContentGraph<
           requestGraphEdgeTypes.invalidated_by_delete,
         )) {
           didInvalidate = true;
-          this.invalidateNode(connectedNode, FILE_DELETE);
+          invalidateNode(connectedNode, FILE_DELETE);
         }
 
         // Delete the file node since it doesn't exist anymore.
         // This ensures that files that don't exist aren't sent
         // to requests as invalidations for future requests.
-        this.removeNode(nodeId);
+        this.removeNode(nodeId, removeOrphans);
       }
 
       let configKeyNodes = this.configKeyNodes.get(_filePath);
@@ -1030,18 +1090,22 @@ export class RequestGraph extends ContentGraph<
               nodeId,
               requestGraphEdgeTypes.invalidated_by_update,
             )) {
-              this.invalidateNode(
+              invalidateNode(
                 connectedNode,
                 type === 'delete' ? FILE_DELETE : FILE_UPDATE,
               );
             }
             didInvalidate = true;
-            this.removeNode(nodeId);
+            this.removeNode(nodeId, removeOrphans);
           }
         }
       }
     }
 
+    if (getFeatureFlag('fixQuadraticCacheInvalidation')) {
+      cleanUpOrphans(this);
+    }
+
     let duration = Date.now() - startTime;
     logger.verbose({
       origin: '@atlaspack/core',
@@ -1050,6 +1114,8 @@ export class RequestGraph extends ContentGraph<
         trackableEvent: 'fsevent_response_time',
         duration,
         predictedTime,
+        numberOfEvents: events.length,
+        numberOfInvalidatedNodes: invalidatedNodes.size,
       },
     });
 
@@ -1608,6 +1674,15 @@ async function loadRequestGraph(options): Async<RequestGraph> {
   let timeout;
   const snapshotKey = `snapshot-${cacheKey}`;
   const snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt');
+
+  logger.verbose({
+    origin: '@atlaspack/core',
+    message: 'Loading request graph',
+    meta: {
+      cacheKey,
+      snapshotKey,
+    },
+  });
   if (await options.cache.hasLargeBlob(requestGraphKey)) {
     try {
       let {requestGraph} = await readAndDeserializeRequestGraph(
@@ -1663,8 +1738,18 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     }
   }
 
+  logger.verbose({
+    origin: '@atlaspack/core',
+    message:
+      'Cache entry for request tracker was not found, initializing a clean cache.',
+    meta: {
+      cacheKey,
+      snapshotKey,
+    },
+  });
   return new RequestGraph();
 }
+
 function logErrorOnBailout(
   options: AtlaspackOptions,
   snapshotPath: string,
@@ -1695,3 +1780,24 @@ function logErrorOnBailout(
     });
   }
 }
+
+export function cleanUpOrphans<N, E: number>(graph: Graph<N, E>): NodeId[] {
+  if (graph.rootNodeId == null) {
+    return [];
+  }
+
+  const reachableNodes = new Set();
+  graph.traverse(nodeId => {
+    reachableNodes.add(nodeId);
+  });
+
+  const removedNodeIds = [];
+  graph.nodes.forEach((_node, nodeId) => {
+    if (!reachableNodes.has(nodeId)) {
+      removedNodeIds.push(nodeId);
+      graph.removeNode(nodeId);
+    }
+  });
+
+  return removedNodeIds;
+}

package/src/SymbolPropagation.js

@@ -16,7 +16,7 @@ import nullthrows from 'nullthrows';
 import {setEqual} from '@atlaspack/utils';
 import logger from '@atlaspack/logger';
 import {md, convertSourceLocationToHighlight} from '@atlaspack/diagnostic';
-import {BundleBehavior} from './types';
+import {BundleBehavior, Priority} from './types';
 import {fromProjectPathRelative, fromProjectPath} from './projectPath';
 
 export function propagateSymbols({
@@ -101,7 +101,10 @@ export function propagateSymbols({
         namespaceReexportedSymbols.add('*');
       } else {
         for (let incomingDep of incomingDeps) {
-          if (incomingDep.value.symbols == null) {
+          if (
+            incomingDep.value.symbols == null ||
+            incomingDep.value.priority === Priority.conditional
+          ) {
             if (incomingDep.value.sourceAssetId == null) {
               // The root dependency on non-library builds
               isEntry = true;

package/src/Transformation.js

@@ -33,7 +33,6 @@ import ThrowableDiagnostic, {
   type Diagnostic,
 } from '@atlaspack/diagnostic';
 import {SOURCEMAP_EXTENSIONS} from '@atlaspack/utils';
-import {hashString} from '@atlaspack/rust';
 
 import {createDependency} from './Dependency';
 import AtlaspackConfig from './AtlaspackConfig';
@@ -231,16 +230,9 @@ export default class Transformation {
     // Prefer `isSource` originating from the AssetRequest.
     let isSource = isSourceOverride ?? summarizedIsSource;
 
-    // If the transformer request passed code, use a hash in addition
-    // to the filename as the base for the id to ensure it is unique.
-    let idBase = fromProjectPathRelative(filePath);
-    if (code != null) {
-      idBase += hashString(code);
-    }
     return new UncommittedAsset({
-      idBase,
       value: createAsset(this.options.projectRoot, {
-        idBase,
+        code,
         filePath,
         isSource,
         type: path.extname(fromProjectPathRelative(filePath)).slice(1),

package/src/UncommittedAsset.js

@@ -39,7 +39,7 @@ type UncommittedAssetOptions = {|
   mapBuffer?: ?Buffer,
   ast?: ?AST,
   isASTDirty?: ?boolean,
-  idBase?: ?string,
+  code?: ?string,
   invalidations?: Invalidations,
 |};
 
@@ -52,7 +52,7 @@ export default class UncommittedAsset {
   map: ?SourceMap;
   ast: ?AST;
   isASTDirty: boolean;
-  idBase: ?string;
+  code: ?string;
   invalidations: Invalidations;
   generate: ?() => Promise<GenerateOutput>;
 
@@ -63,7 +63,7 @@ export default class UncommittedAsset {
     mapBuffer,
     ast,
     isASTDirty,
-    idBase,
+    code,
     invalidations,
   }: UncommittedAssetOptions) {
     this.value = value;
@@ -72,7 +72,7 @@ export default class UncommittedAsset {
     this.mapBuffer = mapBuffer;
     this.ast = ast;
     this.isASTDirty = isASTDirty || false;
-    this.idBase = idBase;
+    this.code = code;
     this.invalidations = invalidations || createInvalidations();
   }
 
@@ -358,7 +358,7 @@ export default class UncommittedAsset {
 
     let asset = new UncommittedAsset({
       value: createAsset(this.options.projectRoot, {
-        idBase: this.idBase,
+        code: this.code,
         filePath: this.value.filePath,
         type: result.type,
         bundleBehavior:
@@ -405,7 +405,7 @@ export default class UncommittedAsset {
       ast: result.ast,
       isASTDirty: result.ast === this.ast ? this.isASTDirty : true,
       mapBuffer: result.map ? result.map.toBuffer() : null,
-      idBase: this.idBase,
+      code: this.code,
       invalidations: this.invalidations,
     });
 

package/src/Validation.js

@@ -17,7 +17,6 @@ import PluginOptions from './public/PluginOptions';
 import summarizeRequest from './summarizeRequest';
 import {fromProjectPath, fromProjectPathRelative} from './projectPath';
 import {PluginTracer} from '@atlaspack/profiler';
-import {hashString} from '@atlaspack/rust';
 
 export type ValidationOpts = {|
   config: AtlaspackConfig,
@@ -197,14 +196,9 @@ export default class Validation {
       },
     );
 
-    // If the transformer request passed code rather than a filename,
-    // use a hash as the base for the id to ensure it is unique.
-    let idBase =
-      code != null ? hashString(code) : fromProjectPathRelative(filePath);
     return new UncommittedAsset({
-      idBase,
       value: createAsset(this.options.projectRoot, {
-        idBase,
+        code,
         filePath: filePath,
         isSource,
         type: path.extname(fromProjectPathRelative(filePath)).slice(1),