metro 0.71.0 → 0.71.1

package/src/DeltaBundler/graphOperations.js

@@ -7,16 +7,44 @@
  *
  * @format
  */
+
+/**
+ * Portions of this code are based on the Synchronous Cycle Collection
+ * algorithm described in:
+ *
+ * David F. Bacon and V. T. Rajan. 2001. Concurrent Cycle Collection in
+ * Reference Counted Systems. In Proceedings of the 15th European Conference on
+ * Object-Oriented Programming (ECOOP '01). Springer-Verlag, Berlin,
+ * Heidelberg, 207–235.
+ *
+ * Notable differences from the algorithm in the paper:
+ * 1. Our implementation uses the inverseDependencies set (which we already
+ *    have to maintain) instead of a separate refcount variable. A module's
+ *    reference count is equal to the size of its inverseDependencies set, plus
+ *    1 if it's an entry point of the graph.
+ * 2. We keep the "root buffer" (possibleCycleRoots) free of duplicates by
+ *    making it a Set, instead of storing a "buffered" flag on each node.
+ * 3. On top of tracking edges between nodes, we also count references between
+ *    nodes and entries in the importBundleNames set.
+ */
 "use strict";
 
-const nullthrows = require("nullthrows"); // Private state for the graph that persists between operations.
+const invariant = require("invariant");
+
+const nullthrows = require("nullthrows"); // TODO: Convert to a Flow enum
 
 function createGraph(options) {
   return {
     ...options,
     dependencies: new Map(),
     importBundleNames: new Set(),
-    privateState: {},
+    privateState: {
+      gc: {
+        color: new Map(),
+        possibleCycleRoots: new Set(),
+        importBundleRefs: new Map(),
+      },
+    },
   };
 }
 
@@ -55,7 +83,7 @@ async function traverseDependencies(paths, graph, options) {
     added: new Set(),
     modified: new Set(),
     deleted: new Set(),
-    inverseDependencies: new Map(),
+    earlyInverseDependencies: new Map(),
   };
   const internalOptions = getInternalOptions(options);
 
@@ -72,6 +100,7 @@ async function traverseDependencies(paths, graph, options) {
     }
   }
 
+  collectCycles(graph, delta);
   const added = new Map();
 
   for (const path of delta.added) {
@@ -99,9 +128,26 @@ async function initialTraverseDependencies(graph, options) {
     added: new Set(),
     modified: new Set(),
     deleted: new Set(),
-    inverseDependencies: new Map(),
+    earlyInverseDependencies: new Map(),
   };
   const internalOptions = getInternalOptions(options);
+  invariant(
+    graph.dependencies.size === 0,
+    "initialTraverseDependencies called on nonempty graph"
+  );
+  invariant(
+    graph.importBundleNames.size === 0,
+    "initialTraverseDependencies called on nonempty graph"
+  );
+  graph.privateState.gc.color.clear();
+  graph.privateState.gc.possibleCycleRoots.clear();
+  graph.privateState.gc.importBundleRefs.clear();
+
+  for (const path of graph.entryPoints) {
+    // Each entry point implicitly has a refcount of 1, so mark them all black.
+    graph.privateState.gc.color.set(path, "black");
+  }
+
   await Promise.all(
     [...graph.entryPoints].map((path) =>
       traverseDependenciesForSingleFile(path, graph, delta, internalOptions)
@@ -135,264 +181,181 @@ async function processModule(path, graph, delta, options) {
     options
   );
   const previousModule = graph.dependencies.get(path) || {
-    inverseDependencies: delta.inverseDependencies.get(path) || new Set(),
+    inverseDependencies: delta.earlyInverseDependencies.get(path) || new Set(),
     path,
   };
   const previousDependencies = previousModule.dependencies || new Map(); // Update the module information.
 
   const module = {
     ...previousModule,
-    dependencies: new Map(),
+    dependencies: new Map(previousDependencies),
     getSource: result.getSource,
     output: result.output,
   };
-  graph.dependencies.set(module.path, module);
-
-  for (const [relativePath, dependency] of currentDependencies) {
-    module.dependencies.set(relativePath, dependency);
-  }
+  graph.dependencies.set(module.path, module); // Diff dependencies (1/2): remove dependencies that have changed or been removed.
 
-  Array.from(previousDependencies.entries())
-    .filter(
-      ([relativePath, dependency]) =>
-        !currentDependencies.has(relativePath) ||
-        nullthrows(currentDependencies.get(relativePath)).absolutePath !==
-          dependency.absolutePath
-    )
-    .forEach(([relativePath, dependency]) =>
-      removeDependency(module, dependency.absolutePath, graph, delta, new Set())
-    ); // Check all the module dependencies and start traversing the tree from each
-  // added and removed dependency, to get all the modules that have to be added
-  // and removed from the dependency graph.
-
-  const promises = [];
+  for (const [relativePath, prevDependency] of previousDependencies) {
+    const curDependency = currentDependencies.get(relativePath);
 
-  for (const [relativePath, dependency] of currentDependencies) {
-    if (!options.shallow) {
-      if (
-        options.experimentalImportBundleSupport &&
-        dependency.data.data.asyncType != null
-      ) {
-        graph.importBundleNames.add(dependency.absolutePath);
-      } else if (
-        !previousDependencies.has(relativePath) ||
-        nullthrows(previousDependencies.get(relativePath)).absolutePath !==
-          dependency.absolutePath
-      ) {
-        promises.push(
-          addDependency(module, dependency.absolutePath, graph, delta, options)
-        );
-      }
+    if (
+      !curDependency ||
+      curDependency.absolutePath !== prevDependency.absolutePath ||
+      (options.experimentalImportBundleSupport &&
+        curDependency.data.data.asyncType !==
+          prevDependency.data.data.asyncType)
+    ) {
+      removeDependency(
+        module,
+        relativePath,
+        prevDependency,
+        graph,
+        delta,
+        options
+      );
     }
-  }
-
-  try {
-    await Promise.all(promises);
-  } catch (err) {
-    // If there is an error, restore the previous dependency list.
-    // This ensures we don't skip over them during the next traversal attempt.
-    // $FlowFixMe[cannot-write]
-    module.dependencies = previousDependencies;
-    throw err;
-  }
-
-  return module;
-}
-
-async function addDependency(parentModule, path, graph, delta, options) {
-  // The new dependency was already in the graph, we don't need to do anything.
-  const existingModule = graph.dependencies.get(path);
-
-  if (existingModule) {
-    existingModule.inverseDependencies.add(parentModule.path);
-    return;
-  } // This module is being transformed at the moment in parallel, so we should
-  // only mark its parent as an inverse dependency.
-
-  const inverse = delta.inverseDependencies.get(path);
-
-  if (inverse) {
-    inverse.add(parentModule.path);
-    return;
-  }
+  } // Diff dependencies (2/2): add dependencies that have changed or been added.
 
-  if (delta.deleted.has(path)) {
-    // Mark the addition by clearing a prior deletion.
-    delta.deleted.delete(path);
-  } else {
-    // Mark the addition in the added set.
-    delta.added.add(path);
-    delta.modified.delete(path);
-  }
-
-  delta.inverseDependencies.set(path, new Set([parentModule.path]));
-  options.onDependencyAdd();
-  const module = await processModule(path, graph, delta, options);
-  graph.dependencies.set(module.path, module);
-  module.inverseDependencies.add(parentModule.path);
-  options.onDependencyAdded();
-}
-/**
- * Recursively look up `inverseDependencies` until it is empty,
- * returning a set of paths for the last module that does not have
- * `inverseDependencies`.
- */
-
-function getAllTopLevelInverseDependencies(
-  inverseDependencies,
-  graph,
-  currModule,
-  visited
-) {
-  if (visited.has(currModule)) {
-    return new Set();
-  }
+  const promises = [];
 
-  visited.add(currModule);
+  for (const [relativePath, curDependency] of currentDependencies) {
+    const prevDependency = previousDependencies.get(relativePath);
 
-  if (!inverseDependencies.size) {
-    return new Set([currModule]);
+    if (
+      !prevDependency ||
+      prevDependency.absolutePath !== curDependency.absolutePath ||
+      (options.experimentalImportBundleSupport &&
+        prevDependency.data.data.asyncType !==
+          curDependency.data.data.asyncType)
+    ) {
+      promises.push(
+        addDependency(
+          module,
+          relativePath,
+          curDependency,
+          graph,
+          delta,
+          options
+        )
+      );
+    }
   }
 
-  return Array.from(inverseDependencies)
-    .filter((inverseDep) => graph.dependencies.has(inverseDep))
-    .reduce((acc, inverseDep) => {
-      const mod = graph.dependencies.get(inverseDep);
+  await Promise.all(promises); // Replace dependencies with the correctly-ordered version. As long as all
+  // the above promises have resolved, this will be the same map but without
+  // the added nondeterminism of promise resolution order. Because this
+  // assignment does not add or remove edges, it does NOT invalidate any of the
+  // garbage collection state.
+  // Catch obvious errors with a cheap assertion.
 
-      if (!mod) {
-        return acc;
-      }
+  invariant(
+    module.dependencies.size === currentDependencies.size,
+    "Failed to add the correct dependencies"
+  ); // $FlowFixMe[cannot-write]
 
-      getAllTopLevelInverseDependencies(
-        mod.inverseDependencies,
-        graph,
-        inverseDep,
-        visited
-      ).forEach((x) => {
-        acc.add(x);
-      });
-      return acc;
-    }, new Set());
+  module.dependencies = currentDependencies;
+  return module;
 }
-/**
- * Given `inverseDependencies`, tracing back inverse dependencies to
- * see if it only leads back to `parentModule`.
- */
 
-function canSafelyRemoveFromParentModule(
-  inverseDependencies,
+async function addDependency(
   parentModule,
+  relativePath,
+  dependency,
   graph,
-  canBeRemovedSafely,
-  delta
+  delta,
+  options
 ) {
-  const visited = new Set();
-  const topInverseDependencies = getAllTopLevelInverseDependencies(
-    inverseDependencies,
-    graph,
-    "", // current module name
-    visited
-  );
-
-  if (!topInverseDependencies.size) {
-    /**
-     * This happens when parentModule and inverseDependencies have a circular dependency.
-     * This will eventually become an empty set due to the `visited` Set being the
-     * base case for the recursive call.
-     */
-    return true;
-  }
+  const path = dependency.absolutePath; // The module may already exist, in which case we just need to update some
+  // bookkeeping instead of adding a new node to the graph.
+
+  let module = graph.dependencies.get(path);
+
+  if (options.shallow) {
+    // Don't add a node for the module if the graph is shallow (single-module).
+  } else if (
+    options.experimentalImportBundleSupport &&
+    dependency.data.data.asyncType != null
+  ) {
+    // Don't add a node for the module if we are traversing async dependencies
+    // lazily (and this is an async dependency). Instead, record it in
+    // importBundleNames.
+    incrementImportBundleReference(dependency, graph);
+  } else {
+    if (!module) {
+      // Add a new node to the graph.
+      const earlyInverseDependencies = delta.earlyInverseDependencies.get(path);
+
+      if (earlyInverseDependencies) {
+        // This module is being transformed at the moment in parallel, so we
+        // should only mark its parent as an inverse dependency.
+        earlyInverseDependencies.add(parentModule.path);
+      } else {
+        if (delta.deleted.has(path)) {
+          // Mark the addition by clearing a prior deletion.
+          delta.deleted.delete(path);
+        } else {
+          // Mark the addition in the added set.
+          delta.added.add(path);
+          delta.modified.delete(path);
+        }
+
+        delta.earlyInverseDependencies.set(path, new Set([parentModule.path]));
+        options.onDependencyAdd();
+        module = await processModule(path, graph, delta, options);
+        options.onDependencyAdded();
+        graph.dependencies.set(module.path, module);
+      }
+    }
 
-  const undeletedInverseDependencies = Array.from(
-    topInverseDependencies
-  ).filter((x) => graph.dependencies.has(x));
-  /**
-   * We can only mark the `visited` Set of modules to be safely removable if
-   * 1. We do not have top a level module to compare with parentModule.
-   *   This can happen when trying to see if we can safely remove from
-   *   a module that was deleted. This is why we filtered them out with `delta.deleted`
-   * 2. We have one top module and it is parentModule
-   */
-
-  const canSafelyRemove =
-    !undeletedInverseDependencies.length ||
-    (undeletedInverseDependencies.length === 1 &&
-      undeletedInverseDependencies[0] === parentModule);
-
-  if (canSafelyRemove) {
-    visited.forEach((mod) => {
-      canBeRemovedSafely.add(mod);
-    });
-  }
+    if (module) {
+      // We either added a new node to the graph, or we're updating an existing one.
+      module.inverseDependencies.add(parentModule.path);
+      markModuleInUse(module, graph);
+    }
+  } // Always update the parent's dependency map.
+  // This means the parent's dependencies can get desynced from
+  // inverseDependencies and the other fields in the case of lazy edges.
+  // Not an optimal representation :(
 
-  return canSafelyRemove;
+  parentModule.dependencies.set(relativePath, dependency);
 }
 
 function removeDependency(
   parentModule,
-  absolutePath,
+  relativePath,
+  dependency,
   graph,
-  delta, // We use `canBeRemovedSafely` set to keep track of visited
-  // module(s) that we're sure can be removed. This will skip expensive
-  // inverse dependency traversals.
-  canBeRemovedSafely = new Set()
+  delta,
+  options
 ) {
+  parentModule.dependencies.delete(relativePath);
+  const { absolutePath } = dependency;
+
+  if (
+    options.experimentalImportBundleSupport &&
+    dependency.data.data.asyncType != null
+  ) {
+    decrementImportBundleReference(dependency, graph);
+  }
+
   const module = graph.dependencies.get(absolutePath);
 
   if (!module) {
     return;
   }
 
-  module.inverseDependencies.delete(parentModule.path); // Even if there are modules still using parentModule, we want to ensure
-  // there is no circular dependency. Thus, we check if it can be safely removed
-  // by tracing back the inverseDependencies.
-
-  if (!canBeRemovedSafely.has(module.path)) {
-    if (
-      module.inverseDependencies.size &&
-      !canSafelyRemoveFromParentModule(
-        module.inverseDependencies,
-        module.path,
-        graph,
-        canBeRemovedSafely,
-        delta
-      )
-    ) {
-      return;
-    }
-  }
+  module.inverseDependencies.delete(parentModule.path);
 
-  if (delta.added.has(module.path)) {
-    // Mark the deletion by clearing a prior addition.
-    delta.added.delete(module.path);
+  if (
+    module.inverseDependencies.size > 0 ||
+    graph.entryPoints.has(absolutePath)
+  ) {
+    // The reference count has decreased, but not to zero.
+    // NOTE: Each entry point implicitly has a refcount of 1.
+    markAsPossibleCycleRoot(module, graph);
   } else {
-    // Mark the deletion in the deleted set.
-    delta.deleted.add(module.path);
-    delta.modified.delete(module.path);
-  } // This module is not used anywhere else! We can clear it from the bundle.
-  // Clean up all the state associated with this module in order to correctly
-  // re-add it if we encounter it again.
-
-  graph.dependencies.delete(module.path);
-  delta.inverseDependencies.delete(module.path); // Now we need to iterate through the module dependencies in order to
-  // clean up everything (we cannot read the module because it may have
-  // been deleted).
-
-  Array.from(module.dependencies.values())
-    .filter(
-      (dependency) =>
-        graph.dependencies.has(dependency.absolutePath) &&
-        dependency.absolutePath !== parentModule.path
-    )
-    .forEach((dependency) =>
-      removeDependency(
-        module,
-        dependency.absolutePath,
-        graph,
-        delta,
-        canBeRemovedSafely
-      )
-    );
+    // The reference count has decreased to zero.
+    releaseModule(module, graph, delta, options);
+  }
 }
 
 function resolveDependencies(parentPath, dependencies, options) {
@@ -472,6 +435,196 @@ function reorderDependencies(graph, module, orderedDependencies, options) {
     reorderDependencies(graph, childModule, orderedDependencies, options);
   });
 }
+/** Garbage collection functions */
+// Add an entry to importBundleNames (or increase the reference count of an existing one)
+
+function incrementImportBundleReference(dependency, graph) {
+  var _graph$privateState$g;
+
+  const { absolutePath } = dependency;
+  graph.privateState.gc.importBundleRefs.set(
+    absolutePath,
+    ((_graph$privateState$g =
+      graph.privateState.gc.importBundleRefs.get(absolutePath)) !== null &&
+    _graph$privateState$g !== void 0
+      ? _graph$privateState$g
+      : 0) + 1
+  );
+  graph.importBundleNames.add(absolutePath);
+} // Decrease the reference count of an entry in importBundleNames (and delete it if necessary)
+
+function decrementImportBundleReference(dependency, graph) {
+  const { absolutePath } = dependency;
+  const prevRefCount = nullthrows(
+    graph.privateState.gc.importBundleRefs.get(absolutePath)
+  );
+  invariant(
+    prevRefCount > 0,
+    "experimentalImportBundleSupport: import bundle refcount not valid"
+  );
+  graph.privateState.gc.importBundleRefs.set(absolutePath, prevRefCount - 1);
+
+  if (prevRefCount === 1) {
+    graph.privateState.gc.importBundleRefs.delete(absolutePath);
+    graph.importBundleNames.delete(absolutePath);
+  }
+} // Mark a module as in use (ref count >= 1)
+
+function markModuleInUse(module, graph) {
+  graph.privateState.gc.color.set(module.path, "black");
+} // Delete an unreachable module from the graph immediately, unless it's queued
+// for later deletion as a potential cycle root. Delete the module's outbound
+// edges.
+// Called when the reference count of a module has reached 0.
+
+function releaseModule(module, graph, delta, options) {
+  for (const [relativePath, dependency] of module.dependencies) {
+    removeDependency(module, relativePath, dependency, graph, delta, options);
+  }
+
+  graph.privateState.gc.color.set(module.path, "black");
+
+  if (!graph.privateState.gc.possibleCycleRoots.has(module.path)) {
+    freeModule(module, graph, delta);
+  }
+} // Delete an unreachable module from the graph.
+
+function freeModule(module, graph, delta) {
+  if (delta.added.has(module.path)) {
+    // Mark the deletion by clearing a prior addition.
+    delta.added.delete(module.path);
+  } else {
+    // Mark the deletion in the deleted set.
+    delta.deleted.add(module.path);
+    delta.modified.delete(module.path);
+  } // This module is not used anywhere else! We can clear it from the bundle.
+  // Clean up all the state associated with this module in order to correctly
+  // re-add it if we encounter it again.
+
+  graph.dependencies.delete(module.path);
+  delta.earlyInverseDependencies.delete(module.path);
+  graph.privateState.gc.possibleCycleRoots.delete(module.path);
+  graph.privateState.gc.color.delete(module.path);
+} // Mark a module as a possible cycle root
+
+function markAsPossibleCycleRoot(module, graph) {
+  if (nullthrows(graph.privateState.gc.color.get(module.path)) !== "purple") {
+    graph.privateState.gc.color.set(module.path, "purple");
+    graph.privateState.gc.possibleCycleRoots.add(module.path);
+  }
+} // Collect any unreachable cycles in the graph.
+
+function collectCycles(graph, delta) {
+  // Mark recursively from roots (trial deletion)
+  for (const path of graph.privateState.gc.possibleCycleRoots) {
+    const module = nullthrows(graph.dependencies.get(path));
+    const color = nullthrows(graph.privateState.gc.color.get(path));
+
+    if (color === "purple") {
+      markGray(module, graph);
+    } else {
+      graph.privateState.gc.possibleCycleRoots.delete(path);
+
+      if (
+        color === "black" &&
+        module.inverseDependencies.size === 0 &&
+        !graph.entryPoints.has(path)
+      ) {
+        freeModule(module, graph, delta);
+      }
+    }
+  } // Scan recursively from roots (undo unsuccessful trial deletions)
+
+  for (const path of graph.privateState.gc.possibleCycleRoots) {
+    const module = nullthrows(graph.dependencies.get(path));
+    scan(module, graph);
+  } // Collect recursively from roots (free unreachable cycles)
+
+  for (const path of graph.privateState.gc.possibleCycleRoots) {
+    graph.privateState.gc.possibleCycleRoots.delete(path);
+    const module = nullthrows(graph.dependencies.get(path));
+    collectWhite(module, graph, delta);
+  }
+}
+
+function markGray(module, graph) {
+  const color = nullthrows(graph.privateState.gc.color.get(module.path));
+
+  if (color !== "gray") {
+    graph.privateState.gc.color.set(module.path, "gray");
+
+    for (const dependency of module.dependencies.values()) {
+      const childModule = nullthrows(
+        graph.dependencies.get(dependency.absolutePath)
+      ); // The inverse dependency will be restored during the scan phase if this module remains live.
+
+      childModule.inverseDependencies.delete(module.path);
+      markGray(childModule, graph);
+    }
+  }
+}
+
+function scan(module, graph) {
+  const color = nullthrows(graph.privateState.gc.color.get(module.path));
+
+  if (color === "gray") {
+    if (
+      module.inverseDependencies.size > 0 ||
+      graph.entryPoints.has(module.path)
+    ) {
+      scanBlack(module, graph);
+    } else {
+      graph.privateState.gc.color.set(module.path, "white");
+
+      for (const dependency of module.dependencies.values()) {
+        const childModule = nullthrows(
+          graph.dependencies.get(dependency.absolutePath)
+        );
+        scan(childModule, graph);
+      }
+    }
+  }
+}
+
+function scanBlack(module, graph) {
+  graph.privateState.gc.color.set(module.path, "black");
+
+  for (const dependency of module.dependencies.values()) {
+    const childModule = nullthrows(
+      graph.dependencies.get(dependency.absolutePath)
+    ); // The inverse dependency must have been deleted during the mark phase.
+
+    childModule.inverseDependencies.add(module.path);
+    const childColor = nullthrows(
+      graph.privateState.gc.color.get(childModule.path)
+    );
+
+    if (childColor !== "black") {
+      scanBlack(childModule, graph);
+    }
+  }
+}
+
+function collectWhite(module, graph, delta) {
+  const color = nullthrows(graph.privateState.gc.color.get(module.path));
+
+  if (
+    color === "white" &&
+    !graph.privateState.gc.possibleCycleRoots.has(module.path)
+  ) {
+    graph.privateState.gc.color.set(module.path, "black");
+
+    for (const dependency of module.dependencies.values()) {
+      const childModule = nullthrows(
+        graph.dependencies.get(dependency.absolutePath)
+      );
+      collectWhite(childModule, graph, delta);
+    }
+
+    freeModule(module, graph, delta);
+  }
+}
+/** End of garbage collection functions */
 
 module.exports = {
   createGraph,