metro 0.70.2 → 0.71.1

package/src/DeltaBundler/graphOperations.js

@@ -0,0 +1,634 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ *
+ * @format
+ */
+
+/**
+ * Portions of this code are based on the Synchronous Cycle Collection
+ * algorithm described in:
+ *
+ * David F. Bacon and V. T. Rajan. 2001. Concurrent Cycle Collection in
+ * Reference Counted Systems. In Proceedings of the 15th European Conference on
+ * Object-Oriented Programming (ECOOP '01). Springer-Verlag, Berlin,
+ * Heidelberg, 207–235.
+ *
+ * Notable differences from the algorithm in the paper:
+ * 1. Our implementation uses the inverseDependencies set (which we already
+ *    have to maintain) instead of a separate refcount variable. A module's
+ *    reference count is equal to the size of its inverseDependencies set, plus
+ *    1 if it's an entry point of the graph.
+ * 2. We keep the "root buffer" (possibleCycleRoots) free of duplicates by
+ *    making it a Set, instead of storing a "buffered" flag on each node.
+ * 3. On top of tracking edges between nodes, we also count references between
+ *    nodes and entries in the importBundleNames set.
+ */
+"use strict";
+
+const invariant = require("invariant");
+
+const nullthrows = require("nullthrows"); // TODO: Convert to a Flow enum
+
+function createGraph(options) {
+  return {
+    ...options,
+    dependencies: new Map(),
+    importBundleNames: new Set(),
+    privateState: {
+      gc: {
+        color: new Map(),
+        possibleCycleRoots: new Set(),
+        importBundleRefs: new Map(),
+      },
+    },
+  };
+}
+
+function getInternalOptions({
+  transform,
+  resolve,
+  onProgress,
+  experimentalImportBundleSupport,
+  shallow,
+}) {
+  let numProcessed = 0;
+  let total = 0;
+  return {
+    experimentalImportBundleSupport,
+    transform,
+    resolve,
+    onDependencyAdd: () => onProgress && onProgress(numProcessed, ++total),
+    onDependencyAdded: () => onProgress && onProgress(++numProcessed, total),
+    shallow,
+  };
+}
+/**
+ * Dependency Traversal logic for the Delta Bundler. This method calculates
+ * the modules that should be included in the bundle by traversing the
+ * dependency graph.
+ * Instead of traversing the whole graph each time, it just calculates the
+ * difference between runs by only traversing the added/removed dependencies.
+ * To do so, it uses the passed passed graph dependencies and it mutates it.
+ * The paths parameter contains the absolute paths of the root files that the
+ * method should traverse. Normally, these paths should be the modified files
+ * since the last traversal.
+ */
+
+async function traverseDependencies(paths, graph, options) {
+  const delta = {
+    added: new Set(),
+    modified: new Set(),
+    deleted: new Set(),
+    earlyInverseDependencies: new Map(),
+  };
+  const internalOptions = getInternalOptions(options);
+
+  for (const path of paths) {
+    // Start traversing from modules that are already part of the dependency graph.
+    if (graph.dependencies.get(path)) {
+      delta.modified.add(path);
+      await traverseDependenciesForSingleFile(
+        path,
+        graph,
+        delta,
+        internalOptions
+      );
+    }
+  }
+
+  collectCycles(graph, delta);
+  const added = new Map();
+
+  for (const path of delta.added) {
+    added.set(path, nullthrows(graph.dependencies.get(path)));
+  }
+
+  const modified = new Map();
+
+  for (const path of delta.modified) {
+    // Only report a module as modified if we're not already reporting it as added.
+    if (!delta.added.has(path)) {
+      modified.set(path, nullthrows(graph.dependencies.get(path)));
+    }
+  }
+
+  return {
+    added,
+    modified,
+    deleted: delta.deleted,
+  };
+}
+
+async function initialTraverseDependencies(graph, options) {
+  const delta = {
+    added: new Set(),
+    modified: new Set(),
+    deleted: new Set(),
+    earlyInverseDependencies: new Map(),
+  };
+  const internalOptions = getInternalOptions(options);
+  invariant(
+    graph.dependencies.size === 0,
+    "initialTraverseDependencies called on nonempty graph"
+  );
+  invariant(
+    graph.importBundleNames.size === 0,
+    "initialTraverseDependencies called on nonempty graph"
+  );
+  graph.privateState.gc.color.clear();
+  graph.privateState.gc.possibleCycleRoots.clear();
+  graph.privateState.gc.importBundleRefs.clear();
+
+  for (const path of graph.entryPoints) {
+    // Each entry point implicitly has a refcount of 1, so mark them all black.
+    graph.privateState.gc.color.set(path, "black");
+  }
+
+  await Promise.all(
+    [...graph.entryPoints].map((path) =>
+      traverseDependenciesForSingleFile(path, graph, delta, internalOptions)
+    )
+  );
+  reorderGraph(graph, {
+    shallow: options.shallow,
+  });
+  return {
+    added: graph.dependencies,
+    modified: new Map(),
+    deleted: new Set(),
+  };
+}
+
+async function traverseDependenciesForSingleFile(path, graph, delta, options) {
+  options.onDependencyAdd();
+  await processModule(path, graph, delta, options);
+  options.onDependencyAdded();
+}
+
+async function processModule(path, graph, delta, options) {
+  // Transform the file via the given option.
+  // TODO: Unbind the transform method from options
+  const result = await options.transform(path); // Get the absolute path of all sub-dependencies (some of them could have been
+  // moved but maintain the same relative path).
+
+  const currentDependencies = resolveDependencies(
+    path,
+    result.dependencies,
+    options
+  );
+  const previousModule = graph.dependencies.get(path) || {
+    inverseDependencies: delta.earlyInverseDependencies.get(path) || new Set(),
+    path,
+  };
+  const previousDependencies = previousModule.dependencies || new Map(); // Update the module information.
+
+  const module = {
+    ...previousModule,
+    dependencies: new Map(previousDependencies),
+    getSource: result.getSource,
+    output: result.output,
+  };
+  graph.dependencies.set(module.path, module); // Diff dependencies (1/2): remove dependencies that have changed or been removed.
+
+  for (const [relativePath, prevDependency] of previousDependencies) {
+    const curDependency = currentDependencies.get(relativePath);
+
+    if (
+      !curDependency ||
+      curDependency.absolutePath !== prevDependency.absolutePath ||
+      (options.experimentalImportBundleSupport &&
+        curDependency.data.data.asyncType !==
+          prevDependency.data.data.asyncType)
+    ) {
+      removeDependency(
+        module,
+        relativePath,
+        prevDependency,
+        graph,
+        delta,
+        options
+      );
+    }
+  } // Diff dependencies (2/2): add dependencies that have changed or been added.
+
+  const promises = [];
+
+  for (const [relativePath, curDependency] of currentDependencies) {
+    const prevDependency = previousDependencies.get(relativePath);
+
+    if (
+      !prevDependency ||
+      prevDependency.absolutePath !== curDependency.absolutePath ||
+      (options.experimentalImportBundleSupport &&
+        prevDependency.data.data.asyncType !==
+          curDependency.data.data.asyncType)
+    ) {
+      promises.push(
+        addDependency(
+          module,
+          relativePath,
+          curDependency,
+          graph,
+          delta,
+          options
+        )
+      );
+    }
+  }
+
+  await Promise.all(promises); // Replace dependencies with the correctly-ordered version. As long as all
+  // the above promises have resolved, this will be the same map but without
+  // the added nondeterminism of promise resolution order. Because this
+  // assignment does not add or remove edges, it does NOT invalidate any of the
+  // garbage collection state.
+  // Catch obvious errors with a cheap assertion.
+
+  invariant(
+    module.dependencies.size === currentDependencies.size,
+    "Failed to add the correct dependencies"
+  ); // $FlowFixMe[cannot-write]
+
+  module.dependencies = currentDependencies;
+  return module;
+}
+
+async function addDependency(
+  parentModule,
+  relativePath,
+  dependency,
+  graph,
+  delta,
+  options
+) {
+  const path = dependency.absolutePath; // The module may already exist, in which case we just need to update some
+  // bookkeeping instead of adding a new node to the graph.
+
+  let module = graph.dependencies.get(path);
+
+  if (options.shallow) {
+    // Don't add a node for the module if the graph is shallow (single-module).
+  } else if (
+    options.experimentalImportBundleSupport &&
+    dependency.data.data.asyncType != null
+  ) {
+    // Don't add a node for the module if we are traversing async dependencies
+    // lazily (and this is an async dependency). Instead, record it in
+    // importBundleNames.
+    incrementImportBundleReference(dependency, graph);
+  } else {
+    if (!module) {
+      // Add a new node to the graph.
+      const earlyInverseDependencies = delta.earlyInverseDependencies.get(path);
+
+      if (earlyInverseDependencies) {
+        // This module is being transformed at the moment in parallel, so we
+        // should only mark its parent as an inverse dependency.
+        earlyInverseDependencies.add(parentModule.path);
+      } else {
+        if (delta.deleted.has(path)) {
+          // Mark the addition by clearing a prior deletion.
+          delta.deleted.delete(path);
+        } else {
+          // Mark the addition in the added set.
+          delta.added.add(path);
+          delta.modified.delete(path);
+        }
+
+        delta.earlyInverseDependencies.set(path, new Set([parentModule.path]));
+        options.onDependencyAdd();
+        module = await processModule(path, graph, delta, options);
+        options.onDependencyAdded();
+        graph.dependencies.set(module.path, module);
+      }
+    }
+
+    if (module) {
+      // We either added a new node to the graph, or we're updating an existing one.
+      module.inverseDependencies.add(parentModule.path);
+      markModuleInUse(module, graph);
+    }
+  } // Always update the parent's dependency map.
+  // This means the parent's dependencies can get desynced from
+  // inverseDependencies and the other fields in the case of lazy edges.
+  // Not an optimal representation :(
+
+  parentModule.dependencies.set(relativePath, dependency);
+}
+
+function removeDependency(
+  parentModule,
+  relativePath,
+  dependency,
+  graph,
+  delta,
+  options
+) {
+  parentModule.dependencies.delete(relativePath);
+  const { absolutePath } = dependency;
+
+  if (
+    options.experimentalImportBundleSupport &&
+    dependency.data.data.asyncType != null
+  ) {
+    decrementImportBundleReference(dependency, graph);
+  }
+
+  const module = graph.dependencies.get(absolutePath);
+
+  if (!module) {
+    return;
+  }
+
+  module.inverseDependencies.delete(parentModule.path);
+
+  if (
+    module.inverseDependencies.size > 0 ||
+    graph.entryPoints.has(absolutePath)
+  ) {
+    // The reference count has decreased, but not to zero.
+    // NOTE: Each entry point implicitly has a refcount of 1.
+    markAsPossibleCycleRoot(module, graph);
+  } else {
+    // The reference count has decreased to zero.
+    releaseModule(module, graph, delta, options);
+  }
+}
+
+function resolveDependencies(parentPath, dependencies, options) {
+  const resolve = (parentPath, result) => {
+    const relativePath = result.name;
+
+    try {
+      return [
+        relativePath,
+        {
+          absolutePath: options.resolve(parentPath, relativePath),
+          data: result,
+        },
+      ];
+    } catch (error) {
+      // Ignore unavailable optional dependencies. They are guarded
+      // with a try-catch block and will be handled during runtime.
+      if (result.data.isOptional !== true) {
+        throw error;
+      }
+    }
+
+    return undefined;
+  };
+
+  const resolved = dependencies.reduce((list, result) => {
+    const resolvedPath = resolve(parentPath, result);
+
+    if (resolvedPath) {
+      list.push(resolvedPath);
+    }
+
+    return list;
+  }, []);
+  return new Map(resolved);
+}
+/**
+ * Re-traverse the dependency graph in DFS order to reorder the modules and
+ * guarantee the same order between runs. This method mutates the passed graph.
+ */
+
+function reorderGraph(graph, options) {
+  const orderedDependencies = new Map();
+  graph.entryPoints.forEach((entryPoint) => {
+    const mainModule = graph.dependencies.get(entryPoint);
+
+    if (!mainModule) {
+      throw new ReferenceError("Module not registered in graph: " + entryPoint);
+    }
+
+    reorderDependencies(graph, mainModule, orderedDependencies, options);
+  });
+  graph.dependencies = orderedDependencies;
+}
+
+function reorderDependencies(graph, module, orderedDependencies, options) {
+  if (module.path) {
+    if (orderedDependencies.has(module.path)) {
+      return;
+    }
+
+    orderedDependencies.set(module.path, module);
+  }
+
+  module.dependencies.forEach((dependency) => {
+    const path = dependency.absolutePath;
+    const childModule = graph.dependencies.get(path);
+
+    if (!childModule) {
+      if (dependency.data.data.asyncType != null || options.shallow) {
+        return;
+      } else {
+        throw new ReferenceError("Module not registered in graph: " + path);
+      }
+    }
+
+    reorderDependencies(graph, childModule, orderedDependencies, options);
+  });
+}
+/** Garbage collection functions */
+// Add an entry to importBundleNames (or increase the reference count of an existing one)
+
+function incrementImportBundleReference(dependency, graph) {
+  var _graph$privateState$g;
+
+  const { absolutePath } = dependency;
+  graph.privateState.gc.importBundleRefs.set(
+    absolutePath,
+    ((_graph$privateState$g =
+      graph.privateState.gc.importBundleRefs.get(absolutePath)) !== null &&
+    _graph$privateState$g !== void 0
+      ? _graph$privateState$g
+      : 0) + 1
+  );
+  graph.importBundleNames.add(absolutePath);
+} // Decrease the reference count of an entry in importBundleNames (and delete it if necessary)
+
+function decrementImportBundleReference(dependency, graph) {
+  const { absolutePath } = dependency;
+  const prevRefCount = nullthrows(
+    graph.privateState.gc.importBundleRefs.get(absolutePath)
+  );
+  invariant(
+    prevRefCount > 0,
+    "experimentalImportBundleSupport: import bundle refcount not valid"
+  );
+  graph.privateState.gc.importBundleRefs.set(absolutePath, prevRefCount - 1);
+
+  if (prevRefCount === 1) {
+    graph.privateState.gc.importBundleRefs.delete(absolutePath);
+    graph.importBundleNames.delete(absolutePath);
+  }
+} // Mark a module as in use (ref count >= 1)
+
+function markModuleInUse(module, graph) {
+  graph.privateState.gc.color.set(module.path, "black");
+} // Delete an unreachable module from the graph immediately, unless it's queued
+// for later deletion as a potential cycle root. Delete the module's outbound
+// edges.
+// Called when the reference count of a module has reached 0.
+
+function releaseModule(module, graph, delta, options) {
+  for (const [relativePath, dependency] of module.dependencies) {
+    removeDependency(module, relativePath, dependency, graph, delta, options);
+  }
+
+  graph.privateState.gc.color.set(module.path, "black");
+
+  if (!graph.privateState.gc.possibleCycleRoots.has(module.path)) {
+    freeModule(module, graph, delta);
+  }
+} // Delete an unreachable module from the graph.
+
+function freeModule(module, graph, delta) {
+  if (delta.added.has(module.path)) {
+    // Mark the deletion by clearing a prior addition.
+    delta.added.delete(module.path);
+  } else {
+    // Mark the deletion in the deleted set.
+    delta.deleted.add(module.path);
+    delta.modified.delete(module.path);
+  } // This module is not used anywhere else! We can clear it from the bundle.
+  // Clean up all the state associated with this module in order to correctly
+  // re-add it if we encounter it again.
+
+  graph.dependencies.delete(module.path);
+  delta.earlyInverseDependencies.delete(module.path);
+  graph.privateState.gc.possibleCycleRoots.delete(module.path);
+  graph.privateState.gc.color.delete(module.path);
+} // Mark a module as a possible cycle root
+
+function markAsPossibleCycleRoot(module, graph) {
+  if (nullthrows(graph.privateState.gc.color.get(module.path)) !== "purple") {
+    graph.privateState.gc.color.set(module.path, "purple");
+    graph.privateState.gc.possibleCycleRoots.add(module.path);
+  }
+} // Collect any unreachable cycles in the graph.
+
+function collectCycles(graph, delta) {
+  // Mark recursively from roots (trial deletion)
+  for (const path of graph.privateState.gc.possibleCycleRoots) {
+    const module = nullthrows(graph.dependencies.get(path));
+    const color = nullthrows(graph.privateState.gc.color.get(path));
+
+    if (color === "purple") {
+      markGray(module, graph);
+    } else {
+      graph.privateState.gc.possibleCycleRoots.delete(path);
+
+      if (
+        color === "black" &&
+        module.inverseDependencies.size === 0 &&
+        !graph.entryPoints.has(path)
+      ) {
+        freeModule(module, graph, delta);
+      }
+    }
+  } // Scan recursively from roots (undo unsuccessful trial deletions)
+
+  for (const path of graph.privateState.gc.possibleCycleRoots) {
+    const module = nullthrows(graph.dependencies.get(path));
+    scan(module, graph);
+  } // Collect recursively from roots (free unreachable cycles)
+
+  for (const path of graph.privateState.gc.possibleCycleRoots) {
+    graph.privateState.gc.possibleCycleRoots.delete(path);
+    const module = nullthrows(graph.dependencies.get(path));
+    collectWhite(module, graph, delta);
+  }
+}
+
+function markGray(module, graph) {
+  const color = nullthrows(graph.privateState.gc.color.get(module.path));
+
+  if (color !== "gray") {
+    graph.privateState.gc.color.set(module.path, "gray");
+
+    for (const dependency of module.dependencies.values()) {
+      const childModule = nullthrows(
+        graph.dependencies.get(dependency.absolutePath)
+      ); // The inverse dependency will be restored during the scan phase if this module remains live.
+
+      childModule.inverseDependencies.delete(module.path);
+      markGray(childModule, graph);
+    }
+  }
+}
+
+function scan(module, graph) {
+  const color = nullthrows(graph.privateState.gc.color.get(module.path));
+
+  if (color === "gray") {
+    if (
+      module.inverseDependencies.size > 0 ||
+      graph.entryPoints.has(module.path)
+    ) {
+      scanBlack(module, graph);
+    } else {
+      graph.privateState.gc.color.set(module.path, "white");
+
+      for (const dependency of module.dependencies.values()) {
+        const childModule = nullthrows(
+          graph.dependencies.get(dependency.absolutePath)
+        );
+        scan(childModule, graph);
+      }
+    }
+  }
+}
+
+function scanBlack(module, graph) {
+  graph.privateState.gc.color.set(module.path, "black");
+
+  for (const dependency of module.dependencies.values()) {
+    const childModule = nullthrows(
+      graph.dependencies.get(dependency.absolutePath)
+    ); // The inverse dependency must have been deleted during the mark phase.
+
+    childModule.inverseDependencies.add(module.path);
+    const childColor = nullthrows(
+      graph.privateState.gc.color.get(childModule.path)
+    );
+
+    if (childColor !== "black") {
+      scanBlack(childModule, graph);
+    }
+  }
+}
+
+function collectWhite(module, graph, delta) {
+  const color = nullthrows(graph.privateState.gc.color.get(module.path));
+
+  if (
+    color === "white" &&
+    !graph.privateState.gc.possibleCycleRoots.has(module.path)
+  ) {
+    graph.privateState.gc.color.set(module.path, "black");
+
+    for (const dependency of module.dependencies.values()) {
+      const childModule = nullthrows(
+        graph.dependencies.get(dependency.absolutePath)
+      );
+      collectWhite(childModule, graph, delta);
+    }
+
+    freeModule(module, graph, delta);
+  }
+}
+/** End of garbage collection functions */
+
+module.exports = {
+  createGraph,
+  initialTraverseDependencies,
+  traverseDependencies,
+  reorderGraph,
+};