metro-file-map 0.83.3 → 0.84.0

package/src/lib/FileProcessor.js

@@ -6,40 +6,50 @@ Object.defineProperty(exports, "__esModule", {
 exports.FileProcessor = void 0;
 var _constants = _interopRequireDefault(require("../constants"));
 var _worker = require("../worker");
+var _RootPathUtils = require("./RootPathUtils");
 var _jestWorker = require("jest-worker");
 var _path = require("path");
 function _interopRequireDefault(e) {
   return e && e.__esModule ? e : { default: e };
 }
 const debug = require("debug")("Metro:FileMap");
-const NODE_MODULES = _path.sep + "node_modules" + _path.sep;
+const NODE_MODULES_SEP = "node_modules" + _path.sep;
 const MAX_FILES_PER_WORKER = 100;
 class FileProcessor {
-  #dependencyExtractor;
-  #enableHastePackages;
-  #hasteImplModulePath;
-  #enableWorkerThreads;
   #maxFilesPerWorker;
   #maxWorkers;
   #perfLogger;
-  #workerArgs;
+  #pluginWorkers;
   #inBandWorker;
+  #rootPathUtils;
   constructor(opts) {
-    this.#dependencyExtractor = opts.dependencyExtractor;
-    this.#enableHastePackages = opts.enableHastePackages;
-    this.#enableWorkerThreads = opts.enableWorkerThreads;
-    this.#hasteImplModulePath = opts.hasteImplModulePath;
     this.#maxFilesPerWorker = opts.maxFilesPerWorker ?? MAX_FILES_PER_WORKER;
     this.#maxWorkers = opts.maxWorkers;
-    this.#workerArgs = {};
-    this.#inBandWorker = new _worker.Worker(this.#workerArgs);
+    this.#pluginWorkers = opts.pluginWorkers ?? [];
+    this.#inBandWorker = new _worker.Worker({
+      plugins: this.#pluginWorkers.map((plugin) => plugin.worker),
+    });
     this.#perfLogger = opts.perfLogger;
+    this.#rootPathUtils = new _RootPathUtils.RootPathUtils(opts.rootDir);
   }
   async processBatch(files, req) {
     const errors = [];
+    const workerJobs = files
+      .map(([normalFilePath, fileMetadata]) => {
+        const maybeWorkerInput = this.#getWorkerInput(
+          normalFilePath,
+          fileMetadata,
+          req,
+        );
+        if (!maybeWorkerInput) {
+          return null;
+        }
+        return [maybeWorkerInput, fileMetadata];
+      })
+      .filter(Boolean);
     const numWorkers = Math.min(
       this.#maxWorkers,
-      Math.ceil(files.length / this.#maxFilesPerWorker),
+      Math.ceil(workerJobs.length / this.#maxFilesPerWorker),
     );
     const batchWorker = this.#getBatchWorker(numWorkers);
     if (req.maybeReturnContent) {
@@ -48,21 +58,17 @@ class FileProcessor {
       );
     }
     await Promise.all(
-      files.map(([absolutePath, fileMetadata]) => {
-        const maybeWorkerInput = this.#getWorkerInput(
-          absolutePath,
-          fileMetadata,
-          req,
-        );
-        if (!maybeWorkerInput) {
-          return null;
-        }
+      workerJobs.map(([workerInput, fileMetadata]) => {
         return batchWorker
-          .processFile(maybeWorkerInput)
-          .then((reply) => processWorkerReply(reply, fileMetadata))
+          .processFile(workerInput)
+          .then((reply) =>
+            processWorkerReply(reply, workerInput.pluginsToRun, fileMetadata),
+          )
           .catch((error) =>
             errors.push({
-              absolutePath,
+              normalFilePath: this.#rootPathUtils.absoluteToNormal(
+                workerInput.filePath,
+              ),
               error: normalizeWorkerError(error),
             }),
           );
@@ -73,43 +79,60 @@ class FileProcessor {
       errors,
     };
   }
-  processRegularFile(absolutePath, fileMetadata, req) {
-    const workerInput = this.#getWorkerInput(absolutePath, fileMetadata, req);
+  processRegularFile(normalPath, fileMetadata, req) {
+    const workerInput = this.#getWorkerInput(normalPath, fileMetadata, req);
     return workerInput
       ? {
           content: processWorkerReply(
             this.#inBandWorker.processFile(workerInput),
+            workerInput.pluginsToRun,
             fileMetadata,
           ),
         }
       : null;
   }
-  #getWorkerInput(absolutePath, fileMetadata, req) {
+  #getWorkerInput(normalPath, fileMetadata, req) {
+    if (fileMetadata[_constants.default.SYMLINK] !== 0) {
+      return null;
+    }
     const computeSha1 =
       req.computeSha1 && fileMetadata[_constants.default.SHA1] == null;
-    const { computeDependencies, maybeReturnContent } = req;
-    if (absolutePath.includes(NODE_MODULES)) {
+    const { maybeReturnContent } = req;
+    const nodeModulesIdx = normalPath.indexOf(NODE_MODULES_SEP);
+    const isNodeModules =
+      nodeModulesIdx === 0 ||
+      (nodeModulesIdx > 0 && normalPath[nodeModulesIdx - 1] === _path.sep);
+    const pluginsToRun =
+      this.#pluginWorkers?.reduce((prev, plugin, idx) => {
+        if (
+          plugin.filter({
+            isNodeModules,
+            normalPath,
+          })
+        ) {
+          prev.push(idx);
+        }
+        return prev;
+      }, []) ?? [];
+    if (!computeSha1 && pluginsToRun.length === 0) {
+      return null;
+    }
+    if (isNodeModules) {
       if (computeSha1) {
         return {
-          computeDependencies: false,
           computeSha1: true,
-          dependencyExtractor: null,
-          enableHastePackages: false,
-          filePath: absolutePath,
-          hasteImplModulePath: null,
+          filePath: this.#rootPathUtils.normalToAbsolute(normalPath),
           maybeReturnContent,
+          pluginsToRun,
         };
       }
       return null;
     }
     return {
-      computeDependencies,
       computeSha1,
-      dependencyExtractor: this.#dependencyExtractor,
-      enableHastePackages: this.#enableHastePackages,
-      filePath: absolutePath,
-      hasteImplModulePath: this.#hasteImplModulePath,
+      filePath: this.#rootPathUtils.normalToAbsolute(normalPath),
       maybeReturnContent,
+      pluginsToRun,
     };
   }
   #getBatchWorker(numWorkers) {
@@ -120,21 +143,21 @@ class FileProcessor {
       };
     }
     const workerPath = require.resolve("../worker");
-    debug(
-      "Creating worker farm of %d worker %s",
-      numWorkers,
-      this.#enableWorkerThreads ? "threads" : "processes",
-    );
+    debug("Creating worker farm of %d worker threads", numWorkers);
     this.#perfLogger?.point("initWorkers_start");
     const jestWorker = new _jestWorker.Worker(workerPath, {
       exposedMethods: ["processFile"],
       maxRetries: 3,
       numWorkers,
-      enableWorkerThreads: this.#enableWorkerThreads,
+      enableWorkerThreads: true,
       forkOptions: {
         execArgv: [],
       },
-      setupArgs: [this.#workerArgs],
+      setupArgs: [
+        {
+          plugins: this.#pluginWorkers.map((plugin) => plugin.worker),
+        },
+      ],
     });
     this.#perfLogger?.point("initWorkers_end");
     this.#perfLogger = null;
@@ -143,15 +166,14 @@ class FileProcessor {
   async end() {}
 }
 exports.FileProcessor = FileProcessor;
-function processWorkerReply(metadata, fileMetadata) {
+function processWorkerReply(metadata, pluginsRun, fileMetadata) {
   fileMetadata[_constants.default.VISITED] = 1;
-  const metadataId = metadata.id;
-  if (metadataId != null) {
-    fileMetadata[_constants.default.ID] = metadataId;
+  const pluginData = metadata.pluginData;
+  if (pluginData) {
+    for (const [i, pluginIdx] of pluginsRun.entries()) {
+      fileMetadata[_constants.default.PLUGINDATA + pluginIdx] = pluginData[i];
+    }
   }
-  fileMetadata[_constants.default.DEPENDENCIES] = metadata.dependencies
-    ? metadata.dependencies.join(_constants.default.DEPENDENCY_DELIM)
-    : "";
   if (metadata.sha1 != null) {
     fileMetadata[_constants.default.SHA1] = metadata.sha1;
   }

package/src/lib/FileProcessor.js.flow

@@ -10,6 +10,7 @@
  */
 
 import type {
+  FileMapPluginWorker,
   FileMetadata,
   PerfLogger,
   WorkerMessage,
@@ -19,22 +20,18 @@ import type {
 
 import H from '../constants';
 import {Worker} from '../worker';
+import {RootPathUtils} from './RootPathUtils';
 import {Worker as JestWorker} from 'jest-worker';
 import {sep} from 'path';
 
 // eslint-disable-next-line import/no-commonjs
 const debug = require('debug')('Metro:FileMap');
 
-type ProcessFileRequest = $ReadOnly<{
+type ProcessFileRequest = Readonly<{
   /**
    * Populate metadata[H.SHA1] with the SHA1 of the file's contents.
    */
   computeSha1: boolean,
-  /**
-   * Populate metadata[H.DEPENDENCIES] with unresolved dependency specifiers
-   * using the dependencyExtractor provided to the constructor.
-   */
-  computeDependencies: boolean,
   /**
    * Only if processing has already required reading the file's contents, return
    * the contents as a Buffer - null otherwise. Not supported for batches.
@@ -51,55 +48,64 @@ interface MaybeCodedError extends Error {
   code?: string;
 }
 
-const NODE_MODULES = sep + 'node_modules' + sep;
+const NODE_MODULES_SEP = 'node_modules' + sep;
 const MAX_FILES_PER_WORKER = 100;
 
 export class FileProcessor {
-  #dependencyExtractor: ?string;
-  #enableHastePackages: boolean;
-  #hasteImplModulePath: ?string;
-  #enableWorkerThreads: boolean;
   #maxFilesPerWorker: number;
   #maxWorkers: number;
   #perfLogger: ?PerfLogger;
-  #workerArgs: WorkerSetupArgs;
+  #pluginWorkers: ReadonlyArray<FileMapPluginWorker>;
   #inBandWorker: Worker;
+  #rootPathUtils: RootPathUtils;
 
   constructor(
-    opts: $ReadOnly<{
-      dependencyExtractor: ?string,
-      enableHastePackages: boolean,
-      enableWorkerThreads: boolean,
-      hasteImplModulePath: ?string,
+    opts: Readonly<{
       maxFilesPerWorker?: ?number,
       maxWorkers: number,
+      pluginWorkers: ?ReadonlyArray<FileMapPluginWorker>,
       perfLogger: ?PerfLogger,
+      rootDir: string,
     }>,
   ) {
-    this.#dependencyExtractor = opts.dependencyExtractor;
-    this.#enableHastePackages = opts.enableHastePackages;
-    this.#enableWorkerThreads = opts.enableWorkerThreads;
-    this.#hasteImplModulePath = opts.hasteImplModulePath;
     this.#maxFilesPerWorker = opts.maxFilesPerWorker ?? MAX_FILES_PER_WORKER;
     this.#maxWorkers = opts.maxWorkers;
-    this.#workerArgs = {};
-    this.#inBandWorker = new Worker(this.#workerArgs);
+    this.#pluginWorkers = opts.pluginWorkers ?? [];
+    this.#inBandWorker = new Worker({
+      plugins: this.#pluginWorkers.map(plugin => plugin.worker),
+    });
     this.#perfLogger = opts.perfLogger;
+    this.#rootPathUtils = new RootPathUtils(opts.rootDir);
   }
 
   async processBatch(
-    files: $ReadOnlyArray<[string /*absolutePath*/, FileMetadata]>,
+    files: ReadonlyArray<[string /*relativePath*/, FileMetadata]>,
     req: ProcessFileRequest,
   ): Promise<{
     errors: Array<{
-      absolutePath: string,
+      normalFilePath: string,
       error: MaybeCodedError,
     }>,
   }> {
     const errors = [];
+
+    const workerJobs = files
+      .map(([normalFilePath, fileMetadata]) => {
+        const maybeWorkerInput = this.#getWorkerInput(
+          normalFilePath,
+          fileMetadata,
+          req,
+        );
+        if (!maybeWorkerInput) {
+          return null;
+        }
+        return [maybeWorkerInput, fileMetadata];
+      })
+      .filter(Boolean);
+
     const numWorkers = Math.min(
       this.#maxWorkers,
-      Math.ceil(files.length / this.#maxFilesPerWorker),
+      Math.ceil(workerJobs.length / this.#maxFilesPerWorker),
     );
     const batchWorker = this.#getBatchWorker(numWorkers);
 
@@ -110,20 +116,19 @@ export class FileProcessor {
     }
 
     await Promise.all(
-      files.map(([absolutePath, fileMetadata]) => {
-        const maybeWorkerInput = this.#getWorkerInput(
-          absolutePath,
-          fileMetadata,
-          req,
-        );
-        if (!maybeWorkerInput) {
-          return null;
-        }
+      workerJobs.map(([workerInput, fileMetadata]) => {
         return batchWorker
-          .processFile(maybeWorkerInput)
-          .then(reply => processWorkerReply(reply, fileMetadata))
+          .processFile(workerInput)
+          .then(reply =>
+            processWorkerReply(reply, workerInput.pluginsToRun, fileMetadata),
+          )
           .catch(error =>
-            errors.push({absolutePath, error: normalizeWorkerError(error)}),
+            errors.push({
+              normalFilePath: this.#rootPathUtils.absoluteToNormal(
+                workerInput.filePath,
+              ),
+              error: normalizeWorkerError(error),
+            }),
           );
       }),
     );
@@ -132,15 +137,16 @@ export class FileProcessor {
   }
 
   processRegularFile(
-    absolutePath: string,
+    normalPath: string,
     fileMetadata: FileMetadata,
     req: ProcessFileRequest,
   ): ?{content: ?Buffer} {
-    const workerInput = this.#getWorkerInput(absolutePath, fileMetadata, req);
+    const workerInput = this.#getWorkerInput(normalPath, fileMetadata, req);
     return workerInput
       ? {
           content: processWorkerReply(
             this.#inBandWorker.processFile(workerInput),
+            workerInput.pluginsToRun,
             fileMetadata,
           ),
         }
@@ -148,43 +154,60 @@ export class FileProcessor {
   }
 
   #getWorkerInput(
-    absolutePath: string,
+    normalPath: string,
     fileMetadata: FileMetadata,
     req: ProcessFileRequest,
   ): ?WorkerMessage {
+    if (fileMetadata[H.SYMLINK] !== 0) {
+      // Only process regular files
+      return null;
+    }
+
     const computeSha1 = req.computeSha1 && fileMetadata[H.SHA1] == null;
+    const {maybeReturnContent} = req;
+
+    const nodeModulesIdx = normalPath.indexOf(NODE_MODULES_SEP);
+    // Path may begin 'node_modules/' or contain '/node_modules/'.
+    const isNodeModules =
+      nodeModulesIdx === 0 ||
+      (nodeModulesIdx > 0 && normalPath[nodeModulesIdx - 1] === sep);
+
+    // Indices of plugins with a passing filter
+    const pluginsToRun =
+      this.#pluginWorkers?.reduce((prev, plugin, idx) => {
+        if (plugin.filter({isNodeModules, normalPath})) {
+          prev.push(idx);
+        }
+        return prev;
+      }, [] as Array<number>) ?? [];
 
-    const {computeDependencies, maybeReturnContent} = req;
+    if (!computeSha1 && pluginsToRun.length === 0) {
+      // Nothing to process
+      return null;
+    }
 
-    // Use a cheaper worker configuration for node_modules files, because we
-    // never care about extracting dependencies, and they may never be Haste
-    // modules or packages.
+    // Use a cheaper worker configuration for node_modules files, because
+    // they may never be Haste modules or packages.
     //
     // Note that we'd only expect node_modules files to reach this point if
     // retainAllFiles is true, or they're touched during watch mode.
-    if (absolutePath.includes(NODE_MODULES)) {
+    if (isNodeModules) {
       if (computeSha1) {
         return {
-          computeDependencies: false,
           computeSha1: true,
-          dependencyExtractor: null,
-          enableHastePackages: false,
-          filePath: absolutePath,
-          hasteImplModulePath: null,
+          filePath: this.#rootPathUtils.normalToAbsolute(normalPath),
           maybeReturnContent,
+          pluginsToRun,
         };
       }
       return null;
     }
 
     return {
-      computeDependencies,
       computeSha1,
-      dependencyExtractor: this.#dependencyExtractor,
-      enableHastePackages: this.#enableHastePackages,
-      filePath: absolutePath,
-      hasteImplModulePath: this.#hasteImplModulePath,
+      filePath: this.#rootPathUtils.normalToAbsolute(normalPath),
       maybeReturnContent,
+      pluginsToRun,
     };
   }
 
@@ -200,11 +223,7 @@ export class FileProcessor {
       };
     }
     const workerPath = require.resolve('../worker');
-    debug(
-      'Creating worker farm of %d worker %s',
-      numWorkers,
-      this.#enableWorkerThreads ? 'threads' : 'processes',
-    );
+    debug('Creating worker farm of %d worker threads', numWorkers);
     this.#perfLogger?.point('initWorkers_start');
     const jestWorker = new JestWorker<{
       processFile: WorkerMessage => Promise<WorkerMetadata>,
@@ -212,14 +231,18 @@ export class FileProcessor {
       exposedMethods: ['processFile'],
       maxRetries: 3,
       numWorkers,
-      enableWorkerThreads: this.#enableWorkerThreads,
+      enableWorkerThreads: true,
       forkOptions: {
         // Don't pass Node arguments down to workers. In particular, avoid
         // unnecessarily registering Babel when we're running Metro from
         // source (our worker is plain CommonJS).
         execArgv: [],
       },
-      setupArgs: [this.#workerArgs],
+      setupArgs: [
+        {
+          plugins: this.#pluginWorkers.map(plugin => plugin.worker),
+        } as WorkerSetupArgs,
+      ],
     });
     this.#perfLogger?.point('initWorkers_end');
     // Only log worker init once
@@ -232,20 +255,18 @@ export class FileProcessor {
 
 function processWorkerReply(
   metadata: WorkerMetadata,
+  pluginsRun: ReadonlyArray<number>,
   fileMetadata: FileMetadata,
 ) {
   fileMetadata[H.VISITED] = 1;
-
-  const metadataId = metadata.id;
-
-  if (metadataId != null) {
-    fileMetadata[H.ID] = metadataId;
+  const pluginData = metadata.pluginData;
+  if (pluginData) {
+    for (const [i, pluginIdx] of pluginsRun.entries()) {
+      // $FlowFixMe[invalid-tuple-index]
+      fileMetadata[H.PLUGINDATA + pluginIdx] = pluginData[i];
+    }
   }
 
-  fileMetadata[H.DEPENDENCIES] = metadata.dependencies
-    ? metadata.dependencies.join(H.DEPENDENCY_DELIM)
-    : '';
-
   if (metadata.sha1 != null) {
     fileMetadata[H.SHA1] = metadata.sha1;
   }

package/src/lib/RootPathUtils.js

@@ -6,28 +6,32 @@ Object.defineProperty(exports, "__esModule", {
 exports.RootPathUtils = void 0;
 var _invariant = _interopRequireDefault(require("invariant"));
 var path = _interopRequireWildcard(require("path"));
-function _getRequireWildcardCache(e) {
-  if ("function" != typeof WeakMap) return null;
-  var r = new WeakMap(),
-    t = new WeakMap();
-  return (_getRequireWildcardCache = function (e) {
-    return e ? t : r;
-  })(e);
-}
-function _interopRequireWildcard(e, r) {
-  if (!r && e && e.__esModule) return e;
-  if (null === e || ("object" != typeof e && "function" != typeof e))
-    return { default: e };
-  var t = _getRequireWildcardCache(r);
-  if (t && t.has(e)) return t.get(e);
-  var n = { __proto__: null },
-    a = Object.defineProperty && Object.getOwnPropertyDescriptor;
-  for (var u in e)
-    if ("default" !== u && {}.hasOwnProperty.call(e, u)) {
-      var i = a ? Object.getOwnPropertyDescriptor(e, u) : null;
-      i && (i.get || i.set) ? Object.defineProperty(n, u, i) : (n[u] = e[u]);
+function _interopRequireWildcard(e, t) {
+  if ("function" == typeof WeakMap)
+    var r = new WeakMap(),
+      n = new WeakMap();
+  return (_interopRequireWildcard = function (e, t) {
+    if (!t && e && e.__esModule) return e;
+    var o,
+      i,
+      f = { __proto__: null, default: e };
+    if (null === e || ("object" != typeof e && "function" != typeof e))
+      return f;
+    if ((o = t ? n : r)) {
+      if (o.has(e)) return o.get(e);
+      o.set(e, f);
     }
-  return ((n.default = e), t && t.set(e, n), n);
+    for (const t in e)
+      "default" !== t &&
+        {}.hasOwnProperty.call(e, t) &&
+        ((i =
+          (o = Object.defineProperty) &&
+          Object.getOwnPropertyDescriptor(e, t)) &&
+        (i.get || i.set)
+          ? o(f, t, i)
+          : (f[t] = e[t]));
+    return f;
+  })(e, t);
 }
 function _interopRequireDefault(e) {
   return e && e.__esModule ? e : { default: e };

package/src/lib/RootPathUtils.js.flow

@@ -4,8 +4,8 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  *
- * @format
  * @flow strict
+ * @format
  */
 
 import invariant from 'invariant';
@@ -48,8 +48,8 @@ const CURRENT_FRAGMENT = '.' + path.sep;
 
 export class RootPathUtils {
   #rootDir: string;
-  #rootDirnames: $ReadOnlyArray<string>;
-  #rootParts: $ReadOnlyArray<string>;
+  #rootDirnames: ReadonlyArray<string>;
+  #rootParts: ReadonlyArray<string>;
   #rootDepth: number;
 
   constructor(rootDir: string) {
@@ -80,7 +80,7 @@ export class RootPathUtils {
     return this.#rootParts[this.#rootParts.length - 1 - n];
   }
 
-  getParts(): $ReadOnlyArray<string> {
+  getParts(): ReadonlyArray<string> {
     return this.#rootParts;
   }