metro-file-map 0.83.4 → 0.83.6

package/src/index.d.ts

@@ -0,0 +1,182 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @noformat
+ * @oncall react_native
+ * @generated SignedSource<<806d228988308075b7a911c3dfb513d3>>
+ *
+ * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js
+ * Original file: packages/metro-file-map/src/index.js
+ * To regenerate, run:
+ *   js1 build metro-ts-defs (internal) OR
+ *   yarn run build-ts-defs (OSS) 
+ */
+
+import type {
+  BuildParameters,
+  BuildResult,
+  CacheData,
+  CacheManagerFactory,
+  ChangeEventMetadata,
+  Console,
+  FileData,
+  FileSystem,
+  HasteMapData,
+  HasteMapItem,
+  HType,
+  InputFileMapPlugin,
+  PerfLoggerFactory,
+} from './flow-types';
+
+import EventEmitter from 'events';
+
+export type {
+  BuildParameters,
+  BuildResult,
+  CacheData,
+  ChangeEventMetadata,
+  FileData,
+  FileMap,
+  FileSystem,
+  HasteMapData,
+  HasteMapItem,
+  InputFileMapPlugin,
+};
+export type InputOptions = Readonly<{
+  computeSha1?: null | undefined | boolean;
+  enableSymlinks?: null | undefined | boolean;
+  extensions: ReadonlyArray<string>;
+  forceNodeFilesystemAPI?: null | undefined | boolean;
+  ignorePattern?: null | undefined | RegExp;
+  plugins?: ReadonlyArray<InputFileMapPlugin>;
+  retainAllFiles: boolean;
+  rootDir: string;
+  roots: ReadonlyArray<string>;
+  cacheManagerFactory?: null | undefined | CacheManagerFactory;
+  console?: Console;
+  healthCheck: HealthCheckOptions;
+  maxFilesPerWorker?: null | undefined | number;
+  maxWorkers: number;
+  perfLoggerFactory?: null | undefined | PerfLoggerFactory;
+  resetCache?: null | undefined | boolean;
+  useWatchman?: null | undefined | boolean;
+  watch?: null | undefined | boolean;
+  watchmanDeferStates?: ReadonlyArray<string>;
+}>;
+type HealthCheckOptions = Readonly<{
+  enabled: boolean;
+  interval: number;
+  timeout: number;
+  filePrefix: string;
+}>;
+export {DiskCacheManager} from './cache/DiskCacheManager';
+export {default as DependencyPlugin} from './plugins/DependencyPlugin';
+export type {DependencyPluginOptions} from './plugins/DependencyPlugin';
+export {DuplicateHasteCandidatesError} from './plugins/haste/DuplicateHasteCandidatesError';
+export {HasteConflictsError} from './plugins/haste/HasteConflictsError';
+export {default as HastePlugin} from './plugins/HastePlugin';
+export type {HasteMap} from './flow-types';
+export type {HealthCheckResult} from './Watcher';
+export type {
+  CacheManager,
+  CacheManagerFactory,
+  CacheManagerFactoryOptions,
+  CacheManagerWriteOptions,
+  ChangeEvent,
+  DependencyExtractor,
+  WatcherStatus,
+} from './flow-types';
+/**
+ * FileMap includes a JavaScript implementation of Facebook's haste module system.
+ *
+ * This implementation is inspired by https://github.com/facebook/node-haste
+ * and was built with for high-performance in large code repositories with
+ * hundreds of thousands of files. This implementation is scalable and provides
+ * predictable performance.
+ *
+ * Because the file map creation and synchronization is critical to startup
+ * performance and most tasks are blocked by I/O this class makes heavy use of
+ * synchronous operations. It uses worker processes for parallelizing file
+ * access and metadata extraction.
+ *
+ * The data structures created by `metro-file-map` can be used directly from the
+ * cache without further processing. The metadata objects in the `files` and
+ * `map` objects contain cross-references: a metadata object from one can look
+ * up the corresponding metadata object in the other map. Note that in most
+ * projects, the number of files will be greater than the number of haste
+ * modules one module can refer to many files based on platform extensions.
+ *
+ * type CacheData = {
+ *   clocks: WatchmanClocks,
+ *   files: {[filepath: string]: FileMetadata},
+ *   map: {[id: string]: HasteMapItem},
+ *   mocks: {[id: string]: string},
+ * }
+ *
+ * // Watchman clocks are used for query synchronization and file system deltas.
+ * type WatchmanClocks = {[filepath: string]: string};
+ *
+ * type FileMetadata = {
+ *   id: ?string, // used to look up module metadata objects in `map`.
+ *   mtime: number, // check for outdated files.
+ *   size: number, // size of the file in bytes.
+ *   visited: boolean, // whether the file has been parsed or not.
+ *   dependencies: Array<string>, // all relative dependencies of this file.
+ *   sha1: ?string, // SHA-1 of the file, if requested via options.
+ *   symlink: ?(1 | 0 | string), // Truthy if symlink, string is target
+ * };
+ *
+ * // Modules can be targeted to a specific platform based on the file name.
+ * // Example: platform.ios.js and Platform.android.js will both map to the same
+ * // `Platform` module. The platform should be specified during resolution.
+ * type HasteMapItem = {[platform: string]: ModuleMetadata};
+ *
+ * //
+ * type ModuleMetadata = {
+ *   path: string, // the path to look up the file object in `files`.
+ *   type: string, // the module type (either `package` or `module`).
+ * };
+ *
+ * Note that the data structures described above are conceptual only. The actual
+ * implementation uses arrays and constant keys for metadata storage. Instead of
+ * `{id: 'flatMap', mtime: 3421, size: 42, visited: true, dependencies: []}` the real
+ * representation is similar to `['flatMap', 3421, 42, 1, []]` to save storage space
+ * and reduce parse and write time of a big JSON blob.
+ *
+ * The FileMap is created as follows:
+ *  1. read data from the cache or create an empty structure.
+ *
+ *  2. crawl the file system.
+ *     * empty cache: crawl the entire file system.
+ *     * cache available:
+ *       * if watchman is available: get file system delta changes.
+ *       * if watchman is unavailable: crawl the entire file system.
+ *     * build metadata objects for every file. This builds the `files` part of
+ *       the `FileMap`.
+ *
+ *  3. visit and extract metadata from changed files, including sha1,
+ *     depedendencies, and any plugins.
+ *     * this is done in parallel over worker processes to improve performance.
+ *     * the worst case is to visit all files.
+ *     * the best case is no file system access and retrieving all data from
+ *       the cache.
+ *     * the average case is a small number of changed files.
+ *
+ *  4. serialize the new `FileMap` in a cache file.
+ *
+ */
+declare class FileMap extends EventEmitter {
+  static create(options: InputOptions): FileMap;
+  constructor(options: InputOptions);
+  build(): Promise<BuildResult>;
+  /**
+   * 1. read data from the cache or create an empty structure.
+   */
+  read(): Promise<null | undefined | CacheData>;
+  end(): Promise<void>;
+  static H: HType;
+}
+export default FileMap;

package/src/index.js

@@ -40,6 +40,7 @@ var _checkWatchmanCapabilities = _interopRequireDefault(
   require("./lib/checkWatchmanCapabilities"),
 );
 var _FileProcessor = require("./lib/FileProcessor");
+var _FileSystemChangeAggregator = require("./lib/FileSystemChangeAggregator");
 var _normalizePathSeparatorsToPosix = _interopRequireDefault(
   require("./lib/normalizePathSeparatorsToPosix"),
 );
@@ -52,7 +53,6 @@ var _Watcher = require("./Watcher");
 var _events = _interopRequireDefault(require("events"));
 var _fs = require("fs");
 var _invariant = _interopRequireDefault(require("invariant"));
-var _nullthrows = _interopRequireDefault(require("nullthrows"));
 var path = _interopRequireWildcard(require("path"));
 var _perf_hooks = require("perf_hooks");
 var _DependencyPlugin = _interopRequireDefault(
@@ -69,10 +69,7 @@ function _interopRequireWildcard(e, t) {
     if (!t && e && e.__esModule) return e;
     var o,
       i,
-      f = {
-        __proto__: null,
-        default: e,
-      };
+      f = { __proto__: null, default: e };
     if (null === e || ("object" != typeof e && "function" != typeof e))
       return f;
     if ((o = t ? n : r)) {
@@ -92,11 +89,7 @@ function _interopRequireWildcard(e, t) {
   })(e, t);
 }
 function _interopRequireDefault(e) {
-  return e && e.__esModule
-    ? e
-    : {
-        default: e,
-      };
+  return e && e.__esModule ? e : { default: e };
 }
 const debug = require("debug")("Metro:FileMap");
 const CACHE_BREAKER = "11";
@@ -277,7 +270,7 @@ class FileMap extends _events.default {
                     };
                   },
                   fileIterator: (opts) =>
-                    mapIterator(
+                    mapIterable(
                       fileSystem.metadataIterator(opts),
                       ({ baseName, canonicalPath, metadata }) => ({
                         baseName,
@@ -291,21 +284,21 @@ class FileMap extends _events.default {
             ),
           ),
         ]);
-        await this.#applyFileDelta(fileSystem, plugins, fileDelta);
+        const actualChanges = await this.#applyFileDelta(
+          fileSystem,
+          plugins,
+          fileDelta,
+        );
+        const changeSize = actualChanges.getSize();
         plugins.forEach(({ plugin }) => plugin.assertValid());
         const watchmanClocks = new Map(fileDelta.clocks ?? []);
         await this.#takeSnapshotAndPersist(
           fileSystem,
           watchmanClocks,
           plugins,
-          fileDelta.changedFiles,
-          fileDelta.removedFiles,
-        );
-        debug(
-          "Finished mapping files (%d changes, %d removed).",
-          fileDelta.changedFiles.size,
-          fileDelta.removedFiles.size,
+          changeSize > 0,
         );
+        debug("Finished mapping files (%d changes).", changeSize);
         await this.#watch(fileSystem, watchmanClocks, plugins);
         return {
           fileSystem,
@@ -375,10 +368,9 @@ class FileMap extends _events.default {
     });
     const watcher = this.#watcher;
     watcher.on("status", (status) => this.emit("status", status));
-    return watcher.crawl().then((result) => {
-      this.#startupPerfLogger?.point("buildFileDelta_end");
-      return result;
-    });
+    const result = await watcher.crawl();
+    this.#startupPerfLogger?.point("buildFileDelta_end");
+    return result;
   }
   #maybeReadLink(normalPath, fileMetadata) {
     if (fileMetadata[_constants.default.SYMLINK] === 1) {
@@ -395,14 +387,11 @@ class FileMap extends _events.default {
     this.#startupPerfLogger?.point("applyFileDelta_start");
     const { changedFiles, removedFiles } = delta;
     this.#startupPerfLogger?.point("applyFileDelta_preprocess_start");
-    const missingFiles = new Set();
     this.#startupPerfLogger?.point("applyFileDelta_remove_start");
-    const removed = [];
+    const changeAggregator =
+      new _FileSystemChangeAggregator.FileSystemChangeAggregator();
     for (const relativeFilePath of removedFiles) {
-      const metadata = fileSystem.remove(relativeFilePath);
-      if (metadata) {
-        removed.push([relativeFilePath, metadata]);
-      }
+      fileSystem.remove(relativeFilePath, changeAggregator);
     }
     this.#startupPerfLogger?.point("applyFileDelta_remove_end");
     const readLinkPromises = [];
@@ -418,12 +407,12 @@ class FileMap extends _events.default {
         const maybeReadLink = this.#maybeReadLink(normalFilePath, fileData);
         if (maybeReadLink) {
           readLinkPromises.push(
-            maybeReadLink.catch((error) =>
+            maybeReadLink.catch((error) => {
               readLinkErrors.push({
                 normalFilePath,
                 error,
-              }),
-            ),
+              });
+            }),
           );
         }
       }
@@ -448,39 +437,34 @@ class FileMap extends _events.default {
       readLinkErrors,
     )) {
       if (["ENOENT", "EACCESS"].includes(error.code)) {
-        missingFiles.add(normalFilePath);
+        delta.changedFiles.delete(normalFilePath);
+        fileSystem.remove(normalFilePath, changeAggregator);
       } else {
         throw error;
       }
     }
-    for (const relativeFilePath of missingFiles) {
-      changedFiles.delete(relativeFilePath);
-      const metadata = fileSystem.remove(relativeFilePath);
-      if (metadata) {
-        removed.push([relativeFilePath, metadata]);
-      }
-    }
     this.#startupPerfLogger?.point("applyFileDelta_missing_end");
     this.#startupPerfLogger?.point("applyFileDelta_add_start");
-    fileSystem.bulkAddOrModify(changedFiles);
+    fileSystem.bulkAddOrModify(changedFiles, changeAggregator);
     this.#startupPerfLogger?.point("applyFileDelta_add_end");
     this.#startupPerfLogger?.point("applyFileDelta_updatePlugins_start");
-    await Promise.all([
-      plugins.map(({ plugin, dataIdx }) => {
-        const mapFn =
-          dataIdx != null
-            ? ([relativePath, fileData]) => [relativePath, fileData[dataIdx]]
-            : ([relativePath, fileData]) => [relativePath, null];
-        return plugin.bulkUpdate({
-          addedOrModified: mapIterator(changedFiles.entries(), mapFn),
-          removed: mapIterator(removed.values(), mapFn),
-        });
-      }),
-    ]);
+    this.#plugins.forEach(({ plugin, dataIdx }) => {
+      plugin.onChanged(
+        changeAggregator.getMappedView(
+          dataIdx != null ? (metadata) => metadata[dataIdx] : () => null,
+        ),
+      );
+    });
     this.#startupPerfLogger?.point("applyFileDelta_updatePlugins_end");
     this.#startupPerfLogger?.point("applyFileDelta_end");
+    return changeAggregator;
   }
-  async #takeSnapshotAndPersist(fileSystem, clocks, plugins, changed, removed) {
+  async #takeSnapshotAndPersist(
+    fileSystem,
+    clocks,
+    plugins,
+    changedSinceCacheRead,
+  ) {
     this.#startupPerfLogger?.point("persist_start");
     await this.#cacheManager.write(
       () => ({
@@ -494,7 +478,7 @@ class FileMap extends _events.default {
         ),
       }),
       {
-        changedSinceCacheRead: changed.size + removed.size > 0,
+        changedSinceCacheRead,
         eventSource: {
           onChange: (cb) => {
             this.on("change", cb);
@@ -520,14 +504,46 @@ class FileMap extends _events.default {
     }
     const hasWatchedExtension = (filePath) =>
       this.#options.extensions.some((ext) => filePath.endsWith(ext));
-    let changeQueue = Promise.resolve();
     let nextEmit = null;
     const emitChange = () => {
-      if (nextEmit == null || nextEmit.eventsQueue.length === 0) {
+      if (nextEmit == null) {
+        return;
+      }
+      const { events, firstEventTimestamp, firstEnqueuedTimestamp } = nextEmit;
+      const changeAggregator =
+        new _FileSystemChangeAggregator.FileSystemChangeAggregator();
+      for (const event of events) {
+        const { relativeFilePath, clock } = event;
+        if (event.type === "delete") {
+          fileSystem.remove(relativeFilePath, changeAggregator);
+        } else {
+          fileSystem.addOrModify(
+            relativeFilePath,
+            event.metadata,
+            changeAggregator,
+          );
+        }
+        this.#updateClock(clocks, clock);
+      }
+      const changeSize = changeAggregator.getSize();
+      if (changeSize === 0) {
+        nextEmit = null;
         return;
       }
-      const { eventsQueue, firstEventTimestamp, firstEnqueuedTimestamp } =
-        nextEmit;
+      const _netChange = changeAggregator.getView();
+      this.#plugins.forEach(({ plugin, dataIdx }) => {
+        plugin.onChanged(
+          changeAggregator.getMappedView(
+            dataIdx != null ? (metadata) => metadata[dataIdx] : () => null,
+          ),
+        );
+      });
+      const toPublicMetadata = (metadata) => ({
+        isSymlink: metadata[_constants.default.SYMLINK] !== 0,
+        modifiedTime: metadata[_constants.default.MTIME] ?? null,
+      });
+      const changesWithMetadata =
+        changeAggregator.getMappedView(toPublicMetadata);
       const hmrPerfLogger = this.#options.perfLoggerFactory?.("HMR", {
         key: this.#getNextChangeID(),
       });
@@ -541,20 +557,23 @@ class FileMap extends _events.default {
         hmrPerfLogger.point("waitingForChangeInterval_end");
         hmrPerfLogger.annotate({
           int: {
-            eventsQueueLength: eventsQueue.length,
+            changeSize,
           },
         });
         hmrPerfLogger.point("fileChange_start");
       }
       const changeEvent = {
-        eventsQueue,
+        changes: changesWithMetadata,
         logger: hmrPerfLogger,
+        rootDir: this.#options.rootDir,
       };
       this.emit("change", changeEvent);
       nextEmit = null;
     };
+    let changeQueue = Promise.resolve();
     const onChange = (change) => {
       if (
+        change.event !== "recrawl" &&
         change.metadata &&
         (change.metadata.type === "d" ||
           (change.metadata.type === "f" &&
@@ -572,62 +591,36 @@ class FileMap extends _events.default {
       }
       const relativeFilePath =
         this.#pathUtils.absoluteToNormal(absoluteFilePath);
-      const linkStats = fileSystem.linkStats(relativeFilePath);
-      if (
-        change.event === "touch" &&
-        linkStats != null &&
-        change.metadata.modifiedTime != null &&
-        linkStats.modifiedTime === change.metadata.modifiedTime
-      ) {
-        return;
-      }
-      const eventTypeToEmit =
-        change.event === "touch"
-          ? linkStats == null
-            ? "add"
-            : "change"
-          : "delete";
       const onChangeStartTime =
         _perf_hooks.performance.timeOrigin + _perf_hooks.performance.now();
+      const enqueueEvent = (event) => {
+        nextEmit ??= {
+          events: [],
+          firstEnqueuedTimestamp:
+            _perf_hooks.performance.timeOrigin + _perf_hooks.performance.now(),
+          firstEventTimestamp: onChangeStartTime,
+        };
+        nextEmit.events.push(event);
+      };
       changeQueue = changeQueue
         .then(async () => {
           if (
             nextEmit != null &&
-            nextEmit.eventsQueue.find(
+            nextEmit.events.find(
               (event) =>
-                event.type === eventTypeToEmit &&
-                event.filePath === absoluteFilePath &&
+                event.type === change.event &&
+                event.relativeFilePath === relativeFilePath &&
                 ((!event.metadata && !change.metadata) ||
                   (event.metadata &&
                     change.metadata &&
-                    event.metadata.modifiedTime != null &&
+                    event.metadata[_constants.default.MTIME] != null &&
                     change.metadata.modifiedTime != null &&
-                    event.metadata.modifiedTime ===
+                    event.metadata[_constants.default.MTIME] ===
                       change.metadata.modifiedTime)),
             )
           ) {
             return null;
           }
-          const linkStats = fileSystem.linkStats(relativeFilePath);
-          const enqueueEvent = (metadata) => {
-            const event = {
-              filePath: absoluteFilePath,
-              metadata,
-              type: eventTypeToEmit,
-            };
-            if (nextEmit == null) {
-              nextEmit = {
-                eventsQueue: [event],
-                firstEnqueuedTimestamp:
-                  _perf_hooks.performance.timeOrigin +
-                  _perf_hooks.performance.now(),
-                firstEventTimestamp: onChangeStartTime,
-              };
-            } else {
-              nextEmit.eventsQueue.push(event);
-            }
-            return null;
-          };
           if (change.event === "touch") {
             (0, _invariant.default)(
               change.metadata.size != null,
@@ -654,40 +647,65 @@ class FileMap extends _events.default {
                   },
                 );
               }
-              fileSystem.addOrModify(relativeFilePath, fileMetadata);
-              this.#updateClock(clocks, change.clock);
-              plugins.forEach(({ plugin, dataIdx }) =>
-                dataIdx != null
-                  ? plugin.onNewOrModifiedFile(
-                      relativeFilePath,
-                      fileMetadata[dataIdx],
-                    )
-                  : plugin.onNewOrModifiedFile(relativeFilePath),
-              );
-              enqueueEvent(change.metadata);
+              enqueueEvent({
+                clock: change.clock,
+                relativeFilePath,
+                metadata: fileMetadata,
+                type: change.event,
+              });
             } catch (e) {
               if (!["ENOENT", "EACCESS"].includes(e.code)) {
                 throw e;
               }
             }
           } else if (change.event === "delete") {
-            if (linkStats == null) {
+            enqueueEvent({
+              clock: change.clock,
+              relativeFilePath,
+              type: "delete",
+            });
+          } else if (change.event === "recrawl") {
+            emitChange();
+            const absoluteDirPath = path.join(
+              change.root,
+              (0, _normalizePathSeparatorsToSystem.default)(
+                change.relativePath,
+              ),
+            );
+            const subpath = this.#pathUtils.absoluteToNormal(absoluteDirPath);
+            const watcher = this.#watcher;
+            (0, _invariant.default)(
+              watcher != null,
+              "Watcher must be initialized",
+            );
+            const crawlResult = await watcher.recrawl(subpath, fileSystem);
+            if (
+              crawlResult.changedFiles.size === 0 &&
+              crawlResult.removedFiles.size === 0
+            ) {
               return null;
             }
-            const metadata = (0, _nullthrows.default)(
-              fileSystem.remove(relativeFilePath),
+            const recrawlChangeAggregator = await this.#applyFileDelta(
+              fileSystem,
+              this.#plugins,
+              crawlResult,
             );
             this.#updateClock(clocks, change.clock);
-            plugins.forEach(({ plugin, dataIdx }) =>
-              dataIdx != null
-                ? plugin.onRemovedFile(relativeFilePath, metadata[dataIdx])
-                : plugin.onRemovedFile(relativeFilePath),
-            );
-            enqueueEvent({
-              modifiedTime: null,
-              size: null,
-              type: linkStats.fileType,
+            if (recrawlChangeAggregator.getSize() === 0) {
+              return null;
+            }
+            const toPublicMetadata = (metadata) => ({
+              isSymlink: metadata[_constants.default.SYMLINK] !== 0,
+              modifiedTime: metadata[_constants.default.MTIME] ?? null,
             });
+            const changesWithMetadata =
+              recrawlChangeAggregator.getMappedView(toPublicMetadata);
+            const changeEvent = {
+              changes: changesWithMetadata,
+              logger: null,
+              rootDir: this.#options.rootDir,
+            };
+            this.emit("change", changeEvent);
           } else {
             throw new Error(
               `metro-file-map: Unrecognized event type from watcher: ${change.event}`,
@@ -787,11 +805,9 @@ class FileMap extends _events.default {
   static H = _constants.default;
 }
 exports.default = FileMap;
-const mapIterator = (it, fn) =>
-  "map" in it
-    ? it.map(fn)
-    : (function* mapped() {
-        for (const item of it) {
-          yield fn(item);
-        }
-      })();
+const mapIterable = (it, fn) =>
+  (function* mapped() {
+    for (const item of it) {
+      yield fn(item);
+    }
+  })();