metro-file-map 0.83.4 → 0.83.6

package/src/index.js.flow

@@ -17,12 +17,13 @@ import type {
   CacheManagerFactory,
   CacheManagerFactoryOptions,
   CanonicalPath,
+  ChangedFileMetadata,
   ChangeEvent,
   ChangeEventClock,
   ChangeEventMetadata,
   Console,
   CrawlerOptions,
-  EventsQueue,
+  CrawlResult,
   FileData,
   FileMapPlugin,
   FileMapPluginWorker,
@@ -31,6 +32,7 @@ import type {
   HasteMapData,
   HasteMapItem,
   HType,
+  InputFileMapPlugin,
   MutableFileSystem,
   Path,
   PerfLogger,
@@ -44,6 +46,7 @@ import {DiskCacheManager} from './cache/DiskCacheManager';
 import H from './constants';
 import checkWatchmanCapabilities from './lib/checkWatchmanCapabilities';
 import {FileProcessor} from './lib/FileProcessor';
+import {FileSystemChangeAggregator} from './lib/FileSystemChangeAggregator';
 import normalizePathSeparatorsToPosix from './lib/normalizePathSeparatorsToPosix';
 import normalizePathSeparatorsToSystem from './lib/normalizePathSeparatorsToSystem';
 import {RootPathUtils} from './lib/RootPathUtils';
@@ -52,7 +55,6 @@ import {Watcher} from './Watcher';
 import EventEmitter from 'events';
 import {promises as fsPromises} from 'fs';
 import invariant from 'invariant';
-import nullthrows from 'nullthrows';
 import * as path from 'path';
 import {performance} from 'perf_hooks';
 
@@ -69,6 +71,7 @@ export type {
   FileSystem,
   HasteMapData,
   HasteMapItem,
+  InputFileMapPlugin,
 };
 
 export type InputOptions = Readonly<{
@@ -77,7 +80,7 @@ export type InputOptions = Readonly<{
   extensions: ReadonlyArray<string>,
   forceNodeFilesystemAPI?: ?boolean,
   ignorePattern?: ?RegExp,
-  plugins?: ReadonlyArray<AnyFileMapPlugin>,
+  plugins?: ReadonlyArray<InputFileMapPlugin>,
   retainAllFiles: boolean,
   rootDir: string,
   roots: ReadonlyArray<string>,
@@ -111,12 +114,24 @@ type InternalOptions = Readonly<{
   watchmanDeferStates: ReadonlyArray<string>,
 }>;
 
-// $FlowFixMe[unclear-type] Plugin types cannot be known statically
-type AnyFileMapPlugin = FileMapPlugin<any, any>;
 type IndexedPlugin = Readonly<{
-  plugin: AnyFileMapPlugin,
+  // $FlowFixMe[unclear-type] Plugin types cannot be known statically
+  plugin: FileMapPlugin<any, any>,
   dataIdx: ?number,
 }>;
+type InternalEnqueuedEvent = Readonly<
+  | {
+      clock: ?ChangeEventClock,
+      relativeFilePath: string,
+      metadata: FileMetadata,
+      type: 'touch',
+    }
+  | {
+      clock: ?ChangeEventClock,
+      relativeFilePath: string,
+      type: 'delete',
+    },
+>;
 
 export {DiskCacheManager} from './cache/DiskCacheManager';
 export {default as DependencyPlugin} from './plugins/DependencyPlugin';
@@ -421,7 +436,7 @@ export default class FileMap extends EventEmitter {
                     };
                   },
                   fileIterator: opts =>
-                    mapIterator(
+                    mapIterable(
                       fileSystem.metadataIterator(opts),
                       ({baseName, canonicalPath, metadata}) => ({
                         baseName,
@@ -437,7 +452,13 @@ export default class FileMap extends EventEmitter {
         ]);
 
         // Update `fileSystem` and plugins based on the file delta.
-        await this.#applyFileDelta(fileSystem, plugins, fileDelta);
+        const actualChanges = await this.#applyFileDelta(
+          fileSystem,
+          plugins,
+          fileDelta,
+        );
+
+        const changeSize = actualChanges.getSize();
 
         // Validate plugins before persisting them.
         plugins.forEach(({plugin}) => plugin.assertValid());
@@ -447,14 +468,9 @@ export default class FileMap extends EventEmitter {
           fileSystem,
           watchmanClocks,
           plugins,
-          fileDelta.changedFiles,
-          fileDelta.removedFiles,
-        );
-        debug(
-          'Finished mapping files (%d changes, %d removed).',
-          fileDelta.changedFiles.size,
-          fileDelta.removedFiles.size,
+          changeSize > 0,
         );
+        debug('Finished mapping files (%d changes).', changeSize);
 
         await this.#watch(fileSystem, watchmanClocks, plugins);
         return {fileSystem};
@@ -492,11 +508,7 @@ export default class FileMap extends EventEmitter {
    */
   async #buildFileDelta(
     previousState: CrawlerOptions['previousState'],
-  ): Promise<{
-    removedFiles: Set<CanonicalPath>,
-    changedFiles: FileData,
-    clocks?: WatchmanClocks,
-  }> {
+  ): Promise<CrawlResult> {
     this.#startupPerfLogger?.point('buildFileDelta_start');
 
     const {
@@ -540,10 +552,9 @@ export default class FileMap extends EventEmitter {
 
     watcher.on('status', status => this.emit('status', status));
 
-    return watcher.crawl().then(result => {
-      this.#startupPerfLogger?.point('buildFileDelta_end');
-      return result;
-    });
+    const result = await watcher.crawl();
+    this.#startupPerfLogger?.point('buildFileDelta_end');
+    return result;
   }
 
   #maybeReadLink(normalPath: Path, fileMetadata: FileMetadata): ?Promise<void> {
@@ -568,25 +579,20 @@ export default class FileMap extends EventEmitter {
       removedFiles: ReadonlySet<CanonicalPath>,
       clocks?: WatchmanClocks,
     }>,
-  ): Promise<void> {
+  ): Promise<FileSystemChangeAggregator> {
     this.#startupPerfLogger?.point('applyFileDelta_start');
     const {changedFiles, removedFiles} = delta;
     this.#startupPerfLogger?.point('applyFileDelta_preprocess_start');
-    const missingFiles: Set<string> = new Set();
-
     // Remove files first so that we don't mistake moved modules
     // modules as duplicates.
     this.#startupPerfLogger?.point('applyFileDelta_remove_start');
-    const removed: Array<[string, FileMetadata]> = [];
+    const changeAggregator = new FileSystemChangeAggregator();
     for (const relativeFilePath of removedFiles) {
-      const metadata = fileSystem.remove(relativeFilePath);
-      if (metadata) {
-        removed.push([relativeFilePath, metadata]);
-      }
+      fileSystem.remove(relativeFilePath, changeAggregator);
     }
     this.#startupPerfLogger?.point('applyFileDelta_remove_end');
 
-    const readLinkPromises = [];
+    const readLinkPromises: Array<Promise<void>> = [];
     const readLinkErrors: Array<{
       normalFilePath: string,
       error: Error & {code?: string},
@@ -606,9 +612,9 @@ export default class FileMap extends EventEmitter {
         const maybeReadLink = this.#maybeReadLink(normalFilePath, fileData);
         if (maybeReadLink) {
           readLinkPromises.push(
-            maybeReadLink.catch(error =>
-              readLinkErrors.push({normalFilePath, error}),
-            ),
+            maybeReadLink.catch(error => {
+              readLinkErrors.push({normalFilePath, error});
+            }),
           );
         }
       }
@@ -647,43 +653,32 @@ export default class FileMap extends EventEmitter {
       /* $FlowFixMe[incompatible-type] Error exposed after improved typing of
        * Array.{includes,indexOf,lastIndexOf} */
       if (['ENOENT', 'EACCESS'].includes(error.code)) {
-        missingFiles.add(normalFilePath);
+        delta.changedFiles.delete(normalFilePath);
+        fileSystem.remove(normalFilePath, changeAggregator);
       } else {
         // Anything else is fatal.
         throw error;
       }
     }
-    for (const relativeFilePath of missingFiles) {
-      changedFiles.delete(relativeFilePath);
-      const metadata = fileSystem.remove(relativeFilePath);
-      if (metadata) {
-        removed.push([relativeFilePath, metadata]);
-      }
-    }
+
     this.#startupPerfLogger?.point('applyFileDelta_missing_end');
 
     this.#startupPerfLogger?.point('applyFileDelta_add_start');
-    fileSystem.bulkAddOrModify(changedFiles);
+    fileSystem.bulkAddOrModify(changedFiles, changeAggregator);
     this.#startupPerfLogger?.point('applyFileDelta_add_end');
 
     this.#startupPerfLogger?.point('applyFileDelta_updatePlugins_start');
-
-    await Promise.all([
-      plugins.map(({plugin, dataIdx}) => {
-        const mapFn: (
-          [CanonicalPath, FileMetadata],
-        ) => [CanonicalPath, unknown] =
-          dataIdx != null
-            ? ([relativePath, fileData]) => [relativePath, fileData[dataIdx]]
-            : ([relativePath, fileData]) => [relativePath, null];
-        return plugin.bulkUpdate({
-          addedOrModified: mapIterator(changedFiles.entries(), mapFn),
-          removed: mapIterator(removed.values(), mapFn),
-        });
-      }),
-    ]);
+    this.#plugins.forEach(({plugin, dataIdx}) => {
+      plugin.onChanged(
+        changeAggregator.getMappedView(
+          dataIdx != null ? metadata => metadata[dataIdx] : () => null,
+        ),
+      );
+    });
     this.#startupPerfLogger?.point('applyFileDelta_updatePlugins_end');
     this.#startupPerfLogger?.point('applyFileDelta_end');
+
+    return changeAggregator;
   }
 
   /**
@@ -693,8 +688,7 @@ export default class FileMap extends EventEmitter {
     fileSystem: FileSystem,
     clocks: WatchmanClocks,
     plugins: ReadonlyArray<IndexedPlugin>,
-    changed: FileData,
-    removed: Set<CanonicalPath>,
+    changedSinceCacheRead: boolean,
   ) {
     this.#startupPerfLogger?.point('persist_start');
     await this.#cacheManager.write(
@@ -709,7 +703,7 @@ export default class FileMap extends EventEmitter {
         ),
       }),
       {
-        changedSinceCacheRead: changed.size + removed.size > 0,
+        changedSinceCacheRead,
         eventSource: {
           onChange: cb => {
             // Inform the cache about changes to internal state, including:
@@ -748,20 +742,68 @@ export default class FileMap extends EventEmitter {
     const hasWatchedExtension = (filePath: string) =>
       this.#options.extensions.some(ext => filePath.endsWith(ext));
 
-    let changeQueue: Promise<null | void> = Promise.resolve();
     let nextEmit: ?{
-      eventsQueue: EventsQueue,
+      events: Array<InternalEnqueuedEvent>,
       firstEventTimestamp: number,
       firstEnqueuedTimestamp: number,
     } = null;
 
     const emitChange = () => {
-      if (nextEmit == null || nextEmit.eventsQueue.length === 0) {
+      if (nextEmit == null) {
         // Nothing to emit
         return;
       }
-      const {eventsQueue, firstEventTimestamp, firstEnqueuedTimestamp} =
-        nextEmit;
+      const {events, firstEventTimestamp, firstEnqueuedTimestamp} = nextEmit;
+
+      const changeAggregator = new FileSystemChangeAggregator();
+
+      // Process a sequence of events. Note that preserving ordering is
+      // important here - a file may be both removed and added in the same
+      // batch.
+      // `changeAggregator` flattens this over time into the net change from
+      // this sequence.
+      for (const event of events) {
+        const {relativeFilePath, clock} = event;
+        if (event.type === 'delete') {
+          fileSystem.remove(relativeFilePath, changeAggregator);
+        } else {
+          fileSystem.addOrModify(
+            relativeFilePath,
+            event.metadata,
+            changeAggregator,
+          );
+        }
+        this.#updateClock(clocks, clock);
+      }
+
+      const changeSize = changeAggregator.getSize();
+
+      if (changeSize === 0) {
+        // We had events, but they've exactly cancelled each other out, reset
+        // so that timers are correct for the next change.
+        nextEmit = null;
+        return;
+      }
+
+      const _netChange = changeAggregator.getView();
+      this.#plugins.forEach(({plugin, dataIdx}) => {
+        plugin.onChanged(
+          changeAggregator.getMappedView(
+            dataIdx != null ? metadata => metadata[dataIdx] : () => null,
+          ),
+        );
+      });
+
+      const toPublicMetadata = (
+        metadata: Readonly<FileMetadata>,
+      ): ChangedFileMetadata => ({
+        isSymlink: metadata[H.SYMLINK] !== 0,
+        modifiedTime: metadata[H.MTIME] ?? null,
+      });
+
+      const changesWithMetadata =
+        changeAggregator.getMappedView(toPublicMetadata);
+
       const hmrPerfLogger = this.#options.perfLoggerFactory?.('HMR', {
         key: this.#getNextChangeID(),
       });
@@ -771,21 +813,24 @@ export default class FileMap extends EventEmitter {
           timestamp: firstEnqueuedTimestamp,
         });
         hmrPerfLogger.point('waitingForChangeInterval_end');
-        hmrPerfLogger.annotate({
-          int: {eventsQueueLength: eventsQueue.length},
-        });
+        hmrPerfLogger.annotate({int: {changeSize}});
         hmrPerfLogger.point('fileChange_start');
       }
       const changeEvent: ChangeEvent = {
-        eventsQueue,
+        changes: changesWithMetadata,
         logger: hmrPerfLogger,
+        rootDir: this.#options.rootDir,
       };
       this.emit('change', changeEvent);
       nextEmit = null;
     };
 
+    let changeQueue: Promise<null | void> = Promise.resolve();
+
     const onChange = (change: WatcherBackendChangeEvent) => {
+      // Recrawl events bypass normal filtering - they trigger a full subdirectory scan
       if (
+        change.event !== 'recrawl' &&
         change.metadata &&
         // Ignore all directory events
         (change.metadata.type === 'd' ||
@@ -811,73 +856,38 @@ export default class FileMap extends EventEmitter {
 
       const relativeFilePath =
         this.#pathUtils.absoluteToNormal(absoluteFilePath);
-      const linkStats = fileSystem.linkStats(relativeFilePath);
-
-      // The file has been accessed, not modified. If the modified time is
-      // null, then it is assumed that the watcher does not have capabilities
-      // to detect modified time, and change processing proceeds.
-      if (
-        change.event === 'touch' &&
-        linkStats != null &&
-        change.metadata.modifiedTime != null &&
-        linkStats.modifiedTime === change.metadata.modifiedTime
-      ) {
-        return;
-      }
-
-      // Emitted events, unlike memoryless backend events, specify 'add' or
-      // 'change' instead of 'touch'.
-      const eventTypeToEmit =
-        change.event === 'touch'
-          ? linkStats == null
-            ? 'add'
-            : 'change'
-          : 'delete';
 
       const onChangeStartTime = performance.timeOrigin + performance.now();
 
+      const enqueueEvent = (event: InternalEnqueuedEvent) => {
+        nextEmit ??= {
+          events: [],
+          firstEnqueuedTimestamp: performance.timeOrigin + performance.now(),
+          firstEventTimestamp: onChangeStartTime,
+        };
+        nextEmit.events.push(event);
+      };
+
       changeQueue = changeQueue
         .then(async () => {
           // If we get duplicate events for the same file, ignore them.
           if (
             nextEmit != null &&
-            nextEmit.eventsQueue.find(
+            nextEmit.events.find(
               event =>
-                event.type === eventTypeToEmit &&
-                event.filePath === absoluteFilePath &&
+                event.type === change.event &&
+                event.relativeFilePath === relativeFilePath &&
                 ((!event.metadata && !change.metadata) ||
                   (event.metadata &&
                     change.metadata &&
-                    event.metadata.modifiedTime != null &&
+                    event.metadata[H.MTIME] != null &&
                     change.metadata.modifiedTime != null &&
-                    event.metadata.modifiedTime ===
-                      change.metadata.modifiedTime)),
+                    event.metadata[H.MTIME] === change.metadata.modifiedTime)),
             )
           ) {
             return null;
           }
 
-          const linkStats = fileSystem.linkStats(relativeFilePath);
-
-          const enqueueEvent = (metadata: ChangeEventMetadata) => {
-            const event = {
-              filePath: absoluteFilePath,
-              metadata,
-              type: eventTypeToEmit,
-            };
-            if (nextEmit == null) {
-              nextEmit = {
-                eventsQueue: [event],
-                firstEnqueuedTimestamp:
-                  performance.timeOrigin + performance.now(),
-                firstEventTimestamp: onChangeStartTime,
-              };
-            } else {
-              nextEmit.eventsQueue.push(event);
-            }
-            return null;
-          };
-
           // If the file was added or modified,
           // parse it and update the file map.
           if (change.event === 'touch') {
@@ -907,17 +917,12 @@ export default class FileMap extends EventEmitter {
                   },
                 );
               }
-              fileSystem.addOrModify(relativeFilePath, fileMetadata);
-              this.#updateClock(clocks, change.clock);
-              plugins.forEach(({plugin, dataIdx}) =>
-                dataIdx != null
-                  ? plugin.onNewOrModifiedFile(
-                      relativeFilePath,
-                      fileMetadata[dataIdx],
-                    )
-                  : plugin.onNewOrModifiedFile(relativeFilePath),
-              );
-              enqueueEvent(change.metadata);
+              enqueueEvent({
+                clock: change.clock,
+                relativeFilePath,
+                metadata: fileMetadata,
+                type: change.event,
+              });
             } catch (e) {
               if (!['ENOENT', 'EACCESS'].includes(e.code)) {
                 throw e;
@@ -930,26 +935,68 @@ export default class FileMap extends EventEmitter {
               //   point.
             }
           } else if (change.event === 'delete') {
-            if (linkStats == null) {
-              // Don't emit deletion events for files we weren't retaining.
-              // This is expected for deletion of an ignored file.
+            enqueueEvent({
+              clock: change.clock,
+              relativeFilePath,
+              type: 'delete',
+            });
+          } else if (change.event === 'recrawl') {
+            // Recrawl event: flush pending changes and re-crawl the directory
+            emitChange();
+
+            // The relativePath is relative to the watcher root (change.root),
+            // but we need a path relative to rootDir for the recrawl.
+            const absoluteDirPath = path.join(
+              change.root,
+              normalizePathSeparatorsToSystem(change.relativePath),
+            );
+            const subpath = this.#pathUtils.absoluteToNormal(absoluteDirPath);
+
+            // Crawl the specific subdirectory
+            const watcher = this.#watcher;
+            invariant(watcher != null, 'Watcher must be initialized');
+            const crawlResult = await watcher.recrawl(subpath, fileSystem);
+
+            // Skip if no changes
+            if (
+              crawlResult.changedFiles.size === 0 &&
+              crawlResult.removedFiles.size === 0
+            ) {
               return null;
             }
-            // We've already checked linkStats != null above, so the file
-            // exists in the file map and remove should always return metadata.
-            const metadata = nullthrows(fileSystem.remove(relativeFilePath));
-            this.#updateClock(clocks, change.clock);
-            plugins.forEach(({plugin, dataIdx}) =>
-              dataIdx != null
-                ? plugin.onRemovedFile(relativeFilePath, metadata[dataIdx])
-                : plugin.onRemovedFile(relativeFilePath),
+
+            // Reuse the same batch processing logic as build()
+            const recrawlChangeAggregator = await this.#applyFileDelta(
+              fileSystem,
+              this.#plugins,
+              crawlResult,
             );
 
-            enqueueEvent({
-              modifiedTime: null,
-              size: null,
-              type: linkStats.fileType,
+            // Update clock if provided
+            this.#updateClock(clocks, change.clock);
+
+            // Skip emit if no changes after processing
+            if (recrawlChangeAggregator.getSize() === 0) {
+              return null;
+            }
+
+            // Emit changes directly
+            const toPublicMetadata = (
+              metadata: Readonly<FileMetadata>,
+            ): ChangedFileMetadata => ({
+              isSymlink: metadata[H.SYMLINK] !== 0,
+              modifiedTime: metadata[H.MTIME] ?? null,
             });
+
+            const changesWithMetadata =
+              recrawlChangeAggregator.getMappedView(toPublicMetadata);
+
+            const changeEvent: ChangeEvent = {
+              changes: changesWithMetadata,
+              logger: null,
+              rootDir: this.#options.rootDir,
+            };
+            this.emit('change', changeEvent);
           } else {
             throw new Error(
               `metro-file-map: Unrecognized event type from watcher: ${change.event}`,
@@ -1060,11 +1107,9 @@ export default class FileMap extends EventEmitter {
 }
 
 // TODO: Replace with it.map() from Node 22+
-const mapIterator: <T, S>(Iterator<T>, (T) => S) => Iterable<S> = (it, fn) =>
-  'map' in it
-    ? it.map(fn)
-    : (function* mapped() {
-        for (const item of it) {
-          yield fn(item);
-        }
-      })();
+const mapIterable: <T, S>(Iterable<T>, (T) => S) => Iterator<S> = (it, fn) =>
+  (function* mapped() {
+    for (const item of it) {
+      yield fn(item);
+    }
+  })();

package/src/lib/FileProcessor.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @noformat
+ * @oncall react_native
+ * @generated SignedSource<<2ea213f753eef5de14cb8a27f68b9fa2>>
+ *
+ * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js
+ * Original file: packages/metro-file-map/src/lib/FileProcessor.js
+ * To regenerate, run:
+ *   js1 build metro-ts-defs (internal) OR
+ *   yarn run build-ts-defs (OSS) 
+ */
+
+import type {
+  FileMapPluginWorker,
+  FileMetadata,
+  PerfLogger,
+} from '../flow-types';
+
+type ProcessFileRequest = Readonly<{
+  /**
+   * Populate metadata[H.SHA1] with the SHA1 of the file's contents.
+   */
+  computeSha1: boolean;
+  /**
+   * Only if processing has already required reading the file's contents, return
+   * the contents as a Buffer - null otherwise. Not supported for batches.
+   */
+  maybeReturnContent: boolean;
+}>;
+interface MaybeCodedError extends Error {
+  code?: string;
+}
+export declare class FileProcessor {
+  constructor(
+    opts: Readonly<{
+      maxFilesPerWorker?: null | undefined | number;
+      maxWorkers: number;
+      pluginWorkers: null | undefined | ReadonlyArray<FileMapPluginWorker>;
+      perfLogger: null | undefined | PerfLogger;
+      rootDir: string;
+    }>,
+  );
+  processBatch(
+    files: ReadonlyArray<[string, FileMetadata]>,
+    req: ProcessFileRequest,
+  ): Promise<{
+    errors: Array<{normalFilePath: string; error: MaybeCodedError}>;
+  }>;
+  processRegularFile(
+    normalPath: string,
+    fileMetadata: FileMetadata,
+    req: ProcessFileRequest,
+  ): null | undefined | {content: null | undefined | Buffer};
+  end(): Promise<void>;
+}

package/src/lib/FileProcessor.js

@@ -10,11 +10,7 @@ var _RootPathUtils = require("./RootPathUtils");
 var _jestWorker = require("jest-worker");
 var _path = require("path");
 function _interopRequireDefault(e) {
-  return e && e.__esModule
-    ? e
-    : {
-        default: e,
-      };
+  return e && e.__esModule ? e : { default: e };
 }
 const debug = require("debug")("Metro:FileMap");
 const NODE_MODULES_SEP = "node_modules" + _path.sep;