metro-file-map 0.72.3 → 0.73.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "metro-file-map",
-  "version": "0.72.3",
+  "version": "0.73.0",
   "description": "[Experimental] - 🚇 File crawling, watching and mapping for Metro",
   "main": "src/index.js",
   "repository": {
@@ -30,6 +30,6 @@
     "slash": "^3.0.0"
   },
   "optionalDependencies": {
-    "fsevents": "^2.1.2"
+    "fsevents": "^2.3.2"
   }
 }

package/src/ModuleMap.js

@@ -63,6 +63,7 @@ function _interopRequireDefault(obj) {
  *
  *
  * @format
+ * @oncall react_native
  */
 const EMPTY_OBJ = {};
 const EMPTY_MAP = new Map();

package/src/ModuleMap.js.flow

@@ -6,6 +6,7 @@
  *
  * @flow strict-local
  * @format
+ * @oncall react_native
  */
 
 import type {

package/src/cache/DiskCacheManager.js

@@ -29,6 +29,7 @@ function _interopRequireDefault(obj) {
  *
  *
  * @format
+ * @oncall react_native
  */
 const DEFAULT_PREFIX = "metro-file-map";
 const DEFAULT_DIRECTORY = (0, _os.tmpdir)();
@@ -66,9 +67,14 @@ class DiskCacheManager {
       return (0, _v.deserialize)(
         (0, _gracefulFs.readFileSync)(this._cachePath)
       );
-    } catch {}
+    } catch (e) {
+      if ((e === null || e === void 0 ? void 0 : e.code) === "ENOENT") {
+        // Cache file not found - not considered an error.
+        return null;
+      } // Rethrow anything else.
 
-    return null;
+      throw e;
+    }
   }
 
   async write(dataSnapshot, { changed, removed }) {

package/src/cache/DiskCacheManager.js.flow

@@ -6,6 +6,7 @@
  *
  * @flow strict-local
  * @format
+ * @oncall react_native
  */
 
 import type {
@@ -68,8 +69,14 @@ export class DiskCacheManager implements CacheManager {
   async read(): Promise<?InternalData> {
     try {
       return deserialize(readFileSync(this._cachePath));
-    } catch {}
-    return null;
+    } catch (e) {
+      if (e?.code === 'ENOENT') {
+        // Cache file not found - not considered an error.
+        return null;
+      }
+      // Rethrow anything else.
+      throw e;
+    }
   }
 
   async write(

package/src/crawlers/node.js

@@ -64,6 +64,7 @@ function _interopRequireDefault(obj) {
  *
  *
  * @format
+ * @oncall react_native
  */
 async function hasNativeFindSupport(forceNodeFilesystemAPI) {
   if (forceNodeFilesystemAPI) {

package/src/crawlers/node.js.flow

@@ -6,6 +6,7 @@
  *
  * @flow strict-local
  * @format
+ * @oncall react_native
  */
 
 import type {

package/src/crawlers/{watchman.js → watchman/index.js}

@@ -1,13 +1,17 @@
 "use strict";
 
-var _constants = _interopRequireDefault(require("../constants"));
+var _planQuery = require("./planQuery");
 
-var fastPath = _interopRequireWildcard(require("../lib/fast_path"));
+var _constants = _interopRequireDefault(require("../../constants"));
+
+var fastPath = _interopRequireWildcard(require("../../lib/fast_path"));
 
 var _normalizePathSep = _interopRequireDefault(
-  require("../lib/normalizePathSep")
+  require("../../lib/normalizePathSep")
 );
 
+var _invariant = _interopRequireDefault(require("invariant"));
+
 var path = _interopRequireWildcard(require("path"));
 
 function _getRequireWildcardCache(nodeInterop) {
@@ -64,8 +68,9 @@ function _interopRequireDefault(obj) {
  *
  *
  * @format
+ * @oncall react_native
  */
-const watchman = require("fb-watchman"); // $FlowFixMe[unclear-type] - Improve fb-watchman types to cover our uses
+const watchman = require("fb-watchman");
 
 const WATCHMAN_WARNING_INITIAL_DELAY_MILLISECONDS = 10000;
 const WATCHMAN_WARNING_INTERVAL_MILLISECONDS = 20000;
@@ -77,68 +82,30 @@ function makeWatchmanError(error) {
     `is running for this project. See ${watchmanURL}.`;
   return error;
 }
-/**
- * Wrap watchman capabilityCheck method as a promise.
- *
- * @param client watchman client
- * @param caps capabilities to verify
- * @returns a promise resolving to a list of verified capabilities
- */
-
-async function capabilityCheck(client, caps) {
-  return new Promise((resolve, reject) => {
-    client.capabilityCheck(
-      // @ts-expect-error: incorrectly typed
-      caps,
-      (error, response) => {
-        if (error != null || response == null) {
-          reject(
-            error !== null && error !== void 0
-              ? error
-              : new Error("capabilityCheck: Response missing")
-          );
-        } else {
-          resolve(response);
-        }
-      }
-    );
-  });
-}
 
-module.exports = async function watchmanCrawl(options) {
-  var _options$abortSignal;
-
-  const fields = ["name", "exists", "mtime_ms", "size"];
-  const { data, extensions, ignore, rootDir, roots, perfLogger } = options;
-  const clocks = data.clocks;
+module.exports = async function watchmanCrawl({
+  abortSignal,
+  computeSha1,
+  data,
+  enableSymlinks,
+  extensions,
+  ignore,
+  rootDir,
+  roots,
+  perfLogger,
+}) {
   perfLogger === null || perfLogger === void 0
     ? void 0
     : perfLogger.point("watchmanCrawl_start");
+  const clocks = data.clocks;
   const client = new watchman.Client();
-  (_options$abortSignal = options.abortSignal) === null ||
-  _options$abortSignal === void 0
-    ? void 0
-    : _options$abortSignal.addEventListener("abort", () => client.end());
-  perfLogger === null || perfLogger === void 0
+  abortSignal === null || abortSignal === void 0
     ? void 0
-    : perfLogger.point("watchmanCrawl/negotiateCapabilities_start"); // https://facebook.github.io/watchman/docs/capabilities.html
-  // Check adds about ~28ms
-
-  const capabilities = await capabilityCheck(client, {
-    // If a required capability is missing then an error will be thrown,
-    // we don't need this assertion, so using optional instead.
-    optional: ["suffix-set"],
+    : abortSignal.addEventListener("abort", () => client.end());
+  let clientError;
+  client.on("error", (error) => {
+    clientError = makeWatchmanError(error);
   });
-  const suffixExpression =
-    capabilities !== null &&
-    capabilities !== void 0 &&
-    capabilities.capabilities["suffix-set"] // If available, use the optimized `suffix-set` operation:
-      ? // https://facebook.github.io/watchman/docs/expr/suffix.html#suffix-set
-        ["suffix", extensions]
-      : ["anyof", ...extensions.map((extension) => ["suffix", extension])];
-  let clientError; // $FlowFixMe[prop-missing] - Client is not typed as an EventEmitter
-
-  client.on("error", (error) => (clientError = makeWatchmanError(error)));
   let didLogWatchmanWaitMessage = false; // $FlowFixMe[unclear-type] - Fix to use fb-watchman types
 
   const cmd = async (command, ...args) => {
@@ -171,18 +138,6 @@ module.exports = async function watchmanCrawl(options) {
     }
   };
 
-  if (options.computeSha1) {
-    const { capabilities } = await cmd("list-capabilities");
-
-    if (capabilities.indexOf("field-content.sha1hex") !== -1) {
-      fields.push("content.sha1hex");
-    }
-  }
-
-  perfLogger === null || perfLogger === void 0
-    ? void 0
-    : perfLogger.point("watchmanCrawl/negotiateCapabilities_end");
-
   async function getWatchmanRoots(roots) {
     perfLogger === null || perfLogger === void 0
       ? void 0
@@ -250,61 +205,13 @@ module.exports = async function watchmanCrawl(options) {
                   [`watchmanCrawl/query_${index}_has_clock`]: since != null,
                 },
               });
-          const query = {
-            fields,
-            expression: [
-              "allof", // Match regular files only. Different Watchman generators treat
-              // symlinks differently, so this ensures consistent results.
-              ["type", "f"],
-            ],
-          };
-          /**
-           * Watchman "query planner".
-           *
-           * Watchman file queries consist of 1 or more generators that feed
-           * files through the expression evaluator.
-           *
-           * Strategy:
-           * 1. Select the narrowest possible generator so that the expression
-           *    evaluator has fewer candidates to process.
-           * 2. Evaluate expressions from narrowest to broadest.
-           * 3. Don't use an expression to recheck a condition that the
-           *    generator already guarantees.
-           * 4. Compose expressions to avoid combinatorial explosions in the
-           *    number of terms.
-           *
-           * The ordering of generators/filters, from narrow to broad, is:
-           * - since          = O(changes)
-           * - glob / dirname = O(files in a subtree of the repo)
-           * - suffix         = O(files in the repo)
-           *
-           * We assume that file extensions are ~uniformly distributed in the
-           * repo but Haste map projects are focused on a handful of
-           * directories. Therefore `glob` < `suffix`.
-           */
-
-          let queryGenerator;
-
-          if (since != null) {
-            // Use the `since` generator and filter by both path and extension.
-            query.since = since;
-            queryGenerator = "since";
-            query.expression.push(
-              ["anyof", ...directoryFilters.map((dir) => ["dirname", dir])],
-              suffixExpression
-            );
-          } else if (directoryFilters.length > 0) {
-            // Use the `glob` generator and filter only by extension.
-            query.glob = directoryFilters.map((directory) => `${directory}/**`);
-            query.glob_includedotfiles = true;
-            queryGenerator = "glob";
-            query.expression.push(suffixExpression);
-          } else {
-            // Use the `suffix` generator with no path/extension filtering.
-            query.suffix = extensions;
-            queryGenerator = "suffix";
-          }
-
+          const { query, queryGenerator } = (0, _planQuery.planQuery)({
+            since,
+            extensions,
+            directoryFilters,
+            includeSha1: computeSha1,
+            includeSymlinks: enableSymlinks,
+          });
           perfLogger === null || perfLogger === void 0
             ? void 0
             : perfLogger.annotate({
@@ -435,6 +342,11 @@ module.exports = async function watchmanCrawl(options) {
     );
 
     for (const fileData of response.files) {
+      if (fileData.symlink_target != null) {
+        // TODO: Process symlinks
+        continue;
+      }
+
       const filePath =
         fsRoot + path.sep + (0, _normalizePathSep.default)(fileData.name);
       const relativeFilePath = fastPath.relative(rootDir, filePath);
@@ -456,11 +368,13 @@ module.exports = async function watchmanCrawl(options) {
           }
         }
       } else if (!ignore(filePath)) {
+        const { mtime_ms, size } = fileData;
+        (0, _invariant.default)(
+          mtime_ms != null && size != null,
+          "missing file data in watchman response"
+        );
         const mtime =
-          typeof fileData.mtime_ms === "number"
-            ? fileData.mtime_ms
-            : fileData.mtime_ms.toNumber();
-        const size = fileData.size;
+          typeof mtime_ms === "number" ? mtime_ms : mtime_ms.toNumber();
         let sha1hex = fileData["content.sha1hex"];
 
         if (typeof sha1hex !== "string" || sha1hex.length !== 40) {

package/src/crawlers/{watchman.js.flow → watchman/index.js.flow}

@@ -6,6 +6,7 @@
  *
  * @flow strict-local
  * @format
+ * @oncall react_native
  */
 
 import type {
@@ -14,55 +15,20 @@ import type {
   FileMetaData,
   InternalData,
   Path,
-} from '../flow-types';
-
-import H from '../constants';
-import * as fastPath from '../lib/fast_path';
-import normalizePathSep from '../lib/normalizePathSep';
+} from '../../flow-types';
+import type {WatchmanQueryResponse, WatchmanWatchResponse} from 'fb-watchman';
+
+import {planQuery} from './planQuery';
+import H from '../../constants';
+import * as fastPath from '../../lib/fast_path';
+import normalizePathSep from '../../lib/normalizePathSep';
+import invariant from 'invariant';
 import * as path from 'path';
 
 const watchman = require('fb-watchman');
 
-// $FlowFixMe[unclear-type] - Improve fb-watchman types to cover our uses
-type WatchmanQuery = any;
-
 type WatchmanRoots = Map<string, Array<string>>;
 
-type WatchmanListCapabilitiesResponse = {
-  capabilities: Array<string>,
-};
-
-type WatchmanCapabilityCheckResponse = {
-  // { 'suffix-set': true }
-  capabilities: $ReadOnly<{[string]: boolean}>,
-  // '2021.06.07.00'
-  version: string,
-};
-
-type WatchmanWatchProjectResponse = {
-  watch: string,
-  relative_path: string,
-};
-
-type WatchmanQueryResponse = {
-  warning?: string,
-  is_fresh_instance: boolean,
-  version: string,
-  clock:
-    | string
-    | {
-        scm: {'mergebase-with': string, mergebase: string},
-        clock: string,
-      },
-  files: Array<{
-    name: string,
-    exists: boolean,
-    mtime_ms: number | {toNumber: () => number},
-    size: number,
-    'content.sha1hex'?: string,
-  }>,
-};
-
 const WATCHMAN_WARNING_INITIAL_DELAY_MILLISECONDS = 10000;
 const WATCHMAN_WARNING_INTERVAL_MILLISECONDS = 20000;
 
@@ -75,65 +41,32 @@ function makeWatchmanError(error: Error): Error {
   return error;
 }
 
-/**
- * Wrap watchman capabilityCheck method as a promise.
- *
- * @param client watchman client
- * @param caps capabilities to verify
- * @returns a promise resolving to a list of verified capabilities
- */
-async function capabilityCheck(
-  client: watchman.Client,
-  caps: $ReadOnly<{optional?: $ReadOnlyArray<string>}>,
-): Promise<WatchmanCapabilityCheckResponse> {
-  return new Promise((resolve, reject) => {
-    client.capabilityCheck(
-      // @ts-expect-error: incorrectly typed
-      caps,
-      (error, response) => {
-        if (error != null || response == null) {
-          reject(error ?? new Error('capabilityCheck: Response missing'));
-        } else {
-          resolve(response);
-        }
-      },
-    );
-  });
-}
-
-module.exports = async function watchmanCrawl(
-  options: CrawlerOptions,
-): Promise<{
+module.exports = async function watchmanCrawl({
+  abortSignal,
+  computeSha1,
+  data,
+  enableSymlinks,
+  extensions,
+  ignore,
+  rootDir,
+  roots,
+  perfLogger,
+}: CrawlerOptions): Promise<{
   changedFiles?: FileData,
   removedFiles: FileData,
   hasteMap: InternalData,
 }> {
-  const fields = ['name', 'exists', 'mtime_ms', 'size'];
-  const {data, extensions, ignore, rootDir, roots, perfLogger} = options;
+  perfLogger?.point('watchmanCrawl_start');
+
   const clocks = data.clocks;
 
-  perfLogger?.point('watchmanCrawl_start');
   const client = new watchman.Client();
-  options.abortSignal?.addEventListener('abort', () => client.end());
-
-  perfLogger?.point('watchmanCrawl/negotiateCapabilities_start');
-  // https://facebook.github.io/watchman/docs/capabilities.html
-  // Check adds about ~28ms
-  const capabilities = await capabilityCheck(client, {
-    // If a required capability is missing then an error will be thrown,
-    // we don't need this assertion, so using optional instead.
-    optional: ['suffix-set'],
-  });
-
-  const suffixExpression = capabilities?.capabilities['suffix-set']
-    ? // If available, use the optimized `suffix-set` operation:
-      // https://facebook.github.io/watchman/docs/expr/suffix.html#suffix-set
-      ['suffix', extensions]
-    : ['anyof', ...extensions.map(extension => ['suffix', extension])];
+  abortSignal?.addEventListener('abort', () => client.end());
 
   let clientError;
-  // $FlowFixMe[prop-missing] - Client is not typed as an EventEmitter
-  client.on('error', error => (clientError = makeWatchmanError(error)));
+  client.on('error', error => {
+    clientError = makeWatchmanError(error);
+  });
 
   let didLogWatchmanWaitMessage = false;
 
@@ -163,18 +96,6 @@ module.exports = async function watchmanCrawl(
     }
   };
 
-  if (options.computeSha1) {
-    const {capabilities} = await cmd<WatchmanListCapabilitiesResponse>(
-      'list-capabilities',
-    );
-
-    if (capabilities.indexOf('field-content.sha1hex') !== -1) {
-      fields.push('content.sha1hex');
-    }
-  }
-
-  perfLogger?.point('watchmanCrawl/negotiateCapabilities_end');
-
   async function getWatchmanRoots(
     roots: $ReadOnlyArray<Path>,
   ): Promise<WatchmanRoots> {
@@ -183,7 +104,7 @@ module.exports = async function watchmanCrawl(
     await Promise.all(
       roots.map(async (root, index) => {
         perfLogger?.point(`watchmanCrawl/watchProject_${index}_start`);
-        const response = await cmd<WatchmanWatchProjectResponse>(
+        const response = await cmd<WatchmanWatchResponse>(
           'watch-project',
           root,
         );
@@ -235,61 +156,13 @@ module.exports = async function watchmanCrawl(
             },
           });
 
-          const query: WatchmanQuery = {
-            fields,
-            expression: [
-              'allof',
-              // Match regular files only. Different Watchman generators treat
-              // symlinks differently, so this ensures consistent results.
-              ['type', 'f'],
-            ],
-          };
-
-          /**
-           * Watchman "query planner".
-           *
-           * Watchman file queries consist of 1 or more generators that feed
-           * files through the expression evaluator.
-           *
-           * Strategy:
-           * 1. Select the narrowest possible generator so that the expression
-           *    evaluator has fewer candidates to process.
-           * 2. Evaluate expressions from narrowest to broadest.
-           * 3. Don't use an expression to recheck a condition that the
-           *    generator already guarantees.
-           * 4. Compose expressions to avoid combinatorial explosions in the
-           *    number of terms.
-           *
-           * The ordering of generators/filters, from narrow to broad, is:
-           * - since          = O(changes)
-           * - glob / dirname = O(files in a subtree of the repo)
-           * - suffix         = O(files in the repo)
-           *
-           * We assume that file extensions are ~uniformly distributed in the
-           * repo but Haste map projects are focused on a handful of
-           * directories. Therefore `glob` < `suffix`.
-           */
-          let queryGenerator: ?string;
-          if (since != null) {
-            // Use the `since` generator and filter by both path and extension.
-            query.since = since;
-            queryGenerator = 'since';
-            query.expression.push(
-              ['anyof', ...directoryFilters.map(dir => ['dirname', dir])],
-              suffixExpression,
-            );
-          } else if (directoryFilters.length > 0) {
-            // Use the `glob` generator and filter only by extension.
-            query.glob = directoryFilters.map(directory => `${directory}/**`);
-            query.glob_includedotfiles = true;
-            queryGenerator = 'glob';
-
-            query.expression.push(suffixExpression);
-          } else {
-            // Use the `suffix` generator with no path/extension filtering.
-            query.suffix = extensions;
-            queryGenerator = 'suffix';
-          }
+          const {query, queryGenerator} = planQuery({
+            since,
+            extensions,
+            directoryFilters,
+            includeSha1: computeSha1,
+            includeSymlinks: enableSymlinks,
+          });
 
           perfLogger?.annotate({
             string: {
@@ -392,6 +265,10 @@ module.exports = async function watchmanCrawl(
     );
 
     for (const fileData of response.files) {
+      if (fileData.symlink_target != null) {
+        // TODO: Process symlinks
+        continue;
+      }
       const filePath = fsRoot + path.sep + normalizePathSep(fileData.name);
       const relativeFilePath = fastPath.relative(rootDir, filePath);
       const existingFileData = data.files.get(relativeFilePath);
@@ -414,11 +291,13 @@ module.exports = async function watchmanCrawl(
           }
         }
       } else if (!ignore(filePath)) {
+        const {mtime_ms, size} = fileData;
+        invariant(
+          mtime_ms != null && size != null,
+          'missing file data in watchman response',
+        );
         const mtime =
-          typeof fileData.mtime_ms === 'number'
-            ? fileData.mtime_ms
-            : fileData.mtime_ms.toNumber();
-        const size = fileData.size;
+          typeof mtime_ms === 'number' ? mtime_ms : mtime_ms.toNumber();
 
         let sha1hex = fileData['content.sha1hex'];
         if (typeof sha1hex !== 'string' || sha1hex.length !== 40) {