metro-file-map 0.81.1 → 0.81.2

package/package.json

@@ -1,8 +1,16 @@
 {
   "name": "metro-file-map",
-  "version": "0.81.1",
+  "version": "0.81.2",
   "description": "[Experimental] - 🚇 File crawling, watching and mapping for Metro",
   "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./package.json": "./package.json",
+    "./private/*": "./src/*.js",
+    "./src": "./src/index.js",
+    "./src/*.js": "./src/*.js",
+    "./src/*": "./src/*.js"
+  },
   "repository": {
     "type": "git",
     "url": "git@github.com:facebook/metro.git"
@@ -18,7 +26,7 @@
     "flow-enums-runtime": "^0.0.6",
     "graceful-fs": "^4.2.4",
     "invariant": "^2.2.4",
-    "jest-worker": "^29.6.3",
+    "jest-worker": "^29.7.0",
     "micromatch": "^4.0.4",
     "nullthrows": "^1.1.1",
     "walker": "^1.0.7"

package/src/cache/DiskCacheManager.d.ts

@@ -12,7 +12,7 @@ import type {
   BuildParameters,
   CacheData,
   CacheManager,
-  FileData,
+  CacheManagerWriteOptions,
 } from '../flow-types';
 
 export interface DiskCacheConfig {
@@ -31,7 +31,8 @@ export class DiskCacheManager implements CacheManager {
   getCacheFilePath(): string;
   read(): Promise<CacheData | null>;
   write(
-    dataSnapshot: CacheData,
-    {changed, removed}: Readonly<{changed: FileData; removed: FileData}>,
+    getSnapshot: () => CacheData,
+    opts: CacheManagerWriteOptions,
   ): Promise<void>;
+  end(): Promise<void>;
 }

package/src/cache/DiskCacheManager.js

@@ -7,22 +7,42 @@ exports.DiskCacheManager = void 0;
 var _rootRelativeCacheKeys = _interopRequireDefault(
   require("../lib/rootRelativeCacheKeys")
 );
-var _gracefulFs = require("graceful-fs");
+var _fs = require("fs");
 var _os = require("os");
 var _path = _interopRequireDefault(require("path"));
+var _timers = require("timers");
 var _v = require("v8");
 function _interopRequireDefault(e) {
   return e && e.__esModule ? e : { default: e };
 }
+const debug = require("debug")("Metro:FileMapCache");
 const DEFAULT_PREFIX = "metro-file-map";
 const DEFAULT_DIRECTORY = (0, _os.tmpdir)();
+const DEFAULT_AUTO_SAVE_DEBOUNCE_MS = 5000;
 class DiskCacheManager {
-  constructor({ buildParameters, cacheDirectory, cacheFilePrefix }) {
-    this._cachePath = DiskCacheManager.getCacheFilePath(
+  #autoSaveOpts;
+  #cachePath;
+  #debounceTimeout = null;
+  #writePromise = Promise.resolve();
+  #hasUnwrittenChanges = false;
+  #tryWrite;
+  #stopListening;
+  constructor(
+    { buildParameters },
+    { autoSave = {}, cacheDirectory, cacheFilePrefix }
+  ) {
+    this.#cachePath = DiskCacheManager.getCacheFilePath(
       buildParameters,
       cacheFilePrefix,
       cacheDirectory
     );
+    if (autoSave) {
+      const { debounceMs = DEFAULT_AUTO_SAVE_DEBOUNCE_MS } =
+        autoSave === true ? {} : autoSave;
+      this.#autoSaveOpts = {
+        debounceMs,
+      };
+    }
   }
   static getCacheFilePath(buildParameters, cacheFilePrefix, cacheDirectory) {
     const { rootDirHash, relativeConfigHash } = (0,
@@ -35,13 +55,11 @@ class DiskCacheManager {
     );
   }
   getCacheFilePath() {
-    return this._cachePath;
+    return this.#cachePath;
   }
   async read() {
     try {
-      return (0, _v.deserialize)(
-        (0, _gracefulFs.readFileSync)(this._cachePath)
-      );
+      return (0, _v.deserialize)(await _fs.promises.readFile(this.#cachePath));
     } catch (e) {
       if (e?.code === "ENOENT") {
         return null;
@@ -49,13 +67,53 @@ class DiskCacheManager {
       throw e;
     }
   }
-  async write(dataSnapshot, { changed, removed }) {
-    if (changed.size > 0 || removed.size > 0) {
-      (0, _gracefulFs.writeFileSync)(
-        this._cachePath,
-        (0, _v.serialize)(dataSnapshot)
-      );
+  async write(
+    getSnapshot,
+    { changedSinceCacheRead, eventSource, onWriteError }
+  ) {
+    const tryWrite = (this.#tryWrite = () => {
+      this.#writePromise = this.#writePromise
+        .then(async () => {
+          if (!this.#hasUnwrittenChanges) {
+            return;
+          }
+          const data = getSnapshot();
+          this.#hasUnwrittenChanges = false;
+          await _fs.promises.writeFile(
+            this.#cachePath,
+            (0, _v.serialize)(data)
+          );
+          debug("Written cache to %s", this.#cachePath);
+        })
+        .catch(onWriteError);
+      return this.#writePromise;
+    });
+    if (this.#autoSaveOpts) {
+      const autoSave = this.#autoSaveOpts;
+      this.#stopListening?.();
+      this.#stopListening = eventSource.onChange(() => {
+        this.#hasUnwrittenChanges = true;
+        if (this.#debounceTimeout) {
+          this.#debounceTimeout.refresh();
+        } else {
+          this.#debounceTimeout = (0, _timers.setTimeout)(
+            () => tryWrite(),
+            autoSave.debounceMs
+          ).unref();
+        }
+      });
+    }
+    if (changedSinceCacheRead) {
+      this.#hasUnwrittenChanges = true;
+      await tryWrite();
+    }
+  }
+  async end() {
+    if (this.#debounceTimeout) {
+      (0, _timers.clearTimeout)(this.#debounceTimeout);
     }
+    this.#stopListening?.();
+    await this.#tryWrite?.();
   }
 }
 exports.DiskCacheManager = DiskCacheManager;

package/src/cache/DiskCacheManager.js.flow

@@ -12,38 +12,59 @@
 import type {
   BuildParameters,
   CacheData,
-  CacheDelta,
   CacheManager,
+  CacheManagerFactoryOptions,
+  CacheManagerWriteOptions,
 } from '../flow-types';
 
 import rootRelativeCacheKeys from '../lib/rootRelativeCacheKeys';
-import {readFileSync, writeFileSync} from 'graceful-fs';
+import {promises as fsPromises} from 'fs';
 import {tmpdir} from 'os';
 import path from 'path';
+import {Timeout, clearTimeout, setTimeout} from 'timers';
 import {deserialize, serialize} from 'v8';
 
+const debug = require('debug')('Metro:FileMapCache');
+
+type AutoSaveOptions = $ReadOnly<{
+  debounceMs: number,
+}>;
+
 type DiskCacheConfig = {
-  buildParameters: BuildParameters,
+  autoSave?: Partial<AutoSaveOptions> | boolean,
   cacheFilePrefix?: ?string,
   cacheDirectory?: ?string,
 };
 
 const DEFAULT_PREFIX = 'metro-file-map';
 const DEFAULT_DIRECTORY = tmpdir();
+const DEFAULT_AUTO_SAVE_DEBOUNCE_MS = 5000;
 
 export class DiskCacheManager implements CacheManager {
-  _cachePath: string;
-
-  constructor({
-    buildParameters,
-    cacheDirectory,
-    cacheFilePrefix,
-  }: DiskCacheConfig) {
-    this._cachePath = DiskCacheManager.getCacheFilePath(
+  +#autoSaveOpts: ?AutoSaveOptions;
+  +#cachePath: string;
+  #debounceTimeout: ?Timeout = null;
+  #writePromise: Promise<void> = Promise.resolve();
+  #hasUnwrittenChanges: boolean = false;
+  #tryWrite: ?() => Promise<void>;
+  #stopListening: ?() => void;
+
+  constructor(
+    {buildParameters}: CacheManagerFactoryOptions,
+    {autoSave = {}, cacheDirectory, cacheFilePrefix}: DiskCacheConfig,
+  ) {
+    this.#cachePath = DiskCacheManager.getCacheFilePath(
       buildParameters,
       cacheFilePrefix,
       cacheDirectory,
     );
+
+    // Normalise auto-save options.
+    if (autoSave) {
+      const {debounceMs = DEFAULT_AUTO_SAVE_DEBOUNCE_MS} =
+        autoSave === true ? {} : autoSave;
+      this.#autoSaveOpts = {debounceMs};
+    }
   }
 
   static getCacheFilePath(
@@ -63,12 +84,12 @@ export class DiskCacheManager implements CacheManager {
   }
 
   getCacheFilePath(): string {
-    return this._cachePath;
+    return this.#cachePath;
   }
 
   async read(): Promise<?CacheData> {
     try {
-      return deserialize(readFileSync(this._cachePath));
+      return deserialize(await fsPromises.readFile(this.#cachePath));
     } catch (e) {
       if (e?.code === 'ENOENT') {
         // Cache file not found - not considered an error.
@@ -80,11 +101,64 @@ export class DiskCacheManager implements CacheManager {
   }
 
   async write(
-    dataSnapshot: CacheData,
-    {changed, removed}: CacheDelta,
+    getSnapshot: () => CacheData,
+    {
+      changedSinceCacheRead,
+      eventSource,
+      onWriteError,
+    }: CacheManagerWriteOptions,
   ): Promise<void> {
-    if (changed.size > 0 || removed.size > 0) {
-      writeFileSync(this._cachePath, serialize(dataSnapshot));
+    // Initialise a writer function using a promise queue to ensure writes are
+    // sequenced.
+    const tryWrite = (this.#tryWrite = () => {
+      this.#writePromise = this.#writePromise
+        .then(async () => {
+          if (!this.#hasUnwrittenChanges) {
+            return;
+          }
+          const data = getSnapshot();
+          this.#hasUnwrittenChanges = false;
+          await fsPromises.writeFile(this.#cachePath, serialize(data));
+          debug('Written cache to %s', this.#cachePath);
+        })
+        .catch(onWriteError);
+      return this.#writePromise;
+    });
+
+    // Set up auto-save on changes, if enabled.
+    if (this.#autoSaveOpts) {
+      const autoSave = this.#autoSaveOpts;
+      this.#stopListening?.();
+      this.#stopListening = eventSource.onChange(() => {
+        this.#hasUnwrittenChanges = true;
+        if (this.#debounceTimeout) {
+          this.#debounceTimeout.refresh();
+        } else {
+          this.#debounceTimeout = setTimeout(
+            () => tryWrite(),
+            autoSave.debounceMs,
+          ).unref();
+        }
+      });
     }
+
+    // Write immediately if state has changed since the cache was read.
+    if (changedSinceCacheRead) {
+      this.#hasUnwrittenChanges = true;
+      await tryWrite();
+    }
+  }
+
+  async end() {
+    // Clear any timers
+    if (this.#debounceTimeout) {
+      clearTimeout(this.#debounceTimeout);
+    }
+
+    // Remove event listeners
+    this.#stopListening?.();
+
+    // Flush unwritten changes to disk (no-op if no changes)
+    await this.#tryWrite?.();
   }
 }

package/src/flow-types.d.ts

@@ -42,22 +42,55 @@ export interface BuildResult {
 
 export interface CacheData {
   readonly clocks: WatchmanClocks;
-  readonly mocks: RawMockMap;
-  readonly files: FileData;
+  readonly fileSystemData: unknown;
+  readonly plugins: ReadonlyMap<string, unknown>;
 }
 
 export interface CacheManager {
+  /**
+   * Called during startup to load initial state, if available. Provided to
+   * a crawler, which will return the delta between the initial state and the
+   * current file system state.
+   */
   read(): Promise<CacheData | null>;
+
+  /**
+   * Called when metro-file-map `build()` has applied changes returned by the
+   * crawler - i.e. internal state reflects the current file system state.
+   *
+   * getSnapshot may be retained and called at any time before end(), such as
+   * in response to eventSource 'change' events.
+   */
   write(
-    dataSnapshot: CacheData,
-    delta: Readonly<{changed: FileData; removed: FileData}>,
+    getSnapshot: () => CacheData,
+    opts: CacheManagerWriteOptions,
   ): Promise<void>;
+
+  /**
+   * The last call that will be made to this CacheManager. Any handles should
+   * be closed by the time this settles.
+   */
+  end(): Promise<void>;
+}
+
+export interface CacheManagerEventSource {
+  onChange(listener: () => void): () => void /* unsubscribe */;
 }
 
 export type CacheManagerFactory = (
-  buildParameters: BuildParameters,
+  options: CacheManagerFactoryOptions,
 ) => CacheManager;
 
+export type CacheManagerFactoryOptions = {
+  buildParameters: BuildParameters;
+};
+
+export type CacheManagerWriteOptions = {
+  changedSinceCacheRead: boolean;
+  eventSource: CacheManagerEventSource;
+  onWriteError: (e: Error) => void;
+};
+
 export interface ChangeEvent {
   logger: RootPerfLogger | null;
   eventsQueue: EventsQueue;

package/src/flow-types.js.flow

@@ -47,24 +47,55 @@ export type BuildResult = {
 
 export type CacheData = $ReadOnly<{
   clocks: WatchmanClocks,
-  mocks: ?RawMockMap,
   fileSystemData: mixed,
-}>;
-
-export type CacheDelta = $ReadOnly<{
-  changed: $ReadOnlyMap<CanonicalPath, FileMetaData>,
-  removed: $ReadOnlySet<CanonicalPath>,
+  plugins: $ReadOnlyMap<string, mixed>,
 }>;
 
 export interface CacheManager {
+  /**
+   * Called during startup to load initial state, if available. Provided to
+   * a crawler, which will return the delta between the initial state and the
+   * current file system state.
+   */
   read(): Promise<?CacheData>;
-  write(dataSnapshot: CacheData, delta: CacheDelta): Promise<void>;
+
+  /**
+   * Called when metro-file-map `build()` has applied changes returned by the
+   * crawler - i.e. internal state reflects the current file system state.
+   *
+   * getSnapshot may be retained and called at any time before end(), such as
+   * in response to eventSource 'change' events.
+   */
+  write(
+    getSnapshot: () => CacheData,
+    opts: CacheManagerWriteOptions,
+  ): Promise<void>;
+
+  /**
+   * The last call that will be made to this CacheManager. Any handles should
+   * be closed by the time this settles.
+   */
+  end(): Promise<void>;
+}
+
+export interface CacheManagerEventSource {
+  onChange(listener: () => void): () => void /* unsubscribe */;
 }
 
 export type CacheManagerFactory = (
-  buildParameters: BuildParameters,
+  options: CacheManagerFactoryOptions,
 ) => CacheManager;
 
+export type CacheManagerFactoryOptions = $ReadOnly<{
+  buildParameters: BuildParameters,
+}>;
+
+export type CacheManagerWriteOptions = $ReadOnly<{
+  changedSinceCacheRead: boolean,
+  eventSource: CacheManagerEventSource,
+  onWriteError: Error => void,
+}>;
+
 // A path that is
 //  - Relative to the contextual `rootDir`
 //  - Normalised (no extraneous '.' or '..')
@@ -128,6 +159,46 @@ export type EventsQueue = Array<{
   type: string,
 }>;
 
+export type FileMapDelta = $ReadOnly<{
+  removed: Iterable<[CanonicalPath, FileMetaData]>,
+  addedOrModified: Iterable<[CanonicalPath, FileMetaData]>,
+}>;
+
+interface FileSystemState {
+  metadataIterator(
+    opts: $ReadOnly<{
+      includeNodeModules: boolean,
+      includeSymlinks: boolean,
+    }>,
+  ): Iterable<{
+    baseName: string,
+    canonicalPath: string,
+    metadata: FileMetaData,
+  }>;
+}
+
+export type FileMapPluginInitOptions<SerializableState> = $ReadOnly<{
+  files: FileSystemState,
+  pluginState: ?SerializableState,
+}>;
+
+type V8Serializable = interface {};
+
+export interface FileMapPlugin<SerializableState = V8Serializable> {
+  +name: string;
+  initialize(
+    initOptions: FileMapPluginInitOptions<SerializableState>,
+  ): Promise<void>;
+  assertValid(): void;
+  bulkUpdate(delta: FileMapDelta): Promise<void>;
+  getSerializableSnapshot(): SerializableState;
+  onRemovedFile(relativeFilePath: string, fileMetadata: FileMetaData): void;
+  onNewOrModifiedFile(
+    relativeFilePath: string,
+    fileMetadata: FileMetaData,
+  ): void;
+}
+
 export type HType = {
   ID: 0,
   MTIME: 1,
@@ -177,6 +248,7 @@ export interface FileSystem {
   getModuleName(file: Path): ?string;
   getSerializableSnapshot(): CacheData['fileSystemData'];
   getSha1(file: Path): ?string;
+  getOrComputeSha1(file: Path): Promise<?{sha1: string, content?: Buffer}>;
 
   /**
    * Given a start path (which need not exist), a subpath and type, and
@@ -311,14 +383,28 @@ export interface MutableFileSystem extends FileSystem {
 
 export type Path = string;
 
+export type ProcessFileFunction = (
+  absolutePath: string,
+  metadata: FileMetaData,
+  request: $ReadOnly<{computeSha1: boolean}>,
+) => Promise<?Buffer>;
+
 export type RawMockMap = $ReadOnly<{
-  duplicates: Map<string, Set<string>>,
-  mocks: Map<string, Path>,
+  duplicates: Map<
+    string, // posix-separated mock name
+    Set<string>, // posix-separated, project-relative paths
+  >,
+  mocks: Map<
+    string, // posix-separated mock name
+    Path, // posix-separated, project-relative path
+  >,
+  version: number,
 }>;
 
 export type ReadOnlyRawMockMap = $ReadOnly<{
   duplicates: $ReadOnlyMap<string, $ReadOnlySet<string>>,
   mocks: $ReadOnlyMap<string, Path>,
+  version: number,
 }>;
 
 export interface WatcherBackend {
@@ -329,15 +415,22 @@ export interface WatcherBackend {
   stopWatching(): Promise<void>;
 }
 
+export type ChangeEventClock = [
+  string /* absolute watch root */,
+  string /* opaque clock */,
+];
+
 export type WatcherBackendChangeEvent =
   | $ReadOnly<{
       event: 'touch',
+      clock?: ChangeEventClock,
       relativePath: string,
       root: string,
       metadata: ChangeEventMetadata,
     }>
   | $ReadOnly<{
       event: 'delete',
+      clock?: ChangeEventClock,
       relativePath: string,
       root: string,
       metadata?: void,
@@ -353,14 +446,14 @@ export type WorkerMessage = $ReadOnly<{
   computeSha1: boolean,
   dependencyExtractor?: ?string,
   enableHastePackages: boolean,
-  rootDir: string,
   filePath: string,
   hasteImplModulePath?: ?string,
+  maybeReturnContent: boolean,
 }>;
 
 export type WorkerMetadata = $ReadOnly<{
   dependencies?: ?$ReadOnlyArray<string>,
   id?: ?string,
-  module?: ?HasteMapItemMetaData,
   sha1?: ?string,
+  content?: ?Buffer,
 }>;