@nomicfoundation/hardhat-utils 3.0.0-next.0 → 3.0.0-next.10

package/src/fs.ts

@@ -1,7 +1,15 @@
+import type { JsonTypes, ParsedElementInfo } from "@streamparser/json-node";
+import type { FileHandle } from "node:fs/promises";
+
 import fsPromises from "node:fs/promises";
+import { tmpdir } from "node:os";
 import path from "node:path";
+import { pipeline } from "node:stream/promises";
+
+import { JSONParser } from "@streamparser/json-node";
+import { JsonStreamStringify } from "json-stream-stringify";
 
-import { ensureError } from "./error.js";
+import { ensureError, ensureNodeErrnoExceptionError } from "./error.js";
 import {
   FileNotFoundError,
   FileSystemAccessError,
@@ -24,7 +32,7 @@ export async function getRealPath(absolutePath: string): Promise<string> {
   try {
     return await fsPromises.realpath(path.normalize(absolutePath));
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePath, e);
     }
@@ -39,6 +47,7 @@ export async function getRealPath(absolutePath: string): Promise<string> {
  *
  * @param dirFrom The absolute path of the directory to start the search from.
  * @param matches A function to filter files (not directories).
+ * @param directoryFilter A function to filter which directories to recurse into
  * @returns An array of absolute paths. Each file has its true case, except
  *  for the initial dirFrom part, which preserves the given casing.
  *  No order is guaranteed. If dirFrom doesn't exist `[]` is returned.
@@ -48,6 +57,7 @@ export async function getRealPath(absolutePath: string): Promise<string> {
 export async function getAllFilesMatching(
   dirFrom: string,
   matches?: (absolutePathToFile: string) => boolean,
+  directoryFilter?: (absolutePathToDir: string) => boolean,
 ): Promise<string[]> {
   const dirContent = await readdirOrEmpty(dirFrom);
 
@@ -55,7 +65,18 @@ export async function getAllFilesMatching(
     dirContent.map(async (file) => {
       const absolutePathToFile = path.join(dirFrom, file);
       if (await isDirectory(absolutePathToFile)) {
-        return getAllFilesMatching(absolutePathToFile, matches);
+        if (
+          directoryFilter === undefined ||
+          directoryFilter(absolutePathToFile)
+        ) {
+          return getAllFilesMatching(
+            absolutePathToFile,
+            matches,
+            directoryFilter,
+          );
+        }
+
+        return [];
       } else if (matches === undefined || matches(absolutePathToFile)) {
         return absolutePathToFile;
       } else {
@@ -158,7 +179,7 @@ export async function isDirectory(absolutePath: string): Promise<boolean> {
   try {
     return (await fsPromises.lstat(absolutePath)).isDirectory();
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePath, e);
     }
@@ -182,8 +203,81 @@ export async function readJsonFile<T>(absolutePathToFile: string): Promise<T> {
   try {
     return JSON.parse(content.toString());
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureError(e);
+    throw new InvalidFileFormatError(absolutePathToFile, e);
+  }
+}
+
+/**
+ * Reads a JSON file as a stream and parses it. The encoding used is "utf8".
+ * This function should be used when parsing very large JSON files.
+ *
+ * @param absolutePathToFile The path to the file.
+ * @returns The parsed JSON object.
+ * @throws FileNotFoundError if the file doesn't exist.
+ * @throws InvalidFileFormatError if the file is not a valid JSON file.
+ * @throws IsDirectoryError if the path is a directory instead of a file.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function readJsonFileAsStream<T>(
+  absolutePathToFile: string,
+): Promise<T> {
+  let fileHandle: FileHandle | undefined;
+
+  try {
+    fileHandle = await fsPromises.open(absolutePathToFile, "r");
+
+    const fileReadStream = fileHandle.createReadStream();
+
+    // NOTE: We set a separator to disable self-closing to be able to use the parser
+    // in the stream.pipeline context; see https://github.com/juanjoDiaz/streamparser-json/issues/47
+    const jsonParser = new JSONParser({
+      separator: "",
+    });
+
+    const result: T | undefined = await pipeline(
+      fileReadStream,
+      jsonParser,
+      async (
+        elements: AsyncIterable<ParsedElementInfo.ParsedElementInfo>,
+      ): Promise<any | undefined> => {
+        let value: JsonTypes.JsonPrimitive | JsonTypes.JsonStruct | undefined;
+        for await (const element of elements) {
+          value = element.value;
+        }
+        return value;
+      },
+    );
+
+    if (result === undefined) {
+      throw new Error("No data");
+    }
+
+    return result;
+  } catch (e) {
+    ensureError(e);
+
+    // If the code is defined, we assume the error to be related to the file system
+    if ("code" in e) {
+      if (e.code === "ENOENT") {
+        throw new FileNotFoundError(absolutePathToFile, e);
+      }
+
+      if (e.code === "EISDIR") {
+        throw new IsDirectoryError(absolutePathToFile, e);
+      }
+
+      // If the code is defined, we assume the error to be related to the file system
+      if (e.code !== undefined) {
+        throw new FileSystemAccessError(absolutePathToFile, e);
+      }
+    }
+
+    // Otherwise, we assume the error to be related to the file formatting
     throw new InvalidFileFormatError(absolutePathToFile, e);
+  } finally {
+    // Explicitly closing the file handle to fully release the underlying resources
+    await fileHandle?.close();
   }
 }
 
@@ -204,13 +298,66 @@ export async function writeJsonFile<T>(
   try {
     content = JSON.stringify(object, null, 2);
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureError(e);
     throw new JsonSerializationError(absolutePathToFile, e);
   }
 
   await writeUtf8File(absolutePathToFile, content);
 }
 
+/**
+ * Writes an object to a JSON file as stream. The encoding used is "utf8" and the file is overwritten.
+ * If part of the path doesn't exist, it will be created.
+ * This function should be used when stringifying very large JSON objects.
+ *
+ * @param absolutePathToFile The path to the file. If the file exists, it will be overwritten.
+ * @param object The object to write.
+ * @throws JsonSerializationError if the object can't be serialized to JSON.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function writeJsonFileAsStream<T>(
+  absolutePathToFile: string,
+  object: T,
+): Promise<void> {
+  const dirPath = path.dirname(absolutePathToFile);
+  const dirExists = await exists(dirPath);
+  if (!dirExists) {
+    await mkdir(dirPath);
+  }
+
+  let fileHandle: FileHandle | undefined;
+
+  try {
+    fileHandle = await fsPromises.open(absolutePathToFile, "w");
+
+    const jsonStream = new JsonStreamStringify(object);
+    const fileWriteStream = fileHandle.createWriteStream();
+
+    await pipeline(jsonStream, fileWriteStream);
+  } catch (e) {
+    ensureError(e);
+    // if the directory was created, we should remove it
+    if (dirExists === false) {
+      try {
+        await remove(dirPath);
+        // we don't want to override the original error
+      } catch (_error) {}
+    }
+
+    // If the code is defined, we assume the error to be related to the file system
+    if ("code" in e && e.code !== undefined) {
+      throw new FileSystemAccessError(e.message, e);
+    }
+
+    // Otherwise, we assume the error to be related to the file formatting
+    throw new JsonSerializationError(absolutePathToFile, e);
+  } finally {
+    // NOTE: Historically, not closing the file handle caused issues on Windows,
+    // for example, when trying to move the file previously written to by this function
+    await fileHandle?.close();
+  }
+}
+
 /**
  * Reads a file and returns its content as a string. The encoding used is "utf8".
  *
@@ -226,7 +373,7 @@ export async function readUtf8File(
   try {
     return await fsPromises.readFile(absolutePathToFile, { encoding: "utf8" });
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
 
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePathToFile, e);
@@ -268,13 +415,13 @@ export async function writeUtf8File(
       flag,
     });
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     // if the directory was created, we should remove it
     if (dirExists === false) {
       try {
         await remove(dirPath);
         // we don't want to override the original error
-      } catch (err) {}
+      } catch (_error) {}
     }
 
     if (e.code === "ENOENT") {
@@ -306,7 +453,7 @@ export async function readBinaryFile(
     const buffer = await fsPromises.readFile(absolutePathToFile);
     return new Uint8Array(buffer);
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
 
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePathToFile, e);
@@ -333,7 +480,7 @@ export async function readdir(absolutePathToDir: string): Promise<string[]> {
   try {
     return await fsPromises.readdir(absolutePathToDir);
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePathToDir, e);
     }
@@ -373,7 +520,7 @@ export async function mkdir(absolutePath: string): Promise<void> {
   try {
     await fsPromises.mkdir(absolutePath, { recursive: true });
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     throw new FileSystemAccessError(e.message, e);
   }
 }
@@ -384,6 +531,22 @@ export async function mkdir(absolutePath: string): Promise<void> {
  */
 export const ensureDir: typeof mkdir = mkdir;
 
+/**
+ * Creates a temporary directory with the specified prefix.
+ *
+ * @param prefix The prefix to use for the temporary directory.
+ * @returns The absolute path to the created temporary directory.
+ * @throws FileSystemAccessError for any error.
+ */
+export async function mkdtemp(prefix: string): Promise<string> {
+  try {
+    return await fsPromises.mkdtemp(path.join(tmpdir(), prefix));
+  } catch (e) {
+    ensureNodeErrnoExceptionError(e);
+    throw new FileSystemAccessError(e.message, e);
+  }
+}
+
 /**
  * Retrieves the last change time of a file or directory's properties.
  * This includes changes to the file's metadata or contents.
@@ -398,7 +561,7 @@ export async function getChangeTime(absolutePath: string): Promise<Date> {
     const stats = await fsPromises.stat(absolutePath);
     return stats.ctime;
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePath, e);
     }
@@ -420,7 +583,7 @@ export async function getAccessTime(absolutePath: string): Promise<Date> {
     const stats = await fsPromises.stat(absolutePath);
     return stats.atime;
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePath, e);
     }
@@ -442,7 +605,7 @@ export async function getFileSize(absolutePath: string): Promise<number> {
     const stats = await fsPromises.stat(absolutePath);
     return stats.size;
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePath, e);
     }
@@ -461,7 +624,7 @@ export async function exists(absolutePath: string): Promise<boolean> {
   try {
     await fsPromises.access(absolutePath);
     return true;
-  } catch (e) {
+  } catch (_error) {
     return false;
   }
 }
@@ -480,7 +643,7 @@ export async function copy(source: string, destination: string): Promise<void> {
   try {
     await fsPromises.copyFile(source, destination);
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       if (!(await exists(source))) {
         throw new FileNotFoundError(source, e);
@@ -524,7 +687,7 @@ export async function move(source: string, destination: string): Promise<void> {
   try {
     await fsPromises.rename(source, destination);
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       if (!(await exists(source))) {
         throw new FileNotFoundError(source, e);
@@ -537,7 +700,9 @@ export async function move(source: string, destination: string): Promise<void> {
     // On linux, trying to move a non-empty directory will throw ENOTEMPTY,
     // while on Windows it will throw EPERM.
     if (e.code === "ENOTEMPTY" || e.code === "EPERM") {
-      throw new DirectoryNotEmptyError(destination, e);
+      if (await isDirectory(source)) {
+        throw new DirectoryNotEmptyError(destination, e);
+      }
     }
 
     throw new FileSystemAccessError(e.message, e);
@@ -557,9 +722,10 @@ export async function remove(absolutePath: string): Promise<void> {
       recursive: true,
       force: true,
       maxRetries: 3,
+      retryDelay: 300,
     });
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     throw new FileSystemAccessError(e.message, e);
   }
 }
@@ -579,7 +745,7 @@ export async function chmod(
   try {
     await fsPromises.chmod(absolutePath, mode);
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       throw new FileNotFoundError(absolutePath, e);
     }
@@ -616,7 +782,7 @@ export async function emptyDir(absolutePath: string): Promise<void> {
     isDir = stats.isDirectory();
     mode = stats.mode;
   } catch (e) {
-    ensureError<NodeJS.ErrnoException>(e);
+    ensureNodeErrnoExceptionError(e);
     if (e.code === "ENOENT") {
       await mkdir(absolutePath);
       return;
@@ -631,7 +797,7 @@ export async function emptyDir(absolutePath: string): Promise<void> {
 
   await remove(absolutePath);
   await mkdir(absolutePath);
-  // eslint-disable-next-line no-bitwise -- Bitwise as common in fs permissions
+  // eslint-disable-next-line no-bitwise -- Bitwise is common in fs permissions
   await chmod(absolutePath, mode & 0o777);
 }
 

package/src/global-dir.ts

@@ -1,7 +1,6 @@
+import { ensureDir } from "./fs.js";
 import { generatePaths, HARDHAT_PACKAGE_NAME } from "./internal/global-dir.js";
 
-import { ensureDir } from "@nomicfoundation/hardhat-utils/fs";
-
 /**
  * Returns the configuration directory path for a given package (defaults to "hardhat").
  * Ensures that the directory exists before returning the path.

package/src/internal/lang.ts

@@ -1,5 +1,7 @@
 import type rfdcT from "rfdc";
 
+import { isObject } from "../lang.js";
+
 let clone: ReturnType<typeof rfdcT> | null = null;
 export async function getDeepCloneFunction(): Promise<<T>(input: T) => T> {
   const { default: rfdc } = await import("rfdc");
@@ -10,3 +12,39 @@ export async function getDeepCloneFunction(): Promise<<T>(input: T) => T> {
 
   return clone;
 }
+
+export function deepMergeImpl<T extends object, U extends object>(
+  target: T,
+  source: U,
+): T & U {
+  /* eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+  -- The result is expected to include properties from both target and source,
+  but initially only target is spread in, so a cast is needed. */
+  const result = { ...target } as T & U;
+
+  /*  eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+  -- TypeScript cannot infer the correct union of string and symbol keys, but all keys come from U */
+  const keys = [
+    ...Object.keys(source),
+    ...Object.getOwnPropertySymbols(source),
+  ] as Array<keyof U>;
+
+  for (const key of keys) {
+    if (
+      isObject(source[key]) &&
+      // Only merge recursively objects that are not class instances
+      Object.getPrototypeOf(source[key]) === Object.prototype
+    ) {
+      /* eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      -- The call signature expects the second argument to be of type U; the type is correct, but TypeScript can't infer it here. */
+      result[key] = deepMergeImpl(result[key] ?? {}, source[key] as U) as (T &
+        U)[Extract<keyof U, string>];
+    } else {
+      /* eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      --  Cast required because TypeScript can't guarantee that a dynamic key from `U` exists in `T & U` or has the correct value type. */
+      result[key] = source[key] as (T & U)[Extract<keyof U, string>];
+    }
+  }
+
+  return result;
+}

package/src/lang.ts

@@ -1,4 +1,4 @@
-import { getDeepCloneFunction } from "./internal/lang.js";
+import { deepMergeImpl, getDeepCloneFunction } from "./internal/lang.js";
 
 /**
  * Creates a deep clone of the provided value.
@@ -25,6 +25,31 @@ export async function deepEqual<T>(x: T, y: T): Promise<boolean> {
   return _deepEqual(x, y);
 }
 
+/**
+ * Deeply merges two objects.
+ *
+ * @remarks
+ * - Arrays or `undefined` values are not valid inputs.
+ * - Functions: If a function exists in both the target and source, the source function overwrites the target.
+ * - Symbol properties: Symbol-keyed properties are merged just like string keys.
+ * - Class instances: Class instances are not merged recursively. If a class instance exists in the source, it will replace the one in the target.
+ *
+ * @param target The target object to merge into.
+ * @param source The source object to merge from.
+ * @returns A new object containing the deeply merged properties.
+ *
+ * @example
+ * deepMerge({ a: { b: 1 } }, { a: { c: 2 } }) // => { a: { b: 1, c: 2 } }
+ *
+ * deepMerge({ a: { fn: () => "from target" } }, { a: { fn: () => "from source" } }) // => { a: { fn: () => "from source" } }
+ */
+export function deepMerge<T extends object, U extends object>(
+  target: T,
+  source: U,
+): T & U {
+  return deepMergeImpl(target, source);
+}
+
 /**
  * Checks if a value is an object. This function returns false for arrays.
  *

package/src/request.ts

@@ -23,7 +23,7 @@ import {
   handleError,
 } from "./internal/request.js";
 
-export const DEFAULT_TIMEOUT_IN_MILLISECONDS = 30_000;
+export const DEFAULT_TIMEOUT_IN_MILLISECONDS = 300_000; // Aligned with unidici
 export const DEFAULT_MAX_REDIRECTS = 10;
 export const DEFAULT_POOL_MAX_CONNECTIONS = 128;
 export const DEFAULT_USER_AGENT = "Hardhat";

package/src/synchronization.ts

@@ -4,8 +4,8 @@
 // ATTENTION: in the current implementation, there's still a risk of two processes running simultaneously.
 // For example, if processA has locked the mutex and is running, processB will wait.
 // During this wait, processB continuously checks the elapsed time since the mutex lock file was created.
-// If an excessive amount of time has passed, processB will assume ownership of the mutex to avoid stale locks.
-// However, there's a possibility that processB might take ownership because the mutex creation file is outdated, even though processA is still running
+// If an excessive amount of time has passed, processB will assume ownership of the mutex to prevent stale locks, even if processA is still running.
+// As a result, two processes will be running simultaneously in what is theoretically a mutex-locked section.
 
 import fs from "node:fs";
 import os from "node:os";
@@ -13,7 +13,7 @@ import path from "node:path";
 
 import debug from "debug";
 
-import { ensureError } from "./error.js";
+import { ensureNodeErrnoExceptionError } from "./error.js";
 import { FileSystemAccessError } from "./errors/fs.js";
 import { sleep } from "./lang.js";
 
@@ -62,7 +62,7 @@ export class MultiProcessMutex {
       fs.writeFileSync(this.#mutexFilePath, "", { flag: "wx+" });
       return true;
     } catch (e) {
-      ensureError<NodeJS.ErrnoException>(e);
+      ensureNodeErrnoExceptionError(e);
 
       if (e.code === "EEXIST") {
         // File already exists, so the mutex is already acquired
@@ -91,7 +91,7 @@ export class MultiProcessMutex {
     try {
       fileStat = fs.statSync(this.#mutexFilePath);
     } catch (e) {
-      ensureError<NodeJS.ErrnoException>(e);
+      ensureNodeErrnoExceptionError(e);
 
       if (e.code === "ENOENT") {
         // The file might have been deleted by another process while this function was trying to access it.
@@ -113,7 +113,7 @@ export class MultiProcessMutex {
       log(`Deleting mutex file at path '${this.#mutexFilePath}'`);
       fs.unlinkSync(this.#mutexFilePath);
     } catch (e) {
-      ensureError<NodeJS.ErrnoException>(e);
+      ensureNodeErrnoExceptionError(e);
 
       if (e.code === "ENOENT") {
         // The file might have been deleted by another process while this function was trying to access it.