hardhat 2.11.0-dev.0 → 2.11.0

package/src/internal/solidity/compiler/downloader.ts

@@ -1,21 +1,16 @@
-import chalk from "chalk";
-import debug from "debug";
+import path from "path";
 import fsExtra from "fs-extra";
+import debug from "debug";
 import os from "os";
-import path from "path";
-
-import { HardhatError } from "../../core/errors";
+import { execFile } from "child_process";
+import { download } from "../../util/download";
+import { assertHardhatInvariant, HardhatError } from "../../core/errors";
 import { ERRORS } from "../../core/errors-list";
+import { Mutex } from "../../vendor/await-semaphore";
 
-export interface CompilerBuild {
-  path: string;
-  version: string;
-  build: string;
-  longVersion: string;
-  keccak256: string;
-  urls: string[];
-  platform: CompilerPlatform;
-}
+const log = debug("hardhat:core:solidity:downloader");
+
+const COMPILER_REPOSITORY_URL = "https://binaries.soliditylang.org";
 
 export enum CompilerPlatform {
   LINUX = "linux-amd64",
@@ -24,304 +19,337 @@ export enum CompilerPlatform {
   WASM = "wasm",
 }
 
-interface CompilerPath {
-  compilerPath: string; // absolute path
+export interface Compiler {
+  version: string;
+  longVersion: string;
+  compilerPath: string;
+  isSolcJs: boolean;
+}
+
+interface CompilerBuild {
+  path: string;
+  version: string;
+  build: string;
+  longVersion: string;
+  keccak256: string;
+  urls: string[];
   platform: CompilerPlatform;
 }
 
-export interface CompilersList {
+interface CompilerList {
   builds: CompilerBuild[];
-  releases: {
-    [version: string]: string;
-  };
+  releases: { [version: string]: string };
   latestRelease: string;
 }
 
-const log = debug("hardhat:core:solidity:downloader");
-
-const COMPILER_FILES_DIR_URL_SOLC = "https://binaries.soliditylang.org/";
-
-async function downloadFile(
-  url: string,
-  destinationFile: string
-): Promise<void> {
-  const { download } = await import("../../util/download");
-  log(`Downloading from ${url} to ${destinationFile}`);
-  await download(url, destinationFile);
+/**
+ * A compiler downloader which must be specialized per-platform. It can't and
+ * shouldn't support multiple platforms at the same time.
+ */
+export interface ICompilerDownloader {
+  /**
+   * Returns true if the compiler has been downloaded.
+   *
+   * This function access the filesystem, but doesn't modify it.
+   */
+  isCompilerDownloaded(version: string): Promise<boolean>;
+
+  /**
+   * Downloads the compiler for a given version, which can later be obtained
+   * with getCompiler.
+   */
+  downloadCompiler(version: string): Promise<void>;
+
+  /**
+   * Returns the compiler, which MUST be downloaded before calling this function.
+   *
+   * Returns undefined if the compiler has been downloaded but can't be run.
+   *
+   * This function access the filesystem, but doesn't modify it.
+   */
+  getCompiler(version: string): Promise<Compiler | undefined>;
 }
 
-type CompilerDownloaderOptions = Partial<{
-  download: (url: string, destinationFile: string) => Promise<void>;
-  forceSolcJs: boolean;
-}>;
+/**
+ * Default implementation of ICompilerDownloader.
+ *
+ * Important things to note:
+ *   1. If a compiler version is not found, this downloader may fail.
+ *    1.1. It only re-downloads the list of compilers once every X time.
+ *      1.1.1 If a user tries to download a new compiler before X amount of time
+ *      has passed since its release, they may need to clean the cache, as
+ *      indicated in the error messages.
+ */
+export class CompilerDownloader implements ICompilerDownloader {
+  public static getCompilerPlatform(): CompilerPlatform {
+    switch (os.platform()) {
+      case "win32":
+        return CompilerPlatform.WINDOWS;
+      case "linux":
+        return CompilerPlatform.LINUX;
+      case "darwin":
+        return CompilerPlatform.MACOS;
+      default:
+        return CompilerPlatform.WASM;
+    }
+  }
 
-export class CompilerDownloader {
-  private readonly _download: (
-    url: string,
-    destinationFile: string
-  ) => Promise<void>;
-  private readonly _forceSolcJs: boolean;
+  private static _downloaderPerPlatform: Map<string, CompilerDownloader> =
+    new Map();
 
-  constructor(
-    private readonly _compilersDir: string,
-    options: CompilerDownloaderOptions = {}
+  public static getConcurrencySafeDownloader(
+    platform: CompilerPlatform,
+    compilersDir: string
   ) {
-    this._download = options.download ?? downloadFile;
-    this._forceSolcJs = options.forceSolcJs ?? false;
-  }
+    const key = platform + compilersDir;
 
-  public async isCompilerDownloaded(version: string): Promise<boolean> {
-    const compilerBuild = await this.getCompilerBuild(version);
-    const downloadedFilePath = this._getDownloadedFilePath(compilerBuild);
+    if (!this._downloaderPerPlatform.has(key)) {
+      this._downloaderPerPlatform.set(
+        key,
+        new CompilerDownloader(platform, compilersDir)
+      );
+    }
 
-    return this._fileExists(downloadedFilePath);
+    return this._downloaderPerPlatform.get(key)!;
   }
 
-  public async verifyCompiler(
-    compilerBuild: CompilerBuild,
-    downloadedFilePath: string
-  ) {
-    const ethereumjsUtil = await import("@ignored/util");
-    const { keccak256 } = await import("ethereum-cryptography/keccak");
-
-    const expectedKeccak256 = compilerBuild.keccak256;
-    const compiler = await fsExtra.readFile(downloadedFilePath);
+  public static defaultCompilerListCachePeriod = 3_600_00;
+  private readonly _mutex = new Mutex();
 
-    const compilerKeccak256 = ethereumjsUtil.bufferToHex(
-      ethereumjsUtil.arrToBufArr(
-        keccak256(ethereumjsUtil.bufArrToArr(compiler))
-      )
-    );
+  /**
+   * Use CompilerDownloader.getConcurrencySafeDownloader instead
+   */
+  constructor(
+    private readonly _platform: CompilerPlatform,
+    private readonly _compilersDir: string,
+    private readonly _compilerListCachePeriodMs = CompilerDownloader.defaultCompilerListCachePeriod,
+    private readonly _downloadFunction: typeof download = download
+  ) {}
 
-    if (expectedKeccak256 !== compilerKeccak256) {
-      await fsExtra.unlink(downloadedFilePath);
-      await fsExtra.unlink(this.getCompilersListPath(compilerBuild.platform));
+  public async isCompilerDownloaded(version: string): Promise<boolean> {
+    const build = await this._getCompilerBuild(version);
 
-      throw new HardhatError(ERRORS.SOLC.INVALID_DOWNLOAD, {
-        remoteVersion: compilerBuild.version,
-      });
+    if (build === undefined) {
+      return false;
     }
-  }
 
-  public async getDownloadedCompilerPath(
-    version: string
-  ): Promise<CompilerPath | undefined> {
-    const { default: AdmZip } = await import("adm-zip");
+    const downloadPath = this._getCompilerBinaryPathFromBuild(build);
 
-    try {
-      const compilerBuild = await this.getCompilerBuild(version);
+    return fsExtra.pathExists(downloadPath);
+  }
 
-      let downloadedFilePath = this._getDownloadedFilePath(compilerBuild);
+  public async downloadCompiler(version: string): Promise<void> {
+    await this._mutex.use(async () => {
+      let build = await this._getCompilerBuild(version);
+
+      if (build === undefined && (await this._shouldDownloadCompilerList())) {
+        try {
+          await this._downloadCompilerList();
+        } catch (e: any) {
+          throw new HardhatError(
+            ERRORS.SOLC.VERSION_LIST_DOWNLOAD_FAILED,
+            {},
+            e
+          );
+        }
 
-      if (!(await this._fileExists(downloadedFilePath))) {
-        await this.downloadCompiler(compilerBuild, downloadedFilePath);
+        build = await this._getCompilerBuild(version);
       }
 
-      await this.verifyCompiler(compilerBuild, downloadedFilePath);
-
-      if (
-        compilerBuild.platform === CompilerPlatform.LINUX ||
-        compilerBuild.platform === CompilerPlatform.MACOS
-      ) {
-        fsExtra.chmodSync(downloadedFilePath, 0o755);
-      } else if (compilerBuild.platform === CompilerPlatform.WINDOWS) {
-        // some window builds are zipped, some are not
-        if (downloadedFilePath.endsWith(".zip")) {
-          const zip = new AdmZip(downloadedFilePath);
-          zip.extractAllTo(
-            path.join(this._compilersDir, compilerBuild.version)
-          );
-          downloadedFilePath = path.join(
-            this._compilersDir,
-            compilerBuild.version,
-            "solc.exe"
-          );
-        }
+      if (build === undefined) {
+        throw new HardhatError(ERRORS.SOLC.INVALID_VERSION, { version });
       }
 
-      return {
-        compilerPath: downloadedFilePath,
-        platform: compilerBuild.platform,
-      };
-    } catch (e) {
-      if (e instanceof Error) {
-        if (HardhatError.isHardhatError(e)) {
-          throw e;
-        }
-        console.warn(
-          chalk.yellow(
-            `There was an unexpected problem downloading the compiler: ${e.message}`
-          )
+      let downloadPath: string;
+      try {
+        downloadPath = await this._downloadCompiler(build);
+      } catch (e: any) {
+        throw new HardhatError(
+          ERRORS.SOLC.DOWNLOAD_FAILED,
+          {
+            remoteVersion: build.longVersion,
+          },
+          e
         );
       }
-    }
-  }
 
-  public async getCompilersList(
-    platform: CompilerPlatform,
-    pendingRetries: number = 3
-  ): Promise<CompilersList> {
-    if (!(await this.compilersListExists(platform))) {
-      await this.downloadCompilersList(platform);
-    }
-
-    try {
-      return await fsExtra.readJSON(this.getCompilersListPath(platform));
-    } catch (error) {
-      // if parsing throws a syntax error, redownload and parse once more
-      if (!(error instanceof SyntaxError) || pendingRetries === 0) {
-        // eslint-disable-next-line @nomiclabs/hardhat-internal-rules/only-hardhat-error
-        throw error;
+      const verified = await this._verifyCompilerDownload(build, downloadPath);
+      if (!verified) {
+        throw new HardhatError(ERRORS.SOLC.INVALID_DOWNLOAD, {
+          remoteVersion: build.longVersion,
+        });
       }
 
-      // remove the malformed list and retry
-      await fsExtra.remove(this.getCompilersListPath(platform));
-      return this.getCompilersList(platform, pendingRetries - 1);
-    }
+      await this._postProcessCompilerDownload(build, downloadPath);
+    });
   }
 
-  public async getCompilerBuild(version: string): Promise<CompilerBuild> {
-    const platform = this._getCurrentPlatform();
+  public async getCompiler(version: string): Promise<Compiler | undefined> {
+    const build = await this._getCompilerBuild(version);
 
-    if (await this._versionExists(version, platform)) {
-      try {
-        return await this._getCompilerBuildByPlatform(version, platform);
-      } catch {
-        log("Couldn't download native compiler, using solcjs instead");
-      }
+    assertHardhatInvariant(
+      build !== undefined,
+      "Trying to get a compiler before it was downloaded"
+    );
+
+    const compilerPath = this._getCompilerBinaryPathFromBuild(build);
+
+    assertHardhatInvariant(
+      await fsExtra.pathExists(compilerPath),
+      "Trying to get a compiler before it was downloaded"
+    );
+
+    if (await fsExtra.pathExists(this._getCompilerDoesntWorkFile(build))) {
+      return undefined;
     }
 
-    return this._getCompilerBuildByPlatform(version, CompilerPlatform.WASM);
+    return {
+      version,
+      longVersion: build.longVersion,
+      compilerPath,
+      isSolcJs: this._platform === CompilerPlatform.WASM,
+    };
   }
 
-  public async downloadCompilersList(
-    platform: CompilerPlatform = this._getCurrentPlatform()
-  ) {
-    try {
-      await this._download(
-        getCompilerListURL(platform),
-        this.getCompilersListPath(platform)
-      );
-    } catch (error: any) {
-      throw new HardhatError(
-        ERRORS.SOLC.VERSION_LIST_DOWNLOAD_FAILED,
-        {},
-        error
-      );
+  private async _getCompilerBuild(
+    version: string
+  ): Promise<CompilerBuild | undefined> {
+    const listPath = this._getCompilerListPath();
+    if (!(await fsExtra.pathExists(listPath))) {
+      return undefined;
     }
+
+    const list = await this._readCompilerList(listPath);
+    return list.builds.find((b) => b.version === version);
   }
 
-  public async downloadCompiler(
-    compilerBuild: CompilerBuild,
-    downloadedFilePath: string
-  ) {
-    log(
-      `Downloading compiler version ${compilerBuild.version} platform ${compilerBuild.platform}`
-    );
+  private _getCompilerListPath(): string {
+    return path.join(this._compilersDir, this._platform, "list.json");
+  }
 
-    const compilerUrl = getCompilerURL(
-      compilerBuild.platform,
-      compilerBuild.path
-    );
+  private async _readCompilerList(listPath: string): Promise<CompilerList> {
+    return fsExtra.readJSON(listPath);
+  }
 
-    try {
-      await this._download(compilerUrl, downloadedFilePath);
-    } catch (error) {
-      throw new HardhatError(
-        ERRORS.SOLC.DOWNLOAD_FAILED,
-        {
-          remoteVersion: compilerBuild.version,
-        },
-        error as Error
-      );
-    }
+  private _getCompilerDownloadPathFromBuild(build: CompilerBuild): string {
+    return path.join(this._compilersDir, this._platform, build.path);
   }
 
-  public async compilersListExists(platform: CompilerPlatform) {
-    return fsExtra.pathExists(this.getCompilersListPath(platform));
+  private _getCompilerBinaryPathFromBuild(build: CompilerBuild): string {
+    const downloadPath = this._getCompilerDownloadPathFromBuild(build);
+
+    if (
+      this._platform !== CompilerPlatform.WINDOWS ||
+      !downloadPath.endsWith(".zip")
+    ) {
+      return downloadPath;
+    }
+
+    return path.join(this._compilersDir, build.version, "solc.exe");
   }
 
-  public getCompilersListPath(platform: CompilerPlatform) {
-    return path.join(this._compilersDir, platform, "list.json");
+  private _getCompilerDoesntWorkFile(build: CompilerBuild): string {
+    return `${this._getCompilerBinaryPathFromBuild(build)}.does.not.work`;
   }
 
-  private _getDownloadedFilePath(compilerBuild: CompilerBuild): string {
-    return path.join(
-      this._compilersDir,
-      compilerBuild.platform,
-      compilerBuild.path
-    );
+  private async _shouldDownloadCompilerList(): Promise<boolean> {
+    const listPath = this._getCompilerListPath();
+    if (!(await fsExtra.pathExists(listPath))) {
+      return true;
+    }
+
+    const stats = await fsExtra.stat(listPath);
+    const age = new Date().valueOf() - stats.ctimeMs;
+
+    return age > this._compilerListCachePeriodMs;
   }
 
-  private async _fetchVersionPath(
-    version: string,
-    platform: CompilerPlatform
-  ): Promise<string | undefined> {
-    const compilersListExisted = await this.compilersListExists(platform);
-    let list = await this.getCompilersList(platform);
-    let compilerBuildPath = list.releases[version];
+  private async _downloadCompilerList(): Promise<void> {
+    log(`Downloading compiler list for platform ${this._platform}`);
+    const url = `${COMPILER_REPOSITORY_URL}/${this._platform}/list.json`;
+    const downloadPath = this._getCompilerListPath();
 
-    // We may need to re-download the compilers list.
-    if (compilerBuildPath === undefined && compilersListExisted) {
-      await fsExtra.unlink(this.getCompilersListPath(platform));
+    await this._downloadFunction(url, downloadPath);
+  }
 
-      list = await this.getCompilersList(platform);
-      compilerBuildPath = list.releases[version];
-    }
+  private async _downloadCompiler(build: CompilerBuild): Promise<string> {
+    log(`Downloading compiler ${build.longVersion}`);
+    const url = `${COMPILER_REPOSITORY_URL}/${this._platform}/${build.path}`;
+    const downloadPath = this._getCompilerDownloadPathFromBuild(build);
 
-    return compilerBuildPath;
+    await this._downloadFunction(url, downloadPath);
+
+    return downloadPath;
   }
 
-  private async _versionExists(
-    version: string,
-    platform: CompilerPlatform
+  private async _verifyCompilerDownload(
+    build: CompilerBuild,
+    downloadPath: string
   ): Promise<boolean> {
-    const versionPath = await this._fetchVersionPath(version, platform);
-    return versionPath !== undefined;
-  }
+    const ethereumjsUtil = await import("@nomicfoundation/ethereumjs-util");
+    const { keccak256 } = await import("../../util/keccak");
 
-  private async _getCompilerBuildByPlatform(
-    version: string,
-    platform: CompilerPlatform
-  ): Promise<CompilerBuild> {
-    const compilerBuildPath = await this._fetchVersionPath(version, platform);
-    const list = await this.getCompilersList(platform);
-    const compilerBuild = list.builds.find((b) => b.path === compilerBuildPath);
+    const expectedKeccak256 = build.keccak256;
+    const compiler = await fsExtra.readFile(downloadPath);
 
-    if (compilerBuild === undefined) {
-      throw new HardhatError(ERRORS.SOLC.INVALID_VERSION, { version });
+    const compilerKeccak256 = ethereumjsUtil.bufferToHex(keccak256(compiler));
+
+    if (expectedKeccak256 !== compilerKeccak256) {
+      await fsExtra.unlink(downloadPath);
+      return false;
     }
 
-    compilerBuild.platform = platform;
-    return compilerBuild;
+    return true;
   }
 
-  private async _fileExists(filePath: string) {
-    return fsExtra.pathExists(filePath);
-  }
+  private async _postProcessCompilerDownload(
+    build: CompilerBuild,
+    downloadPath: string
+  ): Promise<void> {
+    if (this._platform === CompilerPlatform.WASM) {
+      return;
+    }
 
-  private _getCurrentPlatform(): CompilerPlatform {
-    if (this._forceSolcJs) {
-      return CompilerPlatform.WASM;
+    if (
+      this._platform === CompilerPlatform.LINUX ||
+      this._platform === CompilerPlatform.MACOS
+    ) {
+      fsExtra.chmodSync(downloadPath, 0o755);
+    } else if (
+      this._platform === CompilerPlatform.WINDOWS &&
+      downloadPath.endsWith(".zip")
+    ) {
+      // some window builds are zipped, some are not
+      const { default: AdmZip } = await import("adm-zip");
+
+      const solcFolder = path.join(this._compilersDir, build.version);
+      await fsExtra.ensureDir(solcFolder);
+
+      const zip = new AdmZip(downloadPath);
+      zip.extractAllTo(solcFolder);
     }
 
-    switch (os.platform()) {
-      case "win32":
-        return CompilerPlatform.WINDOWS;
-      case "linux":
-        return CompilerPlatform.LINUX;
-      case "darwin":
-        return CompilerPlatform.MACOS;
-      default:
-        return CompilerPlatform.WASM;
+    log("Checking native solc binary");
+    const nativeSolcWorks = await this._checkNativeSolc(build);
+
+    if (nativeSolcWorks) {
+      return;
     }
-  }
-}
 
-function getCompilerURL(platform: CompilerPlatform, filePath: string) {
-  return `${COMPILER_FILES_DIR_URL_SOLC}${platform}/${filePath}`;
-}
+    await fsExtra.createFile(this._getCompilerDoesntWorkFile(build));
+  }
 
-function getCompilerListURL(platform: CompilerPlatform) {
-  return getCompilerURL(platform, "list.json");
+  private _checkNativeSolc(build: CompilerBuild) {
+    const solcPath = this._getCompilerBinaryPathFromBuild(build);
+    return new Promise((resolve) => {
+      try {
+        const process = execFile(solcPath, ["--version"]);
+        process.on("exit", (code) => {
+          resolve(code === 0);
+        });
+      } catch {
+        resolve(false);
+      }
+    });
+  }
 }

package/src/internal/solidity/compiler/index.ts

@@ -1,12 +1,17 @@
-import { exec } from "child_process";
+import { execFile } from "child_process";
 import * as fs from "fs";
+import { CompilerInput, CompilerOutput } from "../../../types";
 
-export class Compiler {
+export interface ICompiler {
+  compile(input: CompilerInput): Promise<CompilerOutput>;
+}
+
+export class Compiler implements ICompiler {
   private _loadedSolc?: any;
 
   constructor(private _pathToSolcJs: string) {}
 
-  public async compile(input: any) {
+  public async compile(input: CompilerInput) {
     const solc = await this.getSolc();
 
     const jsonOutput = solc.compile(JSON.stringify(input));
@@ -52,13 +57,14 @@ export class Compiler {
   }
 }
 
-export class NativeCompiler {
+export class NativeCompiler implements ICompiler {
   constructor(private _pathToSolc: string) {}
 
-  public async compile(input: any) {
+  public async compile(input: CompilerInput) {
     const output: string = await new Promise((resolve, reject) => {
-      const process = exec(
-        `${this._pathToSolc} --standard-json`,
+      const process = execFile(
+        this._pathToSolc,
+        [`--standard-json`],
         {
           maxBuffer: 1024 * 1024 * 500,
         },

package/src/internal/solidity/dependencyGraph.ts

@@ -9,9 +9,11 @@ export class DependencyGraph implements taskTypes.DependencyGraph {
   ): Promise<DependencyGraph> {
     const graph = new DependencyGraph();
 
-    for (const resolvedFile of resolvedFiles) {
-      await graph._addDependenciesFrom(resolver, resolvedFile);
-    }
+    await Promise.all(
+      resolvedFiles.map((resolvedFile) =>
+        graph._addDependenciesFrom(resolver, resolvedFile)
+      )
+    );
 
     return graph;
   }
@@ -167,11 +169,13 @@ export class DependencyGraph implements taskTypes.DependencyGraph {
     this._resolvedFiles.set(file.sourceName, file);
     this._dependenciesPerFile.set(file.sourceName, dependencies);
 
-    for (const imp of file.content.imports) {
-      const dependency = await resolver.resolveImport(file, imp);
-      dependencies.add(dependency);
+    await Promise.all(
+      file.content.imports.map(async (imp) => {
+        const dependency = await resolver.resolveImport(file, imp);
+        dependencies.add(dependency);
 
-      await this._addDependenciesFrom(resolver, dependency);
-    }
+        await this._addDependenciesFrom(resolver, dependency);
+      })
+    );
   }
 }