nx 20.5.0 → 20.6.0-beta.1

package/src/native/index.d.ts

@@ -37,12 +37,13 @@ export declare class ImportResult {
 
 export declare class NxCache {
   cacheDirectory: string
-  constructor(workspaceRoot: string, cachePath: string, dbConnection: ExternalObject<NxDbConnection>, linkTaskDetails?: boolean | undefined | null)
+  constructor(workspaceRoot: string, cachePath: string, dbConnection: ExternalObject<NxDbConnection>, linkTaskDetails?: boolean | undefined | null, maxCacheSize?: number | undefined | null)
   get(hash: string): CachedResult | null
   put(hash: string, terminalOutput: string, outputs: Array<string>, code: number): void
-  applyRemoteCacheResults(hash: string, result: CachedResult): void
+  applyRemoteCacheResults(hash: string, result: CachedResult, outputs: Array<string>): void
   getTaskOutputsPath(hash: string): string
-  copyFilesFromCache(cachedResult: CachedResult, outputs: Array<string>): void
+  getCacheSize(): number
+  copyFilesFromCache(cachedResult: CachedResult, outputs: Array<string>): number
   removeOldCacheRecords(): void
   checkCacheFsInSync(): boolean
 }
@@ -112,13 +113,14 @@ export interface CachedResult {
   code: number
   terminalOutput: string
   outputsPath: string
+  size?: number
 }
 
 export declare export function closeDbConnection(connection: ExternalObject<NxDbConnection>): void
 
 export declare export function connectToNxDb(cacheDir: string, nxVersion: string, dbName?: string | undefined | null): ExternalObject<NxDbConnection>
 
-export declare export function copy(src: string, dest: string): void
+export declare export function copy(src: string, dest: string): number
 
 export interface DepsOutputsInput {
   dependentTasksOutputFiles: string
@@ -165,6 +167,8 @@ export declare export function findImports(projectFileMap: Record<string, Array<
 
 export declare export function getBinaryTarget(): string
 
+export declare export function getDefaultMaxCacheSize(cachePath: string): number
+
 /**
  * Expands the given outputs into a list of existing files.
  * This is used when hashing outputs

package/src/native/native-bindings.js

@@ -379,6 +379,7 @@ module.exports.EventType = nativeBinding.EventType
 module.exports.expandOutputs = nativeBinding.expandOutputs
 module.exports.findImports = nativeBinding.findImports
 module.exports.getBinaryTarget = nativeBinding.getBinaryTarget
+module.exports.getDefaultMaxCacheSize = nativeBinding.getDefaultMaxCacheSize
 module.exports.getFilesForOutputs = nativeBinding.getFilesForOutputs
 module.exports.getTransformableOutputs = nativeBinding.getTransformableOutputs
 module.exports.hashArray = nativeBinding.hashArray

package/src/tasks-runner/cache.d.ts

@@ -15,6 +15,7 @@ export declare function dbCacheEnabled(nxJson?: NxJsonConfiguration): boolean;
 export declare function getCache(options: DefaultTasksRunnerOptions): DbCache | Cache;
 export declare class DbCache {
     private readonly options;
+    private nxJson;
     private cache;
     private remoteCache;
     private remoteCachePromise;
@@ -25,9 +26,10 @@ export declare class DbCache {
     });
     init(): Promise<void>;
     get(task: Task): Promise<CachedResult | null>;
+    getUsedCacheSpace(): number;
     private applyRemoteCacheResults;
     put(task: Task, terminalOutput: string | null, outputs: string[], code: number): Promise<void>;
-    copyFilesFromCache(_: string, cachedResult: CachedResult, outputs: string[]): Promise<void>;
+    copyFilesFromCache(_: string, cachedResult: CachedResult, outputs: string[]): Promise<number>;
     removeOldCacheRecords(): void;
     temporaryOutputPath(task: Task): string;
     private getRemoteCache;
@@ -66,3 +68,14 @@ export declare class Cache {
     private createCacheDir;
     private createTerminalOutputsDir;
 }
+/**
+ * Converts a string representation of a max cache size to a number.
+ *
+ * e.g. '1GB' -> 1024 * 1024 * 1024
+ *      '1MB' -> 1024 * 1024
+ *      '1KB' -> 1024
+ *
+ * @param maxCacheSize Max cache size as specified in nx.json
+ */
+export declare function parseMaxCacheSize(maxCacheSize: string | number): number | undefined;
+export declare function formatCacheSize(maxCacheSize: number, decimals?: number): string;

package/src/tasks-runner/cache.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Cache = exports.DbCache = void 0;
 exports.dbCacheEnabled = dbCacheEnabled;
 exports.getCache = getCache;
+exports.parseMaxCacheSize = parseMaxCacheSize;
+exports.formatCacheSize = formatCacheSize;
 const workspace_root_1 = require("../utils/workspace-root");
 const path_1 = require("path");
 const perf_hooks_1 = require("perf_hooks");
@@ -68,7 +70,10 @@ function getCache(options) {
 class DbCache {
     constructor(options) {
         this.options = options;
-        this.cache = new native_1.NxCache(workspace_root_1.workspaceRoot, cache_directory_1.cacheDir, (0, db_connection_1.getDbConnection)());
+        this.nxJson = (0, nx_json_1.readNxJson)();
+        this.cache = new native_1.NxCache(workspace_root_1.workspaceRoot, cache_directory_1.cacheDir, (0, db_connection_1.getDbConnection)(), undefined, this.nxJson.maxCacheSize !== undefined
+            ? parseMaxCacheSize(this.nxJson.maxCacheSize)
+            : (0, native_1.getDefaultMaxCacheSize)(cache_directory_1.cacheDir));
         this.isVerbose = process.env.NX_VERBOSE_LOGGING === 'true';
     }
     async init() {
@@ -91,7 +96,7 @@ class DbCache {
             // attempt remote cache
             const res = await this.remoteCache.retrieve(task.hash, this.cache.cacheDirectory);
             if (res) {
-                this.applyRemoteCacheResults(task.hash, res);
+                this.applyRemoteCacheResults(task.hash, res, task.outputs);
                 return {
                     ...res,
                     remote: true,
@@ -105,8 +110,11 @@ class DbCache {
             return null;
         }
     }
-    applyRemoteCacheResults(hash, res) {
-        return this.cache.applyRemoteCacheResults(hash, res);
+    getUsedCacheSpace() {
+        return this.cache.getCacheSize();
+    }
+    applyRemoteCacheResults(hash, res, outputs) {
+        return this.cache.applyRemoteCacheResults(hash, res, outputs);
     }
     async put(task, terminalOutput, outputs, code) {
         return tryAndRetry(async () => {
@@ -455,3 +463,54 @@ function tryAndRetry(fn) {
     };
     return _try();
 }
+/**
+ * Converts a string representation of a max cache size to a number.
+ *
+ * e.g. '1GB' -> 1024 * 1024 * 1024
+ *      '1MB' -> 1024 * 1024
+ *      '1KB' -> 1024
+ *
+ * @param maxCacheSize Max cache size as specified in nx.json
+ */
+function parseMaxCacheSize(maxCacheSize) {
+    if (maxCacheSize === null || maxCacheSize === undefined) {
+        return undefined;
+    }
+    let regexResult = maxCacheSize
+        // Covers folks who accidentally specify as a number rather than a string
+        .toString()
+        // Match a number followed by an optional unit (KB, MB, GB), with optional whitespace between the number and unit
+        .match(/^(?<size>[\d|.]+)\s?((?<unit>[KMG]?B)?)$/);
+    if (!regexResult) {
+        throw new Error(`Invalid max cache size specified in nx.json: ${maxCacheSize}. Must be a number followed by an optional unit (KB, MB, GB)`);
+    }
+    let sizeString = regexResult.groups.size;
+    let unit = regexResult.groups.unit;
+    if ([...sizeString].filter((c) => c === '.').length > 1) {
+        throw new Error(`Invalid max cache size specified in nx.json: ${maxCacheSize} (multiple decimal points in size)`);
+    }
+    let size = parseFloat(sizeString);
+    if (isNaN(size)) {
+        throw new Error(`Invalid max cache size specified in nx.json: ${maxCacheSize} (${sizeString} is not a number)`);
+    }
+    switch (unit) {
+        case 'KB':
+            return size * 1024;
+        case 'MB':
+            return size * 1024 * 1024;
+        case 'GB':
+            return size * 1024 * 1024 * 1024;
+        default:
+            return size;
+    }
+}
+function formatCacheSize(maxCacheSize, decimals = 2) {
+    const exponents = ['B', 'KB', 'MB', 'GB'];
+    let exponent = 0;
+    let size = maxCacheSize;
+    while (size >= 1024 && exponent < exponents.length - 1) {
+        size /= 1024;
+        exponent++;
+    }
+    return `${size.toFixed(decimals)} ${exponents[exponent]}`;
+}

package/src/tasks-runner/task-orchestrator.js

@@ -131,7 +131,12 @@ class TaskOrchestrator {
         if (!cachedResult || cachedResult.code !== 0)
             return null;
         const outputs = task.outputs;
-        const shouldCopyOutputsFromCache = !!outputs.length &&
+        const shouldCopyOutputsFromCache = 
+        // No output files to restore
+        !!outputs.length &&
+            // Remote caches are restored to output dirs when applied
+            !cachedResult.remote &&
+            // Output files have not been touched since last run
             (await this.shouldCopyOutputsFromCache(outputs, task.hash));
         if (shouldCopyOutputsFromCache) {
             await this.cache.copyFilesFromCache(task.hash, cachedResult, outputs);