nx 21.0.0-canary.20250206-8bd0bcd → 21.0.0-canary.20250416-caa26a7

package/src/tasks-runner/cache.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Cache = exports.DbCache = void 0;
 exports.dbCacheEnabled = dbCacheEnabled;
 exports.getCache = getCache;
+exports.parseMaxCacheSize = parseMaxCacheSize;
+exports.formatCacheSize = formatCacheSize;
 const workspace_root_1 = require("../utils/workspace-root");
 const path_1 = require("path");
 const perf_hooks_1 = require("perf_hooks");
@@ -68,7 +70,10 @@ function getCache(options) {
 class DbCache {
     constructor(options) {
         this.options = options;
-        this.cache = new native_1.NxCache(workspace_root_1.workspaceRoot, cache_directory_1.cacheDir, (0, db_connection_1.getDbConnection)());
+        this.nxJson = (0, nx_json_1.readNxJson)();
+        this.cache = new native_1.NxCache(workspace_root_1.workspaceRoot, cache_directory_1.cacheDir, (0, db_connection_1.getDbConnection)(), undefined, this.nxJson.maxCacheSize !== undefined
+            ? parseMaxCacheSize(this.nxJson.maxCacheSize)
+            : (0, native_1.getDefaultMaxCacheSize)(cache_directory_1.cacheDir));
         this.isVerbose = process.env.NX_VERBOSE_LOGGING === 'true';
     }
     async init() {
@@ -83,6 +88,7 @@ class DbCache {
         if (res) {
             return {
                 ...res,
+                terminalOutput: res.terminalOutput ?? '',
                 remote: false,
             };
         }
@@ -91,9 +97,10 @@ class DbCache {
             // attempt remote cache
             const res = await this.remoteCache.retrieve(task.hash, this.cache.cacheDirectory);
             if (res) {
-                this.applyRemoteCacheResults(task.hash, res);
+                this.applyRemoteCacheResults(task.hash, res, task.outputs);
                 return {
                     ...res,
+                    terminalOutput: res.terminalOutput ?? '',
                     remote: true,
                 };
             }
@@ -105,8 +112,11 @@ class DbCache {
             return null;
         }
     }
-    applyRemoteCacheResults(hash, res) {
-        return this.cache.applyRemoteCacheResults(hash, res);
+    getUsedCacheSpace() {
+        return this.cache.getCacheSize();
+    }
+    applyRemoteCacheResults(hash, res, outputs) {
+        return this.cache.applyRemoteCacheResults(hash, res, outputs);
     }
     async put(task, terminalOutput, outputs, code) {
         return tryAndRetry(async () => {
@@ -155,26 +165,49 @@ class DbCache {
             }
         }
         else {
-            return ((await this.getPowerpackS3Cache()) ??
-                (await this.getPowerpackSharedCache()) ??
-                (await this.getPowerpackGcsCache()) ??
-                (await this.getPowerpackAzureCache()) ??
+            return ((await this.getS3Cache()) ??
+                (await this.getSharedCache()) ??
+                (await this.getGcsCache()) ??
+                (await this.getAzureCache()) ??
+                this.getHttpCache() ??
                 null);
         }
     }
-    getPowerpackS3Cache() {
-        return this.getPowerpackCache('@nx/powerpack-s3-cache');
-    }
-    getPowerpackSharedCache() {
-        return this.getPowerpackCache('@nx/powerpack-shared-fs-cache');
-    }
-    getPowerpackGcsCache() {
-        return this.getPowerpackCache('@nx/powerpack-gcs-cache');
-    }
-    getPowerpackAzureCache() {
-        return this.getPowerpackCache('@nx/powerpack-azure-cache');
+    async getS3Cache() {
+        const cache = await this.resolveRemoteCache('@nx/s3-cache');
+        if (cache)
+            return cache;
+        return this.resolveRemoteCache('@nx/powerpack-s3-cache');
+    }
+    async getSharedCache() {
+        const cache = await this.resolveRemoteCache('@nx/shared-fs-cache');
+        if (cache)
+            return cache;
+        return this.resolveRemoteCache('@nx/powerpack-shared-fs-cache');
+    }
+    async getGcsCache() {
+        const cache = await this.resolveRemoteCache('@nx/gcs-cache');
+        if (cache)
+            return cache;
+        return this.resolveRemoteCache('@nx/powerpack-gcs-cache');
+    }
+    async getAzureCache() {
+        const cache = await this.resolveRemoteCache('@nx/azure-cache');
+        if (cache)
+            return cache;
+        return this.resolveRemoteCache('@nx/powerpack-azure-cache');
+    }
+    getHttpCache() {
+        if (process.env.NX_SELF_HOSTED_REMOTE_CACHE_SERVER) {
+            if (native_1.IS_WASM) {
+                logger_1.logger.warn('The HTTP remote cache is not yet supported in the wasm build of Nx.');
+                return null;
+            }
+            return new native_1.HttpRemoteCache();
+        }
+        return null;
     }
-    async getPowerpackCache(pkg) {
+    async resolveRemoteCache(pkg) {
         let getRemoteCache = null;
         try {
             getRemoteCache = (await Promise.resolve(`${this.resolvePackage(pkg)}`).then(s => require(s))).getRemoteCache;
@@ -455,3 +488,54 @@ function tryAndRetry(fn) {
     };
     return _try();
 }
+/**
+ * Converts a string representation of a max cache size to a number.
+ *
+ * e.g. '1GB' -> 1024 * 1024 * 1024
+ *      '1MB' -> 1024 * 1024
+ *      '1KB' -> 1024
+ *
+ * @param maxCacheSize Max cache size as specified in nx.json
+ */
+function parseMaxCacheSize(maxCacheSize) {
+    if (maxCacheSize === null || maxCacheSize === undefined) {
+        return undefined;
+    }
+    let regexResult = maxCacheSize
+        // Covers folks who accidentally specify as a number rather than a string
+        .toString()
+        // Match a number followed by an optional unit (KB, MB, GB), with optional whitespace between the number and unit
+        .match(/^(?<size>[\d|.]+)\s?((?<unit>[KMG]?B)?)$/);
+    if (!regexResult) {
+        throw new Error(`Invalid max cache size specified in nx.json: ${maxCacheSize}. Must be a number followed by an optional unit (KB, MB, GB)`);
+    }
+    let sizeString = regexResult.groups.size;
+    let unit = regexResult.groups.unit;
+    if ([...sizeString].filter((c) => c === '.').length > 1) {
+        throw new Error(`Invalid max cache size specified in nx.json: ${maxCacheSize} (multiple decimal points in size)`);
+    }
+    let size = parseFloat(sizeString);
+    if (isNaN(size)) {
+        throw new Error(`Invalid max cache size specified in nx.json: ${maxCacheSize} (${sizeString} is not a number)`);
+    }
+    switch (unit) {
+        case 'KB':
+            return size * 1024;
+        case 'MB':
+            return size * 1024 * 1024;
+        case 'GB':
+            return size * 1024 * 1024 * 1024;
+        default:
+            return size;
+    }
+}
+function formatCacheSize(maxCacheSize, decimals = 2) {
+    const exponents = ['B', 'KB', 'MB', 'GB'];
+    let exponent = 0;
+    let size = maxCacheSize;
+    while (size >= 1024 && exponent < exponents.length - 1) {
+        size /= 1024;
+        exponent++;
+    }
+    return `${size.toFixed(decimals)} ${exponents[exponent]}`;
+}

package/src/tasks-runner/create-task-graph.d.ts

@@ -11,6 +11,9 @@ export declare class ProcessTasks {
     readonly dependencies: {
         [k: string]: string[];
     };
+    readonly continuousDependencies: {
+        [k: string]: string[];
+    };
     private readonly allTargetNames;
     constructor(extraTargetDependencies: TargetDependencies, projectGraph: ProjectGraph);
     processTasks(projectNames: string[], targets: string[], configuration: string, overrides: Object, excludeTaskDependencies: boolean): string[];
@@ -21,7 +24,6 @@ export declare class ProcessTasks {
     private createDummyTask;
     createTask(id: string, project: ProjectGraphProjectNode, target: string, resolvedConfiguration: string | undefined, overrides: Object): Task;
     resolveConfiguration(project: ProjectGraphProjectNode, target: string, configuration: string | undefined): string;
-    getId(project: string, target: string, configuration: string | undefined): string;
 }
 export declare function createTaskGraph(projectGraph: ProjectGraph, extraTargetDependencies: TargetDependencies, projectNames: string[], targets: string[], configuration: string | undefined, overrides: Object, excludeTaskDependencies?: boolean): TaskGraph;
 export declare function mapTargetDefaultsToDependencies(defaults: TargetDefaults | undefined): TargetDependencies;
@@ -38,3 +40,4 @@ export declare function filterDummyTasks(dependencies: {
 export declare function getNonDummyDeps(currentTask: string, dependencies: {
     [k: string]: string[];
 }, cycles?: Set<string>, seen?: Set<string>): string[];
+export declare function createTaskId(project: string, target: string, configuration: string | undefined): string;

package/src/tasks-runner/create-task-graph.js

@@ -5,6 +5,7 @@ exports.createTaskGraph = createTaskGraph;
 exports.mapTargetDefaultsToDependencies = mapTargetDefaultsToDependencies;
 exports.filterDummyTasks = filterDummyTasks;
 exports.getNonDummyDeps = getNonDummyDeps;
+exports.createTaskId = createTaskId;
 const utils_1 = require("./utils");
 const project_graph_utils_1 = require("../utils/project-graph-utils");
 const output_1 = require("../utils/output");
@@ -17,6 +18,7 @@ class ProcessTasks {
         this.seen = new Set();
         this.tasks = {};
         this.dependencies = {};
+        this.continuousDependencies = {};
         const allTargetNames = new Set();
         for (const projectName in projectGraph.nodes) {
             const project = projectGraph.nodes[projectName];
@@ -32,10 +34,11 @@ class ProcessTasks {
                 const project = this.projectGraph.nodes[projectName];
                 if (targets.length === 1 || project.data.targets[target]) {
                     const resolvedConfiguration = this.resolveConfiguration(project, target, configuration);
-                    const id = this.getId(projectName, target, resolvedConfiguration);
+                    const id = createTaskId(projectName, target, resolvedConfiguration);
                     const task = this.createTask(id, project, target, resolvedConfiguration, overrides);
                     this.tasks[task.id] = task;
                     this.dependencies[task.id] = [];
+                    this.continuousDependencies[task.id] = [];
                 }
             }
         }
@@ -50,11 +53,15 @@ class ProcessTasks {
                 if (!initialTasks[t]) {
                     delete this.tasks[t];
                     delete this.dependencies[t];
+                    delete this.continuousDependencies[t];
                 }
             }
             for (let d of Object.keys(this.dependencies)) {
                 this.dependencies[d] = this.dependencies[d].filter((dd) => !!initialTasks[dd]);
             }
+            for (let d of Object.keys(this.continuousDependencies)) {
+                this.continuousDependencies[d] = this.continuousDependencies[d].filter((dd) => !!initialTasks[dd]);
+            }
         }
         filterDummyTasks(this.dependencies);
         for (const taskId of Object.keys(this.dependencies)) {
@@ -64,7 +71,16 @@ class ProcessTasks {
                 ];
             }
         }
-        return Object.keys(this.dependencies).filter((d) => this.dependencies[d].length === 0);
+        filterDummyTasks(this.continuousDependencies);
+        for (const taskId of Object.keys(this.continuousDependencies)) {
+            if (this.continuousDependencies[taskId].length > 0) {
+                this.continuousDependencies[taskId] = [
+                    ...new Set(this.continuousDependencies[taskId].filter((d) => d !== taskId)).values(),
+                ];
+            }
+        }
+        return Object.keys(this.tasks).filter((d) => this.dependencies[d].length === 0 &&
+            this.continuousDependencies[d].length === 0);
     }
     processTask(task, projectUsedToDeriveDependencies, configuration, overrides) {
         const seenKey = `${task.id}-${projectUsedToDeriveDependencies}`;
@@ -105,16 +121,22 @@ class ProcessTasks {
         const selfProject = this.projectGraph.nodes[projectName];
         if ((0, project_graph_utils_1.projectHasTarget)(selfProject, dependencyConfig.target)) {
             const resolvedConfiguration = this.resolveConfiguration(selfProject, dependencyConfig.target, configuration);
-            const selfTaskId = this.getId(selfProject.name, dependencyConfig.target, resolvedConfiguration);
-            if (task.id !== selfTaskId) {
-                this.dependencies[task.id].push(selfTaskId);
-            }
+            const selfTaskId = createTaskId(selfProject.name, dependencyConfig.target, resolvedConfiguration);
             if (!this.tasks[selfTaskId]) {
                 const newTask = this.createTask(selfTaskId, selfProject, dependencyConfig.target, resolvedConfiguration, taskOverrides);
                 this.tasks[selfTaskId] = newTask;
                 this.dependencies[selfTaskId] = [];
+                this.continuousDependencies[selfTaskId] = [];
                 this.processTask(newTask, newTask.target.project, configuration, overrides);
             }
+            if (task.id !== selfTaskId) {
+                if (this.tasks[selfTaskId].continuous) {
+                    this.continuousDependencies[task.id].push(selfTaskId);
+                }
+                else {
+                    this.dependencies[task.id].push(selfTaskId);
+                }
+            }
         }
     }
     processTasksForDependencies(projectUsedToDeriveDependencies, dependencyConfig, configuration, task, taskOverrides, overrides) {
@@ -128,25 +150,33 @@ class ProcessTasks {
                 continue;
             if ((0, project_graph_utils_1.projectHasTarget)(depProject, dependencyConfig.target)) {
                 const resolvedConfiguration = this.resolveConfiguration(depProject, dependencyConfig.target, configuration);
-                const depTargetId = this.getId(depProject.name, dependencyConfig.target, resolvedConfiguration);
+                const depTargetId = createTaskId(depProject.name, dependencyConfig.target, resolvedConfiguration);
+                const depTargetConfiguration = this.projectGraph.nodes[depProject.name].data.targets[dependencyConfig.target];
                 if (task.id !== depTargetId) {
-                    this.dependencies[task.id].push(depTargetId);
+                    if (depTargetConfiguration.continuous) {
+                        this.continuousDependencies[task.id].push(depTargetId);
+                    }
+                    else {
+                        this.dependencies[task.id].push(depTargetId);
+                    }
                 }
                 if (!this.tasks[depTargetId]) {
                     const newTask = this.createTask(depTargetId, depProject, dependencyConfig.target, resolvedConfiguration, taskOverrides);
                     this.tasks[depTargetId] = newTask;
                     this.dependencies[depTargetId] = [];
+                    this.continuousDependencies[depTargetId] = [];
                     this.processTask(newTask, newTask.target.project, configuration, overrides);
                 }
             }
             else {
                 // Create a dummy task for task.target.project... which simulates if depProject had dependencyConfig.target
-                const dummyId = this.getId(depProject.name, task.target.project +
+                const dummyId = createTaskId(depProject.name, task.target.project +
                     '__' +
                     dependencyConfig.target +
                     DUMMY_TASK_TARGET, undefined);
                 this.dependencies[task.id].push(dummyId);
                 this.dependencies[dummyId] ??= [];
+                this.continuousDependencies[dummyId] ??= [];
                 const noopTask = this.createDummyTask(dummyId, task);
                 this.processTask(noopTask, depProject.name, configuration, overrides);
             }
@@ -179,6 +209,7 @@ class ProcessTasks {
             outputs: (0, utils_1.getOutputs)(this.projectGraph.nodes, qualifiedTarget, interpolatedOverrides),
             cache: project.data.targets[target].cache,
             parallelism: project.data.targets[target].parallelism ?? true,
+            continuous: project.data.targets[target].continuous ?? false,
         };
     }
     resolveConfiguration(project, target, configuration) {
@@ -188,13 +219,6 @@ class ProcessTasks {
             ? configuration
             : defaultConfiguration;
     }
-    getId(project, target, configuration) {
-        let id = `${project}:${target}`;
-        if (configuration) {
-            id += `:${configuration}`;
-        }
-        return id;
-    }
 }
 exports.ProcessTasks = ProcessTasks;
 function createTaskGraph(projectGraph, extraTargetDependencies, projectNames, targets, configuration, overrides, excludeTaskDependencies = false) {
@@ -204,6 +228,7 @@ function createTaskGraph(projectGraph, extraTargetDependencies, projectNames, ta
         roots,
         tasks: p.tasks,
         dependencies: p.dependencies,
+        continuousDependencies: p.continuousDependencies,
     };
 }
 function mapTargetDefaultsToDependencies(defaults) {
@@ -268,3 +293,10 @@ function getNonDummyDeps(currentTask, dependencies, cycles, seen = new Set()) {
         return [currentTask];
     }
 }
+function createTaskId(project, target, configuration) {
+    let id = `${project}:${target}`;
+    if (configuration) {
+        id += `:${configuration}`;
+    }
+    return id;
+}

package/src/tasks-runner/default-tasks-runner.js

@@ -54,26 +54,17 @@ class RemoteCacheV2 {
 }
 exports.RemoteCacheV2 = RemoteCacheV2;
 const defaultTasksRunner = async (tasks, options, context) => {
-    if (options['parallel'] === 'false' ||
-        options['parallel'] === false) {
-        options['parallel'] = 1;
-    }
-    else if (options['parallel'] === 'true' ||
-        options['parallel'] === true ||
-        options['parallel'] === undefined ||
-        options['parallel'] === '') {
-        options['parallel'] = Number(options['maxParallel'] || 3);
-    }
-    await options.lifeCycle.startCommand();
+    const threadCount = (0, task_orchestrator_1.getThreadCount)(options, context.taskGraph);
+    await options.lifeCycle.startCommand(threadCount);
     try {
-        return await runAllTasks(tasks, options, context);
+        return await runAllTasks(options, context);
     }
     finally {
         await options.lifeCycle.endCommand();
     }
 };
 exports.defaultTasksRunner = defaultTasksRunner;
-async function runAllTasks(tasks, options, context) {
+async function runAllTasks(options, context) {
     const orchestrator = new task_orchestrator_1.TaskOrchestrator(context.hasher, context.initiatingProject, context.projectGraph, context.taskGraph, context.nxJson, options, context.nxArgs?.nxBail, context.daemon, context.nxArgs?.outputStyle);
     return orchestrator.run();
 }

package/src/tasks-runner/forked-process-task-runner.d.ts

@@ -1,26 +1,29 @@
 import { DefaultTasksRunnerOptions } from './default-tasks-runner';
 import { Batch } from './tasks-schedule';
-import { BatchResults } from './batch/batch-messages';
 import { Task, TaskGraph } from '../config/task-graph';
+import { PseudoTtyProcess } from './pseudo-terminal';
+import { ProjectGraph } from '../config/project-graph';
+import { BatchProcess } from './running-tasks/batch-process';
+import { RunningTask } from './running-tasks/running-task';
 export declare class ForkedProcessTaskRunner {
     private readonly options;
+    private readonly tuiEnabled;
     cliPath: string;
     private readonly verbose;
     private processes;
-    private pseudoTerminal;
-    constructor(options: DefaultTasksRunnerOptions);
+    private finishedProcesses;
+    private pseudoTerminals;
+    constructor(options: DefaultTasksRunnerOptions, tuiEnabled: boolean);
     init(): Promise<void>;
-    forkProcessForBatch({ executorName, taskGraph: batchTaskGraph }: Batch, fullTaskGraph: TaskGraph, env: NodeJS.ProcessEnv): Promise<BatchResults>;
+    forkProcessForBatch({ executorName, taskGraph: batchTaskGraph }: Batch, projectGraph: ProjectGraph, fullTaskGraph: TaskGraph, env: NodeJS.ProcessEnv): Promise<BatchProcess>;
+    cleanUpBatchProcesses(): void;
     forkProcessLegacy(task: Task, { temporaryOutputPath, streamOutput, pipeOutput, taskGraph, env, }: {
         temporaryOutputPath: string;
         streamOutput: boolean;
         pipeOutput: boolean;
         taskGraph: TaskGraph;
         env: NodeJS.ProcessEnv;
-    }): Promise<{
-        code: number;
-        terminalOutput: string;
-    }>;
+    }): Promise<RunningTask>;
     forkProcess(task: Task, { temporaryOutputPath, streamOutput, taskGraph, env, disablePseudoTerminal, }: {
         temporaryOutputPath: string;
         streamOutput: boolean;
@@ -28,15 +31,11 @@ export declare class ForkedProcessTaskRunner {
         taskGraph: TaskGraph;
         env: NodeJS.ProcessEnv;
         disablePseudoTerminal: boolean;
-    }): Promise<{
-        code: number;
-        terminalOutput: string;
-    }>;
+    }): Promise<RunningTask | PseudoTtyProcess>;
+    private createPseudoTerminal;
     private forkProcessWithPseudoTerminal;
-    private forkProcessPipeOutputCapture;
     private forkProcessWithPrefixAndNotTTY;
     private forkProcessDirectOutputCapture;
-    private readTerminalOutput;
     private writeTerminalOutput;
     private setupProcessEventListeners;
 }