nx 20.8.0-rc.0 → 20.8.1

package/src/native/index.js

@@ -1,5 +1,5 @@
 const { join, basename } = require('path');
-const { copyFileSync, existsSync, mkdirSync } = require('fs');
+const { copyFileSync, existsSync, mkdirSync, renameSync } = require('fs');
 const Module = require('module');
 const { nxVersion } = require('../utils/versions');
 const { getNativeFileCacheLocation } = require('./native-file-cache-location');
@@ -71,14 +71,28 @@ Module._load = function (request, parent, isMain) {
 
     // we copy the file to a workspace-scoped tmp directory and prefix with nxVersion to avoid stale files being loaded
     const nativeFileCacheLocation = getNativeFileCacheLocation();
+    // This is a path to copy to, not the one that gets loaded
+    const tmpTmpFile = join(
+      nativeFileCacheLocation,
+      nxVersion + '-' + Math.random() + fileName
+    );
+    // This is the path that will get loaded
     const tmpFile = join(nativeFileCacheLocation, nxVersion + '-' + fileName);
+
+    // If the file to be loaded already exists, just load it
     if (existsSync(tmpFile)) {
       return originalLoad.apply(this, [tmpFile, parent, isMain]);
     }
     if (!existsSync(nativeFileCacheLocation)) {
       mkdirSync(nativeFileCacheLocation, { recursive: true });
     }
-    copyFileSync(nativeLocation, tmpFile);
+    // First copy to a unique location for each process
+    copyFileSync(nativeLocation, tmpTmpFile);
+
+    // Then rename to the final location
+    renameSync(tmpTmpFile, tmpFile);
+
+    // Load from the final location
     return originalLoad.apply(this, [tmpFile, parent, isMain]);
   } else {
     // call the original _load function for everything else

package/src/tasks-runner/forked-process-task-runner.d.ts

@@ -8,10 +8,12 @@ export declare class ForkedProcessTaskRunner {
     cliPath: string;
     private readonly verbose;
     private processes;
+    private finishedProcesses;
     private pseudoTerminal;
     constructor(options: DefaultTasksRunnerOptions);
     init(): Promise<void>;
     forkProcessForBatch({ executorName, taskGraph: batchTaskGraph }: Batch, projectGraph: ProjectGraph, fullTaskGraph: TaskGraph, env: NodeJS.ProcessEnv): Promise<BatchResults>;
+    cleanUpBatchProcesses(): void;
     forkProcessLegacy(task: Task, { temporaryOutputPath, streamOutput, pipeOutput, taskGraph, env, }: {
         temporaryOutputPath: string;
         streamOutput: boolean;

package/src/tasks-runner/forked-process-task-runner.js

@@ -20,6 +20,7 @@ class ForkedProcessTaskRunner {
         this.cliPath = (0, utils_1.getCliPath)();
         this.verbose = process.env.NX_VERBOSE_LOGGING === 'true';
         this.processes = new Set();
+        this.finishedProcesses = new Set();
         this.pseudoTerminal = pseudo_terminal_1.PseudoTerminal.isSupported()
             ? (0, pseudo_terminal_1.getPseudoTerminal)()
             : null;
@@ -33,6 +34,7 @@ class ForkedProcessTaskRunner {
     // TODO: vsavkin delegate terminal output printing
     forkProcessForBatch({ executorName, taskGraph: batchTaskGraph }, projectGraph, fullTaskGraph, env) {
         return new Promise((res, rej) => {
+            let p;
             try {
                 const count = Object.keys(batchTaskGraph.tasks).length;
                 if (count > 1) {
@@ -42,7 +44,7 @@ class ForkedProcessTaskRunner {
                     const args = (0, utils_1.getPrintableCommandArgsForTask)(Object.values(batchTaskGraph.tasks)[0]);
                     output_1.output.logCommand(args.join(' '));
                 }
-                const p = (0, child_process_1.fork)(workerPath, {
+                p = (0, child_process_1.fork)(workerPath, {
                     stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
                     env,
                 });
@@ -52,20 +54,18 @@ class ForkedProcessTaskRunner {
                     if (code === null)
                         code = (0, exit_codes_1.signalToCode)(signal);
                     if (code !== 0) {
-                        const results = {};
-                        for (const rootTaskId of batchTaskGraph.roots) {
-                            results[rootTaskId] = {
-                                success: false,
-                                terminalOutput: '',
-                            };
-                        }
                         rej(new Error(`"${executorName}" exited unexpectedly with code: ${code}`));
                     }
                 });
+                p.on('error', (err) => {
+                    this.processes.delete(p);
+                    rej(err || new Error(`"${executorName}" exited unexpectedly`));
+                });
                 p.on('message', (message) => {
                     switch (message.type) {
                         case batch_messages_1.BatchMessageType.CompleteBatchExecution: {
                             res(message.results);
+                            this.finishedProcesses.add(p);
                             break;
                         }
                         case batch_messages_1.BatchMessageType.RunTasks: {
@@ -90,9 +90,23 @@ class ForkedProcessTaskRunner {
             }
             catch (e) {
                 rej(e);
+                if (p) {
+                    this.processes.delete(p);
+                    p.kill();
+                }
             }
         });
     }
+    cleanUpBatchProcesses() {
+        if (this.finishedProcesses.size > 0) {
+            this.finishedProcesses.forEach((p) => {
+                if ('connected' in p ? p.connected : p.isAlive) {
+                    p.kill();
+                }
+            });
+            this.finishedProcesses.clear();
+        }
+    }
     async forkProcessLegacy(task, { temporaryOutputPath, streamOutput, pipeOutput, taskGraph, env, }) {
         return pipeOutput
             ? await this.forkProcessPipeOutputCapture(task, {
@@ -157,6 +171,7 @@ class ForkedProcessTaskRunner {
         });
         return new Promise((res) => {
             p.onExit((code) => {
+                this.processes.delete(p);
                 // If the exit code is greater than 128, it's a special exit code for a signal
                 if (code >= 128) {
                     process.exit(code);
@@ -227,7 +242,7 @@ class ForkedProcessTaskRunner {
                 p.stderr.on('data', (chunk) => {
                     outWithErr.push(chunk.toString());
                 });
-                p.on('exit', (code, signal) => {
+                p.once('exit', (code, signal) => {
                     this.processes.delete(p);
                     if (code === null)
                         code = (0, exit_codes_1.signalToCode)(signal);
@@ -272,7 +287,8 @@ class ForkedProcessTaskRunner {
                     taskGraph,
                     isVerbose: this.verbose,
                 });
-                p.on('exit', (code, signal) => {
+                p.once('exit', (code, signal) => {
+                    this.processes.delete(p);
                     if (code === null)
                         code = (0, exit_codes_1.signalToCode)(signal);
                     // we didn't print any output as we were running the command
@@ -317,8 +333,7 @@ class ForkedProcessTaskRunner {
                 process.send(message);
             });
         }
-        // When the nx process gets a message, it will be sent into the task's process
-        process.on('message', (message) => {
+        const messageHandler = (message) => {
             // this.publisher.publish(message.toString());
             if (this.pseudoTerminal) {
                 this.pseudoTerminal.sendMessageToChildren(message);
@@ -328,39 +343,34 @@ class ForkedProcessTaskRunner {
                     p.send(message);
                 }
             });
-        });
-        // Terminate any task processes on exit
-        process.on('exit', () => {
+        };
+        // When the nx process gets a message, it will be sent into the task's process
+        process.on('message', messageHandler);
+        const cleanUp = (signal) => {
             this.processes.forEach((p) => {
                 if ('connected' in p ? p.connected : p.isAlive) {
-                    p.kill();
+                    p.kill(signal);
                 }
             });
+            process.off('message', messageHandler);
+            this.cleanUpBatchProcesses();
+        };
+        // Terminate any task processes on exit
+        process.once('exit', () => {
+            cleanUp();
         });
-        process.on('SIGINT', () => {
-            this.processes.forEach((p) => {
-                if ('connected' in p ? p.connected : p.isAlive) {
-                    p.kill('SIGTERM');
-                }
-            });
+        process.once('SIGINT', () => {
+            cleanUp('SIGTERM');
             // we exit here because we don't need to write anything to cache.
             process.exit((0, exit_codes_1.signalToCode)('SIGINT'));
         });
-        process.on('SIGTERM', () => {
-            this.processes.forEach((p) => {
-                if ('connected' in p ? p.connected : p.isAlive) {
-                    p.kill('SIGTERM');
-                }
-            });
+        process.once('SIGTERM', () => {
+            cleanUp('SIGTERM');
             // no exit here because we expect child processes to terminate which
             // will store results to the cache and will terminate this process
         });
-        process.on('SIGHUP', () => {
-            this.processes.forEach((p) => {
-                if ('connected' in p ? p.connected : p.isAlive) {
-                    p.kill('SIGTERM');
-                }
-            });
+        process.once('SIGHUP', () => {
+            cleanUp('SIGTERM');
             // no exit here because we expect child processes to terminate which
             // will store results to the cache and will terminate this process
         });

package/src/tasks-runner/task-orchestrator.js

@@ -171,6 +171,7 @@ class TaskOrchestrator {
             results.push(...batchResults);
         }
         await this.postRunSteps(tasks, results, doNotSkipCache, { groupId });
+        this.forkedProcessTaskRunner.cleanUpBatchProcesses();
         const tasksCompleted = taskEntries.filter(([taskId]) => this.completedTasks[taskId]);
         // Batch is still not done, run it again
         if (tasksCompleted.length !== taskEntries.length) {

package/src/tasks-runner/utils.js

@@ -362,6 +362,8 @@ function getSerializedArgsForTask(task, isVerbose) {
 function shouldStreamOutput(task, initiatingProject) {
     if (process.env.NX_STREAM_OUTPUT === 'true')
         return true;
+    if (process.env.NX_STREAM_OUTPUT === 'false')
+        return false;
     if (longRunningTask(task))
         return true;
     if (task.target.project === initiatingProject)