nx 23.2.0-beta.0 → 23.2.0-beta.10

package/dist/src/tasks-runner/running-tasks/batch-process.js

@@ -1,8 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BatchProcess = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
 const native_1 = require("../../native");
+const cache_directory_1 = require("../../utils/cache-directory");
 const exit_codes_1 = require("../../utils/exit-codes");
+const output_1 = require("../../utils/output");
 const batch_messages_1 = require("../batch/batch-messages");
 class BatchProcess {
     constructor(childProcess, executorName) {
@@ -12,6 +16,18 @@ class BatchProcess {
         this.batchResultsCallbacks = [];
         this.taskResultsCallbacks = [];
         this.outputCallbacks = [];
+        /**
+         * Set once the capture is released. A chunk can still arrive after that —
+         * stdout delivers past the exit event — and reopening then would mint a
+         * second numbered file that nothing ever cleans up.
+         */
+        this.capturedOutputDiscarded = false;
+        /**
+         * Set when writing the capture failed. Distinct from discarded: the bytes
+         * already on disk are still the best record of the batch, so the file is kept
+         * and rendered, while everything after the failure goes live to the terminal.
+         */
+        this.capturedOutputFailed = false;
         this.childProcess.on('message', (message) => {
             switch (message.type) {
                 case batch_messages_1.BatchMessageType.CompleteTask: {
@@ -47,24 +63,39 @@ class BatchProcess {
         // Capture stdout output
         if (this.childProcess.stdout) {
             this.childProcess.stdout.on('data', (chunk) => {
-                const output = chunk.toString();
-                // Maintain current terminal output behavior
-                process.stdout.write(chunk);
+                const text = chunk.toString();
+                // When batch output is being folded, the live copy is suppressed to
+                // keep each group contiguous; it is retained (see capturedOutputPath)
+                // for the renderings that need it. Otherwise, maintain
+                // current terminal output behavior. These chunks are forwarded raw and
+                // routinely end mid-line, so they go through `output` to keep its line
+                // tracking accurate for whatever prints next.
+                if ((0, output_1.shouldGroupBatchOutput)()) {
+                    this.capture(chunk);
+                }
+                else {
+                    output_1.output.writeTaskOutputChunk(chunk);
+                }
                 // Notify callbacks for TUI
                 for (const cb of this.outputCallbacks) {
-                    cb(output);
+                    cb(text);
                 }
             });
         }
         // Capture stderr output
         if (this.childProcess.stderr) {
             this.childProcess.stderr.on('data', (chunk) => {
-                const output = chunk.toString();
-                // Maintain current terminal output behavior
-                process.stderr.write(chunk);
+                const text = chunk.toString();
+                if ((0, output_1.shouldGroupBatchOutput)()) {
+                    this.capture(chunk, process.stderr);
+                }
+                else {
+                    // Maintain current terminal output behavior
+                    output_1.output.writeTaskOutputChunk(chunk, process.stderr);
+                }
                 // Notify callbacks for TUI
                 for (const cb of this.outputCallbacks) {
-                    cb(output);
+                    cb(text);
                 }
             });
         }
@@ -81,6 +112,127 @@ class BatchProcess {
     onOutput(cb) {
         this.outputCallbacks.push(cb);
     }
+    capture(chunk, stream = process.stdout) {
+        // Only a released capture stops recording; a handed-over one keeps
+        // appending until the fold is rendered.
+        if (this.capturedOutputDiscarded) {
+            return;
+        }
+        if (this.capturedOutputFailed) {
+            // Capture is over, but the output is not optional. Everything from here
+            // goes straight to the terminal, on the stream it arrived on.
+            output_1.output.writeTaskOutputChunk(chunk, stream);
+            return;
+        }
+        // Hoisted so the catch can tell how much of this chunk reached the file: a
+        // `writeSync` that fails partway leaves a prefix on disk, and replaying the
+        // whole chunk live would print that prefix twice - once now, once when the
+        // fold reads the file.
+        const bytes = typeof chunk === 'string' ? Buffer.from(chunk) : chunk;
+        let written = 0;
+        try {
+            if (this.capturedOutputFd === undefined) {
+                const dir = (0, path_1.join)(cache_directory_1.workspaceDataDirectory, 'batch-outputs');
+                (0, fs_1.mkdirSync)(dir, { recursive: true });
+                const name = `${this.executorName.replace(/[^a-zA-Z0-9]+/g, '-')}-${process.pid}-${++BatchProcess.capturedOutputCount}.log`;
+                const path = (0, path_1.join)(dir, name);
+                // Assigned only once the file is actually open, so a path always names
+                // an fd rather than a file that failed to open.
+                const fd = (0, fs_1.openSync)(path, 'w');
+                this.capturedOutputPath = path;
+                this.capturedOutputFd = fd;
+            }
+            // Written synchronously so the file is complete the moment the batch ends,
+            // with no flush to sequence against the read that renders the fold.
+            while (written < bytes.length) {
+                written += (0, fs_1.writeSync)(this.capturedOutputFd, bytes, written);
+            }
+        }
+        catch (e) {
+            // This runs inside a stream 'data' handler, where a throw is an uncaught
+            // exception rather than something the orchestrator's try can see - so a
+            // full disk or a read-only data directory would take down a run that
+            // otherwise had nothing wrong with it. The capture is an optimization
+            // that the common path discards anyway, so give it up and put the bytes
+            // back on the terminal instead of losing both them and the run.
+            // Latch on a flag of its own rather than reusing the discard flag: that
+            // one makes `capture` return early, which would drop every later chunk
+            // on the floor - captured nowhere and printed nowhere.
+            this.capturedOutputFailed = true;
+            // Close the fd but keep the file. Whatever was written before the failure
+            // is still the head of the batch's log, which is where a compiler's first
+            // non-cascading errors live; unlinking it here would throw away the most
+            // useful bytes precisely when the disk is in trouble.
+            this.closeCapturedOutput();
+            // Forward only what did not reach the file, and do it before warning.
+            // `output.warn` writes to the terminal as well, so if that is the thing
+            // failing, the bytes this handler was handed must already be out - losing
+            // the warning is survivable, losing task output is what this path exists
+            // to prevent.
+            const unwritten = bytes.subarray(written);
+            if (unwritten.length > 0) {
+                output_1.output.writeTaskOutputChunk(unwritten, stream);
+            }
+            output_1.output.warn({
+                title: `Could not capture batch output for ${this.executorName}`,
+                bodyLines: [
+                    e.message,
+                    'Streaming the rest of it live instead; the fold holds what was captured first.',
+                ],
+            });
+        }
+    }
+    /**
+     * Path to the file holding everything held back from the live stream under
+     * log grouping, or undefined if nothing was captured. Used to render the whole
+     * batch as one fold, so output no task claimed is not lost.
+     *
+     * The file is deliberately left open rather than closed here. A worker's
+     * stdout can deliver after its exit event — which is what `getResults()`
+     * settles on — and leaving the fd open keeps such a chunk appending to this
+     * same file instead of minting a second numbered one that nothing cleans up.
+     * The caller reads the file once, synchronously, while rendering the fold, so
+     * anything arriving after that read is not shown; writes are unbuffered, so
+     * the read always sees a complete prefix of what has arrived by then.
+     */
+    getCapturedOutputPath() {
+        return this.capturedOutputPath;
+    }
+    /** Releases the capture file. Safe to call more than once. */
+    discardCapturedOutput() {
+        this.capturedOutputDiscarded = true;
+        this.releaseCapturedOutput();
+    }
+    /**
+     * Closes the fd and unlinks the file, leaving no path behind for a caller to
+     * read. Tolerates a partially-initialized capture, since it also runs when
+     * opening or writing the file is what failed.
+     */
+    releaseCapturedOutput() {
+        this.closeCapturedOutput();
+        if (this.capturedOutputPath) {
+            try {
+                (0, fs_1.rmSync)(this.capturedOutputPath, { force: true });
+            }
+            catch {
+                // `force` covers a missing file but not a locked one: on Windows an fd
+                // that failed to close still holds a share lock. This runs from
+                // `runBatch`'s finally, where throwing would replace the batch's real
+                // result with a cleanup error, so a stale file is the lesser outcome.
+            }
+            this.capturedOutputPath = undefined;
+        }
+    }
+    /** Closes the fd, keeping the file and its path readable. */
+    closeCapturedOutput() {
+        if (this.capturedOutputFd !== undefined) {
+            try {
+                (0, fs_1.closeSync)(this.capturedOutputFd);
+            }
+            catch { }
+            this.capturedOutputFd = undefined;
+        }
+    }
     async getResults() {
         return Promise.race([
             new Promise((_, rej) => {
@@ -108,3 +260,4 @@ class BatchProcess {
     }
 }
 exports.BatchProcess = BatchProcess;
+BatchProcess.capturedOutputCount = 0;

package/dist/src/tasks-runner/task-env.d.ts

@@ -1,5 +1,14 @@
 import { ProjectGraph } from '../config/project-graph';
 import { Task } from '../config/task-graph';
+/**
+ * Resolves the FORCE_COLOR value for forked child processes.
+ *
+ * When the user sets FORCE_COLOR=0, bin/nx.ts deletes it from process.env
+ * (workaround for picocolors treating "0" as truthy) and saves the original
+ * value in NX_ORIGINAL_FORCE_COLOR. Without this check, the undefined
+ * FORCE_COLOR would default to 'true', re-enabling colors in all children.
+ */
+export declare function getForceColorForChild(): string;
 export declare function getEnvVariablesForBatchProcess(skipNxCache: boolean, captureStderr: boolean): NodeJS.ProcessEnv;
 export declare function getTaskSpecificEnv(task: Task, graph: ProjectGraph): NodeJS.ProcessEnv;
 export declare function getEnvVariablesForTask(task: Task, taskSpecificEnv: NodeJS.ProcessEnv, forceColor: string, skipNxCache: boolean, captureStderr: boolean, outputPath: string, streamOutput: boolean): NodeJS.ProcessEnv;

package/dist/src/tasks-runner/task-env.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.getForceColorForChild = getForceColorForChild;
 exports.getEnvVariablesForBatchProcess = getEnvVariablesForBatchProcess;
 exports.getTaskSpecificEnv = getTaskSpecificEnv;
 exports.getEnvVariablesForTask = getEnvVariablesForTask;
@@ -11,13 +12,33 @@ const dotenv_expand_1 = require("dotenv-expand");
 const node_path_1 = require("node:path");
 const workspace_root_1 = require("../utils/workspace-root");
 const task_env_paths_1 = require("./task-env-paths");
+/**
+ * Resolves the FORCE_COLOR value for forked child processes.
+ *
+ * When the user sets FORCE_COLOR=0, bin/nx.ts deletes it from process.env
+ * (workaround for picocolors treating "0" as truthy) and saves the original
+ * value in NX_ORIGINAL_FORCE_COLOR. Without this check, the undefined
+ * FORCE_COLOR would default to 'true', re-enabling colors in all children.
+ */
+function getForceColorForChild() {
+    if (process.env.FORCE_COLOR !== undefined) {
+        return process.env.FORCE_COLOR;
+    }
+    if (process.env.NX_ORIGINAL_FORCE_COLOR === '0') {
+        return '0';
+    }
+    return 'true';
+}
 function getEnvVariablesForBatchProcess(skipNxCache, captureStderr) {
-    return {
+    const res = {
         // User Process Env Variables override Dotenv Variables
         ...process.env,
         // Nx Env Variables overrides everything
-        ...getNxEnvVariablesForForkedProcess(process.env.FORCE_COLOR === undefined ? 'true' : process.env.FORCE_COLOR, skipNxCache, captureStderr),
+        ...getNxEnvVariablesForForkedProcess(getForceColorForChild(), skipNxCache, captureStderr),
     };
+    // NX_ORIGINAL_FORCE_COLOR is an internal signal and should not leak into child processes
+    delete res.NX_ORIGINAL_FORCE_COLOR;
+    return res;
 }
 // The orchestrator now calls this eagerly during the coordinator pre-hash
 // in addition to processTask (and again in hashBatchTasks), so the same
@@ -58,6 +79,9 @@ function getEnvVariablesForTask(task, taskSpecificEnv, forceColor, skipNxCache,
     }
     // we don't reset NX_BASE or NX_HEAD because those are set by the user and should be preserved
     delete res.NX_SET_CLI;
+    // NX_ORIGINAL_FORCE_COLOR is an internal signal used by getForceColorForChild()
+    // and should not leak into child processes
+    delete res.NX_ORIGINAL_FORCE_COLOR;
     return res;
 }
 function getNxEnvVariablesForForkedProcess(forceColor, skipNxCache, captureStderr, outputPath, streamOutput) {
@@ -142,10 +166,13 @@ function getOwnerTargetForTask(task, graph) {
         for (const targets of Object.values(project.data.metadata.targetGroups)) {
             if (targets.includes(task.target.target)) {
                 for (const target of targets) {
-                    if (project.data.targets[target].metadata?.nonAtomizedTarget) {
+                    // a target group can name a target the project does not define —
+                    // inference plugins build the group and the targets separately, so
+                    // the two can disagree. skip the phantom rather than dereferencing it.
+                    if (project.data.targets[target]?.metadata?.nonAtomizedTarget) {
                         return [
                             target,
-                            project.data.targets[target].metadata?.nonAtomizedTarget,
+                            project.data.targets[target]?.metadata?.nonAtomizedTarget,
                         ];
                     }
                 }

package/dist/src/tasks-runner/task-orchestrator.d.ts

@@ -35,11 +35,18 @@ export declare class TaskOrchestrator {
     private reverseTaskDeps;
     private initializingTaskIds;
     private processedTasks;
+    private cacheMissedHashes;
     private completedTasks;
     private waitingForTasks;
     private pendingDiscreteWorkers;
     private groups;
     private continuousTasksStarted;
+    /**
+     * How many folds each batch id has rendered. A batch that reports a strict
+     * subset of its tasks is re-run under the same id, so one id can produce more
+     * than one fold and the redirect lines have to point at the right one.
+     */
+    private batchFoldRenders;
     private bailed;
     private resolveStopPromise;
     private stopRequested;
@@ -100,7 +107,9 @@ export declare class TaskOrchestrator {
      * The coordinator relies on this running unconditionally (when cache
      * is enabled): tasks dispatched in step 5 via runTaskDirectly skip
      * their own cache lookup on the assumption that this has already
-     * confirmed them as misses. Don't add length-based bails.
+     * confirmed them as misses. Excluding cacheMissedHashes preserves that
+     * invariant — every dispatched hash was queried exactly once — but
+     * don't add other length-based bails.
      */
     private resolveCachedTasksBulk;
     /**
@@ -115,6 +124,68 @@ export declare class TaskOrchestrator {
     private hashBatchTasks;
     applyFromCacheOrRunBatch(doNotSkipCache: boolean, batch: Batch, groupId: number): Promise<TaskResult[]>;
     private runBatch;
+    /**
+     * Rendering a batch's output must never change the batch's results. A throw
+     * from the printer would otherwise land in `runBatch`'s own error handling:
+     * on the resolved path it rewrites every task to `failure` with the printer's
+     * stack as its output — reporting a green build red to the life cycles and Nx
+     * Cloud — and on the crash path it escapes `runBatch`, replacing the built
+     * failure results with the printer's error. Both call sites degrade to a
+     * warning here instead.
+     */
+    private renderBatchOutputSafely;
+    /**
+     * Prints a completed batch's output once, under log grouping. Live forwarding
+     * was suppressed while grouping, so this is the only copy — which is why the
+     * requested output style has to reach this path rather than stopping at the
+     * life cycle.
+     *
+     * Two things are rendered, and they answer different questions.
+     *
+     * Every task always renders through the life cycle, exactly as in a non-batch
+     * run - failures in full, successes collapsed to a line for run-many, plus the
+     * initiating project in full for run-one. That is what attributes output to a
+     * task, and it is the only place some of it exists: `@nx/jest` synthesizes
+     * each task's `terminalOutput` from an aggregated result and never writes
+     * those per-project summaries to the worker's stdio at all.
+     *
+     * The worker's whole captured log is rendered as a fold above them when the
+     * run asked for full output, or when any task failed or was stopped. A
+     * diagnostic that explains a failure is routinely one no task claimed:
+     * `@nx/maven`'s batch impl writes its exit-code dump and failed-task outputs
+     * to the worker's stderr via `console.error`, and the Maven JVM it spawns
+     * points slf4j at `System.out` because its own stderr carries the result
+     * protocol - two layers, two streams, both captured and neither attributed to
+     * a task - and `@nx/gradle` emits configuration-phase errors before the first
+     * `> Task :x:y` header tells it which task to attribute to. Both catch their
+     * own crash and backfill task results, so the batch resolves and lands here
+     * rather than in the caller's failure path.
+     *
+     * Rendering both duplicates some bytes, deliberately. `@nx/maven` and
+     * `@nx/gradle` tee each task's output into the worker's stdio on the way to
+     * `terminalOutput`, so a failing task's body appears in the fold and again in
+     * its own block. That is bounded on the default style, where successes
+     * collapse to a line each and a crashed batch backfills a short
+     * `e.toString()` rather than a body, so the case with the largest fold
+     * duplicates the least. Under a full-output style it is not bounded: every
+     * task prints in full beside a log that already contains it, which is the
+     * price of that style asking for everything. What it buys either way is
+     * attribution the fold cannot express. The
+     * alternative, letting the fold replace per-task rendering, silently dropped
+     * `@nx/jest`'s summaries and is what this shape exists to avoid.
+     *
+     * A batch that never reported results is handled by the caller instead.
+     */
+    private printGroupedBatchOutput;
+    /**
+     * Renders a batch's whole output as one fold, plus — unless `redirectLines`
+     * is off — a line per task pointing at it. The fold is labelled with the
+     * executor and a run-unique id (the same
+     * executor can run more than one batch), rather than an arbitrary task. Safe
+     * to write to `output` directly: grouping implies GitHub Actions implies a
+     * non-TTY, static lifecycle.
+     */
+    private printBatchFold;
     /**
      * Bulk-resolve cache hits for a set of tasks: fetch cached entries,
      * copy outputs as needed, fire lifecycle, and return the TaskResults