nx 23.3.0-beta.0 → 23.3.0-beta.2

package/dist/src/tasks-runner/running-tasks/batch-process.js

@@ -2,32 +2,72 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BatchProcess = void 0;
 const fs_1 = require("fs");
+const crypto_1 = require("crypto");
 const path_1 = require("path");
 const native_1 = require("../../native");
-const cache_directory_1 = require("../../utils/cache-directory");
+const cache_1 = require("../cache");
 const exit_codes_1 = require("../../utils/exit-codes");
 const output_1 = require("../../utils/output");
 const batch_messages_1 = require("../batch/batch-messages");
 class BatchProcess {
-    constructor(childProcess, executorName) {
+    constructor(childProcess, executorName, 
+    /**
+     * Whether the active output style puts task bytes on the terminal at all.
+     * `summary` does not, and its life cycle cannot enforce that here: this
+     * class writes to `output` directly from a stream handler, so without being
+     * told it would print a whole batch into a run that asked for log paths.
+     *
+     * False makes this capture rather than print. Suppressing the write alone
+     * would drop the bytes entirely off GitHub Actions, where nothing else
+     * captures them - and the worker's own crash output is exactly what no task
+     * ever claims, so it would exist nowhere.
+     */
+    printsOutput = true, 
+    /** Labels the capture file so it is identifiable in `batchOutputs/`. */
+    batchId = executorName) {
         this.childProcess = childProcess;
         this.executorName = executorName;
+        this.printsOutput = printsOutput;
+        this.batchId = batchId;
         this.exitCallbacks = [];
         this.batchResultsCallbacks = [];
         this.taskResultsCallbacks = [];
         this.outputCallbacks = [];
+        /** Sources paused for backpressure, awaiting a 'drain' that may never come. */
+        this.pausedSources = new Set();
         /**
          * Set once the capture is released. A chunk can still arrive after that —
-         * stdout delivers past the exit event — and reopening then would mint a
-         * second numbered file that nothing ever cleans up.
+         * stdout delivers past the exit event — and writing then would reach a file
+         * the renderer has already read.
          */
         this.capturedOutputDiscarded = false;
         /**
-         * Set when writing the capture failed. Distinct from discarded: the bytes
-         * already on disk are still the best record of the batch, so the file is kept
-         * and rendered, while everything after the failure goes live to the terminal.
+         * Set when the capture file could not be written. Whatever reached it is kept
+         * and still rendered: that is the head of the batch's log, where a compiler's
+         * first non-cascading errors are. Everything after it goes to the terminal -
+         * see `capture`.
          */
         this.capturedOutputFailed = false;
+        /** Set once the failure has been reported, so it is said once per batch. */
+        this.capturedOutputWarned = false;
+        /** Set before `end()`: a write after it would destroy the unflushed tail. */
+        this.capturedOutputClosing = false;
+        /**
+         * Makes the capture file unique across every batch in the process, which
+         * `batchId` alone does not: `TasksSchedule.batchCounters` is per instance, so
+         * a second orchestrator in the same process - `runDiscreteTasks` and
+         * `runContinuousTasks`, not the CLI - re-mints `<executor> 1`, and the pid
+         * cannot separate them.
+         */
+        this.captureSeq = ++BatchProcess.captureSeq;
+        /**
+         * Separates this capture from one minted by a DIFFERENT process, which
+         * `captureSeq` cannot: it is a per-process static that restarts at 1. These
+         * files now outlive their batch and `cacheDir` can be shared across checkouts
+         * of a repo, so without this a recycled pid truncates a log an earlier run's
+         * summary is still pointing at.
+         */
+        this.captureNonce = (0, crypto_1.randomBytes)(4).toString('hex');
         this.childProcess.on('message', (message) => {
             switch (message.type) {
                 case batch_messages_1.BatchMessageType.CompleteTask: {
@@ -70,8 +110,8 @@ class BatchProcess {
                 // current terminal output behavior. These chunks are forwarded raw and
                 // routinely end mid-line, so they go through `output` to keep its line
                 // tracking accurate for whatever prints next.
-                if ((0, output_1.shouldGroupBatchOutput)()) {
-                    this.capture(chunk);
+                if ((0, output_1.shouldGroupBatchOutput)() || !this.printsOutput) {
+                    this.capture(chunk, this.childProcess.stdout);
                 }
                 else {
                     output_1.output.writeTaskOutputChunk(chunk);
@@ -86,8 +126,8 @@ class BatchProcess {
         if (this.childProcess.stderr) {
             this.childProcess.stderr.on('data', (chunk) => {
                 const text = chunk.toString();
-                if ((0, output_1.shouldGroupBatchOutput)()) {
-                    this.capture(chunk, process.stderr);
+                if ((0, output_1.shouldGroupBatchOutput)() || !this.printsOutput) {
+                    this.capture(chunk, this.childProcess.stderr);
                 }
                 else {
                     // Maintain current terminal output behavior
@@ -112,88 +152,144 @@ class BatchProcess {
     onOutput(cb) {
         this.outputCallbacks.push(cb);
     }
-    capture(chunk, stream = process.stdout) {
-        // Only a released capture stops recording; a handed-over one keeps
-        // appending until the fold is rendered.
-        if (this.capturedOutputDiscarded) {
+    capture(chunk, source) {
+        // A closed or released capture drops the chunk; see `getCapturedOutputPath`.
+        if (this.capturedOutputDiscarded || this.capturedOutputClosing) {
             return;
         }
+        // A failed one does not. The style withheld these bytes on the promise that
+        // they are readable elsewhere; once the file is unwritable that promise
+        // cannot be kept, so they go to the terminal even under a style that prints
+        // nothing. Losing the format is survivable, losing task output is what this
+        // path exists to prevent.
         if (this.capturedOutputFailed) {
-            // Capture is over, but the output is not optional. Everything from here
-            // goes straight to the terminal, on the stream it arrived on.
-            output_1.output.writeTaskOutputChunk(chunk, stream);
+            this.forwardUncaptured(chunk, source);
+            return;
+        }
+        const stream = this.openCapturedOutput();
+        if (!stream) {
+            // Opening is what failed, so nothing of this chunk reached disk.
+            this.forwardUncaptured(chunk, source);
+            return;
+        }
+        // A full write buffer pauses the worker rather than growing in this
+        // process. Tracked, because only 'drain' resumes it and a stream that
+        // errors never emits one - see `resumeCapturedSources`.
+        if (!stream.write(chunk) && source) {
+            this.pausedSources.add(source);
+            source.pause();
+            stream.once('drain', () => {
+                this.pausedSources.delete(source);
+                source.resume();
+            });
+        }
+    }
+    /**
+     * Puts bytes the capture could not take back on the terminal, on the stream
+     * they arrived on.
+     */
+    forwardUncaptured(chunk, source) {
+        output_1.output.writeTaskOutputChunk(chunk, source === this.childProcess.stderr ? process.stderr : process.stdout);
+    }
+    /**
+     * Names the failure once. A stream reports it asynchronously, so the chunk
+     * that tripped it may or may not have reached disk - unlike a `writeSync`,
+     * there is no byte count to tell. Every chunk after it is forwarded.
+     */
+    warnCaptureFailed(reason) {
+        if (this.capturedOutputWarned) {
             return;
         }
-        // Hoisted so the catch can tell how much of this chunk reached the file: a
-        // `writeSync` that fails partway leaves a prefix on disk, and replaying the
-        // whole chunk live would print that prefix twice - once now, once when the
-        // fold reads the file.
-        const bytes = typeof chunk === 'string' ? Buffer.from(chunk) : chunk;
-        let written = 0;
+        this.capturedOutputWarned = true;
+        output_1.output.warn({
+            title: `Could not capture batch output for ${this.executorName}`,
+            bodyLines: [reason, 'Streaming the rest of it live instead.'],
+        });
+    }
+    /**
+     * Lets the worker write again after the capture has stopped.
+     *
+     * Without this a failed capture wedges the whole run: the source stays
+     * paused, so the worker blocks once its stdout pipe fills, never sends its
+     * results, and `getResults` waits on a batch that can no longer finish.
+     */
+    resumeCapturedSources() {
+        const resumed = [...this.pausedSources];
+        for (const source of resumed) {
+            source.resume();
+        }
+        this.pausedSources.clear();
+        return resumed;
+    }
+    openCapturedOutput() {
+        if (this.capturedOutputStream) {
+            return this.capturedOutputStream;
+        }
         try {
-            if (this.capturedOutputFd === undefined) {
-                const dir = (0, path_1.join)(cache_directory_1.workspaceDataDirectory, 'batch-outputs');
-                (0, fs_1.mkdirSync)(dir, { recursive: true });
-                const name = `${this.executorName.replace(/[^a-zA-Z0-9]+/g, '-')}-${process.pid}-${++BatchProcess.capturedOutputCount}.log`;
-                const path = (0, path_1.join)(dir, name);
-                // Assigned only once the file is actually open, so a path always names
-                // an fd rather than a file that failed to open.
-                const fd = (0, fs_1.openSync)(path, 'w');
-                this.capturedOutputPath = path;
-                this.capturedOutputFd = fd;
-            }
-            // Written synchronously so the file is complete the moment the batch ends,
-            // with no flush to sequence against the read that renders the fold.
-            while (written < bytes.length) {
-                written += (0, fs_1.writeSync)(this.capturedOutputFd, bytes, written);
-            }
+            // `batchId` names the file for a human reading the directory;
+            // `captureSeq` separates batches within this process and `captureNonce`
+            // separates it from every other process - see both fields.
+            const key = `${this.batchId.replace(/[^a-zA-Z0-9]+/g, '-')}-${process.pid}-${this.captureSeq}-${this.captureNonce}`;
+            const path = (0, cache_1.batchOutputPathForKey)(key);
+            (0, fs_1.mkdirSync)((0, path_1.dirname)(path), { recursive: true });
+            const stream = (0, fs_1.createWriteStream)(path);
+            // Mandatory, not defensive: a write error reaches a stream as an 'error'
+            // event, and an unhandled one is an uncaught exception that would take
+            // down a run with nothing else wrong with it. Stop capturing, keep what
+            // already reached the file — the head of the batch's log, where a
+            // compiler's first non-cascading errors are — and put everything after
+            // it on the terminal.
+            stream.on('error', (e) => {
+                this.capturedOutputFailed = true;
+                this.resumeCapturedSources();
+                this.warnCaptureFailed(e.message);
+            });
+            this.capturedOutputPath = path;
+            this.capturedOutputStream = stream;
         }
         catch (e) {
-            // This runs inside a stream 'data' handler, where a throw is an uncaught
-            // exception rather than something the orchestrator's try can see - so a
-            // full disk or a read-only data directory would take down a run that
-            // otherwise had nothing wrong with it. The capture is an optimization
-            // that the common path discards anyway, so give it up and put the bytes
-            // back on the terminal instead of losing both them and the run.
-            // Latch on a flag of its own rather than reusing the discard flag: that
-            // one makes `capture` return early, which would drop every later chunk
-            // on the floor - captured nowhere and printed nowhere.
+            // mkdir is the only synchronous throw here; the stream reports its own
+            // failures through the handler above.
             this.capturedOutputFailed = true;
-            // Close the fd but keep the file. Whatever was written before the failure
-            // is still the head of the batch's log, which is where a compiler's first
-            // non-cascading errors live; unlinking it here would throw away the most
-            // useful bytes precisely when the disk is in trouble.
-            this.closeCapturedOutput();
-            // Forward only what did not reach the file, and do it before warning.
-            // `output.warn` writes to the terminal as well, so if that is the thing
-            // failing, the bytes this handler was handed must already be out - losing
-            // the warning is survivable, losing task output is what this path exists
-            // to prevent.
-            const unwritten = bytes.subarray(written);
-            if (unwritten.length > 0) {
-                output_1.output.writeTaskOutputChunk(unwritten, stream);
-            }
-            output_1.output.warn({
-                title: `Could not capture batch output for ${this.executorName}`,
-                bodyLines: [
-                    e.message,
-                    'Streaming the rest of it live instead; the fold holds what was captured first.',
-                ],
-            });
+            this.warnCaptureFailed(e.message);
+        }
+        return this.capturedOutputStream;
+    }
+    /**
+     * Flushes and closes the capture so the file is complete before a caller
+     * reads it.
+     */
+    async flushCapturedOutput() {
+        // Before the early return, and unconditionally: `end()` means a pending
+        // 'drain' will never arrive, so a source paused for backpressure has
+        // nothing left to resume it. Measured: 20/20 runs reach 'finish' with the
+        // source still paused.
+        const resumed = this.resumeCapturedSources();
+        // `resume()` delivers on the next tick while `end()` takes effect in this
+        // one, so ending here would strand everything the source still holds - the
+        // tail of the log, which is where a build tool puts what went wrong.
+        // Bounded, because a source that never empties must not hold the run open.
+        for (let turn = 0; turn < BatchProcess.MAX_DRAIN_TURNS &&
+            resumed.some((source) => source.readableLength > 0); turn++) {
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+        this.capturedOutputClosing = true;
+        // The loop's own writes can re-pause a source.
+        this.resumeCapturedSources();
+        const stream = this.capturedOutputStream;
+        if (!stream || stream.destroyed || stream.writableEnded) {
+            return;
         }
+        await new Promise((resolve) => stream.end(resolve));
     }
     /**
      * Path to the file holding everything held back from the live stream under
      * log grouping, or undefined if nothing was captured. Used to render the whole
      * batch as one fold, so output no task claimed is not lost.
      *
-     * The file is deliberately left open rather than closed here. A worker's
-     * stdout can deliver after its exit event — which is what `getResults()`
-     * settles on — and leaving the fd open keeps such a chunk appending to this
-     * same file instead of minting a second numbered one that nothing cleans up.
-     * The caller reads the file once, synchronously, while rendering the fold, so
-     * anything arriving after that read is not shown; writes are unbuffered, so
-     * the read always sees a complete prefix of what has arrived by then.
+     * Not valid to read before `flushCapturedOutput`; that flush ends the stream,
+     * so a chunk arriving afterwards - a worker's stdout can deliver past the
+     * exit event `getResults()` settles on - is dropped rather than appended.
      */
     getCapturedOutputPath() {
         return this.capturedOutputPath;
@@ -204,9 +300,9 @@ class BatchProcess {
         this.releaseCapturedOutput();
     }
     /**
-     * Closes the fd and unlinks the file, leaving no path behind for a caller to
-     * read. Tolerates a partially-initialized capture, since it also runs when
-     * opening or writing the file is what failed.
+     * Destroys the stream and unlinks the file, leaving no path behind for a
+     * caller to read. Tolerates a partially-initialized capture, since it also
+     * runs when opening or writing the file is what failed.
      */
     releaseCapturedOutput() {
         this.closeCapturedOutput();
@@ -223,14 +319,12 @@ class BatchProcess {
             this.capturedOutputPath = undefined;
         }
     }
-    /** Closes the fd, keeping the file and its path readable. */
+    /** Destroys the stream, dropping anything still buffered. The file stays. */
     closeCapturedOutput() {
-        if (this.capturedOutputFd !== undefined) {
-            try {
-                (0, fs_1.closeSync)(this.capturedOutputFd);
-            }
-            catch { }
-            this.capturedOutputFd = undefined;
+        this.resumeCapturedSources();
+        if (this.capturedOutputStream) {
+            this.capturedOutputStream.destroy();
+            this.capturedOutputStream = undefined;
         }
     }
     // Resolves on the CompleteBatchExecution message, not process exit: the
@@ -263,4 +357,6 @@ class BatchProcess {
     }
 }
 exports.BatchProcess = BatchProcess;
-BatchProcess.capturedOutputCount = 0;
+BatchProcess.captureSeq = 0;
+/** Turns `flushCapturedOutput` waits for a resumed source to empty. */
+BatchProcess.MAX_DRAIN_TURNS = 100;

package/dist/src/tasks-runner/task-orchestrator.d.ts

@@ -1,3 +1,4 @@
+import type { OutputStyle } from '../command-line/yargs-utils/shared-options';
 import { NxJsonConfiguration } from '../config/nx-json';
 import { ProjectGraph } from '../config/project-graph';
 import { Task, TaskGraph } from '../config/task-graph';
@@ -20,7 +21,10 @@ export declare class TaskOrchestrator {
     private readonly options;
     private readonly bail;
     private readonly daemon;
-    private readonly outputStyle;
+    /** What the user named; undefined means they named nothing. */
+    private readonly specifiedOutputStyle;
+    /** What this run renders with, after defaults. */
+    private readonly resolvedOutputStyle;
     private readonly fullTaskGraph;
     private taskDetails;
     private cache;
@@ -57,7 +61,14 @@ export declare class TaskOrchestrator {
     private continuousTaskExitHandled;
     private cleanupPromise;
     private signalHandlers;
-    constructor(hasher: TaskHasher, initiatingProject: string | undefined, initiatingTasks: Task[], projectGraph: ProjectGraph, taskGraph: TaskGraph, nxJson: NxJsonConfiguration, options: NxArgs & DefaultTasksRunnerOptions, bail: boolean, daemon: DaemonClient, outputStyle: string, fullTaskGraph?: TaskGraph);
+    private tasksWithPersistedOutput;
+    /** Batches whose worker log was handed to the life cycle, so it must survive. */
+    private announcedBatchLogs;
+    constructor(hasher: TaskHasher, initiatingProject: string | undefined, initiatingTasks: Task[], projectGraph: ProjectGraph, taskGraph: TaskGraph, nxJson: NxJsonConfiguration, options: NxArgs & DefaultTasksRunnerOptions, bail: boolean, daemon: DaemonClient, 
+    /** What the user named; undefined means they named nothing. */
+    specifiedOutputStyle: OutputStyle | undefined, 
+    /** What this run renders with, after defaults. */
+    resolvedOutputStyle: OutputStyle, fullTaskGraph?: TaskGraph);
     init(): Promise<void>;
     run(): Promise<{
         [k: string]: TaskStatus;
@@ -125,6 +136,15 @@ export declare class TaskOrchestrator {
     private hashBatchTasks;
     applyFromCacheOrRunBatch(doNotSkipCache: boolean, batch: Batch, groupId: number): Promise<TaskResult[]>;
     private runBatch;
+    /**
+     * Tells the life cycle where the batch worker's own log is, when something
+     * failed and a style that prints nothing would otherwise leave it unread.
+     *
+     * The path is announced rather than the bytes copied into a task: one worker
+     * produces one log, it explains the batch rather than any single task, and a
+     * task's own output file is read back verbatim on a cache hit.
+     */
+    private announceBatchWorkerLog;
     /**
      * Rendering a batch's output must never change the batch's results. A throw
      * from the printer would otherwise land in `runBatch`'s own error handling:
@@ -229,6 +249,48 @@ export declare class TaskOrchestrator {
     startContinuousTask(task: Task, groupId: number): Promise<RunningTask | SharedRunningTask>;
     private preRunSteps;
     private postRunSteps;
+    /**
+     * The results `cache.put` will write, in the shape it wants them. Pulled out
+     * of postRunSteps so the terminal-output backstop can be told exactly which
+     * tasks the cache is already writing a file for.
+     */
+    private resultsToCache;
+    /**
+     * Guarantee that every task that actually ran leaves its terminal output at
+     * `<cacheDir>/terminalOutputs/<hash>`, whatever its cache setting, output
+     * style, or batch membership. That path is what `--output-style=summary`
+     * addresses each failure by, what the DB cache reads back on a hit
+     * (`build_cached_result`), and what Nx Cloud falls back to reading when a
+     * task's in-memory output is undefined, so a task that reaches a terminal
+     * state without a file there is a dangling reference.
+     *
+     * Batch tasks are why this exists. Their output only ever arrives over IPC,
+     * so the sole thing that ever put it on disk was `cache.put` — leaving a
+     * `cache:false` batch task (the shape CI runs under `NX_BATCH_MODE`) with no
+     * file at all.
+     *
+     * Written exactly once per task: tasks whose runner owns the file are
+     * recorded in `tasksWithPersistedOutput`, and tasks `cache.put` is about to
+     * write are skipped here. Cache hits are skipped by status — their output was
+     * read from this very file — except a replayed cached *failure*, which comes
+     * back as `failure` and is rewritten with the bytes it was just read with.
+     * Skipped tasks never ran, so there is nothing to write.
+     *
+     * Every file written without a cache entry behind it is then registered with
+     * the cache, because `removeOldCacheRecords` only collects hashes it finds in
+     * the database — an unregistered file would never be cleaned up. That covers
+     * files this method wrote AND ones a runner wrote, which are equally
+     * invisible to the GC.
+     */
+    private persistTerminalOutputs;
+    /**
+     * Register a continuous task's terminal output with the cache so the GC can
+     * collect it. Continuous tasks never reach postRunSteps — their file is
+     * written by whichever runner ran them, as the process exits — so this is
+     * the only place their hash gets recorded. Read off disk rather than from a
+     * result, because a continuous task completes without one.
+     */
+    private recordContinuousTerminalOutput;
     private scheduleNextTasksAndReleaseThreads;
     private complete;
     /**