@depup/nx 23.2.1-depup.7 → 23.3.0-depup.0

package/dist/src/tasks-runner/task-orchestrator.js

@@ -12,7 +12,6 @@ const project_graph_1 = require("../project-graph/project-graph");
 const run_commands_impl_1 = require("../executors/run-commands/run-commands.impl");
 const hash_task_1 = require("../hasher/hash-task");
 const task_graph_utils_1 = require("./task-graph-utils");
-const task_hasher_1 = require("../hasher/task-hasher");
 const native_1 = require("../native");
 const db_connection_1 = require("../utils/db-connection");
 const exit_codes_1 = require("../utils/exit-codes");
@@ -37,9 +36,24 @@ const utils_1 = require("./utils");
 function resolveBatchTaskStatus(result) {
     return result.status ?? (result.success ? 'success' : 'failure');
 }
+/** Whether a path names a file with bytes in it. */
+function hasContent(path) {
+    try {
+        return (0, fs_1.statSync)(path).size > 0;
+    }
+    catch {
+        return false;
+    }
+}
 class TaskOrchestrator {
     // endregion internal state
-    constructor(hasher, initiatingProject, initiatingTasks, projectGraph, taskGraph, nxJson, options, bail, daemon, outputStyle, fullTaskGraph = taskGraph) {
+    constructor(hasher, initiatingProject, initiatingTasks, projectGraph, taskGraph, nxJson, options, bail, daemon, 
+    /** What the user named; undefined means they named nothing. */
+    specifiedOutputStyle, 
+    /** What this run renders with, after defaults. */
+    resolvedOutputStyle, fullTaskGraph = taskGraph, 
+    /** Tasks the up-front pass left to hash once the tasks they read from have run. */
+    deferredTaskIds) {
         this.hasher = hasher;
         this.initiatingProject = initiatingProject;
         this.initiatingTasks = initiatingTasks;
@@ -49,8 +63,10 @@ class TaskOrchestrator {
         this.options = options;
         this.bail = bail;
         this.daemon = daemon;
-        this.outputStyle = outputStyle;
+        this.specifiedOutputStyle = specifiedOutputStyle;
+        this.resolvedOutputStyle = resolvedOutputStyle;
         this.fullTaskGraph = fullTaskGraph;
+        this.deferredTaskIds = deferredTaskIds;
         this.taskDetails = (0, hash_task_1.getTaskDetails)();
         this.cache = (0, cache_1.getCache)(this.options);
         this.tuiEnabled = (0, is_tui_enabled_1.isTuiEnabled)();
@@ -85,6 +101,9 @@ class TaskOrchestrator {
         // miss waiting for a worker slot is re-queried (including the remote
         // retrieval) on every coordinator cycle.
         this.cacheMissedHashes = new Set();
+        // The files the cache just wrote or restored, by task id, for postRunSteps
+        // to hand the daemon instead of it walking the outputs again.
+        this.copiedOutputFiles = new Map();
         this.completedTasks = new Map();
         this.waitingForTasks = [];
         this.pendingDiscreteWorkers = new Set();
@@ -106,6 +125,13 @@ class TaskOrchestrator {
         this.continuousTaskExitHandled = new Map();
         this.cleanupPromise = null;
         this.signalHandlers = [];
+        // Tasks whose runner already owns the write to
+        // `<cacheDir>/terminalOutputs/<hash>` — forked processes write it as they
+        // exit, and the paths that never spawn one write it inline. The backstop in
+        // postRunSteps consults this so a task's output is written exactly once.
+        this.tasksWithPersistedOutput = new Set();
+        /** Batches whose worker log was handed to the life cycle, so it must survive. */
+        this.announcedBatchLogs = new Set();
     }
     async init() {
         this.setupSignalHandlers();
@@ -158,6 +184,10 @@ class TaskOrchestrator {
         perf_hooks_1.performance.measure('task-execution', 'task-execution:start', 'task-execution:end');
         if (!this.stopRequested) {
             this.cache.removeOldCacheRecords();
+            // Free function, not a cache method: `BatchProcess` writes these logs
+            // whichever cache implementation is active. It no-ops under WASM, where
+            // the native half does not exist.
+            (0, cache_1.sweepBatchOutputs)();
         }
         await this.cleanup();
         await this.dispose();
@@ -377,7 +407,9 @@ class TaskOrchestrator {
         // Copy outputs in parallel for tasks that need it.
         await Promise.all(cacheHits.map(async ({ task, cachedResult }) => {
             if (shouldCopyMap.get(task.hash)) {
-                await this.cache.copyFilesFromCache(task.hash, cachedResult, task.outputs);
+                const files = await this.cache.copyFilesFromCache(task.hash, cachedResult, task.outputs);
+                if (files)
+                    this.copiedOutputFiles.set(task.id, files);
             }
         }));
         // Derive status, print terminal output, build results.
@@ -452,8 +484,8 @@ class TaskOrchestrator {
      * Hash all batch tasks and resolve cache hits topologically.
      *
      * Walks the task graph level by level. Every task gets a preliminary hash
-     * (so startTasks always has a valid hash for Cloud). Tasks with depsOutputs
-     * whose deps weren't cached are ineligible for cache lookup but still
+     * (so startTasks always has a valid hash for Cloud). Tasks the up-front
+     * pass deferred, whose deps weren't cached, are ineligible for cache lookup but still
      * receive a preliminary hash — they'll be re-hashed after execution.
      */
     async applyBatchCachedResults(batch, doNotSkipCache, groupId) {
@@ -473,8 +505,7 @@ class TaskOrchestrator {
             for (const task of rootTasks) {
                 const depIds = batch.taskGraph.dependencies[task.id];
                 const hasNonCachedDep = depIds.some((id) => nonCachedTaskIds.has(id));
-                if (hasNonCachedDep &&
-                    (0, task_hasher_1.getInputs)(task, this.projectGraph, this.nxJson).depsOutputs.length > 0) {
+                if (hasNonCachedDep && this.deferredTaskIds?.has(task.id)) {
                     nonCachedTaskIds.add(task.id);
                     needsRehashAfterExecution.add(task.id);
                 }
@@ -530,7 +561,7 @@ class TaskOrchestrator {
             await Promise.all(nonCachedTasks.map((task) => this.options.lifeCycle.scheduleTask(task)));
             await this.preRunSteps(nonCachedTasks, { groupId });
         }
-        // Phase 2: Run non-cached tasks, then re-hash depsOutputs tasks
+        // Phase 2: Run non-cached tasks, then re-hash tasks that read their outputs
         const taskIdsToSkip = cachedResults.map((r) => r.task.id);
         let batchResults = [];
         if (taskIdsToSkip.length < tasks.length) {
@@ -543,7 +574,7 @@ class TaskOrchestrator {
                 executorName: batch.executorName,
                 taskGraph: runGraph,
             }, this.batchEnv, groupId);
-            // Re-hash depsOutputs tasks — their dep outputs are now on disk
+            // Re-hash tasks that read dep outputs — those outputs are now on disk
             const tasksToRehash = batchResults
                 .filter((r) => needsRehashAfterExecution.has(r.task.id) &&
                 (r.status === 'success' || r.status === 'failure'))
@@ -586,7 +617,7 @@ class TaskOrchestrator {
         const runBatchStart = perf_hooks_1.performance.mark('TaskOrchestrator-run-batch:start');
         let batchProcess;
         try {
-            batchProcess = await this.forkedProcessTaskRunner.forkProcessForBatch(batch, this.projectGraph, this.fullTaskGraph, env);
+            batchProcess = await this.forkedProcessTaskRunner.forkProcessForBatch(batch, this.projectGraph, this.fullTaskGraph, env, (0, output_1.printsTaskOutput)(this.resolvedOutputStyle));
             // Stream output from batch process to the batch
             batchProcess.onOutput((output) => {
                 this.options.lifeCycle.appendBatchOutput?.(batch.id, output);
@@ -633,9 +664,13 @@ class TaskOrchestrator {
                     terminalOutput: result.terminalOutput,
                 };
             });
+            // The capture stream buffers, so both readers below would otherwise be
+            // racing bytes that have not reached the file yet.
+            await batchProcess.flushCapturedOutput();
             if ((0, output_1.shouldGroupBatchOutput)()) {
                 this.renderBatchOutputSafely(batch.id, () => this.printGroupedBatchOutput(batch, taskResults, batchProcess.getCapturedOutputPath()));
             }
+            this.announceBatchWorkerLog(batch.id, taskResults, batchProcess.getCapturedOutputPath());
             return taskResults;
         }
         catch (e) {
@@ -652,15 +687,19 @@ class TaskOrchestrator {
                     terminalOutput: isBatchStopping ? '' : (e.stack ?? e.message ?? ''),
                 };
             });
+            await batchProcess?.flushCapturedOutput();
+            this.announceBatchWorkerLog(batch.id, taskResults, batchProcess?.getCapturedOutputPath());
             // The worker died without reporting results, so nothing was attributed to
             // a task and no per-task output ran. Everything it wrote went to
-            // stdout/stderr, held back under log grouping — surface it as one fold.
-            // Outside grouping it already streamed live. This matters just as much
-            // when the batch was stopped: every task is marked stopped whether or not
-            // it finished, so the log is the only record of what got through. Only
-            // the exit-code error is dropped there, since it restates the
-            // cancellation.
-            if ((0, output_1.shouldGroupBatchOutput)()) {
+            // stdout/stderr, held back under grouping - surface it as one fold. Under
+            // a style that prints nothing there is no fold; the log is announced by
+            // path above instead. Outside both, it already streamed live. This
+            // matters just as much when the batch was stopped: every task
+            // is marked stopped whether or not it finished, so the log is the only
+            // record of what got through. Only the exit-code error is dropped there,
+            // since it restates the cancellation.
+            if ((0, output_1.shouldGroupBatchOutput)() &&
+                (0, output_1.printsTaskOutput)(this.resolvedOutputStyle)) {
                 const capturedOutputPath = batchProcess?.getCapturedOutputPath();
                 const trailer = isBatchStopping ? undefined : e.message;
                 if (capturedOutputPath || trailer) {
@@ -673,11 +712,42 @@ class TaskOrchestrator {
             return taskResults;
         }
         finally {
-            batchProcess?.discardCapturedOutput();
+            // Kept only when something failed and the log was announced; an all-green
+            // batch's chatter explains nothing and would sit in `batchOutputs/` until
+            // the sweep.
+            if (!this.announcedBatchLogs.has(batch.id)) {
+                batchProcess?.discardCapturedOutput();
+            }
             const runBatchEnd = perf_hooks_1.performance.mark('TaskOrchestrator-run-batch:end');
             perf_hooks_1.performance.measure('TaskOrchestrator-run-batch', runBatchStart.name, runBatchEnd.name);
         }
     }
+    /**
+     * Tells the life cycle where the batch worker's own log is, when something
+     * failed and a style that prints nothing would otherwise leave it unread.
+     *
+     * The path is announced rather than the bytes copied into a task: one worker
+     * produces one log, it explains the batch rather than any single task, and a
+     * task's own output file is read back verbatim on a cache hit.
+     */
+    announceBatchWorkerLog(batchId, results, capturedOutputPath) {
+        if (!capturedOutputPath || (0, output_1.printsTaskOutput)(this.resolvedOutputStyle)) {
+            return;
+        }
+        const failed = results.some((r) => r.status === 'failure' || r.status === 'stopped');
+        if (!failed) {
+            return;
+        }
+        // The path is minted when the file is opened, before any byte reaches it,
+        // so a capture that failed on its first write leaves one that exists and is
+        // empty. Announcing that offers the reader an address holding nothing.
+        // Checked after the flush, so the size is what a reader will actually see.
+        if (!hasContent(capturedOutputPath)) {
+            return;
+        }
+        this.announcedBatchLogs.add(batchId);
+        this.options.lifeCycle.batchOutputAvailable?.(batchId, capturedOutputPath);
+    }
     /**
      * Rendering a batch's output must never change the batch's results. A throw
      * from the printer would otherwise land in `runBatch`'s own error handling:
@@ -741,17 +811,20 @@ class TaskOrchestrator {
      * A batch that never reported results is handled by the caller instead.
      */
     printGroupedBatchOutput(batch, taskResults, capturedOutputPath) {
-        // Read from the same field the streaming decision uses. `this.options` has
-        // its own `outputStyle`, merged from `nx.json`'s tasksRunnerOptions, so the
-        // two disagree whenever a style is configured there but not named on the
-        // command line - and `init-tasks-runner` passes a populated `options` with
-        // no style argument at all, so on that path only `options` can carry one.
+        // Reads `resolvedOutputStyle` - what the run actually renders with. The
+        // streaming decision deliberately reads `specifiedOutputStyle` instead,
+        // since it has to tell a named style from an inferred default. `this.options`
+        // has its own `outputStyle`, merged from `nx.json`'s tasksRunnerOptions, so
+        // the two disagree whenever a style is configured there but not named on the
+        // command line.
         const printsFullOutput = (0, output_1.printsFullTaskOutput)({
             verbose: this.options.verbose,
-            outputStyle: this.outputStyle,
+            outputStyle: this.resolvedOutputStyle,
         });
         const batchOwnsTheDiagnostic = taskResults.some((r) => r.status === 'failure' || r.status === 'stopped');
-        if ((printsFullOutput || batchOwnsTheDiagnostic) && capturedOutputPath) {
+        if ((0, output_1.printsTaskOutput)(this.resolvedOutputStyle) &&
+            (printsFullOutput || batchOwnsTheDiagnostic) &&
+            capturedOutputPath) {
             // No redirect lines: every task renders itself below, so there is nothing
             // to redirect anyone to.
             this.printBatchFold(batch, taskResults, { capturedOutputPath }, { redirectLines: false });
@@ -866,6 +939,10 @@ class TaskOrchestrator {
         if (this.completedTasks.has(task.id))
             return;
         const terminalOutput = e?.message ?? '';
+        // The worker rejected, so whatever the runner was going to leave on disk
+        // either never landed or can't be trusted. Hand the file back to
+        // postRunSteps so the failure itself is what the task's output path holds.
+        this.tasksWithPersistedOutput.delete(task.id);
         this.options.lifeCycle.printTaskTerminalOutput(task, 'failure', terminalOutput);
         await this.postRunSteps([{ task, status: 'failure', terminalOutput }], doNotSkipCache, groupId);
     }
@@ -880,7 +957,12 @@ class TaskOrchestrator {
         await this.preRunSteps([task], { groupId });
         const pipeOutput = await this.pipeOutputCapture(task);
         const temporaryOutputPath = this.cache.temporaryOutputPath(task);
-        const streamOutput = (0, output_1.isStaticOutputStyle)(this.outputStyle)
+        // `summary` prints log paths rather than logs, and `shouldStreamOutput`
+        // would otherwise stream the initiating project's output in full - the one
+        // task a run-one is most likely to have. Continuous tasks are deliberately
+        // NOT suppressed the same way; see `startContinuousTask`.
+        const streamOutput = (0, output_1.isStaticOutputStyle)(this.specifiedOutputStyle) ||
+            !(0, output_1.printsTaskOutput)(this.resolvedOutputStyle)
             ? false
             : (0, utils_1.shouldStreamOutput)(task, this.initiatingProject);
         const env = pipeOutput
@@ -943,7 +1025,7 @@ class TaskOrchestrator {
                 };
                 const runningTask = await (0, run_commands_impl_1.runCommands)(runCommandsOptions, {
                     root: workspace_root_1.workspaceRoot, // only root is needed in runCommands
-                }, task.id);
+                }, task);
                 this.runningRunCommandsTasks.set(task.id, runningTask);
                 runningTask.onExit(() => {
                     this.runningRunCommandsTasks.delete(task.id);
@@ -970,13 +1052,18 @@ class TaskOrchestrator {
                         });
                     }
                 }
-                if (!streamOutput && !shouldPrefix) {
-                    // TODO: shouldn't this be checking if the task is continuous before writing anything to disk or calling printTaskTerminalOutput?
-                    runningTask.onExit((code, terminalOutput) => {
+                runningTask.onExit((code, terminalOutput) => {
+                    if (!streamOutput && !shouldPrefix) {
                         this.options.lifeCycle.printTaskTerminalOutput(task, code === 0 ? 'success' : 'failure', terminalOutput);
+                    }
+                    // A continuous task never reaches postRunSteps, so exiting is its
+                    // only chance to leave its output on disk. A discrete one is written
+                    // there instead, whatever its output style — writing here too would
+                    // just duplicate it.
+                    if (task.continuous) {
                         (0, fs_1.writeFileSync)(temporaryOutputPath, terminalOutput);
-                    });
-                }
+                    }
+                });
                 return runningTask;
             }
             catch (e) {
@@ -988,6 +1075,7 @@ class TaskOrchestrator {
                 }
                 const terminalOutput = e.stack ?? e.message ?? '';
                 (0, fs_1.writeFileSync)(temporaryOutputPath, terminalOutput);
+                this.tasksWithPersistedOutput.add(task.id);
                 return new noop_child_process_1.NoopChildProcess({
                     code: 1,
                     terminalOutput,
@@ -996,6 +1084,7 @@ class TaskOrchestrator {
         }
         else if (targetConfiguration.executor === 'nx:noop') {
             (0, fs_1.writeFileSync)(temporaryOutputPath, '');
+            this.tasksWithPersistedOutput.add(task.id);
             return new noop_child_process_1.NoopChildProcess({
                 code: 0,
                 terminalOutput: '',
@@ -1004,6 +1093,12 @@ class TaskOrchestrator {
         else {
             // cache prep
             const runningTask = await this.runTaskInForkedProcess(task, env, pipeOutput, temporaryOutputPath, streamOutput);
+            // Every forked path leaves the file behind whatever the output style:
+            // the pseudo-terminal and prefixed processes write it when they exit,
+            // the direct-output capture has the child write it itself, and the
+            // fork-failure fallback writes the error. Continuous tasks depend on
+            // that — they never reach postRunSteps.
+            this.tasksWithPersistedOutput.add(task.id);
             if (this.tuiEnabled) {
                 if (runningTask instanceof pseudo_terminal_1.PseudoTtyProcess) {
                     // This is an external of a the pseudo terminal where a task is running and can be passed to the TUI
@@ -1056,9 +1151,14 @@ class TaskOrchestrator {
             if (process.env.NX_VERBOSE_LOGGING === 'true') {
                 console.error(e);
             }
+            const terminalOutput = e.stack ?? e.message ?? '';
+            // No process ever started, so nothing else will write the file. Record
+            // why, so the task's terminal output path resolves to the fork error
+            // rather than to nothing.
+            (0, fs_1.writeFileSync)(temporaryOutputPath, terminalOutput);
             return new noop_child_process_1.NoopChildProcess({
                 code: 1,
-                terminalOutput: e.stack ?? e.message ?? '',
+                terminalOutput,
             });
         }
     }
@@ -1091,7 +1191,13 @@ class TaskOrchestrator {
         const pipeOutput = await this.pipeOutputCapture(task);
         // obtain metadata
         const temporaryOutputPath = this.cache.temporaryOutputPath(task);
-        const streamOutput = (0, output_1.isStaticOutputStyle)(this.outputStyle)
+        // Deliberately not gated on `printsTaskOutput`, unlike `runTaskDirectly`.
+        // `SummaryTerminalOutputLifeCycle.printTaskTerminalOutput` is a no-op and it
+        // only reports at `endCommand`, which never runs for a task that does not
+        // end - so suppressing here would leave a `nx serve` under `summary` with a
+        // permanently silent terminal and no file to read yet. Streaming is the only
+        // channel a continuous task has.
+        const streamOutput = (0, output_1.isStaticOutputStyle)(this.specifiedOutputStyle)
             ? false
             : (0, utils_1.shouldStreamOutput)(task, this.initiatingProject);
         let env = pipeOutput
@@ -1108,6 +1214,9 @@ class TaskOrchestrator {
         this.continuousTaskExitHandled.set(task.id, new Promise((resolve) => {
             childProcess.onExit(async (code) => {
                 await this.handleContinuousTaskExit(code, task, groupId, true);
+                // Registered after the runner's own exit handler, which is what
+                // wrote the file.
+                this.recordContinuousTerminalOutput(task);
                 resolve();
             });
         }));
@@ -1125,52 +1234,143 @@ class TaskOrchestrator {
     }
     async postRunSteps(results, shouldCache, groupId) {
         const now = Date.now();
-        const tasksToRecord = [];
-        for (const { task, status } of results) {
+        for (const { task } of results) {
             // Only set endTime as fallback (batch provides timing via result.task)
             task.endTime ??= now;
+        }
+        // Caller decides whether these results should be written to the cache.
+        // Cache replays pass false so a replayed failure (reported as 'failure' so
+        // it counts as a failed run) isn't re-written to the cache on every replay.
+        const resultsToCache = shouldCache && !this.stopRequested ? this.resultsToCache(results) : [];
+        // Resolved before the cache writes so the two never write the same file.
+        this.persistTerminalOutputs(results, resultsToCache);
+        if (resultsToCache.length > 0) {
+            // cache the results
+            perf_hooks_1.performance.mark('cache-results-start');
+            await Promise.all(resultsToCache.map(async ({ task, code, terminalOutput, outputs }) => {
+                const files = await this.cache.put(task, terminalOutput, outputs, code);
+                if (files)
+                    this.copiedOutputFiles.set(task.id, files);
+            }));
+            perf_hooks_1.performance.mark('cache-results-end');
+            perf_hooks_1.performance.measure('cache-results', 'cache-results-start', 'cache-results-end');
+        }
+        const tasksToRecord = [];
+        for (const { task, status } of results) {
+            const files = this.copiedOutputFiles.get(task.id);
+            this.copiedOutputFiles.delete(task.id);
             // Skip recording for tasks whose outputs already match the cache —
             // the daemon already has the correct hash recorded.
             if (!this.stopRequested &&
                 task.outputs.length > 0 &&
                 status !== 'local-cache-kept-existing') {
-                tasksToRecord.push({ outputs: task.outputs, hash: task.hash });
+                tasksToRecord.push({ outputs: task.outputs, hash: task.hash, files });
             }
         }
         if (tasksToRecord.length > 0) {
             await this.recordOutputsHashBatch(tasksToRecord);
         }
-        // Caller decides whether these results should be written to the cache.
-        // Cache replays pass false so a replayed failure (reported as 'failure' so
-        // it counts as a failed run) isn't re-written to the cache on every replay.
-        if (shouldCache && !this.stopRequested) {
-            // cache the results
-            perf_hooks_1.performance.mark('cache-results-start');
-            await Promise.all(results
-                .filter(({ status }) => status !== 'local-cache' &&
-                status !== 'local-cache-kept-existing' &&
-                status !== 'remote-cache' &&
-                status !== 'skipped' &&
-                status !== 'stopped')
-                .map((result) => ({
-                ...result,
-                code: result.status === 'local-cache' ||
-                    result.status === 'local-cache-kept-existing' ||
-                    result.status === 'remote-cache' ||
-                    result.status === 'success'
-                    ? 0
-                    : 1,
-                outputs: result.task.outputs,
-            }))
-                .filter(({ task, code }) => this.shouldCacheTaskResult(task, code))
-                .filter(({ terminalOutput, outputs }) => terminalOutput || outputs)
-                .map(async ({ task, code, terminalOutput, outputs }) => this.cache.put(task, terminalOutput, outputs, code)));
-            perf_hooks_1.performance.mark('cache-results-end');
-            perf_hooks_1.performance.measure('cache-results', 'cache-results-start', 'cache-results-end');
-        }
         await this.complete(results, groupId);
         await this.scheduleNextTasksAndReleaseThreads();
     }
+    /**
+     * The results `cache.put` will write, in the shape it wants them. Pulled out
+     * of postRunSteps so the terminal-output backstop can be told exactly which
+     * tasks the cache is already writing a file for.
+     */
+    resultsToCache(results) {
+        return results
+            .filter(({ status }) => status !== 'local-cache' &&
+            status !== 'local-cache-kept-existing' &&
+            status !== 'remote-cache' &&
+            status !== 'skipped' &&
+            status !== 'stopped')
+            .map((result) => ({
+            ...result,
+            code: result.status === 'local-cache' ||
+                result.status === 'local-cache-kept-existing' ||
+                result.status === 'remote-cache' ||
+                result.status === 'success'
+                ? 0
+                : 1,
+            outputs: result.task.outputs,
+        }))
+            .filter(({ task, code }) => this.shouldCacheTaskResult(task, code))
+            .filter(({ terminalOutput, outputs }) => terminalOutput || outputs);
+    }
+    /**
+     * Guarantee that every task that actually ran leaves its terminal output at
+     * `<cacheDir>/terminalOutputs/<hash>`, whatever its cache setting, output
+     * style, or batch membership. That path is what `--output-style=summary`
+     * addresses each failure by, what the DB cache reads back on a hit
+     * (`build_cached_result`), and what Nx Cloud falls back to reading when a
+     * task's in-memory output is undefined, so a task that reaches a terminal
+     * state without a file there is a dangling reference.
+     *
+     * Batch tasks are why this exists. Their output only ever arrives over IPC,
+     * so the sole thing that ever put it on disk was `cache.put` — leaving a
+     * `cache:false` batch task (the shape CI runs under `NX_BATCH_MODE`) with no
+     * file at all.
+     *
+     * Written exactly once per task: tasks whose runner owns the file are
+     * recorded in `tasksWithPersistedOutput`, and tasks `cache.put` is about to
+     * write are skipped here. Cache hits are skipped by status — their output was
+     * read from this very file — except a replayed cached *failure*, which comes
+     * back as `failure` and is rewritten with the bytes it was just read with.
+     * Skipped tasks never ran, so there is nothing to write.
+     *
+     * Every file written without a cache entry behind it is then registered with
+     * the cache, because `removeOldCacheRecords` only collects hashes it finds in
+     * the database — an unregistered file would never be cleaned up. That covers
+     * files this method wrote AND ones a runner wrote, which are equally
+     * invisible to the GC.
+     */
+    persistTerminalOutputs(results, resultsToCache) {
+        const writtenByCache = new Set(resultsToCache.map(({ task }) => task.id));
+        const toRecord = [];
+        for (const { task, status, terminalOutput } of results) {
+            if (terminalOutput === undefined ||
+                // A task that failed before it was hashed has no path to write to.
+                !task.hash ||
+                status === 'skipped' ||
+                status === 'local-cache' ||
+                status === 'local-cache-kept-existing' ||
+                status === 'remote-cache' ||
+                // `cache.put` writes the file and records the hash itself.
+                writtenByCache.has(task.id)) {
+                continue;
+            }
+            if (!this.tasksWithPersistedOutput.has(task.id)) {
+                this.tasksWithPersistedOutput.add(task.id);
+                (0, fs_1.writeFileSync)(this.cache.temporaryOutputPath(task), terminalOutput);
+            }
+            toRecord.push({
+                hash: task.hash,
+                size: Buffer.byteLength(terminalOutput),
+            });
+        }
+        if (toRecord.length > 0) {
+            this.cache.recordTerminalOutputs(toRecord);
+        }
+    }
+    /**
+     * Register a continuous task's terminal output with the cache so the GC can
+     * collect it. Continuous tasks never reach postRunSteps — their file is
+     * written by whichever runner ran them, as the process exits — so this is
+     * the only place their hash gets recorded. Read off disk rather than from a
+     * result, because a continuous task completes without one.
+     */
+    recordContinuousTerminalOutput(task) {
+        if (!task.hash)
+            return;
+        try {
+            const { size } = (0, fs_1.statSync)(this.cache.temporaryOutputPath(task));
+            this.cache.recordTerminalOutputs([{ hash: task.hash, size }]);
+        }
+        catch {
+            // Exit handler: a throw here would be an unhandled rejection.
+        }
+    }
     async scheduleNextTasksAndReleaseThreads() {
         if (this.stopRequested) {
             this.waitingForTasks.forEach((f) => f(null));

package/dist/src/tasks-runner/tasks-runner.d.ts

@@ -19,6 +19,8 @@ export type TasksRunner<T = unknown> = (tasks: Task[], options: T, context?: {
     taskGraph?: TaskGraph;
     hasher?: TaskHasher;
     daemon?: DaemonClient;
+    /** Tasks hashed only once the tasks they read outputs from have run. */
+    deferredTaskIds?: ReadonlySet<string>;
 }) => any | Promise<{
     [id: string]: TaskStatus;
 }>;

package/dist/src/tasks-runner/terminal-output-path.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Where a task's terminal output lives on disk, for callers that have a hash but
+ * no cache instance. Mirrors the native layout (`get_task_outputs_path_internal`
+ * in cache.rs) and the legacy cache's `terminalOutputsDir`; both resolve
+ * `<cacheDir>/terminalOutputs/<hash>`.
+ *
+ * Its own module rather than part of `cache.ts` so the graph and plugin paths
+ * can resolve a path without pulling the cache — and the database, cloud and
+ * task-IO modules behind it — into their import graph.
+ */
+export declare function terminalOutputPathForHash(hash: string): string;

package/dist/src/tasks-runner/terminal-output-path.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.terminalOutputPathForHash = terminalOutputPathForHash;
+const path_1 = require("path");
+const cache_directory_1 = require("../utils/cache-directory");
+/**
+ * Where a task's terminal output lives on disk, for callers that have a hash but
+ * no cache instance. Mirrors the native layout (`get_task_outputs_path_internal`
+ * in cache.rs) and the legacy cache's `terminalOutputsDir`; both resolve
+ * `<cacheDir>/terminalOutputs/<hash>`.
+ *
+ * Its own module rather than part of `cache.ts` so the graph and plugin paths
+ * can resolve a path without pulling the cache — and the database, cloud and
+ * task-IO modules behind it — into their import graph.
+ */
+function terminalOutputPathForHash(hash) {
+    return (0, path_1.join)(cache_directory_1.cacheDir, 'terminalOutputs', hash);
+}

package/dist/src/tasks-runner/utils.d.ts

@@ -49,6 +49,11 @@ export declare function getExecutorForTask(task: Task, projects: Record<string,
     isNxExecutor: boolean;
 };
 export declare function getCustomHasher(task: Task, projects: Record<string, ProjectConfiguration>): CustomHasher | null;
+/**
+ * `taskGraph` pruned to `required`, which must already hold its dependencies.
+ * Unknown ids are ignored: a sync generator can remove a task after selection.
+ */
+export declare function pruneToSelectedTasks(taskGraph: TaskGraph, required: Iterable<string>): TaskGraph;
 export declare function removeTasksFromTaskGraph(graph: TaskGraph, ids: string[]): TaskGraph;
 export declare function calculateReverseDeps(taskGraph: TaskGraph): Record<string, string[]>;
 export declare function getCliPath(): string;

package/dist/src/tasks-runner/utils.js

@@ -17,6 +17,7 @@ exports.getExecutorNameForTask = getExecutorNameForTask;
 exports.expandInitiatingTasksThroughNoop = expandInitiatingTasksThroughNoop;
 exports.getExecutorForTask = getExecutorForTask;
 exports.getCustomHasher = getCustomHasher;
+exports.pruneToSelectedTasks = pruneToSelectedTasks;
 exports.removeTasksFromTaskGraph = removeTasksFromTaskGraph;
 exports.calculateReverseDeps = calculateReverseDeps;
 exports.getCliPath = getCliPath;
@@ -221,7 +222,10 @@ function getOutputsForTargetAndConfiguration(taskTargetOrTask, overridesOrNode,
         validateOutputs(targetConfiguration.outputs);
         const result = new Set();
         for (const output of targetConfiguration.outputs) {
-            const interpolatedOutput = interpolate(output, {
+            // A leading `!` is negation, not part of the path: strip it so
+            // `{workspaceRoot}` still sits where interpolate requires it.
+            const isNegated = output.startsWith('!');
+            const interpolatedOutput = interpolate(isNegated ? output.substring(1) : output, {
                 projectRoot: node.data.root,
                 projectName: node.name,
                 project: { ...node.data, name: node.name }, // this is legacy
@@ -229,7 +233,7 @@ function getOutputsForTargetAndConfiguration(taskTargetOrTask, overridesOrNode,
             });
             if (!!interpolatedOutput &&
                 !interpolatedOutput.match(/{(projectRoot|workspaceRoot|(options.*))}/)) {
-                result.add(interpolatedOutput);
+                result.add(isNegated ? `!${interpolatedOutput}` : interpolatedOutput);
             }
         }
         return Array.from(result);
@@ -341,6 +345,14 @@ function getCustomHasher(task, projects) {
     const factory = getExecutorForTask(task, projects).hasherFactory;
     return factory ? factory() : null;
 }
+/**
+ * `taskGraph` pruned to `required`, which must already hold its dependencies.
+ * Unknown ids are ignored: a sync generator can remove a task after selection.
+ */
+function pruneToSelectedTasks(taskGraph, required) {
+    const keep = new Set(required);
+    return removeTasksFromTaskGraph(taskGraph, Object.keys(taskGraph.tasks).filter((id) => !keep.has(id)));
+}
 function removeTasksFromTaskGraph(graph, ids) {
     const newGraph = removeIdsFromTaskGraph(graph, ids, graph.tasks);
     return {