nx 22.7.0-beta.12 → 22.7.0-beta.13

package/dist/src/tasks-runner/task-orchestrator.js

@@ -8,6 +8,7 @@ const fs_1 = require("fs");
 const pc = tslib_1.__importStar(require("picocolors"));
 const path_1 = require("path");
 const perf_hooks_1 = require("perf_hooks");
+const project_graph_1 = require("../project-graph/project-graph");
 const run_commands_impl_1 = require("../executors/run-commands/run-commands.impl");
 const hash_task_1 = require("../hasher/hash-task");
 const task_graph_utils_1 = require("./task-graph-utils");
@@ -45,18 +46,22 @@ class TaskOrchestrator {
         this.taskDetails = (0, hash_task_1.getTaskDetails)();
         this.cache = (0, cache_1.getCache)(this.options);
         this.tuiEnabled = (0, is_tui_enabled_1.isTuiEnabled)();
+        // Derived from projectGraph once — passed to getExecutorForTask /
+        // getCustomHasher so they don't have to re-walk the graph per call.
+        this.projects = (0, project_graph_1.readProjectsConfigurationFromProjectGraph)(this.projectGraph).projects;
         this.forkedProcessTaskRunner = new forked_process_task_runner_1.ForkedProcessTaskRunner(this.options, this.tuiEnabled);
         this.runningTasksService = !native_1.IS_WASM
             ? new native_1.RunningTasksService((0, db_connection_1.getLocalDbConnection)())
             : null;
-        this.tasksSchedule = new tasks_schedule_1.TasksSchedule(this.projectGraph, this.taskGraph, this.options);
+        this.tasksSchedule = new tasks_schedule_1.TasksSchedule(this.projectGraph, this.projects, this.taskGraph, this.options);
         // region internal state
         this.batchEnv = (0, task_env_1.getEnvVariablesForBatchProcess)(this.options.skipNxCache, this.options.captureStderr);
         this.reverseTaskDeps = (0, utils_1.calculateReverseDeps)(this.taskGraph);
         this.initializingTaskIds = new Set(this.initiatingTasks.map((t) => t.id));
         this.processedTasks = new Map();
-        this.completedTasks = {};
+        this.completedTasks = new Map();
         this.waitingForTasks = [];
+        this.pendingDiscreteWorkers = new Set();
         this.groups = [];
         this.continuousTasksStarted = 0;
         this.bailed = false;
@@ -89,34 +94,31 @@ class TaskOrchestrator {
         await this.init();
         perf_hooks_1.performance.mark('task-execution:start');
         const { discrete, continuous, total } = getThreadPoolSize(this.options, this.taskGraph);
-        const threads = [];
         process.stdout.setMaxListeners(total + events_1.defaultMaxListeners);
         process.stderr.setMaxListeners(total + events_1.defaultMaxListeners);
         process.setMaxListeners(total + events_1.defaultMaxListeners);
-        // initial seeding of the queue
-        for (let i = 0; i < discrete; ++i) {
-            threads.push(this.executeDiscreteTaskLoop());
-        }
+        const doNotSkipCache = this.options.skipNxCache === false ||
+            this.options.skipNxCache === undefined;
+        // Start continuous task loops (these run independently)
+        const continuousLoops = [];
         for (let i = 0; i < continuous; ++i) {
-            threads.push(this.executeContinuousTaskLoop(continuous));
+            continuousLoops.push(this.executeContinuousTaskLoop(continuous));
         }
+        // Set up forced shutdown handler
+        const shutdownPromise = this.tuiEnabled
+            ? new Promise((resolve) => {
+                this.options.lifeCycle.registerForcedShutdownCallback(() => {
+                    this.stopRequested = true;
+                    resolve(undefined);
+                });
+            })
+            : new Promise((resolve) => {
+                this.resolveStopPromise = resolve;
+            });
+        const coordinatorLoop = this.executeCoordinatorLoop(doNotSkipCache, discrete);
         await Promise.race([
-            Promise.all(threads),
-            ...(this.tuiEnabled
-                ? [
-                    new Promise((resolve) => {
-                        this.options.lifeCycle.registerForcedShutdownCallback(() => {
-                            // The user force quit the TUI with ctrl+c, so proceed onto cleanup
-                            this.stopRequested = true;
-                            resolve(undefined);
-                        });
-                    }),
-                ]
-                : [
-                    new Promise((resolve) => {
-                        this.resolveStopPromise = resolve;
-                    }),
-                ]),
+            Promise.all([coordinatorLoop, ...continuousLoops]),
+            shutdownPromise,
         ]);
         perf_hooks_1.performance.mark('task-execution:end');
         perf_hooks_1.performance.measure('task-execution', 'task-execution:start', 'task-execution:end');
@@ -124,20 +126,57 @@ class TaskOrchestrator {
             this.cache.removeOldCacheRecords();
         }
         await this.cleanup();
-        return this.completedTasks;
+        // Public API (defaultTasksRunner) returns a plain object keyed by
+        // task id. Internal state is a Map for faster lookup.
+        return Object.fromEntries(this.completedTasks);
     }
     nextBatch() {
         return this.tasksSchedule.nextBatch();
     }
-    async executeDiscreteTaskLoop() {
-        const doNotSkipCache = this.options.skipNxCache === false ||
-            this.options.skipNxCache === undefined;
+    /**
+     * Coordinator loop. All batch operations (hashing, cache resolution)
+     * happen on this single thread — no races. Cache misses are dispatched
+     * as fire-and-forget workers. Workers signal completion via
+     * scheduleNextTasksAndReleaseThreads which wakes all waiting loops.
+     *
+     * Safety: the dispatch phase (step 5) is fully synchronous — no
+     * worker can run during it. So all tasks picked up by nextTask()
+     * are guaranteed to be in processedTasks from step 1.
+     */
+    async executeCoordinatorLoop(doNotSkipCache, parallelism) {
         while (true) {
-            // completed all the tasks
-            if (!this.tasksSchedule.hasTasks() || this.bailed || this.stopRequested) {
-                return null;
+            if (this.bailed || this.stopRequested)
+                break;
+            // 1. Hash BEFORE processAll so processTask sees hashes set, and so
+            //    resolveCachedTasksBulk can look them up in the cache. Each task
+            //    is hashed with its own task-specific env (project/target .env
+            //    files, custom hasher env reads) — the shared batchEnv would
+            //    compute a different cache key than the single-task path and
+            //    risk stale cache reuse after env changes.
+            {
+                const { scheduledTasks } = this.tasksSchedule.getAllScheduledTasks();
+                const unhashed = scheduledTasks
+                    .map((id) => this.taskGraph.tasks[id])
+                    .filter((t) => !t.hash &&
+                    this.taskGraph.dependencies[t.id].every((depId) => this.completedTasks.has(depId)));
+                if (unhashed.length > 1) {
+                    const perTaskEnvs = {};
+                    for (const task of unhashed) {
+                        perTaskEnvs[task.id] = (0, task_env_1.getTaskSpecificEnv)(task, this.projectGraph);
+                    }
+                    await (0, hash_task_1.hashTasks)(this.hasher, this.projectGraph, this.taskGraphForHashing, perTaskEnvs, this.taskDetails, unhashed);
+                }
+            }
+            // 2. Bulk-resolve cache hits before processTask — avoids N
+            //    lifecycle calls for tasks that will be resolved from cache.
+            if (doNotSkipCache) {
+                const resolved = await this.resolveCachedTasksBulk();
+                if (resolved)
+                    continue;
             }
+            // 3. Process remaining scheduled tasks (cache misses + non-cacheable).
             this.processAllScheduledTasks();
+            // 4. Handle batch executors
             const batch = this.nextBatch();
             if (batch) {
                 const groupId = this.closeGroup();
@@ -145,14 +184,20 @@ class TaskOrchestrator {
                 this.openGroup(groupId);
                 continue;
             }
-            const task = this.tasksSchedule.nextTask((t) => !t.continuous);
-            if (task) {
+            // 5. Dispatch cache misses as individual workers
+            while (this.pendingDiscreteWorkers.size < parallelism) {
+                const task = this.tasksSchedule.nextTask((t) => !t.continuous);
+                if (!task)
+                    break;
                 const groupId = this.closeGroup();
-                await this.applyFromCacheOrRunTask(doNotSkipCache, task, groupId);
-                this.openGroup(groupId);
-                continue;
+                this.dispatchDiscreteWorker(doNotSkipCache, task, groupId);
             }
-            // block until some other task completes, then try again
+            // 6. Nothing left to dispatch and nothing in flight — done.
+            if (!this.tasksSchedule.hasTasks() &&
+                this.pendingDiscreteWorkers.size === 0) {
+                break;
+            }
+            // 7. Wait for a worker to finish (woken by scheduleNextTasksAndReleaseThreads)
             await new Promise((res) => this.waitingForTasks.push(res));
         }
     }
@@ -186,14 +231,6 @@ class TaskOrchestrator {
             await new Promise((res) => this.waitingForTasks.push(res));
         }
     }
-    processTasks(taskIds) {
-        for (const taskId of taskIds) {
-            // Task is already handled or being handled
-            if (!this.processedTasks.has(taskId)) {
-                this.processedTasks.set(taskId, this.processTask(taskId));
-            }
-        }
-    }
     // region Processing Scheduled Tasks
     async processTask(taskId) {
         const task = this.taskGraph.tasks[taskId];
@@ -206,41 +243,117 @@ class TaskOrchestrator {
     }
     processAllScheduledTasks() {
         const { scheduledTasks } = this.tasksSchedule.getAllScheduledTasks();
-        this.processTasks(scheduledTasks);
+        for (const taskId of scheduledTasks) {
+            // Task is already handled or being handled
+            if (!this.processedTasks.has(taskId)) {
+                this.processedTasks.set(taskId, this.processTask(taskId));
+            }
+        }
     }
     // endregion Processing Scheduled Tasks
     // region Applying Cache
     async applyCachedResults(tasks) {
         const cacheableTasks = tasks.filter((t) => (0, utils_1.isCacheableTask)(t, this.options));
-        const res = await Promise.all(cacheableTasks.map((t) => this.applyCachedResult(t)));
-        return res.filter((r) => r !== null);
+        if (cacheableTasks.length === 0)
+            return [];
+        const cacheHits = await this.fetchCacheHits(cacheableTasks);
+        if (cacheHits.length === 0)
+            return [];
+        return this.finalizeCacheHits(cacheHits);
+    }
+    /**
+     * Batch cache lookup + filter to successful entries. Handles both
+     * local (one rarray SQL call) and remote (parallel HTTP retrievals)
+     * inside DbCache.getBatch.
+     */
+    async fetchCacheHits(tasks) {
+        const batchResults = await this.cache.getBatch(tasks);
+        const cacheHits = [];
+        for (const task of tasks) {
+            const cachedResult = batchResults.get(task.hash);
+            if (cachedResult && cachedResult.code === 0) {
+                cacheHits.push({ task, cachedResult });
+            }
+        }
+        return cacheHits;
     }
-    async applyCachedResult(task) {
-        const cachedResult = await this.cache.get(task);
-        if (!cachedResult || cachedResult.code !== 0)
-            return null;
-        const outputs = task.outputs;
-        const shouldCopyOutputsFromCache = 
-        // No output files to restore
-        !!outputs.length &&
-            // Remote caches are restored to output dirs when applied and using db cache
-            (!cachedResult.remote || !(0, cache_1.dbCacheEnabled)()) &&
-            // Output files have not been touched since last run
-            (await this.shouldCopyOutputsFromCache(outputs, task.hash));
-        if (shouldCopyOutputsFromCache) {
-            await this.cache.copyFilesFromCache(task.hash, cachedResult, outputs);
+    /**
+     * For each confirmed cache hit: decide whether to copy outputs from
+     * the cache (skipping if the on-disk outputs already match the
+     * recorded hash), copy in parallel, derive the task status, print
+     * terminal output, and return the assembled results.
+     */
+    async finalizeCacheHits(cacheHits) {
+        // Batch-check which tasks need outputs copied from cache. Remote
+        // cache entries come pre-restored to their output dirs when the
+        // db cache is on, so we only check ones that aren't.
+        const usingDbCache = (0, cache_1.dbCacheEnabled)();
+        const tasksNeedingOutputCheck = cacheHits.filter(({ task, cachedResult }) => task.outputs.length > 0 && (!cachedResult.remote || !usingDbCache));
+        const shouldCopyMap = await this.shouldCopyOutputsFromCacheBatch(tasksNeedingOutputCheck.map(({ task }) => ({
+            outputs: task.outputs,
+            hash: task.hash,
+        })));
+        // Copy outputs in parallel for tasks that need it.
+        await Promise.all(cacheHits.map(async ({ task, cachedResult }) => {
+            if (shouldCopyMap.get(task.hash)) {
+                await this.cache.copyFilesFromCache(task.hash, cachedResult, task.outputs);
+            }
+        }));
+        // Derive status, print terminal output, build results.
+        const results = [];
+        for (const { task, cachedResult } of cacheHits) {
+            const shouldCopy = shouldCopyMap.get(task.hash) ?? false;
+            const status = cachedResult.remote
+                ? 'remote-cache'
+                : shouldCopy
+                    ? 'local-cache'
+                    : 'local-cache-kept-existing';
+            this.options.lifeCycle.printTaskTerminalOutput(task, status, cachedResult.terminalOutput);
+            results.push({
+                task,
+                code: cachedResult.code,
+                status,
+                terminalOutput: cachedResult.terminalOutput,
+            });
+        }
+        return results;
+    }
+    /**
+     * Coordinator wrapper around {@link resolveCachedTasks}: peeks at
+     * scheduledTasks (without removing anything from the schedule),
+     * filters to cacheable hashed discrete candidates, and delegates the
+     * cache fetch + lifecycle to the public method. Returns true if any
+     * tasks were resolved.
+     *
+     * The coordinator relies on this running unconditionally (when cache
+     * is enabled): tasks dispatched in step 5 via runTaskDirectly skip
+     * their own cache lookup on the assumption that this has already
+     * confirmed them as misses. Don't add length-based bails.
+     */
+    async resolveCachedTasksBulk() {
+        const { scheduledTasks } = this.tasksSchedule.getAllScheduledTasks();
+        const candidates = [];
+        for (const id of scheduledTasks) {
+            const task = this.taskGraph.tasks[id];
+            if (task.hash &&
+                !task.continuous &&
+                (0, utils_1.isCacheableTask)(task, this.options)) {
+                candidates.push(task);
+            }
+        }
+        if (candidates.length === 0)
+            return false;
+        // postRunSteps → complete() → tasksSchedule.complete() will filter
+        // resolved hits out of scheduledTasks before we return, so there's
+        // no need to mutate the schedule here.
+        const groupId = this.closeGroup();
+        try {
+            const results = await this.resolveCachedTasks(true, candidates, groupId);
+            return results.length > 0;
+        }
+        finally {
+            this.openGroup(groupId);
         }
-        const status = cachedResult.remote
-            ? 'remote-cache'
-            : shouldCopyOutputsFromCache
-                ? 'local-cache'
-                : 'local-cache-kept-existing';
-        this.options.lifeCycle.printTaskTerminalOutput(task, status, cachedResult.terminalOutput);
-        return {
-            code: cachedResult.code,
-            task,
-            status,
-        };
     }
     // endregion Applying Cache
     // region Batch
@@ -298,7 +411,15 @@ class TaskOrchestrator {
         return { cachedResults, needsRehashAfterExecution };
     }
     async hashBatchTasks(tasks) {
-        await (0, hash_task_1.hashTasks)(this.hasher, this.projectGraph, this.taskGraphForHashing, this.batchEnv, this.taskDetails, tasks);
+        // Batch executors run every task in the same forked process, but
+        // each task still has its own .env files / custom-hasher env — use
+        // task-specific env for hashing so the cache key matches the
+        // single-task path.
+        const perTaskEnvs = {};
+        for (const task of tasks) {
+            perTaskEnvs[task.id] = (0, task_env_1.getTaskSpecificEnv)(task, this.projectGraph);
+        }
+        await (0, hash_task_1.hashTasks)(this.hasher, this.projectGraph, this.taskGraphForHashing, perTaskEnvs, this.taskDetails, tasks);
     }
     async applyFromCacheOrRunBatch(doNotSkipCache, batch, groupId) {
         const applyFromCacheOrRunBatchStart = perf_hooks_1.performance.mark('TaskOrchestrator-apply-from-cache-or-run-batch:start');
@@ -341,12 +462,12 @@ class TaskOrchestrator {
         }
         // Update batch status based on all task results
         const hasFailures = taskEntries.some(([taskId]) => {
-            const status = this.completedTasks[taskId];
+            const status = this.completedTasks.get(taskId);
             return status === 'failure' || status === 'skipped';
         });
         this.options.lifeCycle.setBatchStatus?.(batch.id, hasFailures ? "Failure" /* BatchStatus.Failure */ : "Success" /* BatchStatus.Success */);
         this.forkedProcessTaskRunner.cleanUpBatchProcesses();
-        const tasksCompleted = taskEntries.filter(([taskId]) => this.completedTasks[taskId]);
+        const tasksCompleted = taskEntries.filter(([taskId]) => this.completedTasks.has(taskId));
         // Batch is still not done, run it again
         if (tasksCompleted.length !== taskEntries.length) {
             await this.applyFromCacheOrRunBatch(doNotSkipCache, {
@@ -420,52 +541,120 @@ class TaskOrchestrator {
     }
     // endregion Batch
     // region Single Task
-    async applyFromCacheOrRunTask(doNotSkipCache, task, groupId) {
+    /**
+     * Bulk-resolve cache hits for a set of tasks: fetch cached entries,
+     * copy outputs as needed, fire lifecycle, and return the TaskResults
+     * for the hits. Tasks that aren't in the cache (or aren't cacheable)
+     * are silently omitted from the return value — callers are responsible
+     * for running those via {@link runTaskDirectly}.
+     *
+     * Fires scheduleTask lifecycle for hits that haven't been through
+     * processAllScheduledTasks yet. That's a coordinator gap-filler and
+     * a no-op for callers that pre-process the schedule.
+     *
+     * The caller provides `groupId` — cache hits share one slot since they
+     * don't actually compete for parallelism.
+     */
+    async resolveCachedTasks(doNotSkipCache, tasks, groupId) {
+        if (!doNotSkipCache || tasks.length === 0)
+            return [];
+        const cacheableTasks = tasks.filter((t) => (0, utils_1.isCacheableTask)(t, this.options));
+        if (cacheableTasks.length === 0)
+            return [];
+        const cacheHits = await this.fetchCacheHits(cacheableTasks);
+        if (cacheHits.length === 0)
+            return [];
+        // scheduleTask lifecycle for hits the coordinator resolved before
+        // processAllScheduledTasks could fire it. No-op for callers that
+        // already ran processAllScheduledTasks (every hit is in processedTasks).
+        await Promise.all(cacheHits
+            .filter(({ task }) => !this.processedTasks.has(task.id))
+            .map(({ task }) => this.options.lifeCycle.scheduleTask(task)));
+        const hitTasks = cacheHits.map((h) => h.task);
+        await this.preRunSteps(hitTasks, { groupId });
+        const results = await this.finalizeCacheHits(cacheHits);
+        await this.postRunSteps(results, doNotSkipCache, { groupId });
+        return results;
+    }
+    /**
+     * Fire a discrete-task worker and track it in pendingDiscreteWorkers until
+     * it settles. Uses runTaskDirectly (not applyFromCacheOrRun*) because
+     * resolveCachedTasksBulk already confirmed this task is a cache miss —
+     * another lookup would re-query the DB and (for Nx Cloud users) repeat
+     * the remote HTTP retrieval.
+     */
+    dispatchDiscreteWorker(doNotSkipCache, task, groupId) {
+        const worker = this.runTaskDirectly(doNotSkipCache, task, groupId)
+            .catch((e) => this.handleDiscreteWorkerFailure(doNotSkipCache, task, groupId, e))
+            .finally(() => {
+            this.openGroup(groupId);
+            this.pendingDiscreteWorkers.delete(worker);
+            // Wake coordinator — the delete above may satisfy the exit condition
+            // (pendingDiscreteWorkers.size === 0) that was missed when
+            // scheduleNextTasksAndReleaseThreads fired earlier.
+            this.waitingForTasks.forEach((f) => f(null));
+            this.waitingForTasks.length = 0;
+        });
+        this.pendingDiscreteWorkers.add(worker);
+    }
+    /**
+     * Route a worker rejection (e.g. remote cache errors) through the normal
+     * failure path instead of letting it become an unhandled promise. Guard
+     * against double-finalize: completeTasks() populates `completedTasks`,
+     * so a rejection arriving after postRunSteps has already finalized the
+     * task must not run postRunSteps again.
+     */
+    async handleDiscreteWorkerFailure(doNotSkipCache, task, groupId, e) {
+        if (this.completedTasks.has(task.id))
+            return;
+        const terminalOutput = e?.message ?? '';
+        this.options.lifeCycle.printTaskTerminalOutput(task, 'failure', terminalOutput);
+        await this.postRunSteps([{ task, status: 'failure', terminalOutput }], doNotSkipCache, { groupId });
+    }
+    /**
+     * Spawn and wait on a task's child process, unconditionally — no cache
+     * lookup. Callers must have already confirmed the task is a cache miss
+     * (or disabled caching entirely).
+     */
+    async runTaskDirectly(doNotSkipCache, task, groupId) {
         // Wait for task to be processed
         const taskSpecificEnv = await this.processedTasks.get(task.id);
         await this.preRunSteps([task], { groupId });
         const pipeOutput = await this.pipeOutputCapture(task);
-        // obtain metadata
         const temporaryOutputPath = this.cache.temporaryOutputPath(task);
         const streamOutput = this.outputStyle === 'static'
             ? false
             : (0, utils_1.shouldStreamOutput)(task, this.initiatingProject);
-        let env = pipeOutput
+        const env = pipeOutput
             ? (0, task_env_1.getEnvVariablesForTask)(task, taskSpecificEnv, process.env.FORCE_COLOR === undefined
                 ? 'true'
                 : process.env.FORCE_COLOR, this.options.skipNxCache, this.options.captureStderr, null, null)
             : (0, task_env_1.getEnvVariablesForTask)(task, taskSpecificEnv, undefined, this.options.skipNxCache, this.options.captureStderr, temporaryOutputPath, streamOutput);
-        let results = doNotSkipCache ? await this.applyCachedResults([task]) : [];
-        // the task wasn't cached
         let resolveDiscreteExit;
-        if (results.length === 0) {
-            const discreteExitHandled = new Promise((r) => (resolveDiscreteExit = r));
-            this.discreteTaskExitHandled.set(task.id, discreteExitHandled);
-            const childProcess = await this.runTask(task, streamOutput, env, temporaryOutputPath, pipeOutput);
-            this.runningDiscreteTasks.set(task.id, {
-                runningTask: childProcess,
-                stopping: false,
-            });
-            const { code, terminalOutput } = await childProcess.getResults();
-            const isStopping = this.runningDiscreteTasks.get(task.id)?.stopping ?? false;
-            this.runningDiscreteTasks.delete(task.id);
-            results.push({
-                task,
-                code,
-                status: isStopping ? 'stopped' : code === 0 ? 'success' : 'failure',
-                terminalOutput,
-            });
-        }
+        const discreteExitHandled = new Promise((r) => (resolveDiscreteExit = r));
+        this.discreteTaskExitHandled.set(task.id, discreteExitHandled);
+        const childProcess = await this.runTask(task, streamOutput, env, temporaryOutputPath, pipeOutput);
+        this.runningDiscreteTasks.set(task.id, {
+            runningTask: childProcess,
+            stopping: false,
+        });
+        const { code, terminalOutput } = await childProcess.getResults();
+        const isStopping = this.runningDiscreteTasks.get(task.id)?.stopping ?? false;
+        this.runningDiscreteTasks.delete(task.id);
+        const result = {
+            task,
+            code,
+            status: isStopping ? 'stopped' : code === 0 ? 'success' : 'failure',
+            terminalOutput,
+        };
         try {
-            await this.postRunSteps(results, doNotSkipCache, { groupId });
+            await this.postRunSteps([result], doNotSkipCache, { groupId });
         }
         finally {
-            if (resolveDiscreteExit) {
-                this.discreteTaskExitHandled.delete(task.id);
-                resolveDiscreteExit();
-            }
+            this.discreteTaskExitHandled.delete(task.id);
+            resolveDiscreteExit();
         }
-        return results[0];
+        return result;
     }
     async runTask(task, streamOutput, env, temporaryOutputPath, pipeOutput) {
         const shouldPrefix = streamOutput &&
@@ -475,7 +664,7 @@ class TaskOrchestrator {
         if (process.env.NX_RUN_COMMANDS_DIRECTLY !== 'false' &&
             targetConfiguration.executor === 'nx:run-commands') {
             try {
-                const { schema } = (0, utils_1.getExecutorForTask)(task, this.projectGraph);
+                const { schema } = (0, utils_1.getExecutorForTask)(task, this.projects);
                 const combinedOptions = (0, params_1.combineOptionsForExecutor)(task.overrides, task.target.configuration ?? targetConfiguration.defaultConfiguration, targetConfiguration, schema, task.target.project, (0, path_1.relative)(task.projectRoot ?? workspace_root_1.workspaceRoot, process.cwd()), process.env.NX_VERBOSE_LOGGING === 'true');
                 if (combinedOptions.env) {
                     env = {
@@ -680,13 +869,21 @@ class TaskOrchestrator {
     }
     async postRunSteps(results, doNotSkipCache, { groupId }) {
         const now = Date.now();
-        for (const { task } of results) {
+        const tasksToRecord = [];
+        for (const { task, status } of results) {
             // Only set endTime as fallback (batch provides timing via result.task)
             task.endTime ??= now;
-            if (!this.stopRequested) {
-                await this.recordOutputsHash(task);
+            // Skip recording for tasks whose outputs already match the cache —
+            // the daemon already has the correct hash recorded.
+            if (!this.stopRequested &&
+                task.outputs.length > 0 &&
+                status !== 'local-cache-kept-existing') {
+                tasksToRecord.push({ outputs: task.outputs, hash: task.hash });
             }
         }
+        if (tasksToRecord.length > 0) {
+            await this.recordOutputsHashBatch(tasksToRecord);
+        }
         if (doNotSkipCache && !this.stopRequested) {
             // cache the results
             perf_hooks_1.performance.mark('cache-results-start');
@@ -744,7 +941,7 @@ class TaskOrchestrator {
         const taskIds = [];
         for (const { task, status, terminalOutput } of results) {
             taskIds.push(task.id);
-            if (this.completedTasks[task.id] === undefined && status !== 'skipped') {
+            if (!this.completedTasks.has(task.id) && status !== 'skipped') {
                 tasksToReport.push({
                     task,
                     status,
@@ -766,9 +963,9 @@ class TaskOrchestrator {
         // 3. Set completedTasks + update TUI + collect dependent tasks to skip
         const dependentTasksToSkip = [];
         for (const { task, status, displayStatus } of results) {
-            if (this.completedTasks[task.id] !== undefined)
+            if (this.completedTasks.has(task.id))
                 continue;
-            this.completedTasks[task.id] = status;
+            this.completedTasks.set(task.id, status);
             if (this.tuiEnabled) {
                 this.options.lifeCycle.setTaskStatus(task.id, displayStatus ?? (0, native_1.parseTaskStatus)(status));
             }
@@ -814,7 +1011,7 @@ class TaskOrchestrator {
             if (this.tuiEnabled) {
                 return true;
             }
-            const { schema } = (0, utils_1.getExecutorForTask)(task, this.projectGraph);
+            const { schema } = (0, utils_1.getExecutorForTask)(task, this.projects);
             return (schema.outputCapture === 'pipe' ||
                 process.env.NX_STREAM_OUTPUT === 'true');
         }
@@ -837,23 +1034,33 @@ class TaskOrchestrator {
     openGroup(id) {
         this.groups[id] = false;
     }
-    async shouldCopyOutputsFromCache(outputs, hash) {
+    async shouldCopyOutputsFromCacheBatch(tasks) {
+        const resultMap = new Map();
+        if (tasks.length === 0)
+            return resultMap;
         if (this.daemon?.enabled()) {
-            return !(await this.daemon.outputsHashesMatch(outputs, hash));
+            const matches = await this.daemon.outputsHashesMatchBatch(tasks);
+            for (let i = 0; i < tasks.length; i++) {
+                resultMap.set(tasks[i].hash, !matches[i]);
+            }
         }
         else {
-            return true;
+            // No daemon → can't verify on-disk outputs, always copy.
+            for (const task of tasks) {
+                resultMap.set(task.hash, true);
+            }
         }
+        return resultMap;
     }
-    async recordOutputsHash(task) {
+    async recordOutputsHashBatch(entries) {
         if (this.daemon?.enabled()) {
-            return this.daemon.recordOutputsHash(task.outputs, task.hash);
+            return this.daemon.recordOutputsHashBatch(entries);
         }
     }
     // endregion utils
     async handleContinuousTaskExit(code, task, groupId, ownsRunningTasksService) {
         // If cleanup already completed this task, nothing left to do
-        if (this.completedTasks[task.id] !== undefined) {
+        if (this.completedTasks.has(task.id)) {
             return;
         }
         const stoppingReason = this.runningContinuousTasks.get(task.id)?.stoppingReason;
@@ -867,7 +1074,7 @@ class TaskOrchestrator {
         }
     }
     async completeContinuousTask(task, groupId, ownsRunningTasksService, reason) {
-        if (this.completedTasks[task.id] !== undefined)
+        if (this.completedTasks.has(task.id))
             return;
         this.runningContinuousTasks.delete(task.id);
         if (ownsRunningTasksService) {

package/dist/src/tasks-runner/tasks-schedule.d.ts

@@ -1,6 +1,7 @@
 import { DefaultTasksRunnerOptions } from './default-tasks-runner';
 import { Task, TaskGraph } from '../config/task-graph';
 import { ProjectGraph } from '../config/project-graph';
+import { ProjectConfiguration } from '../config/workspace-json-project-json';
 export interface Batch {
     id: string;
     executorName: string;
@@ -8,6 +9,7 @@ export interface Batch {
 }
 export declare class TasksSchedule {
     private readonly projectGraph;
+    private readonly projects;
     private readonly taskGraph;
     private readonly options;
     private notScheduledTaskGraph;
@@ -22,7 +24,7 @@ export declare class TasksSchedule {
     private estimatedTaskTimings;
     private projectDependencies;
     private batchCounters;
-    constructor(projectGraph: ProjectGraph, taskGraph: TaskGraph, options: DefaultTasksRunnerOptions);
+    constructor(projectGraph: ProjectGraph, projects: Record<string, ProjectConfiguration>, taskGraph: TaskGraph, options: DefaultTasksRunnerOptions);
     init(): Promise<void>;
     scheduleNextTasks(): Promise<void>;
     hasTasks(): boolean;
@@ -35,7 +37,8 @@ export declare class TasksSchedule {
     nextBatch(): Batch;
     getIncompleteTasks(): Task[];
     private scheduleTasks;
-    private scheduleTask;
+    private scheduleTaskBatch;
+    private sortScheduledTasks;
     private scheduleBatches;
     private scheduleBatch;
     private processTaskForBatches;