nx 23.3.0 → 24.0.0-beta.1

package/dist/src/tasks-runner/task-orchestrator.js

@@ -22,6 +22,8 @@ const cache_1 = require("./cache");
 const forked_process_task_runner_1 = require("./forked-process-task-runner");
 const is_tui_enabled_1 = require("./is-tui-enabled");
 const pseudo_terminal_1 = require("./pseudo-terminal");
+const probes_1 = require("./readiness/probes");
+const ready_when_1 = require("./readiness/ready-when");
 const noop_child_process_1 = require("./running-tasks/noop-child-process");
 const output_prefix_1 = require("./running-tasks/output-prefix");
 const shared_running_task_1 = require("./running-tasks/shared-running-task");
@@ -45,6 +47,31 @@ function hasContent(path) {
         return false;
     }
 }
+function createReadinessState(onSettled) {
+    const state = {
+        settled: false,
+        abort: new AbortController(),
+    };
+    state.promise = new Promise((resolve, reject) => {
+        state.resolve = () => {
+            if (state.settled)
+                return;
+            state.settled = true;
+            resolve();
+            onSettled('ready');
+        };
+        state.reject = (error) => {
+            if (state.settled)
+                return;
+            state.settled = true;
+            reject(error);
+            onSettled('failed');
+        };
+    });
+    // A producer with no waiter still settles; that must not be unhandled
+    state.promise.catch(() => { });
+    return state;
+}
 class TaskOrchestrator {
     // endregion internal state
     constructor(hasher, initiatingProject, initiatingTasks, projectGraph, taskGraph, nxJson, options, bail, daemon, 
@@ -78,13 +105,17 @@ class TaskOrchestrator {
             ? new native_1.RunningTasksService((0, db_connection_1.getLocalDbConnection)())
             : null;
         this.taskInvocationTracker = !native_1.IS_WASM
-            ? new native_1.TaskInvocationTracker((0, db_connection_1.getLocalDbConnection)(), Number(process.env.NX_INVOCATION_ROOT_PID ?? process.pid))
+            ? new native_1.TaskInvocationTracker((0, db_connection_1.getLocalDbConnection)(), (0, task_env_1.getInvocationRootPid)(), (0, task_env_1.getInvocationAncestorPids)())
             : null;
-        // Tracks tasks registered by THIS process so that recursive code paths
-        // (e.g. applyFromCacheOrRunBatch looping on incomplete batches) don't
-        // re-register and trip the DB uniqueness constraint.
+        // Task ids this process has already registered. Recursive code paths (e.g.
+        // applyFromCacheOrRunBatch looping on incomplete batches) re-enter the
+        // detector for the same task; skipping them saves a DB round trip. A task id
+        // is unique within one process, so it is enough to identify the row.
         this.registeredInvocations = new Set();
-        this.tasksSchedule = new tasks_schedule_1.TasksSchedule(this.projectGraph, this.projects, this.taskGraph, this.options);
+        this.tasksSchedule = new tasks_schedule_1.TasksSchedule(this.projectGraph, this.projects, this.taskGraph, this.options, this.fullTaskGraph, {
+            readinessElsewhere: (producerId) => this.runningTasksService?.getTaskReadiness(producerId) ?? null,
+            onReadinessHold: (producerId) => this.onReadinessHold(producerId),
+        });
         // region internal state
         this.batchEnv = (0, task_env_1.getEnvVariablesForBatchProcess)(this.options.skipNxCache, this.options.captureStderr);
         this.reverseTaskDeps = (0, utils_1.calculateReverseDeps)(this.taskGraph);
@@ -123,6 +154,14 @@ class TaskOrchestrator {
         this.runningDiscreteTasks = new Map();
         this.discreteTaskExitHandled = new Map();
         this.continuousTaskExitHandled = new Map();
+        // Keyed by producer id: a producer this process started or shares
+        this.readiness = new Map();
+        // Producers that failed their check while still running: the exit must not
+        // report them again, and the saved log gets the reason after the transcript
+        this.readinessFailures = new Map();
+        this.readinessFailureReasons = new Map();
+        this.readinessSettled = false;
+        this.waitingLogged = new Set();
         this.cleanupPromise = null;
         this.signalHandlers = [];
         // Tasks whose runner already owns the write to
@@ -212,6 +251,7 @@ class TaskOrchestrator {
         while (true) {
             if (this.bailed || this.stopRequested)
                 break;
+            await this.scheduleAfterReadiness();
             // 1. Hash BEFORE processAll so processTask sees hashes set, and so
             //    resolveCachedTasksBulk can look them up in the cache. Each task
             //    is hashed with its own task-specific env (project/target .env
@@ -223,7 +263,7 @@ class TaskOrchestrator {
                 const unhashed = scheduledTasks
                     .map((id) => this.taskGraph.tasks[id])
                     .filter((t) => !t.hash &&
-                    this.taskGraph.dependencies[t.id].every((depId) => this.completedTasks.has(depId)));
+                    this.taskGraph.dependencies[t.id].every(({ id }) => this.completedTasks.has(id)));
                 if (unhashed.length > 0) {
                     const perTaskEnvs = {};
                     for (const task of unhashed) {
@@ -238,6 +278,11 @@ class TaskOrchestrator {
                 const resolved = await this.resolveCachedTasksBulk();
                 if (resolved)
                     continue;
+                // A pass from a continuous loop during the waits above may have
+                // admitted tasks this iteration never looked up; they must not reach
+                // step 5 without a confirmed miss
+                if (this.hasUnresolvedCacheableTask())
+                    continue;
             }
             // 3. Process remaining scheduled tasks (cache misses + non-cacheable).
             this.processAllScheduledTasks();
@@ -263,6 +308,10 @@ class TaskOrchestrator {
                 break;
             }
             // 7. Wait for a worker to finish (woken by scheduleNextTasksAndReleaseThreads)
+            // A verdict that landed during this iteration found nothing parked,
+            // so run its pass instead of parking
+            if (this.readinessSettled)
+                continue;
             await new Promise((res) => this.waitingForTasks.push(res));
         }
     }
@@ -272,6 +321,7 @@ class TaskOrchestrator {
             if (!this.tasksSchedule.hasTasks() || this.bailed || this.stopRequested) {
                 return null;
             }
+            await this.scheduleAfterReadiness();
             this.processAllScheduledTasks();
             const task = this.tasksSchedule.nextTask((t) => t.continuous);
             if (task) {
@@ -293,9 +343,29 @@ class TaskOrchestrator {
                 return null;
             }
             // block until some other task completes, then try again
+            if (this.readinessSettled)
+                continue;
             await new Promise((res) => this.waitingForTasks.push(res));
         }
     }
+    hasUnresolvedCacheableTask() {
+        return this.tasksSchedule
+            .getAllScheduledTasks()
+            .scheduledTasks.some((id) => {
+            const task = this.taskGraph.tasks[id];
+            return (!task.continuous &&
+                task.cache &&
+                (!task.hash || !this.cacheMissedHashes.has(task.hash)));
+        });
+    }
+    // A verdict releases tasks the schedule has not admitted yet. Whichever
+    // loop wakes first runs the pass; the coordinator may be inside a batch.
+    async scheduleAfterReadiness() {
+        if (this.readinessSettled) {
+            this.readinessSettled = false;
+            await this.scheduleNextTasksAndReleaseThreads();
+        }
+    }
     // region Processing Scheduled Tasks
     async processTask(taskId) {
         const task = this.taskGraph.tasks[taskId];
@@ -316,39 +386,59 @@ class TaskOrchestrator {
         }
     }
     /**
-     * Registers a task invocation and checks for loops across nested Nx processes.
-     * Uses the task_invocations DB table keyed by root PID. registerTask() throws
-     * on unique constraint violation when a parent Nx process already registered
-     * this task — indicating an infinite loop.
+     * Registers a task invocation and checks for loops across nested Nx
+     * processes. registerTask() returns the invocation chain only when an
+     * *ancestor* Nx process is already running this task — a genuine loop.
+     * Sibling processes running the same task are legitimate and register
+     * without complaint.
+     *
+     * The overrides hash rides along with the task id so the tracker can tell an
+     * ancestor re-invoking a task with different arguments from one repeating
+     * the same invocation.
      */
     detectTaskInvocationLoop(task) {
         if (!this.taskInvocationTracker)
             return;
         if (this.registeredInvocations.has(task.id))
             return;
+        let chain;
         try {
-            this.taskInvocationTracker.registerTask(process.pid, task.id);
+            chain = this.taskInvocationTracker.registerTask(process.pid, task.id, (0, utils_1.hashTaskOverrides)(task));
+        }
+        catch {
+            // Loop detection is diagnostic only; a DB failure must not fail the run
+            // or be mistaken for a loop.
+            return;
+        }
+        if (!chain) {
             this.registeredInvocations.add(task.id);
+            return;
+        }
+        const chainDisplay = chain.map((r) => r.taskId).join(' -> ');
+        output_1.output.error({
+            title: 'Recursive task invocation detected',
+            bodyLines: [
+                `Nx detected a recursive loop of task invocations:`,
+                ``,
+                `  ${chainDisplay} -> ${task.id}`,
+                ``,
+                `Task "${task.id}" was already invoked by a parent Nx process in this chain.`,
+                `This typically happens when a task's command (e.g., "nx ${task.target.target} ${task.target.project}")`,
+                `triggers a chain of tasks that eventually re-invokes itself.`,
+                ``,
+                `To fix this, review the command configuration for the tasks in the chain above.`,
+            ],
+        });
+        process.exit(1);
+    }
+    releaseTaskInvocation(task) {
+        if (!this.registeredInvocations.delete(task.id))
+            return;
+        try {
+            this.taskInvocationTracker?.unregisterTask(process.pid, task.id);
         }
         catch {
-            // Unique constraint violation — task already invoked by an ancestor Nx process
-            const chain = this.taskInvocationTracker.getInvocationChain();
-            const chainDisplay = chain.map((r) => r.taskId).join(' -> ');
-            output_1.output.error({
-                title: 'Recursive task invocation detected',
-                bodyLines: [
-                    `Nx detected a recursive loop of task invocations:`,
-                    ``,
-                    `  ${chainDisplay} -> ${task.id}`,
-                    ``,
-                    `Task "${task.id}" was already invoked by a parent Nx process in this chain.`,
-                    `This typically happens when a task's command (e.g., "nx ${task.target.target} ${task.target.project}")`,
-                    `triggers a chain of tasks that eventually re-invokes itself.`,
-                    ``,
-                    `To fix this, review the command configuration for the tasks in the chain above.`,
-                ],
-            });
-            process.exit(1);
+            // Diagnostic only, like registration. A leftover row is swept as stale.
         }
     }
     // endregion Processing Scheduled Tasks
@@ -503,8 +593,7 @@ class TaskOrchestrator {
             await this.hashBatchTasks(rootTasks);
             const eligible = [];
             for (const task of rootTasks) {
-                const depIds = batch.taskGraph.dependencies[task.id];
-                const hasNonCachedDep = depIds.some((id) => nonCachedTaskIds.has(id));
+                const hasNonCachedDep = batch.taskGraph.dependencies[task.id].some(({ id }) => nonCachedTaskIds.has(id));
                 if (hasNonCachedDep && this.deferredTaskIds?.has(task.id)) {
                     nonCachedTaskIds.add(task.id);
                     needsRehashAfterExecution.add(task.id);
@@ -553,17 +642,19 @@ class TaskOrchestrator {
             taskIds: Object.keys(batch.taskGraph.tasks),
         });
         const { cachedResults, needsRehashAfterExecution } = await this.applyBatchCachedResults(batch, doNotSkipCache, groupId);
-        // Schedule and start non-cached tasks (cached tasks were already
-        // started and completed inside applyBatchCachedResults)
-        const cachedTaskIds = new Set(cachedResults.map((r) => r.task.id));
-        const nonCachedTasks = tasks.filter((t) => !cachedTaskIds.has(t.id));
+        // Cached tasks were started and completed inside applyBatchCachedResults;
+        // a member skipped meanwhile (a dependency failed) must not run either
+        const nonCachedTasks = tasks.filter((t) => !this.completedTasks.has(t.id));
         if (nonCachedTasks.length > 0) {
             await Promise.all(nonCachedTasks.map((task) => this.options.lifeCycle.scheduleTask(task)));
             await this.preRunSteps(nonCachedTasks, { groupId });
         }
         // Phase 2: Run non-cached tasks, then re-hash tasks that read their outputs
-        const taskIdsToSkip = cachedResults.map((r) => r.task.id);
         let batchResults = [];
+        // Read again: a member can be skipped while the hooks above ran
+        const taskIdsToSkip = tasks
+            .filter((t) => this.completedTasks.has(t.id))
+            .map((t) => t.id);
         if (taskIdsToSkip.length < tasks.length) {
             const runGraph = (0, utils_1.removeTasksFromTaskGraph)(batch.taskGraph, taskIdsToSkip);
             for (const task of Object.values(runGraph.tasks)) {
@@ -923,8 +1014,7 @@ class TaskOrchestrator {
             // Wake coordinator — the delete above may satisfy the exit condition
             // (pendingDiscreteWorkers.size === 0) that was missed when
             // scheduleNextTasksAndReleaseThreads fired earlier.
-            this.waitingForTasks.forEach((f) => f(null));
-            this.waitingForTasks.length = 0;
+            this.releaseThreads();
         });
         this.pendingDiscreteWorkers.add(worker);
     }
@@ -947,13 +1037,36 @@ class TaskOrchestrator {
         await this.postRunSteps([{ task, status: 'failure', terminalOutput }], doNotSkipCache, groupId);
     }
     /**
-     * Spawn and wait on a task's child process, unconditionally — no cache
-     * lookup. Callers must have already confirmed the task is a cache miss
-     * (or disabled caching entirely).
+     * Spawn and wait on a task's child process once its ready dependencies are
+     * ready, with no cache lookup. Callers must have already confirmed the task
+     * is a cache miss (or disabled caching entirely).
      */
     async runTaskDirectly(doNotSkipCache, task, groupId) {
         // Wait for task to be processed
         const taskSpecificEnv = await this.processedTasks.get(task.id);
+        const waitError = await this.waitForReadyDependencies(task).then(() => null, (e) => e);
+        // Skipped by a failed dependency's propagation while waiting
+        if (this.completedTasks.has(task.id)) {
+            return {
+                task,
+                code: 1,
+                status: this.completedTasks.get(task.id),
+                terminalOutput: '',
+            };
+        }
+        if (waitError instanceof ready_when_1.ProducerFailedError) {
+            await this.skipWaiter(task, groupId);
+            return { task, code: 1, status: 'skipped', terminalOutput: '' };
+        }
+        if (waitError) {
+            await this.failTaskBeforeStart(task, groupId, waitError, doNotSkipCache);
+            return {
+                task,
+                code: 1,
+                status: 'failure',
+                terminalOutput: waitError.message,
+            };
+        }
         await this.preRunSteps([task], { groupId });
         const pipeOutput = await this.pipeOutputCapture(task);
         const temporaryOutputPath = this.cache.temporaryOutputPath(task);
@@ -1165,11 +1278,28 @@ class TaskOrchestrator {
     async startContinuousTask(task, groupId) {
         if (this.runningTasksService &&
             this.runningTasksService.getRunningTasks([task.id]).length) {
+            let readyWhen;
+            try {
+                readyWhen = this.getReadyWhen(task);
+            }
+            catch (e) {
+                return this.failTaskBeforeStart(task, groupId, e);
+            }
             await this.preRunSteps([task], { groupId });
             if (this.tuiEnabled) {
                 this.options.lifeCycle.setTaskStatus(task.id, 8 /* NativeTaskStatus.Shared */);
             }
             const runningTask = new shared_running_task_1.SharedRunningTask(this.runningTasksService, task.id);
+            if (readyWhen) {
+                const state = this.armReadiness(task.id);
+                this.pollReadinessRow(task, readyWhen, state.abort.signal, true).then(() => {
+                    if (!state.abort.signal.aborted)
+                        state.resolve();
+                }, (e) => {
+                    if (!state.abort.signal.aborted)
+                        this.failReadiness(task, state, e);
+                });
+            }
             this.runningContinuousTasks.set(task.id, {
                 runningTask,
                 groupId,
@@ -1183,34 +1313,72 @@ class TaskOrchestrator {
             }));
             // task is already running by another process, we schedule the next tasks
             // and release the threads
+            this.tasksSchedule.markContinuousTaskStarted(task.id);
             await this.scheduleNextTasksAndReleaseThreads();
             return runningTask;
         }
-        const taskSpecificEnv = await this.processedTasks.get(task.id);
-        await this.preRunSteps([task], { groupId });
-        const pipeOutput = await this.pipeOutputCapture(task);
-        // obtain metadata
-        const temporaryOutputPath = this.cache.temporaryOutputPath(task);
-        // Deliberately not gated on `printsTaskOutput`, unlike `runTaskDirectly`.
-        // `SummaryTerminalOutputLifeCycle.printTaskTerminalOutput` is a no-op and it
-        // only reports at `endCommand`, which never runs for a task that does not
-        // end - so suppressing here would leave a `nx serve` under `summary` with a
-        // permanently silent terminal and no file to read yet. Streaming is the only
-        // channel a continuous task has.
-        const streamOutput = (0, output_1.isStaticOutputStyle)(this.specifiedOutputStyle)
-            ? false
-            : (0, utils_1.shouldStreamOutput)(task, this.initiatingProject);
-        let env = pipeOutput
-            ? (0, task_env_1.getEnvVariablesForTask)(task, taskSpecificEnv, (0, task_env_1.getForceColorForChild)(), this.options.skipNxCache, this.options.captureStderr, null, null)
-            : (0, task_env_1.getEnvVariablesForTask)(task, taskSpecificEnv, undefined, this.options.skipNxCache, this.options.captureStderr, temporaryOutputPath, streamOutput);
-        this.detectTaskInvocationLoop(task);
-        const childProcess = await this.runTask(task, streamOutput, env, temporaryOutputPath, pipeOutput);
+        // Claim the task before doing any of the work below. getRunningTasks()
+        // above is a check-then-act: until this row exists, a sibling Nx process
+        // asking the same question is told the task is not running and starts a
+        // duplicate of it.
         this.runningTasksService?.addRunningTask(task.id);
-        this.runningContinuousTasks.set(task.id, {
-            runningTask: childProcess,
-            groupId,
-            ownsRunningTasksService: true,
-        });
+        let taskSpecificEnv;
+        let readyWhen;
+        try {
+            taskSpecificEnv = await this.processedTasks.get(task.id);
+            readyWhen = this.getReadyWhen(task);
+            await this.waitForReadyDependencies(task);
+        }
+        catch (e) {
+            this.runningTasksService?.removeRunningTask(task.id);
+            if (e instanceof ready_when_1.ProducerFailedError) {
+                await this.skipWaiter(task, groupId);
+                return new noop_child_process_1.NoopChildProcess({ code: 1, terminalOutput: '' });
+            }
+            return this.failTaskBeforeStart(task, groupId, e);
+        }
+        // Skipped by a failed dependency's propagation while waiting
+        if (this.completedTasks.has(task.id)) {
+            this.runningTasksService?.removeRunningTask(task.id);
+            return new noop_child_process_1.NoopChildProcess({ code: 1, terminalOutput: '' });
+        }
+        let childProcess;
+        try {
+            await this.preRunSteps([task], { groupId });
+            const pipeOutput = await this.pipeOutputCapture(task);
+            // obtain metadata
+            const temporaryOutputPath = this.cache.temporaryOutputPath(task);
+            // Deliberately not gated on `printsTaskOutput`, unlike `runTaskDirectly`.
+            // `SummaryTerminalOutputLifeCycle.printTaskTerminalOutput` is a no-op and it
+            // only reports at `endCommand`, which never runs for a task that does not
+            // end - so suppressing here would leave a `nx serve` under `summary` with a
+            // permanently silent terminal and no file to read yet. Streaming is the only
+            // channel a continuous task has.
+            const streamOutput = (0, output_1.isStaticOutputStyle)(this.specifiedOutputStyle)
+                ? false
+                : (0, utils_1.shouldStreamOutput)(task, this.initiatingProject);
+            let env = pipeOutput
+                ? (0, task_env_1.getEnvVariablesForTask)(task, taskSpecificEnv, (0, task_env_1.getForceColorForChild)(), this.options.skipNxCache, this.options.captureStderr, null, null)
+                : (0, task_env_1.getEnvVariablesForTask)(task, taskSpecificEnv, undefined, this.options.skipNxCache, this.options.captureStderr, temporaryOutputPath, streamOutput);
+            this.detectTaskInvocationLoop(task);
+            childProcess = await this.runTask(task, streamOutput, env, temporaryOutputPath, pipeOutput);
+            this.tasksSchedule.markContinuousTaskStarted(task.id);
+            if (readyWhen) {
+                this.startReadinessProbe(task, readyWhen, childProcess);
+            }
+            this.runningContinuousTasks.set(task.id, {
+                runningTask: childProcess,
+                groupId,
+                ownsRunningTasksService: true,
+            });
+        }
+        catch (e) {
+            // Nothing owns the claim until runningContinuousTasks holds it — the
+            // release paths (completeContinuousTask, the signal handler) both iterate
+            // that map. Drop it here or siblings wait on a task no one is running.
+            this.runningTasksService?.removeRunningTask(task.id);
+            throw e;
+        }
         this.continuousTaskExitHandled.set(task.id, new Promise((resolve) => {
             childProcess.onExit(async (code) => {
                 await this.handleContinuousTaskExit(code, task, groupId, true);
@@ -1224,6 +1392,194 @@ class TaskOrchestrator {
         return childProcess;
     }
     // endregion Single Task
+    // region Readiness
+    getReadyWhen(task) {
+        return (0, ready_when_1.getReadyWhen)(task, this.projectGraph, this.projects, workspace_root_1.workspaceRoot);
+    }
+    async waitForReadyDependencies(task) {
+        const producerIds = (0, ready_when_1.getReadyProducerIds)(task, this.fullTaskGraph);
+        for (const producerId of producerIds) {
+            const producer = this.fullTaskGraph.tasks[producerId];
+            const readyWhen = this.getReadyWhen(producer);
+            if (!readyWhen) {
+                if (process.env.NX_VERBOSE_LOGGING === 'true') {
+                    console.log(`Task "${producer.id}" declares no "readyWhen", so "${task.id}" runs once it has started.`);
+                }
+                continue;
+            }
+            // A producer in this task graph is held by dispatch and settled by its
+            // probe or row poll; one outside it (an agent's flat graph) has only its row
+            if (this.taskGraph.tasks[producerId]) {
+                await this.readinessOf(producerId).promise;
+            }
+            else {
+                this.onReadinessHold(producerId);
+                await this.pollReadinessRow(producer, readyWhen);
+            }
+        }
+    }
+    onReadinessHold(producerId) {
+        if (!this.waitingLogged.has(producerId) &&
+            !this.tuiEnabled &&
+            (0, output_1.printsTaskOutput)(this.resolvedOutputStyle)) {
+            this.waitingLogged.add(producerId);
+            console.log(`Waiting for "${producerId}" to be ready...`);
+        }
+    }
+    // A row absent from the start means no Nx process this one can see owns the
+    // producer, unless the caller already saw the owner: then it exited. A row
+    // that disappears mid-wait means the producer exited.
+    async pollReadinessRow(producer, readyWhen, signal, ownerObserved = false) {
+        const deadline = Date.now() + readyWhen.timeout;
+        let lastStatus = null;
+        while (true) {
+            if (signal?.aborted) {
+                throw (0, ready_when_1.notReadyError)(producer.id, 'exited');
+            }
+            if (this.stopRequested || this.bailed) {
+                throw (0, ready_when_1.notReadyError)(producer.id, 'was stopped');
+            }
+            const status = this.runningTasksService?.getTaskReadiness(producer.id) ?? null;
+            if (status === null) {
+                if (lastStatus !== null || ownerObserved) {
+                    throw (0, ready_when_1.notReadyError)(producer.id, 'exited');
+                }
+                return;
+            }
+            if (status !== lastStatus) {
+                lastStatus = status;
+                this.options.lifeCycle.setTaskReadiness?.(producer.id, status);
+            }
+            if (status === 1 /* TaskReadiness.Ready */) {
+                return;
+            }
+            if (status === 2 /* TaskReadiness.Failed */) {
+                throw new ready_when_1.ProducerFailedError(producer.id);
+            }
+            if (Date.now() >= deadline) {
+                throw (0, ready_when_1.readinessTimeoutError)(producer.id, readyWhen);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+    }
+    startReadinessProbe(task, readyWhen, runningTask) {
+        const state = this.armReadiness(task.id);
+        this.options.lifeCycle.setTaskReadiness?.(task.id, 0 /* TaskReadiness.Pending */);
+        (0, probes_1.waitForReadiness)(readyWhen, {
+            taskId: task.id,
+            runningTask,
+            cwd: workspace_root_1.workspaceRoot,
+            signal: state.abort.signal,
+        }).then(() => {
+            if (state.abort.signal.aborted)
+                return;
+            this.recordReadiness(task, 1 /* TaskReadiness.Ready */);
+            state.resolve();
+        }, (e) => {
+            if (state.abort.signal.aborted)
+                return;
+            this.recordReadiness(task, 2 /* TaskReadiness.Failed */);
+            this.failReadiness(task, state, e);
+        });
+    }
+    // A verdict against a producer something in this run waits for fails the
+    // producer, and the skip propagation covers the waiters. Without a waiter
+    // the producer is left alone, as a plain `nx serve` with a probe would be.
+    async failReadiness(task, state, e) {
+        const message = e?.message ?? String(e);
+        const entry = this.runningContinuousTasks.get(task.id);
+        if (entry &&
+            !(e instanceof ready_when_1.NotReadyError) &&
+            this.hasReadyWaiter(task.id)) {
+            if (this.tuiEnabled) {
+                this.options.lifeCycle.appendTaskOutput(task.id, `\r\n${message}\r\n`, false);
+            }
+            else {
+                console.error(message);
+            }
+            this.readinessFailureReasons.set(task.id, message);
+            const completion = this.complete([{ task, status: 'failure', terminalOutput: message }], entry.groupId);
+            this.readinessFailures.set(task.id, completion);
+            try {
+                await completion;
+            }
+            finally {
+                this.readinessFailures.delete(task.id);
+            }
+        }
+        else if (process.env.NX_VERBOSE_LOGGING === 'true') {
+            console.error(message);
+        }
+        state.reject(e);
+    }
+    // On an agent the waiter runs in another process with its own flat graph
+    hasReadyWaiter(producerId) {
+        return Object.values(this.fullTaskGraph.tasks).some((task) => (0, ready_when_1.getReadyProducerIds)(task, this.fullTaskGraph).includes(producerId));
+    }
+    // The producer is not in this task graph, so no skip propagates to the waiter
+    async skipWaiter(task, groupId) {
+        if (!this.completedTasks.has(task.id)) {
+            await this.complete([{ task, status: 'skipped' }], groupId);
+        }
+    }
+    // The probe's verdict stands for local waiters even when the row for
+    // other processes cannot be written
+    recordReadiness(task, status) {
+        this.options.lifeCycle.setTaskReadiness?.(task.id, status);
+        try {
+            this.runningTasksService?.setTaskReadiness(task.id, status);
+        }
+        catch (e) {
+            if (process.env.NX_VERBOSE_LOGGING === 'true') {
+                console.error(`Failed to record readiness of "${task.id}":`, e);
+            }
+        }
+    }
+    readinessOf(taskId) {
+        return this.readiness.get(taskId) ?? this.armReadiness(taskId);
+    }
+    // Replaces a state settled by an earlier run so a restart re-arms it
+    armReadiness(taskId) {
+        let state = this.readiness.get(taskId);
+        if (!state || state.settled) {
+            this.tasksSchedule.markReadinessPending(taskId);
+            state = createReadinessState((outcome) => {
+                if (outcome === 'ready') {
+                    this.tasksSchedule.markReady(taskId);
+                }
+                else {
+                    this.tasksSchedule.markReadinessFailed(taskId);
+                }
+                this.readinessSettled = true;
+                this.releaseThreads();
+            });
+            this.readiness.set(taskId, state);
+        }
+        return state;
+    }
+    abortReadiness(taskId, reason) {
+        const state = this.readiness.get(taskId);
+        if (!state)
+            return;
+        state.abort.abort();
+        state.reject((0, ready_when_1.notReadyError)(taskId, reason));
+    }
+    abortAllReadiness() {
+        for (const taskId of this.readiness.keys()) {
+            this.abortReadiness(taskId, 'was stopped');
+        }
+    }
+    async failTaskBeforeStart(task, groupId, e, doNotSkipCache = false) {
+        this.abortReadiness(task.id, 'failed');
+        // Already skipped by a failed dependency's propagation
+        if (!this.completedTasks.has(task.id)) {
+            // Lifecycles pair endTasks with startTasks, so start it before failing it
+            await this.preRunSteps([task], { groupId });
+            await this.handleDiscreteWorkerFailure(doNotSkipCache, task, groupId, e);
+        }
+        return new noop_child_process_1.NoopChildProcess({ code: 1, terminalOutput: e?.message ?? '' });
+    }
+    // endregion Readiness
     // region Lifecycle
     async preRunSteps(tasks, metadata) {
         const now = Date.now();
@@ -1364,7 +1720,13 @@ class TaskOrchestrator {
         if (!task.hash)
             return;
         try {
-            const { size } = (0, fs_1.statSync)(this.cache.temporaryOutputPath(task));
+            const outputPath = this.cache.temporaryOutputPath(task);
+            // The runner's exit handler wrote the child's transcript alone
+            const reason = this.readinessFailureReasons.get(task.id);
+            if (reason) {
+                (0, fs_1.appendFileSync)(outputPath, `\n${reason}\n`);
+            }
+            const { size } = (0, fs_1.statSync)(outputPath);
             this.cache.recordTerminalOutputs([{ hash: task.hash, size }]);
         }
         catch {
@@ -1373,12 +1735,13 @@ class TaskOrchestrator {
     }
     async scheduleNextTasksAndReleaseThreads() {
         if (this.stopRequested) {
-            this.waitingForTasks.forEach((f) => f(null));
-            this.waitingForTasks.length = 0;
+            this.releaseThreads();
             return;
         }
         await this.tasksSchedule.scheduleNextTasks();
-        // release blocked threads
+        this.releaseThreads();
+    }
+    releaseThreads() {
         this.waitingForTasks.forEach((f) => f(null));
         this.waitingForTasks.length = 0;
     }
@@ -1425,8 +1788,7 @@ class TaskOrchestrator {
             if (this.completedTasks.has(task.id))
                 continue;
             this.completedTasks.set(task.id, status);
-            this.taskInvocationTracker?.unregisterTask(task.id);
-            this.registeredInvocations.delete(task.id);
+            this.releaseTaskInvocation(task);
             if (this.tuiEnabled) {
                 this.options.lifeCycle.setTaskStatus(task.id, displayStatus ?? (0, native_1.parseTaskStatus)(status));
             }
@@ -1437,6 +1799,7 @@ class TaskOrchestrator {
                     // mark the execution as bailed which will stop all further execution
                     // only the tasks that are currently running will finish
                     this.bailed = true;
+                    this.abortAllReadiness();
                 }
                 else {
                     // Collect reverse deps to skip
@@ -1467,6 +1830,10 @@ class TaskOrchestrator {
             if (process.env.NX_NATIVE_COMMAND_RUNNER !== 'false') {
                 return true;
             }
+            // The log probe reads the captured output
+            if (this.getReadyWhen(task)?.kind === 'logMatches') {
+                return true;
+            }
             // When TUI is enabled, we need to use pipe output capture to support
             // progressive output streaming via the onOutput callback
             if (this.tuiEnabled) {
@@ -1520,8 +1887,13 @@ class TaskOrchestrator {
     }
     // endregion utils
     async handleContinuousTaskExit(code, task, groupId, ownsRunningTasksService) {
-        // If cleanup already completed this task, nothing left to do
+        await this.readinessFailures.get(task.id);
+        // Completed by cleanup, or failed its readiness check while still running
         if (this.completedTasks.has(task.id)) {
+            if (this.runningContinuousTasks.delete(task.id) &&
+                ownsRunningTasksService) {
+                this.runningTasksService?.removeRunningTask(task.id);
+            }
             return;
         }
         const stoppingReason = this.runningContinuousTasks.get(task.id)?.stoppingReason;
@@ -1542,15 +1914,20 @@ class TaskOrchestrator {
     isContinuousTaskNeeded(taskId) {
         return this.tasksSchedule
             .getIncompleteTasks()
-            .some((t) => this.taskGraph.continuousDependencies[t.id]?.includes(taskId));
+            .some((t) => this.taskGraph.continuousDependencies[t.id]?.some((edge) => edge.id === taskId));
     }
     async completeContinuousTask(task, groupId, ownsRunningTasksService, reason) {
-        if (this.completedTasks.has(task.id))
-            return;
+        await this.readinessFailures.get(task.id);
         this.runningContinuousTasks.delete(task.id);
         if (ownsRunningTasksService) {
             this.runningTasksService?.removeRunningTask(task.id);
         }
+        // A probe that does not need the child (url, port, command) could still
+        // pass during completion and release dependents of an exited producer
+        this.readiness.get(task.id)?.abort.abort();
+        // Failed its readiness check while still running
+        if (this.completedTasks.has(task.id))
+            return;
         task.endTime = Date.now();
         if (reason === 'fulfilled') {
             await this.complete([
@@ -1567,6 +1944,11 @@ class TaskOrchestrator {
         else {
             await this.complete([{ task, status: 'stopped' }], groupId);
         }
+        // After the skip propagation above, so a waiter is reported as skipped
+        // rather than as failed
+        this.abortReadiness(task.id, 'exited');
+        // Skipped dependents leave nothing in flight to wake the loops
+        this.releaseThreads();
     }
     async cleanup() {
         if (this.cleanupPromise) {
@@ -1576,6 +1958,8 @@ class TaskOrchestrator {
         return this.cleanupPromise;
     }
     async performCleanup() {
+        // Waiters must not sit out a probe once the run is stopping
+        this.abortAllReadiness();
         // Mark all running tasks for intentional stop
         const reason = this.stopRequested ? 'interrupted' : 'fulfilled';
         for (const entry of this.runningContinuousTasks.values()) {
@@ -1716,7 +2100,7 @@ class TaskOrchestrator {
             }
             const continuousDependencies = this.taskGraph.continuousDependencies[task.id];
             for (const continuousDependency of continuousDependencies) {
-                neededContinuousTasks.add(continuousDependency);
+                neededContinuousTasks.add(continuousDependency.id);
             }
         }
         for (const [taskId, entry] of this.runningContinuousTasks) {