nx 20.8.4 → 22.7.7

package/dist/src/tasks-runner/tasks-runner.d.ts

@@ -0,0 +1,24 @@
+import { NxJsonConfiguration } from '../config/nx-json';
+import { ProjectGraph } from '../config/project-graph';
+import { Task, TaskGraph } from '../config/task-graph';
+import { NxArgs } from '../utils/command-line-utils';
+import { TaskHasher } from '../hasher/task-hasher';
+import { DaemonClient } from '../daemon/client/client';
+export type TaskStatus = 'success' | 'failure' | 'skipped' | 'stopped' | 'local-cache-kept-existing' | 'local-cache' | 'remote-cache';
+/**
+ * `any | Promise<{ [id: string]: TaskStatus }>`
+ * will change to Promise<{ [id: string]: TaskStatus }> after Nx 15 is released.
+ */
+export type TasksRunner<T = unknown> = (tasks: Task[], options: T, context?: {
+    target?: string;
+    initiatingProject?: string | null;
+    initiatingTasks: Task[];
+    projectGraph: ProjectGraph;
+    nxJson: NxJsonConfiguration;
+    nxArgs: NxArgs;
+    taskGraph?: TaskGraph;
+    hasher?: TaskHasher;
+    daemon?: DaemonClient;
+}) => any | Promise<{
+    [id: string]: TaskStatus;
+}>;

package/dist/src/tasks-runner/tasks-schedule.d.ts

@@ -0,0 +1,48 @@
+import { DefaultTasksRunnerOptions } from './default-tasks-runner';
+import { Task, TaskGraph } from '../config/task-graph';
+import { ProjectGraph } from '../config/project-graph';
+import { ProjectConfiguration } from '../config/workspace-json-project-json';
+export interface Batch {
+    id: string;
+    executorName: string;
+    taskGraph: TaskGraph;
+}
+export declare class TasksSchedule {
+    private readonly projectGraph;
+    private readonly projects;
+    private readonly taskGraph;
+    private readonly options;
+    private notScheduledTaskGraph;
+    private reverseTaskDeps;
+    private reverseProjectGraph;
+    private taskHistory;
+    private scheduledBatches;
+    private scheduledTasks;
+    private runningTasks;
+    private completedTasks;
+    private scheduleRequestsExecutionChain;
+    private estimatedTaskTimings;
+    private projectDependencies;
+    private batchCounters;
+    constructor(projectGraph: ProjectGraph, projects: Record<string, ProjectConfiguration>, taskGraph: TaskGraph, options: DefaultTasksRunnerOptions);
+    init(): Promise<void>;
+    scheduleNextTasks(): Promise<void>;
+    hasTasks(): boolean;
+    complete(taskIds: string[]): void;
+    getAllScheduledTasks(): {
+        scheduledTasks: string[];
+        scheduledBatches: Batch[];
+    };
+    nextTask(filter?: (task: Task) => boolean): Task;
+    nextBatch(): Batch;
+    getIncompleteTasks(): Task[];
+    private scheduleTasks;
+    private scheduleTaskBatch;
+    private sortScheduledTasks;
+    private scheduleBatches;
+    private scheduleBatch;
+    private processTaskForBatches;
+    private canBatchTaskBeScheduled;
+    private canBeScheduled;
+    getEstimatedTaskTimings(): Record<string, number>;
+}

package/dist/src/tasks-runner/tasks-schedule.js

@@ -0,0 +1,258 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TasksSchedule = void 0;
+const utils_1 = require("./utils");
+const project_graph_utils_1 = require("../utils/project-graph-utils");
+const operators_1 = require("../project-graph/operators");
+const task_history_1 = require("../utils/task-history");
+class TasksSchedule {
+    constructor(projectGraph, projects, taskGraph, options) {
+        this.projectGraph = projectGraph;
+        this.projects = projects;
+        this.taskGraph = taskGraph;
+        this.options = options;
+        this.notScheduledTaskGraph = this.taskGraph;
+        this.reverseTaskDeps = (0, utils_1.calculateReverseDeps)(this.taskGraph);
+        this.reverseProjectGraph = (0, operators_1.reverse)(this.projectGraph);
+        this.taskHistory = (0, task_history_1.getTaskHistory)();
+        this.scheduledBatches = [];
+        this.scheduledTasks = [];
+        this.runningTasks = new Set();
+        this.completedTasks = new Set();
+        this.scheduleRequestsExecutionChain = Promise.resolve();
+        this.estimatedTaskTimings = {};
+        this.projectDependencies = {};
+        this.batchCounters = {};
+    }
+    async init() {
+        if (this.taskHistory) {
+            this.estimatedTaskTimings =
+                await this.taskHistory.getEstimatedTaskTimings(Object.values(this.taskGraph.tasks).map((t) => t.target));
+        }
+        for (const project of Object.values(this.taskGraph.tasks).map((t) => t.target.project)) {
+            this.projectDependencies[project] ??= (0, project_graph_utils_1.findAllProjectNodeDependencies)(project, this.reverseProjectGraph).length;
+        }
+    }
+    async scheduleNextTasks() {
+        this.scheduleRequestsExecutionChain =
+            this.scheduleRequestsExecutionChain.then(() => this.scheduleTasks());
+        await this.scheduleRequestsExecutionChain;
+    }
+    hasTasks() {
+        return (this.scheduledBatches.length +
+            this.scheduledTasks.length +
+            Object.keys(this.notScheduledTaskGraph.tasks).length !==
+            0);
+    }
+    complete(taskIds) {
+        for (const taskId of taskIds) {
+            this.completedTasks.add(taskId);
+            this.runningTasks.delete(taskId);
+            delete this.reverseTaskDeps[taskId];
+        }
+        const removedSet = new Set(taskIds);
+        this.scheduledTasks = this.scheduledTasks.filter((id) => !removedSet.has(id));
+        for (const [key, deps] of Object.entries(this.reverseTaskDeps)) {
+            this.reverseTaskDeps[key] = deps.filter((d) => !removedSet.has(d));
+        }
+        this.notScheduledTaskGraph = (0, utils_1.removeTasksFromTaskGraph)(this.notScheduledTaskGraph, taskIds);
+    }
+    getAllScheduledTasks() {
+        return {
+            scheduledTasks: this.scheduledTasks,
+            scheduledBatches: this.scheduledBatches,
+        };
+    }
+    nextTask(filter) {
+        if (this.scheduledTasks.length === 0) {
+            return null;
+        }
+        if (!filter) {
+            return this.taskGraph.tasks[this.scheduledTasks.shift()];
+        }
+        const idx = this.scheduledTasks.findIndex((id) => filter(this.taskGraph.tasks[id]));
+        if (idx === -1) {
+            return null;
+        }
+        const [taskId] = this.scheduledTasks.splice(idx, 1);
+        return this.taskGraph.tasks[taskId];
+    }
+    nextBatch() {
+        return this.scheduledBatches.length > 0
+            ? this.scheduledBatches.shift()
+            : null;
+    }
+    getIncompleteTasks() {
+        const incompleteTasks = [];
+        for (const taskId in this.taskGraph.tasks) {
+            if (!this.completedTasks.has(taskId)) {
+                incompleteTasks.push(this.taskGraph.tasks[taskId]);
+            }
+        }
+        return incompleteTasks;
+    }
+    async scheduleTasks() {
+        // Try to schedule batches unless --batch=false explicitly
+        // Individual tasks will be filtered by preferBatch in processTaskForBatches
+        // NX_BATCH_MODE env var is also checked for backward compatibility
+        if (this.options.batch !== false && process.env.NX_BATCH_MODE !== 'false') {
+            await this.scheduleBatches();
+        }
+        const toSchedule = [];
+        for (let root of this.notScheduledTaskGraph.roots) {
+            if (this.canBeScheduled(root)) {
+                toSchedule.push(root);
+                // Mark as running so canBeScheduled gates on parallelism for
+                // subsequent roots in this same scheduling pass.
+                this.runningTasks.add(root);
+            }
+        }
+        if (toSchedule.length > 0) {
+            this.scheduleTaskBatch(toSchedule);
+        }
+    }
+    scheduleTaskBatch(taskIds) {
+        this.notScheduledTaskGraph = (0, utils_1.removeTasksFromTaskGraph)(this.notScheduledTaskGraph, taskIds);
+        for (const taskId of taskIds) {
+            this.scheduledTasks.push(taskId);
+            this.runningTasks.add(taskId);
+        }
+        this.sortScheduledTasks();
+    }
+    sortScheduledTasks() {
+        // NOTE: sort task by most dependent on first
+        this.scheduledTasks.sort((taskId1, taskId2) => {
+            // First compare the length of task dependencies.
+            const taskDifference = this.reverseTaskDeps[taskId2].length -
+                this.reverseTaskDeps[taskId1].length;
+            if (taskDifference !== 0) {
+                return taskDifference;
+            }
+            // Tie-breaker for tasks with equal number of task dependencies.
+            // Most likely tasks with no dependencies such as test
+            const project1 = this.taskGraph.tasks[taskId1].target.project;
+            const project2 = this.taskGraph.tasks[taskId2].target.project;
+            const project1NodeDependencies = this.projectDependencies[project1];
+            const project2NodeDependencies = this.projectDependencies[project2];
+            const dependenciesDiff = project2NodeDependencies - project1NodeDependencies;
+            if (dependenciesDiff !== 0) {
+                return dependenciesDiff;
+            }
+            const task1Timing = this.estimatedTaskTimings[taskId1];
+            const task2Timing = this.estimatedTaskTimings[taskId2];
+            // Tasks with no historical timing run first (unknown duration = assume long)
+            const has1 = task1Timing != null && task1Timing !== 0;
+            const has2 = task2Timing != null && task2Timing !== 0;
+            if (!has1 && !has2)
+                return 0;
+            if (!has1)
+                return -1;
+            if (!has2)
+                return 1;
+            return task2Timing - task1Timing;
+        });
+    }
+    async scheduleBatches() {
+        const batchMap = {};
+        for (const root of this.notScheduledTaskGraph.roots) {
+            const rootTask = this.notScheduledTaskGraph.tasks[root];
+            const executorName = (0, utils_1.getExecutorNameForTask)(rootTask, this.projectGraph);
+            await this.processTaskForBatches(batchMap, rootTask, executorName, true, new Set());
+        }
+        for (const [executorName, taskGraph] of Object.entries(batchMap)) {
+            this.scheduleBatch(executorName, taskGraph);
+        }
+    }
+    scheduleBatch(executorName, taskGraph) {
+        // Generate batch ID with incrementing counter
+        if (!this.batchCounters[executorName]) {
+            this.batchCounters[executorName] = 0;
+        }
+        this.batchCounters[executorName]++;
+        const batchId = `${executorName} ${this.batchCounters[executorName]}`;
+        // Create a new task graph without the tasks that are being scheduled as part of this batch
+        this.notScheduledTaskGraph = (0, utils_1.removeTasksFromTaskGraph)(this.notScheduledTaskGraph, Object.keys(taskGraph.tasks));
+        this.scheduledBatches.push({ id: batchId, executorName, taskGraph });
+    }
+    async processTaskForBatches(batches, task, rootExecutorName, isRoot, visitedInBatch) {
+        // Skip if already processed in this batch - prevents redundant traversals
+        if (visitedInBatch.has(task.id)) {
+            return;
+        }
+        if (!this.canBatchTaskBeScheduled(task, batches[rootExecutorName])) {
+            return;
+        }
+        const { batchImplementationFactory, preferBatch } = (0, utils_1.getExecutorForTask)(task, this.projects);
+        const executorName = (0, utils_1.getExecutorNameForTask)(task, this.projectGraph);
+        if (rootExecutorName !== executorName) {
+            return;
+        }
+        if (!batchImplementationFactory) {
+            return;
+        }
+        // Check if we should batch this task:
+        // - If --batch is explicitly true or NX_BATCH_MODE=true, batch everything with batchImplementation
+        // - If --batch is not set (undefined) and NX_BATCH_MODE is not 'true', only batch if preferBatch is true
+        // - If --batch is explicitly false, we never get here (early return in scheduleTasks)
+        const batchForced = this.options.batch === true || process.env.NX_BATCH_MODE === 'true';
+        if (!batchForced && !preferBatch) {
+            return;
+        }
+        // Mark as visited only after all checks pass and we're actually adding to batch
+        // This ensures tasks can be added if they pass checks from any path
+        visitedInBatch.add(task.id);
+        const batch = (batches[rootExecutorName] =
+            batches[rootExecutorName] ??
+                {
+                    tasks: {},
+                    dependencies: {},
+                    continuousDependencies: {},
+                    roots: [],
+                });
+        batch.tasks[task.id] = task;
+        batch.dependencies[task.id] =
+            this.notScheduledTaskGraph.dependencies[task.id];
+        batch.continuousDependencies[task.id] =
+            this.notScheduledTaskGraph.continuousDependencies[task.id];
+        if (isRoot) {
+            batch.roots.push(task.id);
+        }
+        for (const dep of this.reverseTaskDeps[task.id]) {
+            const depTask = this.taskGraph.tasks[dep];
+            await this.processTaskForBatches(batches, depTask, rootExecutorName, false, visitedInBatch);
+        }
+    }
+    canBatchTaskBeScheduled(task, batchTaskGraph) {
+        // task self needs to support parallelism (undefined defaults to parallel)
+        // all deps have either completed or belong to the same batch
+        return (task.parallelism !== false &&
+            this.taskGraph.dependencies[task.id].every((id) => this.completedTasks.has(id) || !!batchTaskGraph?.tasks[id]));
+    }
+    canBeScheduled(taskId) {
+        const hasDependenciesCompleted = this.taskGraph.dependencies[taskId].every((id) => this.completedTasks.has(id));
+        const hasContinuousDependenciesStarted = this.taskGraph.continuousDependencies[taskId].every((id) => this.runningTasks.has(id));
+        // if dependencies have not completed, cannot schedule
+        if (!hasDependenciesCompleted || !hasContinuousDependenciesStarted) {
+            return false;
+        }
+        // if there are no running tasks, can schedule anything
+        if (this.runningTasks.size === 0) {
+            return true;
+        }
+        const runningTasksNotSupportParallelism = Array.from(this.runningTasks).some((taskId) => {
+            return this.taskGraph.tasks[taskId].parallelism === false;
+        });
+        if (runningTasksNotSupportParallelism) {
+            // if any running tasks do not support parallelism, no other tasks can be scheduled
+            return false;
+        }
+        else {
+            // if all running tasks support parallelism, can only schedule task with parallelism
+            return this.taskGraph.tasks[taskId].parallelism !== false;
+        }
+    }
+    getEstimatedTaskTimings() {
+        return this.estimatedTaskTimings;
+    }
+}
+exports.TasksSchedule = TasksSchedule;

package/dist/src/tasks-runner/utils.d.ts

@@ -0,0 +1,61 @@
+import { CustomHasher, ExecutorConfig } from '../config/misc-interfaces';
+import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph';
+import { Task, TaskGraph } from '../config/task-graph';
+import { ProjectConfiguration, TargetConfiguration, TargetDependencyConfig } from '../config/workspace-json-project-json';
+export type NormalizedTargetDependencyConfig = TargetDependencyConfig & {
+    projects: string[];
+};
+export declare function getDependencyConfigs({ project, target }: {
+    project: string;
+    target: string;
+}, extraTargetDependencies: Record<string, (TargetDependencyConfig | string)[]>, projectGraph: ProjectGraph, allTargetNames: string[]): NormalizedTargetDependencyConfig[] | undefined;
+export declare function normalizeDependencyConfigDefinition(definition: string | TargetDependencyConfig, currentProject: string, graph: ProjectGraph, allTargetNames: string[]): NormalizedTargetDependencyConfig[];
+export declare function normalizeDependencyConfigProjects(dependencyConfig: TargetDependencyConfig, currentProject: string, graph: ProjectGraph): NormalizedTargetDependencyConfig;
+export declare function expandDependencyConfigSyntaxSugar(dependencyConfigString: string | TargetDependencyConfig, graph: ProjectGraph, currentProject?: string): TargetDependencyConfig;
+export declare function expandWildcardTargetConfiguration(dependencyConfig: NormalizedTargetDependencyConfig, allTargetNames: string[]): NormalizedTargetDependencyConfig[];
+export declare function readProjectAndTargetFromTargetString(targetString: string, projects: Record<string, ProjectGraphProjectNode>, currentProject?: string): {
+    projects?: string[];
+    target: string;
+};
+export declare function getOutputs(p: Record<string, ProjectGraphProjectNode>, target: Task['target'], overrides: Task['overrides']): string[];
+export declare function normalizeTargetDependencyWithStringProjects(dependencyConfig: TargetDependencyConfig): Omit<TargetDependencyConfig, 'projects'> & {
+    projects: string[];
+};
+export declare function validateOutputs(outputs: string[]): void;
+export declare function transformLegacyOutputs(projectRoot: string, outputs: string[]): string[];
+/**
+ * @deprecated Pass the target and overrides instead. This will be removed in v20.
+ */
+export declare function getOutputsForTargetAndConfiguration(task: Task, node: ProjectGraphProjectNode): string[];
+export declare function getOutputsForTargetAndConfiguration(target: Task['target'] | Task, overrides: Task['overrides'] | ProjectGraphProjectNode, node: ProjectGraphProjectNode): string[];
+export declare function interpolate(template: string, data: any): string;
+export declare function getTargetConfigurationForTask(task: Task, projectGraph: ProjectGraph): TargetConfiguration | undefined;
+export declare function getExecutorNameForTask(task: Task, projectGraph: ProjectGraph): string;
+/**
+ * Expand a set of initiating task IDs by walking through any `nx:noop` tasks
+ * and replacing them with their direct dependencies + continuous dependencies.
+ * Non-noop tasks are kept as-is; cycles are safe.
+ *
+ * An `nx:noop` executor returns immediately, so if it is the only thing
+ * anchoring a continuous child, the child gets killed by
+ * `cleanUpUnneededContinuousTasks` the moment the noop completes. Treating the
+ * noop's dependencies as the real anchors preserves the intended orchestration.
+ */
+export declare function expandInitiatingTasksThroughNoop(initiatingTasks: Task[], taskGraph: TaskGraph, projectGraph: ProjectGraph): Set<string>;
+export declare function getExecutorForTask(task: Task, projects: Record<string, ProjectConfiguration>): ExecutorConfig & {
+    isNgCompat: boolean;
+    isNxExecutor: boolean;
+};
+export declare function getCustomHasher(task: Task, projects: Record<string, ProjectConfiguration>): CustomHasher | null;
+export declare function removeTasksFromTaskGraph(graph: TaskGraph, ids: string[]): TaskGraph;
+export declare function calculateReverseDeps(taskGraph: TaskGraph): Record<string, string[]>;
+export declare function getCliPath(): string;
+export declare function getPrintableCommandArgsForTask(task: Task): string[];
+export declare function getSerializedArgsForTask(task: Task, isVerbose: boolean): string[];
+export declare function shouldStreamOutput(task: Task, initiatingProject: string | null): boolean;
+export declare function isCacheableTask(task: Task, options: {
+    cacheableOperations?: string[] | null;
+    cacheableTargets?: string[] | null;
+}): boolean;
+export declare function unparse(options: Object): string[];
+export declare function createTaskId(project: string, target: string, configuration: string | undefined): string;