nx 19.4.0-canary.20240626-3a2e8d4 → 19.4.0-canary.20240628-336d371

package/src/plugins/js/utils/register.js

@@ -7,7 +7,45 @@ const swcNodeInstalled = packageIsInstalled('@swc-node/register');
 const tsNodeInstalled = packageIsInstalled('ts-node/register');
 let ts;
 let isTsEsmLoaderRegistered = false;
+/**
+ * tsx is a utility to run TypeScript files in node which is growing in popularity:
+ * https://tsx.is
+ *
+ * Behind the scenes it is invoking node with relevant --require and --import flags.
+ *
+ * If the user is invoking Nx via a script which is being invoked via tsx, then we
+ * do not need to register any transpiler at all as the environment will have already
+ * been configured by tsx. In fact, registering a transpiler such as ts-node or swc
+ * in this case causes issues.
+ *
+ * Because node is being invoked by tsx, the tsx binary does not end up in the final
+ * process.argv and so we need to check a few possible things to account for usage
+ * via different package managers (e.g. pnpm does not set process._ to tsx, but rather
+ * pnpm itself, modern yarn does not set process._ at all etc.).
+ */
+const isInvokedByTsx = (() => {
+    if (process.env._?.endsWith(`${path_1.sep}tsx`)) {
+        return true;
+    }
+    const requireArgs = [];
+    const importArgs = [];
+    (process.execArgv ?? []).forEach((arg, i) => {
+        if (arg === '-r' || arg === '--require') {
+            requireArgs.push(process.execArgv[i + 1]);
+        }
+        if (arg === '--import') {
+            importArgs.push(process.execArgv[i + 1]);
+        }
+    });
+    const isTsxPath = (p) => p.includes(`${path_1.sep}tsx${path_1.sep}`);
+    return (requireArgs.some((a) => isTsxPath(a)) ||
+        importArgs.some((a) => isTsxPath(a)));
+})();
 function registerTsProject(path, configFilename) {
+    // See explanation alongside isInvokedByTsx declaration
+    if (isInvokedByTsx) {
+        return () => { };
+    }
     const tsConfigPath = configFilename ? (0, path_1.join)(path, configFilename) : path;
     const compilerOptions = readCompilerOptions(tsConfigPath);
     const cleanupFunctions = [

package/src/plugins/project-json/build-nodes/package-json-next-to-project-json.js

@@ -5,6 +5,7 @@ const path_1 = require("path");
 const fs_1 = require("fs");
 const fileutils_1 = require("../../../utils/fileutils");
 const package_json_1 = require("../../../utils/package-json");
+const package_json_workspaces_1 = require("../../package-json-workspaces");
 // TODO: Remove this one day, this should not need to be done.
 exports.PackageJsonProjectsNextToProjectJsonPlugin = {
     // Its not a problem if plugins happen to have same name, and this
@@ -27,8 +28,14 @@ exports.PackageJsonProjectsNextToProjectJsonPlugin = {
 exports.default = exports.PackageJsonProjectsNextToProjectJsonPlugin;
 function createProjectFromPackageJsonNextToProjectJson(projectJsonPath, workspaceRoot) {
     const root = (0, path_1.dirname)(projectJsonPath);
-    const packageJsonPath = (0, path_1.join)(workspaceRoot, root, 'package.json');
-    if (!(0, fs_1.existsSync)(packageJsonPath)) {
+    const relativePackageJsonPath = (0, path_1.join)(root, 'package.json');
+    const packageJsonPath = (0, path_1.join)(workspaceRoot, relativePackageJsonPath);
+    const readJson = (f) => (0, fileutils_1.readJsonFile)((0, path_1.join)(workspaceRoot, f));
+    // Do not create projects for package.json files
+    // that are part of the package manager workspaces
+    // Those package.json files will be processed later on
+    const matcher = (0, package_json_workspaces_1.buildPackageJsonWorkspacesMatcher)(workspaceRoot, readJson);
+    if (!(0, fs_1.existsSync)(packageJsonPath) || matcher(relativePackageJsonPath)) {
         return null;
     }
     try {

package/src/plugins/target-defaults/target-defaults-plugin.d.ts

@@ -17,6 +17,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     executor?: undefined;
     options?: undefined;
@@ -31,6 +39,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
 } | {
@@ -44,6 +60,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
 } | {
@@ -52,6 +76,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
     options?: undefined;
@@ -66,6 +98,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
 } | {

package/src/project-graph/plugins/internal-api.js

@@ -76,7 +76,7 @@ async function loadNxPlugins(plugins, root = workspace_root_1.workspaceRoot) {
     plugins = await normalizePlugins(plugins, root);
     const cleanupFunctions = [];
     for (const plugin of plugins) {
-        const [loadedPluginPromise, cleanup] = loadingMethod(plugin, root);
+        const [loadedPluginPromise, cleanup] = await loadingMethod(plugin, root);
         result.push(loadedPluginPromise);
         cleanupFunctions.push(cleanup);
     }

package/src/project-graph/plugins/isolation/index.d.ts

@@ -1,3 +1,3 @@
 import { PluginConfiguration } from '../../../config/nx-json';
 import { LoadedNxPlugin } from '../internal-api';
-export declare function loadNxPluginInIsolation(plugin: PluginConfiguration, root?: string): readonly [Promise<LoadedNxPlugin>, () => void];
+export declare function loadNxPluginInIsolation(plugin: PluginConfiguration, root?: string): Promise<readonly [Promise<LoadedNxPlugin>, () => void]>;

package/src/project-graph/plugins/isolation/index.js

@@ -3,25 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadNxPluginInIsolation = void 0;
 const workspace_root_1 = require("../../../utils/workspace-root");
 const plugin_pool_1 = require("./plugin-pool");
-/**
- * Used to ensure 1 plugin : 1 worker
- */
-const remotePluginCache = new Map();
-function loadNxPluginInIsolation(plugin, root = workspace_root_1.workspaceRoot) {
-    const cacheKey = JSON.stringify(plugin);
-    if (remotePluginCache.has(cacheKey)) {
-        return remotePluginCache.get(cacheKey);
-    }
-    const [loadingPlugin, cleanup] = (0, plugin_pool_1.loadRemoteNxPlugin)(plugin, root);
-    // We clean up plugin workers when Nx process completes.
-    const val = [
+async function loadNxPluginInIsolation(plugin, root = workspace_root_1.workspaceRoot) {
+    const [loadingPlugin, cleanup] = await (0, plugin_pool_1.loadRemoteNxPlugin)(plugin, root);
+    return [
         loadingPlugin,
         () => {
             cleanup();
-            remotePluginCache.delete(cacheKey);
         },
     ];
-    remotePluginCache.set(cacheKey, val);
-    return val;
 }
 exports.loadNxPluginInIsolation = loadNxPluginInIsolation;

package/src/project-graph/plugins/isolation/messaging.d.ts

@@ -1,9 +1,11 @@
 /// <reference types="node" />
+/// <reference types="node" />
 import { ProjectGraph, ProjectGraphProcessorContext } from '../../../config/project-graph';
 import { PluginConfiguration } from '../../../config/nx-json';
-import { CreateDependenciesContext, CreateMetadataContext, CreateNodesContext } from '../public-api';
+import { CreateDependenciesContext, CreateMetadataContext, CreateNodesContextV2 } from '../public-api';
 import { LoadedNxPlugin } from '../internal-api';
 import { Serializable } from 'child_process';
+import { Socket } from 'net';
 export interface PluginWorkerLoadMessage {
     type: 'load';
     payload: {
@@ -31,7 +33,7 @@ export interface PluginWorkerCreateNodesMessage {
     type: 'createNodes';
     payload: {
         configFiles: string[];
-        context: CreateNodesContext;
+        context: CreateNodesContextV2;
         tx: string;
     };
 }
@@ -112,9 +114,10 @@ export declare function isPluginWorkerMessage(message: Serializable): message is
 export declare function isPluginWorkerResult(message: Serializable): message is PluginWorkerResult;
 type MaybePromise<T> = T | Promise<T>;
 type MessageHandlerReturn<T extends PluginWorkerMessage | PluginWorkerResult> = T extends PluginWorkerResult ? MaybePromise<PluginWorkerMessage | void> : MaybePromise<PluginWorkerResult | void>;
-export declare function consumeMessage<T extends PluginWorkerMessage | PluginWorkerResult>(raw: T, handlers: {
+export declare function consumeMessage<T extends PluginWorkerMessage | PluginWorkerResult>(socket: Socket, raw: T, handlers: {
     [K in T['type']]: (payload: Extract<T, {
         type: K;
     }>['payload']) => MessageHandlerReturn<T>;
 }): Promise<void>;
+export declare function sendMessageOverSocket(socket: Socket, message: PluginWorkerMessage | PluginWorkerResult): void;
 export {};

package/src/project-graph/plugins/isolation/messaging.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.consumeMessage = exports.isPluginWorkerResult = exports.isPluginWorkerMessage = void 0;
+exports.sendMessageOverSocket = exports.consumeMessage = exports.isPluginWorkerResult = exports.isPluginWorkerMessage = void 0;
 function isPluginWorkerMessage(message) {
     return (typeof message === 'object' &&
         'type' in message &&
@@ -10,6 +10,7 @@ function isPluginWorkerMessage(message) {
             'createNodes',
             'createDependencies',
             'processProjectGraph',
+            'createMetadata',
         ].includes(message.type));
 }
 exports.isPluginWorkerMessage = isPluginWorkerMessage;
@@ -22,19 +23,24 @@ function isPluginWorkerResult(message) {
             'createNodesResult',
             'createDependenciesResult',
             'processProjectGraphResult',
+            'createMetadataResult',
         ].includes(message.type));
 }
 exports.isPluginWorkerResult = isPluginWorkerResult;
 // Takes a message and a map of handlers and calls the appropriate handler
 // type safe and requires all handlers to be handled
-async function consumeMessage(raw, handlers) {
+async function consumeMessage(socket, raw, handlers) {
     const message = raw;
     const handler = handlers[message.type];
     if (handler) {
         const response = await handler(message.payload);
         if (response) {
-            process.send(response);
+            sendMessageOverSocket(socket, response);
         }
     }
 }
 exports.consumeMessage = consumeMessage;
+function sendMessageOverSocket(socket, message) {
+    socket.write(JSON.stringify(message) + String.fromCodePoint(4));
+}
+exports.sendMessageOverSocket = sendMessageOverSocket;

package/src/project-graph/plugins/isolation/plugin-pool.d.ts

@@ -1,3 +1,3 @@
 import { PluginConfiguration } from '../../../config/nx-json';
 import { LoadedNxPlugin } from '../internal-api';
-export declare function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string): [Promise<LoadedNxPlugin>, () => void];
+export declare function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string): Promise<[Promise<LoadedNxPlugin>, () => void]>;

package/src/project-graph/plugins/isolation/plugin-pool.js

@@ -3,55 +3,50 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadRemoteNxPlugin = void 0;
 const child_process_1 = require("child_process");
 const path = require("path");
+const net_1 = require("net");
 // TODO (@AgentEnder): After scoped verbose logging is implemented, re-add verbose logs here.
 // import { logger } from '../../utils/logger';
 const internal_api_1 = require("../internal-api");
+const socket_utils_1 = require("../../../daemon/socket-utils");
+const consume_messages_from_socket_1 = require("../../../utils/consume-messages-from-socket");
+const exit_codes_1 = require("../../../utils/exit-codes");
 const messaging_1 = require("./messaging");
 const cleanupFunctions = new Set();
 const pluginNames = new Map();
-function loadRemoteNxPlugin(plugin, root) {
-    // this should only really be true when running unit tests within
-    // the Nx repo. We still need to start the worker in this case,
-    // but its typescript.
-    const isWorkerTypescript = path.extname(__filename) === '.ts';
-    const workerPath = path.join(__dirname, 'plugin-worker');
-    const env = {
-        ...process.env,
-        ...(isWorkerTypescript
-            ? {
-                // Ensures that the worker uses the same tsconfig as the main process
-                TS_NODE_PROJECT: path.join(__dirname, '../../../../tsconfig.lib.json'),
-            }
-            : {}),
-    };
-    const worker = (0, child_process_1.fork)(workerPath, [], {
-        stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
-        env,
-        execArgv: [
-            ...process.execArgv,
-            // If the worker is typescript, we need to register ts-node
-            ...(isWorkerTypescript ? ['-r', 'ts-node/register'] : []),
-        ],
-    });
-    worker.send({ type: 'load', payload: { plugin, root } });
-    // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`);
+const MAX_MESSAGE_WAIT = 1000 * 60 * 5; // 5 minutes
+const nxPluginWorkerCache = (global['nxPluginWorkerCache'] ??= new Map());
+async function loadRemoteNxPlugin(plugin, root) {
+    const cacheKey = JSON.stringify({ plugin, root });
+    if (nxPluginWorkerCache.has(cacheKey)) {
+        return [nxPluginWorkerCache.get(cacheKey), () => { }];
+    }
+    const { worker, socket } = await startPluginWorker();
     const pendingPromises = new Map();
     const exitHandler = createWorkerExitHandler(worker, pendingPromises);
     const cleanupFunction = () => {
         worker.off('exit', exitHandler);
+        socket.destroy();
         shutdownPluginWorker(worker);
+        nxPluginWorkerCache.delete(cacheKey);
     };
     cleanupFunctions.add(cleanupFunction);
-    return [
-        new Promise((res, rej) => {
-            worker.on('message', createWorkerHandler(worker, pendingPromises, res, rej));
-            worker.on('exit', exitHandler);
-        }),
-        () => {
-            cleanupFunction();
-            cleanupFunctions.delete(cleanupFunction);
-        },
-    ];
+    const pluginPromise = new Promise((res, rej) => {
+        (0, messaging_1.sendMessageOverSocket)(socket, {
+            type: 'load',
+            payload: { plugin, root },
+        });
+        // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`);
+        const loadTimeout = setTimeout(() => {
+            rej(new Error('Plugin worker timed out when loading plugin:' + plugin));
+        }, MAX_MESSAGE_WAIT);
+        socket.on('data', (0, consume_messages_from_socket_1.consumeMessagesFromSocket)(createWorkerHandler(worker, pendingPromises, (val) => {
+            clearTimeout(loadTimeout);
+            res(val);
+        }, rej, socket)));
+        worker.on('exit', exitHandler);
+    });
+    nxPluginWorkerCache.set(cacheKey, pluginPromise);
+    return [pluginPromise, cleanupFunction];
 }
 exports.loadRemoteNxPlugin = loadRemoteNxPlugin;
 function shutdownPluginWorker(worker) {
@@ -68,13 +63,15 @@ function shutdownPluginWorker(worker) {
  * @param onloadError Rejecter for RemotePlugin promise
  * @returns Function to handle messages from the worker
  */
-function createWorkerHandler(worker, pending, onload, onloadError) {
+function createWorkerHandler(worker, pending, onload, onloadError, socket) {
     let pluginName;
-    return function (message) {
+    let txId = 0;
+    return function (raw) {
+        const message = JSON.parse(raw);
         if (!(0, messaging_1.isPluginWorkerResult)(message)) {
             return;
         }
-        return (0, messaging_1.consumeMessage)(message, {
+        return (0, messaging_1.consumeMessage)(socket, message, {
             'load-result': (result) => {
                 if (result.success) {
                     const { name, createNodesPattern, include, exclude } = result;
@@ -88,9 +85,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             ? [
                                 createNodesPattern,
                                 (configFiles, ctx) => {
-                                    const tx = pluginName + ':createNodes:' + performance.now();
+                                    const tx = pluginName + worker.pid + ':createNodes:' + txId++;
                                     return registerPendingPromise(tx, pending, () => {
-                                        worker.send({
+                                        (0, messaging_1.sendMessageOverSocket)(socket, {
                                             type: 'createNodes',
                                             payload: { configFiles, context: ctx, tx },
                                         });
@@ -100,9 +97,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         createDependencies: result.hasCreateDependencies
                             ? (ctx) => {
-                                const tx = pluginName + ':createDependencies:' + performance.now();
+                                const tx = pluginName + worker.pid + ':createDependencies:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'createDependencies',
                                         payload: { context: ctx, tx },
                                     });
@@ -111,9 +108,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         processProjectGraph: result.hasProcessProjectGraph
                             ? (graph, ctx) => {
-                                const tx = pluginName + ':processProjectGraph:' + performance.now();
+                                const tx = pluginName + worker.pid + ':processProjectGraph:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'processProjectGraph',
                                         payload: { graph, ctx, tx },
                                     });
@@ -122,9 +119,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         createMetadata: result.hasCreateMetadata
                             ? (graph, ctx) => {
-                                const tx = pluginName + ':createMetadata:' + performance.now();
+                                const tx = pluginName + worker.pid + ':createMetadata:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'createMetadata',
                                         payload: { graph, context: ctx, tx },
                                     });
@@ -183,19 +180,31 @@ function createWorkerExitHandler(worker, pendingPromises) {
         }
     };
 }
-process.on('exit', () => {
+let cleanedUp = false;
+const exitHandler = () => {
     for (const fn of cleanupFunctions) {
         fn();
     }
+    cleanedUp = true;
+};
+process.on('exit', exitHandler);
+process.on('SIGINT', () => {
+    exitHandler();
+    process.exit((0, exit_codes_1.signalToCode)('SIGINT'));
 });
+process.on('SIGTERM', exitHandler);
 function registerPendingPromise(tx, pending, callback) {
-    let resolver, rejector;
+    let resolver, rejector, timeout;
     const promise = new Promise((res, rej) => {
-        resolver = res;
         rejector = rej;
+        resolver = res;
+        timeout = setTimeout(() => {
+            rej(new Error(`Plugin worker timed out when processing message ${tx}`));
+        }, MAX_MESSAGE_WAIT);
         callback();
     }).finally(() => {
         pending.delete(tx);
+        clearTimeout(timeout);
     });
     pending.set(tx, {
         promise,
@@ -204,3 +213,70 @@ function registerPendingPromise(tx, pending, callback) {
     });
     return promise;
 }
+global.nxPluginWorkerCount ??= 0;
+async function startPluginWorker() {
+    // this should only really be true when running unit tests within
+    // the Nx repo. We still need to start the worker in this case,
+    // but its typescript.
+    const isWorkerTypescript = path.extname(__filename) === '.ts';
+    const workerPath = path.join(__dirname, 'plugin-worker');
+    const env = {
+        ...process.env,
+        ...(isWorkerTypescript
+            ? {
+                // Ensures that the worker uses the same tsconfig as the main process
+                TS_NODE_PROJECT: path.join(__dirname, '../../../../tsconfig.lib.json'),
+            }
+            : {}),
+    };
+    const ipcPath = (0, socket_utils_1.getPluginOsSocketPath)([process.pid, global.nxPluginWorkerCount++].join('-'));
+    const worker = (0, child_process_1.spawn)(process.execPath, [
+        ...(isWorkerTypescript ? ['--require', 'ts-node/register'] : []),
+        workerPath,
+        ipcPath,
+    ], {
+        stdio: process.stdout.isTTY ? 'inherit' : 'ignore',
+        env,
+        detached: true,
+        shell: false,
+        windowsHide: true,
+    });
+    worker.unref();
+    let attempts = 0;
+    return new Promise((resolve, reject) => {
+        const id = setInterval(async () => {
+            const socket = await isServerAvailable(ipcPath);
+            if (socket) {
+                socket.unref();
+                clearInterval(id);
+                resolve({
+                    worker,
+                    socket,
+                });
+            }
+            else if (attempts > 1000) {
+                // daemon fails to start, the process probably exited
+                // we print the logs and exit the client
+                reject('Failed to start plugin worker.');
+            }
+            else {
+                attempts++;
+            }
+        }, 10);
+    });
+}
+function isServerAvailable(ipcPath) {
+    return new Promise((resolve) => {
+        try {
+            const socket = (0, net_1.connect)(ipcPath, () => {
+                resolve(socket);
+            });
+            socket.once('error', () => {
+                resolve(false);
+            });
+        }
+        catch (err) {
+            resolve(false);
+        }
+    });
+}