nx 19.4.0-canary.20240626-3a2e8d4 → 19.4.0-rc.0

package/src/plugins/target-defaults/target-defaults-plugin.d.ts

@@ -17,6 +17,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     executor?: undefined;
     options?: undefined;
@@ -31,6 +39,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
 } | {
@@ -44,6 +60,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
 } | {
@@ -52,6 +76,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
     options?: undefined;
@@ -66,6 +98,14 @@ export declare function getTargetInfo(target: string, projectJsonTargets: Record
         [x: string]: any;
         description?: string;
         technologies?: string[];
+        nonAtomizedTarget?: string;
+        help?: {
+            command: string;
+            example: {
+                options?: Record<string, unknown>;
+                args?: string[];
+            };
+        };
     };
     command?: undefined;
 } | {

package/src/project-graph/plugins/internal-api.js

@@ -76,7 +76,7 @@ async function loadNxPlugins(plugins, root = workspace_root_1.workspaceRoot) {
     plugins = await normalizePlugins(plugins, root);
     const cleanupFunctions = [];
     for (const plugin of plugins) {
-        const [loadedPluginPromise, cleanup] = loadingMethod(plugin, root);
+        const [loadedPluginPromise, cleanup] = await loadingMethod(plugin, root);
         result.push(loadedPluginPromise);
         cleanupFunctions.push(cleanup);
     }

package/src/project-graph/plugins/isolation/index.d.ts

@@ -1,3 +1,3 @@
 import { PluginConfiguration } from '../../../config/nx-json';
 import { LoadedNxPlugin } from '../internal-api';
-export declare function loadNxPluginInIsolation(plugin: PluginConfiguration, root?: string): readonly [Promise<LoadedNxPlugin>, () => void];
+export declare function loadNxPluginInIsolation(plugin: PluginConfiguration, root?: string): Promise<readonly [Promise<LoadedNxPlugin>, () => void]>;

package/src/project-graph/plugins/isolation/index.js

@@ -3,25 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadNxPluginInIsolation = void 0;
 const workspace_root_1 = require("../../../utils/workspace-root");
 const plugin_pool_1 = require("./plugin-pool");
-/**
- * Used to ensure 1 plugin : 1 worker
- */
-const remotePluginCache = new Map();
-function loadNxPluginInIsolation(plugin, root = workspace_root_1.workspaceRoot) {
-    const cacheKey = JSON.stringify(plugin);
-    if (remotePluginCache.has(cacheKey)) {
-        return remotePluginCache.get(cacheKey);
-    }
-    const [loadingPlugin, cleanup] = (0, plugin_pool_1.loadRemoteNxPlugin)(plugin, root);
-    // We clean up plugin workers when Nx process completes.
-    const val = [
+async function loadNxPluginInIsolation(plugin, root = workspace_root_1.workspaceRoot) {
+    const [loadingPlugin, cleanup] = await (0, plugin_pool_1.loadRemoteNxPlugin)(plugin, root);
+    return [
         loadingPlugin,
         () => {
             cleanup();
-            remotePluginCache.delete(cacheKey);
         },
     ];
-    remotePluginCache.set(cacheKey, val);
-    return val;
 }
 exports.loadNxPluginInIsolation = loadNxPluginInIsolation;

package/src/project-graph/plugins/isolation/messaging.d.ts

@@ -1,9 +1,11 @@
 /// <reference types="node" />
+/// <reference types="node" />
 import { ProjectGraph, ProjectGraphProcessorContext } from '../../../config/project-graph';
 import { PluginConfiguration } from '../../../config/nx-json';
-import { CreateDependenciesContext, CreateMetadataContext, CreateNodesContext } from '../public-api';
+import { CreateDependenciesContext, CreateMetadataContext, CreateNodesContextV2 } from '../public-api';
 import { LoadedNxPlugin } from '../internal-api';
 import { Serializable } from 'child_process';
+import { Socket } from 'net';
 export interface PluginWorkerLoadMessage {
     type: 'load';
     payload: {
@@ -31,7 +33,7 @@ export interface PluginWorkerCreateNodesMessage {
     type: 'createNodes';
     payload: {
         configFiles: string[];
-        context: CreateNodesContext;
+        context: CreateNodesContextV2;
         tx: string;
     };
 }
@@ -112,9 +114,10 @@ export declare function isPluginWorkerMessage(message: Serializable): message is
 export declare function isPluginWorkerResult(message: Serializable): message is PluginWorkerResult;
 type MaybePromise<T> = T | Promise<T>;
 type MessageHandlerReturn<T extends PluginWorkerMessage | PluginWorkerResult> = T extends PluginWorkerResult ? MaybePromise<PluginWorkerMessage | void> : MaybePromise<PluginWorkerResult | void>;
-export declare function consumeMessage<T extends PluginWorkerMessage | PluginWorkerResult>(raw: T, handlers: {
+export declare function consumeMessage<T extends PluginWorkerMessage | PluginWorkerResult>(socket: Socket, raw: T, handlers: {
     [K in T['type']]: (payload: Extract<T, {
         type: K;
     }>['payload']) => MessageHandlerReturn<T>;
 }): Promise<void>;
+export declare function sendMessageOverSocket(socket: Socket, message: PluginWorkerMessage | PluginWorkerResult): void;
 export {};

package/src/project-graph/plugins/isolation/messaging.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.consumeMessage = exports.isPluginWorkerResult = exports.isPluginWorkerMessage = void 0;
+exports.sendMessageOverSocket = exports.consumeMessage = exports.isPluginWorkerResult = exports.isPluginWorkerMessage = void 0;
 function isPluginWorkerMessage(message) {
     return (typeof message === 'object' &&
         'type' in message &&
@@ -10,6 +10,7 @@ function isPluginWorkerMessage(message) {
             'createNodes',
             'createDependencies',
             'processProjectGraph',
+            'createMetadata',
         ].includes(message.type));
 }
 exports.isPluginWorkerMessage = isPluginWorkerMessage;
@@ -22,19 +23,24 @@ function isPluginWorkerResult(message) {
             'createNodesResult',
             'createDependenciesResult',
             'processProjectGraphResult',
+            'createMetadataResult',
         ].includes(message.type));
 }
 exports.isPluginWorkerResult = isPluginWorkerResult;
 // Takes a message and a map of handlers and calls the appropriate handler
 // type safe and requires all handlers to be handled
-async function consumeMessage(raw, handlers) {
+async function consumeMessage(socket, raw, handlers) {
     const message = raw;
     const handler = handlers[message.type];
     if (handler) {
         const response = await handler(message.payload);
         if (response) {
-            process.send(response);
+            sendMessageOverSocket(socket, response);
         }
     }
 }
 exports.consumeMessage = consumeMessage;
+function sendMessageOverSocket(socket, message) {
+    socket.write(JSON.stringify(message) + String.fromCodePoint(4));
+}
+exports.sendMessageOverSocket = sendMessageOverSocket;

package/src/project-graph/plugins/isolation/plugin-pool.d.ts

@@ -1,3 +1,3 @@
 import { PluginConfiguration } from '../../../config/nx-json';
 import { LoadedNxPlugin } from '../internal-api';
-export declare function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string): [Promise<LoadedNxPlugin>, () => void];
+export declare function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string): Promise<[Promise<LoadedNxPlugin>, () => void]>;

package/src/project-graph/plugins/isolation/plugin-pool.js

@@ -3,55 +3,50 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadRemoteNxPlugin = void 0;
 const child_process_1 = require("child_process");
 const path = require("path");
+const net_1 = require("net");
 // TODO (@AgentEnder): After scoped verbose logging is implemented, re-add verbose logs here.
 // import { logger } from '../../utils/logger';
 const internal_api_1 = require("../internal-api");
+const socket_utils_1 = require("../../../daemon/socket-utils");
+const consume_messages_from_socket_1 = require("../../../utils/consume-messages-from-socket");
+const exit_codes_1 = require("../../../utils/exit-codes");
 const messaging_1 = require("./messaging");
 const cleanupFunctions = new Set();
 const pluginNames = new Map();
-function loadRemoteNxPlugin(plugin, root) {
-    // this should only really be true when running unit tests within
-    // the Nx repo. We still need to start the worker in this case,
-    // but its typescript.
-    const isWorkerTypescript = path.extname(__filename) === '.ts';
-    const workerPath = path.join(__dirname, 'plugin-worker');
-    const env = {
-        ...process.env,
-        ...(isWorkerTypescript
-            ? {
-                // Ensures that the worker uses the same tsconfig as the main process
-                TS_NODE_PROJECT: path.join(__dirname, '../../../../tsconfig.lib.json'),
-            }
-            : {}),
-    };
-    const worker = (0, child_process_1.fork)(workerPath, [], {
-        stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
-        env,
-        execArgv: [
-            ...process.execArgv,
-            // If the worker is typescript, we need to register ts-node
-            ...(isWorkerTypescript ? ['-r', 'ts-node/register'] : []),
-        ],
-    });
-    worker.send({ type: 'load', payload: { plugin, root } });
-    // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`);
+const MAX_MESSAGE_WAIT = 1000 * 60 * 5; // 5 minutes
+const nxPluginWorkerCache = (global['nxPluginWorkerCache'] ??= new Map());
+async function loadRemoteNxPlugin(plugin, root) {
+    const cacheKey = JSON.stringify({ plugin, root });
+    if (nxPluginWorkerCache.has(cacheKey)) {
+        return [nxPluginWorkerCache.get(cacheKey), () => { }];
+    }
+    const { worker, socket } = await startPluginWorker();
     const pendingPromises = new Map();
     const exitHandler = createWorkerExitHandler(worker, pendingPromises);
     const cleanupFunction = () => {
         worker.off('exit', exitHandler);
+        socket.destroy();
         shutdownPluginWorker(worker);
+        nxPluginWorkerCache.delete(cacheKey);
     };
     cleanupFunctions.add(cleanupFunction);
-    return [
-        new Promise((res, rej) => {
-            worker.on('message', createWorkerHandler(worker, pendingPromises, res, rej));
-            worker.on('exit', exitHandler);
-        }),
-        () => {
-            cleanupFunction();
-            cleanupFunctions.delete(cleanupFunction);
-        },
-    ];
+    const pluginPromise = new Promise((res, rej) => {
+        (0, messaging_1.sendMessageOverSocket)(socket, {
+            type: 'load',
+            payload: { plugin, root },
+        });
+        // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`);
+        const loadTimeout = setTimeout(() => {
+            rej(new Error('Plugin worker timed out when loading plugin:' + plugin));
+        }, MAX_MESSAGE_WAIT);
+        socket.on('data', (0, consume_messages_from_socket_1.consumeMessagesFromSocket)(createWorkerHandler(worker, pendingPromises, (val) => {
+            clearTimeout(loadTimeout);
+            res(val);
+        }, rej, socket)));
+        worker.on('exit', exitHandler);
+    });
+    nxPluginWorkerCache.set(cacheKey, pluginPromise);
+    return [pluginPromise, cleanupFunction];
 }
 exports.loadRemoteNxPlugin = loadRemoteNxPlugin;
 function shutdownPluginWorker(worker) {
@@ -68,13 +63,15 @@ function shutdownPluginWorker(worker) {
  * @param onloadError Rejecter for RemotePlugin promise
  * @returns Function to handle messages from the worker
  */
-function createWorkerHandler(worker, pending, onload, onloadError) {
+function createWorkerHandler(worker, pending, onload, onloadError, socket) {
     let pluginName;
-    return function (message) {
+    let txId = 0;
+    return function (raw) {
+        const message = JSON.parse(raw);
         if (!(0, messaging_1.isPluginWorkerResult)(message)) {
             return;
         }
-        return (0, messaging_1.consumeMessage)(message, {
+        return (0, messaging_1.consumeMessage)(socket, message, {
             'load-result': (result) => {
                 if (result.success) {
                     const { name, createNodesPattern, include, exclude } = result;
@@ -88,9 +85,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             ? [
                                 createNodesPattern,
                                 (configFiles, ctx) => {
-                                    const tx = pluginName + ':createNodes:' + performance.now();
+                                    const tx = pluginName + worker.pid + ':createNodes:' + txId++;
                                     return registerPendingPromise(tx, pending, () => {
-                                        worker.send({
+                                        (0, messaging_1.sendMessageOverSocket)(socket, {
                                             type: 'createNodes',
                                             payload: { configFiles, context: ctx, tx },
                                         });
@@ -100,9 +97,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         createDependencies: result.hasCreateDependencies
                             ? (ctx) => {
-                                const tx = pluginName + ':createDependencies:' + performance.now();
+                                const tx = pluginName + worker.pid + ':createDependencies:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'createDependencies',
                                         payload: { context: ctx, tx },
                                     });
@@ -111,9 +108,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         processProjectGraph: result.hasProcessProjectGraph
                             ? (graph, ctx) => {
-                                const tx = pluginName + ':processProjectGraph:' + performance.now();
+                                const tx = pluginName + worker.pid + ':processProjectGraph:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'processProjectGraph',
                                         payload: { graph, ctx, tx },
                                     });
@@ -122,9 +119,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         createMetadata: result.hasCreateMetadata
                             ? (graph, ctx) => {
-                                const tx = pluginName + ':createMetadata:' + performance.now();
+                                const tx = pluginName + worker.pid + ':createMetadata:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'createMetadata',
                                         payload: { graph, context: ctx, tx },
                                     });
@@ -183,19 +180,31 @@ function createWorkerExitHandler(worker, pendingPromises) {
         }
     };
 }
-process.on('exit', () => {
+let cleanedUp = false;
+const exitHandler = () => {
     for (const fn of cleanupFunctions) {
         fn();
     }
+    cleanedUp = true;
+};
+process.on('exit', exitHandler);
+process.on('SIGINT', () => {
+    exitHandler();
+    process.exit((0, exit_codes_1.signalToCode)('SIGINT'));
 });
+process.on('SIGTERM', exitHandler);
 function registerPendingPromise(tx, pending, callback) {
-    let resolver, rejector;
+    let resolver, rejector, timeout;
     const promise = new Promise((res, rej) => {
-        resolver = res;
         rejector = rej;
+        resolver = res;
+        timeout = setTimeout(() => {
+            rej(new Error(`Plugin worker timed out when processing message ${tx}`));
+        }, MAX_MESSAGE_WAIT);
         callback();
     }).finally(() => {
         pending.delete(tx);
+        clearTimeout(timeout);
     });
     pending.set(tx, {
         promise,
@@ -204,3 +213,70 @@ function registerPendingPromise(tx, pending, callback) {
     });
     return promise;
 }
+global.nxPluginWorkerCount ??= 0;
+async function startPluginWorker() {
+    // this should only really be true when running unit tests within
+    // the Nx repo. We still need to start the worker in this case,
+    // but its typescript.
+    const isWorkerTypescript = path.extname(__filename) === '.ts';
+    const workerPath = path.join(__dirname, 'plugin-worker');
+    const env = {
+        ...process.env,
+        ...(isWorkerTypescript
+            ? {
+                // Ensures that the worker uses the same tsconfig as the main process
+                TS_NODE_PROJECT: path.join(__dirname, '../../../../tsconfig.lib.json'),
+            }
+            : {}),
+    };
+    const ipcPath = (0, socket_utils_1.getPluginOsSocketPath)([process.pid, global.nxPluginWorkerCount++].join('-'));
+    const worker = (0, child_process_1.spawn)(process.execPath, [
+        ...(isWorkerTypescript ? ['--require', 'ts-node/register'] : []),
+        workerPath,
+        ipcPath,
+    ], {
+        stdio: process.stdout.isTTY ? 'inherit' : 'ignore',
+        env,
+        detached: true,
+        shell: false,
+        windowsHide: true,
+    });
+    worker.unref();
+    let attempts = 0;
+    return new Promise((resolve, reject) => {
+        const id = setInterval(async () => {
+            const socket = await isServerAvailable(ipcPath);
+            if (socket) {
+                socket.unref();
+                clearInterval(id);
+                resolve({
+                    worker,
+                    socket,
+                });
+            }
+            else if (attempts > 1000) {
+                // daemon fails to start, the process probably exited
+                // we print the logs and exit the client
+                reject('Failed to start plugin worker.');
+            }
+            else {
+                attempts++;
+            }
+        }, 10);
+    });
+}
+function isServerAvailable(ipcPath) {
+    return new Promise((resolve) => {
+        try {
+            const socket = (0, net_1.connect)(ipcPath, () => {
+                resolve(socket);
+            });
+            socket.once('error', () => {
+                resolve(false);
+            });
+        }
+        catch (err) {
+            resolve(false);
+        }
+    });
+}