nx 19.4.0-canary.20240626-3a2e8d4 → 19.4.0-canary.20240627-c2c6a13

package/src/project-graph/plugins/isolation/messaging.d.ts

@@ -1,9 +1,11 @@
 /// <reference types="node" />
+/// <reference types="node" />
 import { ProjectGraph, ProjectGraphProcessorContext } from '../../../config/project-graph';
 import { PluginConfiguration } from '../../../config/nx-json';
-import { CreateDependenciesContext, CreateMetadataContext, CreateNodesContext } from '../public-api';
+import { CreateDependenciesContext, CreateMetadataContext, CreateNodesContextV2 } from '../public-api';
 import { LoadedNxPlugin } from '../internal-api';
 import { Serializable } from 'child_process';
+import { Socket } from 'net';
 export interface PluginWorkerLoadMessage {
     type: 'load';
     payload: {
@@ -31,7 +33,7 @@ export interface PluginWorkerCreateNodesMessage {
     type: 'createNodes';
     payload: {
         configFiles: string[];
-        context: CreateNodesContext;
+        context: CreateNodesContextV2;
         tx: string;
     };
 }
@@ -112,9 +114,10 @@ export declare function isPluginWorkerMessage(message: Serializable): message is
 export declare function isPluginWorkerResult(message: Serializable): message is PluginWorkerResult;
 type MaybePromise<T> = T | Promise<T>;
 type MessageHandlerReturn<T extends PluginWorkerMessage | PluginWorkerResult> = T extends PluginWorkerResult ? MaybePromise<PluginWorkerMessage | void> : MaybePromise<PluginWorkerResult | void>;
-export declare function consumeMessage<T extends PluginWorkerMessage | PluginWorkerResult>(raw: T, handlers: {
+export declare function consumeMessage<T extends PluginWorkerMessage | PluginWorkerResult>(socket: Socket, raw: T, handlers: {
     [K in T['type']]: (payload: Extract<T, {
         type: K;
     }>['payload']) => MessageHandlerReturn<T>;
 }): Promise<void>;
+export declare function sendMessageOverSocket(socket: Socket, message: PluginWorkerMessage | PluginWorkerResult): void;
 export {};

package/src/project-graph/plugins/isolation/messaging.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.consumeMessage = exports.isPluginWorkerResult = exports.isPluginWorkerMessage = void 0;
+exports.sendMessageOverSocket = exports.consumeMessage = exports.isPluginWorkerResult = exports.isPluginWorkerMessage = void 0;
 function isPluginWorkerMessage(message) {
     return (typeof message === 'object' &&
         'type' in message &&
@@ -10,6 +10,7 @@ function isPluginWorkerMessage(message) {
             'createNodes',
             'createDependencies',
             'processProjectGraph',
+            'createMetadata',
         ].includes(message.type));
 }
 exports.isPluginWorkerMessage = isPluginWorkerMessage;
@@ -22,19 +23,24 @@ function isPluginWorkerResult(message) {
             'createNodesResult',
             'createDependenciesResult',
             'processProjectGraphResult',
+            'createMetadataResult',
         ].includes(message.type));
 }
 exports.isPluginWorkerResult = isPluginWorkerResult;
 // Takes a message and a map of handlers and calls the appropriate handler
 // type safe and requires all handlers to be handled
-async function consumeMessage(raw, handlers) {
+async function consumeMessage(socket, raw, handlers) {
     const message = raw;
     const handler = handlers[message.type];
     if (handler) {
         const response = await handler(message.payload);
         if (response) {
-            process.send(response);
+            sendMessageOverSocket(socket, response);
         }
     }
 }
 exports.consumeMessage = consumeMessage;
+function sendMessageOverSocket(socket, message) {
+    socket.write(JSON.stringify(message) + String.fromCodePoint(4));
+}
+exports.sendMessageOverSocket = sendMessageOverSocket;

package/src/project-graph/plugins/isolation/plugin-pool.d.ts

@@ -1,3 +1,3 @@
 import { PluginConfiguration } from '../../../config/nx-json';
 import { LoadedNxPlugin } from '../internal-api';
-export declare function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string): [Promise<LoadedNxPlugin>, () => void];
+export declare function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string): Promise<[Promise<LoadedNxPlugin>, () => void]>;

package/src/project-graph/plugins/isolation/plugin-pool.js

@@ -6,52 +6,46 @@ const path = require("path");
 // TODO (@AgentEnder): After scoped verbose logging is implemented, re-add verbose logs here.
 // import { logger } from '../../utils/logger';
 const internal_api_1 = require("../internal-api");
+const socket_utils_1 = require("../../../daemon/socket-utils");
+const consume_messages_from_socket_1 = require("../../../utils/consume-messages-from-socket");
 const messaging_1 = require("./messaging");
+const net_1 = require("net");
 const cleanupFunctions = new Set();
 const pluginNames = new Map();
-function loadRemoteNxPlugin(plugin, root) {
-    // this should only really be true when running unit tests within
-    // the Nx repo. We still need to start the worker in this case,
-    // but its typescript.
-    const isWorkerTypescript = path.extname(__filename) === '.ts';
-    const workerPath = path.join(__dirname, 'plugin-worker');
-    const env = {
-        ...process.env,
-        ...(isWorkerTypescript
-            ? {
-                // Ensures that the worker uses the same tsconfig as the main process
-                TS_NODE_PROJECT: path.join(__dirname, '../../../../tsconfig.lib.json'),
-            }
-            : {}),
-    };
-    const worker = (0, child_process_1.fork)(workerPath, [], {
-        stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
-        env,
-        execArgv: [
-            ...process.execArgv,
-            // If the worker is typescript, we need to register ts-node
-            ...(isWorkerTypescript ? ['-r', 'ts-node/register'] : []),
-        ],
-    });
-    worker.send({ type: 'load', payload: { plugin, root } });
-    // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`);
+const MAX_MESSAGE_WAIT = 1000 * 60 * 5; // 5 minutes
+const nxPluginWorkerCache = (global['nxPluginWorkerCache'] ??= new Map());
+async function loadRemoteNxPlugin(plugin, root) {
+    const cacheKey = JSON.stringify({ plugin, root });
+    if (nxPluginWorkerCache.has(cacheKey)) {
+        return [nxPluginWorkerCache.get(cacheKey), () => { }];
+    }
+    const { worker, socket } = await startPluginWorker();
     const pendingPromises = new Map();
     const exitHandler = createWorkerExitHandler(worker, pendingPromises);
     const cleanupFunction = () => {
         worker.off('exit', exitHandler);
+        socket.destroy();
         shutdownPluginWorker(worker);
+        nxPluginWorkerCache.delete(cacheKey);
     };
     cleanupFunctions.add(cleanupFunction);
-    return [
-        new Promise((res, rej) => {
-            worker.on('message', createWorkerHandler(worker, pendingPromises, res, rej));
-            worker.on('exit', exitHandler);
-        }),
-        () => {
-            cleanupFunction();
-            cleanupFunctions.delete(cleanupFunction);
-        },
-    ];
+    const pluginPromise = new Promise((res, rej) => {
+        (0, messaging_1.sendMessageOverSocket)(socket, {
+            type: 'load',
+            payload: { plugin, root },
+        });
+        // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`);
+        const loadTimeout = setTimeout(() => {
+            rej(new Error('Plugin worker timed out when loading plugin:' + plugin));
+        }, MAX_MESSAGE_WAIT);
+        socket.on('data', (0, consume_messages_from_socket_1.consumeMessagesFromSocket)(createWorkerHandler(worker, pendingPromises, (val) => {
+            clearTimeout(loadTimeout);
+            res(val);
+        }, rej, socket)));
+        worker.on('exit', exitHandler);
+    });
+    nxPluginWorkerCache.set(cacheKey, pluginPromise);
+    return [pluginPromise, cleanupFunction];
 }
 exports.loadRemoteNxPlugin = loadRemoteNxPlugin;
 function shutdownPluginWorker(worker) {
@@ -68,13 +62,15 @@ function shutdownPluginWorker(worker) {
  * @param onloadError Rejecter for RemotePlugin promise
  * @returns Function to handle messages from the worker
  */
-function createWorkerHandler(worker, pending, onload, onloadError) {
+function createWorkerHandler(worker, pending, onload, onloadError, socket) {
     let pluginName;
-    return function (message) {
+    let txId = 0;
+    return function (raw) {
+        const message = JSON.parse(raw);
         if (!(0, messaging_1.isPluginWorkerResult)(message)) {
             return;
         }
-        return (0, messaging_1.consumeMessage)(message, {
+        return (0, messaging_1.consumeMessage)(socket, message, {
             'load-result': (result) => {
                 if (result.success) {
                     const { name, createNodesPattern, include, exclude } = result;
@@ -88,9 +84,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             ? [
                                 createNodesPattern,
                                 (configFiles, ctx) => {
-                                    const tx = pluginName + ':createNodes:' + performance.now();
+                                    const tx = pluginName + worker.pid + ':createNodes:' + txId++;
                                     return registerPendingPromise(tx, pending, () => {
-                                        worker.send({
+                                        (0, messaging_1.sendMessageOverSocket)(socket, {
                                             type: 'createNodes',
                                             payload: { configFiles, context: ctx, tx },
                                         });
@@ -100,9 +96,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         createDependencies: result.hasCreateDependencies
                             ? (ctx) => {
-                                const tx = pluginName + ':createDependencies:' + performance.now();
+                                const tx = pluginName + worker.pid + ':createDependencies:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'createDependencies',
                                         payload: { context: ctx, tx },
                                     });
@@ -111,9 +107,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         processProjectGraph: result.hasProcessProjectGraph
                             ? (graph, ctx) => {
-                                const tx = pluginName + ':processProjectGraph:' + performance.now();
+                                const tx = pluginName + worker.pid + ':processProjectGraph:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'processProjectGraph',
                                         payload: { graph, ctx, tx },
                                     });
@@ -122,9 +118,9 @@ function createWorkerHandler(worker, pending, onload, onloadError) {
                             : undefined,
                         createMetadata: result.hasCreateMetadata
                             ? (graph, ctx) => {
-                                const tx = pluginName + ':createMetadata:' + performance.now();
+                                const tx = pluginName + worker.pid + ':createMetadata:' + txId++;
                                 return registerPendingPromise(tx, pending, () => {
-                                    worker.send({
+                                    (0, messaging_1.sendMessageOverSocket)(socket, {
                                         type: 'createMetadata',
                                         payload: { graph, context: ctx, tx },
                                     });
@@ -183,19 +179,30 @@ function createWorkerExitHandler(worker, pendingPromises) {
         }
     };
 }
-process.on('exit', () => {
+let cleanedUp = false;
+const exitHandler = () => {
+    if (cleanedUp)
+        return;
     for (const fn of cleanupFunctions) {
         fn();
     }
-});
+    cleanedUp = true;
+};
+process.on('exit', exitHandler);
+process.on('SIGINT', exitHandler);
+process.on('SIGTERM', exitHandler);
 function registerPendingPromise(tx, pending, callback) {
-    let resolver, rejector;
+    let resolver, rejector, timeout;
     const promise = new Promise((res, rej) => {
-        resolver = res;
         rejector = rej;
+        resolver = res;
+        timeout = setTimeout(() => {
+            rej(new Error(`Plugin worker timed out when processing message ${tx}`));
+        }, MAX_MESSAGE_WAIT);
         callback();
     }).finally(() => {
         pending.delete(tx);
+        clearTimeout(timeout);
     });
     pending.set(tx, {
         promise,
@@ -204,3 +211,70 @@ function registerPendingPromise(tx, pending, callback) {
     });
     return promise;
 }
+global.nxPluginWorkerCount ??= 0;
+async function startPluginWorker() {
+    // this should only really be true when running unit tests within
+    // the Nx repo. We still need to start the worker in this case,
+    // but its typescript.
+    const isWorkerTypescript = path.extname(__filename) === '.ts';
+    const workerPath = path.join(__dirname, 'plugin-worker');
+    const env = {
+        ...process.env,
+        ...(isWorkerTypescript
+            ? {
+                // Ensures that the worker uses the same tsconfig as the main process
+                TS_NODE_PROJECT: path.join(__dirname, '../../../../tsconfig.lib.json'),
+            }
+            : {}),
+    };
+    const ipcPath = (0, socket_utils_1.getPluginOsSocketPath)([process.pid, global.nxPluginWorkerCount++].join('-'));
+    const worker = (0, child_process_1.fork)(workerPath, [ipcPath], {
+        stdio: process.stdout.isTTY ? 'inherit' : 'ignore',
+        env,
+        execArgv: [
+            ...process.execArgv,
+            // If the worker is typescript, we need to register ts-node
+            ...(isWorkerTypescript ? ['-r', 'ts-node/register'] : []),
+        ],
+        detached: true,
+    });
+    worker.disconnect();
+    worker.unref();
+    let attempts = 0;
+    return new Promise((resolve, reject) => {
+        const id = setInterval(async () => {
+            const socket = await isServerAvailable(ipcPath);
+            if (socket) {
+                socket.unref();
+                clearInterval(id);
+                resolve({
+                    worker,
+                    socket,
+                });
+            }
+            else if (attempts > 1000) {
+                // daemon fails to start, the process probably exited
+                // we print the logs and exit the client
+                reject('Failed to start plugin worker.');
+            }
+            else {
+                attempts++;
+            }
+        }, 10);
+    });
+}
+function isServerAvailable(ipcPath) {
+    return new Promise((resolve) => {
+        try {
+            const socket = (0, net_1.connect)(ipcPath, () => {
+                resolve(socket);
+            });
+            socket.once('error', () => {
+                resolve(false);
+            });
+        }
+        catch (err) {
+            resolve(false);
+        }
+    });
+}

package/src/project-graph/plugins/isolation/plugin-worker.js

@@ -3,116 +3,137 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const messaging_1 = require("./messaging");
 const loader_1 = require("../loader");
 const serializable_error_1 = require("../../../utils/serializable-error");
+const net_1 = require("net");
+const consume_messages_from_socket_1 = require("../../../utils/consume-messages-from-socket");
+const fs_1 = require("fs");
 if (process.env.NX_PERF_LOGGING === 'true') {
     require('../../../utils/perf-logging');
 }
 global.NX_GRAPH_CREATION = true;
 let plugin;
-process.on('message', async (message) => {
-    if (!(0, messaging_1.isPluginWorkerMessage)(message)) {
-        return;
-    }
-    return (0, messaging_1.consumeMessage)(message, {
-        load: async ({ plugin: pluginConfiguration, root }) => {
-            process.chdir(root);
-            try {
-                const [promise] = (0, loader_1.loadNxPlugin)(pluginConfiguration, root);
-                plugin = await promise;
-                return {
-                    type: 'load-result',
-                    payload: {
-                        name: plugin.name,
-                        include: plugin.include,
-                        exclude: plugin.exclude,
-                        createNodesPattern: plugin.createNodes?.[0],
-                        hasCreateDependencies: 'createDependencies' in plugin && !!plugin.createDependencies,
-                        hasProcessProjectGraph: 'processProjectGraph' in plugin && !!plugin.processProjectGraph,
-                        hasCreateMetadata: 'createMetadata' in plugin && !!plugin.createMetadata,
-                        success: true,
-                    },
-                };
-            }
-            catch (e) {
-                return {
-                    type: 'load-result',
-                    payload: {
-                        success: false,
-                        error: (0, serializable_error_1.createSerializableError)(e),
-                    },
-                };
-            }
-        },
-        createNodes: async ({ configFiles, context, tx }) => {
-            try {
-                const result = await plugin.createNodes[1](configFiles, context);
-                return {
-                    type: 'createNodesResult',
-                    payload: { result, success: true, tx },
-                };
-            }
-            catch (e) {
-                return {
-                    type: 'createNodesResult',
-                    payload: {
-                        success: false,
-                        error: (0, serializable_error_1.createSerializableError)(e),
-                        tx,
-                    },
-                };
-            }
-        },
-        createDependencies: async ({ context, tx }) => {
-            try {
-                const result = await plugin.createDependencies(context);
-                return {
-                    type: 'createDependenciesResult',
-                    payload: { dependencies: result, success: true, tx },
-                };
-            }
-            catch (e) {
-                return {
-                    type: 'createDependenciesResult',
-                    payload: {
-                        success: false,
-                        error: (0, serializable_error_1.createSerializableError)(e),
-                        tx,
-                    },
-                };
-            }
-        },
-        processProjectGraph: async ({ graph, ctx, tx }) => {
-            try {
-                const result = await plugin.processProjectGraph(graph, ctx);
-                return {
-                    type: 'processProjectGraphResult',
-                    payload: { graph: result, success: true, tx },
-                };
-            }
-            catch (e) {
-                return {
-                    type: 'processProjectGraphResult',
-                    payload: {
-                        success: false,
-                        error: (0, serializable_error_1.createSerializableError)(e),
-                        tx,
-                    },
-                };
-            }
-        },
-        createMetadata: async ({ graph, context, tx }) => {
-            try {
-                const result = await plugin.createMetadata(graph, context);
-                return {
-                    type: 'createMetadataResult',
-                    payload: { metadata: result, success: true, tx },
-                };
-            }
-            catch (e) {
-                return {
-                    type: 'createMetadataResult',
-                    payload: { success: false, error: e.stack, tx },
-                };
-            }
-        },
-    });
+const socketPath = process.argv[2];
+const server = (0, net_1.createServer)((socket) => {
+    socket.on('data', (0, consume_messages_from_socket_1.consumeMessagesFromSocket)((raw) => {
+        const message = JSON.parse(raw.toString());
+        if (!(0, messaging_1.isPluginWorkerMessage)(message)) {
+            return;
+        }
+        return (0, messaging_1.consumeMessage)(socket, message, {
+            load: async ({ plugin: pluginConfiguration, root }) => {
+                process.chdir(root);
+                try {
+                    const [promise] = (0, loader_1.loadNxPlugin)(pluginConfiguration, root);
+                    plugin = await promise;
+                    return {
+                        type: 'load-result',
+                        payload: {
+                            name: plugin.name,
+                            include: plugin.include,
+                            exclude: plugin.exclude,
+                            createNodesPattern: plugin.createNodes?.[0],
+                            hasCreateDependencies: 'createDependencies' in plugin && !!plugin.createDependencies,
+                            hasProcessProjectGraph: 'processProjectGraph' in plugin &&
+                                !!plugin.processProjectGraph,
+                            hasCreateMetadata: 'createMetadata' in plugin && !!plugin.createMetadata,
+                            success: true,
+                        },
+                    };
+                }
+                catch (e) {
+                    return {
+                        type: 'load-result',
+                        payload: {
+                            success: false,
+                            error: (0, serializable_error_1.createSerializableError)(e),
+                        },
+                    };
+                }
+            },
+            createNodes: async ({ configFiles, context, tx }) => {
+                try {
+                    const result = await plugin.createNodes[1](configFiles, context);
+                    return {
+                        type: 'createNodesResult',
+                        payload: { result, success: true, tx },
+                    };
+                }
+                catch (e) {
+                    return {
+                        type: 'createNodesResult',
+                        payload: {
+                            success: false,
+                            error: (0, serializable_error_1.createSerializableError)(e),
+                            tx,
+                        },
+                    };
+                }
+            },
+            createDependencies: async ({ context, tx }) => {
+                try {
+                    const result = await plugin.createDependencies(context);
+                    return {
+                        type: 'createDependenciesResult',
+                        payload: { dependencies: result, success: true, tx },
+                    };
+                }
+                catch (e) {
+                    return {
+                        type: 'createDependenciesResult',
+                        payload: {
+                            success: false,
+                            error: (0, serializable_error_1.createSerializableError)(e),
+                            tx,
+                        },
+                    };
+                }
+            },
+            processProjectGraph: async ({ graph, ctx, tx }) => {
+                try {
+                    const result = await plugin.processProjectGraph(graph, ctx);
+                    return {
+                        type: 'processProjectGraphResult',
+                        payload: { graph: result, success: true, tx },
+                    };
+                }
+                catch (e) {
+                    return {
+                        type: 'processProjectGraphResult',
+                        payload: {
+                            success: false,
+                            error: (0, serializable_error_1.createSerializableError)(e),
+                            tx,
+                        },
+                    };
+                }
+            },
+            createMetadata: async ({ graph, context, tx }) => {
+                try {
+                    const result = await plugin.createMetadata(graph, context);
+                    return {
+                        type: 'createMetadataResult',
+                        payload: { metadata: result, success: true, tx },
+                    };
+                }
+                catch (e) {
+                    return {
+                        type: 'createMetadataResult',
+                        payload: { success: false, error: e.stack, tx },
+                    };
+                }
+            },
+        });
+    }));
 });
+server.listen(socketPath);
+const exitHandler = (exitCode) => () => {
+    server.close();
+    try {
+        (0, fs_1.unlinkSync)(socketPath);
+    }
+    catch (e) { }
+    process.exit(exitCode);
+};
+const events = ['SIGINT', 'SIGTERM', 'SIGQUIT', 'exit'];
+events.forEach((event) => process.once(event, exitHandler(0)));
+process.once('uncaughtException', exitHandler(1));
+process.once('unhandledRejection', exitHandler(1));

package/src/project-graph/utils/retrieve-workspace-files.d.ts

@@ -9,12 +9,12 @@ import { LoadedNxPlugin } from '../plugins/internal-api';
  * @param nxJson
  */
 export declare function retrieveWorkspaceFiles(workspaceRoot: string, projectRootMap: Record<string, string>): Promise<{
-    allWorkspaceFiles: import("nx/src/devkit-exports").FileData[];
+    allWorkspaceFiles: import("../file-utils").FileData[];
     fileMap: {
         projectFileMap: ProjectFiles;
-        nonProjectFiles: import("nx/src/native").FileData[];
+        nonProjectFiles: import("../../native").FileData[];
     };
-    rustReferences: import("nx/src/native").NxWorkspaceFilesExternals;
+    rustReferences: import("../../native").NxWorkspaceFilesExternals;
 }>;
 /**
  * Walk through the workspace and return `ProjectConfigurations`. Only use this if the projectFileMap is not needed.

package/src/tasks-runner/task-env.d.ts

@@ -6,3 +6,16 @@ export declare function getEnvVariablesForTask(task: Task, taskSpecificEnv: Node
     [x: string]: string;
     TZ?: string;
 };
+/**
+ * This function loads a .env file and expands the variables in it.
+ * It is going to override existing environmentVariables.
+ * @param filename
+ * @param environmentVariables
+ */
+export declare function loadAndExpandDotEnvFile(filename: string, environmentVariables: NodeJS.ProcessEnv, override?: boolean): import("dotenv-expand").DotenvExpandOutput;
+/**
+ * This function unloads a .env file and removes the variables in it from the environmentVariables.
+ * @param filename
+ * @param environmentVariables
+ */
+export declare function unloadDotEnvFile(filename: string, environmentVariables: NodeJS.ProcessEnv, override?: boolean): void;