@atlaspack/workers 2.14.31 → 2.14.32

package/dist/child.js

@@ -0,0 +1,290 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Child = void 0;
+const assert_1 = __importDefault(require("assert"));
+const nullthrows_1 = __importDefault(require("nullthrows"));
+const build_cache_1 = require("@atlaspack/build-cache");
+const logger_1 = __importStar(require("@atlaspack/logger"));
+const diagnostic_1 = __importStar(require("@atlaspack/diagnostic"));
+const bus_1 = __importDefault(require("./bus"));
+const profiler_1 = require("@atlaspack/profiler");
+const Handle_1 = __importDefault(require("./Handle"));
+// /flow-to-ts helpers
+// The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
+const Handle = Handle_1.default;
+class Child {
+    constructor(ChildBackend) {
+        this.callQueue = [];
+        this.maxConcurrentCalls = 10;
+        this.responseId = 0;
+        this.responseQueue = new Map();
+        // @ts-expect-error TS2749
+        this.handles = new Map();
+        this.sharedReferences = new Map();
+        this.sharedReferencesByValue = new Map();
+        this.workerApi = {
+            callMaster: (request, awaitResponse = true) => this.addCall(request, awaitResponse),
+            // @ts-expect-error TS2749
+            createReverseHandle: (fn) => this.createReverseHandle(fn),
+            // @ts-expect-error TS2749
+            runHandle: (handle, args) => this.workerApi.callMaster({ handle: handle.id, args }, true),
+            getSharedReference: (ref) => this.sharedReferences.get(ref),
+            resolveSharedReference: (value) => this.sharedReferencesByValue.get(value),
+        };
+        this.child = new ChildBackend((m) => {
+            this.messageListener(m);
+        }, () => this.handleEnd());
+        // Monitior all logging events inside this child process and forward to
+        // the main process via the bus.
+        this.loggerDisposable = logger_1.default.onLog((event) => {
+            bus_1.default.emit('logEvent', event);
+        });
+        // .. and do the same for trace events
+        this.tracerDisposable = profiler_1.tracer.onTrace((event) => {
+            bus_1.default.emit('traceEvent', event);
+        });
+    }
+    messageListener(message) {
+        if (message.type === 'response') {
+            return this.handleResponse(message);
+        }
+        else if (message.type === 'request') {
+            return this.handleRequest(message);
+        }
+    }
+    send(data) {
+        this.child.send(data);
+    }
+    async childInit(module, childId) {
+        this.module = require(module);
+        this.childId = childId;
+        if (this.module.childInit != null) {
+            await this.module.childInit();
+        }
+    }
+    async handleRequest(data) {
+        let { idx, method, args, handle: handleId } = data;
+        let child = (0, nullthrows_1.default)(data.child);
+        const responseFromContent = (content) => ({
+            idx,
+            child,
+            type: 'response',
+            contentType: 'data',
+            content,
+        });
+        const errorResponseFromError = (e) => ({
+            idx,
+            child,
+            type: 'response',
+            contentType: 'error',
+            content: (0, diagnostic_1.anyToDiagnostic)(e),
+        });
+        let result;
+        if (handleId != null) {
+            try {
+                let fn = (0, nullthrows_1.default)(this.handles.get(handleId)?.fn);
+                result = responseFromContent(fn(...args));
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        else if (method === 'childInit') {
+            try {
+                let [moduleName, childOptions] = args;
+                if (childOptions.shouldPatchConsole) {
+                    (0, logger_1.patchConsole)();
+                }
+                else {
+                    (0, logger_1.unpatchConsole)();
+                }
+                if (childOptions.shouldTrace) {
+                    profiler_1.tracer.enable();
+                }
+                result = responseFromContent(await this.childInit(moduleName, child));
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        else if (method === 'startProfile') {
+            this.profiler = new profiler_1.SamplingProfiler();
+            try {
+                result = responseFromContent(await this.profiler.startProfiling());
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        else if (method === 'endProfile') {
+            try {
+                let res = this.profiler ? await this.profiler.stopProfiling() : null;
+                result = responseFromContent(res);
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        else if (method === 'takeHeapSnapshot') {
+            try {
+                let v8 = require('v8');
+                result = responseFromContent(v8.writeHeapSnapshot('heap-' +
+                    args[0] +
+                    '-' +
+                    (this.childId ? 'worker' + this.childId : 'main') +
+                    '.heapsnapshot'));
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        else if (method === 'createSharedReference') {
+            let [ref, _value] = args;
+            let value = _value instanceof ArrayBuffer
+                ? // In the case the value is pre-serialized as a buffer,
+                    // deserialize it.
+                    (0, build_cache_1.deserialize)(Buffer.from(_value))
+                : _value;
+            this.sharedReferences.set(ref, value);
+            this.sharedReferencesByValue.set(value, ref);
+            result = responseFromContent(null);
+        }
+        else if (method === 'deleteSharedReference') {
+            let ref = args[0];
+            let value = this.sharedReferences.get(ref);
+            this.sharedReferencesByValue.delete(value);
+            this.sharedReferences.delete(ref);
+            result = responseFromContent(null);
+        }
+        else {
+            try {
+                result = responseFromContent(
+                // @ts-expect-error TS2538
+                await this.module[method](this.workerApi, ...args));
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        try {
+            this.send(result);
+        }
+        catch (e) {
+            result = this.send(errorResponseFromError(e));
+        }
+    }
+    handleResponse(data) {
+        let idx = (0, nullthrows_1.default)(data.idx);
+        let contentType = data.contentType;
+        let content = data.content;
+        let call = (0, nullthrows_1.default)(this.responseQueue.get(idx));
+        if (contentType === 'error') {
+            (0, assert_1.default)(typeof content !== 'string');
+            call.reject(new diagnostic_1.default({ diagnostic: content }));
+        }
+        else {
+            call.resolve(content);
+        }
+        this.responseQueue.delete(idx);
+        // Process the next call
+        this.processQueue();
+    }
+    // Keep in mind to make sure responses to these calls are JSON.Stringify safe
+    addCall(request, awaitResponse = true) {
+        let call = {
+            ...request,
+            type: 'request',
+            child: this.childId,
+            // @ts-expect-error TS2322
+            awaitResponse,
+            resolve: () => { },
+            reject: () => { },
+        };
+        let promise;
+        if (awaitResponse) {
+            promise = new Promise((resolve, reject) => {
+                call.resolve = resolve;
+                call.reject = reject;
+            });
+        }
+        this.callQueue.push(call);
+        this.processQueue();
+        return promise ?? Promise.resolve();
+    }
+    sendRequest(call) {
+        let idx;
+        if (call.awaitResponse) {
+            idx = this.responseId++;
+            this.responseQueue.set(idx, call);
+        }
+        this.send({
+            idx,
+            child: call.child,
+            type: call.type,
+            location: call.location,
+            handle: call.handle,
+            method: call.method,
+            args: call.args,
+            awaitResponse: call.awaitResponse,
+        });
+    }
+    processQueue() {
+        if (!this.callQueue.length) {
+            return;
+        }
+        if (this.responseQueue.size < this.maxConcurrentCalls) {
+            // @ts-expect-error TS2345
+            this.sendRequest(this.callQueue.shift());
+        }
+    }
+    handleEnd() {
+        this.loggerDisposable.dispose();
+        this.tracerDisposable.dispose();
+    }
+    // @ts-expect-error TS2749
+    createReverseHandle(fn) {
+        let handle = new Handle({
+            fn,
+            childId: this.childId,
+        });
+        this.handles.set(handle.id, handle);
+        return handle;
+    }
+}
+exports.Child = Child;

package/dist/childState.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.child = void 0;
+exports.setChild = setChild;
+// This file is imported by both the WorkerFarm and child implementation.
+// When a worker is inited, it sets the state in this file.
+// This way, WorkerFarm can access the state without directly importing the child code.
+exports.child = null;
+function setChild(c) {
+    exports.child = c;
+}

package/dist/cpuCount.js

@@ -0,0 +1,72 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectRealCores = detectRealCores;
+exports.default = getCores;
+const os_1 = __importDefault(require("os"));
+const child_process_1 = require("child_process");
+const exec = (command) => {
+    try {
+        let stdout = (0, child_process_1.execSync)(command, {
+            encoding: 'utf8',
+            // This prevents the command from outputting to the console
+            stdio: [null, null, null],
+        });
+        return stdout.trim();
+    }
+    catch (e) {
+        return '';
+    }
+};
+function detectRealCores() {
+    let platform = os_1.default.platform();
+    let amount = 0;
+    if (platform === 'linux') {
+        amount = parseInt(exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l'), 10);
+    }
+    else if (platform === 'darwin') {
+        amount = parseInt(exec('sysctl -n hw.physicalcpu_max'), 10);
+    }
+    else if (platform === 'win32') {
+        const str = exec('wmic cpu get NumberOfCores').match(/\d+/g);
+        if (str !== null) {
+            amount = parseInt(str.filter((n) => n !== '')[0], 10);
+        }
+    }
+    if (!amount || amount <= 0) {
+        throw new Error('Could not detect cpu count!');
+    }
+    return amount;
+}
+// @ts-expect-error TS7034
+let cores;
+function getCores(bypassCache = false) {
+    // Do not re-run commands if we already have the count...
+    // @ts-expect-error TS7005
+    if (cores && !bypassCache) {
+        return cores;
+    }
+    // @ts-expect-error TS2339
+    if (process.browser) {
+        cores = navigator.hardwareConcurrency / 2;
+    }
+    // @ts-expect-error TS7005
+    if (!cores) {
+        try {
+            cores = detectRealCores();
+        }
+        catch (e) {
+            // Guess the amount of real cores
+            cores = os_1.default
+                .cpus()
+                .filter((cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1).length;
+        }
+    }
+    // Another fallback
+    if (!cores) {
+        cores = 1;
+    }
+    return cores;
+}

package/dist/index.js

@@ -0,0 +1,44 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Handle = exports.bus = void 0;
+const assert_1 = __importDefault(require("assert"));
+const WorkerFarm_1 = __importDefault(require("./WorkerFarm"));
+const logger_1 = __importDefault(require("@atlaspack/logger"));
+const bus_1 = __importDefault(require("./bus"));
+exports.bus = bus_1.default;
+const profiler_1 = require("@atlaspack/profiler");
+if (!WorkerFarm_1.default.isWorker()) {
+    // Forward all logger events originating from workers into the main process
+    bus_1.default.on('logEvent', (e) => {
+        switch (e.level) {
+            case 'info':
+                logger_1.default.info(e.diagnostics);
+                break;
+            case 'progress':
+                (0, assert_1.default)(typeof e.message === 'string');
+                logger_1.default.progress(e.message);
+                break;
+            case 'verbose':
+                logger_1.default.verbose(e.diagnostics);
+                break;
+            case 'warn':
+                logger_1.default.warn(e.diagnostics);
+                break;
+            case 'error':
+                logger_1.default.error(e.diagnostics);
+                break;
+            default:
+                throw new Error('Unknown log level');
+        }
+    });
+    // Forward all trace events originating from workers into the main process
+    bus_1.default.on('traceEvent', (e) => {
+        profiler_1.tracer.trace(e);
+    });
+}
+exports.default = WorkerFarm_1.default;
+var WorkerFarm_2 = require("./WorkerFarm");
+Object.defineProperty(exports, "Handle", { enumerable: true, get: function () { return WorkerFarm_2.Handle; } });

package/dist/process/ProcessChild.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const build_cache_1 = require("@atlaspack/build-cache");
+const nullthrows_1 = __importDefault(require("nullthrows"));
+const child_1 = require("../child");
+const childState_1 = require("../childState");
+// @ts-expect-error TS2420
+class ProcessChild {
+    constructor(onMessage, onExit) {
+        if (!process.send) {
+            throw new Error('Only create ProcessChild instances in a worker!');
+        }
+        this.onMessage = onMessage;
+        this.onExit = onExit;
+        // @ts-expect-error TS2345
+        process.on('message', (data) => this.handleMessage(data));
+    }
+    handleMessage(data) {
+        if (data === 'die') {
+            return this.stop();
+        }
+        this.onMessage((0, build_cache_1.deserialize)(Buffer.from(data, 'base64')));
+    }
+    send(data) {
+        let processSend = (0, nullthrows_1.default)(process.send).bind(process);
+        // @ts-expect-error TS7006
+        processSend((0, build_cache_1.serialize)(data).toString('base64'), (err) => {
+            if (err && err instanceof Error) {
+                // @ts-expect-error TS2339
+                if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
+                    // IPC connection closed
+                    // no need to keep the worker running if it can't send or receive data
+                    return this.stop();
+                }
+            }
+        });
+    }
+    stop() {
+        this.onExit(0);
+        process.exit();
+    }
+}
+exports.default = ProcessChild;
+// @ts-expect-error TS2345
+(0, childState_1.setChild)(new child_1.Child(ProcessChild));

package/dist/process/ProcessWorker.js

@@ -0,0 +1,68 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WORKER_PATH = void 0;
+const child_process_1 = __importDefault(require("child_process"));
+const path_1 = __importDefault(require("path"));
+const build_cache_1 = require("@atlaspack/build-cache");
+exports.WORKER_PATH = path_1.default.join(__dirname, 'ProcessChild.js');
+if (process.env.ATLASPACK_REGISTER_USE_SRC === 'true') {
+    exports.WORKER_PATH = path_1.default.join(__dirname, 'ProcessChild.ts');
+}
+// @ts-expect-error TS2420
+class ProcessWorker {
+    constructor(execArgv, onMessage, onError, onExit) {
+        this.processQueue = true;
+        this.sendQueue = [];
+        this.execArgv = execArgv;
+        this.onMessage = onMessage;
+        this.onError = onError;
+        this.onExit = onExit;
+    }
+    start() {
+        this.child = child_process_1.default.fork(exports.WORKER_PATH, process.argv, {
+            execArgv: this.execArgv,
+            env: process.env,
+            cwd: process.cwd(),
+        });
+        this.child.on('message', (data) => {
+            this.onMessage((0, build_cache_1.deserialize)(Buffer.from(data, 'base64')));
+        });
+        this.child.once('exit', this.onExit);
+        this.child.on('error', this.onError);
+        return Promise.resolve();
+    }
+    async stop() {
+        this.child.send('die');
+        let forceKill = setTimeout(() => this.child.kill('SIGINT'), 500);
+        await new Promise((resolve) => {
+            this.child.once('exit', resolve);
+        });
+        clearTimeout(forceKill);
+    }
+    send(data) {
+        if (!this.processQueue) {
+            this.sendQueue.push(data);
+            return;
+        }
+        let result = this.child.send((0, build_cache_1.serialize)(data).toString('base64'), (error) => {
+            if (error && error instanceof Error) {
+                // Ignore this, the workerfarm handles child errors
+                return;
+            }
+            this.processQueue = true;
+            if (this.sendQueue.length > 0) {
+                let queueCopy = this.sendQueue.slice(0);
+                this.sendQueue = [];
+                queueCopy.forEach((entry) => this.send(entry));
+            }
+        });
+        if (!result || /^win/.test(process.platform)) {
+            // Queue is handling too much messages throttle it
+            this.processQueue = false;
+        }
+    }
+}
+exports.default = ProcessWorker;

package/dist/threads/ThreadsChild.js

@@ -0,0 +1,31 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_threads_1 = require("worker_threads");
+const build_cache_1 = require("@atlaspack/build-cache");
+const nullthrows_1 = __importDefault(require("nullthrows"));
+const child_1 = require("../child");
+const childState_1 = require("../childState");
+// @ts-expect-error TS2420
+class ThreadsChild {
+    constructor(onMessage, onExit) {
+        if (worker_threads_1.isMainThread || !worker_threads_1.parentPort) {
+            throw new Error('Only create ThreadsChild instances in a worker!');
+        }
+        this.onMessage = onMessage;
+        this.onExit = onExit;
+        worker_threads_1.parentPort.on('message', (data) => this.handleMessage(data));
+        worker_threads_1.parentPort.on('close', this.onExit);
+    }
+    handleMessage(data) {
+        this.onMessage((0, build_cache_1.restoreDeserializedObject)(data));
+    }
+    send(data) {
+        (0, nullthrows_1.default)(worker_threads_1.parentPort).postMessage((0, build_cache_1.prepareForSerialization)(data));
+    }
+}
+exports.default = ThreadsChild;
+// @ts-expect-error TS2345
+(0, childState_1.setChild)(new child_1.Child(ThreadsChild));

package/dist/threads/ThreadsWorker.js

@@ -0,0 +1,47 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WORKER_PATH = void 0;
+const worker_threads_1 = require("worker_threads");
+const path_1 = __importDefault(require("path"));
+const build_cache_1 = require("@atlaspack/build-cache");
+exports.WORKER_PATH = path_1.default.join(__dirname, 'ThreadsChild.js');
+if (process.env.ATLASPACK_REGISTER_USE_SRC === 'true') {
+    exports.WORKER_PATH = path_1.default.join(__dirname, 'ThreadsChild.ts');
+}
+// @ts-expect-error TS2420
+class ThreadsWorker {
+    constructor(execArgv, onMessage, onError, onExit) {
+        this.execArgv = execArgv;
+        this.onMessage = onMessage;
+        this.onError = onError;
+        this.onExit = onExit;
+    }
+    start() {
+        this.worker = new worker_threads_1.Worker(exports.WORKER_PATH, {
+            execArgv: this.execArgv,
+            env: process.env,
+        });
+        this.worker.on('message', (data) => this.handleMessage(data));
+        this.worker.on('error', this.onError);
+        this.worker.on('exit', this.onExit);
+        return new Promise((resolve) => {
+            this.worker.on('online', resolve);
+        });
+    }
+    stop() {
+        // In node 12, this returns a promise, but previously it accepted a callback
+        // TODO: Pass a callback in earlier versions of Node
+        // @ts-expect-error TS2322
+        return Promise.resolve(this.worker.terminate());
+    }
+    handleMessage(data) {
+        this.onMessage((0, build_cache_1.restoreDeserializedObject)(data));
+    }
+    send(data) {
+        this.worker.postMessage((0, build_cache_1.prepareForSerialization)(data));
+    }
+}
+exports.default = ThreadsWorker;

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/web/WebChild.js

@@ -0,0 +1,34 @@
+"use strict";
+/* eslint-env worker*/
+Object.defineProperty(exports, "__esModule", { value: true });
+const build_cache_1 = require("@atlaspack/build-cache");
+const child_1 = require("../child");
+const childState_1 = require("../childState");
+// @ts-expect-error TS2420
+class WebChild {
+    constructor(onMessage, onExit) {
+        if (!(typeof WorkerGlobalScope !== 'undefined' &&
+            self instanceof WorkerGlobalScope)) {
+            throw new Error('Only create WebChild instances in a worker!');
+        }
+        this.onMessage = onMessage;
+        this.onExit = onExit;
+        self.addEventListener('message', ({ data }) => {
+            if (data === 'stop') {
+                this.onExit(0);
+                self.postMessage('stopped');
+            }
+            this.handleMessage(data);
+        });
+        self.postMessage('online');
+    }
+    handleMessage(data) {
+        this.onMessage((0, build_cache_1.restoreDeserializedObject)(data));
+    }
+    send(data) {
+        self.postMessage((0, build_cache_1.prepareForSerialization)(data));
+    }
+}
+exports.default = WebChild;
+// @ts-expect-error TS2345
+(0, childState_1.setChild)(new child_1.Child(WebChild));