@atlaspack/workers 2.14.31 → 2.14.32

package/dist/WorkerFarm.js

@@ -0,0 +1,541 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Handle = void 0;
+const bus = __importStar(require("./bus"));
+const assert_1 = __importDefault(require("assert"));
+const nullthrows_1 = __importDefault(require("nullthrows"));
+const events_1 = __importDefault(require("events"));
+const build_cache_1 = require("@atlaspack/build-cache");
+const diagnostic_1 = __importStar(require("@atlaspack/diagnostic"));
+const Worker_1 = __importDefault(require("./Worker"));
+const cpuCount_1 = __importDefault(require("./cpuCount"));
+const Handle_1 = __importDefault(require("./Handle"));
+exports.Handle = Handle_1.default;
+const childState_1 = require("./childState");
+const backend_1 = require("./backend");
+const profiler_1 = require("@atlaspack/profiler");
+const fs_1 = __importDefault(require("fs"));
+const logger_1 = __importDefault(require("@atlaspack/logger"));
+let referenceId = 1;
+const DEFAULT_MAX_CONCURRENT_CALLS = 30;
+/**
+ * workerPath should always be defined inside farmOptions
+ */
+class WorkerFarm extends events_1.default {
+    constructor(farmOptions = {}) {
+        super();
+        this.callQueue = [];
+        this.ending = false;
+        this.warmWorkers = 0;
+        this.readyWorkers = 0;
+        this.workers = new Map();
+        this.handles = new Map();
+        this.sharedReferences = new Map();
+        this.sharedReferencesByValue = new Map();
+        this.serializedSharedReferences = new Map();
+        this.workerApi = {
+            callMaster: async (request, awaitResponse = true) => {
+                let result = await this.processRequest({
+                    ...request,
+                    // @ts-expect-error TS2322
+                    awaitResponse,
+                });
+                return (0, build_cache_1.deserialize)((0, build_cache_1.serialize)(result));
+            },
+            createReverseHandle: (fn) => this.createReverseHandle(fn),
+            callChild: (childId, request) => new Promise((resolve, reject) => {
+                (0, nullthrows_1.default)(this.workers.get(childId)).call({
+                    ...request,
+                    resolve,
+                    reject,
+                    retries: 0,
+                });
+            }),
+            runHandle: (handle, args) => this.workerApi.callChild((0, nullthrows_1.default)(handle.childId), {
+                handle: handle.id,
+                args,
+            }),
+            getSharedReference: (ref) => this.sharedReferences.get(ref),
+            resolveSharedReference: (value) => this.sharedReferencesByValue.get(value),
+        };
+        this.options = {
+            maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
+            maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(farmOptions.shouldTrace ? 1 : DEFAULT_MAX_CONCURRENT_CALLS),
+            forcedKillTime: 500,
+            warmWorkers: false,
+            useLocalWorker: true, // TODO: setting this to false makes some tests fail, figure out why
+            backend: (0, backend_1.detectBackend)(),
+            ...farmOptions,
+        };
+        if (!this.options.workerPath) {
+            throw new Error('Please provide a worker path!');
+        }
+        this.localWorker = require(this.options.workerPath);
+        // @ts-expect-error TS2322
+        this.localWorkerInit =
+            this.localWorker.childInit != null ? this.localWorker.childInit() : null;
+        this.run = this.createHandle('run');
+        // Worker thread stdout is by default piped into the process stdout, if there are enough worker
+        // threads to exceed the default listener limit, then anything else piping into stdout will trigger
+        // the `MaxListenersExceededWarning`, so we should ensure the max listeners is at least equal to the
+        // number of workers + 1 for the main thread.
+        //
+        // Note this can't be fixed easily where other things pipe into stdout -  even after starting > 10 worker
+        // threads `process.stdout.getMaxListeners()` will still return 10, however adding another pipe into `stdout`
+        // will give the warning with `<worker count + 1>` as the number of listeners.
+        process.stdout?.setMaxListeners(Math.max(process.stdout.getMaxListeners(), WorkerFarm.getNumWorkers() + 1));
+        this.startMaxWorkers();
+    }
+    warmupWorker(method, args) {
+        // Workers are already stopping
+        if (this.ending) {
+            return;
+        }
+        // Workers are not warmed up yet.
+        // Send the job to a remote worker in the background,
+        // but use the result from the local worker - it will be faster.
+        let promise = this.addCall(method, [...args, true]);
+        if (promise) {
+            promise
+                .then(() => {
+                this.warmWorkers++;
+                if (this.warmWorkers >= this.workers.size) {
+                    this.emit('warmedup');
+                }
+            })
+                .catch(() => { });
+        }
+    }
+    shouldStartRemoteWorkers() {
+        return (this.options.maxConcurrentWorkers > 0 || !this.options.useLocalWorker);
+    }
+    createHandle(method, useMainThread = false) {
+        if (!this.options.useLocalWorker) {
+            useMainThread = false;
+        }
+        return async (...args) => {
+            // Child process workers are slow to start (~600ms).
+            // While we're waiting, just run on the main thread.
+            // This significantly speeds up startup time.
+            if (this.shouldUseRemoteWorkers() && !useMainThread) {
+                return this.addCall(method, [...args, false]);
+            }
+            else {
+                if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
+                    this.warmupWorker(method, args);
+                }
+                let processedArgs;
+                if (!useMainThread) {
+                    processedArgs = (0, build_cache_1.restoreDeserializedObject)((0, build_cache_1.prepareForSerialization)([...args, false]));
+                }
+                else {
+                    processedArgs = args;
+                }
+                if (this.localWorkerInit != null) {
+                    await this.localWorkerInit;
+                    this.localWorkerInit = null;
+                }
+                return this.localWorker[method](this.workerApi, ...processedArgs);
+            }
+        };
+    }
+    onError(error, worker) {
+        // Handle ipc errors
+        if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
+            // @ts-expect-error TS2322
+            return this.stopWorker(worker);
+        }
+        else {
+            logger_1.default.error(error, '@atlaspack/workers');
+        }
+    }
+    startChild() {
+        let worker = new Worker_1.default({
+            forcedKillTime: this.options.forcedKillTime,
+            backend: this.options.backend,
+            shouldPatchConsole: this.options.shouldPatchConsole,
+            shouldTrace: this.options.shouldTrace,
+            sharedReferences: this.sharedReferences,
+        });
+        worker.fork((0, nullthrows_1.default)(this.options.workerPath));
+        worker.on('request', (data) => this.processRequest(data, worker));
+        worker.on('ready', () => {
+            this.readyWorkers++;
+            if (this.readyWorkers === this.options.maxConcurrentWorkers) {
+                this.emit('ready');
+            }
+            this.processQueue();
+        });
+        worker.on('response', () => this.processQueue());
+        worker.on('error', (err) => this.onError(err, worker));
+        worker.once('exit', () => this.stopWorker(worker));
+        this.workers.set(worker.id, worker);
+    }
+    async stopWorker(worker) {
+        if (!worker.stopped) {
+            this.workers.delete(worker.id);
+            worker.isStopping = true;
+            if (worker.calls.size) {
+                for (let call of worker.calls.values()) {
+                    call.retries++;
+                    this.callQueue.unshift(call);
+                }
+            }
+            worker.calls.clear();
+            await worker.stop();
+            // Process any requests that failed and start a new worker
+            this.processQueue();
+        }
+    }
+    processQueue() {
+        if (this.ending || !this.callQueue.length)
+            return;
+        if (this.workers.size < this.options.maxConcurrentWorkers) {
+            this.startChild();
+        }
+        let workers = [...this.workers.values()].sort((a, b) => a.calls.size - b.calls.size);
+        for (let worker of workers) {
+            if (!this.callQueue.length) {
+                break;
+            }
+            if (!worker.ready || worker.stopped || worker.isStopping) {
+                continue;
+            }
+            if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
+                // @ts-expect-error TS2345
+                this.callWorker(worker, this.callQueue.shift());
+            }
+        }
+    }
+    async callWorker(worker, call) {
+        for (let ref of this.sharedReferences.keys()) {
+            if (!worker.sentSharedReferences.has(ref)) {
+                await worker.sendSharedReference(ref, this.getSerializedSharedReference(ref));
+            }
+        }
+        worker.call(call);
+    }
+    async processRequest(data, worker) {
+        let { method, args, location, awaitResponse, idx, handle: handleId } = data;
+        let mod;
+        if (handleId != null) {
+            mod = (0, nullthrows_1.default)(this.handles.get(handleId)?.fn);
+        }
+        else if (location) {
+            // @ts-expect-error TS2339
+            if (process.browser) {
+                if (location === '@atlaspack/workers/bus') {
+                    mod = bus;
+                }
+                else {
+                    throw new Error('No dynamic require possible: ' + location);
+                }
+            }
+            else {
+                mod = require(location);
+            }
+        }
+        else {
+            throw new Error('Unknown request');
+        }
+        const responseFromContent = (content) => ({
+            idx,
+            type: 'response',
+            contentType: 'data',
+            content,
+        });
+        const errorResponseFromError = (e) => ({
+            idx,
+            type: 'response',
+            contentType: 'error',
+            content: (0, diagnostic_1.anyToDiagnostic)(e),
+        });
+        let result;
+        if (method == null) {
+            try {
+                // @ts-expect-error TS2488
+                result = responseFromContent(await mod(...args));
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        else {
+            // ESModule default interop
+            if (mod.__esModule && !mod[method] && mod.default) {
+                mod = mod.default;
+            }
+            try {
+                result = responseFromContent(await mod[method](...args));
+            }
+            catch (e) {
+                result = errorResponseFromError(e);
+            }
+        }
+        if (awaitResponse) {
+            if (worker) {
+                worker.send(result);
+            }
+            else {
+                if (result.contentType === 'error') {
+                    throw new diagnostic_1.default({ diagnostic: result.content });
+                }
+                return result.content;
+            }
+        }
+    }
+    addCall(method, args) {
+        if (this.ending) {
+            throw new Error('Cannot add a worker call if workerfarm is ending.');
+        }
+        return new Promise((resolve, reject) => {
+            this.callQueue.push({
+                method,
+                args: args,
+                retries: 0,
+                resolve,
+                reject,
+            });
+            this.processQueue();
+        });
+    }
+    async end() {
+        this.ending = true;
+        await Promise.all(Array.from(this.workers.values()).map((worker) => this.stopWorker(worker)));
+        for (let handle of this.handles.values()) {
+            handle.dispose();
+        }
+        this.handles = new Map();
+        this.sharedReferences = new Map();
+        this.sharedReferencesByValue = new Map();
+        this.ending = false;
+    }
+    startMaxWorkers() {
+        // Starts workers until the maximum is reached
+        if (this.workers.size < this.options.maxConcurrentWorkers) {
+            let toStart = this.options.maxConcurrentWorkers - this.workers.size;
+            while (toStart--) {
+                this.startChild();
+            }
+        }
+    }
+    shouldUseRemoteWorkers() {
+        return (!this.options.useLocalWorker ||
+            ((this.warmWorkers >= this.workers.size || !this.options.warmWorkers) &&
+                this.options.maxConcurrentWorkers > 0));
+    }
+    createReverseHandle(fn) {
+        let handle = new Handle_1.default({ fn });
+        this.handles.set(handle.id, handle);
+        return handle;
+    }
+    createSharedReference(value, isCacheable = true) {
+        let ref = referenceId++;
+        this.sharedReferences.set(ref, value);
+        this.sharedReferencesByValue.set(value, ref);
+        if (!isCacheable) {
+            this.serializedSharedReferences.set(ref, null);
+        }
+        return {
+            ref,
+            dispose: () => {
+                this.sharedReferences.delete(ref);
+                this.sharedReferencesByValue.delete(value);
+                this.serializedSharedReferences.delete(ref);
+                let promises = [];
+                for (let worker of this.workers.values()) {
+                    if (!worker.sentSharedReferences.has(ref)) {
+                        continue;
+                    }
+                    worker.sentSharedReferences.delete(ref);
+                    promises.push(new Promise((resolve, reject) => {
+                        worker.call({
+                            method: 'deleteSharedReference',
+                            args: [ref],
+                            resolve,
+                            reject,
+                            skipReadyCheck: true,
+                            retries: 0,
+                        });
+                    }));
+                }
+                return Promise.all(promises);
+            },
+        };
+    }
+    getSerializedSharedReference(ref) {
+        let cached = this.serializedSharedReferences.get(ref);
+        if (cached) {
+            return cached;
+        }
+        let value = this.sharedReferences.get(ref);
+        let buf = (0, build_cache_1.serialize)(value).buffer;
+        // If the reference was created with the isCacheable option set to false,
+        // serializedSharedReferences will contain `null` as the value.
+        if (cached !== null) {
+            // @ts-expect-error TS2345
+            this.serializedSharedReferences.set(ref, buf);
+        }
+        // @ts-expect-error TS2322
+        return buf;
+    }
+    async startProfile() {
+        let promises = [];
+        for (let worker of this.workers.values()) {
+            promises.push(new Promise((resolve, reject) => {
+                worker.call({
+                    method: 'startProfile',
+                    args: [],
+                    resolve,
+                    reject,
+                    retries: 0,
+                    skipReadyCheck: true,
+                });
+            }));
+        }
+        this.profiler = new profiler_1.SamplingProfiler();
+        promises.push(this.profiler.startProfiling());
+        await Promise.all(promises);
+    }
+    async endProfile() {
+        if (!this.profiler) {
+            return;
+        }
+        let promises = [this.profiler.stopProfiling()];
+        let names = ['Master'];
+        for (let worker of this.workers.values()) {
+            names.push('Worker ' + worker.id);
+            promises.push(new Promise((resolve, reject) => {
+                worker.call({
+                    method: 'endProfile',
+                    args: [],
+                    resolve,
+                    reject,
+                    retries: 0,
+                    skipReadyCheck: true,
+                });
+            }));
+        }
+        var profiles = await Promise.all(promises);
+        let trace = new profiler_1.Trace();
+        let filename = `profile-${getTimeId()}.trace`;
+        let stream = trace.pipe(fs_1.default.createWriteStream(filename));
+        for (let profile of profiles) {
+            // @ts-expect-error TS2345
+            trace.addCPUProfile(names.shift(), profile);
+        }
+        trace.flush();
+        await new Promise((resolve) => {
+            stream.once('finish', resolve);
+        });
+        logger_1.default.info({
+            origin: '@atlaspack/workers',
+            message: (0, diagnostic_1.md) `Wrote profile to ${filename}`,
+        });
+    }
+    async callAllWorkers(method, args) {
+        let promises = [];
+        for (let worker of this.workers.values()) {
+            promises.push(new Promise((resolve, reject) => {
+                worker.call({
+                    method,
+                    args,
+                    resolve,
+                    reject,
+                    retries: 0,
+                });
+            }));
+        }
+        promises.push(this.localWorker[method](this.workerApi, ...args));
+        await Promise.all(promises);
+    }
+    async takeHeapSnapshot() {
+        let snapshotId = getTimeId();
+        try {
+            let snapshotPaths = await Promise.all([...this.workers.values()].map((worker) => new Promise((resolve, reject) => {
+                worker.call({
+                    method: 'takeHeapSnapshot',
+                    args: [snapshotId],
+                    resolve,
+                    reject,
+                    retries: 0,
+                    skipReadyCheck: true,
+                });
+            })));
+            logger_1.default.info({
+                origin: '@atlaspack/workers',
+                message: (0, diagnostic_1.md) `Wrote heap snapshots to the following paths:\n${snapshotPaths.join('\n')}`,
+            });
+        }
+        catch {
+            logger_1.default.error({
+                origin: '@atlaspack/workers',
+                message: 'Unable to take heap snapshots. Note: requires Node 11.13.0+',
+            });
+        }
+    }
+    static getNumWorkers() {
+        return process.env.ATLASPACK_WORKERS
+            ? parseInt(process.env.ATLASPACK_WORKERS, 10)
+            : Math.min(4, Math.ceil((0, cpuCount_1.default)() / 2));
+    }
+    static isWorker() {
+        return !!childState_1.child;
+    }
+    static getWorkerApi() {
+        (0, assert_1.default)(childState_1.child != null, 'WorkerFarm.getWorkerApi can only be called within workers');
+        return childState_1.child.workerApi;
+    }
+    static getConcurrentCallsPerWorker(defaultValue = DEFAULT_MAX_CONCURRENT_CALLS) {
+        return (
+        // @ts-expect-error TS2345
+        parseInt(process.env.ATLASPACK_MAX_CONCURRENT_CALLS, 10) || defaultValue);
+    }
+}
+exports.default = WorkerFarm;
+function getTimeId() {
+    let now = new Date();
+    return (String(now.getFullYear()) +
+        String(now.getMonth() + 1).padStart(2, '0') +
+        String(now.getDate()).padStart(2, '0') +
+        '-' +
+        String(now.getHours()).padStart(2, '0') +
+        String(now.getMinutes()).padStart(2, '0') +
+        String(now.getSeconds()).padStart(2, '0'));
+}

package/dist/backend.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectBackend = detectBackend;
+exports.getWorkerBackend = getWorkerBackend;
+// /flow-to-ts helpers
+function detectBackend() {
+    // @ts-expect-error TS2339
+    if (process.browser)
+        return 'web';
+    switch (process.env.ATLASPACK_WORKER_BACKEND) {
+        case 'threads':
+        case 'process':
+            return process.env.ATLASPACK_WORKER_BACKEND;
+    }
+    try {
+        require('worker_threads');
+        return 'threads';
+    }
+    catch (err) {
+        return 'process';
+    }
+}
+function getWorkerBackend(backend) {
+    switch (backend) {
+        case 'threads':
+            return require('./threads/ThreadsWorker').default;
+        case 'process':
+            return require('./process/ProcessWorker').default;
+        case 'web':
+            return require('./web/WebWorker').default;
+        default:
+            throw new Error(`Invalid backend: ${backend}`);
+    }
+}

package/dist/bus.js

@@ -0,0 +1,24 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const events_1 = __importDefault(require("events"));
+const childState_1 = require("./childState");
+class Bus extends events_1.default {
+    emit(event, ...args) {
+        if (childState_1.child) {
+            childState_1.child.workerApi.callMaster({
+                // @ts-expect-error TS2339
+                location: process.browser ? '@atlaspack/workers/bus' : __filename,
+                method: 'emit',
+                args: [event, ...args],
+            }, false);
+            return true;
+        }
+        else {
+            return super.emit(event, ...args);
+        }
+    }
+}
+exports.default = new Bus();