@atlaspack/workers 2.14.0 → 2.14.1-canary.6

package/lib/child.js

@@ -0,0 +1,284 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.Child = void 0;
+function _assert() {
+  const data = _interopRequireDefault(require("assert"));
+  _assert = function () {
+    return data;
+  };
+  return data;
+}
+function _nullthrows() {
+  const data = _interopRequireDefault(require("nullthrows"));
+  _nullthrows = function () {
+    return data;
+  };
+  return data;
+}
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
+function _logger() {
+  const data = _interopRequireWildcard(require("@atlaspack/logger"));
+  _logger = function () {
+    return data;
+  };
+  return data;
+}
+function _diagnostic() {
+  const data = _interopRequireWildcard(require("@atlaspack/diagnostic"));
+  _diagnostic = function () {
+    return data;
+  };
+  return data;
+}
+var _bus = _interopRequireDefault(require("./bus"));
+function _profiler() {
+  const data = require("@atlaspack/profiler");
+  _profiler = function () {
+    return data;
+  };
+  return data;
+}
+var _Handle2 = _interopRequireDefault(require("./Handle"));
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+// The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
+const Handle = _Handle2.default;
+class Child {
+  callQueue = [];
+  maxConcurrentCalls = 10;
+  responseId = 0;
+  responseQueue = new Map();
+  handles = new Map();
+  sharedReferences = new Map();
+  sharedReferencesByValue = new Map();
+  constructor(ChildBackend) {
+    this.child = new ChildBackend(m => {
+      this.messageListener(m);
+    }, () => this.handleEnd());
+
+    // Monitior all logging events inside this child process and forward to
+    // the main process via the bus.
+    this.loggerDisposable = _logger().default.onLog(event => {
+      _bus.default.emit('logEvent', event);
+    });
+    // .. and do the same for trace events
+    this.tracerDisposable = _profiler().tracer.onTrace(event => {
+      _bus.default.emit('traceEvent', event);
+    });
+  }
+  workerApi = {
+    callMaster: (request, awaitResponse = true) => this.addCall(request, awaitResponse),
+    createReverseHandle: fn => this.createReverseHandle(fn),
+    runHandle: (handle, args) => this.workerApi.callMaster({
+      handle: handle.id,
+      args
+    }, true),
+    getSharedReference: ref => this.sharedReferences.get(ref),
+    resolveSharedReference: value => this.sharedReferencesByValue.get(value)
+  };
+  messageListener(message) {
+    if (message.type === 'response') {
+      return this.handleResponse(message);
+    } else if (message.type === 'request') {
+      return this.handleRequest(message);
+    }
+  }
+  send(data) {
+    this.child.send(data);
+  }
+  async childInit(module, childId) {
+    // $FlowFixMe
+    this.module = require(module);
+    this.childId = childId;
+    if (this.module.childInit != null) {
+      await this.module.childInit();
+    }
+  }
+  async handleRequest(data) {
+    let {
+      idx,
+      method,
+      args,
+      handle: handleId
+    } = data;
+    let child = (0, _nullthrows().default)(data.child);
+    const responseFromContent = content => ({
+      idx,
+      child,
+      type: 'response',
+      contentType: 'data',
+      content
+    });
+    const errorResponseFromError = e => ({
+      idx,
+      child,
+      type: 'response',
+      contentType: 'error',
+      content: (0, _diagnostic().anyToDiagnostic)(e)
+    });
+    let result;
+    if (handleId != null) {
+      try {
+        var _this$handles$get;
+        let fn = (0, _nullthrows().default)((_this$handles$get = this.handles.get(handleId)) === null || _this$handles$get === void 0 ? void 0 : _this$handles$get.fn);
+        result = responseFromContent(fn(...args));
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'childInit') {
+      try {
+        let [moduleName, childOptions] = args;
+        if (childOptions.shouldPatchConsole) {
+          (0, _logger().patchConsole)();
+        } else {
+          (0, _logger().unpatchConsole)();
+        }
+        if (childOptions.shouldTrace) {
+          _profiler().tracer.enable();
+        }
+        result = responseFromContent(await this.childInit(moduleName, child));
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'startProfile') {
+      this.profiler = new (_profiler().SamplingProfiler)();
+      try {
+        result = responseFromContent(await this.profiler.startProfiling());
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'endProfile') {
+      try {
+        let res = this.profiler ? await this.profiler.stopProfiling() : null;
+        result = responseFromContent(res);
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'takeHeapSnapshot') {
+      try {
+        let v8 = require('v8');
+        result = responseFromContent(v8.writeHeapSnapshot('heap-' + args[0] + '-' + (this.childId ? 'worker' + this.childId : 'main') + '.heapsnapshot'));
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'createSharedReference') {
+      let [ref, _value] = args;
+      let value = _value instanceof ArrayBuffer ?
+      // In the case the value is pre-serialized as a buffer,
+      // deserialize it.
+      (0, _buildCache().deserialize)(Buffer.from(_value)) : _value;
+      this.sharedReferences.set(ref, value);
+      this.sharedReferencesByValue.set(value, ref);
+      result = responseFromContent(null);
+    } else if (method === 'deleteSharedReference') {
+      let ref = args[0];
+      let value = this.sharedReferences.get(ref);
+      this.sharedReferencesByValue.delete(value);
+      this.sharedReferences.delete(ref);
+      result = responseFromContent(null);
+    } else {
+      try {
+        result = responseFromContent(
+        // $FlowFixMe
+        await this.module[method](this.workerApi, ...args));
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    }
+    try {
+      this.send(result);
+    } catch (e) {
+      result = this.send(errorResponseFromError(e));
+    }
+  }
+  handleResponse(data) {
+    let idx = (0, _nullthrows().default)(data.idx);
+    let contentType = data.contentType;
+    let content = data.content;
+    let call = (0, _nullthrows().default)(this.responseQueue.get(idx));
+    if (contentType === 'error') {
+      (0, _assert().default)(typeof content !== 'string');
+      call.reject(new (_diagnostic().default)({
+        diagnostic: content
+      }));
+    } else {
+      call.resolve(content);
+    }
+    this.responseQueue.delete(idx);
+
+    // Process the next call
+    this.processQueue();
+  }
+
+  // Keep in mind to make sure responses to these calls are JSON.Stringify safe
+  addCall(request, awaitResponse = true) {
+    // $FlowFixMe
+    let call = {
+      ...request,
+      type: 'request',
+      child: this.childId,
+      // $FlowFixMe Added in Flow 0.121.0 upgrade in #4381
+      awaitResponse,
+      resolve: () => {},
+      reject: () => {}
+    };
+    let promise;
+    if (awaitResponse) {
+      promise = new Promise((resolve, reject) => {
+        call.resolve = resolve;
+        call.reject = reject;
+      });
+    }
+    this.callQueue.push(call);
+    this.processQueue();
+    return promise ?? Promise.resolve();
+  }
+  sendRequest(call) {
+    let idx;
+    if (call.awaitResponse) {
+      idx = this.responseId++;
+      this.responseQueue.set(idx, call);
+    }
+    this.send({
+      idx,
+      child: call.child,
+      type: call.type,
+      location: call.location,
+      handle: call.handle,
+      method: call.method,
+      args: call.args,
+      awaitResponse: call.awaitResponse
+    });
+  }
+  processQueue() {
+    if (!this.callQueue.length) {
+      return;
+    }
+    if (this.responseQueue.size < this.maxConcurrentCalls) {
+      this.sendRequest(this.callQueue.shift());
+    }
+  }
+  handleEnd() {
+    this.loggerDisposable.dispose();
+    this.tracerDisposable.dispose();
+  }
+  createReverseHandle(fn) {
+    let handle = new Handle({
+      fn,
+      childId: this.childId
+    });
+    this.handles.set(handle.id, handle);
+    return handle;
+  }
+}
+exports.Child = Child;

package/lib/childState.js

@@ -0,0 +1,14 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.child = void 0;
+exports.setChild = setChild;
+// This file is imported by both the WorkerFarm and child implementation.
+// When a worker is inited, it sets the state in this file.
+// This way, WorkerFarm can access the state without directly importing the child code.
+let child = exports.child = null;
+function setChild(c) {
+  exports.child = child = c;
+}

package/lib/cpuCount.js

@@ -0,0 +1,79 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = getCores;
+exports.detectRealCores = detectRealCores;
+function _os() {
+  const data = _interopRequireDefault(require("os"));
+  _os = function () {
+    return data;
+  };
+  return data;
+}
+function _child_process() {
+  const data = require("child_process");
+  _child_process = function () {
+    return data;
+  };
+  return data;
+}
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+const exec = command => {
+  try {
+    let stdout = (0, _child_process().execSync)(command, {
+      encoding: 'utf8',
+      // This prevents the command from outputting to the console
+      stdio: [null, null, null]
+    });
+    return stdout.trim();
+  } catch (e) {
+    return '';
+  }
+};
+function detectRealCores() {
+  let platform = _os().default.platform();
+  let amount = 0;
+  if (platform === 'linux') {
+    amount = parseInt(exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l'), 10);
+  } else if (platform === 'darwin') {
+    amount = parseInt(exec('sysctl -n hw.physicalcpu_max'), 10);
+  } else if (platform === 'win32') {
+    const str = exec('wmic cpu get NumberOfCores').match(/\d+/g);
+    if (str !== null) {
+      amount = parseInt(str.filter(n => n !== '')[0], 10);
+    }
+  }
+  if (!amount || amount <= 0) {
+    throw new Error('Could not detect cpu count!');
+  }
+  return amount;
+}
+let cores;
+function getCores(bypassCache = false) {
+  // Do not re-run commands if we already have the count...
+  if (cores && !bypassCache) {
+    return cores;
+  }
+
+  // $FlowFixMe
+  if (process.browser) {
+    // eslint-disable-next-line no-undef
+    cores = navigator.hardwareConcurrency / 2;
+  }
+  if (!cores) {
+    try {
+      cores = detectRealCores();
+    } catch (e) {
+      // Guess the amount of real cores
+      cores = _os().default.cpus().filter((cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1).length;
+    }
+  }
+
+  // Another fallback
+  if (!cores) {
+    cores = 1;
+  }
+  return cores;
+}

package/lib/index.js

@@ -0,0 +1,75 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "Handle", {
+  enumerable: true,
+  get: function () {
+    return _WorkerFarm.Handle;
+  }
+});
+Object.defineProperty(exports, "bus", {
+  enumerable: true,
+  get: function () {
+    return _bus.default;
+  }
+});
+exports.default = void 0;
+function _assert() {
+  const data = _interopRequireDefault(require("assert"));
+  _assert = function () {
+    return data;
+  };
+  return data;
+}
+var _WorkerFarm = _interopRequireWildcard(require("./WorkerFarm"));
+function _logger() {
+  const data = _interopRequireDefault(require("@atlaspack/logger"));
+  _logger = function () {
+    return data;
+  };
+  return data;
+}
+var _bus = _interopRequireDefault(require("./bus"));
+function _profiler() {
+  const data = require("@atlaspack/profiler");
+  _profiler = function () {
+    return data;
+  };
+  return data;
+}
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+if (!_WorkerFarm.default.isWorker()) {
+  // Forward all logger events originating from workers into the main process
+  _bus.default.on('logEvent', e => {
+    switch (e.level) {
+      case 'info':
+        _logger().default.info(e.diagnostics);
+        break;
+      case 'progress':
+        (0, _assert().default)(typeof e.message === 'string');
+        _logger().default.progress(e.message);
+        break;
+      case 'verbose':
+        _logger().default.verbose(e.diagnostics);
+        break;
+      case 'warn':
+        _logger().default.warn(e.diagnostics);
+        break;
+      case 'error':
+        _logger().default.error(e.diagnostics);
+        break;
+      default:
+        throw new Error('Unknown log level');
+    }
+  });
+
+  // Forward all trace events originating from workers into the main process
+  _bus.default.on('traceEvent', e => {
+    _profiler().tracer.trace(e);
+  });
+}
+var _default = exports.default = _WorkerFarm.default;

package/lib/process/ProcessChild.js

@@ -0,0 +1,58 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
+function _nullthrows() {
+  const data = _interopRequireDefault(require("nullthrows"));
+  _nullthrows = function () {
+    return data;
+  };
+  return data;
+}
+var _child = require("../child");
+var _childState = require("../childState");
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+class ProcessChild {
+  constructor(onMessage, onExit) {
+    if (!process.send) {
+      throw new Error('Only create ProcessChild instances in a worker!');
+    }
+    this.onMessage = onMessage;
+    this.onExit = onExit;
+    process.on('message', data => this.handleMessage(data));
+  }
+  handleMessage(data) {
+    if (data === 'die') {
+      return this.stop();
+    }
+    this.onMessage((0, _buildCache().deserialize)(Buffer.from(data, 'base64')));
+  }
+  send(data) {
+    let processSend = (0, _nullthrows().default)(process.send).bind(process);
+    processSend((0, _buildCache().serialize)(data).toString('base64'), err => {
+      if (err && err instanceof Error) {
+        // $FlowFixMe[prop-missing]
+        if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
+          // IPC connection closed
+          // no need to keep the worker running if it can't send or receive data
+          return this.stop();
+        }
+      }
+    });
+  }
+  stop() {
+    this.onExit(0);
+    process.exit();
+  }
+}
+exports.default = ProcessChild;
+(0, _childState.setChild)(new _child.Child(ProcessChild));

package/lib/process/ProcessWorker.js

@@ -0,0 +1,83 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+function _child_process() {
+  const data = _interopRequireDefault(require("child_process"));
+  _child_process = function () {
+    return data;
+  };
+  return data;
+}
+function _path() {
+  const data = _interopRequireDefault(require("path"));
+  _path = function () {
+    return data;
+  };
+  return data;
+}
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+const WORKER_PATH = _path().default.join(__dirname, './ProcessChild.js');
+class ProcessWorker {
+  processQueue = true;
+  sendQueue = [];
+  constructor(execArgv, onMessage, onError, onExit) {
+    this.execArgv = execArgv;
+    this.onMessage = onMessage;
+    this.onError = onError;
+    this.onExit = onExit;
+  }
+  start() {
+    this.child = _child_process().default.fork(WORKER_PATH, process.argv, {
+      execArgv: this.execArgv,
+      env: process.env,
+      cwd: process.cwd()
+    });
+    this.child.on('message', data => {
+      this.onMessage((0, _buildCache().deserialize)(Buffer.from(data, 'base64')));
+    });
+    this.child.once('exit', this.onExit);
+    this.child.on('error', this.onError);
+    return Promise.resolve();
+  }
+  async stop() {
+    this.child.send('die');
+    let forceKill = setTimeout(() => this.child.kill('SIGINT'), 500);
+    await new Promise(resolve => {
+      this.child.once('exit', resolve);
+    });
+    clearTimeout(forceKill);
+  }
+  send(data) {
+    if (!this.processQueue) {
+      this.sendQueue.push(data);
+      return;
+    }
+    let result = this.child.send((0, _buildCache().serialize)(data).toString('base64'), error => {
+      if (error && error instanceof Error) {
+        // Ignore this, the workerfarm handles child errors
+        return;
+      }
+      this.processQueue = true;
+      if (this.sendQueue.length > 0) {
+        let queueCopy = this.sendQueue.slice(0);
+        this.sendQueue = [];
+        queueCopy.forEach(entry => this.send(entry));
+      }
+    });
+    if (!result || /^win/.test(process.platform)) {
+      // Queue is handling too much messages throttle it
+      this.processQueue = false;
+    }
+  }
+}
+exports.default = ProcessWorker;

package/lib/threads/ThreadsChild.js

@@ -0,0 +1,49 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+function _worker_threads() {
+  const data = require("worker_threads");
+  _worker_threads = function () {
+    return data;
+  };
+  return data;
+}
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
+function _nullthrows() {
+  const data = _interopRequireDefault(require("nullthrows"));
+  _nullthrows = function () {
+    return data;
+  };
+  return data;
+}
+var _child = require("../child");
+var _childState = require("../childState");
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+class ThreadsChild {
+  constructor(onMessage, onExit) {
+    if (_worker_threads().isMainThread || !_worker_threads().parentPort) {
+      throw new Error('Only create ThreadsChild instances in a worker!');
+    }
+    this.onMessage = onMessage;
+    this.onExit = onExit;
+    _worker_threads().parentPort.on('message', data => this.handleMessage(data));
+    _worker_threads().parentPort.on('close', this.onExit);
+  }
+  handleMessage(data) {
+    this.onMessage((0, _buildCache().restoreDeserializedObject)(data));
+  }
+  send(data) {
+    (0, _nullthrows().default)(_worker_threads().parentPort).postMessage((0, _buildCache().prepareForSerialization)(data));
+  }
+}
+exports.default = ThreadsChild;
+(0, _childState.setChild)(new _child.Child(ThreadsChild));

package/lib/threads/ThreadsWorker.js

@@ -0,0 +1,61 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+function _worker_threads() {
+  const data = require("worker_threads");
+  _worker_threads = function () {
+    return data;
+  };
+  return data;
+}
+function _path() {
+  const data = _interopRequireDefault(require("path"));
+  _path = function () {
+    return data;
+  };
+  return data;
+}
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+const WORKER_PATH = _path().default.join(__dirname, './ThreadsChild.js');
+class ThreadsWorker {
+  constructor(execArgv, onMessage, onError, onExit) {
+    this.execArgv = execArgv;
+    this.onMessage = onMessage;
+    this.onError = onError;
+    this.onExit = onExit;
+  }
+  start() {
+    this.worker = new (_worker_threads().Worker)(WORKER_PATH, {
+      execArgv: this.execArgv,
+      env: process.env
+    });
+    this.worker.on('message', data => this.handleMessage(data));
+    this.worker.on('error', this.onError);
+    this.worker.on('exit', this.onExit);
+    return new Promise(resolve => {
+      this.worker.on('online', resolve);
+    });
+  }
+  stop() {
+    // In node 12, this returns a promise, but previously it accepted a callback
+    // TODO: Pass a callback in earlier versions of Node
+    return Promise.resolve(this.worker.terminate());
+  }
+  handleMessage(data) {
+    this.onMessage((0, _buildCache().restoreDeserializedObject)(data));
+  }
+  send(data) {
+    this.worker.postMessage((0, _buildCache().prepareForSerialization)(data));
+  }
+}
+exports.default = ThreadsWorker;