@parcel/workers 2.0.0-nightly.130 → 2.0.0-nightly.1303

package/lib/WorkerFarm.js

@@ -11,107 +11,129 @@ Object.defineProperty(exports, "Handle", {
 });
 exports.default = void 0;
 
-var _assert = _interopRequireDefault(require("assert"));
+function _assert() {
+  const data = _interopRequireDefault(require("assert"));
 
-var _nullthrows = _interopRequireDefault(require("nullthrows"));
+  _assert = function () {
+    return data;
+  };
 
-var _events = _interopRequireDefault(require("events"));
+  return data;
+}
 
-var _core = require("@parcel/core");
+function _nullthrows() {
+  const data = _interopRequireDefault(require("nullthrows"));
 
-var _diagnostic = _interopRequireWildcard(require("@parcel/diagnostic"));
+  _nullthrows = function () {
+    return data;
+  };
 
-var _Worker = _interopRequireDefault(require("./Worker"));
+  return data;
+}
 
-var _cpuCount = _interopRequireDefault(require("./cpuCount"));
+function _events() {
+  const data = _interopRequireDefault(require("events"));
 
-var _Handle = _interopRequireDefault(require("./Handle"));
+  _events = function () {
+    return data;
+  };
 
-var _childState = require("./childState");
+  return data;
+}
 
-var _backend = require("./backend");
+function _core() {
+  const data = require("@parcel/core");
 
-var _Profiler = _interopRequireDefault(require("./Profiler"));
+  _core = function () {
+    return data;
+  };
 
-var _Trace = _interopRequireDefault(require("./Trace"));
+  return data;
+}
 
-var _fs = _interopRequireDefault(require("fs"));
+function _diagnostic() {
+  const data = _interopRequireWildcard(require("@parcel/diagnostic"));
 
-var _logger = _interopRequireDefault(require("@parcel/logger"));
+  _diagnostic = function () {
+    return data;
+  };
 
-function _getRequireWildcardCache() { if (typeof WeakMap !== "function") return null; var cache = new WeakMap(); _getRequireWildcardCache = function () { return cache; }; return cache; }
+  return data;
+}
 
-function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+var _Worker = _interopRequireDefault(require("./Worker"));
 
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+var _cpuCount = _interopRequireDefault(require("./cpuCount"));
 
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); keys.push.apply(keys, symbols); } return keys; }
+var _Handle = _interopRequireDefault(require("./Handle"));
 
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+var _childState = require("./childState");
 
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+var _backend = require("./backend");
 
-let profileId = 1;
-let referenceId = 1;
+function _profiler() {
+  const data = require("@parcel/profiler");
 
-/**
- * workerPath should always be defined inside farmOptions
- */
-class WorkerFarm extends _events.default {
-  constructor(farmOptions = {}) {
-    super();
+  _profiler = function () {
+    return data;
+  };
 
-    _defineProperty(this, "callQueue", []);
+  return data;
+}
 
-    _defineProperty(this, "ending", false);
+function _fs() {
+  const data = _interopRequireDefault(require("fs"));
 
-    _defineProperty(this, "localWorker", void 0);
+  _fs = function () {
+    return data;
+  };
 
-    _defineProperty(this, "options", void 0);
+  return data;
+}
 
-    _defineProperty(this, "run", void 0);
+function _logger() {
+  const data = _interopRequireDefault(require("@parcel/logger"));
 
-    _defineProperty(this, "warmWorkers", 0);
+  _logger = function () {
+    return data;
+  };
 
-    _defineProperty(this, "workers", new Map());
+  return data;
+}
 
-    _defineProperty(this, "handles", new Map());
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
 
-    _defineProperty(this, "sharedReferences", new Map());
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
 
-    _defineProperty(this, "sharedReferencesByValue", new Map());
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
-    _defineProperty(this, "profiler", void 0);
+let referenceId = 1;
 
-    _defineProperty(this, "workerApi", {
-      callMaster: async (request, awaitResponse = true) => {
-        // $FlowFixMe
-        let result = await this.processRequest(_objectSpread({}, request, {
-          awaitResponse
-        }));
-        return (0, _core.deserialize)((0, _core.serialize)(result));
-      },
-      createReverseHandle: fn => this.createReverseHandle(fn),
-      callChild: (childId, request) => new Promise((resolve, reject) => {
-        (0, _nullthrows.default)(this.workers.get(childId)).call(_objectSpread({}, request, {
-          resolve,
-          reject,
-          retries: 0
-        }));
-      }),
-      getSharedReference: ref => this.sharedReferences.get(ref),
-      resolveSharedReference: value => this.sharedReferencesByValue.get(value)
-    });
+/**
+ * workerPath should always be defined inside farmOptions
+ */
+class WorkerFarm extends _events().default {
+  callQueue = [];
+  ending = false;
+  warmWorkers = 0;
+  workers = new Map();
+  handles = new Map();
+  sharedReferences = new Map();
+  sharedReferencesByValue = new Map();
+  serializedSharedReferences = new Map();
 
-    this.options = _objectSpread({
+  constructor(farmOptions = {}) {
+    super();
+    this.options = {
       maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
       maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(),
       forcedKillTime: 500,
       warmWorkers: false,
       useLocalWorker: true,
       // TODO: setting this to false makes some tests fail, figure out why
-      backend: (0, _backend.detectBackend)()
-    }, farmOptions);
+      backend: (0, _backend.detectBackend)(),
+      ...farmOptions
+    };
 
     if (!this.options.workerPath) {
       throw new Error('Please provide a worker path!');
@@ -119,10 +141,44 @@ class WorkerFarm extends _events.default {
 
 
     this.localWorker = require(this.options.workerPath);
-    this.run = this.createHandle('run');
+    this.localWorkerInit = this.localWorker.childInit != null ? this.localWorker.childInit() : null;
+    this.run = this.createHandle('run'); // Worker thread stdout is by default piped into the process stdout, if there are enough worker
+    // threads to exceed the default listener limit, then anything else piping into stdout will trigger
+    // the `MaxListenersExceededWarning`, so we should ensure the max listeners is at least equal to the
+    // number of workers + 1 for the main thread.
+    //
+    // Note this can't be fixed easily where other things pipe into stdout -  even after starting > 10 worker
+    // threads `process.stdout.getMaxListeners()` will still return 10, however adding another pipe into `stdout`
+    // will give the warning with `<worker count + 1>` as the number of listeners.
+
+    process.stdout.setMaxListeners(Math.max(process.stdout.getMaxListeners(), WorkerFarm.getNumWorkers() + 1));
     this.startMaxWorkers();
   }
 
+  workerApi = {
+    callMaster: async (request, awaitResponse = true) => {
+      // $FlowFixMe
+      let result = await this.processRequest({ ...request,
+        awaitResponse
+      });
+      return (0, _core().deserialize)((0, _core().serialize)(result));
+    },
+    createReverseHandle: fn => this.createReverseHandle(fn),
+    callChild: (childId, request) => new Promise((resolve, reject) => {
+      (0, _nullthrows().default)(this.workers.get(childId)).call({ ...request,
+        resolve,
+        reject,
+        retries: 0
+      });
+    }),
+    runHandle: (handle, args) => this.workerApi.callChild((0, _nullthrows().default)(handle.childId), {
+      handle: handle.id,
+      args
+    }),
+    getSharedReference: ref => this.sharedReferences.get(ref),
+    resolveSharedReference: value => this.sharedReferencesByValue.get(value)
+  };
+
   warmupWorker(method, args) {
     // Workers are already stopping
     if (this.ending) {
@@ -149,19 +205,35 @@ class WorkerFarm extends _events.default {
     return this.options.maxConcurrentWorkers > 0 || !this.options.useLocalWorker;
   }
 
-  createHandle(method) {
-    return (...args) => {
+  createHandle(method, useMainThread = false) {
+    if (!this.options.useLocalWorker) {
+      useMainThread = false;
+    }
+
+    return async (...args) => {
       // Child process workers are slow to start (~600ms).
       // While we're waiting, just run on the main thread.
       // This significantly speeds up startup time.
-      if (this.shouldUseRemoteWorkers()) {
+      if (this.shouldUseRemoteWorkers() && !useMainThread) {
         return this.addCall(method, [...args, false]);
       } else {
         if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
           this.warmupWorker(method, args);
         }
 
-        let processedArgs = (0, _core.restoreDeserializedObject)((0, _core.prepareForSerialization)([...args, false]));
+        let processedArgs;
+
+        if (!useMainThread) {
+          processedArgs = (0, _core().restoreDeserializedObject)((0, _core().prepareForSerialization)([...args, false]));
+        } else {
+          processedArgs = args;
+        }
+
+        if (this.localWorkerInit != null) {
+          await this.localWorkerInit;
+          this.localWorkerInit = null;
+        }
+
         return this.localWorker[method](this.workerApi, ...processedArgs);
       }
     };
@@ -171,6 +243,8 @@ class WorkerFarm extends _events.default {
     // Handle ipc errors
     if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
       return this.stopWorker(worker);
+    } else {
+      _logger().default.error(error, '@parcel/workers');
     }
   }
 
@@ -178,9 +252,11 @@ class WorkerFarm extends _events.default {
     let worker = new _Worker.default({
       forcedKillTime: this.options.forcedKillTime,
       backend: this.options.backend,
-      patchConsole: this.options.patchConsole
+      shouldPatchConsole: this.options.shouldPatchConsole,
+      shouldTrace: this.options.shouldTrace,
+      sharedReferences: this.sharedReferences
     });
-    worker.fork((0, _nullthrows.default)(this.options.workerPath));
+    worker.fork((0, _nullthrows().default)(this.options.workerPath));
     worker.on('request', data => this.processRequest(data, worker));
     worker.on('ready', () => this.processQueue());
     worker.on('response', () => this.processQueue());
@@ -215,7 +291,9 @@ class WorkerFarm extends _events.default {
       this.startChild();
     }
 
-    for (let worker of this.workers.values()) {
+    let workers = [...this.workers.values()].sort((a, b) => a.calls.size - b.calls.size);
+
+    for (let worker of workers) {
       if (!this.callQueue.length) {
         break;
       }
@@ -225,9 +303,19 @@ class WorkerFarm extends _events.default {
       }
 
       if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
-        worker.call(this.callQueue.shift());
+        this.callWorker(worker, this.callQueue.shift());
+      }
+    }
+  }
+
+  async callWorker(worker, call) {
+    for (let ref of this.sharedReferences.keys()) {
+      if (!worker.sentSharedReferences.has(ref)) {
+        await worker.sendSharedReference(ref, this.getSerializedSharedReference(ref));
       }
     }
+
+    worker.call(call);
   }
 
   async processRequest(data, worker) {
@@ -242,7 +330,9 @@ class WorkerFarm extends _events.default {
     let mod;
 
     if (handleId != null) {
-      mod = (0, _nullthrows.default)(this.handles.get(handleId)).fn;
+      var _this$handles$get;
+
+      mod = (0, _nullthrows().default)((_this$handles$get = this.handles.get(handleId)) === null || _this$handles$get === void 0 ? void 0 : _this$handles$get.fn);
     } else if (location) {
       // $FlowFixMe this must be dynamic
       mod = require(location);
@@ -261,27 +351,26 @@ class WorkerFarm extends _events.default {
       idx,
       type: 'response',
       contentType: 'error',
-      content: (0, _diagnostic.anyToDiagnostic)(e)
+      content: (0, _diagnostic().anyToDiagnostic)(e)
     });
 
     let result;
 
     if (method == null) {
       try {
-        result = responseFromContent((await mod(...args)));
+        result = responseFromContent(await mod(...args));
       } catch (e) {
         result = errorResponseFromError(e);
       }
     } else {
       // ESModule default interop
-      // $FlowFixMe
       if (mod.__esModule && !mod[method] && mod.default) {
         mod = mod.default;
       }
 
       try {
         // $FlowFixMe
-        result = responseFromContent((await mod[method](...args)));
+        result = responseFromContent(await mod[method](...args));
       } catch (e) {
         result = errorResponseFromError(e);
       }
@@ -292,7 +381,7 @@ class WorkerFarm extends _events.default {
         worker.send(result);
       } else {
         if (result.contentType === 'error') {
-          throw new _diagnostic.default({
+          throw new (_diagnostic().default)({
             diagnostic: result.content
           });
         }
@@ -321,6 +410,7 @@ class WorkerFarm extends _events.default {
 
   async end() {
     this.ending = true;
+    await Promise.all(Array.from(this.workers.values()).map(worker => this.stopWorker(worker)));
 
     for (let handle of this.handles.values()) {
       handle.dispose();
@@ -329,7 +419,6 @@ class WorkerFarm extends _events.default {
     this.handles = new Map();
     this.sharedReferences = new Map();
     this.sharedReferencesByValue = new Map();
-    await Promise.all(Array.from(this.workers.values()).map(worker => this.stopWorker(worker)));
     this.ending = false;
   }
 
@@ -350,46 +439,42 @@ class WorkerFarm extends _events.default {
 
   createReverseHandle(fn) {
     let handle = new _Handle.default({
-      fn,
-      workerApi: this.workerApi
+      fn
     });
     this.handles.set(handle.id, handle);
     return handle;
   }
 
-  async createSharedReference(value) {
+  createSharedReference(value, isCacheable = true) {
     let ref = referenceId++;
     this.sharedReferences.set(ref, value);
     this.sharedReferencesByValue.set(value, ref);
-    let promises = [];
 
-    for (let worker of this.workers.values()) {
-      promises.push(new Promise((resolve, reject) => {
-        worker.call({
-          method: 'createSharedReference',
-          args: [ref, value],
-          resolve,
-          reject,
-          retries: 0
-        });
-      }));
+    if (!isCacheable) {
+      this.serializedSharedReferences.set(ref, null);
     }
 
-    await Promise.all(promises);
     return {
       ref,
       dispose: () => {
         this.sharedReferences.delete(ref);
         this.sharedReferencesByValue.delete(value);
+        this.serializedSharedReferences.delete(ref);
         let promises = [];
 
         for (let worker of this.workers.values()) {
+          if (!worker.sentSharedReferences.has(ref)) {
+            continue;
+          }
+
+          worker.sentSharedReferences.delete(ref);
           promises.push(new Promise((resolve, reject) => {
             worker.call({
               method: 'deleteSharedReference',
               args: [ref],
               resolve,
               reject,
+              skipReadyCheck: true,
               retries: 0
             });
           }));
@@ -400,6 +485,24 @@ class WorkerFarm extends _events.default {
     };
   }
 
+  getSerializedSharedReference(ref) {
+    let cached = this.serializedSharedReferences.get(ref);
+
+    if (cached) {
+      return cached;
+    }
+
+    let value = this.sharedReferences.get(ref);
+    let buf = (0, _core().serialize)(value).buffer; // If the reference was created with the isCacheable option set to false,
+    // serializedSharedReferences will contain `null` as the value.
+
+    if (cached !== null) {
+      this.serializedSharedReferences.set(ref, buf);
+    }
+
+    return buf;
+  }
+
   async startProfile() {
     let promises = [];
 
@@ -410,12 +513,13 @@ class WorkerFarm extends _events.default {
           args: [],
           resolve,
           reject,
-          retries: 0
+          retries: 0,
+          skipReadyCheck: true
         });
       }));
     }
 
-    this.profiler = new _Profiler.default();
+    this.profiler = new (_profiler().SamplingProfiler)();
     promises.push(this.profiler.startProfiling());
     await Promise.all(promises);
   }
@@ -436,15 +540,16 @@ class WorkerFarm extends _events.default {
           args: [],
           resolve,
           reject,
-          retries: 0
+          retries: 0,
+          skipReadyCheck: true
         });
       }));
     }
 
     var profiles = await Promise.all(promises);
-    let trace = new _Trace.default();
-    let filename = `profile-${profileId++}.trace`;
-    let stream = trace.pipe(_fs.default.createWriteStream(filename));
+    let trace = new (_profiler().Trace)();
+    let filename = `profile-${getTimeId()}.trace`;
+    let stream = trace.pipe(_fs().default.createWriteStream(filename));
 
     for (let profile of profiles) {
       trace.addCPUProfile(names.shift(), profile);
@@ -455,14 +560,60 @@ class WorkerFarm extends _events.default {
       stream.once('finish', resolve);
     });
 
-    _logger.default.info({
+    _logger().default.info({
       origin: '@parcel/workers',
-      message: `Wrote profile to ${filename}`
+      message: (0, _diagnostic().md)`Wrote profile to ${filename}`
     });
   }
 
+  async callAllWorkers(method, args) {
+    let promises = [];
+
+    for (let worker of this.workers.values()) {
+      promises.push(new Promise((resolve, reject) => {
+        worker.call({
+          method,
+          args,
+          resolve,
+          reject,
+          retries: 0
+        });
+      }));
+    }
+
+    promises.push(this.localWorker[method](this.workerApi, ...args));
+    await Promise.all(promises);
+  }
+
+  async takeHeapSnapshot() {
+    let snapshotId = getTimeId();
+
+    try {
+      let snapshotPaths = await Promise.all([...this.workers.values()].map(worker => new Promise((resolve, reject) => {
+        worker.call({
+          method: 'takeHeapSnapshot',
+          args: [snapshotId],
+          resolve,
+          reject,
+          retries: 0,
+          skipReadyCheck: true
+        });
+      })));
+
+      _logger().default.info({
+        origin: '@parcel/workers',
+        message: (0, _diagnostic().md)`Wrote heap snapshots to the following paths:\n${snapshotPaths.join('\n')}`
+      });
+    } catch {
+      _logger().default.error({
+        origin: '@parcel/workers',
+        message: 'Unable to take heap snapshots. Note: requires Node 11.13.0+'
+      });
+    }
+  }
+
   static getNumWorkers() {
-    return process.env.PARCEL_WORKERS ? parseInt(process.env.PARCEL_WORKERS, 10) : (0, _cpuCount.default)();
+    return process.env.PARCEL_WORKERS ? parseInt(process.env.PARCEL_WORKERS, 10) : Math.ceil((0, _cpuCount.default)() / 2);
   }
 
   static isWorker() {
@@ -470,14 +621,19 @@ class WorkerFarm extends _events.default {
   }
 
   static getWorkerApi() {
-    (0, _assert.default)(_childState.child != null, 'WorkerFarm.getWorkerApi can only be called within workers');
+    (0, _assert().default)(_childState.child != null, 'WorkerFarm.getWorkerApi can only be called within workers');
     return _childState.child.workerApi;
   }
 
   static getConcurrentCallsPerWorker() {
-    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 5;
+    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 30;
   }
 
 }
 
-exports.default = WorkerFarm;
+exports.default = WorkerFarm;
+
+function getTimeId() {
+  let now = new Date();
+  return String(now.getFullYear()) + String(now.getMonth() + 1).padStart(2, '0') + String(now.getDate()).padStart(2, '0') + '-' + String(now.getHours()).padStart(2, '0') + String(now.getMinutes()).padStart(2, '0') + String(now.getSeconds()).padStart(2, '0');
+}

package/lib/bus.js

@@ -5,13 +5,21 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = void 0;
 
-var _events = _interopRequireDefault(require("events"));
+function _events() {
+  const data = _interopRequireDefault(require("events"));
+
+  _events = function () {
+    return data;
+  };
+
+  return data;
+}
 
 var _childState = require("./childState");
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
-class Bus extends _events.default {
+class Bus extends _events().default {
   emit(event, ...args) {
     if (_childState.child) {
       _childState.child.workerApi.callMaster({