@parcel/workers 2.0.0-nightly.151 → 2.0.0-nightly.1511

package/src/WorkerFarm.js

@@ -11,6 +11,8 @@ import type {
 } from './types';
 import type {HandleFunction} from './Handle';
 
+import * as coreWorker from './core-worker';
+import * as bus from './bus';
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 import EventEmitter from 'events';
@@ -20,20 +22,20 @@ import {
   restoreDeserializedObject,
   serialize,
 } from '@parcel/core';
-import ThrowableDiagnostic, {anyToDiagnostic} from '@parcel/diagnostic';
+import ThrowableDiagnostic, {anyToDiagnostic, md} from '@parcel/diagnostic';
 import Worker, {type WorkerCall} from './Worker';
 import cpuCount from './cpuCount';
 import Handle from './Handle';
 import {child} from './childState';
 import {detectBackend} from './backend';
-import Profiler from './Profiler';
-import Trace from './Trace';
+import {SamplingProfiler, Trace} from '@parcel/profiler';
 import fs from 'fs';
 import logger from '@parcel/logger';
 
-let profileId = 1;
 let referenceId = 1;
 
+export opaque type SharedReference = number;
+
 export type FarmOptions = {|
   maxConcurrentWorkers: number,
   maxConcurrentCallsPerWorker: number,
@@ -42,23 +44,27 @@ export type FarmOptions = {|
   warmWorkers: boolean,
   workerPath?: FilePath,
   backend: BackendType,
-  patchConsole?: boolean,
+  shouldPatchConsole?: boolean,
+  shouldTrace?: boolean,
 |};
 
-type WorkerModule = {|
+type WorkerModule = {
   +[string]: (...args: Array<mixed>) => Promise<mixed>,
-|};
+  ...
+};
 
 export type WorkerApi = {|
   callMaster(CallRequest, ?boolean): Promise<mixed>,
   createReverseHandle(fn: HandleFunction): Handle,
-  getSharedReference(ref: number): mixed,
-  resolveSharedReference(value: mixed): ?number,
+  getSharedReference(ref: SharedReference): mixed,
+  resolveSharedReference(value: mixed): ?SharedReference,
   callChild?: (childId: number, request: HandleCallRequest) => Promise<mixed>,
 |};
 
 export {Handle};
 
+const DEFAULT_MAX_CONCURRENT_CALLS: number = 30;
+
 /**
  * workerPath should always be defined inside farmOptions
  */
@@ -67,20 +73,25 @@ export default class WorkerFarm extends EventEmitter {
   callQueue: Array<WorkerCall> = [];
   ending: boolean = false;
   localWorker: WorkerModule;
+  localWorkerInit: ?Promise<void>;
   options: FarmOptions;
   run: HandleFunction;
   warmWorkers: number = 0;
+  readyWorkers: number = 0;
   workers: Map<number, Worker> = new Map();
   handles: Map<number, Handle> = new Map();
-  sharedReferences: Map<number, mixed> = new Map();
-  sharedReferencesByValue: Map<mixed, number> = new Map();
-  profiler: ?Profiler;
+  sharedReferences: Map<SharedReference, mixed> = new Map();
+  sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
+  serializedSharedReferences: Map<SharedReference, ?ArrayBuffer> = new Map();
+  profiler: ?SamplingProfiler;
 
   constructor(farmOptions: $Shape<FarmOptions> = {}) {
     super();
     this.options = {
       maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
-      maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(),
+      maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(
+        farmOptions.shouldTrace ? 1 : DEFAULT_MAX_CONCURRENT_CALLS,
+      ),
       forcedKillTime: 500,
       warmWorkers: false,
       useLocalWorker: true, // TODO: setting this to false makes some tests fail, figure out why
@@ -92,14 +103,54 @@ export default class WorkerFarm extends EventEmitter {
       throw new Error('Please provide a worker path!');
     }
 
-    // $FlowFixMe this must be dynamic
-    this.localWorker = require(this.options.workerPath);
+    // $FlowFixMe
+    if (process.browser) {
+      if (this.options.workerPath === '@parcel/core/src/worker.js') {
+        this.localWorker = coreWorker;
+      } else {
+        throw new Error(
+          'No dynamic require possible: ' + this.options.workerPath,
+        );
+      }
+    } else {
+      // $FlowFixMe this must be dynamic
+      this.localWorker = require(this.options.workerPath);
+    }
+
+    this.localWorkerInit =
+      this.localWorker.childInit != null ? this.localWorker.childInit() : null;
+
     this.run = this.createHandle('run');
 
+    // Worker thread stdout is by default piped into the process stdout, if there are enough worker
+    // threads to exceed the default listener limit, then anything else piping into stdout will trigger
+    // the `MaxListenersExceededWarning`, so we should ensure the max listeners is at least equal to the
+    // number of workers + 1 for the main thread.
+    //
+    // Note this can't be fixed easily where other things pipe into stdout -  even after starting > 10 worker
+    // threads `process.stdout.getMaxListeners()` will still return 10, however adding another pipe into `stdout`
+    // will give the warning with `<worker count + 1>` as the number of listeners.
+    process.stdout?.setMaxListeners(
+      Math.max(
+        process.stdout.getMaxListeners(),
+        WorkerFarm.getNumWorkers() + 1,
+      ),
+    );
+
     this.startMaxWorkers();
   }
 
-  workerApi = {
+  workerApi: {|
+    callChild: (childId: number, request: HandleCallRequest) => Promise<mixed>,
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: HandleFunction) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} = {
     callMaster: async (
       request: CallRequest,
       awaitResponse: ?boolean = true,
@@ -122,7 +173,13 @@ export default class WorkerFarm extends EventEmitter {
           retries: 0,
         });
       }),
-    getSharedReference: (ref: number) => this.sharedReferences.get(ref),
+    runHandle: (handle: Handle, args: Array<any>): Promise<mixed> =>
+      this.workerApi.callChild(nullthrows(handle.childId), {
+        handle: handle.id,
+        args,
+      }),
+    getSharedReference: (ref: SharedReference) =>
+      this.sharedReferences.get(ref),
     resolveSharedReference: (value: mixed) =>
       this.sharedReferencesByValue.get(value),
   };
@@ -155,30 +212,46 @@ export default class WorkerFarm extends EventEmitter {
     );
   }
 
-  createHandle(method: string): HandleFunction {
-    return (...args) => {
+  createHandle(method: string, useMainThread: boolean = false): HandleFunction {
+    if (!this.options.useLocalWorker) {
+      useMainThread = false;
+    }
+
+    return async (...args) => {
       // Child process workers are slow to start (~600ms).
       // While we're waiting, just run on the main thread.
       // This significantly speeds up startup time.
-      if (this.shouldUseRemoteWorkers()) {
+      if (this.shouldUseRemoteWorkers() && !useMainThread) {
         return this.addCall(method, [...args, false]);
       } else {
         if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
           this.warmupWorker(method, args);
         }
 
-        let processedArgs = restoreDeserializedObject(
-          prepareForSerialization([...args, false]),
-        );
+        let processedArgs;
+        if (!useMainThread) {
+          processedArgs = restoreDeserializedObject(
+            prepareForSerialization([...args, false]),
+          );
+        } else {
+          processedArgs = args;
+        }
+
+        if (this.localWorkerInit != null) {
+          await this.localWorkerInit;
+          this.localWorkerInit = null;
+        }
         return this.localWorker[method](this.workerApi, ...processedArgs);
       }
     };
   }
 
-  onError(error: ErrorWithCode, worker: Worker) {
+  onError(error: ErrorWithCode, worker: Worker): void | Promise<void> {
     // Handle ipc errors
     if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
       return this.stopWorker(worker);
+    } else {
+      logger.error(error, '@parcel/workers');
     }
   }
 
@@ -186,14 +259,22 @@ export default class WorkerFarm extends EventEmitter {
     let worker = new Worker({
       forcedKillTime: this.options.forcedKillTime,
       backend: this.options.backend,
-      patchConsole: this.options.patchConsole,
+      shouldPatchConsole: this.options.shouldPatchConsole,
+      shouldTrace: this.options.shouldTrace,
+      sharedReferences: this.sharedReferences,
     });
 
     worker.fork(nullthrows(this.options.workerPath));
 
     worker.on('request', data => this.processRequest(data, worker));
 
-    worker.on('ready', () => this.processQueue());
+    worker.on('ready', () => {
+      this.readyWorkers++;
+      if (this.readyWorkers === this.options.maxConcurrentWorkers) {
+        this.emit('ready');
+      }
+      this.processQueue();
+    });
     worker.on('response', () => this.processQueue());
 
     worker.on('error', err => this.onError(err, worker));
@@ -231,7 +312,11 @@ export default class WorkerFarm extends EventEmitter {
       this.startChild();
     }
 
-    for (let worker of this.workers.values()) {
+    let workers = [...this.workers.values()].sort(
+      (a, b) => a.calls.size - b.calls.size,
+    );
+
+    for (let worker of workers) {
       if (!this.callQueue.length) {
         break;
       }
@@ -241,9 +326,22 @@ export default class WorkerFarm extends EventEmitter {
       }
 
       if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
-        worker.call(this.callQueue.shift());
+        this.callWorker(worker, this.callQueue.shift());
+      }
+    }
+  }
+
+  async callWorker(worker: Worker, call: WorkerCall): Promise<void> {
+    for (let ref of this.sharedReferences.keys()) {
+      if (!worker.sentSharedReferences.has(ref)) {
+        await worker.sendSharedReference(
+          ref,
+          this.getSerializedSharedReference(ref),
+        );
       }
     }
+
+    worker.call(call);
   }
 
   async processRequest(
@@ -255,10 +353,19 @@ export default class WorkerFarm extends EventEmitter {
     let {method, args, location, awaitResponse, idx, handle: handleId} = data;
     let mod;
     if (handleId != null) {
-      mod = nullthrows(this.handles.get(handleId)).fn;
+      mod = nullthrows(this.handles.get(handleId)?.fn);
     } else if (location) {
-      // $FlowFixMe this must be dynamic
-      mod = require(location);
+      // $FlowFixMe
+      if (process.browser) {
+        if (location === '@parcel/workers/src/bus.js') {
+          mod = (bus: any);
+        } else {
+          throw new Error('No dynamic require possible: ' + location);
+        }
+      } else {
+        // $FlowFixMe this must be dynamic
+        mod = require(location);
+      }
     } else {
       throw new Error('Unknown request');
     }
@@ -286,7 +393,6 @@ export default class WorkerFarm extends EventEmitter {
       }
     } else {
       // ESModule default interop
-      // $FlowFixMe
       if (mod.__esModule && !mod[method] && mod.default) {
         mod = mod.default;
       }
@@ -331,6 +437,10 @@ export default class WorkerFarm extends EventEmitter {
   async end(): Promise<void> {
     this.ending = true;
 
+    await Promise.all(
+      Array.from(this.workers.values()).map(worker => this.stopWorker(worker)),
+    );
+
     for (let handle of this.handles.values()) {
       handle.dispose();
     }
@@ -338,9 +448,6 @@ export default class WorkerFarm extends EventEmitter {
     this.sharedReferences = new Map();
     this.sharedReferencesByValue = new Map();
 
-    await Promise.all(
-      Array.from(this.workers.values()).map(worker => this.stopWorker(worker)),
-    );
     this.ending = false;
   }
 
@@ -362,40 +469,37 @@ export default class WorkerFarm extends EventEmitter {
     );
   }
 
-  createReverseHandle(fn: HandleFunction) {
-    let handle = new Handle({fn, workerApi: this.workerApi});
+  createReverseHandle(fn: HandleFunction): Handle {
+    let handle = new Handle({fn});
     this.handles.set(handle.id, handle);
     return handle;
   }
 
-  async createSharedReference(value: mixed) {
+  createSharedReference(
+    value: mixed,
+    isCacheable: boolean = true,
+  ): {|ref: SharedReference, dispose(): Promise<mixed>|} {
     let ref = referenceId++;
     this.sharedReferences.set(ref, value);
     this.sharedReferencesByValue.set(value, ref);
-    let promises = [];
-    for (let worker of this.workers.values()) {
-      promises.push(
-        new Promise((resolve, reject) => {
-          worker.call({
-            method: 'createSharedReference',
-            args: [ref, value],
-            resolve,
-            reject,
-            retries: 0,
-          });
-        }),
-      );
+    if (!isCacheable) {
+      this.serializedSharedReferences.set(ref, null);
     }
 
-    await Promise.all(promises);
-
     return {
       ref,
       dispose: () => {
         this.sharedReferences.delete(ref);
         this.sharedReferencesByValue.delete(value);
+        this.serializedSharedReferences.delete(ref);
+
         let promises = [];
         for (let worker of this.workers.values()) {
+          if (!worker.sentSharedReferences.has(ref)) {
+            continue;
+          }
+
+          worker.sentSharedReferences.delete(ref);
           promises.push(
             new Promise((resolve, reject) => {
               worker.call({
@@ -403,6 +507,7 @@ export default class WorkerFarm extends EventEmitter {
                 args: [ref],
                 resolve,
                 reject,
+                skipReadyCheck: true,
                 retries: 0,
               });
             }),
@@ -413,6 +518,24 @@ export default class WorkerFarm extends EventEmitter {
     };
   }
 
+  getSerializedSharedReference(ref: SharedReference): ArrayBuffer {
+    let cached = this.serializedSharedReferences.get(ref);
+    if (cached) {
+      return cached;
+    }
+
+    let value = this.sharedReferences.get(ref);
+    let buf = serialize(value).buffer;
+
+    // If the reference was created with the isCacheable option set to false,
+    // serializedSharedReferences will contain `null` as the value.
+    if (cached !== null) {
+      this.serializedSharedReferences.set(ref, buf);
+    }
+
+    return buf;
+  }
+
   async startProfile() {
     let promises = [];
     for (let worker of this.workers.values()) {
@@ -424,12 +547,13 @@ export default class WorkerFarm extends EventEmitter {
             resolve,
             reject,
             retries: 0,
+            skipReadyCheck: true,
           });
         }),
       );
     }
 
-    this.profiler = new Profiler();
+    this.profiler = new SamplingProfiler();
 
     promises.push(this.profiler.startProfiling());
     await Promise.all(promises);
@@ -453,6 +577,7 @@ export default class WorkerFarm extends EventEmitter {
             resolve,
             reject,
             retries: 0,
+            skipReadyCheck: true,
           });
         }),
       );
@@ -460,7 +585,7 @@ export default class WorkerFarm extends EventEmitter {
 
     var profiles = await Promise.all(promises);
     let trace = new Trace();
-    let filename = `profile-${profileId++}.trace`;
+    let filename = `profile-${getTimeId()}.trace`;
     let stream = trace.pipe(fs.createWriteStream(filename));
 
     for (let profile of profiles) {
@@ -474,21 +599,84 @@ export default class WorkerFarm extends EventEmitter {
 
     logger.info({
       origin: '@parcel/workers',
-      message: `Wrote profile to ${filename}`,
+      message: md`Wrote profile to ${filename}`,
     });
   }
 
-  static getNumWorkers() {
+  async callAllWorkers(method: string, args: Array<any>) {
+    let promises = [];
+    for (let worker of this.workers.values()) {
+      promises.push(
+        new Promise((resolve, reject) => {
+          worker.call({
+            method,
+            args,
+            resolve,
+            reject,
+            retries: 0,
+          });
+        }),
+      );
+    }
+
+    promises.push(this.localWorker[method](this.workerApi, ...args));
+    await Promise.all(promises);
+  }
+
+  async takeHeapSnapshot() {
+    let snapshotId = getTimeId();
+
+    try {
+      let snapshotPaths = await Promise.all(
+        [...this.workers.values()].map(
+          worker =>
+            new Promise((resolve, reject) => {
+              worker.call({
+                method: 'takeHeapSnapshot',
+                args: [snapshotId],
+                resolve,
+                reject,
+                retries: 0,
+                skipReadyCheck: true,
+              });
+            }),
+        ),
+      );
+
+      logger.info({
+        origin: '@parcel/workers',
+        message: md`Wrote heap snapshots to the following paths:\n${snapshotPaths.join(
+          '\n',
+        )}`,
+      });
+    } catch {
+      logger.error({
+        origin: '@parcel/workers',
+        message: 'Unable to take heap snapshots. Note: requires Node 11.13.0+',
+      });
+    }
+  }
+
+  static getNumWorkers(): number {
     return process.env.PARCEL_WORKERS
       ? parseInt(process.env.PARCEL_WORKERS, 10)
-      : cpuCount();
+      : Math.min(4, Math.ceil(cpuCount() / 2));
   }
 
-  static isWorker() {
+  static isWorker(): boolean {
     return !!child;
   }
 
-  static getWorkerApi() {
+  static getWorkerApi(): {|
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: (...args: Array<any>) => mixed) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} {
     invariant(
       child != null,
       'WorkerFarm.getWorkerApi can only be called within workers',
@@ -496,7 +684,24 @@ export default class WorkerFarm extends EventEmitter {
     return child.workerApi;
   }
 
-  static getConcurrentCallsPerWorker() {
-    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 5;
+  static getConcurrentCallsPerWorker(
+    defaultValue?: number = DEFAULT_MAX_CONCURRENT_CALLS,
+  ): number {
+    return (
+      parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || defaultValue
+    );
   }
 }
+
+function getTimeId() {
+  let now = new Date();
+  return (
+    String(now.getFullYear()) +
+    String(now.getMonth() + 1).padStart(2, '0') +
+    String(now.getDate()).padStart(2, '0') +
+    '-' +
+    String(now.getHours()).padStart(2, '0') +
+    String(now.getMinutes()).padStart(2, '0') +
+    String(now.getSeconds()).padStart(2, '0')
+  );
+}

package/src/backend.js

@@ -2,6 +2,9 @@
 import type {BackendType, WorkerImpl} from './types';
 
 export function detectBackend(): BackendType {
+  // $FlowFixMe
+  if (process.browser) return 'web';
+
   switch (process.env.PARCEL_WORKER_BACKEND) {
     case 'threads':
     case 'process':
@@ -22,6 +25,8 @@ export function getWorkerBackend(backend: BackendType): Class<WorkerImpl> {
       return require('./threads/ThreadsWorker').default;
     case 'process':
       return require('./process/ProcessWorker').default;
+    case 'web':
+      return require('./web/WebWorker').default;
     default:
       throw new Error(`Invalid backend: ${backend}`);
   }

package/src/bus.js

@@ -7,7 +7,8 @@ class Bus extends EventEmitter {
     if (child) {
       child.workerApi.callMaster(
         {
-          location: __filename,
+          // $FlowFixMe
+          location: process.browser ? '@parcel/workers/src/bus.js' : __filename,
           method: 'emit',
           args: [event, ...args],
         },
@@ -20,4 +21,4 @@ class Bus extends EventEmitter {
   }
 }
 
-export default new Bus();
+export default (new Bus(): Bus);