@parcel/workers 2.0.0-nightly.122 → 2.0.0-nightly.1222

package/src/WorkerFarm.js

@@ -20,7 +20,7 @@ import {
   restoreDeserializedObject,
   serialize,
 } from '@parcel/core';
-import ThrowableDiagnostic, {anyToDiagnostic} from '@parcel/diagnostic';
+import ThrowableDiagnostic, {anyToDiagnostic, md} from '@parcel/diagnostic';
 import Worker, {type WorkerCall} from './Worker';
 import cpuCount from './cpuCount';
 import Handle from './Handle';
@@ -31,9 +31,10 @@ import Trace from './Trace';
 import fs from 'fs';
 import logger from '@parcel/logger';
 
-let profileId = 1;
 let referenceId = 1;
 
+export opaque type SharedReference = number;
+
 export type FarmOptions = {|
   maxConcurrentWorkers: number,
   maxConcurrentCallsPerWorker: number,
@@ -42,7 +43,7 @@ export type FarmOptions = {|
   warmWorkers: boolean,
   workerPath?: FilePath,
   backend: BackendType,
-  patchConsole?: boolean,
+  shouldPatchConsole?: boolean,
 |};
 
 type WorkerModule = {|
@@ -52,8 +53,8 @@ type WorkerModule = {|
 export type WorkerApi = {|
   callMaster(CallRequest, ?boolean): Promise<mixed>,
   createReverseHandle(fn: HandleFunction): Handle,
-  getSharedReference(ref: number): mixed,
-  resolveSharedReference(value: mixed): ?number,
+  getSharedReference(ref: SharedReference): mixed,
+  resolveSharedReference(value: mixed): ?SharedReference,
   callChild?: (childId: number, request: HandleCallRequest) => Promise<mixed>,
 |};
 
@@ -67,13 +68,15 @@ export default class WorkerFarm extends EventEmitter {
   callQueue: Array<WorkerCall> = [];
   ending: boolean = false;
   localWorker: WorkerModule;
+  localWorkerInit: ?Promise<void>;
   options: FarmOptions;
   run: HandleFunction;
   warmWorkers: number = 0;
   workers: Map<number, Worker> = new Map();
   handles: Map<number, Handle> = new Map();
-  sharedReferences: Map<number, mixed> = new Map();
-  sharedReferencesByValue: Map<mixed, number> = new Map();
+  sharedReferences: Map<SharedReference, mixed> = new Map();
+  sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
+  serializedSharedReferences: Map<SharedReference, ?ArrayBuffer> = new Map();
   profiler: ?Profiler;
 
   constructor(farmOptions: $Shape<FarmOptions> = {}) {
@@ -94,12 +97,39 @@ export default class WorkerFarm extends EventEmitter {
 
     // $FlowFixMe this must be dynamic
     this.localWorker = require(this.options.workerPath);
+    this.localWorkerInit =
+      this.localWorker.childInit != null ? this.localWorker.childInit() : null;
     this.run = this.createHandle('run');
 
+    // Worker thread stdout is by default piped into the process stdout, if there are enough worker
+    // threads to exceed the default listener limit, then anything else piping into stdout will trigger
+    // the `MaxListenersExceededWarning`, so we should ensure the max listeners is at least equal to the
+    // number of workers + 1 for the main thread.
+    //
+    // Note this can't be fixed easily where other things pipe into stdout -  even after starting > 10 worker
+    // threads `process.stdout.getMaxListeners()` will still return 10, however adding another pipe into `stdout`
+    // will give the warning with `<worker count + 1>` as the number of listeners.
+    process.stdout.setMaxListeners(
+      Math.max(
+        process.stdout.getMaxListeners(),
+        WorkerFarm.getNumWorkers() + 1,
+      ),
+    );
+
     this.startMaxWorkers();
   }
 
-  workerApi = {
+  workerApi: {|
+    callChild: (childId: number, request: HandleCallRequest) => Promise<mixed>,
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: HandleFunction) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} = {
     callMaster: async (
       request: CallRequest,
       awaitResponse: ?boolean = true,
@@ -122,7 +152,13 @@ export default class WorkerFarm extends EventEmitter {
           retries: 0,
         });
       }),
-    getSharedReference: (ref: number) => this.sharedReferences.get(ref),
+    runHandle: (handle: Handle, args: Array<any>): Promise<mixed> =>
+      this.workerApi.callChild(nullthrows(handle.childId), {
+        handle: handle.id,
+        args,
+      }),
+    getSharedReference: (ref: SharedReference) =>
+      this.sharedReferences.get(ref),
     resolveSharedReference: (value: mixed) =>
       this.sharedReferencesByValue.get(value),
   };
@@ -155,30 +191,42 @@ export default class WorkerFarm extends EventEmitter {
     );
   }
 
-  createHandle(method: string): HandleFunction {
-    return (...args) => {
+  createHandle(method: string, useMainThread: boolean = false): HandleFunction {
+    return async (...args) => {
       // Child process workers are slow to start (~600ms).
       // While we're waiting, just run on the main thread.
       // This significantly speeds up startup time.
-      if (this.shouldUseRemoteWorkers()) {
+      if (this.shouldUseRemoteWorkers() && !useMainThread) {
         return this.addCall(method, [...args, false]);
       } else {
         if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
           this.warmupWorker(method, args);
         }
 
-        let processedArgs = restoreDeserializedObject(
-          prepareForSerialization([...args, false]),
-        );
+        let processedArgs;
+        if (!useMainThread) {
+          processedArgs = restoreDeserializedObject(
+            prepareForSerialization([...args, false]),
+          );
+        } else {
+          processedArgs = args;
+        }
+
+        if (this.localWorkerInit != null) {
+          await this.localWorkerInit;
+          this.localWorkerInit = null;
+        }
         return this.localWorker[method](this.workerApi, ...processedArgs);
       }
     };
   }
 
-  onError(error: ErrorWithCode, worker: Worker) {
+  onError(error: ErrorWithCode, worker: Worker): void | Promise<void> {
     // Handle ipc errors
     if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
       return this.stopWorker(worker);
+    } else {
+      logger.error(error, '@parcel/workers');
     }
   }
 
@@ -186,7 +234,8 @@ export default class WorkerFarm extends EventEmitter {
     let worker = new Worker({
       forcedKillTime: this.options.forcedKillTime,
       backend: this.options.backend,
-      patchConsole: this.options.patchConsole,
+      shouldPatchConsole: this.options.shouldPatchConsole,
+      sharedReferences: this.sharedReferences,
     });
 
     worker.fork(nullthrows(this.options.workerPath));
@@ -231,7 +280,11 @@ export default class WorkerFarm extends EventEmitter {
       this.startChild();
     }
 
-    for (let worker of this.workers.values()) {
+    let workers = [...this.workers.values()].sort(
+      (a, b) => a.calls.size - b.calls.size,
+    );
+
+    for (let worker of workers) {
       if (!this.callQueue.length) {
         break;
       }
@@ -241,11 +294,24 @@ export default class WorkerFarm extends EventEmitter {
       }
 
       if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
-        worker.call(this.callQueue.shift());
+        this.callWorker(worker, this.callQueue.shift());
       }
     }
   }
 
+  async callWorker(worker: Worker, call: WorkerCall): Promise<void> {
+    for (let ref of this.sharedReferences.keys()) {
+      if (!worker.sentSharedReferences.has(ref)) {
+        await worker.sendSharedReference(
+          ref,
+          this.getSerializedSharedReference(ref),
+        );
+      }
+    }
+
+    worker.call(call);
+  }
+
   async processRequest(
     data: {|
       location: FilePath,
@@ -255,7 +321,7 @@ export default class WorkerFarm extends EventEmitter {
     let {method, args, location, awaitResponse, idx, handle: handleId} = data;
     let mod;
     if (handleId != null) {
-      mod = nullthrows(this.handles.get(handleId)).fn;
+      mod = nullthrows(this.handles.get(handleId)?.fn);
     } else if (location) {
       // $FlowFixMe this must be dynamic
       mod = require(location);
@@ -286,7 +352,6 @@ export default class WorkerFarm extends EventEmitter {
       }
     } else {
       // ESModule default interop
-      // $FlowFixMe
       if (mod.__esModule && !mod[method] && mod.default) {
         mod = mod.default;
       }
@@ -331,6 +396,10 @@ export default class WorkerFarm extends EventEmitter {
   async end(): Promise<void> {
     this.ending = true;
 
+    await Promise.all(
+      Array.from(this.workers.values()).map(worker => this.stopWorker(worker)),
+    );
+
     for (let handle of this.handles.values()) {
       handle.dispose();
     }
@@ -338,9 +407,6 @@ export default class WorkerFarm extends EventEmitter {
     this.sharedReferences = new Map();
     this.sharedReferencesByValue = new Map();
 
-    await Promise.all(
-      Array.from(this.workers.values()).map(worker => this.stopWorker(worker)),
-    );
     this.ending = false;
   }
 
@@ -362,40 +428,37 @@ export default class WorkerFarm extends EventEmitter {
     );
   }
 
-  createReverseHandle(fn: HandleFunction) {
-    let handle = new Handle({fn, workerApi: this.workerApi});
+  createReverseHandle(fn: HandleFunction): Handle {
+    let handle = new Handle({fn});
     this.handles.set(handle.id, handle);
     return handle;
   }
 
-  async createSharedReference(value: mixed) {
+  createSharedReference(
+    value: mixed,
+    isCacheable: boolean = true,
+  ): {|ref: SharedReference, dispose(): Promise<mixed>|} {
     let ref = referenceId++;
     this.sharedReferences.set(ref, value);
     this.sharedReferencesByValue.set(value, ref);
-    let promises = [];
-    for (let worker of this.workers.values()) {
-      promises.push(
-        new Promise((resolve, reject) => {
-          worker.call({
-            method: 'createSharedReference',
-            args: [ref, value],
-            resolve,
-            reject,
-            retries: 0,
-          });
-        }),
-      );
+    if (!isCacheable) {
+      this.serializedSharedReferences.set(ref, null);
     }
 
-    await Promise.all(promises);
-
     return {
       ref,
       dispose: () => {
         this.sharedReferences.delete(ref);
         this.sharedReferencesByValue.delete(value);
+        this.serializedSharedReferences.delete(ref);
+
         let promises = [];
         for (let worker of this.workers.values()) {
+          if (!worker.sentSharedReferences.has(ref)) {
+            continue;
+          }
+
+          worker.sentSharedReferences.delete(ref);
           promises.push(
             new Promise((resolve, reject) => {
               worker.call({
@@ -403,6 +466,7 @@ export default class WorkerFarm extends EventEmitter {
                 args: [ref],
                 resolve,
                 reject,
+                skipReadyCheck: true,
                 retries: 0,
               });
             }),
@@ -413,6 +477,24 @@ export default class WorkerFarm extends EventEmitter {
     };
   }
 
+  getSerializedSharedReference(ref: SharedReference): ArrayBuffer {
+    let cached = this.serializedSharedReferences.get(ref);
+    if (cached) {
+      return cached;
+    }
+
+    let value = this.sharedReferences.get(ref);
+    let buf = serialize(value).buffer;
+
+    // If the reference was created with the isCacheable option set to false,
+    // serializedSharedReferences will contain `null` as the value.
+    if (cached !== null) {
+      this.serializedSharedReferences.set(ref, buf);
+    }
+
+    return buf;
+  }
+
   async startProfile() {
     let promises = [];
     for (let worker of this.workers.values()) {
@@ -424,6 +506,7 @@ export default class WorkerFarm extends EventEmitter {
             resolve,
             reject,
             retries: 0,
+            skipReadyCheck: true,
           });
         }),
       );
@@ -453,6 +536,7 @@ export default class WorkerFarm extends EventEmitter {
             resolve,
             reject,
             retries: 0,
+            skipReadyCheck: true,
           });
         }),
       );
@@ -460,7 +544,7 @@ export default class WorkerFarm extends EventEmitter {
 
     var profiles = await Promise.all(promises);
     let trace = new Trace();
-    let filename = `profile-${profileId++}.trace`;
+    let filename = `profile-${getTimeId()}.trace`;
     let stream = trace.pipe(fs.createWriteStream(filename));
 
     for (let profile of profiles) {
@@ -474,21 +558,84 @@ export default class WorkerFarm extends EventEmitter {
 
     logger.info({
       origin: '@parcel/workers',
-      message: `Wrote profile to ${filename}`,
+      message: md`Wrote profile to ${filename}`,
     });
   }
 
-  static getNumWorkers() {
+  async callAllWorkers(method: string, args: Array<any>) {
+    let promises = [];
+    for (let worker of this.workers.values()) {
+      promises.push(
+        new Promise((resolve, reject) => {
+          worker.call({
+            method,
+            args,
+            resolve,
+            reject,
+            retries: 0,
+          });
+        }),
+      );
+    }
+
+    promises.push(this.localWorker[method](this.workerApi, ...args));
+    await Promise.all(promises);
+  }
+
+  async takeHeapSnapshot() {
+    let snapshotId = getTimeId();
+
+    try {
+      let snapshotPaths = await Promise.all(
+        [...this.workers.values()].map(
+          worker =>
+            new Promise((resolve, reject) => {
+              worker.call({
+                method: 'takeHeapSnapshot',
+                args: [snapshotId],
+                resolve,
+                reject,
+                retries: 0,
+                skipReadyCheck: true,
+              });
+            }),
+        ),
+      );
+
+      logger.info({
+        origin: '@parcel/workers',
+        message: md`Wrote heap snapshots to the following paths:\n${snapshotPaths.join(
+          '\n',
+        )}`,
+      });
+    } catch {
+      logger.error({
+        origin: '@parcel/workers',
+        message: 'Unable to take heap snapshots. Note: requires Node 11.13.0+',
+      });
+    }
+  }
+
+  static getNumWorkers(): number {
     return process.env.PARCEL_WORKERS
       ? parseInt(process.env.PARCEL_WORKERS, 10)
-      : cpuCount();
+      : Math.ceil(cpuCount() / 2);
   }
 
-  static isWorker() {
+  static isWorker(): boolean {
     return !!child;
   }
 
-  static getWorkerApi() {
+  static getWorkerApi(): {|
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: (...args: Array<any>) => mixed) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} {
     invariant(
       child != null,
       'WorkerFarm.getWorkerApi can only be called within workers',
@@ -496,7 +643,20 @@ export default class WorkerFarm extends EventEmitter {
     return child.workerApi;
   }
 
-  static getConcurrentCallsPerWorker() {
-    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 5;
+  static getConcurrentCallsPerWorker(): number {
+    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 30;
   }
 }
+
+function getTimeId() {
+  let now = new Date();
+  return (
+    String(now.getFullYear()) +
+    String(now.getMonth() + 1).padStart(2, '0') +
+    String(now.getDate()).padStart(2, '0') +
+    '-' +
+    String(now.getHours()).padStart(2, '0') +
+    String(now.getMinutes()).padStart(2, '0') +
+    String(now.getSeconds()).padStart(2, '0')
+  );
+}

package/src/bus.js

@@ -20,4 +20,4 @@ class Bus extends EventEmitter {
   }
 }
 
-export default new Bus();
+export default (new Bus(): Bus);

package/src/child.js

@@ -9,16 +9,20 @@ import type {
   WorkerResponse,
   ChildImpl,
 } from './types';
-import type {IDisposable} from '@parcel/types';
-import type {WorkerApi} from './WorkerFarm';
+import type {Async, IDisposable} from '@parcel/types';
+import type {SharedReference} from './WorkerFarm';
 
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 import Logger, {patchConsole, unpatchConsole} from '@parcel/logger';
 import ThrowableDiagnostic, {anyToDiagnostic} from '@parcel/diagnostic';
+import {deserialize} from '@parcel/core';
 import bus from './bus';
 import Profiler from './Profiler';
-import Handle from './Handle';
+import _Handle from './Handle';
+
+// The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
+const Handle = _Handle;
 
 type ChildCall = WorkerRequest & {|
   resolve: (result: Promise<any> | any) => void,
@@ -30,20 +34,21 @@ export class Child {
   childId: ?number;
   maxConcurrentCalls: number = 10;
   module: ?any;
-  responseId = 0;
+  responseId: number = 0;
   responseQueue: Map<number, ChildCall> = new Map();
   loggerDisposable: IDisposable;
   child: ChildImpl;
   profiler: ?Profiler;
-  workerApi: WorkerApi;
   handles: Map<number, Handle> = new Map();
-  sharedReferences: Map<number, mixed> = new Map();
-  sharedReferencesByValue: Map<mixed, number> = new Map();
+  sharedReferences: Map<SharedReference, mixed> = new Map();
+  sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
 
   constructor(ChildBackend: Class<ChildImpl>) {
     this.child = new ChildBackend(
-      this.messageListener.bind(this),
-      this.handleEnd.bind(this),
+      m => {
+        this.messageListener(m);
+      },
+      () => this.handleEnd(),
     );
 
     // Monitior all logging events inside this child process and forward to
@@ -53,19 +58,31 @@ export class Child {
     });
   }
 
-  workerApi = {
+  workerApi: {|
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: (...args: Array<any>) => mixed) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} = {
     callMaster: (
       request: CallRequest,
       awaitResponse: ?boolean = true,
     ): Promise<mixed> => this.addCall(request, awaitResponse),
     createReverseHandle: (fn: (...args: Array<any>) => mixed): Handle =>
       this.createReverseHandle(fn),
-    getSharedReference: (ref: number) => this.sharedReferences.get(ref),
+    runHandle: (handle: Handle, args: Array<any>): Promise<mixed> =>
+      this.workerApi.callMaster({handle: handle.id, args}, true),
+    getSharedReference: (ref: SharedReference) =>
+      this.sharedReferences.get(ref),
     resolveSharedReference: (value: mixed) =>
       this.sharedReferencesByValue.get(value),
   };
 
-  messageListener(message: WorkerMessage): void | Promise<void> {
+  messageListener(message: WorkerMessage): Async<void> {
     if (message.type === 'response') {
       return this.handleResponse(message);
     } else if (message.type === 'request') {
@@ -77,10 +94,14 @@ export class Child {
     this.child.send(data);
   }
 
-  childInit(module: string, childId: number): void {
+  async childInit(module: string, childId: number): Promise<void> {
     // $FlowFixMe this must be dynamic
     this.module = require(module);
     this.childId = childId;
+
+    if (this.module.childInit != null) {
+      await this.module.childInit();
+    }
   }
 
   async handleRequest(data: WorkerRequest): Promise<void> {
@@ -106,7 +127,7 @@ export class Child {
     let result;
     if (handleId != null) {
       try {
-        let fn = nullthrows(this.handles.get(handleId)).fn;
+        let fn = nullthrows(this.handles.get(handleId)?.fn);
         result = responseFromContent(fn(...args));
       } catch (e) {
         result = errorResponseFromError(e);
@@ -114,13 +135,13 @@ export class Child {
     } else if (method === 'childInit') {
       try {
         let [moduleName, childOptions] = args;
-        if (childOptions.patchConsole) {
+        if (childOptions.shouldPatchConsole) {
           patchConsole();
         } else {
           unpatchConsole();
         }
 
-        result = responseFromContent(this.childInit(moduleName, child));
+        result = responseFromContent(await this.childInit(moduleName, child));
       } catch (e) {
         result = errorResponseFromError(e);
       }
@@ -138,13 +159,37 @@ export class Child {
       } catch (e) {
         result = errorResponseFromError(e);
       }
+    } else if (method === 'takeHeapSnapshot') {
+      try {
+        let v8 = require('v8');
+        result = responseFromContent(
+          v8.writeHeapSnapshot(
+            'heap-' +
+              args[0] +
+              '-' +
+              (this.childId ? 'worker' + this.childId : 'main') +
+              '.heapsnapshot',
+          ),
+        );
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
     } else if (method === 'createSharedReference') {
-      let [ref, value] = args;
+      let [ref, _value] = args;
+      let value =
+        _value instanceof ArrayBuffer
+          ? // In the case the value is pre-serialized as a buffer,
+            // deserialize it.
+            deserialize(Buffer.from(_value))
+          : _value;
       this.sharedReferences.set(ref, value);
       this.sharedReferencesByValue.set(value, ref);
       result = responseFromContent(null);
     } else if (method === 'deleteSharedReference') {
-      this.sharedReferences.delete(args[0]);
+      let ref = args[0];
+      let value = this.sharedReferences.get(ref);
+      this.sharedReferencesByValue.delete(value);
+      this.sharedReferences.delete(ref);
       result = responseFromContent(null);
     } else {
       try {
@@ -157,7 +202,11 @@ export class Child {
       }
     }
 
-    this.send(result);
+    try {
+      this.send(result);
+    } catch (e) {
+      result = this.send(errorResponseFromError(e));
+    }
   }
 
   handleResponse(data: WorkerResponse): void {
@@ -189,6 +238,7 @@ export class Child {
       ...request,
       type: 'request',
       child: this.childId,
+      // $FlowFixMe Added in Flow 0.121.0 upgrade in #4381
       awaitResponse,
       resolve: () => {},
       reject: () => {},
@@ -241,10 +291,9 @@ export class Child {
     this.loggerDisposable.dispose();
   }
 
-  createReverseHandle(fn: (...args: Array<any>) => mixed) {
+  createReverseHandle(fn: (...args: Array<any>) => mixed): Handle {
     let handle = new Handle({
       fn,
-      workerApi: this.workerApi,
       childId: this.childId,
     });
     this.handles.set(handle.id, handle);