@parcel/workers 2.0.0-nightly.97 → 2.1.0

package/src/WorkerFarm.js

@@ -20,7 +20,7 @@ import {
   restoreDeserializedObject,
   serialize,
 } from '@parcel/core';
-import ThrowableDiagnostic, {anyToDiagnostic} from '@parcel/diagnostic';
+import ThrowableDiagnostic, {anyToDiagnostic, md} from '@parcel/diagnostic';
 import Worker, {type WorkerCall} from './Worker';
 import cpuCount from './cpuCount';
 import Handle from './Handle';
@@ -31,9 +31,10 @@ import Trace from './Trace';
 import fs from 'fs';
 import logger from '@parcel/logger';
 
-let profileId = 1;
 let referenceId = 1;
 
+export opaque type SharedReference = number;
+
 export type FarmOptions = {|
   maxConcurrentWorkers: number,
   maxConcurrentCallsPerWorker: number,
@@ -42,7 +43,7 @@ export type FarmOptions = {|
   warmWorkers: boolean,
   workerPath?: FilePath,
   backend: BackendType,
-  patchConsole?: boolean,
+  shouldPatchConsole?: boolean,
 |};
 
 type WorkerModule = {|
@@ -52,8 +53,8 @@ type WorkerModule = {|
 export type WorkerApi = {|
   callMaster(CallRequest, ?boolean): Promise<mixed>,
   createReverseHandle(fn: HandleFunction): Handle,
-  getSharedReference(ref: number): mixed,
-  resolveSharedReference(value: mixed): ?number,
+  getSharedReference(ref: SharedReference): mixed,
+  resolveSharedReference(value: mixed): ?SharedReference,
   callChild?: (childId: number, request: HandleCallRequest) => Promise<mixed>,
 |};
 
@@ -67,13 +68,14 @@ export default class WorkerFarm extends EventEmitter {
   callQueue: Array<WorkerCall> = [];
   ending: boolean = false;
   localWorker: WorkerModule;
+  localWorkerInit: ?Promise<void>;
   options: FarmOptions;
   run: HandleFunction;
   warmWorkers: number = 0;
   workers: Map<number, Worker> = new Map();
   handles: Map<number, Handle> = new Map();
-  sharedReferences: Map<number, mixed> = new Map();
-  sharedReferencesByValue: Map<mixed, number> = new Map();
+  sharedReferences: Map<SharedReference, mixed> = new Map();
+  sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
   profiler: ?Profiler;
 
   constructor(farmOptions: $Shape<FarmOptions> = {}) {
@@ -94,12 +96,24 @@ export default class WorkerFarm extends EventEmitter {
 
     // $FlowFixMe this must be dynamic
     this.localWorker = require(this.options.workerPath);
+    this.localWorkerInit =
+      this.localWorker.childInit != null ? this.localWorker.childInit() : null;
     this.run = this.createHandle('run');
 
     this.startMaxWorkers();
   }
 
-  workerApi = {
+  workerApi: {|
+    callChild: (childId: number, request: HandleCallRequest) => Promise<mixed>,
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: HandleFunction) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} = {
     callMaster: async (
       request: CallRequest,
       awaitResponse: ?boolean = true,
@@ -122,7 +136,13 @@ export default class WorkerFarm extends EventEmitter {
           retries: 0,
         });
       }),
-    getSharedReference: (ref: number) => this.sharedReferences.get(ref),
+    runHandle: (handle: Handle, args: Array<any>): Promise<mixed> =>
+      this.workerApi.callChild(nullthrows(handle.childId), {
+        handle: handle.id,
+        args,
+      }),
+    getSharedReference: (ref: SharedReference) =>
+      this.sharedReferences.get(ref),
     resolveSharedReference: (value: mixed) =>
       this.sharedReferencesByValue.get(value),
   };
@@ -156,7 +176,7 @@ export default class WorkerFarm extends EventEmitter {
   }
 
   createHandle(method: string): HandleFunction {
-    return (...args) => {
+    return async (...args) => {
       // Child process workers are slow to start (~600ms).
       // While we're waiting, just run on the main thread.
       // This significantly speeds up startup time.
@@ -170,15 +190,22 @@ export default class WorkerFarm extends EventEmitter {
         let processedArgs = restoreDeserializedObject(
           prepareForSerialization([...args, false]),
         );
+
+        if (this.localWorkerInit != null) {
+          await this.localWorkerInit;
+          this.localWorkerInit = null;
+        }
         return this.localWorker[method](this.workerApi, ...processedArgs);
       }
     };
   }
 
-  onError(error: ErrorWithCode, worker: Worker) {
+  onError(error: ErrorWithCode, worker: Worker): void | Promise<void> {
     // Handle ipc errors
     if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
       return this.stopWorker(worker);
+    } else {
+      logger.error(error, '@parcel/workers');
     }
   }
 
@@ -186,7 +213,8 @@ export default class WorkerFarm extends EventEmitter {
     let worker = new Worker({
       forcedKillTime: this.options.forcedKillTime,
       backend: this.options.backend,
-      patchConsole: this.options.patchConsole,
+      shouldPatchConsole: this.options.shouldPatchConsole,
+      sharedReferences: this.sharedReferences,
     });
 
     worker.fork(nullthrows(this.options.workerPath));
@@ -231,7 +259,11 @@ export default class WorkerFarm extends EventEmitter {
       this.startChild();
     }
 
-    for (let worker of this.workers.values()) {
+    let workers = [...this.workers.values()].sort(
+      (a, b) => a.calls.size - b.calls.size,
+    );
+
+    for (let worker of workers) {
       if (!this.callQueue.length) {
         break;
       }
@@ -255,7 +287,7 @@ export default class WorkerFarm extends EventEmitter {
     let {method, args, location, awaitResponse, idx, handle: handleId} = data;
     let mod;
     if (handleId != null) {
-      mod = nullthrows(this.handles.get(handleId)).fn;
+      mod = nullthrows(this.handles.get(handleId)?.fn);
     } else if (location) {
       // $FlowFixMe this must be dynamic
       mod = require(location);
@@ -286,7 +318,6 @@ export default class WorkerFarm extends EventEmitter {
       }
     } else {
       // ESModule default interop
-      // $FlowFixMe
       if (mod.__esModule && !mod[method] && mod.default) {
         mod = mod.default;
       }
@@ -331,6 +362,10 @@ export default class WorkerFarm extends EventEmitter {
   async end(): Promise<void> {
     this.ending = true;
 
+    await Promise.all(
+      Array.from(this.workers.values()).map(worker => this.stopWorker(worker)),
+    );
+
     for (let handle of this.handles.values()) {
       handle.dispose();
     }
@@ -338,9 +373,6 @@ export default class WorkerFarm extends EventEmitter {
     this.sharedReferences = new Map();
     this.sharedReferencesByValue = new Map();
 
-    await Promise.all(
-      Array.from(this.workers.values()).map(worker => this.stopWorker(worker)),
-    );
     this.ending = false;
   }
 
@@ -362,29 +394,28 @@ export default class WorkerFarm extends EventEmitter {
     );
   }
 
-  createReverseHandle(fn: HandleFunction) {
-    let handle = new Handle({fn, workerApi: this.workerApi});
+  createReverseHandle(fn: HandleFunction): Handle {
+    let handle = new Handle({fn});
     this.handles.set(handle.id, handle);
     return handle;
   }
 
-  async createSharedReference(value: mixed) {
+  async createSharedReference(
+    value: mixed,
+    // An optional, pre-serialized representation of the value to be used
+    // in its place.
+    buffer?: Buffer,
+  ): Promise<{|ref: SharedReference, dispose(): Promise<mixed>|}> {
     let ref = referenceId++;
     this.sharedReferences.set(ref, value);
     this.sharedReferencesByValue.set(value, ref);
+
+    let toSend = buffer ? buffer.buffer : value;
     let promises = [];
     for (let worker of this.workers.values()) {
-      promises.push(
-        new Promise((resolve, reject) => {
-          worker.call({
-            method: 'createSharedReference',
-            args: [ref, value],
-            resolve,
-            reject,
-            retries: 0,
-          });
-        }),
-      );
+      if (worker.ready) {
+        promises.push(worker.sendSharedReference(ref, toSend));
+      }
     }
 
     await Promise.all(promises);
@@ -403,6 +434,7 @@ export default class WorkerFarm extends EventEmitter {
                 args: [ref],
                 resolve,
                 reject,
+                skipReadyCheck: true,
                 retries: 0,
               });
             }),
@@ -424,6 +456,7 @@ export default class WorkerFarm extends EventEmitter {
             resolve,
             reject,
             retries: 0,
+            skipReadyCheck: true,
           });
         }),
       );
@@ -453,6 +486,7 @@ export default class WorkerFarm extends EventEmitter {
             resolve,
             reject,
             retries: 0,
+            skipReadyCheck: true,
           });
         }),
       );
@@ -460,7 +494,7 @@ export default class WorkerFarm extends EventEmitter {
 
     var profiles = await Promise.all(promises);
     let trace = new Trace();
-    let filename = `profile-${profileId++}.trace`;
+    let filename = `profile-${getTimeId()}.trace`;
     let stream = trace.pipe(fs.createWriteStream(filename));
 
     for (let profile of profiles) {
@@ -474,21 +508,84 @@ export default class WorkerFarm extends EventEmitter {
 
     logger.info({
       origin: '@parcel/workers',
-      message: `Wrote profile to ${filename}`,
+      message: md`Wrote profile to ${filename}`,
     });
   }
 
-  static getNumWorkers() {
+  async callAllWorkers(method: string, args: Array<any>) {
+    let promises = [];
+    for (let worker of this.workers.values()) {
+      promises.push(
+        new Promise((resolve, reject) => {
+          worker.call({
+            method,
+            args,
+            resolve,
+            reject,
+            retries: 0,
+          });
+        }),
+      );
+    }
+
+    promises.push(this.localWorker[method](this.workerApi, ...args));
+    await Promise.all(promises);
+  }
+
+  async takeHeapSnapshot() {
+    let snapshotId = getTimeId();
+
+    try {
+      let snapshotPaths = await Promise.all(
+        [...this.workers.values()].map(
+          worker =>
+            new Promise((resolve, reject) => {
+              worker.call({
+                method: 'takeHeapSnapshot',
+                args: [snapshotId],
+                resolve,
+                reject,
+                retries: 0,
+                skipReadyCheck: true,
+              });
+            }),
+        ),
+      );
+
+      logger.info({
+        origin: '@parcel/workers',
+        message: md`Wrote heap snapshots to the following paths:\n${snapshotPaths.join(
+          '\n',
+        )}`,
+      });
+    } catch {
+      logger.error({
+        origin: '@parcel/workers',
+        message: 'Unable to take heap snapshots. Note: requires Node 11.13.0+',
+      });
+    }
+  }
+
+  static getNumWorkers(): number {
     return process.env.PARCEL_WORKERS
       ? parseInt(process.env.PARCEL_WORKERS, 10)
-      : cpuCount();
+      : Math.ceil(cpuCount() / 2);
   }
 
-  static isWorker() {
+  static isWorker(): boolean {
     return !!child;
   }
 
-  static getWorkerApi() {
+  static getWorkerApi(): {|
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: (...args: Array<any>) => mixed) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} {
     invariant(
       child != null,
       'WorkerFarm.getWorkerApi can only be called within workers',
@@ -496,7 +593,20 @@ export default class WorkerFarm extends EventEmitter {
     return child.workerApi;
   }
 
-  static getConcurrentCallsPerWorker() {
-    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 5;
+  static getConcurrentCallsPerWorker(): number {
+    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 30;
   }
 }
+
+function getTimeId() {
+  let now = new Date();
+  return (
+    String(now.getFullYear()) +
+    String(now.getMonth() + 1).padStart(2, '0') +
+    String(now.getDate()).padStart(2, '0') +
+    '-' +
+    String(now.getHours()).padStart(2, '0') +
+    String(now.getMinutes()).padStart(2, '0') +
+    String(now.getSeconds()).padStart(2, '0')
+  );
+}

package/src/backend.js

@@ -2,6 +2,12 @@
 import type {BackendType, WorkerImpl} from './types';
 
 export function detectBackend(): BackendType {
+  switch (process.env.PARCEL_WORKER_BACKEND) {
+    case 'threads':
+    case 'process':
+      return process.env.PARCEL_WORKER_BACKEND;
+  }
+
   try {
     require('worker_threads');
     return 'threads';

package/src/bus.js

@@ -20,4 +20,4 @@ class Bus extends EventEmitter {
   }
 }
 
-export default new Bus();
+export default (new Bus(): Bus);

package/src/child.js

@@ -9,16 +9,20 @@ import type {
   WorkerResponse,
   ChildImpl,
 } from './types';
-import type {IDisposable} from '@parcel/types';
-import type {WorkerApi} from './WorkerFarm';
+import type {Async, IDisposable} from '@parcel/types';
+import type {SharedReference, WorkerApi} from './WorkerFarm';
 
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 import Logger, {patchConsole, unpatchConsole} from '@parcel/logger';
 import ThrowableDiagnostic, {anyToDiagnostic} from '@parcel/diagnostic';
+import {deserialize} from '@parcel/core';
 import bus from './bus';
 import Profiler from './Profiler';
-import Handle from './Handle';
+import _Handle from './Handle';
+
+// The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
+const Handle = _Handle;
 
 type ChildCall = WorkerRequest & {|
   resolve: (result: Promise<any> | any) => void,
@@ -30,20 +34,22 @@ export class Child {
   childId: ?number;
   maxConcurrentCalls: number = 10;
   module: ?any;
-  responseId = 0;
+  responseId: number = 0;
   responseQueue: Map<number, ChildCall> = new Map();
   loggerDisposable: IDisposable;
   child: ChildImpl;
   profiler: ?Profiler;
   workerApi: WorkerApi;
   handles: Map<number, Handle> = new Map();
-  sharedReferences: Map<number, mixed> = new Map();
-  sharedReferencesByValue: Map<mixed, number> = new Map();
+  sharedReferences: Map<SharedReference, mixed> = new Map();
+  sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
 
   constructor(ChildBackend: Class<ChildImpl>) {
     this.child = new ChildBackend(
-      this.messageListener.bind(this),
-      this.handleEnd.bind(this),
+      m => {
+        this.messageListener(m);
+      },
+      () => this.handleEnd(),
     );
 
     // Monitior all logging events inside this child process and forward to
@@ -53,19 +59,31 @@ export class Child {
     });
   }
 
-  workerApi = {
+  workerApi: {|
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: (...args: Array<any>) => mixed) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} = {
     callMaster: (
       request: CallRequest,
       awaitResponse: ?boolean = true,
     ): Promise<mixed> => this.addCall(request, awaitResponse),
     createReverseHandle: (fn: (...args: Array<any>) => mixed): Handle =>
       this.createReverseHandle(fn),
-    getSharedReference: (ref: number) => this.sharedReferences.get(ref),
+    runHandle: (handle: Handle, args: Array<any>): Promise<mixed> =>
+      this.workerApi.callMaster({handle: handle.id, args}, true),
+    getSharedReference: (ref: SharedReference) =>
+      this.sharedReferences.get(ref),
     resolveSharedReference: (value: mixed) =>
       this.sharedReferencesByValue.get(value),
   };
 
-  messageListener(message: WorkerMessage): void | Promise<void> {
+  messageListener(message: WorkerMessage): Async<void> {
     if (message.type === 'response') {
       return this.handleResponse(message);
     } else if (message.type === 'request') {
@@ -77,10 +95,14 @@ export class Child {
     this.child.send(data);
   }
 
-  childInit(module: string, childId: number): void {
+  async childInit(module: string, childId: number): Promise<void> {
     // $FlowFixMe this must be dynamic
     this.module = require(module);
     this.childId = childId;
+
+    if (this.module.childInit != null) {
+      await this.module.childInit();
+    }
   }
 
   async handleRequest(data: WorkerRequest): Promise<void> {
@@ -106,7 +128,7 @@ export class Child {
     let result;
     if (handleId != null) {
       try {
-        let fn = nullthrows(this.handles.get(handleId)).fn;
+        let fn = nullthrows(this.handles.get(handleId)?.fn);
         result = responseFromContent(fn(...args));
       } catch (e) {
         result = errorResponseFromError(e);
@@ -114,13 +136,13 @@ export class Child {
     } else if (method === 'childInit') {
       try {
         let [moduleName, childOptions] = args;
-        if (childOptions.patchConsole) {
+        if (childOptions.shouldPatchConsole) {
           patchConsole();
         } else {
           unpatchConsole();
         }
 
-        result = responseFromContent(this.childInit(moduleName, child));
+        result = responseFromContent(await this.childInit(moduleName, child));
       } catch (e) {
         result = errorResponseFromError(e);
       }
@@ -138,13 +160,38 @@ export class Child {
       } catch (e) {
         result = errorResponseFromError(e);
       }
+    } else if (method === 'takeHeapSnapshot') {
+      try {
+        let v8 = require('v8');
+        result = responseFromContent(
+          // $FlowFixMe
+          v8.writeHeapSnapshot(
+            'heap-' +
+              args[0] +
+              '-' +
+              (this.childId ? 'worker' + this.childId : 'main') +
+              '.heapsnapshot',
+          ),
+        );
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
     } else if (method === 'createSharedReference') {
-      let [ref, value] = args;
+      let [ref, _value] = args;
+      let value =
+        _value instanceof ArrayBuffer
+          ? // In the case the value is pre-serialized as a buffer,
+            // deserialize it.
+            deserialize(Buffer.from(_value))
+          : _value;
       this.sharedReferences.set(ref, value);
       this.sharedReferencesByValue.set(value, ref);
       result = responseFromContent(null);
     } else if (method === 'deleteSharedReference') {
-      this.sharedReferences.delete(args[0]);
+      let ref = args[0];
+      let value = this.sharedReferences.get(ref);
+      this.sharedReferencesByValue.delete(value);
+      this.sharedReferences.delete(ref);
       result = responseFromContent(null);
     } else {
       try {
@@ -157,7 +204,11 @@ export class Child {
       }
     }
 
-    this.send(result);
+    try {
+      this.send(result);
+    } catch (e) {
+      result = this.send(errorResponseFromError(e));
+    }
   }
 
   handleResponse(data: WorkerResponse): void {
@@ -189,6 +240,7 @@ export class Child {
       ...request,
       type: 'request',
       child: this.childId,
+      // $FlowFixMe Added in Flow 0.121.0 upgrade in #4381
       awaitResponse,
       resolve: () => {},
       reject: () => {},
@@ -241,10 +293,9 @@ export class Child {
     this.loggerDisposable.dispose();
   }
 
-  createReverseHandle(fn: (...args: Array<any>) => mixed) {
+  createReverseHandle(fn: (...args: Array<any>) => mixed): Handle {
     let handle = new Handle({
       fn,
-      workerApi: this.workerApi,
       childId: this.childId,
     });
     this.handles.set(handle.id, handle);

package/src/cpuCount.js

@@ -26,6 +26,11 @@ export function detectRealCores(): number {
     );
   } else if (platform === 'darwin') {
     amount = parseInt(exec('sysctl -n hw.physicalcpu_max'), 10);
+  } else if (platform === 'win32') {
+    const str = exec('wmic cpu get NumberOfCores').match(/\d+/g);
+    if (str !== null) {
+      amount = parseInt(str.filter(n => n !== '')[0], 10);
+    }
   }
 
   if (!amount || amount <= 0) {
@@ -36,9 +41,8 @@ export function detectRealCores(): number {
 }
 
 let cores;
-export default function getCores(bypassCache?: boolean = false) {
+export default function getCores(bypassCache?: boolean = false): number {
   // Do not re-run commands if we already have the count...
-  // $FlowFixMe
   if (cores && !bypassCache) {
     return cores;
   }
@@ -49,8 +53,9 @@ export default function getCores(bypassCache?: boolean = false) {
     // Guess the amount of real cores
     cores = os
       .cpus()
-      .filter((cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1)
-      .length;
+      .filter(
+        (cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1,
+      ).length;
   }
 
   // Another fallback

package/src/index.js

@@ -34,4 +34,4 @@ if (!WorkerFarm.isWorker()) {
 export default WorkerFarm;
 export {bus};
 export {Handle} from './WorkerFarm';
-export type {WorkerApi, FarmOptions} from './WorkerFarm';
+export type {WorkerApi, FarmOptions, SharedReference} from './WorkerFarm';

package/src/process/ProcessChild.js

@@ -25,7 +25,7 @@ export default class ProcessChild implements ChildImpl {
     process.on('message', data => this.handleMessage(data));
   }
 
-  handleMessage(data: string) {
+  handleMessage(data: string): void {
     if (data === 'die') {
       return this.stop();
     }

package/src/process/ProcessWorker.js

@@ -34,7 +34,7 @@ export default class ProcessWorker implements WorkerImpl {
     this.onExit = onExit;
   }
 
-  start() {
+  start(): Promise<void> {
     this.child = childProcess.fork(WORKER_PATH, process.argv, {
       execArgv: this.execArgv,
       env: process.env,

package/src/threads/ThreadsWorker.js

@@ -32,7 +32,7 @@ export default class ThreadsWorker implements WorkerImpl {
     this.onExit = onExit;
   }
 
-  start() {
+  start(): Promise<void> {
     this.worker = new Worker(WORKER_PATH, {
       execArgv: this.execArgv,
       env: process.env,
@@ -47,7 +47,7 @@ export default class ThreadsWorker implements WorkerImpl {
     });
   }
 
-  stop() {
+  stop(): Promise<void> {
     // In node 12, this returns a promise, but previously it accepted a callback
     // TODO: Pass a callback in earlier versions of Node
     return Promise.resolve(this.worker.terminate());

package/test/cpuCount.test.js

@@ -3,7 +3,7 @@ import os from 'os';
 
 import getCores, {detectRealCores} from '../src/cpuCount';
 
-describe('cpuCount', function() {
+describe('cpuCount', function () {
   it('Should be able to detect real cpu count', () => {
     // Windows not supported as getting the cpu count takes a couple seconds...
     if (os.platform() === 'win32') return;

package/test/integration/workerfarm/console.js

@@ -1,4 +1,4 @@
-const WorkerFarm = require('../../../').default;
+const WorkerFarm = require('../../../src/WorkerFarm').default;
 
 function run() {
   if (WorkerFarm.isWorker()) {