@parcel/workers 2.0.0-beta.3 → 2.0.0-dev.1510

package/src/Worker.js

@@ -23,6 +23,7 @@ type WorkerOpts = {|
   forcedKillTime: number,
   backend: BackendType,
   shouldPatchConsole?: boolean,
+  shouldTrace?: boolean,
   sharedReferences: $ReadOnlyMap<SharedReference, mixed>,
 |};
 
@@ -31,7 +32,7 @@ export default class Worker extends EventEmitter {
   +options: WorkerOpts;
   worker: WorkerImpl;
   id: number = WORKER_ID++;
-  sharedReferences: $ReadOnlyMap<SharedReference, mixed> = new Map();
+  sentSharedReferences: Set<SharedReference> = new Set();
 
   calls: Map<number, WorkerCall> = new Map();
   exitCode: ?number = null;
@@ -47,18 +48,25 @@ export default class Worker extends EventEmitter {
   }
 
   async fork(forkModule: FilePath) {
-    let filteredArgs = process.execArgv.filter(
-      v => !/^--(debug|inspect|max-old-space-size=)/.test(v),
-    );
-
-    for (let i = 0; i < filteredArgs.length; i++) {
-      let arg = filteredArgs[i];
-      if (
-        (arg === '-r' || arg === '--require') &&
-        filteredArgs[i + 1] === '@parcel/register'
-      ) {
-        filteredArgs.splice(i, 2);
-        i--;
+    let filteredArgs = [];
+    if (process.execArgv) {
+      filteredArgs = process.execArgv.filter(
+        v =>
+          !/^--(debug|inspect|no-opt|max-old-space-size=|max-semi-space-size=|expose-gc)/.test(
+            v,
+          ),
+      );
+
+      for (let i = 0; i < filteredArgs.length; i++) {
+        let arg = filteredArgs[i];
+        let isArgWithParam =
+          ((arg === '-r' || arg === '--require') &&
+            filteredArgs[i + 1] === '@parcel/register') ||
+          arg === '--title';
+        if (isArgWithParam) {
+          filteredArgs.splice(i, 2);
+          i--;
+        }
       }
     }
 
@@ -67,7 +75,7 @@ export default class Worker extends EventEmitter {
       // arg parsing logic adapted from https://stackoverflow.com/a/46946420/2352201
       let opts = [''];
       let quote = false;
-      for (let c of nullthrows(process.env.NODE_OPTIONS.match(/\\?.|^$/g))) {
+      for (let c of nullthrows(process.env.NODE_OPTIONS.match(/.|^$/g))) {
         if (c === '"') {
           quote = !quote;
         } else if (!quote && c === ' ') {
@@ -107,6 +115,7 @@ export default class Worker extends EventEmitter {
           forkModule,
           {
             shouldPatchConsole: !!this.options.shouldPatchConsole,
+            shouldTrace: !!this.options.shouldTrace,
           },
         ],
         retries: 0,
@@ -135,6 +144,7 @@ export default class Worker extends EventEmitter {
   }
 
   sendSharedReference(ref: SharedReference, value: mixed): Promise<any> {
+    this.sentSharedReferences.add(ref);
     return new Promise((resolve, reject) => {
       this.call({
         method: 'createSharedReference',

package/src/WorkerFarm.js

@@ -11,6 +11,8 @@ import type {
 } from './types';
 import type {HandleFunction} from './Handle';
 
+import * as coreWorker from './core-worker';
+import * as bus from './bus';
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 import EventEmitter from 'events';
@@ -26,8 +28,7 @@ import cpuCount from './cpuCount';
 import Handle from './Handle';
 import {child} from './childState';
 import {detectBackend} from './backend';
-import Profiler from './Profiler';
-import Trace from './Trace';
+import {SamplingProfiler, Trace} from '@parcel/profiler';
 import fs from 'fs';
 import logger from '@parcel/logger';
 
@@ -44,11 +45,13 @@ export type FarmOptions = {|
   workerPath?: FilePath,
   backend: BackendType,
   shouldPatchConsole?: boolean,
+  shouldTrace?: boolean,
 |};
 
-type WorkerModule = {|
+type WorkerModule = {
   +[string]: (...args: Array<mixed>) => Promise<mixed>,
-|};
+  ...
+};
 
 export type WorkerApi = {|
   callMaster(CallRequest, ?boolean): Promise<mixed>,
@@ -60,6 +63,8 @@ export type WorkerApi = {|
 
 export {Handle};
 
+const DEFAULT_MAX_CONCURRENT_CALLS: number = 30;
+
 /**
  * workerPath should always be defined inside farmOptions
  */
@@ -68,20 +73,25 @@ export default class WorkerFarm extends EventEmitter {
   callQueue: Array<WorkerCall> = [];
   ending: boolean = false;
   localWorker: WorkerModule;
+  localWorkerInit: ?Promise<void>;
   options: FarmOptions;
   run: HandleFunction;
   warmWorkers: number = 0;
+  readyWorkers: number = 0;
   workers: Map<number, Worker> = new Map();
   handles: Map<number, Handle> = new Map();
   sharedReferences: Map<SharedReference, mixed> = new Map();
   sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
-  profiler: ?Profiler;
+  serializedSharedReferences: Map<SharedReference, ?ArrayBuffer> = new Map();
+  profiler: ?SamplingProfiler;
 
   constructor(farmOptions: $Shape<FarmOptions> = {}) {
     super();
     this.options = {
       maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
-      maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(),
+      maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(
+        farmOptions.shouldTrace ? 1 : DEFAULT_MAX_CONCURRENT_CALLS,
+      ),
       forcedKillTime: 500,
       warmWorkers: false,
       useLocalWorker: true, // TODO: setting this to false makes some tests fail, figure out why
@@ -93,10 +103,40 @@ export default class WorkerFarm extends EventEmitter {
       throw new Error('Please provide a worker path!');
     }
 
-    // $FlowFixMe this must be dynamic
-    this.localWorker = require(this.options.workerPath);
+    // $FlowFixMe
+    if (process.browser) {
+      if (this.options.workerPath === '@parcel/core/src/worker.js') {
+        this.localWorker = coreWorker;
+      } else {
+        throw new Error(
+          'No dynamic require possible: ' + this.options.workerPath,
+        );
+      }
+    } else {
+      // $FlowFixMe this must be dynamic
+      this.localWorker = require(this.options.workerPath);
+    }
+
+    this.localWorkerInit =
+      this.localWorker.childInit != null ? this.localWorker.childInit() : null;
+
     this.run = this.createHandle('run');
 
+    // Worker thread stdout is by default piped into the process stdout, if there are enough worker
+    // threads to exceed the default listener limit, then anything else piping into stdout will trigger
+    // the `MaxListenersExceededWarning`, so we should ensure the max listeners is at least equal to the
+    // number of workers + 1 for the main thread.
+    //
+    // Note this can't be fixed easily where other things pipe into stdout -  even after starting > 10 worker
+    // threads `process.stdout.getMaxListeners()` will still return 10, however adding another pipe into `stdout`
+    // will give the warning with `<worker count + 1>` as the number of listeners.
+    process.stdout?.setMaxListeners(
+      Math.max(
+        process.stdout.getMaxListeners(),
+        WorkerFarm.getNumWorkers() + 1,
+      ),
+    );
+
     this.startMaxWorkers();
   }
 
@@ -172,21 +212,35 @@ export default class WorkerFarm extends EventEmitter {
     );
   }
 
-  createHandle(method: string): HandleFunction {
-    return (...args) => {
+  createHandle(method: string, useMainThread: boolean = false): HandleFunction {
+    if (!this.options.useLocalWorker) {
+      useMainThread = false;
+    }
+
+    return async (...args) => {
       // Child process workers are slow to start (~600ms).
       // While we're waiting, just run on the main thread.
       // This significantly speeds up startup time.
-      if (this.shouldUseRemoteWorkers()) {
+      if (this.shouldUseRemoteWorkers() && !useMainThread) {
         return this.addCall(method, [...args, false]);
       } else {
         if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
           this.warmupWorker(method, args);
         }
 
-        let processedArgs = restoreDeserializedObject(
-          prepareForSerialization([...args, false]),
-        );
+        let processedArgs;
+        if (!useMainThread) {
+          processedArgs = restoreDeserializedObject(
+            prepareForSerialization([...args, false]),
+          );
+        } else {
+          processedArgs = args;
+        }
+
+        if (this.localWorkerInit != null) {
+          await this.localWorkerInit;
+          this.localWorkerInit = null;
+        }
         return this.localWorker[method](this.workerApi, ...processedArgs);
       }
     };
@@ -206,6 +260,7 @@ export default class WorkerFarm extends EventEmitter {
       forcedKillTime: this.options.forcedKillTime,
       backend: this.options.backend,
       shouldPatchConsole: this.options.shouldPatchConsole,
+      shouldTrace: this.options.shouldTrace,
       sharedReferences: this.sharedReferences,
     });
 
@@ -213,7 +268,13 @@ export default class WorkerFarm extends EventEmitter {
 
     worker.on('request', data => this.processRequest(data, worker));
 
-    worker.on('ready', () => this.processQueue());
+    worker.on('ready', () => {
+      this.readyWorkers++;
+      if (this.readyWorkers === this.options.maxConcurrentWorkers) {
+        this.emit('ready');
+      }
+      this.processQueue();
+    });
     worker.on('response', () => this.processQueue());
 
     worker.on('error', err => this.onError(err, worker));
@@ -265,11 +326,24 @@ export default class WorkerFarm extends EventEmitter {
       }
 
       if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
-        worker.call(this.callQueue.shift());
+        this.callWorker(worker, this.callQueue.shift());
       }
     }
   }
 
+  async callWorker(worker: Worker, call: WorkerCall): Promise<void> {
+    for (let ref of this.sharedReferences.keys()) {
+      if (!worker.sentSharedReferences.has(ref)) {
+        await worker.sendSharedReference(
+          ref,
+          this.getSerializedSharedReference(ref),
+        );
+      }
+    }
+
+    worker.call(call);
+  }
+
   async processRequest(
     data: {|
       location: FilePath,
@@ -281,8 +355,17 @@ export default class WorkerFarm extends EventEmitter {
     if (handleId != null) {
       mod = nullthrows(this.handles.get(handleId)?.fn);
     } else if (location) {
-      // $FlowFixMe this must be dynamic
-      mod = require(location);
+      // $FlowFixMe
+      if (process.browser) {
+        if (location === '@parcel/workers/src/bus.js') {
+          mod = (bus: any);
+        } else {
+          throw new Error('No dynamic require possible: ' + location);
+        }
+      } else {
+        // $FlowFixMe this must be dynamic
+        mod = require(location);
+      }
     } else {
       throw new Error('Unknown request');
     }
@@ -392,33 +475,31 @@ export default class WorkerFarm extends EventEmitter {
     return handle;
   }
 
-  async createSharedReference(
+  createSharedReference(
     value: mixed,
-    // An optional, pre-serialized representation of the value to be used
-    // in its place.
-    buffer?: Buffer,
-  ): Promise<{|ref: SharedReference, dispose(): Promise<mixed>|}> {
+    isCacheable: boolean = true,
+  ): {|ref: SharedReference, dispose(): Promise<mixed>|} {
     let ref = referenceId++;
     this.sharedReferences.set(ref, value);
     this.sharedReferencesByValue.set(value, ref);
-
-    let toSend = buffer ? buffer.buffer : value;
-    let promises = [];
-    for (let worker of this.workers.values()) {
-      if (worker.ready) {
-        promises.push(worker.sendSharedReference(ref, toSend));
-      }
+    if (!isCacheable) {
+      this.serializedSharedReferences.set(ref, null);
     }
 
-    await Promise.all(promises);
-
     return {
       ref,
       dispose: () => {
         this.sharedReferences.delete(ref);
         this.sharedReferencesByValue.delete(value);
+        this.serializedSharedReferences.delete(ref);
+
         let promises = [];
         for (let worker of this.workers.values()) {
+          if (!worker.sentSharedReferences.has(ref)) {
+            continue;
+          }
+
+          worker.sentSharedReferences.delete(ref);
           promises.push(
             new Promise((resolve, reject) => {
               worker.call({
@@ -437,6 +518,24 @@ export default class WorkerFarm extends EventEmitter {
     };
   }
 
+  getSerializedSharedReference(ref: SharedReference): ArrayBuffer {
+    let cached = this.serializedSharedReferences.get(ref);
+    if (cached) {
+      return cached;
+    }
+
+    let value = this.sharedReferences.get(ref);
+    let buf = serialize(value).buffer;
+
+    // If the reference was created with the isCacheable option set to false,
+    // serializedSharedReferences will contain `null` as the value.
+    if (cached !== null) {
+      this.serializedSharedReferences.set(ref, buf);
+    }
+
+    return buf;
+  }
+
   async startProfile() {
     let promises = [];
     for (let worker of this.workers.values()) {
@@ -454,7 +553,7 @@ export default class WorkerFarm extends EventEmitter {
       );
     }
 
-    this.profiler = new Profiler();
+    this.profiler = new SamplingProfiler();
 
     promises.push(this.profiler.startProfiling());
     await Promise.all(promises);
@@ -561,7 +660,7 @@ export default class WorkerFarm extends EventEmitter {
   static getNumWorkers(): number {
     return process.env.PARCEL_WORKERS
       ? parseInt(process.env.PARCEL_WORKERS, 10)
-      : Math.ceil(cpuCount() / 2);
+      : Math.min(4, Math.ceil(cpuCount() / 2));
   }
 
   static isWorker(): boolean {
@@ -585,8 +684,12 @@ export default class WorkerFarm extends EventEmitter {
     return child.workerApi;
   }
 
-  static getConcurrentCallsPerWorker(): number {
-    return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 30;
+  static getConcurrentCallsPerWorker(
+    defaultValue?: number = DEFAULT_MAX_CONCURRENT_CALLS,
+  ): number {
+    return (
+      parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || defaultValue
+    );
   }
 }
 

package/src/backend.js

@@ -2,6 +2,9 @@
 import type {BackendType, WorkerImpl} from './types';
 
 export function detectBackend(): BackendType {
+  // $FlowFixMe
+  if (process.browser) return 'web';
+
   switch (process.env.PARCEL_WORKER_BACKEND) {
     case 'threads':
     case 'process':
@@ -22,6 +25,8 @@ export function getWorkerBackend(backend: BackendType): Class<WorkerImpl> {
       return require('./threads/ThreadsWorker').default;
     case 'process':
       return require('./process/ProcessWorker').default;
+    case 'web':
+      return require('./web/WebWorker').default;
     default:
       throw new Error(`Invalid backend: ${backend}`);
   }

package/src/bus.js

@@ -7,7 +7,8 @@ class Bus extends EventEmitter {
     if (child) {
       child.workerApi.callMaster(
         {
-          location: __filename,
+          // $FlowFixMe
+          location: process.browser ? '@parcel/workers/src/bus.js' : __filename,
           method: 'emit',
           args: [event, ...args],
         },

package/src/child.js

@@ -10,15 +10,16 @@ import type {
   ChildImpl,
 } from './types';
 import type {Async, IDisposable} from '@parcel/types';
-import type {SharedReference, WorkerApi} from './WorkerFarm';
+import type {SharedReference} from './WorkerFarm';
 
+import * as coreWorker from './core-worker';
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 import Logger, {patchConsole, unpatchConsole} from '@parcel/logger';
 import ThrowableDiagnostic, {anyToDiagnostic} from '@parcel/diagnostic';
 import {deserialize} from '@parcel/core';
 import bus from './bus';
-import Profiler from './Profiler';
+import {SamplingProfiler, tracer} from '@parcel/profiler';
 import _Handle from './Handle';
 
 // The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
@@ -37,17 +38,19 @@ export class Child {
   responseId: number = 0;
   responseQueue: Map<number, ChildCall> = new Map();
   loggerDisposable: IDisposable;
+  tracerDisposable: IDisposable;
   child: ChildImpl;
-  profiler: ?Profiler;
-  workerApi: WorkerApi;
+  profiler: ?SamplingProfiler;
   handles: Map<number, Handle> = new Map();
   sharedReferences: Map<SharedReference, mixed> = new Map();
   sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
 
   constructor(ChildBackend: Class<ChildImpl>) {
     this.child = new ChildBackend(
-      this.messageListener.bind(this),
-      this.handleEnd.bind(this),
+      m => {
+        this.messageListener(m);
+      },
+      () => this.handleEnd(),
     );
 
     // Monitior all logging events inside this child process and forward to
@@ -55,6 +58,10 @@ export class Child {
     this.loggerDisposable = Logger.onLog(event => {
       bus.emit('logEvent', event);
     });
+    // .. and do the same for trace events
+    this.tracerDisposable = tracer.onTrace(event => {
+      bus.emit('traceEvent', event);
+    });
   }
 
   workerApi: {|
@@ -93,10 +100,23 @@ export class Child {
     this.child.send(data);
   }
 
-  childInit(module: string, childId: number): void {
-    // $FlowFixMe this must be dynamic
-    this.module = require(module);
+  async childInit(module: string, childId: number): Promise<void> {
+    // $FlowFixMe
+    if (process.browser) {
+      if (module === '@parcel/core/src/worker.js') {
+        this.module = coreWorker;
+      } else {
+        throw new Error('No dynamic require possible: ' + module);
+      }
+    } else {
+      // $FlowFixMe this must be dynamic
+      this.module = require(module);
+    }
     this.childId = childId;
+
+    if (this.module.childInit != null) {
+      await this.module.childInit();
+    }
   }
 
   async handleRequest(data: WorkerRequest): Promise<void> {
@@ -136,12 +156,16 @@ export class Child {
           unpatchConsole();
         }
 
-        result = responseFromContent(this.childInit(moduleName, child));
+        if (childOptions.shouldTrace) {
+          tracer.enable();
+        }
+
+        result = responseFromContent(await this.childInit(moduleName, child));
       } catch (e) {
         result = errorResponseFromError(e);
       }
     } else if (method === 'startProfile') {
-      this.profiler = new Profiler();
+      this.profiler = new SamplingProfiler();
       try {
         result = responseFromContent(await this.profiler.startProfiling());
       } catch (e) {
@@ -158,7 +182,6 @@ export class Child {
       try {
         let v8 = require('v8');
         result = responseFromContent(
-          // $FlowFixMe
           v8.writeHeapSnapshot(
             'heap-' +
               args[0] +
@@ -285,6 +308,7 @@ export class Child {
 
   handleEnd(): void {
     this.loggerDisposable.dispose();
+    this.tracerDisposable.dispose();
   }
 
   createReverseHandle(fn: (...args: Array<any>) => mixed): Handle {

package/src/core-worker.browser.js

@@ -0,0 +1,3 @@
+// @flow
+// eslint-disable-next-line monorepo/no-internal-import
+module.exports = require('@parcel/core/src/worker.js');

package/src/core-worker.js

@@ -0,0 +1,2 @@
+// This is used only in browser builds
+module.exports = {};

package/src/cpuCount.js

@@ -47,14 +47,23 @@ export default function getCores(bypassCache?: boolean = false): number {
     return cores;
   }
 
-  try {
-    cores = detectRealCores();
-  } catch (e) {
-    // Guess the amount of real cores
-    cores = os
-      .cpus()
-      .filter((cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1)
-      .length;
+  // $FlowFixMe
+  if (process.browser) {
+    // eslint-disable-next-line no-undef
+    cores = navigator.hardwareConcurrency / 2;
+  }
+
+  if (!cores) {
+    try {
+      cores = detectRealCores();
+    } catch (e) {
+      // Guess the amount of real cores
+      cores = os
+        .cpus()
+        .filter(
+          (cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1,
+        ).length;
+    }
   }
 
   // Another fallback

package/src/index.js

@@ -1,9 +1,10 @@
 // @flow
-import type {LogEvent} from '@parcel/types';
+import type {TraceEvent, LogEvent} from '@parcel/types';
 import invariant from 'assert';
 import WorkerFarm from './WorkerFarm';
 import Logger from '@parcel/logger';
 import bus from './bus';
+import {tracer} from '@parcel/profiler';
 
 if (!WorkerFarm.isWorker()) {
   // Forward all logger events originating from workers into the main process
@@ -29,6 +30,11 @@ if (!WorkerFarm.isWorker()) {
         throw new Error('Unknown log level');
     }
   });
+
+  // Forward all trace events originating from workers into the main process
+  bus.on('traceEvent', (e: TraceEvent) => {
+    tracer.trace(e);
+  });
 }
 
 export default WorkerFarm;

package/src/process/ProcessChild.js

@@ -37,6 +37,7 @@ export default class ProcessChild implements ChildImpl {
     let processSend = nullthrows(process.send).bind(process);
     processSend(serialize(data).toString('base64'), err => {
       if (err && err instanceof Error) {
+        // $FlowFixMe[prop-missing]
         if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
           // IPC connection closed
           // no need to keep the worker running if it can't send or receive data

package/src/types.js

@@ -65,4 +65,4 @@ export interface ChildImpl {
   send(data: WorkerMessage): void;
 }
 
-export type BackendType = 'threads' | 'process';
+export type BackendType = 'threads' | 'process' | 'web';

package/src/web/WebChild.js

@@ -0,0 +1,50 @@
+// @flow
+/* eslint-env worker*/
+
+import type {
+  ChildImpl,
+  MessageHandler,
+  ExitHandler,
+  WorkerMessage,
+} from '../types';
+import {setChild} from '../childState';
+import {Child} from '../child';
+import {prepareForSerialization, restoreDeserializedObject} from '@parcel/core';
+
+export default class WebChild implements ChildImpl {
+  onMessage: MessageHandler;
+  onExit: ExitHandler;
+
+  constructor(onMessage: MessageHandler, onExit: ExitHandler) {
+    if (
+      !(
+        typeof WorkerGlobalScope !== 'undefined' &&
+        self instanceof WorkerGlobalScope
+      )
+    ) {
+      throw new Error('Only create WebChild instances in a worker!');
+    }
+
+    this.onMessage = onMessage;
+    this.onExit = onExit;
+    self.addEventListener('message', ({data}: MessageEvent) => {
+      if (data === 'stop') {
+        this.onExit(0);
+        self.postMessage('stopped');
+      }
+      // $FlowFixMe assume WorkerMessage as data
+      this.handleMessage(data);
+    });
+    self.postMessage('online');
+  }
+
+  handleMessage(data: WorkerMessage) {
+    this.onMessage(restoreDeserializedObject(data));
+  }
+
+  send(data: WorkerMessage) {
+    self.postMessage(prepareForSerialization(data));
+  }
+}
+
+setChild(new Child(WebChild));