@parcel/workers 2.0.0-nightly.146 → 2.0.0-nightly.1470

package/src/child.js

@@ -9,16 +9,21 @@ import type {
   WorkerResponse,
   ChildImpl,
 } from './types';
-import type {IDisposable} from '@parcel/types';
-import type {WorkerApi} from './WorkerFarm';
+import type {Async, IDisposable} from '@parcel/types';
+import type {SharedReference} from './WorkerFarm';
 
+import * as coreWorker from './core-worker';
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
 import Logger, {patchConsole, unpatchConsole} from '@parcel/logger';
 import ThrowableDiagnostic, {anyToDiagnostic} from '@parcel/diagnostic';
+import {deserialize} from '@parcel/core';
 import bus from './bus';
-import Profiler from './Profiler';
-import Handle from './Handle';
+import {SamplingProfiler, tracer} from '@parcel/profiler';
+import _Handle from './Handle';
+
+// The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
+const Handle = _Handle;
 
 type ChildCall = WorkerRequest & {|
   resolve: (result: Promise<any> | any) => void,
@@ -30,20 +35,22 @@ export class Child {
   childId: ?number;
   maxConcurrentCalls: number = 10;
   module: ?any;
-  responseId = 0;
+  responseId: number = 0;
   responseQueue: Map<number, ChildCall> = new Map();
   loggerDisposable: IDisposable;
+  tracerDisposable: IDisposable;
   child: ChildImpl;
-  profiler: ?Profiler;
-  workerApi: WorkerApi;
+  profiler: ?SamplingProfiler;
   handles: Map<number, Handle> = new Map();
-  sharedReferences: Map<number, mixed> = new Map();
-  sharedReferencesByValue: Map<mixed, number> = new Map();
+  sharedReferences: Map<SharedReference, mixed> = new Map();
+  sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
 
   constructor(ChildBackend: Class<ChildImpl>) {
     this.child = new ChildBackend(
-      this.messageListener.bind(this),
-      this.handleEnd.bind(this),
+      m => {
+        this.messageListener(m);
+      },
+      () => this.handleEnd(),
     );
 
     // Monitior all logging events inside this child process and forward to
@@ -51,21 +58,37 @@ export class Child {
     this.loggerDisposable = Logger.onLog(event => {
       bus.emit('logEvent', event);
     });
+    // .. and do the same for trace events
+    this.tracerDisposable = tracer.onTrace(event => {
+      bus.emit('traceEvent', event);
+    });
   }
 
-  workerApi = {
+  workerApi: {|
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: (...args: Array<any>) => mixed) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} = {
     callMaster: (
       request: CallRequest,
       awaitResponse: ?boolean = true,
     ): Promise<mixed> => this.addCall(request, awaitResponse),
     createReverseHandle: (fn: (...args: Array<any>) => mixed): Handle =>
       this.createReverseHandle(fn),
-    getSharedReference: (ref: number) => this.sharedReferences.get(ref),
+    runHandle: (handle: Handle, args: Array<any>): Promise<mixed> =>
+      this.workerApi.callMaster({handle: handle.id, args}, true),
+    getSharedReference: (ref: SharedReference) =>
+      this.sharedReferences.get(ref),
     resolveSharedReference: (value: mixed) =>
       this.sharedReferencesByValue.get(value),
   };
 
-  messageListener(message: WorkerMessage): void | Promise<void> {
+  messageListener(message: WorkerMessage): Async<void> {
     if (message.type === 'response') {
       return this.handleResponse(message);
     } else if (message.type === 'request') {
@@ -77,10 +100,23 @@ export class Child {
     this.child.send(data);
   }
 
-  childInit(module: string, childId: number): void {
-    // $FlowFixMe this must be dynamic
-    this.module = require(module);
+  async childInit(module: string, childId: number): Promise<void> {
+    // $FlowFixMe
+    if (process.browser) {
+      if (module === '@parcel/core/src/worker.js') {
+        this.module = coreWorker;
+      } else {
+        throw new Error('No dynamic require possible: ' + module);
+      }
+    } else {
+      // $FlowFixMe this must be dynamic
+      this.module = require(module);
+    }
     this.childId = childId;
+
+    if (this.module.childInit != null) {
+      await this.module.childInit();
+    }
   }
 
   async handleRequest(data: WorkerRequest): Promise<void> {
@@ -106,7 +142,7 @@ export class Child {
     let result;
     if (handleId != null) {
       try {
-        let fn = nullthrows(this.handles.get(handleId)).fn;
+        let fn = nullthrows(this.handles.get(handleId)?.fn);
         result = responseFromContent(fn(...args));
       } catch (e) {
         result = errorResponseFromError(e);
@@ -114,18 +150,22 @@ export class Child {
     } else if (method === 'childInit') {
       try {
         let [moduleName, childOptions] = args;
-        if (childOptions.patchConsole) {
+        if (childOptions.shouldPatchConsole) {
           patchConsole();
         } else {
           unpatchConsole();
         }
 
-        result = responseFromContent(this.childInit(moduleName, child));
+        if (childOptions.shouldTrace) {
+          tracer.enable();
+        }
+
+        result = responseFromContent(await this.childInit(moduleName, child));
       } catch (e) {
         result = errorResponseFromError(e);
       }
     } else if (method === 'startProfile') {
-      this.profiler = new Profiler();
+      this.profiler = new SamplingProfiler();
       try {
         result = responseFromContent(await this.profiler.startProfiling());
       } catch (e) {
@@ -138,13 +178,37 @@ export class Child {
       } catch (e) {
         result = errorResponseFromError(e);
       }
+    } else if (method === 'takeHeapSnapshot') {
+      try {
+        let v8 = require('v8');
+        result = responseFromContent(
+          v8.writeHeapSnapshot(
+            'heap-' +
+              args[0] +
+              '-' +
+              (this.childId ? 'worker' + this.childId : 'main') +
+              '.heapsnapshot',
+          ),
+        );
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
     } else if (method === 'createSharedReference') {
-      let [ref, value] = args;
+      let [ref, _value] = args;
+      let value =
+        _value instanceof ArrayBuffer
+          ? // In the case the value is pre-serialized as a buffer,
+            // deserialize it.
+            deserialize(Buffer.from(_value))
+          : _value;
       this.sharedReferences.set(ref, value);
       this.sharedReferencesByValue.set(value, ref);
       result = responseFromContent(null);
     } else if (method === 'deleteSharedReference') {
-      this.sharedReferences.delete(args[0]);
+      let ref = args[0];
+      let value = this.sharedReferences.get(ref);
+      this.sharedReferencesByValue.delete(value);
+      this.sharedReferences.delete(ref);
       result = responseFromContent(null);
     } else {
       try {
@@ -157,7 +221,11 @@ export class Child {
       }
     }
 
-    this.send(result);
+    try {
+      this.send(result);
+    } catch (e) {
+      result = this.send(errorResponseFromError(e));
+    }
   }
 
   handleResponse(data: WorkerResponse): void {
@@ -189,6 +257,7 @@ export class Child {
       ...request,
       type: 'request',
       child: this.childId,
+      // $FlowFixMe Added in Flow 0.121.0 upgrade in #4381
       awaitResponse,
       resolve: () => {},
       reject: () => {},
@@ -239,12 +308,12 @@ export class Child {
 
   handleEnd(): void {
     this.loggerDisposable.dispose();
+    this.tracerDisposable.dispose();
   }
 
-  createReverseHandle(fn: (...args: Array<any>) => mixed) {
+  createReverseHandle(fn: (...args: Array<any>) => mixed): Handle {
     let handle = new Handle({
       fn,
-      workerApi: this.workerApi,
       childId: this.childId,
     });
     this.handles.set(handle.id, handle);

package/src/core-worker.browser.js

@@ -0,0 +1,3 @@
+// @flow
+// eslint-disable-next-line monorepo/no-internal-import
+module.exports = require('@parcel/core/src/worker.js');

package/src/core-worker.js

@@ -0,0 +1,2 @@
+// This is used only in browser builds
+module.exports = {};

package/src/cpuCount.js

@@ -26,6 +26,11 @@ export function detectRealCores(): number {
     );
   } else if (platform === 'darwin') {
     amount = parseInt(exec('sysctl -n hw.physicalcpu_max'), 10);
+  } else if (platform === 'win32') {
+    const str = exec('wmic cpu get NumberOfCores').match(/\d+/g);
+    if (str !== null) {
+      amount = parseInt(str.filter(n => n !== '')[0], 10);
+    }
   }
 
   if (!amount || amount <= 0) {
@@ -36,21 +41,29 @@ export function detectRealCores(): number {
 }
 
 let cores;
-export default function getCores(bypassCache?: boolean = false) {
+export default function getCores(bypassCache?: boolean = false): number {
   // Do not re-run commands if we already have the count...
-  // $FlowFixMe
   if (cores && !bypassCache) {
     return cores;
   }
 
-  try {
-    cores = detectRealCores();
-  } catch (e) {
-    // Guess the amount of real cores
-    cores = os
-      .cpus()
-      .filter((cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1)
-      .length;
+  // $FlowFixMe
+  if (process.browser) {
+    // eslint-disable-next-line no-undef
+    cores = navigator.hardwareConcurrency / 2;
+  }
+
+  if (!cores) {
+    try {
+      cores = detectRealCores();
+    } catch (e) {
+      // Guess the amount of real cores
+      cores = os
+        .cpus()
+        .filter(
+          (cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1,
+        ).length;
+    }
   }
 
   // Another fallback

package/src/index.js

@@ -1,9 +1,10 @@
 // @flow
-import type {LogEvent} from '@parcel/types';
+import type {TraceEvent, LogEvent} from '@parcel/types';
 import invariant from 'assert';
 import WorkerFarm from './WorkerFarm';
 import Logger from '@parcel/logger';
 import bus from './bus';
+import {tracer} from '@parcel/profiler';
 
 if (!WorkerFarm.isWorker()) {
   // Forward all logger events originating from workers into the main process
@@ -29,9 +30,14 @@ if (!WorkerFarm.isWorker()) {
         throw new Error('Unknown log level');
     }
   });
+
+  // Forward all trace events originating from workers into the main process
+  bus.on('traceEvent', (e: TraceEvent) => {
+    tracer.trace(e);
+  });
 }
 
 export default WorkerFarm;
 export {bus};
 export {Handle} from './WorkerFarm';
-export type {WorkerApi, FarmOptions} from './WorkerFarm';
+export type {WorkerApi, FarmOptions, SharedReference} from './WorkerFarm';

package/src/process/ProcessChild.js

@@ -25,7 +25,7 @@ export default class ProcessChild implements ChildImpl {
     process.on('message', data => this.handleMessage(data));
   }
 
-  handleMessage(data: string) {
+  handleMessage(data: string): void {
     if (data === 'die') {
       return this.stop();
     }
@@ -37,6 +37,7 @@ export default class ProcessChild implements ChildImpl {
     let processSend = nullthrows(process.send).bind(process);
     processSend(serialize(data).toString('base64'), err => {
       if (err && err instanceof Error) {
+        // $FlowFixMe[prop-missing]
         if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
           // IPC connection closed
           // no need to keep the worker running if it can't send or receive data

package/src/process/ProcessWorker.js

@@ -34,7 +34,7 @@ export default class ProcessWorker implements WorkerImpl {
     this.onExit = onExit;
   }
 
-  start() {
+  start(): Promise<void> {
     this.child = childProcess.fork(WORKER_PATH, process.argv, {
       execArgv: this.execArgv,
       env: process.env,

package/src/threads/ThreadsWorker.js

@@ -32,7 +32,7 @@ export default class ThreadsWorker implements WorkerImpl {
     this.onExit = onExit;
   }
 
-  start() {
+  start(): Promise<void> {
     this.worker = new Worker(WORKER_PATH, {
       execArgv: this.execArgv,
       env: process.env,
@@ -47,7 +47,7 @@ export default class ThreadsWorker implements WorkerImpl {
     });
   }
 
-  stop() {
+  stop(): Promise<void> {
     // In node 12, this returns a promise, but previously it accepted a callback
     // TODO: Pass a callback in earlier versions of Node
     return Promise.resolve(this.worker.terminate());

package/src/types.js

@@ -65,4 +65,4 @@ export interface ChildImpl {
   send(data: WorkerMessage): void;
 }
 
-export type BackendType = 'threads' | 'process';
+export type BackendType = 'threads' | 'process' | 'web';

package/src/web/WebChild.js

@@ -0,0 +1,50 @@
+// @flow
+/* eslint-env worker*/
+
+import type {
+  ChildImpl,
+  MessageHandler,
+  ExitHandler,
+  WorkerMessage,
+} from '../types';
+import {setChild} from '../childState';
+import {Child} from '../child';
+import {prepareForSerialization, restoreDeserializedObject} from '@parcel/core';
+
+export default class WebChild implements ChildImpl {
+  onMessage: MessageHandler;
+  onExit: ExitHandler;
+
+  constructor(onMessage: MessageHandler, onExit: ExitHandler) {
+    if (
+      !(
+        typeof WorkerGlobalScope !== 'undefined' &&
+        self instanceof WorkerGlobalScope
+      )
+    ) {
+      throw new Error('Only create WebChild instances in a worker!');
+    }
+
+    this.onMessage = onMessage;
+    this.onExit = onExit;
+    self.addEventListener('message', ({data}: MessageEvent) => {
+      if (data === 'stop') {
+        this.onExit(0);
+        self.postMessage('stopped');
+      }
+      // $FlowFixMe assume WorkerMessage as data
+      this.handleMessage(data);
+    });
+    self.postMessage('online');
+  }
+
+  handleMessage(data: WorkerMessage) {
+    this.onMessage(restoreDeserializedObject(data));
+  }
+
+  send(data: WorkerMessage) {
+    self.postMessage(prepareForSerialization(data));
+  }
+}
+
+setChild(new Child(WebChild));

package/src/web/WebWorker.js

@@ -0,0 +1,85 @@
+// @flow
+
+import type {
+  WorkerImpl,
+  MessageHandler,
+  ErrorHandler,
+  ExitHandler,
+  WorkerMessage,
+} from '../types';
+import {prepareForSerialization, restoreDeserializedObject} from '@parcel/core';
+import {makeDeferredWithPromise} from '@parcel/utils';
+
+let id = 0;
+
+export default class WebWorker implements WorkerImpl {
+  execArgv: Object;
+  onMessage: MessageHandler;
+  onError: ErrorHandler;
+  onExit: ExitHandler;
+  worker: Worker;
+  stopping: ?Promise<void>;
+
+  constructor(
+    execArgv: Object,
+    onMessage: MessageHandler,
+    onError: ErrorHandler,
+    onExit: ExitHandler,
+  ) {
+    this.execArgv = execArgv;
+    this.onMessage = onMessage;
+    this.onError = onError;
+    this.onExit = onExit;
+  }
+
+  start(): Promise<void> {
+    // $FlowFixMe[incompatible-call]
+    this.worker = new Worker(new URL('./WebChild.js', import.meta.url), {
+      name: `Parcel Worker ${id++}`,
+      type: 'module',
+    });
+
+    let {deferred, promise} = makeDeferredWithPromise();
+
+    this.worker.onmessage = ({data}) => {
+      if (data === 'online') {
+        deferred.resolve();
+        return;
+      }
+
+      // $FlowFixMe assume WorkerMessage as data
+      this.handleMessage(data);
+    };
+    this.worker.onerror = this.onError;
+    // Web workers can't crash or intentionally stop on their own, apart from stop() below
+    // this.worker.on('exit', this.onExit);
+
+    return promise;
+  }
+
+  stop(): Promise<void> {
+    if (!this.stopping) {
+      this.stopping = (async () => {
+        this.worker.postMessage('stop');
+        let {deferred, promise} = makeDeferredWithPromise();
+        this.worker.addEventListener('message', ({data}: MessageEvent) => {
+          if (data === 'stopped') {
+            deferred.resolve();
+          }
+        });
+        await promise;
+        this.worker.terminate();
+        this.onExit(0);
+      })();
+    }
+    return this.stopping;
+  }
+
+  handleMessage(data: WorkerMessage) {
+    this.onMessage(restoreDeserializedObject(data));
+  }
+
+  send(data: WorkerMessage) {
+    this.worker.postMessage(prepareForSerialization(data));
+  }
+}

package/test/cpuCount.test.js

@@ -3,7 +3,7 @@ import os from 'os';
 
 import getCores, {detectRealCores} from '../src/cpuCount';
 
-describe('cpuCount', function() {
+describe('cpuCount', function () {
   it('Should be able to detect real cpu count', () => {
     // Windows not supported as getting the cpu count takes a couple seconds...
     if (os.platform() === 'win32') return;

package/test/integration/workerfarm/console.js

@@ -1,4 +1,4 @@
-const WorkerFarm = require('../../../').default;
+const WorkerFarm = require('../../../src/WorkerFarm').default;
 
 function run() {
   if (WorkerFarm.isWorker()) {

package/test/integration/workerfarm/logging.js

@@ -1,4 +1,4 @@
-const WorkerFarm = require('../../../').default;
+const WorkerFarm = require('../../../src/WorkerFarm').default;
 const Logger = require('@parcel/logger').default;
 
 function run() {

package/test/integration/workerfarm/reverse-handle.js

@@ -1,5 +1,5 @@
-function run(_, handle) {
-  return handle();
+function run(workerApi, handle) {
+  return workerApi.runHandle(handle, []);
 }
 
 exports.run = run;

package/test/workerfarm.js

@@ -1,8 +1,8 @@
 import Logger from '@parcel/logger';
 import assert from 'assert';
-import WorkerFarm from '../';
+import WorkerFarm from '../src';
 
-describe('WorkerFarm', function() {
+describe('WorkerFarm', function () {
   this.timeout(30000);
 
   it('Should start up workers', async () => {
@@ -132,7 +132,7 @@ describe('WorkerFarm', function() {
     await workerfarm.end();
   });
 
-  it('Forwards stdio from the child process and levels event source if patchConsole is true', async () => {
+  it('Forwards stdio from the child process and levels event source if shouldPatchConsole is true', async () => {
     let events = [];
     let logDisposable = Logger.onLog(event => events.push(event));
 
@@ -140,7 +140,7 @@ describe('WorkerFarm', function() {
       warmWorkers: true,
       useLocalWorker: false,
       workerPath: require.resolve('./integration/workerfarm/console.js'),
-      patchConsole: true,
+      shouldPatchConsole: true,
     });
 
     await workerfarm.run();
@@ -291,7 +291,7 @@ describe('WorkerFarm', function() {
     assert.equal(result, 'Shared reference does not exist');
   });
 
-  it('should resolve shared references in workers', async () => {
+  it('Should resolve shared references in workers', async () => {
     let workerfarm = new WorkerFarm({
       warmWorkers: true,
       useLocalWorker: false,

package/lib/Profiler.js

@@ -1,70 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-
-var _assert = _interopRequireDefault(require("assert"));
-
-var _diagnostic = _interopRequireDefault(require("@parcel/diagnostic"));
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
-class Profiler {
-  constructor() {
-    _defineProperty(this, "session", void 0);
-  }
-
-  startProfiling() {
-    let inspector;
-
-    try {
-      inspector = require('inspector');
-    } catch (err) {
-      throw new _diagnostic.default({
-        diagnostic: {
-          message: `The inspector module isn't available`,
-          origin: '@parcel/workers',
-          hints: ['Disable build profiling']
-        }
-      });
-    }
-
-    this.session = new inspector.Session();
-    this.session.connect();
-    return Promise.all([this.sendCommand('Profiler.setSamplingInterval', {
-      interval: 100
-    }), this.sendCommand('Profiler.enable'), this.sendCommand('Profiler.start')]);
-  }
-
-  sendCommand(method, params) {
-    (0, _assert.default)(this.session != null);
-    return new Promise((resolve, reject) => {
-      this.session.post(method, params, (err, params) => {
-        if (err == null) {
-          resolve(params);
-        } else {
-          reject(err);
-        }
-      });
-    });
-  }
-
-  destroy() {
-    if (this.session != null) {
-      this.session.disconnect();
-    }
-  }
-
-  async stopProfiling() {
-    let res = await this.sendCommand('Profiler.stop');
-    this.destroy();
-    return res.profile;
-  }
-
-}
-
-exports.default = Profiler;