@atlaspack/workers 2.12.1-canary.3354

package/src/backend.js

@@ -0,0 +1,33 @@
+// @flow
+import type {BackendType, WorkerImpl} from './types';
+
+export function detectBackend(): BackendType {
+  // $FlowFixMe
+  if (process.browser) return 'web';
+
+  switch (process.env.ATLASPACK_WORKER_BACKEND) {
+    case 'threads':
+    case 'process':
+      return process.env.ATLASPACK_WORKER_BACKEND;
+  }
+
+  try {
+    require('worker_threads');
+    return 'threads';
+  } catch (err) {
+    return 'process';
+  }
+}
+
+export function getWorkerBackend(backend: BackendType): Class<WorkerImpl> {
+  switch (backend) {
+    case 'threads':
+      return require('./threads/ThreadsWorker').default;
+    case 'process':
+      return require('./process/ProcessWorker').default;
+    case 'web':
+      return require('./web/WebWorker').default;
+    default:
+      throw new Error(`Invalid backend: ${backend}`);
+  }
+}

package/src/bus.js

@@ -0,0 +1,26 @@
+// @flow
+import EventEmitter from 'events';
+import {child} from './childState';
+
+class Bus extends EventEmitter {
+  emit(event: string, ...args: Array<any>): boolean {
+    if (child) {
+      child.workerApi.callMaster(
+        {
+          // $FlowFixMe
+          location: process.browser
+            ? '@atlaspack/workers/src/bus.js'
+            : __filename,
+          method: 'emit',
+          args: [event, ...args],
+        },
+        false,
+      );
+      return true;
+    } else {
+      return super.emit(event, ...args);
+    }
+  }
+}
+
+export default (new Bus(): Bus);

package/src/child.js

@@ -0,0 +1,322 @@
+// @flow
+
+import type {
+  CallRequest,
+  WorkerDataResponse,
+  WorkerErrorResponse,
+  WorkerMessage,
+  WorkerRequest,
+  WorkerResponse,
+  ChildImpl,
+} from './types';
+import type {Async, IDisposable} from '@atlaspack/types-internal';
+import type {SharedReference} from './WorkerFarm';
+
+import * as coreWorker from './core-worker';
+import invariant from 'assert';
+import nullthrows from 'nullthrows';
+import Logger, {patchConsole, unpatchConsole} from '@atlaspack/logger';
+import ThrowableDiagnostic, {anyToDiagnostic} from '@atlaspack/diagnostic';
+import {deserialize} from '@atlaspack/core';
+import bus from './bus';
+import {SamplingProfiler, tracer} from '@atlaspack/profiler';
+import _Handle from './Handle';
+
+// The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
+const Handle = _Handle;
+
+type ChildCall = WorkerRequest & {|
+  resolve: (result: Promise<any> | any) => void,
+  reject: (error: any) => void,
+|};
+
+export class Child {
+  callQueue: Array<ChildCall> = [];
+  childId: ?number;
+  maxConcurrentCalls: number = 10;
+  module: ?any;
+  responseId: number = 0;
+  responseQueue: Map<number, ChildCall> = new Map();
+  loggerDisposable: IDisposable;
+  tracerDisposable: IDisposable;
+  child: ChildImpl;
+  profiler: ?SamplingProfiler;
+  handles: Map<number, Handle> = new Map();
+  sharedReferences: Map<SharedReference, mixed> = new Map();
+  sharedReferencesByValue: Map<mixed, SharedReference> = new Map();
+
+  constructor(ChildBackend: Class<ChildImpl>) {
+    this.child = new ChildBackend(
+      m => {
+        this.messageListener(m);
+      },
+      () => this.handleEnd(),
+    );
+
+    // Monitior all logging events inside this child process and forward to
+    // the main process via the bus.
+    this.loggerDisposable = Logger.onLog(event => {
+      bus.emit('logEvent', event);
+    });
+    // .. and do the same for trace events
+    this.tracerDisposable = tracer.onTrace(event => {
+      bus.emit('traceEvent', event);
+    });
+  }
+
+  workerApi: {|
+    callMaster: (
+      request: CallRequest,
+      awaitResponse?: ?boolean,
+    ) => Promise<mixed>,
+    createReverseHandle: (fn: (...args: Array<any>) => mixed) => Handle,
+    getSharedReference: (ref: SharedReference) => mixed,
+    resolveSharedReference: (value: mixed) => void | SharedReference,
+    runHandle: (handle: Handle, args: Array<any>) => Promise<mixed>,
+  |} = {
+    callMaster: (
+      request: CallRequest,
+      awaitResponse: ?boolean = true,
+    ): Promise<mixed> => this.addCall(request, awaitResponse),
+    createReverseHandle: (fn: (...args: Array<any>) => mixed): Handle =>
+      this.createReverseHandle(fn),
+    runHandle: (handle: Handle, args: Array<any>): Promise<mixed> =>
+      this.workerApi.callMaster({handle: handle.id, args}, true),
+    getSharedReference: (ref: SharedReference) =>
+      this.sharedReferences.get(ref),
+    resolveSharedReference: (value: mixed) =>
+      this.sharedReferencesByValue.get(value),
+  };
+
+  messageListener(message: WorkerMessage): Async<void> {
+    if (message.type === 'response') {
+      return this.handleResponse(message);
+    } else if (message.type === 'request') {
+      return this.handleRequest(message);
+    }
+  }
+
+  send(data: WorkerMessage): void {
+    this.child.send(data);
+  }
+
+  async childInit(module: string, childId: number): Promise<void> {
+    // $FlowFixMe
+    if (process.browser) {
+      if (module === '@atlaspack/core/src/worker.js') {
+        this.module = coreWorker;
+      } else {
+        throw new Error('No dynamic require possible: ' + module);
+      }
+    } else {
+      // $FlowFixMe this must be dynamic
+      this.module = require(module);
+    }
+    this.childId = childId;
+
+    if (this.module.childInit != null) {
+      await this.module.childInit();
+    }
+  }
+
+  async handleRequest(data: WorkerRequest): Promise<void> {
+    let {idx, method, args, handle: handleId} = data;
+    let child = nullthrows(data.child);
+
+    const responseFromContent = (content: any): WorkerDataResponse => ({
+      idx,
+      child,
+      type: 'response',
+      contentType: 'data',
+      content,
+    });
+
+    const errorResponseFromError = (e: Error): WorkerErrorResponse => ({
+      idx,
+      child,
+      type: 'response',
+      contentType: 'error',
+      content: anyToDiagnostic(e),
+    });
+
+    let result;
+    if (handleId != null) {
+      try {
+        let fn = nullthrows(this.handles.get(handleId)?.fn);
+        result = responseFromContent(fn(...args));
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'childInit') {
+      try {
+        let [moduleName, childOptions] = args;
+        if (childOptions.shouldPatchConsole) {
+          patchConsole();
+        } else {
+          unpatchConsole();
+        }
+
+        if (childOptions.shouldTrace) {
+          tracer.enable();
+        }
+
+        result = responseFromContent(await this.childInit(moduleName, child));
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'startProfile') {
+      this.profiler = new SamplingProfiler();
+      try {
+        result = responseFromContent(await this.profiler.startProfiling());
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'endProfile') {
+      try {
+        let res = this.profiler ? await this.profiler.stopProfiling() : null;
+        result = responseFromContent(res);
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'takeHeapSnapshot') {
+      try {
+        let v8 = require('v8');
+        result = responseFromContent(
+          v8.writeHeapSnapshot(
+            'heap-' +
+              args[0] +
+              '-' +
+              (this.childId ? 'worker' + this.childId : 'main') +
+              '.heapsnapshot',
+          ),
+        );
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    } else if (method === 'createSharedReference') {
+      let [ref, _value] = args;
+      let value =
+        _value instanceof ArrayBuffer
+          ? // In the case the value is pre-serialized as a buffer,
+            // deserialize it.
+            deserialize(Buffer.from(_value))
+          : _value;
+      this.sharedReferences.set(ref, value);
+      this.sharedReferencesByValue.set(value, ref);
+      result = responseFromContent(null);
+    } else if (method === 'deleteSharedReference') {
+      let ref = args[0];
+      let value = this.sharedReferences.get(ref);
+      this.sharedReferencesByValue.delete(value);
+      this.sharedReferences.delete(ref);
+      result = responseFromContent(null);
+    } else {
+      try {
+        result = responseFromContent(
+          // $FlowFixMe
+          await this.module[method](this.workerApi, ...args),
+        );
+      } catch (e) {
+        result = errorResponseFromError(e);
+      }
+    }
+
+    try {
+      this.send(result);
+    } catch (e) {
+      result = this.send(errorResponseFromError(e));
+    }
+  }
+
+  handleResponse(data: WorkerResponse): void {
+    let idx = nullthrows(data.idx);
+    let contentType = data.contentType;
+    let content = data.content;
+    let call = nullthrows(this.responseQueue.get(idx));
+
+    if (contentType === 'error') {
+      invariant(typeof content !== 'string');
+      call.reject(new ThrowableDiagnostic({diagnostic: content}));
+    } else {
+      call.resolve(content);
+    }
+
+    this.responseQueue.delete(idx);
+
+    // Process the next call
+    this.processQueue();
+  }
+
+  // Keep in mind to make sure responses to these calls are JSON.Stringify safe
+  addCall(
+    request: CallRequest,
+    awaitResponse: ?boolean = true,
+  ): Promise<mixed> {
+    // $FlowFixMe
+    let call: ChildCall = {
+      ...request,
+      type: 'request',
+      child: this.childId,
+      // $FlowFixMe Added in Flow 0.121.0 upgrade in #4381
+      awaitResponse,
+      resolve: () => {},
+      reject: () => {},
+    };
+
+    let promise;
+    if (awaitResponse) {
+      promise = new Promise((resolve, reject) => {
+        call.resolve = resolve;
+        call.reject = reject;
+      });
+    }
+
+    this.callQueue.push(call);
+    this.processQueue();
+
+    return promise ?? Promise.resolve();
+  }
+
+  sendRequest(call: ChildCall): void {
+    let idx;
+    if (call.awaitResponse) {
+      idx = this.responseId++;
+      this.responseQueue.set(idx, call);
+    }
+
+    this.send({
+      idx,
+      child: call.child,
+      type: call.type,
+      location: call.location,
+      handle: call.handle,
+      method: call.method,
+      args: call.args,
+      awaitResponse: call.awaitResponse,
+    });
+  }
+
+  processQueue(): void {
+    if (!this.callQueue.length) {
+      return;
+    }
+
+    if (this.responseQueue.size < this.maxConcurrentCalls) {
+      this.sendRequest(this.callQueue.shift());
+    }
+  }
+
+  handleEnd(): void {
+    this.loggerDisposable.dispose();
+    this.tracerDisposable.dispose();
+  }
+
+  createReverseHandle(fn: (...args: Array<any>) => mixed): Handle {
+    let handle = new Handle({
+      fn,
+      childId: this.childId,
+    });
+    this.handles.set(handle.id, handle);
+    return handle;
+  }
+}

package/src/childState.js

@@ -0,0 +1,10 @@
+// @flow
+import type {Child} from './child';
+
+// This file is imported by both the WorkerFarm and child implementation.
+// When a worker is inited, it sets the state in this file.
+// This way, WorkerFarm can access the state without directly importing the child code.
+export let child: ?Child = null;
+export function setChild(c: Child) {
+  child = c;
+}

package/src/core-worker.browser.js

@@ -0,0 +1,3 @@
+// @flow
+// eslint-disable-next-line monorepo/no-internal-import
+module.exports = require('@atlaspack/core/src/worker.js');

package/src/core-worker.js

@@ -0,0 +1,2 @@
+// This is used only in browser builds
+module.exports = {};

package/src/cpuCount.js

@@ -0,0 +1,75 @@
+// @flow
+import os from 'os';
+import {execSync} from 'child_process';
+
+const exec = (command: string): string => {
+  try {
+    let stdout = execSync(command, {
+      encoding: 'utf8',
+      // This prevents the command from outputting to the console
+      stdio: [null, null, null],
+    });
+    return stdout.trim();
+  } catch (e) {
+    return '';
+  }
+};
+
+export function detectRealCores(): number {
+  let platform = os.platform();
+  let amount = 0;
+
+  if (platform === 'linux') {
+    amount = parseInt(
+      exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l'),
+      10,
+    );
+  } else if (platform === 'darwin') {
+    amount = parseInt(exec('sysctl -n hw.physicalcpu_max'), 10);
+  } else if (platform === 'win32') {
+    const str = exec('wmic cpu get NumberOfCores').match(/\d+/g);
+    if (str !== null) {
+      amount = parseInt(str.filter(n => n !== '')[0], 10);
+    }
+  }
+
+  if (!amount || amount <= 0) {
+    throw new Error('Could not detect cpu count!');
+  }
+
+  return amount;
+}
+
+let cores;
+export default function getCores(bypassCache?: boolean = false): number {
+  // Do not re-run commands if we already have the count...
+  if (cores && !bypassCache) {
+    return cores;
+  }
+
+  // $FlowFixMe
+  if (process.browser) {
+    // eslint-disable-next-line no-undef
+    cores = navigator.hardwareConcurrency / 2;
+  }
+
+  if (!cores) {
+    try {
+      cores = detectRealCores();
+    } catch (e) {
+      // Guess the amount of real cores
+      cores = os
+        .cpus()
+        .filter(
+          (cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1,
+        ).length;
+    }
+  }
+
+  // Another fallback
+  if (!cores) {
+    cores = 1;
+  }
+
+  return cores;
+}

package/src/index.js

@@ -0,0 +1,43 @@
+// @flow
+import type {TraceEvent, LogEvent} from '@atlaspack/types-internal';
+import invariant from 'assert';
+import WorkerFarm from './WorkerFarm';
+import Logger from '@atlaspack/logger';
+import bus from './bus';
+import {tracer} from '@atlaspack/profiler';
+
+if (!WorkerFarm.isWorker()) {
+  // Forward all logger events originating from workers into the main process
+  bus.on('logEvent', (e: LogEvent) => {
+    switch (e.level) {
+      case 'info':
+        Logger.info(e.diagnostics);
+        break;
+      case 'progress':
+        invariant(typeof e.message === 'string');
+        Logger.progress(e.message);
+        break;
+      case 'verbose':
+        Logger.verbose(e.diagnostics);
+        break;
+      case 'warn':
+        Logger.warn(e.diagnostics);
+        break;
+      case 'error':
+        Logger.error(e.diagnostics);
+        break;
+      default:
+        throw new Error('Unknown log level');
+    }
+  });
+
+  // Forward all trace events originating from workers into the main process
+  bus.on('traceEvent', (e: TraceEvent) => {
+    tracer.trace(e);
+  });
+}
+
+export default WorkerFarm;
+export {bus};
+export {Handle} from './WorkerFarm';
+export type {WorkerApi, FarmOptions, SharedReference} from './WorkerFarm';

package/src/process/ProcessChild.js

@@ -0,0 +1,56 @@
+// @flow
+
+import type {
+  ChildImpl,
+  MessageHandler,
+  ExitHandler,
+  WorkerMessage,
+} from '../types';
+import nullthrows from 'nullthrows';
+import {setChild} from '../childState';
+import {Child} from '../child';
+import {serialize, deserialize} from '@atlaspack/core';
+
+export default class ProcessChild implements ChildImpl {
+  onMessage: MessageHandler;
+  onExit: ExitHandler;
+
+  constructor(onMessage: MessageHandler, onExit: ExitHandler) {
+    if (!process.send) {
+      throw new Error('Only create ProcessChild instances in a worker!');
+    }
+
+    this.onMessage = onMessage;
+    this.onExit = onExit;
+    process.on('message', data => this.handleMessage(data));
+  }
+
+  handleMessage(data: string): void {
+    if (data === 'die') {
+      return this.stop();
+    }
+
+    this.onMessage(deserialize(Buffer.from(data, 'base64')));
+  }
+
+  send(data: WorkerMessage) {
+    let processSend = nullthrows(process.send).bind(process);
+    processSend(serialize(data).toString('base64'), err => {
+      if (err && err instanceof Error) {
+        // $FlowFixMe[prop-missing]
+        if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
+          // IPC connection closed
+          // no need to keep the worker running if it can't send or receive data
+          return this.stop();
+        }
+      }
+    });
+  }
+
+  stop() {
+    this.onExit(0);
+    process.exit();
+  }
+}
+
+setChild(new Child(ProcessChild));

package/src/process/ProcessWorker.js

@@ -0,0 +1,91 @@
+// @flow
+
+import type {
+  WorkerImpl,
+  MessageHandler,
+  ErrorHandler,
+  ExitHandler,
+  WorkerMessage,
+} from '../types';
+import childProcess, {type ChildProcess} from 'child_process';
+import path from 'path';
+import {serialize, deserialize} from '@atlaspack/core';
+
+const WORKER_PATH = path.join(__dirname, 'ProcessChild.js');
+
+export default class ProcessWorker implements WorkerImpl {
+  execArgv: Object;
+  onMessage: MessageHandler;
+  onError: ErrorHandler;
+  onExit: ExitHandler;
+  child: ChildProcess;
+  processQueue: boolean = true;
+  sendQueue: Array<any> = [];
+
+  constructor(
+    execArgv: Object,
+    onMessage: MessageHandler,
+    onError: ErrorHandler,
+    onExit: ExitHandler,
+  ) {
+    this.execArgv = execArgv;
+    this.onMessage = onMessage;
+    this.onError = onError;
+    this.onExit = onExit;
+  }
+
+  start(): Promise<void> {
+    this.child = childProcess.fork(WORKER_PATH, process.argv, {
+      execArgv: this.execArgv,
+      env: process.env,
+      cwd: process.cwd(),
+    });
+
+    this.child.on('message', (data: string) => {
+      this.onMessage(deserialize(Buffer.from(data, 'base64')));
+    });
+
+    this.child.once('exit', this.onExit);
+    this.child.on('error', this.onError);
+
+    return Promise.resolve();
+  }
+
+  async stop() {
+    this.child.send('die');
+
+    let forceKill = setTimeout(() => this.child.kill('SIGINT'), 500);
+    await new Promise(resolve => {
+      this.child.once('exit', resolve);
+    });
+
+    clearTimeout(forceKill);
+  }
+
+  send(data: WorkerMessage) {
+    if (!this.processQueue) {
+      this.sendQueue.push(data);
+      return;
+    }
+
+    let result = this.child.send(serialize(data).toString('base64'), error => {
+      if (error && error instanceof Error) {
+        // Ignore this, the workerfarm handles child errors
+        return;
+      }
+
+      this.processQueue = true;
+
+      if (this.sendQueue.length > 0) {
+        let queueCopy = this.sendQueue.slice(0);
+        this.sendQueue = [];
+        queueCopy.forEach(entry => this.send(entry));
+      }
+    });
+
+    if (!result || /^win/.test(process.platform)) {
+      // Queue is handling too much messages throttle it
+      this.processQueue = false;
+    }
+  }
+}