@atlaspack/core 2.12.1-canary.3486 → 2.12.1-canary.3488

package/lib/atlaspack-v3/AtlaspackV3.js

@@ -4,20 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.AtlaspackV3 = void 0;
-function _path() {
-  const data = _interopRequireDefault(require("path"));
-  _path = function () {
-    return data;
-  };
-  return data;
-}
-function _worker_threads() {
-  const data = require("worker_threads");
-  _worker_threads = function () {
-    return data;
-  };
-  return data;
-}
+var _WorkerPool = require("./WorkerPool");
 function _rust() {
   const data = require("@atlaspack/rust");
   _rust = function () {
@@ -25,8 +12,6 @@ function _rust() {
   };
   return data;
 }
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-const WORKER_PATH = _path().default.join(__dirname, 'worker', 'index.js');
 class AtlaspackV3 {
   constructor({
     fs,
@@ -50,23 +35,16 @@ class AtlaspackV3 {
     });
   }
   async buildAssetGraph() {
-    const [workers, registerWorker] = this.#createWorkers();
+    const workerIds = [];
     let result = await this._internal.buildAssetGraph({
-      registerWorker
+      registerWorker: tx_worker => {
+        // $FlowFixMe
+        const workerId = _WorkerPool.workerPool.registerWorker(tx_worker);
+        workerIds.push(workerId);
+      }
     });
-    for (const worker of workers) worker.terminate();
+    _WorkerPool.workerPool.releaseWorkers(workerIds);
     return result;
   }
-  #createWorkers() {
-    const workers = [];
-    return [workers, tx_worker => {
-      let worker = new (_worker_threads().Worker)(WORKER_PATH, {
-        workerData: {
-          tx_worker
-        }
-      });
-      workers.push(worker);
-    }];
-  }
 }
 exports.AtlaspackV3 = AtlaspackV3;

package/lib/atlaspack-v3/WorkerPool.js

@@ -0,0 +1,136 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.WorkerPool = void 0;
+exports.waitForMessage = waitForMessage;
+exports.workerPool = void 0;
+function _path() {
+  const data = _interopRequireDefault(require("path"));
+  _path = function () {
+    return data;
+  };
+  return data;
+}
+function _worker_threads() {
+  const data = require("worker_threads");
+  _worker_threads = function () {
+    return data;
+  };
+  return data;
+}
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+/*!
+ * Atlaspack V3 delegates work to node.js worker threads.
+ *
+ * Starting-up each worker is relatively expensive, in particular when atlaspack
+ * is running in development mode, in which case each worker will transpile the
+ * project on startup.
+ *
+ * This "WorkerPool" mitigates this problem by reusing worker threads across
+ * builds.
+ */
+const WORKER_PATH = _path().default.join(__dirname, 'worker', 'index.js');
+function waitForMessage(worker, type) {
+  return new Promise(resolve => {
+    const onMessage = message => {
+      if (message.type === type) {
+        resolve(message);
+        worker.off('message', onMessage);
+      }
+    };
+    worker.on('message', onMessage);
+  });
+}
+class WorkerPool {
+  #workerPool = [];
+  #usedWorkers = new Set();
+  #workerPath;
+  constructor(workerPath = WORKER_PATH) {
+    this.#workerPath = workerPath;
+
+    // $FlowFixMe
+    this[Symbol.dispose] = () => {
+      this.shutdown();
+    };
+  }
+
+  /**
+   * Find a worker thread that is free to use or create a new one.
+   *
+   * Then register the `tx_worker` channel ID with the worker thread.
+   */
+  registerWorker(tx_worker) {
+    const availableIndex = this.#workerPool.findIndex((worker, index) => !this.#usedWorkers.has(index));
+    const [workerId, worker] = availableIndex !== -1 ? [availableIndex, this.#workerPool[availableIndex]] : this.#createWorker();
+    this.#bootWorker(worker, tx_worker).catch(err => {
+      // eslint-disable-next-line no-console
+      console.error('Worker failed, retrying to create it...', err);
+      this.#workerPool[workerId] = new (_worker_threads().Worker)(WORKER_PATH, {});
+      this.#bootWorker(this.#workerPool[workerId], tx_worker).catch(err => {
+        // eslint-disable-next-line no-console
+        console.error('Worker failed to start. The build may hang', err);
+      });
+    });
+    this.#usedWorkers.add(workerId);
+    return workerId;
+  }
+
+  /**
+   * Release a set of workers back into the pool for re-use
+   */
+  releaseWorkers(ids) {
+    for (let id of ids) {
+      this.#usedWorkers.delete(id);
+    }
+  }
+
+  /**
+   * Terminate all worker threads and reset state.
+   */
+  shutdown() {
+    for (let worker of this.#workerPool) {
+      worker.terminate();
+    }
+    this.#usedWorkers.clear();
+    this.#workerPool = [];
+  }
+  getStats() {
+    return {
+      totalWorkers: this.#workerPool.length,
+      workersInUse: this.#usedWorkers.size
+    };
+  }
+
+  /**
+   * Get the worker thread. Used for testing.
+   */
+  getWorker(workerId) {
+    if (!this.#workerPool[workerId]) {
+      throw new Error('Worker does not exist');
+    }
+    return this.#workerPool[workerId];
+  }
+  async #bootWorker(worker, tx_worker) {
+    const timeout = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error('Worker failed to register in time'));
+      }, 5000);
+    });
+    const workerReady = waitForMessage(worker, 'workerRegistered');
+    worker.postMessage({
+      type: 'registerWorker',
+      tx_worker
+    });
+    await Promise.race([workerReady, timeout]);
+  }
+  #createWorker() {
+    const worker = new (_worker_threads().Worker)(this.#workerPath, {});
+    const workerId = this.#workerPool.length;
+    this.#workerPool.push(worker);
+    return [workerId, worker];
+  }
+}
+exports.WorkerPool = WorkerPool;
+const workerPool = exports.workerPool = new WorkerPool();

package/lib/atlaspack-v3/worker/worker.js

@@ -265,4 +265,29 @@ class AtlaspackWorker {
   });
 }
 exports.AtlaspackWorker = AtlaspackWorker;
-napi().registerWorker(_worker_threads().workerData.tx_worker, new AtlaspackWorker());
+const worker = new AtlaspackWorker();
+_worker_threads().parentPort === null || _worker_threads().parentPort === void 0 || _worker_threads().parentPort.on('message', event => {
+  if (event.type === 'registerWorker') {
+    try {
+      napi().registerWorker(event.tx_worker, worker);
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.error('Registering worker failed... This might mean atlaspack is getting shut-down before the worker registered', err);
+      _worker_threads().parentPort === null || _worker_threads().parentPort === void 0 || _worker_threads().parentPort.postMessage({
+        type: 'workerError',
+        error: err
+      });
+    }
+    _worker_threads().parentPort === null || _worker_threads().parentPort === void 0 || _worker_threads().parentPort.postMessage({
+      type: 'workerRegistered'
+    });
+  } else if (event.type === 'probeStatus') {
+    _worker_threads().parentPort.postMessage({
+      type: 'status',
+      status: 'ok'
+    });
+  }
+});
+_worker_threads().parentPort === null || _worker_threads().parentPort === void 0 || _worker_threads().parentPort.postMessage({
+  type: 'workerLoaded'
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atlaspack/core",
-  "version": "2.12.1-canary.3486+ed60801bc",
+  "version": "2.12.1-canary.3488+bacf813b1",
   "license": "(MIT OR Apache-2.0)",
   "publishConfig": {
     "access": "public"
@@ -20,21 +20,21 @@
     "check-ts": "tsc --noEmit index.d.ts"
   },
   "dependencies": {
-    "@atlaspack/build-cache": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/cache": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/diagnostic": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/events": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/feature-flags": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/fs": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/graph": "3.2.1-canary.3486+ed60801bc",
-    "@atlaspack/logger": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/package-manager": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/plugin": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/profiler": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/rust": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/types": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/utils": "2.12.1-canary.3486+ed60801bc",
-    "@atlaspack/workers": "2.12.1-canary.3486+ed60801bc",
+    "@atlaspack/build-cache": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/cache": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/diagnostic": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/events": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/feature-flags": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/fs": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/graph": "3.2.1-canary.3488+bacf813b1",
+    "@atlaspack/logger": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/package-manager": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/plugin": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/profiler": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/rust": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/types": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/utils": "2.12.1-canary.3488+bacf813b1",
+    "@atlaspack/workers": "2.12.1-canary.3488+bacf813b1",
     "@mischnic/json-sourcemap": "^0.1.0",
     "@parcel/source-map": "^2.1.1",
     "base-x": "^3.0.8",
@@ -65,5 +65,5 @@
   "browser": {
     "./src/serializerCore.js": "./src/serializerCore.browser.js"
   },
-  "gitHead": "ed60801bcda8951391d16e375ecb0e33d0d5ad15"
+  "gitHead": "bacf813b1fd2c475991643425e6e1d1545791424"
 }

package/src/atlaspack-v3/AtlaspackV3.js

@@ -1,11 +1,8 @@
 // @flow
 
-import path from 'path';
-import {Worker} from 'worker_threads';
+import {workerPool} from './WorkerPool';
 import {AtlaspackNapi, type AtlaspackNapiOptions} from '@atlaspack/rust';
 
-const WORKER_PATH = path.join(__dirname, 'worker', 'index.js');
-
 export type AtlaspackV3Options = {|
   fs?: AtlaspackNapiOptions['fs'],
   nodeWorkers?: number,
@@ -42,29 +39,17 @@ export class AtlaspackV3 {
   }
 
   async buildAssetGraph(): Promise<any> {
-    const [workers, registerWorker] = this.#createWorkers();
-
+    const workerIds = [];
     let result = await this._internal.buildAssetGraph({
-      registerWorker,
+      registerWorker: (tx_worker) => {
+        // $FlowFixMe
+        const workerId = workerPool.registerWorker(tx_worker);
+        workerIds.push(workerId);
+      },
     });
 
-    for (const worker of workers) worker.terminate();
-    return result;
-  }
-
-  #createWorkers() {
-    const workers = [];
+    workerPool.releaseWorkers(workerIds);
 
-    return [
-      workers,
-      (tx_worker) => {
-        let worker = new Worker(WORKER_PATH, {
-          workerData: {
-            tx_worker,
-          },
-        });
-        workers.push(worker);
-      },
-    ];
+    return result;
   }
 }

package/src/atlaspack-v3/WorkerPool.js

@@ -0,0 +1,129 @@
+// @flow strict-local
+/*!
+ * Atlaspack V3 delegates work to node.js worker threads.
+ *
+ * Starting-up each worker is relatively expensive, in particular when atlaspack
+ * is running in development mode, in which case each worker will transpile the
+ * project on startup.
+ *
+ * This "WorkerPool" mitigates this problem by reusing worker threads across
+ * builds.
+ */
+import path from 'path';
+import {Worker} from 'worker_threads';
+
+const WORKER_PATH = path.join(__dirname, 'worker', 'index.js');
+
+export function waitForMessage<T>(worker: Worker, type: string): Promise<T> {
+  return new Promise((resolve) => {
+    const onMessage = (message: T & {|type: string|}) => {
+      if (message.type === type) {
+        resolve(message);
+        worker.off('message', onMessage);
+      }
+    };
+    worker.on('message', onMessage);
+  });
+}
+
+export class WorkerPool {
+  #workerPool: Worker[] = [];
+  #usedWorkers: Set<number> = new Set();
+  #workerPath: string;
+
+  constructor(workerPath: string = WORKER_PATH) {
+    this.#workerPath = workerPath;
+
+    // $FlowFixMe
+    this[Symbol.dispose] = () => {
+      this.shutdown();
+    };
+  }
+
+  /**
+   * Find a worker thread that is free to use or create a new one.
+   *
+   * Then register the `tx_worker` channel ID with the worker thread.
+   */
+  registerWorker(tx_worker: number): number {
+    const availableIndex = this.#workerPool.findIndex(
+      (worker, index) => !this.#usedWorkers.has(index),
+    );
+
+    const [workerId, worker] =
+      availableIndex !== -1
+        ? [availableIndex, this.#workerPool[availableIndex]]
+        : this.#createWorker();
+
+    this.#bootWorker(worker, tx_worker).catch((err) => {
+      // eslint-disable-next-line no-console
+      console.error('Worker failed, retrying to create it...', err);
+      this.#workerPool[workerId] = new Worker(WORKER_PATH, {});
+      this.#bootWorker(this.#workerPool[workerId], tx_worker).catch((err) => {
+        // eslint-disable-next-line no-console
+        console.error('Worker failed to start. The build may hang', err);
+      });
+    });
+
+    this.#usedWorkers.add(workerId);
+
+    return workerId;
+  }
+
+  /**
+   * Release a set of workers back into the pool for re-use
+   */
+  releaseWorkers(ids: number[]) {
+    for (let id of ids) {
+      this.#usedWorkers.delete(id);
+    }
+  }
+
+  /**
+   * Terminate all worker threads and reset state.
+   */
+  shutdown() {
+    for (let worker of this.#workerPool) {
+      worker.terminate();
+    }
+    this.#usedWorkers.clear();
+    this.#workerPool = [];
+  }
+
+  getStats(): {|totalWorkers: number, workersInUse: number|} {
+    return {
+      totalWorkers: this.#workerPool.length,
+      workersInUse: this.#usedWorkers.size,
+    };
+  }
+
+  /**
+   * Get the worker thread. Used for testing.
+   */
+  getWorker(workerId: number): Worker {
+    if (!this.#workerPool[workerId]) {
+      throw new Error('Worker does not exist');
+    }
+    return this.#workerPool[workerId];
+  }
+
+  async #bootWorker(worker: Worker, tx_worker: number): Promise<void> {
+    const timeout = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error('Worker failed to register in time'));
+      }, 5000);
+    });
+    const workerReady = waitForMessage(worker, 'workerRegistered');
+    worker.postMessage({type: 'registerWorker', tx_worker});
+    await Promise.race([workerReady, timeout]);
+  }
+
+  #createWorker(): [number, Worker] {
+    const worker = new Worker(this.#workerPath, {});
+    const workerId = this.#workerPool.length;
+    this.#workerPool.push(worker);
+    return [workerId, worker];
+  }
+}
+
+export const workerPool: WorkerPool = new WorkerPool();

package/src/atlaspack-v3/worker/worker.js

@@ -10,7 +10,7 @@ import type {
   FilePath,
   FileSystem,
 } from '@atlaspack/types';
-import {workerData} from 'worker_threads';
+import {parentPort} from 'worker_threads';
 import * as module from 'module';
 
 import {
@@ -273,7 +273,28 @@ export class AtlaspackWorker {
   });
 }
 
-napi.registerWorker(workerData.tx_worker, new AtlaspackWorker());
+const worker = new AtlaspackWorker();
+parentPort?.on('message', (event) => {
+  if (event.type === 'registerWorker') {
+    try {
+      napi.registerWorker(event.tx_worker, worker);
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.error(
+        'Registering worker failed... This might mean atlaspack is getting shut-down before the worker registered',
+        err,
+      );
+      parentPort?.postMessage({type: 'workerError', error: err});
+    }
+    parentPort?.postMessage({type: 'workerRegistered'});
+  } else if (event.type === 'probeStatus') {
+    parentPort.postMessage({
+      type: 'status',
+      status: 'ok',
+    });
+  }
+});
+parentPort?.postMessage({type: 'workerLoaded'});
 
 type ResolverState<T> = {|
   resolver: Resolver<T>,

package/test/atlaspack-v3/WorkerPool.test.js

@@ -0,0 +1,121 @@
+// @flow strict-local
+
+import path from 'path';
+import {Worker} from 'worker_threads';
+import {WorkerPool, waitForMessage} from '../../src/atlaspack-v3/WorkerPool';
+import assert from 'assert';
+
+function probeStatus(worker: Worker) {
+  const response = waitForMessage(worker, 'status');
+  worker.postMessage({type: 'probeStatus'});
+  return response;
+}
+
+describe('WorkerPool', () => {
+  it('can create workers and will send them the tx_worker value', async () => {
+    const workerPool = new WorkerPool(path.join(__dirname, 'worker.js'));
+    const workerId = workerPool.registerWorker(0);
+    const worker = workerPool.getWorker(workerId);
+    const status = await probeStatus(worker);
+
+    assert.deepEqual(status, {
+      type: 'status',
+      status: 'test-status-ok',
+      receivedMessages: [
+        {
+          type: 'registerWorker',
+          tx_worker: 0,
+        },
+      ],
+    });
+  });
+
+  it('when a worker is created, it is tracked', () => {
+    const workerPool = new WorkerPool(path.join(__dirname, 'worker.js'));
+    const w1 = workerPool.registerWorker(0);
+    const w2 = workerPool.registerWorker(0);
+    const w3 = workerPool.registerWorker(0);
+    assert.notEqual(w1, w2);
+    assert.notEqual(w2, w3);
+    assert.notEqual(w1, w3);
+
+    assert.deepEqual(workerPool.getStats(), {
+      totalWorkers: 3,
+      workersInUse: 3,
+    });
+  });
+
+  it('workers can be released and will then be reused', async () => {
+    const workerPool = new WorkerPool(path.join(__dirname, 'worker.js'));
+    const worker1 = workerPool.registerWorker(0);
+    workerPool.registerWorker(5);
+    assert.deepEqual(workerPool.getStats(), {
+      totalWorkers: 2,
+      workersInUse: 2,
+    });
+
+    // Release the worker
+    workerPool.releaseWorkers([worker1]);
+    assert.deepEqual(workerPool.getStats(), {
+      totalWorkers: 2,
+      workersInUse: 1,
+    });
+
+    const worker3 = workerPool.registerWorker(33);
+    assert.equal(worker1, worker3);
+
+    const worker = workerPool.getWorker(worker3);
+    const status = await probeStatus(worker);
+    assert.deepEqual(status, {
+      type: 'status',
+      status: 'test-status-ok',
+      receivedMessages: [
+        {
+          type: 'registerWorker',
+          tx_worker: 0,
+        },
+        {
+          type: 'registerWorker',
+          tx_worker: 33,
+        },
+      ],
+    });
+  });
+
+  describe('shutdown', () => {
+    it('terminates all workers', async () => {
+      const workerPool = new WorkerPool(path.join(__dirname, 'worker.js'));
+      const worker1Id = workerPool.registerWorker(0);
+      const worker2Id = workerPool.registerWorker(0);
+      const worker1 = workerPool.getWorker(worker1Id);
+      const worker2 = workerPool.getWorker(worker2Id);
+
+      const worker1Exit = new Promise((resolve) => {
+        worker1.on('exit', () => {
+          resolve(null);
+        });
+      });
+      const worker2Exit = new Promise((resolve) => {
+        worker2.on('exit', () => {
+          resolve(null);
+        });
+      });
+
+      workerPool.shutdown();
+      assert.throws(() => {
+        workerPool.getWorker(worker1Id);
+      });
+      assert.throws(() => {
+        workerPool.getWorker(worker2Id);
+      });
+
+      await worker1Exit;
+      await worker2Exit;
+
+      assert.deepEqual(workerPool.getStats(), {
+        totalWorkers: 0,
+        workersInUse: 0,
+      });
+    });
+  });
+});

package/test/atlaspack-v3/worker.js

@@ -0,0 +1,23 @@
+const {parentPort} = require('worker_threads');
+
+const receivedMessages = [];
+
+parentPort.on('message', (message) => {
+  if (message.type === 'registerWorker') {
+    parentPort.postMessage({
+      type: 'workerRegistered',
+    });
+  } else if (message.type === 'probeStatus') {
+    parentPort.postMessage({
+      type: 'status',
+      status: 'test-status-ok',
+      receivedMessages,
+    });
+  }
+
+  receivedMessages.push(message);
+  parentPort.postMessage({
+    type: 'received',
+    message,
+  });
+});