@unpackjs/core 1.5.3 → 1.6.0

package/dist/thread-loader/WorkerPool.js

@@ -0,0 +1,387 @@
+import { createRequire } from 'node:module';
+var require = createRequire(import.meta['url']);
+
+import path from "path";
+import { fileURLToPath } from "url";
+var getFilename = () => fileURLToPath(import.meta.url);
+var getDirname = () => path.dirname(getFilename());
+var __dirname = /* @__PURE__ */ getDirname();
+var __filename = /* @__PURE__ */ getFilename();
+import childProcess from "node:child_process";
+const asyncMapSeries = require("neo-async/mapSeries.js");
+const asyncQueue = require("neo-async/queue.js");
+import WorkerError from "./WorkerError.js";
+import readBuffer from "./readBuffer.js";
+import { replacer, reviver } from "./serializer.js";
+const workerPath = require.resolve("./worker");
+let workerId = 0;
+class PoolWorker {
+  constructor(options, onJobDone) {
+    this.disposed = false;
+    this.nextJobId = 0;
+    this.jobs = /* @__PURE__ */ Object.create(null);
+    this.activeJobs = 0;
+    this.onJobDone = onJobDone;
+    this.id = workerId;
+    workerId += 1;
+    const sanitizedNodeArgs = (options.nodeArgs || []).filter((opt) => !!opt);
+    this.worker = childProcess.spawn(
+      process.execPath,
+      [].concat(sanitizedNodeArgs).concat(workerPath, options.parallelJobs),
+      {
+        detached: true,
+        stdio: ["ignore", "pipe", "pipe", "pipe", "pipe"]
+      }
+    );
+    this.worker.unref();
+    if (!this.worker.stdio) {
+      throw new Error(
+        `Failed to create the worker pool with workerId: ${workerId} and ${""}configuration: ${JSON.stringify(
+          options
+        )}. Please verify if you hit the OS open files limit.`
+      );
+    }
+    const [, , , readPipe, writePipe] = this.worker.stdio;
+    this.readPipe = readPipe;
+    this.writePipe = writePipe;
+    this.listenStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+    this.readNextMessage();
+  }
+  listenStdOutAndErrFromWorker(workerStdout, workerStderr) {
+    if (workerStdout) {
+      workerStdout.on("data", this.writeToStdout);
+    }
+    if (workerStderr) {
+      workerStderr.on("data", this.writeToStderr);
+    }
+  }
+  ignoreStdOutAndErrFromWorker(workerStdout, workerStderr) {
+    if (workerStdout) {
+      workerStdout.removeListener("data", this.writeToStdout);
+    }
+    if (workerStderr) {
+      workerStderr.removeListener("data", this.writeToStderr);
+    }
+  }
+  writeToStdout(data) {
+    if (!this.disposed) {
+      process.stdout.write(data);
+    }
+  }
+  writeToStderr(data) {
+    if (!this.disposed) {
+      process.stderr.write(data);
+    }
+  }
+  run(data, callback) {
+    const jobId = this.nextJobId;
+    this.nextJobId += 1;
+    this.jobs[jobId] = { data, callback };
+    this.activeJobs += 1;
+    this.writeJson({
+      type: "job",
+      id: jobId,
+      data
+    });
+  }
+  warmup(requires) {
+    this.writeJson({
+      type: "warmup",
+      requires
+    });
+  }
+  writeJson(data) {
+    const lengthBuffer = Buffer.alloc(4);
+    const messageBuffer = Buffer.from(JSON.stringify(data, replacer), "utf-8");
+    lengthBuffer.writeInt32BE(messageBuffer.length, 0);
+    this.writePipe.write(lengthBuffer);
+    this.writePipe.write(messageBuffer);
+  }
+  writeEnd() {
+    const lengthBuffer = Buffer.alloc(4);
+    lengthBuffer.writeInt32BE(0, 0);
+    this.writePipe.write(lengthBuffer);
+  }
+  readNextMessage() {
+    this.state = "read length";
+    this.readBuffer(4, (lengthReadError, lengthBuffer) => {
+      if (lengthReadError) {
+        console.error(`Failed to communicate with worker (read length) ${lengthReadError}`);
+        return;
+      }
+      this.state = "length read";
+      const length = lengthBuffer.readInt32BE(0);
+      this.state = "read message";
+      this.readBuffer(length, (messageError, messageBuffer) => {
+        if (messageError) {
+          console.error(`Failed to communicate with worker (read message) ${messageError}`);
+          return;
+        }
+        this.state = "message read";
+        const messageString = messageBuffer.toString("utf-8");
+        const message = JSON.parse(messageString, reviver);
+        this.state = "process message";
+        this.onWorkerMessage(message, (err) => {
+          if (err) {
+            console.error(`Failed to communicate with worker (process message) ${err}`);
+            return;
+          }
+          this.state = "soon next";
+          setImmediate(() => this.readNextMessage());
+        });
+      });
+    });
+  }
+  onWorkerMessage(message, finalCallback) {
+    const { type, id } = message;
+    switch (type) {
+      case "job": {
+        const { data, error, result } = message;
+        asyncMapSeries(
+          data,
+          (length, callback) => this.readBuffer(length, callback),
+          (eachErr, buffers) => {
+            const { callback: jobCallback } = this.jobs[id];
+            const callback = (err, arg) => {
+              if (jobCallback) {
+                delete this.jobs[id];
+                this.activeJobs -= 1;
+                this.onJobDone();
+                if (err) {
+                  jobCallback(err instanceof Error ? err : new Error(err), arg);
+                } else {
+                  jobCallback(null, arg);
+                }
+              }
+              finalCallback();
+            };
+            if (eachErr) {
+              callback(eachErr);
+              return;
+            }
+            let bufferPosition = 0;
+            if (result.result) {
+              result.result = result.result.map((r) => {
+                if (r.buffer) {
+                  const buffer = buffers[bufferPosition];
+                  bufferPosition += 1;
+                  if (r.string) {
+                    return buffer.toString("utf-8");
+                  }
+                  return buffer;
+                }
+                return r.data;
+              });
+            }
+            if (error) {
+              callback(this.fromErrorObj(error), result);
+              return;
+            }
+            callback(null, result);
+          }
+        );
+        break;
+      }
+      case "loadModule": {
+        const { request, questionId } = message;
+        const { data } = this.jobs[id];
+        data.loadModule(request, (error, source, sourceMap) => {
+          this.writeJson({
+            type: "result",
+            id: questionId,
+            error: error ? {
+              message: error.message,
+              details: error.details,
+              missing: error.missing
+            } : null,
+            result: [
+              source,
+              sourceMap
+              // TODO: Serialize module?
+              // module,
+            ]
+          });
+        });
+        finalCallback();
+        break;
+      }
+      case "resolve": {
+        const { context, request, options, questionId } = message;
+        const { data } = this.jobs[id];
+        if (options) {
+          data.getResolve(options)(context, request, (error, result) => {
+            this.writeJson({
+              type: "result",
+              id: questionId,
+              error: error ? {
+                message: error.message,
+                details: error.details,
+                missing: error.missing
+              } : null,
+              result
+            });
+          });
+        } else {
+          data.resolve(context, request, (error, result) => {
+            this.writeJson({
+              type: "result",
+              id: questionId,
+              error: error ? {
+                message: error.message,
+                details: error.details,
+                missing: error.missing
+              } : null,
+              result
+            });
+          });
+        }
+        finalCallback();
+        break;
+      }
+      case "emitWarning": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        jobData.emitWarning(this.fromErrorObj(data));
+        finalCallback();
+        break;
+      }
+      case "emitError": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        jobData.emitError(this.fromErrorObj(data));
+        finalCallback();
+        break;
+      }
+      case "getLogger": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        if (!Object.hasOwnProperty.call(jobData.loggers, data.name)) {
+          jobData.loggers[data.name] = jobData.getLogger(data.name);
+        }
+        finalCallback();
+        break;
+      }
+      case "logger": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        jobData.loggers[data.name][data.severity](data.message);
+        finalCallback();
+        break;
+      }
+      default: {
+        console.error(`Unexpected worker message ${type} in WorkerPool.`);
+        finalCallback();
+        break;
+      }
+    }
+  }
+  fromErrorObj(arg) {
+    let obj;
+    if (typeof arg === "string") {
+      obj = { message: arg };
+    } else {
+      obj = arg;
+    }
+    return new WorkerError(obj, this.id);
+  }
+  readBuffer(length, callback) {
+    readBuffer(this.readPipe, length, callback);
+  }
+  dispose() {
+    if (!this.disposed) {
+      this.disposed = true;
+      this.ignoreStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+      this.writeEnd();
+    }
+  }
+}
+class WorkerPool {
+  constructor(options) {
+    this.options = options || {};
+    this.numberOfWorkers = options.numberOfWorkers;
+    this.poolTimeout = options.poolTimeout;
+    this.workerNodeArgs = options.workerNodeArgs;
+    this.workerParallelJobs = options.workerParallelJobs;
+    this.workers = /* @__PURE__ */ new Set();
+    this.activeJobs = 0;
+    this.timeout = null;
+    this.poolQueue = asyncQueue(this.distributeJob.bind(this), options.poolParallelJobs);
+    this.terminated = false;
+    this.setupLifeCycle();
+  }
+  isAbleToRun() {
+    return !this.terminated;
+  }
+  terminate() {
+    if (this.terminated) {
+      return;
+    }
+    this.terminated = true;
+    this.poolQueue.kill();
+    this.disposeWorkers(true);
+  }
+  setupLifeCycle() {
+    process.on("exit", () => {
+      this.terminate();
+    });
+  }
+  run(data, callback) {
+    if (this.timeout) {
+      clearTimeout(this.timeout);
+      this.timeout = null;
+    }
+    this.activeJobs += 1;
+    this.poolQueue.push(data, callback);
+  }
+  distributeJob(data, callback) {
+    let bestWorker;
+    for (const worker of this.workers) {
+      if (!bestWorker || worker.activeJobs < bestWorker.activeJobs) {
+        bestWorker = worker;
+      }
+    }
+    if (bestWorker && (bestWorker.activeJobs === 0 || this.workers.size >= this.numberOfWorkers)) {
+      bestWorker.run(data, callback);
+      return;
+    }
+    const newWorker = this.createWorker();
+    newWorker.run(data, callback);
+  }
+  createWorker() {
+    const newWorker = new PoolWorker(
+      {
+        nodeArgs: this.workerNodeArgs,
+        parallelJobs: this.workerParallelJobs
+      },
+      () => this.onJobDone()
+    );
+    this.workers.add(newWorker);
+    return newWorker;
+  }
+  warmup(requires) {
+    while (this.workers.size < this.numberOfWorkers) {
+      this.createWorker().warmup(requires);
+    }
+  }
+  onJobDone() {
+    this.activeJobs -= 1;
+    if (this.activeJobs === 0 && Number.isFinite(this.poolTimeout)) {
+      this.timeout = setTimeout(() => this.disposeWorkers(), this.poolTimeout);
+    }
+  }
+  disposeWorkers(fromTerminate) {
+    if (!this.options.poolRespawn && !fromTerminate) {
+      this.terminate();
+      return;
+    }
+    if (this.activeJobs === 0 || fromTerminate) {
+      for (const worker of this.workers) {
+        worker.dispose();
+      }
+      this.workers.clear();
+    }
+  }
+}
+export {
+  WorkerPool as default
+};

package/dist/thread-loader/index.cjs

@@ -0,0 +1,100 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var thread_loader_exports = {};
+__export(thread_loader_exports, {
+  pitch: () => pitch,
+  warmup: () => warmup
+});
+module.exports = __toCommonJS(thread_loader_exports);
+var import_workerPools = require("./workerPools.cjs");
+function pitch() {
+  const options = this.getOptions();
+  const workerPool = (0, import_workerPools.getPool)(options);
+  if (!workerPool.isAbleToRun()) {
+    return;
+  }
+  const callback = this.async();
+  workerPool.run(
+    {
+      loaders: this.loaders.slice(this.loaderIndex + 1).map((l) => {
+        return {
+          loader: l.path,
+          options: l.options,
+          ident: l.ident
+        };
+      }),
+      _compiler: {
+        fsStartTime: this._compiler.fsStartTime,
+        options: { plugins: [] }
+      },
+      _compilation: {
+        outputOptions: {
+          hashSalt: this._compilation.outputOptions.hashSalt,
+          hashFunction: this._compilation.outputOptions.hashFunction,
+          hashDigest: this._compilation.outputOptions.hashDigest,
+          hashDigestLength: this._compilation.outputOptions.hashDigestLength
+        },
+        options: {
+          devtool: this._compilation?.options?.devtool
+        }
+      },
+      resourcePath: this.resourcePath,
+      resource: this.resourcePath + (this.resourceQuery || ""),
+      sourceMap: this.sourceMap,
+      emitError: this.emitError,
+      emitWarning: this.emitWarning,
+      loadModule: this.loadModule,
+      resolve: this.resolve,
+      getResolve: this.getResolve,
+      target: this.target,
+      minimize: this.minimize,
+      resourceQuery: this.resourceQuery,
+      optionsContext: this.rootContext || this.options.context,
+      rootContext: this.rootContext,
+      getLogger: this.getLogger,
+      loggers: {}
+    },
+    (err, r) => {
+      if (r) {
+        r.fileDependencies.forEach((d) => this.addDependency(d));
+        r.contextDependencies.forEach((d) => this.addContextDependency(d));
+        r.missingDependencies.forEach((d) => this.addMissingDependency(d));
+        r.buildDependencies.forEach(
+          (d) => (
+            // Compatibility with webpack v4
+            this.addBuildDependency ? this.addBuildDependency(d) : this.addDependency(d)
+          )
+        );
+      }
+      if (err) {
+        callback(err);
+        return;
+      }
+      callback(null, ...r.result);
+    }
+  );
+}
+function warmup(options, requires) {
+  const workerPool = (0, import_workerPools.getPool)(options);
+  workerPool.warmup(requires);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  pitch,
+  warmup
+});

package/dist/thread-loader/index.js

@@ -0,0 +1,85 @@
+import { createRequire } from 'node:module';
+var require = createRequire(import.meta['url']);
+
+import path from "path";
+import { fileURLToPath } from "url";
+var getFilename = () => fileURLToPath(import.meta.url);
+var getDirname = () => path.dirname(getFilename());
+var __dirname = /* @__PURE__ */ getDirname();
+var __filename = /* @__PURE__ */ getFilename();
+import { getPool } from "./workerPools.js";
+function pitch() {
+  const options = this.getOptions();
+  const workerPool = getPool(options);
+  if (!workerPool.isAbleToRun()) {
+    return;
+  }
+  const callback = this.async();
+  workerPool.run(
+    {
+      loaders: this.loaders.slice(this.loaderIndex + 1).map((l) => {
+        return {
+          loader: l.path,
+          options: l.options,
+          ident: l.ident
+        };
+      }),
+      _compiler: {
+        fsStartTime: this._compiler.fsStartTime,
+        options: { plugins: [] }
+      },
+      _compilation: {
+        outputOptions: {
+          hashSalt: this._compilation.outputOptions.hashSalt,
+          hashFunction: this._compilation.outputOptions.hashFunction,
+          hashDigest: this._compilation.outputOptions.hashDigest,
+          hashDigestLength: this._compilation.outputOptions.hashDigestLength
+        },
+        options: {
+          devtool: this._compilation?.options?.devtool
+        }
+      },
+      resourcePath: this.resourcePath,
+      resource: this.resourcePath + (this.resourceQuery || ""),
+      sourceMap: this.sourceMap,
+      emitError: this.emitError,
+      emitWarning: this.emitWarning,
+      loadModule: this.loadModule,
+      resolve: this.resolve,
+      getResolve: this.getResolve,
+      target: this.target,
+      minimize: this.minimize,
+      resourceQuery: this.resourceQuery,
+      optionsContext: this.rootContext || this.options.context,
+      rootContext: this.rootContext,
+      getLogger: this.getLogger,
+      loggers: {}
+    },
+    (err, r) => {
+      if (r) {
+        r.fileDependencies.forEach((d) => this.addDependency(d));
+        r.contextDependencies.forEach((d) => this.addContextDependency(d));
+        r.missingDependencies.forEach((d) => this.addMissingDependency(d));
+        r.buildDependencies.forEach(
+          (d) => (
+            // Compatibility with webpack v4
+            this.addBuildDependency ? this.addBuildDependency(d) : this.addDependency(d)
+          )
+        );
+      }
+      if (err) {
+        callback(err);
+        return;
+      }
+      callback(null, ...r.result);
+    }
+  );
+}
+function warmup(options, requires) {
+  const workerPool = getPool(options);
+  workerPool.warmup(requires);
+}
+export {
+  pitch,
+  warmup
+};

package/dist/thread-loader/readBuffer.cjs

@@ -0,0 +1,55 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var readBuffer_exports = {};
+__export(readBuffer_exports, {
+  default: () => readBuffer
+});
+module.exports = __toCommonJS(readBuffer_exports);
+function readBuffer(pipe, length, callback) {
+  if (length === 0) {
+    callback(null, Buffer.alloc(0));
+    return;
+  }
+  let remainingLength = length;
+  const buffers = [];
+  const readChunk = () => {
+    const onChunk = (arg) => {
+      let chunk = arg;
+      let overflow;
+      if (chunk.length > remainingLength) {
+        overflow = chunk.slice(remainingLength);
+        chunk = chunk.slice(0, remainingLength);
+        remainingLength = 0;
+      } else {
+        remainingLength -= chunk.length;
+      }
+      buffers.push(chunk);
+      if (remainingLength === 0) {
+        pipe.removeListener("data", onChunk);
+        pipe.pause();
+        if (overflow) {
+          pipe.unshift(overflow);
+        }
+        callback(null, Buffer.concat(buffers, length));
+      }
+    };
+    pipe.on("data", onChunk);
+    pipe.resume();
+  };
+  readChunk();
+}

package/dist/thread-loader/readBuffer.js

@@ -0,0 +1,45 @@
+import { createRequire } from 'node:module';
+var require = createRequire(import.meta['url']);
+
+import path from "path";
+import { fileURLToPath } from "url";
+var getFilename = () => fileURLToPath(import.meta.url);
+var getDirname = () => path.dirname(getFilename());
+var __dirname = /* @__PURE__ */ getDirname();
+var __filename = /* @__PURE__ */ getFilename();
+function readBuffer(pipe, length, callback) {
+  if (length === 0) {
+    callback(null, Buffer.alloc(0));
+    return;
+  }
+  let remainingLength = length;
+  const buffers = [];
+  const readChunk = () => {
+    const onChunk = (arg) => {
+      let chunk = arg;
+      let overflow;
+      if (chunk.length > remainingLength) {
+        overflow = chunk.slice(remainingLength);
+        chunk = chunk.slice(0, remainingLength);
+        remainingLength = 0;
+      } else {
+        remainingLength -= chunk.length;
+      }
+      buffers.push(chunk);
+      if (remainingLength === 0) {
+        pipe.removeListener("data", onChunk);
+        pipe.pause();
+        if (overflow) {
+          pipe.unshift(overflow);
+        }
+        callback(null, Buffer.concat(buffers, length));
+      }
+    };
+    pipe.on("data", onChunk);
+    pipe.resume();
+  };
+  readChunk();
+}
+export {
+  readBuffer as default
+};