@unpackjs/core 1.5.4 → 1.6.1

package/dist/run/index.cjs

@@ -0,0 +1,23 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var run_exports = {};
+module.exports = __toCommonJS(run_exports);
+__reExport(run_exports, require("./build.cjs"), module.exports);
+__reExport(run_exports, require("./dev.cjs"), module.exports);
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ...require("./build.cjs"),
+  ...require("./dev.cjs")
+});

package/dist/run/index.d.ts

@@ -0,0 +1,3 @@
+export * from './build';
+export * from './dev';
+//# sourceMappingURL=index.d.ts.map

package/dist/run/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/run/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAA;AACvB,cAAc,OAAO,CAAA"}

package/dist/run/index.js

@@ -0,0 +1,11 @@
+import { createRequire } from 'node:module';
+var require = createRequire(import.meta['url']);
+
+import path from "path";
+import { fileURLToPath } from "url";
+var getFilename = () => fileURLToPath(import.meta.url);
+var getDirname = () => path.dirname(getFilename());
+var __dirname = /* @__PURE__ */ getDirname();
+var __filename = /* @__PURE__ */ getFilename();
+export * from "./build.js";
+export * from "./dev.js";

package/dist/thread-loader/WorkerError.cjs

@@ -0,0 +1,41 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var WorkerError_exports = {};
+__export(WorkerError_exports, {
+  default: () => WorkerError_default
+});
+module.exports = __toCommonJS(WorkerError_exports);
+const stack = (err, worker, workerId) => {
+  const originError = (err.stack || "").split("\n").filter((line) => line.trim().startsWith("at"));
+  const workerError = worker.split("\n").filter((line) => line.trim().startsWith("at"));
+  const diff = workerError.slice(0, workerError.length - originError.length).join("\n");
+  originError.unshift(diff);
+  originError.unshift(err.message);
+  originError.unshift(`Thread Loader (Worker ${workerId})`);
+  return originError.join("\n");
+};
+class WorkerError extends Error {
+  constructor(err, workerId) {
+    super(err);
+    this.name = err.name;
+    this.message = err.message;
+    Error.captureStackTrace(this, this.constructor);
+    this.stack = stack(err, this.stack, workerId);
+  }
+}
+var WorkerError_default = WorkerError;

package/dist/thread-loader/WorkerError.js

@@ -0,0 +1,31 @@
+import { createRequire } from 'node:module';
+var require = createRequire(import.meta['url']);
+
+import path from "path";
+import { fileURLToPath } from "url";
+var getFilename = () => fileURLToPath(import.meta.url);
+var getDirname = () => path.dirname(getFilename());
+var __dirname = /* @__PURE__ */ getDirname();
+var __filename = /* @__PURE__ */ getFilename();
+const stack = (err, worker, workerId) => {
+  const originError = (err.stack || "").split("\n").filter((line) => line.trim().startsWith("at"));
+  const workerError = worker.split("\n").filter((line) => line.trim().startsWith("at"));
+  const diff = workerError.slice(0, workerError.length - originError.length).join("\n");
+  originError.unshift(diff);
+  originError.unshift(err.message);
+  originError.unshift(`Thread Loader (Worker ${workerId})`);
+  return originError.join("\n");
+};
+class WorkerError extends Error {
+  constructor(err, workerId) {
+    super(err);
+    this.name = err.name;
+    this.message = err.message;
+    Error.captureStackTrace(this, this.constructor);
+    this.stack = stack(err, this.stack, workerId);
+  }
+}
+var WorkerError_default = WorkerError;
+export {
+  WorkerError_default as default
+};

package/dist/thread-loader/WorkerPool.cjs

@@ -0,0 +1,407 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var WorkerPool_exports = {};
+__export(WorkerPool_exports, {
+  default: () => WorkerPool
+});
+module.exports = __toCommonJS(WorkerPool_exports);
+var import_node_child_process = __toESM(require("node:child_process"));
+var import_WorkerError = __toESM(require("./WorkerError.cjs"));
+var import_readBuffer = __toESM(require("./readBuffer.cjs"));
+var import_serializer = require("./serializer.cjs");
+const asyncMapSeries = require("neo-async/mapSeries.js");
+const asyncQueue = require("neo-async/queue.js");
+const workerPath = require.resolve("./worker");
+let workerId = 0;
+class PoolWorker {
+  constructor(options, onJobDone) {
+    this.disposed = false;
+    this.nextJobId = 0;
+    this.jobs = /* @__PURE__ */ Object.create(null);
+    this.activeJobs = 0;
+    this.onJobDone = onJobDone;
+    this.id = workerId;
+    workerId += 1;
+    const sanitizedNodeArgs = (options.nodeArgs || []).filter((opt) => !!opt);
+    this.worker = import_node_child_process.default.spawn(
+      process.execPath,
+      [].concat(sanitizedNodeArgs).concat(workerPath, options.parallelJobs),
+      {
+        detached: true,
+        stdio: ["ignore", "pipe", "pipe", "pipe", "pipe"]
+      }
+    );
+    this.worker.unref();
+    if (!this.worker.stdio) {
+      throw new Error(
+        `Failed to create the worker pool with workerId: ${workerId} and ${""}configuration: ${JSON.stringify(
+          options
+        )}. Please verify if you hit the OS open files limit.`
+      );
+    }
+    const [, , , readPipe, writePipe] = this.worker.stdio;
+    this.readPipe = readPipe;
+    this.writePipe = writePipe;
+    this.listenStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+    this.readNextMessage();
+  }
+  listenStdOutAndErrFromWorker(workerStdout, workerStderr) {
+    if (workerStdout) {
+      workerStdout.on("data", this.writeToStdout);
+    }
+    if (workerStderr) {
+      workerStderr.on("data", this.writeToStderr);
+    }
+  }
+  ignoreStdOutAndErrFromWorker(workerStdout, workerStderr) {
+    if (workerStdout) {
+      workerStdout.removeListener("data", this.writeToStdout);
+    }
+    if (workerStderr) {
+      workerStderr.removeListener("data", this.writeToStderr);
+    }
+  }
+  writeToStdout(data) {
+    if (!this.disposed) {
+      process.stdout.write(data);
+    }
+  }
+  writeToStderr(data) {
+    if (!this.disposed) {
+      process.stderr.write(data);
+    }
+  }
+  run(data, callback) {
+    const jobId = this.nextJobId;
+    this.nextJobId += 1;
+    this.jobs[jobId] = { data, callback };
+    this.activeJobs += 1;
+    this.writeJson({
+      type: "job",
+      id: jobId,
+      data
+    });
+  }
+  warmup(requires) {
+    this.writeJson({
+      type: "warmup",
+      requires
+    });
+  }
+  writeJson(data) {
+    const lengthBuffer = Buffer.alloc(4);
+    const messageBuffer = Buffer.from(JSON.stringify(data, import_serializer.replacer), "utf-8");
+    lengthBuffer.writeInt32BE(messageBuffer.length, 0);
+    this.writePipe.write(lengthBuffer);
+    this.writePipe.write(messageBuffer);
+  }
+  writeEnd() {
+    const lengthBuffer = Buffer.alloc(4);
+    lengthBuffer.writeInt32BE(0, 0);
+    this.writePipe.write(lengthBuffer);
+  }
+  readNextMessage() {
+    this.state = "read length";
+    this.readBuffer(4, (lengthReadError, lengthBuffer) => {
+      if (lengthReadError) {
+        console.error(`Failed to communicate with worker (read length) ${lengthReadError}`);
+        return;
+      }
+      this.state = "length read";
+      const length = lengthBuffer.readInt32BE(0);
+      this.state = "read message";
+      this.readBuffer(length, (messageError, messageBuffer) => {
+        if (messageError) {
+          console.error(`Failed to communicate with worker (read message) ${messageError}`);
+          return;
+        }
+        this.state = "message read";
+        const messageString = messageBuffer.toString("utf-8");
+        const message = JSON.parse(messageString, import_serializer.reviver);
+        this.state = "process message";
+        this.onWorkerMessage(message, (err) => {
+          if (err) {
+            console.error(`Failed to communicate with worker (process message) ${err}`);
+            return;
+          }
+          this.state = "soon next";
+          setImmediate(() => this.readNextMessage());
+        });
+      });
+    });
+  }
+  onWorkerMessage(message, finalCallback) {
+    const { type, id } = message;
+    switch (type) {
+      case "job": {
+        const { data, error, result } = message;
+        asyncMapSeries(
+          data,
+          (length, callback) => this.readBuffer(length, callback),
+          (eachErr, buffers) => {
+            const { callback: jobCallback } = this.jobs[id];
+            const callback = (err, arg) => {
+              if (jobCallback) {
+                delete this.jobs[id];
+                this.activeJobs -= 1;
+                this.onJobDone();
+                if (err) {
+                  jobCallback(err instanceof Error ? err : new Error(err), arg);
+                } else {
+                  jobCallback(null, arg);
+                }
+              }
+              finalCallback();
+            };
+            if (eachErr) {
+              callback(eachErr);
+              return;
+            }
+            let bufferPosition = 0;
+            if (result.result) {
+              result.result = result.result.map((r) => {
+                if (r.buffer) {
+                  const buffer = buffers[bufferPosition];
+                  bufferPosition += 1;
+                  if (r.string) {
+                    return buffer.toString("utf-8");
+                  }
+                  return buffer;
+                }
+                return r.data;
+              });
+            }
+            if (error) {
+              callback(this.fromErrorObj(error), result);
+              return;
+            }
+            callback(null, result);
+          }
+        );
+        break;
+      }
+      case "loadModule": {
+        const { request, questionId } = message;
+        const { data } = this.jobs[id];
+        data.loadModule(request, (error, source, sourceMap) => {
+          this.writeJson({
+            type: "result",
+            id: questionId,
+            error: error ? {
+              message: error.message,
+              details: error.details,
+              missing: error.missing
+            } : null,
+            result: [
+              source,
+              sourceMap
+              // TODO: Serialize module?
+              // module,
+            ]
+          });
+        });
+        finalCallback();
+        break;
+      }
+      case "resolve": {
+        const { context, request, options, questionId } = message;
+        const { data } = this.jobs[id];
+        if (options) {
+          data.getResolve(options)(context, request, (error, result) => {
+            this.writeJson({
+              type: "result",
+              id: questionId,
+              error: error ? {
+                message: error.message,
+                details: error.details,
+                missing: error.missing
+              } : null,
+              result
+            });
+          });
+        } else {
+          data.resolve(context, request, (error, result) => {
+            this.writeJson({
+              type: "result",
+              id: questionId,
+              error: error ? {
+                message: error.message,
+                details: error.details,
+                missing: error.missing
+              } : null,
+              result
+            });
+          });
+        }
+        finalCallback();
+        break;
+      }
+      case "emitWarning": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        jobData.emitWarning(this.fromErrorObj(data));
+        finalCallback();
+        break;
+      }
+      case "emitError": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        jobData.emitError(this.fromErrorObj(data));
+        finalCallback();
+        break;
+      }
+      case "getLogger": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        if (!Object.hasOwnProperty.call(jobData.loggers, data.name)) {
+          jobData.loggers[data.name] = jobData.getLogger(data.name);
+        }
+        finalCallback();
+        break;
+      }
+      case "logger": {
+        const { data } = message;
+        const { data: jobData } = this.jobs[id];
+        jobData.loggers[data.name][data.severity](data.message);
+        finalCallback();
+        break;
+      }
+      default: {
+        console.error(`Unexpected worker message ${type} in WorkerPool.`);
+        finalCallback();
+        break;
+      }
+    }
+  }
+  fromErrorObj(arg) {
+    let obj;
+    if (typeof arg === "string") {
+      obj = { message: arg };
+    } else {
+      obj = arg;
+    }
+    return new import_WorkerError.default(obj, this.id);
+  }
+  readBuffer(length, callback) {
+    (0, import_readBuffer.default)(this.readPipe, length, callback);
+  }
+  dispose() {
+    if (!this.disposed) {
+      this.disposed = true;
+      this.ignoreStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+      this.writeEnd();
+    }
+  }
+}
+class WorkerPool {
+  constructor(options) {
+    this.options = options || {};
+    this.numberOfWorkers = options.numberOfWorkers;
+    this.poolTimeout = options.poolTimeout;
+    this.workerNodeArgs = options.workerNodeArgs;
+    this.workerParallelJobs = options.workerParallelJobs;
+    this.workers = /* @__PURE__ */ new Set();
+    this.activeJobs = 0;
+    this.timeout = null;
+    this.poolQueue = asyncQueue(this.distributeJob.bind(this), options.poolParallelJobs);
+    this.terminated = false;
+    this.setupLifeCycle();
+  }
+  isAbleToRun() {
+    return !this.terminated;
+  }
+  terminate() {
+    if (this.terminated) {
+      return;
+    }
+    this.terminated = true;
+    this.poolQueue.kill();
+    this.disposeWorkers(true);
+  }
+  setupLifeCycle() {
+    process.on("exit", () => {
+      this.terminate();
+    });
+  }
+  run(data, callback) {
+    if (this.timeout) {
+      clearTimeout(this.timeout);
+      this.timeout = null;
+    }
+    this.activeJobs += 1;
+    this.poolQueue.push(data, callback);
+  }
+  distributeJob(data, callback) {
+    let bestWorker;
+    for (const worker of this.workers) {
+      if (!bestWorker || worker.activeJobs < bestWorker.activeJobs) {
+        bestWorker = worker;
+      }
+    }
+    if (bestWorker && (bestWorker.activeJobs === 0 || this.workers.size >= this.numberOfWorkers)) {
+      bestWorker.run(data, callback);
+      return;
+    }
+    const newWorker = this.createWorker();
+    newWorker.run(data, callback);
+  }
+  createWorker() {
+    const newWorker = new PoolWorker(
+      {
+        nodeArgs: this.workerNodeArgs,
+        parallelJobs: this.workerParallelJobs
+      },
+      () => this.onJobDone()
+    );
+    this.workers.add(newWorker);
+    return newWorker;
+  }
+  warmup(requires) {
+    while (this.workers.size < this.numberOfWorkers) {
+      this.createWorker().warmup(requires);
+    }
+  }
+  onJobDone() {
+    this.activeJobs -= 1;
+    if (this.activeJobs === 0 && Number.isFinite(this.poolTimeout)) {
+      this.timeout = setTimeout(() => this.disposeWorkers(), this.poolTimeout);
+    }
+  }
+  disposeWorkers(fromTerminate) {
+    if (!this.options.poolRespawn && !fromTerminate) {
+      this.terminate();
+      return;
+    }
+    if (this.activeJobs === 0 || fromTerminate) {
+      for (const worker of this.workers) {
+        worker.dispose();
+      }
+      this.workers.clear();
+    }
+  }
+}