@tachybase/module-worker-thread 0.23.8

package/dist/server/worker.js

@@ -0,0 +1,157 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var worker_exports = {};
+__export(worker_exports, {
+  main: () => main
+});
+module.exports = __toCommonJS(worker_exports);
+var import_worker_threads = require("worker_threads");
+var import_logger = require("@tachybase/logger");
+var import_module_collection = __toESM(require("@tachybase/module-collection"));
+var import_module_user = __toESM(require("@tachybase/module-user"));
+var import_server = require("@tachybase/server");
+var import_utils = require("@tachybase/utils");
+var import_workerTypes = require("./workerTypes");
+const loggerOptions = {
+  system: {
+    transports: (0, import_logger.getLoggerTransport)(),
+    level: (0, import_logger.getLoggerLevel)()
+    // TODO: 后续考虑是否提到info
+  },
+  request: {
+    transports: (0, import_logger.getLoggerTransport)(),
+    level: (0, import_logger.getLoggerLevel)()
+  }
+};
+const handleWorkerMessages = (app) => {
+  if (import_worker_threads.isMainThread) {
+    return;
+  }
+  import_worker_threads.parentPort.postMessage({
+    event: import_workerTypes.WorkerEvent.Started
+  });
+  import_worker_threads.parentPort.on("message", async (message) => {
+    app.logger.info("[worker] input", message);
+    const { reqId, event } = message;
+    if (event === import_workerTypes.WorkerEvent.PluginMethod) {
+      try {
+        const { plugin, method, params } = message.values;
+        const appPlugin = app.pm.get(plugin);
+        const result = await appPlugin[method](params);
+        app.logger.info(`[worker] output ${result}`);
+        import_worker_threads.parentPort.postMessage({
+          event,
+          reqId,
+          result
+        });
+      } catch (error) {
+        app.logger.error("[worker] error", error);
+        import_worker_threads.parentPort.postMessage({
+          event,
+          reqId,
+          error
+        });
+      }
+    } else {
+      app.logger.error("[worker] invalid message", message);
+      import_worker_threads.parentPort.postMessage({
+        event,
+        reqId,
+        error: "invalid message"
+      });
+    }
+  });
+};
+const main = async () => {
+  var _a, _b, _c;
+  let app;
+  const appName = import_worker_threads.workerData.appName || "main";
+  try {
+    const applicationOptions = {
+      name: `worker-${appName}-${(0, import_utils.uid)()}`,
+      database: import_worker_threads.workerData.databaseOptions,
+      logger: loggerOptions
+    };
+    app = new import_server.Application(applicationOptions);
+    app.logger.info("[worker] app boot");
+    import_utils.Container.reset();
+    import_utils.Container.set({ id: "db", value: app.db });
+    import_utils.Container.set({ id: "app", value: app });
+    import_utils.Container.set({ id: "logger", value: app.logger });
+    await loadPlugins(app);
+    app.logger.info("[worker] app has been started");
+    handleWorkerMessages(app);
+  } catch (err) {
+    (_a = app == null ? void 0 : app.logger) == null ? void 0 : _a.error("[worker] Failed to start worker:", err);
+    try {
+      await ((_b = app == null ? void 0 : app.db) == null ? void 0 : _b.close());
+    } catch (cleanupErr) {
+      (_c = app == null ? void 0 : app.logger) == null ? void 0 : _c.error("[worker] Cleanup failed:", cleanupErr);
+    }
+    process.exit(-1);
+  }
+};
+async function loadPlugins(app) {
+  await app.pm.initPlugins();
+  const userPluginName = app.pm.get(import_module_user.default).name;
+  const loadPlugins2 = [userPluginName];
+  for (const [P, plugin] of app.pm.getPlugins()) {
+    if (plugin.name.startsWith("field-")) {
+      loadPlugins2.push(plugin.name);
+    }
+  }
+  loadPlugins2.push(app.pm.get(import_module_collection.default).name);
+  for (const pluginName of loadPlugins2) {
+    const plugin = app.pm.get(pluginName);
+    try {
+      await plugin.beforeLoad();
+    } catch (error) {
+      app.logger.error(`Failed to execute beforeLoad for plugin ${pluginName}:`, error);
+      throw error;
+    }
+  }
+  for (const [P, plugin] of app.pm.getPlugins()) {
+    if (!plugin.enabled) {
+      continue;
+    }
+    await plugin.loadCollections();
+    if (!loadPlugins2.includes(plugin.name)) {
+      continue;
+    }
+    await plugin.load();
+    for (const feature of plugin.featureInstances) {
+      await feature.load();
+    }
+  }
+  await app.db.getRepository("collections").load();
+}
+main();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  main
+});

package/dist/server/workerManager.d.ts

@@ -0,0 +1,49 @@
+import { Application } from '@tachybase/server';
+import { WorkerEventInputPluginMethod } from './workerTypes';
+export declare class WorkerManager {
+    app: Application;
+    get available(): boolean;
+    private isProd;
+    private workerNum;
+    private workerList;
+    private currentWorkerIndex;
+    private busyWorkers;
+    private busyWorkerSet;
+    private cache;
+    private errorRecoveryTimes;
+    private static readonly timeoutSecond;
+    private getPluginMethodKey;
+    private getGlobalKey;
+    private databaseOptions;
+    constructor(app: Application, workerNum?: number);
+    getPresetWorkerNum(): number;
+    getCurrentWorkerNum(): number;
+    getBusyWorkerNum(): number;
+    initWorkers(): Promise<void>;
+    private addWorker;
+    clear(): Promise<void>;
+    resetWorkerNum(targetNum: number): Promise<void>;
+    /**
+     * 检查并更新并发计数
+     * @param key - 用于缓存的键
+     * @param limit - 并发限制
+     * @param cache - 缓存对象
+     * @returns 是否超过并发限制
+     */
+    private checkAndUpdateConcurrency;
+    /**
+     * 减少并发计数
+     * @param key - 用于缓存的键
+     * @param cache - 缓存对象
+     */
+    private decrementConcurrencyCount;
+    /**
+     * 运行插件方法
+     */
+    callPluginMethod(values: WorkerEventInputPluginMethod): Promise<unknown>;
+    private handleWorkerCompletion;
+    /**
+     * 运行通用方法
+     */
+    private callMethod;
+}

package/dist/server/workerManager.js

@@ -0,0 +1,343 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var workerManager_exports = {};
+__export(workerManager_exports, {
+  WorkerManager: () => WorkerManager
+});
+module.exports = __toCommonJS(workerManager_exports);
+var import_crypto = require("crypto");
+var import_path = __toESM(require("path"));
+var import_worker_threads = require("worker_threads");
+var import_utils = require("@tachybase/utils");
+var import_constants = require("./constants");
+var import_workerTypes = require("./workerTypes");
+function copyBasicTypes(obj) {
+  const result = {};
+  for (const key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      const value = obj[key];
+      if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+        result[key] = value;
+      }
+    }
+  }
+  return result;
+}
+class WorkerManager {
+  constructor(app, workerNum) {
+    this.app = app;
+    if (!import_worker_threads.isMainThread) {
+      return;
+    }
+    if (workerNum !== void 0) {
+      this.workerNum = workerNum;
+    }
+    this.databaseOptions = copyBasicTypes(app.db.options);
+  }
+  get available() {
+    return this.workerList.length > 0;
+  }
+  isProd = true;
+  workerNum = 0;
+  workerList = [];
+  currentWorkerIndex = 0;
+  busyWorkers = /* @__PURE__ */ new WeakMap();
+  busyWorkerSet = /* @__PURE__ */ new Set();
+  cache = /* @__PURE__ */ new Map();
+  errorRecoveryTimes = 0;
+  // 默认超时时间
+  static timeoutSecond = import_constants.WORKER_TIMEOUT;
+  getPluginMethodKey(values) {
+    return `worker:pluginMethod:${values.plugin}:${values.method}`;
+  }
+  getGlobalKey() {
+    return `worker:global`;
+  }
+  databaseOptions;
+  getPresetWorkerNum() {
+    return this.workerNum;
+  }
+  getCurrentWorkerNum() {
+    return this.workerList.length;
+  }
+  getBusyWorkerNum() {
+    return this.busyWorkerSet.size;
+  }
+  async initWorkers() {
+    if (!import_worker_threads.isMainThread) {
+      return;
+    }
+    this.isProd = await (0, import_utils.fsExists)(import_path.default.resolve(__dirname, `${import_constants.WORKER_FILE}.js`));
+    await Promise.all(Array.from({ length: this.workerNum }).map(() => this.addWorker()));
+  }
+  async addWorker() {
+    let worker;
+    if (this.isProd) {
+      worker = new import_worker_threads.Worker(import_path.default.resolve(__dirname, `${import_constants.WORKER_FILE}.js`), {
+        workerData: {
+          appName: this.app.name,
+          databaseOptions: this.databaseOptions
+        }
+      });
+    } else {
+      worker = new import_worker_threads.Worker(import_path.default.resolve(__dirname, "../../worker-starter.mjs"), {
+        workerData: {
+          scriptPath: import_path.default.resolve(__dirname, `${import_constants.WORKER_FILE}.ts`),
+          appName: this.app.name,
+          databaseOptions: this.databaseOptions
+        }
+      });
+    }
+    await new Promise((resolve, reject) => {
+      worker.on("message", (message) => {
+        if (message.event === "started") {
+          this.workerList.push(worker);
+          resolve();
+        }
+      });
+      worker.on("error", (error) => {
+        reject(error);
+      });
+      worker.on("exit", async (code) => {
+        if (code === 0 || code === 1) {
+          return;
+        }
+        this.errorRecoveryTimes++;
+        this.app.logger.error(`Worker stopped with exit code ${code}, times: ${this.errorRecoveryTimes}`);
+        if (this.errorRecoveryTimes >= import_constants.WORKER_ERROR_RETRY) {
+          this.app.logger.error(`Worker error for ${this.errorRecoveryTimes} times`);
+          return;
+        }
+        this.workerList = this.workerList.filter((w) => w !== worker);
+        if (this.workerList.length < this.workerNum) {
+          await this.addWorker();
+        }
+        this.errorRecoveryTimes = 0;
+      });
+    });
+  }
+  async clear() {
+    if (!import_worker_threads.isMainThread) {
+      return;
+    }
+    await Promise.all(this.workerList.map((worker) => worker.terminate()));
+    this.workerList = [];
+    this.workerNum = 0;
+    this.currentWorkerIndex = 0;
+    this.busyWorkers = /* @__PURE__ */ new WeakMap();
+    this.busyWorkerSet = /* @__PURE__ */ new Set();
+    this.errorRecoveryTimes = 0;
+    this.cache.clear();
+  }
+  async resetWorkerNum(targetNum) {
+    var _a;
+    if (!import_worker_threads.isMainThread) {
+      return;
+    }
+    this.workerNum = targetNum;
+    if (targetNum < this.workerList.length) {
+      const excessWorkers = this.workerList.length - targetNum;
+      let terminatedCount = 0;
+      for (let i = this.workerList.length - 1; i >= 0 && terminatedCount < excessWorkers; i--) {
+        const worker = this.workerList[i];
+        if (!((_a = this.busyWorkers.get(worker)) == null ? void 0 : _a.size)) {
+          this.workerList.splice(i, 1);
+          worker.terminate();
+          terminatedCount++;
+        }
+      }
+    } else if (targetNum > this.workerList.length) {
+      const additionalWorkers = targetNum - this.workerList.length;
+      for (let i = 0; i < additionalWorkers; i++) {
+        this.addWorker();
+      }
+    }
+  }
+  /**
+   * 检查并更新并发计数
+   * @param key - 用于缓存的键
+   * @param limit - 并发限制
+   * @param cache - 缓存对象
+   * @returns 是否超过并发限制
+   */
+  checkAndUpdateConcurrency(key, limit, cache) {
+    const now = Date.now();
+    let countInfo = cache.get(key);
+    if (countInfo && countInfo.lastTime) {
+      const diff = now - countInfo.lastTime.getTime();
+      if (diff > 1e3 * WorkerManager.timeoutSecond) {
+        countInfo.count = 0;
+        this.app.logger.debug(`[worker] Reset concurrency count for ${key} due to timeout`);
+      }
+    }
+    if (countInfo && countInfo.count >= limit) {
+      this.app.logger.debug(`[worker] Concurrency limit reached for ${key}: ${countInfo.count}/${limit}`);
+      return true;
+    } else {
+      if (!countInfo) {
+        countInfo = {
+          lastTime: new Date(now),
+          count: 1
+        };
+      } else {
+        countInfo.lastTime = new Date(now);
+        countInfo.count++;
+      }
+      cache.set(key, countInfo);
+      return false;
+    }
+  }
+  /**
+   * 减少并发计数
+   * @param key - 用于缓存的键
+   * @param cache - 缓存对象
+   */
+  decrementConcurrencyCount(key, cache) {
+    const countInfo = cache.get(key);
+    if (countInfo) {
+      countInfo.count--;
+      if (countInfo.count < 0) {
+        cache.delete(key);
+      } else {
+        countInfo.lastTime = /* @__PURE__ */ new Date();
+        cache.set(key, countInfo);
+      }
+    }
+  }
+  /**
+   * 运行插件方法
+   */
+  async callPluginMethod(values) {
+    if (values.globalConcurrency) {
+      const exceeded = this.checkAndUpdateConcurrency(this.getGlobalKey(), values.globalConcurrency, this.cache);
+      if (exceeded) {
+        throw new Error("Global method call limit exceeded");
+      }
+    }
+    if (values.concurrency) {
+      const exceeded = this.checkAndUpdateConcurrency(this.getPluginMethodKey(values), values.concurrency, this.cache);
+      if (exceeded) {
+        throw new Error("Plugin method call limit exceeded");
+      }
+    }
+    const result = await this.callMethod(import_workerTypes.WorkerEvent.PluginMethod, {
+      ...values,
+      plugin: this.app.pm.get(values.plugin).name
+    });
+    if (values.globalConcurrency) {
+      this.decrementConcurrencyCount(this.getGlobalKey(), this.cache);
+    }
+    if (values.concurrency) {
+      this.decrementConcurrencyCount(this.getPluginMethodKey(values), this.cache);
+    }
+    return result;
+  }
+  handleWorkerCompletion(worker, reqId, resolve, result, reject, error) {
+    const busySet = this.busyWorkers.get(worker);
+    if (busySet) {
+      busySet.delete(reqId);
+      if (!busySet.size) {
+        this.busyWorkers.delete(worker);
+        this.busyWorkerSet.delete(worker);
+        if (this.workerList.length > this.workerNum) {
+          this.workerList = this.workerList.filter((w) => w !== worker);
+          worker.terminate();
+        }
+      }
+    }
+    if (error) {
+      reject(error);
+    } else {
+      resolve(result);
+    }
+  }
+  /**
+   * 运行通用方法
+   */
+  async callMethod(workerEvent, values) {
+    if (this.workerList.length === 0) {
+      throw new Error("No available workers");
+    }
+    const reqId = (0, import_crypto.randomUUID)();
+    if (this.currentWorkerIndex >= this.workerList.length) {
+      this.currentWorkerIndex = 0;
+    }
+    const worker = this.workerList[this.currentWorkerIndex];
+    this.currentWorkerIndex = (this.currentWorkerIndex + 1) % this.workerList.length;
+    if (!this.busyWorkers.has(worker)) {
+      this.busyWorkers.set(worker, /* @__PURE__ */ new Set());
+      this.busyWorkerSet.add(worker);
+    }
+    this.busyWorkers.get(worker).add(reqId);
+    return new Promise((resolve, reject) => {
+      let timeout;
+      let handleMessage;
+      let handleError;
+      handleMessage = (message) => {
+        if (message.reqId === reqId) {
+          clearTimeout(timeout);
+          worker.off("message", handleMessage);
+          worker.off("error", handleError);
+          this.handleWorkerCompletion(worker, reqId, resolve, message.result);
+        }
+      };
+      handleError = (error) => {
+        clearTimeout(timeout);
+        worker.off("message", handleMessage);
+        worker.off("error", handleError);
+        this.handleWorkerCompletion(worker, reqId, resolve, void 0, reject, error);
+      };
+      timeout = setTimeout(
+        () => {
+          worker.off("message", handleMessage);
+          worker.off("error", handleError);
+          this.handleWorkerCompletion(
+            worker,
+            reqId,
+            resolve,
+            void 0,
+            reject,
+            new Error("callPluginMethod timed out")
+          );
+        },
+        (values.timeoutSecond || WorkerManager.timeoutSecond) * 1e3
+      );
+      worker.on("message", handleMessage);
+      worker.on("error", handleError);
+      worker.postMessage({
+        reqId,
+        event: workerEvent,
+        values
+      });
+    });
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  WorkerManager
+});

package/dist/server/workerTypes.d.ts

@@ -0,0 +1,25 @@
+import type { Constructor, Plugin } from '@tachybase/server';
+export declare enum WorkerEvent {
+    Started = "started",
+    PluginMethod = "PluginMethod"
+}
+export type WorkerEventInputDefault = {
+    timeoutSecond?: number;
+    concurrency?: number;
+    globalConcurrency?: number;
+};
+export type WorkerEventInputPluginMethod<T = any> = WorkerEventInputDefault & {
+    plugin: Constructor<Plugin> | string;
+    method: string;
+    params: T;
+};
+export type WorkerEventInput = WorkerEventInputPluginMethod;
+export type WorkerWebInfo = {
+    preset: number;
+    current: number;
+    busy: number;
+};
+export type callPluginMethodInfo = {
+    lastTime: Date;
+    count: number;
+};

package/dist/server/workerTypes.js

@@ -0,0 +1,31 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var workerTypes_exports = {};
+__export(workerTypes_exports, {
+  WorkerEvent: () => WorkerEvent
+});
+module.exports = __toCommonJS(workerTypes_exports);
+var WorkerEvent = /* @__PURE__ */ ((WorkerEvent2) => {
+  WorkerEvent2["Started"] = "started";
+  WorkerEvent2["PluginMethod"] = "PluginMethod";
+  return WorkerEvent2;
+})(WorkerEvent || {});
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  WorkerEvent
+});

package/dist/server/workerWebController.d.ts

@@ -0,0 +1,5 @@
+import { Context, Next } from '@tachybase/actions';
+export declare class WorkerWebController {
+    info(ctx: Context, next: Next): Promise<void>;
+    preset(ctx: Context, next: Next): Promise<void>;
+}