@nocobase/plugin-async-task-manager 2.0.0-alpha.9 → 2.0.0-beta.10

package/dist/server/base-task-manager.js

@@ -36,14 +36,19 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var base_task_manager_exports = {};
 __export(base_task_manager_exports, {
-  BaseTaskManager: () => BaseTaskManager
+  BaseTaskManager: () => BaseTaskManager,
+  ConcurrencyMonitorDelegate: () => ConcurrencyMonitorDelegate
 });
 module.exports = __toCommonJS(base_task_manager_exports);
 var import_lodash = require("lodash");
 var import_constants = require("../common/constants");
 var import_crypto = require("crypto");
 var import_plugin = __toESM(require("./plugin"));
+var import_base_concurrency_monitor = require("./base-concurrency-monitor");
 const WORKER_JOB_ASYNC_TASK_PROCESS = "async-task:process";
+const CONCURRENCY = process.env.ASYNC_TASK_MAX_CONCURRENCY ? Number.parseInt(process.env.ASYNC_TASK_MAX_CONCURRENCY, 10) : 3;
+const CONCURRENCY_MODE = process.env.ASYNC_TASK_CONCURRENCY_MODE ?? "app";
+const PROCESS_CONCURRENCY_MONITOR = new import_base_concurrency_monitor.BaseConcurrencyMonitor(CONCURRENCY);
 class BaseTaskManager {
   taskTypes = /* @__PURE__ */ new Map();
   tasks = /* @__PURE__ */ new Map();
@@ -53,13 +58,25 @@ class BaseTaskManager {
   logger;
   app;
   progressThrottles = /* @__PURE__ */ new Map();
-  concurrency = process.env.ASYNC_TASK_MAX_CONCURRENCY ? Number.parseInt(process.env.ASYNC_TASK_MAX_CONCURRENCY, 10) : 3;
-  idle = () => {
-    return this.app.serving(WORKER_JOB_ASYNC_TASK_PROCESS) && this.tasks.size < this.concurrency;
-  };
-  onQueueTask = async ({ id }) => {
+  concurrencyMonitor = new ConcurrencyMonitorDelegate();
+  get concurrency() {
+    return this.concurrencyMonitor.concurrency;
+  }
+  set concurrency(concurrency) {
+    this.concurrencyMonitor.concurrency = concurrency;
+  }
+  idle = () => this.app.serving(WORKER_JOB_ASYNC_TASK_PROCESS) && this.concurrencyMonitor.idle();
+  onQueueTask = async ({ id }, { queueOptions }) => {
     const task = await this.prepareTask(id);
-    await this.runTask(task);
+    if (!this.concurrencyMonitor.increase(task.record.id)) {
+      this.enqueueTask(task, queueOptions);
+      return;
+    }
+    try {
+      await this.runTask(task);
+    } finally {
+      this.concurrencyMonitor.decrease(task.record.id);
+    }
   };
   onTaskProgress = (item) => {
     const userId = item.createdById;
@@ -105,6 +122,7 @@ class BaseTaskManager {
     if (task.doneAt) {
       this.progressThrottles.delete(task.id);
       this.tasks.delete(task.id);
+      this.concurrencyMonitor.decrease(task.id);
     }
     if (task.status === import_constants.TASK_STATUS.SUCCEEDED) {
       this.app.emit("workflow:dispatch");
@@ -113,6 +131,7 @@ class BaseTaskManager {
   onTaskAfterDelete = (task) => {
     this.tasks.delete(task.id);
     this.progressThrottles.delete(task.id);
+    this.concurrencyMonitor.decrease(task.id);
     const userId = task.createdById;
     if (userId) {
       this.app.emit("ws:sendToUser", {
@@ -130,31 +149,44 @@ class BaseTaskManager {
   cleanup = async () => {
     this.logger.debug("Running cleanup for completed tasks...");
     const TaskRepo = this.app.db.getRepository("asyncTasks");
-    const tasksToCleanup = await TaskRepo.find({
-      fields: ["id"],
-      filter: {
-        $or: [
-          {
-            status: [import_constants.TASK_STATUS.SUCCEEDED, import_constants.TASK_STATUS.FAILED],
-            doneAt: {
-              $lt: new Date(Date.now() - this.cleanupDelay)
+    try {
+      const tasksToCleanup = await TaskRepo.find({
+        fields: ["id"],
+        filter: {
+          $or: [
+            {
+              status: [import_constants.TASK_STATUS.SUCCEEDED, import_constants.TASK_STATUS.FAILED],
+              doneAt: {
+                $lt: new Date(Date.now() - this.cleanupDelay)
+              }
+            },
+            {
+              status: import_constants.TASK_STATUS.CANCELED
             }
-          },
-          {
-            status: import_constants.TASK_STATUS.CANCELED
-          }
-        ]
+          ]
+        }
+      });
+      this.logger.debug(`Found ${tasksToCleanup.length} tasks to cleanup`);
+      if (tasksToCleanup.length) {
+        for (const task of tasksToCleanup) {
+          this.tasks.delete(task.id);
+          this.progressThrottles.delete(task.id);
+          this.concurrencyMonitor.decrease(task.id);
+        }
+        await TaskRepo.destroy({
+          filterByTk: tasksToCleanup.map((task) => task.id),
+          individualHooks: true
+        });
+      }
+    } catch (error) {
+      this.logger.error(`DB error during cleanup: ${error.message}. Will stop cleanup timer.`, { error });
+      if (error.name === "SequelizeConnectionAcquireTimeoutError") {
+        if (this.cleanupTimer) {
+          clearInterval(this.cleanupTimer);
+          this.cleanupTimer = null;
+        }
       }
-    });
-    this.logger.debug(`Found ${tasksToCleanup.length} tasks to cleanup`);
-    for (const task of tasksToCleanup) {
-      this.tasks.delete(task.id);
-      this.progressThrottles.delete(task.id);
     }
-    await TaskRepo.destroy({
-      filterByTk: tasksToCleanup.map((task) => task.id),
-      individualHooks: true
-    });
   };
   getThrottledProgressEmitter(taskId, userId) {
     if (!this.progressThrottles.has(taskId)) {
@@ -314,7 +346,33 @@ class BaseTaskManager {
     }
   }
 }
+class ConcurrencyMonitorDelegate {
+  constructor(mode = CONCURRENCY_MODE, appConcurrencyMonitor = new import_base_concurrency_monitor.BaseConcurrencyMonitor(CONCURRENCY), processConcurrencyMonitor = PROCESS_CONCURRENCY_MONITOR) {
+    this.mode = mode;
+    this.appConcurrencyMonitor = appConcurrencyMonitor;
+    this.processConcurrencyMonitor = processConcurrencyMonitor;
+  }
+  get concurrencyMonitor() {
+    return this.mode === "process" ? this.processConcurrencyMonitor : this.appConcurrencyMonitor;
+  }
+  idle() {
+    return this.concurrencyMonitor.idle();
+  }
+  get concurrency() {
+    return this.concurrencyMonitor.concurrency;
+  }
+  set concurrency(concurrency) {
+    this.concurrencyMonitor.concurrency = concurrency;
+  }
+  increase(taskId) {
+    return this.concurrencyMonitor.increase(taskId);
+  }
+  decrease(taskId) {
+    this.concurrencyMonitor.decrease(taskId);
+  }
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  BaseTaskManager
+  BaseTaskManager,
+  ConcurrencyMonitorDelegate
 });

package/dist/server/command-task-type.d.ts

@@ -9,6 +9,7 @@
 /// <reference types="node" />
 import { Worker } from 'worker_threads';
 import { TaskType } from './task-type';
+export declare function parseArgv(list: string[]): any;
 export declare class CommandTaskType extends TaskType {
     static type: string;
     workerThread: Worker;

package/dist/server/command-task-type.js

@@ -36,7 +36,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var command_task_type_exports = {};
 __export(command_task_type_exports, {
-  CommandTaskType: () => CommandTaskType
+  CommandTaskType: () => CommandTaskType,
+  parseArgv: () => parseArgv
 });
 module.exports = __toCommonJS(command_task_type_exports);
 var import_async_task_manager = require("./interfaces/async-task-manager");
@@ -44,6 +45,51 @@ var import_node_process = __toESM(require("node:process"));
 var import_worker_threads = require("worker_threads");
 var import_path = __toESM(require("path"));
 var import_task_type = require("./task-type");
+const getResourceLimitsFromEnv = () => {
+  let resourceLimitsUndefined = true;
+  const resourceLimits = {};
+  if (import_node_process.default.env.ASYNC_TASK_WORKER_MAX_OLD) {
+    resourceLimits.maxOldGenerationSizeMb = Number.parseInt(import_node_process.default.env.ASYNC_TASK_WORKER_MAX_OLD, 10);
+    resourceLimitsUndefined = false;
+  }
+  if (import_node_process.default.env.ASYNC_TASK_WORKER_MAX_YOUNG) {
+    resourceLimits.maxYoungGenerationSizeMb = Number.parseInt(import_node_process.default.env.ASYNC_TASK_WORKER_MAX_YOUNG, 10);
+    resourceLimitsUndefined = false;
+  }
+  return resourceLimitsUndefined ? void 0 : resourceLimits;
+};
+const RESOURCE_LIMITS = getResourceLimitsFromEnv();
+function parseArgv(list) {
+  const argv = {};
+  for (const item of list) {
+    const match = item.match(/^--([^=]+)=(.*)$/);
+    if (match) {
+      const key = match[1];
+      let value = match[2];
+      if (value.startsWith("{") || value.startsWith("[")) {
+        try {
+          value = JSON.parse(value);
+        } catch (err) {
+        }
+      } else {
+        if (value === "true") {
+          value = true;
+        } else if (value === "false") {
+          value = false;
+        }
+      }
+      argv[key] = value;
+      continue;
+    }
+    const parts = item.split(":");
+    if (parts.length === 2) {
+      const command = parts[0];
+      const commandValue = parts[1];
+      argv[command] = commandValue;
+    }
+  }
+  return argv;
+}
 class CommandTaskType extends import_task_type.TaskType {
   static type = "command";
   workerThread;
@@ -55,6 +101,7 @@ class CommandTaskType extends import_task_type.TaskType {
   async execute() {
     var _a;
     const { argv } = this.record.params;
+    const parsedArgv = parseArgv(argv);
     const isDev = (((_a = import_node_process.default.argv[1]) == null ? void 0 : _a.endsWith(".ts")) || import_node_process.default.argv[1].includes("tinypool")) ?? false;
     const appRoot = import_node_process.default.env.APP_PACKAGE_ROOT || "packages/core/app";
     const workerPath = import_path.default.resolve(import_node_process.default.cwd(), appRoot, isDev ? "src/index.ts" : "lib/index.js");
@@ -73,8 +120,10 @@ class CommandTaskType extends import_task_type.TaskType {
           },
           env: {
             ...import_node_process.default.env,
-            WORKER_MODE: "-"
-          }
+            WORKER_MODE: "-",
+            ...parsedArgv.app && parsedArgv.app !== "main" ? { STARTUP_SUBAPP: parsedArgv.app } : {}
+          },
+          resourceLimits: RESOURCE_LIMITS
         });
         this.workerThread = worker;
         (_b = this.logger) == null ? void 0 : _b.debug(`Worker created successfully for task ${this.record.id}`);
@@ -128,5 +177,6 @@ class CommandTaskType extends import_task_type.TaskType {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  CommandTaskType
+  CommandTaskType,
+  parseArgv
 });

package/dist/server/interfaces/concurrency-monitor.d.ts

@@ -0,0 +1,16 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { TaskId } from '../../common/types';
+export type ConcurrencyMode = 'app' | 'process';
+export interface ConcurrencyMonitor {
+    idle(): boolean;
+    concurrency: number;
+    increase(taskId: TaskId): boolean;
+    decrease(taskId: TaskId): void;
+}

package/dist/server/interfaces/concurrency-monitor.js

@@ -0,0 +1,24 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var concurrency_monitor_exports = {};
+module.exports = __toCommonJS(concurrency_monitor_exports);

package/dist/server/task-type.js

@@ -116,11 +116,12 @@ class TaskType {
     try {
       const result = await this.execute();
       (_c = this.logger) == null ? void 0 : _c.info(`Task ${this.record.id} completed successfully with result: ${JSON.stringify(result)}`);
-      await this.record.update({
+      this.record.set({
         status: import_constants.TASK_STATUS.SUCCEEDED,
         doneAt: /* @__PURE__ */ new Date(),
         result
       });
+      await this.record.save();
     } catch (error) {
       if (error instanceof import_async_task_manager.CancelError) {
         (_d = this.logger) == null ? void 0 : _d.info(`Task ${this.record.id} was cancelled during execution`);
@@ -129,7 +130,7 @@ class TaskType {
         await this.record.update({
           status: import_constants.TASK_STATUS.FAILED,
           doneAt: /* @__PURE__ */ new Date(),
-          error: error.toString()
+          result: { ...error, message: error.message }
         });
         (_e = this.logger) == null ? void 0 : _e.error(`Task ${this.record.id} failed with error: ${error.message}`);
         throw error;

package/package.json

@@ -1,10 +1,12 @@
 {
   "name": "@nocobase/plugin-async-task-manager",
   "displayName": "Async task manager",
+  "displayName.ru-RU": "Менеджер асинхронных задач",
   "displayName.zh-CN": "异步任务管理器",
   "description": "Manage and monitor asynchronous tasks such as data import/export. Support task progress tracking and notification.",
+  "description.ru-RU": "Управление асинхронными задачами и мониторинг (например, импорт/экспорт данных). Поддержка отслеживания прогресса и уведомлений о задачах.",
   "description.zh-CN": "管理和监控数据导入导出等异步任务。支持任务进度跟踪和通知。",
-  "version": "2.0.0-alpha.9",
+  "version": "2.0.0-beta.10",
   "main": "dist/server/index.js",
   "peerDependencies": {
     "@nocobase/client": "2.x",
@@ -15,5 +17,5 @@
   "dependencies": {
     "p-queue": "^6.6.2"
   },
-  "gitHead": "4a9acf96f21a3aa35bccbd188b942595b09da0a9"
+  "gitHead": "9943dc4b0fdedcac3f304714b58635ae441e2560"
 }