@nocobase/plugin-workflow 2.1.0-beta.9 → 2.2.0-alpha.1

package/dist/server/ExecutionTimeoutManager.d.ts

@@ -0,0 +1,46 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Transactionable } from '@nocobase/database';
+import type PluginWorkflowServer from './Plugin';
+import type { ExecutionModel } from './types';
+export default class ExecutionTimeoutManager {
+    private readonly plugin;
+    private readonly timers;
+    private scanTimer;
+    private nextExpiresAtTimer;
+    private nextExpiresAt;
+    private scanning;
+    private stopped;
+    constructor(plugin: PluginWorkflowServer);
+    getTimeout(execution: ExecutionModel): number;
+    getExpiresAt(execution: ExecutionModel, startedAt: Date): Date;
+    load(): Promise<void>;
+    unload(): Promise<void>;
+    isExpired(execution: ExecutionModel, now?: Date): boolean;
+    getRemainingMs(execution: ExecutionModel, now?: Date): number;
+    abort(execution: ExecutionModel, options?: Transactionable): Promise<boolean>;
+    abortExecutionIfExpired(execution: ExecutionModel, options?: Transactionable): Promise<boolean>;
+    private abortExecutionIfExpiredWithLock;
+    clear(executionId: number | string): void;
+    shouldContinue(execution: ExecutionModel, options?: Transactionable): Promise<boolean>;
+    /**
+     * Owner-only per-execution timer. Only call from code paths that have acquired
+     * local execution ownership, such as Dispatcher/Processor processing paths.
+     */
+    scheduleExecutionTimeout(execution: ExecutionModel): void;
+    invalidateNextExpiresAtIfMatches(expiresAt?: Date | null): void;
+    private scanExpiredExecutions;
+    private scheduleScan;
+    private scheduleNextExpiresAtTimer;
+    private scheduleNextExpiresAtIfEarlier;
+    private handleNextExpiresAtTimeout;
+    private clearNextExpiresAtTimer;
+    private clearExecutionTimeout;
+    private handleExecutionTimeout;
+}

package/dist/server/ExecutionTimeoutManager.js

@@ -0,0 +1,320 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ExecutionTimeoutManager_exports = {};
+__export(ExecutionTimeoutManager_exports, {
+  default: () => ExecutionTimeoutManager
+});
+module.exports = __toCommonJS(ExecutionTimeoutManager_exports);
+var import_database = require("@nocobase/database");
+var import_constants = require("./constants");
+var import_utils = require("./utils");
+const LOAD_BATCH_SIZE = 1e3;
+const DEFAULT_SCAN_INTERVAL = 3e4;
+const SCAN_JITTER = 5e3;
+const MAX_TIMER_DELAY = 2147483647;
+const EXECUTION_LOCK_TIMEOUT = 6e4;
+class ExecutionTimeoutManager {
+  constructor(plugin) {
+    this.plugin = plugin;
+  }
+  timers = /* @__PURE__ */ new Map();
+  scanTimer = null;
+  nextExpiresAtTimer = null;
+  nextExpiresAt = null;
+  scanning = null;
+  stopped = true;
+  getTimeout(execution) {
+    var _a, _b;
+    const timeout = Number(((_b = (_a = execution.workflow) == null ? void 0 : _a.options) == null ? void 0 : _b.timeout) ?? 0);
+    return Number.isFinite(timeout) && timeout > 0 ? timeout : 0;
+  }
+  getExpiresAt(execution, startedAt) {
+    const timeout = this.getTimeout(execution);
+    return timeout > 0 ? new Date(startedAt.getTime() + timeout) : null;
+  }
+  async load() {
+    this.stopped = false;
+    await this.scanExpiredExecutions();
+    await this.scheduleNextExpiresAtTimer();
+    this.scheduleScan();
+  }
+  async unload() {
+    var _a;
+    this.stopped = true;
+    if (this.scanTimer) {
+      clearTimeout(this.scanTimer);
+      this.scanTimer = null;
+    }
+    if (this.nextExpiresAtTimer) {
+      clearTimeout(this.nextExpiresAtTimer);
+      this.nextExpiresAtTimer = null;
+      this.nextExpiresAt = null;
+    }
+    for (const timer of this.timers.values()) {
+      clearTimeout(timer);
+    }
+    this.timers.clear();
+    await ((_a = this.scanning) == null ? void 0 : _a.catch(() => {
+    }));
+  }
+  isExpired(execution, now = /* @__PURE__ */ new Date()) {
+    return !!execution.expiresAt && execution.expiresAt.getTime() <= now.getTime();
+  }
+  getRemainingMs(execution, now = /* @__PURE__ */ new Date()) {
+    if (!execution.expiresAt) {
+      return null;
+    }
+    return execution.expiresAt.getTime() - now.getTime();
+  }
+  async abort(execution, options = {}) {
+    const aborted = await (0, import_utils.abortExecution)(this.plugin, execution, {
+      ...options,
+      reason: import_constants.EXECUTION_REASON.TIMEOUT
+    });
+    if (aborted) {
+      this.clearExecutionTimeout(execution.id);
+    }
+    return aborted;
+  }
+  async abortExecutionIfExpired(execution, options = {}) {
+    if (execution.status !== import_constants.EXECUTION_STATUS.STARTED || !this.isExpired(execution)) {
+      return false;
+    }
+    return this.abort(execution, options);
+  }
+  async abortExecutionIfExpiredWithLock(executionOrId) {
+    const executionId = typeof executionOrId === "object" ? executionOrId.id : executionOrId;
+    const lock = await this.plugin.app.lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(executionId), EXECUTION_LOCK_TIMEOUT);
+    return lock.runExclusive(async () => {
+      const execution = await this.plugin.db.getRepository("executions").findOne({
+        filterByTk: executionId
+      });
+      if (!execution) {
+        return false;
+      }
+      return this.abortExecutionIfExpired(execution);
+    }, EXECUTION_LOCK_TIMEOUT);
+  }
+  clear(executionId) {
+    this.clearExecutionTimeout(executionId);
+  }
+  async shouldContinue(execution, options = {}) {
+    if (execution.status !== import_constants.EXECUTION_STATUS.STARTED) {
+      return false;
+    }
+    if (!this.isExpired(execution)) {
+      return true;
+    }
+    await this.abortExecutionIfExpired(execution, options);
+    return false;
+  }
+  /**
+   * Owner-only per-execution timer. Only call from code paths that have acquired
+   * local execution ownership, such as Dispatcher/Processor processing paths.
+   */
+  scheduleExecutionTimeout(execution) {
+    const id = String(execution.id);
+    this.clearExecutionTimeout(id);
+    if (execution.status !== import_constants.EXECUTION_STATUS.STARTED || !execution.expiresAt) {
+      return;
+    }
+    this.scheduleNextExpiresAtIfEarlier(execution.expiresAt);
+    const remaining = this.getRemainingMs(execution);
+    if (remaining == null) {
+      return;
+    }
+    if (remaining <= 0) {
+      setImmediate(() => {
+        this.handleExecutionTimeout(id).catch((error) => {
+          this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) timeout handling failed`, {
+            error
+          });
+        });
+      });
+      return;
+    }
+    this.timers.set(
+      id,
+      setTimeout(
+        () => {
+          if (execution.expiresAt && execution.expiresAt.getTime() > Date.now()) {
+            this.scheduleExecutionTimeout(execution);
+            return;
+          }
+          this.handleExecutionTimeout(id).catch((error) => {
+            this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) timeout handling failed`, {
+              error
+            });
+          });
+        },
+        Math.min(remaining, MAX_TIMER_DELAY)
+      )
+    );
+  }
+  invalidateNextExpiresAtIfMatches(expiresAt) {
+    if (!expiresAt || !this.nextExpiresAt || this.nextExpiresAt.getTime() !== expiresAt.getTime()) {
+      return;
+    }
+    this.clearNextExpiresAtTimer();
+    this.scheduleNextExpiresAtTimer().catch((error) => {
+      this.plugin.getLogger("timeout").error(`workflow execution timeout one-shot refresh failed`, { error });
+    });
+  }
+  async scanExpiredExecutions() {
+    if (this.scanning) {
+      return this.scanning;
+    }
+    if (this.stopped) {
+      return;
+    }
+    this.scanning = (async () => {
+      const ExecutionModelClass = this.plugin.db.getModel("executions");
+      const executions = await ExecutionModelClass.findAll({
+        attributes: ["id", "workflowId", "status", "expiresAt"],
+        where: {
+          status: import_constants.EXECUTION_STATUS.STARTED,
+          expiresAt: {
+            [import_database.Op.not]: null,
+            [import_database.Op.lt]: /* @__PURE__ */ new Date()
+          }
+        },
+        order: [
+          ["expiresAt", "ASC"],
+          ["id", "ASC"]
+        ],
+        limit: LOAD_BATCH_SIZE
+      });
+      for (const execution of executions) {
+        await this.abortExecutionIfExpiredWithLock(execution);
+      }
+    })();
+    try {
+      await this.scanning;
+    } finally {
+      this.scanning = null;
+    }
+  }
+  scheduleScan() {
+    if (this.stopped) {
+      return;
+    }
+    if (this.scanTimer) {
+      clearTimeout(this.scanTimer);
+    }
+    this.scanTimer = setTimeout(
+      async () => {
+        this.scanTimer = null;
+        if (this.stopped) {
+          return;
+        }
+        try {
+          await this.scanExpiredExecutions();
+          this.scheduleNextExpiresAtTimer();
+        } catch (error) {
+          this.plugin.getLogger("timeout").error(`workflow execution timeout scan failed`, { error });
+        } finally {
+          this.scheduleScan();
+        }
+      },
+      DEFAULT_SCAN_INTERVAL + Math.floor(Math.random() * SCAN_JITTER)
+    );
+  }
+  async scheduleNextExpiresAtTimer() {
+    if (this.stopped) {
+      return;
+    }
+    const ExecutionModelClass = this.plugin.db.getModel("executions");
+    const execution = await ExecutionModelClass.findOne({
+      attributes: ["id", "expiresAt"],
+      where: {
+        status: import_constants.EXECUTION_STATUS.STARTED,
+        expiresAt: {
+          [import_database.Op.not]: null
+        }
+      },
+      order: [
+        ["expiresAt", "ASC"],
+        ["id", "ASC"]
+      ]
+    });
+    if (!(execution == null ? void 0 : execution.expiresAt)) {
+      this.clearNextExpiresAtTimer();
+      return;
+    }
+    this.scheduleNextExpiresAtIfEarlier(execution.expiresAt, true);
+  }
+  scheduleNextExpiresAtIfEarlier(expiresAt, force = false) {
+    if (this.stopped) {
+      return;
+    }
+    if (!force && this.nextExpiresAt && this.nextExpiresAt.getTime() <= expiresAt.getTime()) {
+      return;
+    }
+    this.clearNextExpiresAtTimer();
+    this.nextExpiresAt = expiresAt;
+    const remaining = expiresAt.getTime() - Date.now();
+    const delay = Math.max(0, Math.min(remaining, MAX_TIMER_DELAY));
+    this.nextExpiresAtTimer = setTimeout(() => {
+      this.handleNextExpiresAtTimeout(expiresAt).catch((error) => {
+        this.plugin.getLogger("timeout").error(`workflow execution timeout one-shot failed`, { error });
+      });
+    }, delay);
+  }
+  async handleNextExpiresAtTimeout(expiresAt) {
+    this.nextExpiresAtTimer = null;
+    this.nextExpiresAt = null;
+    if (expiresAt.getTime() > Date.now()) {
+      this.scheduleNextExpiresAtIfEarlier(expiresAt, true);
+      return;
+    }
+    await this.scanExpiredExecutions();
+    await this.scheduleNextExpiresAtTimer();
+  }
+  clearNextExpiresAtTimer() {
+    if (this.nextExpiresAtTimer) {
+      clearTimeout(this.nextExpiresAtTimer);
+      this.nextExpiresAtTimer = null;
+    }
+    this.nextExpiresAt = null;
+  }
+  clearExecutionTimeout(executionId) {
+    const id = String(executionId);
+    const timer = this.timers.get(id);
+    if (!timer) {
+      return;
+    }
+    clearTimeout(timer);
+    this.timers.delete(id);
+  }
+  async handleExecutionTimeout(executionId) {
+    this.clearExecutionTimeout(executionId);
+    if (this.plugin.abortRunningExecution(executionId, import_constants.EXECUTION_REASON.TIMEOUT)) {
+      return;
+    }
+    await this.abortExecutionIfExpiredWithLock(executionId);
+  }
+}

package/dist/server/Plugin.d.ts

@@ -13,10 +13,13 @@ import { Registry } from '@nocobase/utils';
 import { Logger } from '@nocobase/logger';
 import Dispatcher, { EventOptions } from './Dispatcher';
 import Processor from './Processor';
+import ExecutionTimeoutManager from './ExecutionTimeoutManager';
+import RunningExecutionRegistry from './RunningExecutionRegistry';
 import { CustomFunction } from './functions';
 import Trigger from './triggers';
 import { InstructionInterface } from './instructions';
 import type { ExecutionModel, WorkflowModel } from './types';
+import type { Transaction } from '@nocobase/database';
 type ID = number | string;
 export declare const WORKER_JOB_WORKFLOW_PROCESS = "workflow:process";
 export default class PluginWorkflowServer extends Plugin {
@@ -26,6 +29,8 @@ export default class PluginWorkflowServer extends Plugin {
     enabledCache: Map<number, WorkflowModel>;
     snowflake: Snowflake;
     private dispatcher;
+    readonly timeoutManager: ExecutionTimeoutManager;
+    readonly runningExecutionRegistry: RunningExecutionRegistry;
     get channelPendingExecution(): string;
     private loggerCache;
     private meter;
@@ -38,6 +43,12 @@ export default class PluginWorkflowServer extends Plugin {
     private onBeforeStop;
     handleSyncMessage(message: any): Promise<void>;
     serving(): boolean;
+    abortExecutionIfExpired(execution: ExecutionModel, options?: {
+        transaction?: Transaction;
+    }): Promise<boolean>;
+    registerRunningExecution(executionId: number | string, abort: (reason?: string) => void): void;
+    unregisterRunningExecution(executionId: number | string): void;
+    abortRunningExecution(executionId: number | string, reason?: string): boolean;
     /**
      * @experimental
      */
@@ -56,6 +67,7 @@ export default class PluginWorkflowServer extends Plugin {
     }): void;
     private initTriggers;
     private initInstructions;
+    private registerErrorHandlers;
     beforeLoad(): Promise<void>;
     /**
      * @internal
@@ -64,6 +76,7 @@ export default class PluginWorkflowServer extends Plugin {
     private toggle;
     trigger(workflow: WorkflowModel, context: object, options?: EventOptions): void | Promise<Processor | null>;
     run(pending: Parameters<Dispatcher['run']>[0]): Promise<void>;
+    dispatch(): void;
     resume(job: any): Promise<void>;
     /**
      * Start a deferred execution
@@ -76,10 +89,11 @@ export default class PluginWorkflowServer extends Plugin {
      * @experimental
      * @param {string} dataSourceName
      * @param {Transaction} transaction
-     * @param {boolean} create
-     * @returns {Trasaction}
+     * @param {boolean} create Create a new transaction when the input transaction does not belong to this data source.
+     * @returns {Transaction}
      */
-    useDataSourceTransaction(dataSourceName: string, transaction: any, create?: boolean): any;
+    useDataSourceTransaction(dataSourceName?: string, transaction?: Transaction | null, create?: false): Transaction | undefined;
+    useDataSourceTransaction(dataSourceName: string | undefined, transaction: Transaction | null | undefined, create: true): Transaction | Promise<Transaction> | undefined;
     /**
      * @experimental
      */

package/dist/server/Plugin.js

@@ -42,13 +42,17 @@ __export(Plugin_exports, {
 module.exports = __toCommonJS(Plugin_exports);
 var import_path = __toESM(require("path"));
 var import_nodejs_snowflake = require("nodejs-snowflake");
-var import_lru_cache = __toESM(require("lru-cache"));
+var import_lru_cache = require("lru-cache");
 var import_database = require("@nocobase/database");
 var import_server = require("@nocobase/server");
 var import_utils = require("@nocobase/utils");
 var import_Dispatcher = __toESM(require("./Dispatcher"));
 var import_Processor = __toESM(require("./Processor"));
+var import_ExecutionTimeoutManager = __toESM(require("./ExecutionTimeoutManager"));
+var import_RunningExecutionRegistry = __toESM(require("./RunningExecutionRegistry"));
 var import_actions = __toESM(require("./actions"));
+var import_nodes = require("./actions/nodes");
+var import_workflows = require("./actions/workflows");
 var import_functions = __toESM(require("./functions"));
 var import_CollectionTrigger = __toESM(require("./triggers/CollectionTrigger"));
 var import_ScheduleTrigger = __toESM(require("./triggers/ScheduleTrigger"));
@@ -70,6 +74,8 @@ class PluginWorkflowServer extends import_server.Plugin {
   enabledCache = /* @__PURE__ */ new Map();
   snowflake;
   dispatcher = new import_Dispatcher.default(this);
+  timeoutManager = new import_ExecutionTimeoutManager.default(this);
+  runningExecutionRegistry = new import_RunningExecutionRegistry.default();
   get channelPendingExecution() {
     return `${this.name}.pendingExecution`;
   }
@@ -164,9 +170,9 @@ class PluginWorkflowServer extends import_server.Plugin {
       }
     }
     this.checker = setInterval(() => {
-      this.getLogger("dispatcher").debug(`(cycling) check for queueing executions`);
       this.dispatcher.dispatch();
     }, 3e5);
+    await this.timeoutManager.load();
     this.app.on("workflow:dispatch", () => {
       this.app.logger.info("workflow:dispatch");
       this.dispatcher.dispatch();
@@ -179,6 +185,7 @@ class PluginWorkflowServer extends import_server.Plugin {
     if (this.checker) {
       clearInterval(this.checker);
     }
+    await this.timeoutManager.unload();
     await this.dispatcher.beforeStop();
     this.app.logger.info(`stopping workflow plugin before app (${this.app.name}) shutdown...`);
     for (const workflow of this.enabledCache.values()) {
@@ -212,6 +219,18 @@ class PluginWorkflowServer extends import_server.Plugin {
   serving() {
     return this.app.serving(WORKER_JOB_WORKFLOW_PROCESS);
   }
+  async abortExecutionIfExpired(execution, options = {}) {
+    return this.timeoutManager.abortExecutionIfExpired(execution, options);
+  }
+  registerRunningExecution(executionId, abort) {
+    this.runningExecutionRegistry.register(executionId, { abort });
+  }
+  unregisterRunningExecution(executionId) {
+    this.runningExecutionRegistry.unregister(executionId);
+  }
+  abortRunningExecution(executionId, reason) {
+    return this.runningExecutionRegistry.abort(executionId, reason);
+  }
   /**
    * @experimental
    */
@@ -280,6 +299,29 @@ class PluginWorkflowServer extends import_server.Plugin {
       this.registerInstruction(name, instruction);
     }
   }
+  registerErrorHandlers() {
+    const PluginErrorHandler = this.app.pm.get("error-handler");
+    if (PluginErrorHandler == null ? void 0 : PluginErrorHandler.errorHandler) {
+      PluginErrorHandler.errorHandler.register(
+        (err) => err instanceof import_nodes.NodeValidationError || err.name === "NodeValidationError",
+        (err, ctx) => {
+          ctx.status = err.status;
+          ctx.body = {
+            errors: Object.values(err.errors).map((message) => ({ message }))
+          };
+        }
+      );
+      PluginErrorHandler.errorHandler.register(
+        (err) => err instanceof import_workflows.WorkflowValidationError || err.name === "WorkflowValidationError",
+        (err, ctx) => {
+          ctx.status = err.status;
+          ctx.body = {
+            errors: Object.values(err.errors).map((message) => ({ message }))
+          };
+        }
+      );
+    }
+  }
   async beforeLoad() {
     this.db.registerRepositories({
       WorkflowRepository: import_WorkflowRepository.default
@@ -302,10 +344,11 @@ class PluginWorkflowServer extends import_server.Plugin {
     this.initTriggers(options.triggers);
     this.initInstructions(options.instructions);
     (0, import_functions.default)(this, options.functions);
+    this.registerErrorHandlers();
     this.functions.register("instanceId", () => this.app.instanceId);
     this.functions.register("epoch", () => 1605024e3);
     this.functions.register("genSnowflakeId", () => this.app.snowflakeIdGenerator.generate());
-    this.loggerCache = new import_lru_cache.default({
+    this.loggerCache = new import_lru_cache.LRUCache({
       max: 20,
       updateAgeOnGet: true,
       dispose(logger) {
@@ -350,7 +393,6 @@ class PluginWorkflowServer extends import_server.Plugin {
       actions: ["workflows:list"]
     });
     this.app.acl.allow("userWorkflowTasks", "listMine", "loggedIn");
-    this.app.acl.allow("*", ["trigger"], "loggedIn");
     db.on("workflows.beforeSave", this.onBeforeSave);
     db.on("workflows.afterCreate", this.onAfterCreate);
     db.on("workflows.afterUpdate", this.onAfterUpdate);
@@ -409,6 +451,9 @@ class PluginWorkflowServer extends import_server.Plugin {
   async run(pending) {
     return this.dispatcher.run(pending);
   }
+  dispatch() {
+    return this.dispatcher.dispatch();
+  }
   async resume(job) {
     return this.dispatcher.resume(job);
   }
@@ -432,15 +477,12 @@ class PluginWorkflowServer extends import_server.Plugin {
     }
     return trigger.execute(workflow, values, options);
   }
-  /**
-   * @experimental
-   * @param {string} dataSourceName
-   * @param {Transaction} transaction
-   * @param {boolean} create
-   * @returns {Trasaction}
-   */
   useDataSourceTransaction(dataSourceName = "main", transaction, create = false) {
-    const { db } = this.app.dataSourceManager.dataSources.get(dataSourceName).collectionManager;
+    const dataSource = this.app.dataSourceManager.dataSources.get(dataSourceName);
+    if (!dataSource) {
+      throw new Error(`data source ${dataSourceName} is not found`);
+    }
+    const { db } = dataSource.collectionManager;
     if (!db) {
       return;
     }