@nocobase/plugin-workflow 2.1.0-beta.37 → 2.1.0-beta.38

package/dist/server/ExecutionTimeoutManager.js

@@ -0,0 +1,312 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ExecutionTimeoutManager_exports = {};
+__export(ExecutionTimeoutManager_exports, {
+  default: () => ExecutionTimeoutManager
+});
+module.exports = __toCommonJS(ExecutionTimeoutManager_exports);
+var import_database = require("@nocobase/database");
+var import_constants = require("./constants");
+var import_utils = require("./utils");
+const LOAD_BATCH_SIZE = 1e3;
+const DEFAULT_SCAN_INTERVAL = 3e4;
+const SCAN_JITTER = 5e3;
+const MAX_TIMER_DELAY = 2147483647;
+class ExecutionTimeoutManager {
+  constructor(plugin) {
+    this.plugin = plugin;
+  }
+  timers = /* @__PURE__ */ new Map();
+  scanTimer = null;
+  nextExpiresAtTimer = null;
+  nextExpiresAt = null;
+  scanning = null;
+  stopped = true;
+  getTimeout(execution) {
+    var _a, _b;
+    const timeout = Number(((_b = (_a = execution.workflow) == null ? void 0 : _a.options) == null ? void 0 : _b.timeout) ?? 0);
+    return Number.isFinite(timeout) && timeout > 0 ? timeout : 0;
+  }
+  getExpiresAt(execution, startedAt) {
+    const timeout = this.getTimeout(execution);
+    return timeout > 0 ? new Date(startedAt.getTime() + timeout) : null;
+  }
+  async load() {
+    this.stopped = false;
+    await this.scanExpiredExecutions();
+    await this.scheduleNextExpiresAtTimer();
+    this.scheduleScan();
+  }
+  async unload() {
+    var _a;
+    this.stopped = true;
+    if (this.scanTimer) {
+      clearTimeout(this.scanTimer);
+      this.scanTimer = null;
+    }
+    if (this.nextExpiresAtTimer) {
+      clearTimeout(this.nextExpiresAtTimer);
+      this.nextExpiresAtTimer = null;
+      this.nextExpiresAt = null;
+    }
+    for (const timer of this.timers.values()) {
+      clearTimeout(timer);
+    }
+    this.timers.clear();
+    await ((_a = this.scanning) == null ? void 0 : _a.catch(() => {
+    }));
+  }
+  isExpired(execution, now = /* @__PURE__ */ new Date()) {
+    return !!execution.expiresAt && execution.expiresAt.getTime() <= now.getTime();
+  }
+  getRemainingMs(execution, now = /* @__PURE__ */ new Date()) {
+    if (!execution.expiresAt) {
+      return null;
+    }
+    return execution.expiresAt.getTime() - now.getTime();
+  }
+  async abort(execution, options = {}) {
+    const aborted = await (0, import_utils.abortExecution)(this.plugin, execution, {
+      ...options,
+      reason: import_constants.EXECUTION_REASON.TIMEOUT
+    });
+    if (aborted) {
+      this.clearExecutionTimeout(execution.id);
+    }
+    return aborted;
+  }
+  async abortExecutionIfExpired(execution, options = {}) {
+    if (execution.status !== import_constants.EXECUTION_STATUS.STARTED || !this.isExpired(execution)) {
+      return false;
+    }
+    return this.abort(execution, options);
+  }
+  clear(executionId) {
+    this.clearExecutionTimeout(executionId);
+  }
+  async shouldContinue(execution, options = {}) {
+    if (execution.status !== import_constants.EXECUTION_STATUS.STARTED) {
+      return false;
+    }
+    if (!this.isExpired(execution)) {
+      return true;
+    }
+    await this.abortExecutionIfExpired(execution, options);
+    return false;
+  }
+  /**
+   * Owner-only per-execution timer. Only call from code paths that have acquired
+   * local execution ownership, such as Dispatcher/Processor processing paths.
+   */
+  scheduleExecutionTimeout(execution) {
+    const id = String(execution.id);
+    this.clearExecutionTimeout(id);
+    if (execution.status !== import_constants.EXECUTION_STATUS.STARTED || !execution.expiresAt) {
+      return;
+    }
+    this.scheduleNextExpiresAtIfEarlier(execution.expiresAt);
+    const remaining = this.getRemainingMs(execution);
+    if (remaining == null) {
+      return;
+    }
+    if (remaining <= 0) {
+      setImmediate(() => {
+        this.handleExecutionTimeout(id).catch((error) => {
+          this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) timeout handling failed`, {
+            error
+          });
+        });
+      });
+      return;
+    }
+    this.timers.set(
+      id,
+      setTimeout(
+        () => {
+          if (execution.expiresAt && execution.expiresAt.getTime() > Date.now()) {
+            this.scheduleExecutionTimeout(execution);
+            return;
+          }
+          this.handleExecutionTimeout(id).catch((error) => {
+            this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) timeout handling failed`, {
+              error
+            });
+          });
+        },
+        Math.min(remaining, MAX_TIMER_DELAY)
+      )
+    );
+  }
+  invalidateNextExpiresAtIfMatches(expiresAt) {
+    if (!expiresAt || !this.nextExpiresAt || this.nextExpiresAt.getTime() !== expiresAt.getTime()) {
+      return;
+    }
+    this.clearNextExpiresAtTimer();
+    this.scheduleNextExpiresAtTimer().catch((error) => {
+      this.plugin.getLogger("timeout").error(`workflow execution timeout one-shot refresh failed`, { error });
+    });
+  }
+  async scanExpiredExecutions() {
+    if (this.scanning) {
+      return this.scanning;
+    }
+    if (this.stopped) {
+      return;
+    }
+    this.scanning = (async () => {
+      const ExecutionModelClass = this.plugin.db.getModel("executions");
+      const executions = await ExecutionModelClass.findAll({
+        attributes: ["id", "workflowId", "status", "expiresAt"],
+        where: {
+          status: import_constants.EXECUTION_STATUS.STARTED,
+          expiresAt: {
+            [import_database.Op.not]: null,
+            [import_database.Op.lt]: /* @__PURE__ */ new Date()
+          }
+        },
+        order: [
+          ["expiresAt", "ASC"],
+          ["id", "ASC"]
+        ],
+        limit: LOAD_BATCH_SIZE
+      });
+      for (const execution of executions) {
+        await this.abortExecutionIfExpired(execution);
+      }
+    })();
+    try {
+      await this.scanning;
+    } finally {
+      this.scanning = null;
+    }
+  }
+  scheduleScan() {
+    if (this.stopped) {
+      return;
+    }
+    if (this.scanTimer) {
+      clearTimeout(this.scanTimer);
+    }
+    this.scanTimer = setTimeout(
+      async () => {
+        this.scanTimer = null;
+        if (this.stopped) {
+          return;
+        }
+        try {
+          await this.scanExpiredExecutions();
+          this.scheduleNextExpiresAtTimer();
+        } catch (error) {
+          this.plugin.getLogger("timeout").error(`workflow execution timeout scan failed`, { error });
+        } finally {
+          this.scheduleScan();
+        }
+      },
+      DEFAULT_SCAN_INTERVAL + Math.floor(Math.random() * SCAN_JITTER)
+    );
+  }
+  async scheduleNextExpiresAtTimer() {
+    if (this.stopped) {
+      return;
+    }
+    const ExecutionModelClass = this.plugin.db.getModel("executions");
+    const execution = await ExecutionModelClass.findOne({
+      attributes: ["id", "expiresAt"],
+      where: {
+        status: import_constants.EXECUTION_STATUS.STARTED,
+        expiresAt: {
+          [import_database.Op.not]: null
+        }
+      },
+      order: [
+        ["expiresAt", "ASC"],
+        ["id", "ASC"]
+      ]
+    });
+    if (!(execution == null ? void 0 : execution.expiresAt)) {
+      this.clearNextExpiresAtTimer();
+      return;
+    }
+    this.scheduleNextExpiresAtIfEarlier(execution.expiresAt, true);
+  }
+  scheduleNextExpiresAtIfEarlier(expiresAt, force = false) {
+    if (this.stopped) {
+      return;
+    }
+    if (!force && this.nextExpiresAt && this.nextExpiresAt.getTime() <= expiresAt.getTime()) {
+      return;
+    }
+    this.clearNextExpiresAtTimer();
+    this.nextExpiresAt = expiresAt;
+    const remaining = expiresAt.getTime() - Date.now();
+    const delay = Math.max(0, Math.min(remaining, MAX_TIMER_DELAY));
+    this.nextExpiresAtTimer = setTimeout(() => {
+      this.handleNextExpiresAtTimeout(expiresAt).catch((error) => {
+        this.plugin.getLogger("timeout").error(`workflow execution timeout one-shot failed`, { error });
+      });
+    }, delay);
+  }
+  async handleNextExpiresAtTimeout(expiresAt) {
+    this.nextExpiresAtTimer = null;
+    this.nextExpiresAt = null;
+    if (expiresAt.getTime() > Date.now()) {
+      this.scheduleNextExpiresAtIfEarlier(expiresAt, true);
+      return;
+    }
+    await this.scanExpiredExecutions();
+    await this.scheduleNextExpiresAtTimer();
+  }
+  clearNextExpiresAtTimer() {
+    if (this.nextExpiresAtTimer) {
+      clearTimeout(this.nextExpiresAtTimer);
+      this.nextExpiresAtTimer = null;
+    }
+    this.nextExpiresAt = null;
+  }
+  clearExecutionTimeout(executionId) {
+    const id = String(executionId);
+    const timer = this.timers.get(id);
+    if (!timer) {
+      return;
+    }
+    clearTimeout(timer);
+    this.timers.delete(id);
+  }
+  async handleExecutionTimeout(executionId) {
+    this.clearExecutionTimeout(executionId);
+    if (this.plugin.abortRunningExecution(executionId, import_constants.EXECUTION_REASON.TIMEOUT)) {
+      return;
+    }
+    const execution = await this.plugin.db.getRepository("executions").findOne({
+      filterByTk: executionId
+    });
+    if (!execution) {
+      return;
+    }
+    await this.abortExecutionIfExpired(execution);
+  }
+}

package/dist/server/Plugin.d.ts

@@ -13,10 +13,13 @@ import { Registry } from '@nocobase/utils';
 import { Logger } from '@nocobase/logger';
 import Dispatcher, { EventOptions } from './Dispatcher';
 import Processor from './Processor';
+import ExecutionTimeoutManager from './ExecutionTimeoutManager';
+import RunningExecutionRegistry from './RunningExecutionRegistry';
 import { CustomFunction } from './functions';
 import Trigger from './triggers';
 import { InstructionInterface } from './instructions';
 import type { ExecutionModel, WorkflowModel } from './types';
+import type { Transaction } from '@nocobase/database';
 type ID = number | string;
 export declare const WORKER_JOB_WORKFLOW_PROCESS = "workflow:process";
 export default class PluginWorkflowServer extends Plugin {
@@ -26,6 +29,8 @@ export default class PluginWorkflowServer extends Plugin {
     enabledCache: Map<number, WorkflowModel>;
     snowflake: Snowflake;
     private dispatcher;
+    readonly timeoutManager: ExecutionTimeoutManager;
+    readonly runningExecutionRegistry: RunningExecutionRegistry;
     get channelPendingExecution(): string;
     private loggerCache;
     private meter;
@@ -38,6 +43,12 @@ export default class PluginWorkflowServer extends Plugin {
     private onBeforeStop;
     handleSyncMessage(message: any): Promise<void>;
     serving(): boolean;
+    abortExecutionIfExpired(execution: ExecutionModel, options?: {
+        transaction?: Transaction;
+    }): Promise<boolean>;
+    registerRunningExecution(executionId: number | string, abort: (reason?: string) => void): void;
+    unregisterRunningExecution(executionId: number | string): void;
+    abortRunningExecution(executionId: number | string, reason?: string): boolean;
     /**
      * @experimental
      */
@@ -65,6 +76,7 @@ export default class PluginWorkflowServer extends Plugin {
     private toggle;
     trigger(workflow: WorkflowModel, context: object, options?: EventOptions): void | Promise<Processor | null>;
     run(pending: Parameters<Dispatcher['run']>[0]): Promise<void>;
+    dispatch(): void;
     resume(job: any): Promise<void>;
     /**
      * Start a deferred execution

package/dist/server/Plugin.js

@@ -48,6 +48,8 @@ var import_server = require("@nocobase/server");
 var import_utils = require("@nocobase/utils");
 var import_Dispatcher = __toESM(require("./Dispatcher"));
 var import_Processor = __toESM(require("./Processor"));
+var import_ExecutionTimeoutManager = __toESM(require("./ExecutionTimeoutManager"));
+var import_RunningExecutionRegistry = __toESM(require("./RunningExecutionRegistry"));
 var import_actions = __toESM(require("./actions"));
 var import_nodes = require("./actions/nodes");
 var import_workflows = require("./actions/workflows");
@@ -72,6 +74,8 @@ class PluginWorkflowServer extends import_server.Plugin {
   enabledCache = /* @__PURE__ */ new Map();
   snowflake;
   dispatcher = new import_Dispatcher.default(this);
+  timeoutManager = new import_ExecutionTimeoutManager.default(this);
+  runningExecutionRegistry = new import_RunningExecutionRegistry.default();
   get channelPendingExecution() {
     return `${this.name}.pendingExecution`;
   }
@@ -166,9 +170,9 @@ class PluginWorkflowServer extends import_server.Plugin {
       }
     }
     this.checker = setInterval(() => {
-      this.getLogger("dispatcher").debug(`(cycling) check for queueing executions`);
       this.dispatcher.dispatch();
     }, 3e5);
+    await this.timeoutManager.load();
     this.app.on("workflow:dispatch", () => {
       this.app.logger.info("workflow:dispatch");
       this.dispatcher.dispatch();
@@ -181,6 +185,7 @@ class PluginWorkflowServer extends import_server.Plugin {
     if (this.checker) {
       clearInterval(this.checker);
     }
+    await this.timeoutManager.unload();
     await this.dispatcher.beforeStop();
     this.app.logger.info(`stopping workflow plugin before app (${this.app.name}) shutdown...`);
     for (const workflow of this.enabledCache.values()) {
@@ -214,6 +219,18 @@ class PluginWorkflowServer extends import_server.Plugin {
   serving() {
     return this.app.serving(WORKER_JOB_WORKFLOW_PROCESS);
   }
+  async abortExecutionIfExpired(execution, options = {}) {
+    return this.timeoutManager.abortExecutionIfExpired(execution, options);
+  }
+  registerRunningExecution(executionId, abort) {
+    this.runningExecutionRegistry.register(executionId, { abort });
+  }
+  unregisterRunningExecution(executionId) {
+    this.runningExecutionRegistry.unregister(executionId);
+  }
+  abortRunningExecution(executionId, reason) {
+    return this.runningExecutionRegistry.abort(executionId, reason);
+  }
   /**
    * @experimental
    */
@@ -434,6 +451,9 @@ class PluginWorkflowServer extends import_server.Plugin {
   async run(pending) {
     return this.dispatcher.run(pending);
   }
+  dispatch() {
+    return this.dispatcher.dispatch();
+  }
   async resume(job) {
     return this.dispatcher.resume(job);
   }

package/dist/server/Processor.d.ts

@@ -6,14 +6,29 @@
  * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
+/// <reference types="node" />
 import { Transaction, Transactionable } from '@nocobase/database';
 import { Logger } from '@nocobase/logger';
 import type Plugin from './Plugin';
+import { IJob } from './instructions';
 import type { ExecutionModel, FlowNodeModel, JobModel } from './types';
+export type ProcessorRunOptions = {
+    rerun?: true;
+    signal?: AbortSignal;
+};
+export type ProcessorRerunOptions = {
+    nodeId?: string | number;
+    overwrite?: boolean;
+};
 export interface ProcessorOptions extends Transactionable {
     plugin: Plugin;
     [key: string]: any;
 }
+export type BackgroundAbortHandle = {
+    signal: AbortSignal;
+    dispose: () => void;
+    throwIfAborted: () => void;
+};
 export default class Processor {
     execution: ExecutionModel;
     options: ProcessorOptions;
@@ -31,11 +46,11 @@ export default class Processor {
     /**
      * @experimental
      */
-    transaction: Transaction;
+    transaction?: Transaction | null;
     /**
      * @experimental
      */
-    mainTransaction: Transaction;
+    mainTransaction?: Transaction | null;
     /**
      * @experimental
      */
@@ -47,29 +62,70 @@ export default class Processor {
     private jobsMapByNodeKey;
     private jobResultsMapByNodeKey;
     private jobsToSave;
+    private rerunContext;
     /**
      * @experimental
      */
     lastSavedJob: JobModel | null;
+    abortController: AbortController;
+    timeoutGuard: NodeJS.Timeout | null;
+    private runningRegistered;
+    private abortReason;
+    private aborted;
     constructor(execution: ExecutionModel, options: ProcessorOptions);
+    get abortSignal(): AbortSignal;
+    setTimeoutGuard(ms: number): void;
+    abortExecution(reason?: string): void;
+    isTimeoutAborted(): boolean;
+    /**
+     * Create an independent abort handle for background work that outlives this processor's
+     * run loop (e.g. fire-and-forget instructions that resume the job later). It mirrors the
+     * current abort state and sets its own timer based on the execution's `expiresAt`, so the
+     * timeout still applies after the processor has exited its synchronous run.
+     *
+     * The caller must invoke `dispose()` once the background work settles to release the timer
+     * and the abort listener.
+     */
+    createBackgroundAbortHandle(): BackgroundAbortHandle;
+    /**
+     * Reload a job and return it only when it is still pending, otherwise `null`. Background
+     * work uses this before resuming so it never overwrites a job that another path (timeout
+     * abort, a competing resume) has already settled.
+     */
+    findPendingJob(jobId: number | string): Promise<JobModel | null>;
     private makeNodes;
     private makeJobs;
     prepare(): Promise<void>;
-    start(): Promise<void>;
+    start(): Promise<any>;
     resume(job: JobModel): Promise<void>;
+    resolveRerun(options?: ProcessorRerunOptions): {
+        node: FlowNodeModel;
+        input: JobModel | {
+            result: any;
+        };
+        targetJob: JobModel;
+    };
+    rerun(options?: ProcessorRerunOptions): Promise<any>;
+    private getRerunNode;
+    private getRerunInput;
     private exec;
-    run(node: any, input?: any): any;
-    end(node: any, job: JobModel): Promise<any>;
+    run(node: FlowNodeModel, input?: JobModel | {
+        result: unknown;
+    }, options?: ProcessorRunOptions): Promise<any>;
+    end(node: FlowNodeModel, job: JobModel): Promise<any>;
     private recall;
     exit(s?: number | true): Promise<any>;
     /**
      * @experimental
      */
-    saveJob(payload: JobModel | Record<string, any>): JobModel;
+    saveJob(payload: JobModel | IJob): JobModel;
     /**
      * @experimental
      */
     getBranches(node: FlowNodeModel): FlowNodeModel[];
+    private enterRunningState;
+    private shouldContinueExecution;
+    private leaveRunningState;
     /**
      * @experimental
      * find the first node in current branch
@@ -79,7 +135,7 @@ export default class Processor {
      * @experimental
      * find the node start current branch
      */
-    findBranchParentNode(node: FlowNodeModel): FlowNodeModel | null;
+    findBranchParentNode(node?: FlowNodeModel): FlowNodeModel | null;
     /**
      * @experimental
      */
@@ -102,7 +158,7 @@ export default class Processor {
                 [key: string]: any;
             };
             $system: {};
-            $scopes: {};
+            $scopes: Record<string, any>;
             $env: {};
         };
         $context: any;
@@ -110,7 +166,7 @@ export default class Processor {
             [key: string]: any;
         };
         $system: {};
-        $scopes: {};
+        $scopes: Record<string, any>;
         $env: {};
     };
     /**