@nocobase/plugin-workflow 2.1.0-alpha.4 → 2.1.0-alpha.45

package/dist/server/Dispatcher.js

@@ -29,22 +29,23 @@ __export(Dispatcher_exports, {
   default: () => Dispatcher
 });
 module.exports = __toCommonJS(Dispatcher_exports);
-var import_crypto = require("crypto");
+var import_node_crypto = require("node:crypto");
 var import_sequelize = require("sequelize");
 var import_constants = require("./constants");
 var import_Plugin = require("./Plugin");
+var import_utils = require("./utils");
 class Dispatcher {
   constructor(plugin) {
     this.plugin = plugin;
-    this.prepare = this.prepare.bind(this);
   }
   ready = false;
   executing = null;
+  saving = null;
   pending = [];
   events = [];
   eventsCount = 0;
   get idle() {
-    return this.ready && !this.executing && !this.pending.length && !this.events.length;
+    return this.ready && !this.executing && !this.saving && !this.pending.length && !this.events.length;
   }
   onQueueExecution = async (event) => {
     const ExecutionRepo = this.plugin.db.getRepository("executions");
@@ -52,11 +53,10 @@ class Dispatcher {
       filterByTk: event.executionId
     });
     if (!execution || execution.dispatched) {
-      this.plugin.getLogger("dispatcher").info(`execution (${event.executionId}) from queue not found or not in queueing status, skip`);
       return;
     }
     this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) received from queue, adding to pending list`);
-    this.run({ execution });
+    await this.run({ execution });
   };
   setReady(ready) {
     this.ready = ready;
@@ -96,11 +96,55 @@ class Dispatcher {
     this.eventsCount = this.events.length;
     logger.info(`new event triggered, now events: ${this.events.length}`);
     logger.debug(`event data:`, { context });
-    if (this.events.length > 1) {
-      logger.info(`new event is pending to be prepared after previous preparation is finished`);
+    this.saveEvent();
+  }
+  saveEvent() {
+    if (this.saving) {
       return;
     }
-    setImmediate(this.prepare);
+    this.saving = (async () => {
+      try {
+        while (this.events.length) {
+          if (this.executing && this.plugin.db.options.dialect === "sqlite") {
+            await this.executing;
+          }
+          const event = this.events.shift();
+          this.eventsCount = this.events.length;
+          if (!event) continue;
+          const logger = this.plugin.getLogger(event[0].id);
+          logger.info(`preparing execution for event`);
+          try {
+            const execution = await this.createExecution(...event);
+            if (!execution.dispatched) {
+              if (this.plugin.serving() && !this.executing && !this.pending.length) {
+                logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
+                this.pending.push({ execution });
+              } else {
+                logger.info(
+                  `instance is not serving as worker or local pending list is not empty, sending execution (${execution.id}) to queue`
+                );
+                try {
+                  await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, {
+                    executionId: execution.id
+                  });
+                } catch (qErr) {
+                  logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
+                }
+              }
+            }
+          } catch (error) {
+            logger.error(`failed to create execution:`, { error });
+          }
+        }
+      } finally {
+        this.saving = null;
+        if (this.events.length) {
+          this.saveEvent();
+        } else {
+          this.dispatch();
+        }
+      }
+    })();
   }
   async resume(job) {
     let { execution } = job;
@@ -108,27 +152,37 @@ class Dispatcher {
       execution = await job.getExecution();
     }
     this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) resuming from job (${job.id}) added to pending list`);
-    this.run({ execution, job, loaded: true });
+    await this.run({ execution, job });
   }
   async start(execution) {
     if (execution.status) {
       return;
     }
     this.plugin.getLogger(execution.workflowId).info(`starting deferred execution (${execution.id})`);
-    this.run({ execution, loaded: true });
+    await this.run({ execution });
   }
   async beforeStop() {
     this.ready = false;
-    if (this.events.length) {
-      await this.prepare();
-    }
-    if (this.executing) {
-      await this.executing;
+    this.plugin.getLogger("dispatcher").info("app is stopping, draining local queues...");
+    while (this.saving || this.executing || this.events.length || this.pending.length) {
+      if (this.saving) {
+        await this.saving;
+      }
+      if (this.executing) {
+        await this.executing;
+      }
+      if (this.events.length && !this.saving) {
+        this.saveEvent();
+      }
+      if (this.pending.length && !this.executing) {
+        this.dispatch();
+      }
+      await new Promise((resolve) => setImmediate(resolve));
     }
+    this.plugin.getLogger("dispatcher").info("local queues drained");
   }
   dispatch() {
-    if (!this.ready) {
-      this.plugin.getLogger("dispatcher").warn(`app is not ready, new dispatching will be ignored`);
+    if (!this.ready && !this.pending.length && !this.events.length) {
       return;
     }
     if (this.executing) {
@@ -136,30 +190,36 @@ class Dispatcher {
       return;
     }
     if (this.events.length) {
-      return this.prepare();
+      this.saveEvent();
+      return;
     }
     this.executing = (async () => {
       let next = null;
-      let execution = null;
-      if (this.pending.length) {
-        const pending = this.pending.shift();
-        execution = pending.loaded ? pending.execution : await this.acquirePendingExecution(pending.execution);
+      const pending = this.pending.shift() ?? null;
+      if (pending || this.ready && this.plugin.serving()) {
+        const execution = await this.prepare((pending == null ? void 0 : pending.execution) ?? null, {
+          immediate: pending == null ? void 0 : pending.immediate
+        });
         if (execution) {
-          next = [execution, pending.job];
+          next = [execution, pending == null ? void 0 : pending.job, pending == null ? void 0 : pending.rerun];
+        }
+        if (pending && next) {
           this.plugin.getLogger(next[0].workflowId).info(`pending execution (${next[0].id}) ready to process`);
         }
       } else {
-        if (this.plugin.serving()) {
-          execution = await this.acquireQueueingExecution();
-          if (execution) {
-            next = [execution];
-          }
-        } else {
-          this.plugin.getLogger("dispatcher").warn(`${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance, new dispatching will be ignored`);
-        }
+        this.plugin.getLogger("dispatcher").warn(
+          `${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance or app not ready, new dispatching will be ignored`
+        );
       }
       if (next) {
-        await this.process(...next);
+        try {
+          await this.process(next[0], next[1], { rerun: next[2] });
+        } catch (error) {
+          this.plugin.getLogger(next[0].workflowId).error(`execution (${next[0].id}) process failed`, { error });
+          if (pending && (0, import_utils.isLockAcquireError)(error)) {
+            this.pending.unshift({ ...pending, execution: next[0], immediate: true });
+          }
+        }
       }
       setImmediate(() => {
         this.executing = null;
@@ -171,7 +231,10 @@ class Dispatcher {
     })();
   }
   async run(pending) {
-    this.pending.push(pending);
+    this.pending.push({
+      ...pending,
+      immediate: !this.executing && !this.pending.length && !this.saving && !this.events.length
+    });
     this.dispatch();
   }
   async triggerSync(workflow, context, { deferred, ...options } = {}) {
@@ -179,13 +242,23 @@ class Dispatcher {
     try {
       execution = await this.createExecution(workflow, context, options);
     } catch (err) {
-      this.plugin.getLogger(workflow.id).error(`creating execution failed: ${err.message}`, err);
+      if (err instanceof Error) {
+        this.plugin.getLogger(workflow.id).error(`creating execution failed: ${err.message}`, err);
+      }
       return null;
     }
     try {
-      return this.process(execution, null, options);
+      const entered = await this.prepare(execution, {
+        transaction: this.plugin.useDataSourceTransaction("main", options.transaction)
+      });
+      if (!entered) {
+        return null;
+      }
+      return this.process(entered, void 0, options);
     } catch (err) {
-      this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) error: ${err.message}`, err);
+      if (err instanceof Error) {
+        this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) error: ${err.message}`, err);
+      }
     }
     return null;
   }
@@ -195,9 +268,9 @@ class Dispatcher {
     if (!triggerValid) {
       return false;
     }
-    const { stack } = options;
+    const { stack = [] } = options;
     let valid = true;
-    if ((stack == null ? void 0 : stack.length) > 0) {
+    if (stack == null ? void 0 : stack.length) {
       const existed = await workflow.countExecutions({
         where: {
           id: stack
@@ -207,7 +280,7 @@ class Dispatcher {
       const limitCount = workflow.options.stackLimit || 1;
       if (existed >= limitCount) {
         this.plugin.getLogger(workflow.id).warn(
-          `workflow ${workflow.id} has already been triggered in stacks executions (${stack}), and max call coont is ${limitCount}, newly triggering will be skipped.`
+          `workflow ${workflow.id} has already been triggered in stacks executions (${stack}), and max call count is ${limitCount}, newly triggering will be skipped.`
         );
         valid = false;
       }
@@ -219,13 +292,21 @@ class Dispatcher {
     const { deferred } = options;
     const transaction = await this.plugin.useDataSourceTransaction("main", options.transaction, true);
     const sameTransaction = options.transaction === transaction;
-    const valid = await this.validateEvent(workflow, context, { ...options, transaction });
+    let stack = options.stack;
+    if (options.parentExecutionId && !stack) {
+      const parentExecution = await this.plugin.db.getRepository("executions").findOne({
+        filterByTk: options.parentExecutionId,
+        transaction
+      });
+      stack = parentExecution ? [...parentExecution.stack ?? [], parentExecution.id] : [];
+    }
+    const valid = await this.validateEvent(workflow, context, { ...options, stack, transaction });
     if (!valid) {
       if (!sameTransaction) {
         await transaction.commit();
       }
       (_a = options.onTriggerFail) == null ? void 0 : _a.call(options, workflow, context, options);
-      return Promise.reject(new Error("event is not valid"));
+      throw new Error("event is not valid");
     }
     let execution;
     try {
@@ -233,8 +314,9 @@ class Dispatcher {
         {
           context,
           key: workflow.key,
-          eventKey: options.eventKey ?? (0, import_crypto.randomUUID)(),
-          stack: options.stack,
+          eventKey: options.eventKey ?? (0, import_node_crypto.randomUUID)(),
+          stack,
+          parentExecutionId: options.parentExecutionId ?? null,
           dispatched: deferred ?? false,
           status: deferred ? import_constants.EXECUTION_STATUS.STARTED : import_constants.EXECUTION_STATUS.QUEUEING,
           manually: options.manually
@@ -268,136 +350,135 @@ class Dispatcher {
     execution.workflow = workflow;
     return execution;
   }
-  prepare = async () => {
-    if (this.executing && this.plugin.db.options.dialect === "sqlite") {
-      await this.executing;
-    }
-    const event = this.events.shift();
-    this.eventsCount = this.events.length;
-    if (!event) {
-      this.plugin.getLogger("dispatcher").info(`events queue is empty, no need to prepare`);
-      return;
-    }
-    const logger = this.plugin.getLogger(event[0].id);
-    logger.info(`preparing execution for event`);
+  async prepare(input, options = {}) {
+    const transaction = options.transaction;
+    const ownTransaction = !transaction;
+    const tx = transaction || await this.plugin.db.sequelize.transaction({
+      isolationLevel: this.plugin.db.options.dialect === "sqlite" ? void 0 : import_sequelize.Transaction.ISOLATION_LEVELS.REPEATABLE_READ
+    });
+    const logger = input ? this.plugin.getLogger(input.workflowId) : this.plugin.getLogger("dispatcher");
     try {
-      const execution = await this.createExecution(...event);
-      if (!(execution == null ? void 0 : execution.dispatched)) {
-        if (this.plugin.serving() && !this.executing && !this.pending.length) {
-          logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
-          this.pending.push({ execution });
+      let execution = input;
+      if (execution) {
+        if (!options.immediate || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
+          await execution.reload({ transaction: tx });
+        }
+      } else {
+        execution = await this.plugin.db.getRepository("executions").findOne({
+          filter: {
+            dispatched: false,
+            "workflow.enabled": true
+          },
+          sort: "id",
+          transaction: tx,
+          lock: tx.LOCK.UPDATE,
+          skipLocked: true
+        });
+        if (execution) {
+          this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) fetched from db`);
         } else {
-          logger.info(
-            `instance is not serving as worker or local pending list is not empty, sending execution (${execution.id}) to queue`
-          );
-          try {
-            await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, {
-              executionId: execution.id
-            });
-          } catch (qErr) {
-            logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
-          }
+          this.plugin.getLogger("dispatcher").debug(`no execution in db queued to process`);
+        }
+      }
+      if (!execution) {
+        if (ownTransaction) {
+          await tx.commit();
         }
+        return null;
+      }
+      const entered = await this.enter(execution, tx);
+      if (ownTransaction) {
+        await tx.commit();
       }
+      return entered;
     } catch (error) {
-      logger.error(`failed to create execution:`, { error });
-    }
-    if (this.events.length) {
-      await this.prepare();
-    } else {
-      this.plugin.getLogger("dispatcher").info("no more events need to be prepared, dispatching...");
-      if (this.executing) {
-        await this.executing;
+      if (ownTransaction) {
+        await tx.rollback();
       }
-      this.dispatch();
+      if (error instanceof Error) {
+        logger.error(`entering execution failed: ${error.message}`, { error });
+      }
+      return null;
     }
-  };
-  async acquirePendingExecution(execution) {
-    const logger = this.plugin.getLogger(execution.workflowId);
-    const isolationLevel = this.plugin.db.options.dialect === "sqlite" ? [][0] : import_sequelize.Transaction.ISOLATION_LEVELS.REPEATABLE_READ;
-    let fetched = execution;
-    try {
-      await this.plugin.db.sequelize.transaction({ isolationLevel }, async (transaction) => {
-        const ExecutionModelClass = this.plugin.db.getModel("executions");
-        const [affected] = await ExecutionModelClass.update(
-          { dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED },
-          {
-            where: {
-              id: execution.id,
-              dispatched: false
-            },
-            transaction
-          }
-        );
-        if (!affected) {
-          fetched = null;
-          return;
-        }
-        await execution.reload({ transaction });
+  }
+  async enter(execution, transaction) {
+    const workflow = execution.workflow || this.plugin.enabledCache.get(execution.workflowId) || await execution.getWorkflow({ transaction });
+    if (!workflow) {
+      this.plugin.getLogger(execution.workflowId).warn(`workflow (${execution.workflowId}) not found for execution`, {
+        workflowId: execution.workflowId,
+        executionId: execution.id
       });
-    } catch (error) {
-      logger.error(`acquiring pending execution failed: ${error.message}`, { error });
     }
-    return fetched;
-  }
-  async acquireQueueingExecution() {
-    const isolationLevel = this.plugin.db.options.dialect === "sqlite" ? [][0] : import_sequelize.Transaction.ISOLATION_LEVELS.REPEATABLE_READ;
-    let fetched = null;
-    try {
-      await this.plugin.db.sequelize.transaction(
-        {
-          isolationLevel
-        },
-        async (transaction) => {
-          const execution = await this.plugin.db.getRepository("executions").findOne({
-            filter: {
-              dispatched: false,
-              "workflow.enabled": true
-            },
-            sort: "id",
-            transaction
-          });
-          if (execution) {
-            this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) fetched from db`);
-            await execution.update(
-              {
-                dispatched: true,
-                status: import_constants.EXECUTION_STATUS.STARTED
-              },
-              { transaction }
-            );
-            execution.workflow = this.plugin.enabledCache.get(execution.workflowId);
-            fetched = execution;
-          } else {
-            this.plugin.getLogger("dispatcher").debug(`no execution in db queued to process`);
-          }
-        }
-      );
-    } catch (error) {
-      this.plugin.getLogger("dispatcher").error(`fetching execution from db failed: ${error.message}`, { error });
+    if (execution.status && execution.status !== import_constants.EXECUTION_STATUS.STARTED) {
+      return null;
     }
-    return fetched;
-  }
-  async process(execution, job, options = {}) {
-    var _a, _b;
-    const logger = this.plugin.getLogger(execution.workflowId);
+    if (execution.dispatched && execution.status === import_constants.EXECUTION_STATUS.STARTED && execution.startedAt) {
+      execution.workflow = workflow;
+      return execution;
+    }
+    const values = {
+      dispatched: true,
+      status: import_constants.EXECUTION_STATUS.STARTED
+    };
+    const where = {
+      id: execution.id,
+      status: execution.status ?? null
+    };
     if (!execution.dispatched) {
-      const transaction = await this.plugin.useDataSourceTransaction("main", options.transaction);
-      await execution.update({ dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED }, { transaction });
-      logger.info(`execution (${execution.id}) from pending list updated to started`);
+      where.dispatched = false;
+    }
+    if (!execution.startedAt) {
+      const startedAt = /* @__PURE__ */ new Date();
+      values.startedAt = startedAt;
+      execution.workflow = workflow;
+      values.expiresAt = this.plugin.timeoutManager.getExpiresAt(execution, startedAt);
+      where.startedAt = null;
+    }
+    const ExecutionModelClass = this.plugin.db.getModel("executions");
+    const [affected] = await ExecutionModelClass.update(values, {
+      where,
+      transaction
+    });
+    if (!affected) {
+      return null;
     }
-    const processor = this.plugin.createProcessor(execution, options);
-    logger.info(`execution (${execution.id}) ${job ? "resuming" : "starting"}...`);
-    try {
-      await (job ? processor.resume(job) : processor.start());
-      logger.info(`execution (${execution.id}) finished with status: ${execution.status}`);
-      logger.debug(`execution (${execution.id}) details:`, { execution });
-      if (execution.status && ((_b = (_a = execution.workflow.options) == null ? void 0 : _a.deleteExecutionOnStatus) == null ? void 0 : _b.includes(execution.status))) {
-        await execution.destroy({ transaction: processor.mainTransaction });
+    await execution.reload({ transaction });
+    execution.workflow = workflow;
+    return execution;
+  }
+  async process(execution, job = null, options = {}) {
+    const { rerun, ...processorOptions } = options;
+    const logger = this.plugin.getLogger(execution.workflowId);
+    const run = async () => {
+      var _a, _b, _c;
+      if (!execution.dispatched) {
+        const transaction = await this.plugin.useDataSourceTransaction("main", processorOptions.transaction);
+        await execution.update({ dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED }, { transaction });
+        logger.info(`execution (${execution.id}) from pending list updated to started`);
       }
-    } catch (err) {
-      logger.error(`execution (${execution.id}) error: ${err.message}`, err);
+      this.plugin.timeoutManager.scheduleExecutionTimeout(execution);
+      const processor = this.plugin.createProcessor(execution, processorOptions);
+      logger.info(`execution (${execution.id}) ${rerun ? "rerunning" : job ? "resuming" : "starting"}...`);
+      try {
+        await (rerun ? processor.rerun(rerun) : job ? processor.resume(job) : processor.start());
+        logger.info(`execution (${execution.id}) finished with status: ${execution.status}`);
+        logger.debug(`execution (${execution.id}) details:`, { execution });
+        if (execution.status && ((_c = (_b = (_a = execution.workflow) == null ? void 0 : _a.options) == null ? void 0 : _b.deleteExecutionOnStatus) == null ? void 0 : _c.includes(execution.status))) {
+          await execution.destroy({ transaction: processor.mainTransaction });
+        }
+      } catch (err) {
+        if (err instanceof Error) {
+          logger.error(`execution (${execution.id}) error: ${err.message}`, err);
+        }
+      }
+      return processor;
+    };
+    const lock = await this.plugin.app.lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(execution.id), 6e4);
+    try {
+      return await lock.runExclusive(run, 6e4);
+    } catch (error) {
+      logger.error(`execution (${execution.id}) could not acquire process lock`, { error });
+      throw error;
     }
-    return processor;
   }
 }

package/dist/server/ExecutionTimeoutManager.d.ts

@@ -0,0 +1,45 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Transactionable } from '@nocobase/database';
+import type PluginWorkflowServer from './Plugin';
+import type { ExecutionModel } from './types';
+export default class ExecutionTimeoutManager {
+    private readonly plugin;
+    private readonly timers;
+    private scanTimer;
+    private nextExpiresAtTimer;
+    private nextExpiresAt;
+    private scanning;
+    private stopped;
+    constructor(plugin: PluginWorkflowServer);
+    getTimeout(execution: ExecutionModel): number;
+    getExpiresAt(execution: ExecutionModel, startedAt: Date): Date;
+    load(): Promise<void>;
+    unload(): Promise<void>;
+    isExpired(execution: ExecutionModel, now?: Date): boolean;
+    getRemainingMs(execution: ExecutionModel, now?: Date): number;
+    abort(execution: ExecutionModel, options?: Transactionable): Promise<boolean>;
+    abortExecutionIfExpired(execution: ExecutionModel, options?: Transactionable): Promise<boolean>;
+    clear(executionId: number | string): void;
+    shouldContinue(execution: ExecutionModel, options?: Transactionable): Promise<boolean>;
+    /**
+     * Owner-only per-execution timer. Only call from code paths that have acquired
+     * local execution ownership, such as Dispatcher/Processor processing paths.
+     */
+    scheduleExecutionTimeout(execution: ExecutionModel): void;
+    invalidateNextExpiresAtIfMatches(expiresAt?: Date | null): void;
+    private scanExpiredExecutions;
+    private scheduleScan;
+    private scheduleNextExpiresAtTimer;
+    private scheduleNextExpiresAtIfEarlier;
+    private handleNextExpiresAtTimeout;
+    private clearNextExpiresAtTimer;
+    private clearExecutionTimeout;
+    private handleExecutionTimeout;
+}