@nocobase/plugin-workflow 2.1.0-beta.8 → 2.2.0-alpha.1

package/dist/server/Dispatcher.js

@@ -29,22 +29,24 @@ __export(Dispatcher_exports, {
   default: () => Dispatcher
 });
 module.exports = __toCommonJS(Dispatcher_exports);
-var import_crypto = require("crypto");
+var import_node_crypto = require("node:crypto");
 var import_sequelize = require("sequelize");
 var import_constants = require("./constants");
 var import_Plugin = require("./Plugin");
+var import_utils = require("./utils");
+const EXECUTION_ACQUIRE_MAX_ATTEMPTS = 5;
 class Dispatcher {
   constructor(plugin) {
     this.plugin = plugin;
-    this.prepare = this.prepare.bind(this);
   }
   ready = false;
   executing = null;
+  saving = null;
   pending = [];
   events = [];
   eventsCount = 0;
   get idle() {
-    return this.ready && !this.executing && !this.pending.length && !this.events.length;
+    return this.ready && !this.executing && !this.saving && !this.pending.length && !this.events.length;
   }
   onQueueExecution = async (event) => {
     const ExecutionRepo = this.plugin.db.getRepository("executions");
@@ -52,11 +54,10 @@ class Dispatcher {
       filterByTk: event.executionId
     });
     if (!execution || execution.dispatched) {
-      this.plugin.getLogger("dispatcher").info(`execution (${event.executionId}) from queue not found or not in queueing status, skip`);
       return;
     }
     this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) received from queue, adding to pending list`);
-    this.run({ execution });
+    await this.run({ execution });
   };
   setReady(ready) {
     this.ready = ready;
@@ -96,11 +97,55 @@ class Dispatcher {
     this.eventsCount = this.events.length;
     logger.info(`new event triggered, now events: ${this.events.length}`);
     logger.debug(`event data:`, { context });
-    if (this.events.length > 1) {
-      logger.info(`new event is pending to be prepared after previous preparation is finished`);
+    this.saveEvent();
+  }
+  saveEvent() {
+    if (this.saving) {
       return;
     }
-    setImmediate(this.prepare);
+    this.saving = (async () => {
+      try {
+        while (this.events.length) {
+          if (this.executing && this.plugin.db.options.dialect === "sqlite") {
+            await this.executing;
+          }
+          const event = this.events.shift();
+          this.eventsCount = this.events.length;
+          if (!event) continue;
+          const logger = this.plugin.getLogger(event[0].id);
+          logger.info(`preparing execution for event`);
+          try {
+            const execution = await this.createExecution(...event);
+            if (!execution.dispatched) {
+              if (this.plugin.serving() && !this.executing && !this.pending.length) {
+                logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
+                this.pending.push({ execution });
+              } else {
+                logger.info(
+                  `instance is not serving as worker or local pending list is not empty, sending execution (${execution.id}) to queue`
+                );
+                try {
+                  await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, {
+                    executionId: execution.id
+                  });
+                } catch (qErr) {
+                  logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
+                }
+              }
+            }
+          } catch (error) {
+            logger.error(`failed to create execution:`, { error });
+          }
+        }
+      } finally {
+        this.saving = null;
+        if (this.events.length) {
+          this.saveEvent();
+        } else {
+          this.dispatch();
+        }
+      }
+    })();
   }
   async resume(job) {
     let { execution } = job;
@@ -108,27 +153,37 @@ class Dispatcher {
       execution = await job.getExecution();
     }
     this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) resuming from job (${job.id}) added to pending list`);
-    this.run({ execution, job, loaded: true });
+    await this.run({ execution, job });
   }
   async start(execution) {
     if (execution.status) {
       return;
     }
     this.plugin.getLogger(execution.workflowId).info(`starting deferred execution (${execution.id})`);
-    this.run({ execution, loaded: true });
+    await this.run({ execution });
   }
   async beforeStop() {
     this.ready = false;
-    if (this.events.length) {
-      await this.prepare();
-    }
-    if (this.executing) {
-      await this.executing;
+    this.plugin.getLogger("dispatcher").info("app is stopping, draining local queues...");
+    while (this.saving || this.executing || this.events.length || this.pending.length) {
+      if (this.saving) {
+        await this.saving;
+      }
+      if (this.executing) {
+        await this.executing;
+      }
+      if (this.events.length && !this.saving) {
+        this.saveEvent();
+      }
+      if (this.pending.length && !this.executing) {
+        this.dispatch();
+      }
+      await new Promise((resolve) => setImmediate(resolve));
     }
+    this.plugin.getLogger("dispatcher").info("local queues drained");
   }
   dispatch() {
-    if (!this.ready) {
-      this.plugin.getLogger("dispatcher").warn(`app is not ready, new dispatching will be ignored`);
+    if (!this.ready && !this.pending.length && !this.events.length) {
       return;
     }
     if (this.executing) {
@@ -136,30 +191,36 @@ class Dispatcher {
       return;
     }
     if (this.events.length) {
-      return this.prepare();
+      this.saveEvent();
+      return;
     }
     this.executing = (async () => {
       let next = null;
-      let execution = null;
-      if (this.pending.length) {
-        const pending = this.pending.shift();
-        execution = pending.loaded ? pending.execution : await this.acquirePendingExecution(pending.execution);
+      const pending = this.pending.shift() ?? null;
+      if (pending || this.ready && this.plugin.serving()) {
+        const execution = await this.prepare((pending == null ? void 0 : pending.execution) ?? null, {
+          immediate: pending == null ? void 0 : pending.immediate
+        });
         if (execution) {
-          next = [execution, pending.job];
+          next = [execution, pending == null ? void 0 : pending.job, pending == null ? void 0 : pending.rerun];
+        }
+        if (pending && next) {
           this.plugin.getLogger(next[0].workflowId).info(`pending execution (${next[0].id}) ready to process`);
         }
       } else {
-        if (this.plugin.serving()) {
-          execution = await this.acquireQueueingExecution();
-          if (execution) {
-            next = [execution];
-          }
-        } else {
-          this.plugin.getLogger("dispatcher").warn(`${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance, new dispatching will be ignored`);
-        }
+        this.plugin.getLogger("dispatcher").warn(
+          `${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance or app not ready, new dispatching will be ignored`
+        );
       }
       if (next) {
-        await this.process(...next);
+        try {
+          await this.process(next[0], next[1], { rerun: next[2] });
+        } catch (error) {
+          this.plugin.getLogger(next[0].workflowId).error(`execution (${next[0].id}) process failed`, { error });
+          if (pending && (0, import_utils.isLockAcquireError)(error)) {
+            this.pending.unshift({ ...pending, execution: next[0], immediate: true });
+          }
+        }
       }
       setImmediate(() => {
         this.executing = null;
@@ -171,7 +232,10 @@ class Dispatcher {
     })();
   }
   async run(pending) {
-    this.pending.push(pending);
+    this.pending.push({
+      ...pending,
+      immediate: !this.executing && !this.pending.length && !this.saving && !this.events.length
+    });
     this.dispatch();
   }
   async triggerSync(workflow, context, { deferred, ...options } = {}) {
@@ -179,13 +243,21 @@ class Dispatcher {
     try {
       execution = await this.createExecution(workflow, context, options);
     } catch (err) {
-      this.plugin.getLogger(workflow.id).error(`creating execution failed: ${err.message}`, err);
+      if (err instanceof Error) {
+        this.plugin.getLogger(workflow.id).error(`creating execution failed: ${err.message}`, err);
+      }
       return null;
     }
     try {
-      return this.process(execution, null, options);
+      const entered = await this.prepare(execution);
+      if (!entered) {
+        return null;
+      }
+      return this.process(entered, void 0, options);
     } catch (err) {
-      this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) error: ${err.message}`, err);
+      if (err instanceof Error) {
+        this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) error: ${err.message}`, err);
+      }
     }
     return null;
   }
@@ -195,19 +267,18 @@ class Dispatcher {
     if (!triggerValid) {
       return false;
     }
-    const { stack } = options;
+    const { stack = [] } = options;
     let valid = true;
-    if ((stack == null ? void 0 : stack.length) > 0) {
+    if (stack == null ? void 0 : stack.length) {
       const existed = await workflow.countExecutions({
         where: {
           id: stack
-        },
-        transaction: options.transaction
+        }
       });
       const limitCount = workflow.options.stackLimit || 1;
       if (existed >= limitCount) {
         this.plugin.getLogger(workflow.id).warn(
-          `workflow ${workflow.id} has already been triggered in stacks executions (${stack}), and max call coont is ${limitCount}, newly triggering will be skipped.`
+          `workflow ${workflow.id} has already been triggered in stacks executions (${stack}), and max call count is ${limitCount}, newly triggering will be skipped.`
         );
         valid = false;
       }
@@ -217,187 +288,235 @@ class Dispatcher {
   async createExecution(workflow, context, options) {
     var _a;
     const { deferred } = options;
-    const transaction = await this.plugin.useDataSourceTransaction("main", options.transaction, true);
-    const sameTransaction = options.transaction === transaction;
-    const valid = await this.validateEvent(workflow, context, { ...options, transaction });
+    let stack = options.stack;
+    if (options.parentExecutionId && !stack) {
+      const parentExecution = await this.plugin.db.getRepository("executions").findOne({
+        filterByTk: options.parentExecutionId
+      });
+      stack = parentExecution ? [...parentExecution.stack ?? [], parentExecution.id] : [];
+    }
+    const valid = await this.validateEvent(workflow, context, { ...options, stack });
     if (!valid) {
-      if (!sameTransaction) {
-        await transaction.commit();
-      }
       (_a = options.onTriggerFail) == null ? void 0 : _a.call(options, workflow, context, options);
-      return Promise.reject(new Error("event is not valid"));
-    }
-    let execution;
-    try {
-      execution = await workflow.createExecution(
-        {
-          context,
-          key: workflow.key,
-          eventKey: options.eventKey ?? (0, import_crypto.randomUUID)(),
-          stack: options.stack,
-          dispatched: deferred ?? false,
-          status: deferred ? import_constants.EXECUTION_STATUS.STARTED : import_constants.EXECUTION_STATUS.QUEUEING,
-          manually: options.manually
-        },
-        { transaction }
-      );
-    } catch (err) {
-      if (!sameTransaction) {
-        await transaction.rollback();
-      }
-      throw err;
-    }
+      throw new Error("event is not valid");
+    }
+    const execution = await workflow.createExecution({
+      context,
+      key: workflow.key,
+      eventKey: options.eventKey ?? (0, import_node_crypto.randomUUID)(),
+      stack,
+      parentExecutionId: options.parentExecutionId ?? null,
+      dispatched: deferred ?? false,
+      status: deferred ? import_constants.EXECUTION_STATUS.STARTED : import_constants.EXECUTION_STATUS.QUEUEING,
+      manually: options.manually
+    });
     this.plugin.getLogger(workflow.id).info(`execution of workflow ${workflow.id} created as ${execution.id}`);
     if (!workflow.stats) {
-      workflow.stats = await workflow.getStats({ transaction });
+      workflow.stats = await workflow.getStats();
     }
-    await workflow.stats.increment("executed", { transaction });
+    await workflow.stats.increment("executed");
     if (this.plugin.db.options.dialect !== "postgres") {
-      await workflow.stats.reload({ transaction });
+      await workflow.stats.reload();
     }
     if (!workflow.versionStats) {
-      workflow.versionStats = await workflow.getVersionStats({ transaction });
+      workflow.versionStats = await workflow.getVersionStats();
     }
-    await workflow.versionStats.increment("executed", { transaction });
+    await workflow.versionStats.increment("executed");
     if (this.plugin.db.options.dialect !== "postgres") {
-      await workflow.versionStats.reload({ transaction });
-    }
-    if (!sameTransaction) {
-      await transaction.commit();
+      await workflow.versionStats.reload();
     }
     execution.workflow = workflow;
     return execution;
   }
-  prepare = async () => {
-    if (this.executing && this.plugin.db.options.dialect === "sqlite") {
-      await this.executing;
-    }
-    const event = this.events.shift();
-    this.eventsCount = this.events.length;
-    if (!event) {
-      this.plugin.getLogger("dispatcher").info(`events queue is empty, no need to prepare`);
-      return;
+  async prepare(input, options = {}) {
+    const logger = input ? this.plugin.getLogger(input.workflowId) : this.plugin.getLogger("dispatcher");
+    if (options.transaction) {
+      try {
+        const { execution } = await this.acquireExecution(input, options, options.transaction);
+        return execution;
+      } catch (error) {
+        if (error instanceof Error) {
+          logger.error(`entering execution failed: ${error.message}`, { error });
+        }
+        return null;
+      }
     }
-    const logger = this.plugin.getLogger(event[0].id);
-    logger.info(`preparing execution for event`);
+    let result = null;
     try {
-      const execution = await this.createExecution(...event);
-      if (!(execution == null ? void 0 : execution.dispatched)) {
-        if (this.plugin.serving() && !this.executing && !this.pending.length) {
-          logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
-          this.pending.push({ execution });
-        } else {
-          logger.info(
-            `instance is not serving as worker or local pending list is not empty, sending execution (${execution.id}) to queue`
-          );
+      await this.acquireWithRetry(
+        async () => {
+          const tx = await this.plugin.db.sequelize.transaction({
+            isolationLevel: this.plugin.db.options.dialect === "sqlite" ? void 0 : import_sequelize.Transaction.ISOLATION_LEVELS.REPEATABLE_READ
+          });
           try {
-            await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, {
-              executionId: execution.id
-            });
-          } catch (qErr) {
-            logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
+            const { execution, shouldRetry } = await this.acquireExecution(input, options, tx);
+            await tx.commit();
+            result = execution;
+            return shouldRetry;
+          } catch (error) {
+            await tx.rollback();
+            if (this.isConcurrentAcquireError(error)) {
+              throw error;
+            }
+            if (error instanceof Error) {
+              logger.error(`entering execution failed: ${error.message}`, { error });
+            }
+            result = null;
+            return false;
           }
+        },
+        {
+          logger,
+          conflictMessage: input ? `acquiring pending execution (${input.id}) conflicted with another worker, retrying` : `acquiring execution conflicted with another worker, retrying`,
+          maxAttemptsMessage: input ? `acquiring pending execution (${input.id}) reached max retry attempts` : `acquiring execution reached max retry attempts, will retry on next dispatch`
         }
-      }
+      );
     } catch (error) {
-      logger.error(`failed to create execution:`, { error });
-    }
-    if (this.events.length) {
-      await this.prepare();
-    } else {
-      this.plugin.getLogger("dispatcher").info("no more events need to be prepared, dispatching...");
-      if (this.executing) {
-        await this.executing;
+      if (error instanceof Error) {
+        logger.error(`acquiring execution failed: ${error.message}`, { error });
       }
-      this.dispatch();
+      return null;
     }
-  };
-  async acquirePendingExecution(execution) {
-    const logger = this.plugin.getLogger(execution.workflowId);
-    const isolationLevel = this.plugin.db.options.dialect === "sqlite" ? [][0] : import_sequelize.Transaction.ISOLATION_LEVELS.REPEATABLE_READ;
-    let fetched = execution;
-    try {
-      await this.plugin.db.sequelize.transaction({ isolationLevel }, async (transaction) => {
-        const ExecutionModelClass = this.plugin.db.getModel("executions");
-        const [affected] = await ExecutionModelClass.update(
-          { dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED },
-          {
-            where: {
-              id: execution.id,
-              dispatched: false
-            },
-            transaction
-          }
-        );
-        if (!affected) {
-          fetched = null;
-          return;
-        }
+    return result;
+  }
+  async acquireExecution(input, options, transaction) {
+    let execution = input;
+    if (execution) {
+      if (!options.immediate || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
         await execution.reload({ transaction });
+      }
+    } else {
+      execution = await this.plugin.db.getRepository("executions").findOne({
+        filter: {
+          dispatched: false,
+          "workflow.enabled": true
+        },
+        sort: "id",
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+        skipLocked: true
       });
-    } catch (error) {
-      logger.error(`acquiring pending execution failed: ${error.message}`, { error });
+      if (execution) {
+        this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) fetched from db`);
+      } else {
+        this.plugin.getLogger("dispatcher").debug(`no execution in db queued to process`);
+      }
+    }
+    if (!execution) {
+      return { execution: null, shouldRetry: false };
     }
-    return fetched;
+    const entered = await this.enter(execution, transaction);
+    const shouldRetry = !input && !entered;
+    return { execution: entered, shouldRetry };
   }
-  async acquireQueueingExecution() {
-    const isolationLevel = this.plugin.db.options.dialect === "sqlite" ? [][0] : import_sequelize.Transaction.ISOLATION_LEVELS.REPEATABLE_READ;
-    let fetched = null;
-    try {
-      await this.plugin.db.sequelize.transaction(
-        {
-          isolationLevel
-        },
-        async (transaction) => {
-          const execution = await this.plugin.db.getRepository("executions").findOne({
-            filter: {
-              dispatched: false,
-              "workflow.enabled": true
-            },
-            sort: "id",
-            transaction
-          });
-          if (execution) {
-            this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) fetched from db`);
-            await execution.update(
-              {
-                dispatched: true,
-                status: import_constants.EXECUTION_STATUS.STARTED
-              },
-              { transaction }
-            );
-            execution.workflow = this.plugin.enabledCache.get(execution.workflowId);
-            fetched = execution;
-          } else {
-            this.plugin.getLogger("dispatcher").debug(`no execution in db queued to process`);
-          }
+  async acquireWithRetry(acquire, options) {
+    for (let attempt = 1; attempt <= EXECUTION_ACQUIRE_MAX_ATTEMPTS; attempt++) {
+      let shouldRetry = false;
+      try {
+        shouldRetry = await acquire();
+      } catch (error) {
+        if (!this.isConcurrentAcquireError(error)) {
+          throw error;
         }
-      );
-    } catch (error) {
-      this.plugin.getLogger("dispatcher").error(`fetching execution from db failed: ${error.message}`, { error });
+        shouldRetry = true;
+        options.logger.warn(options.conflictMessage, { error });
+      }
+      if (!shouldRetry) {
+        break;
+      }
+      if (attempt >= EXECUTION_ACQUIRE_MAX_ATTEMPTS) {
+        options.logger.warn(options.maxAttemptsMessage);
+        break;
+      }
     }
-    return fetched;
   }
-  async process(execution, job, options = {}) {
-    var _a, _b;
-    const logger = this.plugin.getLogger(execution.workflowId);
+  async enter(execution, transaction) {
+    const workflow = execution.workflow || this.plugin.enabledCache.get(execution.workflowId) || await execution.getWorkflow({ transaction });
+    if (!workflow) {
+      this.plugin.getLogger(execution.workflowId).warn(`workflow (${execution.workflowId}) not found for execution`, {
+        workflowId: execution.workflowId,
+        executionId: execution.id
+      });
+    }
+    if (execution.status && execution.status !== import_constants.EXECUTION_STATUS.STARTED) {
+      return null;
+    }
+    if (execution.dispatched && execution.status === import_constants.EXECUTION_STATUS.STARTED && execution.startedAt) {
+      execution.workflow = workflow;
+      return execution;
+    }
+    const values = {
+      dispatched: true,
+      status: import_constants.EXECUTION_STATUS.STARTED
+    };
+    const where = {
+      id: execution.id,
+      status: execution.status ?? null
+    };
     if (!execution.dispatched) {
-      const transaction = await this.plugin.useDataSourceTransaction("main", options.transaction);
-      await execution.update({ dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED }, { transaction });
-      logger.info(`execution (${execution.id}) from pending list updated to started`);
+      where.dispatched = false;
+    }
+    if (!execution.startedAt) {
+      const startedAt = /* @__PURE__ */ new Date();
+      values.startedAt = startedAt;
+      execution.workflow = workflow;
+      values.expiresAt = this.plugin.timeoutManager.getExpiresAt(execution, startedAt);
+      where.startedAt = null;
+    }
+    const ExecutionModelClass = this.plugin.db.getModel("executions");
+    const [affected] = await ExecutionModelClass.update(values, {
+      where,
+      transaction
+    });
+    if (!affected) {
+      return null;
     }
-    const processor = this.plugin.createProcessor(execution, options);
-    logger.info(`execution (${execution.id}) ${job ? "resuming" : "starting"}...`);
-    try {
-      await (job ? processor.resume(job) : processor.start());
-      logger.info(`execution (${execution.id}) finished with status: ${execution.status}`);
-      logger.debug(`execution (${execution.id}) details:`, { execution });
-      if (execution.status && ((_b = (_a = execution.workflow.options) == null ? void 0 : _a.deleteExecutionOnStatus) == null ? void 0 : _b.includes(execution.status))) {
-        await execution.destroy({ transaction: processor.mainTransaction });
+    await execution.reload({ transaction });
+    execution.workflow = workflow;
+    return execution;
+  }
+  isConcurrentAcquireError(error) {
+    var _a, _b, _c, _d;
+    if (!(error instanceof Error)) {
+      return false;
+    }
+    const databaseError = error;
+    const code = ((_a = databaseError.parent) == null ? void 0 : _a.code) || ((_b = databaseError.original) == null ? void 0 : _b.code);
+    const errno = ((_c = databaseError.parent) == null ? void 0 : _c.errno) || ((_d = databaseError.original) == null ? void 0 : _d.errno);
+    return code === "40001" || code === "40P01" || code === "ER_LOCK_DEADLOCK" || errno === 1205 || errno === 1213;
+  }
+  async process(execution, job = null, options = {}) {
+    const { rerun, ...processorOptions } = options;
+    const logger = this.plugin.getLogger(execution.workflowId);
+    const run = async () => {
+      var _a, _b, _c;
+      if (!execution.dispatched) {
+        await execution.update({ dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED });
+        logger.info(`execution (${execution.id}) from pending list updated to started`);
       }
-    } catch (err) {
-      logger.error(`execution (${execution.id}) error: ${err.message}`, err);
+      this.plugin.timeoutManager.scheduleExecutionTimeout(execution);
+      const processor = this.plugin.createProcessor(execution, processorOptions);
+      logger.info(`execution (${execution.id}) ${rerun ? "rerunning" : job ? "resuming" : "starting"}...`);
+      try {
+        await (rerun ? processor.rerun(rerun) : job ? processor.resume(job) : processor.start());
+        logger.info(`execution (${execution.id}) finished with status: ${execution.status}`);
+        logger.debug(`execution (${execution.id}) details:`, { execution });
+        if (execution.status && ((_c = (_b = (_a = execution.workflow) == null ? void 0 : _a.options) == null ? void 0 : _b.deleteExecutionOnStatus) == null ? void 0 : _c.includes(execution.status))) {
+          await execution.destroy();
+        }
+      } catch (err) {
+        if (err instanceof Error) {
+          logger.error(`execution (${execution.id}) error: ${err.message}`, err);
+        }
+      }
+      return processor;
+    };
+    const lock = await this.plugin.app.lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(execution.id), 6e4);
+    try {
+      return await lock.runExclusive(run, 6e4);
+    } catch (error) {
+      logger.error(`execution (${execution.id}) could not acquire process lock`, { error });
+      throw error;
     }
-    return processor;
   }
 }