@nocobase/plugin-workflow 2.2.0-alpha.1 → 2.2.0-alpha.10

package/dist/server/Dispatcher.js

@@ -35,29 +35,52 @@ var import_constants = require("./constants");
 var import_Plugin = require("./Plugin");
 var import_utils = require("./utils");
 const EXECUTION_ACQUIRE_MAX_ATTEMPTS = 5;
+const QUEUE_TASK_MAX_ATTEMPTS = 3;
+const RECOVERY_BATCH_SIZE = 100;
+const RECOVERY_LOCK_TTL = 3e5;
 class Dispatcher {
   constructor(plugin) {
     this.plugin = plugin;
   }
   ready = false;
   executing = null;
+  recovering = null;
   saving = null;
-  pending = [];
   events = [];
   eventsCount = 0;
   get idle() {
-    return this.ready && !this.executing && !this.saving && !this.pending.length && !this.events.length;
+    return this.ready && !this.executing && !this.saving && !this.events.length;
   }
-  onQueueExecution = async (event) => {
-    const ExecutionRepo = this.plugin.db.getRepository("executions");
-    const execution = await ExecutionRepo.findOne({
-      filterByTk: event.executionId
-    });
-    if (!execution || execution.dispatched) {
-      return;
+  onQueueTask = async (event) => {
+    var _a;
+    const task = event;
+    let next = null;
+    const previous = this.executing;
+    const executing = (async () => {
+      await (previous == null ? void 0 : previous.catch(() => void 0));
+      this.plugin.getLogger("dispatcher").info(`workflow queue task for execution (${task.executionId}) received from queue`);
+      next = await this.resolveTask(task);
+      if (!next) {
+        return;
+      }
+      this.plugin.getLogger(next[0].workflowId).info(`queued execution (${next[0].id}) ready to process`);
+      await this.process(next[0], next[1], { rerun: next[2] });
+    })();
+    this.executing = executing;
+    try {
+      await executing;
+    } catch (error) {
+      const workflowId = ((_a = next == null ? void 0 : next[0]) == null ? void 0 : _a.workflowId) ?? task.executionId;
+      this.plugin.getLogger(workflowId).error(`workflow queue task failed`, { error });
+      throw error;
+    } finally {
+      if (this.executing === executing) {
+        this.executing = null;
+      }
+      if (this.events.length) {
+        this.saveEvent();
+      }
     }
-    this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) received from queue, adding to pending list`);
-    await this.run({ execution });
   };
   setReady(ready) {
     this.ready = ready;
@@ -70,7 +93,7 @@ class Dispatcher {
     if (!this.ready) {
       logger.warn(`app is not ready, event of workflow ${workflow.id} will be ignored`);
       logger.debug(`ignored event data:`, context);
-      return;
+      return this.handleTriggerFail(workflow, context, options, new Error("app is not ready"));
     }
     if (!options.force && !options.manually && !workflow.enabled) {
       logger.warn(`workflow ${workflow.id} is not enabled, event will be ignored`);
@@ -87,7 +110,7 @@ class Dispatcher {
     }
     if (context == null) {
       logger.warn(`workflow ${workflow.id} event data context is null, event will be ignored`);
-      return;
+      return this.handleTriggerFail(workflow, context, options, new Error("event context is null"));
     }
     if (options.manually || this.plugin.isWorkflowSync(workflow)) {
       return this.triggerSync(workflow, context, options);
@@ -117,20 +140,12 @@ class Dispatcher {
           try {
             const execution = await this.createExecution(...event);
             if (!execution.dispatched) {
-              if (this.plugin.serving() && !this.executing && !this.pending.length) {
-                logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
-                this.pending.push({ execution });
-              } else {
-                logger.info(
-                  `instance is not serving as worker or local pending list is not empty, sending execution (${execution.id}) to queue`
-                );
-                try {
-                  await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, {
-                    executionId: execution.id
-                  });
-                } catch (qErr) {
-                  logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
-                }
+              try {
+                await this.enqueue({
+                  executionId: execution.id
+                });
+              } catch (qErr) {
+                logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
               }
             }
           } catch (error) {
@@ -141,31 +156,17 @@ class Dispatcher {
         this.saving = null;
         if (this.events.length) {
           this.saveEvent();
-        } else {
-          this.dispatch();
         }
       }
     })();
   }
-  async resume(job) {
-    let { execution } = job;
-    if (!execution) {
-      execution = await job.getExecution();
-    }
-    this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) resuming from job (${job.id}) added to pending list`);
-    await this.run({ execution, job });
-  }
-  async start(execution) {
-    if (execution.status) {
-      return;
-    }
-    this.plugin.getLogger(execution.workflowId).info(`starting deferred execution (${execution.id})`);
-    await this.run({ execution });
-  }
   async beforeStop() {
     this.ready = false;
     this.plugin.getLogger("dispatcher").info("app is stopping, draining local queues...");
-    while (this.saving || this.executing || this.events.length || this.pending.length) {
+    if (this.recovering) {
+      await this.recovering;
+    }
+    while (this.saving || this.executing || this.events.length) {
       if (this.saving) {
         await this.saving;
       }
@@ -175,68 +176,131 @@ class Dispatcher {
       if (this.events.length && !this.saving) {
         this.saveEvent();
       }
-      if (this.pending.length && !this.executing) {
-        this.dispatch();
-      }
       await new Promise((resolve) => setImmediate(resolve));
     }
     this.plugin.getLogger("dispatcher").info("local queues drained");
   }
-  dispatch() {
-    if (!this.ready && !this.pending.length && !this.events.length) {
+  async recover(options = {}) {
+    if (!this.ready) {
       return;
     }
-    if (this.executing) {
-      this.plugin.getLogger("dispatcher").warn(`workflow executing is not finished, new dispatching will be ignored`);
+    if (this.recovering) {
+      await this.recovering;
       return;
     }
-    if (this.events.length) {
-      this.saveEvent();
-      return;
+    const recovering = this.recoverQueueTasks(options.gracePeriod ?? 0);
+    this.recovering = recovering;
+    try {
+      await recovering;
+    } finally {
+      if (this.recovering === recovering) {
+        this.recovering = null;
+      }
     }
-    this.executing = (async () => {
-      let next = null;
-      const pending = this.pending.shift() ?? null;
-      if (pending || this.ready && this.plugin.serving()) {
-        const execution = await this.prepare((pending == null ? void 0 : pending.execution) ?? null, {
-          immediate: pending == null ? void 0 : pending.immediate
-        });
-        if (execution) {
-          next = [execution, pending == null ? void 0 : pending.job, pending == null ? void 0 : pending.rerun];
-        }
-        if (pending && next) {
-          this.plugin.getLogger(next[0].workflowId).info(`pending execution (${next[0].id}) ready to process`);
-        }
-      } else {
-        this.plugin.getLogger("dispatcher").warn(
-          `${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance or app not ready, new dispatching will be ignored`
+  }
+  async recoverQueueTasks(gracePeriod) {
+    const logger = this.plugin.getLogger("dispatcher");
+    try {
+      if (!this.plugin.serving()) {
+        logger.warn(
+          `${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance, workflow queue recovery will be ignored`
         );
+        return;
       }
-      if (next) {
-        try {
-          await this.process(next[0], next[1], { rerun: next[2] });
-        } catch (error) {
-          this.plugin.getLogger(next[0].workflowId).error(`execution (${next[0].id}) process failed`, { error });
-          if (pending && (0, import_utils.isLockAcquireError)(error)) {
-            this.pending.unshift({ ...pending, execution: next[0], immediate: true });
-          }
+      let lock;
+      try {
+        lock = await this.plugin.app.lockManager.tryAcquire(`workflow:recover:${this.plugin.app.name}`);
+      } catch (error) {
+        if ((0, import_utils.isLockAcquireError)(error)) {
+          logger.debug(`workflow queue recovery is already running on another instance`);
+          return;
         }
+        throw error;
       }
-      setImmediate(() => {
-        this.executing = null;
-        if (next || this.pending.length) {
-          this.plugin.getLogger("dispatcher").debug(`last process finished, will do another dispatch`);
-          this.dispatch();
+      await lock.runExclusive(async () => {
+        let executionCursor = null;
+        while (this.ready) {
+          const executions = await this.findQueueingExecutions(executionCursor, gracePeriod);
+          if (!executions.length) {
+            break;
+          }
+          for (const execution of executions) {
+            if (!this.ready) {
+              return;
+            }
+            await this.enqueue({ executionId: execution.id });
+          }
+          executionCursor = executions[executions.length - 1].id;
         }
-      });
-    })();
+      }, RECOVERY_LOCK_TTL);
+    } catch (error) {
+      logger.error(`workflow queue recovery failed`, { error });
+    }
   }
-  async run(pending) {
-    this.pending.push({
-      ...pending,
-      immediate: !this.executing && !this.pending.length && !this.saving && !this.events.length
+  async enqueue(task) {
+    await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, task, {
+      maxRetries: QUEUE_TASK_MAX_ATTEMPTS - 1
     });
-    this.dispatch();
+  }
+  async loadJob(jobId) {
+    if (jobId == null) {
+      return null;
+    }
+    return this.plugin.db.getRepository("jobs").findOne({
+      filterByTk: jobId
+    });
+  }
+  async findQueueingExecutions(afterId, gracePeriod) {
+    const executions = await this.plugin.db.getRepository("executions").find({
+      filter: {
+        ...afterId == null ? {} : { id: { $gt: afterId } },
+        ...gracePeriod > 0 ? { createdAt: { $lt: new Date(Date.now() - gracePeriod) } } : {},
+        dispatched: false,
+        status: import_constants.EXECUTION_STATUS.QUEUEING,
+        startedAt: null,
+        "workflow.enabled": true
+      },
+      sort: "id",
+      limit: RECOVERY_BATCH_SIZE
+    });
+    for (const execution of executions) {
+      this.plugin.getLogger(execution.workflowId).info(`queueing execution (${execution.id}) found, publishing task`);
+    }
+    return executions;
+  }
+  async resolveTask(task) {
+    const executionInput = await this.plugin.db.getRepository("executions").findOne({
+      filterByTk: task.executionId
+    });
+    if (!executionInput) {
+      this.plugin.getLogger("dispatcher").warn(`execution (${task.executionId}) not found, queue task ignored`);
+      return null;
+    }
+    if (task.jobId == null && task.rerun == null && executionInput.status === import_constants.EXECUTION_STATUS.STARTED && executionInput.startedAt) {
+      this.plugin.getLogger(executionInput.workflowId).warn(`execution (${executionInput.id}) has already started, start task ignored`);
+      return null;
+    }
+    let job = null;
+    if (task.jobId != null) {
+      job = await this.loadJob(task.jobId);
+      if (!job) {
+        this.plugin.getLogger(executionInput.workflowId).warn(`job (${task.jobId}) not found, resume ignored`);
+        return null;
+      }
+      if (String(job.executionId) !== String(executionInput.id)) {
+        this.plugin.getLogger(executionInput.workflowId).warn(`job (${job.id}) does not belong to execution (${executionInput.id}), resume ignored`);
+        return null;
+      }
+    }
+    const execution = await this.prepare(executionInput);
+    if (!execution) {
+      return null;
+    }
+    if (job) {
+      job.execution = execution;
+      return [execution, job];
+    }
+    return [execution, void 0, task.rerun];
   }
   async triggerSync(workflow, context, { deferred, ...options } = {}) {
     let execution;
@@ -285,8 +349,15 @@ class Dispatcher {
     }
     return valid;
   }
-  async createExecution(workflow, context, options) {
+  async handleTriggerFail(workflow, context, options, error) {
     var _a;
+    try {
+      await ((_a = options.onTriggerFail) == null ? void 0 : _a.call(options, workflow, context, options, error));
+    } catch (triggerFailError) {
+      this.plugin.getLogger(workflow.id).error(`trigger failure callback failed`, { error: triggerFailError });
+    }
+  }
+  async createExecution(workflow, context, options) {
     const { deferred } = options;
     let stack = options.stack;
     if (options.parentExecutionId && !stack) {
@@ -295,21 +366,34 @@ class Dispatcher {
       });
       stack = parentExecution ? [...parentExecution.stack ?? [], parentExecution.id] : [];
     }
-    const valid = await this.validateEvent(workflow, context, { ...options, stack });
+    let valid;
+    try {
+      valid = await this.validateEvent(workflow, context, { ...options, stack });
+    } catch (error) {
+      await this.handleTriggerFail(workflow, context, options, error);
+      throw error;
+    }
     if (!valid) {
-      (_a = options.onTriggerFail) == null ? void 0 : _a.call(options, workflow, context, options);
-      throw new Error("event is not valid");
-    }
-    const execution = await workflow.createExecution({
-      context,
-      key: workflow.key,
-      eventKey: options.eventKey ?? (0, import_node_crypto.randomUUID)(),
-      stack,
-      parentExecutionId: options.parentExecutionId ?? null,
-      dispatched: deferred ?? false,
-      status: deferred ? import_constants.EXECUTION_STATUS.STARTED : import_constants.EXECUTION_STATUS.QUEUEING,
-      manually: options.manually
-    });
+      const error = new Error("event is not valid");
+      await this.handleTriggerFail(workflow, context, options, error);
+      throw error;
+    }
+    let execution;
+    try {
+      execution = await workflow.createExecution({
+        context,
+        key: workflow.key,
+        eventKey: options.eventKey ?? (0, import_node_crypto.randomUUID)(),
+        stack,
+        parentExecutionId: options.parentExecutionId ?? null,
+        dispatched: deferred ?? false,
+        status: deferred ? import_constants.EXECUTION_STATUS.STARTED : import_constants.EXECUTION_STATUS.QUEUEING,
+        manually: options.manually
+      });
+    } catch (error) {
+      await this.handleTriggerFail(workflow, context, options, error);
+      throw error;
+    }
     this.plugin.getLogger(workflow.id).info(`execution of workflow ${workflow.id} created as ${execution.id}`);
     if (!workflow.stats) {
       workflow.stats = await workflow.getStats();
@@ -329,11 +413,10 @@ class Dispatcher {
     return execution;
   }
   async prepare(input, options = {}) {
-    const logger = input ? this.plugin.getLogger(input.workflowId) : this.plugin.getLogger("dispatcher");
+    const logger = this.plugin.getLogger(input.workflowId);
     if (options.transaction) {
       try {
-        const { execution } = await this.acquireExecution(input, options, options.transaction);
-        return execution;
+        return await this.acquireExecution(input, options, options.transaction);
       } catch (error) {
         if (error instanceof Error) {
           logger.error(`entering execution failed: ${error.message}`, { error });
@@ -346,13 +429,12 @@ class Dispatcher {
       await this.acquireWithRetry(
         async () => {
           const tx = await this.plugin.db.sequelize.transaction({
-            isolationLevel: this.plugin.db.options.dialect === "sqlite" ? void 0 : import_sequelize.Transaction.ISOLATION_LEVELS.REPEATABLE_READ
+            isolationLevel: this.plugin.db.options.dialect === "sqlite" ? void 0 : import_sequelize.Transaction.ISOLATION_LEVELS.READ_COMMITTED
           });
           try {
-            const { execution, shouldRetry } = await this.acquireExecution(input, options, tx);
+            const execution = await this.acquireExecution(input, options, tx);
             await tx.commit();
             result = execution;
-            return shouldRetry;
           } catch (error) {
             await tx.rollback();
             if (this.isConcurrentAcquireError(error)) {
@@ -362,13 +444,12 @@ class Dispatcher {
               logger.error(`entering execution failed: ${error.message}`, { error });
             }
             result = null;
-            return false;
           }
         },
         {
           logger,
-          conflictMessage: input ? `acquiring pending execution (${input.id}) conflicted with another worker, retrying` : `acquiring execution conflicted with another worker, retrying`,
-          maxAttemptsMessage: input ? `acquiring pending execution (${input.id}) reached max retry attempts` : `acquiring execution reached max retry attempts, will retry on next dispatch`
+          conflictMessage: `acquiring queue task execution (${input.id}) conflicted with another worker, retrying`,
+          maxAttemptsMessage: `acquiring queue task execution (${input.id}) reached max retry attempts`
         }
       );
     } catch (error) {
@@ -380,53 +461,25 @@ class Dispatcher {
     return result;
   }
   async acquireExecution(input, options, transaction) {
-    let execution = input;
-    if (execution) {
-      if (!options.immediate || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
-        await execution.reload({ transaction });
-      }
-    } else {
-      execution = await this.plugin.db.getRepository("executions").findOne({
-        filter: {
-          dispatched: false,
-          "workflow.enabled": true
-        },
-        sort: "id",
-        transaction,
-        lock: transaction.LOCK.UPDATE,
-        skipLocked: true
-      });
-      if (execution) {
-        this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) fetched from db`);
-      } else {
-        this.plugin.getLogger("dispatcher").debug(`no execution in db queued to process`);
-      }
+    if (!options.immediate || input.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
+      await input.reload({ transaction });
     }
-    if (!execution) {
-      return { execution: null, shouldRetry: false };
-    }
-    const entered = await this.enter(execution, transaction);
-    const shouldRetry = !input && !entered;
-    return { execution: entered, shouldRetry };
+    return this.enter(input, transaction);
   }
   async acquireWithRetry(acquire, options) {
     for (let attempt = 1; attempt <= EXECUTION_ACQUIRE_MAX_ATTEMPTS; attempt++) {
-      let shouldRetry = false;
       try {
-        shouldRetry = await acquire();
+        await acquire();
+        break;
       } catch (error) {
         if (!this.isConcurrentAcquireError(error)) {
           throw error;
         }
-        shouldRetry = true;
         options.logger.warn(options.conflictMessage, { error });
-      }
-      if (!shouldRetry) {
-        break;
-      }
-      if (attempt >= EXECUTION_ACQUIRE_MAX_ATTEMPTS) {
-        options.logger.warn(options.maxAttemptsMessage);
-        break;
+        if (attempt >= EXECUTION_ACQUIRE_MAX_ATTEMPTS) {
+          options.logger.warn(options.maxAttemptsMessage);
+          break;
+        }
       }
     }
   }
@@ -492,7 +545,7 @@ class Dispatcher {
       var _a, _b, _c;
       if (!execution.dispatched) {
         await execution.update({ dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED });
-        logger.info(`execution (${execution.id}) from pending list updated to started`);
+        logger.info(`execution (${execution.id}) from workflow queue updated to started`);
       }
       this.plugin.timeoutManager.scheduleExecutionTimeout(execution);
       const processor = this.plugin.createProcessor(execution, processorOptions);
@@ -511,7 +564,8 @@ class Dispatcher {
       }
       return processor;
     };
-    const lock = await this.plugin.app.lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(execution.id), 6e4);
+    const lockManager = this.plugin.app.lockManager;
+    const lock = await lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(execution.id), 6e4);
     try {
       return await lock.runExclusive(run, 6e4);
     } catch (error) {

package/dist/server/ExecutionTimeoutManager.js

@@ -81,10 +81,10 @@ class ExecutionTimeoutManager {
     await ((_a = this.scanning) == null ? void 0 : _a.catch(() => {
     }));
   }
-  isExpired(execution, now = /* @__PURE__ */ new Date()) {
+  isExpired(execution, now = new Date(Date.now())) {
     return !!execution.expiresAt && execution.expiresAt.getTime() <= now.getTime();
   }
-  getRemainingMs(execution, now = /* @__PURE__ */ new Date()) {
+  getRemainingMs(execution, now = new Date(Date.now())) {
     if (!execution.expiresAt) {
       return null;
     }
@@ -108,7 +108,8 @@ class ExecutionTimeoutManager {
   }
   async abortExecutionIfExpiredWithLock(executionOrId) {
     const executionId = typeof executionOrId === "object" ? executionOrId.id : executionOrId;
-    const lock = await this.plugin.app.lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(executionId), EXECUTION_LOCK_TIMEOUT);
+    const lockManager = this.plugin.app.lockManager;
+    const lock = await lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(executionId), EXECUTION_LOCK_TIMEOUT);
     return lock.runExclusive(async () => {
       const execution = await this.plugin.db.getRepository("executions").findOne({
         filterByTk: executionId

package/dist/server/Plugin.d.ts

@@ -11,14 +11,14 @@ import { Transactionable } from 'sequelize';
 import { Plugin } from '@nocobase/server';
 import { Registry } from '@nocobase/utils';
 import { Logger } from '@nocobase/logger';
-import Dispatcher, { EventOptions } from './Dispatcher';
-import Processor from './Processor';
+import { EventOptions } from './Dispatcher';
+import Processor, { ProcessorRerunOptions } from './Processor';
 import ExecutionTimeoutManager from './ExecutionTimeoutManager';
 import RunningExecutionRegistry from './RunningExecutionRegistry';
 import { CustomFunction } from './functions';
 import Trigger from './triggers';
 import { InstructionInterface } from './instructions';
-import type { ExecutionModel, WorkflowModel } from './types';
+import type { ExecutionModel, JobModel, WorkflowModel } from './types';
 import type { Transaction } from '@nocobase/database';
 type ID = number | string;
 export declare const WORKER_JOB_WORKFLOW_PROCESS = "workflow:process";
@@ -74,16 +74,25 @@ export default class PluginWorkflowServer extends Plugin {
      */
     load(): Promise<void>;
     private toggle;
-    trigger(workflow: WorkflowModel, context: object, options?: EventOptions): void | Promise<Processor | null>;
-    run(pending: Parameters<Dispatcher['run']>[0]): Promise<void>;
-    dispatch(): void;
-    resume(job: any): Promise<void>;
+    trigger(workflow: WorkflowModel, context: object, options?: EventOptions): void | Promise<Processor | null | void>;
+    rerun(execution: ExecutionModel | ID, rerun?: ProcessorRerunOptions): Promise<void>;
+    recover(): Promise<void>;
+    /**
+     * Persist the current job state and publish a poke that evaluates that state.
+     *
+     * PENDING resume calls are valid for instructions such as delay and sequential approval.
+     * Callers that aggregate multiple user actions must only call this method when their
+     * atomic update changes the job from PENDING to a terminal status.
+     * Persistence and queue publication failures are logged and rethrown to awaiting callers.
+     */
+    resume(job: JobModel): Promise<void>;
     /**
      * Start a deferred execution
      * @experimental
      */
-    start(execution: ExecutionModel): Promise<void>;
+    start(execution: ExecutionModel | ID): Promise<void>;
     createProcessor(execution: ExecutionModel, options?: {}): Processor;
+    private getModelId;
     execute(workflow: WorkflowModel, values: any, options?: EventOptions): Promise<void | Processor>;
     /**
      * @experimental