@nocobase/plugin-workflow 2.2.0-beta.15 → 2.2.0-beta.16

package/dist/server/Dispatcher.js

@@ -35,30 +35,52 @@ var import_constants = require("./constants");
 var import_Plugin = require("./Plugin");
 var import_utils = require("./utils");
 const EXECUTION_ACQUIRE_MAX_ATTEMPTS = 5;
-const PENDING_DISPATCH_MAX_ATTEMPTS = 3;
+const QUEUE_TASK_MAX_ATTEMPTS = 3;
+const RECOVERY_BATCH_SIZE = 100;
+const RECOVERY_LOCK_TTL = 3e5;
 class Dispatcher {
   constructor(plugin) {
     this.plugin = plugin;
   }
   ready = false;
   executing = null;
+  recovering = null;
   saving = null;
-  pending = [];
   events = [];
   eventsCount = 0;
   get idle() {
-    return this.ready && !this.executing && !this.saving && !this.pending.length && !this.events.length;
+    return this.ready && !this.executing && !this.saving && !this.events.length;
   }
-  onQueueExecution = async (event) => {
-    const ExecutionRepo = this.plugin.db.getRepository("executions");
-    const execution = await ExecutionRepo.findOne({
-      filterByTk: event.executionId
-    });
-    if (!execution || execution.dispatched || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
-      return;
+  onQueueTask = async (event) => {
+    var _a;
+    const task = event;
+    let next = null;
+    const previous = this.executing;
+    const executing = (async () => {
+      await (previous == null ? void 0 : previous.catch(() => void 0));
+      this.plugin.getLogger("dispatcher").info(`workflow queue task for execution (${task.executionId}) received from queue`);
+      next = await this.resolveTask(task);
+      if (!next) {
+        return;
+      }
+      this.plugin.getLogger(next[0].workflowId).info(`queued execution (${next[0].id}) ready to process`);
+      await this.process(next[0], next[1], { rerun: next[2] });
+    })();
+    this.executing = executing;
+    try {
+      await executing;
+    } catch (error) {
+      const workflowId = ((_a = next == null ? void 0 : next[0]) == null ? void 0 : _a.workflowId) ?? task.executionId;
+      this.plugin.getLogger(workflowId).error(`workflow queue task failed`, { error });
+      throw error;
+    } finally {
+      if (this.executing === executing) {
+        this.executing = null;
+      }
+      if (this.events.length) {
+        this.saveEvent();
+      }
     }
-    this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) received from queue, adding to pending list`);
-    await this.run({ execution });
   };
   setReady(ready) {
     this.ready = ready;
@@ -118,20 +140,12 @@ class Dispatcher {
           try {
             const execution = await this.createExecution(...event);
             if (!execution.dispatched) {
-              if (this.plugin.serving() && !this.executing && !this.pending.length) {
-                logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
-                this.pending.push({ execution });
-              } else {
-                logger.info(
-                  `instance is not serving as worker or local pending list is not empty, sending execution (${execution.id}) to queue`
-                );
-                try {
-                  await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, {
-                    executionId: execution.id
-                  });
-                } catch (qErr) {
-                  logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
-                }
+              try {
+                await this.enqueue({
+                  executionId: execution.id
+                });
+              } catch (qErr) {
+                logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
               }
             }
           } catch (error) {
@@ -142,31 +156,17 @@ class Dispatcher {
         this.saving = null;
         if (this.events.length) {
           this.saveEvent();
-        } else {
-          this.dispatch();
         }
       }
     })();
   }
-  async resume(job) {
-    let { execution } = job;
-    if (!execution) {
-      execution = await job.getExecution();
-    }
-    this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) resuming from job (${job.id}) added to pending list`);
-    await this.run({ execution, job });
-  }
-  async start(execution) {
-    if (execution.status) {
-      return;
-    }
-    this.plugin.getLogger(execution.workflowId).info(`starting deferred execution (${execution.id})`);
-    await this.run({ execution });
-  }
   async beforeStop() {
     this.ready = false;
     this.plugin.getLogger("dispatcher").info("app is stopping, draining local queues...");
-    while (this.saving || this.executing || this.events.length || this.pending.length) {
+    if (this.recovering) {
+      await this.recovering;
+    }
+    while (this.saving || this.executing || this.events.length) {
       if (this.saving) {
         await this.saving;
       }
@@ -176,85 +176,131 @@ class Dispatcher {
       if (this.events.length && !this.saving) {
         this.saveEvent();
       }
-      if (this.pending.length && !this.executing) {
-        this.dispatch();
-      }
       await new Promise((resolve) => setImmediate(resolve));
     }
     this.plugin.getLogger("dispatcher").info("local queues drained");
   }
-  dispatch() {
-    if (!this.ready && !this.pending.length && !this.events.length) {
+  async recover(options = {}) {
+    if (!this.ready) {
       return;
     }
-    if (this.executing) {
-      this.plugin.getLogger("dispatcher").warn(`workflow executing is not finished, new dispatching will be ignored`);
+    if (this.recovering) {
+      await this.recovering;
       return;
     }
-    if (this.events.length) {
-      this.saveEvent();
-      return;
+    const recovering = this.recoverQueueTasks(options.gracePeriod ?? 0);
+    this.recovering = recovering;
+    try {
+      await recovering;
+    } finally {
+      if (this.recovering === recovering) {
+        this.recovering = null;
+      }
     }
-    this.executing = (async () => {
-      let next = null;
-      let pending = null;
+  }
+  async recoverQueueTasks(gracePeriod) {
+    const logger = this.plugin.getLogger("dispatcher");
+    try {
+      if (!this.plugin.serving()) {
+        logger.warn(
+          `${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance, workflow queue recovery will be ignored`
+        );
+        return;
+      }
+      let lock;
       try {
-        pending = this.pending.shift() ?? null;
-        if (pending || this.ready && this.plugin.serving()) {
-          const execution = await this.prepare((pending == null ? void 0 : pending.execution) ?? null, {
-            immediate: pending == null ? void 0 : pending.immediate
-          });
-          if (execution) {
-            next = [execution, pending == null ? void 0 : pending.job, pending == null ? void 0 : pending.rerun];
-          }
-          if (pending && next) {
-            this.plugin.getLogger(next[0].workflowId).info(`pending execution (${next[0].id}) ready to process`);
-          }
-        } else {
-          this.plugin.getLogger("dispatcher").warn(
-            `${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance or app not ready, new dispatching will be ignored`
-          );
+        lock = await this.plugin.app.lockManager.tryAcquire(`workflow:recover:${this.plugin.app.name}`);
+      } catch (error) {
+        if ((0, import_utils.isLockAcquireError)(error)) {
+          logger.debug(`workflow queue recovery is already running on another instance`);
+          return;
         }
-        if (next) {
-          try {
-            await this.process(next[0], next[1], { rerun: next[2] });
-          } catch (error) {
-            this.plugin.getLogger(next[0].workflowId).error(`execution (${next[0].id}) process failed`, { error });
-            if (pending && (0, import_utils.isLockAcquireError)(error)) {
-              this.pending.unshift({ ...pending, execution: next[0], immediate: true });
-            }
+        throw error;
+      }
+      await lock.runExclusive(async () => {
+        let executionCursor = null;
+        while (this.ready) {
+          const executions = await this.findQueueingExecutions(executionCursor, gracePeriod);
+          if (!executions.length) {
+            break;
           }
-        }
-      } catch (error) {
-        this.plugin.getLogger("dispatcher").error(`workflow dispatch failed`, { error });
-        if (pending) {
-          const dispatchAttempts = (pending.dispatchAttempts ?? 0) + 1;
-          if (dispatchAttempts < PENDING_DISPATCH_MAX_ATTEMPTS) {
-            this.pending.push({ ...pending, dispatchAttempts });
-          } else {
-            this.plugin.getLogger(pending.execution.workflowId).error(`pending execution (${pending.execution.id}) dispatch failed, local retry limit reached`, {
-              error,
-              dispatchAttempts
-            });
+          for (const execution of executions) {
+            if (!this.ready) {
+              return;
+            }
+            await this.enqueue({ executionId: execution.id });
           }
+          executionCursor = executions[executions.length - 1].id;
         }
-      } finally {
-        setImmediate(() => {
-          this.executing = null;
-          if (next || this.pending.length) {
-            this.plugin.getLogger("dispatcher").debug(`last process finished, will do another dispatch`);
-            this.dispatch();
-          }
-        });
-      }
-    })();
+      }, RECOVERY_LOCK_TTL);
+    } catch (error) {
+      logger.error(`workflow queue recovery failed`, { error });
+    }
+  }
+  async enqueue(task) {
+    await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, task, {
+      maxRetries: QUEUE_TASK_MAX_ATTEMPTS - 1
+    });
   }
-  async run(pending) {
-    this.pending.push({
-      ...pending,
-      immediate: !this.executing && !this.pending.length && !this.saving && !this.events.length
+  async loadJob(jobId) {
+    if (jobId == null) {
+      return null;
+    }
+    return this.plugin.db.getRepository("jobs").findOne({
+      filterByTk: jobId
     });
-    this.dispatch();
+  }
+  async findQueueingExecutions(afterId, gracePeriod) {
+    const executions = await this.plugin.db.getRepository("executions").find({
+      filter: {
+        ...afterId == null ? {} : { id: { $gt: afterId } },
+        ...gracePeriod > 0 ? { createdAt: { $lt: new Date(Date.now() - gracePeriod) } } : {},
+        dispatched: false,
+        status: import_constants.EXECUTION_STATUS.QUEUEING,
+        startedAt: null,
+        "workflow.enabled": true
+      },
+      sort: "id",
+      limit: RECOVERY_BATCH_SIZE
+    });
+    for (const execution of executions) {
+      this.plugin.getLogger(execution.workflowId).info(`queueing execution (${execution.id}) found, publishing task`);
+    }
+    return executions;
+  }
+  async resolveTask(task) {
+    const executionInput = await this.plugin.db.getRepository("executions").findOne({
+      filterByTk: task.executionId
+    });
+    if (!executionInput) {
+      this.plugin.getLogger("dispatcher").warn(`execution (${task.executionId}) not found, queue task ignored`);
+      return null;
+    }
+    if (task.jobId == null && task.rerun == null && executionInput.status === import_constants.EXECUTION_STATUS.STARTED && executionInput.startedAt) {
+      this.plugin.getLogger(executionInput.workflowId).warn(`execution (${executionInput.id}) has already started, start task ignored`);
+      return null;
+    }
+    let job = null;
+    if (task.jobId != null) {
+      job = await this.loadJob(task.jobId);
+      if (!job) {
+        this.plugin.getLogger(executionInput.workflowId).warn(`job (${task.jobId}) not found, resume ignored`);
+        return null;
+      }
+      if (String(job.executionId) !== String(executionInput.id)) {
+        this.plugin.getLogger(executionInput.workflowId).warn(`job (${job.id}) does not belong to execution (${executionInput.id}), resume ignored`);
+        return null;
+      }
+    }
+    const execution = await this.prepare(executionInput);
+    if (!execution) {
+      return null;
+    }
+    if (job) {
+      job.execution = execution;
+      return [execution, job];
+    }
+    return [execution, void 0, task.rerun];
   }
   async triggerSync(workflow, context, { deferred, ...options } = {}) {
     let execution;
@@ -367,7 +413,7 @@ class Dispatcher {
     return execution;
   }
   async prepare(input, options = {}) {
-    const logger = input ? this.plugin.getLogger(input.workflowId) : this.plugin.getLogger("dispatcher");
+    const logger = this.plugin.getLogger(input.workflowId);
     if (options.transaction) {
       try {
         return await this.acquireExecution(input, options, options.transaction);
@@ -402,8 +448,8 @@ class Dispatcher {
         },
         {
           logger,
-          conflictMessage: input ? `acquiring pending execution (${input.id}) conflicted with another worker, retrying` : `acquiring execution conflicted with another worker, retrying`,
-          maxAttemptsMessage: input ? `acquiring pending execution (${input.id}) reached max retry attempts` : `acquiring execution reached max retry attempts, will retry on next dispatch`
+          conflictMessage: `acquiring queue task execution (${input.id}) conflicted with another worker, retrying`,
+          maxAttemptsMessage: `acquiring queue task execution (${input.id}) reached max retry attempts`
         }
       );
     } catch (error) {
@@ -415,39 +461,10 @@ class Dispatcher {
     return result;
   }
   async acquireExecution(input, options, transaction) {
-    let execution = input;
-    if (execution) {
-      if (!options.immediate || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
-        await execution.reload({ transaction });
-      }
-    } else {
-      const workflowIds = [...this.plugin.enabledCache.keys()];
-      if (!workflowIds.length) {
-        this.plugin.getLogger("dispatcher").debug(`no enabled workflow to process`);
-        return null;
-      }
-      execution = await this.plugin.db.getRepository("executions").findOne({
-        filter: {
-          dispatched: false,
-          status: import_constants.EXECUTION_STATUS.QUEUEING,
-          startedAt: null,
-          workflowId: workflowIds
-        },
-        sort: "id",
-        transaction,
-        lock: transaction.LOCK.UPDATE,
-        skipLocked: true
-      });
-      if (execution) {
-        this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) fetched from db`);
-      } else {
-        this.plugin.getLogger("dispatcher").debug(`no execution in db queued to process`);
-      }
-    }
-    if (!execution) {
-      return null;
+    if (!options.immediate || input.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
+      await input.reload({ transaction });
     }
-    return this.enter(execution, transaction);
+    return this.enter(input, transaction);
   }
   async acquireWithRetry(acquire, options) {
     for (let attempt = 1; attempt <= EXECUTION_ACQUIRE_MAX_ATTEMPTS; attempt++) {
@@ -528,7 +545,7 @@ class Dispatcher {
       var _a, _b, _c;
       if (!execution.dispatched) {
         await execution.update({ dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED });
-        logger.info(`execution (${execution.id}) from pending list updated to started`);
+        logger.info(`execution (${execution.id}) from workflow queue updated to started`);
       }
       this.plugin.timeoutManager.scheduleExecutionTimeout(execution);
       const processor = this.plugin.createProcessor(execution, processorOptions);

package/dist/server/Plugin.d.ts

@@ -11,16 +11,39 @@ import { Transactionable } from 'sequelize';
 import { Plugin } from '@nocobase/server';
 import { Registry } from '@nocobase/utils';
 import { Logger } from '@nocobase/logger';
-import Dispatcher, { EventOptions } from './Dispatcher';
-import Processor from './Processor';
+import { EventOptions } from './Dispatcher';
+import Processor, { ProcessorRerunOptions } from './Processor';
 import ExecutionTimeoutManager from './ExecutionTimeoutManager';
 import RunningExecutionRegistry from './RunningExecutionRegistry';
 import { CustomFunction } from './functions';
 import Trigger from './triggers';
 import { InstructionInterface } from './instructions';
-import type { ExecutionModel, WorkflowModel } from './types';
+import type { ExecutionModel, JobModel, WorkflowModel } from './types';
 import type { Transaction } from '@nocobase/database';
 type ID = number | string;
+type TaskStats = {
+    pending: number;
+    all: number;
+};
+export type TaskStatsRow = {
+    userId: number;
+    workflowKey: string;
+    type: string;
+    pending: number;
+    all: number;
+};
+export type RepairTaskStatsOptions = {
+    userIds?: number[];
+    workflowKeys?: string[];
+    types?: string[];
+    transaction?: Transaction;
+    silent?: boolean;
+};
+export type TaskStatsProvider = {
+    collectTaskStats: (options: RepairTaskStatsOptions, context: {
+        plugin: PluginWorkflowServer;
+    }) => Promise<TaskStatsRow[]>;
+};
 export declare const WORKER_JOB_WORKFLOW_PROCESS = "workflow:process";
 export default class PluginWorkflowServer extends Plugin {
     instructions: Registry<InstructionInterface>;
@@ -35,6 +58,7 @@ export default class PluginWorkflowServer extends Plugin {
     private loggerCache;
     private meter;
     private checker;
+    private taskStatsProviders;
     private onBeforeSave;
     private onAfterCreate;
     private onAfterUpdate;
@@ -75,15 +99,24 @@ export default class PluginWorkflowServer extends Plugin {
     load(): Promise<void>;
     private toggle;
     trigger(workflow: WorkflowModel, context: object, options?: EventOptions): void | Promise<Processor | null | void>;
-    run(pending: Parameters<Dispatcher['run']>[0]): Promise<void>;
-    dispatch(): void;
-    resume(job: any): Promise<void>;
+    rerun(execution: ExecutionModel | ID, rerun?: ProcessorRerunOptions): Promise<void>;
+    recover(): Promise<void>;
+    /**
+     * Persist the current job state and publish a poke that evaluates that state.
+     *
+     * PENDING resume calls are valid for instructions such as delay and sequential approval.
+     * Callers that aggregate multiple user actions must only call this method when their
+     * atomic update changes the job from PENDING to a terminal status.
+     * Persistence and queue publication failures are logged and rethrown to awaiting callers.
+     */
+    resume(job: JobModel): Promise<void>;
     /**
      * Start a deferred execution
      * @experimental
      */
-    start(execution: ExecutionModel): Promise<void>;
+    start(execution: ExecutionModel | ID): Promise<void>;
     createProcessor(execution: ExecutionModel, options?: {}): Processor;
+    private getModelId;
     execute(workflow: WorkflowModel, values: any, options?: EventOptions): Promise<void | Processor>;
     /**
      * @experimental
@@ -94,12 +127,25 @@ export default class PluginWorkflowServer extends Plugin {
      */
     useDataSourceTransaction(dataSourceName?: string, transaction?: Transaction | null, create?: false): Transaction | undefined;
     useDataSourceTransaction(dataSourceName: string | undefined, transaction: Transaction | null | undefined, create: true): Transaction | Promise<Transaction> | undefined;
+    registerTaskStatsProvider(type: string, provider: TaskStatsProvider): void;
+    getWorkflowIdsByKey(workflowKey: string, transaction?: Transaction): Promise<any>;
+    private buildTaskStatsFilter;
+    private normalizeTaskStats;
+    private upsertTaskStatsRow;
+    refreshUserWorkflowTaskTypeStats(userId: number, type: string, { transaction }?: Transactionable): Promise<any>;
+    refreshUserWorkflowTaskWorkflowStats(userId: number, workflowKey: string, { transaction }?: Transactionable): Promise<any>;
+    private sendTaskWorkflowStatsUpdated;
+    updateTaskStatsByWorkflow(input: {
+        userId: number;
+        workflowKey: string;
+        type: string;
+        stats: TaskStats;
+    }, { transaction }?: Transactionable): Promise<void>;
+    repairTaskStats(options?: RepairTaskStatsOptions): Promise<void>;
     /**
      * @experimental
+     * @deprecated Use updateTaskStatsByWorkflow() when workflowKey is available.
      */
-    updateTasksStats(userId: number, type: string, stats: {
-        pending: number;
-        all: number;
-    }, { transaction }: Transactionable): Promise<void>;
+    updateTasksStats(userId: number, type: string, stats: TaskStats, { transaction }: Transactionable): Promise<void>;
 }
 export {};