@nocobase/plugin-workflow 1.7.0-beta.9 → 1.8.0-alpha.1

package/dist/server/Plugin.d.ts

@@ -6,6 +6,7 @@
  * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
+import { Snowflake } from 'nodejs-snowflake';
 import { Transactionable } from 'sequelize';
 import { Plugin } from '@nocobase/server';
 import { Registry } from '@nocobase/utils';
@@ -14,7 +15,7 @@ import Processor from './Processor';
 import { CustomFunction } from './functions';
 import Trigger from './triggers';
 import { InstructionInterface } from './instructions';
-import type { ExecutionModel, WorkflowModel, WorkflowTaskModel } from './types';
+import type { ExecutionModel, WorkflowModel } from './types';
 type ID = number | string;
 export type EventOptions = {
     eventKey?: string;
@@ -31,6 +32,7 @@ export default class PluginWorkflowServer extends Plugin {
     triggers: Registry<Trigger>;
     functions: Registry<CustomFunction>;
     enabledCache: Map<number, WorkflowModel>;
+    snowflake: Snowflake;
     private ready;
     private executing;
     private pending;
@@ -39,7 +41,13 @@ export default class PluginWorkflowServer extends Plugin {
     private loggerCache;
     private meter;
     private checker;
+    private onQueueExecution;
     private onBeforeSave;
+    private onAfterCreate;
+    private onAfterUpdate;
+    private onAfterDestroy;
+    private onAfterStart;
+    private onBeforeStop;
     handleSyncMessage(message: any): Promise<void>;
     /**
      * @experimental
@@ -91,6 +99,9 @@ export default class PluginWorkflowServer extends Plugin {
     /**
      * @experimental
      */
-    toggleTaskStatus(task: WorkflowTaskModel, done: boolean, { transaction }: Transactionable): Promise<void>;
+    updateTasksStats(userId: number, type: string, stats: {
+        pending: number;
+        all: number;
+    }, { transaction }: Transactionable): Promise<void>;
 }
 export {};

package/dist/server/Plugin.js

@@ -41,6 +41,7 @@ __export(Plugin_exports, {
 module.exports = __toCommonJS(Plugin_exports);
 var import_path = __toESM(require("path"));
 var import_crypto = require("crypto");
+var import_nodejs_snowflake = require("nodejs-snowflake");
 var import_sequelize = require("sequelize");
 var import_lru_cache = __toESM(require("lru-cache"));
 var import_database = require("@nocobase/database");
@@ -60,12 +61,12 @@ var import_DestroyInstruction = __toESM(require("./instructions/DestroyInstructi
 var import_QueryInstruction = __toESM(require("./instructions/QueryInstruction"));
 var import_UpdateInstruction = __toESM(require("./instructions/UpdateInstruction"));
 var import_WorkflowRepository = __toESM(require("./repositories/WorkflowRepository"));
-var import_WorkflowTasksRepository = __toESM(require("./repositories/WorkflowTasksRepository"));
 class PluginWorkflowServer extends import_server.Plugin {
   instructions = new import_utils.Registry();
   triggers = new import_utils.Registry();
   functions = new import_utils.Registry();
   enabledCache = /* @__PURE__ */ new Map();
+  snowflake;
   ready = false;
   executing = null;
   pending = [];
@@ -74,8 +75,28 @@ class PluginWorkflowServer extends import_server.Plugin {
   loggerCache;
   meter = null;
   checker = null;
-  onBeforeSave = async (instance, { transaction }) => {
+  onQueueExecution = async (event) => {
+    const ExecutionRepo = this.db.getRepository("executions");
+    const execution = await ExecutionRepo.findOne({
+      filterByTk: event.executionId
+    });
+    if (!execution || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
+      return;
+    }
+    this.getLogger(execution.workflowId).info(
+      `execution (${execution.id}) received from queue, adding to pending list`
+    );
+    this.pending.push([execution]);
+    this.dispatch();
+  };
+  onBeforeSave = async (instance, { transaction, cycling }) => {
+    if (cycling) {
+      return;
+    }
     const Model = instance.constructor;
+    if (!instance.key) {
+      instance.set("key", (0, import_utils.uid)());
+    }
     if (instance.enabled) {
       instance.set("current", true);
     }
@@ -91,18 +112,95 @@ class PluginWorkflowServer extends import_server.Plugin {
     });
     if (!previous) {
       instance.set("current", true);
-    }
-    if (instance.current && previous) {
+    } else if (instance.current) {
       await previous.update(
         { enabled: false, current: null },
         {
           transaction,
-          hooks: false
+          cycling: true
         }
       );
       this.toggle(previous, false, { transaction });
     }
   };
+  onAfterCreate = async (model, { transaction }) => {
+    const WorkflowStatsModel = this.db.getModel("workflowStats");
+    let stats = await WorkflowStatsModel.findOne({
+      where: { key: model.key },
+      transaction
+    });
+    if (!stats) {
+      stats = await model.createStats({ executed: 0 }, { transaction });
+    }
+    model.stats = stats;
+    model.versionStats = await model.createVersionStats({ id: model.id }, { transaction });
+    if (model.enabled) {
+      this.toggle(model, true, { transaction });
+    }
+  };
+  onAfterUpdate = async (model, { transaction }) => {
+    model.stats = await model.getStats({ transaction });
+    model.versionStats = await model.getVersionStats({ transaction });
+    this.toggle(model, model.enabled, { transaction });
+  };
+  onAfterDestroy = async (model, { transaction }) => {
+    this.toggle(model, false, { transaction });
+    const TaskRepo = this.db.getRepository("workflowTasks");
+    await TaskRepo.destroy({
+      filter: {
+        workflowId: model.id
+      },
+      transaction
+    });
+  };
+  // [Life Cycle]:
+  //   * load all workflows in db
+  //   * add all hooks for enabled workflows
+  //   * add hooks for create/update[enabled]/delete workflow to add/remove specific hooks
+  onAfterStart = async () => {
+    this.ready = true;
+    const collection = this.db.getCollection("workflows");
+    const workflows = await collection.repository.find({
+      filter: { enabled: true },
+      appends: ["stats", "versionStats"]
+    });
+    for (const workflow of workflows) {
+      if (!workflow.stats) {
+        workflow.stats = await workflow.createStats({ executed: 0 });
+      }
+      if (!workflow.versionStats) {
+        workflow.versionStats = await workflow.createVersionStats({ executed: 0 });
+      }
+      this.toggle(workflow, true, { silent: true });
+    }
+    this.checker = setInterval(() => {
+      this.getLogger("dispatcher").info(`(cycling) check for queueing executions`);
+      this.dispatch();
+    }, 3e5);
+    this.app.on("workflow:dispatch", () => {
+      this.app.logger.info("workflow:dispatch");
+      this.dispatch();
+    });
+    this.getLogger("dispatcher").info("(starting) check for queueing executions");
+    this.dispatch();
+    this.ready = true;
+  };
+  onBeforeStop = async () => {
+    this.app.logger.info(`stopping workflow plugin before app (${this.app.name}) shutdown...`);
+    for (const workflow of this.enabledCache.values()) {
+      this.toggle(workflow, false, { silent: true });
+    }
+    this.ready = false;
+    if (this.events.length) {
+      await this.prepare();
+    }
+    if (this.executing) {
+      await this.executing;
+    }
+    if (this.checker) {
+      clearInterval(this.checker);
+    }
+  };
   async handleSyncMessage(message) {
     if (message.type === "statusChange") {
       if (message.enabled) {
@@ -193,8 +291,18 @@ class PluginWorkflowServer extends import_server.Plugin {
   }
   async beforeLoad() {
     this.db.registerRepositories({
-      WorkflowRepository: import_WorkflowRepository.default,
-      WorkflowTasksRepository: import_WorkflowTasksRepository.default
+      WorkflowRepository: import_WorkflowRepository.default
+    });
+    const PluginRepo = this.db.getRepository("applicationPlugins");
+    const pluginRecord = await PluginRepo.findOne({
+      filter: { name: this.name }
+    });
+    this.snowflake = new import_nodejs_snowflake.Snowflake({
+      custom_epoch: pluginRecord == null ? void 0 : pluginRecord.createdAt.getTime()
+    });
+    this.app.eventQueue.subscribe(`${this.name}.pendingExecution`, {
+      idle: () => !this.executing && !this.pending.length && !this.events.length,
+      process: this.onQueueExecution
     });
   }
   /**
@@ -230,77 +338,22 @@ class PluginWorkflowServer extends import_server.Plugin {
         "flow_nodes:update",
         "flow_nodes:destroy",
         "flow_nodes:test",
-        "jobs:get"
+        "jobs:get",
+        "workflowCategories:*"
       ]
     });
     this.app.acl.registerSnippet({
       name: "ui.workflows",
       actions: ["workflows:list"]
     });
-    this.app.acl.allow("workflowTasks", "countMine", "loggedIn");
+    this.app.acl.allow("userWorkflowTasks", "listMine", "loggedIn");
     this.app.acl.allow("*", ["trigger"], "loggedIn");
-    this.db.addMigrations({
-      namespace: this.name,
-      directory: import_path.default.resolve(__dirname, "migrations"),
-      context: {
-        plugin: this
-      }
-    });
     db.on("workflows.beforeSave", this.onBeforeSave);
-    db.on("workflows.afterCreate", (model, { transaction }) => {
-      if (model.enabled) {
-        this.toggle(model, true, { transaction });
-      }
-    });
-    db.on(
-      "workflows.afterUpdate",
-      (model, { transaction }) => this.toggle(model, model.enabled, { transaction })
-    );
-    db.on("workflows.afterDestroy", async (model, { transaction }) => {
-      this.toggle(model, false, { transaction });
-      const TaskRepo = this.db.getRepository("workflowTasks");
-      await TaskRepo.destroy({
-        filter: {
-          workflowId: model.id
-        },
-        transaction
-      });
-    });
-    this.app.on("afterStart", async () => {
-      this.ready = true;
-      const collection = db.getCollection("workflows");
-      const workflows = await collection.repository.find({
-        filter: { enabled: true }
-      });
-      workflows.forEach((workflow) => {
-        this.toggle(workflow, true, { silent: true });
-      });
-      this.checker = setInterval(() => {
-        this.getLogger("dispatcher").info(`(cycling) check for queueing executions`);
-        this.dispatch();
-      }, 3e5);
-      this.app.on("workflow:dispatch", () => {
-        this.app.logger.info("workflow:dispatch");
-        this.dispatch();
-      });
-      this.getLogger("dispatcher").info("(starting) check for queueing executions");
-      this.dispatch();
-    });
-    this.app.on("beforeStop", async () => {
-      for (const workflow of this.enabledCache.values()) {
-        this.toggle(workflow, false, { silent: true });
-      }
-      this.ready = false;
-      if (this.events.length) {
-        await this.prepare();
-      }
-      if (this.executing) {
-        await this.executing;
-      }
-      if (this.checker) {
-        clearInterval(this.checker);
-      }
-    });
+    db.on("workflows.afterCreate", this.onAfterCreate);
+    db.on("workflows.afterUpdate", this.onAfterUpdate);
+    db.on("workflows.afterDestroy", this.onAfterDestroy);
+    this.app.on("afterStart", this.onAfterStart);
+    this.app.on("beforeStop", this.onBeforeStop);
   }
   toggle(workflow, enable, { silent, transaction } = {}) {
     const type = workflow.get("type");
@@ -314,11 +367,14 @@ class PluginWorkflowServer extends import_server.Plugin {
       const prev = workflow.previous();
       if (prev.config) {
         trigger.off({ ...workflow.get(), ...prev });
+        this.getLogger(workflow.id).info(`toggle OFF workflow ${workflow.id} based on configuration before updated`);
       }
       trigger.on(workflow);
+      this.getLogger(workflow.id).info(`toggle ON workflow ${workflow.id}`);
       this.enabledCache.set(workflow.id, workflow);
     } else {
       trigger.off(workflow);
+      this.getLogger(workflow.id).info(`toggle OFF workflow ${workflow.id}`);
       this.enabledCache.delete(workflow.id);
     }
     if (!silent) {
@@ -470,21 +526,20 @@ class PluginWorkflowServer extends import_server.Plugin {
       throw err;
     }
     this.getLogger(workflow.id).info(`execution of workflow ${workflow.id} created as ${execution.id}`);
-    await workflow.increment(["executed", "allExecuted"], { transaction });
+    if (!workflow.stats) {
+      workflow.stats = await workflow.getStats({ transaction });
+    }
+    await workflow.stats.increment("executed", { transaction });
     if (this.db.options.dialect !== "postgres") {
-      await workflow.reload({ transaction });
+      await workflow.stats.reload({ transaction });
+    }
+    if (!workflow.versionStats) {
+      workflow.versionStats = await workflow.getVersionStats({ transaction });
+    }
+    await workflow.versionStats.increment("executed", { transaction });
+    if (this.db.options.dialect !== "postgres") {
+      await workflow.versionStats.reload({ transaction });
     }
-    await workflow.constructor.update(
-      {
-        allExecuted: workflow.allExecuted
-      },
-      {
-        where: {
-          key: workflow.key
-        },
-        transaction
-      }
-    );
     if (!sameTransaction) {
       await transaction.commit();
     }
@@ -505,8 +560,14 @@ class PluginWorkflowServer extends import_server.Plugin {
     logger.info(`preparing execution for event`);
     try {
       const execution = await this.createExecution(...event);
-      if ((execution == null ? void 0 : execution.status) === import_constants.EXECUTION_STATUS.QUEUEING && !this.executing && !this.pending.length) {
-        this.pending.push([execution]);
+      if ((execution == null ? void 0 : execution.status) === import_constants.EXECUTION_STATUS.QUEUEING) {
+        if (!this.executing && !this.pending.length) {
+          logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
+          this.pending.push([execution]);
+        } else {
+          logger.info(`local pending list is not empty, sending execution (${execution.id}) to queue`);
+          this.app.eventQueue.publish(`${this.name}.pendingExecution`, { executionId: execution.id });
+        }
       }
     } catch (error) {
       logger.error(`failed to create execution:`, { error });
@@ -638,36 +699,38 @@ class PluginWorkflowServer extends import_server.Plugin {
   /**
    * @experimental
    */
-  async toggleTaskStatus(task, done, { transaction }) {
+  async updateTasksStats(userId, type, stats = { pending: 0, all: 0 }, { transaction }) {
     const { db } = this.app;
-    const repository = db.getRepository("workflowTasks");
-    if (done) {
-      await repository.destroy({
-        filter: {
-          type: task.type,
-          key: `${task.key}`
+    const repository = db.getRepository("userWorkflowTasks");
+    let record = await repository.findOne({
+      filter: {
+        userId,
+        type
+      },
+      transaction
+    });
+    if (record) {
+      await record.update(
+        {
+          stats
         },
-        transaction
-      });
+        { transaction }
+      );
     } else {
-      await repository.updateOrCreate({
-        filterKeys: ["key", "type"],
-        values: task,
+      record = await repository.create({
+        values: {
+          userId,
+          type,
+          stats
+        },
         transaction
       });
     }
-    if (task.userId) {
-      const counts = await repository.countAll({
-        where: {
-          userId: task.userId,
-          workflowId: { [import_database.Op.ne]: null }
-        },
-        transaction
-      }) || [];
+    if (userId) {
       this.app.emit("ws:sendToTag", {
         tagKey: "userId",
-        tagValue: `${task.userId}`,
-        message: { type: "workflow:tasks:updated", payload: counts }
+        tagValue: `${userId}`,
+        message: { type: "workflow:tasks:updated", payload: record.get() }
       });
     }
   }

package/dist/server/Processor.d.ts

@@ -44,16 +44,9 @@ export default class Processor {
      * @experimental
      */
     nodesMap: Map<number, FlowNodeModel>;
-    /**
-     * @experimental
-     */
-    jobsMap: Map<number, JobModel>;
-    /**
-     * @experimental
-     */
-    jobsMapByNodeKey: {
-        [key: string]: any;
-    };
+    private jobsMapByNodeKey;
+    private jobResultsMapByNodeKey;
+    private jobsToSave;
     /**
      * @experimental
      */
@@ -72,7 +65,7 @@ export default class Processor {
     /**
      * @experimental
      */
-    saveJob(payload: JobModel | Record<string, any>): Promise<JobModel>;
+    saveJob(payload: JobModel | Record<string, any>): JobModel;
     /**
      * @experimental
      */

package/dist/server/Processor.js

@@ -78,14 +78,9 @@ class Processor {
    * @experimental
    */
   nodesMap = /* @__PURE__ */ new Map();
-  /**
-   * @experimental
-   */
-  jobsMap = /* @__PURE__ */ new Map();
-  /**
-   * @experimental
-   */
   jobsMapByNodeKey = {};
+  jobResultsMapByNodeKey = {};
+  jobsToSave = /* @__PURE__ */ new Map();
   /**
    * @experimental
    */
@@ -107,9 +102,9 @@ class Processor {
   }
   makeJobs(jobs) {
     jobs.forEach((job) => {
-      this.jobsMap.set(job.id, job);
       const node = this.nodesMap.get(job.nodeId);
-      this.jobsMapByNodeKey[node.key] = job.result;
+      this.jobsMapByNodeKey[node.key] = job;
+      this.jobResultsMapByNodeKey[node.key] = job.result;
     });
   }
   async prepare() {
@@ -123,11 +118,13 @@ class Processor {
       execution.workflow = plugin.enabledCache.get(execution.workflowId) || await execution.getWorkflow({ transaction });
     }
     const nodes = await execution.workflow.getNodes({ transaction });
+    execution.workflow.nodes = nodes;
     this.makeNodes(nodes);
     const jobs = await execution.getJobs({
       order: [["id", "ASC"]],
       transaction
     });
+    execution.jobs = jobs;
     this.makeJobs(jobs);
   }
   async start() {
@@ -179,11 +176,10 @@ class Processor {
       }
     }
     if (!(job instanceof import_database.Model)) {
-      job.upstreamId = prevJob instanceof import_database.Model ? prevJob.get("id") : null;
       job.nodeId = node.id;
       job.nodeKey = node.key;
     }
-    const savedJob = await this.saveJob(job);
+    const savedJob = this.saveJob(job);
     this.logger.info(
       `execution (${this.execution.id}) run instruction [${node.type}] for node (${node.id}) finished as status: ${savedJob.status}`
     );
@@ -230,6 +226,30 @@ class Processor {
     return this.exec(instruction.resume.bind(instruction), node, job);
   }
   async exit(s) {
+    if (this.jobsToSave.size) {
+      const newJobs = [];
+      for (const job of this.jobsToSave.values()) {
+        if (job.isNewRecord) {
+          newJobs.push(job);
+        } else {
+          await job.save({ transaction: this.mainTransaction });
+        }
+      }
+      if (newJobs.length) {
+        const JobsModel = this.options.plugin.db.getModel("jobs");
+        await JobsModel.bulkCreate(
+          newJobs.map((job) => job.toJSON()),
+          {
+            transaction: this.mainTransaction,
+            returning: false
+          }
+        );
+        for (const job of newJobs) {
+          job.isNewRecord = false;
+        }
+      }
+      this.jobsToSave.clear();
+    }
     if (typeof s === "number") {
       const status = this.constructor.StatusMap[s] ?? Math.sign(s);
       await this.execution.update({ status }, { transaction: this.mainTransaction });
@@ -240,32 +260,29 @@ class Processor {
     this.logger.info(`execution (${this.execution.id}) exiting with status ${this.execution.status}`);
     return null;
   }
-  // TODO(optimize)
   /**
    * @experimental
    */
-  async saveJob(payload) {
+  saveJob(payload) {
     const { database } = this.execution.constructor;
-    const { mainTransaction: transaction } = this;
     const { model } = database.getCollection("jobs");
     let job;
     if (payload instanceof model) {
-      job = await payload.save({ transaction });
-    } else if (payload.id) {
-      job = await model.findByPk(payload.id, { transaction });
-      await job.update(payload, { transaction });
+      job = payload;
+      job.set("updatedAt", /* @__PURE__ */ new Date());
     } else {
-      job = await model.create(
-        {
-          ...payload,
-          executionId: this.execution.id
-        },
-        { transaction }
-      );
+      job = model.build({
+        ...payload,
+        id: this.options.plugin.snowflake.getUniqueID().toString(),
+        createdAt: /* @__PURE__ */ new Date(),
+        updatedAt: /* @__PURE__ */ new Date(),
+        executionId: this.execution.id
+      });
     }
-    this.jobsMap.set(job.id, job);
+    this.jobsToSave.set(job.id, job);
     this.lastSavedJob = job;
-    this.jobsMapByNodeKey[job.nodeKey] = job.result;
+    this.jobsMapByNodeKey[job.nodeKey] = job;
+    this.jobResultsMapByNodeKey[job.nodeKey] = job.result;
     return job;
   }
   /**
@@ -319,31 +336,19 @@ class Processor {
    * @experimental
    */
   findBranchParentJob(job, node) {
-    for (let j = job; j; j = this.jobsMap.get(j.upstreamId)) {
-      if (j.nodeId === node.id) {
-        return j;
-      }
-    }
-    return null;
+    return this.jobsMapByNodeKey[node.key];
   }
   /**
    * @experimental
    */
   findBranchLastJob(node, job) {
-    const allJobs = Array.from(this.jobsMap.values());
+    const allJobs = Object.values(this.jobsMapByNodeKey);
     const branchJobs = [];
     for (let n = this.findBranchEndNode(node); n && n !== node.upstream; n = n.upstream) {
       branchJobs.push(...allJobs.filter((item) => item.nodeId === n.id));
     }
-    branchJobs.sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime());
-    for (let i = branchJobs.length - 1; i >= 0; i -= 1) {
-      for (let j = branchJobs[i]; j && j.id !== job.id; j = this.jobsMap.get(j.upstreamId)) {
-        if (j.upstreamId === job.id) {
-          return branchJobs[i];
-        }
-      }
-    }
-    return null;
+    branchJobs.sort((a, b) => a.updatedAt.getTime() - b.updatedAt.getTime());
+    return branchJobs[branchJobs.length - 1] || null;
   }
   /**
    * @experimental
@@ -362,12 +367,12 @@ class Processor {
     for (let n = includeSelfScope ? node : this.findBranchParentNode(node); n; n = this.findBranchParentNode(n)) {
       const instruction = this.options.plugin.instructions.get(n.type);
       if (typeof (instruction == null ? void 0 : instruction.getScope) === "function") {
-        $scopes[n.id] = $scopes[n.key] = instruction.getScope(n, this.jobsMapByNodeKey[n.key], this);
+        $scopes[n.id] = $scopes[n.key] = instruction.getScope(n, this.jobResultsMapByNodeKey[n.key], this);
       }
     }
     return {
       $context: this.execution.context,
-      $jobsMapByNodeKey: this.jobsMapByNodeKey,
+      $jobsMapByNodeKey: this.jobResultsMapByNodeKey,
       $system: systemFns,
       $scopes,
       $env: this.options.plugin.app.environment.getVariables()

package/dist/server/actions/index.js

@@ -42,7 +42,7 @@ module.exports = __toCommonJS(actions_exports);
 var workflows = __toESM(require("./workflows"));
 var nodes = __toESM(require("./nodes"));
 var executions = __toESM(require("./executions"));
-var workflowTasks = __toESM(require("./workflowTasks"));
+var userWorkflowTasks = __toESM(require("./userWorkflowTasks"));
 function make(name, mod) {
   return Object.keys(mod).reduce(
     (result, key) => ({
@@ -64,6 +64,6 @@ function actions_default({ app }) {
       test: nodes.test
     }),
     ...make("executions", executions),
-    ...make("workflowTasks", workflowTasks)
+    ...make("userWorkflowTasks", userWorkflowTasks)
   });
 }