@nocobase/plugin-workflow 0.10.1-alpha.1 → 0.11.0-alpha.1

package/src/index.ts

@@ -0,0 +1 @@
+export { default } from './server';

package/src/server/Plugin.ts

@@ -0,0 +1,351 @@
+import path from 'path';
+
+import winston from 'winston';
+import LRUCache from 'lru-cache';
+
+import { Op } from '@nocobase/database';
+import { Plugin } from '@nocobase/server';
+import { Registry } from '@nocobase/utils';
+
+import initFields from './fields';
+import initActions from './actions';
+import { EXECUTION_STATUS } from './constants';
+import initInstructions, { Instruction } from './instructions';
+import Processor from './Processor';
+import initTriggers, { Trigger } from './triggers';
+import initFunctions, { CustomFunction } from './functions';
+import { createLogger, Logger, LoggerOptions, getLoggerLevel, getLoggerFilePath } from '@nocobase/logger';
+
+import type { WorkflowModel, ExecutionModel, JobModel } from './types';
+
+type Pending = [ExecutionModel, JobModel?];
+
+type ID = number | string;
+export default class WorkflowPlugin extends Plugin {
+  instructions: Registry<Instruction> = new Registry();
+  triggers: Registry<Trigger> = new Registry();
+  functions: Registry<CustomFunction> = new Registry();
+  private executing = false;
+  private pending: Pending[] = [];
+  private events: [WorkflowModel, any, { context?: any }][] = [];
+
+  private loggerCache: LRUCache<string, Logger>;
+
+  getLogger(workflowId: ID): Logger {
+    const now = new Date();
+    const date = `${now.getFullYear()}-${`0${now.getMonth() + 1}`.slice(-2)}-${`0${now.getDate()}`.slice(-2)}`;
+    const key = `${date}-${workflowId}}`;
+    if (this.loggerCache.has(key)) {
+      return this.loggerCache.get(key);
+    }
+
+    const logger = createLogger({
+      transports: [
+        'console',
+        new winston.transports.File({
+          filename: getLoggerFilePath('workflows', date, `${workflowId}.log`),
+          level: getLoggerLevel(),
+        }),
+      ],
+    } as LoggerOptions);
+
+    this.loggerCache.set(key, logger);
+
+    return logger;
+  }
+
+  onBeforeSave = async (instance: WorkflowModel, options) => {
+    const Model = <typeof WorkflowModel>instance.constructor;
+
+    if (instance.enabled) {
+      instance.set('current', true);
+    } else if (!instance.current) {
+      const count = await Model.count({
+        where: {
+          key: instance.key,
+        },
+        transaction: options.transaction,
+      });
+      if (!count) {
+        instance.set('current', true);
+      }
+    }
+
+    if (!instance.changed('enabled') || !instance.enabled) {
+      return;
+    }
+
+    const previous = await Model.findOne({
+      where: {
+        key: instance.key,
+        current: true,
+        id: {
+          [Op.ne]: instance.id,
+        },
+      },
+      transaction: options.transaction,
+    });
+
+    if (previous) {
+      // NOTE: set to `null` but not `false` will not violate the unique index
+      await previous.update(
+        { enabled: false, current: null },
+        {
+          transaction: options.transaction,
+          hooks: false,
+        },
+      );
+
+      this.toggle(previous, false);
+    }
+  };
+
+  async load() {
+    const { db, options } = this;
+
+    initFields(this);
+    initActions(this);
+    initTriggers(this, options.triggers);
+    initInstructions(this, options.instructions);
+    initFunctions(this, options.functions);
+
+    this.loggerCache = new LRUCache({
+      max: 20,
+      updateAgeOnGet: true,
+      dispose(logger) {
+        (<Logger>logger).end();
+      },
+    });
+
+    this.app.acl.registerSnippet({
+      name: `pm.${this.name}.workflows`,
+      actions: [
+        'workflows:*',
+        'workflows.nodes:*',
+        'executions:list',
+        'executions:get',
+        'flow_nodes:update',
+        'flow_nodes:destroy',
+      ],
+    });
+
+    this.app.acl.allow('users_jobs', ['list', 'get', 'submit'], 'loggedIn');
+
+    await db.import({
+      directory: path.resolve(__dirname, 'collections'),
+    });
+
+    this.db.addMigrations({
+      namespace: 'workflow',
+      directory: path.resolve(__dirname, 'migrations'),
+      context: {
+        plugin: this,
+      },
+    });
+
+    db.on('workflows.beforeSave', this.onBeforeSave);
+    db.on('workflows.afterSave', (model: WorkflowModel) => this.toggle(model));
+    db.on('workflows.afterDestroy', (model: WorkflowModel) => this.toggle(model, false));
+
+    // [Life Cycle]:
+    //   * load all workflows in db
+    //   * add all hooks for enabled workflows
+    //   * add hooks for create/update[enabled]/delete workflow to add/remove specific hooks
+    this.app.on('beforeStart', async () => {
+      const collection = db.getCollection('workflows');
+      const workflows = await collection.repository.find({
+        filter: { enabled: true },
+      });
+
+      workflows.forEach((workflow: WorkflowModel) => {
+        this.toggle(workflow);
+      });
+    });
+
+    this.app.on('afterStart', () => {
+      // check for not started executions
+      this.dispatch();
+    });
+
+    this.app.on('beforeStop', async () => {
+      const collection = db.getCollection('workflows');
+      const workflows = await collection.repository.find({
+        filter: { enabled: true },
+      });
+
+      workflows.forEach((workflow: WorkflowModel) => {
+        this.toggle(workflow, false);
+      });
+    });
+  }
+
+  toggle(workflow: WorkflowModel, enable?: boolean) {
+    const type = workflow.get('type');
+    const trigger = this.triggers.get(type);
+    if (typeof enable !== 'undefined' ? enable : workflow.get('enabled')) {
+      // NOTE: remove previous listener if config updated
+      const prev = workflow.previous();
+      if (prev.config) {
+        trigger.off({ ...workflow.get(), ...prev });
+      }
+      trigger.on(workflow);
+    } else {
+      trigger.off(workflow);
+    }
+  }
+
+  public trigger(workflow: WorkflowModel, context: { [key: string]: any }, options: { context?: any } = {}): void {
+    // `null` means not to trigger
+    if (context == null) {
+      return;
+    }
+
+    this.events.push([workflow, context, options]);
+
+    this.getLogger(workflow.id).info(`new event triggered, now events: ${this.events.length}`);
+    this.getLogger(workflow.id).debug(`event data:`, {
+      data: context,
+    });
+
+    if (this.events.length > 1) {
+      return;
+    }
+
+    // NOTE: no await for quick return
+    setTimeout(this.prepare);
+  }
+
+  private prepare = async () => {
+    const event = this.events.shift();
+    if (!event) {
+      return;
+    }
+    const [workflow, context, options] = event;
+
+    let valid = true;
+    if (options.context?.executionId) {
+      // NOTE: no transaction here for read-uncommitted execution
+      const existed = await workflow.countExecutions({
+        where: {
+          id: options.context.executionId,
+        },
+      });
+
+      if (existed) {
+        this.getLogger(workflow.id).warn(
+          `workflow ${workflow.id} has already been triggered in same execution (${options.context.executionId}), and newly triggering will be skipped.`,
+        );
+
+        valid = false;
+      }
+    }
+
+    if (valid) {
+      const execution = await this.db.sequelize.transaction(async (transaction) => {
+        const execution = await workflow.createExecution(
+          {
+            context,
+            key: workflow.key,
+            status: EXECUTION_STATUS.QUEUEING,
+            useTransaction: workflow.useTransaction,
+          },
+          { transaction },
+        );
+
+        await workflow.increment('executed', { transaction });
+
+        await (<typeof WorkflowModel>workflow.constructor).increment('allExecuted', {
+          where: {
+            key: workflow.key,
+          },
+          transaction,
+        });
+
+        execution.workflow = workflow;
+
+        return execution;
+      });
+
+      this.getLogger(workflow.id).debug(`execution of workflow ${workflow.id} created as ${execution.id}`, {
+        data: execution.context,
+      });
+
+      // NOTE: cache first execution for most cases
+      if (!this.executing && !this.pending.length) {
+        this.pending.push([execution]);
+      }
+    }
+
+    if (this.events.length) {
+      await this.prepare();
+    } else {
+      this.dispatch();
+    }
+  };
+
+  public async resume(job) {
+    if (!job.execution) {
+      job.execution = await job.getExecution();
+    }
+
+    this.pending.push([job.execution, job]);
+    this.dispatch();
+  }
+
+  private async dispatch() {
+    if (this.executing) {
+      return;
+    }
+
+    this.executing = true;
+
+    let next: Pending | null = null;
+    // resuming has high priority
+    if (this.pending.length) {
+      next = this.pending.shift() as Pending;
+    } else {
+      const execution = (await this.db.getRepository('executions').findOne({
+        filter: {
+          status: EXECUTION_STATUS.QUEUEING,
+        },
+        appends: ['workflow'],
+        sort: 'createdAt',
+      })) as ExecutionModel;
+      if (execution && execution.workflow.enabled) {
+        next = [execution];
+      }
+    }
+    if (next) {
+      this.process(...next);
+    } else {
+      this.executing = false;
+    }
+  }
+
+  private async process(execution: ExecutionModel, job?: JobModel) {
+    if (execution.status === EXECUTION_STATUS.QUEUEING) {
+      await execution.update({ status: EXECUTION_STATUS.STARTED });
+    }
+
+    const processor = this.createProcessor(execution);
+
+    this.getLogger(execution.workflowId).info(`execution (${execution.id}) ${job ? 'resuming' : 'starting'}...`);
+
+    try {
+      await (job ? processor.resume(job) : processor.start());
+      this.getLogger(execution.workflowId).info(
+        `execution (${execution.id}) finished with status: ${execution.status}`,
+      );
+    } catch (err) {
+      this.getLogger(execution.workflowId).error(`execution (${execution.id}) error: ${err.message}`, err);
+    }
+
+    this.executing = false;
+
+    this.dispatch();
+  }
+
+  public createProcessor(execution: ExecutionModel, options = {}): Processor {
+    return new Processor(execution, { ...options, plugin: this });
+  }
+}

package/src/server/Processor.ts

@@ -0,0 +1,354 @@
+import { Model, Transaction, Transactionable } from '@nocobase/database';
+import { appendArrayColumn } from '@nocobase/evaluators';
+import { Logger } from '@nocobase/logger';
+import { parse } from '@nocobase/utils';
+import Plugin from '.';
+import { EXECUTION_STATUS, JOB_STATUS } from './constants';
+import { Runner } from './instructions';
+import type { ExecutionModel, FlowNodeModel, JobModel } from './types';
+
+export interface ProcessorOptions extends Transactionable {
+  plugin: Plugin;
+}
+
+export default class Processor {
+  static StatusMap = {
+    [JOB_STATUS.PENDING]: EXECUTION_STATUS.STARTED,
+    [JOB_STATUS.RESOLVED]: EXECUTION_STATUS.RESOLVED,
+    [JOB_STATUS.FAILED]: EXECUTION_STATUS.FAILED,
+    [JOB_STATUS.ERROR]: EXECUTION_STATUS.ERROR,
+    [JOB_STATUS.ABORTED]: EXECUTION_STATUS.ABORTED,
+    [JOB_STATUS.CANCELED]: EXECUTION_STATUS.CANCELED,
+    [JOB_STATUS.REJECTED]: EXECUTION_STATUS.REJECTED,
+  };
+
+  logger: Logger;
+
+  transaction?: Transaction;
+
+  nodes: FlowNodeModel[] = [];
+  nodesMap = new Map<number, FlowNodeModel>();
+  jobsMap = new Map<number, JobModel>();
+  jobsMapByNodeId: { [key: number]: any } = {};
+
+  constructor(public execution: ExecutionModel, public options: ProcessorOptions) {
+    this.logger = options.plugin.getLogger(execution.workflowId);
+  }
+
+  // make dual linked nodes list then cache
+  private makeNodes(nodes: FlowNodeModel[] = []) {
+    this.nodes = nodes;
+
+    nodes.forEach((node) => {
+      this.nodesMap.set(node.id, node);
+    });
+
+    nodes.forEach((node) => {
+      if (node.upstreamId) {
+        node.upstream = this.nodesMap.get(node.upstreamId) as FlowNodeModel;
+      }
+
+      if (node.downstreamId) {
+        node.downstream = this.nodesMap.get(node.downstreamId) as FlowNodeModel;
+      }
+    });
+  }
+
+  private makeJobs(jobs: Array<JobModel>) {
+    jobs.forEach((job) => {
+      this.jobsMap.set(job.id, job);
+      // TODO: should consider cycle, and from previous job
+      this.jobsMapByNodeId[job.nodeId] = job.result;
+    });
+  }
+
+  private async getTransaction() {
+    if (!this.execution.useTransaction) {
+      return;
+    }
+
+    const { options } = this;
+
+    // @ts-ignore
+    return options.transaction && !options.transaction.finished
+      ? options.transaction
+      : await options.plugin.db.sequelize.transaction();
+  }
+
+  public async prepare() {
+    const transaction = await this.getTransaction();
+    this.transaction = transaction;
+
+    const { execution } = this;
+    if (!execution.workflow) {
+      execution.workflow = await execution.getWorkflow({ transaction });
+    }
+
+    const nodes = await execution.workflow.getNodes({ transaction });
+
+    this.makeNodes(nodes);
+
+    const jobs = await execution.getJobs({
+      order: [['id', 'ASC']],
+      transaction,
+    });
+
+    this.makeJobs(jobs);
+  }
+
+  public async start() {
+    const { execution } = this;
+    if (execution.status !== EXECUTION_STATUS.STARTED) {
+      throw new Error(`execution was ended with status ${execution.status}`);
+    }
+    await this.prepare();
+    if (this.nodes.length) {
+      const head = this.nodes.find((item) => !item.upstream);
+      await this.run(head, { result: execution.context });
+    } else {
+      await this.exit(JOB_STATUS.RESOLVED);
+    }
+  }
+
+  public async resume(job: JobModel) {
+    const { execution } = this;
+    if (execution.status !== EXECUTION_STATUS.STARTED) {
+      throw new Error(`execution was ended with status ${execution.status}`);
+    }
+    await this.prepare();
+    const node = this.nodesMap.get(job.nodeId);
+    await this.recall(node, job);
+  }
+
+  private async commit() {
+    // @ts-ignore
+    if (this.transaction && (!this.options.transaction || this.options.transaction.finished)) {
+      await this.transaction.commit();
+    }
+  }
+
+  private async exec(instruction: Runner, node: FlowNodeModel, prevJob) {
+    let job;
+    try {
+      // call instruction to get result and status
+      this.logger.info(`execution (${this.execution.id}) run instruction [${node.type}] for node (${node.id})`);
+      this.logger.debug(`config of node`, { data: node.config });
+      job = await instruction(node, prevJob, this);
+      if (!job) {
+        return null;
+      }
+    } catch (err) {
+      // for uncaught error, set to error
+      this.logger.error(
+        `execution (${this.execution.id}) run instruction [${node.type}] for node (${node.id}) failed: `,
+        { error: err },
+      );
+      job = {
+        result:
+          err instanceof Error
+            ? { message: err.message, stack: process.env.NODE_ENV === 'production' ? [] : err.stack }
+            : err,
+        status: JOB_STATUS.ERROR,
+      };
+      // if previous job is from resuming
+      if (prevJob && prevJob.nodeId === node.id) {
+        prevJob.set(job);
+        job = prevJob;
+      }
+    }
+
+    if (!(job instanceof Model)) {
+      job.upstreamId = prevJob instanceof Model ? prevJob.get('id') : null;
+      job.nodeId = node.id;
+    }
+    const savedJob = await this.saveJob(job);
+
+    this.logger.info(
+      `execution (${this.execution.id}) run instruction [${node.type}] for node (${node.id}) finished as status: ${savedJob.status}`,
+    );
+    this.logger.debug(`result of node`, { data: savedJob.result });
+
+    if (savedJob.status === JOB_STATUS.RESOLVED && node.downstream) {
+      // run next node
+      this.logger.debug(`run next node (${node.downstreamId})`);
+      return this.run(node.downstream, savedJob);
+    }
+
+    // all nodes in scope have been executed
+    return this.end(node, savedJob);
+  }
+
+  public async run(node, input?) {
+    const { instructions } = this.options.plugin;
+    const instruction = instructions.get(node.type);
+    if (typeof instruction.run !== 'function') {
+      return Promise.reject(new Error('`run` should be implemented for customized execution of the node'));
+    }
+
+    return this.exec(instruction.run.bind(instruction), node, input);
+  }
+
+  // parent node should take over the control
+  public async end(node, job: JobModel) {
+    this.logger.debug(`branch ended at node (${node.id})`);
+    const parentNode = this.findBranchParentNode(node);
+    // no parent, means on main flow
+    if (parentNode) {
+      this.logger.debug(`not on main, recall to parent entry node (${node.id})})`);
+      await this.recall(parentNode, job);
+      return job;
+    }
+
+    // really done for all nodes
+    // * should mark execution as done with last job status
+    return this.exit(job.status);
+  }
+
+  async recall(node, job) {
+    const { instructions } = this.options.plugin;
+    const instruction = instructions.get(node.type);
+    if (typeof instruction.resume !== 'function') {
+      return Promise.reject(new Error('`resume` should be implemented'));
+    }
+
+    return this.exec(instruction.resume.bind(instruction), node, job);
+  }
+
+  async exit(s?: number) {
+    if (typeof s === 'number') {
+      const status = (<typeof Processor>this.constructor).StatusMap[s] ?? Math.sign(s);
+      await this.execution.update({ status }, { transaction: this.transaction });
+    }
+    this.logger.info(`execution (${this.execution.id}) exiting with status ${this.execution.status}`);
+    await this.commit();
+    return null;
+  }
+
+  // TODO(optimize)
+  async saveJob(payload) {
+    const { database } = <typeof ExecutionModel>this.execution.constructor;
+    const { model } = database.getCollection('jobs');
+    let job;
+    if (payload instanceof model) {
+      job = await payload.save({ transaction: this.transaction });
+    } else if (payload.id) {
+      job = await model.findByPk(payload.id);
+      await job.update(payload, {
+        transaction: this.transaction,
+      });
+    } else {
+      job = await model.create(
+        {
+          ...payload,
+          executionId: this.execution.id,
+        },
+        {
+          transaction: this.transaction,
+        },
+      );
+    }
+    this.jobsMap.set(job.id, job);
+    this.jobsMapByNodeId[job.nodeId] = job.result;
+
+    return job;
+  }
+
+  getBranches(node: FlowNodeModel): FlowNodeModel[] {
+    return this.nodes
+      .filter((item) => item.upstream === node && item.branchIndex !== null)
+      .sort((a, b) => Number(a.branchIndex) - Number(b.branchIndex));
+  }
+
+  // find the first node in current branch
+  findBranchStartNode(node: FlowNodeModel, parent?: FlowNodeModel): FlowNodeModel | null {
+    for (let n = node; n; n = n.upstream) {
+      if (!parent) {
+        if (n.branchIndex !== null) {
+          return n;
+        }
+      } else {
+        if (n.upstream === parent) {
+          return n;
+        }
+      }
+    }
+    return null;
+  }
+
+  // find the node start current branch
+  findBranchParentNode(node: FlowNodeModel): FlowNodeModel | null {
+    for (let n = node; n; n = n.upstream) {
+      if (n.branchIndex !== null) {
+        return n.upstream;
+      }
+    }
+    return null;
+  }
+
+  findBranchEndNode(node: FlowNodeModel): FlowNodeModel | null {
+    for (let n = node; n; n = n.downstream) {
+      if (!n.downstream) {
+        return n;
+      }
+    }
+    return null;
+  }
+
+  findBranchParentJob(job: JobModel, node: FlowNodeModel): JobModel | null {
+    for (let j: JobModel | undefined = job; j; j = this.jobsMap.get(j.upstreamId)) {
+      if (j.nodeId === node.id) {
+        return j;
+      }
+    }
+    return null;
+  }
+
+  findBranchLastJob(node: FlowNodeModel): JobModel | null {
+    for (let n = this.findBranchEndNode(node); n && n !== node.upstream; n = n.upstream) {
+      const jobs = Array.from(this.jobsMap.values())
+        .filter((item) => item.nodeId === n.id)
+        .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime());
+      if (jobs.length) {
+        return jobs[jobs.length - 1];
+      }
+    }
+    return null;
+  }
+
+  public getScope(node?) {
+    const systemFns = {};
+    const scope = {
+      execution: this.execution,
+      node,
+    };
+    for (const [name, fn] of this.options.plugin.functions.getEntities()) {
+      systemFns[name] = fn.bind(scope);
+    }
+
+    const $scopes = {};
+    if (node) {
+      for (let n = this.findBranchParentNode(node); n; n = this.findBranchParentNode(n)) {
+        const instruction = this.options.plugin.instructions.get(n.type);
+        if (typeof instruction.getScope === 'function') {
+          $scopes[n.id] = instruction.getScope(n, this.jobsMapByNodeId[n.id], this);
+        }
+      }
+    }
+
+    return {
+      $context: this.execution.context,
+      $jobsMapByNodeId: this.jobsMapByNodeId,
+      $system: systemFns,
+      $scopes,
+    };
+  }
+
+  public getParsedValue(value, node?) {
+    const template = parse(value);
+    const scope = this.getScope(node);
+    template.parameters.forEach(({ key }) => {
+      appendArrayColumn(scope, key);
+    });
+    return template(scope);
+  }
+}