@certd/pipeline 1.0.0

package/.eslintrc

@@ -0,0 +1,21 @@
+{
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "plugin:@typescript-eslint/recommended",
+    "plugin:prettier/recommended",
+    "prettier"
+  ],
+  "env": {
+    "mocha": true
+  },
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": "off",
+    "@typescript-eslint/ban-ts-ignore": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+//    "no-unused-expressions": "off",
+    "max-len": [0, 160, 2, { "ignoreUrls": true }]
+  }
+}

package/.mocharc.json

@@ -0,0 +1,5 @@
+{
+  "extension": ["ts"],
+  "spec": "test/**/*.test.ts",
+  "require": "ts-node/register"
+}

package/.prettierrc

@@ -0,0 +1,3 @@
+{
+  "printWidth": 160
+}

package/README.md

@@ -0,0 +1,16 @@
+# Vue 3 + TypeScript + Vite
+
+This template should help get you started developing with Vue 3 and TypeScript in Vite. The template uses Vue 3 `<script setup>` SFCs, check out the [script setup docs](https://v3.vuejs.org/api/sfc-script-setup.html#sfc-script-setup) to learn more.
+
+## Recommended IDE Setup
+
+- [VS Code](https://code.visualstudio.com/) + [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar)
+
+## Type Support For `.vue` Imports in TS
+
+Since TypeScript cannot handle type information for `.vue` imports, they are shimmed to be a generic Vue component type by default. In most cases this is fine if you don't really care about component prop types outside of templates. However, if you wish to get actual prop types in `.vue` imports (for example to get props validation when using manual `h(...)` calls), you can enable Volar's Take Over mode by following these steps:
+
+1. Run `Extensions: Show Built-in Extensions` from VS Code's command palette, look for `TypeScript and JavaScript Language Features`, then right click and select `Disable (Workspace)`. By default, Take Over mode will enable itself if the default TypeScript extension is disabled.
+2. Reload the VS Code window by running `Developer: Reload Window` from the command palette.
+
+You can learn more about Take Over mode [here](https://github.com/johnsoncodehk/volar/discussions/471).

package/index.ts

@@ -0,0 +1 @@
+export * from "./src";

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@certd/pipeline",
+  "private": false,
+  "version": "1.0.0",
+  "main": "./dist/pipeline.umd.js",
+  "module": "./dist/pipeline.mjs",
+  "types": "./dist/d/index.d.ts",
+  "publishConfig": {
+    "main": "./dist/pipeline.umd.js",
+    "module": "./dist/pipeline.mjs",
+    "types": "./dist/d/index.d.ts"
+  },
+  "scripts": {
+    "dev": "vite",
+    "build": "vue-tsc --noEmit && vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@certd/acme-client": "^1.0.0",
+    "axios": "^0.21.1",
+    "dayjs": "^1.11.6",
+    "lodash": "^4.17.21",
+    "log4js": "^6.3.0",
+    "node-forge": "^0.10.0",
+    "qs": "^6.9.4",
+    "reflect-metadata": "^0.1.13"
+  },
+  "devDependencies": {
+    "@alicloud/cs20151215": "^3.0.3",
+    "@alicloud/openapi-client": "^0.4.0",
+    "@alicloud/pop-core": "^1.7.10",
+    "@midwayjs/bootstrap": "^3.9.1",
+    "@midwayjs/cache": "^3.9.0",
+    "@midwayjs/cli": "^1.3.21",
+    "@midwayjs/core": "^3.0.0",
+    "@midwayjs/decorator": "^3.0.0",
+    "@midwayjs/koa": "^3.9.0",
+    "@midwayjs/logger": "^2.17.0",
+    "@midwayjs/typeorm": "^3.9.0",
+    "@midwayjs/validate": "^3.9.0",
+    "@rollup/plugin-typescript": "^11.0.0",
+    "@types/chai": "^4.3.3",
+    "@types/lodash": "^4.14.194",
+    "@types/mocha": "^10.0.0",
+    "@types/node-forge": "^1.3.0",
+    "@typescript-eslint/eslint-plugin": "^5.38.1",
+    "@typescript-eslint/parser": "^5.38.1",
+    "chai": "^4.3.6",
+    "eslint": "^8.24.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-prettier": "^4.2.1",
+    "mocha": "^10.1.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.8.4",
+    "vite": "^3.1.0",
+    "vue-tsc": "^0.38.9"
+  },
+  "gitHead": "5950e1cae7cf30ebfc5128c15c7d1b0d101cbbb8"
+}

package/src/access/api.ts

@@ -0,0 +1,18 @@
+import { Registrable } from "../registry";
+import { FormItemProps } from "../d.ts";
+
+export type AccessInputDefine = FormItemProps & {
+  title: string;
+  required?: boolean;
+};
+export type AccessDefine = Registrable & {
+  input?: {
+    [key: string]: AccessInputDefine;
+  };
+};
+export interface IAccessService {
+  getById(id: any): Promise<any>;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IAccess {}

package/src/access/decorator.ts

@@ -0,0 +1,39 @@
+// src/decorator/memoryCache.decorator.ts
+import { AccessDefine, AccessInputDefine } from "./api";
+import { Decorator } from "../decorator";
+import _ from "lodash";
+import { accessRegistry } from "./registry";
+
+// 提供一个唯一 key
+export const ACCESS_CLASS_KEY = "pipeline:access";
+export const ACCESS_INPUT_KEY = "pipeline:access:input";
+
+export function IsAccess(define: AccessDefine): ClassDecorator {
+  return (target: any) => {
+    target = Decorator.target(target);
+
+    const inputs: any = {};
+    const properties = Decorator.getClassProperties(target);
+    for (const property in properties) {
+      const input = Reflect.getMetadata(ACCESS_INPUT_KEY, target, property);
+      if (input) {
+        inputs[property] = input;
+      }
+    }
+    _.merge(define, { input: inputs });
+    Reflect.defineMetadata(ACCESS_CLASS_KEY, define, target);
+    target.define = define;
+    accessRegistry.register(define.name, {
+      define,
+      target,
+    });
+  };
+}
+
+export function AccessInput(input?: AccessInputDefine): PropertyDecorator {
+  return (target, propertyKey) => {
+    target = Decorator.target(target, propertyKey);
+    // const _type = Reflect.getMetadata("design:type", target, propertyKey);
+    Reflect.defineMetadata(ACCESS_INPUT_KEY, input, target, propertyKey);
+  };
+}

package/src/access/index.ts

@@ -0,0 +1,3 @@
+export * from "./api";
+export * from "./registry";
+export * from "./decorator";

package/src/access/registry.ts

@@ -0,0 +1,4 @@
+import { Registry } from "../registry";
+
+// @ts-ignore
+export const accessRegistry = new Registry();

package/src/context/index.ts

@@ -0,0 +1,6 @@
+import { AxiosInstance } from "axios";
+import { IContext } from "../core";
+
+export type HttpClient = AxiosInstance;
+export type UserContext = IContext;
+export type PipelineContext = IContext;

package/src/core/context.ts

@@ -0,0 +1,88 @@
+import { IStorage, MemoryStorage } from "./storage";
+const CONTEXT_VERSION_KEY = "contextVersion";
+export interface IContext {
+  getInt(key: string): Promise<number>;
+  get(key: string): Promise<string | null>;
+  set(key: string, value: string): Promise<void>;
+  getObj<T = any>(key: string): Promise<T | null>;
+  setObj<T = any>(key: string, value: T): Promise<void>;
+  updateVersion(): Promise<void>;
+  initVersion(): Promise<void>;
+}
+
+export class ContextFactory {
+  storage: IStorage;
+  memoryStorage: IStorage;
+
+  constructor(storage: IStorage) {
+    this.storage = storage;
+    this.memoryStorage = new MemoryStorage();
+  }
+
+  getContext(scope: string, namespace: string): IContext {
+    const context = new StorageContext(scope, namespace, this.storage);
+    return context;
+  }
+
+  getMemoryContext(scope: string, namespace: string): IContext {
+    const context = new StorageContext(scope, namespace, this.memoryStorage);
+    return context;
+  }
+}
+
+export class StorageContext implements IContext {
+  storage: IStorage;
+  namespace: string;
+  scope: string;
+
+  _version = 0;
+  _initialVersion = 0;
+  constructor(scope: string, namespace: string, storage: IStorage) {
+    this.storage = storage;
+    this.scope = scope;
+    this.namespace = namespace;
+  }
+
+  async initVersion() {
+    const version = await this.getInt(CONTEXT_VERSION_KEY);
+    this._initialVersion = version;
+    this._version = version;
+  }
+
+  async updateVersion() {
+    if (this._version === this._initialVersion) {
+      this._version++;
+    }
+
+    await this.set(CONTEXT_VERSION_KEY, this._version.toString());
+  }
+
+  async get(key: string) {
+    const version = key === CONTEXT_VERSION_KEY ? 0 : this._version;
+    return await this.storage.get(this.scope, this.namespace, version.toString(), key);
+  }
+  async set(key: string, value: string) {
+    const version = key === CONTEXT_VERSION_KEY ? 0 : this._version;
+    return await this.storage.set(this.scope, this.namespace, version.toString(), key, value);
+  }
+
+  async getInt(key: string): Promise<number> {
+    const str = await this.get(key);
+    if (str) {
+      return parseInt(str);
+    }
+    return 0;
+  }
+  async getObj<T = any>(key: string): Promise<T | null> {
+    const str = await this.get(key);
+    if (str) {
+      const store = JSON.parse(str);
+      return store.value;
+    }
+    return null;
+  }
+
+  async setObj<T = any>(key: string, value: T) {
+    await this.set(key, JSON.stringify({ value }));
+  }
+}

package/src/core/executor.ts

@@ -0,0 +1,207 @@
+import { ConcurrencyStrategy, Pipeline, ResultType, Runnable, RunStrategy, Stage, Step, Task } from "../d.ts";
+import _ from "lodash";
+import { RunHistory, RunnableCollection } from "./run-history";
+import { AbstractTaskPlugin, PluginDefine, pluginRegistry } from "../plugin";
+import { ContextFactory, IContext } from "./context";
+import { IStorage } from "./storage";
+import { logger } from "../utils/util.log";
+import { Logger } from "log4js";
+import { request } from "../utils/util.request";
+import { IAccessService } from "../access";
+import { RegistryItem } from "../registry";
+import { Decorator } from "../decorator";
+
+export class Executor {
+  userId: any;
+  pipeline: Pipeline;
+  runtime!: RunHistory;
+  accessService: IAccessService;
+  contextFactory: ContextFactory;
+  logger: Logger;
+  pipelineContext!: IContext;
+  lastStatusMap!: RunnableCollection;
+  onChanged: (history: RunHistory) => void;
+  constructor(options: {
+    userId: any;
+    pipeline: Pipeline;
+    storage: IStorage;
+    onChanged: (history: RunHistory) => Promise<void>;
+    accessService: IAccessService;
+  }) {
+    this.pipeline = _.cloneDeep(options.pipeline);
+    this.onChanged = async (history: RunHistory) => {
+      await options.onChanged(history);
+    };
+    this.accessService = options.accessService;
+    this.userId = options.userId;
+    this.pipeline.userId = this.userId;
+    this.contextFactory = new ContextFactory(options.storage);
+    this.logger = logger;
+    this.pipelineContext = this.contextFactory.getContext("pipeline", this.pipeline.id);
+  }
+
+  async init() {
+    const lastRuntime = await this.pipelineContext.getObj(`lastRuntime`);
+    this.lastStatusMap = new RunnableCollection(lastRuntime?.pipeline);
+  }
+
+  async run(runtimeId: any = 0, triggerType: string) {
+    try {
+      await this.init();
+      const trigger = { type: triggerType };
+      // 读取last
+      this.runtime = new RunHistory(runtimeId, trigger, this.pipeline);
+      this.logger.info(`pipeline.${this.pipeline.id}  start`);
+      await this.runWithHistory(this.pipeline, "pipeline", async () => {
+        await this.runStages(this.pipeline);
+      });
+    } catch (e) {
+      this.logger.error("pipeline 执行失败", e);
+    } finally {
+      await this.pipelineContext.setObj("lastRuntime", this.runtime);
+      this.logger.info(`pipeline.${this.pipeline.id}  end`);
+    }
+  }
+
+  async runWithHistory(runnable: Runnable, runnableType: string, run: () => Promise<void>) {
+    runnable.runnableType = runnableType;
+    this.runtime.start(runnable);
+    await this.onChanged(this.runtime);
+
+    if (runnable.strategy?.runStrategy === RunStrategy.SkipWhenSucceed) {
+      //如果是成功后跳过策略
+      const lastNode = this.lastStatusMap.get(runnable.id);
+      const lastResult = lastNode?.status?.status;
+      const lastInput = JSON.stringify(lastNode?.status?.input);
+      let inputChanged = false;
+      if (runnableType === "step") {
+        const step = runnable as Step;
+        const input = JSON.stringify(step.input);
+        if (input != null && lastInput !== input) {
+          //参数有变化
+          inputChanged = true;
+        }
+      }
+      if (lastResult != null && lastResult === ResultType.success && !inputChanged) {
+        this.runtime.skip(runnable);
+        await this.onChanged(this.runtime);
+        return ResultType.skip;
+      }
+    }
+    try {
+      await run();
+      this.runtime.success(runnable);
+      await this.onChanged(this.runtime);
+      return ResultType.success;
+    } catch (e: any) {
+      this.runtime.error(runnable, e);
+      await this.onChanged(this.runtime);
+      throw e;
+    } finally {
+      this.runtime.finally(runnable);
+    }
+  }
+
+  private async runStages(pipeline: Pipeline) {
+    const resList: ResultType[] = [];
+    for (const stage of pipeline.stages) {
+      const res: ResultType = await this.runWithHistory(stage, "stage", async () => {
+        await this.runStage(stage);
+      });
+      resList.push(res);
+    }
+    return this.compositionResultType(resList);
+  }
+
+  async runStage(stage: Stage) {
+    const runnerList = [];
+    for (const task of stage.tasks) {
+      const runner = this.runWithHistory(task, "task", async () => {
+        await this.runTask(task);
+      });
+      runnerList.push(runner);
+    }
+
+    let resList: ResultType[] = [];
+    if (stage.concurrency === ConcurrencyStrategy.Parallel) {
+      resList = await Promise.all(runnerList);
+    } else {
+      for (let i = 0; i < runnerList.length; i++) {
+        const runner = runnerList[i];
+        resList[i] = await runner;
+      }
+    }
+    return this.compositionResultType(resList);
+  }
+
+  compositionResultType(resList: ResultType[]) {
+    let hasSuccess = false;
+    for (const type of resList) {
+      if (type === ResultType.error) {
+        return ResultType.error;
+      }
+      if (type === ResultType.success) {
+        hasSuccess = true;
+      }
+    }
+    if (hasSuccess) {
+      return ResultType.success;
+    }
+    return ResultType.error;
+  }
+
+  private async runTask(task: Task) {
+    const resList: ResultType[] = [];
+    for (const step of task.steps) {
+      step.runnableType = "step";
+      const res: ResultType = await this.runWithHistory(step, "step", async () => {
+        await this.runStep(step);
+      });
+      resList.push(res);
+    }
+    return this.compositionResultType(resList);
+  }
+
+  private async runStep(step: Step) {
+    const lastStatus = this.lastStatusMap.get(step.id);
+    //执行任务
+    const plugin: RegistryItem<AbstractTaskPlugin> = pluginRegistry.get(step.type);
+
+    // @ts-ignore
+    const instance: ITaskPlugin = new plugin.target();
+    // @ts-ignore
+    const define: PluginDefine = plugin.define;
+    //从outputContext读取输入参数
+    Decorator.inject(define.input, instance, step.input, (item, key) => {
+      if (item.component?.name === "pi-output-selector") {
+        const contextKey = step.input[key];
+        if (contextKey != null) {
+          step.input[key] = this.runtime.context[contextKey];
+        }
+      }
+    });
+
+    const context: any = {
+      logger: this.runtime._loggers[step.id],
+      accessService: this.accessService,
+      pipelineContext: this.pipelineContext,
+      lastStatus,
+      userContext: this.contextFactory.getContext("user", this.userId),
+      http: request,
+    };
+    Decorator.inject(define.autowire, instance, context);
+
+    await instance.onInstance();
+    await instance.execute();
+
+    if (instance.result.clearLastStatus) {
+      this.lastStatusMap.clear();
+    }
+    //输出到output context
+    _.forEach(define.output, (item, key) => {
+      step!.status!.output[key] = instance[key];
+      const stepOutputKey = `step.${step.id}.${key}`;
+      this.runtime.context[stepOutputKey] = instance[key];
+    });
+  }
+}

package/src/core/index.ts

@@ -0,0 +1,4 @@
+export * from "./executor";
+export * from "./run-history";
+export * from "./context";
+export * from "./storage";

package/src/core/run-history.ts

@@ -0,0 +1,158 @@
+import { Context, HistoryResult, Pipeline, ResultType, Runnable, RunnableMap, Stage, Step, Task } from "../d.ts";
+import _ from "lodash";
+import { buildLogger } from "../utils/util.log";
+import { Logger } from "log4js";
+
+export type HistoryStatus = {
+  result: HistoryResult;
+  logs: string[];
+};
+
+export type RunTrigger = {
+  type: string; // user , timer
+};
+
+export function NewRunHistory(obj: any) {
+  const history = new RunHistory(obj.id, obj.trigger, obj.pipeline);
+  history.context = obj.context;
+  history.logs = obj.logs;
+  history._loggers = obj.loggers;
+  return history;
+}
+export class RunHistory {
+  id!: string;
+  //运行时上下文变量
+  context: Context = {};
+  pipeline!: Pipeline;
+  logs: {
+    [runnableId: string]: string[];
+  } = {};
+  _loggers: {
+    [runnableId: string]: Logger;
+  } = {};
+  trigger!: RunTrigger;
+
+  constructor(runtimeId: any, trigger: RunTrigger, pipeline: Pipeline) {
+    this.id = runtimeId;
+    this.pipeline = pipeline;
+    this.trigger = trigger;
+  }
+
+  start(runnable: Runnable): HistoryResult {
+    const now = new Date().getTime();
+    this.logs[runnable.id] = [];
+    this._loggers[runnable.id] = buildLogger((text) => {
+      this.logs[runnable.id].push(text);
+    });
+    const input = (runnable as Step).input;
+    const status: HistoryResult = {
+      output: {},
+      input: _.cloneDeep(input),
+      status: ResultType.start,
+      startTime: now,
+      result: ResultType.start,
+    };
+    runnable.status = status;
+    this.log(runnable, `开始执行`);
+    return status;
+  }
+
+  success(runnable: Runnable) {
+    const now = new Date().getTime();
+    const status = runnable.status;
+    _.merge(status, {
+      status: "success",
+      endTime: now,
+      result: "success",
+    });
+    this.log(runnable, `执行成功`);
+  }
+
+  skip(runnable: Runnable) {
+    const now = new Date().getTime();
+    const status = runnable.status;
+    _.merge(status, {
+      status: "success",
+      endTime: now,
+      result: "skip",
+    });
+    this.log(runnable, `跳过`);
+  }
+
+  error(runnable: Runnable, e: Error) {
+    const now = new Date().getTime();
+    const status = runnable.status;
+    _.merge(status, {
+      status: ResultType.error,
+      endTime: now,
+      result: ResultType.error,
+      message: e.message,
+    });
+
+    this.logError(runnable, e);
+  }
+
+  log(runnable: Runnable, text: string) {
+    // @ts-ignore
+    this._loggers[runnable.id].info(`[${runnable.title}]<id:${runnable.id}> [${runnable.runnableType}]`, text);
+  }
+
+  logError(runnable: Runnable, e: Error) {
+    // @ts-ignore
+    this._loggers[runnable.id].error(`[${runnable.title}]<id:${runnable.id}> [${runnable.runnableType}]`, e);
+  }
+
+  finally(runnable: Runnable) {
+    //
+  }
+}
+
+export class RunnableCollection {
+  private collection: RunnableMap = {};
+  private pipeline!: Pipeline;
+  constructor(pipeline?: Pipeline) {
+    if (!pipeline) {
+      return;
+    }
+    this.pipeline = pipeline;
+    const map = this.toMap(pipeline);
+    this.collection = map;
+  }
+
+  private each<T extends Runnable>(list: T[], exec: (item: Runnable) => void) {
+    list.forEach((item) => {
+      exec(item);
+      if (item.runnableType === "pipeline") {
+        // @ts-ignore
+        this.each<Stage>(item.stages, exec);
+      } else if (item.runnableType === "stage") {
+        // @ts-ignore
+        this.each<Task>(item.tasks, exec);
+      } else if (item.runnableType === "task") {
+        // @ts-ignore
+        this.each<Step>(item.steps, exec);
+      }
+    });
+  }
+  private toMap(pipeline: Pipeline) {
+    const map: RunnableMap = {};
+
+    this.each(pipeline.stages, (item) => {
+      map[item.id] = item;
+    });
+    return map;
+  }
+
+  get(id: string): Runnable | undefined {
+    return this.collection[id];
+  }
+
+  clear() {
+    if (!this.pipeline) {
+      return;
+    }
+    this.each(this.pipeline.stages, (item) => {
+      item.status = undefined;
+    });
+  }
+}