filecat 5.32.0 → 5.32.2

package/build/server/main/domain/file/workflow/workflow.service.js

@@ -0,0 +1,659 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.workflowService = exports.WorkflowService = exports.work_children = exports.workflow_logger = exports.send_ws_type = void 0;
+const WsData_1 = require("../../../../common/frame/WsData");
+const file_req_1 = require("../../../../common/req/file.req");
+const setting_service_1 = require("../../setting/setting.service");
+const path_1 = __importDefault(require("path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const user_service_1 = require("../../user/user.service");
+const user_req_1 = require("../../../../common/req/user.req");
+const base_data_util_1 = require("../../data/basedata/base_data_util");
+const StringUtil_1 = require("../../../../common/StringUtil");
+const ValueUtil_1 = require("../../../../common/ValueUtil");
+const FileUtil_1 = require("../FileUtil");
+const workflow_util_1 = require("./workflow.util");
+const workflow_process_1 = require("./workflow.process");
+const common_util_1 = require("../../../../common/common.util");
+const readYamlFile = require('read-yaml-file');
+const Mustache = require('mustache');
+var send_ws_type;
+(function (send_ws_type) {
+    send_ws_type[send_ws_type["done"] = 0] = "done";
+    send_ws_type[send_ws_type["new_log"] = 1] = "new_log";
+})(send_ws_type || (exports.send_ws_type = send_ws_type = {}));
+class workflow_logger {
+    constructor(instance) {
+        this.wss_call_set = new Set();
+        this.instance = instance;
+    }
+    send_wss(type, ...wss) {
+        const r_list = [];
+        if (type !== send_ws_type.new_log) {
+            work_children.get_workflow_realtime_tree_list(r_list, this.instance);
+        }
+        for (const w of wss) {
+            const result = new WsData_1.WsData(WsData_1.CmdType.workflow_realtime_one_rsq);
+            result.context = {
+                done: type === undefined ? undefined : type === send_ws_type.done,
+                list: r_list,
+                new_log: this.running_log
+            };
+            w.sendData(result.encode());
+        }
+    }
+    send_all_wss(type) {
+        if (this.wss_call_set) {
+            this.send_wss(type, ...this.wss_call_set.values());
+        }
+    }
+    add_wss(wss) {
+        if (!this.wss_call_set.has(wss)) {
+            this.wss_call_set.add(wss);
+        }
+        this.send_wss(undefined, wss);
+    }
+    remove_wss(wss) {
+        if (!this.wss_call_set) {
+            this.wss_call_set.delete(wss);
+        }
+    }
+}
+exports.workflow_logger = workflow_logger;
+class work_children {
+    constructor(yaml_path = undefined, workflow_dir_path = undefined) {
+        this.running_type = file_req_1.running_type.running;
+        this.import_files_map = new Map();
+        this.jobs_map = new Map();
+        this.done_jobs = new Set();
+        this.need_job_set = new Set();
+        this.pty_shell_set = new Set();
+        this.worker_children_use_yml_map = new Map();
+        this.yaml_path = yaml_path;
+        if (workflow_dir_path) {
+            this.workflow_dir_path = workflow_dir_path;
+        }
+        this.logger = new workflow_logger(this);
+    }
+    static get_workflow_realtime_tree_list(r_list, worker) {
+        for (const it of worker.jobs_map.values()) {
+            const v = {
+                name: it.name,
+                extra_data: { running_type: it.running_type },
+                code: it.code,
+                children: []
+            };
+            if (it.code !== 0 && it.steps) {
+                const children = [];
+                for (const step of it.steps ?? []) {
+                    let name = '';
+                    if (step.run) {
+                        name = step.run;
+                    }
+                    else if (step["use-yml"]) {
+                        name = step["use-yml"];
+                    }
+                    else if (step["run-js"]) {
+                        name = step["run-js"];
+                    }
+                    else if (step["sleep"]) {
+                        name = `sleep ${step.sleep}`;
+                    }
+                    else if (step["runs"]) {
+                        name = JSON.stringify(step.runs);
+                    }
+                    const p = {
+                        name: name,
+                        extra_data: { running_type: step.running_type },
+                        code: step.code,
+                        children: []
+                    };
+                    const children_worker = worker.worker_children_use_yml_map.get(step['use-yml']);
+                    if (children_worker) {
+                        this.get_workflow_realtime_tree_list(p.children, children_worker);
+                    }
+                    children.push(p);
+                }
+                v.children = children;
+            }
+            r_list.push(v);
+        }
+    }
+    async init(param) {
+        if (param?.yaml_path) {
+            this.yaml_path = param.yaml_path;
+        }
+        if (param?.workflow_logger) {
+            this.logger = param.workflow_logger;
+        }
+        this.filename = path_1.default.basename(this.yaml_path);
+        const yaml_data = param?.yaml_data ?? await readYamlFile(this.yaml_path);
+        this.env = yaml_data.env ?? {};
+        if (param.pre_env) {
+            for (const key of Object.keys(param.pre_env)) {
+                this.env[key] = param.pre_env[key];
+            }
+        }
+        this.env['filecat_user_id'] = param.filecat_user_id;
+        this.env['filecat_user_name'] = param.filecat_user_name;
+        this.env['filecat_user_note'] = param.filecat_user_note;
+        let user_id = `${yaml_data.user_id || yaml_data['user-id'] || ""}`;
+        if (!user_id) {
+            user_id = user_service_1.userService.get_user_id(`${yaml_data.username}`);
+        }
+        if (!user_id) {
+            throw "user not exists";
+        }
+        this.user_id = user_id;
+        if (param?.yaml_path)
+            user_service_1.userService.check_user_path_by_user_id(this.user_id, param?.yaml_path);
+        user_service_1.userService.check_user_auth_by_user_id(this.user_id, user_req_1.UserAuth.workflow_exe_user, {
+            auto_throw: true,
+            root_check: true
+        });
+        if (!this.workflow_dir_path) {
+            let find_p = path_1.default.dirname(this.yaml_path);
+            let have_find = false;
+            while (true) {
+                if (await FileUtil_1.FileUtil.access(path_1.default.join(find_p, file_req_1.workflow_dir_name))) {
+                    have_find = true;
+                    break;
+                }
+                const p = path_1.default.join(find_p, "..");
+                if (p === find_p) {
+                    have_find = false;
+                    break;
+                }
+                find_p = p;
+                if (!user_service_1.userService.check_user_path_by_user_id(this.user_id, find_p, false)) {
+                    have_find = false;
+                    break;
+                }
+            }
+            if (have_find) {
+                this.workflow_dir_path = path_1.default.join(find_p, file_req_1.workflow_dir_name);
+            }
+            else {
+                this.workflow_dir_path = path_1.default.join(path_1.default.dirname(this.yaml_path), file_req_1.workflow_dir_name);
+                await fs_extra_1.default.ensureDir(this.workflow_dir_path);
+            }
+        }
+        if (yaml_data.jobs) {
+            for (const key of Object.keys(yaml_data.jobs)) {
+                const v = yaml_data.jobs[key];
+                v.key = key;
+                this.jobs_map.set(v.key, v);
+            }
+        }
+        if (yaml_data['import-files']) {
+            for (const p of yaml_data['import-files']) {
+                let p1 = p;
+                if (!path_1.default.isAbsolute(p)) {
+                    p1 = path_1.default.join(path_1.default.dirname(this.yaml_path), p);
+                }
+                user_service_1.userService.check_user_path_by_user_id(this.user_id, p1);
+                const d = await readYamlFile(p1);
+                this.import_files_map.set(d.name, { yaml_path: p1, yaml_data: d });
+            }
+        }
+        yaml_data['run-name'] = Mustache.render(`${yaml_data['run-name'] ?? ""}`, this.env ?? {});
+        this["run-name"] = `${yaml_data['run-name'] ?? ""}`;
+        this.name = `${yaml_data.name}`;
+        if (param?.env) {
+            for (const key of Object.keys(param.env)) {
+                this.env[key] = param.env[key];
+            }
+        }
+        if (yaml_data['file-env']) {
+            for (const key of Object.keys(yaml_data['file-env'])) {
+                let p1 = yaml_data['file-env'][key];
+                if (!path_1.default.isAbsolute(p1)) {
+                    p1 = path_1.default.join(path_1.default.dirname(this.yaml_path), p1);
+                }
+                user_service_1.userService.check_user_path_by_user_id(this.user_id, p1);
+                this.env[key] = (await FileUtil_1.FileUtil.readFileSync(p1)).toString();
+            }
+        }
+        return this;
+    }
+    async run_jobs(return_steps = false) {
+        const start_time = Date.now();
+        await new Promise(resolve => {
+            this.all_job_resolve = resolve;
+            const v_List = [...this.jobs_map.values()];
+            for (let i = 0; i < v_List.length; i++) {
+                const job = v_List[i];
+                this.run_job(job).then(() => {
+                });
+            }
+        });
+        const all_jobs = [];
+        let failed = false;
+        for (const it of this.jobs_map.values()) {
+            all_jobs.push(it);
+            if (it.code !== 0 && it.code !== undefined) {
+                failed = true;
+            }
+        }
+        this.close(failed ? file_req_1.running_type.fail : file_req_1.running_type.success);
+        if (return_steps) {
+            return { all_jobs, failed };
+        }
+        this.logger.send_all_wss(failed === true ? send_ws_type.done : undefined);
+        this.running_type = failed === false ? file_req_1.running_type.success : file_req_1.running_type.fail;
+        try {
+            const base_data_util = new base_data_util_1.Base_data_util({ base_dir: this.workflow_dir_path });
+            await base_data_util.init();
+            await base_data_util.insert(JSON.stringify({
+                all_jobs
+            }), JSON.stringify({
+                name: this.name,
+                "run-name": this["run-name"],
+                is_success: failed === false,
+                timestamp: (0, ValueUtil_1.formatter_time)(new Date()),
+                duration: `${((Date.now() - start_time) / 1000).toFixed(2)} s`
+            }));
+        }
+        catch (error) {
+            console.log('插入数据库失败', error);
+        }
+    }
+    done_fail_job_handle(job, fail_message) {
+        job.code = -1;
+        this.logger.send_all_wss();
+        job.message = fail_message;
+        this.close();
+        this.all_job_resolve("ok");
+    }
+    async run_job(job) {
+        try {
+            let job_can_run = false;
+            if (!job.if) {
+                job_can_run = true;
+            }
+            else {
+                job_can_run = await workflow_util_1.workflow_util.run_js(job.if, this.env);
+            }
+            if (job_can_run) {
+                workflow_util_1.workflow_util.handle_job_pre(job, this);
+                if (job['need-jobs']) {
+                    if (!workflow_util_1.workflow_util.all_jobs_done(job['need-jobs'], this)) {
+                        this.need_job_set.add(job);
+                        return;
+                    }
+                }
+                job.running_type = file_req_1.running_type.running;
+                const start_time = Date.now();
+                const process_runner = new workflow_process_1.WorkflowProcess(this, job);
+                this.pty_shell_set.add(process_runner.pty);
+                for (let i = 0; i < job.steps.length; i++) {
+                    const step = job.steps[i];
+                    if (step.if) {
+                        if (!await workflow_util_1.workflow_util.run_js(step.if, this.env)) {
+                            this.logger.send_all_wss();
+                            continue;
+                        }
+                    }
+                    step.running_type = file_req_1.running_type.running;
+                    this.logger.send_all_wss();
+                    if (step['sleep']) {
+                        await common_util_1.CommonUtil.sleep(step['sleep']);
+                    }
+                    const step_start_time = Date.now();
+                    try {
+                        if (step["run-js"]) {
+                            await workflow_util_1.workflow_util.run_code_js_by_step(step, job, this.env, true);
+                        }
+                        else if (step["use-yml"]) {
+                            const yaml = this.import_files_map.get(step["use-yml"]);
+                            if (step['with-env']) {
+                                for (const key of Object.keys(step['with-env'])) {
+                                    step['with-env'][key] = Mustache.render(`${step['with-env'][key] ?? ""}`, this.env);
+                                }
+                            }
+                            const worker = new work_children(undefined, this.workflow_dir_path);
+                            await worker.init({
+                                env: step['with-env'] ?? this.env,
+                                yaml_data: yaml.yaml_data,
+                                yaml_path: yaml.yaml_path,
+                                filecat_user_name: this.env['filecat_user_name'],
+                                filecat_user_id: this.env['filecat_user_id'],
+                                filecat_user_note: this.env['filecat_user_note'],
+                                workflow_logger: this.logger
+                            });
+                            this.worker_children_use_yml_map.set(step["use-yml"], worker);
+                            const r = await worker.run_jobs(true);
+                            step.use_job_children_list = r.all_jobs;
+                            if (r.failed) {
+                                step.code = -1;
+                                this.logger.send_all_wss();
+                                throw "";
+                            }
+                            step.code = 0;
+                            this.logger.send_all_wss();
+                        }
+                        else if (step.run != null || step.runs != null) {
+                            step.code = await process_runner.run_step(step);
+                            if (step.code !== 0) {
+                                throw step.message;
+                            }
+                        }
+                        else {
+                            step.code = 0;
+                        }
+                    }
+                    catch (e) {
+                        if (step['catch-js'] != null && await workflow_util_1.workflow_util.run_js(step['catch-js'], this.env)) {
+                        }
+                        else {
+                            throw e;
+                        }
+                    }
+                    finally {
+                        step.duration = `${((Date.now() - step_start_time) / 1000).toFixed(2)} s`;
+                    }
+                    if (step['then-log']) {
+                        const log = Mustache.render(`${step['then-log'] ?? ""}`, this.env);
+                        this.logger.running_log = log;
+                        if (!step.message) {
+                            step.message = '';
+                        }
+                        step.message += log;
+                        this.logger.send_all_wss();
+                        if (step['then-log-file']) {
+                            const p = step['then-log-file'];
+                            let p1 = p;
+                            if (!path_1.default.isAbsolute(p)) {
+                                p1 = path_1.default.join(path_1.default.dirname(this.yaml_path), p);
+                            }
+                            user_service_1.userService.check_user_path_by_user_id(this.user_id, p1);
+                            await FileUtil_1.FileUtil.writeFileSync(p1, log);
+                        }
+                    }
+                    if (step["while"] != null && this.running_type === file_req_1.running_type.running && await workflow_util_1.workflow_util.run_js(step["while"], this.env)) {
+                        i--;
+                        await common_util_1.CommonUtil.sleep(100);
+                        continue;
+                    }
+                    this.logger.send_all_wss();
+                }
+                this.pty_shell_set.delete(process_runner.pty);
+                job.duration = `${((Date.now() - start_time) / 1000).toFixed(2)} s`;
+            }
+            else {
+                job.duration = `0 s`;
+            }
+            if (job.while != null && this.running_type === file_req_1.running_type.running && await workflow_util_1.workflow_util.run_js(job.while, this.env)) {
+                workflow_util_1.workflow_util.reset_all_step_status(job);
+                await common_util_1.CommonUtil.sleep(100);
+                this.run_job(job);
+                console.log('run again job');
+                return;
+            }
+            job.code = 0;
+            this.done_jobs.add(job.key);
+            this.logger.send_all_wss();
+            if (this.need_job_set.size) {
+                const v_List = [...this.need_job_set];
+                for (let i = 0; i < v_List.length; i++) {
+                    const job = v_List[i];
+                    this.need_job_set.delete(job);
+                    this.run_job(job).catch(e => {
+                        throw e;
+                    }).then(() => {
+                    });
+                }
+            }
+            if (this.done_jobs.size === this.jobs_map.size) {
+                this.all_job_resolve("ok");
+            }
+        }
+        catch (error) {
+            if (job['catch-js'] != null && await workflow_util_1.workflow_util.run_js(job['catch-js'], this.env)) {
+            }
+            else {
+                console.log('workflows job error ', error);
+                this.done_fail_job_handle(job, typeof error === 'string' ? error : error?.message ?? JSON.stringify(error));
+            }
+        }
+    }
+    close(type) {
+        this.running_type = type ?? file_req_1.running_type.fail;
+        for (const it of this.pty_shell_set) {
+            it.close();
+        }
+        for (const worker of this.worker_children_use_yml_map.values()) {
+            worker.close(type);
+        }
+        this.all_job_resolve("ok");
+    }
+}
+exports.work_children = work_children;
+const work_exec_map = new Map();
+const realtime_user_dir_wss_map = new Map();
+class WorkflowService {
+    async workflow_get_pre_inputs(path) {
+        const list = [];
+        const yml_data = await readYamlFile(path);
+        if (yml_data.inputs && typeof yml_data.inputs === "object") {
+            for (const key of Object.keys(yml_data.inputs)) {
+                const v = yml_data.inputs[key];
+                list.push({
+                    key: key,
+                    description: v.description,
+                    required: v.required,
+                    default: v.default,
+                    options: v.options
+                });
+            }
+        }
+        return list;
+    }
+    async workflow_exec(data) {
+        const token = data.wss.token;
+        const pojo = data.context;
+        const root_path = setting_service_1.settingService.getFileRootPath(token);
+        const file_path = path_1.default.join(root_path, decodeURIComponent(pojo.path));
+        const user_info = user_service_1.userService.get_user_info_by_token(token);
+        if (pojo.run_type === file_req_1.WorkRunType.start) {
+            await this.exec_file(file_path, user_info, pojo.inputs);
+        }
+        else if (pojo.run_type === file_req_1.WorkRunType.stop) {
+            const worker = work_exec_map.get(file_path);
+            if (!worker)
+                throw "no task  exists";
+            worker.close();
+            work_exec_map.delete(file_path);
+            this.online_change_push();
+        }
+    }
+    async exec_file(file_path, user_info, inputs) {
+        if (work_exec_map.get(file_path))
+            throw "Workflow exec task already exists";
+        const worker = new work_children(file_path);
+        work_exec_map.set(file_path, worker);
+        try {
+            const pre_env = {};
+            if (inputs) {
+                for (const it of inputs) {
+                    pre_env[it.key] = it.default;
+                }
+            }
+            await worker.init({
+                filecat_user_id: user_service_1.userService.get_user_id(user_info.username),
+                filecat_user_name: user_info.username,
+                filecat_user_note: user_info.note,
+                pre_env
+            });
+        }
+        catch (e) {
+            worker.running_type = file_req_1.running_type.fail;
+            this.online_change_push();
+            work_exec_map.delete(file_path);
+            throw e;
+        }
+        this.online_change_push();
+        worker.run_jobs().then(e => {
+            this.online_change_push();
+            work_exec_map.delete(file_path);
+        }).catch(e => {
+            this.online_change_push();
+            worker.running_type = file_req_1.running_type.fail;
+            console.log('任务执行失败', e);
+            work_exec_map.delete(file_path);
+            worker.close();
+        });
+    }
+    async workflow_realtime_one(data) {
+        const token = data.wss.token;
+        const pojo = data.context;
+        const root_path = setting_service_1.settingService.getFileRootPath(token);
+        const file_path = path_1.default.join(root_path, decodeURIComponent(pojo.filename_path));
+        user_service_1.userService.check_user_path(data.wss.token, file_path);
+        const v = work_exec_map.get(file_path);
+        if (!v) {
+            return;
+        }
+        v.logger.add_wss(data.wss);
+        data.wss.setClose(() => {
+            v.logger.remove_wss(data.wss);
+        });
+    }
+    async workflow_get(data) {
+        const token = data.wss.token;
+        const pojo = data.context;
+        const root_path = setting_service_1.settingService.getFileRootPath(token);
+        const dir_path = path_1.default.join(root_path, decodeURIComponent(pojo.dir_path), file_req_1.workflow_dir_name);
+        user_service_1.userService.check_user_path(data.wss.token, dir_path);
+        const basedata = new base_data_util_1.Base_data_util({ base_dir: dir_path });
+        const r = new file_req_1.WorkflowGetRsq();
+        if (pojo.index !== undefined) {
+            r.one_data = await basedata.find_one_by_index(pojo.index);
+            return r;
+        }
+        const running_list = [];
+        for (const it of work_exec_map.values()) {
+            running_list.push({
+                meta: {
+                    is_running: true,
+                    name: it.name,
+                    "run-name": it["run-name"]
+                },
+                index: -1
+            });
+        }
+        r.list = [...running_list, ...await basedata.find_page(pojo.page_num, pojo.page_size, true)];
+        r.total = await basedata.find_num();
+        return r;
+    }
+    async workflow_search_by_run_name(data) {
+        const token = data.wss.token;
+        const pojo = data.context;
+        const root_path = setting_service_1.settingService.getFileRootPath(token);
+        const dir_path = path_1.default.join(root_path, decodeURIComponent(pojo.dir_path), file_req_1.workflow_dir_name);
+        user_service_1.userService.check_user_path(data.wss.token, dir_path);
+        const basedata = new base_data_util_1.Base_data_util({ base_dir: dir_path });
+        const r = new file_req_1.WorkflowGetRsq();
+        const regex = new RegExp(pojo.search_name);
+        r.list = await basedata.find_list((index, meta) => {
+            if (!meta)
+                return false;
+            return regex.test(JSON.parse(meta)['run-name']);
+        });
+        r.total = await basedata.find_num();
+        return r;
+    }
+    workflow_realtime(data) {
+        const token = data.wss.token;
+        const pojo = data.context;
+        const wss = data.wss;
+        const root_path = setting_service_1.settingService.getFileRootPath(token);
+        const dir_path = path_1.default.join(root_path, pojo.dir_path);
+        user_service_1.userService.check_user_path(data.wss.token, dir_path);
+        const runing_filename_list = [];
+        const parent = (0, StringUtil_1.removeTrailingPath)(decodeURIComponent(dir_path));
+        for (const key of work_exec_map.keys()) {
+            const child = (0, StringUtil_1.removeTrailingPath)(path_1.default.dirname(key));
+            const p = work_exec_map.get(key);
+            if (parent === child && p.running_type === file_req_1.running_type.running) {
+                runing_filename_list.push(p.filename);
+            }
+        }
+        const pre_path = wss.dataMap.get("pre_path");
+        if (pre_path) {
+            let set = realtime_user_dir_wss_map.get(pre_path);
+            if (set) {
+                set.delete(wss);
+            }
+        }
+        let set = realtime_user_dir_wss_map.get(parent);
+        if (!set) {
+            set = new Set();
+            realtime_user_dir_wss_map.set(parent, set);
+        }
+        set.add(wss);
+        wss.setClose(() => {
+            set.delete(wss);
+        });
+        wss.dataMap.set("pre_path", parent);
+        const rsq = new file_req_1.WorkFlowRealTimeRsq();
+        rsq.running_file_list = runing_filename_list;
+        return rsq;
+    }
+    online_change_push() {
+        const map = new Map();
+        const faile_map = new Map();
+        const run_map = new Map();
+        for (const it of work_exec_map.values()) {
+            const key = it.yaml_path;
+            const dir_path = (0, StringUtil_1.removeTrailingPath)(path_1.default.dirname(key));
+            let list;
+            if (it.running_type === file_req_1.running_type.fail) {
+                list = faile_map.get(dir_path);
+                if (!list) {
+                    list = [];
+                    faile_map.set(dir_path, list);
+                }
+            }
+            else if (it.running_type === file_req_1.running_type.success) {
+                list = map.get(dir_path);
+                if (!list) {
+                    list = [];
+                    map.set(dir_path, list);
+                }
+            }
+            else {
+                list = run_map.get(dir_path);
+                if (!list) {
+                    list = [];
+                    run_map.set(dir_path, list);
+                }
+            }
+            list.push(work_exec_map.get(key).filename);
+        }
+        for (const key of realtime_user_dir_wss_map.keys()) {
+            const result = new WsData_1.WsData(WsData_1.CmdType.workflow_realtime);
+            const rsq = new file_req_1.WorkFlowRealTimeRsq();
+            result.context = rsq;
+            if (map.has(key)) {
+                rsq.sucess_file_list = map.get(key);
+            }
+            if (faile_map.has(key)) {
+                rsq.failed_file_list = faile_map.get(key);
+            }
+            if (run_map.has(key)) {
+                rsq.running_file_list = run_map.get(key);
+            }
+            const user_wss_set = realtime_user_dir_wss_map.get(key);
+            for (const v of user_wss_set) {
+                v.sendData(result.encode());
+            }
+        }
+    }
+}
+exports.WorkflowService = WorkflowService;
+exports.workflowService = new WorkflowService();
+//# sourceMappingURL=workflow.service.js.map