filecat 5.32.2 → 5.34.0

package/build/server/main/domain/file/workflow/workflow.service.js

@@ -1,659 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.workflowService = exports.WorkflowService = exports.work_children = exports.workflow_logger = exports.send_ws_type = void 0;
-const WsData_1 = require("../../../../common/frame/WsData");
-const file_req_1 = require("../../../../common/req/file.req");
-const setting_service_1 = require("../../setting/setting.service");
-const path_1 = __importDefault(require("path"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const user_service_1 = require("../../user/user.service");
-const user_req_1 = require("../../../../common/req/user.req");
-const base_data_util_1 = require("../../data/basedata/base_data_util");
-const StringUtil_1 = require("../../../../common/StringUtil");
-const ValueUtil_1 = require("../../../../common/ValueUtil");
-const FileUtil_1 = require("../FileUtil");
-const workflow_util_1 = require("./workflow.util");
-const workflow_process_1 = require("./workflow.process");
-const common_util_1 = require("../../../../common/common.util");
-const readYamlFile = require('read-yaml-file');
-const Mustache = require('mustache');
-var send_ws_type;
-(function (send_ws_type) {
-    send_ws_type[send_ws_type["done"] = 0] = "done";
-    send_ws_type[send_ws_type["new_log"] = 1] = "new_log";
-})(send_ws_type || (exports.send_ws_type = send_ws_type = {}));
-class workflow_logger {
-    constructor(instance) {
-        this.wss_call_set = new Set();
-        this.instance = instance;
-    }
-    send_wss(type, ...wss) {
-        const r_list = [];
-        if (type !== send_ws_type.new_log) {
-            work_children.get_workflow_realtime_tree_list(r_list, this.instance);
-        }
-        for (const w of wss) {
-            const result = new WsData_1.WsData(WsData_1.CmdType.workflow_realtime_one_rsq);
-            result.context = {
-                done: type === undefined ? undefined : type === send_ws_type.done,
-                list: r_list,
-                new_log: this.running_log
-            };
-            w.sendData(result.encode());
-        }
-    }
-    send_all_wss(type) {
-        if (this.wss_call_set) {
-            this.send_wss(type, ...this.wss_call_set.values());
-        }
-    }
-    add_wss(wss) {
-        if (!this.wss_call_set.has(wss)) {
-            this.wss_call_set.add(wss);
-        }
-        this.send_wss(undefined, wss);
-    }
-    remove_wss(wss) {
-        if (!this.wss_call_set) {
-            this.wss_call_set.delete(wss);
-        }
-    }
-}
-exports.workflow_logger = workflow_logger;
-class work_children {
-    constructor(yaml_path = undefined, workflow_dir_path = undefined) {
-        this.running_type = file_req_1.running_type.running;
-        this.import_files_map = new Map();
-        this.jobs_map = new Map();
-        this.done_jobs = new Set();
-        this.need_job_set = new Set();
-        this.pty_shell_set = new Set();
-        this.worker_children_use_yml_map = new Map();
-        this.yaml_path = yaml_path;
-        if (workflow_dir_path) {
-            this.workflow_dir_path = workflow_dir_path;
-        }
-        this.logger = new workflow_logger(this);
-    }
-    static get_workflow_realtime_tree_list(r_list, worker) {
-        for (const it of worker.jobs_map.values()) {
-            const v = {
-                name: it.name,
-                extra_data: { running_type: it.running_type },
-                code: it.code,
-                children: []
-            };
-            if (it.code !== 0 && it.steps) {
-                const children = [];
-                for (const step of it.steps ?? []) {
-                    let name = '';
-                    if (step.run) {
-                        name = step.run;
-                    }
-                    else if (step["use-yml"]) {
-                        name = step["use-yml"];
-                    }
-                    else if (step["run-js"]) {
-                        name = step["run-js"];
-                    }
-                    else if (step["sleep"]) {
-                        name = `sleep ${step.sleep}`;
-                    }
-                    else if (step["runs"]) {
-                        name = JSON.stringify(step.runs);
-                    }
-                    const p = {
-                        name: name,
-                        extra_data: { running_type: step.running_type },
-                        code: step.code,
-                        children: []
-                    };
-                    const children_worker = worker.worker_children_use_yml_map.get(step['use-yml']);
-                    if (children_worker) {
-                        this.get_workflow_realtime_tree_list(p.children, children_worker);
-                    }
-                    children.push(p);
-                }
-                v.children = children;
-            }
-            r_list.push(v);
-        }
-    }
-    async init(param) {
-        if (param?.yaml_path) {
-            this.yaml_path = param.yaml_path;
-        }
-        if (param?.workflow_logger) {
-            this.logger = param.workflow_logger;
-        }
-        this.filename = path_1.default.basename(this.yaml_path);
-        const yaml_data = param?.yaml_data ?? await readYamlFile(this.yaml_path);
-        this.env = yaml_data.env ?? {};
-        if (param.pre_env) {
-            for (const key of Object.keys(param.pre_env)) {
-                this.env[key] = param.pre_env[key];
-            }
-        }
-        this.env['filecat_user_id'] = param.filecat_user_id;
-        this.env['filecat_user_name'] = param.filecat_user_name;
-        this.env['filecat_user_note'] = param.filecat_user_note;
-        let user_id = `${yaml_data.user_id || yaml_data['user-id'] || ""}`;
-        if (!user_id) {
-            user_id = user_service_1.userService.get_user_id(`${yaml_data.username}`);
-        }
-        if (!user_id) {
-            throw "user not exists";
-        }
-        this.user_id = user_id;
-        if (param?.yaml_path)
-            user_service_1.userService.check_user_path_by_user_id(this.user_id, param?.yaml_path);
-        user_service_1.userService.check_user_auth_by_user_id(this.user_id, user_req_1.UserAuth.workflow_exe_user, {
-            auto_throw: true,
-            root_check: true
-        });
-        if (!this.workflow_dir_path) {
-            let find_p = path_1.default.dirname(this.yaml_path);
-            let have_find = false;
-            while (true) {
-                if (await FileUtil_1.FileUtil.access(path_1.default.join(find_p, file_req_1.workflow_dir_name))) {
-                    have_find = true;
-                    break;
-                }
-                const p = path_1.default.join(find_p, "..");
-                if (p === find_p) {
-                    have_find = false;
-                    break;
-                }
-                find_p = p;
-                if (!user_service_1.userService.check_user_path_by_user_id(this.user_id, find_p, false)) {
-                    have_find = false;
-                    break;
-                }
-            }
-            if (have_find) {
-                this.workflow_dir_path = path_1.default.join(find_p, file_req_1.workflow_dir_name);
-            }
-            else {
-                this.workflow_dir_path = path_1.default.join(path_1.default.dirname(this.yaml_path), file_req_1.workflow_dir_name);
-                await fs_extra_1.default.ensureDir(this.workflow_dir_path);
-            }
-        }
-        if (yaml_data.jobs) {
-            for (const key of Object.keys(yaml_data.jobs)) {
-                const v = yaml_data.jobs[key];
-                v.key = key;
-                this.jobs_map.set(v.key, v);
-            }
-        }
-        if (yaml_data['import-files']) {
-            for (const p of yaml_data['import-files']) {
-                let p1 = p;
-                if (!path_1.default.isAbsolute(p)) {
-                    p1 = path_1.default.join(path_1.default.dirname(this.yaml_path), p);
-                }
-                user_service_1.userService.check_user_path_by_user_id(this.user_id, p1);
-                const d = await readYamlFile(p1);
-                this.import_files_map.set(d.name, { yaml_path: p1, yaml_data: d });
-            }
-        }
-        yaml_data['run-name'] = Mustache.render(`${yaml_data['run-name'] ?? ""}`, this.env ?? {});
-        this["run-name"] = `${yaml_data['run-name'] ?? ""}`;
-        this.name = `${yaml_data.name}`;
-        if (param?.env) {
-            for (const key of Object.keys(param.env)) {
-                this.env[key] = param.env[key];
-            }
-        }
-        if (yaml_data['file-env']) {
-            for (const key of Object.keys(yaml_data['file-env'])) {
-                let p1 = yaml_data['file-env'][key];
-                if (!path_1.default.isAbsolute(p1)) {
-                    p1 = path_1.default.join(path_1.default.dirname(this.yaml_path), p1);
-                }
-                user_service_1.userService.check_user_path_by_user_id(this.user_id, p1);
-                this.env[key] = (await FileUtil_1.FileUtil.readFileSync(p1)).toString();
-            }
-        }
-        return this;
-    }
-    async run_jobs(return_steps = false) {
-        const start_time = Date.now();
-        await new Promise(resolve => {
-            this.all_job_resolve = resolve;
-            const v_List = [...this.jobs_map.values()];
-            for (let i = 0; i < v_List.length; i++) {
-                const job = v_List[i];
-                this.run_job(job).then(() => {
-                });
-            }
-        });
-        const all_jobs = [];
-        let failed = false;
-        for (const it of this.jobs_map.values()) {
-            all_jobs.push(it);
-            if (it.code !== 0 && it.code !== undefined) {
-                failed = true;
-            }
-        }
-        this.close(failed ? file_req_1.running_type.fail : file_req_1.running_type.success);
-        if (return_steps) {
-            return { all_jobs, failed };
-        }
-        this.logger.send_all_wss(failed === true ? send_ws_type.done : undefined);
-        this.running_type = failed === false ? file_req_1.running_type.success : file_req_1.running_type.fail;
-        try {
-            const base_data_util = new base_data_util_1.Base_data_util({ base_dir: this.workflow_dir_path });
-            await base_data_util.init();
-            await base_data_util.insert(JSON.stringify({
-                all_jobs
-            }), JSON.stringify({
-                name: this.name,
-                "run-name": this["run-name"],
-                is_success: failed === false,
-                timestamp: (0, ValueUtil_1.formatter_time)(new Date()),
-                duration: `${((Date.now() - start_time) / 1000).toFixed(2)} s`
-            }));
-        }
-        catch (error) {
-            console.log('插入数据库失败', error);
-        }
-    }
-    done_fail_job_handle(job, fail_message) {
-        job.code = -1;
-        this.logger.send_all_wss();
-        job.message = fail_message;
-        this.close();
-        this.all_job_resolve("ok");
-    }
-    async run_job(job) {
-        try {
-            let job_can_run = false;
-            if (!job.if) {
-                job_can_run = true;
-            }
-            else {
-                job_can_run = await workflow_util_1.workflow_util.run_js(job.if, this.env);
-            }
-            if (job_can_run) {
-                workflow_util_1.workflow_util.handle_job_pre(job, this);
-                if (job['need-jobs']) {
-                    if (!workflow_util_1.workflow_util.all_jobs_done(job['need-jobs'], this)) {
-                        this.need_job_set.add(job);
-                        return;
-                    }
-                }
-                job.running_type = file_req_1.running_type.running;
-                const start_time = Date.now();
-                const process_runner = new workflow_process_1.WorkflowProcess(this, job);
-                this.pty_shell_set.add(process_runner.pty);
-                for (let i = 0; i < job.steps.length; i++) {
-                    const step = job.steps[i];
-                    if (step.if) {
-                        if (!await workflow_util_1.workflow_util.run_js(step.if, this.env)) {
-                            this.logger.send_all_wss();
-                            continue;
-                        }
-                    }
-                    step.running_type = file_req_1.running_type.running;
-                    this.logger.send_all_wss();
-                    if (step['sleep']) {
-                        await common_util_1.CommonUtil.sleep(step['sleep']);
-                    }
-                    const step_start_time = Date.now();
-                    try {
-                        if (step["run-js"]) {
-                            await workflow_util_1.workflow_util.run_code_js_by_step(step, job, this.env, true);
-                        }
-                        else if (step["use-yml"]) {
-                            const yaml = this.import_files_map.get(step["use-yml"]);
-                            if (step['with-env']) {
-                                for (const key of Object.keys(step['with-env'])) {
-                                    step['with-env'][key] = Mustache.render(`${step['with-env'][key] ?? ""}`, this.env);
-                                }
-                            }
-                            const worker = new work_children(undefined, this.workflow_dir_path);
-                            await worker.init({
-                                env: step['with-env'] ?? this.env,
-                                yaml_data: yaml.yaml_data,
-                                yaml_path: yaml.yaml_path,
-                                filecat_user_name: this.env['filecat_user_name'],
-                                filecat_user_id: this.env['filecat_user_id'],
-                                filecat_user_note: this.env['filecat_user_note'],
-                                workflow_logger: this.logger
-                            });
-                            this.worker_children_use_yml_map.set(step["use-yml"], worker);
-                            const r = await worker.run_jobs(true);
-                            step.use_job_children_list = r.all_jobs;
-                            if (r.failed) {
-                                step.code = -1;
-                                this.logger.send_all_wss();
-                                throw "";
-                            }
-                            step.code = 0;
-                            this.logger.send_all_wss();
-                        }
-                        else if (step.run != null || step.runs != null) {
-                            step.code = await process_runner.run_step(step);
-                            if (step.code !== 0) {
-                                throw step.message;
-                            }
-                        }
-                        else {
-                            step.code = 0;
-                        }
-                    }
-                    catch (e) {
-                        if (step['catch-js'] != null && await workflow_util_1.workflow_util.run_js(step['catch-js'], this.env)) {
-                        }
-                        else {
-                            throw e;
-                        }
-                    }
-                    finally {
-                        step.duration = `${((Date.now() - step_start_time) / 1000).toFixed(2)} s`;
-                    }
-                    if (step['then-log']) {
-                        const log = Mustache.render(`${step['then-log'] ?? ""}`, this.env);
-                        this.logger.running_log = log;
-                        if (!step.message) {
-                            step.message = '';
-                        }
-                        step.message += log;
-                        this.logger.send_all_wss();
-                        if (step['then-log-file']) {
-                            const p = step['then-log-file'];
-                            let p1 = p;
-                            if (!path_1.default.isAbsolute(p)) {
-                                p1 = path_1.default.join(path_1.default.dirname(this.yaml_path), p);
-                            }
-                            user_service_1.userService.check_user_path_by_user_id(this.user_id, p1);
-                            await FileUtil_1.FileUtil.writeFileSync(p1, log);
-                        }
-                    }
-                    if (step["while"] != null && this.running_type === file_req_1.running_type.running && await workflow_util_1.workflow_util.run_js(step["while"], this.env)) {
-                        i--;
-                        await common_util_1.CommonUtil.sleep(100);
-                        continue;
-                    }
-                    this.logger.send_all_wss();
-                }
-                this.pty_shell_set.delete(process_runner.pty);
-                job.duration = `${((Date.now() - start_time) / 1000).toFixed(2)} s`;
-            }
-            else {
-                job.duration = `0 s`;
-            }
-            if (job.while != null && this.running_type === file_req_1.running_type.running && await workflow_util_1.workflow_util.run_js(job.while, this.env)) {
-                workflow_util_1.workflow_util.reset_all_step_status(job);
-                await common_util_1.CommonUtil.sleep(100);
-                this.run_job(job);
-                console.log('run again job');
-                return;
-            }
-            job.code = 0;
-            this.done_jobs.add(job.key);
-            this.logger.send_all_wss();
-            if (this.need_job_set.size) {
-                const v_List = [...this.need_job_set];
-                for (let i = 0; i < v_List.length; i++) {
-                    const job = v_List[i];
-                    this.need_job_set.delete(job);
-                    this.run_job(job).catch(e => {
-                        throw e;
-                    }).then(() => {
-                    });
-                }
-            }
-            if (this.done_jobs.size === this.jobs_map.size) {
-                this.all_job_resolve("ok");
-            }
-        }
-        catch (error) {
-            if (job['catch-js'] != null && await workflow_util_1.workflow_util.run_js(job['catch-js'], this.env)) {
-            }
-            else {
-                console.log('workflows job error ', error);
-                this.done_fail_job_handle(job, typeof error === 'string' ? error : error?.message ?? JSON.stringify(error));
-            }
-        }
-    }
-    close(type) {
-        this.running_type = type ?? file_req_1.running_type.fail;
-        for (const it of this.pty_shell_set) {
-            it.close();
-        }
-        for (const worker of this.worker_children_use_yml_map.values()) {
-            worker.close(type);
-        }
-        this.all_job_resolve("ok");
-    }
-}
-exports.work_children = work_children;
-const work_exec_map = new Map();
-const realtime_user_dir_wss_map = new Map();
-class WorkflowService {
-    async workflow_get_pre_inputs(path) {
-        const list = [];
-        const yml_data = await readYamlFile(path);
-        if (yml_data.inputs && typeof yml_data.inputs === "object") {
-            for (const key of Object.keys(yml_data.inputs)) {
-                const v = yml_data.inputs[key];
-                list.push({
-                    key: key,
-                    description: v.description,
-                    required: v.required,
-                    default: v.default,
-                    options: v.options
-                });
-            }
-        }
-        return list;
-    }
-    async workflow_exec(data) {
-        const token = data.wss.token;
-        const pojo = data.context;
-        const root_path = setting_service_1.settingService.getFileRootPath(token);
-        const file_path = path_1.default.join(root_path, decodeURIComponent(pojo.path));
-        const user_info = user_service_1.userService.get_user_info_by_token(token);
-        if (pojo.run_type === file_req_1.WorkRunType.start) {
-            await this.exec_file(file_path, user_info, pojo.inputs);
-        }
-        else if (pojo.run_type === file_req_1.WorkRunType.stop) {
-            const worker = work_exec_map.get(file_path);
-            if (!worker)
-                throw "no task  exists";
-            worker.close();
-            work_exec_map.delete(file_path);
-            this.online_change_push();
-        }
-    }
-    async exec_file(file_path, user_info, inputs) {
-        if (work_exec_map.get(file_path))
-            throw "Workflow exec task already exists";
-        const worker = new work_children(file_path);
-        work_exec_map.set(file_path, worker);
-        try {
-            const pre_env = {};
-            if (inputs) {
-                for (const it of inputs) {
-                    pre_env[it.key] = it.default;
-                }
-            }
-            await worker.init({
-                filecat_user_id: user_service_1.userService.get_user_id(user_info.username),
-                filecat_user_name: user_info.username,
-                filecat_user_note: user_info.note,
-                pre_env
-            });
-        }
-        catch (e) {
-            worker.running_type = file_req_1.running_type.fail;
-            this.online_change_push();
-            work_exec_map.delete(file_path);
-            throw e;
-        }
-        this.online_change_push();
-        worker.run_jobs().then(e => {
-            this.online_change_push();
-            work_exec_map.delete(file_path);
-        }).catch(e => {
-            this.online_change_push();
-            worker.running_type = file_req_1.running_type.fail;
-            console.log('任务执行失败', e);
-            work_exec_map.delete(file_path);
-            worker.close();
-        });
-    }
-    async workflow_realtime_one(data) {
-        const token = data.wss.token;
-        const pojo = data.context;
-        const root_path = setting_service_1.settingService.getFileRootPath(token);
-        const file_path = path_1.default.join(root_path, decodeURIComponent(pojo.filename_path));
-        user_service_1.userService.check_user_path(data.wss.token, file_path);
-        const v = work_exec_map.get(file_path);
-        if (!v) {
-            return;
-        }
-        v.logger.add_wss(data.wss);
-        data.wss.setClose(() => {
-            v.logger.remove_wss(data.wss);
-        });
-    }
-    async workflow_get(data) {
-        const token = data.wss.token;
-        const pojo = data.context;
-        const root_path = setting_service_1.settingService.getFileRootPath(token);
-        const dir_path = path_1.default.join(root_path, decodeURIComponent(pojo.dir_path), file_req_1.workflow_dir_name);
-        user_service_1.userService.check_user_path(data.wss.token, dir_path);
-        const basedata = new base_data_util_1.Base_data_util({ base_dir: dir_path });
-        const r = new file_req_1.WorkflowGetRsq();
-        if (pojo.index !== undefined) {
-            r.one_data = await basedata.find_one_by_index(pojo.index);
-            return r;
-        }
-        const running_list = [];
-        for (const it of work_exec_map.values()) {
-            running_list.push({
-                meta: {
-                    is_running: true,
-                    name: it.name,
-                    "run-name": it["run-name"]
-                },
-                index: -1
-            });
-        }
-        r.list = [...running_list, ...await basedata.find_page(pojo.page_num, pojo.page_size, true)];
-        r.total = await basedata.find_num();
-        return r;
-    }
-    async workflow_search_by_run_name(data) {
-        const token = data.wss.token;
-        const pojo = data.context;
-        const root_path = setting_service_1.settingService.getFileRootPath(token);
-        const dir_path = path_1.default.join(root_path, decodeURIComponent(pojo.dir_path), file_req_1.workflow_dir_name);
-        user_service_1.userService.check_user_path(data.wss.token, dir_path);
-        const basedata = new base_data_util_1.Base_data_util({ base_dir: dir_path });
-        const r = new file_req_1.WorkflowGetRsq();
-        const regex = new RegExp(pojo.search_name);
-        r.list = await basedata.find_list((index, meta) => {
-            if (!meta)
-                return false;
-            return regex.test(JSON.parse(meta)['run-name']);
-        });
-        r.total = await basedata.find_num();
-        return r;
-    }
-    workflow_realtime(data) {
-        const token = data.wss.token;
-        const pojo = data.context;
-        const wss = data.wss;
-        const root_path = setting_service_1.settingService.getFileRootPath(token);
-        const dir_path = path_1.default.join(root_path, pojo.dir_path);
-        user_service_1.userService.check_user_path(data.wss.token, dir_path);
-        const runing_filename_list = [];
-        const parent = (0, StringUtil_1.removeTrailingPath)(decodeURIComponent(dir_path));
-        for (const key of work_exec_map.keys()) {
-            const child = (0, StringUtil_1.removeTrailingPath)(path_1.default.dirname(key));
-            const p = work_exec_map.get(key);
-            if (parent === child && p.running_type === file_req_1.running_type.running) {
-                runing_filename_list.push(p.filename);
-            }
-        }
-        const pre_path = wss.dataMap.get("pre_path");
-        if (pre_path) {
-            let set = realtime_user_dir_wss_map.get(pre_path);
-            if (set) {
-                set.delete(wss);
-            }
-        }
-        let set = realtime_user_dir_wss_map.get(parent);
-        if (!set) {
-            set = new Set();
-            realtime_user_dir_wss_map.set(parent, set);
-        }
-        set.add(wss);
-        wss.setClose(() => {
-            set.delete(wss);
-        });
-        wss.dataMap.set("pre_path", parent);
-        const rsq = new file_req_1.WorkFlowRealTimeRsq();
-        rsq.running_file_list = runing_filename_list;
-        return rsq;
-    }
-    online_change_push() {
-        const map = new Map();
-        const faile_map = new Map();
-        const run_map = new Map();
-        for (const it of work_exec_map.values()) {
-            const key = it.yaml_path;
-            const dir_path = (0, StringUtil_1.removeTrailingPath)(path_1.default.dirname(key));
-            let list;
-            if (it.running_type === file_req_1.running_type.fail) {
-                list = faile_map.get(dir_path);
-                if (!list) {
-                    list = [];
-                    faile_map.set(dir_path, list);
-                }
-            }
-            else if (it.running_type === file_req_1.running_type.success) {
-                list = map.get(dir_path);
-                if (!list) {
-                    list = [];
-                    map.set(dir_path, list);
-                }
-            }
-            else {
-                list = run_map.get(dir_path);
-                if (!list) {
-                    list = [];
-                    run_map.set(dir_path, list);
-                }
-            }
-            list.push(work_exec_map.get(key).filename);
-        }
-        for (const key of realtime_user_dir_wss_map.keys()) {
-            const result = new WsData_1.WsData(WsData_1.CmdType.workflow_realtime);
-            const rsq = new file_req_1.WorkFlowRealTimeRsq();
-            result.context = rsq;
-            if (map.has(key)) {
-                rsq.sucess_file_list = map.get(key);
-            }
-            if (faile_map.has(key)) {
-                rsq.failed_file_list = faile_map.get(key);
-            }
-            if (run_map.has(key)) {
-                rsq.running_file_list = run_map.get(key);
-            }
-            const user_wss_set = realtime_user_dir_wss_map.get(key);
-            for (const v of user_wss_set) {
-                v.sendData(result.encode());
-            }
-        }
-    }
-}
-exports.WorkflowService = WorkflowService;
-exports.workflowService = new WorkflowService();
-//# sourceMappingURL=workflow.service.js.map