node-automator 1.0.0

package/utils/parse_tool.js

@@ -0,0 +1,236 @@
+const markdown_table = require('markdown-table');
+const xml2js = require('xml-js');
+const xmldom = require('xmldom');
+
+const {
+    json_compacted,
+} = require("./transform_tool");
+var json_stringify = require('json-stable-stringify');
+const parse_markdown_table = require("parse-markdown-table");
+
+const resolveCode = function(doc, node) {
+    return eval(`(${node.strValue})`);
+};
+
+const yamlOptions = {
+    customTags: [
+        { tag: "!fn", class: Function, resolve: resolveCode },
+        { tag: "!re", class: RegExp, resolve: resolveCode },
+    ]
+};
+
+async function parse(content, type, options) {
+    switch (type && type.toLowerCase()) {
+        case "markdown_table_array": {
+            let result = [];
+            const table = await parse_markdown_table.createMarkdownArrayTable(content);
+            result.push(table.headers);
+            for await (const row of table.rows) {
+                if (!row[0].replace(/-/g, "").trim()) {
+                    continue;
+                }
+                result.push(row);
+            }
+            return result;
+        }
+        case "markdown_table":
+        case "markdown_table_object": {
+            let result = [];
+            const table = await parse_markdown_table.createMarkdownObjectTable(content);
+            for await (const row of table) {
+                let values = Object.values(row);
+                if (!values[0].replace(/-/g, "").trim()) {
+                    continue;
+                }
+                result.push(row);
+            }
+            return result;
+        }
+        default: {
+            return parse_sync(content, type, options);
+        }
+    }
+}
+
+function parse_sync(content, type, options) {
+    switch (type && type.toLowerCase()) {
+        case "yml":
+        case "yaml": {
+            let opts = Object.assign({ }, yamlOptions, { merge: true });
+            return require("yaml").parse(content, opts);
+        }
+        case "json": {
+            if (options?.strip_comments) {
+                content = require("strip-json-comments")(content);
+            }
+            return JSON.parse(content);
+        }
+        case "xml": {
+            return new xmldom.DOMParser().parseFromString(content);
+        }
+        case "xml2json": {
+            return xml2js.xml2json(content);
+        }
+        case "xml2js": {
+            return xml2js.xml2js(content);
+        }
+        case "json2xml": {
+            return xml2js.json2xml(content);
+        }
+        case "js2xml": {
+            return xml2js.js2xml(content);
+        }
+        case "lua_assignment": {
+            let ast = require('luaparse').parse(content, {
+                locations: true,
+                comments: true
+            });
+            let ret = {};
+            for (let i = 0, len = ast.body.length; i < len; ++i) {
+                let statement = ast.body[i];
+                if (statement.type == "AssignmentStatement") {
+                    ret[statement.variables[0].name] = eval(statement.init[0].raw);
+                }
+            }
+            return ret;
+        }
+        case "lua": {
+            let ast = require('luaparse').parse(content, {
+                locations: true,
+                comments: true
+            });
+            let all = ast.body.concat(ast.comments).sort((s1, s2) => {
+                return s1.loc.start.line - s2.loc.start.line
+                    || s1.loc.start.column - s2.loc.start.column;
+            });
+            return all;
+        }
+        case "svn_diff": {
+            let ret = require("./svn_diff/diff_parser").parse(content);
+            return ret;
+        }
+        case "sequence": {
+            return content.split(/[^0-9\-]+/).map(v => {
+                let arr = v.split("-");
+                if (arr.length == 2) {
+                    let ret = [];
+                    for (let i = +arr[0]; i <= +arr[1]; ++i) {
+                        ret.push("" + i);
+                    }
+                    return ret;
+                }
+                return v;
+            }).flat().filter(v => v.trim()).map(v => +v).sort((a, b) => a - b);
+        }
+    }
+}
+
+
+async function stringify(data, type, options) {
+    switch (type && type.toLowerCase()) {
+        default: {
+            return stringify_sync(data, type, options);
+        }
+    }
+}
+
+function stringify_sync(data, type, options) {
+    switch (type && type.toLowerCase()) {
+        case "yml":
+        case "yaml": {
+            let opts = Object.assign({ }, yamlOptions, options);
+            return require("yaml").stringify(data, opts);
+        }
+        case "yml_compacted":
+        case "yaml_compacted": {
+            return require("yamljs").stringify(data, options.inline == undefined ? 1 : options.inline, options.space == undefined ? 2 : options.space);
+        }
+        case "json": {
+            options = Object.assign({ space: 4 }, {}, options);
+            let defaultSortKeys = [
+
+            ];
+            defaultSortKeys = Object.assign(defaultSortKeys, options.sort_keys);
+            return json_stringify(data, {
+                space: options.indent != null ? options.indent : 4,
+                cmp: (a, b) => {
+                    let idxa = defaultSortKeys.indexOf(a.key);
+                    let idxb = defaultSortKeys.indexOf(b.key);
+                    if (idxa != idxb) {
+                        if (idxa == -1) {
+                            return 1;
+                        } else {
+                            return -1;
+                        }
+                        return idxa - idxb;
+                    }
+                    let keya = parseInt(a.key);
+                    let keyb = parseInt(b.key);
+                    return keya - keyb;
+                }
+            });
+        }
+        case "json_compacted": {
+            options = Object.assign({ space: 4 }, options);
+            return json_compacted(data, options && options.space, options && options.sort_keys);
+        }
+        case "json_simple": {
+            options = Object.assign({ space: 4 }, options);
+            return JSON.stringify(data, null, options.space);
+        }
+        case "markdown_table":{
+            let records = [];
+            let headers;
+            for (const k in data) {
+                if (Object.hasOwnProperty.call(data, k)) {
+                    const ele = data[k];
+                    if (!headers) {
+                        headers = options && options.headers || Object.keys(ele);
+                        records.push(headers);
+                    }
+                    records.push(headers.map(header => ele[header]));
+                }
+            }
+            return markdown_table(records, options);
+        }
+        case "string": {
+            return data && data.toString();
+        }
+        case "sequence": {
+            let str = "";
+            let delimiter = ",", joint = "-";
+            let in_sequence = false;
+            let prev = null;
+            data.sort((a, b) => a - b).forEach((curr, curr_index, arr) => {
+                if (prev == null) {
+                    str += curr;
+                } else if (prev != curr - 1) {
+                    if (in_sequence) {
+                        str += prev;
+                        in_sequence = false;
+                    }
+                    str += delimiter + curr;
+                } else if (!in_sequence) {
+                    str += joint;
+                    in_sequence = true;
+                }
+                if (curr_index == arr.length - 1 && in_sequence) {
+                    str += curr;
+                }
+                prev = curr;
+            });
+            return str;
+        }
+        default: {
+            // 默认返回自身
+            return data;
+        }
+    }
+}
+
+module.exports = {
+    parse,
+    parse_sync,
+    stringify,
+    stringify_sync,
+};

package/utils/request_tool.js

@@ -0,0 +1,156 @@
+const { get_full_path, remove } = require("./file_tool");
+const transform_tool = require("./transform_tool");
+const { warn, info, log, setLastError } = require("./log_tool");
+const { progress } = require("./display_tool");
+const fs = require("fs");
+const { eval_code, shareData } = require("../commands/share_data");
+const path = require("path");
+const { hash } = require('./hash_tool');
+
+async function doRequest(data) {
+    const request = require("request");
+    const request_progress = require("request-progress");
+    
+    let url = data.src || data?.target?.url;
+    let urlObj = new URL(url);
+    /** 是否启用 http 缓存 */
+    const enable_http_cache = data.enable_cache || false;
+    /** @type {http.RequestOptions} */
+    let options = {
+        method: "GET",
+        rejectUnauthorized: false,
+        timeout: 10 * 60 * 1000, // 超时(ms)
+    };
+    let maxRetryTimes = 0;
+    let leftRetryTimes = maxRetryTimes;
+    options = Object.assign(options, data.options);
+    if (!options.proxy) {
+        options.proxy = shareData.ENV.NODE_AUTOMATOR_PROXY;
+    }
+    /** @type {string} */
+    let http_cache;
+    let cache_name = `[${options.method}]` + hash(url) + "." + (path.basename(urlObj.pathname) || "index.html");
+    let cache_path = path.resolve(shareData.AUTOMATOR_APPDATA, ".raw_cache", cache_name);
+    if (enable_http_cache && fs.existsSync(cache_path)) {
+        http_cache = fs.readFileSync(cache_path);
+    }
+    let result;
+    let useCache = enable_http_cache && http_cache;
+    let quiet = data.quiet;
+    !quiet && info(useCache ? "[缓存]" : `[${options.method}] ${url} `);
+    let hasProgress = false;
+    if (useCache) {
+        result = http_cache;
+        if (data.dst) {
+            let file = fs.createWriteStream(get_full_path(data.dst, "FILE"));
+            file.write(result);
+        }
+    } else {
+        result = await new Promise(resolve => {
+            let sizeTotal = 0;
+            let fileDst = data.dst || shareData.AUTOMATOR_SCRATCH + "/request_tool/" + cache_name;
+            let file = fs.createWriteStream(get_full_path(fileDst, "FILE"));
+            let cacheDir = path.dirname(cache_path);
+            if (!fs.existsSync(cacheDir)) {
+                fs.mkdirSync(cacheDir, {
+                    recursive: true
+                });
+            }
+            let cachedFile;
+            if (enable_http_cache) {
+                cachedFile = fs.createWriteStream(cache_path);
+            }
+            function send_request() {
+                let req = request(url, options, (err, res, result) => {
+                    if (err) {
+                        // 检查重试
+                        if (leftRetryTimes > 0) {
+                            --leftRetryTimes;
+                            !quiet && warn(`[重试] ${url} ${err}`);
+                            send_request();
+                            return;
+                        }
+                        setLastError(`${url} ${err}`);
+                        resolve();
+                        return;
+                    }
+                    sizeTotal = res.client.bytesRead;
+                    !quiet && progress(sizeTotal, sizeTotal, { desc: "完成", format_bytes: true, color: "green" });
+                    // 仅在响应码合法时，写入缓存
+                    if (enable_http_cache) {
+                        if (res.statusCode > 0 && res.statusCode < 400) {
+
+                        } else {
+                            if (fs.existsSync(cache_path)) {
+                                fs.unlinkSync(cache_path);
+                            }
+                        }
+                    }
+                    if (!data.ignore_status_code) {
+                        if (res.statusCode >= 400) {
+                            setLastError(`${options.method} ${url} ${res.statusCode} (${res.statusMessage}) "${typeof res.body == "string" ? res.body : JSON.stringify(res.body)}" `);
+                        }
+                    }
+                    // 返回校验
+                    if (data.response_validator) {
+                        let [ok, errmsg] = eval_code(data.response_validator)(res);
+                        if (!ok) {
+                            setLastError(errmsg || "校验失败");
+                        }
+                    }
+                    switch (data.response_type) {
+                        case "HEADERS": {
+                            resolve(res.headers);
+                            break;
+                        }
+                        case "BODY": {
+                            resolve(res.body);
+                            break;
+                        }
+                        case "STATUS_CODE": {
+                            resolve(res.statusCode);
+                            break;
+                        }
+                        case "RAW": {
+                            resolve(res);
+                            break;
+                        }
+                        case "DEFAULT":
+                        default: {
+                            if (options.method.toLowerCase() == "head") {
+                                resolve(res.headers);
+                            } else {
+                                resolve(result);
+                            } 
+                        }
+                    }
+                });
+                req.pipe(file);
+                if (enable_http_cache) {
+                    req.pipe(cachedFile);
+                }
+                request_progress(
+                    req.on('progress', function (state) {
+                            sizeTotal = state.size.total;
+                            hasProgress = true;
+                            !quiet && progress(state.size.transferred, state.size.total, {
+                                desc: `请求中...(elapsed: ${transform_tool.formatTimeInSec(state.time.elapsed)}, speed: ${transform_tool.formatByteSize(state.speed)}/s, ETA: ${transform_tool.formatTimeInSec(state.time.remaining)})`,
+                                format_bytes: true,
+                                color: "cyan" 
+                            }, true);
+                        }
+                    )
+                );
+            }
+            send_request();
+        });
+    }
+    if (hasProgress) {
+        log("");
+    }
+    return result;
+}
+
+module.exports = {
+    doRequest,
+};

package/utils/scm_tool.js

@@ -0,0 +1,108 @@
+const { option } = require("yargs");
+const { vital, log, warn } = require("./log_tool");
+const { exec_shell } = require("./shell_tool");
+const path = require("path");
+const fs = require("fs");
+const { copy, get_fst_file, get_full_path } = require("./file_tool");
+
+async function svn_is_dirty(cwd) {
+    return await exec_shell({
+        cmd: "svn status . --quiet",
+        cwd: cwd,
+        capture_stdout: true,
+    }, true);
+}
+
+async function git_is_dirty(cwd) {
+    let result = await exec_shell({
+        cmd: "git status .",
+        cwd: cwd,
+        capture_stdout: true,
+    }, true);
+    return result && result.indexOf("nothing to commit, working tree clean") == -1;
+}
+
+async function git_commit(cwd, msg) {
+    await exec_shell({
+        cmd: "git add .",
+        cwd: cwd
+    }, true);
+    await exec_shell({
+        cmd: "git commit -m".split(" ").concat(msg),
+        cwd: cwd,
+    });
+}
+
+async function git_push(cwd, options) {
+    let remote = options.remote || (await exec_shell({cmd: "git remote", capture_stdout: true})).trim();
+    let branch = options.branch || (await exec_shell({cmd: "git symbolic-ref --short -q HEAD", capture_stdout: true})).trim();
+    if (!remote || !branch) {
+        warn(`在 ${cwd} 中未找到 remote 和 branch, 无法 执行 git push`);
+        return;
+    }
+    let cmd = "git push";
+    if (options.remote && options.branch) {
+        cmd = `git push --set-upstream ${options.remote} ${options.branch}`;
+    }
+    await exec_shell({
+        cmd: cmd,
+        cwd: cwd,
+        shell: true,
+    });
+}
+
+async function git_export_diff_head(cwd, dst, base, options) {
+    let diff = await exec_shell({
+        // cmd: "git diff HEAD^ HEAD --stat",
+        // cmd: "git diff --name-only HEAD~1",
+        cmd: "git diff --name-only HEAD^",
+        cwd: cwd,
+        capture_stdout: true,
+    }, true);
+    let git_root = await exec_shell({
+        cmd: "git rev-parse --show-toplevel",
+        cwd: cwd,
+        capture_stdout: true,
+    }, true);
+    git_root = git_root.trim();
+    if (!git_root) {
+        throw ('git根目录未找到');
+    }
+    /** @type {string[]} */
+    let lines = diff.split("\n");
+    /** @type {string[]} */
+    let files = [];
+    for (let line of lines) {
+        let raw_file = line.trim();
+        if (!raw_file)
+            break;
+        let file = path.join(git_root, raw_file);
+        if (fs.existsSync(file)) {
+            files.push(file);
+        }
+    }
+    await copy(files, dst, base, options);
+}
+
+async function svn_export_diff_head(cwd, dst, base, options) {
+    cwd = get_full_path(cwd);
+    dst = get_full_path(dst);
+    let diff = await exec_shell({
+        cmd: 'svn diff -r PREV:HEAD --summarize',
+        cwd: cwd,
+        capture_stdout: true,
+    }, true);
+    let lines = diff.split('\n');
+    let files = lines.filter(v => v).map(v => path.join(cwd, /\w\s+(\S+)/.exec(v)[1]));
+    await copy(files, dst, base, options);
+}
+
+
+module.exports = {
+    svn_is_dirty,
+    git_is_dirty,
+    git_commit,
+    git_push,
+    git_export_diff_head,
+    svn_export_diff_head,
+};