node-automator 1.0.0

package/commands/shell.js

@@ -0,0 +1,13 @@
+const { exec_shell } = require('../utils/shell_tool');
+const { BaseCommand } = require('./base');
+
+class ShellCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        return await exec_shell(data);
+    }
+}
+
+module.exports = {
+    ShellCommand,
+};

package/commands/sleep.js

@@ -0,0 +1,17 @@
+const { BaseCommand } = require("./base");
+
+class SleepCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        const timeout = data.timeout;
+        await new Promise(resolve => {
+            setTimeout(() => {
+                resolve();
+            }, timeout);
+        });
+    }
+}
+
+module.exports = {
+    SleepCommand,
+};

package/commands/store.js

@@ -0,0 +1,13 @@
+const { BaseCommand } = require("./base");
+
+class StoreCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        Object.assign(this.shareData, data);
+        return data;
+    }
+}
+
+module.exports = {
+    StoreCommand,
+};

package/commands/store_optional.js

@@ -0,0 +1,33 @@
+const { number } = require('yargs');
+const { BaseCommand } = require("./base");
+
+class StoreOptionalCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        for (let k in data) {
+            const presentVal = this.shareData[k];
+            if (presentVal == null) {
+                this.shareData[k] = data[k];
+            } else {
+                const expected = typeof data[k];
+                const present = typeof presentVal;
+                if (expected !== present) {
+                    if (expected === "number") {
+                        this.shareData[k] = parseFloat(presentVal);
+                    } else if (expected === "boolean") {
+                        if (presentVal && presentVal !== "false" && presentVal !== "0") {
+                            this.shareData[k] = true;
+                        } else {
+                            this.shareData[k] = false;
+                        }
+                    }
+                }
+            }
+        }
+        return data;
+    }
+}
+
+module.exports = {
+    StoreOptionalCommand,
+};

package/commands/stream.js

@@ -0,0 +1,16 @@
+const { BaseCommand } = require("./base");
+const fs = require("fs");
+const { get_fst_file, get_full_path } = require("../utils/file_tool");
+class StreamCommand extends BaseCommand {
+    async execute() {
+        if (this.selfData.type == "write") {
+            return fs.createWriteStream(get_full_path(this.selfData.dst));
+        } else {
+            return fs.createReadStream(get_fst_file(this.selfData.src));
+        }
+    }
+}
+
+module.exports = {
+    StreamCommand,
+};

package/commands/stringify.js

@@ -0,0 +1,24 @@
+const { stringify } = require("../utils/file_tool");
+const { BaseCommand } = require("./base");
+
+class StringifyCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        let content = this.content;
+        return stringify(content, data.type, data.options);
+    }
+
+    getRequireContent() {
+        return true;
+    }
+
+    getRequiredParams() {
+        return [
+            "type",
+        ];
+    }
+}
+
+module.exports = {
+    StringifyCommand,
+};

package/commands/svn_add.js

@@ -0,0 +1,26 @@
+const { exec_shell } = require("../utils/shell_tool");
+const { BaseCommand } = require("./base");
+const { shareData } = require('./share_data');
+
+class SvnAddCommand extends BaseCommand {
+  async execute() {
+    let data = this.selfData;
+    const status = await exec_shell({
+      cmd: 'svn status',
+      cwd: data.cwd,
+      capture_stdout: true,
+    });
+    for (const line of shareData.processPipes(status, "splitlines|remove_empty")) {
+      if (line[0] != "?") continue;
+      const status_path = line.substring(8);
+      await exec_shell({
+        ...data,
+        cmd: `svn add "${status_path}"`
+      });
+    }
+  }
+}
+
+module.exports = {
+  SvnAddCommand,
+};

package/commands/svn_commit.js

@@ -0,0 +1,49 @@
+const { whisper, info } = require("../utils/log_tool");
+const { svn_is_dirty } = require("../utils/scm_tool");
+const { exec_shell } = require("../utils/shell_tool");
+const { BaseCommand } = require("./base");
+
+class SvnCommitCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        if (!data.force) {
+            let is_dirty = await svn_is_dirty(data.cwd);
+            if (!is_dirty) {
+                whisper(`没有要svn提交的更改 ${data.cwd} ${data.msg}`);
+                return;
+            } else {
+                if (data.remove_missing) {
+                    let missingFiles = is_dirty.split(/[\r\n]{1,2}/).filter(v => v && v[0] == "!").map(v => v.split(/\s+/)[1]);
+                    if (missingFiles.length) {
+                        info(`删除svn丢失文件 ${data.cwd}`);
+                        await exec_shell({
+                            cmd: "svn rm".split(" ").concat(missingFiles),
+                            cwd: data.cwd,
+                        });
+                    }
+                    // 移除文件时要update
+                    await exec_shell({
+                        cmd: "svn update",
+                        cwd: data.cwd,
+                    });
+                }
+                info(`准备svn提交 ${data.cwd} ${data.msg}`);
+                if (data.credential) {
+                    await exec_shell({
+                        cmd: `svn commit --username ${data.credential.username} --password ${data.credential.password}  -m`.split(" ").concat(data.msg),
+                        cwd: data.cwd,
+                    });
+                } else {
+                    await exec_shell({
+                        cmd: "svn commit -m".split(" ").concat(data.msg),
+                        cwd: data.cwd,
+                    });
+                }
+            }
+        }
+    }
+}
+
+module.exports = {
+    SvnCommitCommand,
+};

package/commands/svn_export_diff_head.js

@@ -0,0 +1,13 @@
+const { svn_export_diff_head } = require("../utils/scm_tool");
+const { BaseCommand } = require("./base");
+
+class SVNExportDiffHeadCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        await svn_export_diff_head(data.cwd, data.dst, data.base, data.options);
+    }
+}
+
+module.exports = {
+    SVNExportDiffHeadCommand,
+};

package/commands/svn_is_dirty.js

@@ -0,0 +1,13 @@
+const { svn_is_dirty } = require("../utils/scm_tool");
+const { BaseCommand } = require("./base");
+
+class SvnIsDirtyCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        return await svn_is_dirty(data.cwd);
+    }
+}
+
+module.exports = {
+    SvnIsDirtyCommand,
+};

package/commands/svn_update.js

@@ -0,0 +1,27 @@
+const { exec_shell } = require("../utils/shell_tool");
+const { BaseCommand } = require("./base");
+
+class SvnUpdateCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        if (data.credential) {
+            return await exec_shell({
+                cmd: `svn update --username ${data.credential.username} --password ${data.credential.password}`,
+                cwd: data.cwd,
+                encoding: data.encoding,
+                capture_stdout: data.capture_stdout,
+            });
+        } else {
+            return await exec_shell({
+                cmd: "svn update",
+                cwd: data.cwd,
+                encoding: data.encoding,
+                capture_stdout: data.capture_stdout,
+            });
+        }
+    }
+}
+
+module.exports = {
+    SvnUpdateCommand,
+};

package/commands/switch.js

@@ -0,0 +1,18 @@
+const { info } = require("../utils/log_tool");
+const { BaseCommand } = require("./base");
+
+class SwitchCommand extends BaseCommand {
+    async execute() {
+        let commands_cfg = await this.forward("select", {
+            content: this.selfData.commands_cfgs,
+            preview: "title",
+            quiet: true,
+        });
+        info(`执行命令 ${commands_cfg.title}`);
+        await this.exec(commands_cfg.commands);
+    }
+}
+
+module.exports = {
+    SwitchCommand,
+};

package/commands/temp.js

@@ -0,0 +1,12 @@
+const { BaseCommand } = require("./base");
+const temp = require("temp");
+class TempCommand extends BaseCommand {
+    async execute() {
+        var openFile = temp.openSync(this.selfData.affixes);
+        return openFile.path;
+    }
+}
+
+module.exports = {
+    TempCommand,
+};

package/commands/tencent_cos.js

@@ -0,0 +1,323 @@
+const {
+    get_file_list,
+    get_full_path,
+    format_bytes
+} = require("../utils/file_tool");
+const {
+    success,
+    getPrint,
+    info,
+    warn,
+    whisper
+} = require("../utils/log_tool");
+const {
+    BaseCommand
+} = require("./base");
+const path = require("path");
+const {
+    progress
+} = require("../utils/display_tool");
+const {
+    createWriteStream, readFileSync
+} = require("fs");
+const {
+    getCache,
+    setCache
+} = require("../utils/cache_tool");
+const crypto = require("crypto");
+
+
+class TencentCosCommand extends BaseCommand {
+    async execute() {
+
+        let config = this.selfData.config;
+        let mode = this.selfData.mode;
+
+        return await new Promise((resolve, reject) => {
+
+            // 引入模块
+            var COS = require('cos-nodejs-sdk-v5');
+            // 创建实例
+            var cos = new COS({
+                SecretId: config.secret_id,
+                SecretKey: config.secret_key,
+            });
+
+            // 存储桶名称，由bucketname-appid 组成，appid必须填入，可以在COS控制台查看存储桶名称。 https://console.cloud.tencent.com/cos5/bucket
+            var Bucket = `${config.bucket}-${config.appid}`;
+            // 存储桶Region可以在COS控制台指定存储桶的概览页查看 https://console.cloud.tencent.com/cos5/bucket/ 
+            // 关于地域的详情见 https://cloud.tencent.com/document/product/436/6224
+            var Region = config.region;
+
+            switch (mode) {
+                case "VIEW": {
+                    const prefix = this.selfData.src || "";
+                    let keys = [];
+                    /**
+                     * 
+                     * @param {COS.GetBucketResult} ret 
+                     */
+                    function getPartsComplete(data) {
+                        keys.push(...data.Contents.map(v => v.Key));
+                        progress(keys.length, 0, {
+                            desc: data.NextMarker || "[FINISH]",
+                            depth: 0,
+                            color: "gray",
+                        });
+                        if (data.NextMarker) {
+                            getParts(data.NextMarker);
+                            return;
+                        }
+                        info("");
+                        resolve(keys);
+                    }
+
+                    function getParts(marker = null) {
+                        cos.getBucket({
+                            Bucket: Bucket,
+                            Region: Region,
+                            MaxKeys: 1000,
+                            Marker: marker,
+                            Prefix: prefix || ""
+                        }, (err, data) => {
+                            if (err) {
+                                reject(getPrint(err, true));
+                                return;
+                            }
+                            getPartsComplete(data);
+                        });
+                    }
+                    getParts();
+                    break;
+                }
+                case "DOWNLOAD": {
+                    let src = this.selfData.src;
+                    let dst = get_full_path(this.selfData.dst);
+                    cos.getObject({
+                        Bucket: Bucket,
+                        Region: Region,
+                        Key: src,
+                        onProgress: function (progressData) {
+                            progress(progressData.loaded, progressData.total, {
+                                desc: `Downloading from ${src}  ${format_bytes(progressData.speed)}/s`,
+                                depth: 0,
+                                color: "cyan",
+                                align: true,
+                                format_bytes: true,
+                            }, true);
+                        },
+                        Output: createWriteStream(dst)
+                    }, (err, data) => {
+                        if (err) {
+                            reject(getPrint(err, true));
+                            return;
+                        }
+                        resolve(data);
+                    });
+                    break;
+                }
+                case "UPLOAD_FILE":
+                case "UPLOAD": {
+                    let dst = this.selfData.dst;
+                    let src = get_full_path(this.selfData.src);
+                    info("");
+                    info(`开始上传 ${src} 到 ${dst}`);
+                    // 分片上传
+                    cos.sliceUploadFile({
+                        Bucket: Bucket,
+                        Region: Region,
+                        Key: dst,
+                        onProgress: (progressInfo) => {
+                            progress(progressInfo.loaded, progressInfo.total, {
+                                desc: `Uploading to ${dst} ${format_bytes(progressInfo.speed)}/s`,
+                                depth: 0,
+                                color: "green",
+                                format_bytes: true,
+                                align: true,
+                            }, true);
+                        },
+                        FilePath: src // 本地文件地址，需自行替换
+                    }, function (err, data) {
+                        if (err) {
+                            reject(getPrint(err, true));
+                            return;
+                        }
+                        resolve(data);
+                    });
+                    break;
+                }
+                case "UPLOAD_FILES": {
+                    let dst_folder = this.selfData.dst;
+                    let strategy = this.selfData.options && this.selfData.options.strategy || "lazy";
+                    let concurrency = this.selfData.options && this.selfData.options.concurrency || 10;
+                    let base = this.selfData.base;
+                    whisper(`在 ${base} 中收集要上传的文件...`, undefined, true);
+                    let srcs = get_file_list(this.selfData.src, undefined, true);
+                    let cacheKey = `tencent_cos/${Bucket}/${dst_folder}`.replace(/\//g, ".");
+                    let cacheType = this.selfData.cache_type;
+                    let cacheParam = this.selfData.cache_param;
+                    let cachedFiles = null;
+                    getCache(cacheKey, cacheType, cacheParam).then((_cachedFiles) => {
+                        cachedFiles = _cachedFiles;
+                        info(`cacheKey=${cacheKey}, cacheType=${cacheType}, cacheParam=${cacheParam},cachedFiles=${JSON.stringify(cachedFiles).substring(0, 20)}`);
+                        if (!cachedFiles || !Object.keys(cachedFiles).length) {
+                            const prefix = dst_folder;
+                            let keys = {};
+                            /**
+                             * 
+                             * @param {COS.GetBucketResult} data 
+                             */
+                            function getPartsComplete(data) {
+                                data.Contents.forEach(obj => {
+                                    if (!keys[obj.Key]) {
+                                        keys[obj.Key] = !!obj;// obj.ETag;
+                                    }
+                                });
+                                let size = Object.keys(keys).length;
+                                progress(size, size, {
+                                    desc: `获取已上传文件 ${data.NextMarker || ""}`,
+                                    depth: 0,
+                                    color: "gray",
+                                });
+                                if (data.NextMarker) {
+                                    getParts(data.NextMarker);
+                                    return;
+                                }
+                                info("");
+                                cachedFiles = keys;
+                                setCache(cacheKey, cachedFiles, cacheType, cacheParam);
+                                readyForUpload();
+                            }
+    
+                            function getParts(marker = null) {
+                                cos.getBucket({
+                                    Bucket: Bucket,
+                                    Region: Region,
+                                    MaxKeys: 1000,
+                                    Marker: marker,
+                                    Prefix: prefix || ""
+                                }, (err, data) => {
+                                    if (err) {
+                                        reject(getPrint(err, true));
+                                        return;
+                                    }
+                                    getPartsComplete(data);
+                                });
+                            }
+                            getParts();
+                        } else {
+                            readyForUpload();
+                        }
+                    });
+
+                    function readyForUpload() {
+                        let rawLength = srcs.length;
+                        // 过滤文件
+                        srcs = srcs.filter(src => {
+                            let cachedValue = cachedFiles[path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")];
+                            switch(strategy) {
+                                case "lazy": {
+                                    if (cachedValue) {
+                                        return false;
+                                    }
+                                    break;
+                                }
+                                case "hash": {
+                                    let hash = crypto.createHash("md5").update(readFileSync(src)).digest("hex");
+                                    if (cachedValue == hash) {
+                                        return false;
+                                    }
+                                    break;
+                                }
+                                case "force": {
+                                    
+                                    break;
+                                }
+                                default: {
+                                    throw `Unkown Strategy ${strategy}`;
+                                    break;
+                                }
+                            }
+                            return true;
+                        });
+                        info(`预处理完成!`, "\r");
+                        info(`实际 ${srcs.length}, 忽略 ${rawLength - srcs.length}, 共计 ${rawLength}`);
+                        let from = 0;
+                        function uploadRange(from = 0) {
+                            let tmpSrcs = srcs.slice(from, from + concurrency);
+                            if (tmpSrcs.length == 0) {
+                                resolve(true);
+                                return;
+                            }
+                            let left = tmpSrcs.length;
+                            let files = tmpSrcs.map((src, index) => {
+                                let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/");
+                                let speed = 0;
+                                return {
+                                    Bucket: Bucket,
+                                    Region: Region,
+                                    Key: dst,
+                                    FilePath: src,
+                                    onProgress: (progressInfo) => {
+                                        speed = progressInfo.speed;
+                                        // progress(index + from, srcs.length, {
+                                        //     desc: `Uploading to ${dst} ${progressInfo.speed ? format_bytes(progressInfo.speed) + "/s" : ""}`,
+                                        //     depth: 0,
+                                        //     // format_bytes: true,
+                                        //     color: "green",
+                                        //     align: true,
+                                        // });
+                                    },
+                                    onFileFinish: (err, data) => {
+                                        if (err) {
+                                            // setCache(cacheKey, cachedFiles, cacheType, cacheParam);
+                                            // reject(getPrint(err, true));
+                                            return;
+                                        }
+                                        --left;
+                                        progress(from + (tmpSrcs.length - left), srcs.length, {
+                                            desc: `Uploaded to ${dst} ${format_bytes(speed)}/s`,
+                                            depth: 0,
+                                            // format_bytes: true,
+                                            color: "green",
+                                            align: true,
+                                        }, true);
+                                        // 当前上传的，存储为 hash
+                                        cachedFiles[dst] = crypto.createHash("md5").update(readFileSync(src)).digest("hex");
+                                        if (left == 0) {
+                                            setCache(cacheKey, cachedFiles, cacheType, cacheParam);
+                                            uploadRange(from + concurrency);
+                                        }
+                                    }
+                                };
+                            });
+                            cos.uploadFiles({
+                                files: files,
+                                onProgress: (progressInfo) => {
+                                },
+                                onFileFinish: (err, data) => {
+                                    if (err) {
+                                        setCache(cacheKey, cachedFiles, cacheType, cacheParam);
+                                        reject(getPrint(err, true));
+                                        return;
+                                    }
+                                }
+                            });
+                        }
+
+                        uploadRange();
+                    }
+                    break;
+                }
+                default: {
+                    resolve(false);
+                    break;
+                }
+            }
+        });
+    }
+}
+
+module.exports = {
+    TencentCosCommand,
+};

package/commands/test.js

@@ -0,0 +1,14 @@
+const transform_tool = require("../utils/transform_tool");
+
+const { BaseCommand } = require("./base");
+
+class TestCommand extends BaseCommand {
+    async execute() {
+        let data = this.selfData;
+        await this.forward(data.type, data.data);
+    }
+}
+
+module.exports = {
+    TestCommand,
+};

package/commands/title.js

@@ -0,0 +1,18 @@
+const { BaseCommand } = require("./base");
+class TitleCommand extends BaseCommand {
+    async execute() {
+      this.shareData.TITLE = this.content;
+      await this.forward("shell", {
+        cmd: ["title", this.content],
+        shell: true,
+      });
+    }
+
+    getRequireContent() {
+      return true;
+    }
+}
+
+module.exports = {
+    TitleCommand,
+};

package/commands/tmpdir.js

@@ -0,0 +1,21 @@
+const path = require("path");
+const fs = require("fs");
+const os = require("os");
+const { BaseCommand } = require("./base");
+
+class TmpdirCommand extends BaseCommand {
+    async execute() {
+        return fs.mkdtempSync(path.join(os.tmpdir(), this.selfData.name));
+    }
+
+
+    getRequiredParams() {
+        return [
+            "name",
+        ];
+    }
+}
+
+module.exports = {
+    TmpdirCommand,
+};