node-automator 1.2.0 → 1.3.0

package/commands/ali_oss.js

@@ -0,0 +1,125 @@
+const {
+    get_file_list,
+    get_full_path,
+    format_bytes
+} = require("../utils/file_tool");
+const {
+    success,
+    getPrint,
+    info,
+    warn,
+    whisper
+} = require("../utils/log_tool");
+const {
+    BaseCommand
+} = require("./base");
+const path = require("path");
+const {
+    progress
+} = require("../utils/display_tool");
+const {
+    createWriteStream, readFileSync
+} = require("fs");
+const {
+    getCache,
+    setCache
+} = require("../utils/cache_tool");
+const crypto = require("crypto");
+const { queueAsync } = require('../utils/func_tool');
+
+
+class AliOssCommand extends BaseCommand {
+    async execute() {
+
+        let config = this.selfData.config;
+        let mode = this.selfData.mode;
+
+        return await new Promise((resolve, reject) => {
+            const OSS = require('ali-oss');
+            // 初始化OSS客户端。请将以下参数替换为您自己的配置信息。
+            const client = new OSS({
+                region: config.region, // 示例：'oss-cn-hangzhou'，填写Bucket所在地域。
+                accessKeyId: config.access_key_id, // 确保已设置环境变量OSS_ACCESS_KEY_ID。
+                accessKeySecret: config.access_key_secret, // 确保已设置环境变量OSS_ACCESS_KEY_SECRET。
+                bucket: config.bucket, // 示例：'my-bucket-name'，填写存储空间名称。
+            });
+
+            switch (mode) {
+                case "DOWNLOAD": {
+                    let src = this.selfData.src;
+                    let dst = get_full_path(this.selfData.dst);
+                    client.get(src, dst).then((result) => {
+                        resolve(result);
+                    }).catch((err) => {
+                        reject(err);
+                    });
+                    break;
+                }
+                case "UPLOAD_FILE":
+                case "UPLOAD": {
+                    let dst = this.selfData.dst;
+                    let src = get_full_path(this.selfData.src);
+                    info("");
+                    info(`开始上传 ${src} 到 ${dst}`);
+                    client.put(dst, src).then((result) => {
+                        resolve(result);
+                    })
+                    .catch((err) => {
+                        reject(err);
+                    });
+                    break;
+                }
+                case "UPLOAD_FILES": {
+                    let dst_folder = this.selfData.dst;
+                    let concurrency = this.selfData.options && this.selfData.options.concurrency || 10;
+                    let base = this.selfData.base;
+                    whisper(`在 ${base} 中收集要上传的文件...`, undefined, true);
+                    let srcs = get_file_list(this.selfData.src, undefined, true);
+                    let retryTimes = 0;
+                    let maxRetryTimes = 3;
+                    function uploadFiles(srcs) {
+                        let tasks = srcs.map((src) => {
+                            let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")
+                            return client.put.bind(client, dst, src);
+                        });
+                        queueAsync(tasks, concurrency, (done, total) => {
+                            progress(done, total, {
+                                desc: "上传中...",
+                                depth: 0,
+                                color: "gray",
+                            });
+                        }).then((result) => {
+                            var retrySrcs = [];
+                            for(var i = 0; i < result.length; i++) {
+                                if (result[i].err) {
+                                    retrySrcs.push(srcs[i]);
+                                    warner(`上传文件 ${srcs[i]} 失败: ${result[i].err}`);
+                                }
+                            }
+                            if (retrySrcs.length > 0) {
+                                retryTimes++;
+                                if (retryTimes > maxRetryTimes) {
+                                    reject(`上传文件失败，重试次数超过 ${maxRetryTimes} 次。`);
+                                    return;
+                                }
+                                uploadFiles(retrySrcs);
+                            } else {
+                                resolve(true);
+                            }
+                        });
+                    }
+                    uploadFiles(srcs);
+                    break;
+                }
+                default: {
+                    resolve(false);
+                    break;
+                }
+            }
+        });
+    }
+}
+
+module.exports = {
+    AliOssCommand,
+};

package/commands/mgr.js

@@ -115,6 +115,7 @@ const { ImageCropCommand } = require('./image_crop');
 const { CursorUpCommand } = require('./cursor_up');
 const { RegisterSearchInstallLocationCommand } = require('./register_search_install_location');
 const { DeobfuscateCommand } = require('./deobfuscate');
+const { AliOssCommand } = require('./ali_oss');
 
 const globalData = {
     "executed_cfg": [], // 执行过的配置文件
@@ -471,6 +472,7 @@ function init() {
     register("cursor_up", CursorUpCommand, false);
     register("register_search_install_location", RegisterSearchInstallLocationCommand, false);
     register("deobfuscate", DeobfuscateCommand, false);
+    register("ali_oss", AliOssCommand, false);
 }
 
 module.exports = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-automator",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Execute automation with yaml configuration(compatible with json)",
   "main": "index.js",
   "repository": {
@@ -18,6 +18,7 @@
   "license": "ISC",
   "dependencies": {
     "adm-zip": "^0.5.1",
+    "ali-oss": "^6.20.0",
     "await-event-emitter": "^2.0.2",
     "change-case": "^4.1.2",
     "chardet": "^1.3.0",

package/utils/func_tool.js

@@ -0,0 +1,106 @@
+function queue(fn, limit, trigger, flushTrigger, toggleTrigger) {
+    var argQueues = [];
+    var context = null;
+    var availableCount = limit;
+    var isWaiting = false;
+    var permitAll = false;
+
+    function runQueue() {
+        if (!argQueues.length) return;
+        if (availableCount > 0 || permitAll) {
+            var runNum = permitAll ? argQueues.length : Math.min(availableCount, argQueues.length);
+            var argsToRun = argQueues.splice(0, runNum);
+            argsToRun.forEach(function (item) {
+                fn.apply(context, item);
+            });
+            if (!permitAll) {
+                availableCount -= runNum;
+            }
+        }
+        if (!isWaiting) {
+            isWaiting = true;
+            trigger(reset);
+        }
+    }
+
+    function reset() {
+        isWaiting = false;
+        availableCount = limit;
+        runQueue();
+    }
+
+    function toggle() {
+        permitAll = !permitAll;
+        runQueue();
+    }
+    function flush() {
+        var argsToRun = argQueues.splice(0, argQueues.length);
+        argsToRun.forEach(function (item) {
+            fn.apply(context, item);
+        });
+    }
+    var queued = function () {
+        var args = arguments;
+        argQueues.push(args);
+        context = this;
+        runQueue();
+    };
+    flushTrigger && flushTrigger(flush);
+    toggleTrigger && toggleTrigger(toggle);
+    Object.defineProperty(queued, "status", {
+        get: function() {
+            return {
+                queuedCount: argQueues.length,
+                limit: limit,
+                availableCount: availableCount,
+                isWaiting: isWaiting,
+                permitAll: permitAll,
+            }
+        },
+    });
+    return queued;
+}
+
+function queueAsync(tasks, limit, onProgress) {
+    return new Promise(function (resolve) {
+        var result = [];
+        if (!tasks.length) resolve(result);
+        var index = 0;
+        var numDone = 0;
+        onProgress && onProgress(numDone, tasks.length);
+        function checkDone() {
+            onProgress && onProgress(numDone, tasks.length);
+            if (numDone == tasks.length) {
+                resolve(result);
+                return;
+            }
+            if (index < tasks.length) {
+                _run();
+            }
+        };
+        function _run() {
+            const task = tasks[index++];
+            task().then(function (val) {
+                numDone++;
+                result.push({
+                    val,
+                });
+                checkDone();
+            }).catch(function (err) {
+                numDone++;
+                result.push({
+                    err,
+                });
+                checkDone();
+            });
+        }
+        for(var i = 0; i < tasks.length && i < limit; i++) {
+            _run();
+        }
+    });
+}
+
+module.exports = {
+    queue,
+    queueAsync,
+};

package/utils/request_tool.js

@@ -39,13 +39,14 @@ async function doRequest(data) {
     let quiet = data.quiet;
     !quiet && info(useCache ? "[缓存]" : `[${options.method}] ${url} `);
     let hasProgress = false;
+    let fileDst = get_full_path(data.dst || shareData.AUTOMATOR_SCRATCH + "/request_tool/" + cache_name, "FILE");
     if (useCache) {
         result = http_cache;
+        fs.writeFileSync(fileDst, http_cache);
     } else {
         let chunks = [];
         result = await new Promise(resolve => {
             let sizeTotal = 0;
-            let fileDst = get_full_path(data.dst || shareData.AUTOMATOR_SCRATCH + "/request_tool/" + cache_name, "FILE");
             let cacheDir = path.dirname(cache_path);
             if (!fs.existsSync(cacheDir)) {
                 fs.mkdirSync(cacheDir, {