node-automator 1.3.4 → 1.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/commands/ali_oss.js +78 -30
  2. package/package.json +1 -1
package/commands/ali_oss.js CHANGED
@@ -43,7 +43,6 @@ class AliOssCommand extends BaseCommand {
43
43
  accessKeySecret: config.access_key_secret, // 确保已设置环境变量OSS_ACCESS_KEY_SECRET。
44
44
  bucket: config.bucket, // 示例：'my-bucket-name'，填写存储空间名称。
45
45
  });
46
-
47
46
  switch (mode) {
48
47
  case "DOWNLOAD": {
49
48
  let src = this.selfData.src;
@@ -79,44 +78,93 @@ class AliOssCommand extends BaseCommand {
79
78
  let dst_folder = this.selfData.dst;
80
79
  let concurrency = this.selfData.options && this.selfData.options.concurrency || 10;
81
80
  let timeout = this.selfData.options && this.selfData.options.timeout || 1000 * 60 * 2;
81
+ let strategy = this.selfData.options && this.selfData.options.strategy || "hash";
82
82
  let base = this.selfData.base;
83
83
  whisper(`在 ${base} 中收集要上传的文件...`, undefined, true);
84
84
  let srcs = get_file_list(this.selfData.src, undefined, true);
85
85
  let retryTimes = 0;
86
86
  let maxRetryTimes = 3;
87
- function uploadFiles(srcs) {
88
- let tasks = srcs.map((src) => {
89
- let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")
90
- return client.put.bind(client, dst, src, {
91
- timeout,
87
+ let cacheKey = `ali_oss/${config.bucket}/${dst_folder}`.replace(/\//g, ".");
88
+ let cacheType = this.selfData.cache_type;
89
+ let cacheParam = this.selfData.cache_param;
90
+ let cachedFiles = null;
91
+ getCache(cacheKey, cacheType, cacheParam).then((_cachedFiles) => {
92
+ cachedFiles = _cachedFiles || {};
93
+ info(`cacheKey=${cacheKey}, cacheType=${cacheType}, cacheParam=${cacheParam},cachedFiles=${JSON.stringify(cachedFiles).substring(0, 20)}`);
94
+ function uploadFiles(srcs) {
95
+ let rawLength = srcs.length;
96
+ // 过滤文件
97
+ srcs = srcs.filter(src => {
98
+ let cachedValue = cachedFiles[path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")];
99
+ switch(strategy) {
100
+ case "lazy": {
101
+ if (cachedValue) {
102
+ return false;
103
+ }
104
+ break;
105
+ }
106
+ case "hash": {
107
+ let hash = crypto.createHash("md5").update(readFileSync(src)).digest("hex");
108
+ if (cachedValue == hash) {
109
+ return false;
110
+ }
111
+ break;
112
+ }
113
+ case "force": {
114
+
115
+ break;
116
+ }
117
+ default: {
118
+ throw `Unkown Strategy ${strategy}`;
119
+ break;
120
+ }
121
+ }
122
+ return true;
92
123
  });
93
- });
94
- queueAsync(tasks, concurrency, (done, total, index, indexDone) => {
95
- progress(done, total, {
96
- desc: path.relative(base, srcs[index]) + (indexDone ? " 上传中..." : " 上传成功!"),
97
- depth: 0,
124
+ info(`预处理完成!`, "\r");
125
+ info(`实际 ${srcs.length}, 忽略 ${rawLength - srcs.length}, 共计 ${rawLength}`);
126
+
127
+ let tasks = srcs.map((src) => {
128
+ let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")
129
+ return client.put.bind(client, dst, src, {
130
+ timeout,
131
+ });
98
132
  });
99
- }).then((result) => {
100
- var retrySrcs = [];
101
- for(var i = 0; i < result.length; i++) {
102
- if (result[i].err) {
103
- retrySrcs.push(srcs[i]);
104
- warn(`上传文件 ${path.relative(base, srcs[i])} 失败: ${result[i].err}`);
133
+ queueAsync(tasks, concurrency, (done, total, index, indexDone) => {
134
+ const src = srcs[index];
135
+ if (done) {
136
+ // 当前上传的，存储为 hash
137
+ let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")
138
+ cachedFiles[dst] = crypto.createHash("md5").update(readFileSync(src)).digest("hex");
105
139
  }
106
- }
107
- if (retrySrcs.length > 0) {
108
- retryTimes++;
109
- if (retryTimes > maxRetryTimes) {
110
- reject(`上传文件失败，重试次数超过 ${maxRetryTimes} 次。`);
111
- return;
140
+ progress(done, total, {
141
+ desc: path.relative(base, src) + " " + (indexDone ? "上传成功!" : "上传中..."),
142
+ depth: 0,
143
+ });
144
+ }).then((result) => {
145
+ var retrySrcs = [];
146
+ for(var i = 0; i < result.length; i++) {
147
+ if (result[i].err) {
148
+ retrySrcs.push(srcs[i]);
149
+ warn(`上传文件 ${path.relative(base, srcs[i])} 失败: ${result[i].err}`);
150
+ }
112
151
  }
113
- uploadFiles(retrySrcs);
114
- } else {
115
- resolve(true);
116
- }
117
- });
118
- }
119
- uploadFiles(srcs);
152
+ if (retrySrcs.length > 0) {
153
+ retryTimes++;
154
+ if (retryTimes > maxRetryTimes) {
155
+ setCache(cacheKey, cachedFiles, cacheType, cacheParam);
156
+ reject(`上传文件失败，重试次数超过 ${maxRetryTimes} 次。`);
157
+ return;
158
+ }
159
+ uploadFiles(retrySrcs);
160
+ } else {
161
+ setCache(cacheKey, cachedFiles, cacheType, cacheParam);
162
+ resolve(true);
163
+ }
164
+ });
165
+ }
166
+ uploadFiles(srcs);
167
+ })
120
168
  break;
121
169
  }
122
170
  default: {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "node-automator",
3
- "version": "1.3.4",
3
+ "version": "1.3.6",
4
4
  "description": "Execute automation with yaml configuration(compatible with json)",
5
5
  "main": "index.js",
6
6
  "repository": {