node-automator 1.3.5 → 1.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/commands/ali_oss.js +102 -30
  2. package/package.json +1 -1
package/commands/ali_oss.js CHANGED
@@ -43,7 +43,6 @@ class AliOssCommand extends BaseCommand {
43
43
  accessKeySecret: config.access_key_secret, // 确保已设置环境变量OSS_ACCESS_KEY_SECRET。
44
44
  bucket: config.bucket, // 示例：'my-bucket-name'，填写存储空间名称。
45
45
  });
46
-
47
46
  switch (mode) {
48
47
  case "DOWNLOAD": {
49
48
  let src = this.selfData.src;
@@ -79,44 +78,117 @@ class AliOssCommand extends BaseCommand {
79
78
  let dst_folder = this.selfData.dst;
80
79
  let concurrency = this.selfData.options && this.selfData.options.concurrency || 10;
81
80
  let timeout = this.selfData.options && this.selfData.options.timeout || 1000 * 60 * 2;
81
+ let strategy = this.selfData.options && this.selfData.options.strategy || "hash";
82
82
  let base = this.selfData.base;
83
83
  whisper(`在 ${base} 中收集要上传的文件...`, undefined, true);
84
84
  let srcs = get_file_list(this.selfData.src, undefined, true);
85
85
  let retryTimes = 0;
86
86
  let maxRetryTimes = 3;
87
- function uploadFiles(srcs) {
88
- let tasks = srcs.map((src) => {
89
- let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")
90
- return client.put.bind(client, dst, src, {
91
- timeout,
87
+ let cacheKey = `ali_oss/${config.bucket}/${dst_folder}`.replace(/\//g, ".");
88
+ let cacheType = this.selfData.cache_type;
89
+ let cacheParam = this.selfData.cache_param;
90
+ let cachedFiles = null;
91
+ getCache(cacheKey, cacheType, cacheParam).then((_cachedFiles) => {
92
+ cachedFiles = _cachedFiles;
93
+ info(`cacheKey=${cacheKey}, cacheType=${cacheType}, cacheParam=${cacheParam},cachedFiles=${JSON.stringify(cachedFiles).substring(0, 20)}`);
94
+ if (!cachedFiles || !Object.keys(cachedFiles).length) {
95
+ let marker = null;
96
+ var objs = [];
97
+ async function list () {
98
+ do {
99
+ const result = await client.list({
100
+ prefix: dst_folder + "/",
101
+ marker: marker,
102
+ });
103
+ marker = result.nextMarker;
104
+ objs.push.apply(objs, result.objects);
105
+ } while (marker);
106
+ return objs;
107
+ }
108
+ list().then(objs => {
109
+ cachedFiles = {};
110
+ objs.forEach(obj => {
111
+ cachedFiles[obj.name] = obj.etag.replaceAll("\"", "").toLowerCase();
112
+ });
113
+ setCache(cacheKey, cachedFiles, cacheType, cacheParam);
114
+ uploadFiles(srcs);
92
115
  });
93
- });
94
- queueAsync(tasks, concurrency, (done, total, index, indexDone) => {
95
- progress(done, total, {
96
- desc: path.relative(base, srcs[index]) + " " + (indexDone ? "上传成功!" : "上传中..."),
97
- depth: 0,
116
+ } else {
117
+ uploadFiles(srcs);
118
+ }
119
+ function uploadFiles(srcs) {
120
+ let rawLength = srcs.length;
121
+ // 过滤文件
122
+ srcs = srcs.filter(src => {
123
+ let cachedValue = cachedFiles[path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")];
124
+ switch(strategy) {
125
+ case "lazy": {
126
+ if (cachedValue) {
127
+ return false;
128
+ }
129
+ break;
130
+ }
131
+ case "hash": {
132
+ let hash = crypto.createHash("md5").update(readFileSync(src)).digest("hex");
133
+ if (cachedValue == hash) {
134
+ return false;
135
+ }
136
+ break;
137
+ }
138
+ case "force": {
139
+
140
+ break;
141
+ }
142
+ default: {
143
+ throw `Unkown Strategy ${strategy}`;
144
+ break;
145
+ }
146
+ }
147
+ return true;
148
+ });
149
+ info(`预处理完成!`, "\r");
150
+ info(`实际 ${srcs.length}, 忽略 ${rawLength - srcs.length}, 共计 ${rawLength}`);
151
+
152
+ let tasks = srcs.map((src) => {
153
+ let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")
154
+ return client.put.bind(client, dst, src, {
155
+ timeout,
156
+ });
98
157
  });
99
- }).then((result) => {
100
- var retrySrcs = [];
101
- for(var i = 0; i < result.length; i++) {
102
- if (result[i].err) {
103
- retrySrcs.push(srcs[i]);
104
- warn(`上传文件 ${path.relative(base, srcs[i])} 失败: ${result[i].err}`);
158
+ queueAsync(tasks, concurrency, (done, total, index, indexDone) => {
159
+ const src = srcs[index];
160
+ if (done) {
161
+ // 当前上传的，存储为 hash
162
+ let dst = path.join(dst_folder, path.relative(base, src)).replace(/\\/g, "/")
163
+ cachedFiles[dst] = crypto.createHash("md5").update(readFileSync(src)).digest("hex");
105
164
  }
106
- }
107
- if (retrySrcs.length > 0) {
108
- retryTimes++;
109
- if (retryTimes > maxRetryTimes) {
110
- reject(`上传文件失败，重试次数超过 ${maxRetryTimes} 次。`);
111
- return;
165
+ progress(done, total, {
166
+ desc: path.relative(base, src) + " " + (indexDone ? "上传成功!" : "上传中..."),
167
+ depth: 0,
168
+ });
169
+ }).then((result) => {
170
+ var retrySrcs = [];
171
+ for(var i = 0; i < result.length; i++) {
172
+ if (result[i].err) {
173
+ retrySrcs.push(srcs[i]);
174
+ warn(`上传文件 ${path.relative(base, srcs[i])} 失败: ${result[i].err}`);
175
+ }
112
176
  }
113
- uploadFiles(retrySrcs);
114
- } else {
115
- resolve(true);
116
- }
117
- });
118
- }
119
- uploadFiles(srcs);
177
+ if (retrySrcs.length > 0) {
178
+ retryTimes++;
179
+ if (retryTimes > maxRetryTimes) {
180
+ setCache(cacheKey, cachedFiles, cacheType, cacheParam);
181
+ reject(`上传文件失败，重试次数超过 ${maxRetryTimes} 次。`);
182
+ return;
183
+ }
184
+ uploadFiles(retrySrcs);
185
+ } else {
186
+ setCache(cacheKey, cachedFiles, cacheType, cacheParam);
187
+ resolve(true);
188
+ }
189
+ });
190
+ }
191
+ })
120
192
  break;
121
193
  }
122
194
  default: {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "node-automator",
3
- "version": "1.3.5",
3
+ "version": "1.3.7",
4
4
  "description": "Execute automation with yaml configuration(compatible with json)",
5
5
  "main": "index.js",
6
6
  "repository": {