hw-weapp-compiler 1.0.24 → 1.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/utils/upload.js +67 -37
- package/package.json +2 -2
- package/test/upload-async-client.js +79 -0
package/build/utils/upload.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* 通过增量检测避免重复上传,并展示上传进度。
|
|
6
6
|
*/
|
|
7
7
|
const path = require('path');
|
|
8
|
+
const chalk = require('chalk');
|
|
8
9
|
const OBSClient = require('esdk-obs-nodejs');
|
|
9
10
|
const OSSClient = require('ali-oss');
|
|
10
11
|
const Progress = require('progress');
|
|
@@ -23,6 +24,7 @@ const OBS_PENDING_WARNING_MS = 15 * 1000;
|
|
|
23
24
|
const OBS_TIMEOUT_BUFFER_MS = 5 * 1000;
|
|
24
25
|
|
|
25
26
|
let obsClient;
|
|
27
|
+
let obsClientReady;
|
|
26
28
|
let ossClient;
|
|
27
29
|
let progress;
|
|
28
30
|
let uploadQueue = {};
|
|
@@ -103,18 +105,12 @@ function getObsRuntimeOptions() {
|
|
|
103
105
|
}
|
|
104
106
|
|
|
105
107
|
function createObsRequestLogger(operation, file, key) {
|
|
106
|
-
const startedAt = Date.now();
|
|
107
|
-
const { timeoutSeconds, maxRetryCount, operationTimeoutMs } = getObsRuntimeOptions();
|
|
108
108
|
const displayFile = getDisplayFile(file);
|
|
109
109
|
|
|
110
|
-
console.log(`[OBS] ${operation} started: key=${key}, file=${displayFile}`);
|
|
111
|
-
|
|
112
110
|
const pendingTimer = setTimeout(() => {
|
|
113
111
|
console.warn(
|
|
114
|
-
`[OBS] ${operation}
|
|
115
|
-
`
|
|
116
|
-
`sdkTimeout=${timeoutSeconds}s, maxRetryCount=${maxRetryCount}, ` +
|
|
117
|
-
`operationTimeout=${operationTimeoutMs / 1000}s`,
|
|
112
|
+
`[OBS] ${operation} pending (${OBS_PENDING_WARNING_MS / 1000}s): ` +
|
|
113
|
+
`file=${displayFile}, key=${key}`,
|
|
118
114
|
);
|
|
119
115
|
}, OBS_PENDING_WARNING_MS);
|
|
120
116
|
|
|
@@ -124,19 +120,8 @@ function createObsRequestLogger(operation, file, key) {
|
|
|
124
120
|
}
|
|
125
121
|
|
|
126
122
|
return {
|
|
127
|
-
finish(
|
|
123
|
+
finish() {
|
|
128
124
|
clearTimeout(pendingTimer);
|
|
129
|
-
const elapsed = Date.now() - startedAt;
|
|
130
|
-
const suffix = detail ? `, ${detail}` : '';
|
|
131
|
-
const message = `[OBS] ${operation} ${outcome} in ${elapsed}ms: key=${key}${suffix}`;
|
|
132
|
-
|
|
133
|
-
if (outcome === 'failed') {
|
|
134
|
-
console.error(message);
|
|
135
|
-
} else if (outcome === 'miss') {
|
|
136
|
-
console.warn(message);
|
|
137
|
-
} else {
|
|
138
|
-
console.log(message);
|
|
139
|
-
}
|
|
140
125
|
},
|
|
141
126
|
};
|
|
142
127
|
}
|
|
@@ -149,7 +134,7 @@ function runObsRequest(operation, file, key, request) {
|
|
|
149
134
|
let operationTimer;
|
|
150
135
|
let activeRequest;
|
|
151
136
|
|
|
152
|
-
const finish = (error, result
|
|
137
|
+
const finish = (error, result) => {
|
|
153
138
|
if (settled) {
|
|
154
139
|
return;
|
|
155
140
|
}
|
|
@@ -157,10 +142,7 @@ function runObsRequest(operation, file, key, request) {
|
|
|
157
142
|
settled = true;
|
|
158
143
|
clearTimeout(operationTimer);
|
|
159
144
|
activeRequest = null;
|
|
160
|
-
requestLogger.finish(
|
|
161
|
-
outcome || (error ? 'failed' : 'completed'),
|
|
162
|
-
formatObsError(error || result),
|
|
163
|
-
);
|
|
145
|
+
requestLogger.finish();
|
|
164
146
|
|
|
165
147
|
if (error) {
|
|
166
148
|
reject(error);
|
|
@@ -201,6 +183,37 @@ function runObsRequest(operation, file, key, request) {
|
|
|
201
183
|
});
|
|
202
184
|
}
|
|
203
185
|
|
|
186
|
+
function waitForObsClientReady(client) {
|
|
187
|
+
if (client.util && client.util.signatureContext && client.util.server) {
|
|
188
|
+
return Promise.resolve(client);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const { operationTimeoutMs } = getObsRuntimeOptions();
|
|
192
|
+
const startedAt = Date.now();
|
|
193
|
+
|
|
194
|
+
return new Promise((resolve, reject) => {
|
|
195
|
+
const checkReady = () => {
|
|
196
|
+
if (client.util && client.util.signatureContext && client.util.server) {
|
|
197
|
+
resolve(client);
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (Date.now() - startedAt >= operationTimeoutMs) {
|
|
202
|
+
const error = new Error(
|
|
203
|
+
`OBS client initialization timed out after ${operationTimeoutMs / 1000}s`,
|
|
204
|
+
);
|
|
205
|
+
error.code = 'OBS_CLIENT_INIT_TIMEOUT';
|
|
206
|
+
reject(error);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
setTimeout(checkReady, 10);
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
setTimeout(checkReady, 0);
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
204
217
|
function getObsClient() {
|
|
205
218
|
if (!obsClient) {
|
|
206
219
|
const { timeoutSeconds, maxRetryCount } = getObsRuntimeOptions();
|
|
@@ -209,8 +222,11 @@ function getObsClient() {
|
|
|
209
222
|
timeout: timeoutSeconds,
|
|
210
223
|
max_retry_count: maxRetryCount,
|
|
211
224
|
});
|
|
225
|
+
// esdk-obs-nodejs 3.26.8 将 initFactory 改成了 async,但构造函数没有等待它。
|
|
226
|
+
// 立即请求会在 signatureContext 尚未赋值时破坏 SDK 的 bucket 协商锁。
|
|
227
|
+
obsClientReady = waitForObsClientReady(obsClient);
|
|
212
228
|
}
|
|
213
|
-
return
|
|
229
|
+
return obsClientReady;
|
|
214
230
|
}
|
|
215
231
|
|
|
216
232
|
function getOssClient() {
|
|
@@ -233,10 +249,11 @@ function getOssStat(file) {
|
|
|
233
249
|
return getOssClient().head(getRemoteKey(ossConfig, file));
|
|
234
250
|
}
|
|
235
251
|
|
|
236
|
-
function doObsUpload(file) {
|
|
252
|
+
async function doObsUpload(file) {
|
|
237
253
|
const key = getRemoteKey(obsConfig, file);
|
|
254
|
+
const client = await getObsClient();
|
|
238
255
|
return runObsRequest('upload', file, key, (finish, reportProgress, registerRequest) => {
|
|
239
|
-
|
|
256
|
+
client.putObject(
|
|
240
257
|
{
|
|
241
258
|
Bucket: obsConfig.bucket,
|
|
242
259
|
Key: key,
|
|
@@ -261,8 +278,9 @@ function doObsUpload(file) {
|
|
|
261
278
|
}
|
|
262
279
|
async function getObsStat(file) {
|
|
263
280
|
const key = getRemoteKey(obsConfig, file);
|
|
281
|
+
const client = await getObsClient();
|
|
264
282
|
return runObsRequest('metadata check', file, key, (finish, _reportProgress, registerRequest) => {
|
|
265
|
-
|
|
283
|
+
client.getObjectMetadata(
|
|
266
284
|
{
|
|
267
285
|
Bucket: obsConfig.bucket,
|
|
268
286
|
Key: key,
|
|
@@ -277,7 +295,7 @@ async function getObsStat(file) {
|
|
|
277
295
|
} else if (status < 300) {
|
|
278
296
|
finish(null, result);
|
|
279
297
|
} else {
|
|
280
|
-
finish(result
|
|
298
|
+
finish(result);
|
|
281
299
|
}
|
|
282
300
|
},
|
|
283
301
|
);
|
|
@@ -335,9 +353,18 @@ async function uploadFile(file) {
|
|
|
335
353
|
await doUpload(file);
|
|
336
354
|
setStorage(file, true);
|
|
337
355
|
} catch (uploadError) {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
)
|
|
356
|
+
const detail = `file=${getDisplayFile(file)}, ${formatObsError(uploadError)}`;
|
|
357
|
+
|
|
358
|
+
if (ossConfig) {
|
|
359
|
+
console.error(`[assets] upload failed: ${detail}`);
|
|
360
|
+
} else {
|
|
361
|
+
const message = chalk.red(`[OBS] upload failed: ${detail}`);
|
|
362
|
+
if (progress && progress.stream && progress.stream.isTTY) {
|
|
363
|
+
progress.interrupt(message);
|
|
364
|
+
} else {
|
|
365
|
+
console.error(message);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
341
368
|
}
|
|
342
369
|
} finally {
|
|
343
370
|
uploadQueue[file] = 'completed';
|
|
@@ -403,10 +430,13 @@ function addToUploadQueue(assets) {
|
|
|
403
430
|
}
|
|
404
431
|
});
|
|
405
432
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
`
|
|
409
|
-
|
|
433
|
+
if (ossConfig) {
|
|
434
|
+
console.log(
|
|
435
|
+
`[assets] upload queue: provider=OSS, total=${assets.length}, ` +
|
|
436
|
+
`cached=${cachedCount}, pending=${pendingCount}`,
|
|
437
|
+
);
|
|
438
|
+
}
|
|
439
|
+
|
|
410
440
|
return pendingCount ? checkUpload() : Promise.resolve();
|
|
411
441
|
}
|
|
412
442
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hw-weapp-compiler",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.26",
|
|
4
4
|
"description": "基于 webpack5 的小程序构建工具,兼容 Node.js 14~24",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"bin": {
|
|
@@ -53,6 +53,6 @@
|
|
|
53
53
|
"build": "node ./index.js build",
|
|
54
54
|
"dev": "nodemon --inspect --trace-deprecation --watch build ./index.js dev",
|
|
55
55
|
"simulation": "node ./index.js build -s",
|
|
56
|
-
"test": "node ./test/upload.js"
|
|
56
|
+
"test": "node ./test/upload.js && node ./test/upload-async-client.js"
|
|
57
57
|
}
|
|
58
58
|
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
const assert = require('assert');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const Module = require('module');
|
|
4
|
+
const os = require('os');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
|
|
7
|
+
let metadataCalledBeforeReady = false;
|
|
8
|
+
let metadataCallCount = 0;
|
|
9
|
+
|
|
10
|
+
class AsyncOBSClient {
|
|
11
|
+
constructor(config) {
|
|
12
|
+
this.util = {
|
|
13
|
+
server: null,
|
|
14
|
+
signatureContext: null,
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
// 模拟 esdk-obs-nodejs 3.26.8:initFactory 在第一次 await 后才完成赋值。
|
|
18
|
+
Promise.resolve().then(() => {
|
|
19
|
+
this.util.server = config.server;
|
|
20
|
+
this.util.signatureContext = { signature: 'obs' };
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
getObjectMetadata(_params, callback) {
|
|
25
|
+
metadataCallCount += 1;
|
|
26
|
+
if (!this.util.signatureContext) {
|
|
27
|
+
metadataCalledBeforeReady = true;
|
|
28
|
+
}
|
|
29
|
+
callback(null, { CommonMsg: { Status: 200 } });
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function run() {
|
|
34
|
+
const projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hw-upload-async-client-test-'));
|
|
35
|
+
const originalCwd = process.cwd();
|
|
36
|
+
const originalLoad = Module._load;
|
|
37
|
+
|
|
38
|
+
try {
|
|
39
|
+
fs.mkdirSync(path.join(projectDir, 'dist/assets'), { recursive: true });
|
|
40
|
+
fs.writeFileSync(path.join(projectDir, 'dist/assets/ready.png'), 'ready');
|
|
41
|
+
fs.writeFileSync(
|
|
42
|
+
path.join(projectDir, '.weapp.js'),
|
|
43
|
+
`module.exports = { obsConfig: ${JSON.stringify({
|
|
44
|
+
access_key_id: 'test-ak',
|
|
45
|
+
secret_access_key: 'test-sk',
|
|
46
|
+
server: 'http://obs.example.com',
|
|
47
|
+
bucket: 'test-bucket',
|
|
48
|
+
dir: 'test-dir',
|
|
49
|
+
operation_timeout: 1,
|
|
50
|
+
})} };`,
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
Module._load = function load(request, parent, isMain) {
|
|
54
|
+
if (request === 'esdk-obs-nodejs') {
|
|
55
|
+
return AsyncOBSClient;
|
|
56
|
+
}
|
|
57
|
+
return originalLoad.call(this, request, parent, isMain);
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
process.chdir(projectDir);
|
|
61
|
+
const uploadModule = path.resolve(originalCwd, 'build/utils/upload.js');
|
|
62
|
+
const { addToUploadQueue } = require(uploadModule);
|
|
63
|
+
await addToUploadQueue(['assets/ready.png']);
|
|
64
|
+
assert.strictEqual(metadataCalledBeforeReady, false, 'request started before client was ready');
|
|
65
|
+
assert.strictEqual(metadataCallCount, 1, 'metadata request should run exactly once');
|
|
66
|
+
|
|
67
|
+
// 等待 storage.js 的防抖写入结束后再清理测试目录。
|
|
68
|
+
await new Promise((resolve) => setTimeout(resolve, 400));
|
|
69
|
+
} finally {
|
|
70
|
+
Module._load = originalLoad;
|
|
71
|
+
process.chdir(originalCwd);
|
|
72
|
+
fs.rmSync(projectDir, { recursive: true, force: true });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
run().catch((error) => {
|
|
77
|
+
console.error(error);
|
|
78
|
+
process.exitCode = 1;
|
|
79
|
+
});
|