hw-weapp-compiler 1.0.23 → 1.0.25

package/README.md

@@ -53,6 +53,10 @@ module.exports = {
     server: 'XXX',
     bucket: 'XXX',
     dir: 'weapp',
+    // 可选：单次 SDK 请求超时、重试次数，以及编译器无进展兜底超时（秒）
+    timeout: 15,
+    max_retry_count: 1,
+    operation_timeout: 35,
   },
   ossConfig: {
     region: '<Your region>',

package/build/utils/upload.js

@@ -16,18 +16,236 @@ const { DIST_DIR } = require('../config/constants');
 
 const { obsConfig, ossConfig } = getConfig();
 
+const UPLOAD_CONCURRENCY = 10;
+const DEFAULT_OBS_TIMEOUT_SECONDS = 15;
+const DEFAULT_OBS_MAX_RETRY_COUNT = 1;
+const OBS_PENDING_WARNING_MS = 15 * 1000;
+const OBS_TIMEOUT_BUFFER_MS = 5 * 1000;
+
 let obsClient;
+let obsClientReady;
 let ossClient;
 let progress;
 let uploadQueue = {};
+let uploadRunner;
+
+function getRemoteKey(config, file) {
+  return compatiblePath(path.join(config.dir, path.relative(DIST_DIR, file)));
+}
+
+function getDisplayFile(file) {
+  return compatiblePath(path.relative(process.cwd(), file));
+}
+
+function getObsStatus(value) {
+  return value && value.CommonMsg ? value.CommonMsg.Status : undefined;
+}
+
+function getInvalidObsResponseError(operation) {
+  const error = new Error(`OBS ${operation} returned no HTTP status`);
+  error.code = 'OBS_INVALID_RESPONSE';
+  return error;
+}
+
+function formatObsError(error) {
+  if (!error) {
+    return 'unknown error';
+  }
+
+  const commonMsg = error.CommonMsg || {};
+  const details = {
+    status: commonMsg.Status || error.statusCode,
+    code: commonMsg.Code || error.code,
+    message: commonMsg.Message || error.message,
+    requestId: commonMsg.RequestId || error.requestId,
+  };
+
+  const result = Object.entries(details)
+    .filter(([, value]) => value !== undefined && value !== null && value !== '')
+    .map(([key, value]) => `${key}=${value}`)
+    .join(', ');
+
+  return result || String(error);
+}
+
+function toNonNegativeNumber(value, fallback) {
+  const number = Number(value);
+  return Number.isFinite(number) && number >= 0 ? number : fallback;
+}
+
+function toPositiveNumber(value, fallback) {
+  const number = Number(value);
+  return Number.isFinite(number) && number > 0 ? number : fallback;
+}
+
+function getObsRuntimeOptions() {
+  const timeoutSeconds = toPositiveNumber(
+    obsConfig.timeout,
+    DEFAULT_OBS_TIMEOUT_SECONDS,
+  );
+  const maxRetryCount = toNonNegativeNumber(
+    obsConfig.max_retry_count,
+    DEFAULT_OBS_MAX_RETRY_COUNT,
+  );
+  const defaultOperationTimeoutMs =
+    timeoutSeconds * (maxRetryCount + 1) * 1000 + OBS_TIMEOUT_BUFFER_MS;
+  const operationTimeoutMs = obsConfig.operation_timeout
+    ? toPositiveNumber(
+      obsConfig.operation_timeout,
+      defaultOperationTimeoutMs / 1000,
+    ) * 1000
+    : defaultOperationTimeoutMs;
+
+  return {
+    timeoutSeconds,
+    maxRetryCount,
+    operationTimeoutMs,
+  };
+}
+
+function createObsRequestLogger(operation, file, key) {
+  const startedAt = Date.now();
+  const { timeoutSeconds, maxRetryCount, operationTimeoutMs } = getObsRuntimeOptions();
+  const displayFile = getDisplayFile(file);
+
+  console.log(`[OBS] ${operation} started: key=${key}, file=${displayFile}`);
+
+  const pendingTimer = setTimeout(() => {
+    console.warn(
+      `[OBS] ${operation} is still pending after ${OBS_PENDING_WARNING_MS / 1000}s: key=${key}, ` +
+        `server=${obsConfig.server}, bucket=${obsConfig.bucket}, ` +
+        `sdkTimeout=${timeoutSeconds}s, maxRetryCount=${maxRetryCount}, ` +
+        `operationTimeout=${operationTimeoutMs / 1000}s`,
+    );
+  }, OBS_PENDING_WARNING_MS);
+
+  // 诊断定时器不应单独阻止构建进程退出。
+  if (pendingTimer.unref) {
+    pendingTimer.unref();
+  }
+
+  return {
+    finish(outcome, detail) {
+      clearTimeout(pendingTimer);
+      const elapsed = Date.now() - startedAt;
+      const suffix = detail ? `, ${detail}` : '';
+      const message = `[OBS] ${operation} ${outcome} in ${elapsed}ms: key=${key}${suffix}`;
+
+      if (outcome === 'failed') {
+        console.error(message);
+      } else if (outcome === 'miss') {
+        console.warn(message);
+      } else {
+        console.log(message);
+      }
+    },
+  };
+}
+
+function runObsRequest(operation, file, key, request) {
+  return new Promise((resolve, reject) => {
+    const requestLogger = createObsRequestLogger(operation, file, key);
+    const { operationTimeoutMs } = getObsRuntimeOptions();
+    let settled = false;
+    let operationTimer;
+    let activeRequest;
+
+    const finish = (error, result, outcome) => {
+      if (settled) {
+        return;
+      }
+
+      settled = true;
+      clearTimeout(operationTimer);
+      activeRequest = null;
+      requestLogger.finish(
+        outcome || (error ? 'failed' : 'completed'),
+        formatObsError(error || result),
+      );
+
+      if (error) {
+        reject(error);
+      } else {
+        resolve(result);
+      }
+    };
+
+    const armOperationTimeout = () => {
+      if (settled) {
+        return;
+      }
+      clearTimeout(operationTimer);
+      operationTimer = setTimeout(() => {
+        const requestToAbort = activeRequest;
+        const error = new Error(
+          `OBS ${operation} timed out after ${operationTimeoutMs / 1000}s without progress`,
+        );
+        error.code = 'OBS_OPERATION_TIMEOUT';
+        finish(error);
+        if (requestToAbort && requestToAbort.destroy) {
+          requestToAbort.destroy();
+        }
+      }, operationTimeoutMs);
+    };
+
+    const registerRequest = (clientRequest) => {
+      activeRequest = clientRequest;
+    };
+
+    armOperationTimeout();
+
+    try {
+      request(finish, armOperationTimeout, registerRequest);
+    } catch (error) {
+      finish(error);
+    }
+  });
+}
+
+function waitForObsClientReady(client) {
+  if (client.util && client.util.signatureContext && client.util.server) {
+    return Promise.resolve(client);
+  }
+
+  const { operationTimeoutMs } = getObsRuntimeOptions();
+  const startedAt = Date.now();
+
+  return new Promise((resolve, reject) => {
+    const checkReady = () => {
+      if (client.util && client.util.signatureContext && client.util.server) {
+        resolve(client);
+        return;
+      }
+
+      if (Date.now() - startedAt >= operationTimeoutMs) {
+        const error = new Error(
+          `OBS client initialization timed out after ${operationTimeoutMs / 1000}s`,
+        );
+        error.code = 'OBS_CLIENT_INIT_TIMEOUT';
+        reject(error);
+        return;
+      }
+
+      setImmediate(checkReady);
+    };
+
+    setImmediate(checkReady);
+  });
+}
 
 function getObsClient() {
   if (!obsClient) {
+    const { timeoutSeconds, maxRetryCount } = getObsRuntimeOptions();
     obsClient = new OBSClient({
       ...obsConfig,
+      timeout: timeoutSeconds,
+      max_retry_count: maxRetryCount,
     });
+    // esdk-obs-nodejs 3.26.8 将 initFactory 改成了 async，但构造函数没有等待它。
+    // 立即请求会在 signatureContext 尚未赋值时破坏 SDK 的 bucket 协商锁。
+    obsClientReady = waitForObsClientReady(obsClient);
   }
-  return obsClient;
+  return obsClientReady;
 }
 
 function getOssClient() {
@@ -41,49 +259,62 @@ function getOssClient() {
 
 function doOssUpload(file) {
   return getOssClient().put(
-    compatiblePath(path.join(ossConfig.dir, path.relative(DIST_DIR, file))),
+    getRemoteKey(ossConfig, file),
     file,
   );
 }
 
 function getOssStat(file) {
-  return getOssClient().head(compatiblePath(path.join(ossConfig.dir, path.relative(DIST_DIR, file))));
+  return getOssClient().head(getRemoteKey(ossConfig, file));
 }
 
-function doObsUpload(file) {
-  return new Promise((resolve, reject) => {
-    getObsClient().putObject(
+async function doObsUpload(file) {
+  const key = getRemoteKey(obsConfig, file);
+  const client = await getObsClient();
+  return runObsRequest('upload', file, key, (finish, reportProgress, registerRequest) => {
+    client.putObject(
       {
         Bucket: obsConfig.bucket,
-        Key: compatiblePath(path.join(obsConfig.dir, path.relative(DIST_DIR, file))),
+        Key: key,
         SourceFile: file,
+        ProgressCallback: reportProgress,
+        RequestHook: registerRequest,
       },
       (err, result) => {
+        const status = getObsStatus(result);
         if (err) {
-          reject(err);
-        } else if (result.CommonMsg.Status >= 300) {
-          reject(new Error(`OBS upload failed with status ${result.CommonMsg.Status}`));
+          finish(err);
+        } else if (status === undefined) {
+          finish(getInvalidObsResponseError('upload'));
+        } else if (status >= 300) {
+          finish(result);
         } else {
-          resolve(result);
+          finish(null, result);
         }
       },
     );
   });
 }
 async function getObsStat(file) {
-  return new Promise((resolve, reject) => {
-    getObsClient().getObjectMetadata(
+  const key = getRemoteKey(obsConfig, file);
+  const client = await getObsClient();
+  return runObsRequest('metadata check', file, key, (finish, _reportProgress, registerRequest) => {
+    client.getObjectMetadata(
       {
         Bucket: obsConfig.bucket,
-        Key: compatiblePath(path.join(obsConfig.dir, path.relative(DIST_DIR, file))),
+        Key: key,
+        RequestHook: registerRequest,
       },
       (err, result) => {
+        const status = getObsStatus(result);
         if (err) {
-          reject(err);
-        } else if (result.CommonMsg.Status < 300) {
-          resolve(result);
+          finish(err);
+        } else if (status === undefined) {
+          finish(getInvalidObsResponseError('metadata check'));
+        } else if (status < 300) {
+          finish(null, result);
         } else {
-          reject(result);
+          finish(result, null, 'miss');
         }
       },
     );
@@ -111,8 +342,7 @@ function doUpload(file) {
 }
 
 function updateProgress() {
-  const completed =
-    Object.entries(uploadQueue).filter((item) => item[1] === 'completed').length + 1;
+  const completed = Object.values(uploadQueue).filter((status) => status === 'completed').length;
   const total = Object.keys(uploadQueue).length;
 
   if (!progress) {
@@ -121,49 +351,72 @@ function updateProgress() {
       width: 40,
       clear: true,
     });
+  } else {
+    // watch 构建期间可能继续追加资源，进度条总数需要同步扩容。
+    progress.total = total;
   }
 
   progress.tick();
 
-  if (completed === total) {
-    progress.tick({
-      current: total,
-    });
-    progress = null;
-    uploadQueue = {};
-  }
-
   return `[${completed}/${total}]`;
 }
 
-async function checkUpload() {
-  const files = Object.entries(uploadQueue)
-    .filter((item) => item[1] === false)
-    .splice(0, 10);
-
-  if (files && files.length) {
-    await Promise.all(
-      files.map(async (file) => {
-        uploadQueue[file[0]] = 'uploading';
-        try {
-          await getStat(file[0]);
-          setStorage(file[0], true);
-          updateProgress();
-        } catch (error) {
-          try {
-            await doUpload(file[0]);
-            setStorage(file[0], true);
-          } catch (uploadError) {
-            console.error(`Failed to upload ${file[0]}:`, uploadError);
-          }
-          updateProgress();
-        }
-        uploadQueue[file[0]] = 'completed';
-      }),
-    );
+async function uploadFile(file) {
+  uploadQueue[file] = 'uploading';
 
-    checkUpload();
+  try {
+    await getStat(file);
+    setStorage(file, true);
+  } catch (error) {
+    try {
+      await doUpload(file);
+      setStorage(file, true);
+    } catch (uploadError) {
+      console.error(
+        `[assets] upload failed: file=${getDisplayFile(file)}, ${formatObsError(uploadError)}`,
+      );
+    }
+  } finally {
+    uploadQueue[file] = 'completed';
+    updateProgress();
+  }
+}
+
+function checkUpload() {
+  // done hook 可能在上一次上传尚未结束时再次触发。复用同一个 runner，避免每次构建
+  // 都额外启动 10 个请求，最终耗尽 OBS/HTTP 连接池。
+  if (uploadRunner) {
+    return uploadRunner;
   }
+
+  uploadRunner = (async () => {
+    while (true) {
+      const files = Object.entries(uploadQueue)
+        .filter((item) => item[1] === 'pending')
+        .slice(0, UPLOAD_CONCURRENCY)
+        .map((item) => item[0]);
+
+      if (!files.length) {
+        break;
+      }
+
+      await Promise.all(files.map(uploadFile));
+    }
+  })().finally(() => {
+    uploadRunner = null;
+
+    // 文件状态已经写完后再清空，避免旧实现最后一个文件把已清空的队列重新写回。
+    if (!Object.values(uploadQueue).includes('pending')) {
+      progress = null;
+      uploadQueue = {};
+      return;
+    }
+
+    // 兜住 runner 完成与 watch 新增资源恰好交错的情况。
+    checkUpload();
+  });
+
+  return uploadRunner;
 }
 
 function addToUploadQueue(assets) {
@@ -172,13 +425,26 @@ function addToUploadQueue(assets) {
     return;
   }
 
+  let cachedCount = 0;
+  let pendingCount = 0;
+
   assets.forEach((asset) => {
     const file = path.resolve(DIST_DIR, asset);
+    if (getStorage(file)) {
+      cachedCount += 1;
+      return;
+    }
     if (uploadQueue[file] === undefined) {
-      uploadQueue[file] = false;
+      uploadQueue[file] = 'pending';
+      pendingCount += 1;
     }
   });
-  checkUpload();
+
+  console.log(
+    `[assets] upload queue: provider=${ossConfig ? 'OSS' : 'OBS'}, total=${assets.length}, ` +
+      `cached=${cachedCount}, pending=${pendingCount}`,
+  );
+  return pendingCount ? checkUpload() : Promise.resolve();
 }
 
 module.exports = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hw-weapp-compiler",
-  "version": "1.0.23",
+  "version": "1.0.25",
   "description": "基于 webpack5 的小程序构建工具，兼容 Node.js 14~24",
   "main": "index.js",
   "bin": {
@@ -52,6 +52,7 @@
   "scripts": {
     "build": "node ./index.js build",
     "dev": "nodemon --inspect --trace-deprecation --watch build ./index.js dev",
-    "simulation": "node ./index.js build -s"
+    "simulation": "node ./index.js build -s",
+    "test": "node ./test/upload.js && node ./test/upload-async-client.js"
   }
 }

package/test/upload-async-client.js

@@ -0,0 +1,72 @@
+const assert = require('assert');
+const fs = require('fs');
+const Module = require('module');
+const os = require('os');
+const path = require('path');
+
+class AsyncOBSClient {
+  constructor(config) {
+    this.config = config;
+    this.util = {
+      server: null,
+      signatureContext: null,
+    };
+
+    // 模拟 esdk-obs-nodejs 3.26.8：initFactory 在第一次 await 后才完成赋值。
+    Promise.resolve().then(() => {
+      this.util.server = config.server;
+      this.util.signatureContext = { signature: 'obs' };
+    });
+  }
+
+  getObjectMetadata(_params, callback) {
+    assert(this.util.signatureContext, 'request started before OBS client initialization');
+    callback(null, { CommonMsg: { Status: 200 } });
+  }
+}
+
+async function run() {
+  const projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hw-upload-async-client-test-'));
+  const originalCwd = process.cwd();
+  const originalLoad = Module._load;
+
+  try {
+    fs.mkdirSync(path.join(projectDir, 'dist/assets'), { recursive: true });
+    fs.writeFileSync(path.join(projectDir, 'dist/assets/ready.png'), 'ready');
+    fs.writeFileSync(
+      path.join(projectDir, '.weapp.js'),
+      `module.exports = { obsConfig: ${JSON.stringify({
+        access_key_id: 'test-ak',
+        secret_access_key: 'test-sk',
+        server: 'http://obs.example.com',
+        bucket: 'test-bucket',
+        dir: 'test-dir',
+        operation_timeout: 1,
+      })} };`,
+    );
+
+    Module._load = function load(request, parent, isMain) {
+      if (request === 'esdk-obs-nodejs') {
+        return AsyncOBSClient;
+      }
+      return originalLoad.call(this, request, parent, isMain);
+    };
+
+    process.chdir(projectDir);
+    const uploadModule = path.resolve(originalCwd, 'build/utils/upload.js');
+    const { addToUploadQueue } = require(uploadModule);
+    await addToUploadQueue(['assets/ready.png']);
+
+    // 等待 storage.js 的防抖写入结束后再清理测试目录。
+    await new Promise((resolve) => setTimeout(resolve, 400));
+  } finally {
+    Module._load = originalLoad;
+    process.chdir(originalCwd);
+    fs.rmSync(projectDir, { recursive: true, force: true });
+  }
+}
+
+run().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

package/test/upload.js

@@ -0,0 +1,92 @@
+const assert = require('assert');
+const fs = require('fs');
+const http = require('http');
+const os = require('os');
+const path = require('path');
+
+async function run() {
+  const requests = [];
+  const sockets = new Set();
+  const server = http.createServer((req, res) => {
+    requests.push(`${req.method} ${req.url}`);
+
+    if (req.url.includes('stuck.png')) {
+      return;
+    }
+
+    if (req.method === 'HEAD') {
+      res.statusCode = 404;
+      res.end();
+      return;
+    }
+
+    req.resume();
+    req.on('end', () => {
+      res.statusCode = 200;
+      res.setHeader('etag', 'test-etag');
+      res.end();
+    });
+  });
+  server.on('connection', (socket) => {
+    sockets.add(socket);
+    socket.on('close', () => sockets.delete(socket));
+  });
+
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+
+  const projectDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hw-upload-test-'));
+  const originalCwd = process.cwd();
+  const endpoint = `http://127.0.0.1:${server.address().port}`;
+  const uploadModule = path.resolve(originalCwd, 'build/utils/upload.js');
+
+  try {
+    fs.mkdirSync(path.join(projectDir, 'dist/assets'), { recursive: true });
+    fs.writeFileSync(path.join(projectDir, 'dist/assets/ok.png'), 'ok');
+    fs.writeFileSync(path.join(projectDir, 'dist/assets/stuck.png'), 'stuck');
+    fs.writeFileSync(path.join(projectDir, 'dist/assets/later.png'), 'later');
+    fs.writeFileSync(
+      path.join(projectDir, '.weapp.js'),
+      `module.exports = { obsConfig: ${JSON.stringify({
+        access_key_id: 'test-ak',
+        secret_access_key: 'test-sk',
+        server: endpoint,
+        bucket: 'test-bucket',
+        dir: 'test-dir',
+        timeout: 10,
+        max_retry_count: 0,
+        operation_timeout: 0.1,
+      })} };`,
+    );
+
+    process.chdir(projectDir);
+    const { addToUploadQueue } = require(uploadModule);
+
+    await addToUploadQueue(['assets/ok.png']);
+    assert(requests.some((request) => request.startsWith('HEAD ')));
+    assert(requests.some((request) => request.startsWith('PUT ')));
+
+    const startedAt = Date.now();
+    const stuckRun = addToUploadQueue(['assets/stuck.png']);
+    const sameRun = addToUploadQueue(['assets/later.png']);
+    assert.strictEqual(stuckRun, sameRun, 'watch rebuild should reuse the active queue runner');
+    await sameRun;
+
+    const elapsed = Date.now() - startedAt;
+    assert(elapsed < 2000, `timeout fallback took ${elapsed}ms`);
+    assert(requests.filter((request) => request.includes('stuck.png')).length >= 2);
+    assert(requests.some((request) => request.includes('later.png')));
+
+    // 等待 storage.js 的防抖写入结束后再清理测试目录。
+    await new Promise((resolve) => setTimeout(resolve, 400));
+  } finally {
+    process.chdir(originalCwd);
+    sockets.forEach((socket) => socket.destroy());
+    await new Promise((resolve) => server.close(resolve));
+    fs.rmSync(projectDir, { recursive: true, force: true });
+  }
+}
+
+run().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});