@abtnode/core 1.17.4-beta-20251203-225234-75da41dd → 1.17.4-beta-20251204-152224-243ff54f

package/lib/api/team.js

@@ -3062,7 +3062,11 @@ class TeamAPI extends EventEmitter {
 
       return result;
     } catch (err) {
-      logger.error('Failed to create org', { err, teamDid });
+      logger.error('Failed to create org', err, {
+        teamDid,
+        name: rest.name,
+        userDid: rest.ownerDid || context.user.did || '',
+      });
       throw err;
     }
   }

package/lib/blocklet/manager/disk.js

@@ -45,7 +45,7 @@ const {
   EVENTS,
   USER_PROFILE_SYNC_FIELDS,
 } = require('@abtnode/constant');
-
+const { BLOCKLET_SITE_GROUP_SUFFIX } = require('@abtnode/constant');
 const { getBlockletEngine } = require('@blocklet/meta/lib/engine');
 const {
   isDeletableBlocklet,
@@ -245,7 +245,7 @@ const { installExternalDependencies } = require('../../util/install-external-dep
 const { dockerExecChown } = require('../../util/docker/docker-exec-chown');
 const checkDockerRunHistory = require('../../util/docker/check-docker-run-history');
 const { shouldJobBackoff } = require('../../util/env');
-const ensureBlockletRunning = require('./ensure-blocklet-running');
+const { ensureBlockletRunning } = require('./ensure-blocklet-running');
 
 const { transformNotification } = require('../../util/notification');
 const { generateUserUpdateData } = require('../../util/user');
@@ -1916,7 +1916,7 @@ class DiskBlockletManager extends BaseBlockletManager {
       if (!aliasDomainSite) {
         return null;
       }
-      targetDid = (aliasDomainSite.domain || '').replace('.blocklet-domain-group', '');
+      targetDid = (aliasDomainSite.domain || '').replace(BLOCKLET_SITE_GROUP_SUFFIX, '');
     }
 
     if (!targetDid) {

package/lib/blocklet/manager/ensure-blocklet-running.js

@@ -1,14 +1,22 @@
+/* eslint-disable no-await-in-loop */
 const logger = require('@abtnode/logger')('@abtnode/core:blocklet-status-checker');
 const pAll = require('p-all');
 const { BlockletStatus } = require('@blocklet/constant');
 const sleep = require('@abtnode/util/lib/sleep');
 const { getDisplayName } = require('@blocklet/meta/lib/util');
-const { isValid } = require('@arcblock/did');
-
 const states = require('../../states');
 const { isBlockletPortHealthy, shouldCheckHealthy } = require('../../util/blocklet');
 
-const inProgressStatuses = [BlockletStatus.stopping, BlockletStatus.restarting, BlockletStatus.waiting];
+const inProgressStatuses = [
+  BlockletStatus.stopping,
+  BlockletStatus.restarting,
+  BlockletStatus.waiting,
+  BlockletStatus.starting,
+  BlockletStatus.downloading,
+];
+
+// Restart queue concurrency, 这个改大，容易 blocklet 超时导致启动失败
+const RESTART_CONCURRENCY = 2;
 
 class EnsureBlockletRunning {
   canRunEnsureBlockletRunning = false;
@@ -22,11 +30,9 @@ class EnsureBlockletRunning {
 
   minCheckInterval = 30_000;
 
-  everyBlockletCheckInterval = 2000;
-
-  everyBlockletDoingInterval = 5000;
+  preCheckInterval = 1000;
 
-  fakeRunningToWaitingOnce = false;
+  everyBlockletCheckInterval = 2000;
 
   highLoadCpu = +process.env.ABT_NODE_ENSURE_RUNNING_HIGH_LOAD_CPU || 0.85;
 
@@ -34,27 +40,41 @@ class EnsureBlockletRunning {
 
   highLoadDisk = +process.env.ABT_NODE_ENSURE_RUNNING_HIGH_LOAD_DISK || 0.95;
 
-  // 进行中状态超时时间（毫秒）
-  inProgressTimeout = +process.env.ABT_NODE_ENSURE_RUNNING_IN_PROGRESS_TIMEOUT || 5 * 60 * 1000;
+  // 各个状态的超时阈值（毫秒）
+  // 如果是首次调用（whenCycleCheck 为 false），这些值应该是 0
+  stoppingTimeout = +process.env.ABT_NODE_ENSURE_RUNNING_STOPPING_TIMEOUT || 60 * 1000;
 
-  runningBlocklets = {};
+  restartingTimeout = +process.env.ABT_NODE_ENSURE_RUNNING_RESTARTING_TIMEOUT || 6 * 60 * 1000;
+
+  waitingTimeout = +process.env.ABT_NODE_ENSURE_RUNNING_WAITING_TIMEOUT || 60 * 1000;
 
-  runningRootBlocklets = {};
+  downloadingTimeout = +process.env.ABT_NODE_ENSURE_RUNNING_DOWNLOADING_TIMEOUT || 3 * 60 * 1000;
 
-  fakeRunningBlocklets = {};
+  startingTimeout = +process.env.ABT_NODE_ENSURE_RUNNING_STARTING_TIMEOUT || 6 * 60 * 1000;
 
-  needRestartBlocklets = {};
+  runningBlocklets = {};
 
-  restartingBlocklets = {};
+  rootBlockletsInfo = {};
 
-  errorStartBlocklets = {};
+  progressBlockletsTime = {};
 
   stopped = false;
 
+  // Queue for restarting fake running blocklets
+  restartQueue = [];
+
+  // Set to track queue keys for fast lookup
+  restartQueueKeys = new Set();
+
+  restartQueueProcessing = false;
+
+  // Track pending jobs by componentDid to prevent duplicate processing
+  pendingJobs = {};
+
   // Ease to mock
   isBlockletPortHealthy = isBlockletPortHealthy;
 
-  isBlockletPortHealthyWithRetries = async (blocklet, fastCheck = false) => {
+  isBlockletPortHealthyWithRetries = async (blocklet) => {
     let error;
     if (!this.whenCycleCheck) {
       try {
@@ -81,9 +101,7 @@ class EnsureBlockletRunning {
       } catch (e) {
         error = e;
         // eslint-disable-next-line no-await-in-loop
-        await sleep(
-          fastCheck && this.whenCycleCheck ? this.everyBlockletDoingInterval : this.everyBlockletCheckInterval
-        );
+        await sleep(this.everyBlockletCheckInterval);
       }
     }
     logger.error('blocklet port is not healthy', error);
@@ -106,27 +124,40 @@ class EnsureBlockletRunning {
     this.checkSystemHighLoad = checkSystemHighLoad;
     logger.info('check and fix blocklet status interval', this.checkInterval);
     const task = async () => {
+      await sleep(this.preCheckInterval);
+
+      // 完全停止，后续也不再继续检查
+      if (this.stopped) {
+        logger.info('blocklet status checker stopped');
+        return;
+      }
       // 如果还没进入到需要检查的阶段，则等待 1 秒后继续检查
       if (!this.canRunEnsureBlockletRunning) {
-        await sleep(1000);
         task();
         return;
       }
-      if (this.whenCycleCheck) {
-        await sleep(Math.max(this.checkInterval, this.minCheckInterval));
-      }
+
+      // 首次检查前不等待（whenCycleCheck 为 false）
       try {
         await this.checkAndFix();
+
+        // 每次检查完之后查看消耗的时间
+        await sleep(Math.max(this.checkInterval, this.minCheckInterval));
+        this.whenCycleCheck = true;
       } catch (e) {
         logger.error('check and fix blocklet status failed', e);
+        // 出错时也要等待，避免频繁重试
+        if (this.whenCycleCheck) {
+          await sleep(Math.max(this.checkInterval, this.minCheckInterval));
+        }
       }
       task();
     };
     task();
   };
 
-  getDisplayNameByRootDid = (rootDid) => {
-    const rootBlocklet = this.runningRootBlocklets[rootDid];
+  getDisplayNameByRootDid = async (rootDid) => {
+    const rootBlocklet = this.rootBlockletsInfo[rootDid] || (await this.states.blocklet.getBlocklet(rootDid));
     if (rootBlocklet) {
       return getDisplayName(rootBlocklet);
     }
@@ -137,6 +168,42 @@ class EnsureBlockletRunning {
     return blocklet.meta.title || blocklet.meta.name || blocklet.meta.did;
   };
 
+  /**
+   * Get timeout threshold for a specific status
+   * @param {string} status - Blocklet status
+   * @returns {number} Timeout threshold in milliseconds
+   */
+  getStatusTimeout = (status) => {
+    // 如果是首次调用，所有阈值都是 0
+    if (!this.whenCycleCheck) {
+      return 0;
+    }
+
+    let timeout = 0;
+    switch (status) {
+      case BlockletStatus.stopping:
+        timeout = this.stoppingTimeout;
+        break;
+      case BlockletStatus.restarting:
+        timeout = this.restartingTimeout;
+        break;
+      case BlockletStatus.waiting:
+        timeout = this.waitingTimeout;
+        break;
+      case BlockletStatus.downloading:
+        timeout = this.downloadingTimeout;
+        break;
+      case BlockletStatus.starting:
+        timeout = this.startingTimeout;
+        break;
+      default:
+        timeout = this.downloadingTimeout;
+        break;
+    }
+    // 需要减去检查间隔时间，因为每次检查都会在第二次检查之后才比对时间，最少间隔时间不能小于 waitingTimeout
+    return Math.max(timeout - this.checkInterval, this.waitingTimeout);
+  };
+
   checkAndFix = async () => {
     logger.info('check and fix blocklet status');
     const systemHighLoad = this.checkSystemHighLoad({
@@ -147,50 +214,51 @@ class EnsureBlockletRunning {
 
     if (this.whenCycleCheck && systemHighLoad.isHighLoad) {
       logger.warn('Skip once ensure blocklet running because system high load', systemHighLoad);
-      return;
+      return 0;
     }
 
     this.runningBlocklets = {};
-    this.fakeRunningBlocklets = {};
-    this.needRestartBlocklets = {};
-
     const startTime = Date.now();
     try {
+      this.startRestartQueueProcessor();
       await this.getRunningBlocklets();
       await this.getFakeRunningBlocklets();
-      await this.restartFakeRunningBlocklets();
     } catch (e) {
       logger.error('ensure blocklet status failed', e);
     }
+    const elapsedTime = Date.now() - startTime;
     logger.info(
-      `ensure blocklet status finished in ${Date.now() - startTime}ms. It's server first start: ${!this.whenCycleCheck}`
+      `ensure blocklet status finished in ${elapsedTime}ms. It's server first start: ${!this.whenCycleCheck}`
     );
-    this.runningRootBlocklets = {};
-    this.whenCycleCheck = true;
+    return elapsedTime;
   };
 
   getRunningBlocklets = async () => {
-    const runningStatuses = this.whenCycleCheck
-      ? [BlockletStatus.running, BlockletStatus.waiting]
-      : [BlockletStatus.running, BlockletStatus.waiting, BlockletStatus.error];
-
-    const blocklets = await this.states.blocklet.getBlocklets();
-    for (const rootBlocklet of blocklets) {
-      const { did } = rootBlocklet.meta;
+    const rootBlocklets = await this.states.blocklet.getBlocklets();
+    for (const rootBlocklet of rootBlocklets) {
+      const rootDid = rootBlocklet.appPid || rootBlocklet.meta.did;
       if (rootBlocklet.children) {
         for (const childBlocklet of rootBlocklet.children) {
           const isRunning =
-            runningStatuses.includes(childBlocklet.status) || childBlocklet.greenStatus === BlockletStatus.running;
-          const isInProgress = inProgressStatuses.includes(childBlocklet.status);
+            childBlocklet.status === BlockletStatus.running || childBlocklet.greenStatus === BlockletStatus.running;
+          const isInProgress =
+            inProgressStatuses.includes(childBlocklet.status) || inProgressStatuses.includes(childBlocklet.greenStatus);
+          const isStopped =
+            childBlocklet.status === BlockletStatus.stopped && childBlocklet.greenStatus === BlockletStatus.stopped;
+
+          // 如果处于过 running, 或 stopped，则删除 progressBlockletsTime
+          if (isRunning || isStopped) {
+            delete this.progressBlockletsTime[`${rootDid}-${childBlocklet.meta.did}`];
+          }
           if (isRunning || isInProgress) {
-            if (!this.runningBlocklets[did]) {
-              this.runningBlocklets[did] = [];
+            if (!this.runningBlocklets[rootDid]) {
+              this.runningBlocklets[rootDid] = [];
             }
-            if (this.runningBlocklets[did].find((b) => b.meta.did === childBlocklet.meta.did)) {
+            if (this.runningBlocklets[rootDid].find((child) => child.meta.did === childBlocklet.meta.did)) {
               continue;
             }
-            this.runningBlocklets[did].push(childBlocklet);
-            this.runningRootBlocklets[did] = rootBlocklet;
+            this.runningBlocklets[rootDid].push(childBlocklet);
+            this.rootBlockletsInfo[rootDid] = rootBlocklet;
           }
         }
       }
@@ -201,195 +269,294 @@ class EnsureBlockletRunning {
   getFakeRunningBlocklets = async () => {
     const rootDids = Object.keys(this.runningBlocklets);
     await pAll(
-      rootDids.map((did) => {
+      rootDids.map((rootDid) => {
         return async () => {
-          const blocklets = this.runningBlocklets[did];
+          // runningBlocklets[rootDid] 存储的是该根 blocklet 下的所有子组件（childBlocklets）
+          const childBlocklets = this.runningBlocklets[rootDid];
           // eslint-disable-next-line
+          const fakeDids = [];
           await pAll(
-            blocklets.map((blocklet) => {
+            childBlocklets.map((childBlocklet) => {
               return async () => {
-                if (!shouldCheckHealthy(blocklet)) {
-                  // 如果 blocklet 是不需要启动的，并且不是 running，则设置为 running 状态
-                  if (blocklet.status !== BlockletStatus.running && blocklet.greenStatus !== BlockletStatus.running) {
-                    await this.states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
-                      componentDids: [blocklet.meta.did],
+                if (!shouldCheckHealthy(childBlocklet)) {
+                  // 如果 childBlocklet 是不需要启动的，并且不是 running，则设置为 running 状态
+                  if (
+                    childBlocklet.status !== BlockletStatus.running &&
+                    childBlocklet.greenStatus !== BlockletStatus.running
+                  ) {
+                    await this.states.blocklet.setBlockletStatus(rootDid, BlockletStatus.running, {
+                      componentDids: [childBlocklet.meta.did],
                     });
                   }
                   return;
                 }
 
-                // Skip health check if blocklet is in progress status
-                if (inProgressStatuses.includes(blocklet.status)) {
-                  // Check if it's user-initiated operation
-                  // 如果 operator 是 z 开头的字符串，表示是 did, 跳过健康检查
-                  if (blocklet.operator && isValid(blocklet.operator)) {
-                    logger.info('Skip ensure running check for user-initiated operation', {
-                      did,
-                      componentDid: blocklet.meta.did,
-                      status: blocklet.status,
-                      operator: blocklet.operator,
-                    });
-                    return;
-                  }
+                const isInProgress =
+                  inProgressStatuses.includes(childBlocklet.status) ||
+                  inProgressStatuses.includes(childBlocklet.greenStatus);
 
-                  // Check timeout for daemon-initiated or no operator recorded operations
-                  if (blocklet.inProgressStart) {
-                    const elapsedTime = Date.now() - new Date(blocklet.inProgressStart).getTime();
-                    if (elapsedTime < this.inProgressTimeout) {
-                      logger.info('Skip ensure running check due to timeout not reached', {
-                        did,
-                        componentDid: blocklet.meta.did,
-                        status: blocklet.status,
-                        elapsedTime,
-                        timeout: this.inProgressTimeout,
-                      });
-                      return;
-                    }
+                // 如果处于进行中状态，则记录上次检查时间
+                if (isInProgress) {
+                  const key = `${rootDid}-${childBlocklet.meta.did}`;
+                  if (!this.progressBlockletsTime[key]) {
+                    this.progressBlockletsTime[key] = Date.now();
+                  }
+                  const lastProgressTime = this.progressBlockletsTime[key];
+                  // 首次调用或者超过阈值时间，则认为是 fake running
+                  if (
+                    !this.whenCycleCheck ||
+                    Date.now() - lastProgressTime > this.getStatusTimeout(childBlocklet.status)
+                  ) {
                     logger.warn('InProgress timeout reached, proceeding with health check', {
-                      did,
-                      componentDid: blocklet.meta.did,
-                      status: blocklet.status,
-                      elapsedTime,
-                      timeout: this.inProgressTimeout,
+                      did: rootDid,
+                      componentDid: childBlocklet.meta.did,
+                      status: childBlocklet.status,
+                    });
+                  } else if (this.whenCycleCheck) {
+                    // 如果没有 inProgressStart 时间戳，且非首次调用，跳过检查
+                    logger.info('Skip ensure running check: no inProgressStart timestamp', {
+                      did: rootDid,
+                      componentDid: childBlocklet.meta.did,
+                      status: childBlocklet.status,
                     });
+                    return;
                   }
                 }
 
-                const health = await this.isBlockletPortHealthyWithRetries(
-                  blocklet,
-                  inProgressStatuses.includes(blocklet.status)
-                );
-
-                if (health) {
-                  return;
+                if (!isInProgress) {
+                  const health = await this.isBlockletPortHealthyWithRetries(childBlocklet);
+                  if (health) {
+                    return;
+                  }
                 }
 
-                logger.warn('check blocklet port healthy', did, blocklet.meta.did, 'no healthy');
+                logger.warn('check blocklet port healthy', rootDid, childBlocklet.meta.did, 'no healthy');
 
-                if (!this.fakeRunningBlocklets[did]) {
-                  this.fakeRunningBlocklets[did] = [];
-                }
-                if (this.fakeRunningBlocklets[did].find((b) => b.meta.did === blocklet.meta.did)) {
-                  return;
-                }
-                this.fakeRunningBlocklets[did].push(blocklet);
+                // Add to restart queue immediately
+                fakeDids.push(childBlocklet.meta.did);
               };
             }),
             { concurrency: 10 }
           );
+          if (fakeDids.length > 0) {
+            this.addToRestartQueue(rootDid, fakeDids);
+          }
         };
       }),
       { concurrency: 8 }
     );
+  };
 
-    logger.info('get fake running blocklets', Object.keys(this.fakeRunningBlocklets).length);
+  /**
+   * Add a childBlocklet to the restart queue
+   * @param {string} rootDid - Root blocklet DID
+   * @param {Object} childBlocklet - Child blocklet (component) to restart
+   */
+  addToRestartQueue = (rootDid, dids) => {
+    // Check if job is pending (being processed)
+    if (this.restartQueueKeys.has(rootDid) || this.pendingJobs[rootDid]) {
+      return;
+    }
+
+    this.restartQueue.push({
+      rootDid,
+      componentDids: dids,
+      firstCycle: !this.whenCycleCheck,
+    });
+    this.restartQueueKeys.add(rootDid);
   };
 
-  restartFakeRunningBlocklets = async () => {
-    // blocklet 一组组重启
-    const blockletDids = Object.keys(this.fakeRunningBlocklets);
-    await pAll(
-      blockletDids.map((did) => {
-        return async () => {
-          const blocklets = this.fakeRunningBlocklets[did];
-          const componentDids = blocklets.map((b) => b.meta.did);
-          if (componentDids.length > 0) {
-            const key = `${did}-${componentDids.join('-')}`;
-            this.needRestartBlocklets[key] = true;
-            if (this.restartingBlocklets[key] && this.restartingBlocklets[key] + this.checkInterval < Date.now()) {
-              return;
-            }
-            this.restartingBlocklets[key] = Date.now();
+  // 启动重启队列，保持 4 个 worker 并发处理，如果有一个完成了，则会补充到队列中
+  startRestartQueueProcessor = () => {
+    if (this.restartQueueProcessing) {
+      return;
+    }
+    this.restartQueueProcessing = true;
 
-            const blockletDisplayName = this.getDisplayNameByRootDid(did);
-            const restartTitle = 'Blocklet health check failed';
-            const restartDescription = `Blocklet ${blockletDisplayName} with components ${componentDids.map((v) => this.getDisplayName(blocklets.find((b) => b.meta.did === v))).join(', ')} health check failed, restarting...`;
-            if (this.whenCycleCheck) {
-              this.notification(did, restartTitle, restartDescription, 'warning');
-            }
+    const runWorker = async () => {
+      while (!this.stopped) {
+        const item = this.restartQueue.shift();
+        if (!item) {
+          return;
+        }
 
-            try {
-              logger.info('restart blocklet:', did, componentDids);
-              await this.start({
-                did,
-                componentDids,
-                checkHealthImmediately: true,
-                atomic: true,
-                operator: 'ensure-blocklet-running',
-              });
-              if (this.whenCycleCheck) {
-                this.createAuditLog({
-                  action: 'ensureBlockletRunning',
-                  args: {
-                    teamDid: did,
-                    componentDids,
-                  },
-                  context: {
-                    user: {
-                      did,
-                      role: 'daemon',
-                      blockletDid: did,
-                      fullName: blockletDisplayName,
-                      elevated: false,
-                    },
-                  },
-                  result: {
-                    title: restartTitle,
-                    description: restartDescription,
-                  },
-                });
-              }
-
-              delete this.restartingBlocklets[key];
-            } catch (e) {
-              logger.error('restart blocklet failed', did, componentDids, e);
-              if (!this.errorStartBlocklets[key]) {
-                this.errorStartBlocklets[key] = 0;
-              }
-              this.errorStartBlocklets[key] += 1;
-
-              // 如果重启失败次数超过 3 次，则发送通知, 如果 server 是第一次启动遇到失败，则立刻发送通知
-              if (this.errorStartBlocklets[key] >= 3 || !this.whenCycleCheck) {
-                const title = 'Restart blocklet failed when health check failed';
-                const description = `Restart blocklet ${blockletDisplayName} with components ${componentDids.map((v) => this.getDisplayName(blocklets.find((b) => b.meta.did === v))).join(', ')} failed`;
-                this.notification(did, title, description, 'error');
-                delete this.errorStartBlocklets[key];
-                logger.error('restart many times blocklet failed', did, componentDids, e);
-                try {
-                  // 失败了应该保持 error 状态
-                  await this.states.blocklet.setBlockletStatus(did, BlockletStatus.error, { componentDids });
-                  this.createAuditLog({
-                    action: 'ensureBlockletRunning',
-                    args: {
-                      blockletDisplayName,
-                      teamDid: did,
-                      componentDids,
-                    },
-                    context: {
-                      user: {
-                        did,
-                        role: 'daemon',
-                        blockletDid: did,
-                        fullName: blockletDisplayName,
-                        elevated: false,
-                      },
-                    },
-                    result: {
-                      title,
-                      description,
-                    },
-                  });
-                } catch (err) {
-                  logger.error('ensure blocklet running, create audit log failed', did, componentDids, err);
-                }
-              }
-            }
-          }
-        };
-      }),
-      { concurrency: 8 }
-    );
+        this.restartQueueKeys.delete(item.rootDid);
+
+        try {
+          await this.restartBlockletFromQueue(item);
+        } catch (err) {
+          logger.error('restart blocklet failed', err);
+        }
+      }
+    };
+
+    const processQueue = async () => {
+      while (!this.stopped) {
+        // 防止没有重启队列时，快速空转
+        await sleep(this.preCheckInterval);
+
+        if (this.restartQueue.length === 0) {
+          continue; // 等下一轮
+        }
+
+        // 创建固定数量 worker
+        const workers = [];
+        for (let i = 0; i < RESTART_CONCURRENCY; i++) {
+          workers.push(runWorker());
+        }
+
+        try {
+          await Promise.all(workers);
+        } catch (err) {
+          logger.error('restart queue processor batch failed', err);
+        }
+      }
+    };
+
+    processQueue().catch((err) => {
+      logger.error('restart queue processor failed', err);
+      this.restartQueueProcessing = false;
+    });
+  };
+
+  /**
+   * Create restart notification and audit log context
+   * @param {string} rootDid - Root blocklet DID
+   * @param {Object} childBlocklet - Child blocklet object
+   * @param {string[]} componentDids - Component DIDs
+   * @returns {Object} Context object with displayName, title, description
+   */
+  createRestartContext = async (rootDid, componentDids) => {
+    const blockletDisplayName = await this.getDisplayNameByRootDid(rootDid);
+    const componentNames = componentDids.map((componentDid) => {
+      const child = this.rootBlockletsInfo[rootDid]?.children?.find((bl) => bl.meta.did === componentDid);
+      return child ? this.getDisplayName(child) : componentDid;
+    });
+    const componentNamesStr = componentNames.length === 1 ? componentNames[0] : componentNames.join(', ');
+
+    return {
+      blockletDisplayName,
+      title: 'Blocklet health check failed',
+      description: `Blocklet ${blockletDisplayName} with component${componentNames.length > 1 ? 's' : ''} ${componentNamesStr} health check failed, restarting...`,
+    };
+  };
+
+  /**
+   * Handle restart success
+   * @param {string} key - Queue item key
+   * @param {string} rootDid - Root blocklet DID
+   * @param {string} componentDid - Component DID
+   * @param {Object} context - Restart context
+   */
+  handleRestartSuccess = (rootDid, componentDids, firstCycle, context) => {
+    if (firstCycle) {
+      return;
+    }
+    this.createAuditLog({
+      action: 'ensureBlockletRunning',
+      args: {
+        teamDid: rootDid,
+        componentDids,
+      },
+      context: {
+        user: {
+          did: rootDid,
+          role: 'daemon',
+          blockletDid: rootDid,
+          fullName: context.blockletDisplayName,
+          elevated: false,
+        },
+      },
+      result: {
+        title: context.title,
+        description: context.description,
+      },
+    });
+  };
+
+  /**
+   * Handle restart failure
+   * @param {string} key - Queue item key
+   * @param {string} rootDid - Root blocklet DID
+   * @param {string[]} componentDids - Component DIDs
+   * @param {Object} context - Restart context
+   * @param {Error} error - Error object
+   */
+  handleRestartFailure = (rootDid, componentDids, context, error) => {
+    const title = 'Restart blocklet failed when health check failed';
+    const description = `Ensure blocklet running failed, restart blocklet ${context.blockletDisplayName} with component${componentDids.length > 1 ? 's' : ''} ${componentDids.join(', ')} failed`;
+    this.notification(rootDid, title, description, 'error');
+    logger.error('restart many times blocklet failed', rootDid, componentDids, error);
+    try {
+      this.createAuditLog({
+        action: 'ensureBlockletRunning',
+        args: {
+          blockletDisplayName: context.blockletDisplayName,
+          teamDid: rootDid,
+          componentDids,
+        },
+        context: {
+          user: {
+            did: rootDid,
+            role: 'daemon',
+            blockletDid: rootDid,
+            fullName: context.blockletDisplayName,
+            elevated: false,
+          },
+        },
+        result: {
+          title,
+          description,
+        },
+      });
+    } catch (err) {
+      logger.error('ensure blocklet running, create audit log failed', rootDid, componentDids, err);
+    }
+  };
+
+  /**
+   * Restart a childBlocklet from the queue
+   * @param {Object} item - Queue item with rootDid, componentDids, firstCycle
+   */
+  restartBlockletFromQueue = async ({ rootDid, componentDids, firstCycle }) => {
+    // Set pending status to prevent duplicate processing
+    if (this.pendingJobs[rootDid]) {
+      logger.warn('Skip restart: job is already pending', { rootDid });
+      return;
+    }
+    this.pendingJobs[rootDid] = true;
+
+    try {
+      const context = await this.createRestartContext(rootDid, componentDids);
+      if (!firstCycle) {
+        this.notification(rootDid, context.title, context.description, 'warning');
+      }
+
+      logger.info('restart blocklet:', rootDid, componentDids);
+      await this.start({
+        did: rootDid,
+        componentDids,
+        checkHealthImmediately: true,
+        atomic: true,
+        operator: 'ensure-blocklet-running',
+      });
+      this.handleRestartSuccess(rootDid, componentDids, firstCycle, context);
+    } catch (e) {
+      await this.handleRestartFailure(rootDid, componentDids, context, e);
+    } finally {
+      // Clear pending status after processing
+      delete this.pendingJobs[rootDid];
+      // Clear progress blocklets time
+      for (const componentDid of componentDids) {
+        delete this.progressBlockletsTime[`${rootDid}-${componentDid}`];
+      }
+    }
   };
 }
 
-module.exports = new EnsureBlockletRunning();
+const ensureBlockletRunning = new EnsureBlockletRunning();
+
+module.exports = {
+  ensureBlockletRunning,
+  EnsureBlockletRunning,
+};

package/lib/blocklet/storage/backup/routing-rule.js

@@ -1,5 +1,6 @@
 const { outputJson } = require('fs-extra');
 const { join } = require('path');
+const { BLOCKLET_SITE_GROUP_SUFFIX } = require('@abtnode/constant');
 const states = require('../../../states');
 const { BaseBackup } = require('./base');
 const { getFileObject } = require('../utils/disk');
@@ -24,7 +25,7 @@ class RoutingRuleBackup extends BaseBackup {
    */
   async export() {
     const routingRule = await states.site.findOne({
-      domain: `${this.blocklet.meta.did}.blocklet-domain-group`,
+      domain: `${this.blocklet.meta.did}${BLOCKLET_SITE_GROUP_SUFFIX}`,
     });
 
     await outputJson(this.routingRuleExportPath, routingRule);

package/lib/migrations/1.5.15-site.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-continue */
 /* eslint-disable no-await-in-loop */
 const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+const { BLOCKLET_SITE_GROUP_SUFFIX } = require('@abtnode/constant');
 
 const findBlocklet = (site, blocklets) => {
   //  prefix = /
@@ -101,7 +102,7 @@ module.exports = async ({ states, node, printInfo }) => {
   // generate new blocklet site for every installed blocklet
   const newBlockletSites = {}; // <blockletDid>: <site>
   for (const blocklet of blocklets) {
-    const domain = `${blocklet.meta.did}.blocklet-domain-group`;
+    const domain = `${blocklet.meta.did}${BLOCKLET_SITE_GROUP_SUFFIX}`;
     newBlockletSites[blocklet.meta.did] = {
       domain,
       domainAliases: [],

package/lib/router/manager.js

@@ -56,7 +56,7 @@ const {
   revokeAndDeleteNFTDomainRecord,
   getAvailableGatewayPorts,
 } = require('../util/router');
-const checkDNS = require('../util/check-dns.js');
+const checkDNS = require('../util/check-dns');
 
 const checkPathPrefixInBlackList = (pathPrefix, extraBlackList = []) => {
   const blacklist = [

package/lib/util/blocklet.js

@@ -980,7 +980,7 @@ const startBlockletProcess = async (
       try {
         await promiseSpawn(nextOptions.env.connectInternalDockerNetwork, { mute: true });
       } catch (err) {
-        logger.error('blocklet connect internal docker network failed', { processId: processIdName, error: err });
+        logger.warn('blocklet connect internal docker network failed', { processId: processIdName, error: err });
       }
     }
 

package/lib/util/check-dns.js

@@ -38,8 +38,8 @@ async function checkIsRedirectedBlocklet(domain1, domain2) {
     });
 
     return domain1Blocklet === domain2Blocklet;
-  } catch (err) {
-    logger.error('DNS resolution error:', { domain1, domain2, error: err.message });
+  } catch (error) {
+    logger.error('DNS resolution error:', { domain1, domain2, error });
     return false;
   }
 }
@@ -66,7 +66,7 @@ async function checkDnsAndCname(domain, expectedCname = '') {
       hasCname: false,
       cnameRecords: [],
       isCnameMatch: false,
-      error: error.message,
+      error,
     };
   }
 }

package/lib/validators/org.js

@@ -1,7 +1,7 @@
 const { Joi } = require('@arcblock/validator');
 
 const createOrgInputSchema = Joi.object({
-  name: Joi.string().required().trim().min(1).max(20),
+  name: Joi.string().required().trim().min(1).max(64),
   description: Joi.string().optional().allow('').trim().min(1).max(255),
   ownerDid: Joi.DID().optional().allow('').allow(null),
 });

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.17.4-beta-20251203-225234-75da41dd",
+  "version": "1.17.4-beta-20251204-152224-243ff54f",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -17,21 +17,21 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "Apache-2.0",
   "dependencies": {
-    "@abtnode/analytics": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/auth": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/certificate-manager": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/constant": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/cron": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/db-cache": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/docker-utils": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/logger": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/models": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/queue": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/rbac": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/router-provider": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/static-server": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/timemachine": "1.17.4-beta-20251203-225234-75da41dd",
-    "@abtnode/util": "1.17.4-beta-20251203-225234-75da41dd",
+    "@abtnode/analytics": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/auth": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/certificate-manager": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/constant": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/cron": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/db-cache": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/docker-utils": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/logger": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/models": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/queue": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/rbac": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/router-provider": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/static-server": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/timemachine": "1.17.4-beta-20251204-152224-243ff54f",
+    "@abtnode/util": "1.17.4-beta-20251204-152224-243ff54f",
     "@aigne/aigne-hub": "^0.10.10",
     "@arcblock/did": "^1.27.12",
     "@arcblock/did-connect-js": "^1.27.12",
@@ -43,15 +43,15 @@
     "@arcblock/pm2-events": "^0.0.5",
     "@arcblock/validator": "^1.27.12",
     "@arcblock/vc": "^1.27.12",
-    "@blocklet/constant": "1.17.4-beta-20251203-225234-75da41dd",
+    "@blocklet/constant": "1.17.4-beta-20251204-152224-243ff54f",
     "@blocklet/did-space-js": "^1.2.6",
-    "@blocklet/env": "1.17.4-beta-20251203-225234-75da41dd",
+    "@blocklet/env": "1.17.4-beta-20251204-152224-243ff54f",
     "@blocklet/error": "^0.3.3",
-    "@blocklet/meta": "1.17.4-beta-20251203-225234-75da41dd",
-    "@blocklet/resolver": "1.17.4-beta-20251203-225234-75da41dd",
-    "@blocklet/sdk": "1.17.4-beta-20251203-225234-75da41dd",
-    "@blocklet/server-js": "1.17.4-beta-20251203-225234-75da41dd",
-    "@blocklet/store": "1.17.4-beta-20251203-225234-75da41dd",
+    "@blocklet/meta": "1.17.4-beta-20251204-152224-243ff54f",
+    "@blocklet/resolver": "1.17.4-beta-20251204-152224-243ff54f",
+    "@blocklet/sdk": "1.17.4-beta-20251204-152224-243ff54f",
+    "@blocklet/server-js": "1.17.4-beta-20251204-152224-243ff54f",
+    "@blocklet/store": "1.17.4-beta-20251204-152224-243ff54f",
     "@blocklet/theme": "^3.2.11",
     "@fidm/x509": "^1.2.1",
     "@ocap/mcrypto": "^1.27.12",
@@ -116,5 +116,5 @@
     "express": "^4.18.2",
     "unzipper": "^0.10.11"
   },
-  "gitHead": "d9c14c9b2763e632d2cc8d818dcd288ab08046e7"
+  "gitHead": "090b2f960b834168dfa12b2a559b9256a98be312"
 }