@abtnode/core 1.17.3-beta-20251120-052956-035abea6 → 1.17.3-beta-20251121-135300-8e451e6e

package/lib/blocklet/manager/disk.js

@@ -259,13 +259,13 @@ const { formatEnvironments, getBlockletMeta, validateOwner, isCLI } = util;
 
 const statusLock = new DBCache(() => ({
   prefix: 'blocklet-status-lock',
-  ttl: 1000 * 60,
+  ttl: 120_000,
   ...getAbtNodeRedisAndSQLiteUrl(),
 }));
 
 const startLock = new DBCache(() => ({
   prefix: 'blocklet-start-lock',
-  ttl: process.env.NODE_ENV === 'test' ? 100 : 1000 * 60,
+  ttl: process.env.NODE_ENV === 'test' ? 1_000 : 120_000,
   ...getAbtNodeRedisAndSQLiteUrl(),
 }));
 
@@ -740,6 +740,8 @@ class DiskBlockletManager extends BaseBlockletManager {
     e2eMode,
     context,
     atomic,
+    onStarted,
+    onError,
   }) {
     if (!blocklet.children) {
       return componentDids;
@@ -771,11 +773,17 @@ class DiskBlockletManager extends BaseBlockletManager {
       // start by dependency
       for (const dids of requiredDidsLayers) {
         if (atomic) {
-          await this._start({ blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: dids }, context);
+          await this._start(
+            { blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: dids, onStarted, onError },
+            context
+          );
         } else {
           const tasks = dids.map(
             (x) => () =>
-              this._start({ blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: [x] }, context)
+              this._start(
+                { blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: [x], onStarted, onError },
+                context
+              )
           );
           await pAll(tasks, { concurrency: 4 }).catch((err) => {
             throw new Error(err.errors ? err.errors.join(', ') : err);
@@ -801,8 +809,8 @@ class DiskBlockletManager extends BaseBlockletManager {
     },
     context
   ) {
-    const lockName = `start-not-input-component-dids-${did}`;
-    const shouldLock = !inputComponentDids?.length;
+    const lockName = `${did}-start`;
+    const shouldLock = inputComponentDids?.length === 0;
 
     if (shouldLock) {
       await startLock.acquire(lockName);
@@ -814,8 +822,16 @@ class DiskBlockletManager extends BaseBlockletManager {
         inputComponentDids?.length ? inputComponentDids : blocklet.children.map((x) => x.meta.did)
       );
 
+      try {
+        await this.deleteProcess({ did, componentDids: baseComponentDids, isGreen: true });
+      } catch (error) {
+        logger.warn('Failed to delete process; this warning can be safely ignored.', { error });
+      }
+
+      await this.checkControllerStatus(blocklet, 'start');
+
       // 优先启动资源型组件
-      let componentDids = [];
+      const componentDids = [];
       const nonEntryComponentIds = [];
       for (const b of blocklet.children) {
         if (!b?.meta?.did) {
@@ -844,7 +860,6 @@ class DiskBlockletManager extends BaseBlockletManager {
           operator,
         });
       }
-
       // sync component config before at first to ensure resource component config is ready
       const serverSk = (await states.node.read()).sk;
       await Promise.all(
@@ -853,7 +868,16 @@ class DiskBlockletManager extends BaseBlockletManager {
         )
       );
 
-      componentDids = await this.startRequiredComponents({
+      const doc1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.starting, {
+        componentDids,
+        operator,
+      });
+
+      this.emit(BlockletEvents.statusChange, doc1);
+      const startedBlockletDids = [];
+      const errorBlockletDids = [];
+
+      const notStartedComponentDids = await this.startRequiredComponents({
         componentDids,
         inputComponentDids,
         blocklet,
@@ -862,26 +886,15 @@ class DiskBlockletManager extends BaseBlockletManager {
         e2eMode,
         context,
         atomic,
+        onStarted: (subDid) => {
+          startedBlockletDids.push({ did: subDid });
+        },
+        onError: (subDid, error) => {
+          errorBlockletDids.push({ did: subDid, error });
+        },
       });
 
-      if (atomic || !blocklet.structVersion) {
-        const nextBlocklet = await this._start(
-          { blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids, operator },
-          context
-        );
-
-        // 根据情况更新 route table， 会判断只有包含多 interfaces 的 DID 才会更新 route table
-        if (!['true', '1'].includes(process.env.ABT_NODE_DISABLE_BLUE_GREEN)) {
-          this.emit(BlockletEvents.blurOrGreenStarted, {
-            blocklet: nextBlocklet,
-            componentDids: inputComponentDids,
-            context,
-          });
-        }
-        return nextBlocklet;
-      }
-
-      const tasks = componentDids.map(
+      const tasks = notStartedComponentDids.map(
         (componentDid) => () =>
           this._start(
             {
@@ -891,25 +904,104 @@ class DiskBlockletManager extends BaseBlockletManager {
               e2eMode,
               componentDids: [componentDid],
               operator,
+              onStarted: (subDid) => {
+                startedBlockletDids.push({ did: subDid });
+              },
+              onError: (subDid, error) => {
+                errorBlockletDids.push({ did: subDid, error });
+              },
             },
             context
           )
       );
 
-      const rest = await pAll(tasks, { concurrency: 1 });
+      await pAll(tasks, { concurrency: 6 });
 
       const nextBlocklet = await this.ensureBlocklet(did, { e2eMode });
+      let errorDescription = '';
+      let resultBlocklet = nextBlocklet;
+
+      if (startedBlockletDids.length) {
+        await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
+          componentDids: startedBlockletDids.map((x) => x.did),
+          operator,
+        });
+
+        const finalBlocklet = await this.getBlocklet(did);
+        resultBlocklet = finalBlocklet;
+
+        await this.configSynchronizer.throttledSyncAppConfig(finalBlocklet);
+        const componentsInfo = getComponentsInternalInfo(finalBlocklet);
+        this.emit(BlockletInternalEvents.componentUpdated, {
+          appDid: blocklet.appDid,
+          components: componentsInfo,
+        });
+        this.emit(BlockletInternalEvents.componentStarted, {
+          appDid: blocklet.appDid,
+          components: (componentDids || []).map((x) => ({ did: x })),
+        });
+
+        this.emit(BlockletEvents.statusChange, finalBlocklet);
+        this.emit(BlockletEvents.started, { ...finalBlocklet, componentDids });
+
+        launcher.notifyBlockletStarted(finalBlocklet);
+      }
+
+      if (errorBlockletDids.length) {
+        const error = errorBlockletDids[0];
+        for (const item of errorBlockletDids) {
+          logger.error('Failed to start blocklet', {
+            error: item.error,
+            title: nextBlocklet.meta.title,
+            name: getComponentNamesWithVersion(nextBlocklet, [item.did]),
+          });
+        }
+
+        errorDescription = `${getComponentNamesWithVersion(
+          nextBlocklet,
+          errorBlockletDids.map((x) => x.did)
+        )} start failed for ${getDisplayName(nextBlocklet)}: ${errorBlockletDids.map((x) => x.error.message).join('. ')}`;
+        this._createNotification(did, {
+          title: 'Component start failed',
+          description: errorDescription,
+          entityType: 'blocklet',
+          entityId: did,
+          severity: 'error',
+        });
+
+        await this.deleteProcess({
+          did,
+          componentDids: errorBlockletDids.map((x) => x.did),
+          shouldUpdateBlockletStatus: false,
+        });
+        await states.blocklet.setBlockletStatus(did, BlockletStatus.error, {
+          componentDids: errorBlockletDids.map((x) => x.did),
+          operator,
+        });
+        const finalBlocklet = await this.getBlocklet(did);
+        resultBlocklet = finalBlocklet;
+        this.emit(BlockletEvents.startFailed, {
+          ...finalBlocklet,
+          componentDids: errorBlockletDids.map((x) => x.did),
+          error: { message: error.message },
+        });
+        this.emit(BlockletEvents.statusChange, { ...finalBlocklet, error: { message: error.message } });
+      }
 
       // 根据情况更新 route table， 会判断只有包含多 interfaces 的 DID 才会更新 route table
       if (!['true', '1'].includes(process.env.ABT_NODE_DISABLE_BLUE_GREEN)) {
         this.emit(BlockletEvents.blurOrGreenStarted, {
-          blocklet: nextBlocklet,
+          blocklet: resultBlocklet,
           componentDids: inputComponentDids,
           context,
         });
       }
 
-      return rest[0];
+      if (throwOnError && errorDescription) {
+        throw new Error(errorDescription);
+      }
+
+      return resultBlocklet;
     } finally {
       if (shouldLock) {
         await startLock.releaseLock(lockName);
@@ -926,17 +1018,13 @@ class DiskBlockletManager extends BaseBlockletManager {
       e2eMode = false,
       componentDids,
       operator: _operator,
+      onStarted,
+      onError,
     },
     context
   ) {
     const operator = _operator || context?.user?.did;
 
-    try {
-      await this.deleteProcess({ did, componentDids, isGreen: true });
-    } catch {
-      // do nothing
-    }
-
     logger.info('start blocklet', {
       did: did || inputBlocklet.meta.did,
       componentDids,
@@ -960,7 +1048,6 @@ class DiskBlockletManager extends BaseBlockletManager {
 
     const entryComponentIds = [];
     const nonEntryComponentIds = [];
-    let nonEntryComponentRes;
     const componentDidsSet = new Set(componentDids);
     try {
       await forEachBlocklet(
@@ -996,18 +1083,9 @@ class DiskBlockletManager extends BaseBlockletManager {
         { parallel: true, concurrencyLimit: 4 }
       );
       if (nonEntryComponentIds.length) {
-        const params = {
-          did,
-          context,
-          minConsecutiveTime: 3000,
-          timeout: 5000,
-          componentDids: nonEntryComponentIds,
-        };
-        nonEntryComponentRes = await this._onCheckIfStarted(params, {
-          throwOnError,
-          skipRunningCheck: true,
-          needUpdateBlueStatus: false,
-        });
+        for (const subDid of nonEntryComponentIds) {
+          onStarted?.(subDid);
+        }
       }
     } catch (err) {
       logger.error('Failed to categorize components into entry and non-entry types', { error: err.message });
@@ -1015,7 +1093,7 @@ class DiskBlockletManager extends BaseBlockletManager {
     }
 
     if (!entryComponentIds.length) {
-      return nonEntryComponentRes;
+      return;
     }
 
     try {
@@ -1036,13 +1114,7 @@ class DiskBlockletManager extends BaseBlockletManager {
         }
       }
 
-      const doc1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.starting, {
-        componentDids: entryComponentIds,
-        operator,
-      });
-
       blocklet1.status = BlockletStatus.starting;
-      this.emit(BlockletEvents.statusChange, doc1);
       const blocklet = await ensureAppPortsNotOccupied({
         blocklet: blocklet1,
         componentDids: entryComponentIds,
@@ -1097,8 +1169,6 @@ class DiskBlockletManager extends BaseBlockletManager {
           });
         };
 
-      await this.configSynchronizer.syncAppConfig(blocklet);
-
       await startBlockletProcess(blocklet, {
         ...context,
         preFlight: getHookFn('preFlight'),
@@ -1116,6 +1186,7 @@ class DiskBlockletManager extends BaseBlockletManager {
         checkHealthImmediately,
         componentDids,
       });
+
       const params = {
         did,
         context,
@@ -1124,47 +1195,19 @@ class DiskBlockletManager extends BaseBlockletManager {
         componentDids: entryComponentIds,
       };
 
-      await this._onCheckIfStarted(params, { throwOnError, needUpdateBlueStatus: false });
-      blocklet.status = BlockletStatus.running;
+      await this._onCheckIfStarted(params, { throwOnError });
 
-      return blocklet;
+      for (const subDid of entryComponentIds) {
+        onStarted?.(subDid);
+      }
     } catch (err) {
       const status = await states.blocklet.getBlockletStatus(did);
       if ([BlockletStatus.stopping, BlockletStatus.stopped].includes(status)) {
         logger.info('Failed to start blocklet maybe due to manually stopped');
-        return states.blocklet.getBlocklet(did);
       }
-
-      const error = Array.isArray(err) ? err[0] : err;
-      logger.error('Failed to start blocklet', { error, did, title: blocklet1.meta.title });
-      const description = `${getComponentNamesWithVersion(blocklet1, entryComponentIds)} start failed for ${getDisplayName(
-        blocklet1
-      )}: ${error.message}`;
-      this._createNotification(did, {
-        title: 'Component start failed',
-        description,
-        entityType: 'blocklet',
-        entityId: did,
-        severity: 'error',
-      });
-
-      await this.deleteProcess({ did, componentDids: entryComponentIds });
-      const res = await states.blocklet.setBlockletStatus(did, BlockletStatus.error, {
-        componentDids: entryComponentIds,
-        operator,
-      });
-      this.emit(BlockletEvents.startFailed, {
-        ...res,
-        componentDids: entryComponentIds,
-        error: { message: error.message },
-      });
-      this.emit(BlockletEvents.statusChange, { ...res, error: { message: error.message } });
-
-      if (throwOnError) {
-        throw new Error(description);
+      for (const subDid of entryComponentIds) {
+        onError?.(subDid, err);
       }
-
-      return res;
     }
   }
 
@@ -3590,141 +3633,24 @@ class DiskBlockletManager extends BaseBlockletManager {
     await blueGreenStartBlocklet({ did, componentDids, operator }, context, this, states);
   }
 
-  _onCheckIfStarted = async (
-    jobInfo,
-    { throwOnError, skipRunningCheck = false, isGreen = false, needUpdateBlueStatus = true } = {}
-  ) => {
-    const startedAt = Date.now();
-    const { did, context, minConsecutiveTime = 2000, timeout, componentDids } = jobInfo;
-    const nodeInfo = await states.node.read();
+  _onCheckIfStarted = async (jobInfo, { skipRunningCheck = false, isGreen = false } = {}) => {
+    const { did, minConsecutiveTime = 2000, timeout, componentDids } = jobInfo;
     const blocklet = await this.getBlocklet(did);
 
-    const { meta } = blocklet;
-    const { name } = meta;
-
-    const successBlockletIds = new Set();
-
-    try {
-      if (!skipRunningCheck) {
-        await checkBlockletProcessHealthy(blocklet, {
-          minConsecutiveTime,
-          timeout,
-          componentDids,
-          isGreen,
-          appDid: did,
-          enableDocker: nodeInfo.enableDocker,
-          getBlocklet: () => {
-            return this.getBlocklet(did);
-          },
-          setBlockletRunning: async (componentDid) => {
-            successBlockletIds.add(componentDid);
-            await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
-              componentDids: [componentDid],
-              isGreen,
-            });
-          },
-        });
-      }
+    const nodeInfo = await states.node.read();
 
-      const runningRes = await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
-        componentDids: successBlockletIds.size > 0 ? Array.from(successBlockletIds) : componentDids,
+    if (!skipRunningCheck) {
+      await checkBlockletProcessHealthy(blocklet, {
+        minConsecutiveTime,
+        timeout,
+        componentDids,
         isGreen,
+        appDid: did,
+        enableDocker: nodeInfo.enableDocker,
+        getBlocklet: () => {
+          return this.getBlocklet(did);
+        },
       });
-
-      if (needUpdateBlueStatus) {
-        await states.blocklet.setBlockletStatus(did, BlockletStatus.stopped, {
-          componentDids,
-          isGreen: !isGreen,
-        });
-      }
-
-      const nextBlocklet = await this.getBlocklet(did);
-
-      await this.configSynchronizer.throttledSyncAppConfig(nextBlocklet);
-      const componentsInfo = getComponentsInternalInfo(nextBlocklet);
-      this.emit(BlockletInternalEvents.componentUpdated, {
-        appDid: blocklet.appDid,
-        components: componentsInfo,
-      });
-      this.emit(BlockletInternalEvents.componentStarted, {
-        appDid: blocklet.appDid,
-        components: (componentDids || []).map((x) => ({ did: x })),
-      });
-
-      this.emit(BlockletEvents.statusChange, nextBlocklet);
-      this.emit(BlockletEvents.started, { ...nextBlocklet, componentDids });
-
-      launcher.notifyBlockletStarted(blocklet);
-
-      logger.info('blocklet healthy', { did, name, time: Date.now() - startedAt });
-
-      if (needUpdateBlueStatus) {
-        try {
-          await this.deleteProcess({ did, componentDids, isGreen: !isGreen }, context);
-        } catch {
-          logger.error('delete process failed', { did, componentDids, isGreen });
-        }
-      }
-
-      return runningRes;
-    } catch (error) {
-      const errorBlockletIds = [];
-      for (const componentDid of componentDids) {
-        if (!successBlockletIds.has(componentDid)) {
-          errorBlockletIds.push(componentDid);
-        }
-      }
-      if (errorBlockletIds.length === 0) {
-        return this.getBlocklet(did);
-      }
-
-      const status = await states.blocklet.getBlockletStatus(did);
-      if ([BlockletStatus.stopping, BlockletStatus.stopped].includes(status)) {
-        logger.info(`Check blocklet healthy failing because blocklet is ${fromBlockletStatus(status)}`);
-        return {};
-      }
-
-      logger.error('check blocklet if started failed', { did, name, context, timeout, error });
-
-      await this.deleteProcess({ did, componentDids: errorBlockletIds, isGreen }, context);
-
-      let doc;
-      if (isGreen) {
-        doc = await states.blocklet.setBlockletStatus(did, BlockletStatus.stopped, {
-          componentDids: errorBlockletIds,
-          isGreen,
-        });
-      } else {
-        doc = await states.blocklet.setBlockletStatus(did, BlockletStatus.error, { componentDids: errorBlockletIds });
-      }
-
-      const componentNames = getComponentNamesWithVersion(blocklet, errorBlockletIds);
-      const description = componentNames
-        ? `${getComponentNamesWithVersion(blocklet, errorBlockletIds)} start failed for ${getDisplayName(blocklet)}: ${
-            error.message
-          }`
-        : `${blocklet.meta.title} start failed: ${error.message}`;
-
-      this._createNotification(did, {
-        title: 'Component start failed',
-        description,
-        entityType: 'blocklet',
-        entityId: did,
-        severity: 'error',
-      });
-
-      this.emit(BlockletEvents.startFailed, {
-        ...doc,
-        componentDids: errorBlockletIds,
-        error: { message: error.message },
-      });
-      this.emit(BlockletEvents.statusChange, { ...doc, error });
-
-      if (throwOnError) {
-        throw error;
-      }
-
-      return doc;
     }
   };
 

package/lib/blocklet/manager/helper/blue-green-start-blocklet.js

@@ -10,6 +10,7 @@ const {
 } = require('@blocklet/meta/lib/util');
 const { getBlockletEngine, hasStartEngine } = require('@blocklet/meta/lib/engine');
 const { getComponentsInternalInfo } = require('@blocklet/meta/lib/blocklet');
+const pAll = require('p-all');
 const {
   forEachBlocklet,
   validateBlocklet,
@@ -18,6 +19,7 @@ const {
   getHealthyCheckTimeout,
   getHookArgs,
   shouldSkipComponent,
+  getComponentNamesWithVersion,
 } = require('../../../util/blocklet');
 const { startBlockletProcess } = require('../../../util/blocklet');
 const hooks = require('../../hooks');
@@ -169,143 +171,256 @@ const blueGreenStartBlocklet = async (
 
   const blueGreenComponentIds = await blueGreenGetComponentIds(blocklet1, entryComponentIds);
 
-  // eslint-disable-next-line no-unreachable-loop
+  const startedBlockletDids = [];
+  const errorBlockletDids = [];
+
   for (const item of blueGreenComponentIds) {
     if (!item.componentDids.length) {
       continue;
     }
 
-    try {
-      const doc1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.starting, {
-        componentDids: item.componentDids,
-        operator,
-        isGreen: item.changeToGreen,
-      });
+    await states.blocklet.setBlockletStatus(did, BlockletStatus.starting, {
+      componentDids: item.componentDids,
+      operator,
+      isGreen: item.changeToGreen,
+    });
 
-      const nextBlocklet = await ensureAppPortsNotOccupied({
-        blocklet: blocklet1,
-        componentDids: item.componentDids,
-        states,
-        manager,
-        isGreen: item.changeToGreen,
-      });
+    await ensureAppPortsNotOccupied({
+      blocklet: blocklet1,
+      componentDids: item.componentDids,
+      states,
+      manager,
+      isGreen: item.changeToGreen,
+    });
+  }
 
-      manager.emit(BlockletEvents.statusChange, doc1);
+  const nextBlocklet = await manager.ensureBlocklet(did, { e2eMode });
 
-      const nodeInfo = await states.node.read();
-      const nodeEnvironments = await states.node.getEnvironments();
+  manager.emit(BlockletEvents.statusChange, nextBlocklet);
 
-      // 钩子函数设置
-      const getHookFn =
-        (hookName) =>
-        async (b, { env }) => {
-          const hookArgs = getHookArgs(b);
-          const needRunDocker = await checkNeedRunDocker(b.meta, env, nodeInfo, isExternalBlocklet(nextBlocklet));
-          if (!b.meta.scripts?.[hookName]) {
-            return null;
-          }
-          if (needRunDocker) {
-            return dockerExec({
-              blocklet: nextBlocklet,
-              meta: b.meta,
-              script: b.meta.scripts?.[hookName],
-              hookName,
-              nodeInfo,
-              env,
-              ...hookArgs,
-            });
-          }
-          return hooks[hookName](b, {
-            appDir: b.env.appDir,
-            hooks: Object.assign(b.meta.hooks || {}, b.meta.scripts || {}),
-            env,
-            did, // root blocklet did,
-            teamManager: manager.teamManager,
-            ...hookArgs,
+  // 收集所有任务
+  const tasks = [];
+  for (const item of blueGreenComponentIds) {
+    if (!item.componentDids.length) {
+      continue;
+    }
+
+    for (const subDid of item.componentDids) {
+      tasks.push(async () => {
+        try {
+          const nodeInfo = await states.node.read();
+          const nodeEnvironments = await states.node.getEnvironments();
+
+          // 钩子函数设置
+          const getHookFn =
+            (hookName) =>
+            async (b, { env }) => {
+              const hookArgs = getHookArgs(b);
+              const needRunDocker = await checkNeedRunDocker(b.meta, env, nodeInfo, isExternalBlocklet(nextBlocklet));
+              if (!b.meta.scripts?.[hookName]) {
+                return null;
+              }
+              if (needRunDocker) {
+                return dockerExec({
+                  blocklet: nextBlocklet,
+                  meta: b.meta,
+                  script: b.meta.scripts?.[hookName],
+                  hookName,
+                  nodeInfo,
+                  env,
+                  ...hookArgs,
+                });
+              }
+              return hooks[hookName](b, {
+                appDir: b.env.appDir,
+                hooks: Object.assign(b.meta.hooks || {}, b.meta.scripts || {}),
+                env,
+                did, // root blocklet did,
+                teamManager: manager.teamManager,
+                ...hookArgs,
+              });
+            };
+
+          await startBlockletProcess(nextBlocklet, {
+            ...context,
+            preFlight: getHookFn('preFlight'),
+            preStart: getHookFn('preStart'),
+            postStart: getHookFn('postStart'),
+            nodeEnvironments,
+            nodeInfo,
+            e2eMode,
+            componentDids: [subDid],
+            configSynchronizer: manager.configSynchronizer,
+            isGreen: item.changeToGreen,
+          });
+
+          // 健康检查绿色环境
+          const { startTimeout, minConsecutiveTime } = getHealthyCheckTimeout(nextBlocklet, {
+            checkHealthImmediately,
+            componentDids: [subDid],
           });
-        };
-
-      await startBlockletProcess(nextBlocklet, {
-        ...context,
-        preFlight: getHookFn('preFlight'),
-        preStart: getHookFn('preStart'),
-        postStart: getHookFn('postStart'),
-        nodeEnvironments,
-        nodeInfo,
-        e2eMode,
-        componentDids: item.componentDids,
-        configSynchronizer: manager.configSynchronizer,
-        isGreen: item.changeToGreen,
+
+          await manager._onCheckIfStarted(
+            {
+              did,
+              context,
+              minConsecutiveTime,
+              timeout: startTimeout,
+              componentDids: [subDid],
+            },
+            { throwOnError: true, isGreen: item.changeToGreen }
+          );
+
+          // 收集成功的组件（排除已经在 errorBlockletDids 中的组件）
+          startedBlockletDids.push({ did: subDid, isGreen: item.changeToGreen });
+
+          logger.info('Green environment started successfully', {
+            did,
+            componentDids: [subDid],
+          });
+        } catch (err) {
+          const error = Array.isArray(err) ? err[0] : err;
+          logger.error('Failed to start green environment', { error, did, title: blocklet1.meta.title });
+
+          // 收集失败的组件
+          errorBlockletDids.push({ did: subDid, error, isGreen: item.changeToGreen });
+
+          try {
+            await manager.deleteProcess({ did, componentDids: [subDid], isGreen: item.changeToGreen });
+          } catch (cleanupError) {
+            logger.error('Failed to cleanup green environment', { cleanupError });
+          }
+        }
+      });
+    }
+  }
+
+  await pAll(tasks, { concurrency: 6 });
+
+  const lastBlocklet = await manager.ensureBlocklet(did, { e2eMode });
+  let errorDescription = '';
+
+  // 处理启动失败的组件
+  if (errorBlockletDids.length) {
+    const { error } = errorBlockletDids[0];
+    for (const item of errorBlockletDids) {
+      logger.error('Failed to start blocklet', {
+        error: item.error,
+        title: lastBlocklet.meta.title,
+        name: getComponentNamesWithVersion(lastBlocklet, [item.did]),
+      });
+    }
+
+    errorDescription = `${getComponentNamesWithVersion(
+      lastBlocklet,
+      errorBlockletDids.map((x) => x.did)
+    )} start failed for ${getDisplayName(lastBlocklet)}: ${errorBlockletDids.map((x) => x.error.message).join('. ')}`;
+
+    if (!ignoreErrorNotification) {
+      manager._createNotification(did, {
+        title: 'Blue-Green Deployment: Green Start Failed',
+        description: errorDescription,
+        entityType: 'blocklet',
+        entityId: did,
+        severity: 'error',
       });
+    }
+
+    const greenBlockletDids = errorBlockletDids.filter((x) => x.isGreen).map((x) => x.did);
+    const blueBlockletDids = errorBlockletDids.filter((x) => !x.isGreen).map((x) => x.did);
 
-      // 健康检查绿色环境
-      const { startTimeout, minConsecutiveTime } = getHealthyCheckTimeout(nextBlocklet, {
-        checkHealthImmediately,
-        componentDids: item.componentDids,
+    if (greenBlockletDids.length) {
+      await manager.deleteProcess({
+        did,
+        componentDids: greenBlockletDids,
+        shouldUpdateBlockletStatus: false,
+        isGreen: true,
       });
+      await states.blocklet.setBlockletStatus(did, BlockletStatus.error, {
+        componentDids: greenBlockletDids,
+        operator,
+        isGreen: true,
+      });
+    }
 
-      await manager._onCheckIfStarted(
-        {
-          did,
-          context,
-          minConsecutiveTime,
-          timeout: startTimeout,
-          componentDids: item.componentDids,
-        },
-        { throwOnError: true, isGreen: item.changeToGreen, needUpdateBlueStatus: true }
-      );
-
-      logger.info('Green environment started successfully', {
+    if (blueBlockletDids.length) {
+      await manager.deleteProcess({
         did,
-        componentDids: item.componentDids,
+        componentDids: blueBlockletDids,
+        shouldUpdateBlockletStatus: false,
+        isGreen: false,
       });
-    } catch (err) {
-      const error = Array.isArray(err) ? err[0] : err;
-      logger.error('Failed to start green environment', { error, did, title: blocklet1.meta.title });
-
-      try {
-        await manager.deleteProcess({ did, componentDids: item.componentDids, isGreen: item.changeToGreen });
-        // 如果需要把失败状态设置成 error:
-        // const doc1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.error, {
-        //   componentDids: item.componentDids,
-        //   operator,
-        //   isGreen: item.changeToGreen,
-        // });
-        // manager.emit(BlockletEvents.statusChange, doc1);
-        manager.emit(BlockletEvents.statusChange, blocklet1);
-      } catch (cleanupError) {
-        logger.error('Failed to cleanup green environment', { cleanupError });
-      }
+      await states.blocklet.setBlockletStatus(did, BlockletStatus.error, {
+        componentDids: blueBlockletDids,
+        operator,
+        isGreen: false,
+      });
+    }
 
-      const description = `Green environment start failed for ${getDisplayName(blocklet1)}: ${error.message}`;
-      if (!ignoreErrorNotification) {
-        manager._createNotification(did, {
-          title: 'Blue-Green Deployment: Green Start Failed',
-          description,
-          entityType: 'blocklet',
-          entityId: did,
-          severity: 'error',
-        });
-      }
+    const finalBlocklet = await manager.getBlocklet(did);
+    manager.emit(BlockletEvents.startFailed, {
+      ...finalBlocklet,
+      componentDids: errorBlockletDids.map((x) => x.did),
+      error: { message: error.message },
+    });
+    manager.emit(BlockletEvents.statusChange, { ...finalBlocklet, error: { message: error.message } });
+  }
 
-      if (throwOnError) {
-        throw new Error(description);
-      }
-      throw error;
+  // 处理成功启动的组件
+  if (startedBlockletDids.length) {
+    const startedGreenBlockletDids = startedBlockletDids.filter((x) => x.isGreen).map((x) => x.did);
+    const startedBlueBlockletDids = startedBlockletDids.filter((x) => !x.isGreen).map((x) => x.did);
+
+    if (startedGreenBlockletDids.length) {
+      await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
+        componentDids: startedGreenBlockletDids,
+        operator,
+        isGreen: true,
+      });
+      await manager.deleteProcess({ did, componentDids: startedGreenBlockletDids, isGreen: false });
     }
 
-    const nextBlocklet = await manager.ensureBlocklet(did, { e2eMode });
-    manager.emit(BlockletEvents.blurOrGreenStarted, { blocklet: nextBlocklet, componentDids, context });
+    if (startedBlueBlockletDids.length) {
+      await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
+        componentDids: startedBlueBlockletDids,
+        operator,
+        isGreen: false,
+      });
+      await manager.deleteProcess({ did, componentDids: startedBlueBlockletDids, isGreen: true });
+    }
+
+    const finalBlocklet = await manager.getBlocklet(did);
+
+    await manager.configSynchronizer.throttledSyncAppConfig(finalBlocklet);
+    const componentsInfo = getComponentsInternalInfo(finalBlocklet);
+    manager.emit(BlockletInternalEvents.componentUpdated, {
+      appDid: blocklet1.appDid,
+      components: componentsInfo,
+    });
+    manager.emit(BlockletInternalEvents.componentStarted, {
+      appDid: blocklet1.appDid,
+      components: startedBlockletDids.map((x) => ({ did: x.did })),
+    });
+
+    manager.emit(BlockletEvents.statusChange, finalBlocklet);
+    manager.emit(BlockletEvents.started, {
+      ...finalBlocklet,
+      componentDids: startedBlockletDids.map((x) => x.did),
+    });
   }
 
-  const nextBlocklet = await manager.getBlocklet(did);
-  const componentsInfo = getComponentsInternalInfo(nextBlocklet);
-  manager.emit(BlockletInternalEvents.componentStarted, {
-    appDid: nextBlocklet.appDid,
-    components: componentsInfo,
-  });
+  // 根据情况更新 route table， 会判断只有包含多 interfaces 的 DID 才会更新 route table
+  if (!['true', '1'].includes(process.env.ABT_NODE_DISABLE_BLUE_GREEN)) {
+    manager.emit(BlockletEvents.blurOrGreenStarted, {
+      blocklet: lastBlocklet,
+      componentDids,
+      context,
+    });
+  }
 
-  manager.emit(BlockletEvents.statusChange, nextBlocklet);
+  if (throwOnError && errorDescription) {
+    throw new Error(errorDescription);
+  }
 };
 
 module.exports = {

package/lib/blocklet/manager/helper/blue-green-upgrade-blocklet.js

@@ -158,9 +158,19 @@ const blueGreenUpgradeBlocklet = async (
     return blocklet;
   } catch (err) {
     logger.error('failed to upgrade blocklet', { did, version, name, error: err });
+    const nextBlocklet = await manager.ensureBlocklet(did);
+    for (const child of nextBlocklet.children) {
+      for (const oldChild of oldBlocklet.children) {
+        if (child && oldChild && child.meta.did === oldChild.meta.did) {
+          oldChild.status = child.status;
+          oldChild.greenStatus = child.greenStatus;
+        }
+      }
+    }
     await states.blocklet.upgradeBlocklet({ ...oldBlocklet });
     manager.configSynchronizer.throttledSyncAppConfig(oldBlocklet);
     await manager._updateDependents(did);
+    manager.emit(BlockletEvents.statusChange, { ...oldBlocklet, error: { message: err.message } });
 
     const actionName = action === INSTALL_ACTIONS.INSTALL_COMPONENT ? 'install' : 'upgrade';
     const notificationEvent =

package/lib/states/blocklet.js

@@ -2,7 +2,6 @@
 /* eslint-disable no-await-in-loop */
 /* eslint-disable function-paren-newline */
 /* eslint-disable no-underscore-dangle */
-const pick = require('lodash/pick');
 const omit = require('lodash/omit');
 const uniq = require('lodash/uniq');
 const cloneDeep = require('@abtnode/util/lib/deep-clone');
@@ -15,7 +14,6 @@ const {
   forEachBlocklet,
   forEachBlockletSync,
   forEachComponentV2,
-  forEachComponentV2Sync,
   getBlockletServices,
   hasStartEngine,
 } = require('@blocklet/meta/lib/util');
@@ -591,14 +589,13 @@ class BlockletState extends BaseState {
         return res;
       }
 
-      // update children status
-      forEachComponentV2Sync(doc, (component) => {
+      for (const component of doc.children || []) {
         if (component.meta.group === BlockletGroup.gateway) {
-          return;
+          continue;
         }
 
         if (shouldSkipComponent(component.meta.did, componentDids)) {
-          return;
+          continue;
         }
 
         component[isGreen ? 'greenStatus' : 'status'] = status;
@@ -616,11 +613,16 @@ class BlockletState extends BaseState {
         }
         component.operator = operator;
         component.inProgressStart = Date.now();
-      });
+      }
 
-      const isSetStatus = status === BlockletStatus.downloading || status === BlockletStatus.waiting;
-      const updateData = isSetStatus ? pick(doc, ['status', 'children']) : pick(doc, ['children']);
-      updateData.operator = operator;
+      const shouldSetStatus = status === BlockletStatus.downloading || status === BlockletStatus.waiting;
+      const updateData = {
+        children: doc.children,
+        operator,
+      };
+      if (shouldSetStatus) {
+        updateData.status = status;
+      }
 
       const res = await this.updateBlocklet(did, updateData);
       return res;

package/lib/util/blocklet.js

@@ -781,15 +781,15 @@ const startBlockletProcess = async (
       namespace: 'blocklets',
       name: processIdName,
       cwd,
-      // FIXME @linchen [] does not work, so use () here
       log_date_format: '(YYYY-MM-DD HH:mm:ss)',
       output: path.join(logsDir, 'output.log'),
       error: path.join(logsDir, 'error.log'),
-      wait_ready: process.env.NODE_ENV !== 'test',
+      // wait_ready: process.env.NODE_ENV !== 'test',
+      wait_ready: false,
       listen_timeout: 3000,
       max_memory_restart: `${maxMemoryRestart}M`,
       max_restarts: b.mode === BLOCKLET_MODES.DEVELOPMENT ? 0 : 3,
-      min_uptime: 60 * 1000,
+      min_uptime: 10_000,
       exp_backoff_restart_delay: 300,
       env: omit(
         {
@@ -1142,9 +1142,52 @@ const _checkProcessHealthy = async (
         timeout,
         doConsecutiveCheck: blocklet.mode !== BLOCKLET_MODES.DEVELOPMENT,
         waitTCP: !webInterface,
+        shouldAbort: async () => {
+          // Check if pm2 process exists and is online
+          try {
+            const info = await getProcessInfo(processId, { timeout: 3_000 });
+            const currentStatus = info.pm2_env.status;
+            if (currentStatus !== 'online') {
+              throw new Error(`pm2 process ${processId} status is ${currentStatus}, not online`);
+            }
+          } catch (err) {
+            // If process doesn't exist or has error, abort immediately
+            logger.error('pm2 process check failed in shouldAbort', { appDid, error: err, processId, name });
+            const isProcessNotExist =
+              err.message &&
+              (err.message.includes('not found') ||
+                err.message.includes('does not exist') ||
+                err.message.includes('not running'));
+            if (isProcessNotExist) {
+              throw new Error(`pm2 process ${processId} (${name}) died or does not exist: ${err.message}`);
+            }
+            throw new Error(`pm2 process ${processId} (${name}) check failed: ${err.message}`);
+          }
+        },
       });
     } catch (error) {
-      logger.error('ensure endpoint healthy failed', { appDid, port, minConsecutiveTime, timeout });
+      const isProcessDead =
+        error.message &&
+        (error.message.includes('pm2 process') ||
+          error.message.includes('died') ||
+          error.message.includes('does not exist'));
+      if (isProcessDead) {
+        logger.error('blocklet process died during health check', {
+          appDid,
+          processId,
+          name,
+          port,
+          error: error.message,
+        });
+        throw error;
+      }
+      logger.error('ensure endpoint healthy failed', {
+        appDid,
+        port,
+        minConsecutiveTime,
+        timeout,
+        error: error.message,
+      });
       throw error;
     }
   } catch (error) {

package/lib/util/docker/docker-exec-chown.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const { DBCache, getAbtNodeRedisAndSQLiteUrl } = require('@abtnode/db-cache');
 const promiseSpawn = require('@abtnode/util/lib/promise-spawn');
 const logger = require('@abtnode/logger')('@abtnode/docker-exec-chown');
+const sleep = require('@abtnode/util/lib/sleep');
 const debianChmodDockerfile = require('./debian-chmod-dockerfile');
 const parseDockerName = require('./parse-docker-name');
 const { checkDockerHasImage } = require('./check-docker-has-image');
@@ -65,10 +66,11 @@ async function dockerExecChown({ name, dirs, code = 777, force = false }) {
     await promiseSpawn(
       `docker rm -fv ${realName} > /dev/null 2>&1 || true && docker run --rm --name ${realName} ${volumes} ${image} sh -c '${command}'`,
       {},
-      { timeout: 1000 * 120, retry: 3 }
+      { timeout: 1000 * 120, retry: 0 }
     );
   } finally {
     await promiseSpawn(`docker rm -fv ${realName} > /dev/null 2>&1 || true`, {}, { timeout: 1000 * 10, retry: 3 });
+    await sleep(500);
     await lockFile.releaseLock(realName);
   }
 

package/lib/util/docker/docker-exec.js

@@ -5,6 +5,7 @@ const promiseSpawn = require('@abtnode/util/lib/promise-spawn');
 const { DBCache, getAbtNodeRedisAndSQLiteUrl } = require('@abtnode/db-cache');
 const logger = require('@abtnode/logger')('@abtnode/docker-exec');
 
+const sleep = require('@abtnode/util/lib/sleep');
 const parseDockerOptionsFromPm2 = require('./parse-docker-options-from-pm2');
 const { checkDockerInstalled } = require('./check-docker-installed');
 
@@ -26,7 +27,7 @@ async function dockerExec({
   runScriptParams,
   nodeInfo,
   timeout = 120_000,
-  retry = 3,
+  retry = 0,
   output,
   error,
 }) {
@@ -81,10 +82,11 @@ async function dockerExec({
       { timeout, retry }
     );
   } finally {
-    await lock.releaseLock(lockKey);
     if (nodeInfo.isDockerInstalled) {
       await promiseSpawn(`docker rm -f ${options.env.dockerName} > /dev/null 2>&1 || true`, {}, { timeout: 1000 * 10 });
     }
+    await sleep(500);
+    await lock.releaseLock(lockKey);
   }
   logger.info(`dockerExec ${options.env.dockerName} cost time: ${Date.now() - startTime}ms`);
 }

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.17.3-beta-20251120-052956-035abea6",
+  "version": "1.17.3-beta-20251121-135300-8e451e6e",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -17,21 +17,21 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "Apache-2.0",
   "dependencies": {
-    "@abtnode/analytics": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/auth": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/certificate-manager": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/constant": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/cron": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/db-cache": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/docker-utils": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/logger": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/models": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/queue": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/rbac": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/router-provider": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/static-server": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/timemachine": "1.17.3-beta-20251120-052956-035abea6",
-    "@abtnode/util": "1.17.3-beta-20251120-052956-035abea6",
+    "@abtnode/analytics": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/auth": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/certificate-manager": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/constant": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/cron": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/db-cache": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/docker-utils": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/logger": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/models": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/queue": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/rbac": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/router-provider": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/static-server": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/timemachine": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@abtnode/util": "1.17.3-beta-20251121-135300-8e451e6e",
     "@aigne/aigne-hub": "^0.10.9",
     "@arcblock/did": "^1.27.7",
     "@arcblock/did-connect-js": "^1.27.7",
@@ -43,15 +43,15 @@
     "@arcblock/pm2-events": "^0.0.5",
     "@arcblock/validator": "^1.27.7",
     "@arcblock/vc": "^1.27.7",
-    "@blocklet/constant": "1.17.3-beta-20251120-052956-035abea6",
+    "@blocklet/constant": "1.17.3-beta-20251121-135300-8e451e6e",
     "@blocklet/did-space-js": "^1.2.4",
-    "@blocklet/env": "1.17.3-beta-20251120-052956-035abea6",
+    "@blocklet/env": "1.17.3-beta-20251121-135300-8e451e6e",
     "@blocklet/error": "^0.3.3",
-    "@blocklet/meta": "1.17.3-beta-20251120-052956-035abea6",
-    "@blocklet/resolver": "1.17.3-beta-20251120-052956-035abea6",
-    "@blocklet/sdk": "1.17.3-beta-20251120-052956-035abea6",
-    "@blocklet/server-js": "1.17.3-beta-20251120-052956-035abea6",
-    "@blocklet/store": "1.17.3-beta-20251120-052956-035abea6",
+    "@blocklet/meta": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@blocklet/resolver": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@blocklet/sdk": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@blocklet/server-js": "1.17.3-beta-20251121-135300-8e451e6e",
+    "@blocklet/store": "1.17.3-beta-20251121-135300-8e451e6e",
     "@blocklet/theme": "^3.2.6",
     "@fidm/x509": "^1.2.1",
     "@ocap/mcrypto": "^1.27.7",
@@ -116,5 +116,5 @@
     "express": "^4.18.2",
     "unzipper": "^0.10.11"
   },
-  "gitHead": "fe41f1c21bb8e3efcf42623e9b0d24c77563862a"
+  "gitHead": "e34d7abbe02a8b6eb754b7e2590a9e3d9d663c7d"
 }