@abtnode/core 1.17.3-beta-20251120-052956-035abea6 → 1.17.3-beta-20251123-232619-53258789

package/lib/blocklet/manager/disk.js

@@ -259,13 +259,13 @@ const { formatEnvironments, getBlockletMeta, validateOwner, isCLI } = util;
 
 const statusLock = new DBCache(() => ({
   prefix: 'blocklet-status-lock',
-  ttl: 1000 * 60,
+  ttl: 120_000,
   ...getAbtNodeRedisAndSQLiteUrl(),
 }));
 
 const startLock = new DBCache(() => ({
   prefix: 'blocklet-start-lock',
-  ttl: process.env.NODE_ENV === 'test' ? 100 : 1000 * 60,
+  ttl: process.env.NODE_ENV === 'test' ? 1_000 : 120_000,
   ...getAbtNodeRedisAndSQLiteUrl(),
 }));
 
@@ -740,6 +740,8 @@ class DiskBlockletManager extends BaseBlockletManager {
     e2eMode,
     context,
     atomic,
+    onStarted,
+    onError,
   }) {
     if (!blocklet.children) {
       return componentDids;
@@ -771,11 +773,17 @@ class DiskBlockletManager extends BaseBlockletManager {
       // start by dependency
       for (const dids of requiredDidsLayers) {
         if (atomic) {
-          await this._start({ blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: dids }, context);
+          await this._start(
+            { blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: dids, onStarted, onError },
+            context
+          );
         } else {
           const tasks = dids.map(
             (x) => () =>
-              this._start({ blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: [x] }, context)
+              this._start(
+                { blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids: [x], onStarted, onError },
+                context
+              )
           );
           await pAll(tasks, { concurrency: 4 }).catch((err) => {
             throw new Error(err.errors ? err.errors.join(', ') : err);
@@ -801,8 +809,8 @@ class DiskBlockletManager extends BaseBlockletManager {
     },
     context
   ) {
-    const lockName = `start-not-input-component-dids-${did}`;
-    const shouldLock = !inputComponentDids?.length;
+    const lockName = `${did}-start`;
+    const shouldLock = inputComponentDids?.length === 0;
 
     if (shouldLock) {
       await startLock.acquire(lockName);
@@ -814,8 +822,16 @@ class DiskBlockletManager extends BaseBlockletManager {
         inputComponentDids?.length ? inputComponentDids : blocklet.children.map((x) => x.meta.did)
       );
 
+      try {
+        await this.deleteProcess({ did, componentDids: baseComponentDids, isGreen: true });
+      } catch (error) {
+        logger.warn('Failed to delete process; this warning can be safely ignored.', { error });
+      }
+
+      await this.checkControllerStatus(blocklet, 'start');
+
       // 优先启动资源型组件
-      let componentDids = [];
+      const componentDids = [];
       const nonEntryComponentIds = [];
       for (const b of blocklet.children) {
         if (!b?.meta?.did) {
@@ -844,7 +860,6 @@ class DiskBlockletManager extends BaseBlockletManager {
           operator,
         });
       }
-
       // sync component config before at first to ensure resource component config is ready
       const serverSk = (await states.node.read()).sk;
       await Promise.all(
@@ -853,7 +868,16 @@ class DiskBlockletManager extends BaseBlockletManager {
         )
       );
 
-      componentDids = await this.startRequiredComponents({
+      const doc1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.starting, {
+        componentDids,
+        operator,
+      });
+
+      this.emit(BlockletEvents.statusChange, doc1);
+      const startedBlockletDids = [];
+      const errorBlockletDids = [];
+
+      const notStartedComponentDids = await this.startRequiredComponents({
         componentDids,
         inputComponentDids,
         blocklet,
@@ -862,26 +886,15 @@ class DiskBlockletManager extends BaseBlockletManager {
         e2eMode,
         context,
         atomic,
+        onStarted: (subDid) => {
+          startedBlockletDids.push({ did: subDid });
+        },
+        onError: (subDid, error) => {
+          errorBlockletDids.push({ did: subDid, error });
+        },
       });
 
-      if (atomic || !blocklet.structVersion) {
-        const nextBlocklet = await this._start(
-          { blocklet, throwOnError, checkHealthImmediately, e2eMode, componentDids, operator },
-          context
-        );
-
-        // 根据情况更新 route table， 会判断只有包含多 interfaces 的 DID 才会更新 route table
-        if (!['true', '1'].includes(process.env.ABT_NODE_DISABLE_BLUE_GREEN)) {
-          this.emit(BlockletEvents.blurOrGreenStarted, {
-            blocklet: nextBlocklet,
-            componentDids: inputComponentDids,
-            context,
-          });
-        }
-        return nextBlocklet;
-      }
-
-      const tasks = componentDids.map(
+      const tasks = notStartedComponentDids.map(
         (componentDid) => () =>
           this._start(
             {
@@ -891,25 +904,104 @@ class DiskBlockletManager extends BaseBlockletManager {
               e2eMode,
               componentDids: [componentDid],
               operator,
+              onStarted: (subDid) => {
+                startedBlockletDids.push({ did: subDid });
+              },
+              onError: (subDid, error) => {
+                errorBlockletDids.push({ did: subDid, error });
+              },
             },
             context
           )
       );
 
-      const rest = await pAll(tasks, { concurrency: 1 });
+      await pAll(tasks, { concurrency: 6 });
 
       const nextBlocklet = await this.ensureBlocklet(did, { e2eMode });
+      let errorDescription = '';
+      let resultBlocklet = nextBlocklet;
+
+      if (startedBlockletDids.length) {
+        await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
+          componentDids: startedBlockletDids.map((x) => x.did),
+          operator,
+        });
+
+        const finalBlocklet = await this.getBlocklet(did);
+        resultBlocklet = finalBlocklet;
+
+        await this.configSynchronizer.throttledSyncAppConfig(finalBlocklet);
+        const componentsInfo = getComponentsInternalInfo(finalBlocklet);
+        this.emit(BlockletInternalEvents.componentUpdated, {
+          appDid: blocklet.appDid,
+          components: componentsInfo,
+        });
+        this.emit(BlockletInternalEvents.componentStarted, {
+          appDid: blocklet.appDid,
+          components: (componentDids || []).map((x) => ({ did: x })),
+        });
+
+        this.emit(BlockletEvents.statusChange, finalBlocklet);
+        this.emit(BlockletEvents.started, { ...finalBlocklet, componentDids });
+
+        launcher.notifyBlockletStarted(finalBlocklet);
+      }
+
+      if (errorBlockletDids.length) {
+        const error = errorBlockletDids[0];
+        for (const item of errorBlockletDids) {
+          logger.error('Failed to start blocklet', {
+            error: item.error,
+            title: nextBlocklet.meta.title,
+            name: getComponentNamesWithVersion(nextBlocklet, [item.did]),
+          });
+        }
+
+        errorDescription = `${getComponentNamesWithVersion(
+          nextBlocklet,
+          errorBlockletDids.map((x) => x.did)
+        )} start failed for ${getDisplayName(nextBlocklet)}: ${errorBlockletDids.map((x) => x.error.message).join('. ')}`;
+        this._createNotification(did, {
+          title: 'Component start failed',
+          description: errorDescription,
+          entityType: 'blocklet',
+          entityId: did,
+          severity: 'error',
+        });
+
+        await this.deleteProcess({
+          did,
+          componentDids: errorBlockletDids.map((x) => x.did),
+          shouldUpdateBlockletStatus: false,
+        });
+        await states.blocklet.setBlockletStatus(did, BlockletStatus.error, {
+          componentDids: errorBlockletDids.map((x) => x.did),
+          operator,
+        });
+        const finalBlocklet = await this.getBlocklet(did);
+        resultBlocklet = finalBlocklet;
+        this.emit(BlockletEvents.startFailed, {
+          ...finalBlocklet,
+          componentDids: errorBlockletDids.map((x) => x.did),
+          error: { message: error.message },
+        });
+        this.emit(BlockletEvents.statusChange, { ...finalBlocklet, error: { message: error.message } });
+      }
 
       // 根据情况更新 route table， 会判断只有包含多 interfaces 的 DID 才会更新 route table
       if (!['true', '1'].includes(process.env.ABT_NODE_DISABLE_BLUE_GREEN)) {
         this.emit(BlockletEvents.blurOrGreenStarted, {
-          blocklet: nextBlocklet,
+          blocklet: resultBlocklet,
           componentDids: inputComponentDids,
           context,
         });
       }
 
-      return rest[0];
+      if (throwOnError && errorDescription) {
+        throw new Error(errorDescription);
+      }
+
+      return resultBlocklet;
     } finally {
       if (shouldLock) {
         await startLock.releaseLock(lockName);
@@ -926,17 +1018,13 @@ class DiskBlockletManager extends BaseBlockletManager {
       e2eMode = false,
       componentDids,
       operator: _operator,
+      onStarted,
+      onError,
     },
     context
   ) {
     const operator = _operator || context?.user?.did;
 
-    try {
-      await this.deleteProcess({ did, componentDids, isGreen: true });
-    } catch {
-      // do nothing
-    }
-
     logger.info('start blocklet', {
       did: did || inputBlocklet.meta.did,
       componentDids,
@@ -960,7 +1048,6 @@ class DiskBlockletManager extends BaseBlockletManager {
 
     const entryComponentIds = [];
     const nonEntryComponentIds = [];
-    let nonEntryComponentRes;
     const componentDidsSet = new Set(componentDids);
     try {
       await forEachBlocklet(
@@ -996,18 +1083,9 @@ class DiskBlockletManager extends BaseBlockletManager {
         { parallel: true, concurrencyLimit: 4 }
       );
       if (nonEntryComponentIds.length) {
-        const params = {
-          did,
-          context,
-          minConsecutiveTime: 3000,
-          timeout: 5000,
-          componentDids: nonEntryComponentIds,
-        };
-        nonEntryComponentRes = await this._onCheckIfStarted(params, {
-          throwOnError,
-          skipRunningCheck: true,
-          needUpdateBlueStatus: false,
-        });
+        for (const subDid of nonEntryComponentIds) {
+          onStarted?.(subDid);
+        }
       }
     } catch (err) {
       logger.error('Failed to categorize components into entry and non-entry types', { error: err.message });
@@ -1015,7 +1093,7 @@ class DiskBlockletManager extends BaseBlockletManager {
     }
 
     if (!entryComponentIds.length) {
-      return nonEntryComponentRes;
+      return;
     }
 
     try {
@@ -1036,13 +1114,7 @@ class DiskBlockletManager extends BaseBlockletManager {
         }
       }
 
-      const doc1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.starting, {
-        componentDids: entryComponentIds,
-        operator,
-      });
-
       blocklet1.status = BlockletStatus.starting;
-      this.emit(BlockletEvents.statusChange, doc1);
       const blocklet = await ensureAppPortsNotOccupied({
         blocklet: blocklet1,
         componentDids: entryComponentIds,
@@ -1097,8 +1169,6 @@ class DiskBlockletManager extends BaseBlockletManager {
           });
         };
 
-      await this.configSynchronizer.syncAppConfig(blocklet);
-
       await startBlockletProcess(blocklet, {
         ...context,
         preFlight: getHookFn('preFlight'),
@@ -1116,6 +1186,7 @@ class DiskBlockletManager extends BaseBlockletManager {
         checkHealthImmediately,
         componentDids,
       });
+
       const params = {
         did,
         context,
@@ -1124,47 +1195,19 @@ class DiskBlockletManager extends BaseBlockletManager {
         componentDids: entryComponentIds,
       };
 
-      await this._onCheckIfStarted(params, { throwOnError, needUpdateBlueStatus: false });
-      blocklet.status = BlockletStatus.running;
+      await this._onCheckIfStarted(params, { throwOnError });
 
-      return blocklet;
+      for (const subDid of entryComponentIds) {
+        onStarted?.(subDid);
+      }
     } catch (err) {
       const status = await states.blocklet.getBlockletStatus(did);
       if ([BlockletStatus.stopping, BlockletStatus.stopped].includes(status)) {
         logger.info('Failed to start blocklet maybe due to manually stopped');
-        return states.blocklet.getBlocklet(did);
       }
-
-      const error = Array.isArray(err) ? err[0] : err;
-      logger.error('Failed to start blocklet', { error, did, title: blocklet1.meta.title });
-      const description = `${getComponentNamesWithVersion(blocklet1, entryComponentIds)} start failed for ${getDisplayName(
-        blocklet1
-      )}: ${error.message}`;
-      this._createNotification(did, {
-        title: 'Component start failed',
-        description,
-        entityType: 'blocklet',
-        entityId: did,
-        severity: 'error',
-      });
-
-      await this.deleteProcess({ did, componentDids: entryComponentIds });
-      const res = await states.blocklet.setBlockletStatus(did, BlockletStatus.error, {
-        componentDids: entryComponentIds,
-        operator,
-      });
-      this.emit(BlockletEvents.startFailed, {
-        ...res,
-        componentDids: entryComponentIds,
-        error: { message: error.message },
-      });
-      this.emit(BlockletEvents.statusChange, { ...res, error: { message: error.message } });
-
-      if (throwOnError) {
-        throw new Error(description);
+      for (const subDid of entryComponentIds) {
+        onError?.(subDid, err);
       }
-
-      return res;
     }
   }
 
@@ -1471,13 +1514,19 @@ class DiskBlockletManager extends BaseBlockletManager {
       });
 
       const doc = await this._deleteBlocklet({ did, keepData, keepLogsDir, keepConfigs }, context);
-      this._createNotification(doc.meta.did, {
-        title: 'Blocklet Deleted',
-        description: `Blocklet ${getDisplayName(blocklet)} is deleted.`,
-        entityType: 'blocklet',
-        entityId: doc.meta.did,
-        severity: 'success',
-      });
+      this._createNotification(
+        doc.meta.did,
+        {
+          title: 'Blocklet Deleted',
+          description: `Blocklet ${getDisplayName(blocklet)} is deleted.`,
+          entityType: 'blocklet',
+          entityId: doc.meta.did,
+          severity: 'success',
+        },
+        {
+          skipGetBlocklet: true,
+        }
+      );
       await removeDockerNetwork(parseDockerName(did, 'docker-network'));
       return doc;
     } catch (error) {
@@ -1486,13 +1535,19 @@ class DiskBlockletManager extends BaseBlockletManager {
       const doc = await this._deleteBlocklet({ did, keepData, keepLogsDir, keepConfigs }, context);
       await removeDockerNetwork(parseDockerName(did, 'docker-network'));
 
-      this._createNotification(doc.meta.did, {
-        title: 'Blocklet Deleted',
-        description: `Blocklet ${getDisplayName(blocklet)} is deleted.`,
-        entityType: 'blocklet',
-        entityId: doc.meta.did,
-        severity: 'success',
-      });
+      this._createNotification(
+        doc.meta.did,
+        {
+          title: 'Blocklet Deleted',
+          description: `Blocklet ${getDisplayName(blocklet)} is deleted.`,
+          entityType: 'blocklet',
+          entityId: doc.meta.did,
+          severity: 'success',
+        },
+        {
+          skipGetBlocklet: true,
+        }
+      );
 
       return doc;
     }
@@ -3590,141 +3645,24 @@ class DiskBlockletManager extends BaseBlockletManager {
     await blueGreenStartBlocklet({ did, componentDids, operator }, context, this, states);
   }
 
-  _onCheckIfStarted = async (
-    jobInfo,
-    { throwOnError, skipRunningCheck = false, isGreen = false, needUpdateBlueStatus = true } = {}
-  ) => {
-    const startedAt = Date.now();
-    const { did, context, minConsecutiveTime = 2000, timeout, componentDids } = jobInfo;
-    const nodeInfo = await states.node.read();
+  _onCheckIfStarted = async (jobInfo, { skipRunningCheck = false, isGreen = false } = {}) => {
+    const { did, minConsecutiveTime = 2000, timeout, componentDids } = jobInfo;
     const blocklet = await this.getBlocklet(did);
 
-    const { meta } = blocklet;
-    const { name } = meta;
-
-    const successBlockletIds = new Set();
-
-    try {
-      if (!skipRunningCheck) {
-        await checkBlockletProcessHealthy(blocklet, {
-          minConsecutiveTime,
-          timeout,
-          componentDids,
-          isGreen,
-          appDid: did,
-          enableDocker: nodeInfo.enableDocker,
-          getBlocklet: () => {
-            return this.getBlocklet(did);
-          },
-          setBlockletRunning: async (componentDid) => {
-            successBlockletIds.add(componentDid);
-            await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
-              componentDids: [componentDid],
-              isGreen,
-            });
-          },
-        });
-      }
+    const nodeInfo = await states.node.read();
 
-      const runningRes = await states.blocklet.setBlockletStatus(did, BlockletStatus.running, {
-        componentDids: successBlockletIds.size > 0 ? Array.from(successBlockletIds) : componentDids,
+    if (!skipRunningCheck) {
+      await checkBlockletProcessHealthy(blocklet, {
+        minConsecutiveTime,
+        timeout,
+        componentDids,
         isGreen,
+        appDid: did,
+        enableDocker: nodeInfo.enableDocker,
+        getBlocklet: () => {
+          return this.getBlocklet(did);
+        },
       });
-
-      if (needUpdateBlueStatus) {
-        await states.blocklet.setBlockletStatus(did, BlockletStatus.stopped, {
-          componentDids,
-          isGreen: !isGreen,
-        });
-      }
-
-      const nextBlocklet = await this.getBlocklet(did);
-
-      await this.configSynchronizer.throttledSyncAppConfig(nextBlocklet);
-      const componentsInfo = getComponentsInternalInfo(nextBlocklet);
-      this.emit(BlockletInternalEvents.componentUpdated, {
-        appDid: blocklet.appDid,
-        components: componentsInfo,
-      });
-      this.emit(BlockletInternalEvents.componentStarted, {
-        appDid: blocklet.appDid,
-        components: (componentDids || []).map((x) => ({ did: x })),
-      });
-
-      this.emit(BlockletEvents.statusChange, nextBlocklet);
-      this.emit(BlockletEvents.started, { ...nextBlocklet, componentDids });
-
-      launcher.notifyBlockletStarted(blocklet);
-
-      logger.info('blocklet healthy', { did, name, time: Date.now() - startedAt });
-
-      if (needUpdateBlueStatus) {
-        try {
-          await this.deleteProcess({ did, componentDids, isGreen: !isGreen }, context);
-        } catch {
-          logger.error('delete process failed', { did, componentDids, isGreen });
-        }
-      }
-
-      return runningRes;
-    } catch (error) {
-      const errorBlockletIds = [];
-      for (const componentDid of componentDids) {
-        if (!successBlockletIds.has(componentDid)) {
-          errorBlockletIds.push(componentDid);
-        }
-      }
-      if (errorBlockletIds.length === 0) {
-        return this.getBlocklet(did);
-      }
-
-      const status = await states.blocklet.getBlockletStatus(did);
-      if ([BlockletStatus.stopping, BlockletStatus.stopped].includes(status)) {
-        logger.info(`Check blocklet healthy failing because blocklet is ${fromBlockletStatus(status)}`);
-        return {};
-      }
-
-      logger.error('check blocklet if started failed', { did, name, context, timeout, error });
-
-      await this.deleteProcess({ did, componentDids: errorBlockletIds, isGreen }, context);
-
-      let doc;
-      if (isGreen) {
-        doc = await states.blocklet.setBlockletStatus(did, BlockletStatus.stopped, {
-          componentDids: errorBlockletIds,
-          isGreen,
-        });
-      } else {
-        doc = await states.blocklet.setBlockletStatus(did, BlockletStatus.error, { componentDids: errorBlockletIds });
-      }
-
-      const componentNames = getComponentNamesWithVersion(blocklet, errorBlockletIds);
-      const description = componentNames
-        ? `${getComponentNamesWithVersion(blocklet, errorBlockletIds)} start failed for ${getDisplayName(blocklet)}: ${
-            error.message
-          }`
-        : `${blocklet.meta.title} start failed: ${error.message}`;
-
-      this._createNotification(did, {
-        title: 'Component start failed',
-        description,
-        entityType: 'blocklet',
-        entityId: did,
-        severity: 'error',
-      });
-
-      this.emit(BlockletEvents.startFailed, {
-        ...doc,
-        componentDids: errorBlockletIds,
-        error: { message: error.message },
-      });
-      this.emit(BlockletEvents.statusChange, { ...doc, error });
-
-      if (throwOnError) {
-        throw error;
-      }
-
-      return doc;
     }
   };
 
@@ -5185,7 +5123,7 @@ class DiskBlockletManager extends BaseBlockletManager {
   }
 
   // 目前：Blocklet 的安装消息都会聚合在这里。
-  async _createNotification(did, notification) {
+  async _createNotification(did, notification, { skipGetBlocklet = false } = {}) {
     if (isCLI()) {
       return undefined;
     }
@@ -5194,7 +5132,7 @@ class DiskBlockletManager extends BaseBlockletManager {
       let blockletUrl;
       let blocklet;
       try {
-        blocklet = await this.getBlocklet(did);
+        blocklet = skipGetBlocklet ? null : await this.getBlocklet(did);
         if (blocklet) {
           const urls = blocklet.site?.domainAliases || [];
           const customUrl = urls.find((x) => !x.isProtected)?.value;