@abtnode/core 1.16.11-beta-8e4c6e5c → 1.16.11-beta-0ae58a71

package/lib/blocklet/manager/disk.js

@@ -65,6 +65,8 @@ const {
   BLOCKLET_CONFIGURABLE_KEY,
   RESTORE_PROGRESS_STATUS,
   CHAIN_PROP_MAP_REVERSE,
+  BLOCKLET_CONTROLLER_STATUS,
+  SUSPENDED_REASON,
 } = require('@blocklet/constant');
 const util = require('../../util');
 const {
@@ -104,7 +106,7 @@ const {
   updateBlockletFallbackLogo,
   ensureAppLogo,
   getBlockletDidDomainList,
-  shouldCleanExpiredBlocklet,
+  exceedRedemptionPeriod,
 } = require('../../util/blocklet');
 const states = require('../../states');
 const BaseBlockletManager = require('./base');
@@ -450,6 +452,8 @@ class BlockletManager extends BaseBlockletManager {
     // should check blocklet integrity
     const blocklet = await this.ensureBlocklet(did, { e2eMode });
 
+    await this.checkControllerStatus(blocklet, 'start');
+
     try {
       // blocklet may be manually stopped durning starting
       // so error message would not be sent if blocklet is stopped
@@ -663,6 +667,10 @@ class BlockletManager extends BaseBlockletManager {
   async restart({ did }, context) {
     logger.info('restart blocklet', { did });
 
+    const blocklet = await this.getBlocklet(did);
+
+    await this.checkControllerStatus(blocklet, 'restart');
+
     await states.blocklet.setBlockletStatus(did, BlockletStatus.stopping);
     const result = await states.blocklet.getBlocklet(did);
     this.emit(BlockletEvents.statusChange, result);
@@ -690,6 +698,8 @@ class BlockletManager extends BaseBlockletManager {
   async reload({ did }, context) {
     const blocklet = await this.getBlocklet(did);
 
+    await this.checkControllerStatus(blocklet, 'reload');
+
     await states.blocklet.setBlockletStatus(did, BlockletStatus.stopping);
     await reloadBlockletProcess(blocklet);
     const res = await states.blocklet.setBlockletStatus(did, BlockletStatus.running);
@@ -1495,22 +1505,29 @@ class BlockletManager extends BaseBlockletManager {
       {
         name: 'refresh-accessible-ip',
         time: '0 */10 * * * *', // 10min
+        options: { runOnInit: true },
         fn: async () => {
           const nodeInfo = await states.node.read();
           await refreshAccessibleExternalNodeIp(nodeInfo);
         },
       },
       {
-        name: 'delete-expired-external-blocklet',
+        name: 'check-renewed-blocklet',
+        time: '0 */10 * * * *', // 10min
+        options: { runOnInit: false },
+        fn: () => this.checkRenewedBlocklet(),
+      },
+      {
+        name: 'stop-expired-external-blocklet',
         time: '0 */30 * * * *', // 30min
         options: { runOnInit: false },
-        fn: () => this._deleteExpiredExternalBlocklet(),
+        fn: () => this.stopExpiredBlocklets(),
       },
       {
         name: 'clean-expired-blocklet-data',
         time: '0 */20 0 * * *', // 每天凌晨 0 点的每 20 分钟
         options: { runOnInit: false },
-        fn: () => this._cleanExpiredBlockletData(),
+        fn: () => this.cleanExpiredBlocklets(),
       },
       {
         name: 'record-blocklet-runtime-history',
@@ -2764,71 +2781,102 @@ class BlockletManager extends BaseBlockletManager {
     }
   }
 
-  async _deleteExpiredExternalBlocklet() {
+  async checkRenewedBlocklet() {
     try {
-      logger.info('start check expired external blocklet');
+      logger.info('start check renewed blocklet');
+
+      // 只检查因为过期而被 suspended 的 blocklet
+      const blockletExtras = await states.blockletExtras.find(
+        {
+          'controller.status.value': BLOCKLET_CONTROLLER_STATUS.suspended,
+          'controller.status.reason': SUSPENDED_REASON.expired,
+        },
+        { did: 1, meta: 1, controller: 1 }
+      );
+
+      for (const data of blockletExtras) {
+        const blocklet = await states.blocklet.getBlocklet(data.did);
+        if (!blocklet) {
+          logger.error('blocklet not found', { did: data.did });
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        const assetState = await util.getNFTState(data.controller.chainHost, data.controller.nftId);
+        const isExpired = isNFTExpired(assetState);
+        if (isExpired === false) {
+          logger.info('blocklet is renewed', { did: data.did });
+          await states.blockletExtras.updateByDid(data.did, {
+            ...data.controller,
+            status: { value: BLOCKLET_CONTROLLER_STATUS.normal, reason: '' },
+          });
+
+          logger.info('start to start blocklet', { did: data.did });
+
+          if (![BlockletStatus.starting, BlockletStatus.running].includes(blocklet.status)) {
+            await this.start({ did: data.did });
+            logger.info('start blocklet success', { did: data.did });
+          }
+        }
+      }
+    } catch (error) {
+      logger.error('start check renewed blocklet failed', { error });
+    }
+  }
+
+  async stopExpiredBlocklets() {
+    try {
+      logger.info('start checking expired blocklet');
       const blockletExtras = await states.blockletExtras.find(
         {
           controller: {
             $exists: true,
           },
-          expiredAt: {
-            $exists: false,
-          },
         },
         { did: 1, meta: 1, controller: 1 }
       );
 
-      logger.info('external blocklet count', { count: blockletExtras.length });
+      if (blockletExtras.length === 0) {
+        logger.info('no expired blocklet');
+        return;
+      }
+
+      logger.info('expired blocklet count', { count: blockletExtras.length });
 
       for (const data of blockletExtras) {
         try {
-          if (isEmpty(data.controller)) {
-            logger.info('skip the blocklet without controller', { blockletDid: data.did, controller: data.controller });
-            // eslint-disable-next-line no-continue
-            continue;
-          }
-
           const { did } = data;
           const assetState = await util.getNFTState(data.controller.chainHost, data.controller.nftId);
           const isExpired = isNFTExpired(assetState);
+          const blocklet = await states.blocklet.getBlocklet(did);
 
-          if (isExpired) {
-            logger.info('the blocklet already expired', {
+          if (isExpired && blocklet.status !== BlockletStatus.stopped) {
+            const expiredAt = getNftExpirationDate(assetState);
+            logger.info('the blocklet already expired and will be stopped', {
               blockletDid: did,
               nftId: data.controller.nftId,
+              expiredAt,
             });
 
-            const blocklet = await states.blocklet.getBlocklet(did);
-            if (blocklet) {
-              // 预防 Blocklet 已经删除，但是 blocklet extra data 没有清理的场景
-              await this._backupToDisk({ blocklet });
-              logger.info('backed up the expired blocklet', {
-                blockletDid: did,
-                nftId: data.controller.nftId,
-              });
-
-              await this.delete({ did, keepData: true, keepConfigs: true, keepLogsDir: true });
-              logger.info('the expired blocklet already deleted', {
-                blockletDid: did,
-                nftId: data.controller.nftId,
-              });
-            }
-
-            const expiredAt = getNftExpirationDate(assetState);
-            await states.blockletExtras.updateExpireInfo({ did, expiredAt });
-            logger.info('updated expired blocklet extra info', {
-              nftId: data.controller.nftId,
-              blockletDid: did,
+            await this.stop({ did });
+            await states.blockletExtras.updateByDid(did, {
+              controller: {
+                ...data.controller,
+                status: {
+                  value: BLOCKLET_CONTROLLER_STATUS.suspended,
+                  reason: SUSPENDED_REASON.expired,
+                },
+              },
             });
+            logger.info('the expired blocklet is stopped', { did });
 
-            // 删除 blocklet 后会 reload nginx, 所以这里每次删除一个
+            // 为了减小服务器压力，每次删除一个 blocklet 后，休息 10s
             if (process.env.NODE_ENV !== 'test') {
               await sleep(10 * 1000);
             }
           }
         } catch (error) {
-          logger.error('delete expired blocklet failed', {
+          logger.error('stop expired blocklet failed', {
             blockletDid: data.did,
             nftId: data.controller?.nftId,
             error,
@@ -2842,62 +2890,68 @@ class BlockletManager extends BaseBlockletManager {
     }
   }
 
-  async _cleanExpiredBlockletData() {
+  async cleanExpiredBlocklets() {
     try {
-      logger.info('start clean expired blocklet data');
-      const blockletExtras = await states.blockletExtras.getExpiredList();
+      logger.info('start checking exceeding the redemption period blocklet');
+      const blockletExtras = await states.blockletExtras.find(
+        {
+          'controller.status.value': BLOCKLET_CONTROLLER_STATUS.suspended,
+        },
+        { did: 1, meta: 1, controller: 1 }
+      );
+
       if (blockletExtras.length === 0) {
-        logger.info('no expired blocklet data');
+        logger.info('no exceeding the redemption blocklet need to be cleaned');
         return;
       }
 
-      const tasks = blockletExtras.map(async ({ appDid, did, meta, controller }) => {
-        const blocklet = await this.getBlocklet(did);
-        if (blocklet) {
-          logger.error('skip cleaning the blocklet which is not deleted', { blockletDid: did });
-          return;
-        }
-
-        const assetState = await util.getNFTState(controller.chainHost, controller.nftId);
-        const expirationDate = getNftExpirationDate(assetState);
+      logger.info('exceeding the redemption blocklet count', { count: blockletExtras.length });
 
-        if (!shouldCleanExpiredBlocklet(expirationDate)) {
-          logger.error('skip cleaning the blocklet which is not expired', { blockletDid: did, expirationDate });
-          await blockletExtras.updateExpireInfo({ did, expiredAt: expirationDate });
-          return;
-        }
+      for (const data of blockletExtras) {
+        try {
+          const { did } = data;
+          const assetState = await util.getNFTState(data.controller.chainHost, data.controller.nftId);
+          const expiredAt = getNftExpirationDate(assetState);
 
-        let blockletMeta = meta;
-        if (isEmpty(blockletMeta)) {
-          // 旧版本的 blocklet 没有 meta 信息，这里补充一下
-          blockletMeta = { did, name: did };
-          logger.error('the blocklet original meta is empty', { did, blockletMeta });
-        }
+          if (!exceedRedemptionPeriod(expiredAt)) {
+            logger.error('skip cleaning the non-exceed redemption blocklet', {
+              blockletDid: did,
+              expiredAt,
+            });
+            // eslint-disable-next-line no-continue
+            continue;
+          }
 
-        await this._cleanBlockletData({
-          blocklet: { meta: blockletMeta },
-          keepData: false,
-          keepLogsDir: false,
-          keepConfigs: false,
-        });
-        logger.info(`clean blocklet ${blockletMeta.did} extra data`);
+          logger.info('the blocklet already exceed redemption and will be deleted', {
+            blockletDid: did,
+            nftId: data.controller.nftId,
+            expiredAt,
+          });
 
-        await removeBackup(this.dataDirs.data, appDid);
-        logger.info(`removed blocklet ${blockletMeta.did} backup`);
+          // TODO: 如果绑定了 DID Space 备份到 DID Space
 
-        this.emit(BlockletEvents.dataCleaned, {
-          blocklet: { meta: blockletMeta },
-          keepRouting: false,
-        });
-
-        logger.info(`cleaned expired blocklet blocklet ${blockletMeta.did} data`);
-      });
+          await this.delete({ did, keepData: false, keepConfigs: false, keepLogsDir: false });
+          logger.info('the exceed redemption blocklet already deleted', {
+            blockletDid: did,
+            nftId: data.controller.nftId,
+          });
 
-      await Promise.all(tasks);
+          // 删除 blocklet 后会 reload nginx, 所以这里每次删除一个
+          if (process.env.NODE_ENV !== 'test') {
+            await sleep(10 * 1000);
+          }
+        } catch (error) {
+          logger.error('delete exceed redemption blocklet failed', {
+            blockletDid: data.did,
+            nftId: data.controller?.nftId,
+            error,
+          });
+        }
+      }
 
-      logger.info('clean expired blocklet data done');
+      logger.info('check exceeding the redemption period blocklet end');
     } catch (error) {
-      logger.error('clean expired blocklet data failed', { error });
+      logger.error('checking exceeding the redemption period blocklet failed', { error });
     }
   }
 
@@ -3101,6 +3155,38 @@ class BlockletManager extends BaseBlockletManager {
     }
     await this._setConfigsFromMeta(did);
   }
+
+  async checkControllerStatus(blocklet, action) {
+    if (isEmpty(blocklet.controller)) {
+      return;
+    }
+
+    const assetState = await util.getNFTState(blocklet.controller?.chainHost, blocklet.controller?.nftId);
+
+    if (isNFTExpired(assetState)) {
+      logger.error(`try to ${action} an expired blocklet`, {
+        did: blocklet.meta.did,
+        nftId: blocklet.controller?.nftId,
+      });
+      throw new Error(`can not ${action} an expired blocklet`);
+    }
+
+    if (
+      blocklet.controller.status?.value === BLOCKLET_CONTROLLER_STATUS.suspended &&
+      blocklet.controller.status?.reason === SUSPENDED_REASON.expired
+    ) {
+      // 如果是因为过期被暂停的，续期后, 并且不影响启动
+      await states.blockletExtras.updateByDid(blocklet.meta.did, {
+        controller: {
+          ...blocklet.controller,
+          status: {
+            value: BLOCKLET_CONTROLLER_STATUS.normal,
+            reason: '',
+          },
+        },
+      });
+    }
+  }
 }
 
 module.exports = BlockletManager;

package/lib/states/blocklet-extras.js

@@ -2,15 +2,13 @@
 /* eslint-disable no-underscore-dangle */
 /* eslint-disable consistent-return */
 const logger = require('@abtnode/logger')('@abtnode/core:states:blocklet-extras');
-const { EXPIRED_BLOCKLET_DATA_RETENTION_DAYS } = require('@abtnode/constant');
 const camelCase = require('lodash/camelCase');
 const get = require('lodash/get');
-const dayjs = require('dayjs');
 
 const BaseState = require('./base');
 
 const { mergeConfigs, parseConfigs, encryptConfigs } = require('../blocklet/extras');
-const { validateAddMeta, validateExpiredInfo } = require('../validators/blocklet-extra');
+const { validateAddMeta } = require('../validators/blocklet-extra');
 
 const noop = (k) => (v) => v[k];
 
@@ -213,19 +211,6 @@ class BlockletExtrasState extends BaseState {
     return super.findOne({ did }, { did: 1, controller: 1, meta: 1 });
   }
 
-  async updateExpireInfo({ did, expiredAt } = {}) {
-    const entity = { did, expiredAt };
-    await validateExpiredInfo(entity);
-
-    return this.updateByDid(did, { expiredAt });
-  }
-
-  async getExpiredList() {
-    const deadline = dayjs().subtract(EXPIRED_BLOCKLET_DATA_RETENTION_DAYS, 'days').toISOString();
-    const docs = await super.find({ expiredAt: { $exists: true } });
-    return docs.filter((x) => x.controller?.nftId).filter((x) => x.expiredAt <= deadline);
-  }
-
   encryptSecurityData({ data, _rootDid } = {}) {
     if (!data) {
       return data;

package/lib/util/blocklet.js

@@ -101,6 +101,9 @@ const { validate: validateEngine, get: getEngine } = require('../blocklet/manage
 const isRequirementsSatisfied = require('./requirement');
 const { getDidDomainForBlocklet } = require('./get-domain-for-blocklet');
 const { expandBundle, findInterfacePortByName, prettyURL, getNFTState, templateReplace } = require('./index');
+const externalIpUtil = require('./get-accessible-external-node-ip');
+const { getIpDnsDomainForBlocklet } = require('./get-domain-for-blocklet');
+const { replaceDomainSlot } = require('./index');
 
 const getComponentConfig = (meta) => {
   const components = meta.components || meta.children || [];
@@ -1520,6 +1523,22 @@ const getConfigFromPreferences = (blocklet) => {
   return result;
 };
 
+const getBlockletURLForLauncher = async ({ blocklet, nodeInfo }) => {
+  let nodeIp = await externalIpUtil.getFromCache();
+  if (process.env.ABT_NODE_HOST) {
+    // 开发调试用
+    nodeIp = process.env.ABT_NODE_HOST;
+  }
+
+  if (nodeIp) {
+    const ipEchoDomain = getIpDnsDomainForBlocklet(blocklet);
+    return `https://${replaceDomainSlot({ domain: ipEchoDomain, nodeIp })}`;
+  }
+
+  const didDomain = getDidDomainForBlocklet({ appPid: blocklet.appPid, didDomain: nodeInfo.didDomain });
+  return `https://${didDomain}`;
+};
+
 const consumeServerlessNFT = async ({ nftId, nodeInfo, blocklet }) => {
   try {
     const state = await getNFTState(blocklet.controller.chainHost, nftId);
@@ -1527,13 +1546,15 @@ const consumeServerlessNFT = async ({ nftId, nodeInfo, blocklet }) => {
       throw new Error(`get nft state failed, chainHost: ${blocklet.controller.chainHost}, nftId: ${nftId}`);
     }
 
-    const wallet = getNodeWallet(nodeInfo.sk);
-    const appURL = blocklet.environments.find((item) => item.key === 'BLOCKLET_APP_URL').value;
+    const appURL = await getBlockletURLForLauncher({ blocklet, nodeInfo });
+
+    logger.info('app url when consuming nft', { appURL, nftId });
 
     const body = { nftId, appURL };
 
     const { launcherUrl } = state.data.value;
 
+    const wallet = getNodeWallet(nodeInfo.sk);
     const func = async () => {
       const { data } = await axios.post(joinURL(launcherUrl, '/api/serverless/consume'), body, {
         headers: {
@@ -1931,7 +1952,7 @@ const getBlockletDidDomainList = (blocklet, nodeInfo) => {
   return domainAliases;
 };
 
-const shouldCleanExpiredBlocklet = (expirationDate) => {
+const exceedRedemptionPeriod = (expirationDate) => {
   return dayjs().diff(dayjs(expirationDate), 'day') > EXPIRED_BLOCKLET_DATA_RETENTION_DAYS;
 };
 
@@ -1993,5 +2014,6 @@ module.exports = {
   getFixedBundleSource,
   ensureAppLogo,
   getBlockletDidDomainList,
-  shouldCleanExpiredBlocklet,
+  getBlockletURLForLauncher,
+  exceedRedemptionPeriod,
 };

package/lib/util/get-accessible-external-node-ip.js

@@ -1,6 +1,7 @@
 const joinUrl = require('url-join');
 const axios = require('@abtnode/util/lib/axios');
 const { DEFAULT_IP_DOMAIN, DEFAULT_ADMIN_PATH } = require('@abtnode/constant');
+const logger = require('@abtnode/logger')('@abtnode/core:util:get-accessible-external-node-ip');
 const { get: getIp } = require('./ip');
 
 const getNodeDomain = (ip) => (ip ? DEFAULT_IP_DOMAIN.replace(/^\*/, ip.replace(/\./g, '-')) : '');
@@ -23,6 +24,7 @@ const checkConnected = async ({ ip, nodeInfo }) => {
  */
 const fetch = async (nodeInfo) => {
   const { external } = await getIp();
+  logger.info('refresh external ip:', external);
 
   if (!external) {
     return null;

package/lib/validators/blocklet-extra.js

@@ -13,12 +13,6 @@ const addMeta = Joi.object({
   controller: blockletController.optional(),
 }).required();
 
-const updateExpiredInfo = Joi.object({
-  did: Joi.DID().required(),
-  expiredAt: Joi.string().required(),
-});
-
 module.exports = {
   validateAddMeta: createValidator(addMeta),
-  validateExpiredInfo: createValidator(updateExpiredInfo),
 };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.16.11-beta-8e4c6e5c",
+  "version": "1.16.11-beta-0ae58a71",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,18 +19,18 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "Apache-2.0",
   "dependencies": {
-    "@abtnode/auth": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/certificate-manager": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/constant": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/cron": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/logger": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/models": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/queue": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/rbac": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/router-provider": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/static-server": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/timemachine": "1.16.11-beta-8e4c6e5c",
-    "@abtnode/util": "1.16.11-beta-8e4c6e5c",
+    "@abtnode/auth": "1.16.11-beta-0ae58a71",
+    "@abtnode/certificate-manager": "1.16.11-beta-0ae58a71",
+    "@abtnode/constant": "1.16.11-beta-0ae58a71",
+    "@abtnode/cron": "1.16.11-beta-0ae58a71",
+    "@abtnode/logger": "1.16.11-beta-0ae58a71",
+    "@abtnode/models": "1.16.11-beta-0ae58a71",
+    "@abtnode/queue": "1.16.11-beta-0ae58a71",
+    "@abtnode/rbac": "1.16.11-beta-0ae58a71",
+    "@abtnode/router-provider": "1.16.11-beta-0ae58a71",
+    "@abtnode/static-server": "1.16.11-beta-0ae58a71",
+    "@abtnode/timemachine": "1.16.11-beta-0ae58a71",
+    "@abtnode/util": "1.16.11-beta-0ae58a71",
     "@arcblock/did": "1.18.80",
     "@arcblock/did-auth": "1.18.80",
     "@arcblock/did-ext": "^1.18.80",
@@ -41,10 +41,10 @@
     "@arcblock/pm2-events": "^0.0.5",
     "@arcblock/validator": "^1.18.80",
     "@arcblock/vc": "1.18.80",
-    "@blocklet/constant": "1.16.11-beta-8e4c6e5c",
-    "@blocklet/meta": "1.16.11-beta-8e4c6e5c",
-    "@blocklet/sdk": "1.16.11-beta-8e4c6e5c",
-    "@did-space/client": "^0.2.99",
+    "@blocklet/constant": "1.16.11-beta-0ae58a71",
+    "@blocklet/meta": "1.16.11-beta-0ae58a71",
+    "@blocklet/sdk": "1.16.11-beta-0ae58a71",
+    "@did-space/client": "^0.2.113",
     "@fidm/x509": "^1.2.1",
     "@ocap/mcrypto": "1.18.80",
     "@ocap/util": "1.18.80",
@@ -96,5 +96,5 @@
     "express": "^4.18.2",
     "jest": "^27.5.1"
   },
-  "gitHead": "bd1beb1fb79e075d31359f438a263ee9077d3cd9"
+  "gitHead": "5d044a08b4399eb855563e670af7cbd0d3851e94"
 }