@abtnode/core 1.16.25-beta-85e265d1 → 1.16.25-beta-bc165d9b

package/lib/blocklet/manager/disk.js

@@ -10,7 +10,6 @@ const merge = require('lodash/merge');
 const pick = require('lodash/pick');
 const isEmpty = require('lodash/isEmpty');
 const cloneDeep = require('lodash/cloneDeep');
-const { isNFTExpired, getNftExpirationDate } = require('@abtnode/util/lib/nft');
 const { sign } = require('@arcblock/jwt');
 const sleep = require('@abtnode/util/lib/sleep');
 const getBlockletInfo = require('@blocklet/meta/lib/info');
@@ -103,7 +102,7 @@ const formatContext = require('@abtnode/util/lib/format-context');
 const md5 = require('@abtnode/util/lib/md5');
 const { callFederated } = require('@abtnode/auth/lib/util/federated');
 const pAll = require('p-all');
-const { consumeServerlessNFT, consumeLauncherSession } = require('../../util/launcher');
+const launcher = require('../../util/launcher');
 const util = require('../../util');
 const {
   refresh: refreshAccessibleExternalNodeIp,
@@ -141,7 +140,6 @@ const {
   getProcessState,
   getBlockletStatus,
   shouldSkipComponent,
-  exceedRedemptionPeriod,
   ensureAppPortsNotOccupied,
   getComponentNamesWithVersion,
   updateDidDocument: updateBlockletDocument,
@@ -265,6 +263,7 @@ class DiskBlockletManager extends BaseBlockletManager {
     backupQueue,
     restoreQueue,
     checkUpdateQueue,
+    reportComponentUsageQueue,
     daemon = false,
     teamManager,
   }) {
@@ -280,6 +279,7 @@ class DiskBlockletManager extends BaseBlockletManager {
     this.backupQueue = backupQueue;
     this.restoreQueue = restoreQueue;
     this.checkUpdateQueue = checkUpdateQueue;
+    this.reportComponentUsageQueue = reportComponentUsageQueue;
     this.teamManager = teamManager;
 
     if (isFunction(this.backupQueue.on)) {
@@ -1197,6 +1197,27 @@ class DiskBlockletManager extends BaseBlockletManager {
 
     const newBlocklet = await this.getBlocklet(rootDid);
 
+    if (newBlocklet.controller) {
+      const componentDids = [did];
+      const eventType = 'uninstall';
+
+      this.reportComponentUsageQueue.push({
+        entity: 'blocklet',
+        action: 'reportComponentUsage',
+        did: newBlocklet.meta.did,
+        time: new Date().toISOString(),
+        componentDids,
+        eventType,
+        context,
+      });
+
+      logger.info('pushed reporting uninstall components event job to queue', {
+        did: newBlocklet.meta.did,
+        componentDids,
+        eventType,
+      });
+    }
+
     await this._updateDependents(rootDid);
 
     // support edge case
@@ -2026,6 +2047,10 @@ class DiskBlockletManager extends BaseBlockletManager {
       if (job.action === 'autoCheckUpdate') {
         await this._onCheckForComponentUpdate(job);
       }
+
+      if (job.action === 'reportComponentUsage') {
+        await this._reportComponentUsage(job);
+      }
     }
   }
 
@@ -2070,7 +2095,7 @@ class DiskBlockletManager extends BaseBlockletManager {
         },
         {
           name: 'clean-expired-blocklet-data',
-          time: '0 */20 0 * * *', // 每天凌晨 0 点的每 20 分钟
+          time: '0 10 0 * * *', // 每天凌晨 0:10
           options: { runOnInit: false },
           fn: () => this.cleanExpiredBlocklets(),
         },
@@ -2527,6 +2552,23 @@ class DiskBlockletManager extends BaseBlockletManager {
             context,
             action: postAction,
           });
+
+          const newBlocklet = await this.getBlocklet(did);
+          if (newBlocklet.controller) {
+            const eventType = 'install';
+
+            this.reportComponentUsageQueue.push({
+              entity: 'blocklet',
+              action: 'reportComponentUsage',
+              time: new Date().toISOString(),
+              did,
+              componentDids,
+              eventType: 'install',
+              context,
+            });
+
+            logger.info('pushed reporting install components event job to queue', { did, componentDids, eventType });
+          }
         } catch (err) {
           logger.error('blocklet onUpgrade error', { error: err });
         }
@@ -2926,6 +2968,25 @@ class DiskBlockletManager extends BaseBlockletManager {
     }
   }
 
+  async _reportComponentUsage({ did, componentDids, eventType, time }) {
+    try {
+      logger.info('start report component usage', { did, componentDids, eventType });
+
+      const blocklet = await this.getBlocklet(did);
+      await launcher.reportComponentsEvent({
+        blocklet,
+        dids: componentDids,
+        type: eventType,
+        time: time || new Date().toISOString(),
+      });
+
+      logger.info('report component usage success', { did, componentDids, eventType });
+    } catch (error) {
+      logger.error('report component usage failed', { did, componentDids, eventType, error });
+      throw error; // 一定要 throw 出去，否则会导致队列任务不会重试
+    }
+  }
+
   /**
    * @description send notification to multiple users, and send different notifications according to the user's own language
    * @param {{
@@ -3346,10 +3407,10 @@ class DiskBlockletManager extends BaseBlockletManager {
       if (blocklet.controller && process.env.NODE_ENV !== 'test') {
         let isNFTConsumed = false;
         if (blocklet.controller.launcherSessionId && blocklet.controller.launcherUrl) {
-          await consumeLauncherSession({ params: blocklet.controller, blocklet });
+          await launcher.consumeLauncherSession({ params: blocklet.controller, blocklet });
           isNFTConsumed = true;
         } else if (blocklet.controller.nftId) {
-          await consumeServerlessNFT({ nftId: blocklet.controller.nftId, blocklet });
+          await launcher.consumeServerlessNFT({ nftId: blocklet.controller.nftId, blocklet });
           isNFTConsumed = true;
         }
 
@@ -3921,8 +3982,8 @@ class DiskBlockletManager extends BaseBlockletManager {
           continue;
         }
 
-        const assetState = await util.getNFTState(data.controller.chainHost, data.controller.nftId);
-        const isExpired = isNFTExpired(assetState);
+        const isExpired = await launcher.isBlockletExpired(data.did, data.controller);
+
         if (isExpired === false) {
           logger.info('blocklet is renewed', { did: data.did });
           await states.blockletExtras.updateByDid(data.did, {
@@ -3956,24 +4017,21 @@ class DiskBlockletManager extends BaseBlockletManager {
       );
 
       if (blockletExtras.length === 0) {
-        logger.info('no expired blocklet');
+        logger.info('no serverless blocklet');
         return;
       }
 
-      logger.info('expired blocklet count', { count: blockletExtras.length });
+      logger.info('serverless blocklet count', { count: blockletExtras.length });
 
       for (const data of blockletExtras) {
         try {
           const { did } = data;
-          const assetState = await util.getNFTState(data.controller.chainHost, data.controller.nftId);
-          const isExpired = isNFTExpired(assetState);
+          const isExpired = await launcher.isBlockletExpired(did, data.controller);
 
           if (isExpired) {
-            const expiredAt = getNftExpirationDate(assetState);
             logger.info('the blocklet already expired and will be stopped', {
               blockletDid: did,
               nftId: data.controller.nftId,
-              expiredAt,
             });
 
             // 如果 Blocklet 没停止, 先停止
@@ -4029,22 +4087,43 @@ class DiskBlockletManager extends BaseBlockletManager {
       for (const data of blockletExtras) {
         try {
           const { did } = data;
-          const assetState = await util.getNFTState(data.controller.chainHost, data.controller.nftId);
-          const expiredAt = getNftExpirationDate(assetState);
+          const launcherSession = await launcher.getLauncherSession({
+            launcherUrl: data.controller.launcherUrl,
+            launcherSessionId: data.controller.launcherSessionId,
+          });
+          const isTerminated = launcher.isLaunchSessionTerminated(launcherSession);
+
+          if (!isTerminated) {
+            logger.info('skip cleaning the non-exceed redemption blocklet', {
+              blockletDid: did,
+              controller: data.controller,
+              launcherSession,
+            });
+            continue;
+          }
+
+          if (!launcherSession.terminatedAt) {
+            logger.error('the blocklet launch session does not have terminatedAt, skip', {
+              blockletDid: did,
+              controller: data.controller,
+              launcherSession,
+            });
+            continue;
+          }
 
-          if (!exceedRedemptionPeriod(expiredAt)) {
-            logger.error('skip cleaning the non-exceed redemption blocklet', {
+          // 订阅终止后需要再保留一段时间数据
+          if (!launcher.isDataRetentionExceeded(launcherSession.terminatedAt)) {
+            logger.info('skip cleaning the non-exceed redemption blocklet', {
               blockletDid: did,
-              expiredAt,
+              controller: data.controller,
+              launcherSession,
             });
-            // eslint-disable-next-line no-continue
             continue;
           }
 
           logger.info('the blocklet already exceed redemption and will be deleted', {
             blockletDid: did,
             nftId: data.controller.nftId,
-            expiredAt,
           });
 
           // TODO: 如果绑定了 DID Space 备份到 DID Space
@@ -4275,9 +4354,9 @@ class DiskBlockletManager extends BaseBlockletManager {
       return;
     }
 
-    const assetState = await util.getNFTState(blocklet.controller?.chainHost, blocklet.controller?.nftId);
+    const isExpired = await launcher.isBlockletExpired(blocklet.meta.did, blocklet.controller);
 
-    if (isNFTExpired(assetState)) {
+    if (isExpired) {
       logger.error(`try to ${action} an expired blocklet`, {
         did: blocklet.meta.did,
         nftId: blocklet.controller?.nftId,

package/lib/blocklet/manager/helper/install-application-from-general.js

@@ -53,16 +53,6 @@ const installApplicationFromGeneral = async ({
     throw new Error(`Should not be here: unknown type ${type}`);
   }
 
-  // check install count
-  if (controller?.nftId) {
-    const installedCount = await states.blockletExtras.count({ 'controller.nftId': controller.nftId });
-    if (installedCount >= (controller.appMaxCount || 1)) {
-      throw new Error(
-        `You can only install ${controller.appMaxCount} blocklet with this credential: ${controller.nftId}`
-      );
-    }
-  }
-
   let blockletWalletType;
 
   // create component

package/lib/index.js

@@ -189,6 +189,26 @@ function ABTNode(options) {
     },
   });
 
+  const reportComponentUsageQueue = createQueue({
+    daemon: options.daemon,
+    model: states.job,
+    name: 'report_component_usage_queue',
+    onJob: async (job) => {
+      /* eslint-disable no-use-before-define */
+
+      if (typeof blockletManager.onJob === 'function') {
+        await blockletManager.onJob(job);
+      }
+    },
+    options: {
+      concurrency,
+      maxRetries: 30,
+      retryDelay: 60 * 1000, // retry after 1 minute
+      maxTimeout: 60 * 1000 * 30, // throw timeout error after 15 minutes
+      id: (job) => (job ? md5(`${job.entity}-${job.action}-${job.id}`) : ''),
+    },
+  });
+
   const certManager = new Cert({
     maintainerEmail: DEFAULT_CERTIFICATE_EMAIL,
     dataDir: dataDirs.certManagerModule,
@@ -216,6 +236,7 @@ function ABTNode(options) {
     backupQueue,
     restoreQueue,
     checkUpdateQueue,
+    reportComponentUsageQueue,
     daemon: options.daemon,
     teamManager,
   });

package/lib/util/blocklet.js

@@ -2,7 +2,6 @@
 
 const fs = require('fs-extra');
 const path = require('path');
-const dayjs = require('@abtnode/util/lib/dayjs');
 const shelljs = require('shelljs');
 const os = require('os');
 const tar = require('tar');
@@ -38,7 +37,6 @@ const {
   BLOCKLET_MAX_MEM_LIMIT_IN_MB,
   BLOCKLET_INSTALL_TYPE,
   APP_STRUCT_VERSION,
-  EXPIRED_BLOCKLET_DATA_RETENTION_DAYS,
   NODE_MODES,
 } = require('@abtnode/constant');
 const {
@@ -1804,10 +1802,6 @@ const shouldSkipComponent = (componentDid, whiteList) => {
   return !arr.includes(componentDid);
 };
 
-const exceedRedemptionPeriod = (expirationDate) => {
-  return dayjs().diff(dayjs(expirationDate), 'day') > EXPIRED_BLOCKLET_DATA_RETENTION_DAYS;
-};
-
 const ensureAppPortsNotOccupied = async ({
   blocklet,
   componentDids: inputDids,
@@ -1861,7 +1855,7 @@ const getComponentNamesWithVersion = (app = {}, componentDids = []) => {
 };
 
 const shouldEnableSlpDomain = ({ mode, launcher }) => {
-  if (process.env.NODE_ENV === 'development' && process.env.ABT_NODE_ENABLE_SLP_DOMAIN === 'true') {
+  if (process.env.ABT_NODE_ENABLE_SLP_DOMAIN === 'true') {
     return true;
   }
 
@@ -2048,7 +2042,6 @@ module.exports = {
   getBlockletStatus,
   shouldSkipComponent,
   getBlockletURLForLauncher,
-  exceedRedemptionPeriod,
   ensureAppPortsNotOccupied,
   getComponentNamesWithVersion,
   updateDidDocument,

package/lib/util/launcher.js

@@ -1,7 +1,9 @@
 const path = require('path');
 const fs = require('fs-extra');
 const joinUrl = require('url-join');
+const dayjs = require('@abtnode/util/lib/dayjs');
 const pick = require('lodash/pick');
+const uniq = require('lodash/uniq');
 const isEmpty = require('lodash/isEmpty');
 const {
   getUserAvatarUrl,
@@ -24,6 +26,7 @@ const {
   NODE_DATA_DIR_NAME,
   USER_AVATAR_URL_PREFIX,
   ROLES,
+  LAUNCH_SESSION_STATUS,
 } = require('@abtnode/constant');
 
 const logger = require('@abtnode/logger')('@abtnode/core:util:launcher');
@@ -58,16 +61,59 @@ const consumeServerlessNFT = async ({ nftId, blocklet }) => {
   }
 };
 
+const reportComponentsEvent = async ({ blocklet, dids, type, time }) => {
+  const { controller } = blocklet;
+  const componentDids = uniq(dids);
+
+  const payload = {
+    componentDids,
+    type,
+    time,
+  };
+
+  try {
+    const info = await states.node.read();
+
+    const result = await doRequest(info.sk, {
+      launcherUrl: controller.launcherUrl,
+      pathname: `/api/launches/${controller.launcherSessionId}/events`,
+      payload,
+      method: 'post',
+    });
+
+    logger.info(`reported components event ${type}`, {
+      controller,
+      blockletPid: blocklet.appPid,
+      componentDids,
+      type,
+      result,
+    });
+  } catch (error) {
+    logger.error('report components event failed', { controller, blockletPid: blocklet.appPid, componentDids, type });
+    throw new Error(`report components event failed of blocklet ${blocklet.appPid}`);
+  }
+};
+
 const consumeLauncherSession = async ({ params, blocklet }) => {
   try {
     const info = await states.node.read();
     const { name } = getBlockletInfo(blocklet, info.sk);
     const appUrl = getBlockletURLForLauncher({ blocklet, nodeInfo: info });
 
+    let componentDids = blocklet.children.map((x) => x.meta.did);
+    componentDids = uniq(componentDids);
+
     const result = await doRequest(info.sk, {
       launcherUrl: params.launcherUrl,
       pathname: `/api/launches/${params.launcherSessionId}/installed`,
-      payload: { appDid: blocklet.appPid, appUrl, appName: name, ownerDid: params.ownerDid },
+      payload: {
+        appDid: blocklet.appPid,
+        appUrl,
+        appName: name,
+        installedAt: blocklet.installedAt,
+        ownerDid: params.ownerDid,
+        componentDids,
+      },
       method: 'post',
     });
 
@@ -213,7 +259,15 @@ const getLauncherSession = async ({ launcherUrl, launcherSessionId, external = t
 
   // strip sensitive data if call from external
   if (external && result.launcherSession) {
-    result.launcherSession = pick(result.launcherSession, ['_id', 'status']);
+    result.launcherSession = pick(result.launcherSession, [
+      '_id',
+      'status',
+      'statusText',
+      'subscription',
+      'expirationDate',
+      'terminatedAt',
+      'reservedUntil',
+    ]);
   }
 
   return result;
@@ -240,10 +294,50 @@ const isLauncherSessionConsumed = async (params) => {
   return consumed;
 };
 
+const getLaunchSessionStatus = async (blockletDid, controller) => {
+  logger.info('checking blocklet status', { blockletDid, controller });
+
+  const { launcherSessionId } = controller;
+  // 新版都通过 Launcher 判断过期状态
+  const { error, launcherSession } = await getLauncherSession({
+    launcherSessionId,
+    launcherUrl: controller.launcherUrl || 'https://launcher.arcblock.io/',
+  });
+
+  if (error) {
+    logger.error('get launcher session failed', { error, blockletDid, controller });
+    throw new Error(error);
+  }
+
+  logger.info('get launcher session success', { launcherSessionId, launcherSession });
+  return launcherSession.status;
+};
+
+const isBlockletExpired = async (blockletDid, controller) => {
+  const status = await getLaunchSessionStatus(blockletDid, controller);
+  return [LAUNCH_SESSION_STATUS.expired, LAUNCH_SESSION_STATUS.terminated].includes(status);
+};
+
+const isLaunchSessionTerminated = (session) => session.status === LAUNCH_SESSION_STATUS.terminated;
+
+const isBlockletTerminated = async (blockletDid, controller) => {
+  const status = await getLaunchSessionStatus(blockletDid, controller);
+  return status === LAUNCH_SESSION_STATUS.terminated;
+};
+
+const DATA_RETENTION_DAYS = 30;
+const isDataRetentionExceeded = (terminatedAt) =>
+  !!terminatedAt && dayjs().diff(dayjs(terminatedAt), 'days') > DATA_RETENTION_DAYS;
+
 module.exports = {
   consumeServerlessNFT,
   consumeLauncherSession,
+  reportComponentsEvent,
   setupAppOwner,
   getLauncherSession,
+  isDataRetentionExceeded,
+  isLaunchSessionTerminated,
   isLauncherSessionConsumed,
+  isBlockletExpired,
+  isBlockletTerminated,
 };

package/lib/validators/util.js

@@ -14,7 +14,6 @@ const blockletController = Joi.object({
   nftId: Joi.DID().required(),
   nftOwner: Joi.DID().required(),
   chainHost: Joi.string().uri().required(),
-  appMaxCount: Joi.number().required().min(1),
   launcherUrl: Joi.string().uri().optional(),
   launcherSessionId: Joi.string().optional(),
   ownerDid: Joi.DID().optional(),

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.16.25-beta-85e265d1",
+  "version": "1.16.25-beta-bc165d9b",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,19 +19,19 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "Apache-2.0",
   "dependencies": {
-    "@abtnode/analytics": "1.16.25-beta-85e265d1",
-    "@abtnode/auth": "1.16.25-beta-85e265d1",
-    "@abtnode/certificate-manager": "1.16.25-beta-85e265d1",
-    "@abtnode/constant": "1.16.25-beta-85e265d1",
-    "@abtnode/cron": "1.16.25-beta-85e265d1",
-    "@abtnode/logger": "1.16.25-beta-85e265d1",
-    "@abtnode/models": "1.16.25-beta-85e265d1",
-    "@abtnode/queue": "1.16.25-beta-85e265d1",
-    "@abtnode/rbac": "1.16.25-beta-85e265d1",
-    "@abtnode/router-provider": "1.16.25-beta-85e265d1",
-    "@abtnode/static-server": "1.16.25-beta-85e265d1",
-    "@abtnode/timemachine": "1.16.25-beta-85e265d1",
-    "@abtnode/util": "1.16.25-beta-85e265d1",
+    "@abtnode/analytics": "1.16.25-beta-bc165d9b",
+    "@abtnode/auth": "1.16.25-beta-bc165d9b",
+    "@abtnode/certificate-manager": "1.16.25-beta-bc165d9b",
+    "@abtnode/constant": "1.16.25-beta-bc165d9b",
+    "@abtnode/cron": "1.16.25-beta-bc165d9b",
+    "@abtnode/logger": "1.16.25-beta-bc165d9b",
+    "@abtnode/models": "1.16.25-beta-bc165d9b",
+    "@abtnode/queue": "1.16.25-beta-bc165d9b",
+    "@abtnode/rbac": "1.16.25-beta-bc165d9b",
+    "@abtnode/router-provider": "1.16.25-beta-bc165d9b",
+    "@abtnode/static-server": "1.16.25-beta-bc165d9b",
+    "@abtnode/timemachine": "1.16.25-beta-bc165d9b",
+    "@abtnode/util": "1.16.25-beta-bc165d9b",
     "@arcblock/did": "1.18.113",
     "@arcblock/did-auth": "1.18.113",
     "@arcblock/did-ext": "^1.18.113",
@@ -42,11 +42,11 @@
     "@arcblock/pm2-events": "^0.0.5",
     "@arcblock/validator": "^1.18.113",
     "@arcblock/vc": "1.18.113",
-    "@blocklet/constant": "1.16.25-beta-85e265d1",
-    "@blocklet/env": "1.16.25-beta-85e265d1",
-    "@blocklet/meta": "1.16.25-beta-85e265d1",
-    "@blocklet/resolver": "1.16.25-beta-85e265d1",
-    "@blocklet/sdk": "1.16.25-beta-85e265d1",
+    "@blocklet/constant": "1.16.25-beta-bc165d9b",
+    "@blocklet/env": "1.16.25-beta-bc165d9b",
+    "@blocklet/meta": "1.16.25-beta-bc165d9b",
+    "@blocklet/resolver": "1.16.25-beta-bc165d9b",
+    "@blocklet/sdk": "1.16.25-beta-bc165d9b",
     "@did-space/client": "^0.3.73",
     "@fidm/x509": "^1.2.1",
     "@ocap/mcrypto": "1.18.113",
@@ -102,5 +102,5 @@
     "jest": "^29.7.0",
     "unzipper": "^0.10.11"
   },
-  "gitHead": "ea2dbf135591636c3d1899374b99d66eee61c7d2"
+  "gitHead": "a7895ed39e47c8c6bd10d32317804cc1904714ef"
 }