@abtnode/core 1.8.67 → 1.8.68-beta-500af7e5

package/lib/blocklet/storage/backup/spaces.js

@@ -1,16 +1,22 @@
 /**
  * @typedef {{
- *  did?: string
+ *  did: string
+ *  event: import('events').EventEmitter,
+ *  userDid: string,
+ *  referrer: string,
  * }} SpaceBackupInput
  */
 
 const { isValid } = require('@arcblock/did');
-const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+const { BLOCKLET_CONFIGURABLE_KEY, BlockletEvents } = require('@blocklet/constant');
 const { getBlockletInfo } = require('@blocklet/meta');
-const { SpaceClient, SyncFolderPushCommand } = require('@did-space/client');
+const { SpaceClient, BackupBlockletCommand } = require('@did-space/client');
 const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
-const { join } = require('path');
+const { join, basename } = require('path');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:backup');
+
 const states = require('../../../states');
 const { AuditLogBackup } = require('./audit-log');
 const { BlockletBackup } = require('./blocklet');
@@ -131,12 +137,22 @@ class SpacesBackup {
   }
 
   async export() {
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      did: this.input.did,
+      message: 'Preparing data for backup...',
+      progress: 15,
+    });
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.export();
       })
     );
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      did: this.input.did,
+      message: 'Data ready, start backup...',
+      progress: 20,
+    });
   }
 
   async syncToSpaces() {
@@ -146,22 +162,34 @@ class SpacesBackup {
       wallet,
     });
 
-    // FIXME: 在 Spaces 里面预览出 blocklet 的样式，需要规划一个好的数据结构
-    const { errorCount } = await spaceClient.send(
-      new SyncFolderPushCommand({
+    const { errorCount, message } = await spaceClient.send(
+      new BackupBlockletCommand({
+        appDid: this.blocklet.appDid,
         source: join(this.blockletBackupDir, '/'),
-        target: join('.did-objects', this.blocklet.appDid),
         debug: true,
-        concurrency: 64,
-        retryCount: 100,
+        concurrency: 32,
+        retryCount: 10,
         filter: (object) => {
+          // FIXME: @yejianchao 这里需要更完整的黑名单
           return object.name !== '.DS_Store';
         },
+        onProgress: (data) => {
+          logger.info('backup progress', { appDid: this.input.did, data });
+          const percent = (data.completed * 100) / data.total;
+          this.input.event.emit(BlockletEvents.backupProgress, {
+            did: this.input.did,
+            message: `Uploaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+            // 0.8 是因为上传文件到 spaces 占进度的 80%，+ 20 是因为需要累加之前的进度
+            progress: +Math.ceil(percent * 0.8).toFixed(2) + 20,
+          });
+        },
+        userDid: this.input.userDid,
+        referrer: this.input.referrer,
       })
     );
 
     if (errorCount !== 0) {
-      throw new Error(`Sync to spaces encountered ${errorCount} error`);
+      throw new Error(`Sync to spaces encountered ${errorCount} error: ${message}`);
     }
   }
 

package/lib/blocklet/storage/restore/base.js

@@ -13,14 +13,6 @@ class BaseRestore {
    */
   blockletRestoreDir;
 
-  /**
-   *
-   * @description spaces 的 endpoint
-   * @type {import('@ocap/wallet').WalletObject}
-   * @memberof BaseRestore
-   */
-  blockletWallet;
-
   /**
    *
    * @description server 的数据目录
@@ -41,13 +33,22 @@ class BaseRestore {
    */
   ensureParams(spaces) {
     this.blockletRestoreDir = spaces.blockletRestoreDir;
-    this.blockletWallet = spaces.blockletWallet;
     this.serverDataDir = spaces.serverDataDir;
   }
 
   async import() {
     throw new Error('not implemented');
   }
+
+  /**
+   * Generate params for BlockletManager to install
+   *
+   * @return {object}
+   * @memberof BaseRestore
+   */
+  getInstallParams() {
+    return {};
+  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -2,6 +2,7 @@ const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
 const { cloneDeep, isArray } = require('lodash');
 const { join } = require('path');
 const security = require('@abtnode/util/lib/security');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const { BaseRestore } = require('./base');
 
 class BlockletExtrasRestore extends BaseRestore {
@@ -24,7 +25,6 @@ class BlockletExtrasRestore extends BaseRestore {
      * @type {import('@abtnode/client').BlockletState}
      */
     const blockletExtra = readJSONSync(join(this.blockletRestoreDir, this.filename));
-
     return this.cleanData(blockletExtra);
   }
 
@@ -32,7 +32,6 @@ class BlockletExtrasRestore extends BaseRestore {
    *
    * @description 清理数据并加密
    * @param {import('@abtnode/client').BlockletState} blockletExtraInput
-   * @return {Promise<void>}
    * @memberof BlockletExtrasRestore
    */
   async cleanData(blockletExtraInput) {
@@ -43,7 +42,7 @@ class BlockletExtrasRestore extends BaseRestore {
       const currentBlockletExtra = queue.pop();
 
       // 加解密
-      this.useBlockletDecryptConfigs(currentBlockletExtra.configs);
+      this.decrypt(currentBlockletExtra.configs);
 
       if (currentBlockletExtra?.children) {
         queue.push(...currentBlockletExtra.children);
@@ -55,31 +54,37 @@ class BlockletExtrasRestore extends BaseRestore {
 
   /**
    *
-   * @description 清理数据并加密
+   * @description 解密加密的数据
    * @param {import('@abtnode/client').ConfigEntry[]} configs
    * @return {void}
    * @memberof BlockletExtrasRestore
    */
-  useBlockletDecryptConfigs(configs) {
+  decrypt(configs) {
     if (!configs || !isArray(configs)) {
       return;
     }
 
+    const { appDid, password } = this.input;
     for (const config of configs) {
-      // secure 为 true 的配置才需要被加密保存上次到 did spaces
       if (config.secure) {
-        const encryptByBlocklet = config.value;
-
-        // 再用 blocklet secret 加密，然后才可以上传到 spaces
-        const decryptByBlocklet = security.decrypt(
-          encryptByBlocklet,
-          this.blockletWallet.address,
-          Buffer.from(this.blockletWallet.secretKey)
-        );
-
-        config.value = decryptByBlocklet;
+        const encrypted = config.value;
+        const decrypted = security.decrypt(encrypted, appDid, Buffer.from(password));
+        config.value = decrypted;
       }
     }
+
+    const config = configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
+    if (!config) {
+      throw new Error(`Invalid blocklet backup file, no ${BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK} found`);
+    }
+
+    this.appSk = config.value;
+  }
+
+  getInstallParams() {
+    return {
+      appSk: this.appSk,
+    };
   }
 }
 

package/lib/blocklet/storage/restore/blocklets.js

@@ -1,4 +1,4 @@
-const { existsSync, remove } = require('fs-extra');
+const { existsSync, remove, ensureDirSync } = require('fs-extra');
 const { join } = require('path');
 const fg = require('fast-glob');
 const { BaseRestore } = require('./base');
@@ -10,8 +10,11 @@ class BlockletsRestore extends BaseRestore {
   async import() {
     const blockletsDir = join(this.blockletRestoreDir, this.filename);
 
+    // blockletsDir 可以不存在, 因为还原的 blocklet 可能所有的组件都是来自 store 的
     if (!existsSync(blockletsDir)) {
-      throw new Error(`dir not found: ${blockletsDir}`);
+      // FIXME: 如果文件夹不存在，需要创建文件夹，因为 installFromBackup 未做容错处理
+      ensureDirSync(blockletsDir);
+      return;
     }
 
     const paths = await fg('**/*.zip', {

package/lib/blocklet/storage/restore/spaces.js

@@ -1,16 +1,24 @@
 /**
  * @typedef {{
+ *  appDid: string;
  *  endpoint: string;
- *  blockletSecretKey: string;
+ *  password: Buffer;
+ *  delegation: string;
+ *  wallet: import('@ocap/wallet').WalletObject;
+ *  event: import('events').EventEmitter,
+ *  userDid: string,
+ *  referrer: string,
  * }} SpaceRestoreInput
  */
 
-const { SpaceClient, SyncFolderPullCommand } = require('@did-space/client');
-const { types } = require('@ocap/mcrypto');
-const { fromSecretKey, WalletType } = require('@ocap/wallet');
-const { ensureDirSync, existsSync, rmdirSync } = require('fs-extra');
-const { join } = require('path');
 const validUrl = require('valid-url');
+const { BlockletEvents } = require('@blocklet/constant');
+const { SpaceClient, RestoreBlockletCommand } = require('@did-space/client');
+const { ensureDirSync, existsSync, rmdirSync } = require('fs-extra');
+const { join, basename } = require('path');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:restore');
+
 const { BlockletExtrasRestore } = require('./blocklet-extras');
 const { BlockletsRestore } = require('./blocklets');
 
@@ -37,14 +45,6 @@ class SpacesRestore {
    */
   serverDataDir;
 
-  /**
-   *
-   * @description spaces 的 endpoint
-   * @type {import('@ocap/wallet').WalletObject}
-   * @memberof SpacesRestore
-   */
-  blockletWallet;
-
   storages;
 
   /**
@@ -72,68 +72,75 @@ class SpacesRestore {
 
   async initialize() {
     this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
-    this.blockletWallet = await this.getBlockletWallet();
 
-    if (!this.input.endpoint.includes(this.blockletWallet.address)) {
-      throw new Error(`endpoint and blocklet.appDid(${this.blockletWallet.address}) do not match`);
+    if (!this.input.endpoint.includes(this.input.appDid)) {
+      throw new Error(`endpoint and blocklet.appDid(${this.input.appDid}) do not match`);
     }
 
-    this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.blockletWallet.address);
+    this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.input.appDid);
     if (existsSync(this.blockletRestoreDir)) {
       rmdirSync(this.blockletRestoreDir, { recursive: true });
     }
     ensureDirSync(this.blockletRestoreDir);
   }
 
-  /**
-   *
-   * @returns {Promise<void>}
-   * @memberof SpacesRestore
-   */
   async restore() {
     await this.initialize();
     await this.syncFromSpaces();
     await this.import();
-  }
-
-  async getBlockletWallet() {
-    // @FIXME: blocklet 钱包类型如何得知呢？
-    const wallet = fromSecretKey(this.input.blockletSecretKey, WalletType({ role: types.RoleType.ROLE_APPLICATION }));
 
-    return wallet;
+    return this.storages.map((x) => x.getInstallParams());
   }
 
   async syncFromSpaces() {
-    const { endpoint } = this.input;
-    const wallet = await this.getBlockletWallet();
+    const { endpoint, wallet, delegation } = this.input;
 
     const spaceClient = new SpaceClient({
       endpoint,
+      delegation,
       wallet,
     });
 
-    const { errorCount } = await spaceClient.send(
-      new SyncFolderPullCommand({
-        source: join('.did-objects', this.blockletWallet.address, '/'),
-        target: this.blockletRestoreDir,
+    const { errorCount, message } = await spaceClient.send(
+      new RestoreBlockletCommand({
+        appDid: this.input.appDid,
+        target: join(this.blockletRestoreDir, '/'),
         debug: true,
-        concurrency: 64,
-        retryCount: 100,
+        concurrency: 32,
+        retryCount: 10,
+        onProgress: (data) => {
+          logger.info('restore progress', { appDid: this.input.appDid, data });
+          this.input.event.emit(BlockletEvents.restoreProgress, {
+            did: this.input.appDid,
+            message: `Downloaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+          });
+        },
+
+        userDid: this.input.userDid,
+        referrer: this.input.referrer,
       })
     );
 
     if (errorCount !== 0) {
-      throw new Error(`Sync from spaces encountered ${errorCount} error`);
+      throw new Error(`Sync from spaces encountered ${errorCount} error: ${message}`);
     }
   }
 
   async import() {
+    this.input.event.emit(BlockletEvents.restoreProgress, {
+      did: this.input.appDid,
+      message: 'Preparing to import data...',
+    });
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.import();
       })
     );
+    this.input.event.emit(BlockletEvents.restoreProgress, {
+      did: this.input.appDid,
+      message: 'Importing data successfully...',
+    });
   }
 }
 

package/lib/event.js

@@ -81,7 +81,7 @@ module.exports = ({
     }
   };
 
-  const handleBlockletAdd = async (name, { blocklet, context }) => {
+  const handleBlockletInstall = async (name, { blocklet, context }) => {
     try {
       const changed = await ensureBlockletRouting(blocklet, context);
       if (changed) {
@@ -169,11 +169,17 @@ module.exports = ({
     });
   };
 
+  /**
+   *
+   * @description 事件必须注册在这里才能被发布出去
+   * @param {string} eventName
+   * @param {any} payload
+   */
   const handleBlockletEvent = async (eventName, payload) => {
     const blocklet = payload.blocklet || payload;
 
     if ([BlockletEvents.installed].includes(eventName)) {
-      await handleBlockletAdd(eventName, payload);
+      await handleBlockletInstall(eventName, payload);
 
       try {
         await node.createAuditLog({
@@ -273,6 +279,13 @@ module.exports = ({
     }
   };
 
+  /**
+   *
+   *
+   * @param {*} subject
+   * @param {string} event
+   * @param {(event: string, data: any) => Promise<void> | void} handler
+   */
   const listen = (subject, event, handler) => subject.on(event, (data) => handler(event, data));
 
   [
@@ -291,6 +304,9 @@ module.exports = ({
     BlockletEvents.startFailed,
     BlockletEvents.stopped,
     BlockletEvents.appDidChanged,
+
+    BlockletEvents.backupProgress,
+    BlockletEvents.restoreProgress,
   ].forEach((eventName) => {
     listen(blockletManager, eventName, handleBlockletEvent);
   });

package/lib/index.js

@@ -26,7 +26,7 @@ const Maintain = require('./util/maintain');
 const resetNode = require('./util/reset-node');
 const DiskMonitor = require('./util/disk-monitor');
 const StoreUtil = require('./util/store');
-const createQueue = require('./queue');
+const createQueue = require('./util/queue');
 const createEvents = require('./event');
 const pm2Events = require('./blocklet/manager/pm2-events');
 const { createStateReadyQueue, createStateReadyHandler } = require('./util/ready');
@@ -99,7 +99,25 @@ function ABTNode(options) {
       concurrency,
       maxRetries: 3,
       retryDelay: 5000, // retry after 5 seconds
-      maxTimeout: 60 * 1000 * 15, // throw timeout error after 5 minutes
+      maxTimeout: 60 * 1000 * 15, // throw timeout error after 15 minutes
+      id: (job) => (job ? md5(`${job.entity}-${job.action}-${job.id}`) : ''),
+    },
+  });
+
+  const backupQueue = createQueue({
+    daemon: options.daemon,
+    name: 'backup_queue',
+    dataDir: dataDirs.core,
+    onJob: async (job) => {
+      if (typeof blockletManager.onJob === 'function') {
+        await blockletManager.onJob(job);
+      }
+    },
+    options: {
+      concurrency,
+      maxRetries: 3,
+      retryDelay: 10000, // retry after 10 seconds
+      maxTimeout: 60 * 1000 * 30, // throw timeout error after 30 minutes
       id: (job) => (job ? md5(`${job.entity}-${job.action}-${job.id}`) : ''),
     },
   });
@@ -129,6 +147,7 @@ function ABTNode(options) {
     dataDirs,
     startQueue,
     installQueue,
+    backupQueue,
     daemon: options.daemon,
     teamManager,
   });
@@ -192,7 +211,6 @@ function ABTNode(options) {
 
     // Blocklet manager
     installBlocklet: blockletManager.install.bind(blockletManager),
-    installBlockletFromVc: blockletManager.installBlockletFromVc.bind(blockletManager),
     installComponent: blockletManager.installComponent.bind(blockletManager),
     startBlocklet: blockletManager.start.bind(blockletManager),
     stopBlocklet: blockletManager.stop.bind(blockletManager),

package/lib/router/helper.js

@@ -26,7 +26,6 @@ const {
   NAME_FOR_WELLKNOWN_SITE,
   DEFAULT_HTTP_PORT,
   DEFAULT_HTTPS_PORT,
-  NODE_MODES,
   ROUTING_RULE_TYPES,
   CERTIFICATE_EXPIRES_OFFSET,
   DEFAULT_SERVICE_PATH,
@@ -56,6 +55,7 @@ const {
   findInterfacePortByName,
   getWellknownSitePort,
   getServerDidDomain,
+  isGatewayCacheEnabled,
 } = require('../util');
 const { getIpDnsDomainForBlocklet, getDidDomainForBlocklet } = require('../util/get-domain-for-blocklet');
 const { getFromCache: getAccessibleExternalNodeIp } = require('../util/get-accessible-external-node-ip');
@@ -382,7 +382,7 @@ const ensureBlockletCache = async (sites = [], blocklets) => {
             const clone = cloneDeep(rule);
             clone.from.pathPrefix = joinUrl(rule.from.pathPrefix, cachePrefix);
             clone.to.cacheGroup = 'blockletProxy';
-            clone.to.target = cachePrefix;
+            clone.to.targetPrefix = cachePrefix;
             clone.dynamic = true; // mark as dynamic to avoid redundant generated rules
             cacheRules.push(clone);
           });
@@ -518,7 +518,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const ensureDomainCert = async (domain, url) => {
       const cert = await certManager.getByDomain(domain);
-      if (!cert) {
+      if (!cert || get(cert, 'meta.validTo') <= Date.now()) {
         await downloadCert({
           domain,
           url,
@@ -1115,7 +1115,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
           configDir: path.join(dataDirs.router, providerName),
           httpPort: nodeInfo.routing.httpPort || DEFAULT_HTTP_PORT,
           httpsPort: nodeInfo.routing.httpsPort || DEFAULT_HTTPS_PORT,
-          cacheDisabled: nodeInfo.mode === NODE_MODES.DEBUG,
+          cacheEnabled: isGatewayCacheEnabled(nodeInfo),
         }),
         getRoutingParams: async () => {
           try {
@@ -1220,7 +1220,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
             configDir: path.join(dataDirs.router, info.routing.provider),
             httpPort: info.routing.httpPort || DEFAULT_HTTP_PORT,
             httpsPort: info.routing.httpsPort || DEFAULT_HTTPS_PORT,
-            cacheDisabled: info.mode === NODE_MODES.DEBUG,
+            cacheEnabled: isGatewayCacheEnabled(info),
           });
           await providerInstance.stop();
           logger.info('original router stopped:', { provider: info.routing.provider });
@@ -1408,6 +1408,8 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     deleteRoutingRule,
     addDomainAlias,
     deleteDomainAlias,
+
+    isGatewayCacheEnabled,
   };
 };
 

package/lib/router/index.js

@@ -11,8 +11,11 @@ const {
   GATEWAY_REQ_LIMIT,
 } = require('@abtnode/constant');
 const { BLOCKLET_UI_INTERFACES, BLOCKLET_MODES } = require('@blocklet/constant');
+
 const logger = require('@abtnode/logger')('@abtnode/core:router');
 
+const { isGatewayCacheEnabled } = require('../util');
+
 const expandSites = (sites = []) => {
   const result = [];
 
@@ -117,6 +120,7 @@ class Router {
       services,
       nodeInfo: pick(nodeInfo, ['name', 'version', 'port', 'mode', 'enableWelcomePage', 'routing']),
       requestLimit,
+      cacheEnabled: isGatewayCacheEnabled(nodeInfo),
     });
   }
 
@@ -183,6 +187,10 @@ Router.formatSites = (sites = []) => {
   result.forEach((site) => {
     if (Array.isArray(site.rules) && site.rules.length > 0) {
       const rules = cloneDeep(site.rules);
+
+      let hasRootPathBlockletRule = false;
+      let tmpBlockletRule;
+
       rules.forEach((rule) => {
         if ([ROUTING_RULE_TYPES.BLOCKLET].includes(rule.to.type) === false) {
           return;
@@ -197,6 +205,11 @@ Router.formatSites = (sites = []) => {
         }
 
         if (daemonRule) {
+          if (rule.from.pathPrefix === '/') {
+            hasRootPathBlockletRule = true;
+          }
+          tmpBlockletRule = rule;
+
           // Serve meta js: both prefix and suffix do not contain trailing slash
           // NOTICE: 这里隐含了一个约定
           //   如果安装的 blockletA 和 blockletB 都需要 __blocklet__.js
@@ -244,6 +257,24 @@ Router.formatSites = (sites = []) => {
           });
         }
       });
+
+      // ensure /__blocklet__.js should be proxy to daemon
+      if (daemonRule && !hasRootPathBlockletRule && tmpBlockletRule) {
+        site.rules.push({
+          from: {
+            pathPrefix: '/',
+            groupPathPrefix: '/',
+            pathSuffix: '/__blocklet__.js',
+          },
+          to: {
+            type: ROUTING_RULE_TYPES.DAEMON,
+            port: daemonRule.to.port,
+            did: tmpBlockletRule.to.did,
+            componentId: tmpBlockletRule.to.did,
+            cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletJs' : '',
+          },
+        });
+      }
     }
   });
 

package/lib/states/audit-log.js

@@ -239,9 +239,11 @@ const getLogContent = async (action, args, context, result, info, node) => {
     case 'deleteRoutingRule':
       return `deleted routing rule from ${site}`; // prettier-ignore
     case 'updateGateway': {
-      let message = args.requestLimit.enabled ? `status: enabled, rate: ${args.requestLimit.rate}` : 'status: disabled';
-      message = `update gateway. ${message}`;
-
+      const changes = [
+        args.requestLimit.enabled ? `rate limit: enabled, rate: ${args.requestLimit.rate}` : 'rate limit: disabled',
+        args.cacheEnabled ? `global cache: enabled, rate: ${args.cacheEnabled}` : 'global cache: disabled',
+      ];
+      const message = `update gateway: ${changes.join('; ')}`;
       return message;
     }
     case 'createTransferInvitation':