@abtnode/core 1.8.68-beta-500af7e5 → 1.8.68

package/lib/blocklet/storage/backup/spaces.js

@@ -1,22 +1,16 @@
 /**
  * @typedef {{
- *  did: string
- *  event: import('events').EventEmitter,
- *  userDid: string,
- *  referrer: string,
+ *  did?: string
  * }} SpaceBackupInput
  */
 
 const { isValid } = require('@arcblock/did');
-const { BLOCKLET_CONFIGURABLE_KEY, BlockletEvents } = require('@blocklet/constant');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const { getBlockletInfo } = require('@blocklet/meta');
-const { SpaceClient, BackupBlockletCommand } = require('@did-space/client');
+const { SpaceClient, SyncFolderPushCommand } = require('@did-space/client');
 const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
-const { join, basename } = require('path');
-
-const logger = require('@abtnode/logger')('@abtnode/core:storage:backup');
-
+const { join } = require('path');
 const states = require('../../../states');
 const { AuditLogBackup } = require('./audit-log');
 const { BlockletBackup } = require('./blocklet');
@@ -137,22 +131,12 @@ class SpacesBackup {
   }
 
   async export() {
-    this.input.event.emit(BlockletEvents.backupProgress, {
-      did: this.input.did,
-      message: 'Preparing data for backup...',
-      progress: 15,
-    });
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.export();
       })
     );
-    this.input.event.emit(BlockletEvents.backupProgress, {
-      did: this.input.did,
-      message: 'Data ready, start backup...',
-      progress: 20,
-    });
   }
 
   async syncToSpaces() {
@@ -162,34 +146,22 @@ class SpacesBackup {
       wallet,
     });
 
-    const { errorCount, message } = await spaceClient.send(
-      new BackupBlockletCommand({
-        appDid: this.blocklet.appDid,
+    // FIXME: 在 Spaces 里面预览出 blocklet 的样式，需要规划一个好的数据结构
+    const { errorCount } = await spaceClient.send(
+      new SyncFolderPushCommand({
         source: join(this.blockletBackupDir, '/'),
+        target: join('.did-objects', this.blocklet.appDid),
         debug: true,
-        concurrency: 32,
-        retryCount: 10,
+        concurrency: 64,
+        retryCount: 100,
         filter: (object) => {
-          // FIXME: @yejianchao 这里需要更完整的黑名单
           return object.name !== '.DS_Store';
         },
-        onProgress: (data) => {
-          logger.info('backup progress', { appDid: this.input.did, data });
-          const percent = (data.completed * 100) / data.total;
-          this.input.event.emit(BlockletEvents.backupProgress, {
-            did: this.input.did,
-            message: `Uploaded file ${basename(data.key)} (${data.completed}/${data.total})`,
-            // 0.8 是因为上传文件到 spaces 占进度的 80%，+ 20 是因为需要累加之前的进度
-            progress: +Math.ceil(percent * 0.8).toFixed(2) + 20,
-          });
-        },
-        userDid: this.input.userDid,
-        referrer: this.input.referrer,
       })
     );
 
     if (errorCount !== 0) {
-      throw new Error(`Sync to spaces encountered ${errorCount} error: ${message}`);
+      throw new Error(`Sync to spaces encountered ${errorCount} error`);
     }
   }
 

package/lib/blocklet/storage/restore/base.js

@@ -13,6 +13,14 @@ class BaseRestore {
    */
   blockletRestoreDir;
 
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {import('@ocap/wallet').WalletObject}
+   * @memberof BaseRestore
+   */
+  blockletWallet;
+
   /**
    *
    * @description server 的数据目录
@@ -33,22 +41,13 @@ class BaseRestore {
    */
   ensureParams(spaces) {
     this.blockletRestoreDir = spaces.blockletRestoreDir;
+    this.blockletWallet = spaces.blockletWallet;
     this.serverDataDir = spaces.serverDataDir;
   }
 
   async import() {
     throw new Error('not implemented');
   }
-
-  /**
-   * Generate params for BlockletManager to install
-   *
-   * @return {object}
-   * @memberof BaseRestore
-   */
-  getInstallParams() {
-    return {};
-  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -2,7 +2,6 @@ const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
 const { cloneDeep, isArray } = require('lodash');
 const { join } = require('path');
 const security = require('@abtnode/util/lib/security');
-const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const { BaseRestore } = require('./base');
 
 class BlockletExtrasRestore extends BaseRestore {
@@ -25,6 +24,7 @@ class BlockletExtrasRestore extends BaseRestore {
      * @type {import('@abtnode/client').BlockletState}
      */
     const blockletExtra = readJSONSync(join(this.blockletRestoreDir, this.filename));
+
     return this.cleanData(blockletExtra);
   }
 
@@ -32,6 +32,7 @@ class BlockletExtrasRestore extends BaseRestore {
    *
    * @description 清理数据并加密
    * @param {import('@abtnode/client').BlockletState} blockletExtraInput
+   * @return {Promise<void>}
    * @memberof BlockletExtrasRestore
    */
   async cleanData(blockletExtraInput) {
@@ -42,7 +43,7 @@ class BlockletExtrasRestore extends BaseRestore {
       const currentBlockletExtra = queue.pop();
 
       // 加解密
-      this.decrypt(currentBlockletExtra.configs);
+      this.useBlockletDecryptConfigs(currentBlockletExtra.configs);
 
       if (currentBlockletExtra?.children) {
         queue.push(...currentBlockletExtra.children);
@@ -54,37 +55,31 @@ class BlockletExtrasRestore extends BaseRestore {
 
   /**
    *
-   * @description 解密加密的数据
+   * @description 清理数据并加密
    * @param {import('@abtnode/client').ConfigEntry[]} configs
    * @return {void}
    * @memberof BlockletExtrasRestore
    */
-  decrypt(configs) {
+  useBlockletDecryptConfigs(configs) {
     if (!configs || !isArray(configs)) {
       return;
     }
 
-    const { appDid, password } = this.input;
     for (const config of configs) {
+      // secure 为 true 的配置才需要被加密保存上次到 did spaces
       if (config.secure) {
-        const encrypted = config.value;
-        const decrypted = security.decrypt(encrypted, appDid, Buffer.from(password));
-        config.value = decrypted;
-      }
-    }
+        const encryptByBlocklet = config.value;
 
-    const config = configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
-    if (!config) {
-      throw new Error(`Invalid blocklet backup file, no ${BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK} found`);
-    }
+        // 再用 blocklet secret 加密，然后才可以上传到 spaces
+        const decryptByBlocklet = security.decrypt(
+          encryptByBlocklet,
+          this.blockletWallet.address,
+          Buffer.from(this.blockletWallet.secretKey)
+        );
 
-    this.appSk = config.value;
-  }
-
-  getInstallParams() {
-    return {
-      appSk: this.appSk,
-    };
+        config.value = decryptByBlocklet;
+      }
+    }
   }
 }
 

package/lib/blocklet/storage/restore/blocklets.js

@@ -1,4 +1,4 @@
-const { existsSync, remove, ensureDirSync } = require('fs-extra');
+const { existsSync, remove } = require('fs-extra');
 const { join } = require('path');
 const fg = require('fast-glob');
 const { BaseRestore } = require('./base');
@@ -10,11 +10,8 @@ class BlockletsRestore extends BaseRestore {
   async import() {
     const blockletsDir = join(this.blockletRestoreDir, this.filename);
 
-    // blockletsDir 可以不存在, 因为还原的 blocklet 可能所有的组件都是来自 store 的
     if (!existsSync(blockletsDir)) {
-      // FIXME: 如果文件夹不存在，需要创建文件夹，因为 installFromBackup 未做容错处理
-      ensureDirSync(blockletsDir);
-      return;
+      throw new Error(`dir not found: ${blockletsDir}`);
     }
 
     const paths = await fg('**/*.zip', {

package/lib/blocklet/storage/restore/spaces.js

@@ -1,24 +1,16 @@
 /**
  * @typedef {{
- *  appDid: string;
  *  endpoint: string;
- *  password: Buffer;
- *  delegation: string;
- *  wallet: import('@ocap/wallet').WalletObject;
- *  event: import('events').EventEmitter,
- *  userDid: string,
- *  referrer: string,
+ *  blockletSecretKey: string;
  * }} SpaceRestoreInput
  */
 
-const validUrl = require('valid-url');
-const { BlockletEvents } = require('@blocklet/constant');
-const { SpaceClient, RestoreBlockletCommand } = require('@did-space/client');
+const { SpaceClient, SyncFolderPullCommand } = require('@did-space/client');
+const { types } = require('@ocap/mcrypto');
+const { fromSecretKey, WalletType } = require('@ocap/wallet');
 const { ensureDirSync, existsSync, rmdirSync } = require('fs-extra');
-const { join, basename } = require('path');
-
-const logger = require('@abtnode/logger')('@abtnode/core:storage:restore');
-
+const { join } = require('path');
+const validUrl = require('valid-url');
 const { BlockletExtrasRestore } = require('./blocklet-extras');
 const { BlockletsRestore } = require('./blocklets');
 
@@ -45,6 +37,14 @@ class SpacesRestore {
    */
   serverDataDir;
 
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {import('@ocap/wallet').WalletObject}
+   * @memberof SpacesRestore
+   */
+  blockletWallet;
+
   storages;
 
   /**
@@ -72,75 +72,68 @@ class SpacesRestore {
 
   async initialize() {
     this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
+    this.blockletWallet = await this.getBlockletWallet();
 
-    if (!this.input.endpoint.includes(this.input.appDid)) {
-      throw new Error(`endpoint and blocklet.appDid(${this.input.appDid}) do not match`);
+    if (!this.input.endpoint.includes(this.blockletWallet.address)) {
+      throw new Error(`endpoint and blocklet.appDid(${this.blockletWallet.address}) do not match`);
     }
 
-    this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.input.appDid);
+    this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.blockletWallet.address);
     if (existsSync(this.blockletRestoreDir)) {
       rmdirSync(this.blockletRestoreDir, { recursive: true });
     }
     ensureDirSync(this.blockletRestoreDir);
   }
 
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof SpacesRestore
+   */
   async restore() {
     await this.initialize();
     await this.syncFromSpaces();
     await this.import();
+  }
+
+  async getBlockletWallet() {
+    // @FIXME: blocklet 钱包类型如何得知呢？
+    const wallet = fromSecretKey(this.input.blockletSecretKey, WalletType({ role: types.RoleType.ROLE_APPLICATION }));
 
-    return this.storages.map((x) => x.getInstallParams());
+    return wallet;
   }
 
   async syncFromSpaces() {
-    const { endpoint, wallet, delegation } = this.input;
+    const { endpoint } = this.input;
+    const wallet = await this.getBlockletWallet();
 
     const spaceClient = new SpaceClient({
       endpoint,
-      delegation,
       wallet,
     });
 
-    const { errorCount, message } = await spaceClient.send(
-      new RestoreBlockletCommand({
-        appDid: this.input.appDid,
-        target: join(this.blockletRestoreDir, '/'),
+    const { errorCount } = await spaceClient.send(
+      new SyncFolderPullCommand({
+        source: join('.did-objects', this.blockletWallet.address, '/'),
+        target: this.blockletRestoreDir,
         debug: true,
-        concurrency: 32,
-        retryCount: 10,
-        onProgress: (data) => {
-          logger.info('restore progress', { appDid: this.input.appDid, data });
-          this.input.event.emit(BlockletEvents.restoreProgress, {
-            did: this.input.appDid,
-            message: `Downloaded file ${basename(data.key)} (${data.completed}/${data.total})`,
-          });
-        },
-
-        userDid: this.input.userDid,
-        referrer: this.input.referrer,
+        concurrency: 64,
+        retryCount: 100,
       })
     );
 
     if (errorCount !== 0) {
-      throw new Error(`Sync from spaces encountered ${errorCount} error: ${message}`);
+      throw new Error(`Sync from spaces encountered ${errorCount} error`);
     }
   }
 
   async import() {
-    this.input.event.emit(BlockletEvents.restoreProgress, {
-      did: this.input.appDid,
-      message: 'Preparing to import data...',
-    });
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.import();
       })
     );
-    this.input.event.emit(BlockletEvents.restoreProgress, {
-      did: this.input.appDid,
-      message: 'Importing data successfully...',
-    });
   }
 }
 

package/lib/event.js

@@ -81,7 +81,7 @@ module.exports = ({
     }
   };
 
-  const handleBlockletInstall = async (name, { blocklet, context }) => {
+  const handleBlockletAdd = async (name, { blocklet, context }) => {
     try {
       const changed = await ensureBlockletRouting(blocklet, context);
       if (changed) {
@@ -169,17 +169,11 @@ module.exports = ({
     });
   };
 
-  /**
-   *
-   * @description 事件必须注册在这里才能被发布出去
-   * @param {string} eventName
-   * @param {any} payload
-   */
   const handleBlockletEvent = async (eventName, payload) => {
     const blocklet = payload.blocklet || payload;
 
     if ([BlockletEvents.installed].includes(eventName)) {
-      await handleBlockletInstall(eventName, payload);
+      await handleBlockletAdd(eventName, payload);
 
       try {
         await node.createAuditLog({
@@ -279,13 +273,6 @@ module.exports = ({
     }
   };
 
-  /**
-   *
-   *
-   * @param {*} subject
-   * @param {string} event
-   * @param {(event: string, data: any) => Promise<void> | void} handler
-   */
   const listen = (subject, event, handler) => subject.on(event, (data) => handler(event, data));
 
   [
@@ -304,9 +291,6 @@ module.exports = ({
     BlockletEvents.startFailed,
     BlockletEvents.stopped,
     BlockletEvents.appDidChanged,
-
-    BlockletEvents.backupProgress,
-    BlockletEvents.restoreProgress,
   ].forEach((eventName) => {
     listen(blockletManager, eventName, handleBlockletEvent);
   });

package/lib/index.js

@@ -26,7 +26,7 @@ const Maintain = require('./util/maintain');
 const resetNode = require('./util/reset-node');
 const DiskMonitor = require('./util/disk-monitor');
 const StoreUtil = require('./util/store');
-const createQueue = require('./util/queue');
+const createQueue = require('./queue');
 const createEvents = require('./event');
 const pm2Events = require('./blocklet/manager/pm2-events');
 const { createStateReadyQueue, createStateReadyHandler } = require('./util/ready');
@@ -99,25 +99,7 @@ function ABTNode(options) {
       concurrency,
       maxRetries: 3,
       retryDelay: 5000, // retry after 5 seconds
-      maxTimeout: 60 * 1000 * 15, // throw timeout error after 15 minutes
-      id: (job) => (job ? md5(`${job.entity}-${job.action}-${job.id}`) : ''),
-    },
-  });
-
-  const backupQueue = createQueue({
-    daemon: options.daemon,
-    name: 'backup_queue',
-    dataDir: dataDirs.core,
-    onJob: async (job) => {
-      if (typeof blockletManager.onJob === 'function') {
-        await blockletManager.onJob(job);
-      }
-    },
-    options: {
-      concurrency,
-      maxRetries: 3,
-      retryDelay: 10000, // retry after 10 seconds
-      maxTimeout: 60 * 1000 * 30, // throw timeout error after 30 minutes
+      maxTimeout: 60 * 1000 * 15, // throw timeout error after 5 minutes
       id: (job) => (job ? md5(`${job.entity}-${job.action}-${job.id}`) : ''),
     },
   });
@@ -147,7 +129,6 @@ function ABTNode(options) {
     dataDirs,
     startQueue,
     installQueue,
-    backupQueue,
     daemon: options.daemon,
     teamManager,
   });
@@ -211,6 +192,7 @@ function ABTNode(options) {
 
     // Blocklet manager
     installBlocklet: blockletManager.install.bind(blockletManager),
+    installBlockletFromVc: blockletManager.installBlockletFromVc.bind(blockletManager),
     installComponent: blockletManager.installComponent.bind(blockletManager),
     startBlocklet: blockletManager.start.bind(blockletManager),
     stopBlocklet: blockletManager.stop.bind(blockletManager),

package/lib/router/helper.js

@@ -26,6 +26,7 @@ const {
   NAME_FOR_WELLKNOWN_SITE,
   DEFAULT_HTTP_PORT,
   DEFAULT_HTTPS_PORT,
+  NODE_MODES,
   ROUTING_RULE_TYPES,
   CERTIFICATE_EXPIRES_OFFSET,
   DEFAULT_SERVICE_PATH,
@@ -55,7 +56,6 @@ const {
   findInterfacePortByName,
   getWellknownSitePort,
   getServerDidDomain,
-  isGatewayCacheEnabled,
 } = require('../util');
 const { getIpDnsDomainForBlocklet, getDidDomainForBlocklet } = require('../util/get-domain-for-blocklet');
 const { getFromCache: getAccessibleExternalNodeIp } = require('../util/get-accessible-external-node-ip');
@@ -382,7 +382,7 @@ const ensureBlockletCache = async (sites = [], blocklets) => {
             const clone = cloneDeep(rule);
             clone.from.pathPrefix = joinUrl(rule.from.pathPrefix, cachePrefix);
             clone.to.cacheGroup = 'blockletProxy';
-            clone.to.targetPrefix = cachePrefix;
+            clone.to.target = cachePrefix;
             clone.dynamic = true; // mark as dynamic to avoid redundant generated rules
             cacheRules.push(clone);
           });
@@ -518,7 +518,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const ensureDomainCert = async (domain, url) => {
       const cert = await certManager.getByDomain(domain);
-      if (!cert || get(cert, 'meta.validTo') <= Date.now()) {
+      if (!cert) {
         await downloadCert({
           domain,
           url,
@@ -1115,7 +1115,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
           configDir: path.join(dataDirs.router, providerName),
           httpPort: nodeInfo.routing.httpPort || DEFAULT_HTTP_PORT,
           httpsPort: nodeInfo.routing.httpsPort || DEFAULT_HTTPS_PORT,
-          cacheEnabled: isGatewayCacheEnabled(nodeInfo),
+          cacheDisabled: nodeInfo.mode === NODE_MODES.DEBUG,
         }),
         getRoutingParams: async () => {
           try {
@@ -1220,7 +1220,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
             configDir: path.join(dataDirs.router, info.routing.provider),
             httpPort: info.routing.httpPort || DEFAULT_HTTP_PORT,
             httpsPort: info.routing.httpsPort || DEFAULT_HTTPS_PORT,
-            cacheEnabled: isGatewayCacheEnabled(info),
+            cacheDisabled: info.mode === NODE_MODES.DEBUG,
           });
           await providerInstance.stop();
           logger.info('original router stopped:', { provider: info.routing.provider });
@@ -1408,8 +1408,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     deleteRoutingRule,
     addDomainAlias,
     deleteDomainAlias,
-
-    isGatewayCacheEnabled,
   };
 };
 

package/lib/router/index.js

@@ -11,11 +11,8 @@ const {
   GATEWAY_REQ_LIMIT,
 } = require('@abtnode/constant');
 const { BLOCKLET_UI_INTERFACES, BLOCKLET_MODES } = require('@blocklet/constant');
-
 const logger = require('@abtnode/logger')('@abtnode/core:router');
 
-const { isGatewayCacheEnabled } = require('../util');
-
 const expandSites = (sites = []) => {
   const result = [];
 
@@ -120,7 +117,6 @@ class Router {
       services,
       nodeInfo: pick(nodeInfo, ['name', 'version', 'port', 'mode', 'enableWelcomePage', 'routing']),
       requestLimit,
-      cacheEnabled: isGatewayCacheEnabled(nodeInfo),
     });
   }
 
@@ -187,10 +183,6 @@ Router.formatSites = (sites = []) => {
   result.forEach((site) => {
     if (Array.isArray(site.rules) && site.rules.length > 0) {
       const rules = cloneDeep(site.rules);
-
-      let hasRootPathBlockletRule = false;
-      let tmpBlockletRule;
-
       rules.forEach((rule) => {
         if ([ROUTING_RULE_TYPES.BLOCKLET].includes(rule.to.type) === false) {
           return;
@@ -205,11 +197,6 @@ Router.formatSites = (sites = []) => {
         }
 
         if (daemonRule) {
-          if (rule.from.pathPrefix === '/') {
-            hasRootPathBlockletRule = true;
-          }
-          tmpBlockletRule = rule;
-
           // Serve meta js: both prefix and suffix do not contain trailing slash
           // NOTICE: 这里隐含了一个约定
           //   如果安装的 blockletA 和 blockletB 都需要 __blocklet__.js
@@ -257,24 +244,6 @@ Router.formatSites = (sites = []) => {
           });
         }
       });
-
-      // ensure /__blocklet__.js should be proxy to daemon
-      if (daemonRule && !hasRootPathBlockletRule && tmpBlockletRule) {
-        site.rules.push({
-          from: {
-            pathPrefix: '/',
-            groupPathPrefix: '/',
-            pathSuffix: '/__blocklet__.js',
-          },
-          to: {
-            type: ROUTING_RULE_TYPES.DAEMON,
-            port: daemonRule.to.port,
-            did: tmpBlockletRule.to.did,
-            componentId: tmpBlockletRule.to.did,
-            cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletJs' : '',
-          },
-        });
-      }
     }
   });
 

package/lib/states/audit-log.js

@@ -239,11 +239,9 @@ const getLogContent = async (action, args, context, result, info, node) => {
     case 'deleteRoutingRule':
       return `deleted routing rule from ${site}`; // prettier-ignore
     case 'updateGateway': {
-      const changes = [
-        args.requestLimit.enabled ? `rate limit: enabled, rate: ${args.requestLimit.rate}` : 'rate limit: disabled',
-        args.cacheEnabled ? `global cache: enabled, rate: ${args.cacheEnabled}` : 'global cache: disabled',
-      ];
-      const message = `update gateway: ${changes.join('; ')}`;
+      let message = args.requestLimit.enabled ? `status: enabled, rate: ${args.requestLimit.rate}` : 'status: disabled';
+      message = `update gateway. ${message}`;
+
       return message;
     }
     case 'createTransferInvitation':