@abtnode/core 1.8.68 → 1.8.69-beta-54faead3

package/lib/blocklet/storage/backup/blocklet-extras.js

@@ -1,8 +1,8 @@
-const { removeSync, outputJsonSync, readFileSync } = require('fs-extra');
+const { removeSync, readFileSync, outputJsonSync } = require('fs-extra');
 const { isEmpty, cloneDeep } = require('lodash');
 const { join } = require('path');
 const security = require('@abtnode/util/lib/security');
-const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+
 const states = require('../../../states');
 const { BaseBackup } = require('./base');
 
@@ -11,8 +11,8 @@ class BlockletExtrasBackup extends BaseBackup {
 
   async export() {
     const blockletExtra = await this.getBlockletExtra();
-    removeSync(join(this.blockletBackupDir, this.filename));
-    outputJsonSync(join(this.blockletBackupDir, this.filename), blockletExtra);
+    removeSync(join(this.backupDir, this.filename));
+    outputJsonSync(join(this.backupDir, this.filename), blockletExtra);
   }
 
   /**
@@ -32,35 +32,35 @@ class BlockletExtrasBackup extends BaseBackup {
       throw new Error('blockletExtra cannot be empty');
     }
 
-    return this.cleanDate(blockletExtra);
+    return this.cleanData(blockletExtra);
   }
 
   /**
    *
    * @description 清理数据并加密
-   * @param {import('@abtnode/client').BlockletState} blockletExtraInput
-   * @return {Promise<void>}
+   * @param {import('@abtnode/client').BlockletState} raw
+   * @return {Promise<any>}
    * @memberof BlockletExtrasBackup
    */
-  async cleanDate(blockletExtraInput) {
-    const blockletExtra = cloneDeep(blockletExtraInput);
+  async cleanData(raw) {
+    const blockletExtra = cloneDeep(raw);
 
     const queue = [blockletExtra];
     while (queue.length) {
-      const currentBlockletExtra = queue.pop();
+      const current = queue.pop();
 
       // 删除父 blocklet 的某些数据
-      if (currentBlockletExtra._id) {
-        delete currentBlockletExtra._id;
-        delete currentBlockletExtra.createdAt;
-        delete currentBlockletExtra.updatedAt;
+      if (current._id) {
+        delete current._id;
+        delete current.createdAt;
+        delete current.updatedAt;
       }
 
       // 加解密
-      this.useBlockletEncryptConfigs(currentBlockletExtra.configs);
+      this.encrypt(current.configs);
 
-      if (currentBlockletExtra?.children) {
-        queue.push(...currentBlockletExtra.children);
+      if (current?.children) {
+        queue.push(...current.children);
       }
     }
 
@@ -74,33 +74,16 @@ class BlockletExtrasBackup extends BaseBackup {
    * @return {void}
    * @memberof BlockletExtrasBackup
    */
-  useBlockletEncryptConfigs(configs) {
+  encrypt(configs) {
     if (isEmpty(configs)) {
       return;
     }
 
-    // 准备加解密所需的参数
-    // @see: https://github.com/ArcBlock/blocklet-server/blob/f561ba7290285f2e23dccb6d5323eb4d43c3cc3e/core/state/lib/index.js#L59
-    const dek = readFileSync(join(this.serverDataDir, '.sock'));
-
+    const dk = readFileSync(join(this.serverDir, '.sock'));
     for (const config of configs) {
-      // 置空 blocklet 的密钥，但是保留其他属性，这很重要
-      if (config.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK) {
-        config.value = '';
-      } else if (config.secure) {
-        // secure 为 true 的配置才需要被加密保存上次到 did spaces
-
-        const encryptByServer = config.value;
-        // 先从 blocklet server 解密
-        // @see https://github.com/ArcBlock/blocklet-server/blob/f40338168a66893f325464cea79ae54c43f623b1/core/state/lib/blocklet/extras.js#L139
-        const decryptByServer = security.decrypt(encryptByServer, this.blocklet.meta.did, dek);
-        // 再用 blocklet secret 加密，然后才可以上传到 spaces
-        const encryptByBlocklet = security.encrypt(
-          decryptByServer,
-          this.blockletWallet.address,
-          Buffer.from(this.blockletWallet.secretKey)
-        );
-        config.value = encryptByBlocklet;
+      if (config.secure) {
+        const decrypted = security.decrypt(config.value, this.blocklet.meta.did, dk);
+        config.value = this.securityContext.encrypt(decrypted);
       }
     }
   }

package/lib/blocklet/storage/backup/blocklet.js

@@ -8,45 +8,62 @@ class BlockletBackup extends BaseBackup {
 
   async export() {
     const blocklet = await this.cleanData();
-    removeSync(join(this.blockletBackupDir, this.filename));
-    outputJsonSync(join(this.blockletBackupDir, this.filename), blocklet);
+    removeSync(join(this.backupDir, this.filename));
+    outputJsonSync(join(this.backupDir, this.filename), blocklet);
   }
 
   /**
    * @description 清理数据
    * @see blocklet.db 中需要删除哪些字段呢？ https://github.com/ArcBlock/blocklet-server/issues/6120#issuecomment-1383798348
-   * @return {Promise<void>}
+   * @return {Promise<import('@abtnode/client').BlockletState>}
    * @memberof BlockletBackup
    */
   async cleanData() {
-    const originalBlocklet = cloneDeep(this.blocklet);
+    const clone = cloneDeep(this.blocklet);
 
     /** @type {import('@abtnode/client').ComponentState[]} */
-    const queue = [originalBlocklet];
+    const queue = [clone];
 
     // 广度优先遍历
     while (queue.length) {
-      const currentBlocklet = queue.pop();
+      const current = queue.pop();
 
       // 父组件才需要删除的属性
-      if (currentBlocklet._id) {
-        delete currentBlocklet._id;
-        delete currentBlocklet.createdAt;
-        delete currentBlocklet.startedAt;
-        delete currentBlocklet.installedAt;
+      if (current._id) {
+        delete current._id;
+        delete current.createdAt;
+        delete current.startedAt;
+        delete current.installedAt;
       }
 
       // 子组件和父组件都需要删除的属性
-      delete currentBlocklet.status;
-      delete currentBlocklet.ports;
-      delete currentBlocklet.environments;
+      delete current.status;
+      delete current.ports;
+      delete current.environments;
 
-      if (currentBlocklet.children) {
-        queue.push(...currentBlocklet.children);
+      if (current.children) {
+        queue.push(...current.children);
       }
     }
 
-    return originalBlocklet;
+    return this.encrypt(clone);
+  }
+
+  /**
+   *
+   * @description 清理数据并加密
+   * @param {import('@abtnode/client').BlockletState} info
+   * @memberof BlockletExtrasBackup
+   */
+  encrypt(info) {
+    if (Array.isArray(info.migratedFrom)) {
+      info.migratedFrom = info.migratedFrom.map((x) => {
+        x.appSk = this.securityContext.encrypt(x.appSk);
+        return x;
+      });
+    }
+
+    return info;
   }
 }
 

package/lib/blocklet/storage/backup/blocklets.js

@@ -12,7 +12,7 @@ class BlockletsBackup extends BaseBackup {
    */
   async export() {
     const blockletMetas = this.getBlockletMetas(this.blocklet);
-    const serverBlockletsDir = join(this.serverDataDir, 'blocklets');
+    const serverBlockletsDir = join(this.serverDir, 'blocklets');
 
     const blockletMetasFromLocal = blockletMetas.filter(
       (b) => !validUrl.isHttpUri(b.tarball) && !validUrl.isHttpsUri(b.tarball)
@@ -21,7 +21,7 @@ class BlockletsBackup extends BaseBackup {
     const dirs = [];
     for (const blockletMeta of blockletMetasFromLocal) {
       const sourceDir = join(serverBlockletsDir, blockletMeta.name, blockletMeta.version);
-      const zipPath = join(this.blockletBackupDir, 'blocklets', blockletMeta.name, `${blockletMeta.version}.zip`);
+      const zipPath = join(this.backupDir, 'blocklets', blockletMeta.name, `${blockletMeta.version}.zip`);
       dirs.push({ sourceDir, zipPath });
     }
 

package/lib/blocklet/storage/backup/data.js

@@ -9,8 +9,8 @@ class DataBackup extends BaseBackup {
    * @memberof BlockletsBackup
    */
   async export() {
-    const blockletDataDir = join(this.serverDataDir, 'data', this.blocklet.meta.name);
-    const blockletBackupDataDir = join(this.blockletBackupDir, 'data');
+    const blockletDataDir = join(this.serverDir, 'data', this.blocklet.meta.name);
+    const blockletBackupDataDir = join(this.backupDir, 'data');
 
     await copy(blockletDataDir, blockletBackupDataDir, { overwrite: true });
   }

package/lib/blocklet/storage/backup/logs.js

@@ -5,10 +5,10 @@ const { BaseBackup } = require('./base');
 // note: 可以不需要备份
 class LogsBackup extends BaseBackup {
   async export() {
-    const sourceLogsDir = join(this.serverDataDir, 'logs', this.blocklet.meta.name);
+    const sourceLogsDir = join(this.serverDir, 'logs', this.blocklet.meta.name);
     ensureDirSync(sourceLogsDir);
 
-    const targetLogsDir = join(this.blockletBackupDir, 'logs');
+    const targetLogsDir = join(this.backupDir, 'logs');
 
     await copy(sourceLogsDir, targetLogsDir, {
       overwrite: true,

package/lib/blocklet/storage/backup/routing-rule.js

@@ -11,8 +11,8 @@ class RoutingRuleBackup extends BaseBackup {
       domain: `${this.blocklet.meta.did}.blocklet-domain-group`,
     });
 
-    removeSync(join(this.blockletBackupDir, this.filename));
-    outputJsonSync(join(this.blockletBackupDir, this.filename), routingRule);
+    removeSync(join(this.backupDir, this.filename));
+    outputJsonSync(join(this.backupDir, this.filename), routingRule);
   }
 }
 

package/lib/blocklet/storage/backup/spaces.js

@@ -1,16 +1,34 @@
 /**
  * @typedef {{
- *  did?: string
+ *  appPid: string
+ *  event: import('events').EventEmitter,
+ *  userDid: string,
+ *  referrer: string,
  * }} SpaceBackupInput
+ *
+ * @typedef {{
+ *  signer: import('@ocap/wallet').WalletObject
+ *  encrypt: (v: string) => string,
+ *  decrypt: (v: string) => string,
+ *  delegation: string,
+ * }} SecurityContext
  */
 
 const { isValid } = require('@arcblock/did');
-const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
-const { getBlockletInfo } = require('@blocklet/meta');
-const { SpaceClient, SyncFolderPushCommand } = require('@did-space/client');
+const { BLOCKLET_CONFIGURABLE_KEY, BlockletEvents } = require('@blocklet/constant');
+const { SpaceClient, BackupBlockletCommand } = require('@did-space/client');
 const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
-const { join } = require('path');
+const { join, basename } = require('path');
+const { signV2 } = require('@arcblock/jwt');
+const { Hasher } = require('@ocap/mcrypto');
+const { toBuffer } = require('@ocap/util');
+const { getAppName, getAppDescription } = require('@blocklet/meta/lib/util');
+const getBlockletInfo = require('@blocklet/meta/lib/info');
+const security = require('@abtnode/util/lib/security');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:backup');
+
 const states = require('../../../states');
 const { AuditLogBackup } = require('./audit-log');
 const { BlockletBackup } = require('./blocklet');
@@ -39,7 +57,7 @@ class SpacesBackup {
    * @type {string}
    * @memberof SpacesBackup
    */
-  blockletBackupDir;
+  backupDir;
 
   /**
    *
@@ -47,7 +65,7 @@ class SpacesBackup {
    * @type {string}
    * @memberof SpacesBackup
    */
-  serverDataDir;
+  serverDir;
 
   /**
    *
@@ -55,15 +73,15 @@ class SpacesBackup {
    * @type {string}
    * @memberof SpacesBackup
    */
-  spacesEndpoint;
+  spaceEndpoint;
 
   /**
    *
-   * @description spaces 的 endpoint
-   * @type {import('@ocap/wallet').WalletObject}
+   * @description spaces 安全相关的上下文
+   * @type SecurityContext
    * @memberof SpacesBackup
    */
-  blockletWallet;
+  securityContext;
 
   storages;
 
@@ -91,8 +109,8 @@ class SpacesBackup {
    * @memberof SpacesBackup
    */
   verify(input) {
-    if (isEmpty(input?.did) || !isValid(input?.did)) {
-      throw new Error(`input.did(${input?.did}) is not a valid did`);
+    if (isEmpty(input?.appPid) || !isValid(input?.appPid)) {
+      throw new Error(`input.appPid(${input?.appPid}) is not a valid did`);
     }
   }
 
@@ -108,68 +126,116 @@ class SpacesBackup {
   }
 
   async initialize() {
-    this.blocklet = await states.blocklet.findOne({
-      appDid: this.input.did,
-    });
+    this.blocklet = await states.blocklet.getBlocklet(this.input.appPid);
     if (isEmpty(this.blocklet)) {
       throw new Error('blocklet cannot be empty');
     }
 
-    this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
+    this.serverDir = process.env.ABT_NODE_DATA_DIR;
+    this.backupDir = join(this.serverDir, 'tmp/backup', this.blocklet.appPid);
+    ensureDirSync(this.backupDir);
 
-    this.blockletBackupDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/backup', this.blocklet.meta.did);
-    ensureDirSync(this.blockletBackupDir);
-
-    this.spacesEndpoint = this.blocklet.environments.find(
+    this.spaceEndpoint = this.blocklet.environments.find(
       (e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SPACE_ENDPOINT
     )?.value;
-    if (isEmpty(this.spacesEndpoint)) {
-      throw new Error('spacesEndpoint cannot be empty');
+    if (isEmpty(this.spaceEndpoint)) {
+      throw new Error('spaceEndpoint cannot be empty');
     }
 
-    this.blockletWallet = await this.getBlockletWallet();
+    this.securityContext = await this.getSecurityContext();
   }
 
   async export() {
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      appPid: this.input.appPid,
+      message: 'Preparing data for backup...',
+      progress: 15,
+      completed: false,
+    });
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.export();
       })
     );
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      appPid: this.input.appPid,
+      message: 'Data ready, start backup...',
+      progress: 20,
+      completed: false,
+    });
   }
 
   async syncToSpaces() {
-    const wallet = await this.getBlockletWallet();
     const spaceClient = new SpaceClient({
-      endpoint: this.spacesEndpoint,
-      wallet,
+      endpoint: this.spaceEndpoint,
+      wallet: this.securityContext.signer,
+      delegation: this.securityContext.delegation,
     });
 
-    // FIXME: 在 Spaces 里面预览出 blocklet 的样式，需要规划一个好的数据结构
-    const { errorCount } = await spaceClient.send(
-      new SyncFolderPushCommand({
-        source: join(this.blockletBackupDir, '/'),
-        target: join('.did-objects', this.blocklet.appDid),
+    const { errorCount, message } = await spaceClient.send(
+      new BackupBlockletCommand({
+        appDid: this.blocklet.appPid,
+        appName: getAppName(this.blocklet),
+        appDescription: getAppDescription(this.blocklet),
+        userDid: this.input.userDid,
+        referrer: this.input.referrer,
+        source: join(this.backupDir, '/'),
         debug: true,
-        concurrency: 64,
-        retryCount: 100,
+        concurrency: 32,
+        retryCount: 10,
         filter: (object) => {
+          // FIXME: @yejianchao 这里需要更完整的黑名单
           return object.name !== '.DS_Store';
         },
+        onProgress: (data) => {
+          logger.info('backup progress', { appDid: this.input.appPid, data });
+          const percent = (data.completed * 100) / data.total;
+          this.input.event.emit(BlockletEvents.backupProgress, {
+            appPid: this.input.appPid,
+            message: `Uploaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+            // 0.8 是因为上传文件到 spaces 占进度的 80%，+ 20 是因为需要累加之前的进度
+            progress: +Math.ceil(percent * 0.8).toFixed(2) + 20,
+            completed: false,
+          });
+        },
       })
     );
 
     if (errorCount !== 0) {
-      throw new Error(`Sync to spaces encountered ${errorCount} error`);
+      throw new Error(`Sync to spaces encountered error: ${message}`);
     }
   }
 
-  async getBlockletWallet() {
-    const blockletInfo = await states.blocklet.getBlocklet(this.input.did);
+  async getSecurityContext() {
+    const blocklet = await states.blocklet.getBlocklet(this.input.appPid);
     const nodeInfo = await states.node.read();
-    const { wallet } = getBlockletInfo(blockletInfo, nodeInfo.sk);
-    return wallet;
+
+    const { wallet, permanentWallet } = getBlockletInfo(blocklet, nodeInfo.sk);
+
+    // FIXME: @wangshijun migrated apps can not be restored, need to change core/webapp/api/routes/auth/verify-app-ownership.js
+    const { secretKey, address } = wallet; // we encrypt using latest wallet, not the permanent wallet
+    const password = toBuffer(Hasher.SHA3.hash256(Buffer.concat([secretKey, address].map(toBuffer))));
+    const encrypt = (v) => security.encrypt(v, address, password);
+    const decrypt = (v) => security.decrypt(v, address, password);
+
+    const delegation =
+      permanentWallet.address !== wallet.address
+        ? signV2(permanentWallet.address, permanentWallet.secretKey, {
+            from: permanentWallet.address,
+            to: wallet.address,
+            userPk: permanentWallet.publicKey,
+            permissions: [{ role: 'DIDSpaceAgent', spaces: ['*'] }],
+            exp: Math.floor(new Date().getTime() / 1000) + 60 * 60,
+          })
+        : '';
+
+    return {
+      signer: wallet,
+      delegation,
+      encrypt,
+      decrypt,
+    };
   }
 }
 

package/lib/blocklet/storage/restore/base.js

@@ -11,15 +11,7 @@ class BaseRestore {
    * @type {string}
    * @memberof BaseRestore
    */
-  blockletRestoreDir;
-
-  /**
-   *
-   * @description spaces 的 endpoint
-   * @type {import('@ocap/wallet').WalletObject}
-   * @memberof BaseRestore
-   */
-  blockletWallet;
+  restoreDir;
 
   /**
    *
@@ -27,7 +19,7 @@ class BaseRestore {
    * @type {string}
    * @memberof BaseRestore
    */
-  serverDataDir;
+  serverDir;
 
   constructor(input) {
     this.input = input;
@@ -40,14 +32,34 @@ class BaseRestore {
    * @memberof BaseRestore
    */
   ensureParams(spaces) {
-    this.blockletRestoreDir = spaces.blockletRestoreDir;
-    this.blockletWallet = spaces.blockletWallet;
-    this.serverDataDir = spaces.serverDataDir;
+    this.restoreDir = spaces.restoreDir;
+    this.serverDir = spaces.serverDir;
   }
 
-  async import() {
+  // eslint-disable-next-line
+  async import(params) {
     throw new Error('not implemented');
   }
+
+  /**
+   * Generate params for BlockletManager to install
+   *
+   * @return {object}
+   * @memberof BaseRestore
+   */
+  getInstallParams() {
+    return {};
+  }
+
+  /**
+   * Generate params for other restorers
+   *
+   * @return {object}
+   * @memberof BaseRestore
+   */
+  getImportParams() {
+    return {};
+  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -1,85 +1,85 @@
 const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
-const { cloneDeep, isArray } = require('lodash');
+const { cloneDeep, isEmpty } = require('lodash');
 const { join } = require('path');
 const security = require('@abtnode/util/lib/security');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const { BaseRestore } = require('./base');
 
 class BlockletExtrasRestore extends BaseRestore {
   filename = 'blocklet-extras.json';
 
-  async import() {
-    const blockletExtra = await this.getBlockletExtra();
-    removeSync(join(this.blockletRestoreDir, this.filename));
-    outputJsonSync(join(this.blockletRestoreDir, this.filename), blockletExtra);
+  async import(params) {
+    const extras = this.getExtras();
+    this.cleanExtras(extras, params);
+    removeSync(join(this.restoreDir, this.filename));
+    outputJsonSync(join(this.restoreDir, this.filename), extras);
   }
 
   /**
    *
    * @description
-   * @return {Promise<import('@abtnode/client').BlockletState>}
+   * @return {import('@abtnode/client').BlockletState}
    * @memberof BlockletExtrasRestore
    */
-  async getBlockletExtra() {
+  getExtras() {
     /**
      * @type {import('@abtnode/client').BlockletState}
      */
-    const blockletExtra = readJSONSync(join(this.blockletRestoreDir, this.filename));
-
-    return this.cleanData(blockletExtra);
+    return readJSONSync(join(this.restoreDir, this.filename));
   }
 
   /**
    *
    * @description 清理数据并加密
-   * @param {import('@abtnode/client').BlockletState} blockletExtraInput
-   * @return {Promise<void>}
+   * @param {import('@abtnode/client').BlockletState} raw
    * @memberof BlockletExtrasRestore
    */
-  async cleanData(blockletExtraInput) {
-    const blockletExtra = cloneDeep(blockletExtraInput);
+  async cleanExtras(raw, params) {
+    const blockletExtra = cloneDeep(raw);
 
     const queue = [blockletExtra];
     while (queue.length) {
-      const currentBlockletExtra = queue.pop();
+      const current = queue.pop();
 
       // 加解密
-      this.useBlockletDecryptConfigs(currentBlockletExtra.configs);
+      this.decryptConfigs(current.configs, params);
 
-      if (currentBlockletExtra?.children) {
-        queue.push(...currentBlockletExtra.children);
+      if (current?.children) {
+        queue.push(...current.children);
       }
     }
-
-    return blockletExtra;
   }
 
   /**
    *
-   * @description 清理数据并加密
+   * @description 解密加密的数据
    * @param {import('@abtnode/client').ConfigEntry[]} configs
+   * @param {object} params
    * @return {void}
    * @memberof BlockletExtrasRestore
    */
-  useBlockletDecryptConfigs(configs) {
-    if (!configs || !isArray(configs)) {
+  decryptConfigs(configs, params) {
+    if (isEmpty(configs)) {
       return;
     }
 
+    const { password } = this.input;
     for (const config of configs) {
-      // secure 为 true 的配置才需要被加密保存上次到 did spaces
       if (config.secure) {
-        const encryptByBlocklet = config.value;
-
-        // 再用 blocklet secret 加密，然后才可以上传到 spaces
-        const decryptByBlocklet = security.decrypt(
-          encryptByBlocklet,
-          this.blockletWallet.address,
-          Buffer.from(this.blockletWallet.secretKey)
-        );
-
-        config.value = decryptByBlocklet;
+        config.value = security.decrypt(config.value, params.salt, password);
       }
     }
+
+    const skConfig = configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
+    if (skConfig) {
+      this.appSk = skConfig.value;
+    }
+  }
+
+  getInstallParams() {
+    return {
+      appSk: this.appSk,
+    };
   }
 }