@abtnode/core 1.8.67-beta-794a8082 → 1.8.68-beta-500af7e5

package/lib/blocklet/downloader/bundle-downloader.js

@@ -2,6 +2,7 @@ const { EventEmitter } = require('events');
 const fs = require('fs-extra');
 const { fileURLToPath } = require('url');
 const path = require('path');
+const pick = require('lodash/pick');
 const cloneDeep = require('lodash/cloneDeep');
 const { toBase58 } = require('@ocap/util');
 
@@ -161,7 +162,11 @@ class BundleDownloader extends EventEmitter {
       }
       ctrlStore[rootDid].set(did, cancelCtrl);
 
-      const headers = context.headers ? cloneDeep(context.headers) : {};
+      const headers = pick(context.headers ? cloneDeep(context.headers) : {}, [
+        'x-server-did',
+        'x-server-public-key',
+        'x-server-signature',
+      ]);
       const exist = (context.downloadTokenList || []).find((x) => x.did === did);
       if (exist) {
         headers['x-download-token'] = exist.token;

package/lib/blocklet/manager/disk.js

@@ -4,6 +4,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const flat = require('flat');
 const get = require('lodash/get');
+const merge = require('lodash/merge');
 const pick = require('lodash/pick');
 const cloneDeep = require('lodash/cloneDeep');
 const semver = require('semver');
@@ -111,6 +112,8 @@ const {
   checkDuplicateMountPoint,
   validateStore,
   validateInServerless,
+  isRotatingAppSk,
+  isRotatingAppDid,
 } = require('../../util/blocklet');
 const StoreUtil = require('../../util/store');
 const states = require('../../states');
@@ -193,13 +196,14 @@ class BlockletManager extends BaseBlockletManager {
   /**
    * @param {*} dataDirs generate by ../../util:getDataDirs
    */
-  constructor({ dataDirs, startQueue, installQueue, daemon = false, teamManager }) {
+  constructor({ dataDirs, startQueue, installQueue, backupQueue, daemon = false, teamManager }) {
     super();
 
     this.dataDirs = dataDirs;
     this.installDir = dataDirs.blocklets;
     this.startQueue = startQueue;
     this.installQueue = installQueue;
+    this.backupQueue = backupQueue;
     this.teamManager = teamManager;
 
     // cached installed blocklets for performance
@@ -266,6 +270,7 @@ class BlockletManager extends BaseBlockletManager {
 
     const info = await states.node.read();
 
+    // Note: if you added new header here, please change core/state/lib/blocklet/downloader/bundle-downloader.js to use that header
     context.headers = Object.assign(context?.headers || {}, {
       'x-server-did': info.did,
       'x-server-public-key': info.pk,
@@ -605,32 +610,67 @@ class BlockletManager extends BaseBlockletManager {
   }
 
   /**
-   *
-   *
-   * @param {import('@abtnode/client').BackupToSpacesParams} input
+   * FIXME: @wangshijun create audit log for this
+   * @param {import('@abtnode/client').RequestBackupToSpacesInput} input
    * @memberof BlockletManager
    */
-  async backupToSpaces(input) {
-    const spacesBackup = new SpacesBackup(input);
+  // eslint-disable-next-line no-unused-vars
+  async backupToSpaces({ did }, context) {
+    // add to queue
+    // const ticket = this.backupQueue.push({
+    //   entity: 'blocklet',
+    //   action: 'backup-to-space',
+    //   did,
+    //   context,
+    // });
+
+    // ticket.on('failed', async (err) => {
+    //   logger.error('backup failed', { entity: 'blocklet', did, error: err });
+    //   this.emit('blocklet.backup.failed', { did, err });
+    //   this._createNotification(did, {
+    //     title: 'Blocklet Backup Failed',
+    //     description: `Blocklet backup failed with error: ${err.message || 'queue exception'}`,
+    //     entityType: 'blocklet',
+    //     entityId: did,
+    //     severity: 'error',
+    //   });
+    // });
+
+    const userDid = context.user.did;
+    const { referrer } = context;
+
+    // FIXME: @yejianchao did should be renamed to appDid
+    const spacesBackup = new SpacesBackup({ did, event: this, userDid, referrer });
+    this.emit(BlockletEvents.backupProgress, { did, message: 'Start backup...', progress: 10 });
     await spacesBackup.backup();
+    this.emit(BlockletEvents.backupProgress, { did, completed: true, progress: 100 });
   }
 
   /**
-   *
-   *
+   * FIXME: @linchen support cancel
+   * FIXME: @wangshijun create audit log for this
    * @param {import('@abtnode/client').RequestRestoreFromSpacesInput} input
    * @memberof BlockletManager
    */
-  async restoreFromSpaces(input) {
-    const spacesRestore = new SpacesRestore(input);
-    await spacesRestore.restore();
+  // eslint-disable-next-line no-unused-vars
+  async restoreFromSpaces(input, context) {
+    // FIXME: @yejianchao did should be renamed to appDid
+    this.emit(BlockletEvents.restoreProgress, { did: input.appDid, message: 'Start restore...' });
+
+    const userDid = context.user.did;
+    const { referrer } = context;
+
+    const spacesRestore = new SpacesRestore({ ...input, event: this, userDid, referrer });
+    const params = await spacesRestore.restore();
 
-    // FIXME: 需要改成队列执行，本次失败，下次还需要重试，页面刷新后也能知道最新的状态
+    this.emit(BlockletEvents.restoreProgress, { did: input.appDid, message: 'Installing blocklet...' });
     await this._installFromBackup({
       url: `file://${spacesRestore.blockletRestoreDir}`,
-      appSk: spacesRestore.blockletWallet.secretKey,
       moveDir: true,
+      ...merge(...params),
     });
+
+    this.emit(BlockletEvents.restoreProgress, { did: input.appDid, completed: true });
   }
 
   async restart({ did }, context) {
@@ -1000,6 +1040,7 @@ class BlockletManager extends BaseBlockletManager {
     return blocklets;
   }
 
+  // CAUTION: this method currently only support config by blocklet.meta.did
   // eslint-disable-next-line no-unused-vars
   async config({ did, configs: newConfigs, skipHook, skipDidDocument }, context) {
     if (!Array.isArray(newConfigs)) {
@@ -1031,7 +1072,7 @@ class BlockletManager extends BaseBlockletManager {
           throw new Error(`Cannot set ${x.key} to child blocklet`);
         }
 
-        await validateAppConfig(x, rootDid, states);
+        await validateAppConfig(x, states);
       } else if (!BLOCKLET_CONFIGURABLE_KEY[x.key] && !isPreferenceKey(x)) {
         if (!(blocklet.meta.environments || []).some((y) => y.name === x.key)) {
           // forbid unknown format key
@@ -1054,14 +1095,26 @@ class BlockletManager extends BaseBlockletManager {
       });
     }
 
+    const willAppSkChange = isRotatingAppSk(newConfigs, blocklet.configs, blocklet.externalSk);
+    const willAppDidChange = isRotatingAppDid(newConfigs, blocklet.configs, blocklet.externalSk);
+
     // update db
     await states.blockletExtras.setConfigs(dids, newConfigs);
 
-    const isSkOrWalletTypeChanged = newConfigs.find((item) =>
-      [BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK, BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_WALLET_TYPE].includes(item.key)
-    );
+    if (willAppSkChange) {
+      const info = await states.node.read();
+      const { wallet } = getBlockletInfo(blocklet, info.sk);
+      const migratedFrom = blocklet.migratedFrom || [];
+      await states.blocklet.updateBlocklet(rootDid, {
+        migratedFrom: [
+          ...migratedFrom,
+          { appSk: wallet.secretKey, appDid: wallet.address, at: new Date().toISOString() },
+        ],
+      });
+    }
 
-    if (isSkOrWalletTypeChanged && !skipDidDocument) {
+    // FIXME: @zhenqiang best way to handle the did document: allow all appDids to resolve
+    if (willAppDidChange && !skipDidDocument) {
       await this._updateDidDocument(blocklet);
     }
 
@@ -2064,6 +2117,14 @@ class BlockletManager extends BaseBlockletManager {
     // put BLOCKLET_APP_ID at root level for indexing
     blocklet.appDid = appSystemEnvironments.BLOCKLET_APP_ID;
 
+    if (!blocklet.migratedFrom) {
+      blocklet.migratedFrom = [];
+    }
+    // This can only be set once, can be used for indexing, will not change ever
+    if (!blocklet.appPid) {
+      blocklet.appPid = appSystemEnvironments.BLOCKLET_APP_PID;
+    }
+
     // update state to db
     return states.blocklet.updateBlocklet(did, blocklet);
   }
@@ -2344,7 +2405,7 @@ class BlockletManager extends BaseBlockletManager {
     };
     const meta = validateMeta(rawMeta);
 
-    await states.blocklet.addBlocklet({ meta, source: BlockletSource.custom });
+    await states.blocklet.addBlocklet({ meta, source: BlockletSource.custom, externalSk: !!appSk });
     await this._setConfigsFromMeta(did);
     await this._setAppSk(did, appSk);
 
@@ -2478,6 +2539,7 @@ class BlockletManager extends BaseBlockletManager {
       source: BlockletSource.upload,
       deployedFrom: `Upload by ${context.user.fullName}`,
       children,
+      externalSk: !!appSk,
     });
 
     const action = 'install';
@@ -2812,6 +2874,7 @@ class BlockletManager extends BaseBlockletManager {
         source,
         deployedFrom,
         children,
+        externalSk: !!appSk,
       });
 
       await validateBlocklet(blocklet);

package/lib/blocklet/storage/backup/blocklet-extras.js

@@ -1,8 +1,9 @@
 const { removeSync, outputJsonSync, readFileSync } = require('fs-extra');
 const { isEmpty, cloneDeep } = require('lodash');
 const { join } = require('path');
+const { Hasher } = require('@ocap/mcrypto');
+const { toBuffer } = require('@ocap/util');
 const security = require('@abtnode/util/lib/security');
-const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const states = require('../../../states');
 const { BaseBackup } = require('./base');
 
@@ -32,7 +33,7 @@ class BlockletExtrasBackup extends BaseBackup {
       throw new Error('blockletExtra cannot be empty');
     }
 
-    return this.cleanDate(blockletExtra);
+    return this.cleanData(blockletExtra);
   }
 
   /**
@@ -42,7 +43,7 @@ class BlockletExtrasBackup extends BaseBackup {
    * @return {Promise<void>}
    * @memberof BlockletExtrasBackup
    */
-  async cleanDate(blockletExtraInput) {
+  async cleanData(blockletExtraInput) {
     const blockletExtra = cloneDeep(blockletExtraInput);
 
     const queue = [blockletExtra];
@@ -57,7 +58,7 @@ class BlockletExtrasBackup extends BaseBackup {
       }
 
       // 加解密
-      this.useBlockletEncryptConfigs(currentBlockletExtra.configs);
+      this.encryptBlockletConfigs(currentBlockletExtra.configs);
 
       if (currentBlockletExtra?.children) {
         queue.push(...currentBlockletExtra.children);
@@ -74,33 +75,28 @@ class BlockletExtrasBackup extends BaseBackup {
    * @return {void}
    * @memberof BlockletExtrasBackup
    */
-  useBlockletEncryptConfigs(configs) {
+  encryptBlockletConfigs(configs) {
     if (isEmpty(configs)) {
       return;
     }
 
+    const { secretKey, address } = this.blockletWallet;
+
     // 准备加解密所需的参数
     // @see: https://github.com/ArcBlock/blocklet-server/blob/f561ba7290285f2e23dccb6d5323eb4d43c3cc3e/core/state/lib/index.js#L59
-    const dek = readFileSync(join(this.serverDataDir, '.sock'));
+    const dk = readFileSync(join(this.serverDataDir, '.sock'));
+    const ek = toBuffer(Hasher.SHA3.hash256(Buffer.concat([secretKey, address].map(toBuffer))));
 
     for (const config of configs) {
-      // 置空 blocklet 的密钥，但是保留其他属性，这很重要
-      if (config.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK) {
-        config.value = '';
-      } else if (config.secure) {
+      if (config.secure) {
         // secure 为 true 的配置才需要被加密保存上次到 did spaces
-
         const encryptByServer = config.value;
         // 先从 blocklet server 解密
         // @see https://github.com/ArcBlock/blocklet-server/blob/f40338168a66893f325464cea79ae54c43f623b1/core/state/lib/blocklet/extras.js#L139
-        const decryptByServer = security.decrypt(encryptByServer, this.blocklet.meta.did, dek);
-        // 再用 blocklet secret 加密，然后才可以上传到 spaces
-        const encryptByBlocklet = security.encrypt(
-          decryptByServer,
-          this.blockletWallet.address,
-          Buffer.from(this.blockletWallet.secretKey)
-        );
-        config.value = encryptByBlocklet;
+        const decrypted = security.decrypt(encryptByServer, this.blocklet.meta.did, dk);
+        // 再用 blocklet 信息加密，然后才可以上传到 spaces
+        const encrypted = security.encrypt(decrypted, address, ek);
+        config.value = encrypted;
       }
     }
   }

package/lib/blocklet/storage/backup/spaces.js

@@ -1,16 +1,22 @@
 /**
  * @typedef {{
- *  did?: string
+ *  did: string
+ *  event: import('events').EventEmitter,
+ *  userDid: string,
+ *  referrer: string,
  * }} SpaceBackupInput
  */
 
 const { isValid } = require('@arcblock/did');
-const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+const { BLOCKLET_CONFIGURABLE_KEY, BlockletEvents } = require('@blocklet/constant');
 const { getBlockletInfo } = require('@blocklet/meta');
-const { SpaceClient, SyncFolderPushCommand } = require('@did-space/client');
+const { SpaceClient, BackupBlockletCommand } = require('@did-space/client');
 const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
-const { join } = require('path');
+const { join, basename } = require('path');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:backup');
+
 const states = require('../../../states');
 const { AuditLogBackup } = require('./audit-log');
 const { BlockletBackup } = require('./blocklet');
@@ -131,12 +137,22 @@ class SpacesBackup {
   }
 
   async export() {
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      did: this.input.did,
+      message: 'Preparing data for backup...',
+      progress: 15,
+    });
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.export();
       })
     );
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      did: this.input.did,
+      message: 'Data ready, start backup...',
+      progress: 20,
+    });
   }
 
   async syncToSpaces() {
@@ -146,22 +162,34 @@ class SpacesBackup {
       wallet,
     });
 
-    // FIXME: 在 Spaces 里面预览出 blocklet 的样式，需要规划一个好的数据结构
-    const { errorCount } = await spaceClient.send(
-      new SyncFolderPushCommand({
+    const { errorCount, message } = await spaceClient.send(
+      new BackupBlockletCommand({
+        appDid: this.blocklet.appDid,
         source: join(this.blockletBackupDir, '/'),
-        target: join('.did-objects', this.blocklet.appDid, '/'),
         debug: true,
         concurrency: 32,
         retryCount: 10,
         filter: (object) => {
+          // FIXME: @yejianchao 这里需要更完整的黑名单
           return object.name !== '.DS_Store';
         },
+        onProgress: (data) => {
+          logger.info('backup progress', { appDid: this.input.did, data });
+          const percent = (data.completed * 100) / data.total;
+          this.input.event.emit(BlockletEvents.backupProgress, {
+            did: this.input.did,
+            message: `Uploaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+            // 0.8 是因为上传文件到 spaces 占进度的 80%，+ 20 是因为需要累加之前的进度
+            progress: +Math.ceil(percent * 0.8).toFixed(2) + 20,
+          });
+        },
+        userDid: this.input.userDid,
+        referrer: this.input.referrer,
       })
     );
 
     if (errorCount !== 0) {
-      throw new Error(`Sync to spaces encountered ${errorCount} error`);
+      throw new Error(`Sync to spaces encountered ${errorCount} error: ${message}`);
     }
   }
 

package/lib/blocklet/storage/restore/base.js

@@ -13,14 +13,6 @@ class BaseRestore {
    */
   blockletRestoreDir;
 
-  /**
-   *
-   * @description spaces 的 endpoint
-   * @type {import('@ocap/wallet').WalletObject}
-   * @memberof BaseRestore
-   */
-  blockletWallet;
-
   /**
    *
    * @description server 的数据目录
@@ -41,13 +33,22 @@ class BaseRestore {
    */
   ensureParams(spaces) {
     this.blockletRestoreDir = spaces.blockletRestoreDir;
-    this.blockletWallet = spaces.blockletWallet;
     this.serverDataDir = spaces.serverDataDir;
   }
 
   async import() {
     throw new Error('not implemented');
   }
+
+  /**
+   * Generate params for BlockletManager to install
+   *
+   * @return {object}
+   * @memberof BaseRestore
+   */
+  getInstallParams() {
+    return {};
+  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -2,6 +2,7 @@ const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
 const { cloneDeep, isArray } = require('lodash');
 const { join } = require('path');
 const security = require('@abtnode/util/lib/security');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const { BaseRestore } = require('./base');
 
 class BlockletExtrasRestore extends BaseRestore {
@@ -24,7 +25,6 @@ class BlockletExtrasRestore extends BaseRestore {
      * @type {import('@abtnode/client').BlockletState}
      */
     const blockletExtra = readJSONSync(join(this.blockletRestoreDir, this.filename));
-
     return this.cleanData(blockletExtra);
   }
 
@@ -32,7 +32,6 @@ class BlockletExtrasRestore extends BaseRestore {
    *
    * @description 清理数据并加密
    * @param {import('@abtnode/client').BlockletState} blockletExtraInput
-   * @return {Promise<void>}
    * @memberof BlockletExtrasRestore
    */
   async cleanData(blockletExtraInput) {
@@ -43,7 +42,7 @@ class BlockletExtrasRestore extends BaseRestore {
       const currentBlockletExtra = queue.pop();
 
       // 加解密
-      this.useBlockletDecryptConfigs(currentBlockletExtra.configs);
+      this.decrypt(currentBlockletExtra.configs);
 
       if (currentBlockletExtra?.children) {
         queue.push(...currentBlockletExtra.children);
@@ -55,31 +54,37 @@ class BlockletExtrasRestore extends BaseRestore {
 
   /**
    *
-   * @description 清理数据并加密
+   * @description 解密加密的数据
    * @param {import('@abtnode/client').ConfigEntry[]} configs
    * @return {void}
    * @memberof BlockletExtrasRestore
    */
-  useBlockletDecryptConfigs(configs) {
+  decrypt(configs) {
     if (!configs || !isArray(configs)) {
       return;
     }
 
+    const { appDid, password } = this.input;
     for (const config of configs) {
-      // secure 为 true 的配置才需要被加密保存上次到 did spaces
       if (config.secure) {
-        const encryptByBlocklet = config.value;
-
-        // 再用 blocklet secret 加密，然后才可以上传到 spaces
-        const decryptByBlocklet = security.decrypt(
-          encryptByBlocklet,
-          this.blockletWallet.address,
-          Buffer.from(this.blockletWallet.secretKey)
-        );
-
-        config.value = decryptByBlocklet;
+        const encrypted = config.value;
+        const decrypted = security.decrypt(encrypted, appDid, Buffer.from(password));
+        config.value = decrypted;
       }
     }
+
+    const config = configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
+    if (!config) {
+      throw new Error(`Invalid blocklet backup file, no ${BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK} found`);
+    }
+
+    this.appSk = config.value;
+  }
+
+  getInstallParams() {
+    return {
+      appSk: this.appSk,
+    };
   }
 }
 

package/lib/blocklet/storage/restore/spaces.js

@@ -1,16 +1,24 @@
 /**
  * @typedef {{
+ *  appDid: string;
  *  endpoint: string;
- *  appSk: string;
+ *  password: Buffer;
+ *  delegation: string;
+ *  wallet: import('@ocap/wallet').WalletObject;
+ *  event: import('events').EventEmitter,
+ *  userDid: string,
+ *  referrer: string,
  * }} SpaceRestoreInput
  */
 
-const { SpaceClient, SyncFolderPullCommand } = require('@did-space/client');
-const { types } = require('@ocap/mcrypto');
-const { fromSecretKey, WalletType } = require('@ocap/wallet');
-const { ensureDirSync, existsSync, rmdirSync } = require('fs-extra');
-const { join } = require('path');
 const validUrl = require('valid-url');
+const { BlockletEvents } = require('@blocklet/constant');
+const { SpaceClient, RestoreBlockletCommand } = require('@did-space/client');
+const { ensureDirSync, existsSync, rmdirSync } = require('fs-extra');
+const { join, basename } = require('path');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:restore');
+
 const { BlockletExtrasRestore } = require('./blocklet-extras');
 const { BlockletsRestore } = require('./blocklets');
 
@@ -37,14 +45,6 @@ class SpacesRestore {
    */
   serverDataDir;
 
-  /**
-   *
-   * @description spaces 的 endpoint
-   * @type {import('@ocap/wallet').WalletObject}
-   * @memberof SpacesRestore
-   */
-  blockletWallet;
-
   storages;
 
   /**
@@ -72,68 +72,75 @@ class SpacesRestore {
 
   async initialize() {
     this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
-    this.blockletWallet = await this.getBlockletWallet();
 
-    if (!this.input.endpoint.includes(this.blockletWallet.address)) {
-      throw new Error(`endpoint and blocklet.appDid(${this.blockletWallet.address}) do not match`);
+    if (!this.input.endpoint.includes(this.input.appDid)) {
+      throw new Error(`endpoint and blocklet.appDid(${this.input.appDid}) do not match`);
     }
 
-    this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.blockletWallet.address);
+    this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.input.appDid);
     if (existsSync(this.blockletRestoreDir)) {
       rmdirSync(this.blockletRestoreDir, { recursive: true });
     }
     ensureDirSync(this.blockletRestoreDir);
   }
 
-  /**
-   *
-   * @returns {Promise<void>}
-   * @memberof SpacesRestore
-   */
   async restore() {
     await this.initialize();
     await this.syncFromSpaces();
     await this.import();
-  }
-
-  async getBlockletWallet() {
-    // @FIXME: blocklet 钱包类型如何得知呢？
-    const wallet = fromSecretKey(this.input.appSk, WalletType({ role: types.RoleType.ROLE_APPLICATION }));
 
-    return wallet;
+    return this.storages.map((x) => x.getInstallParams());
   }
 
   async syncFromSpaces() {
-    const { endpoint } = this.input;
-    const wallet = await this.getBlockletWallet();
+    const { endpoint, wallet, delegation } = this.input;
 
     const spaceClient = new SpaceClient({
       endpoint,
+      delegation,
       wallet,
     });
 
-    const { errorCount } = await spaceClient.send(
-      new SyncFolderPullCommand({
-        source: join('.did-objects', this.blockletWallet.address, '/'),
+    const { errorCount, message } = await spaceClient.send(
+      new RestoreBlockletCommand({
+        appDid: this.input.appDid,
         target: join(this.blockletRestoreDir, '/'),
         debug: true,
         concurrency: 32,
         retryCount: 10,
+        onProgress: (data) => {
+          logger.info('restore progress', { appDid: this.input.appDid, data });
+          this.input.event.emit(BlockletEvents.restoreProgress, {
+            did: this.input.appDid,
+            message: `Downloaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+          });
+        },
+
+        userDid: this.input.userDid,
+        referrer: this.input.referrer,
       })
     );
 
     if (errorCount !== 0) {
-      throw new Error(`Sync from spaces encountered ${errorCount} error`);
+      throw new Error(`Sync from spaces encountered ${errorCount} error: ${message}`);
     }
   }
 
   async import() {
+    this.input.event.emit(BlockletEvents.restoreProgress, {
+      did: this.input.appDid,
+      message: 'Preparing to import data...',
+    });
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.import();
       })
     );
+    this.input.event.emit(BlockletEvents.restoreProgress, {
+      did: this.input.appDid,
+      message: 'Importing data successfully...',
+    });
   }
 }
 

package/lib/event.js

@@ -169,6 +169,12 @@ module.exports = ({
     });
   };
 
+  /**
+   *
+   * @description 事件必须注册在这里才能被发布出去
+   * @param {string} eventName
+   * @param {any} payload
+   */
   const handleBlockletEvent = async (eventName, payload) => {
     const blocklet = payload.blocklet || payload;
 
@@ -273,6 +279,13 @@ module.exports = ({
     }
   };
 
+  /**
+   *
+   *
+   * @param {*} subject
+   * @param {string} event
+   * @param {(event: string, data: any) => Promise<void> | void} handler
+   */
   const listen = (subject, event, handler) => subject.on(event, (data) => handler(event, data));
 
   [
@@ -291,6 +304,9 @@ module.exports = ({
     BlockletEvents.startFailed,
     BlockletEvents.stopped,
     BlockletEvents.appDidChanged,
+
+    BlockletEvents.backupProgress,
+    BlockletEvents.restoreProgress,
   ].forEach((eventName) => {
     listen(blockletManager, eventName, handleBlockletEvent);
   });

package/lib/index.js

@@ -26,7 +26,7 @@ const Maintain = require('./util/maintain');
 const resetNode = require('./util/reset-node');
 const DiskMonitor = require('./util/disk-monitor');
 const StoreUtil = require('./util/store');
-const createQueue = require('./queue');
+const createQueue = require('./util/queue');
 const createEvents = require('./event');
 const pm2Events = require('./blocklet/manager/pm2-events');
 const { createStateReadyQueue, createStateReadyHandler } = require('./util/ready');
@@ -99,7 +99,25 @@ function ABTNode(options) {
       concurrency,
       maxRetries: 3,
       retryDelay: 5000, // retry after 5 seconds
-      maxTimeout: 60 * 1000 * 15, // throw timeout error after 5 minutes
+      maxTimeout: 60 * 1000 * 15, // throw timeout error after 15 minutes
+      id: (job) => (job ? md5(`${job.entity}-${job.action}-${job.id}`) : ''),
+    },
+  });
+
+  const backupQueue = createQueue({
+    daemon: options.daemon,
+    name: 'backup_queue',
+    dataDir: dataDirs.core,
+    onJob: async (job) => {
+      if (typeof blockletManager.onJob === 'function') {
+        await blockletManager.onJob(job);
+      }
+    },
+    options: {
+      concurrency,
+      maxRetries: 3,
+      retryDelay: 10000, // retry after 10 seconds
+      maxTimeout: 60 * 1000 * 30, // throw timeout error after 30 minutes
       id: (job) => (job ? md5(`${job.entity}-${job.action}-${job.id}`) : ''),
     },
   });
@@ -129,6 +147,7 @@ function ABTNode(options) {
     dataDirs,
     startQueue,
     installQueue,
+    backupQueue,
     daemon: options.daemon,
     teamManager,
   });

package/lib/states/blocklet.js

@@ -28,6 +28,7 @@ const lock = new Lock('blocklet-port-assign-lock');
 
 const isHex = (str) => /^0x[0-9a-f]+$/i.test(str);
 const getMaxPort = (ports = {}) => Math.max(Object.values(ports).map(Number));
+const getConditions = (did) => [{ 'meta.did': did }, { appDid: did }, { appPid: did }];
 
 const getExternalPortsFromMeta = (meta) =>
   (meta.interfaces || []).map((x) => x.port && x.port.external).filter(Boolean);
@@ -48,7 +49,16 @@ const formatBlocklet = (blocklet, phase, dek) => {
       return;
     }
 
-    ['BLOCKLET_APP_SK'].forEach((key) => {
+    (b.migratedFrom || []).forEach((x) => {
+      if (phase === 'onUpdate' && isHex(x.appSk) === true) {
+        x.appSk = security.encrypt(x.appSk, b.meta.did, dek);
+      }
+      if (phase === 'onRead' && isHex(x.appSk) === false) {
+        x.appSk = security.decrypt(x.appSk, b.meta.did, dek);
+      }
+    });
+
+    ['BLOCKLET_APP_SK', 'BLOCKLET_APP_PSK'].forEach((key) => {
       const env = b.environments.find((x) => x.key === key);
       if (!env) {
         return;
@@ -98,7 +108,7 @@ class BlockletState extends BaseState {
         resolve(null);
       }
 
-      this.findOne({ $or: [{ 'meta.did': did }, { appDid: did }] }, (err, doc) => {
+      this.findOne({ $or: getConditions(did) }, (err, doc) => {
         if (err) {
           return reject(err);
         }
@@ -114,7 +124,7 @@ class BlockletState extends BaseState {
         resolve(null);
       }
 
-      this.findOne({ $or: [{ 'meta.did': did }, { appDid: did }] }, (err, doc) => {
+      this.findOne({ $or: getConditions(did) }, (err, doc) => {
         if (err) {
           return reject(err);
         }
@@ -130,7 +140,7 @@ class BlockletState extends BaseState {
         resolve(null);
       }
 
-      this.findOne({ $or: [{ 'meta.did': did }, { appDid: did }] }, { status: 1 }, (err, doc) => {
+      this.findOne({ $or: getConditions(did) }, { status: 1 }, (err, doc) => {
         if (err) {
           return reject(err);
         }
@@ -146,7 +156,7 @@ class BlockletState extends BaseState {
         resolve(false);
       }
 
-      this.count({ $or: [{ 'meta.did': did }, { appDid: did }] }, (err, count) => {
+      this.count({ $or: getConditions(did) }, (err, count) => {
         if (err) {
           return reject(err);
         }
@@ -199,6 +209,9 @@ class BlockletState extends BaseState {
     deployedFrom = '',
     mode = BLOCKLET_MODES.PRODUCTION,
     children: rawChildren = [],
+    appPid = null, // the permanent appDid, which will not change after initial set
+    migratedFrom = [], // the complete migrate history
+    externalSk = false, // whether sk is managed by some party beside server, such as did-wallet
   } = {}) {
     return this.getBlocklet(meta.did).then(
       (exist) =>
@@ -229,6 +242,7 @@ class BlockletState extends BaseState {
 
             const data = {
               appDid: null, // will updated later when updating blocklet environments
+              appPid,
               mode,
               meta: sanitized,
               status,
@@ -237,6 +251,8 @@ class BlockletState extends BaseState {
               ports,
               environments: [],
               children,
+              migratedFrom,
+              externalSk,
             };
 
             // add to db

package/lib/util/blocklet.js

@@ -305,10 +305,16 @@ const getAppSystemEnvironments = (blocklet, nodeInfo) => {
     })}`;
   }
 
+  const isMigrated = Array.isArray(blocklet.migratedFrom) && blocklet.migratedFrom.length > 0;
+  const appPid = blocklet.appPid || appId;
+  const appPsk = isMigrated ? blocklet.migratedFrom[0].appSk : appSk;
+
   return {
     BLOCKLET_DID: did,
     BLOCKLET_APP_SK: appSk,
     BLOCKLET_APP_ID: appId,
+    BLOCKLET_APP_PSK: appPsk, // permanent sk even the blocklet has been migrated
+    BLOCKLET_APP_PID: appPid, // permanent did even the blocklet has been migrated
     BLOCKLET_APP_NAME: appName,
     BLOCKLET_APP_DESCRIPTION: appDescription,
     BLOCKLET_APP_URL: appUrl,
@@ -1555,11 +1561,63 @@ const createDataArchive = (dataDir, fileName) => {
   });
 };
 
+const isBlockletAppSkUsed = ({ environments, migratedFrom = [] }, appSk) => {
+  const isUsedInEnv = environments.find((e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK)?.value === appSk;
+  const isUsedInHistory = migratedFrom.some((x) => x.appSk === appSk);
+  return isUsedInEnv || isUsedInHistory;
+};
+
+const isRotatingAppSk = (newConfigs, oldConfigs, externalSk) => {
+  const newSk = newConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK === x.key);
+  if (!newSk) {
+    // If no newSk found, we are not rotating the appSk
+    return false;
+  }
+
+  const oldSk = oldConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK === x.key);
+  if (!oldSk) {
+    // If we have no oldSk, we are setting the initial appSk for external managed apps
+    // If we have no oldSk, but we are not external managed apps, we are rotating the appSk
+    return !externalSk;
+  }
+
+  // Otherwise, we must be rotating the appSk
+  // eslint-disable-next-line sonarjs/prefer-single-boolean-return
+  if (oldSk.value !== newSk.value) {
+    return true;
+  }
+
+  return false;
+};
+
+const isRotatingAppDid = (newConfigs, oldConfigs, externalSk) => {
+  if (isRotatingAppSk(newConfigs, oldConfigs, externalSk)) {
+    return true;
+  }
+
+  const newType = newConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_WALLET_TYPE === x.key);
+  const oldType = oldConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_WALLET_TYPE === x.key);
+  if (!newType) {
+    return false;
+  }
+
+  if (!oldType) {
+    return true;
+  }
+
+  // eslint-disable-next-line sonarjs/prefer-single-boolean-return
+  if (oldType !== newType) {
+    return true;
+  }
+
+  return false;
+};
+
 /**
  * this function has side effect on config.value
  * @param {{ key: string, value?: string }} config
  */
-const validateAppConfig = async (config, blockletDid, states) => {
+const validateAppConfig = async (config, states) => {
   const x = config;
 
   // sk should be force secured while other app prop should not be secured
@@ -1577,15 +1635,10 @@ const validateAppConfig = async (config, blockletDid, states) => {
         }
       }
 
-      // Ensure sk is not used by other blocklets, otherwise we may encounter appDid collision
+      // Ensure sk is not used by existing blocklets, otherwise we may encounter appDid collision
       const blocklets = await states.blocklet.getBlocklets({});
-      const others = blocklets.filter((b) => b.meta.did !== blockletDid);
-      if (
-        others.some(
-          (b) => b.environments.find((e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK).value === x.value
-        )
-      ) {
-        throw new Error('Invalid custom blocklet secret key: already used by another blocklet');
+      if (blocklets.some((b) => isBlockletAppSkUsed(b, x.value))) {
+        throw new Error('Invalid custom blocklet secret key: already used by existing blocklet');
       }
     } else {
       delete x.value;
@@ -1769,6 +1822,9 @@ module.exports = {
   getConfigFromPreferences,
   createDataArchive,
   validateAppConfig,
+  isBlockletAppSkUsed,
+  isRotatingAppSk,
+  isRotatingAppDid,
   checkDuplicateAppSk,
   checkDuplicateMountPoint,
   validateStore,

package/lib/webhook/index.js

@@ -5,7 +5,7 @@ const checkURLAccessible = require('@abtnode/util/lib/url-evaluation/check-acces
 const { EVENTS } = require('@abtnode/constant');
 const WebHookSender = require('./sender');
 const WalletSender = require('./sender/wallet');
-const createQueue = require('../queue');
+const createQueue = require('../util/queue');
 const IP = require('../util/ip');
 const states = require('../states');
 const { getBaseUrls } = require('../util');

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.8.67-beta-794a8082",
+  "version": "1.8.68-beta-500af7e5",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,33 +19,33 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/auth": "1.8.67-beta-794a8082",
-    "@abtnode/certificate-manager": "1.8.67-beta-794a8082",
-    "@abtnode/constant": "1.8.67-beta-794a8082",
-    "@abtnode/cron": "1.8.67-beta-794a8082",
-    "@abtnode/db": "1.8.67-beta-794a8082",
-    "@abtnode/logger": "1.8.67-beta-794a8082",
-    "@abtnode/queue": "1.8.67-beta-794a8082",
-    "@abtnode/rbac": "1.8.67-beta-794a8082",
-    "@abtnode/router-provider": "1.8.67-beta-794a8082",
-    "@abtnode/static-server": "1.8.67-beta-794a8082",
-    "@abtnode/timemachine": "1.8.67-beta-794a8082",
-    "@abtnode/util": "1.8.67-beta-794a8082",
-    "@arcblock/did": "1.18.54",
+    "@abtnode/auth": "1.8.68-beta-500af7e5",
+    "@abtnode/certificate-manager": "1.8.68-beta-500af7e5",
+    "@abtnode/constant": "1.8.68-beta-500af7e5",
+    "@abtnode/cron": "1.8.68-beta-500af7e5",
+    "@abtnode/db": "1.8.68-beta-500af7e5",
+    "@abtnode/logger": "1.8.68-beta-500af7e5",
+    "@abtnode/queue": "1.8.68-beta-500af7e5",
+    "@abtnode/rbac": "1.8.68-beta-500af7e5",
+    "@abtnode/router-provider": "1.8.68-beta-500af7e5",
+    "@abtnode/static-server": "1.8.68-beta-500af7e5",
+    "@abtnode/timemachine": "1.8.68-beta-500af7e5",
+    "@abtnode/util": "1.8.68-beta-500af7e5",
+    "@arcblock/did": "1.18.57",
     "@arcblock/did-motif": "^1.1.10",
-    "@arcblock/did-util": "1.18.54",
-    "@arcblock/event-hub": "1.18.54",
-    "@arcblock/jwt": "^1.18.54",
+    "@arcblock/did-util": "1.18.57",
+    "@arcblock/event-hub": "1.18.57",
+    "@arcblock/jwt": "^1.18.57",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/vc": "1.18.54",
-    "@blocklet/constant": "1.8.67-beta-794a8082",
-    "@blocklet/meta": "1.8.67-beta-794a8082",
-    "@blocklet/sdk": "1.8.67-beta-794a8082",
-    "@did-space/client": "^0.1.76",
+    "@arcblock/vc": "1.18.57",
+    "@blocklet/constant": "1.8.68-beta-500af7e5",
+    "@blocklet/meta": "1.8.68-beta-500af7e5",
+    "@blocklet/sdk": "1.8.68-beta-500af7e5",
+    "@did-space/client": "0.1.87-beta-1",
     "@fidm/x509": "^1.2.1",
-    "@ocap/mcrypto": "1.18.54",
-    "@ocap/util": "1.18.54",
-    "@ocap/wallet": "1.18.54",
+    "@ocap/mcrypto": "1.18.57",
+    "@ocap/util": "1.18.57",
+    "@ocap/wallet": "1.18.57",
     "@slack/webhook": "^5.0.4",
     "archiver": "^5.3.1",
     "axios": "^0.27.2",
@@ -89,5 +89,5 @@
     "express": "^4.18.2",
     "jest": "^27.5.1"
   },
-  "gitHead": "f4ad32bea4d80b12971fb6bef941bdbe2af6a834"
+  "gitHead": "9070621373f317a10ff0d289323bf725e30d3521"
 }