@abtnode/core 1.16.0-beta-1d6c582e → 1.16.0-beta-62b42401

package/lib/blocklet/storage/backup/base.js

@@ -1,6 +1,23 @@
+/**
+ * @typedef {{
+ *  appDid: string
+ *  event: import('events').EventEmitter,
+ * }} BaseBackupInput
+ *
+ * @typedef {{
+ *  encrypt: (v: string) => string,
+ *  decrypt: (v: string) => string,
+ * }} BaseSecurityContext
+ */
+
+const { Hasher } = require('@ocap/mcrypto');
+const { toBuffer } = require('@ocap/util');
+const getBlockletInfo = require('@blocklet/meta/lib/info');
+const security = require('@abtnode/util/lib/security');
+
 class BaseBackup {
   /**
-   * @type {import('./spaces').SpaceBackupInput}
+   * @type {BaseBackupInput}
    * @memberof BaseBackup
    */
   input;
@@ -21,8 +38,8 @@ class BaseBackup {
 
   /**
    *
-   * @description spaces 的 endpoint
-   * @type {import('./spaces').SecurityContext}
+   * @description 安全相关的上下文
+   * @type {BaseSecurityContext}
    * @memberof BaseBackup
    */
   securityContext;
@@ -41,20 +58,57 @@ class BaseBackup {
 
   /**
    *
-   *
-   * @param {import('./spaces').SpacesBackup} spacesBackup
+   * @param {BaseBackup} backup
    * @memberof BaseBackup
    */
-  ensureParams(spacesBackup) {
-    this.blocklet = spacesBackup.blocklet;
-    this.serverDir = spacesBackup.serverDir;
-    this.backupDir = spacesBackup.backupDir;
-    this.securityContext = spacesBackup.securityContext;
+  ensureParams(backup) {
+    this.blocklet = backup.blocklet;
+    this.serverDir = backup.serverDir;
+    this.backupDir = backup.backupDir;
+    this.securityContext = backup.securityContext;
   }
 
   async export() {
     throw new Error('not implemented');
   }
+
+  async _getSecurityContext(states) {
+    const blocklet = await states.blocklet.getBlocklet(this.input.appDid);
+    const nodeInfo = await states.node.read();
+
+    const { wallet } = getBlockletInfo(blocklet, nodeInfo.sk);
+
+    const { secretKey, address } = wallet; // we encrypt using latest wallet, not the permanent wallet
+    const password = toBuffer(Hasher.SHA3.hash256(Buffer.concat([secretKey, address].map(toBuffer))));
+    const encrypt = (v) => security.encrypt(v, address, password);
+    const decrypt = (v) => security.decrypt(v, address, password);
+
+    return {
+      signer: wallet,
+      delegation: '',
+      encrypt,
+      decrypt,
+    };
+  }
+
+  /**
+   *
+   * @param {BaseBackup} dataBackup
+   * @param {Array<BaseBackup>} storages
+   * @memberof BaseBackup
+   */
+  async _exportData(dataBackup, storages) {
+    // @note: dataBackup 需要先于 blockletBackup 执行，并且 blockletBackup 与其他 backup的执行可以是无序的
+    dataBackup.ensureParams(this);
+    await dataBackup.export();
+
+    await Promise.all(
+      storages.map((storage) => {
+        storage.ensureParams(this);
+        return storage.export();
+      })
+    );
+  }
 }
 
 module.exports = {

package/lib/blocklet/storage/backup/blocklet.js

@@ -66,7 +66,7 @@ class BlockletBackup extends BaseBackup {
    * @memberof BlockletExtrasBackup
    */
   encrypt(info) {
-    if (Array.isArray(info.migratedFrom)) {
+    if (Array.isArray(info?.migratedFrom)) {
       info.migratedFrom = info.migratedFrom.map((x) => {
         x.appSk = this.securityContext.encrypt(x.appSk);
         return x;

package/lib/blocklet/storage/backup/disk.js

@@ -0,0 +1,132 @@
+const fs = require('fs-extra');
+const { isValid } = require('@arcblock/did');
+const { ensureDirSync } = require('fs-extra');
+const { isEmpty } = require('lodash');
+const { join } = require('path');
+const { getAppName } = require('@blocklet/meta/lib/util');
+
+const states = require('../../../states');
+const { getBackupDirs } = require('../utils/disk');
+const { BaseBackup } = require('./base');
+const { AuditLogBackup } = require('./audit-log');
+const { BlockletBackup } = require('./blocklet');
+const { BlockletExtrasBackup } = require('./blocklet-extras');
+const { BlockletsBackup } = require('./blocklets');
+const { DataBackup } = require('./data');
+const { RoutingRuleBackup } = require('./routing-rule');
+
+class DiskBackup extends BaseBackup {
+  /**
+   *
+   * @type {import('./base').BaseBackupInput}
+   * @memberof DiskBackup
+   */
+  input;
+
+  /**
+   * @description blocklet state 对象
+   * @type {import('@abtnode/client').BlockletState}
+   * @memberof DiskBackup
+   */
+  blocklet;
+
+  /**
+   * @type {string}
+   * @memberof DiskBackup
+   */
+  backupDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof DiskBackup
+   */
+  serverDir;
+
+  /**
+   *
+   * @type {import('./base').BaseSecurityContext}
+   * @memberof DiskBackup
+   */
+  securityContext;
+
+  storages;
+
+  dataBackup;
+
+  /**
+   *
+   * @param {import('./base').BaseBackupInput} input
+   * @memberof DiskBackup
+   */
+  constructor(input) {
+    super(input);
+    this.verify(input);
+    this.input = input;
+    this.storages = [
+      new AuditLogBackup(this.input),
+      new BlockletBackup(this.input),
+      new BlockletsBackup(this.input),
+      new BlockletExtrasBackup(this.input),
+      new RoutingRuleBackup(this.input),
+    ];
+    this.dataBackup = new DataBackup(this.input);
+  }
+
+  /**
+   * @param {import('./base').BaseBackupInput} input
+   * @returns {void}
+   * @memberof DiskBackup
+   */
+  verify(input) {
+    if (isEmpty(input?.appDid) || !isValid(input?.appDid)) {
+      throw new Error(`input.appDid(${input?.appDid}) is not a valid did`);
+    }
+  }
+
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof DiskBackup
+   */
+  async backup() {
+    await this.initialize();
+    await this.addMeta();
+    await this.export();
+  }
+
+  async initialize() {
+    this.blocklet = await states.blocklet.getBlocklet(this.input.appDid);
+    if (isEmpty(this.blocklet)) {
+      throw new Error('blocklet cannot be empty');
+    }
+
+    this.serverDir = process.env.ABT_NODE_DATA_DIR;
+    const { baseBackupDir, backupDir } = getBackupDirs(this.serverDir, this.blocklet.appDid);
+    this.baseBackupDir = baseBackupDir;
+    this.backupDir = backupDir;
+    ensureDirSync(this.backupDir);
+
+    this.securityContext = await this._getSecurityContext(states);
+  }
+
+  async export() {
+    return this._exportData(this.dataBackup, this.storages);
+  }
+
+  async addMeta() {
+    const meta = {
+      appDid: this.blocklet.appDid,
+      appPid: this.blocklet.appPid,
+      name: getAppName(this.blocklet),
+      createdAt: Date.now(),
+    };
+
+    await fs.writeJSON(join(this.baseBackupDir, 'meta.json'), meta);
+  }
+}
+
+module.exports = {
+  DiskBackup,
+};

package/lib/blocklet/storage/backup/spaces.js

@@ -20,15 +20,12 @@ const { SpaceClient, BackupBlockletCommand } = require('@did-space/client');
 const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
 const { join, basename } = require('path');
-const { Hasher } = require('@ocap/mcrypto');
-const { toBuffer } = require('@ocap/util');
 const { getAppName, getAppDescription } = require('@blocklet/meta/lib/util');
-const getBlockletInfo = require('@blocklet/meta/lib/info');
-const security = require('@abtnode/util/lib/security');
 
 const logger = require('@abtnode/logger')('@abtnode/core:storage:backup');
 
 const states = require('../../../states');
+const { BaseBackup } = require('./base');
 const { AuditLogBackup } = require('./audit-log');
 const { BlockletBackup } = require('./blocklet');
 const { BlockletExtrasBackup } = require('./blocklet-extras');
@@ -36,7 +33,7 @@ const { BlockletsBackup } = require('./blocklets');
 const { DataBackup } = require('./data');
 const { RoutingRuleBackup } = require('./routing-rule');
 
-class SpacesBackup {
+class SpacesBackup extends BaseBackup {
   /**
    *
    * @type {SpaceBackupInput}
@@ -84,12 +81,15 @@ class SpacesBackup {
 
   storages;
 
+  dataBackup;
+
   /**
    *
    * @param {SpaceBackupInput} input
    * @memberof SpacesBackup
    */
   constructor(input) {
+    super(input);
     this.verify(input);
     this.input = input;
     this.storages = [
@@ -99,6 +99,7 @@ class SpacesBackup {
       new BlockletExtrasBackup(this.input),
       new RoutingRuleBackup(this.input),
     ];
+    this.dataBackup = new DataBackup(this.input);
   }
 
   /**
@@ -140,7 +141,7 @@ class SpacesBackup {
       throw new Error('spaceEndpoint cannot be empty');
     }
 
-    this.securityContext = await this.getSecurityContext();
+    this.securityContext = await this._getSecurityContext(states);
   }
 
   async export() {
@@ -151,17 +152,7 @@ class SpacesBackup {
       completed: false,
     });
 
-    // @note: dataBackup 需要先于 blockletBackup 执行，并且 blockletBackup 与其他 backup的执行可以是无序的
-    const dataBackup = new DataBackup(this.input);
-    dataBackup.ensureParams(this);
-    await dataBackup.export();
-
-    await Promise.all(
-      this.storages.map((storage) => {
-        storage.ensureParams(this);
-        return storage.export();
-      })
-    );
+    await this._exportData(this.dataBackup, this.storages);
 
     this.input.event.emit(BlockletEvents.backupProgress, {
       appDid: this.input.appDid,
@@ -220,25 +211,6 @@ class SpacesBackup {
       throw new Error(`Sync to spaces encountered error: ${message}`);
     }
   }
-
-  async getSecurityContext() {
-    const blocklet = await states.blocklet.getBlocklet(this.input.appDid);
-    const nodeInfo = await states.node.read();
-
-    const { wallet } = getBlockletInfo(blocklet, nodeInfo.sk);
-
-    const { secretKey, address } = wallet; // we encrypt using latest wallet, not the permanent wallet
-    const password = toBuffer(Hasher.SHA3.hash256(Buffer.concat([secretKey, address].map(toBuffer))));
-    const encrypt = (v) => security.encrypt(v, address, password);
-    const decrypt = (v) => security.decrypt(v, address, password);
-
-    return {
-      signer: wallet,
-      delegation: '',
-      encrypt,
-      decrypt,
-    };
-  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/base.js

@@ -1,13 +1,20 @@
+/**
+ * @typedef {{
+ *  appDid: string; // --> appDid
+ *  password: Buffer; // derived from (appSk, appDid)
+ *  event: import('events').EventEmitter,
+ * }} BaseRestoreInput
+ */
+
 class BaseRestore {
   /**
    *
-   * @type {import('./spaces').SpaceRestoreInput}
+   * @type {BaseRestoreInput}
    * @memberof BaseRestore
    */
   input;
 
   /**
-   * @description 当前 blocklet 的数据目录
    * @type {string}
    * @memberof BaseRestore
    */
@@ -28,12 +35,12 @@ class BaseRestore {
   /**
    *
    *
-   * @param {import('./spaces').SpacesRestore} spaces
+   * @param {BaseRestore} restore
    * @memberof BaseRestore
    */
-  ensureParams(spaces) {
-    this.restoreDir = spaces.restoreDir;
-    this.serverDir = spaces.serverDir;
+  ensureParams(restore) {
+    this.restoreDir = restore.restoreDir;
+    this.serverDir = restore.serverDir;
   }
 
   // eslint-disable-next-line

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -9,8 +9,7 @@ class BlockletExtrasRestore extends BaseRestore {
   filename = 'blocklet-extras.json';
 
   async import(params) {
-    const extras = this.getExtras();
-    this.cleanExtras(extras, params);
+    const extras = this.cleanExtras(this.getExtras(), params);
     removeSync(join(this.restoreDir, this.filename));
     outputJsonSync(join(this.restoreDir, this.filename), extras);
   }
@@ -32,9 +31,10 @@ class BlockletExtrasRestore extends BaseRestore {
    *
    * @description 清理数据并加密
    * @param {import('@abtnode/client').BlockletState} raw
+   * @returns {import('@abtnode/client').BlockletState}
    * @memberof BlockletExtrasRestore
    */
-  async cleanExtras(raw, params) {
+  cleanExtras(raw, params) {
     const blockletExtra = cloneDeep(raw);
 
     const queue = [blockletExtra];
@@ -48,6 +48,8 @@ class BlockletExtrasRestore extends BaseRestore {
         queue.push(...current.children);
       }
     }
+
+    return blockletExtra;
   }
 
   /**

package/lib/blocklet/storage/restore/blocklet.js

@@ -33,7 +33,7 @@ class BlockletRestore extends BaseRestore {
    */
   decrypt(blocklet, params) {
     const { password } = this.input;
-    if (Array.isArray(blocklet.migratedFrom)) {
+    if (Array.isArray(blocklet?.migratedFrom)) {
       blocklet.migratedFrom = blocklet.migratedFrom.map((x) => {
         x.appSk = security.decrypt(x.appSk, params.salt, password);
         return x;

package/lib/blocklet/storage/restore/disk.js

@@ -0,0 +1,104 @@
+const { isValid } = require('@arcblock/did');
+const { isEmpty } = require('lodash');
+const merge = require('lodash/merge');
+const { ensureDirSync, existsSync, rmSync, copy } = require('fs-extra');
+
+const { getBackupDirs } = require('../utils/disk');
+const { BaseRestore } = require('./base');
+const { BlockletExtrasRestore } = require('./blocklet-extras');
+const { BlockletRestore } = require('./blocklet');
+const { BlockletsRestore } = require('./blocklets');
+
+class DiskRestore extends BaseRestore {
+  /**
+   *
+   * @type {import('./base').BaseRestoreInput}
+   * @memberof DiskRestore
+   */
+  input;
+
+  /**
+   * @type {string}
+   * @memberof DiskRestore
+   */
+  restoreDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof DiskRestore
+   */
+  serverDir;
+
+  storages;
+
+  /**
+   *
+   * @param {import('./base').BaseRestoreInput} input
+   * @memberof DiskRestore
+   */
+  constructor(input) {
+    super(input);
+    this.verify(input);
+    this.input = input;
+    this.storages = [
+      new BlockletExtrasRestore(this.input),
+      new BlockletRestore(this.input),
+      new BlockletsRestore(this.input),
+    ];
+  }
+
+  /**
+   *
+   * @param {import('./base').BaseRestoreInput} input
+   * @returns {void}
+   * @memberof DiskRestore
+   */
+  verify(input) {
+    if (isEmpty(input?.appDid) || !isValid(input?.appDid)) {
+      throw new Error(`input.appDid(${input?.appDid}) is not a valid did`);
+    }
+  }
+
+  async initialize() {
+    this.serverDir = process.env.ABT_NODE_DATA_DIR;
+    this.restoreDir = getBackupDirs(this.serverDir, this.input.appDid).restoreDir;
+    if (existsSync(this.restoreDir)) {
+      rmSync(this.restoreDir, { recursive: true });
+    }
+    ensureDirSync(this.restoreDir);
+
+    this.storages.map((x) => x.ensureParams(this));
+  }
+
+  async restore() {
+    await this.initialize();
+    await this.syncFromBackupDir();
+
+    const params = await Promise.all(this.storages.map((x) => x.getImportParams()));
+    await this.import(merge(...params));
+
+    return this.storages.map((x) => x.getInstallParams());
+  }
+
+  async syncFromBackupDir() {
+    const { appDid } = this.input;
+
+    const { backupDir } = getBackupDirs(this.serverDir, appDid);
+
+    await copy(backupDir, this.restoreDir);
+  }
+
+  async import(params) {
+    await Promise.all(
+      this.storages.map((storage) => {
+        return storage.import(params);
+      })
+    );
+  }
+}
+
+module.exports = {
+  DiskRestore,
+};

package/lib/blocklet/storage/restore/spaces.js

@@ -13,18 +13,19 @@
 
 const validUrl = require('valid-url');
 const merge = require('lodash/merge');
-const { BlockletEvents } = require('@blocklet/constant');
+const { BlockletEvents, RESTORE_PROGRESS_STATUS } = require('@blocklet/constant');
 const { SpaceClient, RestoreBlockletCommand } = require('@did-space/client');
 const { ensureDirSync, existsSync, rmSync } = require('fs-extra');
-const { join, basename } = require('path');
+const { join } = require('path');
 
 const logger = require('@abtnode/logger')('@abtnode/core:storage:restore');
 
+const { BaseRestore } = require('./base');
 const { BlockletExtrasRestore } = require('./blocklet-extras');
 const { BlockletRestore } = require('./blocklet');
 const { BlockletsRestore } = require('./blocklets');
 
-class SpacesRestore {
+class SpacesRestore extends BaseRestore {
   /**
    *
    * @type {SpaceRestoreInput}
@@ -55,6 +56,7 @@ class SpacesRestore {
    * @memberof SpacesRestore
    */
   constructor(input) {
+    super(input);
     this.verify(input);
     this.input = input;
     this.storages = [
@@ -120,7 +122,8 @@ class SpacesRestore {
           logger.info('restore progress', { appDid: this.input.appDid, data });
           this.input.event.emit(BlockletEvents.restoreProgress, {
             appDid: this.input.appDid,
-            message: `Downloading file ${basename(data.key)} (${data.completed}/${data.total})`,
+            status: RESTORE_PROGRESS_STATUS.downloading,
+            data,
           });
         },
 
@@ -137,8 +140,9 @@ class SpacesRestore {
   async import(params) {
     this.input.event.emit(BlockletEvents.restoreProgress, {
       appDid: this.input.appDid,
-      message: 'Preparing to import data...',
+      status: RESTORE_PROGRESS_STATUS.importData,
     });
+
     await Promise.all(
       this.storages.map((storage) => {
         return storage.import(params);
@@ -146,7 +150,7 @@ class SpacesRestore {
     );
     this.input.event.emit(BlockletEvents.restoreProgress, {
       appDid: this.input.appDid,
-      message: 'Importing data successfully...',
+      status: RESTORE_PROGRESS_STATUS.importDataSuccess,
     });
   }
 }

package/lib/blocklet/storage/utils/disk.js

@@ -0,0 +1,61 @@
+const path = require('path');
+const fs = require('fs-extra');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:utils:disk');
+
+const backupDirName = '_abtnode/backup';
+const restoreDirName = 'tmp/restore-disk';
+
+const getBackupList = (dataDir) => {
+  const baseBackupDir = path.join(dataDir, backupDirName);
+  const backupList = [];
+  const appDidList = fs.readdirSync(baseBackupDir);
+
+  appDidList.forEach((appDid) => {
+    const metaFile = path.join(baseBackupDir, appDid, 'meta.json');
+    if (fs.existsSync(metaFile)) {
+      try {
+        const meta = fs.readJsonSync(metaFile);
+        if (meta.appDid === appDid) {
+          backupList.push(meta);
+        }
+      } catch (error) {
+        logger.error('read meta.json error', error);
+      }
+    }
+  });
+
+  return backupList;
+};
+
+const removeBackup = async (dataDir, appDid) => {
+  try {
+    const baseBackupDir = path.join(dataDir, backupDirName);
+    const backupDir = path.join(baseBackupDir, appDid);
+    if (fs.existsSync(backupDir)) {
+      await fs.remove(backupDir);
+      return true;
+    }
+    return false;
+  } catch (error) {
+    logger.error('remove backup error', error);
+    return false;
+  }
+};
+
+const getBackupDirs = (serverDir, appDid) => {
+  const baseBackupDir = path.join(serverDir, 'data', backupDirName, appDid);
+  const backupDir = path.join(baseBackupDir, appDid);
+  const restoreDir = path.join(serverDir, restoreDirName, appDid);
+  return {
+    baseBackupDir,
+    backupDir,
+    restoreDir,
+  };
+};
+
+module.exports = {
+  getBackupList,
+  removeBackup,
+  getBackupDirs,
+};

package/lib/event.js

@@ -257,7 +257,13 @@ module.exports = ({
     }
 
     if (
-      ![BlockletEvents.removed, BlockletEvents.dataCleaned].includes(eventName) &&
+      [
+        BlockletEvents.started,
+        BlockletEvents.startFailed,
+        BlockletEvents.stopped,
+        BlockletEvents.reloaded,
+        BlockletEvents.statusChange,
+      ].includes(eventName) &&
       blocklet.status &&
       !isBeforeInstalled(blocklet.status)
     ) {

package/lib/index.js

@@ -230,11 +230,12 @@ function ABTNode(options) {
     deleteBlockletProcess: blockletManager.deleteProcess.bind(blockletManager),
     configPublicToStore: blockletManager.configPublicToStore.bind(blockletManager),
     configNavigations: blockletManager.configNavigations.bind(blockletManager),
+    configOAuth: blockletManager.configOAuth.bind(blockletManager),
     updateWhoCanAccess: blockletManager.updateWhoCanAccess.bind(blockletManager),
     updateComponentTitle: blockletManager.updateComponentTitle.bind(blockletManager),
     updateComponentMountPoint: blockletManager.updateComponentMountPoint.bind(blockletManager),
-    backupToSpaces: blockletManager.backupToSpaces.bind(blockletManager),
-    restoreFromSpaces: blockletManager.restoreFromSpaces.bind(blockletManager),
+    backupBlocklet: blockletManager.backup.bind(blockletManager),
+    restoreBlocklet: blockletManager.restore.bind(blockletManager),
     migrateApplicationToStructV2: blockletManager.migrateApplicationToStructV2.bind(blockletManager),
 
     // For diagnose purpose
@@ -243,6 +244,7 @@ function ABTNode(options) {
     updateBlockletStatus: states.blocklet.setBlockletStatus.bind(states.blocklet),
 
     getBlocklets: blockletManager.list.bind(blockletManager),
+    getBlockletsFromBackup: blockletManager.listBackups.bind(blockletManager),
     getBlocklet: blockletManager.detail.bind(blockletManager),
     getBlockletDiff: blockletManager.diff.bind(blockletManager),
     hasBlocklet: blockletManager.hasBlocklet.bind(blockletManager),