@abtnode/core 1.15.17 → 1.16.0-beta-8ee536d7

package/lib/blocklet/storage/backup/spaces.js

@@ -0,0 +1,240 @@
+/**
+ * @typedef {{
+ *  appDid: string
+ *  event: import('events').EventEmitter,
+ *  userDid: string,
+ *  referrer: string,
+ * }} SpaceBackupInput
+ *
+ * @typedef {{
+ *  signer: import('@ocap/wallet').WalletObject
+ *  encrypt: (v: string) => string,
+ *  decrypt: (v: string) => string,
+ *  delegation: string,
+ * }} SecurityContext
+ */
+
+const { isValid } = require('@arcblock/did');
+const { BLOCKLET_CONFIGURABLE_KEY, BlockletEvents } = require('@blocklet/constant');
+const { SpaceClient, BackupBlockletCommand } = require('@did-space/client');
+const { ensureDirSync } = require('fs-extra');
+const { isEmpty } = require('lodash');
+const { join, basename } = require('path');
+const { Hasher } = require('@ocap/mcrypto');
+const { toBuffer } = require('@ocap/util');
+const { getAppName, getAppDescription } = require('@blocklet/meta/lib/util');
+const getBlockletInfo = require('@blocklet/meta/lib/info');
+const security = require('@abtnode/util/lib/security');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:backup');
+
+const states = require('../../../states');
+const { AuditLogBackup } = require('./audit-log');
+const { BlockletBackup } = require('./blocklet');
+const { BlockletExtrasBackup } = require('./blocklet-extras');
+const { BlockletsBackup } = require('./blocklets');
+const { DataBackup } = require('./data');
+const { RoutingRuleBackup } = require('./routing-rule');
+
+class SpacesBackup {
+  /**
+   *
+   * @type {SpaceBackupInput}
+   * @memberof SpacesBackup
+   */
+  input;
+
+  /**
+   * @description blocklet state 对象
+   * @type {import('@abtnode/client').BlockletState}
+   * @memberof SpacesBackup
+   */
+  blocklet;
+
+  /**
+   * @description 当前 blocklet 的数据目录
+   * @type {string}
+   * @memberof SpacesBackup
+   */
+  backupDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof SpacesBackup
+   */
+  serverDir;
+
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {string}
+   * @memberof SpacesBackup
+   */
+  spaceEndpoint;
+
+  /**
+   *
+   * @description spaces 安全相关的上下文
+   * @type SecurityContext
+   * @memberof SpacesBackup
+   */
+  securityContext;
+
+  storages;
+
+  /**
+   *
+   * @param {SpaceBackupInput} input
+   * @memberof SpacesBackup
+   */
+  constructor(input) {
+    this.verify(input);
+    this.input = input;
+    this.storages = [
+      new AuditLogBackup(this.input),
+      new BlockletBackup(this.input),
+      new BlockletsBackup(this.input),
+      new BlockletExtrasBackup(this.input),
+      new RoutingRuleBackup(this.input),
+      new DataBackup(this.input),
+    ];
+  }
+
+  /**
+   * @param {SpaceBackupInput} input
+   * @returns {void}
+   * @memberof SpacesBackup
+   */
+  verify(input) {
+    if (isEmpty(input?.appDid) || !isValid(input?.appDid)) {
+      throw new Error(`input.appDid(${input?.appDid}) is not a valid did`);
+    }
+  }
+
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof SpacesBackup
+   */
+  async backup() {
+    await this.initialize();
+    await this.export();
+    await this.syncToSpaces();
+  }
+
+  async initialize() {
+    this.blocklet = await states.blocklet.getBlocklet(this.input.appDid);
+    if (isEmpty(this.blocklet)) {
+      throw new Error('blocklet cannot be empty');
+    }
+
+    this.serverDir = process.env.ABT_NODE_DATA_DIR;
+    this.backupDir = join(this.serverDir, 'tmp/backup', this.blocklet.appDid);
+    ensureDirSync(this.backupDir);
+
+    this.spaceEndpoint = this.blocklet.environments.find(
+      (e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SPACE_ENDPOINT
+    )?.value;
+    if (isEmpty(this.spaceEndpoint)) {
+      throw new Error('spaceEndpoint cannot be empty');
+    }
+
+    this.securityContext = await this.getSecurityContext();
+  }
+
+  async export() {
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      appDid: this.input.appDid,
+      message: 'Preparing data for backup...',
+      progress: 15,
+      completed: false,
+    });
+    await Promise.all(
+      this.storages.map((storage) => {
+        storage.ensureParams(this);
+        return storage.export();
+      })
+    );
+    this.input.event.emit(BlockletEvents.backupProgress, {
+      appDid: this.input.appDid,
+      message: 'Data ready, start backup...',
+      progress: 20,
+      completed: false,
+    });
+  }
+
+  async syncToSpaces() {
+    /**
+     * @type {import('@abtnode/client').NodeState}
+     */
+    const node = await states.node.read();
+    const serverDid = node.did;
+
+    const spaceClient = new SpaceClient({
+      endpoint: this.spaceEndpoint,
+      wallet: this.securityContext.signer,
+      delegation: this.securityContext.delegation,
+    });
+
+    const { errorCount, message } = await spaceClient.send(
+      new BackupBlockletCommand({
+        appDid: this.blocklet.appDid,
+        appName: getAppName(this.blocklet),
+        appDescription: getAppDescription(this.blocklet),
+        userDid: this.input.userDid,
+        referrer: this.input.referrer,
+        serverDid,
+        signerDid: this.securityContext.signer.address,
+
+        source: join(this.backupDir, '/'),
+        debug: true,
+        concurrency: 32,
+        retryCount: 10,
+        filter: (object) => {
+          // FIXME: @yejianchao 这里需要更完整的黑名单
+          return object.name !== '.DS_Store';
+        },
+        onProgress: (data) => {
+          logger.info('backup progress', { appDid: this.input.appDid, data });
+          const percent = (data.completed * 100) / data.total;
+          this.input.event.emit(BlockletEvents.backupProgress, {
+            appDid: this.input.appDid,
+            message: `Uploaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+            // 0.8 是因为上传文件到 spaces 占进度的 80%，+ 20 是因为需要累加之前的进度
+            progress: +Math.ceil(percent * 0.8).toFixed(2) + 20,
+            completed: false,
+          });
+        },
+      })
+    );
+
+    if (errorCount !== 0) {
+      throw new Error(`Sync to spaces encountered error: ${message}`);
+    }
+  }
+
+  async getSecurityContext() {
+    const blocklet = await states.blocklet.getBlocklet(this.input.appDid);
+    const nodeInfo = await states.node.read();
+
+    const { wallet } = getBlockletInfo(blocklet, nodeInfo.sk);
+
+    const { secretKey, address } = wallet; // we encrypt using latest wallet, not the permanent wallet
+    const password = toBuffer(Hasher.SHA3.hash256(Buffer.concat([secretKey, address].map(toBuffer))));
+    const encrypt = (v) => security.encrypt(v, address, password);
+    const decrypt = (v) => security.decrypt(v, address, password);
+
+    return {
+      signer: wallet,
+      delegation: '',
+      encrypt,
+      decrypt,
+    };
+  }
+}
+
+module.exports = {
+  SpacesBackup,
+};

package/lib/blocklet/storage/restore/base.js

@@ -0,0 +1,67 @@
+class BaseRestore {
+  /**
+   *
+   * @type {import('./spaces').SpaceRestoreInput}
+   * @memberof BaseRestore
+   */
+  input;
+
+  /**
+   * @description 当前 blocklet 的数据目录
+   * @type {string}
+   * @memberof BaseRestore
+   */
+  restoreDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof BaseRestore
+   */
+  serverDir;
+
+  constructor(input) {
+    this.input = input;
+  }
+
+  /**
+   *
+   *
+   * @param {import('./spaces').SpacesRestore} spaces
+   * @memberof BaseRestore
+   */
+  ensureParams(spaces) {
+    this.restoreDir = spaces.restoreDir;
+    this.serverDir = spaces.serverDir;
+  }
+
+  // eslint-disable-next-line
+  async import(params) {
+    throw new Error('not implemented');
+  }
+
+  /**
+   * Generate params for BlockletManager to install
+   *
+   * @return {object}
+   * @memberof BaseRestore
+   */
+  getInstallParams() {
+    return {};
+  }
+
+  /**
+   * Generate params for other restorers
+   *
+   * @return {object}
+   * @memberof BaseRestore
+   */
+  getImportParams() {
+    return {};
+  }
+}
+
+module.exports = {
+  BaseRestore,
+};

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -0,0 +1,86 @@
+const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
+const { cloneDeep, isEmpty } = require('lodash');
+const { join } = require('path');
+const security = require('@abtnode/util/lib/security');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+const { BaseRestore } = require('./base');
+
+class BlockletExtrasRestore extends BaseRestore {
+  filename = 'blocklet-extras.json';
+
+  async import(params) {
+    const extras = this.getExtras();
+    this.cleanExtras(extras, params);
+    removeSync(join(this.restoreDir, this.filename));
+    outputJsonSync(join(this.restoreDir, this.filename), extras);
+  }
+
+  /**
+   *
+   * @description
+   * @return {import('@abtnode/client').BlockletState}
+   * @memberof BlockletExtrasRestore
+   */
+  getExtras() {
+    /**
+     * @type {import('@abtnode/client').BlockletState}
+     */
+    return readJSONSync(join(this.restoreDir, this.filename));
+  }
+
+  /**
+   *
+   * @description 清理数据并加密
+   * @param {import('@abtnode/client').BlockletState} raw
+   * @memberof BlockletExtrasRestore
+   */
+  async cleanExtras(raw, params) {
+    const blockletExtra = cloneDeep(raw);
+
+    const queue = [blockletExtra];
+    while (queue.length) {
+      const current = queue.pop();
+
+      // 加解密
+      this.decryptConfigs(current.configs, params);
+
+      if (current?.children) {
+        queue.push(...current.children);
+      }
+    }
+  }
+
+  /**
+   *
+   * @description 解密加密的数据
+   * @param {import('@abtnode/client').ConfigEntry[]} configs
+   * @param {object} params
+   * @return {void}
+   * @memberof BlockletExtrasRestore
+   */
+  decryptConfigs(configs, params) {
+    if (isEmpty(configs)) {
+      return;
+    }
+
+    const { password } = this.input;
+    for (const config of configs) {
+      if (config.secure) {
+        config.value = security.decrypt(config.value, params.salt, password);
+      }
+    }
+
+    const skConfig = configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
+    if (skConfig) {
+      this.appSk = skConfig.value;
+    }
+  }
+
+  getInstallParams() {
+    return {
+      appSk: this.appSk,
+    };
+  }
+}
+
+module.exports = { BlockletExtrasRestore };

package/lib/blocklet/storage/restore/blocklet.js

@@ -0,0 +1,56 @@
+const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
+const { join } = require('path');
+const { APP_STRUCT_VERSION } = require('@abtnode/constant');
+const security = require('@abtnode/util/lib/security');
+
+const { BaseRestore } = require('./base');
+
+class BlockletRestore extends BaseRestore {
+  filename = 'blocklet.json';
+
+  async import(params) {
+    const blocklet = this.decrypt(this.getBlocklet(), params);
+    removeSync(join(this.restoreDir, this.filename));
+    outputJsonSync(join(this.restoreDir, this.filename), blocklet);
+  }
+
+  /**
+   *
+   * @description
+   * @return {import('@abtnode/client').BlockletState}
+   * @memberof BlockletRestore
+   */
+  getBlocklet() {
+    return readJSONSync(join(this.restoreDir, this.filename));
+  }
+
+  /**
+   *
+   * @description 解密加密的数据
+   * @param {import('@abtnode/client').BlockletState} blocklet
+   * @return {import('@abtnode/client').BlockletState}
+   * @memberof BlockletRestore
+   */
+  decrypt(blocklet, params) {
+    const { password } = this.input;
+    if (Array.isArray(blocklet.migratedFrom)) {
+      blocklet.migratedFrom = blocklet.migratedFrom.map((x) => {
+        x.appSk = security.decrypt(x.appSk, params.salt, password);
+        return x;
+      });
+    }
+
+    return blocklet;
+  }
+
+  getImportParams() {
+    const blocklet = this.getBlocklet();
+    if (blocklet.structVersion !== APP_STRUCT_VERSION) {
+      throw new Error('Only new version backup can be restored to this server');
+    }
+
+    return { salt: blocklet.appDid };
+  }
+}
+
+module.exports = { BlockletRestore };

package/lib/blocklet/storage/restore/blocklets.js

@@ -0,0 +1,43 @@
+const { existsSync, remove, ensureDirSync } = require('fs-extra');
+const { join } = require('path');
+const fg = require('fast-glob');
+const { BaseRestore } = require('./base');
+const { zipToDir } = require('../utils/zip');
+
+class BlockletsRestore extends BaseRestore {
+  filename = 'blocklets';
+
+  async import() {
+    const blockletsDir = join(this.restoreDir, this.filename);
+
+    if (!existsSync(blockletsDir)) {
+      ensureDirSync(blockletsDir);
+      return;
+    }
+
+    const paths = await fg('**/*.zip', {
+      cwd: blockletsDir,
+      onlyFiles: true,
+      absolute: true,
+    });
+
+    const configs = paths.map((path) => {
+      return {
+        source: path,
+        target: path.replace(/.zip$/, ''),
+      };
+    });
+
+    await Promise.all(
+      configs.map(async ({ source, target }) => {
+        if (!existsSync(target)) {
+          ensureDirSync(target);
+        }
+        await zipToDir(source, target);
+        await remove(source);
+      })
+    );
+  }
+}
+
+module.exports = { BlockletsRestore };

package/lib/blocklet/storage/restore/logs.js

@@ -0,0 +1,21 @@
+const { removeSync, existsSync } = require('fs-extra');
+const { join } = require('path');
+const { BaseRestore } = require('./base');
+const { zipToDir } = require('../utils/zip');
+
+class LogsRestore extends BaseRestore {
+  filename = 'logs.zip';
+
+  async import() {
+    const blockletZipPath = join(this.restoreDir, this.filename);
+
+    if (!existsSync(blockletZipPath)) {
+      throw new Error(`file not found: ${blockletZipPath}`);
+    }
+
+    await zipToDir(blockletZipPath, join(this.restoreDir, 'logs'));
+    removeSync(blockletZipPath);
+  }
+}
+
+module.exports = { LogsRestore };

package/lib/blocklet/storage/restore/spaces.js

@@ -0,0 +1,156 @@
+/**
+ * @typedef {{
+ *  appDid: string; // --> appDid
+ *  endpoint: string;
+ *  password: Buffer; // derived from (appSk, appDid)
+ *  delegation: string; // from appDid --> serverDid for downloading
+ *  wallet: import('@ocap/wallet').WalletObject; // server wallet
+ *  event: import('events').EventEmitter,
+ *  userDid: string,
+ *  referrer: string,
+ * }} SpaceRestoreInput
+ */
+
+const validUrl = require('valid-url');
+const merge = require('lodash/merge');
+const { BlockletEvents } = require('@blocklet/constant');
+const { SpaceClient, RestoreBlockletCommand } = require('@did-space/client');
+const { ensureDirSync, existsSync, rmSync } = require('fs-extra');
+const { join, basename } = require('path');
+
+const logger = require('@abtnode/logger')('@abtnode/core:storage:restore');
+
+const { BlockletExtrasRestore } = require('./blocklet-extras');
+const { BlockletRestore } = require('./blocklet');
+const { BlockletsRestore } = require('./blocklets');
+
+class SpacesRestore {
+  /**
+   *
+   * @type {SpaceRestoreInput}
+   * @memberof SpacesRestore
+   */
+  input;
+
+  /**
+   * @description 当前 blocklet 的数据目录
+   * @type {string}
+   * @memberof SpacesRestore
+   */
+  restoreDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof SpacesRestore
+   */
+  serverDir;
+
+  storages;
+
+  /**
+   *
+   * @param {SpaceRestoreInput} input
+   * @memberof SpacesRestore
+   */
+  constructor(input) {
+    this.verify(input);
+    this.input = input;
+    this.storages = [
+      new BlockletExtrasRestore(this.input),
+      new BlockletRestore(this.input),
+      new BlockletsRestore(this.input),
+    ];
+  }
+
+  /**
+   *
+   * @param {SpaceRestoreInput} input
+   * @returns {void}
+   * @memberof SpacesRestore
+   */
+  verify(input) {
+    if (!validUrl.isWebUri(input.endpoint)) {
+      throw new Error(`endpoint(${input.endpoint}) must be a WebUri`);
+    }
+    if (!input.endpoint.includes(input.appDid)) {
+      throw new Error(`endpoint and blocklet.appDid(${input.appDid}) do not match`);
+    }
+  }
+
+  async initialize() {
+    this.serverDir = process.env.ABT_NODE_DATA_DIR;
+    this.restoreDir = join(this.serverDir, 'tmp/restore', this.input.appDid);
+    if (existsSync(this.restoreDir)) {
+      rmSync(this.restoreDir, { recursive: true });
+    }
+    ensureDirSync(this.restoreDir);
+
+    this.storages.map((x) => x.ensureParams(this));
+  }
+
+  async restore() {
+    await this.initialize();
+    await this.syncFromSpaces();
+
+    const params = await Promise.all(this.storages.map((x) => x.getImportParams()));
+    await this.import(merge(...params));
+
+    return this.storages.map((x) => x.getInstallParams());
+  }
+
+  async syncFromSpaces() {
+    const { endpoint, wallet, delegation } = this.input;
+
+    const spaceClient = new SpaceClient({
+      endpoint,
+      delegation,
+      wallet,
+    });
+
+    const { errorCount, message } = await spaceClient.send(
+      new RestoreBlockletCommand({
+        appDid: this.input.appDid,
+        target: join(this.restoreDir, '/'),
+        debug: true,
+        concurrency: 32,
+        retryCount: 10,
+        onProgress: (data) => {
+          logger.info('restore progress', { appDid: this.input.appDid, data });
+          this.input.event.emit(BlockletEvents.restoreProgress, {
+            appDid: this.input.appDid,
+            message: `Downloaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+          });
+        },
+
+        userDid: this.input.userDid,
+        referrer: this.input.referrer,
+      })
+    );
+
+    if (errorCount !== 0) {
+      throw new Error(`Sync from spaces encountered error: ${message}`);
+    }
+  }
+
+  async import(params) {
+    this.input.event.emit(BlockletEvents.restoreProgress, {
+      appDid: this.input.appDid,
+      message: 'Preparing to import data...',
+    });
+    await Promise.all(
+      this.storages.map((storage) => {
+        return storage.import(params);
+      })
+    );
+    this.input.event.emit(BlockletEvents.restoreProgress, {
+      appDid: this.input.appDid,
+      message: 'Importing data successfully...',
+    });
+  }
+}
+
+module.exports = {
+  SpacesRestore,
+};