@abtnode/core 1.8.63 → 1.8.64-beta-0b5ede51

package/lib/blocklet/storage/backup/spaces.js

@@ -0,0 +1,189 @@
+/**
+ * @typedef {{
+ *  did?: string
+ * }} SpaceBackupInput
+ */
+
+const { isValid } = require('@arcblock/did');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+const { getBlockletInfo } = require('@blocklet/meta');
+const { SpaceClient, SyncFolderPushCommand } = require('@did-space/client');
+const { ensureDirSync, existsSync, rmdirSync, removeSync } = require('fs-extra');
+const { isEmpty } = require('lodash');
+const { join } = require('path');
+const states = require('../../../states');
+const { AuditLogBackup } = require('./audit-log');
+const { BlockletBackup } = require('./blocklet');
+const { BlockletExtrasBackup } = require('./blocklet-extras');
+const { BlockletsBackup } = require('./blocklets');
+const { DataBackup } = require('./data');
+const { LogsBackup } = require('./logs');
+const { RoutingRuleBackup } = require('./routing-rule');
+
+class SpacesBackup {
+  /**
+   *
+   * @type {SpaceBackupInput}
+   * @memberof SpacesBackup
+   */
+  input;
+
+  /**
+   * @description blocklet state 对象
+   * @type {import('@abtnode/client').BlockletState}
+   * @memberof SpacesBackup
+   */
+  blocklet;
+
+  /**
+   * @description 当前 blocklet 的数据目录
+   * @type {string}
+   * @memberof SpacesBackup
+   */
+  blockletBackupDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof SpacesBackup
+   */
+  serverDataDir;
+
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {string}
+   * @memberof SpacesBackup
+   */
+  spacesEndpoint;
+
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {import('@ocap/wallet').WalletObject}
+   * @memberof SpacesBackup
+   */
+  blockletWallet;
+
+  storages;
+
+  /**
+   *
+   * @param {SpaceBackupInput} input
+   * @memberof SpacesBackup
+   */
+  constructor(input) {
+    this.verify(input);
+    this.input = input;
+    this.storages = [
+      new AuditLogBackup(this.input),
+      new LogsBackup(this.input),
+      new BlockletBackup(this.input),
+      new BlockletsBackup(this.input),
+      new BlockletExtrasBackup(this.input),
+      new RoutingRuleBackup(this.input),
+      new DataBackup(this.input),
+    ];
+  }
+
+  /**
+   * @param {SpaceBackupInput} input
+   * @returns {void}
+   * @memberof SpacesBackup
+   */
+  verify(input) {
+    if (isEmpty(input?.did) || !isValid(input?.did)) {
+      throw new Error(`input.did(${input?.did}) is not a valid did`);
+    }
+  }
+
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof SpacesBackup
+   */
+  async backup() {
+    await this.initialize();
+    await this.export();
+    await this.syncToSpaces();
+    await this.destroy();
+  }
+
+  async initialize() {
+    this.blocklet = await states.blocklet.findOne({
+      appDid: this.input.did,
+    });
+    if (isEmpty(this.blocklet)) {
+      throw new Error('blocklet cannot be empty');
+    }
+
+    this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
+
+    this.blockletBackupDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/backup', this.blocklet.meta.did);
+    if (existsSync(this.blockletBackupDir)) {
+      rmdirSync(this.blockletBackupDir, { recursive: true });
+    }
+    ensureDirSync(this.blockletBackupDir);
+
+    this.spacesEndpoint = this.blocklet.environments.find(
+      (e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SPACE_ENDPOINT
+    )?.value;
+    if (isEmpty(this.spacesEndpoint)) {
+      throw new Error('spacesEndpoint cannot be empty');
+    }
+
+    this.blockletWallet = await this.getBlockletWallet();
+  }
+
+  async export() {
+    await Promise.all(
+      this.storages.map((storage) => {
+        storage.ensureParams(this);
+        return storage.export();
+      })
+    );
+  }
+
+  async syncToSpaces() {
+    const wallet = await this.getBlockletWallet();
+    const spaceClient = new SpaceClient({
+      endpoint: this.spacesEndpoint,
+      wallet,
+    });
+
+    // FIXME: 在 Spaces 里面预览出 blocklet 的样式，需要规划一个好的数据结构
+    const { errorCount } = await spaceClient.send(
+      new SyncFolderPushCommand({
+        source: join(this.blockletBackupDir, '/'),
+        target: join('.did-objects', this.blocklet.appDid),
+        debug: true,
+        retryCount: 3,
+        filter: (object) => {
+          return object.name !== '.DS_Store';
+        },
+      })
+    );
+
+    if (errorCount !== 0) {
+      throw new Error(`Sync to spaces encountered ${errorCount} error`);
+    }
+  }
+
+  async getBlockletWallet() {
+    const blockletInfo = await states.blocklet.getBlocklet(this.input.did);
+    const nodeInfo = await states.node.read();
+    const { wallet } = getBlockletInfo(blockletInfo, nodeInfo.sk);
+    return wallet;
+  }
+
+  async destroy() {
+    if (existsSync(this.blockletBackupDir)) {
+      removeSync(this.blockletBackupDir);
+    }
+  }
+}
+
+module.exports = {
+  SpacesBackup,
+};

package/lib/blocklet/storage/restore/base.js

@@ -0,0 +1,55 @@
+class BaseRestore {
+  /**
+   *
+   * @type {import('./spaces').SpaceRestoreInput}
+   * @memberof BaseRestore
+   */
+  input;
+
+  /**
+   * @description 当前 blocklet 的数据目录
+   * @type {string}
+   * @memberof BaseRestore
+   */
+  blockletRestoreDir;
+
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {import('@ocap/wallet').WalletObject}
+   * @memberof BaseRestore
+   */
+  blockletWallet;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof BaseRestore
+   */
+  serverDataDir;
+
+  constructor(input) {
+    this.input = input;
+  }
+
+  /**
+   *
+   *
+   * @param {import('./spaces').SpacesRestore} spaces
+   * @memberof BaseRestore
+   */
+  ensureParams(spaces) {
+    this.blockletRestoreDir = spaces.blockletRestoreDir;
+    this.blockletWallet = spaces.blockletWallet;
+    this.serverDataDir = spaces.serverDataDir;
+  }
+
+  async import() {
+    throw new Error('not implemented');
+  }
+}
+
+module.exports = {
+  BaseRestore,
+};

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -0,0 +1,84 @@
+const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
+const { cloneDeep } = require('lodash');
+const { join } = require('path');
+const security = require('@abtnode/util/lib/security');
+const { BaseRestore } = require('./base');
+
+class BlockletExtrasRestore extends BaseRestore {
+  filename = 'blocklet-extras.json';
+
+  async import() {
+    const blockletExtra = await this.getBlockletExtra();
+    removeSync(join(this.blockletRestoreDir, this.filename));
+    outputJsonSync(join(this.blockletRestoreDir, this.filename), blockletExtra);
+  }
+
+  /**
+   *
+   * @description
+   * @return {Promise<import('@abtnode/client').BlockletState>}
+   * @memberof BlockletExtrasRestore
+   */
+  async getBlockletExtra() {
+    /**
+     * @type {import('@abtnode/client').BlockletState}
+     */
+    const blockletExtra = readJSONSync(join(this.blockletRestoreDir, this.filename));
+
+    return this.cleanData(blockletExtra);
+  }
+
+  /**
+   *
+   * @description 清理数据并加密
+   * @param {import('@abtnode/client').BlockletState} blockletExtraInput
+   * @return {Promise<void>}
+   * @memberof BlockletExtrasRestore
+   */
+  async cleanData(blockletExtraInput) {
+    const blockletExtra = cloneDeep(blockletExtraInput);
+
+    const queue = [blockletExtra];
+    while (queue.length) {
+      const currentBlockletExtra = queue.pop();
+
+      // 加解密
+      this.useBlockletDecryptConfigs(currentBlockletExtra.configs);
+
+      if (currentBlockletExtra?.children) {
+        queue.push(...currentBlockletExtra.children);
+      }
+    }
+
+    return blockletExtra;
+  }
+
+  /**
+   *
+   * @description 清理数据并加密
+   * @param {import('@abtnode/client').ConfigEntry[]} configs
+   * @return {void}
+   * @memberof BlockletExtrasRestore
+   */
+  useBlockletDecryptConfigs(configs) {
+    for (const config of configs) {
+      // secure 为 true 的配置才需要被加密保存上次到 did spaces
+      if (config.secure) {
+        const encryptByBlocklet = config.value;
+
+        // 再用 blocklet secret 加密，然后才可以上传到 spaces
+        const decryptByBlocklet = security.decrypt(
+          encryptByBlocklet,
+          this.blockletWallet.address,
+          Buffer.from(this.blockletWallet.secretKey)
+        );
+
+        config.value = decryptByBlocklet;
+      }
+    }
+
+    return configs;
+  }
+}
+
+module.exports = { BlockletExtrasRestore };

package/lib/blocklet/storage/restore/blocklets.js

@@ -0,0 +1,24 @@
+const { existsSync, removeSync } = require('fs-extra');
+const { join } = require('path');
+const StreamZip = require('node-stream-zip');
+const { BaseRestore } = require('./base');
+
+class BlockletsRestore extends BaseRestore {
+  filename = 'blocklets.zip';
+
+  async import() {
+    const blockletZipPath = join(this.blockletRestoreDir, this.filename);
+
+    if (!existsSync(blockletZipPath)) {
+      throw new Error(`file not found: ${blockletZipPath}`);
+    }
+
+    // eslint-disable-next-line new-cap
+    const zip = new StreamZip.async({ file: blockletZipPath });
+    await zip.extract(null, join(this.blockletRestoreDir, 'blocklets'));
+    await zip.close();
+    removeSync(blockletZipPath);
+  }
+}
+
+module.exports = { BlockletsRestore };

package/lib/blocklet/storage/restore/spaces.js

@@ -0,0 +1,137 @@
+/**
+ * @typedef {{
+ *  endpoint: string;
+ *  blockletSecretKey: string;
+ * }} SpaceRestoreInput
+ */
+
+const { SpaceClient, SyncFolderPullCommand } = require('@did-space/client');
+const { types } = require('@ocap/mcrypto');
+const { fromSecretKey, WalletType } = require('@ocap/wallet');
+const { ensureDirSync, existsSync, rmdirSync } = require('fs-extra');
+const { join } = require('path');
+const validUrl = require('valid-url');
+const { BlockletExtrasRestore } = require('./blocklet-extras');
+const { BlockletsRestore } = require('./blocklets');
+
+class SpacesRestore {
+  /**
+   *
+   * @type {SpaceRestoreInput}
+   * @memberof SpacesRestore
+   */
+  input;
+
+  /**
+   * @description 当前 blocklet 的数据目录
+   * @type {string}
+   * @memberof SpacesRestore
+   */
+  blockletRestoreDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof SpacesRestore
+   */
+  serverDataDir;
+
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {import('@ocap/wallet').WalletObject}
+   * @memberof SpacesRestore
+   */
+  blockletWallet;
+
+  storages;
+
+  /**
+   *
+   * @param {SpaceRestoreInput} input
+   * @memberof SpacesRestore
+   */
+  constructor(input) {
+    this.verify(input);
+    this.input = input;
+    this.storages = [new BlockletExtrasRestore(this.input), new BlockletsRestore(this.input)];
+  }
+
+  /**
+   *
+   * @param {SpaceRestoreInput} input
+   * @returns {void}
+   * @memberof SpacesRestore
+   */
+  verify(input) {
+    if (!validUrl.isWebUri(input.endpoint)) {
+      throw new Error(`endpoint(${input.endpoint}) must be a WebUri`);
+    }
+  }
+
+  async initialize() {
+    this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
+    this.blockletWallet = await this.getBlockletWallet();
+
+    this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.blockletWallet.address);
+    if (existsSync(this.blockletRestoreDir)) {
+      rmdirSync(this.blockletRestoreDir, { recursive: true });
+    }
+    ensureDirSync(this.blockletRestoreDir);
+  }
+
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof SpacesRestore
+   */
+  async restore() {
+    await this.initialize();
+    await this.syncFromSpaces();
+    await this.import();
+  }
+
+  async getBlockletWallet() {
+    // @FIXME: blocklet 钱包类型如何得知呢？
+    const wallet = fromSecretKey(this.input.blockletSecretKey, WalletType({ role: types.RoleType.ROLE_APPLICATION }));
+
+    return wallet;
+  }
+
+  async syncFromSpaces() {
+    const { endpoint } = this.input;
+    const wallet = await this.getBlockletWallet();
+
+    const spaceClient = new SpaceClient({
+      endpoint,
+      wallet,
+    });
+
+    const { errorCount } = await spaceClient.send(
+      new SyncFolderPullCommand({
+        source: join('.did-objects', this.blockletWallet.address, '/'),
+        target: this.blockletRestoreDir,
+        debug: true,
+        retryCount: 3,
+      })
+    );
+
+    if (errorCount !== 0) {
+      throw new Error(`Sync from spaces encountered ${errorCount} error`);
+    }
+  }
+
+  async import() {
+    await Promise.all(
+      this.storages.map((storage) => {
+        storage.ensureParams(this);
+        return storage.import();
+      })
+    );
+  }
+}
+
+module.exports = {
+  SpacesRestore,
+};

package/lib/event.js

@@ -238,7 +238,11 @@ module.exports = ({
       logger.info('take snapshot after updated blocklet app', { event: eventName, did: blocklet.meta.did, hash });
     }
 
-    if (blocklet.status && !isBeforeInstalled(blocklet.status)) {
+    if (
+      ![BlockletEvents.removed, BlockletEvents.dataCleaned].includes(eventName) &&
+      blocklet.status &&
+      !isBeforeInstalled(blocklet.status)
+    ) {
       try {
         await blockletManager.runtimeMonitor.monit(blocklet.meta.did);
       } catch (error) {

package/lib/index.js

@@ -216,6 +216,8 @@ function ABTNode(options) {
     updateWhoCanAccess: blockletManager.updateWhoCanAccess.bind(blockletManager),
     updateComponentTitle: blockletManager.updateComponentTitle.bind(blockletManager),
     updateComponentMountPoint: blockletManager.updateComponentMountPoint.bind(blockletManager),
+    backupToSpaces: blockletManager.backupToSpaces.bind(blockletManager),
+    restoreFromSpaces: blockletManager.restoreFromSpaces.bind(blockletManager),
 
     // For diagnose purpose
     syncBlockletStatus: blockletManager.status.bind(blockletManager),
@@ -259,8 +261,9 @@ function ABTNode(options) {
     createTransferInvitation: teamAPI.createTransferInvitation.bind(teamAPI),
     getInvitation: teamAPI.getInvitation.bind(teamAPI),
     getInvitations: teamAPI.getInvitations.bind(teamAPI),
-    processInvitation: teamAPI.processInvitation.bind(teamAPI),
+    checkInvitation: teamAPI.checkInvitation.bind(teamAPI),
     deleteInvitation: teamAPI.deleteInvitation.bind(teamAPI),
+    closeInvitation: teamAPI.closeInvitation.bind(teamAPI),
 
     // Account
     getUsers: teamAPI.getUsers.bind(teamAPI),

package/lib/router/helper.js

@@ -14,6 +14,7 @@ const getTmpDir = require('@abtnode/util/lib/get-tmp-directory');
 const downloadFile = require('@abtnode/util/lib/download-file');
 const { updateBlockletDocument } = require('@abtnode/util/lib/did-document');
 const getBlockletInfo = require('@blocklet/meta/lib/info');
+const { forEachBlocklet } = require('@blocklet/meta/lib/util');
 const {
   DOMAIN_FOR_DEFAULT_SITE,
   DOMAIN_FOR_IP_SITE_REGEXP,
@@ -768,47 +769,77 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       return false;
     }
 
-    const tasks = (blocklet.meta.interfaces || [])
-      .filter((x) => x.type === BLOCKLET_INTERFACE_TYPE_WELLKNOWN)
-      .map(async (x) => {
-        const pathPrefix = normalizePathPrefix(x.prefix);
-        if (!pathPrefix.startsWith(WELLKNOWN_PATH_PREFIX)) {
-          throw new Error(`Wellknown path prefix must start with: ${WELLKNOWN_PATH_PREFIX}`);
-        }
+    /**
+     * 1. component blocklet 不允许有相同多的 wellknown
+     * 1. wellknown 可以访问的路由:
+     *    /.well-known/xxx
+     *    /{wellknown owner blocklet}/.well-known/xxx
+     */
 
-        const port = findInterfacePortByName(blocklet, x.name);
-        const existedRule = hasRuleByPrefix(wellknownSite, pathPrefix);
+    const isWellknownInterface = (x) => x.type === BLOCKLET_INTERFACE_TYPE_WELLKNOWN;
 
-        const rule = {
-          from: { pathPrefix },
-          to: {
-            did: blocklet.meta.did,
-            port,
-            type: ROUTING_RULE_TYPES.GENERAL_PROXY,
-            interfaceName: x.name,
-          },
-          isProtected: true,
-        };
+    const handler = async (tmpBlocklet, tmpInterface, mountPoint) => {
+      let pathPrefix = normalizePathPrefix(tmpInterface.prefix);
+      if (!pathPrefix.startsWith(WELLKNOWN_PATH_PREFIX)) {
+        throw new Error(`Wellknown path prefix must start with: ${WELLKNOWN_PATH_PREFIX}`);
+      }
 
-        if (!existedRule) {
-          await routerManager.addRoutingRule({ id: wellknownSite.id, rule, skipCheckDynamicBlacklist: true }, context);
-          return true;
-        }
+      const tmpMountPoint = mountPoint || tmpBlocklet.mountPoint;
+      if (tmpMountPoint) {
+        pathPrefix = joinUrl(tmpMountPoint, pathPrefix);
+      }
 
-        // 兼容代码，旧的 rule 没有 to.did 字段
-        if (port !== existedRule.to.port || existedRule.to.did !== blocklet.meta.did) {
-          existedRule.to = rule.to;
+      const port = findInterfacePortByName(tmpBlocklet, tmpInterface.name);
+      const existedRule = hasRuleByPrefix(wellknownSite, pathPrefix);
 
-          await routerManager.updateRoutingRule(
-            { id: wellknownSite.id, rule: existedRule, skipProtectedRuleChecking: true },
-            context
-          );
+      const rule = {
+        from: { pathPrefix },
+        to: {
+          did: tmpBlocklet.meta.did,
+          port,
+          type: ROUTING_RULE_TYPES.GENERAL_PROXY,
+          interfaceName: tmpInterface.name,
+        },
+        isProtected: true,
+      };
 
-          return true;
-        }
+      if (!existedRule) {
+        await routerManager.addRoutingRule({ id: wellknownSite.id, rule, skipCheckDynamicBlacklist: true }, context);
+        return true;
+      }
 
-        return false;
-      });
+      // 兼容代码，旧的 rule 没有 to.did 字段
+      if (port !== existedRule.to.port || existedRule.to.did !== tmpBlocklet.meta.did) {
+        existedRule.to = rule.to;
+
+        await routerManager.updateRoutingRule(
+          { id: wellknownSite.id, rule: existedRule, skipProtectedRuleChecking: true },
+          context
+        );
+
+        return true;
+      }
+
+      return false;
+    };
+
+    const tasks = [];
+
+    forEachBlocklet(
+      blocklet,
+      (b) => {
+        (b.meta.interfaces || []).forEach((item) => {
+          if (isWellknownInterface(item)) {
+            tasks.push(handler(b, item));
+
+            if (!b.mountPoint || b.mountPoint !== '/') {
+              tasks.push(handler(b, item, '/')); // 在站点的根路由下挂载一个
+            }
+          }
+        });
+      },
+      { sync: true }
+    );
 
     const changes = await Promise.all(tasks);
     return changes.some(Boolean);

package/lib/states/blocklet-extras.js

@@ -199,6 +199,10 @@ class BlockletExtrasState extends BaseState {
     return super.update({ did }, { $set: entity }, { upsert: true });
   }
 
+  async getMeta(did) {
+    return super.findOne({ did }, { did: 1, controller: 1, meta: 1 });
+  }
+
   async updateExpireInfo({ did, expiredAt } = {}) {
     const entity = { did, expiredAt };
     await validateExpiredInfo(entity);