@abtnode/core 1.16.0-beta-8ee536d7 → 1.16.0-beta-62b42401

package/lib/blocklet/manager/helper/migrate-application-to-struct-v2.js

@@ -6,7 +6,7 @@ const Client = require('@ocap/client');
 
 const logger = require('@abtnode/logger')('@abtnode/core:migrate-application-to-struct-v2');
 
-const { forEachBlockletSync, getSharedConfigObj, getChainInfo } = require('@blocklet/meta/lib/util');
+const { forEachBlockletSync, forEachChildSync, getSharedConfigObj, getChainInfo } = require('@blocklet/meta/lib/util');
 const { SLOT_FOR_IP_DNS_SITE } = require('@abtnode/constant');
 
 const {
@@ -91,6 +91,31 @@ const fillBlockletData = (data, app, id) => {
 
 const appSystemFiles = ['logo.svg', 'rbac.db', 'session.db', 'user.db', '.assets', BLOCKLET_UPLOADS_DIR];
 
+const getChainHost = (blocklet) => {
+  if (!blocklet) {
+    return 'none';
+  }
+
+  const chainInfo = getChainInfo(blocklet.configObj);
+  if (chainInfo.host !== 'none') {
+    return chainInfo.host;
+  }
+
+  let childChainHost;
+  forEachChildSync(blocklet, (b) => {
+    if (childChainHost) {
+      return;
+    }
+
+    const childChainInfo = getChainInfo(b.configObj);
+    if (childChainInfo.host !== 'none') {
+      childChainHost = childChainInfo.host;
+    }
+  });
+
+  return childChainHost || 'none';
+};
+
 const migrateAppOnChain = async (blocklet, oldSk, newSk) => {
   logger.info('Preparing for on-chain migration', { did: blocklet.meta.did });
   if (!oldSk) {
@@ -104,13 +129,13 @@ const migrateAppOnChain = async (blocklet, oldSk, newSk) => {
   }
 
   // ensure chain host
-  const chainInfo = getChainInfo(blocklet.configObj);
-  if (chainInfo.host === 'none') {
+  const chainHost = getChainHost(blocklet);
+  if (!chainHost || chainHost === 'none') {
     logger.info('on-chain migration aborted because CHAIN_HOST is empty', { did: blocklet.meta.did });
     return;
   }
 
-  logger.info('on-chain migration for chain ', { did: blocklet.meta.did, host: chainInfo.host });
+  logger.info('on-chain migration for chain ', { did: blocklet.meta.did, host: chainHost });
 
   // ensure account changed
   const type = blocklet.configObj.BLOCKLET_WALLET_TYPE;
@@ -122,7 +147,7 @@ const migrateAppOnChain = async (blocklet, oldSk, newSk) => {
   }
 
   // ensure old account exist on chain
-  const client = new Client(chainInfo.host);
+  const client = new Client(chainHost);
   const oldResult = await client.getAccountState({ address: oldWallet.address });
   if (!oldResult.state) {
     logger.info('on-chain migration aborted because oldSk not declared on chain', { did: blocklet.meta.did });
@@ -250,7 +275,7 @@ const migrateApplicationToStructV2 = async ({ did, appSk: newAppSk, context = {}
       // add root component to blockletData
       const { source, deployedFrom } = component;
       let bundleSource;
-      if (source === BlockletSource.store) {
+      if (source === BlockletSource.registry && deployedFrom && component.meta.bundleName) {
         bundleSource = {
           store: component.deployedFrom,
           name: component.meta.bundleName,
@@ -447,4 +472,4 @@ const migrateApplicationToStructV2 = async ({ did, appSk: newAppSk, context = {}
   });
 };
 
-module.exports = { migrateApplicationToStructV2, sortMoveListBySrc };
+module.exports = { migrateApplicationToStructV2, sortMoveListBySrc, getChainHost };

package/lib/blocklet/manager/helper/upgrade-components.js

@@ -13,6 +13,7 @@ const {
   checkStructVersion,
   checkVersionCompatibility,
   validateBlocklet,
+  getFixedBundleSource,
 } = require('../../../util/blocklet');
 
 const check = async ({ did, states }) => {
@@ -24,7 +25,10 @@ const check = async ({ did, states }) => {
   const newChildren = [];
 
   for (const child of newBlocklet.children || []) {
-    if (child.bundleSource) {
+    // There may be dirty data without bundleSource but with source and deployedFrom
+    const bundleSource = getFixedBundleSource(child);
+
+    if (bundleSource) {
       const {
         staticComponents: [newChild],
         dynamicComponents,
@@ -32,7 +36,7 @@ const check = async ({ did, states }) => {
         meta: {
           staticComponents: [
             {
-              source: child.bundleSource,
+              source: bundleSource,
               name: child.meta.name,
               title: child.meta.title,
               mountPoint: child.mountPoint,

package/lib/blocklet/storage/backup/base.js

@@ -1,6 +1,23 @@
+/**
+ * @typedef {{
+ *  appDid: string
+ *  event: import('events').EventEmitter,
+ * }} BaseBackupInput
+ *
+ * @typedef {{
+ *  encrypt: (v: string) => string,
+ *  decrypt: (v: string) => string,
+ * }} BaseSecurityContext
+ */
+
+const { Hasher } = require('@ocap/mcrypto');
+const { toBuffer } = require('@ocap/util');
+const getBlockletInfo = require('@blocklet/meta/lib/info');
+const security = require('@abtnode/util/lib/security');
+
 class BaseBackup {
   /**
-   * @type {import('./spaces').SpaceBackupInput}
+   * @type {BaseBackupInput}
    * @memberof BaseBackup
    */
   input;
@@ -21,8 +38,8 @@ class BaseBackup {
 
   /**
    *
-   * @description spaces 的 endpoint
-   * @type {import('./spaces').SecurityContext}
+   * @description 安全相关的上下文
+   * @type {BaseSecurityContext}
    * @memberof BaseBackup
    */
   securityContext;
@@ -41,20 +58,57 @@ class BaseBackup {
 
   /**
    *
-   *
-   * @param {import('./spaces').SpacesBackup} spacesBackup
+   * @param {BaseBackup} backup
    * @memberof BaseBackup
    */
-  ensureParams(spacesBackup) {
-    this.blocklet = spacesBackup.blocklet;
-    this.serverDir = spacesBackup.serverDir;
-    this.backupDir = spacesBackup.backupDir;
-    this.securityContext = spacesBackup.securityContext;
+  ensureParams(backup) {
+    this.blocklet = backup.blocklet;
+    this.serverDir = backup.serverDir;
+    this.backupDir = backup.backupDir;
+    this.securityContext = backup.securityContext;
   }
 
   async export() {
     throw new Error('not implemented');
   }
+
+  async _getSecurityContext(states) {
+    const blocklet = await states.blocklet.getBlocklet(this.input.appDid);
+    const nodeInfo = await states.node.read();
+
+    const { wallet } = getBlockletInfo(blocklet, nodeInfo.sk);
+
+    const { secretKey, address } = wallet; // we encrypt using latest wallet, not the permanent wallet
+    const password = toBuffer(Hasher.SHA3.hash256(Buffer.concat([secretKey, address].map(toBuffer))));
+    const encrypt = (v) => security.encrypt(v, address, password);
+    const decrypt = (v) => security.decrypt(v, address, password);
+
+    return {
+      signer: wallet,
+      delegation: '',
+      encrypt,
+      decrypt,
+    };
+  }
+
+  /**
+   *
+   * @param {BaseBackup} dataBackup
+   * @param {Array<BaseBackup>} storages
+   * @memberof BaseBackup
+   */
+  async _exportData(dataBackup, storages) {
+    // @note: dataBackup 需要先于 blockletBackup 执行，并且 blockletBackup 与其他 backup的执行可以是无序的
+    dataBackup.ensureParams(this);
+    await dataBackup.export();
+
+    await Promise.all(
+      storages.map((storage) => {
+        storage.ensureParams(this);
+        return storage.export();
+      })
+    );
+  }
 }
 
 module.exports = {

package/lib/blocklet/storage/backup/blocklet.js

@@ -1,6 +1,12 @@
-const { removeSync, outputJsonSync } = require('fs-extra');
+const { removeSync, outputJsonSync, createWriteStream, createReadStream } = require('fs-extra');
 const { cloneDeep } = require('lodash');
-const { join } = require('path');
+const { join, basename } = require('path');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+const isEmpty = require('lodash/isEmpty');
+const streamToPromise = require('stream-to-promise');
+const axios = require('@abtnode/util/lib/axios');
+const isUrl = require('is-url');
+const { getLogoUrl } = require('@abtnode/util/lib/logo');
 const { BaseBackup } = require('./base');
 
 class BlockletBackup extends BaseBackup {
@@ -8,6 +14,10 @@ class BlockletBackup extends BaseBackup {
 
   async export() {
     const blocklet = await this.cleanData();
+
+    const targetLogoPath = await this.writeLogoFile();
+    blocklet.meta.appLogo = basename(targetLogoPath);
+
     removeSync(join(this.backupDir, this.filename));
     outputJsonSync(join(this.backupDir, this.filename), blocklet);
   }
@@ -56,7 +66,7 @@ class BlockletBackup extends BaseBackup {
    * @memberof BlockletExtrasBackup
    */
   encrypt(info) {
-    if (Array.isArray(info.migratedFrom)) {
+    if (Array.isArray(info?.migratedFrom)) {
       info.migratedFrom = info.migratedFrom.map((x) => {
         x.appSk = this.securityContext.encrypt(x.appSk);
         return x;
@@ -65,6 +75,57 @@ class BlockletBackup extends BaseBackup {
 
     return info;
   }
+
+  /**
+   *
+   *
+   * @param {string} target
+   * @returns {Promise<string>}
+   * @memberof DataBackup
+   */
+  async writeLogoFile() {
+    const customLogoSquareUrl = this.blocklet.environments.find(
+      (e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_LOGO_SQUARE
+    )?.value;
+    const appDir = this.blocklet.environments.find((e) => e.key === 'BLOCKLET_APP_DIR')?.value;
+    const logo = this.blocklet?.meta?.logo;
+    const defaultLogoPath = join(this.serverDir, 'data', this.blocklet.meta.name, 'logo.svg');
+
+    const logoUrl = await getLogoUrl({
+      customLogoSquareUrl,
+      appDir,
+      logo,
+      defaultLogoPath,
+    });
+
+    const logoStream = await this.getLogoStream(logoUrl);
+    const targetLogoPath = join(this.backupDir, 'data', basename(logoUrl));
+    await streamToPromise(logoStream.pipe(createWriteStream(targetLogoPath)));
+
+    return targetLogoPath;
+  }
+
+  /**
+   *
+   *
+   * @param {string} logoUrl
+   * @returns {Promise<NodeJS.ReadStream>}
+   * @memberof DataBackup
+   */
+  async getLogoStream(logoUrl) {
+    if (isEmpty(logoUrl)) {
+      throw new Error(`logoUrl(${logoUrl}) cannot be empty`);
+    }
+
+    if (isUrl(logoUrl)) {
+      const res = await axios.get(logoUrl, {
+        responseType: 'stream',
+      });
+      return res.data;
+    }
+
+    return createReadStream(logoUrl);
+  }
 }
 
 module.exports = { BlockletBackup };

package/lib/blocklet/storage/backup/disk.js

@@ -0,0 +1,132 @@
+const fs = require('fs-extra');
+const { isValid } = require('@arcblock/did');
+const { ensureDirSync } = require('fs-extra');
+const { isEmpty } = require('lodash');
+const { join } = require('path');
+const { getAppName } = require('@blocklet/meta/lib/util');
+
+const states = require('../../../states');
+const { getBackupDirs } = require('../utils/disk');
+const { BaseBackup } = require('./base');
+const { AuditLogBackup } = require('./audit-log');
+const { BlockletBackup } = require('./blocklet');
+const { BlockletExtrasBackup } = require('./blocklet-extras');
+const { BlockletsBackup } = require('./blocklets');
+const { DataBackup } = require('./data');
+const { RoutingRuleBackup } = require('./routing-rule');
+
+class DiskBackup extends BaseBackup {
+  /**
+   *
+   * @type {import('./base').BaseBackupInput}
+   * @memberof DiskBackup
+   */
+  input;
+
+  /**
+   * @description blocklet state 对象
+   * @type {import('@abtnode/client').BlockletState}
+   * @memberof DiskBackup
+   */
+  blocklet;
+
+  /**
+   * @type {string}
+   * @memberof DiskBackup
+   */
+  backupDir;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof DiskBackup
+   */
+  serverDir;
+
+  /**
+   *
+   * @type {import('./base').BaseSecurityContext}
+   * @memberof DiskBackup
+   */
+  securityContext;
+
+  storages;
+
+  dataBackup;
+
+  /**
+   *
+   * @param {import('./base').BaseBackupInput} input
+   * @memberof DiskBackup
+   */
+  constructor(input) {
+    super(input);
+    this.verify(input);
+    this.input = input;
+    this.storages = [
+      new AuditLogBackup(this.input),
+      new BlockletBackup(this.input),
+      new BlockletsBackup(this.input),
+      new BlockletExtrasBackup(this.input),
+      new RoutingRuleBackup(this.input),
+    ];
+    this.dataBackup = new DataBackup(this.input);
+  }
+
+  /**
+   * @param {import('./base').BaseBackupInput} input
+   * @returns {void}
+   * @memberof DiskBackup
+   */
+  verify(input) {
+    if (isEmpty(input?.appDid) || !isValid(input?.appDid)) {
+      throw new Error(`input.appDid(${input?.appDid}) is not a valid did`);
+    }
+  }
+
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof DiskBackup
+   */
+  async backup() {
+    await this.initialize();
+    await this.addMeta();
+    await this.export();
+  }
+
+  async initialize() {
+    this.blocklet = await states.blocklet.getBlocklet(this.input.appDid);
+    if (isEmpty(this.blocklet)) {
+      throw new Error('blocklet cannot be empty');
+    }
+
+    this.serverDir = process.env.ABT_NODE_DATA_DIR;
+    const { baseBackupDir, backupDir } = getBackupDirs(this.serverDir, this.blocklet.appDid);
+    this.baseBackupDir = baseBackupDir;
+    this.backupDir = backupDir;
+    ensureDirSync(this.backupDir);
+
+    this.securityContext = await this._getSecurityContext(states);
+  }
+
+  async export() {
+    return this._exportData(this.dataBackup, this.storages);
+  }
+
+  async addMeta() {
+    const meta = {
+      appDid: this.blocklet.appDid,
+      appPid: this.blocklet.appPid,
+      name: getAppName(this.blocklet),
+      createdAt: Date.now(),
+    };
+
+    await fs.writeJSON(join(this.baseBackupDir, 'meta.json'), meta);
+  }
+}
+
+module.exports = {
+  DiskBackup,
+};

package/lib/blocklet/storage/backup/spaces.js

@@ -20,15 +20,12 @@ const { SpaceClient, BackupBlockletCommand } = require('@did-space/client');
 const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
 const { join, basename } = require('path');
-const { Hasher } = require('@ocap/mcrypto');
-const { toBuffer } = require('@ocap/util');
 const { getAppName, getAppDescription } = require('@blocklet/meta/lib/util');
-const getBlockletInfo = require('@blocklet/meta/lib/info');
-const security = require('@abtnode/util/lib/security');
 
 const logger = require('@abtnode/logger')('@abtnode/core:storage:backup');
 
 const states = require('../../../states');
+const { BaseBackup } = require('./base');
 const { AuditLogBackup } = require('./audit-log');
 const { BlockletBackup } = require('./blocklet');
 const { BlockletExtrasBackup } = require('./blocklet-extras');
@@ -36,7 +33,7 @@ const { BlockletsBackup } = require('./blocklets');
 const { DataBackup } = require('./data');
 const { RoutingRuleBackup } = require('./routing-rule');
 
-class SpacesBackup {
+class SpacesBackup extends BaseBackup {
   /**
    *
    * @type {SpaceBackupInput}
@@ -84,12 +81,15 @@ class SpacesBackup {
 
   storages;
 
+  dataBackup;
+
   /**
    *
    * @param {SpaceBackupInput} input
    * @memberof SpacesBackup
    */
   constructor(input) {
+    super(input);
     this.verify(input);
     this.input = input;
     this.storages = [
@@ -98,8 +98,8 @@ class SpacesBackup {
       new BlockletsBackup(this.input),
       new BlockletExtrasBackup(this.input),
       new RoutingRuleBackup(this.input),
-      new DataBackup(this.input),
     ];
+    this.dataBackup = new DataBackup(this.input);
   }
 
   /**
@@ -141,7 +141,7 @@ class SpacesBackup {
       throw new Error('spaceEndpoint cannot be empty');
     }
 
-    this.securityContext = await this.getSecurityContext();
+    this.securityContext = await this._getSecurityContext(states);
   }
 
   async export() {
@@ -151,12 +151,9 @@ class SpacesBackup {
       progress: 15,
       completed: false,
     });
-    await Promise.all(
-      this.storages.map((storage) => {
-        storage.ensureParams(this);
-        return storage.export();
-      })
-    );
+
+    await this._exportData(this.dataBackup, this.storages);
+
     this.input.event.emit(BlockletEvents.backupProgress, {
       appDid: this.input.appDid,
       message: 'Data ready, start backup...',
@@ -201,7 +198,7 @@ class SpacesBackup {
           const percent = (data.completed * 100) / data.total;
           this.input.event.emit(BlockletEvents.backupProgress, {
             appDid: this.input.appDid,
-            message: `Uploaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+            message: `Uploading file ${basename(data.key)} (${data.completed}/${data.total})`,
             // 0.8 是因为上传文件到 spaces 占进度的 80%，+ 20 是因为需要累加之前的进度
             progress: +Math.ceil(percent * 0.8).toFixed(2) + 20,
             completed: false,
@@ -214,25 +211,6 @@ class SpacesBackup {
       throw new Error(`Sync to spaces encountered error: ${message}`);
     }
   }
-
-  async getSecurityContext() {
-    const blocklet = await states.blocklet.getBlocklet(this.input.appDid);
-    const nodeInfo = await states.node.read();
-
-    const { wallet } = getBlockletInfo(blocklet, nodeInfo.sk);
-
-    const { secretKey, address } = wallet; // we encrypt using latest wallet, not the permanent wallet
-    const password = toBuffer(Hasher.SHA3.hash256(Buffer.concat([secretKey, address].map(toBuffer))));
-    const encrypt = (v) => security.encrypt(v, address, password);
-    const decrypt = (v) => security.decrypt(v, address, password);
-
-    return {
-      signer: wallet,
-      delegation: '',
-      encrypt,
-      decrypt,
-    };
-  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/base.js

@@ -1,13 +1,20 @@
+/**
+ * @typedef {{
+ *  appDid: string; // --> appDid
+ *  password: Buffer; // derived from (appSk, appDid)
+ *  event: import('events').EventEmitter,
+ * }} BaseRestoreInput
+ */
+
 class BaseRestore {
   /**
    *
-   * @type {import('./spaces').SpaceRestoreInput}
+   * @type {BaseRestoreInput}
    * @memberof BaseRestore
    */
   input;
 
   /**
-   * @description 当前 blocklet 的数据目录
    * @type {string}
    * @memberof BaseRestore
    */
@@ -28,12 +35,12 @@ class BaseRestore {
   /**
    *
    *
-   * @param {import('./spaces').SpacesRestore} spaces
+   * @param {BaseRestore} restore
    * @memberof BaseRestore
    */
-  ensureParams(spaces) {
-    this.restoreDir = spaces.restoreDir;
-    this.serverDir = spaces.serverDir;
+  ensureParams(restore) {
+    this.restoreDir = restore.restoreDir;
+    this.serverDir = restore.serverDir;
   }
 
   // eslint-disable-next-line

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -9,8 +9,7 @@ class BlockletExtrasRestore extends BaseRestore {
   filename = 'blocklet-extras.json';
 
   async import(params) {
-    const extras = this.getExtras();
-    this.cleanExtras(extras, params);
+    const extras = this.cleanExtras(this.getExtras(), params);
     removeSync(join(this.restoreDir, this.filename));
     outputJsonSync(join(this.restoreDir, this.filename), extras);
   }
@@ -32,9 +31,10 @@ class BlockletExtrasRestore extends BaseRestore {
    *
    * @description 清理数据并加密
    * @param {import('@abtnode/client').BlockletState} raw
+   * @returns {import('@abtnode/client').BlockletState}
    * @memberof BlockletExtrasRestore
    */
-  async cleanExtras(raw, params) {
+  cleanExtras(raw, params) {
     const blockletExtra = cloneDeep(raw);
 
     const queue = [blockletExtra];
@@ -48,6 +48,8 @@ class BlockletExtrasRestore extends BaseRestore {
         queue.push(...current.children);
       }
     }
+
+    return blockletExtra;
   }
 
   /**

package/lib/blocklet/storage/restore/blocklet.js

@@ -33,7 +33,7 @@ class BlockletRestore extends BaseRestore {
    */
   decrypt(blocklet, params) {
     const { password } = this.input;
-    if (Array.isArray(blocklet.migratedFrom)) {
+    if (Array.isArray(blocklet?.migratedFrom)) {
       blocklet.migratedFrom = blocklet.migratedFrom.map((x) => {
         x.appSk = security.decrypt(x.appSk, params.salt, password);
         return x;