@abtnode/core 1.8.65-beta-5405baf2 → 1.8.65-beta-bfcc12ce

package/lib/blocklet/manager/helper/install-from-backup.js

@@ -11,6 +11,14 @@ const logger = require('@abtnode/logger')('@abtnode/core:install-from-backup');
 
 const { validateBlocklet, checkDuplicateAppSk, getAppDirs } = require('../../../util/blocklet');
 
+/**
+ *
+ * @param {{
+ *  manager: import('../disk'),
+ *  states: import('../../../states/index')
+ * }} param0
+ * @returns
+ */
 module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states, manager } = {}) => {
   // TODO: support more url schema feature (http, did-spaces)
   if (!url.startsWith('file://')) {
@@ -24,11 +32,7 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
   }
 
   // parse data from source dir
-
   const srcBundleDirs = await getAppDirs(path.join(dir, 'blocklets'));
-  if (!srcBundleDirs.length) {
-    throw new Error(`bundle dirs does not found in ${dir}`);
-  }
 
   const srcDataDir = path.join(dir, 'data');
 
@@ -61,6 +65,7 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
 
   // FIXME: meta.did and meta.name should be dynamic created when installing multiple blocklet is supported
   const existState = await states.blocklet.hasBlocklet(did);
+
   if (existState) {
     logger.error('blocklet is already exist', { did });
     throw new Error('blocklet is already exist');
@@ -71,6 +76,8 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
     const skConfig = extra.configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
     if (skConfig) {
       skConfig.value = blockletSecretKey;
+      skConfig.secure = true;
+      skConfig.shared = false;
     } else {
       extra.configs.push({
         key: BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK,
@@ -131,6 +138,8 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
         logger.info(`bundle  is ${moveDir ? 'moved' : 'copied'} successfully`, { installDir });
       })
     );
+    // 从 store 下载 blocklet
+    await manager.blockletDownloader.download(state, { skipCheckIntegrity: true });
 
     // FIXME same as copy extra
     const dataDir = path.join(manager.dataDirs.data, appName);

package/lib/blocklet/manager/helper/rollback-cache.js

@@ -0,0 +1,41 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+const DIR_NAME = 'rollback-cache';
+const security = require('@abtnode/util/lib/security');
+
+class RollbackCache {
+  constructor({ dir, dek }) {
+    this.dir = dir;
+    this.dek = dek;
+  }
+
+  async backup({ did, action, oldBlocklet }) {
+    const file = this._getFile(did);
+    const data = this.dek ? security.encrypt(JSON.stringify(oldBlocklet), did, this.dek) : oldBlocklet;
+    await fs.outputJSON(file, { action, oldBlocklet: data });
+  }
+
+  async remove({ did }) {
+    const file = this._getFile(did);
+    await fs.remove(file);
+  }
+
+  async restore({ did }) {
+    const file = this._getFile(did);
+    if (fs.existsSync(file)) {
+      const data = await fs.readJSON(file);
+      if (data?.oldBlocklet && this.dek) {
+        data.oldBlocklet = JSON.parse(security.decrypt(data.oldBlocklet, did, this.dek));
+      }
+      return data;
+    }
+    return null;
+  }
+
+  _getFile(did) {
+    return path.join(this.dir, DIR_NAME, `${did}.json`);
+  }
+}
+
+module.exports = RollbackCache;

package/lib/blocklet/storage/backup/blocklet-extras.js

@@ -75,6 +75,10 @@ class BlockletExtrasBackup extends BaseBackup {
    * @memberof BlockletExtrasBackup
    */
   useBlockletEncryptConfigs(configs) {
+    if (isEmpty(configs)) {
+      return;
+    }
+
     // 准备加解密所需的参数
     // @see: https://github.com/ArcBlock/blocklet-server/blob/f561ba7290285f2e23dccb6d5323eb4d43c3cc3e/core/state/lib/index.js#L59
     const dek = readFileSync(join(this.serverDataDir, '.sock'));

package/lib/blocklet/storage/backup/blocklets.js

@@ -1,7 +1,8 @@
-const { removeSync, existsSync, ensureDirSync, createWriteStream } = require('fs-extra');
-const { join, dirname } = require('path');
-const archiver = require('archiver');
+const { join } = require('path');
+const validUrl = require('valid-url');
 const { BaseBackup } = require('./base');
+const { dirToZip } = require('../utils/zip');
+const { compareAndMove } = require('../utils/hash');
 
 class BlockletsBackup extends BaseBackup {
   /**
@@ -13,21 +14,25 @@ class BlockletsBackup extends BaseBackup {
     const blockletMetas = this.getBlockletMetas(this.blocklet);
     const serverBlockletsDir = join(this.serverDataDir, 'blocklets');
 
+    const blockletMetasFromLocal = blockletMetas.filter(
+      (b) => !validUrl.isHttpUri(b.tarball) && !validUrl.isHttpsUri(b.tarball)
+    );
+
     const dirs = [];
-    for (const blockletMeta of blockletMetas) {
+    for (const blockletMeta of blockletMetasFromLocal) {
       const sourceDir = join(serverBlockletsDir, blockletMeta.name, blockletMeta.version);
-      const destDir = join(blockletMeta.name, blockletMeta.version);
-      dirs.push({ sourceDir, destDir });
+      const zipPath = join(this.blockletBackupDir, 'blocklets', blockletMeta.name, `${blockletMeta.version}.zip`);
+      dirs.push({ sourceDir, zipPath });
     }
 
-    await this.dirsToZip(dirs, join(this.blockletBackupDir, 'blocklets.zip'));
+    await this.dirsToZip(dirs);
   }
 
   /**
    *
    *
    * @param {import('@abtnode/client').BlockletState} blocklet
-   * @returns {Array<{name: string, version: string}>}
+   * @returns {Array<{name: string, version: string, tarball: string}>}
    * @memberof BlockletsBackup
    */
   getBlockletMetas(blocklet) {
@@ -39,6 +44,7 @@ class BlockletsBackup extends BaseBackup {
     metas.push({
       name: blocklet.meta.bundleName,
       version: blocklet.meta.version,
+      tarball: blocklet?.meta?.dist?.tarball,
     });
 
     for (const child of blocklet.children) {
@@ -49,31 +55,19 @@ class BlockletsBackup extends BaseBackup {
   }
 
   /**
-   * @param {Array<{sourceDir: string, destDir: string}>} dirs: /some/folder/to/compress
+   * @param {Array<{sourceDir: string, zipPath: string}>} dirs: /some/folder/to/compress
    * @param {String} zipPath: /path/to/created.zip
    * @returns {Promise}
    * @memberof BlockletsBackup
    */
-  async dirsToZip(dirs, zipPath) {
-    ensureDirSync(dirname(zipPath));
-    if (existsSync(zipPath)) {
-      removeSync(zipPath);
-    }
-
-    const archive = archiver('zip', { zlib: { level: 9 } });
-    const stream = createWriteStream(zipPath);
-
-    return new Promise((resolve, reject) => {
-      archive.on('error', (err) => reject(err));
-      stream.on('close', () => resolve());
-
-      for (const dir of dirs) {
-        archive.directory(dir.sourceDir, dir.destDir);
-      }
-
-      archive.pipe(stream);
-      archive.finalize();
-    });
+  async dirsToZip(dirs) {
+    await Promise.all(
+      dirs.map(async (dir) => {
+        const tempZipPath = `${dir.zipPath}.bak`;
+        await dirToZip(dir.sourceDir, tempZipPath);
+        await compareAndMove(dir.zipPath, tempZipPath);
+      })
+    );
   }
 }
 

package/lib/blocklet/storage/backup/data.js

@@ -1,4 +1,4 @@
-const { copySync } = require('fs-extra');
+const { copy } = require('fs-extra');
 const { join } = require('path');
 const { BaseBackup } = require('./base');
 
@@ -12,7 +12,7 @@ class DataBackup extends BaseBackup {
     const blockletDataDir = join(this.serverDataDir, 'data', this.blocklet.meta.name);
     const blockletBackupDataDir = join(this.blockletBackupDir, 'data');
 
-    copySync(blockletDataDir, blockletBackupDataDir, { overwrite: true });
+    await copy(blockletDataDir, blockletBackupDataDir, { overwrite: true });
   }
 }
 

package/lib/blocklet/storage/backup/logs.js

@@ -1,7 +1,8 @@
-const { ensureDirSync, copySync, statSync } = require('fs-extra');
+const { ensureDirSync, copy, statSync } = require('fs-extra');
 const { join } = require('path');
 const { BaseBackup } = require('./base');
 
+// note: 可以不需要备份
 class LogsBackup extends BaseBackup {
   async export() {
     const sourceLogsDir = join(this.serverDataDir, 'logs', this.blocklet.meta.name);
@@ -9,7 +10,7 @@ class LogsBackup extends BaseBackup {
 
     const targetLogsDir = join(this.blockletBackupDir, 'logs');
 
-    copySync(sourceLogsDir, targetLogsDir, {
+    await copy(sourceLogsDir, targetLogsDir, {
       overwrite: true,
       filter: (src) => {
         return !statSync(src).isSymbolicLink();

package/lib/blocklet/storage/backup/spaces.js

@@ -8,7 +8,7 @@ const { isValid } = require('@arcblock/did');
 const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const { getBlockletInfo } = require('@blocklet/meta');
 const { SpaceClient, SyncFolderPushCommand } = require('@did-space/client');
-const { ensureDirSync, existsSync, rmdirSync, removeSync } = require('fs-extra');
+const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
 const { join } = require('path');
 const states = require('../../../states');
@@ -17,7 +17,6 @@ const { BlockletBackup } = require('./blocklet');
 const { BlockletExtrasBackup } = require('./blocklet-extras');
 const { BlockletsBackup } = require('./blocklets');
 const { DataBackup } = require('./data');
-const { LogsBackup } = require('./logs');
 const { RoutingRuleBackup } = require('./routing-rule');
 
 class SpacesBackup {
@@ -78,7 +77,6 @@ class SpacesBackup {
     this.input = input;
     this.storages = [
       new AuditLogBackup(this.input),
-      new LogsBackup(this.input),
       new BlockletBackup(this.input),
       new BlockletsBackup(this.input),
       new BlockletExtrasBackup(this.input),
@@ -107,7 +105,6 @@ class SpacesBackup {
     await this.initialize();
     await this.export();
     await this.syncToSpaces();
-    await this.destroy();
   }
 
   async initialize() {
@@ -121,9 +118,6 @@ class SpacesBackup {
     this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
 
     this.blockletBackupDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/backup', this.blocklet.meta.did);
-    if (existsSync(this.blockletBackupDir)) {
-      rmdirSync(this.blockletBackupDir, { recursive: true });
-    }
     ensureDirSync(this.blockletBackupDir);
 
     this.spacesEndpoint = this.blocklet.environments.find(
@@ -158,7 +152,8 @@ class SpacesBackup {
         source: join(this.blockletBackupDir, '/'),
         target: join('.did-objects', this.blocklet.appDid),
         debug: true,
-        retryCount: 3,
+        concurrency: 64,
+        retryCount: 100,
         filter: (object) => {
           return object.name !== '.DS_Store';
         },
@@ -176,12 +171,6 @@ class SpacesBackup {
     const { wallet } = getBlockletInfo(blockletInfo, nodeInfo.sk);
     return wallet;
   }
-
-  async destroy() {
-    if (existsSync(this.blockletBackupDir)) {
-      removeSync(this.blockletBackupDir);
-    }
-  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -1,5 +1,5 @@
 const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
-const { cloneDeep } = require('lodash');
+const { cloneDeep, isArray } = require('lodash');
 const { join } = require('path');
 const security = require('@abtnode/util/lib/security');
 const { BaseRestore } = require('./base');
@@ -61,6 +61,13 @@ class BlockletExtrasRestore extends BaseRestore {
    * @memberof BlockletExtrasRestore
    */
   useBlockletDecryptConfigs(configs) {
+    // eslint-disable-next-line no-console
+    console.log('debug', configs, !configs || !isArray(configs));
+
+    if (!configs || !isArray(configs)) {
+      return;
+    }
+
     for (const config of configs) {
       // secure 为 true 的配置才需要被加密保存上次到 did spaces
       if (config.secure) {
@@ -76,8 +83,6 @@ class BlockletExtrasRestore extends BaseRestore {
         config.value = decryptByBlocklet;
       }
     }
-
-    return configs;
   }
 }
 

package/lib/blocklet/storage/restore/blocklets.js

@@ -1,23 +1,41 @@
-const { existsSync, removeSync } = require('fs-extra');
+const { existsSync, remove } = require('fs-extra');
 const { join } = require('path');
-const StreamZip = require('node-stream-zip');
+const fg = require('fast-glob');
 const { BaseRestore } = require('./base');
+const { zipToDir } = require('../utils/zip');
 
 class BlockletsRestore extends BaseRestore {
-  filename = 'blocklets.zip';
+  filename = 'blocklets';
 
   async import() {
-    const blockletZipPath = join(this.blockletRestoreDir, this.filename);
+    const blockletsDir = join(this.blockletRestoreDir, this.filename);
 
-    if (!existsSync(blockletZipPath)) {
-      throw new Error(`file not found: ${blockletZipPath}`);
+    if (!existsSync(blockletsDir)) {
+      throw new Error(`dir not found: ${blockletsDir}`);
     }
 
-    // eslint-disable-next-line new-cap
-    const zip = new StreamZip.async({ file: blockletZipPath });
-    await zip.extract(null, join(this.blockletRestoreDir, 'blocklets'));
-    await zip.close();
-    removeSync(blockletZipPath);
+    const paths = await fg('**/*.zip', {
+      cwd: blockletsDir,
+      onlyFiles: true,
+      absolute: true,
+    });
+
+    /**
+     * @type {{source: string, target: string}}
+     */
+    const zipConfigs = paths.map((path) => {
+      return {
+        source: path,
+        target: path.replace(/.zip$/, ''),
+      };
+    });
+
+    await Promise.all(
+      zipConfigs.map(async (zipConfig) => {
+        await zipToDir(zipConfig.source, zipConfig.target);
+        await remove(zipConfig.source);
+      })
+    );
   }
 }
 

package/lib/blocklet/storage/restore/logs.js

@@ -0,0 +1,21 @@
+const { removeSync, existsSync } = require('fs-extra');
+const { join } = require('path');
+const { BaseRestore } = require('./base');
+const { zipToDir } = require('../utils/zip');
+
+class LogsRestore extends BaseRestore {
+  filename = 'logs.zip';
+
+  async import() {
+    const blockletZipPath = join(this.blockletRestoreDir, this.filename);
+
+    if (!existsSync(blockletZipPath)) {
+      throw new Error(`file not found: ${blockletZipPath}`);
+    }
+
+    await zipToDir(blockletZipPath, join(this.blockletRestoreDir, 'logs'));
+    removeSync(blockletZipPath);
+  }
+}
+
+module.exports = { LogsRestore };

package/lib/blocklet/storage/restore/spaces.js

@@ -74,6 +74,10 @@ class SpacesRestore {
     this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
     this.blockletWallet = await this.getBlockletWallet();
 
+    if (!this.input.endpoint.includes(this.blockletWallet.address)) {
+      throw new Error(`endpoint and blocklet.appDid(${this.blockletWallet.address}) do not match`);
+    }
+
     this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.blockletWallet.address);
     if (existsSync(this.blockletRestoreDir)) {
       rmdirSync(this.blockletRestoreDir, { recursive: true });
@@ -113,7 +117,8 @@ class SpacesRestore {
         source: join('.did-objects', this.blockletWallet.address, '/'),
         target: this.blockletRestoreDir,
         debug: true,
-        retryCount: 3,
+        concurrency: 64,
+        retryCount: 100,
       })
     );
 

package/lib/blocklet/storage/utils/hash.js

@@ -0,0 +1,51 @@
+const { removeSync, existsSync, move, createReadStream } = require('fs-extra');
+const hasha = require('hasha');
+
+/**
+ *
+ *
+ * @param {NodeJS.ReadableStream} stream1
+ * @param {NodeJS.ReadableStream} stream2
+ * @return {Promise<boolean>}
+ */
+async function compareHash(stream1, stream2) {
+  const hash1 = await hasha.fromStream(stream1, {
+    algorithm: 'md5',
+  });
+  const hash2 = await hasha.fromStream(stream2, {
+    algorithm: 'md5',
+  });
+
+  return hash1 === hash2;
+}
+
+/**
+ *
+ * @description 比对新文件和旧文件的hash，旧文件不存在或者hash不同时，使用新文件替换，hash相同，删除新文件
+ * @param {string} oldFilePath
+ * @param {string} newFilePath
+ * @returns {Promise<boolean>}
+ */
+async function compareAndMove(oldFilePath, newFilePath) {
+  if (!existsSync(oldFilePath)) {
+    await move(newFilePath, oldFilePath, { overwrite: true });
+    return;
+  }
+
+  if (!existsSync(newFilePath)) {
+    throw new Error(`newFilePath(${newFilePath}) not found`);
+  }
+
+  const isSame = await compareHash(createReadStream(oldFilePath), createReadStream(newFilePath));
+  if (isSame) {
+    removeSync(newFilePath);
+    return;
+  }
+
+  await move(newFilePath, oldFilePath, { overwrite: true });
+}
+
+module.exports = {
+  compareHash,
+  compareAndMove,
+};

package/lib/blocklet/storage/utils/zip.js

@@ -0,0 +1,43 @@
+const archiver = require('archiver');
+const { ensureDirSync, existsSync, removeSync, createWriteStream } = require('fs-extra');
+const { dirname } = require('path');
+const StreamZip = require('node-stream-zip');
+
+/**
+ *
+ *
+ * @param {string} source ~/abc/ 通常是一个文件夹
+ * @param {string} target abc.zip 压缩包的名称
+ * @return {*}
+ */
+function dirToZip(source, target) {
+  return new Promise((resolve, reject) => {
+    ensureDirSync(dirname(target));
+
+    if (existsSync(target)) {
+      removeSync(target);
+    }
+
+    const output = createWriteStream(target);
+    const archive = archiver('zip', { level: 9 });
+    archive.on('error', (err) => reject(err));
+    output.on('close', () => resolve());
+
+    archive.directory(source, false);
+
+    archive.pipe(output);
+    archive.finalize();
+  });
+}
+
+async function zipToDir(source, target) {
+  // eslint-disable-next-line new-cap
+  const zip = new StreamZip.async({ file: source });
+  await zip.extract(null, target);
+  await zip.close();
+}
+
+module.exports = {
+  dirToZip,
+  zipToDir,
+};