@abtnode/core 1.8.65-beta-f7af64a4 → 1.8.65-beta-bfcc12ce

package/lib/blocklet/storage/backup/blocklets.js

@@ -1,7 +1,8 @@
-const { removeSync, existsSync, ensureDirSync, createWriteStream } = require('fs-extra');
-const { join, dirname } = require('path');
-const archiver = require('archiver');
+const { join } = require('path');
+const validUrl = require('valid-url');
 const { BaseBackup } = require('./base');
+const { dirToZip } = require('../utils/zip');
+const { compareAndMove } = require('../utils/hash');
 
 class BlockletsBackup extends BaseBackup {
   /**
@@ -13,21 +14,25 @@ class BlockletsBackup extends BaseBackup {
     const blockletMetas = this.getBlockletMetas(this.blocklet);
     const serverBlockletsDir = join(this.serverDataDir, 'blocklets');
 
+    const blockletMetasFromLocal = blockletMetas.filter(
+      (b) => !validUrl.isHttpUri(b.tarball) && !validUrl.isHttpsUri(b.tarball)
+    );
+
     const dirs = [];
-    for (const blockletMeta of blockletMetas) {
+    for (const blockletMeta of blockletMetasFromLocal) {
       const sourceDir = join(serverBlockletsDir, blockletMeta.name, blockletMeta.version);
-      const destDir = join(blockletMeta.name, blockletMeta.version);
-      dirs.push({ sourceDir, destDir });
+      const zipPath = join(this.blockletBackupDir, 'blocklets', blockletMeta.name, `${blockletMeta.version}.zip`);
+      dirs.push({ sourceDir, zipPath });
     }
 
-    await this.dirsToZip(dirs, join(this.blockletBackupDir, 'blocklets.zip'));
+    await this.dirsToZip(dirs);
   }
 
   /**
    *
    *
    * @param {import('@abtnode/client').BlockletState} blocklet
-   * @returns {Array<{name: string, version: string}>}
+   * @returns {Array<{name: string, version: string, tarball: string}>}
    * @memberof BlockletsBackup
    */
   getBlockletMetas(blocklet) {
@@ -39,6 +44,7 @@ class BlockletsBackup extends BaseBackup {
     metas.push({
       name: blocklet.meta.bundleName,
       version: blocklet.meta.version,
+      tarball: blocklet?.meta?.dist?.tarball,
     });
 
     for (const child of blocklet.children) {
@@ -49,31 +55,19 @@ class BlockletsBackup extends BaseBackup {
   }
 
   /**
-   * @param {Array<{sourceDir: string, destDir: string}>} dirs: /some/folder/to/compress
+   * @param {Array<{sourceDir: string, zipPath: string}>} dirs: /some/folder/to/compress
    * @param {String} zipPath: /path/to/created.zip
    * @returns {Promise}
    * @memberof BlockletsBackup
    */
-  async dirsToZip(dirs, zipPath) {
-    ensureDirSync(dirname(zipPath));
-    if (existsSync(zipPath)) {
-      removeSync(zipPath);
-    }
-
-    const archive = archiver('zip', { zlib: { level: 9 } });
-    const stream = createWriteStream(zipPath);
-
-    return new Promise((resolve, reject) => {
-      archive.on('error', (err) => reject(err));
-      stream.on('close', () => resolve());
-
-      for (const dir of dirs) {
-        archive.directory(dir.sourceDir, dir.destDir);
-      }
-
-      archive.pipe(stream);
-      archive.finalize();
-    });
+  async dirsToZip(dirs) {
+    await Promise.all(
+      dirs.map(async (dir) => {
+        const tempZipPath = `${dir.zipPath}.bak`;
+        await dirToZip(dir.sourceDir, tempZipPath);
+        await compareAndMove(dir.zipPath, tempZipPath);
+      })
+    );
   }
 }
 

package/lib/blocklet/storage/backup/data.js

@@ -1,4 +1,4 @@
-const { copySync } = require('fs-extra');
+const { copy } = require('fs-extra');
 const { join } = require('path');
 const { BaseBackup } = require('./base');
 
@@ -12,7 +12,7 @@ class DataBackup extends BaseBackup {
     const blockletDataDir = join(this.serverDataDir, 'data', this.blocklet.meta.name);
     const blockletBackupDataDir = join(this.blockletBackupDir, 'data');
 
-    copySync(blockletDataDir, blockletBackupDataDir, { overwrite: true });
+    await copy(blockletDataDir, blockletBackupDataDir, { overwrite: true });
   }
 }
 

package/lib/blocklet/storage/backup/logs.js

@@ -1,7 +1,8 @@
-const { ensureDirSync, copySync, statSync } = require('fs-extra');
+const { ensureDirSync, copy, statSync } = require('fs-extra');
 const { join } = require('path');
 const { BaseBackup } = require('./base');
 
+// note: 可以不需要备份
 class LogsBackup extends BaseBackup {
   async export() {
     const sourceLogsDir = join(this.serverDataDir, 'logs', this.blocklet.meta.name);
@@ -9,7 +10,7 @@ class LogsBackup extends BaseBackup {
 
     const targetLogsDir = join(this.blockletBackupDir, 'logs');
 
-    copySync(sourceLogsDir, targetLogsDir, {
+    await copy(sourceLogsDir, targetLogsDir, {
       overwrite: true,
       filter: (src) => {
         return !statSync(src).isSymbolicLink();

package/lib/blocklet/storage/backup/spaces.js

@@ -8,7 +8,7 @@ const { isValid } = require('@arcblock/did');
 const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 const { getBlockletInfo } = require('@blocklet/meta');
 const { SpaceClient, SyncFolderPushCommand } = require('@did-space/client');
-const { ensureDirSync, existsSync, rmdirSync, removeSync } = require('fs-extra');
+const { ensureDirSync } = require('fs-extra');
 const { isEmpty } = require('lodash');
 const { join } = require('path');
 const states = require('../../../states');
@@ -17,7 +17,6 @@ const { BlockletBackup } = require('./blocklet');
 const { BlockletExtrasBackup } = require('./blocklet-extras');
 const { BlockletsBackup } = require('./blocklets');
 const { DataBackup } = require('./data');
-const { LogsBackup } = require('./logs');
 const { RoutingRuleBackup } = require('./routing-rule');
 
 class SpacesBackup {
@@ -78,7 +77,6 @@ class SpacesBackup {
     this.input = input;
     this.storages = [
       new AuditLogBackup(this.input),
-      new LogsBackup(this.input),
       new BlockletBackup(this.input),
       new BlockletsBackup(this.input),
       new BlockletExtrasBackup(this.input),
@@ -107,7 +105,6 @@ class SpacesBackup {
     await this.initialize();
     await this.export();
     await this.syncToSpaces();
-    await this.destroy();
   }
 
   async initialize() {
@@ -121,9 +118,6 @@ class SpacesBackup {
     this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
 
     this.blockletBackupDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/backup', this.blocklet.meta.did);
-    if (existsSync(this.blockletBackupDir)) {
-      rmdirSync(this.blockletBackupDir, { recursive: true });
-    }
     ensureDirSync(this.blockletBackupDir);
 
     this.spacesEndpoint = this.blocklet.environments.find(
@@ -158,7 +152,8 @@ class SpacesBackup {
         source: join(this.blockletBackupDir, '/'),
         target: join('.did-objects', this.blocklet.appDid),
         debug: true,
-        retryCount: 3,
+        concurrency: 64,
+        retryCount: 100,
         filter: (object) => {
           return object.name !== '.DS_Store';
         },
@@ -176,12 +171,6 @@ class SpacesBackup {
     const { wallet } = getBlockletInfo(blockletInfo, nodeInfo.sk);
     return wallet;
   }
-
-  async destroy() {
-    if (existsSync(this.blockletBackupDir)) {
-      removeSync(this.blockletBackupDir);
-    }
-  }
 }
 
 module.exports = {

package/lib/blocklet/storage/restore/blocklet-extras.js

@@ -1,5 +1,5 @@
 const { removeSync, outputJsonSync, readJSONSync } = require('fs-extra');
-const { cloneDeep } = require('lodash');
+const { cloneDeep, isArray } = require('lodash');
 const { join } = require('path');
 const security = require('@abtnode/util/lib/security');
 const { BaseRestore } = require('./base');
@@ -61,6 +61,13 @@ class BlockletExtrasRestore extends BaseRestore {
    * @memberof BlockletExtrasRestore
    */
   useBlockletDecryptConfigs(configs) {
+    // eslint-disable-next-line no-console
+    console.log('debug', configs, !configs || !isArray(configs));
+
+    if (!configs || !isArray(configs)) {
+      return;
+    }
+
     for (const config of configs) {
       // secure 为 true 的配置才需要被加密保存上次到 did spaces
       if (config.secure) {
@@ -76,8 +83,6 @@ class BlockletExtrasRestore extends BaseRestore {
         config.value = decryptByBlocklet;
       }
     }
-
-    return configs;
   }
 }
 

package/lib/blocklet/storage/restore/blocklets.js

@@ -1,23 +1,41 @@
-const { existsSync, removeSync } = require('fs-extra');
+const { existsSync, remove } = require('fs-extra');
 const { join } = require('path');
-const StreamZip = require('node-stream-zip');
+const fg = require('fast-glob');
 const { BaseRestore } = require('./base');
+const { zipToDir } = require('../utils/zip');
 
 class BlockletsRestore extends BaseRestore {
-  filename = 'blocklets.zip';
+  filename = 'blocklets';
 
   async import() {
-    const blockletZipPath = join(this.blockletRestoreDir, this.filename);
+    const blockletsDir = join(this.blockletRestoreDir, this.filename);
 
-    if (!existsSync(blockletZipPath)) {
-      throw new Error(`file not found: ${blockletZipPath}`);
+    if (!existsSync(blockletsDir)) {
+      throw new Error(`dir not found: ${blockletsDir}`);
     }
 
-    // eslint-disable-next-line new-cap
-    const zip = new StreamZip.async({ file: blockletZipPath });
-    await zip.extract(null, join(this.blockletRestoreDir, 'blocklets'));
-    await zip.close();
-    removeSync(blockletZipPath);
+    const paths = await fg('**/*.zip', {
+      cwd: blockletsDir,
+      onlyFiles: true,
+      absolute: true,
+    });
+
+    /**
+     * @type {{source: string, target: string}}
+     */
+    const zipConfigs = paths.map((path) => {
+      return {
+        source: path,
+        target: path.replace(/.zip$/, ''),
+      };
+    });
+
+    await Promise.all(
+      zipConfigs.map(async (zipConfig) => {
+        await zipToDir(zipConfig.source, zipConfig.target);
+        await remove(zipConfig.source);
+      })
+    );
   }
 }
 

package/lib/blocklet/storage/restore/logs.js

@@ -0,0 +1,21 @@
+const { removeSync, existsSync } = require('fs-extra');
+const { join } = require('path');
+const { BaseRestore } = require('./base');
+const { zipToDir } = require('../utils/zip');
+
+class LogsRestore extends BaseRestore {
+  filename = 'logs.zip';
+
+  async import() {
+    const blockletZipPath = join(this.blockletRestoreDir, this.filename);
+
+    if (!existsSync(blockletZipPath)) {
+      throw new Error(`file not found: ${blockletZipPath}`);
+    }
+
+    await zipToDir(blockletZipPath, join(this.blockletRestoreDir, 'logs'));
+    removeSync(blockletZipPath);
+  }
+}
+
+module.exports = { LogsRestore };

package/lib/blocklet/storage/restore/spaces.js

@@ -74,6 +74,10 @@ class SpacesRestore {
     this.serverDataDir = process.env.ABT_NODE_DATA_DIR;
     this.blockletWallet = await this.getBlockletWallet();
 
+    if (!this.input.endpoint.includes(this.blockletWallet.address)) {
+      throw new Error(`endpoint and blocklet.appDid(${this.blockletWallet.address}) do not match`);
+    }
+
     this.blockletRestoreDir = join(process.env.ABT_NODE_DATA_DIR, 'tmp/restore', this.blockletWallet.address);
     if (existsSync(this.blockletRestoreDir)) {
       rmdirSync(this.blockletRestoreDir, { recursive: true });
@@ -113,7 +117,8 @@ class SpacesRestore {
         source: join('.did-objects', this.blockletWallet.address, '/'),
         target: this.blockletRestoreDir,
         debug: true,
-        retryCount: 3,
+        concurrency: 64,
+        retryCount: 100,
       })
     );
 

package/lib/blocklet/storage/utils/hash.js

@@ -0,0 +1,51 @@
+const { removeSync, existsSync, move, createReadStream } = require('fs-extra');
+const hasha = require('hasha');
+
+/**
+ *
+ *
+ * @param {NodeJS.ReadableStream} stream1
+ * @param {NodeJS.ReadableStream} stream2
+ * @return {Promise<boolean>}
+ */
+async function compareHash(stream1, stream2) {
+  const hash1 = await hasha.fromStream(stream1, {
+    algorithm: 'md5',
+  });
+  const hash2 = await hasha.fromStream(stream2, {
+    algorithm: 'md5',
+  });
+
+  return hash1 === hash2;
+}
+
+/**
+ *
+ * @description 比对新文件和旧文件的hash，旧文件不存在或者hash不同时，使用新文件替换，hash相同，删除新文件
+ * @param {string} oldFilePath
+ * @param {string} newFilePath
+ * @returns {Promise<boolean>}
+ */
+async function compareAndMove(oldFilePath, newFilePath) {
+  if (!existsSync(oldFilePath)) {
+    await move(newFilePath, oldFilePath, { overwrite: true });
+    return;
+  }
+
+  if (!existsSync(newFilePath)) {
+    throw new Error(`newFilePath(${newFilePath}) not found`);
+  }
+
+  const isSame = await compareHash(createReadStream(oldFilePath), createReadStream(newFilePath));
+  if (isSame) {
+    removeSync(newFilePath);
+    return;
+  }
+
+  await move(newFilePath, oldFilePath, { overwrite: true });
+}
+
+module.exports = {
+  compareHash,
+  compareAndMove,
+};

package/lib/blocklet/storage/utils/zip.js

@@ -0,0 +1,43 @@
+const archiver = require('archiver');
+const { ensureDirSync, existsSync, removeSync, createWriteStream } = require('fs-extra');
+const { dirname } = require('path');
+const StreamZip = require('node-stream-zip');
+
+/**
+ *
+ *
+ * @param {string} source ~/abc/ 通常是一个文件夹
+ * @param {string} target abc.zip 压缩包的名称
+ * @return {*}
+ */
+function dirToZip(source, target) {
+  return new Promise((resolve, reject) => {
+    ensureDirSync(dirname(target));
+
+    if (existsSync(target)) {
+      removeSync(target);
+    }
+
+    const output = createWriteStream(target);
+    const archive = archiver('zip', { level: 9 });
+    archive.on('error', (err) => reject(err));
+    output.on('close', () => resolve());
+
+    archive.directory(source, false);
+
+    archive.pipe(output);
+    archive.finalize();
+  });
+}
+
+async function zipToDir(source, target) {
+  // eslint-disable-next-line new-cap
+  const zip = new StreamZip.async({ file: source });
+  await zip.extract(null, target);
+  await zip.close();
+}
+
+module.exports = {
+  dirToZip,
+  zipToDir,
+};

package/lib/router/helper.js

@@ -7,7 +7,7 @@ const get = require('lodash/get');
 const cloneDeep = require('lodash/cloneDeep');
 const isEqual = require('lodash/isEqual');
 const joinUrl = require('url-join');
-const { replaceSlotToIp } = require('@blocklet/meta/lib/util');
+const { replaceSlotToIp, findComponentById, findWebInterface } = require('@blocklet/meta/lib/util');
 const { getProvider } = require('@abtnode/router-provider');
 const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
 const getTmpDir = require('@abtnode/util/lib/get-tmp-directory');
@@ -41,9 +41,11 @@ const {
 const {
   BLOCKLET_DYNAMIC_PATH_PREFIX,
   BLOCKLET_INTERFACE_TYPE_WEB,
+  BLOCKLET_INTERFACE_PUBLIC,
   BLOCKLET_INTERFACE_WELLKNOWN,
   BLOCKLET_INTERFACE_TYPE_WELLKNOWN,
   BlockletEvents,
+  BLOCKLET_MODES,
 } = require('@blocklet/constant');
 
 // eslint-disable-next-line global-require
@@ -206,6 +208,9 @@ const ensureLatestInterfaceInfo = async (sites = []) => {
     }
 
     site.rules = site.rules.map((rule) => {
+      if (rule.dynamic) {
+        return rule;
+      }
       // If a rule already has a target and target is WELLKNOWN_SERVICE_PATH_PREFIX, just return
       // Indicates that the rule id generated by the system for auth service
       if (rule.isProtected && rule.to.target === WELLKNOWN_SERVICE_PATH_PREFIX) {
@@ -279,6 +284,7 @@ const ensureWellknownRule = async (sites) => {
           rule.from.pathPrefix = servicePathPrefix;
           rule.to.target = servicePathPrefix;
           rule.isProtected = true;
+          rule.dynamic = true; // mark as dynamic to avoid redundant generated rules
           site.rules.push(rule);
         }
 
@@ -290,6 +296,7 @@ const ensureWellknownRule = async (sites) => {
           rule.to.cacheGroup = 'blockletProxy';
           rule.to.target = avatarPathPrefix;
           rule.isProtected = true;
+          rule.dynamic = true; // mark as dynamic to avoid redundant generated rules
           site.rules.push(rule);
         }
       }
@@ -312,7 +319,7 @@ const ensureBlockletDid = async (sites) => {
       return site;
     }
 
-    site.blockletDid = site.domain.replace(BLOCKLET_SITE_GROUP_SUFFIX, '');
+    site.blockletDid = getDidFromDomainGroupName(site.domain);
 
     return site;
   });
@@ -329,30 +336,76 @@ const ensureCorsForWebWallet = async (sites) => {
   return sites;
 };
 
-const filterSitesForRemovedBlocklets = async (sites = []) => {
-  const blocklets = await states.blocklet.getBlocklets();
+const filterSitesForRemovedBlocklets = async (sites = [], blocklets) => {
   return sites.filter((site) => {
     if (!site.domain.endsWith(BLOCKLET_SITE_GROUP_SUFFIX)) {
       return true;
     }
 
-    const did = getDidFromDomainGroupName(site.domain);
-    const blocklet = blocklets.find((x) => x.meta.did === did);
-    if (blocklet) {
-      site.mode = blocklet.mode;
-    }
-
+    const blocklet = blocklets.find((x) => x.meta.did === site.blockletDid);
     return !!blocklet;
   });
 };
 
+const ensureBlockletCache = async (sites = [], blocklets) => {
+  return sites
+    .map((site) => {
+      if (!site.domain.endsWith(BLOCKLET_SITE_GROUP_SUFFIX)) {
+        return site;
+      }
+
+      if (site.cacheableGenerated) {
+        return site;
+      }
+
+      // For each rule, get component, check cacheable, clone and push new rule
+      const blocklet = blocklets.find((x) => x.meta.did === site.blockletDid);
+      const cacheRules = [];
+      site.rules
+        .filter(
+          (x) =>
+            x.to.type === ROUTING_RULE_TYPES.BLOCKLET &&
+            x.to.interfaceName === BLOCKLET_INTERFACE_PUBLIC &&
+            x.from.pathPrefix.startsWith(WELLKNOWN_SERVICE_PATH_PREFIX) === false
+        )
+        .forEach((rule) => {
+          const component = findComponentById(blocklet, rule.to.componentId);
+          if (component.mode !== BLOCKLET_MODES.PRODUCTION) {
+            return;
+          }
+          const cacheable = get(findWebInterface(component), 'cacheable', []);
+          cacheable.forEach((cachePrefix) => {
+            const clone = cloneDeep(rule);
+            clone.from.pathPrefix = joinUrl(rule.from.pathPrefix, cachePrefix);
+            clone.to.cacheGroup = 'blockletProxy';
+            clone.to.target = cachePrefix;
+            clone.dynamic = true; // mark as dynamic to avoid redundant generated rules
+            cacheRules.push(clone);
+          });
+        });
+
+      site.rules = site.rules.concat(cacheRules);
+      site.mode = blocklet.mode;
+      site.cacheableGenerated = true;
+
+      return site;
+    })
+    .filter(Boolean);
+};
+
 const ensureLatestInfo = async (sites = [], { withDefaultCors = true } = {}) => {
+  const blocklets = await states.blocklet.getBlocklets();
+
+  // CAUTION: following steps are very important, please do not change the order
   let result = await ensureLatestNodeInfo(sites, { withDefaultCors });
   result = await ensureBlockletDid(result);
+  result = await filterSitesForRemovedBlocklets(sites, blocklets);
+  result = await ensureBlockletCache(result, blocklets);
   result = await ensureWellknownRule(result);
   result = await ensureCorsForWebWallet(result);
+  result = await ensureLatestInterfaceInfo(result);
 
-  return ensureLatestInterfaceInfo(result);
+  return result;
 };
 
 const decompressCertificates = async (source, dest) => {
@@ -1064,7 +1117,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
           try {
             const info = await nodeState.read();
             let { sites } = await readRoutingSites();
-            sites = await filterSitesForRemovedBlocklets(sites);
             sites = await ensureLatestInfo(sites);
             sites = await ensureServiceRule(sites);
             sites = await ensureRootRule(sites);

package/lib/router/index.js

@@ -192,6 +192,10 @@ Router.formatSites = (sites = []) => {
           return;
         }
 
+        if (rule.dynamic) {
+          return;
+        }
+
         if (daemonRule) {
           // Serve meta js: both prefix and suffix do not contain trailing slash
           // NOTICE: 这里隐含了一个约定

package/lib/states/blocklet.js

@@ -92,7 +92,7 @@ class BlockletState extends BaseState {
     this.defaultPort = config.blockletPort || 5555;
   }
 
-  getBlocklet(did) {
+  getBlocklet(did, { decryptSk = true } = {}) {
     return new Promise((resolve, reject) => {
       if (!did) {
         resolve(null);
@@ -103,7 +103,7 @@ class BlockletState extends BaseState {
           return reject(err);
         }
 
-        return resolve(doc ? formatBlocklet(doc, 'onRead', this.config.dek) : null);
+        return resolve(doc ? formatBlocklet(doc, 'onRead', decryptSk ? this.config.dek : null) : null);
       });
     });
   }

package/lib/util/blocklet.js

@@ -931,7 +931,7 @@ const verifyIntegrity = async ({ file, integrity: expected }) => {
 
 /**
  * @param {string} installDir
- * @returns {Array<{ key: <[scope/]name/version>, dir: appDir }>}
+ * @returns {Promise<Array<{ key: string, dir: string }>>} key is <[scope/]name/version>, dir is appDir
  */
 const getAppDirs = async (installDir) => {
   const appDirs = [];