@abtnode/core 1.16.20-beta-e363262e → 1.16.20-beta-bb1cd034

package/lib/blocklet/manager/disk.js

@@ -15,6 +15,7 @@ const { sign } = require('@arcblock/jwt');
 const sleep = require('@abtnode/util/lib/sleep');
 const getBlockletInfo = require('@blocklet/meta/lib/info');
 const joinUrl = require('url-join');
+const { sendToUser } = require('@blocklet/sdk/lib/util/send-notification');
 
 const logger = require('@abtnode/logger')('@abtnode/core:blocklet:manager');
 const {
@@ -94,6 +95,7 @@ const pRetry = require('p-retry');
 
 const isFunction = require('lodash/isFunction');
 const { encode } = require('@abtnode/util/lib/base32');
+const formatContext = require('@abtnode/util/lib/format-context');
 const { consumeServerlessNFT, consumeLauncherSession } = require('../../util/launcher');
 const util = require('../../util');
 const {
@@ -190,6 +192,7 @@ const {
   updateSelectedResources,
 } = require('../project');
 const { callFederated } = require('../../util/federated');
+const { isDnsIpMappingCorrect } = require('../../util/ip');
 
 const { formatEnvironments, getBlockletMeta, validateOwner, isCLI } = util;
 
@@ -270,6 +273,44 @@ class DiskBlockletManager extends BaseBlockletManager {
     }
   }
 
+  async initialize() {
+    await this.ensureAutoBackupJobs();
+  }
+
+  async ensureAutoBackupJobs() {
+    const blocklets = await states.blockletExtras.find({
+      'settings.autoBackup.enabled': true,
+    });
+
+    const info = await states.node.read();
+
+    await Promise.all(
+      blocklets.map(async (x) => {
+        const { did } = x;
+        const jobId = getBackupJobId(did);
+        const job = await this.backupQueue.get(jobId);
+
+        if (job) {
+          return;
+        }
+
+        this.backupQueue.push(
+          {
+            entity: 'blocklet',
+            action: 'backupToSpaces',
+            did,
+            context: formatContext({
+              user: { did: info.did },
+            }),
+          },
+          jobId,
+          true,
+          BACKUPS.JOB.INTERVAL
+        );
+      })
+    );
+  }
+
   // ============================================================================================
   // Public API for Installing/Upgrading Application or Components
   // ============================================================================================
@@ -1395,6 +1436,29 @@ class DiskBlockletManager extends BaseBlockletManager {
     return newState;
   }
 
+  async sendEmail({ did, receiver, email }) {
+    const blocklet = await this.getBlocklet(did);
+    const nodeInfo = await states.node.read();
+    const { permanentWallet } = getBlockletInfo(blocklet, nodeInfo.sk);
+    // HACK: 因为发送方法是按照数组来处理的
+
+    const [result] = await sendToUser(
+      receiver,
+      JSON.parse(email),
+      {
+        appDid: permanentWallet.address,
+        appSk: permanentWallet.secretKey,
+      },
+      undefined,
+      undefined,
+      'send-to-mail'
+    );
+    if (result && result.status === 'fulfilled') {
+      return result.value;
+    }
+    throw new Error(result?.reason || 'failed to send email');
+  }
+
   async configNotification({ did, notification = {} }, context) {
     let newConfig = {};
     try {
@@ -2320,7 +2384,31 @@ class DiskBlockletManager extends BaseBlockletManager {
    * }} { blocklet, context }
    * @memberof BlockletManager
    */
-  async _onBackupToSpaces({ did, context, backupState }) {
+  async _onBackupToSpaces({
+    did,
+    context,
+    backupState = {
+      strategy: BACKUPS.STRATEGY.AUTO,
+    },
+  }) {
+    const blocklet = await states.blocklet.getBlocklet(did);
+    const {
+      appDid,
+      meta: { did: appPid },
+    } = blocklet;
+
+    if (backupState?.strategy === BACKUPS.STRATEGY.AUTO && blocklet.status === BlockletStatus.stopped) {
+      // 自动备份时,应用停止运行就跳过本次备份
+      logger.warn('Skip automatic backups because the application is not running.', { appPid });
+      return;
+    }
+
+    const correct = await isDnsIpMappingCorrect(blocklet.meta.did); // dns/ip 映射错误就跳过本次备份
+    if (!correct) {
+      logger.warn('Skipping automatic backups due to dns mapping errors', { appPid });
+      return;
+    }
+
     const {
       user: { did: userDid, locale },
     } = context;
@@ -2339,12 +2427,6 @@ class DiskBlockletManager extends BaseBlockletManager {
       );
     }
 
-    const blocklet = await states.blocklet.getBlocklet(did);
-    const {
-      appDid,
-      meta: { did: appPid },
-    } = blocklet;
-
     let backup = await states.backup.findOne({ appPid }, {}, { createdAt: -1 });
     if (backup?.status !== BACKUPS.STATUS.PROGRESS) {
       // 创建备份记录

package/lib/blocklet/storage/backup/base.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 /**
  * @typedef {{
  *  appDid: string
@@ -117,7 +118,7 @@ class BaseBackup {
 
   /**
    *
-   * @param {BaseBackup} dataBackup
+   * @param {import('./data').DataBackup} dataBackup
    * @param {Array<BaseBackup>} storages
    * @returns {Promise<SyncObject[]>}
    * @memberof BaseBackup

package/lib/blocklet/storage/backup/blocklets.js

@@ -1,5 +1,6 @@
 const { join } = require('path');
 const validUrl = require('valid-url');
+const pAll = require('p-all');
 const { BaseBackup } = require('./base');
 const { dirToZip } = require('../utils/zip');
 const { compareAndMove } = require('../utils/hash');
@@ -35,7 +36,6 @@ class BlockletsBackup extends BaseBackup {
       const zipPath = join(this.backupDir, 'blocklets', blockletMeta.name, `${blockletMeta.version}.zip`);
       dirs.push({ sourceDir, zipPath });
     }
-
     await this.dirsToZip(dirs);
 
     return {
@@ -76,12 +76,17 @@ class BlockletsBackup extends BaseBackup {
    * @memberof BlockletsBackup
    */
   async dirsToZip(dirs) {
-    await Promise.all(
-      dirs.map(async (dir) => {
-        const tempZipPath = `${dir.zipPath}.bak`;
-        await dirToZip(dir.sourceDir, tempZipPath);
-        await compareAndMove(dir.zipPath, tempZipPath);
-      })
+    await pAll(
+      dirs.map((dir) => {
+        return async () => {
+          const tempZipPath = `${dir.zipPath}.bak`;
+          await dirToZip(dir.sourceDir, tempZipPath);
+          await compareAndMove(dir.zipPath, tempZipPath);
+        };
+      }),
+      {
+        concurrency: 4,
+      }
     );
   }
 

package/lib/blocklet/storage/backup/data.js

@@ -3,10 +3,6 @@ const { join } = require('path');
 const { BaseBackup } = require('./base');
 const { getFolderObjects } = require('../utils/disk');
 
-/**
- * @typedef {import('./base').FileIncludeStats} FileIncludeStats
- */
-
 class DataBackup extends BaseBackup {
   /**
    *
@@ -25,15 +21,11 @@ class DataBackup extends BaseBackup {
    * @return {Promise<import('@did-space/core').Object[]>}
    * @memberof DataBackup
    */
-  collectSyncObjects() {
+  async collectSyncObjects() {
     const blockletDataDir = join(this.serverDir, 'data', this.blocklet.meta.name);
 
-    const objects = getFolderObjects(blockletDataDir).map((x) => {
-      return {
-        ...x,
-        key: join('/data', x.key),
-      };
-    });
+    // @note: 存储到 did-spaces 的应用的 /data 目录下
+    const objects = await getFolderObjects(blockletDataDir, '/data');
 
     return objects;
   }

package/lib/blocklet/storage/utils/disk.js

@@ -1,6 +1,11 @@
+/* eslint-disable no-console */
+
 const fs = require('fs-extra');
 const { join, basename } = require('path');
 const logger = require('@abtnode/logger')('@abtnode/core:storage:utils:disk');
+const FastGlob = require('fast-glob');
+const mapValues = require('lodash/mapValues');
+const xbytes = require('xbytes');
 
 const backupDirName = '_abtnode/backup';
 const restoreDirName = 'tmp/restore-disk';
@@ -94,38 +99,38 @@ function getFolderSize(folderPath) {
 /**
  * @description
  * @param {string} path
- * @return {import('@did-space/core').Object[]}
+ * @param {string} [prefix='']
+ * @return {Promise<Array<import('@did-space/core').Object>>}
  */
-function getFolderObjects(path) {
-  /**
-   * @type {import('@did-space/core').Object[]}
-   */
-  const objects = [];
-  const stack = [path];
-
-  while (stack.length) {
-    const absolutePath = stack.pop();
-    const stats = fs.statSync(absolutePath);
+// eslint-disable-next-line require-await
+async function getFolderObjects(path, prefix = '') {
+  const stream = FastGlob.stream('**', {
+    cwd: path,
+    objectMode: true,
+    stats: true,
+    onlyFiles: true,
+    absolute: true,
+    dot: true,
+    concurrency: 2,
+  });
 
-    if (stats.isFile()) {
-      const key = absolutePath.replace(path, '');
-
-      objects.push({
-        key,
-        name: basename(key),
-        isDir: false,
-        size: stats.size,
-        lastModified: new Date(stats.mtime).getTime(),
-        editable: true,
-        absolutePath,
-      });
-    } else if (stats.isDirectory()) {
-      const files = fs.readdirSync(absolutePath);
-      files.forEach((file) => {
-        const filePath = join(absolutePath, file);
-        stack.push(filePath);
-      });
-    }
+  const objects = [];
+  // eslint-disable-next-line no-unreachable-loop
+  for await (const element of stream) {
+    /**
+     * @type {import('fast-glob').Entry}
+     */
+    const entry = element;
+    const key = entry.path.replace(path, prefix);
+    const { stats } = entry;
+
+    objects.push({
+      key,
+      name: basename(key),
+      size: stats.size,
+      lastModified: new Date(stats.mtime).getTime(),
+      absolutePath: entry.path,
+    });
   }
 
   return objects;
@@ -157,6 +162,11 @@ function getFileObject(absolutePath, prefix = '') {
   };
 }
 
+function formatMemoryUsage() {
+  const memoryUsage = process.memoryUsage();
+  return mapValues(memoryUsage, (x) => xbytes(x));
+}
+
 module.exports = {
   getBackupList,
   removeBackup,
@@ -164,4 +174,5 @@ module.exports = {
   getFolderSize,
   getFolderObjects,
   getFileObject,
+  formatMemoryUsage,
 };

package/lib/index.js

@@ -278,6 +278,7 @@ function ABTNode(options) {
     syncFederated: blockletManager.syncFederated.bind(blockletManager),
     loginFederated: blockletManager.loginFederated.bind(blockletManager),
     configNotification: blockletManager.configNotification.bind(blockletManager),
+    sendEmail: blockletManager.sendEmail.bind(blockletManager),
     updateWhoCanAccess: blockletManager.updateWhoCanAccess.bind(blockletManager),
     updateAppSessionConfig: blockletManager.updateAppSessionConfig.bind(blockletManager),
     updateComponentTitle: blockletManager.updateComponentTitle.bind(blockletManager),
@@ -628,6 +629,15 @@ function ABTNode(options) {
           logger.info('Cron jobs start successfully on daemon start');
         }
       }, 1000);
+
+      blockletManager
+        .initialize()
+        .then(() => {
+          logger.info('blockletManager initialized');
+        })
+        .catch((error) => {
+          logger.error('blockletManager initialize failed', { error });
+        });
     }
   });
 

package/lib/states/index.js

@@ -69,6 +69,7 @@ const init = (dataDirs, config = {}) => {
  *  blockletExtras: import('./blocklet-extras'),
  *  notification: import('./notification'),
  *  job: import('./job'),
+ *  node: import('./node'),
  *  [key: string]: any
  * }}
  */

package/lib/states/node.js

@@ -58,7 +58,7 @@ class NodeState extends BaseState {
   /**
    * Ensure we have an node record in the database
    *
-   * @returns {object} Node document json
+   * @returns {Promise<import('@abtnode/client').NodeState>} Node document json
    * @memberof NodeState
    */
   async read() {

package/lib/util/ip.js

@@ -1,5 +1,12 @@
+const getIP = require('@abtnode/util/lib/get-ip');
 const getIp = require('@abtnode/util/lib/get-ip');
+const { encode } = require('@abtnode/util/lib/base32');
+const { DEFAULT_DID_DOMAIN } = require('@abtnode/constant');
 const logger = require('@abtnode/logger')('@abtnode/core:ip');
+const dns = require('dns');
+const { promisify } = require('util');
+
+const lookup = promisify(dns.lookup);
 
 let cache = null;
 const fetch = async () => {
@@ -23,6 +30,25 @@ const cron = {
   options: { runOnInit: true, runInService: true },
 };
 
+/**
+ * @description
+ * @param {string} did
+ * @return {Promise<boolean>}
+ */
+async function isDnsIpMappingCorrect(did) {
+  try {
+    const { internal, external } = await getIP();
+    const didDomain = `${encode(did)}.${DEFAULT_DID_DOMAIN}`;
+    const { address } = await lookup(didDomain, { rrtype: 'A' });
+
+    return internal === address || external === address;
+  } catch (error) {
+    logger.error(error);
+    return false;
+  }
+}
+
 module.exports.fetch = fetch;
 module.exports.get = get;
 module.exports.cron = cron;
+module.exports.isDnsIpMappingCorrect = isDnsIpMappingCorrect;

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.16.20-beta-e363262e",
+  "version": "1.16.20-beta-bb1cd034",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,39 +19,39 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "Apache-2.0",
   "dependencies": {
-    "@abtnode/analytics": "1.16.20-beta-e363262e",
-    "@abtnode/auth": "1.16.20-beta-e363262e",
-    "@abtnode/certificate-manager": "1.16.20-beta-e363262e",
-    "@abtnode/constant": "1.16.20-beta-e363262e",
-    "@abtnode/cron": "1.16.20-beta-e363262e",
-    "@abtnode/logger": "1.16.20-beta-e363262e",
-    "@abtnode/models": "1.16.20-beta-e363262e",
-    "@abtnode/queue": "1.16.20-beta-e363262e",
-    "@abtnode/rbac": "1.16.20-beta-e363262e",
-    "@abtnode/router-provider": "1.16.20-beta-e363262e",
-    "@abtnode/static-server": "1.16.20-beta-e363262e",
-    "@abtnode/timemachine": "1.16.20-beta-e363262e",
-    "@abtnode/util": "1.16.20-beta-e363262e",
-    "@arcblock/did": "1.18.103",
-    "@arcblock/did-auth": "1.18.103",
-    "@arcblock/did-ext": "^1.18.103",
+    "@abtnode/analytics": "1.16.20-beta-bb1cd034",
+    "@abtnode/auth": "1.16.20-beta-bb1cd034",
+    "@abtnode/certificate-manager": "1.16.20-beta-bb1cd034",
+    "@abtnode/constant": "1.16.20-beta-bb1cd034",
+    "@abtnode/cron": "1.16.20-beta-bb1cd034",
+    "@abtnode/logger": "1.16.20-beta-bb1cd034",
+    "@abtnode/models": "1.16.20-beta-bb1cd034",
+    "@abtnode/queue": "1.16.20-beta-bb1cd034",
+    "@abtnode/rbac": "1.16.20-beta-bb1cd034",
+    "@abtnode/router-provider": "1.16.20-beta-bb1cd034",
+    "@abtnode/static-server": "1.16.20-beta-bb1cd034",
+    "@abtnode/timemachine": "1.16.20-beta-bb1cd034",
+    "@abtnode/util": "1.16.20-beta-bb1cd034",
+    "@arcblock/did": "1.18.105",
+    "@arcblock/did-auth": "1.18.105",
+    "@arcblock/did-ext": "^1.18.105",
     "@arcblock/did-motif": "^1.1.13",
-    "@arcblock/did-util": "1.18.103",
-    "@arcblock/event-hub": "1.18.103",
-    "@arcblock/jwt": "^1.18.103",
+    "@arcblock/did-util": "1.18.105",
+    "@arcblock/event-hub": "1.18.105",
+    "@arcblock/jwt": "^1.18.105",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/validator": "^1.18.103",
-    "@arcblock/vc": "1.18.103",
-    "@blocklet/constant": "1.16.20-beta-e363262e",
-    "@blocklet/env": "1.16.20-beta-e363262e",
-    "@blocklet/meta": "1.16.20-beta-e363262e",
-    "@blocklet/resolver": "1.16.20-beta-e363262e",
-    "@blocklet/sdk": "1.16.20-beta-e363262e",
+    "@arcblock/validator": "^1.18.105",
+    "@arcblock/vc": "1.18.105",
+    "@blocklet/constant": "1.16.20-beta-bb1cd034",
+    "@blocklet/env": "1.16.20-beta-bb1cd034",
+    "@blocklet/meta": "1.16.20-beta-bb1cd034",
+    "@blocklet/resolver": "1.16.20-beta-bb1cd034",
+    "@blocklet/sdk": "1.16.20-beta-bb1cd034",
     "@did-space/client": "^0.3.41",
     "@fidm/x509": "^1.2.1",
-    "@ocap/mcrypto": "1.18.103",
-    "@ocap/util": "1.18.103",
-    "@ocap/wallet": "1.18.103",
+    "@ocap/mcrypto": "1.18.105",
+    "@ocap/util": "1.18.105",
+    "@ocap/wallet": "1.18.105",
     "@slack/webhook": "^5.0.4",
     "archiver": "^5.3.1",
     "axios": "^0.27.2",
@@ -76,6 +76,7 @@
     "lodash": "^4.17.21",
     "lru-cache": "^6.0.0",
     "node-stream-zip": "^1.15.0",
+    "p-all": "3.0.0",
     "p-limit": "^3.1.0",
     "p-retry": "4.6.1",
     "read-last-lines": "^1.8.0",
@@ -101,5 +102,5 @@
     "jest": "^27.5.1",
     "unzipper": "^0.10.11"
   },
-  "gitHead": "479cea04dd620d8194d6221b2594b6a0f390ba76"
+  "gitHead": "b000d880478911fbad7f5b197daf8a9065463ca8"
 }