@abtnode/core 1.15.17 → 1.16.0-beta-b16cb035

package/lib/blocklet/migration.js

@@ -1,68 +1,27 @@
 /* eslint-disable no-await-in-loop */
-const childProcess = require('child_process');
 const fs = require('fs-extra');
 const path = require('path');
 const semver = require('semver');
+const runScript = require('@abtnode/util/lib/run-script');
 
 const { getMigrationScripts: getScripts } = require('../migrations');
 const { getSafeEnv } = require('../util');
 const { name } = require('../../package.json');
 const logger = require('@abtnode/logger')(`${name}:blocklet:migration`); // eslint-disable-line
 
-const _runScript = ({ appDir, env, migrationScript, progress = false }) => {
-  const safeEnv = getSafeEnv(env);
-
-  const child = childProcess.exec(`node ${migrationScript}`, {
-    cwd: appDir,
-    env: safeEnv,
-    stdio: 'inherit',
-  });
-  let hasUnhandledRejection = false;
-
-  if (progress) {
-    child.stdout.pipe(process.stdout);
-    child.stderr.pipe(process.stderr);
-  }
-
-  return new Promise((resolve, reject) => {
-    const errorMessages = [];
-
-    child.stderr.on('data', (err) => {
-      // Check if has unhandledRejection in childProcess
-      // https://stackoverflow.com/questions/32784649/gracefully-handle-errors-in-child-processes-in-nodejs
-      if (err.includes('UnhandledPromiseRejectionWarning')) {
-        hasUnhandledRejection = true;
-      }
-      errorMessages.push(err);
-    });
-
-    child.on('exit', (code) => {
-      if (errorMessages.length > 0) {
-        if (code !== 0 || hasUnhandledRejection) {
-          return reject(new Error(errorMessages.join('\r\n')));
-        }
-
-        if (!progress) {
-          errorMessages.forEach((message) => process.stderr.write(message));
-        }
-      }
-
-      return resolve();
-    });
-  });
-};
-
 async function runScripts({
+  blocklet,
+  appDir,
+  env,
+  oldVersion,
   dbDir,
   backupDir,
   scriptsDir,
   printInfo,
   printSuccess,
   printError,
-  appDir,
-  env,
-  oldVersion,
 }) {
+  // 获取所有待执行的脚本
   let scripts = [];
   try {
     scripts = await getScripts(scriptsDir);
@@ -79,25 +38,36 @@ async function runScripts({
   }
 
   printInfo('pending scripts', pendingScripts);
+  // 执行脚本前需要对数据库文件做一次备份
   try {
     await doBackup({ dbDir, backupDir, printInfo, printSuccess, printError });
   } catch (err) {
     printError(`Failed to backup state db due to ${err.message}, abort!`);
+    // 备份失败时，需要移除备份目录
+    fs.removeSync(backupDir);
     throw err;
   }
 
+  // 按照顺序依次执行 migration 脚本
   for (let i = 0; i < pendingScripts.length; i++) {
     const { script: scriptPath } = pendingScripts[i];
     try {
       printInfo(`Migration script started: ${scriptPath}`);
-      await _runScript({ appDir, env, migrationScript: path.join(scriptsDir, scriptPath) });
+      await runScript(`node ${path.join(scriptsDir, scriptPath)}`, [blocklet.env.processId, 'migration'].join(':'), {
+        cwd: appDir,
+        env: getSafeEnv(env),
+        silent: false,
+      });
       printInfo(`Migration script executed: ${scriptPath}`);
     } catch (migrationErr) {
       printError(`Failed to execute migration script: ${scriptPath}, error: ${migrationErr.message}`);
 
       try {
         await doRestore({ dbDir, backupDir, printInfo, printSuccess, printError });
+        // 恢复备份成功时，需要移除备份目录
+        fs.removeSync(backupDir);
       } catch (restoreErr) {
+        // 如果恢复备份失败，则保留备份目录，恢复数据
         printError(`Failed to restore state db due to: ${restoreErr.message}`);
       }
 
@@ -123,6 +93,7 @@ async function doRestore({ dbDir, backupDir, printInfo, printSuccess }) {
 }
 
 module.exports = async ({
+  blocklet,
   appDir,
   env,
   oldVersion,
@@ -131,6 +102,10 @@ module.exports = async ({
   printSuccess = logger.info,
   printError = logger.error,
 }) => {
+  if (!oldVersion) {
+    return;
+  }
+
   const baseDir = env.BLOCKLET_DATA_DIR;
 
   const scriptsDir = path.join(appDir, 'migration');
@@ -142,15 +117,18 @@ module.exports = async ({
   fs.ensureDirSync(backupDir);
 
   await runScripts({
+    blocklet,
+    appDir,
+    env,
+    oldVersion,
+    newVersion,
     dbDir,
     backupDir,
     scriptsDir,
     printError,
     printInfo,
     printSuccess,
-    appDir,
-    env,
-    oldVersion,
-    newVersion,
   });
+
+  fs.removeSync(backupDir);
 };

package/lib/blocklet/storage/backup/audit-log.js

@@ -0,0 +1,27 @@
+const { removeSync, outputJsonSync } = require('fs-extra');
+const { join } = require('path');
+const states = require('../../../states');
+const { BaseBackup } = require('./base');
+
+/**
+ *
+ *
+ * @class AuditLogBackup
+ */
+class AuditLogBackup extends BaseBackup {
+  filename = 'audit-log.json';
+
+  async export() {
+    /**
+     * @type {import('@abtnode/client').AuditLog}
+     */
+    const auditLogs = await states.auditLog.find({
+      scope: this.blocklet.meta.did,
+    });
+
+    removeSync(join(this.backupDir, this.filename));
+    outputJsonSync(join(this.backupDir, this.filename), auditLogs);
+  }
+}
+
+module.exports = { AuditLogBackup };

package/lib/blocklet/storage/backup/base.js

@@ -0,0 +1,62 @@
+class BaseBackup {
+  /**
+   * @type {import('./spaces').SpaceBackupInput}
+   * @memberof BaseBackup
+   */
+  input;
+
+  /**
+   * @description blocklet state 对象
+   * @type {import('@abtnode/client').BlockletState}
+   * @memberof BaseBackup
+   */
+  blocklet;
+
+  /**
+   * @description 当前 blocklet 的数据目录
+   * @type {string}
+   * @memberof BaseBackup
+   */
+  backupDir;
+
+  /**
+   *
+   * @description spaces 的 endpoint
+   * @type {import('./spaces').SecurityContext}
+   * @memberof BaseBackup
+   */
+  securityContext;
+
+  /**
+   *
+   * @description server 的数据目录
+   * @type {string}
+   * @memberof BaseBackup
+   */
+  serverDir;
+
+  constructor(input) {
+    this.input = input;
+  }
+
+  /**
+   *
+   *
+   * @param {import('./spaces').SpacesBackup} spacesBackup
+   * @memberof BaseBackup
+   */
+  ensureParams(spacesBackup) {
+    this.blocklet = spacesBackup.blocklet;
+    this.serverDir = spacesBackup.serverDir;
+    this.backupDir = spacesBackup.backupDir;
+    this.securityContext = spacesBackup.securityContext;
+  }
+
+  async export() {
+    throw new Error('not implemented');
+  }
+}
+
+module.exports = {
+  BaseBackup,
+};

package/lib/blocklet/storage/backup/blocklet-extras.js

@@ -0,0 +1,92 @@
+const { removeSync, readFileSync, outputJsonSync } = require('fs-extra');
+const { isEmpty, cloneDeep } = require('lodash');
+const { join } = require('path');
+const security = require('@abtnode/util/lib/security');
+
+const states = require('../../../states');
+const { BaseBackup } = require('./base');
+
+class BlockletExtrasBackup extends BaseBackup {
+  filename = 'blocklet-extras.json';
+
+  async export() {
+    const blockletExtra = await this.getBlockletExtra();
+    removeSync(join(this.backupDir, this.filename));
+    outputJsonSync(join(this.backupDir, this.filename), blockletExtra);
+  }
+
+  /**
+   *
+   * @description
+   * @return {Promise<import('@abtnode/client').BlockletState>}
+   * @memberof BlockletExtrasBackup
+   */
+  async getBlockletExtra() {
+    /**
+     * @type {import('@abtnode/client').BlockletState}
+     */
+    const blockletExtra = await states.blockletExtras.findOne({
+      did: this.blocklet.meta.did,
+    });
+    if (isEmpty(blockletExtra)) {
+      throw new Error('blockletExtra cannot be empty');
+    }
+
+    return this.cleanData(blockletExtra);
+  }
+
+  /**
+   *
+   * @description 清理数据并加密
+   * @param {import('@abtnode/client').BlockletState} raw
+   * @return {Promise<any>}
+   * @memberof BlockletExtrasBackup
+   */
+  async cleanData(raw) {
+    const blockletExtra = cloneDeep(raw);
+
+    const queue = [blockletExtra];
+    while (queue.length) {
+      const current = queue.pop();
+
+      // 删除父 blocklet 的某些数据
+      if (current._id) {
+        delete current._id;
+        delete current.createdAt;
+        delete current.updatedAt;
+      }
+
+      // 加解密
+      this.encrypt(current.configs);
+
+      if (current?.children) {
+        queue.push(...current.children);
+      }
+    }
+
+    return blockletExtra;
+  }
+
+  /**
+   *
+   * @description 清理数据并加密
+   * @param {import('@abtnode/client').ConfigEntry[]} configs
+   * @return {void}
+   * @memberof BlockletExtrasBackup
+   */
+  encrypt(configs) {
+    if (isEmpty(configs)) {
+      return;
+    }
+
+    const dk = readFileSync(join(this.serverDir, '.sock'));
+    for (const config of configs) {
+      if (config.secure) {
+        const decrypted = security.decrypt(config.value, this.blocklet.meta.did, dk);
+        config.value = this.securityContext.encrypt(decrypted);
+      }
+    }
+  }
+}
+
+module.exports = { BlockletExtrasBackup };

package/lib/blocklet/storage/backup/blocklet.js

@@ -0,0 +1,70 @@
+const { removeSync, outputJsonSync } = require('fs-extra');
+const { cloneDeep } = require('lodash');
+const { join } = require('path');
+const { BaseBackup } = require('./base');
+
+class BlockletBackup extends BaseBackup {
+  filename = 'blocklet.json';
+
+  async export() {
+    const blocklet = await this.cleanData();
+    removeSync(join(this.backupDir, this.filename));
+    outputJsonSync(join(this.backupDir, this.filename), blocklet);
+  }
+
+  /**
+   * @description 清理数据
+   * @see blocklet.db 中需要删除哪些字段呢？ https://github.com/ArcBlock/blocklet-server/issues/6120#issuecomment-1383798348
+   * @return {Promise<import('@abtnode/client').BlockletState>}
+   * @memberof BlockletBackup
+   */
+  async cleanData() {
+    const clone = cloneDeep(this.blocklet);
+
+    /** @type {import('@abtnode/client').ComponentState[]} */
+    const queue = [clone];
+
+    // 广度优先遍历
+    while (queue.length) {
+      const current = queue.pop();
+
+      // 父组件才需要删除的属性
+      if (current._id) {
+        delete current._id;
+        delete current.createdAt;
+        delete current.startedAt;
+        delete current.installedAt;
+      }
+
+      // 子组件和父组件都需要删除的属性
+      delete current.status;
+      delete current.ports;
+      delete current.environments;
+
+      if (current.children) {
+        queue.push(...current.children);
+      }
+    }
+
+    return this.encrypt(clone);
+  }
+
+  /**
+   *
+   * @description 清理数据并加密
+   * @param {import('@abtnode/client').BlockletState} info
+   * @memberof BlockletExtrasBackup
+   */
+  encrypt(info) {
+    if (Array.isArray(info.migratedFrom)) {
+      info.migratedFrom = info.migratedFrom.map((x) => {
+        x.appSk = this.securityContext.encrypt(x.appSk);
+        return x;
+      });
+    }
+
+    return info;
+  }
+}
+
+module.exports = { BlockletBackup };

package/lib/blocklet/storage/backup/blocklets.js

@@ -0,0 +1,74 @@
+const { join } = require('path');
+const validUrl = require('valid-url');
+const { BaseBackup } = require('./base');
+const { dirToZip } = require('../utils/zip');
+const { compareAndMove } = require('../utils/hash');
+
+class BlockletsBackup extends BaseBackup {
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof BlockletsBackup
+   */
+  async export() {
+    const blockletMetas = this.getBlockletMetas(this.blocklet);
+    const serverBlockletsDir = join(this.serverDir, 'blocklets');
+
+    const blockletMetasFromLocal = blockletMetas.filter(
+      (b) => !validUrl.isHttpUri(b.tarball) && !validUrl.isHttpsUri(b.tarball)
+    );
+
+    const dirs = [];
+    for (const blockletMeta of blockletMetasFromLocal) {
+      const sourceDir = join(serverBlockletsDir, blockletMeta.name, blockletMeta.version);
+      const zipPath = join(this.backupDir, 'blocklets', blockletMeta.name, `${blockletMeta.version}.zip`);
+      dirs.push({ sourceDir, zipPath });
+    }
+
+    await this.dirsToZip(dirs);
+  }
+
+  /**
+   *
+   *
+   * @param {import('@abtnode/client').BlockletState} blocklet
+   * @returns {Array<{name: string, version: string, tarball: string}>}
+   * @memberof BlockletsBackup
+   */
+  getBlockletMetas(blocklet) {
+    if (!blocklet?.meta?.bundleName || !blocklet?.meta?.version) {
+      return [];
+    }
+
+    const metas = [];
+    metas.push({
+      name: blocklet.meta.bundleName,
+      version: blocklet.meta.version,
+      tarball: blocklet?.meta?.dist?.tarball,
+    });
+
+    for (const child of blocklet.children) {
+      metas.push(...this.getBlockletMetas(child));
+    }
+
+    return metas;
+  }
+
+  /**
+   * @param {Array<{sourceDir: string, zipPath: string}>} dirs: /some/folder/to/compress
+   * @param {String} zipPath: /path/to/created.zip
+   * @returns {Promise}
+   * @memberof BlockletsBackup
+   */
+  async dirsToZip(dirs) {
+    await Promise.all(
+      dirs.map(async (dir) => {
+        const tempZipPath = `${dir.zipPath}.bak`;
+        await dirToZip(dir.sourceDir, tempZipPath);
+        await compareAndMove(dir.zipPath, tempZipPath);
+      })
+    );
+  }
+}
+
+module.exports = { BlockletsBackup };

package/lib/blocklet/storage/backup/data.js

@@ -0,0 +1,19 @@
+const { copy } = require('fs-extra');
+const { join } = require('path');
+const { BaseBackup } = require('./base');
+
+class DataBackup extends BaseBackup {
+  /**
+   *
+   * @returns {Promise<void>}
+   * @memberof BlockletsBackup
+   */
+  async export() {
+    const blockletDataDir = join(this.serverDir, 'data', this.blocklet.meta.name);
+    const blockletBackupDataDir = join(this.backupDir, 'data');
+
+    await copy(blockletDataDir, blockletBackupDataDir, { overwrite: true });
+  }
+}
+
+module.exports = { DataBackup };

package/lib/blocklet/storage/backup/logs.js

@@ -0,0 +1,24 @@
+const { ensureDirSync, copy, statSync } = require('fs-extra');
+const { join } = require('path');
+const { BaseBackup } = require('./base');
+
+// note: 可以不需要备份
+class LogsBackup extends BaseBackup {
+  async export() {
+    const sourceLogsDir = join(this.serverDir, 'logs', this.blocklet.meta.name);
+    ensureDirSync(sourceLogsDir);
+
+    const targetLogsDir = join(this.backupDir, 'logs');
+
+    await copy(sourceLogsDir, targetLogsDir, {
+      overwrite: true,
+      filter: (src) => {
+        return !statSync(src).isSymbolicLink();
+      },
+    });
+  }
+}
+
+module.exports = {
+  LogsBackup,
+};

package/lib/blocklet/storage/backup/routing-rule.js

@@ -0,0 +1,19 @@
+const { removeSync, outputJsonSync } = require('fs-extra');
+const { join } = require('path');
+const states = require('../../../states');
+const { BaseBackup } = require('./base');
+
+class RoutingRuleBackup extends BaseBackup {
+  filename = 'routing_rule.json';
+
+  async export() {
+    const routingRule = await states.site.findOne({
+      domain: `${this.blocklet.meta.did}.blocklet-domain-group`,
+    });
+
+    removeSync(join(this.backupDir, this.filename));
+    outputJsonSync(join(this.backupDir, this.filename), routingRule);
+  }
+}
+
+module.exports = { RoutingRuleBackup };