@abtnode/core 1.8.65-beta-f7af64a4 → 1.8.65-beta-bfcc12ce

package/lib/blocklet/downloader/blocklet-downloader.js

@@ -32,7 +32,10 @@ const isMetaFileExist = (dir) => {
  * }}
  * @returns {boolean}
  */
-const needDownload = (component, { installDir, logger = defaultLogger, cachedBundles = [] } = {}) => {
+const needDownload = (
+  component,
+  { installDir, logger = defaultLogger, cachedBundles = [], skipCheckIntegrity } = {}
+) => {
   if (component.mode === BLOCKLET_MODES.DEVELOPMENT) {
     return false;
   }
@@ -62,10 +65,15 @@ const needDownload = (component, { installDir, logger = defaultLogger, cachedBun
     return false;
   }
 
+  if (skipCheckIntegrity) {
+    return false;
+  }
+
   // check integrity
 
   const cachedBundle = cachedBundles.find((x) => x.bundleId === bundleId);
 
+  // FIXME: 不是新安装的组件不应该 check, 不应该 block 安装成功 https://github.com/blocklet/launcher/actions/runs/4184577090/jobs/7250416272
   if (!cachedBundle) {
     return true;
   }
@@ -84,10 +92,12 @@ const needDownload = (component, { installDir, logger = defaultLogger, cachedBun
 class BlockletDownloader extends EventEmitter {
   /**
    * @param {{
+   *   installDir: string,
+   *   downloadDir: string,
    *   cache: {
    *     set: (key: string, value: any) => Promise
    *     get: (key: string) => Promise<any>
-   *   }
+   *   },
    * }}
    */
   constructor({ installDir, downloadDir, cache, logger = defaultLogger }) {
@@ -106,20 +116,25 @@ class BlockletDownloader extends EventEmitter {
 
   /**
    * @param {{}} blocklet
-   * @param {{}} [context={}]
+   * @param {{
+   *   preDownload: ({ downloadList: Array<meta>, downloadComponentIds: Array<string> }) => any
+   *   postDownload: ({ downloadList: Array<meta>, downloadComponentIds: Array<string>, isCancelled: boolean }) => any
+   *   skipCheckIntegrity: boolean
+   * }} [options={}]
    * @return {*}
    */
-  async download(blocklet, context = {}) {
+  async download(blocklet, options = {}) {
     const {
       meta: { name, did },
     } = blocklet;
 
     this.logger.info('Download Blocklet', { name, did });
 
-    const { preDownload = () => {}, postDownload = () => {} } = context;
+    const { preDownload = () => {}, postDownload = () => {}, skipCheckIntegrity } = options;
 
     const { downloadComponentIds, downloadList } = await this.getDownloadList({
       blocklet,
+      skipCheckIntegrity,
     });
 
     await preDownload({ downloadList, downloadComponentIds });
@@ -133,13 +148,15 @@ class BlockletDownloader extends EventEmitter {
       const tasks = [];
       for (const meta of downloadList) {
         const url = meta.dist.tarball;
-        tasks.push(this.bundleDownloader.download(meta, did, url, context));
+        tasks.push(this.bundleDownloader.download(meta, did, url, options));
       }
       const results = await Promise.all(tasks);
 
-      await postDownload({ downloadList, downloadComponentIds });
+      const isCancelled = results.some((x) => x.isCancelled);
+
+      await postDownload({ downloadList, downloadComponentIds, isCancelled });
 
-      if (results.find((x) => x.isCancelled)) {
+      if (isCancelled) {
         return { isCancelled: true };
       }
     } catch (error) {
@@ -160,11 +177,11 @@ class BlockletDownloader extends EventEmitter {
    *   blocklet;
    * }}
    * @returns {{
-   *   downloadList: Array<blockletMeta>;
-   *   downloadComponentIds: Array<string>;
+   *   downloadList: Array<import('@abtnode/client').BlockletMeta>;
+   *   downloadComponentIds: Array<string>; // like: "z8ia1i2yq67x39vruqQTbkVcwCnqRGx8zSPsJ/z8iZwubkNdA1BfEUwc5NJpY2Jnfm7yEbyvmKS"
    * }}
    */
-  async getDownloadList({ blocklet }) {
+  async getDownloadList({ blocklet, skipCheckIntegrity }) {
     const downloadComponentIds = [];
     const metas = {};
 
@@ -178,7 +195,11 @@ class BlockletDownloader extends EventEmitter {
         return;
       }
 
-      const needComponentDownload = needDownload(component, { installDir: this.installDir, cachedBundles });
+      const needComponentDownload = needDownload(component, {
+        installDir: this.installDir,
+        cachedBundles,
+        skipCheckIntegrity,
+      });
       if (!needComponentDownload) {
         this.logger.info(`skip download existed bundle ${bundleId}`, { source: component.source });
         return;

package/lib/blocklet/manager/disk.js

@@ -129,6 +129,7 @@ const { SpacesRestore } = require('../storage/restore/spaces');
 const installFromBackup = require('./helper/install-from-backup');
 const { resolveDownload, resolveDiffDownload } = require('../downloader/resolve-download');
 const BlockletDownloader = require('../downloader/blocklet-downloader');
+const RollbackCache = require('./helper/rollback-cache');
 
 const {
   isInProgress,
@@ -218,6 +219,8 @@ class BlockletManager extends BaseBlockletManager {
       cache: states.cache,
     });
 
+    this._rollbackCache = new RollbackCache({ dir: this.dataDirs.tmp });
+
     if (daemon) {
       blockletPm2Events.on('online', (data) => this._syncPm2Status('online', data.blockletDid));
       blockletPm2Events.on('stop', (data) => this._syncPm2Status('stop', data.blockletDid));
@@ -255,6 +258,10 @@ class BlockletManager extends BaseBlockletManager {
   async install(params, context = {}) {
     logger.debug('install blocklet', { params, context });
 
+    if (!params.controller && context?.user?.controller) {
+      params.controller = context.user.controller;
+    }
+
     const info = await states.node.read();
 
     context.headers = Object.assign(context?.headers || {}, {
@@ -839,12 +846,13 @@ class BlockletManager extends BaseBlockletManager {
     return newBlocklet;
   }
 
+  // eslint-disable-next-line no-unused-vars
   async cancelDownload({ did: inputDid }, context) {
     try {
       await statusLock.acquire();
       const blocklet = await states.blocklet.getBlocklet(inputDid);
       if (!blocklet) {
-        throw new Error('Can not cancel download for non-exist blocklet in database.', { inputDid });
+        throw new Error(`Can not cancel download for non-exist blocklet in database. did: ${inputDid}`);
       }
 
       const { name, did, version } = blocklet.meta;
@@ -854,15 +862,35 @@ class BlockletManager extends BaseBlockletManager {
       }
 
       const job = await this.installQueue.get(did);
-      if (job) {
-        const { postAction, oldBlocklet } = job;
-        await this._rollback(postAction, did, oldBlocklet);
-      }
 
+      // cancel job
       if (blocklet.status === BlockletStatus.downloading) {
-        await this.blockletDownloader.cancelDownload(blocklet.meta.did);
+        try {
+          await this.blockletDownloader.cancelDownload(blocklet.meta.did);
+        } catch (error) {
+          logger.error('failed to exec blockletDownloader.download', { did: blocklet.meta.did, error });
+        }
       } else if (blocklet.status === BlockletStatus.waiting) {
-        await this._cancelWaiting(blocklet.meta, context);
+        try {
+          await this.installQueue.cancel(blocklet.meta.did);
+        } catch (error) {
+          logger.error('failed to cancel waiting', { did: blocklet.meta.did, error });
+        }
+      }
+
+      // rollback
+      if (job) {
+        const { postAction, oldBlocklet } = job;
+        await this._rollback(postAction, did, oldBlocklet);
+      } else {
+        const data = await this._rollbackCache.restore({ did });
+        if (data) {
+          const { action, oldBlocklet } = data;
+          await this._rollback(action, did, oldBlocklet);
+          await this._rollbackCache.remove({ did });
+        } else {
+          throw new Error(`Cannot find rollback data in queue or backup file of blocklet ${inputDid}`);
+        }
       }
 
       logger.info('cancel download blocklet', { did, name, version, status: fromBlockletStatus(blocklet.status) });
@@ -870,7 +898,18 @@ class BlockletManager extends BaseBlockletManager {
       statusLock.release();
       return blocklet;
     } catch (error) {
-      statusLock.release();
+      try {
+        // fallback blocklet status to error
+        const blocklet = await states.blocklet.getBlocklet(inputDid);
+        if (blocklet) {
+          await states.blocklet.setBlockletStatus(blocklet.meta.did, BlockletStatus.error);
+        }
+        statusLock.release();
+      } catch (err) {
+        statusLock.release();
+        logger.error('Failed to fallback blocklet status to error on cancelDownload', { error });
+      }
+
       throw error;
     }
   }
@@ -1425,6 +1464,9 @@ class BlockletManager extends BaseBlockletManager {
 
     this.emit(BlockletEvents.statusChange, newBlocklet);
 
+    // backup rollback data
+    await this._rollbackCache.backup({ did, action, oldBlocklet });
+
     // add to queue
     const ticket = this.installQueue.push(
       {
@@ -1722,7 +1764,7 @@ class BlockletManager extends BaseBlockletManager {
   async onJob(job) {
     if (job.entity === 'blocklet') {
       if (job.action === 'download') {
-        await this.onDownload(job);
+        await this.downloadAndInstall(job);
       }
       if (job.action === 'restart') {
         await this.onRestart(job);
@@ -1747,33 +1789,35 @@ class BlockletManager extends BaseBlockletManager {
    * @return {*}
    * @memberof BlockletManager
    */
-  async onDownload(params) {
-    const { blocklet, context, postAction, oldBlocklet, throwOnError } = params;
+  async downloadAndInstall(params) {
+    const { blocklet, context, postAction, oldBlocklet, throwOnError, skipCheckStatusBeforeDownload } = params;
     const { meta } = blocklet;
     const { name, did, version } = meta;
 
     // check status
-    try {
-      await statusLock.acquire();
-
-      const b0 = await states.blocklet.getBlocklet(did);
-      if (!b0 || ![BlockletStatus.waiting].includes(b0.status)) {
-        if (!b0) {
-          throw new Error('blocklet does not exist before downloading');
-        } else {
-          throw new Error(`blocklet status is invalid before downloading: ${fromBlockletStatus(b0.status)}`);
+    if (!skipCheckStatusBeforeDownload) {
+      try {
+        await statusLock.acquire();
+
+        const b0 = await states.blocklet.getBlocklet(did);
+        if (!b0 || ![BlockletStatus.waiting].includes(b0.status)) {
+          if (!b0) {
+            throw new Error('blocklet does not exist before downloading');
+          } else {
+            throw new Error(`blocklet status is invalid before downloading: ${fromBlockletStatus(b0.status)}`);
+          }
         }
+        statusLock.release();
+      } catch (error) {
+        statusLock.release();
+        logger.error('Check blocklet status failed before downloading', {
+          name,
+          did,
+          error,
+        });
+        await this._rollback(postAction, did, oldBlocklet);
+        return;
       }
-      statusLock.release();
-    } catch (error) {
-      statusLock.release();
-      logger.error('Check blocklet status failed before downloading', {
-        name,
-        did,
-        error,
-      });
-      await this._rollback(postAction, did, oldBlocklet);
-      return;
     }
 
     // download bundle
@@ -1783,8 +1827,16 @@ class BlockletManager extends BaseBlockletManager {
       if (isCancelled) {
         logger.info('Download was canceled', { name, did, version });
 
-        if ((await states.blocklet.getBlockletStatus(did)) === BlockletStatus.downloading) {
-          await this._rollback(postAction, did, oldBlocklet);
+        // rollback on download cancelled
+        await statusLock.acquire();
+        try {
+          if ((await states.blocklet.getBlockletStatus(did)) === BlockletStatus.downloading) {
+            await this._rollback(postAction, did, oldBlocklet);
+          }
+          statusLock.release();
+        } catch (error) {
+          statusLock.release();
+          logger.error('Rollback blocklet failed on download canceled', { postAction, name, did, version, error });
         }
         return;
       }
@@ -1805,10 +1857,16 @@ class BlockletManager extends BaseBlockletManager {
         severity: 'error',
       });
 
+      // rollback on download failed
+      await statusLock.acquire();
       try {
-        await this._rollback(postAction, did, oldBlocklet);
-      } catch (e) {
-        logger.error('Rollback blocklet failed', { postAction, name, did, version });
+        if ((await states.blocklet.getBlockletStatus(did)) === BlockletStatus.downloading) {
+          await this._rollback(postAction, did, oldBlocklet);
+        }
+        statusLock.release();
+      } catch (error) {
+        statusLock.release();
+        logger.error('Rollback blocklet failed on download failed', { postAction, name, did, version, error });
       }
 
       if (throwOnError) {
@@ -2336,13 +2394,20 @@ class BlockletManager extends BaseBlockletManager {
       newBlocklet.deployedFrom = `Upload by ${context.user.fullName}`;
       newBlocklet.children = await this._getChildrenForInstallation(meta);
       await validateBlocklet(newBlocklet);
-      await this._downloadBlocklet(newBlocklet);
 
-      return this._upgradeBlocklet({
+      // backup rollback data
+      const action = 'upgrade';
+      await this._rollbackCache.backup({ did: newBlocklet.meta.did, action, oldBlocklet });
+
+      await this.downloadAndInstall({
+        blocklet: newBlocklet,
         oldBlocklet,
-        newBlocklet,
         context: { ...context, forceStartProcessIds: [getComponentProcessId(newBlocklet)] },
+        throwOnError: true,
+        postAction: action,
+        skipCheckStatusBeforeDownload: true,
       });
+      return this.getBlocklet(newBlocklet.meta.did);
     }
 
     // full deploy
@@ -2362,14 +2427,20 @@ class BlockletManager extends BaseBlockletManager {
       newBlocklet.deployedFrom = `Upload by ${context.user.fullName}`;
       newBlocklet.children = await this._getChildrenForInstallation(meta);
 
-      await validateBlocklet(newBlocklet);
-      await this._downloadBlocklet(newBlocklet);
+      // backup rollback data
+      const action = 'upgrade';
+      await this._rollbackCache.backup({ did: newBlocklet.meta.did, action, oldBlocklet });
 
-      return this._upgradeBlocklet({
+      await validateBlocklet(newBlocklet);
+      await this.downloadAndInstall({
+        blocklet: newBlocklet,
         oldBlocklet,
-        newBlocklet,
         context: { ...context, forceStartProcessIds: [getComponentProcessId(newBlocklet)] },
+        throwOnError: true,
+        postAction: action,
+        skipCheckStatusBeforeDownload: true,
       });
+      return this.getBlocklet(newBlocklet.meta.did);
     }
 
     // full deploy - install
@@ -2383,25 +2454,31 @@ class BlockletManager extends BaseBlockletManager {
       children,
     });
 
+    const action = 'install';
+    const oldState = { extraState: oldExtraState };
     try {
       await this._setConfigsFromMeta(meta.did);
       await validateBlocklet(blocklet);
 
       // check duplicate appSk
       await checkDuplicateAppSk({ did: meta.did, states });
-
-      // download
-      await this._downloadBlocklet(blocklet);
-
-      return this._installBlocklet({
-        did: meta.did,
-        context,
-        oldBlocklet: { extraState: oldExtraState },
-      });
     } catch (error) {
-      await this._rollback('install', meta.did, { extraState: oldExtraState });
+      await this._rollback(action, meta.did, oldState);
       throw error;
     }
+
+    // backup rollback data
+    await this._rollbackCache.backup({ did: meta.did, action, oldBlocklet: oldState });
+
+    await this.downloadAndInstall({
+      blocklet,
+      oldBlocklet: oldState,
+      context,
+      throwOnError: true,
+      postAction: action,
+      skipCheckStatusBeforeDownload: true,
+    });
+    return this.getBlocklet(meta.did);
   }
 
   async _installComponentFromUpload({
@@ -2482,34 +2559,19 @@ class BlockletManager extends BaseBlockletManager {
 
     await this._upsertDynamicNavigation(newBlocklet.meta.did, newChild, { skipNavigation });
 
-    await this._downloadBlocklet(newBlocklet);
-
-    await validateBlocklet(newBlocklet);
+    // backup rollback data
+    const action = 'upgrade';
+    await this._rollbackCache.backup({ did: newBlocklet.meta.did, action, oldBlocklet });
 
-    return this._upgradeBlocklet({
+    await this.downloadAndInstall({
+      blocklet: newBlocklet,
       oldBlocklet,
-      newBlocklet,
       context: { ...context, forceStartProcessIds: [getComponentProcessId(newChild, [newBlocklet])] },
+      throwOnError: true,
+      postAction: action,
+      skipCheckStatusBeforeDownload: true,
     });
-  }
-
-  /**
-   * add to download job queue
-   * @param {string} did blocklet did
-   * @param {object} blocklet object
-   */
-  async download(did, blocklet) {
-    await states.blocklet.setBlockletStatus(did, BlockletStatus.waiting);
-    this.installQueue.push(
-      {
-        entity: 'blocklet',
-        action: 'download',
-        id: did,
-        blocklet: { ...blocklet },
-        postAction: 'install',
-      },
-      did
-    );
+    return this.getBlocklet(newBlocklet.meta.did);
   }
 
   async prune() {
@@ -2732,6 +2794,11 @@ class BlockletManager extends BaseBlockletManager {
       const blocklet1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.waiting);
       this.emit(BlockletEvents.added, blocklet1);
 
+      const action = 'install';
+      const oldBlocklet = {
+        extraState: oldExtraState,
+      };
+
       /** @type {{
        *  blocklet: any;
        *  oldBlocklet: {
@@ -2743,16 +2810,16 @@ class BlockletManager extends BaseBlockletManager {
        * }} */
       const downloadParams = {
         blocklet: { ...blocklet1 },
-        oldBlocklet: {
-          children: children.filter((x) => x.dynamic), // let downloader skip re-downloading dynamic blocklet
-          extraState: oldExtraState,
-        },
         context,
-        postAction: 'install',
+        postAction: action,
+        oldBlocklet,
       };
 
+      // backup rollback data
+      await this._rollbackCache.backup({ did, action, oldBlocklet });
+
       if (sync) {
-        await this.onDownload({ ...downloadParams, throwOnError: true });
+        await this.downloadAndInstall({ ...downloadParams, throwOnError: true });
         return states.blocklet.getBlocklet(did);
       }
 
@@ -2863,8 +2930,11 @@ class BlockletManager extends BaseBlockletManager {
       postAction: action,
     };
 
+    // backup rollback data
+    await this._rollbackCache.backup({ did, action, oldBlocklet });
+
     if (sync) {
-      await this.onDownload({ ...downloadParams, throwOnError: true });
+      await this.downloadAndInstall({ ...downloadParams, throwOnError: true });
       return states.blocklet.getBlocklet(did);
     }
     const ticket = this.installQueue.push(
@@ -2961,6 +3031,8 @@ class BlockletManager extends BaseBlockletManager {
         entityId: did,
         severity: 'success',
       });
+
+      await this._rollbackCache.remove({ did: blocklet.meta.did });
       return blocklet;
     } catch (err) {
       const { meta } = await states.blocklet.getBlocklet(did);
@@ -3087,6 +3159,8 @@ class BlockletManager extends BaseBlockletManager {
       // Update dynamic component meta in blocklet settings
       await this._ensureDynamicChildrenInSettings(blocklet);
 
+      await this._rollbackCache.remove({ did: blocklet.meta.did });
+
       return blocklet;
     } catch (err) {
       const b = await this._rollback(action, did, oldBlocklet);
@@ -3150,9 +3224,11 @@ class BlockletManager extends BaseBlockletManager {
         });
         this.emit(BlockletEvents.statusChange, blocklet1);
       },
-      postDownload: async () => {
-        // since preferences only exist in blocklet bundle, we need to populate then after downloaded
-        await this._setConfigsFromMeta(did);
+      postDownload: async ({ isCancelled }) => {
+        if (!isCancelled) {
+          // since preferences only exist in blocklet bundle, we need to populate then after downloaded
+          await this._setConfigsFromMeta(did);
+        }
       },
     });
   }
@@ -3170,12 +3246,6 @@ class BlockletManager extends BaseBlockletManager {
     }
   }
 
-  // eslint-disable-next-line no-unused-vars
-  async _cancelWaiting(blockletMeta, context) {
-    const { did } = blockletMeta;
-    return this.installQueue.cancel(did);
-  }
-
   /**
    * @param {string} action install, upgrade, downgrade
    * @param {string} did
@@ -3343,7 +3413,7 @@ class BlockletManager extends BaseBlockletManager {
   }
 
   async _getBlockletForInstallation(did) {
-    const blocklet = await states.blocklet.getBlocklet(did);
+    const blocklet = await states.blocklet.getBlocklet(did, { decryptSk: false });
     if (!blocklet) {
       return null;
     }

package/lib/blocklet/manager/helper/install-from-backup.js

@@ -11,6 +11,14 @@ const logger = require('@abtnode/logger')('@abtnode/core:install-from-backup');
 
 const { validateBlocklet, checkDuplicateAppSk, getAppDirs } = require('../../../util/blocklet');
 
+/**
+ *
+ * @param {{
+ *  manager: import('../disk'),
+ *  states: import('../../../states/index')
+ * }} param0
+ * @returns
+ */
 module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states, manager } = {}) => {
   // TODO: support more url schema feature (http, did-spaces)
   if (!url.startsWith('file://')) {
@@ -24,11 +32,7 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
   }
 
   // parse data from source dir
-
   const srcBundleDirs = await getAppDirs(path.join(dir, 'blocklets'));
-  if (!srcBundleDirs.length) {
-    throw new Error(`bundle dirs does not found in ${dir}`);
-  }
 
   const srcDataDir = path.join(dir, 'data');
 
@@ -61,6 +65,7 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
 
   // FIXME: meta.did and meta.name should be dynamic created when installing multiple blocklet is supported
   const existState = await states.blocklet.hasBlocklet(did);
+
   if (existState) {
     logger.error('blocklet is already exist', { did });
     throw new Error('blocklet is already exist');
@@ -133,6 +138,8 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
         logger.info(`bundle  is ${moveDir ? 'moved' : 'copied'} successfully`, { installDir });
       })
     );
+    // 从 store 下载 blocklet
+    await manager.blockletDownloader.download(state, { skipCheckIntegrity: true });
 
     // FIXME same as copy extra
     const dataDir = path.join(manager.dataDirs.data, appName);

package/lib/blocklet/manager/helper/rollback-cache.js

@@ -0,0 +1,41 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+const DIR_NAME = 'rollback-cache';
+const security = require('@abtnode/util/lib/security');
+
+class RollbackCache {
+  constructor({ dir, dek }) {
+    this.dir = dir;
+    this.dek = dek;
+  }
+
+  async backup({ did, action, oldBlocklet }) {
+    const file = this._getFile(did);
+    const data = this.dek ? security.encrypt(JSON.stringify(oldBlocklet), did, this.dek) : oldBlocklet;
+    await fs.outputJSON(file, { action, oldBlocklet: data });
+  }
+
+  async remove({ did }) {
+    const file = this._getFile(did);
+    await fs.remove(file);
+  }
+
+  async restore({ did }) {
+    const file = this._getFile(did);
+    if (fs.existsSync(file)) {
+      const data = await fs.readJSON(file);
+      if (data?.oldBlocklet && this.dek) {
+        data.oldBlocklet = JSON.parse(security.decrypt(data.oldBlocklet, did, this.dek));
+      }
+      return data;
+    }
+    return null;
+  }
+
+  _getFile(did) {
+    return path.join(this.dir, DIR_NAME, `${did}.json`);
+  }
+}
+
+module.exports = RollbackCache;

package/lib/blocklet/storage/backup/blocklet-extras.js

@@ -75,6 +75,10 @@ class BlockletExtrasBackup extends BaseBackup {
    * @memberof BlockletExtrasBackup
    */
   useBlockletEncryptConfigs(configs) {
+    if (isEmpty(configs)) {
+      return;
+    }
+
     // 准备加解密所需的参数
     // @see: https://github.com/ArcBlock/blocklet-server/blob/f561ba7290285f2e23dccb6d5323eb4d43c3cc3e/core/state/lib/index.js#L59
     const dek = readFileSync(join(this.serverDataDir, '.sock'));