@abtnode/core 1.6.13 → 1.6.17

package/lib/blocklet/manager/disk.js

@@ -3,20 +3,20 @@
 const fs = require('fs-extra');
 const path = require('path');
 const get = require('lodash/get');
+const pick = require('lodash/pick');
 const cloneDeep = require('lodash/cloneDeep');
 const semver = require('semver');
 const capitalize = require('lodash/capitalize');
-const diff = require('deep-diff');
 const { Throttle } = require('stream-throttle');
 const LRU = require('lru-cache');
+const joi = require('joi');
 
 const { isValid: isValidDid } = require('@arcblock/did');
 const { verifyPresentation } = require('@arcblock/vc');
-const { toBase58 } = require('@ocap/util');
+const { toBase58, isHex } = require('@ocap/util');
 const { fromSecretKey } = require('@ocap/wallet');
 
 const logger = require('@abtnode/logger')('@abtnode/core:blocklet:manager');
-const hashFiles = require('@abtnode/util/lib/hash-files');
 const downloadFile = require('@abtnode/util/lib/download-file');
 const Lock = require('@abtnode/util/lib/lock');
 const { getVcFromPresentation } = require('@abtnode/util/lib/vc');
@@ -24,9 +24,17 @@ const { updateBlocklet: updateDidDocument } = require('@abtnode/util/lib/did-doc
 const { BLOCKLET_PURCHASE_NFT_TYPE } = require('@abtnode/constant');
 
 const getBlockletEngine = require('@blocklet/meta/lib/engine');
-const { isFreeBlocklet, isDeletableBlocklet, getRequiredMissingConfigs } = require('@blocklet/meta/lib/util');
+const {
+  isFreeBlocklet,
+  isDeletableBlocklet,
+  getRequiredMissingConfigs,
+  hasRunnableComponent,
+} = require('@blocklet/meta/lib/util');
 const validateBlockletEntry = require('@blocklet/meta/lib/entry');
 const { getBlockletInfo } = require('@blocklet/meta/lib');
+const toBlockletDid = require('@blocklet/meta/lib/did');
+const { validateMeta } = require('@blocklet/meta/lib/validate');
+const { update: updateMetaFile } = require('@blocklet/meta/lib/file');
 
 const {
   BlockletStatus,
@@ -37,6 +45,15 @@ const {
   BlockletGroup,
   fromBlockletStatus,
   fromBlockletSource,
+  BLOCKLET_DEFAULT_PORT_NAME,
+  BLOCKLET_INTERFACE_TYPE_WEB,
+  BLOCKLET_INTERFACE_PUBLIC,
+  BLOCKLET_DYNAMIC_PATH_PREFIX,
+  BLOCKLET_INTERFACE_PROTOCOL_HTTP,
+  BLOCKLET_DEFAULT_PATH_REWRITE,
+  BLOCKLET_DEFAULT_VERSION,
+  BLOCKLET_LATEST_SPEC_VERSION,
+  BLOCKLET_META_FILE,
 } = require('@blocklet/meta/lib/constants');
 const util = require('../../util');
 const {
@@ -60,16 +77,19 @@ const {
   getBlockletStatusFromProcess,
   checkBlockletProcessHealthy,
   validateBlocklet,
-  getChildrenMeta,
   statusMap,
   expandTarball,
   verifyIntegrity,
   pruneBlockletBundle,
   getDiskInfo,
   getRuntimeInfo,
-  mergeMeta,
   getUpdateMetaList,
   getRuntimeEnvironments,
+  getSourceFromInstallParams,
+  parseChildren,
+  checkDuplicateComponents,
+  getDiffFiles,
+  getBundleDir,
 } = require('../../util/blocklet');
 const states = require('../../states');
 const BlockletRegistry = require('../registry');
@@ -153,20 +173,43 @@ class BlockletManager extends BaseBlockletManager {
    */
   async install(params, context) {
     logger.debug('install blocklet', { params, context });
-    if (params.url) {
+
+    const source = getSourceFromInstallParams(params);
+
+    if (source === BlockletSource.url) {
       return this._installFromUrl({ url: params.url, sync: params.sync }, context);
     }
 
-    if (params.file) {
+    if (source === BlockletSource.upload) {
       const { file, did, diffVersion, deleteSet } = params;
       return this._installFromUpload({ file, did, diffVersion, deleteSet, context });
     }
 
-    if (params.did) {
-      return this._installFromRegistry({ did: params.did }, context);
+    if (source === BlockletSource.registry) {
+      return this._installFromStore({ did: params.did }, context);
+    }
+
+    if (source === BlockletSource.custom) {
+      return this._installFromCreate({ title: params.title, description: params.description }, context);
+    }
+
+    // should not be here
+    throw new Error('Unknown source');
+  }
+
+  async installComponent({ rootDid, mountPoint, url, file, did, diffVersion, deleteSet }, context = {}) {
+    logger.debug('start install component', { rootDid, mountPoint, url });
+
+    if (file) {
+      return this._installComponentFromUpload({ rootDid, mountPoint, file, did, diffVersion, deleteSet, context });
+    }
+
+    if (url) {
+      return this._installComponentFromUrl({ rootDid, mountPoint, url, context });
     }
 
-    throw new Error('Can only install blocklet from url/file/did');
+    // should not be here
+    throw new Error('Unknown source');
   }
 
   // eslint-disable-next-line no-unused-vars
@@ -189,9 +232,9 @@ class BlockletManager extends BaseBlockletManager {
       }
     }
 
-    meta.isFree = isFree;
+    const blocklet = await states.blocklet.getBlocklet(meta.did);
 
-    return meta;
+    return { meta, isFree, isInstalled: !!blocklet };
   }
 
   async installBlockletFromVc({ vcPresentation, challenge }, context) {
@@ -210,7 +253,7 @@ class BlockletManager extends BaseBlockletManager {
     const did = get(vc, 'credentialSubject.purchased.blocklet.id');
     const registry = urlObject.origin;
 
-    return this._installFromRegistry({ did, registry }, { ...context, blockletPurchaseVerified: true });
+    return this._installFromStore({ did, registry }, { ...context, blockletPurchaseVerified: true });
   }
 
   async start({ did, checkHealthImmediately = false, throwOnError }, context) {
@@ -218,6 +261,10 @@ class BlockletManager extends BaseBlockletManager {
     const blocklet = await this.ensureBlocklet(did);
 
     try {
+      if (!hasRunnableComponent(blocklet)) {
+        throw new Error('No runnable component found');
+      }
+
       // check required config
       const missingProps = getRequiredMissingConfigs(blocklet);
       if (missingProps.length) {
@@ -412,6 +459,22 @@ class BlockletManager extends BaseBlockletManager {
     }
   }
 
+  async deleteComponent({ did, rootDid }, context) {
+    logger.info('delete blocklet component', { did, rootDid });
+
+    const doc = await states.blocklet.getBlocklet(rootDid);
+
+    doc.children = doc.children.filter((x) => x.meta.did !== did);
+    const children = (await this._getDynamicChildrenFromSettings(rootDid)).filter((x) => x.meta.did !== did);
+
+    await states.blocklet.updateBlocklet(rootDid, doc);
+    states.blockletExtras.setSettings(rootDid, { children });
+
+    const newBlocklet = await this.ensureBlocklet(rootDid);
+    this.emit(BlockletEvents.upgraded, { blocklet: newBlocklet, context }); // trigger router refresh
+    return newBlocklet;
+  }
+
   async cancelDownload({ did }, context) {
     await preDownloadLock.acquire();
     try {
@@ -549,6 +612,14 @@ class BlockletManager extends BaseBlockletManager {
         }
       }
 
+      if (x.key === 'BLOCKLET_PASSPORT_COLOR') {
+        if (x.value && x.value !== 'auto') {
+          if (x.value.length !== 7 || !isHex(x.value.slice(-6))) {
+            throw new Error('BLOCKLET_PASSPORT_COLOR must be a hex encoded color, eg. #ffeeaa');
+          }
+        }
+      }
+
       blocklet.configObj[x.key] = x.value;
     }
 
@@ -558,7 +629,6 @@ class BlockletManager extends BaseBlockletManager {
       hooks: Object.assign(blocklet.meta.hooks || {}, blocklet.meta.scripts || {}),
       exitOnError: true,
       env: { ...getRuntimeEnvironments(blocklet, nodeEnvironments), ...blocklet.configObj },
-      notification: states.notification,
       did,
       context,
       progress: blocklet.mode === BLOCKLET_MODES.DEVELOPMENT,
@@ -657,56 +727,45 @@ class BlockletManager extends BaseBlockletManager {
   }
 
   // eslint-disable-next-line no-unused-vars
-  async diff({ did, hashFiles: clientFiles }, context) {
+  async diff({ did, hashFiles: clientFiles, rootDid: inputRootDid }, context) {
     if (!did) {
       throw new Error('did is empty');
     }
+
     if (!clientFiles || !clientFiles.length) {
       throw new Error('hashFiles is empty');
     }
 
-    logger.info('Get blocklet diff', { did, clientFilesNumber: clientFiles.length });
+    const rootDid = inputRootDid || did;
+    const childDid = inputRootDid ? did : '';
+
+    if (childDid === rootDid) {
+      throw new Error('Cannot add self as a component');
+    }
+
+    logger.info('Get blocklet diff', { rootDid, childDid, clientFilesNumber: clientFiles.length });
+
+    const rootBlocklet = await states.blocklet.getBlocklet(rootDid);
+    if (childDid && !rootBlocklet) {
+      throw new Error('Root blocklet does not exist');
+    }
+
+    const state = childDid ? await (rootBlocklet.children || []).find((x) => x.meta.did === childDid) : rootBlocklet;
 
-    const state = await states.blocklet.getBlocklet(did);
     if (!state) {
       return {
         hasBlocklet: false,
       };
     }
+
     if (state.source === BlockletSource.local) {
       throw new Error(`Blocklet ${state.meta.name} is already deployed from local, can not deployed from remote.`);
     }
-    const { name, version } = state.meta;
-    const installDir = path.join(this.installDir, name, version);
-    // eslint-disable-next-line no-param-reassign
-    clientFiles = clientFiles.reduce((obj, item) => {
-      obj[item.file] = item.hash;
-      return obj;
-    }, {});
 
-    const { files } = await hashFiles(installDir, {
-      filter: (x) => x.indexOf('node_modules') === -1,
-      concurrentHash: 1,
-    });
-    logger.info('Get files hash', { filesNum: Object.keys(files).length });
-
-    const addSet = [];
-    const changeSet = [];
-    const deleteSet = [];
-    const diffFiles = diff(files, clientFiles);
-    if (diffFiles) {
-      diffFiles.forEach((item) => {
-        if (item.kind === 'D') {
-          deleteSet.push(item.path[0]);
-        }
-        if (item.kind === 'E') {
-          changeSet.push(item.path[0]);
-        }
-        if (item.kind === 'N') {
-          addSet.push(item.path[0]);
-        }
-      });
-    }
+    const { version } = state.meta;
+    const bundleDir = getBundleDir(this.installDir, state.meta);
+    const { addSet, changeSet, deleteSet } = await getDiffFiles(clientFiles, bundleDir);
+
     logger.info('Diff files', {
       name: state.meta.name,
       did: state.meta.did,
@@ -715,6 +774,7 @@ class BlockletManager extends BaseBlockletManager {
       changeNum: changeSet.length,
       deleteNum: deleteSet.length,
     });
+
     return {
       hasBlocklet: true,
       version,
@@ -726,10 +786,20 @@ class BlockletManager extends BaseBlockletManager {
 
   async checkChildrenForUpdates({ did }) {
     const blocklet = await states.blocklet.getBlocklet(did);
-    const childrenMeta = await getChildrenMeta(blocklet.meta);
+    const newStaticChildren = await parseChildren(blocklet.meta);
+
+    const oldDynamicChildren = await this._getDynamicChildrenFromSettings(did);
+    const noneSourceUrlChildren = oldDynamicChildren.filter((x) => !x.sourceUrl);
+    const dynamicConfig = oldDynamicChildren
+      .filter((x) => x.sourceUrl)
+      .map((x) => ({ resolved: x.sourceUrl, name: x.meta.name, mountPoint: x.mountPoint }));
+    const newDynamicChildren = [...noneSourceUrlChildren, ...(await parseChildren(dynamicConfig, { dynamic: true }))];
+
+    checkDuplicateComponents(newDynamicChildren, newStaticChildren);
+
     const updateList = getUpdateMetaList(
-      blocklet.children.map((x) => x.meta),
-      childrenMeta
+      [...blocklet.children.map((x) => x.meta), ...oldDynamicChildren.map((x) => x.meta)],
+      [...newStaticChildren.map((x) => x.meta), ...newDynamicChildren.map((x) => x.meta)]
     );
 
     if (!updateList.length) {
@@ -739,7 +809,8 @@ class BlockletManager extends BaseBlockletManager {
     // start session
     const { id: updateId } = await states.session.start({
       did,
-      childrenMeta,
+      staticChildren: newStaticChildren,
+      dynamicChildren: newDynamicChildren,
     });
 
     return {
@@ -748,8 +819,18 @@ class BlockletManager extends BaseBlockletManager {
     };
   }
 
-  async updateChildren({ updateId }, context) {
-    const { did, childrenMeta } = await states.session.end(updateId);
+  async updateChildren({ updateId, did: inputDid, children: inputChildren }, context) {
+    let did;
+    let children;
+    if (!updateId && inputDid && inputChildren) {
+      did = inputDid;
+      children = inputChildren;
+    } else {
+      const sessionData = await states.session.end(updateId);
+      did = sessionData.did;
+      const { staticChildren = [], dynamicChildren = [] } = sessionData;
+      children = [...staticChildren, ...dynamicChildren.map((x) => ({ ...x, dynamic: true }))];
+    }
 
     // get old blocklet
     const oldBlocklet = await states.blocklet.getBlocklet(did);
@@ -762,14 +843,13 @@ class BlockletManager extends BaseBlockletManager {
       did,
       name,
       version,
-      children: childrenMeta.map((x) => ({ name: x.name, version: x.version })),
+      children: children.map((x) => ({ name: x.meta.name, version: x.meta.version })),
     });
 
     // new blocklet
     const newBlocklet = await states.blocklet.setBlockletStatus(did, BlockletStatus.waiting);
-    mergeMeta(meta, childrenMeta);
-    newBlocklet.meta = meta;
-    newBlocklet.children = await states.blocklet.getChildrenFromMetas(childrenMeta);
+
+    newBlocklet.children = children;
     await validateBlocklet(newBlocklet);
 
     this.emit(BlockletEvents.statusChange, newBlocklet);
@@ -842,19 +922,19 @@ class BlockletManager extends BaseBlockletManager {
       logger.error('failed to delete blocklet process for dev', { error: err });
     }
 
-    const childrenMeta = await getChildrenMeta(meta);
-    mergeMeta(meta, childrenMeta);
+    const children = await this._getChildren(meta);
     const blocklet = await states.blocklet.addBlocklet({
       did,
       meta,
       source: BlockletSource.local,
       deployedFrom: folder,
       mode: BLOCKLET_MODES.DEVELOPMENT,
-      childrenMeta,
+      children,
     });
     logger.info('add blocklet for dev', { did, version, meta });
 
-    await this._downloadBlocklet(blocklet);
+    const oldBlocklet = { children: children.filter((x) => x.dynamic) }; // let downloader skip re-downloading dynamic blocklet
+    await this._downloadBlocklet(blocklet, oldBlocklet);
     await states.blocklet.setBlockletStatus(did, BlockletStatus.installed);
 
     // Add environments
@@ -997,7 +1077,8 @@ class BlockletManager extends BaseBlockletManager {
         blocklet.diskInfo = await getDiskInfo(blocklet, { useFakeDiskInfo: !diskInfo });
 
         try {
-          const { appId } = blocklet.meta.group === BlockletGroup.gateway ? blocklet.children[0].env : blocklet.env;
+          const app = blocklet.meta.group === BlockletGroup.gateway ? blocklet.children[0].env : blocklet.env;
+          const { appId } = app;
           blocklet.runtimeInfo = await getRuntimeInfo(appId);
           if (blocklet.runtimeInfo.status && shouldUpdateBlockletStatus(blocklet.status)) {
             blocklet.status = statusMap[blocklet.runtimeInfo.status];
@@ -1271,7 +1352,7 @@ class BlockletManager extends BaseBlockletManager {
     }
   }
 
-  async _installFromRegistry({ did, registry }, context) {
+  async _installFromStore({ did, registry }, context) {
     logger.debug('start install blocklet', { did });
     if (!isValidDid(did)) {
       throw new Error('Blocklet did is invalid');
@@ -1333,9 +1414,84 @@ class BlockletManager extends BaseBlockletManager {
     });
   }
 
-  async _installFromUpload({ file, did, diffVersion, deleteSet, context }) {
-    logger.info('install blocklet', { from: 'upload file' });
-    // download
+  async _installComponentFromUrl({ rootDid, mountPoint, url, context }) {
+    const blocklet = await states.blocklet.getBlocklet(rootDid);
+    if (!blocklet) {
+      throw new Error('Root blocklet does not exist');
+    }
+
+    const meta = await getBlockletMetaFromUrl(url);
+
+    if (meta.did === rootDid) {
+      throw new Error('Cannot add self as a component');
+    }
+
+    const newChildren = await parseChildren(blocklet.meta, {
+      children: [
+        {
+          meta,
+          mountPoint,
+          sourceUrl: url,
+        },
+      ],
+      dynamic: true,
+    });
+
+    checkDuplicateComponents(blocklet.children, newChildren);
+
+    return this.updateChildren(
+      {
+        did: rootDid,
+        children: [...blocklet.children, ...newChildren],
+      },
+      context
+    );
+  }
+
+  async _installFromCreate({ title, description }, context = {}) {
+    logger.debug('create blocklet', { title, description });
+
+    await joi.string().label('title').max(20).required().validateAsync(title);
+    await joi.string().label('description').max(80).required().validateAsync(description);
+
+    const { did, name } = await this._findNextCustomBlockletName();
+    const rawMeta = {
+      name,
+      did,
+      title,
+      description,
+      version: BLOCKLET_DEFAULT_VERSION,
+      group: BlockletGroup.gateway,
+      interfaces: [
+        {
+          type: BLOCKLET_INTERFACE_TYPE_WEB,
+          name: BLOCKLET_INTERFACE_PUBLIC,
+          path: BLOCKLET_DEFAULT_PATH_REWRITE,
+          prefix: BLOCKLET_DYNAMIC_PATH_PREFIX,
+          port: BLOCKLET_DEFAULT_PORT_NAME,
+          protocol: BLOCKLET_INTERFACE_PROTOCOL_HTTP,
+        },
+      ],
+      specVersion: BLOCKLET_LATEST_SPEC_VERSION,
+    };
+    const meta = validateMeta(rawMeta);
+
+    await states.blocklet.addBlocklet({
+      did: meta.did,
+      meta,
+      source: BlockletSource.custom,
+    });
+    await this._setConfigs(did);
+
+    // fake install bundle
+    const bundleDir = getBundleDir(this.installDir, meta);
+    fs.mkdirSync(bundleDir, { recursive: true });
+    updateMetaFile(path.join(bundleDir, BLOCKLET_META_FILE), meta);
+
+    return this._installBlocklet({ did, context });
+  }
+
+  async _downloadFromUpload(file) {
     // const { filename, mimetype, encoding, createReadStream } = await file;
     const { filename, createReadStream } = await file;
     const cwd = path.join(this.dataDirs.tmp, 'download');
@@ -1359,6 +1515,13 @@ class BlockletManager extends BaseBlockletManager {
       writeStream.on('finish', resolve);
     });
 
+    return { cwd, tarFile };
+  }
+
+  async _installFromUpload({ file, did, diffVersion, deleteSet, context }) {
+    logger.info('install blocklet', { from: 'upload file' });
+    const { cwd, tarFile } = await this._downloadFromUpload(file);
+
     // diff deploy
     if (did && diffVersion) {
       const oldBlocklet = await states.blocklet.getBlocklet(did);
@@ -1379,14 +1542,12 @@ class BlockletManager extends BaseBlockletManager {
         throw new Error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
       }
 
-      const { meta } = await this._resolveDiffDownload(cwd, tarFile, deleteSet, oldBlocklet);
-      const childrenMeta = await getChildrenMeta(meta);
-      mergeMeta(meta, childrenMeta);
+      const { meta } = await this._resolveDiffDownload(cwd, tarFile, deleteSet, oldBlocklet.meta);
       const newBlocklet = await states.blocklet.getBlocklet(did);
       newBlocklet.meta = meta;
       newBlocklet.source = BlockletSource.upload;
       newBlocklet.deployedFrom = `Upload by ${context.user.did}`;
-      newBlocklet.children = await states.blocklet.getChildrenFromMetas(childrenMeta);
+      newBlocklet.children = await this._getChildren(meta);
       await validateBlocklet(newBlocklet);
       await this._downloadBlocklet(newBlocklet, oldBlocklet);
 
@@ -1401,19 +1562,18 @@ class BlockletManager extends BaseBlockletManager {
     const { meta } = await this._resolveDownload(cwd, tarFile);
     const oldBlocklet = await states.blocklet.getBlocklet(meta.did);
 
+    // full deploy - upgrade
     if (oldBlocklet) {
       if (isInProgress(oldBlocklet.status)) {
         logger.error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
         throw new Error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
       }
 
-      const childrenMeta = await getChildrenMeta(meta);
-      mergeMeta(meta, childrenMeta);
       const newBlocklet = await states.blocklet.getBlocklet(meta.did);
       newBlocklet.meta = meta;
       newBlocklet.source = BlockletSource.upload;
       newBlocklet.deployedFrom = `Upload by ${context.user.did}`;
-      newBlocklet.children = await states.blocklet.getChildrenFromMetas(childrenMeta);
+      newBlocklet.children = await this._getChildren(meta);
 
       await validateBlocklet(newBlocklet);
       await this._downloadBlocklet(newBlocklet, oldBlocklet);
@@ -1425,14 +1585,14 @@ class BlockletManager extends BaseBlockletManager {
       });
     }
 
-    const childrenMeta = await getChildrenMeta(meta);
-    mergeMeta(meta, childrenMeta);
+    // full deploy - install
+    const children = await this._getChildren(meta);
     const blocklet = await states.blocklet.addBlocklet({
       did: meta.did,
       meta,
       source: BlockletSource.upload,
       deployedFrom: `Upload by ${context.user.did}`,
-      childrenMeta,
+      children,
     });
 
     await validateBlocklet(blocklet);
@@ -1440,13 +1600,92 @@ class BlockletManager extends BaseBlockletManager {
     await this._setConfigs(meta.did);
 
     // download
-    await this._downloadBlocklet(blocklet);
+    await this._downloadBlocklet(
+      blocklet,
+      // let downloader skip re-downloading dynamic blocklet
+      { children: children.filter((x) => x.dynamic) }
+    );
     return this._installBlocklet({
       did: meta.did,
       context,
     });
   }
 
+  async _installComponentFromUpload({ rootDid, mountPoint, file, did, diffVersion, deleteSet, context }) {
+    logger.info('install blocklet', { from: 'upload file' });
+    // download
+    const { cwd, tarFile } = await this._downloadFromUpload(file);
+
+    const oldBlocklet = await states.blocklet.getBlocklet(rootDid);
+    if (!oldBlocklet) {
+      throw new Error('Root blocklet does not exist');
+    }
+
+    if (isInProgress(oldBlocklet.status)) {
+      logger.error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
+      throw new Error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
+    }
+
+    let meta;
+    // diff upload
+    if (did && diffVersion) {
+      const oldChild = oldBlocklet.children.find((x) => x.meta.did === did);
+      if (!oldChild) {
+        throw new Error(`Blocklet ${did} not found when diff deploying`);
+      }
+      if (oldChild.meta.version !== diffVersion) {
+        logger.error('Diff deploy: Blocklet version changed', {
+          preVersion: diffVersion,
+          changedVersion: oldChild.meta.version,
+          name: oldChild.meta.name,
+          did: oldChild.meta.did,
+        });
+        throw new Error('Blocklet version changed when diff deploying');
+      }
+
+      meta = (await this._resolveDiffDownload(cwd, tarFile, deleteSet, oldChild.meta)).meta;
+    } else {
+      // full deploy
+      meta = (await this._resolveDownload(cwd, tarFile)).meta;
+    }
+
+    if (meta.did === rootDid) {
+      // should not be here
+      throw new Error('Cannot add self as a component');
+    }
+
+    const newBlocklet = await states.blocklet.getBlocklet(rootDid);
+    const dynamicChildren = await this._getDynamicChildrenFromSettings(rootDid);
+
+    const newChild = {
+      meta,
+      mountPoint,
+      source: BlockletSource.upload,
+      deployedFrom: `Upload by ${context.user.did}`,
+      sourceUrl: '',
+      dynamic: true,
+    };
+    const index = dynamicChildren.findIndex((child) => child.meta.did === meta.did);
+    if (index >= 0) {
+      dynamicChildren.splice(index, 1, newChild);
+    } else {
+      dynamicChildren.push(newChild);
+    }
+
+    const staticChildren = newBlocklet.children.filter((x) => !x.dynamic);
+    checkDuplicateComponents(dynamicChildren, staticChildren);
+
+    newBlocklet.children = [...staticChildren, ...dynamicChildren.map((x) => ({ ...x, dynamic: true }))];
+
+    await validateBlocklet(newBlocklet);
+
+    return this._upgradeBlocklet({
+      oldBlocklet,
+      newBlocklet,
+      context,
+    });
+  }
+
   /**
    * add to download job queue
    * @param {string} did blocklet did
@@ -1481,7 +1720,14 @@ class BlockletManager extends BaseBlockletManager {
 
   async prune() {
     const blocklets = await states.blocklet.getBlocklets();
-    await pruneBlockletBundle(blocklets, this.dataDirs.blocklets);
+    const settings = await states.blockletExtras.listSettings();
+    await pruneBlockletBundle({
+      installDir: this.dataDirs.blocklets,
+      blocklets,
+      blockletSettings: settings
+        .filter((x) => x.settings.children && x.settings.children.length)
+        .map((x) => x.settings),
+    });
   }
 
   async getLatestBlockletVersion({ did, version }) {
@@ -1569,20 +1815,35 @@ class BlockletManager extends BaseBlockletManager {
     blocklets.forEach(run);
   }
 
+  async _getChildren(meta) {
+    const staticChildren = await parseChildren(meta);
+    const dynamicChildren = await parseChildren(meta, {
+      children: await states.blockletExtras.getSettings(meta.did, 'children', []),
+      dynamic: true,
+    });
+    checkDuplicateComponents(dynamicChildren, staticChildren);
+
+    return [...staticChildren, ...dynamicChildren];
+  }
+
+  async _getDynamicChildrenFromSettings(did) {
+    const children = await states.blockletExtras.getSettings(did, 'children', []);
+    return children.map((x) => ({ ...x, dynamic: true }));
+  }
+
   async _install({ meta, source, deployedFrom, context, sync }) {
     validateBlockletMeta(meta, { ensureDist: true });
 
     const { name, did, version } = meta;
 
-    const childrenMeta = await getChildrenMeta(meta);
-    mergeMeta(meta, childrenMeta);
+    const children = await this._getChildren(meta);
     try {
       const blocklet = await states.blocklet.addBlocklet({
         did: meta.did,
         meta,
         source,
         deployedFrom,
-        childrenMeta,
+        children,
       });
 
       await validateBlocklet(blocklet);
@@ -1597,6 +1858,7 @@ class BlockletManager extends BaseBlockletManager {
       // download
       const downloadParams = {
         blocklet: { ...blocklet1 },
+        oldBlocklet: { children: children.filter((x) => x.dynamic) }, // let downloader skip re-downloading dynamic blocklet
         context,
         postAction: 'install',
       };
@@ -1655,7 +1917,7 @@ class BlockletManager extends BaseBlockletManager {
   }
 
   async _upgrade({ meta, source, deployedFrom, context, sync }) {
-    validateBlockletMeta(meta, { ensureDist: true });
+    validateBlockletMeta(meta, { ensureDist: meta.group !== BlockletGroup.gateway });
 
     const { name, version, did } = meta;
 
@@ -1666,12 +1928,11 @@ class BlockletManager extends BaseBlockletManager {
     logger.info(`${action} blocklet`, { did, version });
 
     const newBlocklet = await states.blocklet.setBlockletStatus(did, BlockletStatus.waiting);
-    const childrenMeta = await getChildrenMeta(meta);
-    mergeMeta(meta, childrenMeta);
+
     newBlocklet.meta = meta;
     newBlocklet.source = source;
     newBlocklet.deployedFrom = deployedFrom;
-    newBlocklet.children = await states.blocklet.getChildrenFromMetas(childrenMeta);
+    newBlocklet.children = await this._getChildren(meta);
 
     await validateBlocklet(newBlocklet);
 
@@ -1774,7 +2035,7 @@ class BlockletManager extends BaseBlockletManager {
 
       await ensureBlockletExpanded(meta, downloadDir);
 
-      installDir = path.join(this.installDir, name, version);
+      installDir = getBundleDir(this.installDir, meta);
       if (fs.existsSync(installDir)) {
         fs.removeSync(installDir);
         logger.info('cleanup blocklet upgrade dir', { name, version, installDir });
@@ -1791,7 +2052,7 @@ class BlockletManager extends BaseBlockletManager {
     return { meta, installDir };
   }
 
-  async _resolveDiffDownload(cwd, tarFile, deleteSet, blocklet) {
+  async _resolveDiffDownload(cwd, tarFile, deleteSet, bundle) {
     logger.info('Resolve diff download', { tarFile, cwd });
     const downloadDir = path.join(cwd, `${path.basename(tarFile, path.extname(tarFile))}`);
     const diffDir = `${downloadDir}-diff`;
@@ -1804,7 +2065,7 @@ class BlockletManager extends BaseBlockletManager {
       throw error;
     }
     logger.info('Copy installDir to downloadDir', { installDir: this.installDir, downloadDir });
-    await fs.copy(path.join(this.installDir, blocklet.meta.name, blocklet.meta.version), downloadDir);
+    await fs.copy(getBundleDir(this.installDir, bundle), downloadDir);
     try {
       // delete
       logger.info('Delete files from downloadDir', { fileNum: deleteSet.length });
@@ -1833,11 +2094,11 @@ class BlockletManager extends BaseBlockletManager {
 
       await ensureBlockletExpanded(meta, downloadDir);
 
-      const installDir = path.join(this.installDir, meta.name, meta.version);
-      logger.info('Move downloadDir to installDir', { downloadDir, installDir });
-      await fs.move(downloadDir, installDir, { overwrite: true });
+      const bundleDir = getBundleDir(this.installDir, meta);
+      logger.info('Move downloadDir to installDir', { downloadDir, bundleDir });
+      await fs.move(downloadDir, bundleDir, { overwrite: true });
 
-      return { meta, installDir };
+      return { meta, installDir: bundleDir };
     } catch (error) {
       fs.removeSync(downloadDir);
       fs.removeSync(diffDir);
@@ -1868,7 +2129,7 @@ class BlockletManager extends BaseBlockletManager {
         hooks.preInstall({
           hooks: Object.assign(b.meta.hooks || {}, b.meta.scripts || {}),
           env: { ...nodeEnvironments },
-          appDir: blocklet.env.appDir,
+          appDir: b.env.appDir,
           did, // root blocklet did
           notification: states.notification,
           context,
@@ -1884,7 +2145,7 @@ class BlockletManager extends BaseBlockletManager {
         hooks.postInstall({
           hooks: Object.assign(b.meta.hooks || {}, b.meta.scripts || {}),
           env: getRuntimeEnvironments(b, nodeEnvironments, blocklet),
-          appDir: blocklet.env.appDir,
+          appDir: b.env.appDir,
           did, // root blocklet did
           notification: states.notification,
           context,
@@ -1968,7 +2229,7 @@ class BlockletManager extends BaseBlockletManager {
         hooks.preInstall({
           hooks: Object.assign(b.meta.hooks || {}, b.meta.scripts || {}),
           env: { ...nodeEnvironments },
-          appDir: blocklet.env.appDir,
+          appDir: b.env.appDir,
           did, // root blocklet did
           notification: states.notification,
           context,
@@ -2053,6 +2314,12 @@ class BlockletManager extends BaseBlockletManager {
         logger.error('emit upgrade notification failed', { name, version, error });
       }
 
+      // Update dynamic component meta in blocklet settings
+      const dynamicChildren = blocklet.children
+        .filter((x) => x.dynamic)
+        .map((x) => pick(x, ['meta', 'mountPoint', 'sourceUrl', 'source']));
+      await states.blockletExtras.setSettings(did, { children: dynamicChildren });
+
       return blocklet;
     } catch (err) {
       const b = await this._rollback(action, did, oldBlocklet);
@@ -2259,12 +2526,18 @@ class BlockletManager extends BaseBlockletManager {
 
     for (const child of blocklet.children) {
       const oldChild = oldChildren[child.meta.did];
-      if (!oldChild || oldChild.meta.dist.integrity !== child.meta.dist.integrity) {
+      if (
+        !oldChild ||
+        (![BlockletSource.upload, BlockletSource.local].includes(child.source) &&
+          child.sourceUrl &&
+          get(oldChild, 'meta.dist.integrity') !== get(child, 'meta.dist.integrity'))
+      ) {
         metas.push(child.meta);
       }
     }
 
     try {
+      logger.info('Download Blocklet', { name, did, bundles: metas.map((x) => get(x, 'dist.tarball')) });
       const tasks = [];
       for (const meta of metas) {
         const url = await this.registry.resolveTarballURL({
@@ -2332,7 +2605,7 @@ class BlockletManager extends BaseBlockletManager {
   async _rollback(action, did, oldBlocklet) {
     if (action === 'install') {
       // remove blocklet
-      return this._deleteBlocklet({ did, keepData: false });
+      return this._deleteBlocklet({ did, keepData: true });
     }
 
     if (['upgrade', 'downgrade'].includes(action)) {
@@ -2374,7 +2647,18 @@ class BlockletManager extends BaseBlockletManager {
     const result = await states.blocklet.deleteBlocklet(did);
     logger.info('blocklet removed successfully', { did });
 
-    this.emit(BlockletEvents.removed, { blocklet: result, context });
+    let keepRouting = true;
+    if (keepData === false || keepConfigs === false) {
+      keepRouting = false;
+    }
+
+    this.emit(BlockletEvents.removed, {
+      blocklet: result,
+      context: {
+        ...context,
+        keepRouting,
+      },
+    });
     return blocklet;
   }
 
@@ -2395,6 +2679,20 @@ class BlockletManager extends BaseBlockletManager {
       await states.blockletExtras.delChildConfigs(did, child.meta.did);
     }
   }
+
+  async _findNextCustomBlockletName(leftTimes = 10) {
+    if (leftTimes <= 0) {
+      throw new Error('Generate custom blocklet did too many times');
+    }
+    const number = await states.node.increaseCustomBlockletNumber();
+    const name = `custom-${number}`;
+    const did = toBlockletDid(name);
+    const blocklet = await states.blocklet.getBlocklet(did);
+    if (blocklet) {
+      return this._findNextCustomBlockletName(leftTimes - 1);
+    }
+    return { did, name };
+  }
 }
 
 module.exports = BlockletManager;