@abtnode/core 1.15.17 → 1.16.0-beta-8ee536d7

package/lib/util/blocklet.js

@@ -2,21 +2,46 @@
 
 const fs = require('fs-extra');
 const path = require('path');
+const joinURL = require('url-join');
+const shelljs = require('shelljs');
 const os = require('os');
 const tar = require('tar');
 const get = require('lodash/get');
+const isEmpty = require('lodash/isEmpty');
 const streamToPromise = require('stream-to-promise');
 const { Throttle } = require('stream-throttle');
 const ssri = require('ssri');
-
-const { toHex } = require('@ocap/util');
+const diff = require('deep-diff');
+const createArchive = require('archiver');
+const isUrl = require('is-url');
+const semver = require('semver');
+const axios = require('@abtnode/util/lib/axios');
+const { stableStringify } = require('@arcblock/vc');
+
+const { fromSecretKey, WalletType } = require('@ocap/wallet');
+const { toHex, toBase58, isHex, toDid } = require('@ocap/util');
+const { types } = require('@ocap/mcrypto');
+const { isValid: isValidDid } = require('@arcblock/did');
 const logger = require('@abtnode/logger')('@abtnode/core:util:blocklet');
 const pm2 = require('@abtnode/util/lib/async-pm2');
 const sleep = require('@abtnode/util/lib/sleep');
+const { formatEnv } = require('@abtnode/util/lib/security');
 const ensureEndpointHealthy = require('@abtnode/util/lib/ensure-endpoint-healthy');
 const CustomError = require('@abtnode/util/lib/custom-error');
 const getFolderSize = require('@abtnode/util/lib/get-folder-size');
-const { BLOCKLET_MAX_MEM_LIMIT_IN_MB } = require('@abtnode/constant');
+const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+const hashFiles = require('@abtnode/util/lib/hash-files');
+const {
+  BLOCKLET_MAX_MEM_LIMIT_IN_MB,
+  BLOCKLET_STORE,
+  BLOCKLET_INSTALL_TYPE,
+  BLOCKLET_STORE_DEV,
+  APP_STRUCT_VERSION,
+} = require('@abtnode/constant');
+const formatBackSlash = require('@abtnode/util/lib/format-back-slash');
+
+const SCRIPT_ENGINES_WHITE_LIST = ['npm', 'npx', 'pnpm', 'yarn'];
+
 const {
   BlockletStatus,
   BlockletSource,
@@ -29,27 +54,53 @@ const {
   BLOCKLET_INTERFACE_TYPE_WEB,
   BLOCKLET_CONFIGURABLE_KEY,
   fromBlockletStatus,
-} = require('@blocklet/meta/lib/constants');
-const verifyMultiSig = require('@blocklet/meta/lib/verify-multi-sig');
+  BLOCKLET_PREFERENCE_FILE,
+  BLOCKLET_PREFERENCE_PREFIX,
+} = require('@blocklet/constant');
 const validateBlockletEntry = require('@blocklet/meta/lib/entry');
 const getBlockletEngine = require('@blocklet/meta/lib/engine');
 const getBlockletInfo = require('@blocklet/meta/lib/info');
+const getBlockletWallet = require('@blocklet/meta/lib/wallet');
 const { validateMeta, fixAndValidateService } = require('@blocklet/meta/lib/validate');
-const { forEachBlocklet } = require('@blocklet/meta/lib/util');
+const {
+  forEachBlocklet,
+  getDisplayName,
+  findWebInterface,
+  forEachBlockletSync,
+  forEachChildSync,
+  isComponentBlocklet,
+  getSharedConfigObj,
+  getComponentName,
+  isEnvShareable,
+} = require('@blocklet/meta/lib/util');
+const toBlockletDid = require('@blocklet/meta/lib/did');
+const { titleSchema, descriptionSchema, logoSchema } = require('@blocklet/meta/lib/schema');
+const {
+  getSourceUrlsFromConfig,
+  getBlockletMetaFromUrls,
+  getBlockletMetaFromUrl,
+} = require('@blocklet/meta/lib/util-meta');
+const getComponentProcessId = require('@blocklet/meta/lib/get-component-process-id');
 
 const { validate: validateEngine, get: getEngine } = require('../blocklet/manager/engine');
 
 const isRequirementsSatisfied = require('./requirement');
-const { getServices } = require('./service');
+const { getDidDomainForBlocklet } = require('./get-domain-for-blocklet');
 const {
   isBeforeInstalled,
   expandBundle,
-  getBlockletMetaByUrl,
-  validateUrl,
   findInterfacePortByName,
-  validateBlockletMeta,
+  prettyURL,
+  getNFTState,
+  templateReplace,
 } = require('./index');
 
+const getComponentConfig = (meta) => {
+  const components = meta.components || meta.children || [];
+  const staticComponents = (meta.staticComponents || []).map((x) => ({ ...x, static: true }));
+  return [...components, ...staticComponents];
+};
+
 /**
  * get blocklet engine info, default is node
  * @param {object} blockletMeta blocklet meta
@@ -58,8 +109,7 @@ const {
 const getBlockletEngineNameByPlatform = (blockletMeta) => getBlockletEngine(blockletMeta).interpreter;
 
 const noop = () => {};
-
-const asyncFs = fs.promises;
+const noopAsync = async () => {};
 
 const statusMap = {
   online: BlockletStatus.running,
@@ -69,106 +119,110 @@ const statusMap = {
   stopped: BlockletStatus.stopped,
 };
 
-const nodePrivateEnvs = [
-  // 'NEDB_MULTI_PORT', // FIXME: 排查 abtnode 对外提供的 SDK(比如 @abtnode/queue), SDK 中不要自动使用 NEDB_MULTI_PORT 环境变量
+const PRIVATE_NODE_ENVS = [
+  'NEDB_MULTI_PORT',
   'ABT_NODE_UPDATER_PORT',
   'ABT_NODE_SESSION_TTL',
-  'ABT_NODE_MODE',
   'ABT_NODE_ROUTER_PROVIDER',
   'ABT_NODE_DATA_DIR',
   'ABT_NODE_TOKEN_SECRET',
   'ABT_NODE_SK',
   'ABT_NODE_SESSION_SECRET',
-  'ABT_NODE_NAME',
-  'ABT_NODE_DESCRIPTION',
   'ABT_NODE_BASE_URL',
   'ABT_NODE_LOG_LEVEL',
   'ABT_NODE_LOG_DIR',
+  // in /core/cli/bin/blocklet.js
+  'CLI_MODE',
+  'ABT_NODE_HOME',
+  'PM2_HOME',
+  'ABT_NODE_CONFIG_FILE',
 ];
 
 /**
  * @returns { dataDir, logsDir, cacheDir, appMain, appDir, mainDir, appCwd }
- * dataDir: dataDirs.data/name (root blocklet) or dataDirs.data/name/childName (child blocklet)
+ * dataDir: dataDirs.data/name (root component) or dataDirs.data/name/childName (child component)
  * logsDir: dataDirs.log/name
- * cacheDir: dataDirs.cache/name (root blocklet) or dataDirs.cache/name/childName (child blocklet)
- * appDir: blocklet bundle dir
- * mainDir: appDir (dapp) or appDir/main (static). Used for for static blocklet
- * appMain: app entry file or script (run appMain to start blocklet process)
+ * cacheDir: dataDirs.cache/name (root component) or dataDirs.cache/name/childName (child component)
+ * appDir: component bundle dir
+ * mainDir: appDir (dapp) or appDir/main (static). Used for for static component
+ * appMain: app entry file or script (run appMain to start component process)
  * appCwd: cwd of appMain
  */
-const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false } = {}) => {
-  if (!rootBlocklet) {
-    // eslint-disable-next-line no-param-reassign
-    rootBlocklet = blocklet;
-  }
+const getComponentDirs = (
+  component,
+  { dataDirs, ensure = false, e2eMode = false, validate = false, ancestors = [] } = {}
+) => {
+  // FIXME 这个函数做了太多的事
   // get data dirs
 
-  const { name } = blocklet.meta;
-
-  let logsDir = path.join(dataDirs.logs, name);
-  let dataDir = path.join(dataDirs.data, name);
-  let cacheDir = path.join(dataDirs.cache, name);
-  if (rootBlocklet !== blocklet) {
-    dataDir = path.join(dataDirs.data, rootBlocklet.meta.name, name);
-    cacheDir = path.join(dataDirs.cache, rootBlocklet.meta.name, name);
-    logsDir = path.join(dataDirs.logs, rootBlocklet.meta.name);
-  }
+  const { name: appName } = ancestors.concat(component)[0].meta;
+  const componentName = getComponentName(component, ancestors);
 
-  if (ensure) {
-    try {
-      fs.mkdirSync(dataDir, { recursive: true });
-      fs.mkdirSync(logsDir, { recursive: true });
-      fs.mkdirSync(cacheDir, { recursive: true });
-    } catch (err) {
-      logger.error('make blocklet dir failed', { error: err });
-    }
-  }
+  const logsDir = path.join(dataDirs.logs, appName);
+  const dataDir = path.join(dataDirs.data, componentName);
+  const cacheDir = path.join(dataDirs.cache, componentName);
 
   // get app dirs
 
-  const { version, main, group } = blocklet.meta;
+  const { main, group } = component.meta;
 
-  const startFromDevEntry =
-    blocklet.mode === BLOCKLET_MODES.DEVELOPMENT && blocklet.meta.scripts && blocklet.meta.scripts.dev;
+  let startFromDevEntry = '';
+  if (component.mode === BLOCKLET_MODES.DEVELOPMENT && component.meta.scripts) {
+    startFromDevEntry = component.meta.scripts.dev;
+    if (e2eMode && component.meta.scripts.e2eDev) {
+      startFromDevEntry = component.meta.scripts.e2eDev;
+    }
+  }
 
-  if (!main && !startFromDevEntry && group !== BlockletGroup.gateway) {
-    throw new Error('Incorrect blocklet manifest: missing `main` field');
+  if (validate && !main && !startFromDevEntry && group !== BlockletGroup.gateway) {
+    throw new Error('Incorrect blocklet.yml: missing `main` field');
   }
 
   let appDir = null;
   let mainDir = null;
 
-  if (blocklet.source === BlockletSource.local) {
-    appDir = blocklet.deployedFrom;
+  if (component.source === BlockletSource.local) {
+    appDir = component.deployedFrom;
   } else {
-    appDir = path.join(dataDirs.blocklets, name, version);
+    appDir = getBundleDir(dataDirs.blocklets, component.meta);
   }
 
   if (!appDir) {
     throw new Error('Can not determine blocklet directory, maybe invalid deployment from local blocklets');
   }
 
+  if (ensure) {
+    try {
+      fs.mkdirSync(dataDir, { recursive: true });
+      fs.mkdirSync(logsDir, { recursive: true });
+      fs.mkdirSync(cacheDir, { recursive: true });
+      fs.mkdirSync(appDir, { recursive: true }); // prevent getDiskInfo failed from custom blocklet
+    } catch (err) {
+      logger.error('make blocklet dir failed', { error: err });
+    }
+  }
+
   mainDir = appDir;
 
-  if (!startFromDevEntry && !isBeforeInstalled(rootBlocklet.status)) {
+  if (validate && !startFromDevEntry && !isBeforeInstalled(component.status)) {
     try {
-      validateBlockletEntry(appDir, blocklet.meta);
+      validateBlockletEntry(appDir, component.meta);
     } catch (err) {
       throw new CustomError('BLOCKLET_CORRUPTED', err.message);
     }
   }
 
-  if (!BLOCKLET_GROUPS.includes(group)) {
+  if (validate && !BLOCKLET_GROUPS.includes(group)) {
     throw new CustomError('BLOCKLET_CORRUPTED', `Unsupported blocklet type ${group}`);
   }
 
   let appMain = null;
   let appCwd = null;
   if (startFromDevEntry) {
-    appMain = blocklet.meta.scripts.dev;
+    appMain = startFromDevEntry;
     appCwd = appDir;
   } else if (group === 'dapp') {
-    appMain = getBlockletEngine(blocklet.meta).script || BLOCKLET_ENTRY_FILE;
+    appMain = getBlockletEngine(component.meta).script || BLOCKLET_ENTRY_FILE;
     appCwd = appDir;
   } else if (group === 'static') {
     mainDir = path.join(appDir, main);
@@ -182,44 +236,40 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false } =
 };
 
 /**
- * set 'configs', configObj', 'environmentObj' to blocklet
+ * set 'configs', configObj', 'environmentObj' to blocklet TODO
  * @param {*} blocklet
  * @param {*} configs
  */
 const fillBlockletConfigs = (blocklet, configs) => {
   blocklet.configs = configs || [];
   blocklet.configObj = blocklet.configs.reduce((acc, x) => {
-    acc[x.key] = x.value;
+    acc[x.key] = templateReplace(x.value, blocklet);
     return acc;
   }, {});
-  blocklet.userEnvironments = blocklet.configObj; // deprecated
-  blocklet.environmentObj = (blocklet.environments || []).reduce((acc, x) => {
-    acc[x.key] = x.value;
+  blocklet.environments = blocklet.environments || [];
+  blocklet.environmentObj = blocklet.environments.reduce((acc, x) => {
+    acc[x.key] = templateReplace(x.value, blocklet);
     return acc;
   }, {});
 };
 
 const ensureBlockletExpanded = async (meta, appDir) => {
-  const { main } = meta;
-
-  if (main === BLOCKLET_BUNDLE_FILE) {
-    const bundlePath = path.join(appDir, BLOCKLET_BUNDLE_FILE);
-    if (fs.existsSync(bundlePath)) {
-      try {
-        const nodeModulesPath = path.join(appDir, 'node_modules');
-        if (fs.existsSync(nodeModulesPath)) {
-          fs.removeSync(nodeModulesPath);
-        }
-        await expandBundle(bundlePath, appDir);
-        fs.removeSync(bundlePath);
-      } catch (err) {
-        throw new Error(`Failed to expand blocklet bundle: ${err.message}`);
+  const bundlePath = path.join(appDir, BLOCKLET_BUNDLE_FILE);
+  if (fs.existsSync(bundlePath)) {
+    try {
+      const nodeModulesPath = path.join(appDir, 'node_modules');
+      if (fs.existsSync(nodeModulesPath)) {
+        fs.removeSync(nodeModulesPath);
       }
+      await expandBundle(bundlePath, appDir);
+      fs.removeSync(bundlePath);
+    } catch (err) {
+      throw new Error(`Failed to expand blocklet bundle: ${err.message}`);
     }
   }
 };
 
-const getRootSystemEnvironments = (blocklet, nodeInfo) => {
+const getAppSystemEnvironments = (blocklet, nodeInfo) => {
   const { did, name, title, description } = blocklet.meta;
   const keys = Object.keys(BLOCKLET_CONFIGURABLE_KEY);
   const result = getBlockletInfo(
@@ -236,16 +286,41 @@ const getRootSystemEnvironments = (blocklet, nodeInfo) => {
   const appName = title || name || result.name;
   const appDescription = description || result.description;
 
+  const isMigrated = Array.isArray(blocklet.migratedFrom) && blocklet.migratedFrom.length > 0;
+  const appPid = blocklet.appPid || appId;
+  const appPsk = isMigrated ? blocklet.migratedFrom[0].appSk : appSk;
+
+  /* 获取 did domain 方式:
+   * 1. 先从 site 里读
+   * 2. 如果没有，再拼接
+   */
+
+  let appUrl = '';
+
+  const domainAliases = get(blocklet, 'site.domainAliases') || [];
+  const didDomainAlias = domainAliases.find(
+    (item) => item.value.endsWith(nodeInfo.didDomain) || item.value.endsWith('did.staging.arcblock.io') // did.staging.arcblock.io 是旧 did domain, 但主要存在于比较旧的节点中, 需要做兼容
+  );
+
+  if (didDomainAlias) {
+    appUrl = prettyURL(didDomainAlias.value, true);
+  } else {
+    appUrl = `https://${getDidDomainForBlocklet({ appPid, didDomain: nodeInfo.didDomain })}`;
+  }
+
   return {
-    BLOCKLET_DID: did,
+    BLOCKLET_DID: did, // BLOCKLET_DID is always same as BLOCKLET_APP_PID in structV2 application
     BLOCKLET_APP_SK: appSk,
     BLOCKLET_APP_ID: appId,
+    BLOCKLET_APP_PSK: appPsk, // permanent sk even the blocklet has been migrated
+    BLOCKLET_APP_PID: appPid, // permanent did even the blocklet has been migrated
     BLOCKLET_APP_NAME: appName,
     BLOCKLET_APP_DESCRIPTION: appDescription,
+    BLOCKLET_APP_URL: appUrl,
   };
 };
 
-const getOverwrittenEnvironments = (blocklet, nodeInfo) => {
+const getAppOverwrittenEnvironments = (blocklet, nodeInfo) => {
   const result = {};
   if (!blocklet || !blocklet.configObj) {
     return result;
@@ -278,7 +353,7 @@ const getOverwrittenEnvironments = (blocklet, nodeInfo) => {
   return result;
 };
 
-const getSystemEnvironments = (blocklet) => {
+const getComponentSystemEnvironments = (blocklet) => {
   const { port, ports } = blocklet;
   const portEnvironments = {};
   if (port) {
@@ -287,33 +362,97 @@ const getSystemEnvironments = (blocklet) => {
   if (ports) {
     Object.assign(portEnvironments, ports);
   }
+
   return {
+    BLOCKLET_REAL_DID: blocklet.env.id,
+    BLOCKLET_REAL_NAME: blocklet.env.name,
     BLOCKLET_DATA_DIR: blocklet.env.dataDir,
     BLOCKLET_LOG_DIR: blocklet.env.logsDir,
     BLOCKLET_CACHE_DIR: blocklet.env.cacheDir,
-    BLOCKLET_REAL_DID: blocklet.meta.did,
     BLOCKLET_APP_DIR: blocklet.env.appDir,
     BLOCKLET_MAIN_DIR: blocklet.env.mainDir,
     ...portEnvironments,
   };
 };
 
-const getRuntimeEnvironments = (blocklet, nodeEnvironments) => {
+const getRuntimeEnvironments = (blocklet, nodeEnvironments, ancestors) => {
+  const root = (ancestors || [])[0] || blocklet;
+
   // pm2 will force inject env variables of daemon process to blocklet process
   // we can only rewrite these private env variables to empty
-  const safeNodeEnvironments = nodePrivateEnvs.reduce((o, x) => {
+  const safeNodeEnvironments = PRIVATE_NODE_ENVS.reduce((o, x) => {
     o[x] = '';
     return o;
   }, {});
 
-  return {
+  // get devEnvironments, when blocklet is in dev mode
+  const devEnvironments =
+    blocklet.mode === BLOCKLET_MODES.DEVELOPMENT
+      ? {
+          BLOCKLET_DEV_MOUNT_POINT: blocklet?.mountPoint || '',
+        }
+      : {};
+
+  // BLOCKLET_DEV_PORT should NOT in components of production mode
+  if (process.env.BLOCKLET_DEV_PORT) {
+    devEnvironments.BLOCKLET_DEV_PORT =
+      blocklet.mode === BLOCKLET_MODES.DEVELOPMENT ? process.env.BLOCKLET_DEV_PORT : '';
+  }
+
+  const ports = {};
+  forEachBlockletSync(root, (x) => {
+    const webInterface = findWebInterface(x);
+    if (webInterface && x.environmentObj[webInterface.port]) {
+      ports[x.environmentObj.BLOCKLET_REAL_NAME] = x.environmentObj[webInterface.port];
+    }
+  });
+
+  const mountPoints = [];
+  for (const x of root.children || []) {
+    const mountPoint = {
+      title: x.meta.title,
+      did: x.meta.bundleDid,
+      name: x.meta.bundleName,
+      mountPoint: x.mountPoint || '',
+    };
+
+    const webInterface = findWebInterface(x);
+    if (webInterface && x.environmentObj[webInterface.port]) {
+      mountPoint.port = x.environmentObj[webInterface.port];
+    }
+
+    mountPoints.push(mountPoint);
+  }
+
+  // use index 1 as the path to derive deterministic encryption key for blocklet
+  const tmp = get(nodeEnvironments, 'ABT_NODE_SK')
+    ? getBlockletWallet(blocklet.meta.did, nodeEnvironments.ABT_NODE_SK, undefined, 1)
+    : null;
+
+  const env = {
+    ...blocklet.configObj,
+    ...getSharedConfigObj(blocklet, ancestors),
     ...blocklet.environmentObj,
+    ...devEnvironments,
+    BLOCKLET_WEB_PORTS: JSON.stringify(ports),
+    BLOCKLET_MOUNT_POINTS: JSON.stringify(mountPoints),
+    BLOCKLET_MODE: blocklet.mode || BLOCKLET_MODES.PRODUCTION,
+    BLOCKLET_APP_EK: tmp?.secretKey,
+    BLOCKLET_APP_VERSION: root.meta.version,
     ...nodeEnvironments,
     ...safeNodeEnvironments,
   };
+
+  // ensure all envs are literals and do not contain line breaks
+  Object.keys(env).forEach((key) => {
+    env[key] = formatEnv(env[key]);
+  });
+
+  return env;
 };
 
-const isUsefulError = (err) => err && err.message !== 'process or namespace not found';
+const isUsefulError = (err) =>
+  err && err.message !== 'process or namespace not found' && !/^Process \d+ not found$/.test(err.message);
 
 const getHealthyCheckTimeout = (blocklet, { checkHealthImmediately } = {}) => {
   let minConsecutiveTime = 5000;
@@ -346,68 +485,51 @@ const getHealthyCheckTimeout = (blocklet, { checkHealthImmediately } = {}) => {
   };
 };
 
-const getBlockletMetaFromUrl = async (url) => {
-  const meta = await getBlockletMetaByUrl(url);
-  delete meta.htmlAst;
-
-  validateBlockletMeta(meta, { ensureDist: true });
-
-  try {
-    const { href } = new URL(meta.dist.tarball, url);
-    const tarball = decodeURIComponent(href);
-
-    try {
-      await validateUrl(tarball, ['application/octet-stream', 'application/x-gzip']);
-    } catch (error) {
-      if (!error.message.startsWith('Cannot get content-type')) {
-        throw error;
-      }
-    }
-    logger.info('resolve tarball url base on meta url', { meta: url, tarball });
-
-    meta.dist.tarball = tarball;
-  } catch (err) {
-    throw new Error(`Invalid blocklet meta: dist.tarball is not a valid url ${err.message}`);
-  }
-
-  return meta;
-};
-
 /**
  * Start all precesses of a blocklet
  * @param {*} blocklet should contain env props
  */
-const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironments, nodeInfo } = {}) => {
+const startBlockletProcess = async (
+  blocklet,
+  { preStart = noop, postStart = noopAsync, nodeEnvironments, nodeInfo, e2eMode, skippedProcessIds = [] } = {}
+) => {
   if (!blocklet) {
     throw new Error('blocklet should not be empty');
   }
 
   await forEachBlocklet(
     blocklet,
-    async (b) => {
+    async (b, { ancestors }) => {
       if (b.meta.group === BlockletGroup.gateway) {
         return;
       }
 
-      const { appMain, appId, appCwd, logsDir } = b.env;
+      const { appMain, processId, appCwd, logsDir } = b.env;
+
+      if (skippedProcessIds.includes(processId)) {
+        logger.info(`skip start process ${processId}`);
+        return;
+      }
 
       // get env
-      const env = getRuntimeEnvironments(b, nodeEnvironments);
+      const env = getRuntimeEnvironments(b, nodeEnvironments, ancestors);
 
       // run hook
-      await preStart(b);
+      await preStart(b, { env });
 
       // start process
       const maxMemoryRestart = get(nodeInfo, 'runtimeConfig.blockletMaxMemoryLimit', BLOCKLET_MAX_MEM_LIMIT_IN_MB);
 
       const options = {
         namespace: 'blocklets',
-        name: appId,
-        max_memory_restart: `${maxMemoryRestart}M`,
+        name: processId,
+        cwd: appCwd,
         time: true,
         output: path.join(logsDir, 'output.log'),
         error: path.join(logsDir, 'error.log'),
-        cwd: appCwd,
+        wait_ready: process.env.NODE_ENV !== 'test',
+        listen_timeout: 5000,
+        max_memory_restart: `${maxMemoryRestart}M`,
         max_restarts: b.mode === BLOCKLET_MODES.DEVELOPMENT ? 0 : 3,
         env: {
           ...env,
@@ -417,17 +539,35 @@ const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironment
 
       const clusterMode = get(b.meta, 'capabilities.clusterMode', false);
       if (clusterMode && blocklet.mode !== BLOCKLET_MODES.DEVELOPMENT) {
-        const clusterSize = Number(blocklet.configObj.BLOCKLET_CLUSTER_SIZE) || Number.POSITIVE_INFINITY;
+        const clusterSize = Number(blocklet.configObj.BLOCKLET_CLUSTER_SIZE) || +process.env.ABT_NODE_MAX_CLUSTER_SIZE;
         options.execMode = 'cluster';
         options.mergeLogs = true;
         options.instances = Math.min(os.cpus().length, clusterSize);
       }
 
       if (b.mode === BLOCKLET_MODES.DEVELOPMENT) {
-        options.env.NODE_ENV = 'development';
+        options.env.NODE_ENV = e2eMode ? 'e2e' : 'development';
         options.env.BROWSER = 'none';
         options.env.PORT = options.env[BLOCKLET_DEFAULT_PORT_NAME];
         options.script = appMain;
+
+        if (process.platform === 'win32') {
+          const [cmd, ...args] = options.script.split(' ').filter(Boolean);
+
+          if (!SCRIPT_ENGINES_WHITE_LIST.includes(cmd)) {
+            throw new Error(`${cmd} script is not supported, ${SCRIPT_ENGINES_WHITE_LIST.join(', ')} are supported`);
+          }
+
+          const { stdout: nodejsBinPath } = shelljs.which('node');
+
+          const cmdPath = path.join(path.dirname(nodejsBinPath), 'node_modules', cmd);
+
+          const pkg = JSON.parse(fs.readFileSync(path.join(cmdPath, 'package.json')));
+          const cmdBinPath = pkg.bin[cmd];
+
+          options.script = path.resolve(cmdPath, cmdBinPath);
+          options.args = [...args].join(' ');
+        }
       } else {
         const blockletEngineInfo = getBlockletEngine(b.meta);
         options.args = blockletEngineInfo.args || [];
@@ -435,16 +575,20 @@ const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironment
         const engine = getEngine(blockletEngineInfo.interpreter);
         options.interpreter = engine.interpreter === 'node' ? '' : engine.interpreter;
         options.interpreterArgs = engine.args || '';
-
         options.script = blockletEngineInfo.script || appMain;
-
-        logger.debug('start.blocklet.engine.info', { blockletEngineInfo });
-        logger.debug('start.blocklet.max_memory_restart', { maxMemoryRestart });
       }
 
       await pm2.startAsync(options);
-      logger.info('blocklet started', {
-        appId,
+
+      const status = await getProcessState(processId);
+      if (status === BlockletStatus.error) {
+        throw new Error(`${processId} is not running within 5 seconds`);
+      }
+      logger.info('blocklet started', { processId, status });
+
+      // run hook
+      postStart(b, { env }).catch((err) => {
+        logger.error('blocklet post start failed', { processId, error: err });
       });
     },
     { parallel: true }
@@ -455,12 +599,17 @@ const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironment
  * Stop all precesses of a blocklet
  * @param {*} blocklet should contain env props
  */
-const stopBlockletProcess = async (blocklet, { preStop = noop } = {}) => {
+const stopBlockletProcess = async (blocklet, { preStop = noop, skippedProcessIds = [] } = {}) => {
   await forEachBlocklet(
     blocklet,
-    async (b) => {
-      await preStop(b);
-      await deleteProcess(b.env.appId);
+    async (b, { ancestors }) => {
+      if (skippedProcessIds.includes(b.env.processId)) {
+        logger.info(`skip stop process ${b.env.processId}`);
+        return;
+      }
+
+      await preStop(b, { ancestors });
+      await deleteProcess(b.env.processId);
     },
     { parallel: true }
   );
@@ -470,12 +619,22 @@ const stopBlockletProcess = async (blocklet, { preStop = noop } = {}) => {
  * Delete all precesses of a blocklet
  * @param {*} blocklet should contain env props
  */
-const deleteBlockletProcess = async (blocklet, { preDelete = noop } = {}) => {
+const deleteBlockletProcess = async (blocklet, { preDelete = noop, skippedProcessIds = [] } = {}) => {
   await forEachBlocklet(
     blocklet,
-    async (b) => {
-      await preDelete(b);
-      await deleteProcess(b.env.appId);
+    async (b, { ancestors }) => {
+      // NOTICE: 如果不判断 group, 在 github action 中测试 disk.spec.js 时会报错, 但是在 mac 中跑测试不会报错
+      if (b.meta?.group === BlockletGroup.gateway) {
+        return;
+      }
+
+      if (skippedProcessIds.includes(b.env.processId)) {
+        logger.info(`skip delete process ${b.env.processId}`);
+        return;
+      }
+
+      await preDelete(b, { ancestors });
+      await deleteProcess(b.env.processId);
     },
     { parallel: true }
   );
@@ -492,8 +651,8 @@ const reloadBlockletProcess = async (blocklet) =>
       if (b.meta.group === BlockletGroup.gateway) {
         return;
       }
-      logger.info('reload process', { appId: b.env.appId });
-      await reloadProcess(b.env.appId);
+      logger.info('reload process', { processId: b.env.processId });
+      await reloadProcess(b.env.processId);
     },
     { parallel: false }
   );
@@ -502,12 +661,16 @@ const getBlockletStatusFromProcess = async (blocklet) => {
   const tasks = [];
   await forEachBlocklet(blocklet, (b) => {
     if (b.meta.group !== BlockletGroup.gateway) {
-      tasks.push(getProcessState(b.env.appId));
+      tasks.push(getProcessState(b.env.processId));
     }
   });
 
   const list = await Promise.all(tasks);
 
+  if (!list.length) {
+    return blocklet.status;
+  }
+
   return getRootBlockletStatus(list);
 };
 
@@ -528,11 +691,11 @@ const getRootBlockletStatus = (statusList = []) => {
 };
 
 /**
- * @param {*} appId
+ * @param {*} processId
  * @returns {BlockletStatus}
  */
-const getProcessState = async (appId) => {
-  const info = await getProcessInfo(appId);
+const getProcessState = async (processId) => {
+  const info = await getProcessInfo(processId);
   if (!statusMap[info.pm2_env.status]) {
     logger.error('Cannot find the blocklet status for pm2 status mapping', {
       pm2Status: info.pm2_env.status,
@@ -544,9 +707,9 @@ const getProcessState = async (appId) => {
   return statusMap[info.pm2_env.status];
 };
 
-const getProcessInfo = (appId) =>
+const getProcessInfo = (processId) =>
   new Promise((resolve, reject) => {
-    pm2.describe(appId, async (err, [info]) => {
+    pm2.describe(processId, async (err, [info]) => {
       if (err) {
         logger.error('Failed to get blocklet status from pm2', { error: err });
         return reject(err);
@@ -560,22 +723,20 @@ const getProcessInfo = (appId) =>
     });
   });
 
-const deleteProcess = (appId) =>
+const deleteProcess = (processId) =>
   new Promise((resolve, reject) => {
-    pm2.connect(() => {
-      pm2.delete(appId, async (err) => {
-        if (isUsefulError(err)) {
-          logger.error('blocklet process delete failed', { error: err });
-          return reject(err);
-        }
-        return resolve();
-      });
+    pm2.delete(processId, async (err) => {
+      if (isUsefulError(err)) {
+        logger.error('blocklet process delete failed', { error: err });
+        return reject(err);
+      }
+      return resolve();
     });
   });
 
-const reloadProcess = (appId) =>
+const reloadProcess = (processId) =>
   new Promise((resolve, reject) => {
-    pm2.reload(appId, async (err) => {
+    pm2.reload(processId, async (err) => {
       if (err) {
         if (isUsefulError(err)) {
           logger.error('blocklet reload failed', { error: err });
@@ -586,30 +747,130 @@ const reloadProcess = (appId) =>
     });
   });
 
-const getChildrenMeta = async (meta) => {
-  const children = [];
-  if (meta.children && meta.children.length) {
-    for (const child of meta.children) {
-      const m = await getBlockletMetaFromUrl(child.resolved);
-      if (m.name !== child.name) {
-        logger.error('Resolved child blocklet name does not match in the configuration', {
-          expected: child.name,
-          resolved: m.name,
-        });
-        throw new Error(
-          `Child blocklet name does not match in the configuration. expected: ${child.name}, resolved: ${m.name}`
-        );
+/**
+ * this function has side effect to component (will set component.children: Array<staticComponent> )
+ * this function has side effect to dynamicComponents (will push dynamicComponent in dynamicComponents)
+ *
+ * @param {Component} component
+ * @param {{
+ *   ancestors: Array<{meta}>
+ *   dynamicComponents: Array<{Component}>
+ * }} context
+ * @returns {{
+ *   dynamicComponents: Array<Component>,
+ *   staticComponents: Array<Component>,
+ * }}
+ */
+const parseComponents = async (component, context = {}) => {
+  const { ancestors = [], dynamicComponents = [] } = context;
+  if (ancestors.length > 40) {
+    throw new Error('The depth of component should not exceed 40');
+  }
+
+  const configs = getComponentConfig(component.meta) || [];
+
+  if (!configs || !configs.length) {
+    return {
+      staticComponents: [],
+      dynamicComponents,
+    };
+  }
+
+  const staticComponents = [];
+
+  // FIXME @linchen 改成并行获取
+  for (const config of configs) {
+    const isStatic = !!config.static;
+
+    if (isStatic) {
+      if (!config.name) {
+        throw new Error(`Name does not found in child ${config.name}`);
+      }
+
+      if (!config.mountPoint) {
+        throw new Error(`MountPoint does not found in child ${config.name}`);
       }
-      validateBlockletMeta(m, { ensureDist: true });
-      children.push(m);
     }
+
+    const urls = getSourceUrlsFromConfig(config);
+
+    let rawMeta;
+    try {
+      rawMeta = await getBlockletMetaFromUrls(urls, { logger });
+    } catch (error) {
+      throw new Error(
+        `Failed get component meta. Component: ${rawMeta.title || rawMeta.did}, reason: ${error.message}`
+      );
+    }
+
+    if (rawMeta.group === BlockletGroup.gateway) {
+      throw new Error(`Cannot add gateway component ${rawMeta.title || rawMeta.did}`);
+    }
+
+    validateBlockletMeta(rawMeta, { ensureDist: true });
+
+    if (!isComponentBlocklet(rawMeta)) {
+      throw new Error(`The blocklet cannot be a component: ${rawMeta.title}`);
+    }
+
+    const webInterface = findWebInterface(rawMeta);
+    if (!webInterface) {
+      throw new Error(`Web interface does not found in component ${rawMeta.title || rawMeta.name}`);
+    }
+
+    const meta = ensureMeta(rawMeta, { name: isStatic ? config.name : null });
+
+    // check circular dependencies
+    if (ancestors.map((x) => x.meta?.bundleDid).indexOf(meta.bundleDid) > -1) {
+      throw new Error('Blocklet components have circular dependencies');
+    }
+
+    if (config.title) {
+      meta.title = config.title;
+      meta.title = await titleSchema.validateAsync(config.title);
+    }
+
+    if (config.description) {
+      meta.description = await descriptionSchema.validateAsync(config.description);
+    }
+
+    const mountPoint = isStatic ? config.mountPoint : config.mountPoint || `/${meta.did}`;
+
+    const child = {
+      mountPoint,
+      meta,
+      bundleSource: config.source,
+      dependencies: [],
+    };
+
+    if (isStatic) {
+      staticComponents.push(child);
+    } else {
+      dynamicComponents.push(child);
+      component.dependencies = component.dependencies || [];
+      component.dependencies.push({
+        did: child.meta.bundleDid,
+        required: !!config.required,
+        version: config.source.version || 'latest',
+      });
+    }
+
+    await parseComponents(child, {
+      ancestors: [...ancestors, { meta }],
+      dynamicComponents,
+    });
   }
 
-  return children;
+  component.children = staticComponents;
+
+  return {
+    staticComponents,
+    dynamicComponents,
+  };
 };
 
 const validateBlocklet = (blocklet) =>
-  forEachBlocklet(blocklet, (b) => {
+  forEachBlocklet(blocklet, async (b) => {
     isRequirementsSatisfied(b.meta.requirements);
     validateEngine(getBlockletEngineNameByPlatform(b.meta));
   });
@@ -629,7 +890,7 @@ const checkBlockletProcessHealthy = async (blocklet, { minConsecutiveTime, timeo
 const _checkProcessHealthy = async (blocklet, { minConsecutiveTime, timeout }) => {
   const { meta, ports, env } = blocklet;
   const { name } = meta;
-  const { appId } = env;
+  const { processId } = env;
 
   const webInterface = (meta.interfaces || []).find((x) => x.type === BLOCKLET_INTERFACE_TYPE_WEB);
   if (!webInterface) {
@@ -641,10 +902,10 @@ const _checkProcessHealthy = async (blocklet, { minConsecutiveTime, timeout }) =
     // ensure pm2 status is 'online'
     const getStatus = async () => {
       try {
-        const info = await getProcessInfo(appId);
+        const info = await getProcessInfo(processId);
         return info.pm2_env.status;
       } catch (err) {
-        logger.error('blocklet checkStart error', { error: err, appId, name });
+        logger.error('blocklet checkStart error', { error: err, processId, name });
         return '';
       }
     };
@@ -673,7 +934,7 @@ const _checkProcessHealthy = async (blocklet, { minConsecutiveTime, timeout }) =
       throw error;
     }
   } catch (error) {
-    logger.error('start blocklet failed', { appId, name });
+    logger.error('start blocklet failed', { processId, name });
     throw error;
   }
 };
@@ -703,7 +964,60 @@ const verifyIntegrity = async ({ file, integrity: expected }) => {
   return true;
 };
 
-const pruneBlockletBundle = async (blocklets, installDir) => {
+/**
+ * @param {string} installDir
+ * @returns {Promise<Array<{ key: string, dir: string }>>} key is <[scope/]name/version>, dir is appDir
+ */
+const getAppDirs = async (installDir) => {
+  const appDirs = [];
+
+  const getNextLevel = (level, name) => {
+    if (level === 'root') {
+      if (name.startsWith('@')) {
+        return 'scope';
+      }
+      return 'name';
+    }
+    if (level === 'scope') {
+      return 'name';
+    }
+    if (level === 'name') {
+      return 'version';
+    }
+    throw new Error(`Invalid level ${level}`);
+  };
+
+  const fillAppDirs = async (dir, level = 'root') => {
+    if (level === 'version') {
+      appDirs.push({
+        key: formatBackSlash(path.relative(installDir, dir)),
+        dir,
+      });
+
+      return;
+    }
+
+    const nextDirs = [];
+    for (const x of await fs.promises.readdir(dir)) {
+      if (!fs.lstatSync(path.join(dir, x)).isDirectory()) {
+        logger.error('pruneBlockletBundle: invalid file in bundle storage', { dir, file: x });
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+      nextDirs.push(x);
+    }
+
+    for (const x of nextDirs) {
+      await fillAppDirs(path.join(dir, x), getNextLevel(level, x));
+    }
+  };
+
+  await fillAppDirs(installDir, 'root');
+
+  return appDirs;
+};
+
+const pruneBlockletBundle = async ({ blocklets, installDir, blockletSettings }) => {
   for (const blocklet of blocklets) {
     if (
       [
@@ -714,60 +1028,66 @@ const pruneBlockletBundle = async (blocklets, installDir) => {
       ].includes(blocklet.status)
     ) {
       logger.info('There are blocklet activities in progress, abort pruning', {
-        name: blocklet.meta.name,
+        bundleName: blocklet.meta.bundleName,
         status: fromBlockletStatus(blocklet.status),
       });
       return;
     }
   }
 
-  // blockletMap: { <name/version>: true }
-  const blockletMap = blocklets.reduce((map, b) => {
-    map[`${b.meta.name}/${b.meta.version}`] = true;
-    for (const child of b.children || []) {
-      map[`${child.meta.name}/${child.meta.version}`] = true;
+  // blockletMap: { <[scope/]name/version>: true }
+  const blockletMap = {};
+  for (const blocklet of blocklets) {
+    forEachBlockletSync(blocklet, (component) => {
+      blockletMap[`${component.meta.bundleName}/${component.meta.version}`] = true;
+    });
+  }
+  for (const setting of blockletSettings) {
+    for (const child of setting.children || []) {
+      if (child.status !== BlockletStatus.deleted) {
+        forEachBlockletSync(child, (component) => {
+          blockletMap[`${component.meta.bundleName}/${component.meta.version}`] = true;
+        });
+      }
     }
-    return map;
-  }, {});
-
-  // appDirs: [{ key: <name/version>, dir: appDir }]
-  const appDirs = [];
+  }
 
   // fill appDirs
+  let appDirs = [];
   try {
-    const fillAppDirs = async (dir) => {
-      if (fs.existsSync(path.join(dir, 'blocklet.yml'))) {
-        appDirs.push({
-          key: path.relative(installDir, dir),
-          dir,
-        });
-        return;
-      }
-      const nextDirs = [];
-      for (const x of await asyncFs.readdir(dir)) {
-        const nextDir = path.join(dir, x);
-        // if blocklet.yml does not exist in dir but non-folder file exists in dir, stop finding
-        if (!fs.lstatSync(nextDir).isDirectory()) {
-          logger.error('blocklet.yml does not exist in blocklet bundle dir', { dir });
-          return;
-        }
-        nextDirs.push(nextDir);
-      }
-
-      for (const x of nextDirs) {
-        await fillAppDirs(x);
-      }
-    };
-    await fillAppDirs(installDir);
+    appDirs = await getAppDirs(installDir);
   } catch (error) {
     logger.error('fill app dirs failed', { error });
   }
 
+  const ensureBundleDirRemoved = async (dir) => {
+    const relativeDir = path.relative(installDir, dir);
+    const arr = relativeDir.split(path.sep).filter(Boolean);
+    const { length } = arr;
+    const bundleName = arr[length - 2];
+    const scopeName = length > 2 ? arr[length - 3] : '';
+    const bundleDir = path.join(installDir, scopeName, bundleName);
+    const isDirEmpty = (await fs.promises.readdir(bundleDir)).length === 0;
+    if (isDirEmpty) {
+      logger.info('Remove bundle folder', { bundleDir });
+      await fs.remove(bundleDir);
+    }
+    if (scopeName) {
+      const scopeDir = path.join(installDir, scopeName);
+      const isScopeEmpty = (await fs.promises.readdir(scopeDir)).length === 0;
+      if (isScopeEmpty) {
+        logger.info('Remove scope folder', { scopeDir });
+        await fs.remove(scopeDir);
+      }
+    }
+  };
+
   // remove trash
   for (const app of appDirs) {
     if (!blockletMap[app.key]) {
-      logger.info('Remove blocklet bundle', { dir: app.dir });
+      logger.info('Remove app folder', { dir: app.dir });
       await fs.remove(app.dir);
+      await ensureBundleDirRemoved(app.dir);
     }
   }
 
@@ -794,26 +1114,36 @@ const getDiskInfo = async (blocklet, { useFakeDiskInfo } = {}) => {
   }
 };
 
-const getRuntimeInfo = async (appId) => {
-  const proc = await getProcessInfo(appId);
+const getRuntimeInfo = async (processId) => {
+  const proc = await getProcessInfo(processId);
   return {
     pid: proc.pid,
-    uptime: proc.pm2_env ? Number(proc.pm2_env.pm_uptime) : 0,
+    uptime: proc.pm2_env ? +new Date() - Number(proc.pm2_env.pm_uptime) : 0,
     memoryUsage: proc.monit.memory,
     cpuUsage: proc.monit.cpu,
     status: proc.pm2_env ? proc.pm2_env.status : null,
   };
 };
 
-const mergeMeta = (meta, childrenMeta = []) => {
+/**
+ * merge services
+ * from meta.children[].mountPoints[].services, meta.children[].services
+ * to childrenMeta[].interfaces[].services
+ *
+ * @param {array<child>|object{children:array}} source e.g. [<config>] or { children: [<config>] }
+ * @param {array<meta|{meta}>} childrenMeta e.g. [<meta>] or [{ meta: <meta> }]
+ */
+
+const mergeMeta = (source, childrenMeta = []) => {
   // configMap
   const configMap = {};
-  (meta.children || []).forEach((x) => {
+  (Array.isArray(source) ? source : getComponentConfig(source) || []).forEach((x) => {
     configMap[x.name] = x;
   });
 
   // merge service from config to child meta
-  childrenMeta.forEach((childMeta) => {
+  childrenMeta.forEach((child) => {
+    const childMeta = child.meta || child;
     const config = configMap[childMeta.name];
     if (!config) {
       return;
@@ -839,48 +1169,671 @@ const mergeMeta = (meta, childrenMeta = []) => {
         childInterface.services = services;
       }
     });
+
+    if (config.services) {
+      const childInterface = findWebInterface(childMeta);
+      if (childInterface) {
+        // merge
+        const services = childInterface.services || [];
+        config.services.forEach((x) => {
+          const index = services.findIndex((y) => y.name === x.name);
+          if (index >= 0) {
+            services.splice(index, 1, x);
+          } else {
+            services.push(x);
+          }
+        });
+        childInterface.services = services;
+      }
+    }
   });
 };
 
-const fixAndVerifyBlockletMeta = (meta, did) => {
-  if (meta.did !== did) {
-    throw new Error('Invalid blocklet meta: did does not match');
+const getUpdateMetaList = (oldBlocklet = {}, newBlocklet = {}) => {
+  const oldMap = {};
+  forEachChildSync(oldBlocklet, (b, { id }) => {
+    if (b.bundleSource) {
+      oldMap[id] = b.meta.version;
+    }
+  });
+
+  const res = [];
+
+  forEachChildSync(newBlocklet, (b, { id }) => {
+    if (b.bundleSource && b.meta.version !== oldMap[id]) {
+      res.push({ id, meta: b.meta });
+    }
+  });
+
+  return res;
+};
+
+/**
+ * @returns BLOCKLET_INSTALL_TYPE
+ */
+const getTypeFromInstallParams = (params) => {
+  if (params.type) {
+    if (!Object.values(BLOCKLET_INSTALL_TYPE).includes(params.type)) {
+      throw new Error(`Can only install blocklet from ${Object.values(BLOCKLET_INSTALL_TYPE).join('/')}`);
+    }
+    return params.type;
+  }
+
+  if (params.url) {
+    return BLOCKLET_INSTALL_TYPE.URL;
+  }
+
+  if (params.file) {
+    throw new Error('install from upload is not supported');
+  }
+
+  if (params.did) {
+    return BLOCKLET_INSTALL_TYPE.STORE;
+  }
+
+  if (params.title && params.description) {
+    return BLOCKLET_INSTALL_TYPE.CREATE;
+  }
+
+  throw new Error(`Can only install blocklet from ${Object.values(BLOCKLET_INSTALL_TYPE).join('/')}`);
+};
+
+const checkDuplicateComponents = (components = []) => {
+  const duplicates = components.filter(
+    (item, index) => components.findIndex((x) => x.meta.did === item.meta.did) !== index
+  );
+  if (duplicates.length) {
+    throw new Error(
+      `Cannot add duplicate component${duplicates.length > 1 ? 's' : ''}: ${duplicates
+        .map((x) => getDisplayName(x, true))
+        .join(', ')}`
+    );
+  }
+};
+
+const getDiffFiles = async (inputFiles, sourceDir) => {
+  if (!fs.existsSync(sourceDir)) {
+    throw new Error(`${sourceDir} does not exist`);
+  }
+
+  const files = inputFiles.reduce((obj, item) => {
+    obj[item.file] = item.hash;
+    return obj;
+  }, {});
+
+  const { files: sourceFiles } = await hashFiles(sourceDir, {
+    filter: (x) => x.indexOf('node_modules') === -1,
+    concurrentHash: 1,
+  });
+
+  const addSet = [];
+  const changeSet = [];
+  const deleteSet = [];
+
+  const diffFiles = diff(sourceFiles, files);
+  if (diffFiles) {
+    diffFiles.forEach((item) => {
+      if (item.kind === 'D') {
+        deleteSet.push(item.path[0]);
+      }
+      if (item.kind === 'E') {
+        changeSet.push(item.path[0]);
+      }
+      if (item.kind === 'N') {
+        addSet.push(item.path[0]);
+      }
+    });
+  }
+
+  return {
+    addSet,
+    changeSet,
+    deleteSet,
+  };
+};
+
+const getBundleDir = (installDir, meta) => path.join(installDir, meta.bundleName || meta.name, meta.version);
+
+const needBlockletDownload = (blocklet, oldBlocklet) => {
+  if ([BlockletSource.upload, BlockletSource.local, BlockletSource.custom].includes(blocklet.source)) {
+    return false;
+  }
+
+  if (!get(oldBlocklet, 'meta.dist.integrity')) {
+    return true;
+  }
+
+  return get(oldBlocklet, 'meta.dist.integrity') !== get(blocklet, 'meta.dist.integrity');
+};
+
+const isDidMatchName = (did, name) => {
+  if (isValidDid(did) && name === did) {
+    return true;
+  }
+
+  return toBlockletDid(name) === did;
+};
+
+/**
+ * set bundleDid and bundleMeta in meta
+ * update name and did to server index
+ * in app structure 2.0, application's bundleDid, bundleName, meta, did will be same
+ */
+const ensureMeta = (meta, { name, did } = {}) => {
+  if (name && did && !isDidMatchName(did, name)) {
+    throw new Error(`name does not match with did: ${name}, ${did}`);
   }
 
-  const sanitized = validateMeta(meta, { ensureDist: false, schemaOptions: { noDefaults: true, stripUnknown: true } });
+  const newMeta = {
+    ...meta,
+  };
 
+  if (!newMeta.did || !newMeta.name || !isDidMatchName(newMeta.did, newMeta.name)) {
+    throw new Error(`name does not match with did in meta: ${newMeta.name}, ${newMeta.did}`);
+  }
+
+  if (!meta.bundleDid) {
+    newMeta.bundleDid = meta.did;
+    newMeta.bundleName = meta.name;
+  }
+
+  if (name) {
+    newMeta.name = name;
+    newMeta.did = did || toBlockletDid(name);
+  }
+
+  return newMeta;
+};
+
+const getBlocklet = async ({
+  did,
+  dataDirs,
+  states,
+  e2eMode = false,
+  throwOnNotExist = true,
+  ensureIntegrity = false,
+} = {}) => {
+  if (!did) {
+    throw new Error('Blocklet did does not exist');
+  }
+  if (!isValidDid(did)) {
+    throw new Error(`Blocklet did is invalid: ${did}`);
+  }
+
+  if (!dataDirs) {
+    throw new Error('dataDirs does not exist');
+  }
+
+  if (!states) {
+    throw new Error('states does not exist');
+  }
+
+  const blocklet = await states.blocklet.getBlocklet(did);
+  if (!blocklet) {
+    if (throwOnNotExist || ensureIntegrity) {
+      throw new Error(`can not find blocklet in database by did ${did}`);
+    }
+    return null;
+  }
+
+  // app settings
+  const settings = await states.blockletExtras.getSettings(blocklet.meta.did);
+  blocklet.trustedPassports = get(settings, 'trustedPassports') || [];
+  blocklet.enablePassportIssuance = get(settings, 'enablePassportIssuance', true);
+  blocklet.settings = settings || {};
+
+  const extrasMeta = await states.blockletExtras.getMeta(blocklet.meta.did);
+  if (extrasMeta) {
+    blocklet.controller = extrasMeta.controller;
+  }
+
+  blocklet.settings.storeList = blocklet.settings.storeList || [];
+
+  [BLOCKLET_STORE_DEV, BLOCKLET_STORE].forEach((store) => {
+    if (!blocklet.settings.storeList.find((x) => x.url === store.url)) {
+      blocklet.settings.storeList.unshift({
+        ...store,
+        protected: true,
+      });
+    }
+  });
+
+  // app site
+  blocklet.site = await states.site.findOneByBlocklet(blocklet.meta.did);
+
+  await forEachBlocklet(blocklet, async (component, { id, level, ancestors }) => {
+    // component env
+    component.env = {
+      id,
+      name: getComponentName(component, ancestors),
+      processId: getComponentProcessId(component, ancestors),
+      ...getComponentDirs(component, {
+        dataDirs,
+        ensure: ensureIntegrity,
+        validate: ensureIntegrity,
+        ancestors,
+        e2eMode: level === 0 ? e2eMode : false,
+      }),
+    };
+
+    // component config
+    const configs = await states.blockletExtras.getConfigs([...ancestors.map((x) => x.meta.did), component.meta.did]);
+    fillBlockletConfigs(component, configs);
+  });
+
+  return blocklet;
+};
+
+/**
+ * this function has side effect on environments
+ */
+const ensureEnvDefault = (environments, ancestors) => {
+  // remove default if ancestors has a value
+  const envMap = environments.reduce((o, env) => {
+    o[env.name] = env;
+    return o;
+  }, {});
+
+  for (let i = ancestors.length - 1; i >= 0; i--) {
+    const ancestor = ancestors[i];
+    const aEnvironments = get(ancestor.meta, 'environments', []);
+    const aEnv = aEnvironments.find((x) => envMap[x.name]);
+
+    if (!isEnvShareable(aEnv)) {
+      break;
+    }
+
+    const env = envMap[aEnv.name];
+    if (isEnvShareable(env) && aEnv.default) {
+      env.default = '';
+      break;
+    }
+  }
+
+  return environments;
+};
+
+const fromProperty2Config = (properties = {}, result) => {
+  Object.keys(properties).forEach((key) => {
+    const prop = properties[key];
+    if (prop.properties && ['ArrayTable', 'ArrayCards'].includes(prop['x-component']) === false) {
+      fromProperty2Config(prop.properties, result);
+    } else if (prop['x-decorator'] === 'FormItem') {
+      const secure = prop['x-component'] === 'Password';
+      result.push({
+        default: prop.default || '',
+        description: prop.title || key,
+        name: `${BLOCKLET_PREFERENCE_PREFIX}${key}`,
+        required: prop.required || false,
+        secure,
+        shared: !secure,
+      });
+    }
+  });
+};
+const getConfigFromPreferences = (blocklet) => {
+  const result = [];
+  const schemaFile = path.join(blocklet.env.appDir, BLOCKLET_PREFERENCE_FILE);
+  if (fs.existsSync(schemaFile)) {
+    try {
+      const schema = JSON.parse(fs.readFileSync(schemaFile, 'utf8'));
+      fromProperty2Config(schema.schema?.properties, result);
+    } catch {
+      // do nothing
+    }
+  }
+
+  return result;
+};
+
+const consumeServerlessNFT = async ({ nftId, nodeInfo, blocklet }) => {
   try {
-    const result = verifyMultiSig(sanitized);
-    if (!result) {
-      throw new Error('invalid signature from developer or registry');
+    const state = await getNFTState(blocklet.controller.chainHost, nftId);
+    if (!state) {
+      throw new Error(`get nft state failed, chainHost: ${blocklet.controller.chainHost}, nftId: ${nftId}`);
     }
-  } catch (err) {
-    throw new Error(`Invalid blocklet meta: ${err.message}`);
+
+    const type = WalletType({
+      role: types.RoleType.ROLE_APPLICATION,
+      pk: types.KeyType.ED25519,
+      hash: types.HashType.SHA3,
+    });
+    const wallet = fromSecretKey(nodeInfo.sk, type);
+    const appURL = blocklet.environments.find((item) => item.key === 'BLOCKLET_APP_URL').value;
+
+    const body = { nftId, appURL };
+
+    const { launcherUrl } = state.data.value;
+    const { data } = await axios.post(joinURL(launcherUrl, '/api/serverless/consume'), body, {
+      headers: {
+        'x-sig': toBase58(wallet.sign(stableStringify(body))),
+      },
+    });
+
+    logger.error('consume serverless nft success', { nftId, hash: data.hash });
+  } catch (error) {
+    logger.error('consume serverless nft failed', { nftId, error });
+
+    throw new Error(`consume nft ${nftId} failed`);
   }
+};
 
-  // this step comes last because it has side effects: the meta is changed
-  return fixAndValidateService(meta, getServices());
+const createDataArchive = (dataDir, fileName) => {
+  const zipPath = path.join(os.tmpdir(), fileName);
+  if (fs.existsSync(zipPath)) {
+    fs.removeSync(zipPath);
+  }
+
+  const archive = createArchive('zip', { zlib: { level: 9 } });
+  const stream = fs.createWriteStream(zipPath);
+
+  return new Promise((resolve, reject) => {
+    archive
+      .directory(dataDir, false)
+      .on('error', (err) => reject(err))
+      .pipe(stream);
+
+    stream.on('close', () => resolve(zipPath));
+    archive.finalize();
+  });
+};
+
+const isBlockletAppSkUsed = ({ environments, migratedFrom = [] }, appSk) => {
+  const isUsedInEnv = environments.find((e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK)?.value === appSk;
+  const isUsedInHistory = migratedFrom.some((x) => x.appSk === appSk);
+  return isUsedInEnv || isUsedInHistory;
+};
+
+const isRotatingAppSk = (newConfigs, oldConfigs, externalSk) => {
+  const newSk = newConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK === x.key);
+  if (!newSk) {
+    // If no newSk found, we are not rotating the appSk
+    return false;
+  }
+
+  const oldSk = oldConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK === x.key);
+  if (!oldSk) {
+    // If we have no oldSk, we are setting the initial appSk for external managed apps
+    // If we have no oldSk, but we are not external managed apps, we are rotating the appSk
+    return !externalSk;
+  }
+
+  // Otherwise, we must be rotating the appSk
+  // eslint-disable-next-line sonarjs/prefer-single-boolean-return
+  if (oldSk.value !== newSk.value) {
+    return true;
+  }
+
+  return false;
 };
 
-const getUpdateMetaList = (oldMetas = [], newMetas = []) => {
-  const oldMap = oldMetas.reduce((obj, x) => {
-    if (x.did) {
-      obj[x.did] = x.version;
+const isRotatingAppDid = (newConfigs, oldConfigs, externalSk) => {
+  if (isRotatingAppSk(newConfigs, oldConfigs, externalSk)) {
+    return true;
+  }
+
+  const newType = newConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_WALLET_TYPE === x.key);
+  const oldType = oldConfigs.find((x) => BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_WALLET_TYPE === x.key);
+  if (!newType) {
+    return false;
+  }
+
+  if (!oldType) {
+    return true;
+  }
+
+  // eslint-disable-next-line sonarjs/prefer-single-boolean-return
+  if (oldType !== newType) {
+    return true;
+  }
+
+  return false;
+};
+
+/**
+ * this function has side effect on config.value
+ * @param {{ key: string, value?: string }} config
+ */
+const validateAppConfig = async (config, states) => {
+  const x = config;
+
+  // sk should be force secured while other app prop should not be secured
+  config.secure = x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK;
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK) {
+    if (x.value) {
+      try {
+        fromSecretKey(x.value);
+      } catch {
+        try {
+          fromSecretKey(x.value, 'eth');
+        } catch {
+          throw new Error('Invalid custom blocklet secret key');
+        }
+      }
+
+      // Ensure sk is not used by existing blocklets, otherwise we may encounter appDid collision
+      const blocklets = await states.blocklet.getBlocklets({});
+      if (blocklets.some((b) => isBlockletAppSkUsed(b, x.value))) {
+        throw new Error('Invalid custom blocklet secret key: already used by existing blocklet');
+      }
+    } else {
+      delete x.value;
     }
-    return obj;
-  }, {});
+  }
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_NAME) {
+    x.value = await titleSchema.validateAsync(x.value);
+  }
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_DESCRIPTION) {
+    x.value = await descriptionSchema.validateAsync(x.value);
+  }
+
+  if (
+    [
+      BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_LOGO,
+      BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_LOGO_RECT,
+      BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_LOGO_SQUARE,
+      BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_LOGO_FAVICON,
+    ].includes(x.key)
+  ) {
+    x.value = await logoSchema.validateAsync(x.value);
+  }
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_WALLET_TYPE) {
+    if (['default', 'eth'].includes(x.value) === false) {
+      throw new Error('Invalid blocklet wallet type, only "default" and "eth" are supported');
+    }
+  }
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_DELETABLE) {
+    if (['yes', 'no'].includes(x.value) === false) {
+      throw new Error('BLOCKLET_DELETABLE must be either "yes" or "no"');
+    }
+  }
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_PASSPORT_COLOR) {
+    if (x.value && x.value !== 'auto') {
+      if (x.value.length !== 7 || !isHex(x.value.slice(-6))) {
+        throw new Error('BLOCKLET_PASSPORT_COLOR must be a hex encoded color, eg. #ffeeaa');
+      }
+    }
+  }
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SPACE_ENDPOINT) {
+    if (isEmpty(x.value)) {
+      throw new Error(`${x.key} can not be empty`);
+    }
+
+    if (!isUrl(x.value)) {
+      throw new Error(`${x.key}(${x.value}) is not a valid http address`);
+    }
+  }
+
+  if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SPACES_URL) {
+    if (isEmpty(x.value)) {
+      throw new Error(`${x.key} can not be empty`);
+    }
+
+    if (!isUrl(x.value)) {
+      throw new Error(`${x.key}(${x.value}) is not a valid http address`);
+    }
+  }
+};
+
+const checkDuplicateAppSk = async ({ sk, did, states }) => {
+  if (!sk && !did) {
+    throw new Error('sk and did is empty');
+  }
+
+  let appSk = sk;
+  if (!sk) {
+    const nodeInfo = await states.node.read();
+    const blocklet = await states.blocklet.getBlocklet(did);
+    const configs = await states.blockletExtras.getConfigs([did]);
+    const { wallet } = getBlockletInfo(
+      {
+        meta: blocklet.meta,
+        environments: (configs || []).filter((x) => x.value),
+      },
+      nodeInfo.sk
+    );
+    appSk = wallet.secretKey;
+  }
+
+  const blocklets = await states.blocklet.getBlocklets({});
+  const others = did ? blocklets.filter((b) => b.meta.did !== did) : blocklets;
+
+  const exist = others.find((b) => {
+    const item = (b.environments || []).find((e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
+    return item?.value === toHex(appSk);
+  });
+
+  if (exist) {
+    throw new Error(`blocklet secret key already used by ${exist.meta.title || exist.meta.name}`);
+  }
+};
 
-  return newMetas.filter(({ version, did }) => did && version !== oldMap[did]);
+const checkDuplicateMountPoint = (blocklet, mountPoint) => {
+  const err = new Error(`cannot add duplicate mount point, ${mountPoint || '/'} already exist`);
+  if (
+    blocklet.meta?.group !== BlockletGroup.gateway &&
+    normalizePathPrefix(blocklet.mountPoint) === normalizePathPrefix(mountPoint)
+  ) {
+    throw err;
+  }
+
+  for (const component of blocklet.children || []) {
+    if (normalizePathPrefix(component.mountPoint) === normalizePathPrefix(mountPoint)) {
+      throw err;
+    }
+  }
+};
+
+const validateStore = (nodeInfo, storeUrl) => {
+  if (nodeInfo.mode !== 'serverless') {
+    return;
+  }
+
+  const inStoreList = nodeInfo.blockletRegistryList.find((item) => {
+    const itemURLObj = new URL(item.url);
+    const storeUrlObj = new URL(storeUrl);
+
+    return itemURLObj.host === storeUrlObj.host;
+  });
+
+  if (!inStoreList) {
+    throw new Error('Must be installed from the compliant blocklet store list');
+  }
+};
+
+const validateInServerless = ({ blockletMeta }) => {
+  const { interfaces } = blockletMeta;
+  const externalPortInterfaces = (interfaces || []).filter((item) => !!item.port?.external);
+
+  if (externalPortInterfaces.length > 0) {
+    throw new Error('Blocklets with exposed ports cannot be installed');
+  }
+};
+
+const checkStructVersion = (blocklet) => {
+  if (blocklet.structVersion !== APP_STRUCT_VERSION) {
+    throw new Error('You should migrate the application first');
+  }
+};
+
+const isVersionCompatible = (actualVersion, expectedRange) =>
+  !expectedRange || expectedRange === 'latest' || semver.satisfies(actualVersion, expectedRange);
+
+const checkVersionCompatibility = (components) => {
+  for (const component of components) {
+    // eslint-disable-next-line no-loop-func
+    forEachBlockletSync(component, (x) => {
+      const dependencies = x.dependencies || [];
+      dependencies.forEach((dep) => {
+        const { did, version: expectedRange } = dep;
+        const exist = components.find((y) => y.meta.did === did);
+        if (exist && !isVersionCompatible(exist.meta.version, expectedRange)) {
+          throw new Error(
+            `Check version compatible failed: ${component.meta.title || component.meta.did} expects ${
+              exist.meta.title || exist.meta.did
+            }'s version to be ${expectedRange}, but actual is ${exist.meta.version}`
+          );
+        }
+      });
+    });
+  }
+};
+
+const filterDuplicateComponents = (components = [], currents = []) => {
+  const arr = [];
+
+  components.forEach((component) => {
+    if (currents.some((x) => x.meta.did === component.meta.did)) {
+      return;
+    }
+
+    const index = arr.findIndex((x) => x.meta.did === component.meta.did);
+    if (index > -1) {
+      const exist = arr[index];
+
+      // 选择最小版本:
+      // 如果 com1 和 com2 都依赖某个 component, com1 声明依赖版本 1.0.0, 实际获取版本 1.0.0; com2 声明依赖版本 latest(不限), 实际获取版本 2.0.0 -> 则应该取 1.0.0
+      if (semver.lt(component.meta.version, exist.meta.version)) {
+        arr.splice(index, 1, component);
+      }
+    } else {
+      arr.push(component);
+    }
+  });
+
+  return arr;
+};
+
+const validateBlockletMeta = (meta, opts = {}) => {
+  fixAndValidateService(meta);
+  return validateMeta(meta, { ensureName: true, skipValidateDidName: true, schemaOptions: { ...opts } });
+};
+
+const getBlockletKnownAs = (blocklet) => {
+  const alsoKnownAs = [blocklet.appDid];
+  if (Array.isArray(blocklet.migratedFrom)) {
+    blocklet.migratedFrom.filter((x) => x.appDid !== blocklet.appPid).forEach((x) => alsoKnownAs.push(x.appDid));
+  }
+
+  return alsoKnownAs.filter(Boolean).map(toDid);
 };
 
 module.exports = {
+  consumeServerlessNFT,
   forEachBlocklet,
-  getBlockletMetaFromUrl,
-  getChildrenMeta,
-  getBlockletDirs,
-  getRootSystemEnvironments,
-  getSystemEnvironments,
-  getOverwrittenEnvironments,
+  getBlockletMetaFromUrl: (url) => getBlockletMetaFromUrl(url, { logger }),
+  parseComponents,
+  getComponentDirs,
+  getAppSystemEnvironments,
+  getAppOverwrittenEnvironments,
+  getComponentSystemEnvironments,
   getRuntimeEnvironments,
   validateBlocklet,
   fillBlockletConfigs,
@@ -896,10 +1849,34 @@ module.exports = {
   expandTarball,
   verifyIntegrity,
   statusMap,
+  getAppDirs,
   pruneBlockletBundle,
   getDiskInfo,
   getRuntimeInfo,
   mergeMeta,
-  fixAndVerifyBlockletMeta,
   getUpdateMetaList,
+  getTypeFromInstallParams,
+  findWebInterface,
+  checkDuplicateComponents,
+  filterDuplicateComponents,
+  getDiffFiles,
+  getBundleDir,
+  needBlockletDownload,
+  ensureMeta,
+  getBlocklet,
+  ensureEnvDefault,
+  getConfigFromPreferences,
+  createDataArchive,
+  validateAppConfig,
+  isBlockletAppSkUsed,
+  isRotatingAppSk,
+  isRotatingAppDid,
+  checkDuplicateAppSk,
+  checkDuplicateMountPoint,
+  validateStore,
+  validateInServerless,
+  checkStructVersion,
+  checkVersionCompatibility,
+  validateBlockletMeta,
+  getBlockletKnownAs,
 };