@abtnode/core 1.6.13 → 1.6.17

package/lib/util/blocklet.js

@@ -6,9 +6,11 @@ const os = require('os');
 const tar = require('tar');
 const get = require('lodash/get');
 const intersection = require('lodash/intersection');
+const intersectionBy = require('lodash/intersectionBy');
 const streamToPromise = require('stream-to-promise');
 const { Throttle } = require('stream-throttle');
 const ssri = require('ssri');
+const diff = require('deep-diff');
 
 const { toHex } = require('@ocap/util');
 const logger = require('@abtnode/logger')('@abtnode/core:util:blocklet');
@@ -17,7 +19,10 @@ const sleep = require('@abtnode/util/lib/sleep');
 const ensureEndpointHealthy = require('@abtnode/util/lib/ensure-endpoint-healthy');
 const CustomError = require('@abtnode/util/lib/custom-error');
 const getFolderSize = require('@abtnode/util/lib/get-folder-size');
+const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+const hashFiles = require('@abtnode/util/lib/hash-files');
 const { BLOCKLET_MAX_MEM_LIMIT_IN_MB } = require('@abtnode/constant');
+
 const {
   BlockletStatus,
   BlockletSource,
@@ -29,6 +34,7 @@ const {
   BLOCKLET_DEFAULT_PORT_NAME,
   BLOCKLET_INTERFACE_TYPE_WEB,
   BLOCKLET_CONFIGURABLE_KEY,
+  BLOCKLET_DYNAMIC_PATH_PREFIX,
   fromBlockletStatus,
 } = require('@blocklet/meta/lib/constants');
 const verifyMultiSig = require('@blocklet/meta/lib/verify-multi-sig');
@@ -41,6 +47,7 @@ const { forEachBlocklet } = require('@blocklet/meta/lib/util');
 const { validate: validateEngine, get: getEngine } = require('../blocklet/manager/engine');
 
 const isRequirementsSatisfied = require('./requirement');
+const { getDidDomainForBlocklet } = require('./get-domain-for-blocklet');
 const { getServices } = require('./service');
 const {
   isBeforeInstalled,
@@ -60,8 +67,6 @@ const getBlockletEngineNameByPlatform = (blockletMeta) => getBlockletEngine(bloc
 
 const noop = () => {};
 
-const asyncFs = fs.promises;
-
 const statusMap = {
   online: BlockletStatus.running,
   launching: BlockletStatus.starting,
@@ -74,17 +79,19 @@ const PRIVATE_NODE_ENVS = [
   // 'NEDB_MULTI_PORT', // FIXME: 排查 abtnode 对外提供的 SDK(比如 @abtnode/queue), SDK 中不要自动使用 NEDB_MULTI_PORT 环境变量
   'ABT_NODE_UPDATER_PORT',
   'ABT_NODE_SESSION_TTL',
-  'ABT_NODE_MODE',
   'ABT_NODE_ROUTER_PROVIDER',
   'ABT_NODE_DATA_DIR',
   'ABT_NODE_TOKEN_SECRET',
   'ABT_NODE_SK',
   'ABT_NODE_SESSION_SECRET',
-  'ABT_NODE_NAME',
-  'ABT_NODE_DESCRIPTION',
   'ABT_NODE_BASE_URL',
   'ABT_NODE_LOG_LEVEL',
   'ABT_NODE_LOG_DIR',
+  // in /core/cli/bin/blocklet.js
+  'CLI_MODE',
+  'ABT_NODE_HOME',
+  'PM2_HOME',
+  'ABT_NODE_CONFIG_FILE',
 ];
 
 /**
@@ -115,19 +122,9 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false } =
     logsDir = path.join(dataDirs.logs, rootBlocklet.meta.name);
   }
 
-  if (ensure) {
-    try {
-      fs.mkdirSync(dataDir, { recursive: true });
-      fs.mkdirSync(logsDir, { recursive: true });
-      fs.mkdirSync(cacheDir, { recursive: true });
-    } catch (err) {
-      logger.error('make blocklet dir failed', { error: err });
-    }
-  }
-
   // get app dirs
 
-  const { version, main, group } = blocklet.meta;
+  const { main, group } = blocklet.meta;
 
   const startFromDevEntry =
     blocklet.mode === BLOCKLET_MODES.DEVELOPMENT && blocklet.meta.scripts && blocklet.meta.scripts.dev;
@@ -142,13 +139,24 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false } =
   if (blocklet.source === BlockletSource.local) {
     appDir = blocklet.deployedFrom;
   } else {
-    appDir = path.join(dataDirs.blocklets, name, version);
+    appDir = getBundleDir(dataDirs.blocklets, blocklet.meta);
   }
 
   if (!appDir) {
     throw new Error('Can not determine blocklet directory, maybe invalid deployment from local blocklets');
   }
 
+  if (ensure) {
+    try {
+      fs.mkdirSync(dataDir, { recursive: true });
+      fs.mkdirSync(logsDir, { recursive: true });
+      fs.mkdirSync(cacheDir, { recursive: true });
+      fs.mkdirSync(appDir, { recursive: true }); // prevent getDiskInfo failed from custom blocklet
+    } catch (err) {
+      logger.error('make blocklet dir failed', { error: err });
+    }
+  }
+
   mainDir = appDir;
 
   if (!startFromDevEntry && !isBeforeInstalled(rootBlocklet.status)) {
@@ -243,12 +251,16 @@ const getRootSystemEnvironments = (blocklet, nodeInfo) => {
   const appName = title || name || result.name;
   const appDescription = description || result.description;
 
+  // FIXME: we should use https here when possible, eg, when did-gateway is available
+  const appUrl = `http://${getDidDomainForBlocklet({ appId, didDomain: nodeInfo.didDomain })}`;
+
   return {
     BLOCKLET_DID: did,
     BLOCKLET_APP_SK: appSk,
     BLOCKLET_APP_ID: appId,
     BLOCKLET_APP_NAME: appName,
     BLOCKLET_APP_DESCRIPTION: appDescription,
+    BLOCKLET_APP_URL: appUrl,
   };
 };
 
@@ -428,11 +440,13 @@ const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironment
       const options = {
         namespace: 'blocklets',
         name: appId,
-        max_memory_restart: `${maxMemoryRestart}M`,
+        cwd: appCwd,
         time: true,
         output: path.join(logsDir, 'output.log'),
         error: path.join(logsDir, 'error.log'),
-        cwd: appCwd,
+        wait_ready: process.env.NODE_ENV !== 'test',
+        listen_timeout: 5000,
+        max_memory_restart: `${maxMemoryRestart}M`,
         max_restarts: b.mode === BLOCKLET_MODES.DEVELOPMENT ? 0 : 3,
         env: {
           ...env,
@@ -460,17 +474,17 @@ const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironment
         const engine = getEngine(blockletEngineInfo.interpreter);
         options.interpreter = engine.interpreter === 'node' ? '' : engine.interpreter;
         options.interpreterArgs = engine.args || '';
-
         options.script = blockletEngineInfo.script || appMain;
-
-        logger.debug('start.blocklet.engine.info', { blockletEngineInfo });
-        logger.debug('start.blocklet.max_memory_restart', { maxMemoryRestart });
       }
 
       await pm2.startAsync(options);
-      logger.info('blocklet started', {
-        appId,
-      });
+
+      const status = await getProcessState(appId);
+      if (status === BlockletStatus.error) {
+        throw new Error(`${appId} is not running within 5 seconds`);
+      }
+
+      logger.info('blocklet started', { appId, status });
     },
     { parallel: true }
   );
@@ -533,6 +547,10 @@ const getBlockletStatusFromProcess = async (blocklet) => {
 
   const list = await Promise.all(tasks);
 
+  if (!list.length) {
+    return blocklet.status;
+  }
+
   return getRootBlockletStatus(list);
 };
 
@@ -611,26 +629,76 @@ const reloadProcess = (appId) =>
     });
   });
 
-const getChildrenMeta = async (meta) => {
-  const children = [];
-  if (meta.children && meta.children.length) {
-    for (const child of meta.children) {
-      const m = await getBlockletMetaFromUrl(child.resolved);
-      if (m.name !== child.name) {
-        logger.error('Resolved child blocklet name does not match in the configuration', {
-          expected: child.name,
-          resolved: m.name,
-        });
-        throw new Error(
-          `Child blocklet name does not match in the configuration. expected: ${child.name}, resolved: ${m.name}`
-        );
-      }
-      validateBlockletMeta(m, { ensureDist: true });
-      children.push(m);
+const findWebInterface = (blocklet) => {
+  const meta = blocklet.meta || blocklet || {};
+  const { interfaces = [] } = meta;
+
+  if (!Array.isArray(interfaces)) {
+    return null;
+  }
+
+  return interfaces.find((x) => x.type === BLOCKLET_INTERFACE_TYPE_WEB);
+};
+
+/**
+ * this function has side effect on children
+ */
+const parseChildren = async (src, { children, dynamic } = {}) => {
+  const configs = Array.isArray(src) ? src : src.children || [];
+
+  if (children) {
+    children.forEach((x) => {
+      x.dynamic = !!dynamic;
+    });
+    mergeMeta(configs, children);
+    return children;
+  }
+
+  if (!configs || !configs.length) {
+    return [];
+  }
+
+  const results = [];
+
+  for (const config of configs) {
+    const mountPoint = config.mountPoint || config.mountPoints[0].root.prefix;
+    if (!mountPoint) {
+      throw new Error(`MountPoint does not found in child ${config.name}`);
     }
+
+    const m = await getBlockletMetaFromUrl(config.resolved);
+    if (m.name !== config.name) {
+      logger.error('Resolved child blocklet name does not match in the configuration', {
+        expected: config.name,
+        resolved: m.name,
+      });
+      throw new Error(
+        `Child blocklet name does not match in the configuration. expected: ${config.name}, resolved: ${m.name}`
+      );
+    }
+    validateBlockletMeta(m, { ensureDist: true });
+
+    const webInterface = findWebInterface(m);
+    if (!webInterface) {
+      throw new Error(`Web interface does not found in child ${config.name}`);
+    }
+
+    const rule = webInterface.prefix;
+    if (rule !== BLOCKLET_DYNAMIC_PATH_PREFIX && normalizePathPrefix(rule) !== normalizePathPrefix(mountPoint)) {
+      throw new Error(`Prefix does not match in child ${config.name}. expected: ${rule}, resolved: ${mountPoint}`);
+    }
+
+    results.push({
+      mountPoint,
+      meta: m,
+      dynamic: !!dynamic,
+      sourceUrl: config.resolved,
+    });
   }
 
-  return children;
+  mergeMeta(configs, results);
+
+  return results;
 };
 
 const validateBlocklet = (blocklet) =>
@@ -728,7 +796,7 @@ const verifyIntegrity = async ({ file, integrity: expected }) => {
   return true;
 };
 
-const pruneBlockletBundle = async (blocklets, installDir) => {
+const pruneBlockletBundle = async ({ blocklets, installDir, blockletSettings }) => {
   for (const blocklet of blocklets) {
     if (
       [
@@ -746,53 +814,104 @@ const pruneBlockletBundle = async (blocklets, installDir) => {
     }
   }
 
-  // blockletMap: { <name/version>: true }
-  const blockletMap = blocklets.reduce((map, b) => {
-    map[`${b.meta.name}/${b.meta.version}`] = true;
-    for (const child of b.children || []) {
-      map[`${child.meta.name}/${child.meta.version}`] = true;
+  // blockletMap: { <[scope/]name/version>: true }
+  const blockletMap = {};
+  for (const blocklet of blocklets) {
+    blockletMap[`${blocklet.meta.name}/${blocklet.meta.version}`] = true;
+    for (const child of blocklet.children || []) {
+      blockletMap[`${child.meta.name}/${child.meta.version}`] = true;
     }
-    return map;
-  }, {});
+  }
+  for (const setting of blockletSettings) {
+    for (const child of setting.children || []) {
+      blockletMap[`${child.meta.name}/${child.meta.version}`] = true;
+    }
+  }
 
-  // appDirs: [{ key: <name/version>, dir: appDir }]
+  // appDirs: [{ key: <[scope/]name/version>, dir: appDir }]
   const appDirs = [];
 
   // fill appDirs
   try {
-    const fillAppDirs = async (dir) => {
-      if (fs.existsSync(path.join(dir, 'blocklet.yml'))) {
+    // @return root/scope/bundle/version
+    const getNextLevel = (level, name) => {
+      if (level === 'root') {
+        if (name.startsWith('@')) {
+          return 'scope';
+        }
+        return 'bundle';
+      }
+      if (level === 'scope') {
+        return 'bundle';
+      }
+      if (level === 'bundle') {
+        return 'version';
+      }
+      throw new Error(`Invalid level ${level}`);
+    };
+
+    const fillAppDirs = async (dir, level = 'root') => {
+      if (level === 'version') {
+        if (!fs.existsSync(path.join(dir, 'blocklet.yml'))) {
+          logger.error('blocklet.yml does not exist in blocklet bundle dir', { dir });
+          return;
+        }
+
         appDirs.push({
           key: path.relative(installDir, dir),
           dir,
         });
+
         return;
       }
+
       const nextDirs = [];
-      for (const x of await asyncFs.readdir(dir)) {
-        const nextDir = path.join(dir, x);
-        // if blocklet.yml does not exist in dir but non-folder file exists in dir, stop finding
-        if (!fs.lstatSync(nextDir).isDirectory()) {
-          logger.error('blocklet.yml does not exist in blocklet bundle dir', { dir });
-          return;
+      for (const x of await fs.promises.readdir(dir)) {
+        if (!fs.lstatSync(path.join(dir, x)).isDirectory()) {
+          logger.error('pruneBlockletBundle: invalid file in bundle storage', { dir, file: x });
+          // eslint-disable-next-line no-continue
+          continue;
         }
-        nextDirs.push(nextDir);
+        nextDirs.push(x);
       }
 
       for (const x of nextDirs) {
-        await fillAppDirs(x);
+        await fillAppDirs(path.join(dir, x), getNextLevel(level, x));
       }
     };
-    await fillAppDirs(installDir);
+    await fillAppDirs(installDir, 'root');
   } catch (error) {
     logger.error('fill app dirs failed', { error });
   }
 
+  const ensureBundleDirRemoved = async (dir) => {
+    const relativeDir = path.relative(installDir, dir);
+    const arr = relativeDir.split('/').filter(Boolean);
+    const { length } = arr;
+    const bundleName = arr[length - 2];
+    const scopeName = length > 2 ? arr[length - 3] : '';
+    const bundleDir = path.join(installDir, scopeName, bundleName);
+    const isEmpty = (await fs.promises.readdir(bundleDir)).length === 0;
+    if (isEmpty) {
+      logger.info('Remove bundle folder', { bundleDir });
+      await fs.remove(bundleDir);
+    }
+    if (scopeName) {
+      const scopeDir = path.join(installDir, scopeName);
+      const isScopeEmpty = (await fs.promises.readdir(scopeDir)).length === 0;
+      if (isScopeEmpty) {
+        logger.info('Remove scope folder', { scopeDir });
+        await fs.remove(scopeDir);
+      }
+    }
+  };
+
   // remove trash
   for (const app of appDirs) {
     if (!blockletMap[app.key]) {
-      logger.info('Remove blocklet bundle', { dir: app.dir });
+      logger.info('Remove app folder', { dir: app.dir });
       await fs.remove(app.dir);
+      await ensureBundleDirRemoved(app.dir);
     }
   }
 
@@ -830,15 +949,25 @@ const getRuntimeInfo = async (appId) => {
   };
 };
 
-const mergeMeta = (meta, childrenMeta = []) => {
+/**
+ * merge services
+ * from meta.children[].mountPoints[].services
+ * to childrenMeta[].interfaces[].services
+ *
+ * @param {array<child>|object{children:array}} source e.g. [<config>] or { children: [<config>] }
+ * @param {array<meta|{meta}>} childrenMeta e.g. [<meta>] or [{ meta: <meta> }]
+ */
+
+const mergeMeta = (source, childrenMeta = []) => {
   // configMap
   const configMap = {};
-  (meta.children || []).forEach((x) => {
+  (Array.isArray(source) ? source : source.children || []).forEach((x) => {
     configMap[x.name] = x;
   });
 
   // merge service from config to child meta
-  childrenMeta.forEach((childMeta) => {
+  childrenMeta.forEach((child) => {
+    const childMeta = child.meta || child;
     const config = configMap[childMeta.name];
     if (!config) {
       return;
@@ -898,10 +1027,84 @@ const getUpdateMetaList = (oldMetas = [], newMetas = []) => {
   return newMetas.filter(({ version, did }) => did && version !== oldMap[did]);
 };
 
+const getSourceFromInstallParams = (params) => {
+  if (params.url) {
+    return BlockletSource.url;
+  }
+
+  if (params.file) {
+    return BlockletSource.upload;
+  }
+
+  if (params.did) {
+    return BlockletSource.registry;
+  }
+
+  if (params.title && params.description) {
+    return BlockletSource.custom;
+  }
+
+  throw new Error('Can only install blocklet from store/url/upload/custom');
+};
+
+const checkDuplicateComponents = (dynamicComponents, staticComponents) => {
+  const duplicates = intersectionBy(dynamicComponents, staticComponents, 'meta.did');
+  if (duplicates.length) {
+    throw new Error(
+      `Cannot add duplicate component${duplicates.length > 1 ? 's' : ''}: ${duplicates
+        .map((x) => x.meta.title || x.meta.name)
+        .join(', ')}`
+    );
+  }
+};
+
+const getDiffFiles = async (inputFiles, sourceDir) => {
+  if (!fs.existsSync(sourceDir)) {
+    throw new Error(`${sourceDir} does not exist`);
+  }
+
+  const files = inputFiles.reduce((obj, item) => {
+    obj[item.file] = item.hash;
+    return obj;
+  }, {});
+
+  const { files: sourceFiles } = await hashFiles(sourceDir, {
+    filter: (x) => x.indexOf('node_modules') === -1,
+    concurrentHash: 1,
+  });
+
+  const addSet = [];
+  const changeSet = [];
+  const deleteSet = [];
+
+  const diffFiles = diff(sourceFiles, files);
+  if (diffFiles) {
+    diffFiles.forEach((item) => {
+      if (item.kind === 'D') {
+        deleteSet.push(item.path[0]);
+      }
+      if (item.kind === 'E') {
+        changeSet.push(item.path[0]);
+      }
+      if (item.kind === 'N') {
+        addSet.push(item.path[0]);
+      }
+    });
+  }
+
+  return {
+    addSet,
+    changeSet,
+    deleteSet,
+  };
+};
+
+const getBundleDir = (installDir, bundle) => path.join(installDir, bundle.name, bundle.version);
+
 module.exports = {
   forEachBlocklet,
   getBlockletMetaFromUrl,
-  getChildrenMeta,
+  parseChildren,
   getBlockletDirs,
   getRootSystemEnvironments,
   getSystemEnvironments,
@@ -927,4 +1130,9 @@ module.exports = {
   mergeMeta,
   fixAndVerifyBlockletMeta,
   getUpdateMetaList,
+  getSourceFromInstallParams,
+  findWebInterface,
+  checkDuplicateComponents,
+  getDiffFiles,
+  getBundleDir,
 };

package/lib/util/default-node-config.js

@@ -35,7 +35,7 @@ const defaultNodeConfigs = {
       },
     ],
   },
-  registerUrl: { getDefaultValue: () => NODE_REGISTER_URL }, // removed in 1.5.1
+  registerUrl: { getDefaultValue: () => NODE_REGISTER_URL },
   webWalletUrl: { getDefaultValue: () => WEB_WALLET_URL },
 };
 

package/lib/util/router.js

@@ -0,0 +1,13 @@
+const { BLOCKLET_SITE_GROUP_SUFFIX } = require('@abtnode/constant');
+
+const getBlockletDomainGroupName = (did) => `${did}${BLOCKLET_SITE_GROUP_SUFFIX}`;
+
+const getDidFromDomainGroupName = (name) => {
+  const did = name.replace(BLOCKLET_SITE_GROUP_SUFFIX, '');
+  return did;
+};
+
+module.exports = {
+  getBlockletDomainGroupName,
+  getDidFromDomainGroupName,
+};

package/lib/validators/node.js

@@ -9,14 +9,23 @@ const nodeInfoSchema = Joi.object({
   description: Joi.string()
     .required()
     .messages({ zh: { 'string.empty': '描述不能为空' }, en: { 'string.empty': 'Description cannot be empty' } }),
+  registerUrl: Joi.string()
+    .uri({ scheme: [/https?/] })
+    .label('register url')
+    .allow('')
+    .optional()
+    .messages({
+      zh: { 'string.uriCustomScheme': '应用启动器必须是合法的 URL' },
+      en: { 'string.uriCustomScheme': 'Blocklet Launcher must be a valid URL' },
+    }),
   webWalletUrl: Joi.string()
     .uri({ scheme: [/https?/] })
     .label('web wallet url')
     .allow('')
     .optional()
     .messages({
-      zh: { 'string.uriCustomScheme': 'Web DID Wallet 必须是合法的 URL' },
-      en: { 'string.uriCustomScheme': 'Web DID Wallet must be a valid URL' },
+      zh: { 'string.uriCustomScheme': 'Web Wallet 必须是合法的 URL' },
+      en: { 'string.uriCustomScheme': 'Web Wallet must be a valid URL' },
     }),
   autoUpgrade: Joi.boolean(),
   enableWelcomePage: Joi.boolean(),
@@ -31,16 +40,6 @@ const nodeInfoSchema = Joi.object({
         'number.max': 'Disk usage alert threshold cannot be higher than 99%',
       },
     }),
-  // removed in 1.5.1
-  registerUrl: Joi.string()
-    .uri({ scheme: [/https?/] })
-    .label('register url')
-    .allow('')
-    .optional()
-    .messages({
-      zh: { 'string.uriCustomScheme': '注册地址必须是合法的 URL' },
-      en: { 'string.uriCustomScheme': 'Registry URL must be a valid URL' },
-    }),
 }).options({ stripUnknown: true });
 
 module.exports = {

package/lib/validators/permission.js

@@ -2,7 +2,22 @@
 const JOI = require('joi');
 const { getMultipleLangParams } = require('./util');
 
-const nameSchema = JOI.string().trim().max(64);
+const nameSchema = JOI.string()
+  .trim()
+  .max(64)
+  .custom((name) => {
+    const arr = name.split('_');
+    const formatTip = 'The format of permission name should be "{action}_{resource}", e.g. query_data';
+    if (arr.length > 2) {
+      throw new Error(`Too much "_". ${formatTip}`);
+    }
+
+    if (arr.length < 2) {
+      throw new Error(formatTip);
+    }
+
+    return name;
+  });
 const descriptionSchema = JOI.string().trim().max(600);
 
 const createPermissionSchema = JOI.object({