@abtnode/core 1.6.16 → 1.6.17

package/lib/util/blocklet.js

@@ -6,9 +6,11 @@ const os = require('os');
 const tar = require('tar');
 const get = require('lodash/get');
 const intersection = require('lodash/intersection');
+const intersectionBy = require('lodash/intersectionBy');
 const streamToPromise = require('stream-to-promise');
 const { Throttle } = require('stream-throttle');
 const ssri = require('ssri');
+const diff = require('deep-diff');
 
 const { toHex } = require('@ocap/util');
 const logger = require('@abtnode/logger')('@abtnode/core:util:blocklet');
@@ -17,7 +19,10 @@ const sleep = require('@abtnode/util/lib/sleep');
 const ensureEndpointHealthy = require('@abtnode/util/lib/ensure-endpoint-healthy');
 const CustomError = require('@abtnode/util/lib/custom-error');
 const getFolderSize = require('@abtnode/util/lib/get-folder-size');
+const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+const hashFiles = require('@abtnode/util/lib/hash-files');
 const { BLOCKLET_MAX_MEM_LIMIT_IN_MB } = require('@abtnode/constant');
+
 const {
   BlockletStatus,
   BlockletSource,
@@ -29,6 +34,7 @@ const {
   BLOCKLET_DEFAULT_PORT_NAME,
   BLOCKLET_INTERFACE_TYPE_WEB,
   BLOCKLET_CONFIGURABLE_KEY,
+  BLOCKLET_DYNAMIC_PATH_PREFIX,
   fromBlockletStatus,
 } = require('@blocklet/meta/lib/constants');
 const verifyMultiSig = require('@blocklet/meta/lib/verify-multi-sig');
@@ -61,8 +67,6 @@ const getBlockletEngineNameByPlatform = (blockletMeta) => getBlockletEngine(bloc
 
 const noop = () => {};
 
-const asyncFs = fs.promises;
-
 const statusMap = {
   online: BlockletStatus.running,
   launching: BlockletStatus.starting,
@@ -80,8 +84,6 @@ const PRIVATE_NODE_ENVS = [
   'ABT_NODE_TOKEN_SECRET',
   'ABT_NODE_SK',
   'ABT_NODE_SESSION_SECRET',
-  'ABT_NODE_NAME',
-  'ABT_NODE_DESCRIPTION',
   'ABT_NODE_BASE_URL',
   'ABT_NODE_LOG_LEVEL',
   'ABT_NODE_LOG_DIR',
@@ -120,19 +122,9 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false } =
     logsDir = path.join(dataDirs.logs, rootBlocklet.meta.name);
   }
 
-  if (ensure) {
-    try {
-      fs.mkdirSync(dataDir, { recursive: true });
-      fs.mkdirSync(logsDir, { recursive: true });
-      fs.mkdirSync(cacheDir, { recursive: true });
-    } catch (err) {
-      logger.error('make blocklet dir failed', { error: err });
-    }
-  }
-
   // get app dirs
 
-  const { version, main, group } = blocklet.meta;
+  const { main, group } = blocklet.meta;
 
   const startFromDevEntry =
     blocklet.mode === BLOCKLET_MODES.DEVELOPMENT && blocklet.meta.scripts && blocklet.meta.scripts.dev;
@@ -147,13 +139,24 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false } =
   if (blocklet.source === BlockletSource.local) {
     appDir = blocklet.deployedFrom;
   } else {
-    appDir = path.join(dataDirs.blocklets, name, version);
+    appDir = getBundleDir(dataDirs.blocklets, blocklet.meta);
   }
 
   if (!appDir) {
     throw new Error('Can not determine blocklet directory, maybe invalid deployment from local blocklets');
   }
 
+  if (ensure) {
+    try {
+      fs.mkdirSync(dataDir, { recursive: true });
+      fs.mkdirSync(logsDir, { recursive: true });
+      fs.mkdirSync(cacheDir, { recursive: true });
+      fs.mkdirSync(appDir, { recursive: true }); // prevent getDiskInfo failed from custom blocklet
+    } catch (err) {
+      logger.error('make blocklet dir failed', { error: err });
+    }
+  }
+
   mainDir = appDir;
 
   if (!startFromDevEntry && !isBeforeInstalled(rootBlocklet.status)) {
@@ -437,11 +440,13 @@ const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironment
       const options = {
         namespace: 'blocklets',
         name: appId,
-        max_memory_restart: `${maxMemoryRestart}M`,
+        cwd: appCwd,
         time: true,
         output: path.join(logsDir, 'output.log'),
         error: path.join(logsDir, 'error.log'),
-        cwd: appCwd,
+        wait_ready: process.env.NODE_ENV !== 'test',
+        listen_timeout: 5000,
+        max_memory_restart: `${maxMemoryRestart}M`,
         max_restarts: b.mode === BLOCKLET_MODES.DEVELOPMENT ? 0 : 3,
         env: {
           ...env,
@@ -469,17 +474,17 @@ const startBlockletProcess = async (blocklet, { preStart = noop, nodeEnvironment
         const engine = getEngine(blockletEngineInfo.interpreter);
         options.interpreter = engine.interpreter === 'node' ? '' : engine.interpreter;
         options.interpreterArgs = engine.args || '';
-
         options.script = blockletEngineInfo.script || appMain;
-
-        logger.debug('start.blocklet.engine.info', { blockletEngineInfo });
-        logger.debug('start.blocklet.max_memory_restart', { maxMemoryRestart });
       }
 
       await pm2.startAsync(options);
-      logger.info('blocklet started', {
-        appId,
-      });
+
+      const status = await getProcessState(appId);
+      if (status === BlockletStatus.error) {
+        throw new Error(`${appId} is not running within 5 seconds`);
+      }
+
+      logger.info('blocklet started', { appId, status });
     },
     { parallel: true }
   );
@@ -542,6 +547,10 @@ const getBlockletStatusFromProcess = async (blocklet) => {
 
   const list = await Promise.all(tasks);
 
+  if (!list.length) {
+    return blocklet.status;
+  }
+
   return getRootBlockletStatus(list);
 };
 
@@ -620,26 +629,76 @@ const reloadProcess = (appId) =>
     });
   });
 
-const getChildrenMeta = async (meta) => {
-  const children = [];
-  if (meta.children && meta.children.length) {
-    for (const child of meta.children) {
-      const m = await getBlockletMetaFromUrl(child.resolved);
-      if (m.name !== child.name) {
-        logger.error('Resolved child blocklet name does not match in the configuration', {
-          expected: child.name,
-          resolved: m.name,
-        });
-        throw new Error(
-          `Child blocklet name does not match in the configuration. expected: ${child.name}, resolved: ${m.name}`
-        );
-      }
-      validateBlockletMeta(m, { ensureDist: true });
-      children.push(m);
+const findWebInterface = (blocklet) => {
+  const meta = blocklet.meta || blocklet || {};
+  const { interfaces = [] } = meta;
+
+  if (!Array.isArray(interfaces)) {
+    return null;
+  }
+
+  return interfaces.find((x) => x.type === BLOCKLET_INTERFACE_TYPE_WEB);
+};
+
+/**
+ * this function has side effect on children
+ */
+const parseChildren = async (src, { children, dynamic } = {}) => {
+  const configs = Array.isArray(src) ? src : src.children || [];
+
+  if (children) {
+    children.forEach((x) => {
+      x.dynamic = !!dynamic;
+    });
+    mergeMeta(configs, children);
+    return children;
+  }
+
+  if (!configs || !configs.length) {
+    return [];
+  }
+
+  const results = [];
+
+  for (const config of configs) {
+    const mountPoint = config.mountPoint || config.mountPoints[0].root.prefix;
+    if (!mountPoint) {
+      throw new Error(`MountPoint does not found in child ${config.name}`);
+    }
+
+    const m = await getBlockletMetaFromUrl(config.resolved);
+    if (m.name !== config.name) {
+      logger.error('Resolved child blocklet name does not match in the configuration', {
+        expected: config.name,
+        resolved: m.name,
+      });
+      throw new Error(
+        `Child blocklet name does not match in the configuration. expected: ${config.name}, resolved: ${m.name}`
+      );
     }
+    validateBlockletMeta(m, { ensureDist: true });
+
+    const webInterface = findWebInterface(m);
+    if (!webInterface) {
+      throw new Error(`Web interface does not found in child ${config.name}`);
+    }
+
+    const rule = webInterface.prefix;
+    if (rule !== BLOCKLET_DYNAMIC_PATH_PREFIX && normalizePathPrefix(rule) !== normalizePathPrefix(mountPoint)) {
+      throw new Error(`Prefix does not match in child ${config.name}. expected: ${rule}, resolved: ${mountPoint}`);
+    }
+
+    results.push({
+      mountPoint,
+      meta: m,
+      dynamic: !!dynamic,
+      sourceUrl: config.resolved,
+    });
   }
 
-  return children;
+  mergeMeta(configs, results);
+
+  return results;
 };
 
 const validateBlocklet = (blocklet) =>
@@ -737,7 +796,7 @@ const verifyIntegrity = async ({ file, integrity: expected }) => {
   return true;
 };
 
-const pruneBlockletBundle = async (blocklets, installDir) => {
+const pruneBlockletBundle = async ({ blocklets, installDir, blockletSettings }) => {
   for (const blocklet of blocklets) {
     if (
       [
@@ -755,53 +814,104 @@ const pruneBlockletBundle = async (blocklets, installDir) => {
     }
   }
 
-  // blockletMap: { <name/version>: true }
-  const blockletMap = blocklets.reduce((map, b) => {
-    map[`${b.meta.name}/${b.meta.version}`] = true;
-    for (const child of b.children || []) {
-      map[`${child.meta.name}/${child.meta.version}`] = true;
+  // blockletMap: { <[scope/]name/version>: true }
+  const blockletMap = {};
+  for (const blocklet of blocklets) {
+    blockletMap[`${blocklet.meta.name}/${blocklet.meta.version}`] = true;
+    for (const child of blocklet.children || []) {
+      blockletMap[`${child.meta.name}/${child.meta.version}`] = true;
     }
-    return map;
-  }, {});
+  }
+  for (const setting of blockletSettings) {
+    for (const child of setting.children || []) {
+      blockletMap[`${child.meta.name}/${child.meta.version}`] = true;
+    }
+  }
 
-  // appDirs: [{ key: <name/version>, dir: appDir }]
+  // appDirs: [{ key: <[scope/]name/version>, dir: appDir }]
   const appDirs = [];
 
   // fill appDirs
   try {
-    const fillAppDirs = async (dir) => {
-      if (fs.existsSync(path.join(dir, 'blocklet.yml'))) {
+    // @return root/scope/bundle/version
+    const getNextLevel = (level, name) => {
+      if (level === 'root') {
+        if (name.startsWith('@')) {
+          return 'scope';
+        }
+        return 'bundle';
+      }
+      if (level === 'scope') {
+        return 'bundle';
+      }
+      if (level === 'bundle') {
+        return 'version';
+      }
+      throw new Error(`Invalid level ${level}`);
+    };
+
+    const fillAppDirs = async (dir, level = 'root') => {
+      if (level === 'version') {
+        if (!fs.existsSync(path.join(dir, 'blocklet.yml'))) {
+          logger.error('blocklet.yml does not exist in blocklet bundle dir', { dir });
+          return;
+        }
+
         appDirs.push({
           key: path.relative(installDir, dir),
           dir,
         });
+
         return;
       }
+
       const nextDirs = [];
-      for (const x of await asyncFs.readdir(dir)) {
-        const nextDir = path.join(dir, x);
-        // if blocklet.yml does not exist in dir but non-folder file exists in dir, stop finding
-        if (!fs.lstatSync(nextDir).isDirectory()) {
-          logger.error('blocklet.yml does not exist in blocklet bundle dir', { dir });
-          return;
+      for (const x of await fs.promises.readdir(dir)) {
+        if (!fs.lstatSync(path.join(dir, x)).isDirectory()) {
+          logger.error('pruneBlockletBundle: invalid file in bundle storage', { dir, file: x });
+          // eslint-disable-next-line no-continue
+          continue;
         }
-        nextDirs.push(nextDir);
+        nextDirs.push(x);
       }
 
       for (const x of nextDirs) {
-        await fillAppDirs(x);
+        await fillAppDirs(path.join(dir, x), getNextLevel(level, x));
       }
     };
-    await fillAppDirs(installDir);
+    await fillAppDirs(installDir, 'root');
   } catch (error) {
     logger.error('fill app dirs failed', { error });
   }
 
+  const ensureBundleDirRemoved = async (dir) => {
+    const relativeDir = path.relative(installDir, dir);
+    const arr = relativeDir.split('/').filter(Boolean);
+    const { length } = arr;
+    const bundleName = arr[length - 2];
+    const scopeName = length > 2 ? arr[length - 3] : '';
+    const bundleDir = path.join(installDir, scopeName, bundleName);
+    const isEmpty = (await fs.promises.readdir(bundleDir)).length === 0;
+    if (isEmpty) {
+      logger.info('Remove bundle folder', { bundleDir });
+      await fs.remove(bundleDir);
+    }
+    if (scopeName) {
+      const scopeDir = path.join(installDir, scopeName);
+      const isScopeEmpty = (await fs.promises.readdir(scopeDir)).length === 0;
+      if (isScopeEmpty) {
+        logger.info('Remove scope folder', { scopeDir });
+        await fs.remove(scopeDir);
+      }
+    }
+  };
+
   // remove trash
   for (const app of appDirs) {
     if (!blockletMap[app.key]) {
-      logger.info('Remove blocklet bundle', { dir: app.dir });
+      logger.info('Remove app folder', { dir: app.dir });
       await fs.remove(app.dir);
+      await ensureBundleDirRemoved(app.dir);
     }
   }
 
@@ -839,15 +949,25 @@ const getRuntimeInfo = async (appId) => {
   };
 };
 
-const mergeMeta = (meta, childrenMeta = []) => {
+/**
+ * merge services
+ * from meta.children[].mountPoints[].services
+ * to childrenMeta[].interfaces[].services
+ *
+ * @param {array<child>|object{children:array}} source e.g. [<config>] or { children: [<config>] }
+ * @param {array<meta|{meta}>} childrenMeta e.g. [<meta>] or [{ meta: <meta> }]
+ */
+
+const mergeMeta = (source, childrenMeta = []) => {
   // configMap
   const configMap = {};
-  (meta.children || []).forEach((x) => {
+  (Array.isArray(source) ? source : source.children || []).forEach((x) => {
     configMap[x.name] = x;
   });
 
   // merge service from config to child meta
-  childrenMeta.forEach((childMeta) => {
+  childrenMeta.forEach((child) => {
+    const childMeta = child.meta || child;
     const config = configMap[childMeta.name];
     if (!config) {
       return;
@@ -907,10 +1027,84 @@ const getUpdateMetaList = (oldMetas = [], newMetas = []) => {
   return newMetas.filter(({ version, did }) => did && version !== oldMap[did]);
 };
 
+const getSourceFromInstallParams = (params) => {
+  if (params.url) {
+    return BlockletSource.url;
+  }
+
+  if (params.file) {
+    return BlockletSource.upload;
+  }
+
+  if (params.did) {
+    return BlockletSource.registry;
+  }
+
+  if (params.title && params.description) {
+    return BlockletSource.custom;
+  }
+
+  throw new Error('Can only install blocklet from store/url/upload/custom');
+};
+
+const checkDuplicateComponents = (dynamicComponents, staticComponents) => {
+  const duplicates = intersectionBy(dynamicComponents, staticComponents, 'meta.did');
+  if (duplicates.length) {
+    throw new Error(
+      `Cannot add duplicate component${duplicates.length > 1 ? 's' : ''}: ${duplicates
+        .map((x) => x.meta.title || x.meta.name)
+        .join(', ')}`
+    );
+  }
+};
+
+const getDiffFiles = async (inputFiles, sourceDir) => {
+  if (!fs.existsSync(sourceDir)) {
+    throw new Error(`${sourceDir} does not exist`);
+  }
+
+  const files = inputFiles.reduce((obj, item) => {
+    obj[item.file] = item.hash;
+    return obj;
+  }, {});
+
+  const { files: sourceFiles } = await hashFiles(sourceDir, {
+    filter: (x) => x.indexOf('node_modules') === -1,
+    concurrentHash: 1,
+  });
+
+  const addSet = [];
+  const changeSet = [];
+  const deleteSet = [];
+
+  const diffFiles = diff(sourceFiles, files);
+  if (diffFiles) {
+    diffFiles.forEach((item) => {
+      if (item.kind === 'D') {
+        deleteSet.push(item.path[0]);
+      }
+      if (item.kind === 'E') {
+        changeSet.push(item.path[0]);
+      }
+      if (item.kind === 'N') {
+        addSet.push(item.path[0]);
+      }
+    });
+  }
+
+  return {
+    addSet,
+    changeSet,
+    deleteSet,
+  };
+};
+
+const getBundleDir = (installDir, bundle) => path.join(installDir, bundle.name, bundle.version);
+
 module.exports = {
   forEachBlocklet,
   getBlockletMetaFromUrl,
-  getChildrenMeta,
+  parseChildren,
   getBlockletDirs,
   getRootSystemEnvironments,
   getSystemEnvironments,
@@ -936,4 +1130,9 @@ module.exports = {
   mergeMeta,
   fixAndVerifyBlockletMeta,
   getUpdateMetaList,
+  getSourceFromInstallParams,
+  findWebInterface,
+  checkDuplicateComponents,
+  getDiffFiles,
+  getBundleDir,
 };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.6.16",
+  "version": "1.6.17",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,28 +19,28 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/certificate-manager": "1.6.16",
-    "@abtnode/constant": "1.6.16",
-    "@abtnode/cron": "1.6.16",
-    "@abtnode/db": "1.6.16",
-    "@abtnode/logger": "1.6.16",
-    "@abtnode/queue": "1.6.16",
-    "@abtnode/rbac": "1.6.16",
-    "@abtnode/router-provider": "1.6.16",
-    "@abtnode/static-server": "1.6.16",
-    "@abtnode/timemachine": "1.6.16",
-    "@abtnode/util": "1.6.16",
-    "@arcblock/did": "^1.14.3",
-    "@arcblock/event-hub": "1.14.3",
+    "@abtnode/certificate-manager": "1.6.17",
+    "@abtnode/constant": "1.6.17",
+    "@abtnode/cron": "1.6.17",
+    "@abtnode/db": "1.6.17",
+    "@abtnode/logger": "1.6.17",
+    "@abtnode/queue": "1.6.17",
+    "@abtnode/rbac": "1.6.17",
+    "@abtnode/router-provider": "1.6.17",
+    "@abtnode/static-server": "1.6.17",
+    "@abtnode/timemachine": "1.6.17",
+    "@abtnode/util": "1.6.17",
+    "@arcblock/did": "^1.14.8",
+    "@arcblock/event-hub": "1.14.8",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/vc": "^1.14.3",
-    "@blocklet/meta": "1.6.16",
+    "@arcblock/vc": "^1.14.8",
+    "@blocklet/meta": "1.6.17",
     "@fidm/x509": "^1.2.1",
     "@nedb/core": "^1.2.2",
     "@nedb/multi": "^1.2.2",
-    "@ocap/mcrypto": "^1.14.3",
-    "@ocap/util": "^1.14.3",
-    "@ocap/wallet": "^1.14.3",
+    "@ocap/mcrypto": "^1.14.8",
+    "@ocap/util": "^1.14.8",
+    "@ocap/wallet": "^1.14.8",
     "@slack/webhook": "^5.0.3",
     "axios": "^0.21.4",
     "axon": "^2.0.3",
@@ -57,7 +57,7 @@
     "js-yaml": "^3.14.0",
     "lodash": "^4.17.21",
     "lru-cache": "^6.0.0",
-    "pm2": "^4.2.3",
+    "pm2": "^5.1.2",
     "semver": "^7.3.2",
     "shelljs": "^0.8.4",
     "slugify": "^1.4.6",
@@ -75,5 +75,5 @@
     "express": "^4.17.1",
     "jest": "^27.4.5"
   },
-  "gitHead": "69b08db16aeb75ce23b0e6bb5b9fa396adba2d4b"
+  "gitHead": "d567a622a779eba43c95eaeb4633cb2b276bb7b9"
 }