@abtnode/core 1.7.9 → 1.7.12

package/lib/states/blocklet.js

@@ -9,7 +9,7 @@ const detectPort = require('detect-port');
 const Lock = require('@abtnode/util/lib/lock');
 const security = require('@abtnode/util/lib/security');
 const { fixPerson, fixInterfaces } = require('@blocklet/meta/lib/fix');
-const { getDisplayName } = require('@blocklet/meta/lib/util');
+const { getDisplayName, forEachBlocklet } = require('@blocklet/meta/lib/util');
 const {
   BlockletStatus,
   BlockletSource,
@@ -21,7 +21,7 @@ const {
 const logger = require('@abtnode/logger')('state-blocklet');
 
 const BaseState = require('./base');
-const { forEachBlocklet, checkDuplicateComponents } = require('../util/blocklet');
+const { checkDuplicateComponents, ensureMeta } = require('../util/blocklet');
 const { validateBlockletMeta } = require('../util');
 
 const lock = new Lock('blocklet-port-assign-lock');
@@ -143,7 +143,6 @@ class BlockletState extends BaseState {
   }
 
   addBlocklet({
-    did,
     meta,
     source = BlockletSource.registry,
     status = BlockletStatus.added,
@@ -151,7 +150,7 @@ class BlockletState extends BaseState {
     mode = BLOCKLET_MODES.PRODUCTION,
     children: rawChildren = [],
   } = {}) {
-    return this.getBlocklet(did).then(
+    return this.getBlocklet(meta.did).then(
       (doc) =>
         new Promise(async (resolve, reject) => {
           if (doc) {
@@ -162,7 +161,10 @@ class BlockletState extends BaseState {
           try {
             fixPerson(meta);
             fixInterfaces(meta);
-            const sanitized = validateBlockletMeta(meta);
+            let sanitized = validateBlockletMeta(meta);
+            // bundle info
+            sanitized = ensureMeta(sanitized);
+            sanitized = omit(sanitized, ['htmlAst']);
 
             // get ports
             await lock.acquire();
@@ -180,7 +182,7 @@ class BlockletState extends BaseState {
               {
                 appDid: null, // will updated later when updating blocklet environments
                 mode,
-                meta: omit(sanitized, ['htmlAst']),
+                meta: sanitized,
                 status,
                 source,
                 deployedFrom,
@@ -530,7 +532,7 @@ class BlockletState extends BaseState {
         throw new Error('Cannot add self as a component');
       }
 
-      checkDuplicateComponents([child], newChildren);
+      checkDuplicateComponents([child, ...newChildren]);
 
       newChildren.push({
         mountPoint,

package/lib/states/index.js

@@ -10,6 +10,7 @@ const MigrationState = require('./migration');
 const SessionState = require('./session');
 const ExtrasState = require('./blocklet-extras');
 const CacheState = require('./cache');
+const AuditLogState = require('./audit-log');
 
 const init = (dataDirs, options) => {
   const notificationState = new NotificationState(dataDirs.core, options);
@@ -23,6 +24,7 @@ const init = (dataDirs, options) => {
   const sessionState = new SessionState(dataDirs.core, options);
   const extrasState = new ExtrasState(dataDirs.core, options);
   const cacheState = new CacheState(dataDirs.core, options);
+  const auditLogState = new AuditLogState(dataDirs.core, options);
 
   return {
     node: nodeState,
@@ -36,6 +38,7 @@ const init = (dataDirs, options) => {
     session: sessionState,
     blockletExtras: extrasState,
     cache: cacheState,
+    auditLog: auditLogState,
   };
 };
 

package/lib/states/node.js

@@ -278,6 +278,14 @@ class NodeState extends BaseState {
     await this.update(_id, { $set: { customBlockletNumber: num } });
     return num;
   }
+
+  async updateGateway(gateway) {
+    const [, nodeInfo] = await this.update({}, { $set: { 'routing.requestLimit': gateway.requestLimit } });
+
+    this.emit(EVENTS.RELOAD_GATEWAY, nodeInfo);
+
+    return nodeInfo.routing;
+  }
 }
 
 module.exports = NodeState;

package/lib/util/blocklet.js

@@ -6,7 +6,6 @@ const os = require('os');
 const tar = require('tar');
 const get = require('lodash/get');
 const intersection = require('lodash/intersection');
-const intersectionBy = require('lodash/intersectionBy');
 const streamToPromise = require('stream-to-promise');
 const { Throttle } = require('stream-throttle');
 const ssri = require('ssri');
@@ -43,12 +42,12 @@ const getBlockletEngine = require('@blocklet/meta/lib/engine');
 const getBlockletInfo = require('@blocklet/meta/lib/info');
 const { validateMeta, fixAndValidateService } = require('@blocklet/meta/lib/validate');
 const { forEachBlocklet, isFreeBlocklet, getDisplayName, findWebInterface } = require('@blocklet/meta/lib/util');
+const toBlockletDid = require('@blocklet/meta/lib/did');
 
 const { validate: validateEngine, get: getEngine } = require('../blocklet/manager/engine');
 
 const isRequirementsSatisfied = require('./requirement');
 const { getDidDomainForBlocklet } = require('./get-domain-for-blocklet');
-const { getServices } = require('./service');
 const {
   isBeforeInstalled,
   expandBundle,
@@ -104,7 +103,10 @@ const PRIVATE_NODE_ENVS = [
  * appMain: app entry file or script (run appMain to start blocklet process)
  * appCwd: cwd of appMain
  */
-const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false, e2eMode = false } = {}) => {
+const getBlockletDirs = (
+  blocklet,
+  { rootBlocklet, dataDirs, ensure = false, e2eMode = false, validate = true } = {}
+) => {
   if (!rootBlocklet) {
     // eslint-disable-next-line no-param-reassign
     rootBlocklet = blocklet;
@@ -134,7 +136,7 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false, e2e
     }
   }
 
-  if (!main && !startFromDevEntry && group !== BlockletGroup.gateway) {
+  if (validate && !main && !startFromDevEntry && group !== BlockletGroup.gateway) {
     throw new Error('Incorrect blocklet manifest: missing `main` field');
   }
 
@@ -164,7 +166,7 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false, e2e
 
   mainDir = appDir;
 
-  if (!startFromDevEntry && !isBeforeInstalled(rootBlocklet.status)) {
+  if (validate && !startFromDevEntry && !isBeforeInstalled(rootBlocklet.status)) {
     try {
       validateBlockletEntry(appDir, blocklet.meta);
     } catch (err) {
@@ -172,7 +174,7 @@ const getBlockletDirs = (blocklet, { rootBlocklet, dataDirs, ensure = false, e2e
     }
   }
 
-  if (!BLOCKLET_GROUPS.includes(group)) {
+  if (validate && !BLOCKLET_GROUPS.includes(group)) {
     throw new CustomError('BLOCKLET_CORRUPTED', `Unsupported blocklet type ${group}`);
   }
 
@@ -634,25 +636,30 @@ const reloadProcess = (appId) =>
     });
   });
 
+const parseChildren = (children, parentMeta = {}, { dynamic } = {}) => {
+  const configs = Array.isArray(parentMeta) ? parentMeta : parentMeta.children || [];
+
+  children.forEach((x) => {
+    if (dynamic !== undefined) {
+      x.dynamic = !!dynamic;
+    }
+  });
+
+  mergeMeta(configs, children);
+  return children;
+};
+
 /**
  * this function has side effect on children
  */
-const parseChildren = async (src, { children, dynamic } = {}) => {
+const parseChildrenFromMeta = async (src, { dynamic } = {}) => {
   const configs = Array.isArray(src) ? src : src.children || [];
 
-  if (children) {
-    children.forEach((x) => {
-      x.dynamic = !!dynamic;
-    });
-    mergeMeta(configs, children);
-    return children;
-  }
-
   if (!configs || !configs.length) {
     return [];
   }
 
-  const results = [];
+  const children = [];
 
   for (const config of configs) {
     const mountPoint = config.mountPoint || config.mountPoints[0].root.prefix;
@@ -661,15 +668,6 @@ const parseChildren = async (src, { children, dynamic } = {}) => {
     }
 
     const m = await getBlockletMetaFromUrl(config.resolved);
-    if (m.name !== config.name) {
-      logger.error('Resolved child blocklet name does not match in the configuration', {
-        expected: config.name,
-        resolved: m.name,
-      });
-      throw new Error(
-        `Child blocklet name does not match in the configuration. expected: ${config.name}, resolved: ${m.name}`
-      );
-    }
     validateBlockletMeta(m, { ensureDist: true });
 
     const webInterface = findWebInterface(m);
@@ -682,17 +680,26 @@ const parseChildren = async (src, { children, dynamic } = {}) => {
       throw new Error(`Prefix does not match in child ${config.name}. expected: ${rule}, resolved: ${mountPoint}`);
     }
 
-    results.push({
+    const meta = ensureMeta(m, { name: config.name });
+    if (config.title) {
+      meta.title = config.title;
+    }
+    if (config.description) {
+      meta.description = config.description;
+    }
+
+    const child = {
       mountPoint,
-      meta: m,
-      dynamic: !!dynamic,
+      meta,
       sourceUrl: config.resolved,
-    });
+    };
+
+    children.push(child);
   }
 
-  mergeMeta(configs, results);
+  parseChildren(children, configs, { dynamic });
 
-  return results;
+  return children;
 };
 
 const validateBlocklet = (blocklet) =>
@@ -801,7 +808,7 @@ const pruneBlockletBundle = async ({ blocklets, installDir, blockletSettings })
       ].includes(blocklet.status)
     ) {
       logger.info('There are blocklet activities in progress, abort pruning', {
-        name: blocklet.meta.name,
+        bundleName: blocklet.meta.bundleName,
         status: fromBlockletStatus(blocklet.status),
       });
       return;
@@ -811,14 +818,16 @@ const pruneBlockletBundle = async ({ blocklets, installDir, blockletSettings })
   // blockletMap: { <[scope/]name/version>: true }
   const blockletMap = {};
   for (const blocklet of blocklets) {
-    blockletMap[`${blocklet.meta.name}/${blocklet.meta.version}`] = true;
+    blockletMap[`${blocklet.meta.bundleName}/${blocklet.meta.version}`] = true;
     for (const child of blocklet.children || []) {
-      blockletMap[`${child.meta.name}/${child.meta.version}`] = true;
+      blockletMap[`${child.meta.bundleName}/${child.meta.version}`] = true;
     }
   }
   for (const setting of blockletSettings) {
     for (const child of setting.children || []) {
-      blockletMap[`${child.meta.name}/${child.meta.version}`] = true;
+      if (child.status !== BlockletStatus.deleted) {
+        blockletMap[`${child.meta.bundleName}/${child.meta.version}`] = true;
+      }
     }
   }
 
@@ -1007,11 +1016,7 @@ const mergeMeta = (source, childrenMeta = []) => {
   });
 };
 
-const fixAndVerifyBlockletMeta = (meta, did) => {
-  if (meta.did !== did) {
-    throw new Error('Invalid blocklet meta: did does not match');
-  }
-
+const fixAndVerifyMetaFromStore = (meta) => {
   const sanitized = validateMeta(meta, { ensureDist: false, schemaOptions: { noDefaults: true, stripUnknown: true } });
 
   try {
@@ -1024,7 +1029,7 @@ const fixAndVerifyBlockletMeta = (meta, did) => {
   }
 
   // this step comes last because it has side effects: the meta is changed
-  return fixAndValidateService(meta, getServices());
+  return fixAndValidateService(meta);
 };
 
 const getUpdateMetaList = (oldMetas = [], newMetas = []) => {
@@ -1058,8 +1063,10 @@ const getSourceFromInstallParams = (params) => {
   throw new Error('Can only install blocklet from store/url/upload/custom');
 };
 
-const checkDuplicateComponents = (dynamicComponents, staticComponents) => {
-  const duplicates = intersectionBy(dynamicComponents, staticComponents, 'meta.did');
+const checkDuplicateComponents = (components = []) => {
+  const duplicates = components.filter(
+    (item, index) => components.findIndex((x) => x.meta.did === item.meta.did) !== index
+  );
   if (duplicates.length) {
     throw new Error(
       `Cannot add duplicate component${duplicates.length > 1 ? 's' : ''}: ${duplicates
@@ -1067,6 +1074,13 @@ const checkDuplicateComponents = (dynamicComponents, staticComponents) => {
         .join(', ')}`
     );
   }
+
+  const duplicateMountPoints = components.filter(
+    (item, index) => components.findIndex((x) => x.mountPoint === item.mountPoint) !== index
+  );
+  if (duplicateMountPoints.length) {
+    throw new Error(`mountpoint must be unique: ${duplicateMountPoints.map((x) => x.mountPoint).join(', ')}`);
+  }
 };
 
 const getDiffFiles = async (inputFiles, sourceDir) => {
@@ -1110,7 +1124,7 @@ const getDiffFiles = async (inputFiles, sourceDir) => {
   };
 };
 
-const getBundleDir = (installDir, bundle) => path.join(installDir, bundle.name, bundle.version);
+const getBundleDir = (installDir, meta) => path.join(installDir, meta.bundleName || meta.name, meta.version);
 
 const needBlockletDownload = (blocklet, oldBlocklet) => {
   if ([BlockletSource.upload, BlockletSource.local, BlockletSource.custom].includes(blocklet.source)) {
@@ -1140,9 +1154,51 @@ const verifyPurchase = (meta, opts = {}) => {
   throw new Error('Can not install a non-free blocklet directly');
 };
 
+const findAvailableDid = (meta, siblings) => {
+  const reg = /-(\d+)$/;
+  const match = reg.exec(meta.name);
+
+  const newName = match ? meta.name.replace(reg, `-${Number(match[1]) + 1}`) : `${meta.name}-1`;
+  const newDid = toBlockletDid(newName);
+  const newMeta = { name: newName, did: newDid };
+
+  if (!(siblings || []).some((x) => x.did === newMeta.did)) {
+    return newMeta;
+  }
+
+  return findAvailableDid(newMeta, siblings);
+};
+
+const ensureMeta = (meta, { name, did } = {}) => {
+  if (name && did && toBlockletDid(name) !== did) {
+    throw new Error('name does not match with did');
+  }
+
+  const newMeta = {
+    ...meta,
+  };
+
+  if (!newMeta.did || !newMeta.name || toBlockletDid(newMeta.name) !== newMeta.did) {
+    throw new Error('name does not match with did in meta');
+  }
+
+  if (!meta.bundleDid) {
+    newMeta.bundleDid = meta.did;
+    newMeta.bundleName = meta.name;
+  }
+
+  if (name) {
+    newMeta.name = name;
+    newMeta.did = did || toBlockletDid(name);
+  }
+
+  return newMeta;
+};
+
 module.exports = {
   forEachBlocklet,
   getBlockletMetaFromUrl,
+  parseChildrenFromMeta,
   parseChildren,
   getBlockletDirs,
   getRootSystemEnvironments,
@@ -1167,7 +1223,7 @@ module.exports = {
   getDiskInfo,
   getRuntimeInfo,
   mergeMeta,
-  fixAndVerifyBlockletMeta,
+  fixAndVerifyMetaFromStore,
   getUpdateMetaList,
   getSourceFromInstallParams,
   findWebInterface,
@@ -1176,4 +1232,6 @@ module.exports = {
   getBundleDir,
   needBlockletDownload,
   verifyPurchase,
+  findAvailableDid,
+  ensureMeta,
 };

package/lib/util/index.js

@@ -35,7 +35,6 @@ const DEFAULT_WELLKNOWN_PORT = 8088;
 
 const logger = require('@abtnode/logger')('@abtnode/core:util');
 
-const { getServices } = require('./service');
 const request = require('./request');
 
 const validateOwner = (owner) => {
@@ -475,6 +474,39 @@ const getSafeEnv = (inputEnv, processEnv = process.env) => {
   return mergedEnv;
 };
 
+// For performance sake, we only support 1 param here, and the cache is flushed every minute
+const memoizeAsync = (fn, flushInterval = 60000) => {
+  const cache = new Map();
+  setInterval(() => cache.clear(), flushInterval);
+
+  return (arg) => {
+    if (cache.has(arg)) {
+      return cache.get(arg);
+    }
+
+    // eslint-disable-next-line prefer-spread
+    const result = fn.apply(null, [arg]);
+    cache.set(arg, result);
+
+    return result;
+  };
+};
+
+const getStateCrons = (states) => [
+  {
+    name: 'cleanup-audit-logs',
+    time: '0 0 6 * * *', // cleanup old logs every day at 6am
+    // time: '0 */5 * * * *', // cleanup every 5 minutes
+    options: { runOnInit: false },
+    fn: async () => {
+      const removed = await states.auditLog.remove({
+        createdAt: { $lt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 90) },
+      });
+      logger.info(`Removed ${removed} audit logs`);
+    },
+  },
+];
+
 const lib = {
   validateOwner,
   getProviderFromNodeInfo,
@@ -496,13 +528,9 @@ const lib = {
   ensureDataDirs,
   getDataDirs,
   getBaseUrls,
-  getBlockletMeta: (dir, opts = {}) =>
-    parseBlockletMeta(dir, {
-      serviceMetas: getServices(),
-      ...opts,
-    }),
+  getBlockletMeta: parseBlockletMeta,
   validateBlockletMeta: (meta, opts = {}) => {
-    fixAndValidateService(meta, getServices(), opts.fix);
+    fixAndValidateService(meta);
     return validateMeta(meta, opts);
   },
   getBlockletMetaByUrl,
@@ -515,6 +543,8 @@ const lib = {
   transformIPToDomain,
   getQueueConcurrencyByMem,
   getSafeEnv,
+  memoizeAsync,
+  getStateCrons,
 };
 
 module.exports = lib;

package/lib/util/ip.js

@@ -1,6 +1,9 @@
 const getIp = require('@abtnode/util/lib/get-ip');
 const logger = require('@abtnode/logger')('@abtnode/core:ip');
 
+const doRpc = require('./rpc');
+const { memoizeAsync } = require('./index');
+
 let cache = null;
 const fetch = async () => {
   try {
@@ -22,6 +25,9 @@ const cron = {
   fn: fetch,
 };
 
+const lookup = memoizeAsync((ip) => doRpc({ command: 'lookup', ip }));
+
 module.exports.fetch = fetch;
 module.exports.get = get;
 module.exports.cron = cron;
+module.exports.lookup = lookup;

package/lib/util/rpc.js

@@ -0,0 +1,16 @@
+const axon = require('axon');
+const logger = require('@abtnode/logger')('@abtnode/core:rpc');
+
+const sock = axon.socket('req');
+sock.connect(Number(process.env.ABT_NODE_UPDATER_PORT || 40405), '127.0.0.1');
+
+const doRpc = async (message) =>
+  new Promise((resolve) => {
+    logger.info('send rpc request', message);
+    sock.send(JSON.stringify(message), (result) => {
+      logger.info('receive rpc response', result);
+      resolve(result);
+    });
+  });
+
+module.exports = doRpc;

package/lib/util/ua.js

@@ -0,0 +1,54 @@
+const semver = require('semver');
+const parser = require('ua-parser-js');
+const packageJson = require('../../package.json');
+
+const { memoizeAsync } = require('./index');
+
+const parseWalletUA = (userAgent) => {
+  const ua = (userAgent || '').toString().toLowerCase();
+  let os = '';
+  let version = '';
+  if (ua.indexOf('android') > -1) {
+    os = 'android';
+  } else if (ua.indexOf('darwin') > -1) {
+    os = 'ios';
+  } else if (ua.indexOf('arcwallet') === 0) {
+    os = 'web';
+  }
+
+  const match = ua.split(/\s+/).find((x) => x.startsWith('arcwallet'));
+  if (match) {
+    const tmp = match.split('/');
+    if (tmp.length > 1 && semver.coerce(tmp[1])) {
+      version = semver.coerce(tmp[1]).version;
+    }
+  }
+
+  return { os, version };
+};
+
+const parse = memoizeAsync((ua) => {
+  let result = parser(ua);
+  if (result.browser.name) {
+    return result;
+  }
+
+  result = parseWalletUA(ua);
+  if (result.version) {
+    return {
+      browser: {
+        name: `DID Wallet ${result.os.toUpperCase()}`,
+        version: result.version,
+      },
+    };
+  }
+
+  return {
+    browser: {
+      name: 'CLI',
+      version: packageJson.version,
+    },
+  };
+});
+
+module.exports = { parse };

package/lib/util/upgrade.js

@@ -1,5 +1,4 @@
 /* eslint-disable no-param-reassign */
-const axon = require('axon');
 const semver = require('semver');
 const sleep = require('@abtnode/util/lib/sleep');
 const { NODE_MODES, NODE_UPGRADE_PROGRESS, EVENTS } = require('@abtnode/constant');
@@ -8,21 +7,10 @@ const Lock = require('@abtnode/util/lib/lock');
 const logger = require('@abtnode/logger')('@abtnode/core:upgrade');
 
 const states = require('../states');
+const doRpc = require('./rpc');
 
 const lock = new Lock('node-upgrade-lock');
 
-// Connect to updater and send RPC call
-const sock = axon.socket('req');
-sock.connect(Number(process.env.ABT_NODE_UPDATER_PORT), '127.0.0.1');
-const waitUpdaterRpc = async (message) =>
-  new Promise((resolve) => {
-    logger.info('send command to updater process', message);
-    sock.send(JSON.stringify(message), (result) => {
-      logger.info('receive response from updater process', result);
-      resolve(result);
-    });
-  });
-
 // eslint-disable-next-line no-unused-vars
 const checkNewVersion = async (params, context) => {
   try {
@@ -120,7 +108,7 @@ const doUpgrade = async (session) => {
 
   // 2. install specified version
   if (session.stage === NODE_UPGRADE_PROGRESS.INSTALLING) {
-    await waitUpdaterRpc({ command: 'install', version: to });
+    await doRpc({ command: 'install', version: to });
     logger.info('new version installed for upgrading', { from, to, sessionId });
     await goNextState(NODE_UPGRADE_PROGRESS.RESTARTING);
   }
@@ -131,7 +119,7 @@ const doUpgrade = async (session) => {
     await sleep(3000);
     await goNextState(NODE_UPGRADE_PROGRESS.CLEANUP);
     await sleep(3000);
-    await waitUpdaterRpc({ command: 'restart', dataDir: process.env.ABT_NODE_DATA_DIR });
+    await doRpc({ command: 'restart', dataDir: process.env.ABT_NODE_DATA_DIR });
     await lock.release();
     return; // we should abort here and resume after restart
   }

package/lib/validators/node.js

@@ -1,4 +1,5 @@
 /* eslint-disable newline-per-chained-call */
+const { GATEWAY_REQ_LIMIT } = require('@abtnode/constant');
 const Joi = require('joi');
 const { getMultipleLangParams } = require('./util');
 
@@ -42,6 +43,18 @@ const nodeInfoSchema = Joi.object({
     }),
 }).options({ stripUnknown: true });
 
+const updateGatewaySchema = Joi.object({
+  requestLimit: Joi.object({
+    enabled: Joi.bool().required(),
+    rate: Joi.number()
+      .min(GATEWAY_REQ_LIMIT.min)
+      .max(GATEWAY_REQ_LIMIT.max)
+      .when('requestLimit.enabled', { is: true, then: Joi.required() }),
+    ipHeader: Joi.string().allow('').trim(),
+  }),
+});
+
 module.exports = {
   validateNodeInfo: (entity, context) => nodeInfoSchema.validateAsync(entity, getMultipleLangParams(context)),
+  validateUpdateGateway: (entity, context) => updateGatewaySchema.validateAsync(entity, getMultipleLangParams(context)),
 };