@abtnode/core 1.6.3 → 1.6.7

package/lib/router/index.js

@@ -1,4 +1,5 @@
 const get = require('lodash/get');
+const pick = require('lodash/pick');
 const cloneDeep = require('lodash/cloneDeep');
 const {
   DOMAIN_FOR_DEFAULT_SITE,
@@ -9,7 +10,6 @@ const {
   BLOCKLET_SITE_GROUP_SUFFIX,
 } = require('@abtnode/constant');
 const { BLOCKLET_UI_INTERFACES } = require('@blocklet/meta/lib/constants');
-const { pick } = require('lodash');
 const logger = require('@abtnode/logger')('@abtnode/core:router');
 
 const expandSites = (sites = []) => {

package/lib/router/manager.js

@@ -70,8 +70,9 @@ const normalizeRedirectUrl = (url) => {
 };
 
 class RouterManager extends EventEmitter {
-  constructor() {
+  constructor({ certManager }) {
     super();
+    this.certManager = certManager;
 
     // HACK: do not emit any events from CLI
     if (isCLI()) {
@@ -300,7 +301,10 @@ class RouterManager extends EventEmitter {
     await this.validateRouterConfig('updateRoutingRule', { id, rule });
 
     // update rules
-    const newRules = [...dbSite.rules.filter((x) => x.groupId !== rule.id), ...(await this.getRules(rule))];
+    const newRules = [
+      ...dbSite.rules.filter((x) => x.groupId !== rule.id || x.id !== rule.id), // 有些路由没有 rule.groupId
+      ...(await this.getRules(rule)),
+    ];
 
     const updateResult = await states.site.update({ _id: id }, { $set: { rules: newRules } });
     logger.info('update result', { updateResult });
@@ -363,7 +367,7 @@ class RouterManager extends EventEmitter {
       domain,
     });
     logger.info('add certificate result', { domain: newCert.domain });
-    this.emit('router.certificate.add', { type: 'nginx', data: newCert });
+    this.emit('cert.added', { type: 'nginx', data: newCert });
   }
 
   // eslint-disable-next-line no-unused-vars
@@ -376,7 +380,7 @@ class RouterManager extends EventEmitter {
     const removeResult = await states.certificate.remove({ _id: id });
 
     logger.info('delete certificate', { removeResult, domain: tmpCert.domain });
-    this.emit('router.certificate.remove', { type: 'nginx', data: { domain: tmpCert.domain } });
+    this.emit('cert.removed', { type: 'nginx', data: { domain: tmpCert.domain } });
     return {};
   }
 
@@ -386,7 +390,7 @@ class RouterManager extends EventEmitter {
     this.fixCertificate(entity);
     this.validateCertificate(entity, entity.domain);
     const dbEntity = await states.certificate.upsert(entity);
-    this.emit('router.certificate.updated', { type: 'nginx', data: dbEntity });
+    this.emit('cert.issued', { type: 'nginx', data: dbEntity });
   }
 
   findCertificateByDomain(domain) {
@@ -448,7 +452,7 @@ class RouterManager extends EventEmitter {
   }
 
   async getMatchedCert(domain) {
-    const certs = await states.certificate.find();
+    const certs = await this.certManager.getAll();
     const matchedCert = certs.find((cert) => this.isCertMatchedDomain(cert, domain));
 
     if (matchedCert) {

package/lib/states/base.js

@@ -1,226 +1,17 @@
-const fs = require('fs');
-const path = require('path');
-const util = require('util');
-const { EventEmitter } = require('events');
-const cloneDeep = require('lodash/cloneDeep');
-const DataStore =
-  process.env.NODE_ENV === 'test' ? require('@nedb/core') : require('@nedb/multi')(Number(process.env.NEDB_MULTI_PORT));
+const DB = require('@abtnode/db');
 const logger = require('@abtnode/logger')('@abtnode/core:states');
 
 const { isCLI } = require('../util');
 
-class BaseState extends EventEmitter {
+class BaseState extends DB {
   constructor(baseDir, options) {
-    super();
+    super(baseDir, options);
 
     // HACK: do not emit any events from CLI
     if (isCLI() && process.env.NODE_ENV !== 'test') {
       this.emit = (name) => logger.debug('stopped state db event in CLI', name);
     }
-
-    const dbOptions = options.db || {};
-    this.filename = path.join(baseDir, options.filename);
-    this.options = Object.freeze(cloneDeep(options));
-    this.db = new DataStore({
-      filename: this.filename,
-      timestampData: true,
-      ...dbOptions,
-    });
-
-    logger.info('initialized', { filename: this.filename });
-
-    this.ready = false;
-    this.readyCallbacks = [];
-    this.db.loadDatabase((err) => {
-      if (err) {
-        logger.error(`failed to load disk database ${this.filename}`, { error: err });
-        console.error(err);
-      } else {
-        this.ready = true;
-        if (this.readyCallbacks.length) {
-          this.readyCallbacks.forEach((x) => x());
-        }
-      }
-    });
-
-    this.asyncDB = new Proxy(this.db, {
-      get(target, property) {
-        if (typeof target[property] === 'function') {
-          return util
-            .promisify((...args) => {
-              const cb = args[args.length - 1];
-              const rest = args.slice(0, args.length - 1);
-
-              target[property](...rest, (err, ...result) => {
-                if (err) {
-                  return cb(err);
-                }
-
-                if (result.length === 1) {
-                  return cb(null, result[0]);
-                }
-
-                return cb(null, result);
-              });
-            })
-            .bind(target);
-        }
-
-        return target[property];
-      },
-    });
-  }
-
-  onReady(cb) {
-    if (this.ready) {
-      cb();
-    } else {
-      this.readyCallbacks.push(cb);
-    }
-  }
-
-  createNotification(payload) {
-    if (this.notification && typeof this.notification.create === 'function') {
-      this.notification.create(payload);
-    }
-  }
-
-  paginate(conditions, sort, paging) {
-    const { pageSize: size = 20, page = 1 } = paging || {};
-    const pageSize = Math.min(100, size);
-
-    return new Promise((resolve, reject) => {
-      this.db
-        .find(conditions)
-        .sort(sort)
-        .exec((err, docs) => {
-          if (err) {
-            return reject(err);
-          }
-
-          const pageCount = Math.ceil(docs.length / pageSize);
-          const total = docs.length;
-          const skip = (page - 1) * pageSize;
-          const list = docs.slice(skip, skip + pageSize);
-
-          list.forEach((doc) => {
-            // eslint-disable-next-line no-underscore-dangle
-            doc.id = doc._id;
-          });
-
-          return resolve({
-            list,
-            paging: {
-              total,
-              pageSize,
-              pageCount,
-              page,
-            },
-          });
-        });
-    });
-  }
-
-  async updateById(id, updates, options = {}) {
-    const [, doc] = await this.asyncDB.update({ _id: id }, updates, {
-      multi: false,
-      upsert: false,
-      returnUpdatedDocs: true,
-      ...options,
-    });
-
-    return doc;
-  }
-
-  update(...args) {
-    if (args.length === 0) {
-      throw new Error('param is required by update method');
-    }
-
-    if (typeof args[0] === 'string') {
-      return this.updateById(...args);
-    }
-
-    return this.asyncDB.update(args[0], args[1], { returnUpdatedDocs: true, ...(args[2] || {}) });
-  }
-
-  count(conditions = {}) {
-    return new Promise((resolve, reject) => {
-      this.db.count(conditions, (err, num) => {
-        if (err) {
-          return reject(err);
-        }
-
-        return resolve(num);
-      });
-    });
-  }
-
-  remove(conditions = {}, options = { multi: false }) {
-    return new Promise((resolve, reject) => {
-      this.db.remove(conditions, options, (err, num) => {
-        if (err) {
-          return reject(err);
-        }
-
-        return resolve(num);
-      });
-    });
-  }
-
-  reset() {
-    fs.unlinkSync(this.filename);
-  }
-
-  find(...args) {
-    if (args.length === 0) {
-      return this.asyncDB.find({});
-    }
-
-    return this.asyncDB.find(...args);
-  }
-
-  findOne(...args) {
-    if (args.length === 0) {
-      return this.asyncDB.findOne({});
-    }
-
-    return this.asyncDB.findOne(...args);
-  }
-
-  insert(...args) {
-    return this.asyncDB.insert(...args);
   }
 }
 
-/**
- * Rename _id field name to id, this method has side effects
- * @param {object} entities
- */
-const renameIdFiledName = (entities, from = '_id', to = 'id') => {
-  /* eslint-disable  no-underscore-dangle, no-param-reassign */
-
-  if (!entities) {
-    return entities;
-  }
-
-  const mapEntity = (entity) => {
-    if (entity[from]) {
-      entity[to] = entity[from];
-      delete entity[from];
-    }
-  };
-
-  if (!Array.isArray(entities)) {
-    mapEntity(entities);
-    return entities;
-  }
-
-  entities.forEach(mapEntity);
-
-  return entities;
-};
-
-BaseState.renameIdFiledName = renameIdFiledName;
-
 module.exports = BaseState;

package/lib/states/blocklet-extras.js

@@ -6,7 +6,9 @@ const camelCase = require('lodash/camelCase');
 
 const BaseState = require('./base');
 
-const { mergeConfigs } = require('../blocklet/extras');
+const { mergeConfigs, parseConfigs } = require('../blocklet/extras');
+
+const noop = (k) => (v) => v[k];
 
 class BlockletExtrasState extends BaseState {
   constructor(baseDir, options = {}) {
@@ -17,12 +19,13 @@ class BlockletExtrasState extends BaseState {
       {
         name: 'configs',
         beforeSet: mergeConfigs,
+        afterGet: parseConfigs,
       },
 
       // setting
       {
         name: 'settings',
-        beforeSet: (old, cur) => {
+        beforeSet: ({ old, cur }) => {
           const merged = { ...old, ...cur };
           Object.keys(merged).forEach((key) => {
             if (merged[key] === undefined || merged[key] === null) {
@@ -67,22 +70,23 @@ class BlockletExtrasState extends BaseState {
 
   generateGetFn(extra) {
     return async (did) => {
-      const { name } = extra;
+      const { dek } = this.options;
+      const { name, afterGet = noop('data') } = extra;
       const item = await this.asyncDB.findOne({ did });
-      return item ? item[name] : item;
+      return afterGet({ data: item ? item[name] : item, did, dek });
     };
   }
 
   generateSetFn(extra) {
     return async (did, data) => {
-      const { name, beforeSet } = extra;
-      const hasBeforeSet = beforeSet && typeof beforeSet === 'function';
+      const { dek } = this.options;
+      const { name, beforeSet = noop('cur') } = extra;
       const item = await this.asyncDB.findOne({ did });
 
       if (!item) {
         const insertData = {
           did,
-          [name]: hasBeforeSet ? beforeSet(undefined, data) : data,
+          [name]: beforeSet({ old: undefined, cur: data, did, dek }),
         };
 
         const info = await this.asyncDB.insert(insertData);
@@ -93,7 +97,7 @@ class BlockletExtrasState extends BaseState {
       const itemNameValue = item[name];
       const updated = await this.update(item._id, {
         $set: {
-          [name]: hasBeforeSet ? beforeSet(itemNameValue, data) : data,
+          [name]: beforeSet({ old: itemNameValue, cur: data, did, dek }),
         },
       });
       return updated[name];
@@ -132,22 +136,23 @@ class BlockletExtrasState extends BaseState {
 
   generateGetChildFn(extra) {
     return async (did, childDid) => {
-      const { name } = extra;
+      const { dek } = this.options;
+      const { name, afterGet = noop('data') } = extra;
       const item = await this.asyncDB.findOne({ did });
       const children = (item || {}).children || [];
       const subItem = (children || []).find((x) => x.did === childDid);
-      return subItem ? subItem[name] : null;
+      return afterGet({ data: subItem ? subItem[name] : null, did, dek });
     };
   }
 
   generateSetChildFn(extra) {
     return async (did, childDid, data) => {
-      const { name, beforeSet } = extra;
-      const hasBeforeSet = beforeSet && typeof beforeSet === 'function';
+      const { dek } = this.options;
+      const { name, beforeSet = noop('cur') } = extra;
       const item = await this.asyncDB.findOne({ did });
 
       if (!item) {
-        const newData = hasBeforeSet ? beforeSet(undefined, data) : data;
+        const newData = beforeSet({ old: undefined, cur: data, did, dek });
         const insertData = {
           did,
           children: [
@@ -168,7 +173,7 @@ class BlockletExtrasState extends BaseState {
       const subItem = (children || []).find((x) => x.did === childDid);
 
       if (!subItem) {
-        const newData = hasBeforeSet ? beforeSet(undefined, data) : data;
+        const newData = beforeSet({ old: undefined, cur: data, did, dek });
         await this.update(item._id, {
           $addToSet: {
             children: {
@@ -182,7 +187,7 @@ class BlockletExtrasState extends BaseState {
         return newData;
       }
 
-      const newData = hasBeforeSet ? beforeSet(subItem[name], data) : data;
+      const newData = beforeSet({ old: subItem[name], cur: data, did, dek });
 
       children.forEach((x) => {
         if (x.did === childDid) {

package/lib/states/blocklet.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-async-promise-executor */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable function-paren-newline */
 /* eslint-disable no-underscore-dangle */
@@ -5,6 +6,7 @@ const omit = require('lodash/omit');
 const uniq = require('lodash/uniq');
 const detectPort = require('detect-port');
 const Lock = require('@abtnode/util/lib/lock');
+const security = require('@abtnode/util/lib/security');
 const { fixPerson, fixInterfaces } = require('@blocklet/meta/lib/fix');
 const {
   BlockletStatus,
@@ -27,6 +29,40 @@ const getMaxPort = (ports = {}) => Math.max(Object.values(ports).map(Number));
 const getExternalPortsFromMeta = (meta) =>
   (meta.interfaces || []).map((x) => x.port && x.port.external).filter(Boolean);
 
+const formatBlocklet = (blocklet, phase, dek) => {
+  forEachBlocklet(
+    blocklet,
+    (b) => {
+      if (b.meta) {
+        fixPerson(b.meta);
+        fixInterfaces(b.meta);
+      }
+
+      b.children = b.children || [];
+
+      if (!b.environments || !b.meta || !dek) {
+        return;
+      }
+
+      ['BLOCKLET_APP_SK'].forEach((key) => {
+        const env = b.environments.find((x) => x.key === key);
+        if (!env) {
+          return;
+        }
+        if (phase === 'onUpdate' && env.value.indexOf('0x') === 0) {
+          env.value = security.encrypt(env.value, b.meta.did, dek);
+        }
+        if (phase === 'onRead' && env.value.indexOf('0x') === -1) {
+          env.value = security.decrypt(env.value, b.meta.did, dek);
+        }
+      });
+    },
+    { sync: true }
+  );
+
+  return blocklet;
+};
+
 class BlockletState extends BaseState {
   /**
    * Creates an instance of BlockletState
@@ -48,14 +84,7 @@ class BlockletState extends BaseState {
           return reject(err);
         }
 
-        if (doc) {
-          // TODO: this only exists for backward compatible
-          fixPerson(doc.meta);
-          fixInterfaces(doc.meta);
-          doc.children = doc.children || [];
-        }
-
-        return resolve(doc);
+        return resolve(doc ? formatBlocklet(doc, 'onRead', this.options.dek) : null);
       });
     });
   }
@@ -70,14 +99,7 @@ class BlockletState extends BaseState {
             return reject(err);
           }
 
-          return resolve(
-            docs.filter(Boolean).map((doc) => {
-              // TODO: this only exists for backward compatible
-              fixPerson(doc.meta);
-              fixInterfaces(doc.meta);
-              return doc;
-            })
-          );
+          return resolve(docs.filter(Boolean).map((doc) => formatBlocklet(doc, 'onRead', this.options.dek)));
         });
     });
   }
@@ -114,7 +136,6 @@ class BlockletState extends BaseState {
   } = {}) {
     return this.getBlocklet(did).then(
       (doc) =>
-        // eslint-disable-next-line no-async-promise-executor
         new Promise(async (resolve, reject) => {
           if (doc) {
             reject(new Error('Blocklet already added'));
@@ -165,6 +186,26 @@ class BlockletState extends BaseState {
     );
   }
 
+  updateBlocklet(did, updates) {
+    return this.getBlocklet(did).then(
+      (doc) =>
+        new Promise(async (resolve, reject) => {
+          if (!doc) {
+            reject(new Error('Blocklet does not exist'));
+            return;
+          }
+
+          try {
+            const formatted = formatBlocklet(updates, 'onUpdate', this.options.dek);
+            const newDoc = await this.updateById(doc._id, { $set: formatted });
+            resolve(newDoc);
+          } catch (err) {
+            reject(err);
+          }
+        })
+    );
+  }
+
   upgradeBlocklet({ meta, source, deployedFrom, children } = {}) {
     return this.getBlocklet(meta.did).then(
       (doc) =>

package/lib/states/index.js

@@ -1,4 +1,4 @@
-const BaseState = require('./base');
+const stateFactory = require('@abtnode/db/lib/factory');
 const NodeState = require('./node');
 const ChallengeState = require('./challenge');
 const BlockletState = require('./blocklet');
@@ -12,8 +12,6 @@ const SessionState = require('./session');
 const ExtrasState = require('./blocklet-extras');
 const CacheState = require('./cache');
 
-const states = {};
-
 const init = (dataDirs, options) => {
   const notificationState = new NotificationState(dataDirs.core, options);
   const nodeState = new NodeState(dataDirs.core, options, dataDirs, notificationState);
@@ -28,7 +26,7 @@ const init = (dataDirs, options) => {
   const extrasState = new ExtrasState(dataDirs.core, options);
   const cacheState = new CacheState(dataDirs.core, options);
 
-  Object.assign(states, {
+  return {
     node: nodeState,
     blocklet: blockletState,
     notification: notificationState,
@@ -41,22 +39,7 @@ const init = (dataDirs, options) => {
     session: sessionState,
     blockletExtras: extrasState,
     cache: cacheState,
-  });
+  };
 };
 
-module.exports = new Proxy(
-  {},
-  {
-    get(target, prop) {
-      if (prop === 'init') {
-        return init;
-      }
-
-      if (states[prop] instanceof BaseState) {
-        return states[prop];
-      }
-
-      throw new Error(`State ${String(prop)} may not be initialized`);
-    },
-  }
-);
+module.exports = stateFactory(init);

package/lib/states/node.js

@@ -1,7 +1,9 @@
 /* eslint-disable no-underscore-dangle */
+const semver = require('semver');
 const omit = require('lodash/omit');
 const isEqual = require('lodash/isEqual');
 const isEmpty = require('lodash/isEmpty');
+const security = require('@abtnode/util/lib/security');
 const { isFromPublicKey } = require('@arcblock/did');
 const logger = require('@abtnode/logger')('@abtnode/core:node');
 const { ROUTER_PROVIDER_NONE, NODE_MODES, DISK_ALERT_THRESHOLD_PERCENT } = require('@abtnode/constant');
@@ -58,6 +60,7 @@ class NodeState extends BaseState {
    */
   read() {
     return new Promise((resolve, reject) => {
+      const { nodeDid, dek } = this.options;
       this.db.findOne({ did: this.options.nodeDid }, (err, record) => {
         if (err) {
           // eslint-disable-next-line no-console
@@ -66,6 +69,10 @@ class NodeState extends BaseState {
         }
 
         if (record) {
+          if (dek) {
+            record.sk = security.decrypt(record.sk, record.did, dek);
+          }
+
           return resolve(record);
         }
 
@@ -74,7 +81,6 @@ class NodeState extends BaseState {
           description,
           nodeSk,
           nodePk,
-          nodeDid,
           nodeOwner,
           port,
           version,
@@ -100,7 +106,7 @@ class NodeState extends BaseState {
                 name,
                 description,
                 pk: nodePk,
-                sk: nodeSk,
+                sk: dek ? security.encrypt(nodeSk, nodeDid, dek) : nodeSk,
                 did: nodeDid,
                 initialized,
                 version,
@@ -158,7 +164,7 @@ class NodeState extends BaseState {
 
   cleanupDirtyUpgradeState() {
     return this.read().then((doc) => {
-      if (doc.nextVersion === doc.version) {
+      if (doc.nextVersion && semver.lte(doc.nextVersion, doc.version)) {
         const updates = { nextVersion: '', upgradeSessionId: '' };
 
         // FIXME: this may cause the node exit some mode unexpectedly if it is not being upgraded

package/lib/util/blocklet.js

@@ -192,7 +192,6 @@ const fillBlockletConfigs = (blocklet, configs) => {
     acc[x.key] = x.value;
     return acc;
   }, {});
-  blocklet.userEnvironments = blocklet.configObj; // deprecated
   blocklet.environmentObj = (blocklet.environments || []).reduce((acc, x) => {
     acc[x.key] = x.value;
     return acc;
@@ -308,6 +307,7 @@ const getRuntimeEnvironments = (blocklet, nodeEnvironments) => {
 
   return {
     ...blocklet.environmentObj,
+    ...blocklet.configObj,
     ...nodeEnvironments,
     ...safeNodeEnvironments,
   };

package/lib/util/index.js

@@ -332,6 +332,8 @@ const getDataDirs = (dataDir) => ({
   tmp: path.join(dataDir, 'tmp'),
   blocklets: path.join(dataDir, 'blocklets'),
   services: path.join(dataDir, 'services'),
+  modules: path.join(dataDir, 'modules'),
+  certManagerModule: path.join(dataDir, 'modules', 'certificate-manager'),
 });
 
 // Ensure data dir for Blocklet Server exists