@abtnode/core 1.6.1 → 1.6.5

package/lib/blocklet/registry.js

@@ -3,7 +3,7 @@ const { BlockletGroup } = require('@blocklet/meta/lib/constants');
 const joinURL = require('url-join');
 const get = require('lodash/get');
 const pick = require('lodash/pick');
-const { BLOCKLET_STORE_API_PREFIX } = require('@abtnode/constant');
+const { BLOCKLET_STORE_API_PREFIX, BLOCKLET_STORE_META_PATH } = require('@abtnode/constant');
 
 const { name } = require('../../package.json');
 
@@ -170,7 +170,7 @@ BlockletRegistry.validateRegistryURL = async (registry) => {
 
 BlockletRegistry.getRegistryMeta = async (registry) => {
   try {
-    const url = joinURL(registry, BLOCKLET_STORE_API_PREFIX, `/registry.json?__t__=${Date.now()}`);
+    const url = joinURL(registry, BLOCKLET_STORE_META_PATH, `?__t__=${Date.now()}`);
     const { data } = await request.get(url);
 
     if (!data) {

package/lib/index.js

@@ -1,4 +1,5 @@
-const { listProviders } = require('@abtnode/router-provider');
+const fs = require('fs');
+const path = require('path');
 const md5 = require('@abtnode/util/lib/md5');
 const Cron = require('@abtnode/cron');
 
@@ -9,6 +10,7 @@ const {
   fromBlockletSource,
   toBlockletSource,
 } = require('@blocklet/meta/lib/constants');
+const { listProviders } = require('@abtnode/router-provider');
 
 const RoutingSnapshot = require('./states/routing-snapshot');
 const BlockletRegistry = require('./blocklet/registry');
@@ -53,6 +55,11 @@ function ABTNode(options) {
     throw new Error('Can not initialize ABTNode without dataDir');
   }
 
+  const ekFile = path.join(options.dataDir, '.sock');
+  if (fs.existsSync(ekFile)) {
+    options.dek = fs.readFileSync(ekFile);
+  }
+
   if (typeof options.daemon === 'undefined') {
     options.daemon = false;
   }

package/lib/migrations/1.6.4-security.js

@@ -0,0 +1,59 @@
+/* eslint-disable no-await-in-loop */
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const yaml = require('js-yaml');
+const cloneDeep = require('lodash/cloneDeep');
+const security = require('@abtnode/util/lib/security');
+
+module.exports = async ({ states, configFile, dataDir }) => {
+  if (process.env.CI || process.env.NODE_ENV === 'test') {
+    return;
+  }
+
+  const file = path.join(dataDir, '.sock');
+  if (fs.existsSync(file)) {
+    return;
+  }
+
+  try {
+    fs.writeFileSync(file, crypto.randomBytes(32), { encoding: 'binary', mode: '0600' });
+
+    const config = yaml.safeLoad(fs.readFileSync(configFile).toString(), { json: true });
+    config.node.sk = security.encrypt(config.node.sk, config.node.did, fs.readFileSync(file));
+    fs.writeFileSync(configFile, yaml.dump(config));
+    await states.node.updateNodeInfo({ sk: config.node.sk });
+
+    const items = await states.blockletExtras.find();
+    for (const item of items) {
+      const newConfigs = cloneDeep(item.configs || []).map((c) => {
+        if (c.secure) {
+          c.value = security.encrypt(c.value, item.did, fs.readFileSync(file));
+        }
+
+        return c;
+      });
+
+      const newChildren = cloneDeep(item.children || []).map((x) => {
+        return {
+          ...x,
+          configs: cloneDeep(x.configs || []).map((c) => {
+            if (c.secure) {
+              c.value = security.encrypt(c.value, item.did, fs.readFileSync(file));
+            }
+
+            return c;
+          }),
+        };
+      });
+
+      await states.blockletExtras.update({ did: item.did }, { $set: { configs: newConfigs, children: newChildren } });
+    }
+
+    states.node.compactDatafile();
+    states.blockletExtras.compactDatafile();
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
+};

package/lib/migrations/1.6.5-security.js

@@ -0,0 +1,60 @@
+/* eslint-disable no-await-in-loop */
+const fs = require('fs');
+const path = require('path');
+const uniq = require('lodash/uniq');
+const cloneDeep = require('lodash/cloneDeep');
+const security = require('@abtnode/util/lib/security');
+const { forEachBlocklet } = require('@blocklet/meta/lib/util');
+
+module.exports = async ({ states, dataDir }) => {
+  if (process.env.CI || process.env.NODE_ENV === 'test') {
+    return;
+  }
+
+  const file = path.join(dataDir, '.sock');
+  if (fs.existsSync(file) === false) {
+    return;
+  }
+
+  try {
+    const extras = await states.blockletExtras.find();
+    const blocklets = await states.blocklet.find();
+    for (const blocklet of blocklets) {
+      forEachBlocklet(
+        blocklet,
+        (b) => {
+          let configKeys;
+          const extra = extras.find((x) => x.did === blocklet.meta.did);
+          const rootKeys = (extra.configs || []).map((x) => x.key);
+          if (b.meta.did === blocklet.meta.did) {
+            configKeys = rootKeys;
+          } else {
+            const childExtra = (extra.children || []).find((x) => x.did === b.meta.did);
+            const childKeys = childExtra ? childExtra.configs.map((x) => x.key) : [];
+            configKeys = uniq([...rootKeys, ...childKeys]);
+          }
+
+          b.environments = cloneDeep(b.environments || [])
+            .filter((x) => configKeys.includes(x.key) === false)
+            .map((c) => {
+              if (c.key === 'BLOCKLET_APP_SK') {
+                c.value = security.encrypt(c.value, b.meta.did, fs.readFileSync(file));
+              }
+
+              return c;
+            });
+        },
+        { sync: true }
+      );
+
+      await states.blocklet.updateById(blocklet._id, {
+        $set: { environments: blocklet.environments, children: blocklet.children, migratedFrom: '1.6.5' },
+      });
+    }
+
+    states.blocklet.compactDatafile();
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
+};

package/lib/router/index.js

@@ -1,4 +1,5 @@
 const get = require('lodash/get');
+const pick = require('lodash/pick');
 const cloneDeep = require('lodash/cloneDeep');
 const {
   DOMAIN_FOR_DEFAULT_SITE,
@@ -9,7 +10,6 @@ const {
   BLOCKLET_SITE_GROUP_SUFFIX,
 } = require('@abtnode/constant');
 const { BLOCKLET_UI_INTERFACES } = require('@blocklet/meta/lib/constants');
-const { pick } = require('lodash');
 const logger = require('@abtnode/logger')('@abtnode/core:router');
 
 const expandSites = (sites = []) => {

package/lib/states/base.js

@@ -191,6 +191,14 @@ class BaseState extends EventEmitter {
   insert(...args) {
     return this.asyncDB.insert(...args);
   }
+
+  compactDatafile() {
+    this.db.persistence.compactDatafile((err) => {
+      if (err) {
+        console.error(`failed to compact datafile: ${this.filename}`, err);
+      }
+    });
+  }
 }
 
 /**

package/lib/states/blocklet-extras.js

@@ -6,7 +6,9 @@ const camelCase = require('lodash/camelCase');
 
 const BaseState = require('./base');
 
-const { mergeConfigs } = require('../blocklet/extras');
+const { mergeConfigs, parseConfigs } = require('../blocklet/extras');
+
+const noop = (k) => (v) => v[k];
 
 class BlockletExtrasState extends BaseState {
   constructor(baseDir, options = {}) {
@@ -17,12 +19,13 @@ class BlockletExtrasState extends BaseState {
       {
         name: 'configs',
         beforeSet: mergeConfigs,
+        afterGet: parseConfigs,
       },
 
       // setting
       {
         name: 'settings',
-        beforeSet: (old, cur) => {
+        beforeSet: ({ old, cur }) => {
           const merged = { ...old, ...cur };
           Object.keys(merged).forEach((key) => {
             if (merged[key] === undefined || merged[key] === null) {
@@ -67,22 +70,23 @@ class BlockletExtrasState extends BaseState {
 
   generateGetFn(extra) {
     return async (did) => {
-      const { name } = extra;
+      const { dek } = this.options;
+      const { name, afterGet = noop('data') } = extra;
       const item = await this.asyncDB.findOne({ did });
-      return item ? item[name] : item;
+      return afterGet({ data: item ? item[name] : item, did, dek });
     };
   }
 
   generateSetFn(extra) {
     return async (did, data) => {
-      const { name, beforeSet } = extra;
-      const hasBeforeSet = beforeSet && typeof beforeSet === 'function';
+      const { dek } = this.options;
+      const { name, beforeSet = noop('cur') } = extra;
       const item = await this.asyncDB.findOne({ did });
 
       if (!item) {
         const insertData = {
           did,
-          [name]: hasBeforeSet ? beforeSet(undefined, data) : data,
+          [name]: beforeSet({ old: undefined, cur: data, did, dek }),
         };
 
         const info = await this.asyncDB.insert(insertData);
@@ -93,7 +97,7 @@ class BlockletExtrasState extends BaseState {
       const itemNameValue = item[name];
       const updated = await this.update(item._id, {
         $set: {
-          [name]: hasBeforeSet ? beforeSet(itemNameValue, data) : data,
+          [name]: beforeSet({ old: itemNameValue, cur: data, did, dek }),
         },
       });
       return updated[name];
@@ -132,22 +136,23 @@ class BlockletExtrasState extends BaseState {
 
   generateGetChildFn(extra) {
     return async (did, childDid) => {
-      const { name } = extra;
+      const { dek } = this.options;
+      const { name, afterGet = noop('data') } = extra;
       const item = await this.asyncDB.findOne({ did });
       const children = (item || {}).children || [];
       const subItem = (children || []).find((x) => x.did === childDid);
-      return subItem ? subItem[name] : null;
+      return afterGet({ data: subItem ? subItem[name] : null, did, dek });
     };
   }
 
   generateSetChildFn(extra) {
     return async (did, childDid, data) => {
-      const { name, beforeSet } = extra;
-      const hasBeforeSet = beforeSet && typeof beforeSet === 'function';
+      const { dek } = this.options;
+      const { name, beforeSet = noop('cur') } = extra;
       const item = await this.asyncDB.findOne({ did });
 
       if (!item) {
-        const newData = hasBeforeSet ? beforeSet(undefined, data) : data;
+        const newData = beforeSet({ old: undefined, cur: data, did, dek });
         const insertData = {
           did,
           children: [
@@ -168,7 +173,7 @@ class BlockletExtrasState extends BaseState {
       const subItem = (children || []).find((x) => x.did === childDid);
 
       if (!subItem) {
-        const newData = hasBeforeSet ? beforeSet(undefined, data) : data;
+        const newData = beforeSet({ old: undefined, cur: data, did, dek });
         await this.update(item._id, {
           $addToSet: {
             children: {
@@ -182,7 +187,7 @@ class BlockletExtrasState extends BaseState {
         return newData;
       }
 
-      const newData = hasBeforeSet ? beforeSet(subItem[name], data) : data;
+      const newData = beforeSet({ old: subItem[name], cur: data, did, dek });
 
       children.forEach((x) => {
         if (x.did === childDid) {

package/lib/states/blocklet.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-async-promise-executor */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable function-paren-newline */
 /* eslint-disable no-underscore-dangle */
@@ -5,6 +6,7 @@ const omit = require('lodash/omit');
 const uniq = require('lodash/uniq');
 const detectPort = require('detect-port');
 const Lock = require('@abtnode/util/lib/lock');
+const security = require('@abtnode/util/lib/security');
 const { fixPerson, fixInterfaces } = require('@blocklet/meta/lib/fix');
 const {
   BlockletStatus,
@@ -27,6 +29,40 @@ const getMaxPort = (ports = {}) => Math.max(Object.values(ports).map(Number));
 const getExternalPortsFromMeta = (meta) =>
   (meta.interfaces || []).map((x) => x.port && x.port.external).filter(Boolean);
 
+const formatBlocklet = (blocklet, phase, dek) => {
+  forEachBlocklet(
+    blocklet,
+    (b) => {
+      if (b.meta) {
+        fixPerson(b.meta);
+        fixInterfaces(b.meta);
+      }
+
+      b.children = b.children || [];
+
+      if (!b.environments || !b.meta || !dek) {
+        return;
+      }
+
+      ['BLOCKLET_APP_SK'].forEach((key) => {
+        const env = b.environments.find((x) => x.key === key);
+        if (!env) {
+          return;
+        }
+        if (phase === 'onUpdate' && env.value.indexOf('0x') === 0) {
+          env.value = security.encrypt(env.value, b.meta.did, dek);
+        }
+        if (phase === 'onRead' && env.value.indexOf('0x') === -1) {
+          env.value = security.decrypt(env.value, b.meta.did, dek);
+        }
+      });
+    },
+    { sync: true }
+  );
+
+  return blocklet;
+};
+
 class BlockletState extends BaseState {
   /**
    * Creates an instance of BlockletState
@@ -48,14 +84,7 @@ class BlockletState extends BaseState {
           return reject(err);
         }
 
-        if (doc) {
-          // TODO: this only exists for backward compatible
-          fixPerson(doc.meta);
-          fixInterfaces(doc.meta);
-          doc.children = doc.children || [];
-        }
-
-        return resolve(doc);
+        return resolve(doc ? formatBlocklet(doc, 'onRead', this.options.dek) : null);
       });
     });
   }
@@ -70,14 +99,7 @@ class BlockletState extends BaseState {
             return reject(err);
           }
 
-          return resolve(
-            docs.filter(Boolean).map((doc) => {
-              // TODO: this only exists for backward compatible
-              fixPerson(doc.meta);
-              fixInterfaces(doc.meta);
-              return doc;
-            })
-          );
+          return resolve(docs.filter(Boolean).map((doc) => formatBlocklet(doc, 'onRead', this.options.dek)));
         });
     });
   }
@@ -114,7 +136,6 @@ class BlockletState extends BaseState {
   } = {}) {
     return this.getBlocklet(did).then(
       (doc) =>
-        // eslint-disable-next-line no-async-promise-executor
         new Promise(async (resolve, reject) => {
           if (doc) {
             reject(new Error('Blocklet already added'));
@@ -165,6 +186,26 @@ class BlockletState extends BaseState {
     );
   }
 
+  updateBlocklet(did, updates) {
+    return this.getBlocklet(did).then(
+      (doc) =>
+        new Promise(async (resolve, reject) => {
+          if (!doc) {
+            reject(new Error('Blocklet does not exist'));
+            return;
+          }
+
+          try {
+            const formatted = formatBlocklet(updates, 'onUpdate', this.options.dek);
+            const newDoc = await this.updateById(doc._id, { $set: formatted });
+            resolve(newDoc);
+          } catch (err) {
+            reject(err);
+          }
+        })
+    );
+  }
+
   upgradeBlocklet({ meta, source, deployedFrom, children } = {}) {
     return this.getBlocklet(meta.did).then(
       (doc) =>

package/lib/states/node.js

@@ -2,6 +2,7 @@
 const omit = require('lodash/omit');
 const isEqual = require('lodash/isEqual');
 const isEmpty = require('lodash/isEmpty');
+const security = require('@abtnode/util/lib/security');
 const { isFromPublicKey } = require('@arcblock/did');
 const logger = require('@abtnode/logger')('@abtnode/core:node');
 const { ROUTER_PROVIDER_NONE, NODE_MODES, DISK_ALERT_THRESHOLD_PERCENT } = require('@abtnode/constant');
@@ -58,6 +59,7 @@ class NodeState extends BaseState {
    */
   read() {
     return new Promise((resolve, reject) => {
+      const { nodeDid, dek } = this.options;
       this.db.findOne({ did: this.options.nodeDid }, (err, record) => {
         if (err) {
           // eslint-disable-next-line no-console
@@ -66,6 +68,10 @@ class NodeState extends BaseState {
         }
 
         if (record) {
+          if (dek) {
+            record.sk = security.decrypt(record.sk, record.did, dek);
+          }
+
           return resolve(record);
         }
 
@@ -74,7 +80,6 @@ class NodeState extends BaseState {
           description,
           nodeSk,
           nodePk,
-          nodeDid,
           nodeOwner,
           port,
           version,
@@ -100,7 +105,7 @@ class NodeState extends BaseState {
                 name,
                 description,
                 pk: nodePk,
-                sk: nodeSk,
+                sk: dek ? security.encrypt(nodeSk, nodeDid, dek) : nodeSk,
                 did: nodeDid,
                 initialized,
                 version,

package/lib/util/blocklet.js

@@ -192,7 +192,6 @@ const fillBlockletConfigs = (blocklet, configs) => {
     acc[x.key] = x.value;
     return acc;
   }, {});
-  blocklet.userEnvironments = blocklet.configObj; // deprecated
   blocklet.environmentObj = (blocklet.environments || []).reduce((acc, x) => {
     acc[x.key] = x.value;
     return acc;
@@ -308,6 +307,7 @@ const getRuntimeEnvironments = (blocklet, nodeEnvironments) => {
 
   return {
     ...blocklet.environmentObj,
+    ...blocklet.configObj,
     ...nodeEnvironments,
     ...safeNodeEnvironments,
   };
@@ -798,7 +798,7 @@ const getRuntimeInfo = async (appId) => {
   const proc = await getProcessInfo(appId);
   return {
     pid: proc.pid,
-    uptime: proc.pm2_env ? Number(proc.pm2_env.pm_uptime) : 0,
+    uptime: proc.pm2_env ? +new Date() - Number(proc.pm2_env.pm_uptime) : 0,
     memoryUsage: proc.monit.memory,
     cpuUsage: proc.monit.cpu,
     status: proc.pm2_env ? proc.pm2_env.status : null,

package/lib/util/get-ip-dns-domain-for-blocklet.js

@@ -3,7 +3,7 @@ const { DEFAULT_IP_DNS_DOMAIN_SUFFIX } = require('@abtnode/constant');
 
 const SLOT_FOR_IP_DNS_SITE = '888-888-888-888';
 
-const formatName = (name) => slugify(name.replace(/^[@./-]/, '').replace(/[@./]/g, '-'));
+const formatName = (name) => slugify(name.replace(/^[@./-]/, '').replace(/[@./_]/g, '-'));
 
 const hiddenInterfaceNames = ['public', 'publicUrl'];
 

package/lib/util/requirement.js

@@ -10,7 +10,8 @@ const isSatisfied = (requirements, throwOnUnsatisfied = true) => {
   }
 
   const { platform: actualOs, arch: actualCpu } = process;
-  const { os: expectedOs, cpu: expectedCpu, abtnode: expectedVersion } = Object.assign(defaults, requirements);
+  const { os: expectedOs, cpu: expectedCpu, abtnode, server } = Object.assign(defaults, requirements);
+  const expectedVersion = server || abtnode;
 
   const isOsSatisfied =
     expectedOs === '*' || expectedOs === actualOs || (Array.isArray(expectedOs) && expectedOs.includes(actualOs));

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.6.1",
+  "version": "1.6.5",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,26 +19,26 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/constant": "1.6.1",
-    "@abtnode/cron": "1.6.1",
-    "@abtnode/logger": "1.6.1",
-    "@abtnode/queue": "1.6.1",
-    "@abtnode/rbac": "1.6.1",
-    "@abtnode/router-provider": "1.6.1",
-    "@abtnode/static-server": "1.6.1",
-    "@abtnode/timemachine": "1.6.1",
-    "@abtnode/util": "1.6.1",
-    "@arcblock/did": "^1.13.65",
-    "@arcblock/event-hub": "1.13.65",
+    "@abtnode/constant": "1.6.5",
+    "@abtnode/cron": "1.6.5",
+    "@abtnode/logger": "1.6.5",
+    "@abtnode/queue": "1.6.5",
+    "@abtnode/rbac": "1.6.5",
+    "@abtnode/router-provider": "1.6.5",
+    "@abtnode/static-server": "1.6.5",
+    "@abtnode/timemachine": "1.6.5",
+    "@abtnode/util": "1.6.5",
+    "@arcblock/did": "^1.13.77",
+    "@arcblock/event-hub": "1.13.77",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/vc": "^1.13.65",
-    "@blocklet/meta": "1.6.1",
+    "@arcblock/vc": "^1.13.77",
+    "@blocklet/meta": "1.6.5",
     "@fidm/x509": "^1.2.1",
     "@nedb/core": "^1.2.2",
     "@nedb/multi": "^1.2.2",
-    "@ocap/mcrypto": "^1.13.65",
-    "@ocap/util": "^1.13.65",
-    "@ocap/wallet": "^1.13.65",
+    "@ocap/mcrypto": "^1.13.77",
+    "@ocap/util": "^1.13.77",
+    "@ocap/wallet": "^1.13.77",
     "@slack/webhook": "^5.0.3",
     "axios": "^0.21.4",
     "axon": "^2.0.3",
@@ -73,5 +73,5 @@
     "express": "^4.17.1",
     "jest": "^27.3.1"
   },
-  "gitHead": "5be6776ba9bbfdbff4000fdab7dcb77686b3a9e8"
+  "gitHead": "59da906682f8122797a932640aebbad5a30dfcc3"
 }