@abtnode/core 1.15.17 → 1.16.0-beta-8ee536d7

package/lib/states/blocklet.js

@@ -1,83 +1,227 @@
+/* eslint-disable no-async-promise-executor */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable function-paren-newline */
 /* eslint-disable no-underscore-dangle */
 const omit = require('lodash/omit');
 const uniq = require('lodash/uniq');
+const cloneDeep = require('lodash/cloneDeep');
 const detectPort = require('detect-port');
 const Lock = require('@abtnode/util/lib/lock');
+const security = require('@abtnode/util/lib/security');
 const { fixPerson, fixInterfaces } = require('@blocklet/meta/lib/fix');
+const { getDisplayName, forEachBlocklet, forEachBlockletSync, forEachChildSync } = require('@blocklet/meta/lib/util');
 const {
   BlockletStatus,
   BlockletSource,
   BLOCKLET_MODES,
   BLOCKLET_DEFAULT_PORT_NAME,
   BLOCKLET_INTERFACE_TYPE_SERVICE,
-} = require('@blocklet/meta/lib/constants');
+} = require('@blocklet/constant');
+const { APP_STRUCT_VERSION } = require('@abtnode/constant');
 
 const logger = require('@abtnode/logger')('state-blocklet');
 
 const BaseState = require('./base');
-const { forEachBlocklet } = require('../util/blocklet');
-const { validateBlockletMeta } = require('../util');
+const { checkDuplicateComponents, ensureMeta } = require('../util/blocklet');
+const { validateBlockletMeta } = require('../util/blocklet');
 
 const lock = new Lock('blocklet-port-assign-lock');
 
+const isHex = (str) => /^0x[0-9a-f]+$/i.test(str);
 const getMaxPort = (ports = {}) => Math.max(Object.values(ports).map(Number));
 
+// structV1Did is just for migration purpose and should be removed in the future
+const getConditions = (did) => [{ 'meta.did': did }, { appDid: did }, { appPid: did }, { structV1Did: did }];
+
 const getExternalPortsFromMeta = (meta) =>
   (meta.interfaces || []).map((x) => x.port && x.port.external).filter(Boolean);
 
+const formatBlocklet = (blocklet, phase, dek) => {
+  forEachBlockletSync(blocklet, (b) => {
+    if (b.meta) {
+      fixPerson(b.meta);
+      fixInterfaces(b.meta);
+    }
+
+    if (phase === 'onRead') {
+      b.children = b.children || [];
+      b.environments = b.environments || [];
+    }
+
+    if (!b.environments || !b.meta || !dek) {
+      return;
+    }
+
+    (Array.isArray(b.migratedFrom) ? b.migratedFrom : []).forEach((x) => {
+      if (phase === 'onUpdate' && isHex(x.appSk) === true) {
+        x.appSk = security.encrypt(x.appSk, b.meta.did, dek);
+      }
+      if (phase === 'onRead' && isHex(x.appSk) === false) {
+        x.appSk = security.decrypt(x.appSk, b.meta.did, dek);
+      }
+    });
+
+    ['BLOCKLET_APP_SK', 'BLOCKLET_APP_PSK'].forEach((key) => {
+      const env = b.environments.find((x) => x.key === key);
+      if (!env) {
+        return;
+      }
+      // salt in blocklet state is different from the salt in blocklet-extra state
+      // in blocklet-extra state, salt is app meta did in each component
+      // in blocklet state, salt is component meta did in each component
+      if (phase === 'onUpdate' && isHex(env.value) === true) {
+        env.value = security.encrypt(env.value, b.meta.did, dek);
+      }
+      if (phase === 'onRead' && isHex(env.value) === false) {
+        env.value = security.decrypt(env.value, b.meta.did, dek);
+      }
+    });
+  });
+
+  return blocklet;
+};
+
+const fixChildren = (children) => {
+  if (!children) {
+    return;
+  }
+
+  children.forEach((child) => {
+    child.mode = child.mode || BLOCKLET_MODES.PRODUCTION;
+  });
+};
+
+// type Application = {
+//   appDid: DID,
+//   appPid: DID,
+//   meta: Meta,
+//   source: BlockletSource,
+//   deployedFrom: string,
+//   mode: BLOCKLET_MODES;
+//   status: BlockletStatus,
+//   children: Component,
+//   ports: Ports;
+//   mountPoint: string; // deprecated
+//   migratedFrom: Array<{ appSk: string, appDid: DID, at: Date }>,
+//   externalSk: boolean,
+//   structV1Did: DID, // just for migration purpose and should be removed in the future
+// };
+
+// type Component = {
+//   mountPoint: string;
+//   meta: Meta,
+//   bundleSource: ComponentSource,
+//   source: BlockletSource,
+//   deployedFrom: string,
+//   mode: BLOCKLET_MODES;
+//   status: BlockletStatus,
+//   children: Component,
+//   ports: Ports;
+//   dependents: Array<{
+//     did: DID, // blocklet did
+//     required: boolean,
+//     version: SemverRange,
+//   }>;
+//   dependencies: Array<{
+//     id: string, // format: <did1/did2/did3>
+//     required: boolean,
+//   }>;
+//   dynamic: boolean; // deprecated
+// };
+
+// type Ports = {
+//   [name: string]: Number; // name is in meta.interfaces[].port
+// };
+
 class BlockletState extends BaseState {
   /**
    * Creates an instance of BlockletState
    * @param {string} baseDir
-   * @param {object} options
-   * @param {string} options.blockletPort - from which port to start new blocklets
+   * @param {object} config
+   * @param {string} config.blockletPort - from which port to start new blocklets
    * @memberof BlockletState
    */
-  constructor(baseDir, options = {}) {
-    super(baseDir, { filename: 'blocklet.db', ...options });
+  constructor(baseDir, config = {}) {
+    super(baseDir, { filename: 'blocklet.db', ...config });
 
-    this.defaultPort = options.blockletPort || 5555;
+    this.defaultPort = config.blockletPort || 5555;
   }
 
-  getBlocklet(did) {
+  getBlocklet(did, { decryptSk = true } = {}) {
     return new Promise((resolve, reject) => {
-      this.db.findOne({ 'meta.did': did }, (err, doc) => {
+      if (!did) {
+        resolve(null);
+      }
+
+      this.findOne({ $or: getConditions(did) }, (err, doc) => {
         if (err) {
           return reject(err);
         }
 
-        if (doc) {
-          // TODO: this only exists for backward compatible
-          fixPerson(doc.meta);
-          fixInterfaces(doc.meta);
-          doc.children = doc.children || [];
+        return resolve(doc ? formatBlocklet(doc, 'onRead', decryptSk ? this.config.dek : null) : null);
+      });
+    });
+  }
+
+  getBlockletMetaDid(did) {
+    return new Promise((resolve, reject) => {
+      if (!did) {
+        resolve(null);
+      }
+
+      this.findOne({ $or: getConditions(did) }, (err, doc) => {
+        if (err) {
+          return reject(err);
         }
 
-        return resolve(doc);
+        return resolve(doc ? doc.meta.did : null);
+      });
+    });
+  }
+
+  async getBlockletStatus(did) {
+    return new Promise((resolve, reject) => {
+      if (!did) {
+        resolve(null);
+      }
+
+      this.findOne({ $or: getConditions(did) }, { status: 1 }, (err, doc) => {
+        if (err) {
+          return reject(err);
+        }
+
+        return resolve(doc ? doc.status : null);
+      });
+    });
+  }
+
+  hasBlocklet(did) {
+    return new Promise((resolve, reject) => {
+      if (!did) {
+        resolve(false);
+      }
+
+      this.count({ $or: getConditions(did) }, (err, count) => {
+        if (err) {
+          return reject(err);
+        }
+
+        return resolve(!!count);
       });
     });
   }
 
   getBlocklets(query = {}, projection) {
     return new Promise((resolve, reject) => {
-      this.db
-        .find(query, projection)
-        .sort('-createdAt')
-        .exec((err, docs) => {
+      this.cursor(query)
+        .projection(projection)
+        .sort({ createdAt: -1 })
+        .exec((err, docs = []) => {
           if (err) {
             return reject(err);
           }
 
-          return resolve(
-            docs.filter(Boolean).map((doc) => {
-              // TODO: this only exists for backward compatible
-              fixPerson(doc.meta);
-              fixInterfaces(doc.meta);
-              return doc;
-            })
-          );
+          return resolve(docs.filter(Boolean).map((doc) => formatBlocklet(doc, 'onRead', this.config.dek)));
         });
     });
   }
@@ -91,32 +235,35 @@ class BlockletState extends BaseState {
             return reject(new Error(`Try to remove non-existing blocklet ${did}`));
           }
 
-          this.db.remove({ _id: doc._id }, (err) => {
+          this.remove({ _id: doc._id }, (err) => {
             if (err) {
               return reject(err);
             }
 
             this.emit('remove', doc);
-            return resolve(doc);
+            return resolve(formatBlocklet(doc, 'onRead', this.config.dek));
           });
         })
     );
   }
 
   addBlocklet({
-    did,
     meta,
     source = BlockletSource.registry,
     status = BlockletStatus.added,
-    deployedFrom,
+    deployedFrom = '',
     mode = BLOCKLET_MODES.PRODUCTION,
-    childrenMeta = [],
+    children: rawChildren = [],
+    appPid = null, // the permanent appDid, which will not change after initial set
+    migratedFrom = [], // the complete migrate history
+    // whether sk is managed by some party beside server, such as did-wallet
+    // externalSk is always true in struct V2 blocklet
+    externalSk = true,
   } = {}) {
-    return this.getBlocklet(did).then(
-      (doc) =>
-        // eslint-disable-next-line no-async-promise-executor
+    return this.getBlocklet(meta.did).then(
+      (exist) =>
         new Promise(async (resolve, reject) => {
-          if (doc) {
+          if (exist) {
             reject(new Error('Blocklet already added'));
             return;
           }
@@ -124,39 +271,48 @@ class BlockletState extends BaseState {
           try {
             fixPerson(meta);
             fixInterfaces(meta);
-            const sanitized = validateBlockletMeta(meta);
+            let sanitized = validateBlockletMeta(meta);
+            // bundle info
+            sanitized = ensureMeta(sanitized);
+            sanitized = omit(sanitized, ['htmlAst']);
 
             // get ports
             await lock.acquire();
 
             const ports = await this.getBlockletPorts({ interfaces: sanitized.interfaces || [] });
 
-            const children = await this.fillChildrenPorts(this.getChildrenFromMetas(childrenMeta), {
+            const children = await this.fillChildrenPorts(rawChildren, {
               defaultPort: getMaxPort(ports),
             });
 
+            fixChildren(children);
+
+            const data = {
+              appDid: null, // will updated later when updating blocklet environments
+              appPid,
+              mode,
+              meta: sanitized,
+              status,
+              source,
+              deployedFrom,
+              ports,
+              environments: [],
+              children,
+              migratedFrom,
+              externalSk,
+              structVersion: APP_STRUCT_VERSION,
+            };
+
             // add to db
-            this.db.insert(
-              {
-                mode,
-                meta: omit(sanitized, ['htmlAst']),
-                status,
-                source,
-                deployedFrom,
-                ports,
-                environments: [],
-                children,
-              },
-              (err, newDoc) => {
-                lock.release();
-                if (err) {
-                  return reject(err);
-                }
-
-                this.emit('add', newDoc);
-                return resolve(newDoc);
+            this.insert(data, (err, doc) => {
+              lock.release();
+              if (err) {
+                return reject(err);
               }
-            );
+
+              this.emit('add', doc);
+              return resolve(doc);
+            });
           } catch (err) {
             lock.release();
             reject(err);
@@ -165,7 +321,28 @@ class BlockletState extends BaseState {
     );
   }
 
-  upgradeBlocklet({ meta, source, deployedFrom, children } = {}) {
+  // FIXME: 这个接口比较危险，可能会修改一些本不应该修改的字段，后续需要考虑改进
+  updateBlocklet(did, updates) {
+    return this.getBlocklet(did).then(
+      (doc) =>
+        new Promise(async (resolve, reject) => {
+          if (!doc) {
+            reject(new Error('Blocklet does not exist'));
+            return;
+          }
+
+          try {
+            const formatted = formatBlocklet(cloneDeep(updates), 'onUpdate', this.config.dek);
+            const newDoc = await this.updateById(doc._id, { $set: formatted });
+            resolve(newDoc);
+          } catch (err) {
+            reject(err);
+          }
+        })
+    );
+  }
+
+  upgradeBlocklet({ meta, source, deployedFrom = '', children } = {}) {
     return this.getBlocklet(meta.did).then(
       (doc) =>
         // eslint-disable-next-line no-async-promise-executor
@@ -195,17 +372,18 @@ class BlockletState extends BaseState {
             logger.info('Fill children ports when when upgrading blocklet', { name: doc.meta.name, did: doc.meta.did });
             await this.fillChildrenPorts(children, { oldChildren: doc.children, defaultPort: getMaxPort(ports) });
 
+            fixChildren(children);
+
             // add to db
-            const newDoc = await this.updateById(doc._id, {
-              $set: {
-                meta: omit(sanitized, ['htmlAst']),
-                source,
-                deployedFrom,
-                children,
-                ports,
-              },
+            const newDoc = await this.updateBlocklet(meta.did, {
+              meta: omit(sanitized, ['htmlAst']),
+              source,
+              deployedFrom,
+              children,
+              ports,
             });
             lock.release();
+
             this.emit('upgrade', newDoc);
             resolve(newDoc);
           } catch (err) {
@@ -224,7 +402,7 @@ class BlockletState extends BaseState {
     defaultPort = 0,
   } = {}) {
     try {
-      const blocklets = await this.getBlocklets({}, { port: 1, ports: 1, 'meta.interfaces': 1 });
+      const blocklets = await this.getBlocklets({}, { port: 1, ports: 1, 'meta.interfaces': 1, children: 1 });
 
       const occupiedExternalPorts = new Map();
       const occupiedInternalPorts = new Map();
@@ -331,31 +509,43 @@ class BlockletState extends BaseState {
    * @return {Object} { <did> : { interfaceName } }
    */
   async groupAllInterfaces() {
-    const blocklets = await this.getBlocklets({}, { meta: 1 });
+    const blocklets = await this.getBlocklets({}, { meta: 1, children: 1 });
     const result = {};
-    blocklets.forEach((blocklet) => {
-      const { did, interfaces } = blocklet.meta;
-      if (typeof result[did] === 'undefined') {
-        result[did] = {};
+    const fillResult = (component, { id }) => {
+      const { interfaces } = component.meta;
+      if (typeof result[id] === 'undefined') {
+        result[id] = {};
       }
 
       (interfaces || []).forEach((x) => {
-        result[did][x.name] = x;
+        result[id][x.name] = x;
       });
+    };
+
+    blocklets.forEach((blocklet) => {
+      forEachBlockletSync(blocklet, fillResult);
     });
 
     return result;
   }
 
-  async setBlockletStatus(did, status) {
+  /**
+   * @param {String} did blocklet did
+   * @param {BlockletStatus} status blocklet status
+   *
+   * children status only different with parent before blocklet installation
+   * @param {Array<componentId>} children
+   */
+  async setBlockletStatus(did, status, { children } = {}) {
     if (typeof status === 'undefined') {
       throw new Error('Unsupported blocklet status');
     }
-    const record = await this.getBlocklet(did);
-    // when status change update record
-    if (record.status === status) {
-      return record;
+
+    const doc = await this.getBlocklet(did);
+    if (doc.status === status && !children) {
+      return formatBlocklet(doc, 'onRead', this.config.dek);
     }
+
     const updates = { status, startedAt: undefined, stoppedAt: undefined };
     if (status === BlockletStatus.running) {
       updates.startedAt = new Date();
@@ -367,30 +557,46 @@ class BlockletState extends BaseState {
       updates.stoppedAt = new Date();
     }
 
-    const res = await this.update(record._id, {
-      $set: updates,
+    // update children status
+    forEachChildSync(doc, (child, { id }) => {
+      if (children === 'all') {
+        child.status = status;
+        return;
+      }
+
+      if (!children) {
+        if (
+          ![
+            BlockletStatus.waiting,
+            BlockletStatus.upgrading,
+            BlockletStatus.installing,
+            BlockletStatus.starting,
+          ].includes(status)
+        ) {
+          child.status = status;
+        }
+
+        return;
+      }
+
+      if (children.includes(id)) {
+        child.status = status;
+      }
     });
-    return res;
-  }
 
-  getChildrenFromMetas(childrenMeta) {
-    return childrenMeta.map((x) => ({ meta: x }));
+    updates.children = doc.children;
+    return this.updateBlocklet(did, updates);
   }
 
-  async fillChildrenPorts(children, { defaultPort = 0, oldChildren } = {}) {
+  async fillChildrenPorts(children, { defaultPort = 0, oldChildren, returnMaxPort } = {}) {
     let _maxPort = defaultPort;
     for (const child of children || []) {
       // generate ports
       const childMeta = child.meta;
+      const oldChild = (oldChildren || []).find((x) => x.meta.did === child.meta.did);
 
       // get skipOccupiedCheckPorts
-      let skipOccupiedCheckPorts = [];
-      if (Array.isArray(oldChildren)) {
-        const oldChild = oldChildren.find((x) => x.meta.did === child.meta.did);
-        if (oldChild) {
-          skipOccupiedCheckPorts = getExternalPortsFromMeta(oldChild.meta);
-        }
-      }
+      const skipOccupiedCheckPorts = oldChild ? getExternalPortsFromMeta(oldChild.meta) : [];
 
       const ports = await this.getBlockletPorts({
         interfaces: childMeta.interfaces || [],
@@ -399,22 +605,17 @@ class BlockletState extends BaseState {
       });
       _maxPort = getMaxPort(ports);
 
-      // fill old child's port to new child
-      if (Array.isArray(oldChildren)) {
-        const oldChild = oldChildren.find((x) => x.meta.did === child.meta.did);
-        if (oldChild && oldChild.ports) {
-          logger.info('Merge the previous ports to child blocklet', {
-            did: child.meta.did,
-            name: child.meta.name,
-            oldPorts: oldChild.ports,
-            ports,
-          });
-          Object.keys(ports).forEach((p) => {
-            ports[p] = oldChild.ports[p] || ports[p];
-          });
-          child.ports = ports;
-          continue; // eslint-disable-line
-        }
+      if (oldChild && oldChild.ports) {
+        // fill old child's port to new child
+        logger.info('Merge the previous ports to child blocklet', {
+          did: child.meta.did,
+          name: child.meta.name,
+          oldPorts: oldChild.ports,
+          ports,
+        });
+        Object.keys(ports).forEach((p) => {
+          ports[p] = oldChild.ports[p] || ports[p];
+        });
       }
 
       // assign a new port to child
@@ -422,10 +623,70 @@ class BlockletState extends BaseState {
       child.ports = ports;
     }
 
+    for (const child of children || []) {
+      const oldChild = (oldChildren || []).find((x) => x.meta.did === child.meta.did);
+
+      _maxPort = await this.fillChildrenPorts(child.children || [], {
+        defaultPort: _maxPort,
+        oldChildren: oldChild?.children,
+        returnMaxPort: true,
+      });
+    }
+
+    if (returnMaxPort) {
+      return _maxPort;
+    }
+
     return children;
   }
+
+  async addChildren(did, children) {
+    const parent = await this.getBlocklet(did);
+    if (!parent) {
+      throw new Error('Blocklet does not exist');
+    }
+
+    const oldChildren = parent.children || [];
+
+    const newChildren = [...oldChildren];
+    for (const child of children) {
+      const { meta, mountPoint, bundleSource = null, source = '', deployedFrom = '', mode } = child;
+
+      if (!mountPoint) {
+        throw new Error(`mountPoint is required when adding component ${getDisplayName(child, true)}`);
+      }
+
+      checkDuplicateComponents([child, ...newChildren]);
+
+      newChildren.push({
+        mountPoint,
+        meta,
+        bundleSource,
+        source,
+        deployedFrom,
+        mode,
+        status: BlockletStatus.added,
+        children: child.children,
+      });
+
+      fixChildren(newChildren);
+    }
+
+    // use upgradeBlocklet to assign ports to children and write new data to db
+    return this.upgradeBlocklet({
+      meta: parent.meta,
+      source: parent.source,
+      deployedFrom: parent.deployedFrom,
+      children: newChildren,
+    });
+  }
+
+  async updateStructV1Did(did, v1Did) {
+    return this.updateBlocklet(did, { structV1Did: v1Did });
+  }
 }
 
 BlockletState.BlockletStatus = BlockletStatus;
+BlockletState.formatBlocklet = formatBlocklet;
 
 module.exports = BlockletState;

package/lib/states/cache.js

@@ -5,13 +5,15 @@ const BaseState = require('./base');
  * This db is used to save arbitrary cached data.
  */
 class CacheState extends BaseState {
-  constructor(baseDir, options = {}) {
-    super(baseDir, { filename: 'cache.db', ...options });
+  constructor(baseDir, config = {}) {
+    super(baseDir, { filename: 'cache.db', ...config });
 
-    this.db.ensureIndex({ fieldName: 'key', unique: true }, (error) => {
-      if (error) {
-        logger.error('ensure index failed', { error });
-      }
+    this.onReady(() => {
+      this.ensureIndex({ fieldName: 'key', unique: true }, (error) => {
+        if (error) {
+          logger.error('ensure index failed', { error });
+        }
+      });
     });
   }