@abtnode/core 1.6.15 → 1.6.19

package/lib/blocklet/migration.js

@@ -1,67 +1,25 @@
 /* eslint-disable no-await-in-loop */
-const childProcess = require('child_process');
 const fs = require('fs-extra');
 const path = require('path');
 const semver = require('semver');
+const runScript = require('@abtnode/util/lib/run-script');
 
 const { getMigrationScripts: getScripts } = require('../migrations');
 const { getSafeEnv } = require('../util');
 const { name } = require('../../package.json');
 const logger = require('@abtnode/logger')(`${name}:blocklet:migration`); // eslint-disable-line
 
-const _runScript = ({ appDir, env, migrationScript, progress = false }) => {
-  const safeEnv = getSafeEnv(env);
-
-  const child = childProcess.exec(`node ${migrationScript}`, {
-    cwd: appDir,
-    env: safeEnv,
-    stdio: 'inherit',
-  });
-  let hasUnhandledRejection = false;
-
-  if (progress) {
-    child.stdout.pipe(process.stdout);
-    child.stderr.pipe(process.stderr);
-  }
-
-  return new Promise((resolve, reject) => {
-    const errorMessages = [];
-
-    child.stderr.on('data', (err) => {
-      // Check if has unhandledRejection in childProcess
-      // https://stackoverflow.com/questions/32784649/gracefully-handle-errors-in-child-processes-in-nodejs
-      if (err.includes('UnhandledPromiseRejectionWarning')) {
-        hasUnhandledRejection = true;
-      }
-      errorMessages.push(err);
-    });
-
-    child.on('exit', (code) => {
-      if (errorMessages.length > 0) {
-        if (code !== 0 || hasUnhandledRejection) {
-          return reject(new Error(errorMessages.join('\r\n')));
-        }
-
-        if (!progress) {
-          errorMessages.forEach((message) => process.stderr.write(message));
-        }
-      }
-
-      return resolve();
-    });
-  });
-};
-
 async function runScripts({
+  blocklet,
+  appDir,
+  env,
+  oldVersion,
   dbDir,
   backupDir,
   scriptsDir,
   printInfo,
   printSuccess,
   printError,
-  appDir,
-  env,
-  oldVersion,
 }) {
   let scripts = [];
   try {
@@ -90,7 +48,11 @@ async function runScripts({
     const { script: scriptPath } = pendingScripts[i];
     try {
       printInfo(`Migration script started: ${scriptPath}`);
-      await _runScript({ appDir, env, migrationScript: path.join(scriptsDir, scriptPath) });
+      await runScript(`node ${path.join(scriptsDir, scriptPath)}`, [blocklet.env.appId, 'migration'].join(':'), {
+        cwd: appDir,
+        env: getSafeEnv(env),
+        silent: false,
+      });
       printInfo(`Migration script executed: ${scriptPath}`);
     } catch (migrationErr) {
       printError(`Failed to execute migration script: ${scriptPath}, error: ${migrationErr.message}`);
@@ -123,6 +85,7 @@ async function doRestore({ dbDir, backupDir, printInfo, printSuccess }) {
 }
 
 module.exports = async ({
+  blocklet,
   appDir,
   env,
   oldVersion,
@@ -142,15 +105,16 @@ module.exports = async ({
   fs.ensureDirSync(backupDir);
 
   await runScripts({
+    blocklet,
+    appDir,
+    env,
+    oldVersion,
+    newVersion,
     dbDir,
     backupDir,
     scriptsDir,
     printError,
     printInfo,
     printSuccess,
-    appDir,
-    env,
-    oldVersion,
-    newVersion,
   });
 };

package/lib/event.js

@@ -182,7 +182,7 @@ module.exports = ({
       .catch((error) => logger.error('refresh blocklets failed on initialize the registry', { error }));
 
     // We need update router on some fields change
-    const fields = ['enableWelcomePage', 'webWalletUrl'];
+    const fields = ['enableWelcomePage', 'webWalletUrl', 'registerUrl'];
     const shouldUpdateRouter = fields.some((x) => nodeInfo[x] !== oldInfo[x]);
     if (shouldUpdateRouter) {
       handleRouting(nodeInfo).catch((err) => {

package/lib/index.js

@@ -50,6 +50,7 @@ const { toStatus, fromStatus, ensureDataDirs, getQueueConcurrencyByMem } = requi
  *   version
  *   runtimeConfig: {}
  *   autoUpgrade
+ *   registerUrl
  *   webWalletUrl
  */
 function ABTNode(options) {
@@ -196,16 +197,17 @@ function ABTNode(options) {
     // Blocklet manager
     installBlocklet: blockletManager.install.bind(blockletManager),
     installBlockletFromVc: blockletManager.installBlockletFromVc.bind(blockletManager),
+    installComponent: blockletManager.installComponent.bind(blockletManager),
     startBlocklet: blockletManager.start.bind(blockletManager),
     stopBlocklet: blockletManager.stop.bind(blockletManager),
     reloadBlocklet: blockletManager.reload.bind(blockletManager),
     restartBlocklet: blockletManager.restart.bind(blockletManager),
     deleteBlocklet: blockletManager.delete.bind(blockletManager),
+    deleteComponent: blockletManager.deleteComponent.bind(blockletManager),
     cancelDownloadBlocklet: blockletManager.cancelDownload.bind(blockletManager),
     upgradeBlocklet: blockletManager.upgrade.bind(blockletManager),
     configBlocklet: blockletManager.config.bind(blockletManager),
     devBlocklet: blockletManager.dev.bind(blockletManager),
-    getBlockletStatus: blockletManager.getStatus.bind(blockletManager),
     checkChildBlockletsForUpdates: blockletManager.checkChildrenForUpdates.bind(blockletManager),
     updateChildBlocklets: blockletManager.updateChildren.bind(blockletManager),
     getLatestBlockletVersion: blockletManager.getLatestBlockletVersion.bind(blockletManager),

package/lib/migrations/1.6.17-blocklet-children.js

@@ -0,0 +1,48 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-underscore-dangle */
+
+module.exports = async ({ states, printInfo }) => {
+  printInfo('Try to update blocklet to 1.6.17...');
+  const blockletState = states.blocklet;
+
+  const blocklets = await blockletState.getBlocklets();
+  for (const blocklet of blocklets) {
+    if (!blocklet) {
+      continue;
+    }
+
+    if (!blocklet.children || !blocklet.children.length) {
+      continue;
+    }
+
+    const meta = blocklet.meta || {};
+    const { did } = meta;
+    if (!did) {
+      continue;
+    }
+
+    const children = (blocklet.children || []).map((child) => {
+      if (child.mountPoint) {
+        return child;
+      }
+
+      const config = (meta.children || []).find((x) => x.name === child.meta.name);
+
+      if (
+        config &&
+        config.mountPoints &&
+        config.mountPoints[0] &&
+        config.mountPoints[0].root &&
+        config.mountPoints[0].root.prefix
+      ) {
+        child.mountPoint = config.mountPoints[0].root.prefix;
+      }
+      printInfo(`Set mountPoint: ${child.mountPoint} to child ${child.meta.name} in ${blocklet.meta.name}`);
+
+      return child;
+    });
+
+    await blockletState.update(blocklet._id, { $set: { children } });
+  }
+};

package/lib/router/helper.js

@@ -244,6 +244,9 @@ const ensureLatestNodeInfo = async (sites = [], { withDefaultCors = true } = {})
       site.domain = DOMAIN_FOR_IP_SITE_REGEXP;
 
       if (withDefaultCors) {
+        // Allow CORS from "Install on ABT Node"
+        addCorsToSite(site, info.registerUrl);
+
         // Allow CORS from "Web Wallet"
         addCorsToSite(site, info.webWalletUrl);
       }
@@ -314,7 +317,7 @@ const ensureWellknownRule = async (sites) => {
       rules.forEach((rule) => {
         if (
           rule.to.type !== ROUTING_RULE_TYPES.BLOCKLET || // is not blocklet
-          rule.to.did !== rule.to.realDid // is a component endpoint
+          (rule.to.did !== rule.to.realDid && rule.from.pathPrefix !== '/') // is a component endpoint in sub path
         ) {
           return;
         }
@@ -434,7 +437,8 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const https = get(info, 'routing.https', true);
     const dashboardDomain = get(info, 'routing.dashboardDomain', '');
-    const certDownloadAddress = get(info, 'routing.dashboardCertDownloadAddress', '');
+    const certDownloadAddress =
+      process.env.ABT_NODE_DASHBOARD_CERT_DOWN_URL || get(info, 'routing.dashboardCertDownloadAddress', '');
     if (!https || !dashboardDomain || !certDownloadAddress) {
       return;
     }
@@ -508,9 +512,9 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       return { status: 'existed' };
     }
 
-    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    logger.debug('downloading dashboard ip-domain certificate', { url: downloadUrl, dashboardDomain });
     await updateDashboardCertificate({ checkExpire: false });
-    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    logger.debug('downloaded dashboard ip-domain certificate', { url: downloadUrl, dashboardDomain });
     return { status: 'downloaded' };
   };
 
@@ -656,7 +660,9 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const dashboardDomain = get(info, 'routing.dashboardDomain', '');
     const didDomain = `${info.did.toLowerCase()}.${info.didDomain}`;
-    const dashboardAliasDomains = [dashboardDomain, didDomain]
+    let dashboardAliasDomains = [dashboardDomain, didDomain];
+
+    dashboardAliasDomains = dashboardAliasDomains
       .filter((item) => item && !isExistsInAlias(item))
       .map((item) => ({ value: item, isProtected: true }));
 
@@ -742,18 +748,18 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const existSite = await states.site.findOne({ domain: domainGroup });
     if (!existSite) {
-      const ipEchoDnsDomain = getIpDnsDomainForBlocklet(blocklet, webInterface, nodeInfo.did);
-      const appIdEnv = blocklet.environments.find((e) => e.key === 'BLOCKLET_APP_ID');
-      const domainAliases = [{ value: ipEchoDnsDomain, isProtected: true }];
+      const domainAliases = [];
 
       const didDomain = getDidDomainForBlocklet({
-        appId: appIdEnv.value,
+        name: blocklet.meta.name,
+        daemonDid: nodeInfo.did,
         didDomain: nodeInfo.didDomain,
       });
 
-      if (blocklet.mode !== 'development') {
-        domainAliases.push({ value: didDomain, isProtected: true });
-      }
+      const ipEchoDnsDomain = getIpDnsDomainForBlocklet(blocklet, webInterface, nodeInfo.did);
+
+      // let didDomain in front of ipEchoDnsDomain
+      domainAliases.push({ value: didDomain, isProtected: true }, { value: ipEchoDnsDomain, isProtected: true });
 
       await routerManager.addRoutingSite(
         {

package/lib/router/manager.js

@@ -27,6 +27,7 @@ const {
   BLOCKLET_BUNDLE_FOLDER,
   BLOCKLET_DYNAMIC_PATH_PREFIX,
   BLOCKLET_INTERFACE_TYPE_WEB,
+  BlockletGroup,
 } = require('@blocklet/meta/lib/constants');
 
 const {
@@ -37,6 +38,7 @@ const {
   validateUpdateSite,
 } = require('../validators/router');
 const { getProviderFromNodeInfo, findInterfaceByName, isCLI, findInterfacePortByName } = require('../util');
+const { findWebInterface } = require('../util/blocklet');
 const { attachInterfaceUrls, ensureLatestInfo } = require('./helper');
 const Router = require('./index');
 const states = require('../states');
@@ -219,10 +221,10 @@ class RouterManager extends EventEmitter {
       }
     }
 
-    const updateResult = await states.site.update(
-      { _id: id },
-      { $push: { domainAliases: { value: domainAlias, isProtected: false } } }
-    );
+    // let custom domain in front of protected domain
+    const domainAliases = [{ value: domainAlias, isProtected: false }, ...dbSite.domainAliases];
+
+    const updateResult = await states.site.update({ _id: id }, { $set: { domainAliases } });
     logger.debug('add domain alias update result', { id, updateResult, domainAlias });
 
     const newSite = await states.site.findOne({ _id: id });
@@ -679,49 +681,51 @@ class RouterManager extends EventEmitter {
 
     // get child rules
     const blocklet = await states.blocklet.getBlocklet(rule.to.did);
-    for (const childMeta of blocklet.meta.children || []) {
-      for (const mountPoint of childMeta.mountPoints) {
-        const child = blocklet.children.find((b) => b.meta.name === childMeta.name);
-
-        if (!child) {
-          logger.error(`Child blocklet ${childMeta.name} does not exist`);
-          // eslint-disable-next-line no-continue
-          continue;
-        }
+    for (const child of blocklet.children || []) {
+      const { mountPoint } = child;
+      if (!mountPoint) {
+        logger.error(`mountPoint of child ${child.meta.name} does not exist`);
+        // eslint-disable-next-line no-continue
+        continue;
+      }
 
-        if (mountPoint.root.interfaceName === rule.to.interfaceName) {
-          const pathPrefix = path.join(rule.from.pathPrefix, mountPoint.root.prefix);
-          const isRootPath = pathPrefix === rule.from.pathPrefix;
-          if (isRootPath) {
-            occupied = true;
-          }
+      const childWebInterface = findWebInterface(child);
+      if (!childWebInterface) {
+        logger.error(`web interface of child ${child.meta.name} does not exist`);
+        // eslint-disable-next-line no-continue
+        continue;
+      }
 
-          // if is root path, child rule become root rule
-          const childRule = {
-            id: isRootPath ? rule.id : uuid.v4(),
-            groupId: rule.id,
-            from: {
-              pathPrefix: normalizePathPrefix(pathPrefix),
-              groupPathPrefix: rule.from.pathPrefix,
-            },
-            to: {
-              type: ROUTING_RULE_TYPES.BLOCKLET,
-              port: findInterfacePortByName(child, mountPoint.child.interfaceName),
-              did: rule.to.did, // root blocklet did
-              interfaceName: rule.to.interfaceName, // root blocklet interface
-              realDid: child.meta.did, // child blocklet did
-              realInterfaceName: mountPoint.child.interfaceName,
-            },
-            isProtected: isRootPath ? rule.isProtected : true,
-          };
-
-          rules.push(childRule);
-        }
+      const pathPrefix = path.join(rule.from.pathPrefix, mountPoint);
+      const isRootPath = pathPrefix === rule.from.pathPrefix;
+      if (isRootPath) {
+        occupied = true;
       }
+
+      // if is root path, child rule become root rule
+      const childRule = {
+        id: isRootPath ? rule.id : uuid.v4(),
+        groupId: rule.id,
+        from: {
+          pathPrefix: normalizePathPrefix(pathPrefix),
+          groupPathPrefix: rule.from.pathPrefix,
+        },
+        to: {
+          type: ROUTING_RULE_TYPES.BLOCKLET,
+          port: findInterfacePortByName(child, childWebInterface.name),
+          did: rule.to.did, // root blocklet did
+          interfaceName: rule.to.interfaceName, // root blocklet interface
+          realDid: child.meta.did, // child blocklet did
+          realInterfaceName: childWebInterface.name,
+        },
+        isProtected: isRootPath ? rule.isProtected : true,
+      };
+
+      rules.push(childRule);
     }
 
     // get root rule
-    if (!occupied) {
+    if (!occupied && blocklet.meta.group !== BlockletGroup.gateway) {
       rules.push(rule);
     }
 

package/lib/states/blocklet-extras.js

@@ -3,6 +3,7 @@
 /* eslint-disable consistent-return */
 const logger = require('@abtnode/logger')('state-blocklet-extras');
 const camelCase = require('lodash/camelCase');
+const get = require('lodash/get');
 
 const BaseState = require('./base');
 
@@ -36,16 +37,28 @@ class BlockletExtrasState extends BaseState {
         },
       },
     ];
+
+    this.childExtras = this.extras.filter((x) => ['configs'].includes(x.name));
+
     this.generateExtraFns();
   }
 
+  delete(did) {
+    return this.remove({ did });
+  }
+
   generateExtraFns() {
-    const methods = ['get', 'set', 'del'];
+    const methods = ['get', 'set', 'del', 'list'];
     methods.forEach((method) => {
       this.extras.forEach((extra) => {
         const fn = camelCase(`${method} ${extra.name}`); // getConfigs, getRules
         this[fn] = this.generateExtraFn(method, extra);
+      });
+    });
 
+    const childMethods = ['get', 'set', 'del'];
+    childMethods.forEach((method) => {
+      this.childExtras.forEach((extra) => {
         const childFn = camelCase(`${method} child ${extra.name}`); // getChildConfigs, getChildRules
         this[childFn] = this.generateExtraChildFn(method, extra);
       });
@@ -66,14 +79,22 @@ class BlockletExtrasState extends BaseState {
     if (method === 'del') {
       return this.generateDelFn(extra);
     }
+
+    if (method === 'list') {
+      return this.generateListFn(extra);
+    }
   }
 
   generateGetFn(extra) {
-    return async (did) => {
+    return async (did, path, defaultValue) => {
       const { dek } = this.options;
       const { name, afterGet = noop('data') } = extra;
       const item = await this.asyncDB.findOne({ did });
-      return afterGet({ data: item ? item[name] : item, did, dek });
+      const data = afterGet({ data: item ? item[name] : item, did, dek });
+      if (!path) {
+        return data;
+      }
+      return get(data, path, defaultValue);
     };
   }
 
@@ -118,6 +139,21 @@ class BlockletExtrasState extends BaseState {
     };
   }
 
+  generateListFn(extra) {
+    return async () => {
+      const { dek } = this.options;
+      const { name, afterGet = noop('data') } = extra;
+      const docs = await this.asyncDB.find({});
+      const list = docs
+        .filter((x) => x[name])
+        .map((x) => ({
+          did: x.did,
+          [name]: afterGet({ data: x[name], did: x.did, dek }),
+        }));
+      return list;
+    };
+  }
+
   // generate extra child functions
 
   generateExtraChildFn(method, extra) {

package/lib/states/blocklet.js

@@ -4,6 +4,7 @@
 /* eslint-disable no-underscore-dangle */
 const omit = require('lodash/omit');
 const uniq = require('lodash/uniq');
+const cloneDeep = require('lodash/cloneDeep');
 const detectPort = require('detect-port');
 const Lock = require('@abtnode/util/lib/lock');
 const security = require('@abtnode/util/lib/security');
@@ -19,7 +20,7 @@ const {
 const logger = require('@abtnode/logger')('state-blocklet');
 
 const BaseState = require('./base');
-const { forEachBlocklet } = require('../util/blocklet');
+const { forEachBlocklet, checkDuplicateComponents } = require('../util/blocklet');
 const { validateBlockletMeta } = require('../util');
 
 const lock = new Lock('blocklet-port-assign-lock');
@@ -119,7 +120,7 @@ class BlockletState extends BaseState {
             }
 
             this.emit('remove', doc);
-            return resolve(doc);
+            return resolve(formatBlocklet(doc, 'onRead', this.options.dek));
           });
         })
     );
@@ -130,9 +131,9 @@ class BlockletState extends BaseState {
     meta,
     source = BlockletSource.registry,
     status = BlockletStatus.added,
-    deployedFrom,
+    deployedFrom = '',
     mode = BLOCKLET_MODES.PRODUCTION,
-    childrenMeta = [],
+    children: rawChildren = [],
   } = {}) {
     return this.getBlocklet(did).then(
       (doc) =>
@@ -152,7 +153,7 @@ class BlockletState extends BaseState {
 
             const ports = await this.getBlockletPorts({ interfaces: sanitized.interfaces || [] });
 
-            const children = await this.fillChildrenPorts(this.getChildrenFromMetas(childrenMeta), {
+            const children = await this.fillChildrenPorts(rawChildren, {
               defaultPort: getMaxPort(ports),
             });
 
@@ -196,7 +197,7 @@ class BlockletState extends BaseState {
           }
 
           try {
-            const formatted = formatBlocklet(updates, 'onUpdate', this.options.dek);
+            const formatted = formatBlocklet(cloneDeep(updates), 'onUpdate', this.options.dek);
             const newDoc = await this.updateById(doc._id, { $set: formatted });
             resolve(newDoc);
           } catch (err) {
@@ -206,7 +207,7 @@ class BlockletState extends BaseState {
     );
   }
 
-  upgradeBlocklet({ meta, source, deployedFrom, children } = {}) {
+  upgradeBlocklet({ meta, source, deployedFrom = '', children } = {}) {
     return this.getBlocklet(meta.did).then(
       (doc) =>
         // eslint-disable-next-line no-async-promise-executor
@@ -247,8 +248,10 @@ class BlockletState extends BaseState {
               },
             });
             lock.release();
-            this.emit('upgrade', newDoc);
-            resolve(newDoc);
+
+            const formatted = formatBlocklet(newDoc, 'onRead', this.options.dek);
+            this.emit('upgrade', formatted);
+            resolve(formatted);
           } catch (err) {
             lock.release();
             reject(err);
@@ -388,15 +391,23 @@ class BlockletState extends BaseState {
     return result;
   }
 
-  async setBlockletStatus(did, status) {
+  /**
+   * @param {String} did blocklet did
+   * @param {BlockletStatus} status blocklet status
+   *
+   * children status only different with parent before blocklet installation
+   * @param {Array<{did}>} children
+   */
+  async setBlockletStatus(did, status, { children } = {}) {
     if (typeof status === 'undefined') {
       throw new Error('Unsupported blocklet status');
     }
-    const record = await this.getBlocklet(did);
-    // when status change update record
-    if (record.status === status) {
-      return record;
+
+    const doc = await this.getBlocklet(did);
+    if (doc.status === status && !children) {
+      return formatBlocklet(doc, 'onRead', this.options.dek);
     }
+
     const updates = { status, startedAt: undefined, stoppedAt: undefined };
     if (status === BlockletStatus.running) {
       updates.startedAt = new Date();
@@ -408,14 +419,30 @@ class BlockletState extends BaseState {
       updates.stoppedAt = new Date();
     }
 
-    const res = await this.update(record._id, {
-      $set: updates,
+    // update children status
+    updates.children = doc.children.map((child) => {
+      if (children === 'all') {
+        child.status = status;
+        return child;
+      }
+
+      if (!children) {
+        if (![BlockletStatus.waiting, BlockletStatus.upgrading, BlockletStatus.installing].includes(status)) {
+          child.status = status;
+        }
+
+        return child;
+      }
+
+      const inputChild = children.find((x) => x.did === child.meta.did);
+      if (inputChild) {
+        child.status = status;
+      }
+      return child;
     });
-    return res;
-  }
 
-  getChildrenFromMetas(childrenMeta) {
-    return childrenMeta.map((x) => ({ meta: x }));
+    await this.update(doc._id, { $set: updates });
+    return formatBlocklet({ ...doc, ...updates }, 'onRead', this.options.dek);
   }
 
   async fillChildrenPorts(children, { defaultPort = 0, oldChildren } = {}) {
@@ -465,6 +492,48 @@ class BlockletState extends BaseState {
 
     return children;
   }
+
+  async addChildren(did, children) {
+    const parent = await this.getBlocklet(did);
+    if (!parent) {
+      throw new Error('Blocklet does not exist');
+    }
+
+    const oldChildren = parent.children || [];
+
+    const newChildren = [...oldChildren];
+    for (const child of children) {
+      const { meta, mountPoint, sourceUrl = '', source = '', deployedFrom = '' } = child;
+
+      if (!mountPoint) {
+        throw new Error(`mountPoint is required when adding component ${meta.title || meta.name}`);
+      }
+
+      if (meta.did === parent.meta.did) {
+        throw new Error('Cannot add self as a component');
+      }
+
+      checkDuplicateComponents([child], newChildren);
+
+      newChildren.push({
+        mountPoint,
+        meta,
+        sourceUrl,
+        source,
+        deployedFrom,
+        dynamic: true,
+        status: BlockletStatus.added,
+      });
+    }
+
+    // use upgradeBlocklet to assign ports to children and write new data to db
+    return this.upgradeBlocklet({
+      meta: parent.meta,
+      source: parent.source,
+      deployedFrom: parent.deployedFrom,
+      children: newChildren,
+    });
+  }
 }
 
 BlockletState.BlockletStatus = BlockletStatus;