@abtnode/core 1.6.14 → 1.6.18

package/lib/blocklet/migration.js

@@ -1,67 +1,25 @@
 /* eslint-disable no-await-in-loop */
-const childProcess = require('child_process');
 const fs = require('fs-extra');
 const path = require('path');
 const semver = require('semver');
+const runScript = require('@abtnode/util/lib/run-script');
 
 const { getMigrationScripts: getScripts } = require('../migrations');
 const { getSafeEnv } = require('../util');
 const { name } = require('../../package.json');
 const logger = require('@abtnode/logger')(`${name}:blocklet:migration`); // eslint-disable-line
 
-const _runScript = ({ appDir, env, migrationScript, progress = false }) => {
-  const safeEnv = getSafeEnv(env);
-
-  const child = childProcess.exec(`node ${migrationScript}`, {
-    cwd: appDir,
-    env: safeEnv,
-    stdio: 'inherit',
-  });
-  let hasUnhandledRejection = false;
-
-  if (progress) {
-    child.stdout.pipe(process.stdout);
-    child.stderr.pipe(process.stderr);
-  }
-
-  return new Promise((resolve, reject) => {
-    const errorMessages = [];
-
-    child.stderr.on('data', (err) => {
-      // Check if has unhandledRejection in childProcess
-      // https://stackoverflow.com/questions/32784649/gracefully-handle-errors-in-child-processes-in-nodejs
-      if (err.includes('UnhandledPromiseRejectionWarning')) {
-        hasUnhandledRejection = true;
-      }
-      errorMessages.push(err);
-    });
-
-    child.on('exit', (code) => {
-      if (errorMessages.length > 0) {
-        if (code !== 0 || hasUnhandledRejection) {
-          return reject(new Error(errorMessages.join('\r\n')));
-        }
-
-        if (!progress) {
-          errorMessages.forEach((message) => process.stderr.write(message));
-        }
-      }
-
-      return resolve();
-    });
-  });
-};
-
 async function runScripts({
+  blocklet,
+  appDir,
+  env,
+  oldVersion,
   dbDir,
   backupDir,
   scriptsDir,
   printInfo,
   printSuccess,
   printError,
-  appDir,
-  env,
-  oldVersion,
 }) {
   let scripts = [];
   try {
@@ -90,7 +48,11 @@ async function runScripts({
     const { script: scriptPath } = pendingScripts[i];
     try {
       printInfo(`Migration script started: ${scriptPath}`);
-      await _runScript({ appDir, env, migrationScript: path.join(scriptsDir, scriptPath) });
+      await runScript(`node ${path.join(scriptsDir, scriptPath)}`, [blocklet.env.appId, 'migration'].join(':'), {
+        cwd: appDir,
+        env: getSafeEnv(env),
+        silent: false,
+      });
       printInfo(`Migration script executed: ${scriptPath}`);
     } catch (migrationErr) {
       printError(`Failed to execute migration script: ${scriptPath}, error: ${migrationErr.message}`);
@@ -123,6 +85,7 @@ async function doRestore({ dbDir, backupDir, printInfo, printSuccess }) {
 }
 
 module.exports = async ({
+  blocklet,
   appDir,
   env,
   oldVersion,
@@ -142,15 +105,16 @@ module.exports = async ({
   fs.ensureDirSync(backupDir);
 
   await runScripts({
+    blocklet,
+    appDir,
+    env,
+    oldVersion,
+    newVersion,
     dbDir,
     backupDir,
     scriptsDir,
     printError,
     printInfo,
     printSuccess,
-    appDir,
-    env,
-    oldVersion,
-    newVersion,
   });
 };

package/lib/event.js

@@ -182,7 +182,7 @@ module.exports = ({
       .catch((error) => logger.error('refresh blocklets failed on initialize the registry', { error }));
 
     // We need update router on some fields change
-    const fields = ['enableWelcomePage', 'webWalletUrl'];
+    const fields = ['enableWelcomePage', 'webWalletUrl', 'registerUrl'];
     const shouldUpdateRouter = fields.some((x) => nodeInfo[x] !== oldInfo[x]);
     if (shouldUpdateRouter) {
       handleRouting(nodeInfo).catch((err) => {

package/lib/index.js

@@ -50,6 +50,7 @@ const { toStatus, fromStatus, ensureDataDirs, getQueueConcurrencyByMem } = requi
  *   version
  *   runtimeConfig: {}
  *   autoUpgrade
+ *   registerUrl
  *   webWalletUrl
  */
 function ABTNode(options) {
@@ -196,16 +197,17 @@ function ABTNode(options) {
     // Blocklet manager
     installBlocklet: blockletManager.install.bind(blockletManager),
     installBlockletFromVc: blockletManager.installBlockletFromVc.bind(blockletManager),
+    installComponent: blockletManager.installComponent.bind(blockletManager),
     startBlocklet: blockletManager.start.bind(blockletManager),
     stopBlocklet: blockletManager.stop.bind(blockletManager),
     reloadBlocklet: blockletManager.reload.bind(blockletManager),
     restartBlocklet: blockletManager.restart.bind(blockletManager),
     deleteBlocklet: blockletManager.delete.bind(blockletManager),
+    deleteComponent: blockletManager.deleteComponent.bind(blockletManager),
     cancelDownloadBlocklet: blockletManager.cancelDownload.bind(blockletManager),
     upgradeBlocklet: blockletManager.upgrade.bind(blockletManager),
     configBlocklet: blockletManager.config.bind(blockletManager),
     devBlocklet: blockletManager.dev.bind(blockletManager),
-    getBlockletStatus: blockletManager.getStatus.bind(blockletManager),
     checkChildBlockletsForUpdates: blockletManager.checkChildrenForUpdates.bind(blockletManager),
     updateChildBlocklets: blockletManager.updateChildren.bind(blockletManager),
     getLatestBlockletVersion: blockletManager.getLatestBlockletVersion.bind(blockletManager),
@@ -400,16 +402,6 @@ function ABTNode(options) {
     } else {
       // We should only respond to pm2 events when node is alive
       events.on('node.started', async () => {
-        const info = await states.node.read();
-        certManager
-          .issue({ domain: `${info.did.toLowerCase()}.${info.didDomain}` })
-          .then(() => {
-            logger.info('add issue daemon certificate job');
-          })
-          .catch((error) => {
-            logger.error('issue daemon certificate job failed', { error });
-          });
-
         pm2Events.resume();
         initCron();
       });

package/lib/migrations/1.6.17-blocklet-children.js

@@ -0,0 +1,48 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-underscore-dangle */
+
+module.exports = async ({ states, printInfo }) => {
+  printInfo('Try to update blocklet to 1.6.17...');
+  const blockletState = states.blocklet;
+
+  const blocklets = await blockletState.getBlocklets();
+  for (const blocklet of blocklets) {
+    if (!blocklet) {
+      continue;
+    }
+
+    if (!blocklet.children || !blocklet.children.length) {
+      continue;
+    }
+
+    const meta = blocklet.meta || {};
+    const { did } = meta;
+    if (!did) {
+      continue;
+    }
+
+    const children = (blocklet.children || []).map((child) => {
+      if (child.mountPoint) {
+        return child;
+      }
+
+      const config = (meta.children || []).find((x) => x.name === child.meta.name);
+
+      if (
+        config &&
+        config.mountPoints &&
+        config.mountPoints[0] &&
+        config.mountPoints[0].root &&
+        config.mountPoints[0].root.prefix
+      ) {
+        child.mountPoint = config.mountPoints[0].root.prefix;
+      }
+      printInfo(`Set mountPoint: ${child.mountPoint} to child ${child.meta.name} in ${blocklet.meta.name}`);
+
+      return child;
+    });
+
+    await blockletState.update(blocklet._id, { $set: { children } });
+  }
+};

package/lib/router/helper.js

@@ -55,6 +55,7 @@ const { getFromCache: getAccessibleExternalNodeIp } = require('../util/get-acces
 
 const Router = require('./index');
 const states = require('../states');
+const { getBlockletDomainGroupName, getDidFromDomainGroupName } = require('../util/router');
 
 /**
  * replace 888-888-888-888 with accessible ip for domain
@@ -243,6 +244,9 @@ const ensureLatestNodeInfo = async (sites = [], { withDefaultCors = true } = {})
       site.domain = DOMAIN_FOR_IP_SITE_REGEXP;
 
       if (withDefaultCors) {
+        // Allow CORS from "Install on ABT Node"
+        addCorsToSite(site, info.registerUrl);
+
         // Allow CORS from "Web Wallet"
         addCorsToSite(site, info.webWalletUrl);
       }
@@ -313,7 +317,7 @@ const ensureWellknownRule = async (sites) => {
       rules.forEach((rule) => {
         if (
           rule.to.type !== ROUTING_RULE_TYPES.BLOCKLET || // is not blocklet
-          rule.to.did !== rule.to.realDid // is a component endpoint
+          (rule.to.did !== rule.to.realDid && rule.from.pathPrefix !== '/') // is a component endpoint in sub path
         ) {
           return;
         }
@@ -353,6 +357,18 @@ const ensureCorsForWebWallet = async (sites) => {
   return sites;
 };
 
+const filterSitesForRemovedBlocklets = async (sites = []) => {
+  const blocklets = await states.blocklet.getBlocklets();
+  return sites.filter((site) => {
+    if (!site.domain.endsWith(BLOCKLET_SITE_GROUP_SUFFIX)) {
+      return true;
+    }
+
+    const did = getDidFromDomainGroupName(site.domain);
+    return blocklets.some((x) => x.meta.did === did);
+  });
+};
+
 const ensureLatestInfo = async (sites = [], { withDefaultCors = true } = {}) => {
   let result = await ensureLatestNodeInfo(sites, { withDefaultCors });
   result = await ensureWellknownRule(result);
@@ -421,7 +437,8 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const https = get(info, 'routing.https', true);
     const dashboardDomain = get(info, 'routing.dashboardDomain', '');
-    const certDownloadAddress = get(info, 'routing.dashboardCertDownloadAddress', '');
+    const certDownloadAddress =
+      process.env.ABT_NODE_DASHBOARD_CERT_DOWN_URL || get(info, 'routing.dashboardCertDownloadAddress', '');
     if (!https || !dashboardDomain || !certDownloadAddress) {
       return;
     }
@@ -495,9 +512,9 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       return { status: 'existed' };
     }
 
-    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    logger.debug('downloading dashboard ip-domain certificate', { url: downloadUrl, dashboardDomain });
     await updateDashboardCertificate({ checkExpire: false });
-    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    logger.debug('downloaded dashboard ip-domain certificate', { url: downloadUrl, dashboardDomain });
     return { status: 'downloaded' };
   };
 
@@ -643,7 +660,9 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const dashboardDomain = get(info, 'routing.dashboardDomain', '');
     const didDomain = `${info.did.toLowerCase()}.${info.didDomain}`;
-    const dashboardAliasDomains = [dashboardDomain, didDomain]
+    let dashboardAliasDomains = [dashboardDomain, didDomain];
+
+    dashboardAliasDomains = dashboardAliasDomains
       .filter((item) => item && !isExistsInAlias(item))
       .map((item) => ({ value: item, isProtected: true }));
 
@@ -713,7 +732,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       return `/${str}`.replace(/^\/+/, '/');
     };
 
-    const domainGroup = `${blocklet.meta.did}${BLOCKLET_SITE_GROUP_SUFFIX}`;
+    const domainGroup = getBlockletDomainGroupName(blocklet.meta.did);
 
     const pathPrefix = getPrefix(webInterface.prefix);
     const rule = {
@@ -730,17 +749,15 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     const existSite = await states.site.findOne({ domain: domainGroup });
     if (!existSite) {
       const ipEchoDnsDomain = getIpDnsDomainForBlocklet(blocklet, webInterface, nodeInfo.did);
-      const appIdEnv = blocklet.environments.find((e) => e.key === 'BLOCKLET_APP_ID');
       const domainAliases = [{ value: ipEchoDnsDomain, isProtected: true }];
 
       const didDomain = getDidDomainForBlocklet({
-        appId: appIdEnv.value,
+        name: blocklet.meta.name,
+        daemonDid: nodeInfo.did,
         didDomain: nodeInfo.didDomain,
       });
 
-      if (blocklet.mode !== 'development') {
-        domainAliases.push({ value: didDomain, isProtected: true });
-      }
+      domainAliases.push({ value: didDomain, isProtected: true });
 
       await routerManager.addRoutingSite(
         {
@@ -755,13 +772,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
         context
       );
       logger.info('add routing site', { site: domainGroup });
-      if (
-        process.env.NODE_ENV !== 'development' &&
-        process.env.NODE_ENV !== 'test' &&
-        blocklet.mode !== 'development'
-      ) {
-        await certManager.issue({ domain: didDomain });
-      }
 
       return true;
     }
@@ -1080,7 +1090,10 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     const nodeInfo = await nodeState.read();
 
     const ruleChanged = await routerManager.deleteRoutingRulesItemByDid({ did: blocklet.meta.did }, context);
-    const siteChanged = await _removeBlockletSites(blocklet, nodeInfo, context);
+    let siteChanged;
+    if (!context.keepRouting) {
+      siteChanged = await _removeBlockletSites(blocklet, nodeInfo, context);
+    }
 
     return ruleChanged || siteChanged;
   };
@@ -1140,6 +1153,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
           try {
             const info = await nodeState.read();
             let { sites } = await readRoutingSites();
+            sites = await filterSitesForRemovedBlocklets(sites);
             sites = await ensureLatestInfo(sites);
             sites = await ensureAuthService(sites, blockletManager);
             sites = await ensureServiceRule(sites);

package/lib/router/manager.js

@@ -27,6 +27,7 @@ const {
   BLOCKLET_BUNDLE_FOLDER,
   BLOCKLET_DYNAMIC_PATH_PREFIX,
   BLOCKLET_INTERFACE_TYPE_WEB,
+  BlockletGroup,
 } = require('@blocklet/meta/lib/constants');
 
 const {
@@ -37,6 +38,7 @@ const {
   validateUpdateSite,
 } = require('../validators/router');
 const { getProviderFromNodeInfo, findInterfaceByName, isCLI, findInterfacePortByName } = require('../util');
+const { findWebInterface } = require('../util/blocklet');
 const { attachInterfaceUrls, ensureLatestInfo } = require('./helper');
 const Router = require('./index');
 const states = require('../states');
@@ -186,19 +188,37 @@ class RouterManager extends EventEmitter {
     return dbSite;
   }
 
-  async addDomainAlias({ id, domainAlias }, context = {}) {
+  async addDomainAlias({ id, domainAlias, force }, context = {}) {
     await validateAddDomainAlias(domainAlias, context);
     const dbSite = await states.site.findOne({ _id: id });
     if (!dbSite) {
       throw new Error(`site ${id} does not exist`);
     }
 
-    const siteCount = await states.site.count({
-      $or: [{ domain: domainAlias }, { domainAliases: domainAlias }, { 'domainAliases.value': domainAlias }],
+    // check domain exists in site domain
+    const mainDomainSiteCount = await states.site.count({
+      domain: domainAlias,
     });
 
-    if (siteCount > 0) {
-      throw new Error(`domain alias ${domainAlias} already exists`);
+    if (mainDomainSiteCount > 0) {
+      if (!force) {
+        throw new Error(`${domainAlias} already exists`);
+      } else {
+        throw new Error(`${domainAlias} cannot be forced-added`);
+      }
+    }
+
+    // check domain exists in site alias domain
+    const aliasDomainSite = await states.site.findOne({
+      $or: [{ domainAliases: domainAlias }, { 'domainAliases.value': domainAlias }],
+    });
+
+    if (aliasDomainSite) {
+      if (!force) {
+        throw new Error(`${domainAlias} already exists`);
+      } else {
+        await this.deleteDomainAlias({ id: aliasDomainSite.id, domainAlias });
+      }
     }
 
     const updateResult = await states.site.update(
@@ -661,49 +681,51 @@ class RouterManager extends EventEmitter {
 
     // get child rules
     const blocklet = await states.blocklet.getBlocklet(rule.to.did);
-    for (const childMeta of blocklet.meta.children || []) {
-      for (const mountPoint of childMeta.mountPoints) {
-        const child = blocklet.children.find((b) => b.meta.name === childMeta.name);
-
-        if (!child) {
-          logger.error(`Child blocklet ${childMeta.name} does not exist`);
-          // eslint-disable-next-line no-continue
-          continue;
-        }
+    for (const child of blocklet.children || []) {
+      const { mountPoint } = child;
+      if (!mountPoint) {
+        logger.error(`mountPoint of child ${child.meta.name} does not exist`);
+        // eslint-disable-next-line no-continue
+        continue;
+      }
 
-        if (mountPoint.root.interfaceName === rule.to.interfaceName) {
-          const pathPrefix = path.join(rule.from.pathPrefix, mountPoint.root.prefix);
-          const isRootPath = pathPrefix === rule.from.pathPrefix;
-          if (isRootPath) {
-            occupied = true;
-          }
+      const childWebInterface = findWebInterface(child);
+      if (!childWebInterface) {
+        logger.error(`web interface of child ${child.meta.name} does not exist`);
+        // eslint-disable-next-line no-continue
+        continue;
+      }
 
-          // if is root path, child rule become root rule
-          const childRule = {
-            id: isRootPath ? rule.id : uuid.v4(),
-            groupId: rule.id,
-            from: {
-              pathPrefix: normalizePathPrefix(pathPrefix),
-              groupPathPrefix: rule.from.pathPrefix,
-            },
-            to: {
-              type: ROUTING_RULE_TYPES.BLOCKLET,
-              port: findInterfacePortByName(child, mountPoint.child.interfaceName),
-              did: rule.to.did, // root blocklet did
-              interfaceName: rule.to.interfaceName, // root blocklet interface
-              realDid: child.meta.did, // child blocklet did
-              realInterfaceName: mountPoint.child.interfaceName,
-            },
-            isProtected: isRootPath ? rule.isProtected : true,
-          };
-
-          rules.push(childRule);
-        }
+      const pathPrefix = path.join(rule.from.pathPrefix, mountPoint);
+      const isRootPath = pathPrefix === rule.from.pathPrefix;
+      if (isRootPath) {
+        occupied = true;
       }
+
+      // if is root path, child rule become root rule
+      const childRule = {
+        id: isRootPath ? rule.id : uuid.v4(),
+        groupId: rule.id,
+        from: {
+          pathPrefix: normalizePathPrefix(pathPrefix),
+          groupPathPrefix: rule.from.pathPrefix,
+        },
+        to: {
+          type: ROUTING_RULE_TYPES.BLOCKLET,
+          port: findInterfacePortByName(child, childWebInterface.name),
+          did: rule.to.did, // root blocklet did
+          interfaceName: rule.to.interfaceName, // root blocklet interface
+          realDid: child.meta.did, // child blocklet did
+          realInterfaceName: childWebInterface.name,
+        },
+        isProtected: isRootPath ? rule.isProtected : true,
+      };
+
+      rules.push(childRule);
     }
 
     // get root rule
-    if (!occupied) {
+    if (!occupied && blocklet.meta.group !== BlockletGroup.gateway) {
       rules.push(rule);
     }
 

package/lib/states/blocklet-extras.js

@@ -3,6 +3,7 @@
 /* eslint-disable consistent-return */
 const logger = require('@abtnode/logger')('state-blocklet-extras');
 const camelCase = require('lodash/camelCase');
+const get = require('lodash/get');
 
 const BaseState = require('./base');
 
@@ -36,16 +37,28 @@ class BlockletExtrasState extends BaseState {
         },
       },
     ];
+
+    this.childExtras = this.extras.filter((x) => ['configs'].includes(x.name));
+
     this.generateExtraFns();
   }
 
+  delete(did) {
+    return this.remove({ did });
+  }
+
   generateExtraFns() {
-    const methods = ['get', 'set', 'del'];
+    const methods = ['get', 'set', 'del', 'list'];
     methods.forEach((method) => {
       this.extras.forEach((extra) => {
         const fn = camelCase(`${method} ${extra.name}`); // getConfigs, getRules
         this[fn] = this.generateExtraFn(method, extra);
+      });
+    });
 
+    const childMethods = ['get', 'set', 'del'];
+    childMethods.forEach((method) => {
+      this.childExtras.forEach((extra) => {
         const childFn = camelCase(`${method} child ${extra.name}`); // getChildConfigs, getChildRules
         this[childFn] = this.generateExtraChildFn(method, extra);
       });
@@ -66,14 +79,22 @@ class BlockletExtrasState extends BaseState {
     if (method === 'del') {
       return this.generateDelFn(extra);
     }
+
+    if (method === 'list') {
+      return this.generateListFn(extra);
+    }
   }
 
   generateGetFn(extra) {
-    return async (did) => {
+    return async (did, path, defaultValue) => {
       const { dek } = this.options;
       const { name, afterGet = noop('data') } = extra;
       const item = await this.asyncDB.findOne({ did });
-      return afterGet({ data: item ? item[name] : item, did, dek });
+      const data = afterGet({ data: item ? item[name] : item, did, dek });
+      if (!path) {
+        return data;
+      }
+      return get(data, path, defaultValue);
     };
   }
 
@@ -118,6 +139,21 @@ class BlockletExtrasState extends BaseState {
     };
   }
 
+  generateListFn(extra) {
+    return async () => {
+      const { dek } = this.options;
+      const { name, afterGet = noop('data') } = extra;
+      const docs = await this.asyncDB.find({});
+      const list = docs
+        .filter((x) => x[name])
+        .map((x) => ({
+          did: x.did,
+          [name]: afterGet({ data: x[name], did: x.did, dek }),
+        }));
+      return list;
+    };
+  }
+
   // generate extra child functions
 
   generateExtraChildFn(method, extra) {