@abtnode/core 1.15.17 → 1.16.0-beta-b16cb035

package/lib/router/manager.js

@@ -8,10 +8,11 @@ const path = require('path');
 const os = require('os');
 const fse = require('fs-extra');
 const get = require('lodash/get');
-const { Certificate, PrivateKey } = require('@fidm/x509');
+const toLower = require('lodash/toLower');
 const { EventEmitter } = require('events');
 const uuid = require('uuid');
 const isUrl = require('is-url');
+const joinUrl = require('url-join');
 const cloneDeep = require('lodash/cloneDeep');
 const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
 const logger = require('@abtnode/logger')('@abtnode/core:router:manager');
@@ -27,7 +28,9 @@ const {
   BLOCKLET_BUNDLE_FOLDER,
   BLOCKLET_DYNAMIC_PATH_PREFIX,
   BLOCKLET_INTERFACE_TYPE_WEB,
-} = require('@blocklet/meta/lib/constants');
+  BlockletGroup,
+} = require('@blocklet/constant');
+const { forEachChildSync } = require('@blocklet/meta/lib/util');
 
 const {
   validateAddSite,
@@ -37,7 +40,8 @@ const {
   validateUpdateSite,
 } = require('../validators/router');
 const { getProviderFromNodeInfo, findInterfaceByName, isCLI, findInterfacePortByName } = require('../util');
-const { attachInterfaceUrls, ensureLatestInfo } = require('./helper');
+const { findWebInterface } = require('../util/blocklet');
+const { attachRuntimeDomainAliases, ensureLatestInfo } = require('./helper');
 const Router = require('./index');
 const states = require('../states');
 
@@ -70,8 +74,9 @@ const normalizeRedirectUrl = (url) => {
 };
 
 class RouterManager extends EventEmitter {
-  constructor() {
+  constructor({ certManager }) {
     super();
+    this.certManager = certManager;
 
     // HACK: do not emit any events from CLI
     if (isCLI()) {
@@ -109,7 +114,7 @@ class RouterManager extends EventEmitter {
       for (const rule of newSite.rules) {
         this.fixRootBlockletRule(rule);
         checkPathPrefixInBlackList(rule.from.pathPrefix, dynamicPathBlackList);
-        rules.push(...(await this.getRules(rule)));
+        rules.push(...(await this.getRulesForMutation(rule)));
       }
     }
     newSite.rules = rules;
@@ -121,7 +126,7 @@ class RouterManager extends EventEmitter {
     await this.validateRouterConfig('addRoutingSite', { site: newSite });
 
     const result = await states.site.add(newSite);
-    await attachInterfaceUrls({ sites: result, context, node: states.node });
+    await attachRuntimeDomainAliases({ sites: result, context, node: states.node });
 
     this.emit('router.site.created', result);
     return result;
@@ -150,20 +155,20 @@ class RouterManager extends EventEmitter {
 
   // eslint-disable-next-line no-unused-vars
   async updateRoutingSite(params, context = {}) {
-    await validateUpdateSite(params, context);
-    const site = await states.site.findOne({ _id: params.id });
-    if (!site) {
+    const site = await validateUpdateSite(params, context);
+    const existed = await states.site.findOne({ _id: site.id });
+    if (!existed) {
       throw new Error('Can not update non-existing site');
     }
 
     const updateSet = {};
 
-    if (params.corsAllowedOrigins) {
-      updateSet.corsAllowedOrigins = params.corsAllowedOrigins;
+    if (site.corsAllowedOrigins) {
+      updateSet.corsAllowedOrigins = site.corsAllowedOrigins.filter((x) => x !== '__none__');
     }
 
-    if (params.domain) {
-      const newDomain = params.domain;
+    if (site.domain) {
+      const newDomain = site.domain;
       const ruleCountInDB = await states.site.count({
         $or: [{ domain: newDomain }, { domainAliases: newDomain }, { 'domainAliases.value': newDomain }],
       });
@@ -175,29 +180,47 @@ class RouterManager extends EventEmitter {
       updateSet.domain = newDomain;
     }
 
-    const updated = await states.site.update({ _id: params.id }, { $set: updateSet }, { multi: false, upsert: false });
+    const updated = await states.site.update({ _id: site.id }, { $set: updateSet }, { multi: false, upsert: false });
 
-    logger.info('router.site.updated', { params, updated });
-    this.emit('router.site.updated', params.id);
+    logger.info('router.site.updated', { site, updated });
+    this.emit('router.site.updated', site.id);
 
-    const dbSite = await states.site.findOne({ _id: params.id });
-    await attachInterfaceUrls({ sites: dbSite, context, node: states.node });
+    const dbSite = await states.site.findOne({ _id: site.id });
+    await attachRuntimeDomainAliases({ sites: dbSite, context, node: states.node });
     return dbSite;
   }
 
-  async addDomainAlias({ id, domainAlias }, context = {}) {
-    await validateAddDomainAlias(domainAlias, context);
+  async addDomainAlias({ id, domainAlias: tmpAlias, force }, context = {}) {
+    const domainAlias = await validateAddDomainAlias(tmpAlias, context);
     const dbSite = await states.site.findOne({ _id: id });
     if (!dbSite) {
       throw new Error(`site ${id} does not exist`);
     }
 
-    const siteCount = await states.site.count({
-      $or: [{ domain: domainAlias }, { domainAliases: domainAlias }, { 'domainAliases.value': domainAlias }],
+    // check domain exists in site domain
+    const mainDomainSiteCount = await states.site.count({
+      domain: domainAlias,
     });
 
-    if (siteCount > 0) {
-      throw new Error(`domain alias ${domainAlias} already exists`);
+    if (mainDomainSiteCount > 0) {
+      if (!force) {
+        throw new Error(`${domainAlias} already exists`);
+      } else {
+        throw new Error(`${domainAlias} cannot be forced-added`);
+      }
+    }
+
+    // check domain exists in site alias domain
+    const aliasDomainSite = await states.site.findOne({
+      $or: [{ domainAliases: domainAlias }, { 'domainAliases.value': domainAlias }],
+    });
+
+    if (aliasDomainSite) {
+      if (!force) {
+        throw new Error(`${domainAlias} already exists`);
+      } else {
+        await this.deleteDomainAlias({ id: aliasDomainSite.id, domainAlias });
+      }
     }
 
     const updateResult = await states.site.update(
@@ -207,13 +230,13 @@ class RouterManager extends EventEmitter {
     logger.debug('add domain alias update result', { id, updateResult, domainAlias });
 
     const newSite = await states.site.findOne({ _id: id });
-    await attachInterfaceUrls({ sites: newSite, context, node: states.node });
+    await attachRuntimeDomainAliases({ sites: newSite, context, node: states.node });
 
     return newSite;
   }
 
-  async deleteDomainAlias({ id, domainAlias }, context = {}) {
-    await validateAddDomainAlias(domainAlias, context);
+  async deleteDomainAlias({ id, domainAlias: tmpAlias }, context = {}) {
+    const domainAlias = await validateAddDomainAlias(tmpAlias, context);
     const dbSite = await states.site.findOne({ _id: id });
     if (!dbSite) {
       throw new Error(`site ${id} does not exist`);
@@ -221,22 +244,22 @@ class RouterManager extends EventEmitter {
 
     dbSite.domainAliases = dbSite.domainAliases.filter((x) => {
       if (typeof x === 'string') {
-        return x !== domainAlias;
+        return toLower(x) !== domainAlias;
       }
 
-      return x.value !== domainAlias;
+      return toLower(x.value) !== domainAlias;
     });
 
     const updateResult = await states.site.update({ _id: id }, { $set: { domainAliases: dbSite.domainAliases } });
     logger.debug('remove domain alias update result', { id, updateResult, domainAlias });
 
-    await attachInterfaceUrls({ sites: dbSite, context, node: states.node });
+    await attachRuntimeDomainAliases({ sites: dbSite, context, node: states.node });
 
     return dbSite;
   }
 
-  async addRoutingRule({ id, rule, skipCheckDynamicBlacklist = false }, context = {}) {
-    await validateAddRule({ id, rule }, context);
+  async addRoutingRule({ id, rule: tempRule, skipCheckDynamicBlacklist = false }, context = {}) {
+    const { rule } = await validateAddRule({ id, rule: tempRule }, context);
     const dbSite = await states.site.findOne({ _id: id });
     if (!dbSite) {
       throw new Error(`site ${id} does not exist`);
@@ -260,19 +283,19 @@ class RouterManager extends EventEmitter {
     await this.validateRouterConfig('addRoutingRule', { id, rule });
 
     // add child blocklet rules
-    for (const x of await this.getRules(rule)) {
+    for (const x of await this.getRulesForMutation(rule)) {
       await states.site.addRuleToSite(id, x);
     }
 
     const newSite = await states.site.findOne({ _id: id });
-    await attachInterfaceUrls({ sites: newSite, context, node: states.node });
+    await attachRuntimeDomainAliases({ sites: newSite, context, node: states.node });
 
     this.emit('router.rule.created', newSite);
     return newSite;
   }
 
-  async updateRoutingRule({ id, rule, skipProtectedRuleChecking = false }, context = {}) {
-    await validateEditRule({ id, rule }, context);
+  async updateRoutingRule({ id, rule: tmpRule, skipProtectedRuleChecking = false }, context = {}) {
+    const { rule } = await validateEditRule({ id, rule: tmpRule }, context);
     const dbSite = await states.site.findOne({ _id: id, 'rules.id': rule.id });
     if (!dbSite) {
       throw new Error(`site ${id}, rule ${rule.id} does not exist`);
@@ -300,13 +323,16 @@ class RouterManager extends EventEmitter {
     await this.validateRouterConfig('updateRoutingRule', { id, rule });
 
     // update rules
-    const newRules = [...dbSite.rules.filter((x) => x.groupId !== rule.id), ...(await this.getRules(rule))];
+    const newRules = [
+      ...dbSite.rules.filter((x) => (x.groupId && x.groupId !== rule.id) || x.id !== rule.id), // 有些路由没有 rule.groupId
+      ...(await this.getRulesForMutation(rule)),
+    ];
 
     const updateResult = await states.site.update({ _id: id }, { $set: { rules: newRules } });
     logger.info('update result', { updateResult });
     const newSite = await states.site.findOne({ _id: id });
 
-    await attachInterfaceUrls({ sites: newSite, context, node: states.node });
+    await attachRuntimeDomainAliases({ sites: newSite, context, node: states.node });
 
     this.emit('router.rule.updated', newSite);
 
@@ -335,64 +361,12 @@ class RouterManager extends EventEmitter {
     logger.info('router.rule.removed', { id, ruleId });
     const newSite = await states.site.findOne({ _id: id });
 
-    await attachInterfaceUrls({ sites: newSite, context, node: states.node });
+    await attachRuntimeDomainAliases({ sites: newSite, context, node: states.node });
 
     this.emit('router.rule.removed', newSite);
     return newSite;
   }
 
-  // eslint-disable-next-line no-unused-vars
-  async addCertificate(entity, context = {}) {
-    entity.type = 'nginx';
-    this.fixCertificate(entity);
-
-    this.validateCertificate(entity);
-
-    const info = Certificate.fromPEM(entity.certificate);
-    const domain = get(info, 'subject.commonName', '');
-
-    const cert = await states.certificate.find({ domain });
-    const hasOne = cert && cert.length;
-
-    if (hasOne) {
-      throw new Error('certificate has exists!');
-    }
-
-    const newCert = await states.certificate.upsert({
-      ...entity,
-      domain,
-    });
-    logger.info('add certificate result', { domain: newCert.domain });
-    this.emit('router.certificate.add', { type: 'nginx', data: newCert });
-  }
-
-  // eslint-disable-next-line no-unused-vars
-  async deleteCertificate({ id }, context = {}) {
-    const tmpCert = await states.certificate.find({ _id: id });
-    if (!tmpCert) {
-      throw new Error('certificate does not exist');
-    }
-
-    const removeResult = await states.certificate.remove({ _id: id });
-
-    logger.info('delete certificate', { removeResult, domain: tmpCert.domain });
-    this.emit('router.certificate.remove', { type: 'nginx', data: { domain: tmpCert.domain } });
-    return {};
-  }
-
-  // eslint-disable-next-line no-unused-vars
-  async updateNginxHttpsCert(entity, context = {}) {
-    entity.type = 'nginx';
-    this.fixCertificate(entity);
-    this.validateCertificate(entity, entity.domain);
-    const dbEntity = await states.certificate.upsert(entity);
-    this.emit('router.certificate.updated', { type: 'nginx', data: dbEntity });
-  }
-
-  findCertificateByDomain(domain) {
-    return states.certificate.findOne({ domain });
-  }
-
   // ============================================================================================
   // Internal API that are used by public APIs and called from CLI
   // ============================================================================================
@@ -448,16 +422,16 @@ class RouterManager extends EventEmitter {
   }
 
   async getMatchedCert(domain) {
-    const certs = await states.certificate.find();
+    const certs = await this.certManager.getAllNormal();
     const matchedCert = certs.find((cert) => this.isCertMatchedDomain(cert, domain));
 
     if (matchedCert) {
       return {
         id: matchedCert.id,
         domain: matchedCert.domain,
-        issuer: matchedCert.issuer,
-        validFrom: matchedCert.validFrom,
-        validTo: matchedCert.validTo,
+        issuer: matchedCert.meta.issuer,
+        validFrom: matchedCert.meta.validFrom,
+        validTo: matchedCert.meta.validTo,
       };
     }
 
@@ -522,10 +496,7 @@ class RouterManager extends EventEmitter {
     const info = await states.node.read();
     const httpsEnabled = get(info, 'routing.https', true);
 
-    const selection = { domain: 1, privateKey: 1, certificate: 1 };
-    const certificates = httpsEnabled
-      ? await states.certificate.find({ type: get(info, 'routing.provider', null) }, selection)
-      : [];
+    const certificates = httpsEnabled ? await this.certManager.getAllNormal() : [];
 
     let sites = await states.site.getSites();
     // mutate data by input
@@ -570,14 +541,14 @@ class RouterManager extends EventEmitter {
     // get provider
     const providerName = getProviderFromNodeInfo(info);
     const Provider = getProvider(providerName);
-    const tempPath = path.join(os.tmpdir(), `${providerName}-${Date.now()}`);
-    const provider = new Provider({ configDirectory: tempPath });
+    const tmpDir = path.join(os.tmpdir(), `${providerName}-${Date.now()}`);
+    const provider = new Provider({ configDir: tmpDir });
     const tempRouter = new Router({
       provider,
       getRoutingParams: async () => ({
         sites: await ensureLatestInfo(sites),
         certificates,
-        globalHeaders: get(info, 'routing.headers', {}),
+        commonHeaders: get(info, 'routing.headers', {}),
         services: [], // TODO: do we need to add some item here?
         nodeInfo: info,
       }),
@@ -586,47 +557,14 @@ class RouterManager extends EventEmitter {
     await tempRouter.updateRoutingTable();
     try {
       await tempRouter.validateConfig();
-      await fse.remove(tempPath);
+      await fse.remove(tmpDir);
     } catch (error) {
+      // 如果出错，保留 Nginx 配置文件，方便定位问题
       logger.error('validate router config failed', { error, action, data });
-      await fse.remove(tempPath);
       throw error;
     }
   }
 
-  validateCertificate(cert, domain) {
-    const certificate = Certificate.fromPEM(cert.certificate);
-    const privateKey = PrivateKey.fromPEM(cert.privateKey);
-
-    const data = Buffer.allocUnsafe(100);
-    const signature = privateKey.sign(data, 'sha512');
-    if (!certificate.publicKey.verify(data, signature, 'sha512')) {
-      throw new Error('Invalid certificate: signature verify failed');
-    }
-
-    const certDomain = get(certificate, 'subject.commonName', '');
-    if (domain && domain !== certDomain) {
-      throw new Error('Invalid certificate: domain does not match');
-    }
-
-    const validFrom = get(certificate, 'validFrom', '');
-    if (!validFrom || new Date(validFrom).getTime() > Date.now()) {
-      throw new Error('Invalid certificate: not in valid period');
-    }
-    const validTo = get(certificate, 'validTo', '');
-    if (!validTo || new Date(validTo).getTime() < Date.now()) {
-      throw new Error('Invalid certificate: not in valid period');
-    }
-
-    return certificate;
-  }
-
-  fixCertificate(entity) {
-    // Hack: this logic exists because gql does not allow line breaks in arg values
-    entity.privateKey = entity.privateKey.split('|').join('\n');
-    entity.certificate = entity.certificate.split('|').join('\n');
-  }
-
   fixRootBlockletRule(rule) {
     if (!rule.id) {
       rule.id = uuid.v4();
@@ -634,9 +572,9 @@ class RouterManager extends EventEmitter {
     rule.groupId = rule.id;
     rule.from.pathPrefix = normalizePathPrefix(rule.from.pathPrefix);
     if (rule.to.type === ROUTING_RULE_TYPES.BLOCKLET) {
-      rule.from.groupPathPrefix = rule.from.pathPrefix;
-      rule.to.realDid = rule.to.did;
-      rule.to.realInterfaceName = rule.to.interfaceName;
+      // pathPrefix of root blocklet maybe changed to another prefix than '/', so use old groupPathPrefix first
+      rule.from.groupPathPrefix = rule.from.groupPathPrefix || rule.from.pathPrefix;
+      rule.to.componentId = rule.to.did;
     }
     if (rule.to.url) {
       rule.to.url = normalizeRedirectUrl(rule.to.url);
@@ -645,64 +583,85 @@ class RouterManager extends EventEmitter {
 
   /**
    * get all rules to be add or update to site from root rule
-   * @param {*} rule
+   * @param {*} rootRule
    */
-  async getRules(rule) {
-    if (rule.to.type !== ROUTING_RULE_TYPES.BLOCKLET) {
-      return [rule];
+  async getRulesForMutation(rootRule) {
+    if (rootRule.to.type !== ROUTING_RULE_TYPES.BLOCKLET) {
+      return [rootRule];
     }
 
     const rules = [];
-    let occupied = false;
 
     // get child rules
-    const blocklet = await states.blocklet.getBlocklet(rule.to.did);
-    for (const childMeta of blocklet.meta.children || []) {
-      for (const mountPoint of childMeta.mountPoints) {
-        const child = blocklet.children.find((b) => b.meta.name === childMeta.name);
-
-        if (!child) {
-          logger.error(`Child blocklet ${childMeta.name} does not exist`);
-          // eslint-disable-next-line no-continue
-          continue;
-        }
+    const blocklet = await states.blocklet.getBlocklet(rootRule.to.did);
 
-        if (mountPoint.root.interfaceName === rule.to.interfaceName) {
-          const pathPrefix = path.join(rule.from.pathPrefix, mountPoint.root.prefix);
-          const isRootPath = pathPrefix === rule.from.pathPrefix;
-          if (isRootPath) {
-            occupied = true;
-          }
+    // blocklet may be mounted in relative prefix (for old usage), so blockletPrefix may not be '/'
+    // blocklet prefix is the origin pathPrefix in rootRule
+    const blockletPrefix = normalizePathPrefix(rootRule.from.pathPrefix);
 
-          // if is root path, child rule become root rule
-          const childRule = {
-            id: isRootPath ? rule.id : uuid.v4(),
-            groupId: rule.id,
-            from: {
-              pathPrefix: normalizePathPrefix(pathPrefix),
-              groupPathPrefix: rule.from.pathPrefix,
-            },
-            to: {
-              type: ROUTING_RULE_TYPES.BLOCKLET,
-              port: findInterfacePortByName(child, mountPoint.child.interfaceName),
-              did: rule.to.did, // root blocklet did
-              interfaceName: rule.to.interfaceName, // root blocklet interface
-              realDid: child.meta.did, // child blocklet did
-              realInterfaceName: mountPoint.child.interfaceName,
-            },
-            isProtected: isRootPath ? rule.isProtected : true,
-          };
-
-          rules.push(childRule);
-        }
-      }
-    }
+    // root component's mountPoint may not be '/'
+    const rootComponentPrefix = joinUrl(blockletPrefix, blocklet.mountPoint || '/');
+    rootRule.from.pathPrefix = normalizePathPrefix(rootComponentPrefix);
+
+    const isOccupiable = blocklet.meta.group === BlockletGroup.gateway;
 
-    // get root rule
-    if (!occupied) {
-      rules.push(rule);
+    if (!isOccupiable) {
+      rules.push(rootRule);
     }
 
+    forEachChildSync(blocklet, (component, { id, ancestors }) => {
+      if (component.meta.group === BlockletGroup.gateway) {
+        return;
+      }
+
+      const { mountPoint } = component;
+      if (!mountPoint) {
+        logger.error(`mountPoint of child ${component.meta.name} does not exist`);
+        // eslint-disable-next-line no-continue
+        return;
+      }
+
+      const childWebInterface = findWebInterface(component);
+      if (!childWebInterface) {
+        logger.error(`web interface of child ${component.meta.name} does not exist`);
+        // eslint-disable-next-line no-continue
+        return;
+      }
+
+      const pathPrefix = path.join(
+        blockletPrefix,
+        // level 1 component should be independent with root component
+        ...ancestors.slice(1).map((x) => x.mountPoint || ''),
+        mountPoint
+      );
+
+      const occupied = normalizePathPrefix(pathPrefix) === normalizePathPrefix(rootRule.from.pathPrefix);
+
+      if (occupied && !isOccupiable) {
+        return;
+      }
+
+      // if is root path, child rule become root rule
+      const childRule = {
+        id: occupied ? rootRule.id : uuid.v4(),
+        groupId: rootRule.id,
+        from: {
+          pathPrefix: normalizePathPrefix(pathPrefix),
+          groupPathPrefix: blockletPrefix,
+        },
+        to: {
+          type: ROUTING_RULE_TYPES.BLOCKLET,
+          port: findInterfacePortByName(component, childWebInterface.name),
+          did: rootRule.to.did, // root component did
+          interfaceName: rootRule.to.interfaceName, // root component interface
+          componentId: id,
+        },
+        isProtected: occupied ? rootRule.isProtected : true,
+      };
+
+      rules.push(childRule);
+    });
+
     return rules;
   }
 }

package/lib/states/README.md

@@ -1,3 +1,38 @@
 # State DB
 
-All ABT Node states are managed by files in this folder.
+All Blocklet Server states are managed by files in this folder.
+
+## blocklet
+
+- `meta` meta of bundle _defined in @blocklet/meta/schema_
+  - `did`, `name`: component id, default from source meta, can be changed before install
+  - `title`, `description`: component info, default from source meta, can be changed before install
+  - `bundleDdid`, `bundleName`: bundle id, copy from source meta
+  - `version`: component version
+  - `price`: used for charging
+  - `logo`: component logo
+  - `logoUrl`: component logo (from store)
+  - `interfaces`: for resolving: runtime port; service config; web base prefix; api base prefix
+  - `environments`: component environment
+  - `scripts`: for hook of component lifecycle
+  - `engine`, `timeout`, `group`, `main`: related to process startup
+  - `dist`: bundle source
+  - `theme`: app theme
+  - `navigation`: app navigation`
+- `source`: type of bundle source
+- `bundleSource`: download infomation of bundle srouce
+- `deployedFrom`: extra information of bundle source
+- `appDid`: app instance id _app only_
+- `status`, `startedAt`, `installedAt`, `stoppedAt` app status
+- `deletedAt` _component only_
+- `mode` app mode
+- `ports` runtime port resolved from meta
+  - `<ENV_NAME>`: `<port number>`
+- `environments[]`: env variables generated by server, e.g. BLOCKLET_APP_SK, BLOCKLET_APP_DID
+  - `<key>`: `<value>`
+- `children`: component of app, same structure as parent, can be nested
+- `mountPoint` mount path of web site. _component only_
+- `dynamic` is dynamically added. _component only_
+- `controller`
+- `tokens`
+  - `downloadTokenList`