@abtnode/core 1.6.6 → 1.6.10

package/lib/router/helper.js

@@ -21,15 +21,13 @@ const {
   NAME_FOR_WELLKNOWN_SITE,
   DEFAULT_HTTP_PORT,
   DEFAULT_HTTPS_PORT,
-  DAY_IN_MS,
   NODE_MODES,
   ROUTING_RULE_TYPES,
   CERTIFICATE_EXPIRES_OFFSET,
-  CERTIFICATE_EXPIRES_WARNING_OFFSET,
-  DEFAULT_DAEMON_PORT,
   DEFAULT_SERVICE_PATH,
   SLOT_FOR_IP_DNS_SITE,
   BLOCKLET_SITE_GROUP_SUFFIX,
+  WELLKNOWN_ACME_CHALLENGE_PREFIX,
 } = require('@abtnode/constant');
 const {
   BLOCKLET_DYNAMIC_PATH_PREFIX,
@@ -50,7 +48,7 @@ const {
   getWellknownSitePort,
 } = require('../util');
 const { getServicesFromBlockletInterface } = require('../util/service');
-const getIpDnsDomainForBlocklet = require('../util/get-ip-dns-domain-for-blocklet');
+const { getIpDnsDomainForBlocklet, getDidDomainForBlocklet } = require('../util/get-domain-for-blocklet');
 const { getFromCache: getAccessibleExternalNodeIp } = require('../util/get-accessible-external-node-ip');
 
 const Router = require('./index');
@@ -403,11 +401,10 @@ const decompressCertificates = async (source, dest) => {
   return dest;
 };
 
-module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerManager, blockletManager }) {
+module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerManager, blockletManager, certManager }) {
   const nodeState = states.node;
   const blockletState = states.blocklet;
   const siteState = states.site;
-  const httpsCertState = states.certificate;
   const notification = states.notification;
 
   // site level duplication detection
@@ -468,58 +465,105 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       const certificate = fs.readFileSync(certificateFilePath).toString();
       const privateKey = fs.readFileSync(privateKeyFilePath).toString();
 
-      await routerManager.updateNginxHttpsCert({ domain: dashboardDomain, privateKey, certificate, isProtected: true });
+      await certManager.upsertByDomain({
+        domain: dashboardDomain,
+        privateKey,
+        certificate,
+        isProtected: true,
+      });
       logger.info('dashboard certificate updated');
     } catch (error) {
-      logger.error('Update dashboard certificate failed', { error });
+      logger.error('update dashboard certificate failed', { error });
+      throw error;
     } finally {
       fs.removeSync(destFolder);
     }
   };
 
+  const ensureDashboardCertificate = async () => {
+    const info = await nodeState.read();
+    const downloadUrl = get(info, 'routing.dashboardCertDownloadAddress', '');
+    const dashboardDomain = get(info, 'routing.dashboardDomain', '');
+    if (!dashboardDomain || !downloadUrl) {
+      throw new Error('dashboardCertDownloadAddress and dashboardDomain are not found in the routing configs');
+    }
+
+    const cert = await certManager.getByDomain(dashboardDomain);
+    if (cert) {
+      return { status: 'existed' };
+    }
+
+    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    await updateDashboardCertificate({ checkExpire: false });
+    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    return { status: 'downloaded' };
+  };
+
   const addWellknownSite = async (sites, context) => {
     const site = (sites || []).find((x) => x.name === NAME_FOR_WELLKNOWN_SITE);
 
     try {
-      const pathPrefix = joinUrl(WELLKNOWN_PATH_PREFIX, '/did.json');
+      const info = await nodeState.read();
+      const proxyTarget = {
+        port: info.port,
+        type: ROUTING_RULE_TYPES.GENERAL_PROXY,
+        interfaceName: BLOCKLET_INTERFACE_WELLKNOWN,
+      };
+
       const didResolverWellknownRule = {
-        from: { pathPrefix },
-        to: {
-          port: DEFAULT_DAEMON_PORT,
-          type: ROUTING_RULE_TYPES.GENERAL_PROXY,
-          interfaceName: BLOCKLET_INTERFACE_WELLKNOWN,
-        },
+        from: { pathPrefix: joinUrl(WELLKNOWN_PATH_PREFIX, '/did.json') },
+        to: proxyTarget,
+      };
+
+      const acmeChallengeWellknownRule = {
+        from: { pathPrefix: WELLKNOWN_ACME_CHALLENGE_PREFIX },
+        to: proxyTarget,
       };
 
       if (site) {
-        if (site.rules.find((r) => r.from.pathPrefix === normalizePathPrefix(pathPrefix))) {
-          return false;
+        let changed = false;
+        const exists = (prefix) => !!site.rules.find((r) => r.from.pathPrefix === normalizePathPrefix(prefix));
+
+        if (!exists(didResolverWellknownRule.from.pathPrefix)) {
+          await routerManager.addRoutingRule(
+            {
+              id: site.id,
+              rule: didResolverWellknownRule,
+            },
+            context
+          );
+          changed = true;
         }
 
-        await routerManager.addRoutingRule(
-          {
-            id: site.id,
-            rule: didResolverWellknownRule,
-          },
-          context
-        );
-      } else {
-        await routerManager.addRoutingSite(
-          {
-            site: {
-              domain: DOMAIN_FOR_INTERNAL_SITE,
-              port: await getWellknownSitePort(),
-              name: NAME_FOR_WELLKNOWN_SITE,
-              rules: [didResolverWellknownRule],
-              isProtected: true,
+        if (!exists(acmeChallengeWellknownRule.from.pathPrefix)) {
+          await routerManager.addRoutingRule(
+            {
+              id: site.id,
+              rule: acmeChallengeWellknownRule,
             },
-            skipCheckDynamicBlacklist: true,
-            skipValidation: true,
-          },
-          context
-        );
+            context
+          );
+          changed = true;
+        }
+
+        return changed;
       }
 
+      await routerManager.addRoutingSite(
+        {
+          site: {
+            domain: DOMAIN_FOR_INTERNAL_SITE,
+            port: await getWellknownSitePort(),
+            name: NAME_FOR_WELLKNOWN_SITE,
+            rules: [didResolverWellknownRule, acmeChallengeWellknownRule],
+            isProtected: true,
+          },
+          skipCheckDynamicBlacklist: true,
+          skipValidation: true,
+        },
+        context
+      );
+
       return true;
     } catch (err) {
       console.error('add well-known site failed:', err);
@@ -533,7 +577,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
    *
    * @returns {boolean} if routing changed
    */
-  const ensureDashboardRouting = async (context = {}, output) => {
+  const ensureDashboardRouting = async (context = {}) => {
     const info = await nodeState.read();
 
     const provider = getProviderFromNodeInfo(info);
@@ -579,11 +623,16 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       });
 
     const dashboardDomain = get(info, 'routing.dashboardDomain', '');
-    if (dashboardDomain && !isExistsInAlias(dashboardDomain)) {
+    const didDomain = `${info.did.toLowerCase()}.${info.didDomain}`;
+    const dashboardAliasDomains = [dashboardDomain, didDomain]
+      .filter((item) => item && !isExistsInAlias(item))
+      .map((item) => ({ value: item, isProtected: true }));
+
+    if (dashboardAliasDomains.length > 0) {
       try {
         const result = await siteState.update(
           { _id: dashboardSite.id },
-          { $push: { domainAliases: { value: dashboardDomain, isProtected: true } } }
+          { $push: { domainAliases: { $each: dashboardAliasDomains } } }
         );
 
         updatedResult.push(result);
@@ -618,18 +667,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       updatedResult.push(wellknownRes);
     }
 
-    // Download dashboard certificates if not exists
-    const certDownloadAddress = get(info, 'routing.dashboardCertDownloadAddress', '');
-    if (dashboardDomain && certDownloadAddress) {
-      const cert = await routerManager.findCertificateByDomain(dashboardDomain);
-      if (!cert) {
-        await updateDashboardCertificate({ checkExpire: false });
-        if (typeof output === 'function') {
-          output('Dashboard HTTPS certificate was downloaded successfully!');
-        }
-      }
-    }
-
     if (updatedResult.length) {
       const hash = await takeRoutingSnapshot({ message: 'ensure dashboard routing rules', dryRun: false }, context);
       logger.info('take routing snapshot on ensure dashboard routing rules', { updatedResult, hash });
@@ -658,7 +695,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     };
 
     const domainGroup = `${blocklet.meta.did}${BLOCKLET_SITE_GROUP_SUFFIX}`;
-    const domain = getIpDnsDomainForBlocklet(blocklet, webInterface);
+
     const pathPrefix = getPrefix(webInterface.prefix);
     const rule = {
       from: { pathPrefix },
@@ -673,11 +710,24 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
 
     const existSite = await states.site.findOne({ domain: domainGroup });
     if (!existSite) {
+      const ipEchoDnsDomain = getIpDnsDomainForBlocklet(blocklet, webInterface, nodeInfo.did);
+      const appIdEnv = blocklet.environments.find((e) => e.key === 'BLOCKLET_APP_ID');
+      const domainAliases = [{ value: ipEchoDnsDomain, isProtected: true }];
+
+      const didDomain = getDidDomainForBlocklet({
+        appId: appIdEnv.value,
+        didDomain: nodeInfo.didDomain,
+      });
+
+      if (blocklet.mode !== 'development') {
+        domainAliases.push({ value: didDomain, isProtected: true });
+      }
+
       await routerManager.addRoutingSite(
         {
           site: {
             domain: domainGroup,
-            domainAliases: [{ value: domain, isProtected: true }],
+            domainAliases,
             isProtected: true,
             rules: [rule],
           },
@@ -685,7 +735,14 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
         },
         context
       );
-      logger.info('add routing site', { site: domain });
+      logger.info('add routing site', { site: domainGroup });
+      if (
+        process.env.NODE_ENV !== 'development' &&
+        process.env.NODE_ENV !== 'test' &&
+        blocklet.mode !== 'development'
+      ) {
+        await certManager.issue({ domain: didDomain });
+      }
 
       return true;
     }
@@ -700,13 +757,13 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
         },
         skipProtectedRuleChecking: true,
       });
-      logger.info('update routing rule for site', { site: domain });
+      logger.info('update routing rule for site', { site: domainGroup });
     } else {
       await routerManager.addRoutingRule({
         id: existSite.id,
         rule,
       });
-      logger.info('add routing rule for site', { site: domain });
+      logger.info('add routing rule for site', { site: domainGroup });
     }
 
     return true;
@@ -1068,9 +1125,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
             sites = await ensureAuthService(sites, blockletManager);
             sites = await ensureServiceRule(sites);
 
-            const certificates = httpsEnabled
-              ? await httpsCertState.find({ type: providerName }, { domain: 1, certificate: 1, privateKey: 1 })
-              : [];
+            const certificates = httpsEnabled ? await certManager.getAllNormal() : [];
 
             return {
               sites,
@@ -1093,9 +1148,9 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
         },
       });
 
-      routerManager.on('router.certificate.add', () => routers[providerName].restart());
-      routerManager.on('router.certificate.updated', () => routers[providerName].restart());
-      routerManager.on('router.certificate.remove', () => routers[providerName].restart());
+      certManager.on('cert.added', () => routers[providerName].restart());
+      certManager.on('cert.removed', () => routers[providerName].restart());
+      certManager.on('cert.issued', () => routers[providerName].restart());
 
       await routers[providerName].start();
     }
@@ -1257,18 +1312,21 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
   };
 
   const getCertificates = async () => {
-    const certificates = await httpsCertState.find();
+    const certificates = await certManager.getAll();
     const sites = await getSitesFromSnapshot();
+
+    const isMatch = (cert, domain) =>
+      domain !== DOMAIN_FOR_DEFAULT_SITE && domain && routerManager.isCertMatchedDomain(cert, domain);
+
     return certificates.map((cert) => {
       cert.matchedSites = [];
       sites.forEach((site) => {
-        if (
-          site.domain !== DOMAIN_FOR_DEFAULT_SITE &&
-          site.domain &&
-          routerManager.isCertMatchedDomain(cert, site.domain)
-        ) {
-          cert.matchedSites.push({ id: site.id, domain: site.domain });
-        }
+        const domains = [site.domain, ...(site.domainAliases || []).map((x) => x.value)];
+        domains.forEach((domain) => {
+          if (isMatch(cert, domain)) {
+            cert.matchedSites.push({ id: site.id, domain });
+          }
+        });
       });
 
       return cert;
@@ -1281,42 +1339,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     return { domain, isHttps: !!matchedCert, matchedCert };
   };
 
-  const checkCertificatesExpiration = async () => {
-    const now = Date.now();
-
-    const certificates = await getCertificates();
-    for (let i = 0; i < certificates.length; i++) {
-      const cert = certificates[i];
-      const alreadyExpired = now >= cert.validTo;
-      const aboutToExpire = cert.validTo - now > 0 && cert.validTo - now < CERTIFICATE_EXPIRES_WARNING_OFFSET;
-
-      if (alreadyExpired) {
-        logger.info('send certificate expire notification', { domain: cert.domain });
-        notification.create({
-          title: 'SSL Certificate Expired',
-          description: `Your SSL certificate for domain ${cert.domain} has expired, please update it in Blocklet Server`,
-          severity: 'error',
-          entityType: 'certificate',
-          entityId: cert._id, // eslint-disable-line no-underscore-dangle
-        });
-      } else if (aboutToExpire) {
-        logger.info('send certificate about-expire notification', { domain: cert.domain });
-        const expireInDays = Math.ceil((cert.validTo - now) / DAY_IN_MS);
-        notification.create({
-          title: 'SSL Certificate Expire Warning',
-          description: `Your SSL certificate for domain ${
-            cert.domain
-          } will expire in ${expireInDays} days (on ${new Date(
-            cert.validTo
-          ).toLocaleString()}), please remember to update it in Blocklet Server`,
-          severity: 'warning',
-          entityType: 'certificate',
-          entityId: cert._id, // eslint-disable-line no-underscore-dangle
-        });
-      }
-    }
-  };
-
   return {
     ensureDashboardRouting,
     ensureBlockletRouting,
@@ -1328,9 +1350,10 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     takeRoutingSnapshot,
     getRoutingSites,
     getSnapshotSites,
-    getCertificates,
     getSitesFromSnapshot,
+    getCertificates,
     checkDomain,
+    ensureDashboardCertificate,
 
     getRoutingCrons: () => [
       {
@@ -1339,12 +1362,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
         fn: () => updateDashboardCertificate({ checkExpire: true }),
         options: { runOnInit: true },
       },
-      {
-        name: 'check-certificate-expiration',
-        time: '0 0 9 * * *', // check on 09:00 every day
-        fn: checkCertificatesExpiration,
-        options: { runOnInit: false },
-      },
       {
         name: 'rotate-log-files',
         time: '5 0 0 * * *', // rotate at 05:00 every day

package/lib/router/manager.js

@@ -70,8 +70,9 @@ const normalizeRedirectUrl = (url) => {
 };
 
 class RouterManager extends EventEmitter {
-  constructor() {
+  constructor({ certManager }) {
     super();
+    this.certManager = certManager;
 
     // HACK: do not emit any events from CLI
     if (isCLI()) {
@@ -366,7 +367,7 @@ class RouterManager extends EventEmitter {
       domain,
     });
     logger.info('add certificate result', { domain: newCert.domain });
-    this.emit('router.certificate.add', { type: 'nginx', data: newCert });
+    this.emit('cert.added', { type: 'nginx', data: newCert });
   }
 
   // eslint-disable-next-line no-unused-vars
@@ -379,7 +380,7 @@ class RouterManager extends EventEmitter {
     const removeResult = await states.certificate.remove({ _id: id });
 
     logger.info('delete certificate', { removeResult, domain: tmpCert.domain });
-    this.emit('router.certificate.remove', { type: 'nginx', data: { domain: tmpCert.domain } });
+    this.emit('cert.removed', { type: 'nginx', data: { domain: tmpCert.domain } });
     return {};
   }
 
@@ -389,7 +390,7 @@ class RouterManager extends EventEmitter {
     this.fixCertificate(entity);
     this.validateCertificate(entity, entity.domain);
     const dbEntity = await states.certificate.upsert(entity);
-    this.emit('router.certificate.updated', { type: 'nginx', data: dbEntity });
+    this.emit('cert.issued', { type: 'nginx', data: dbEntity });
   }
 
   findCertificateByDomain(domain) {
@@ -451,16 +452,16 @@ class RouterManager extends EventEmitter {
   }
 
   async getMatchedCert(domain) {
-    const certs = await states.certificate.find();
+    const certs = await this.certManager.getAll();
     const matchedCert = certs.find((cert) => this.isCertMatchedDomain(cert, domain));
 
     if (matchedCert) {
       return {
         id: matchedCert.id,
         domain: matchedCert.domain,
-        issuer: matchedCert.issuer,
-        validFrom: matchedCert.validFrom,
-        validTo: matchedCert.validTo,
+        issuer: matchedCert.meta.issuer,
+        validFrom: matchedCert.meta.validFrom,
+        validTo: matchedCert.meta.validTo,
       };
     }
 

package/lib/states/base.js

@@ -1,234 +1,17 @@
-const fs = require('fs');
-const path = require('path');
-const util = require('util');
-const { EventEmitter } = require('events');
-const cloneDeep = require('lodash/cloneDeep');
-const DataStore =
-  process.env.NODE_ENV === 'test' ? require('@nedb/core') : require('@nedb/multi')(Number(process.env.NEDB_MULTI_PORT));
+const DB = require('@abtnode/db');
 const logger = require('@abtnode/logger')('@abtnode/core:states');
 
 const { isCLI } = require('../util');
 
-class BaseState extends EventEmitter {
+class BaseState extends DB {
   constructor(baseDir, options) {
-    super();
+    super(baseDir, options);
 
     // HACK: do not emit any events from CLI
     if (isCLI() && process.env.NODE_ENV !== 'test') {
       this.emit = (name) => logger.debug('stopped state db event in CLI', name);
     }
-
-    const dbOptions = options.db || {};
-    this.filename = path.join(baseDir, options.filename);
-    this.options = Object.freeze(cloneDeep(options));
-    this.db = new DataStore({
-      filename: this.filename,
-      timestampData: true,
-      ...dbOptions,
-    });
-
-    logger.info('initialized', { filename: this.filename });
-
-    this.ready = false;
-    this.readyCallbacks = [];
-    this.db.loadDatabase((err) => {
-      if (err) {
-        logger.error(`failed to load disk database ${this.filename}`, { error: err });
-        console.error(err);
-      } else {
-        this.ready = true;
-        if (this.readyCallbacks.length) {
-          this.readyCallbacks.forEach((x) => x());
-        }
-      }
-    });
-
-    this.asyncDB = new Proxy(this.db, {
-      get(target, property) {
-        if (typeof target[property] === 'function') {
-          return util
-            .promisify((...args) => {
-              const cb = args[args.length - 1];
-              const rest = args.slice(0, args.length - 1);
-
-              target[property](...rest, (err, ...result) => {
-                if (err) {
-                  return cb(err);
-                }
-
-                if (result.length === 1) {
-                  return cb(null, result[0]);
-                }
-
-                return cb(null, result);
-              });
-            })
-            .bind(target);
-        }
-
-        return target[property];
-      },
-    });
-  }
-
-  onReady(cb) {
-    if (this.ready) {
-      cb();
-    } else {
-      this.readyCallbacks.push(cb);
-    }
-  }
-
-  createNotification(payload) {
-    if (this.notification && typeof this.notification.create === 'function') {
-      this.notification.create(payload);
-    }
-  }
-
-  paginate(conditions, sort, paging) {
-    const { pageSize: size = 20, page = 1 } = paging || {};
-    const pageSize = Math.min(100, size);
-
-    return new Promise((resolve, reject) => {
-      this.db
-        .find(conditions)
-        .sort(sort)
-        .exec((err, docs) => {
-          if (err) {
-            return reject(err);
-          }
-
-          const pageCount = Math.ceil(docs.length / pageSize);
-          const total = docs.length;
-          const skip = (page - 1) * pageSize;
-          const list = docs.slice(skip, skip + pageSize);
-
-          list.forEach((doc) => {
-            // eslint-disable-next-line no-underscore-dangle
-            doc.id = doc._id;
-          });
-
-          return resolve({
-            list,
-            paging: {
-              total,
-              pageSize,
-              pageCount,
-              page,
-            },
-          });
-        });
-    });
-  }
-
-  async updateById(id, updates, options = {}) {
-    const [, doc] = await this.asyncDB.update({ _id: id }, updates, {
-      multi: false,
-      upsert: false,
-      returnUpdatedDocs: true,
-      ...options,
-    });
-
-    return doc;
-  }
-
-  update(...args) {
-    if (args.length === 0) {
-      throw new Error('param is required by update method');
-    }
-
-    if (typeof args[0] === 'string') {
-      return this.updateById(...args);
-    }
-
-    return this.asyncDB.update(args[0], args[1], { returnUpdatedDocs: true, ...(args[2] || {}) });
-  }
-
-  count(conditions = {}) {
-    return new Promise((resolve, reject) => {
-      this.db.count(conditions, (err, num) => {
-        if (err) {
-          return reject(err);
-        }
-
-        return resolve(num);
-      });
-    });
-  }
-
-  remove(conditions = {}, options = { multi: false }) {
-    return new Promise((resolve, reject) => {
-      this.db.remove(conditions, options, (err, num) => {
-        if (err) {
-          return reject(err);
-        }
-
-        return resolve(num);
-      });
-    });
-  }
-
-  reset() {
-    fs.unlinkSync(this.filename);
-  }
-
-  find(...args) {
-    if (args.length === 0) {
-      return this.asyncDB.find({});
-    }
-
-    return this.asyncDB.find(...args);
-  }
-
-  findOne(...args) {
-    if (args.length === 0) {
-      return this.asyncDB.findOne({});
-    }
-
-    return this.asyncDB.findOne(...args);
-  }
-
-  insert(...args) {
-    return this.asyncDB.insert(...args);
-  }
-
-  compactDatafile() {
-    this.db.persistence.compactDatafile((err) => {
-      if (err) {
-        console.error(`failed to compact datafile: ${this.filename}`, err);
-      }
-    });
   }
 }
 
-/**
- * Rename _id field name to id, this method has side effects
- * @param {object} entities
- */
-const renameIdFiledName = (entities, from = '_id', to = 'id') => {
-  /* eslint-disable  no-underscore-dangle, no-param-reassign */
-
-  if (!entities) {
-    return entities;
-  }
-
-  const mapEntity = (entity) => {
-    if (entity[from]) {
-      entity[to] = entity[from];
-      delete entity[from];
-    }
-  };
-
-  if (!Array.isArray(entities)) {
-    mapEntity(entities);
-    return entities;
-  }
-
-  entities.forEach(mapEntity);
-
-  return entities;
-};
-
-BaseState.renameIdFiledName = renameIdFiledName;
-
 module.exports = BaseState;