@abtnode/core 1.6.4 → 1.6.8

package/lib/cert.js

@@ -0,0 +1,116 @@
+const { EventEmitter } = require('events');
+const CertificateManager = require('@abtnode/certificate-manager/sdk/manager');
+const logger = require('@abtnode/logger')('@abtnode/core:cert');
+const states = require('./states');
+
+const onCertExpired = async (cert) => {
+  logger.info('send certificate expire notification', { domain: cert.domain });
+  states.notification.create({
+    title: 'SSL Certificate Expired',
+    description: `Your SSL certificate for domain ${cert.domain} has expired, please update it in Blocklet Server`,
+    severity: 'error',
+    entityType: 'certificate',
+    entityId: cert.id,
+  });
+};
+
+const onCertAboutExpire = (cert) => {
+  logger.info('send certificate about-expire notification', { domain: cert.domain });
+  states.notification.create({
+    title: 'SSL Certificate Expire Warning',
+    description: `Your SSL certificate for domain ${cert.domain} will expire in ${
+      cert.expireInDays
+    } days (on ${new Date(cert.validTo).toLocaleString()}), please remember to update it in Blocklet Server`,
+    severity: 'warning',
+    entityType: 'certificate',
+    entityId: cert.id, // eslint-disable-line no-underscore-dangle
+  });
+};
+
+const onCertIssued = (cert) => {
+  states.notification.create({
+    title: 'Certificate Issued',
+    description: `The ${cert.domain} certificate is issued successfully`,
+    severity: 'success',
+    entityType: 'certificate',
+    entityId: cert.id,
+  });
+};
+
+class Cert extends EventEmitter {
+  constructor({ maintainerEmail, dataDir }) {
+    super();
+
+    this.manager = new CertificateManager({ maintainerEmail, dataDir });
+
+    this.manager.on('cert.issued', (cert) => {
+      this.emit('cert.issued', cert);
+      onCertIssued(cert);
+    });
+
+    this.manager.on('cert.expired', onCertExpired);
+    this.manager.on('cert.about_to_expire', onCertAboutExpire);
+  }
+
+  start() {
+    return this.manager.start();
+  }
+
+  static fixCertificate(entity) {
+    // Hack: this logic exists because gql does not allow line breaks in arg values
+    entity.privateKey = entity.privateKey.split('|').join('\n');
+    entity.certificate = entity.certificate.split('|').join('\n');
+  }
+
+  getAll() {
+    return this.manager.getAll();
+  }
+
+  getAllNormal() {
+    return this.manager.getAllNormal();
+  }
+
+  getByDomain(domain) {
+    return this.manager.getByDomain(domain);
+  }
+
+  async add(data) {
+    if (!data.certificate || !data.privateKey) {
+      throw new Error('certificate and privateKey is required');
+    }
+
+    Cert.fixCertificate(data);
+
+    await this.manager.add(data);
+    logger.info('add certificate result', { name: data.name });
+    this.emit('cert.added', data);
+  }
+
+  async issue({ domain }) {
+    logger.info(`generate certificate for ${domain}`);
+
+    return this.manager.issue(domain);
+  }
+
+  async upsertByDomain(data) {
+    return this.manager.upsertByDomain(data);
+  }
+
+  async update(data) {
+    return this.manager.update(data.id, { name: data.name, public: data.public });
+  }
+
+  async remove({ id }) {
+    await this.manager.remove(id);
+    logger.info('delete certificate', { id });
+    this.emit('cert.removed');
+
+    return {};
+  }
+
+  async addWithoutValidations(data) {
+    return this.manager.addWithoutValidations(data);
+  }
+}
+
+module.exports = Cert;

package/lib/index.js

@@ -11,6 +11,7 @@ const {
   toBlockletSource,
 } = require('@blocklet/meta/lib/constants');
 const { listProviders } = require('@abtnode/router-provider');
+const { DEFAULT_CERTIFICATE_EMAIL } = require('@abtnode/constant');
 
 const RoutingSnapshot = require('./states/routing-snapshot');
 const BlockletRegistry = require('./blocklet/registry');
@@ -22,6 +23,7 @@ const NodeAPI = require('./api/node');
 const TeamAPI = require('./api/team');
 const WebHook = require('./webhook');
 const states = require('./states');
+const Cert = require('./cert');
 
 const IP = require('./util/ip');
 const DomainStatus = require('./util/domain-status');
@@ -108,7 +110,11 @@ function ABTNode(options) {
   // Initialize storage
   states.init(dataDirs, options);
 
-  const routerManager = new RouterManager();
+  const certManager = new Cert({
+    maintainerEmail: DEFAULT_CERTIFICATE_EMAIL,
+    dataDir: dataDirs.certManagerModule,
+  });
+  const routerManager = new RouterManager({ certManager });
   const routingSnapshot = new RoutingSnapshot({
     baseDir: dataDirs.core,
     getRoutingData: async () => {
@@ -117,7 +123,6 @@ function ABTNode(options) {
       return { sites };
     },
   });
-
   const blockletRegistry = new BlockletRegistry();
   const blockletManager = new BlockletManager({
     dataDirs,
@@ -127,6 +132,8 @@ function ABTNode(options) {
     daemon: options.daemon,
   });
 
+  certManager.start(); // 启动证书服务
+
   const {
     handleRouting,
     getRoutingRulesByDid,
@@ -141,10 +148,10 @@ function ABTNode(options) {
     getCertificates,
     getSitesFromSnapshot,
     getRoutingCrons,
-  } = getRouterHelpers({ dataDirs, routingSnapshot, routerManager, blockletManager });
+    ensureDashboardCertificate,
+  } = getRouterHelpers({ dataDirs, routingSnapshot, routerManager, blockletManager, certManager });
 
   const teamManager = new TeamManager({ nodeDid: options.nodeDid, dataDirs, states });
-
   const nodeAPI = new NodeAPI(states.node, blockletRegistry);
   const teamAPI = new TeamAPI({ states, teamManager });
 
@@ -311,6 +318,7 @@ function ABTNode(options) {
     takeRoutingSnapshot,
     getSitesFromSnapshot,
     ensureDashboardRouting,
+    ensureDashboardCertificate,
 
     addDomainAlias: routerManager.addDomainAlias.bind(routerManager),
     deleteDomainAlias: routerManager.deleteDomainAlias.bind(routerManager),
@@ -319,11 +327,13 @@ function ABTNode(options) {
     checkDomains: domainStatus.checkDomainsStatus.bind(domainStatus),
     handleRouting,
 
-    updateNginxHttpsCert: routerManager.updateNginxHttpsCert.bind(routerManager),
+    certManager,
+    updateNginxHttpsCert: certManager.update.bind(certManager), // TODO: 重命名：updateCert
     getCertificates,
-    addCertificate: routerManager.addCertificate.bind(routerManager),
-    deleteCertificate: routerManager.deleteCertificate.bind(routerManager),
-    findCertificateByDomain: routerManager.findCertificateByDomain.bind(routerManager),
+    addCertificate: certManager.add.bind(certManager),
+    deleteCertificate: certManager.remove.bind(certManager),
+    findCertificateByDomain: certManager.getByDomain.bind(certManager),
+    issueLetsEncryptCert: certManager.issue.bind(certManager),
 
     // Access Key
     getAccessKeys: states.accessKey.list.bind(states.accessKey),

package/lib/migrations/1.6.4-security.js

@@ -7,7 +7,7 @@ const cloneDeep = require('lodash/cloneDeep');
 const security = require('@abtnode/util/lib/security');
 
 module.exports = async ({ states, configFile, dataDir }) => {
-  if (process.env.CI) {
+  if (process.env.CI || process.env.NODE_ENV === 'test') {
     return;
   }
 
@@ -34,8 +34,24 @@ module.exports = async ({ states, configFile, dataDir }) => {
         return c;
       });
 
-      await states.blockletExtras.update({ did: item.did }, { $set: { configs: newConfigs } });
+      const newChildren = cloneDeep(item.children || []).map((x) => {
+        return {
+          ...x,
+          configs: cloneDeep(x.configs || []).map((c) => {
+            if (c.secure) {
+              c.value = security.encrypt(c.value, item.did, fs.readFileSync(file));
+            }
+
+            return c;
+          }),
+        };
+      });
+
+      await states.blockletExtras.update({ did: item.did }, { $set: { configs: newConfigs, children: newChildren } });
     }
+
+    states.node.compactDatafile();
+    states.blockletExtras.compactDatafile();
   } catch (err) {
     console.error(err);
     throw err;

package/lib/migrations/1.6.5-security.js

@@ -0,0 +1,60 @@
+/* eslint-disable no-await-in-loop */
+const fs = require('fs');
+const path = require('path');
+const uniq = require('lodash/uniq');
+const cloneDeep = require('lodash/cloneDeep');
+const security = require('@abtnode/util/lib/security');
+const { forEachBlocklet } = require('@blocklet/meta/lib/util');
+
+module.exports = async ({ states, dataDir }) => {
+  if (process.env.CI || process.env.NODE_ENV === 'test') {
+    return;
+  }
+
+  const file = path.join(dataDir, '.sock');
+  if (fs.existsSync(file) === false) {
+    return;
+  }
+
+  try {
+    const extras = await states.blockletExtras.find();
+    const blocklets = await states.blocklet.find();
+    for (const blocklet of blocklets) {
+      forEachBlocklet(
+        blocklet,
+        (b) => {
+          let configKeys;
+          const extra = extras.find((x) => x.did === blocklet.meta.did);
+          const rootKeys = (extra.configs || []).map((x) => x.key);
+          if (b.meta.did === blocklet.meta.did) {
+            configKeys = rootKeys;
+          } else {
+            const childExtra = (extra.children || []).find((x) => x.did === b.meta.did);
+            const childKeys = childExtra ? childExtra.configs.map((x) => x.key) : [];
+            configKeys = uniq([...rootKeys, ...childKeys]);
+          }
+
+          b.environments = cloneDeep(b.environments || [])
+            .filter((x) => configKeys.includes(x.key) === false)
+            .map((c) => {
+              if (c.key === 'BLOCKLET_APP_SK') {
+                c.value = security.encrypt(c.value, b.meta.did, fs.readFileSync(file));
+              }
+
+              return c;
+            });
+        },
+        { sync: true }
+      );
+
+      await states.blocklet.updateById(blocklet._id, {
+        $set: { environments: blocklet.environments, children: blocklet.children, migratedFrom: '1.6.5' },
+      });
+    }
+
+    states.blocklet.compactDatafile();
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
+};

package/lib/migrations/1.6.7-certificate.js

@@ -0,0 +1,30 @@
+module.exports = async ({ states, node, printInfo }) => {
+  printInfo('Migrate certificates...');
+  const certs = await states.certificate.find(
+    {},
+    {
+      domain: 1,
+      privateKey: 1,
+      certificate: 1,
+      createdAt: 1,
+      updatedAt: 1,
+    }
+  );
+
+  const tasks = certs.map((cert) => {
+    const data = {
+      _id: cert.id,
+      domain: cert.domain,
+      privateKey: cert.privateKey,
+      certificate: cert.certificate,
+      createdAt: cert.createdAt,
+      updatedAt: cert.updatedAt,
+      source: 'lets_encrypt',
+      status: 'generated',
+    };
+
+    return node.certManager.addWithoutValidations(data);
+  });
+
+  await Promise.all(tasks);
+};

package/lib/router/helper.js

@@ -21,15 +21,13 @@ const {
   NAME_FOR_WELLKNOWN_SITE,
   DEFAULT_HTTP_PORT,
   DEFAULT_HTTPS_PORT,
-  DAY_IN_MS,
   NODE_MODES,
   ROUTING_RULE_TYPES,
   CERTIFICATE_EXPIRES_OFFSET,
-  CERTIFICATE_EXPIRES_WARNING_OFFSET,
-  DEFAULT_DAEMON_PORT,
   DEFAULT_SERVICE_PATH,
   SLOT_FOR_IP_DNS_SITE,
   BLOCKLET_SITE_GROUP_SUFFIX,
+  WELLKNOWN_ACME_CHALLENGE_PREFIX,
 } = require('@abtnode/constant');
 const {
   BLOCKLET_DYNAMIC_PATH_PREFIX,
@@ -403,11 +401,10 @@ const decompressCertificates = async (source, dest) => {
   return dest;
 };
 
-module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerManager, blockletManager }) {
+module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerManager, blockletManager, certManager }) {
   const nodeState = states.node;
   const blockletState = states.blocklet;
   const siteState = states.site;
-  const httpsCertState = states.certificate;
   const notification = states.notification;
 
   // site level duplication detection
@@ -468,58 +465,105 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       const certificate = fs.readFileSync(certificateFilePath).toString();
       const privateKey = fs.readFileSync(privateKeyFilePath).toString();
 
-      await routerManager.updateNginxHttpsCert({ domain: dashboardDomain, privateKey, certificate, isProtected: true });
+      await certManager.upsertByDomain({
+        domain: dashboardDomain,
+        privateKey,
+        certificate,
+        isProtected: true,
+      });
       logger.info('dashboard certificate updated');
     } catch (error) {
-      logger.error('Update dashboard certificate failed', { error });
+      logger.error('update dashboard certificate failed', { error });
+      throw error;
     } finally {
       fs.removeSync(destFolder);
     }
   };
 
+  const ensureDashboardCertificate = async () => {
+    const info = await nodeState.read();
+    const downloadUrl = get(info, 'routing.dashboardCertDownloadAddress', '');
+    const dashboardDomain = get(info, 'routing.dashboardDomain', '');
+    if (!dashboardDomain || !downloadUrl) {
+      throw new Error('dashboardCertDownloadAddress and dashboardDomain are not found in the routing configs');
+    }
+
+    const cert = await certManager.getByDomain(dashboardDomain);
+    if (cert) {
+      return { status: 'existed' };
+    }
+
+    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    await updateDashboardCertificate({ checkExpire: false });
+    logger.debug('downloading certificate', { url: downloadUrl, dashboardDomain });
+    return { status: 'downloaded' };
+  };
+
   const addWellknownSite = async (sites, context) => {
     const site = (sites || []).find((x) => x.name === NAME_FOR_WELLKNOWN_SITE);
 
     try {
-      const pathPrefix = joinUrl(WELLKNOWN_PATH_PREFIX, '/did.json');
+      const info = await nodeState.read();
+      const proxyTarget = {
+        port: info.port,
+        type: ROUTING_RULE_TYPES.GENERAL_PROXY,
+        interfaceName: BLOCKLET_INTERFACE_WELLKNOWN,
+      };
+
       const didResolverWellknownRule = {
-        from: { pathPrefix },
-        to: {
-          port: DEFAULT_DAEMON_PORT,
-          type: ROUTING_RULE_TYPES.GENERAL_PROXY,
-          interfaceName: BLOCKLET_INTERFACE_WELLKNOWN,
-        },
+        from: { pathPrefix: joinUrl(WELLKNOWN_PATH_PREFIX, '/did.json') },
+        to: proxyTarget,
+      };
+
+      const acmeChallengeWellknownRule = {
+        from: { pathPrefix: WELLKNOWN_ACME_CHALLENGE_PREFIX },
+        to: proxyTarget,
       };
 
       if (site) {
-        if (site.rules.find((r) => r.from.pathPrefix === normalizePathPrefix(pathPrefix))) {
-          return false;
+        let changed = false;
+        const exists = (prefix) => !!site.rules.find((r) => r.from.pathPrefix === normalizePathPrefix(prefix));
+
+        if (!exists(didResolverWellknownRule.from.pathPrefix)) {
+          await routerManager.addRoutingRule(
+            {
+              id: site.id,
+              rule: didResolverWellknownRule,
+            },
+            context
+          );
+          changed = true;
         }
 
-        await routerManager.addRoutingRule(
-          {
-            id: site.id,
-            rule: didResolverWellknownRule,
-          },
-          context
-        );
-      } else {
-        await routerManager.addRoutingSite(
-          {
-            site: {
-              domain: DOMAIN_FOR_INTERNAL_SITE,
-              port: await getWellknownSitePort(),
-              name: NAME_FOR_WELLKNOWN_SITE,
-              rules: [didResolverWellknownRule],
-              isProtected: true,
+        if (!exists(acmeChallengeWellknownRule.from.pathPrefix)) {
+          await routerManager.addRoutingRule(
+            {
+              id: site.id,
+              rule: acmeChallengeWellknownRule,
             },
-            skipCheckDynamicBlacklist: true,
-            skipValidation: true,
-          },
-          context
-        );
+            context
+          );
+          changed = true;
+        }
+
+        return changed;
       }
 
+      await routerManager.addRoutingSite(
+        {
+          site: {
+            domain: DOMAIN_FOR_INTERNAL_SITE,
+            port: await getWellknownSitePort(),
+            name: NAME_FOR_WELLKNOWN_SITE,
+            rules: [didResolverWellknownRule, acmeChallengeWellknownRule],
+            isProtected: true,
+          },
+          skipCheckDynamicBlacklist: true,
+          skipValidation: true,
+        },
+        context
+      );
+
       return true;
     } catch (err) {
       console.error('add well-known site failed:', err);
@@ -533,7 +577,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
    *
    * @returns {boolean} if routing changed
    */
-  const ensureDashboardRouting = async (context = {}, output) => {
+  const ensureDashboardRouting = async (context = {}) => {
     const info = await nodeState.read();
 
     const provider = getProviderFromNodeInfo(info);
@@ -618,18 +662,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       updatedResult.push(wellknownRes);
     }
 
-    // Download dashboard certificates if not exists
-    const certDownloadAddress = get(info, 'routing.dashboardCertDownloadAddress', '');
-    if (dashboardDomain && certDownloadAddress) {
-      const cert = await routerManager.findCertificateByDomain(dashboardDomain);
-      if (!cert) {
-        await updateDashboardCertificate({ checkExpire: false });
-        if (typeof output === 'function') {
-          output('Dashboard HTTPS certificate was downloaded successfully!');
-        }
-      }
-    }
-
     if (updatedResult.length) {
       const hash = await takeRoutingSnapshot({ message: 'ensure dashboard routing rules', dryRun: false }, context);
       logger.info('take routing snapshot on ensure dashboard routing rules', { updatedResult, hash });
@@ -1068,9 +1100,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
             sites = await ensureAuthService(sites, blockletManager);
             sites = await ensureServiceRule(sites);
 
-            const certificates = httpsEnabled
-              ? await httpsCertState.find({ type: providerName }, { domain: 1, certificate: 1, privateKey: 1 })
-              : [];
+            const certificates = httpsEnabled ? await certManager.getAllNormal() : [];
 
             return {
               sites,
@@ -1093,9 +1123,9 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
         },
       });
 
-      routerManager.on('router.certificate.add', () => routers[providerName].restart());
-      routerManager.on('router.certificate.updated', () => routers[providerName].restart());
-      routerManager.on('router.certificate.remove', () => routers[providerName].restart());
+      certManager.on('cert.added', () => routers[providerName].restart());
+      certManager.on('cert.removed', () => routers[providerName].restart());
+      certManager.on('cert.issued', () => routers[providerName].restart());
 
       await routers[providerName].start();
     }
@@ -1257,18 +1287,21 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
   };
 
   const getCertificates = async () => {
-    const certificates = await httpsCertState.find();
+    const certificates = await certManager.getAll();
     const sites = await getSitesFromSnapshot();
+
+    const isMatch = (cert, domain) =>
+      domain !== DOMAIN_FOR_DEFAULT_SITE && domain && routerManager.isCertMatchedDomain(cert, domain);
+
     return certificates.map((cert) => {
       cert.matchedSites = [];
       sites.forEach((site) => {
-        if (
-          site.domain !== DOMAIN_FOR_DEFAULT_SITE &&
-          site.domain &&
-          routerManager.isCertMatchedDomain(cert, site.domain)
-        ) {
-          cert.matchedSites.push({ id: site.id, domain: site.domain });
-        }
+        const domains = [site.domain, ...(site.domainAliases || []).map((x) => x.value)];
+        domains.forEach((domain) => {
+          if (isMatch(cert, domain)) {
+            cert.matchedSites.push({ id: site.id, domain });
+          }
+        });
       });
 
       return cert;
@@ -1281,42 +1314,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     return { domain, isHttps: !!matchedCert, matchedCert };
   };
 
-  const checkCertificatesExpiration = async () => {
-    const now = Date.now();
-
-    const certificates = await getCertificates();
-    for (let i = 0; i < certificates.length; i++) {
-      const cert = certificates[i];
-      const alreadyExpired = now >= cert.validTo;
-      const aboutToExpire = cert.validTo - now > 0 && cert.validTo - now < CERTIFICATE_EXPIRES_WARNING_OFFSET;
-
-      if (alreadyExpired) {
-        logger.info('send certificate expire notification', { domain: cert.domain });
-        notification.create({
-          title: 'SSL Certificate Expired',
-          description: `Your SSL certificate for domain ${cert.domain} has expired, please update it in Blocklet Server`,
-          severity: 'error',
-          entityType: 'certificate',
-          entityId: cert._id, // eslint-disable-line no-underscore-dangle
-        });
-      } else if (aboutToExpire) {
-        logger.info('send certificate about-expire notification', { domain: cert.domain });
-        const expireInDays = Math.ceil((cert.validTo - now) / DAY_IN_MS);
-        notification.create({
-          title: 'SSL Certificate Expire Warning',
-          description: `Your SSL certificate for domain ${
-            cert.domain
-          } will expire in ${expireInDays} days (on ${new Date(
-            cert.validTo
-          ).toLocaleString()}), please remember to update it in Blocklet Server`,
-          severity: 'warning',
-          entityType: 'certificate',
-          entityId: cert._id, // eslint-disable-line no-underscore-dangle
-        });
-      }
-    }
-  };
-
   return {
     ensureDashboardRouting,
     ensureBlockletRouting,
@@ -1328,9 +1325,10 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
     takeRoutingSnapshot,
     getRoutingSites,
     getSnapshotSites,
-    getCertificates,
     getSitesFromSnapshot,
+    getCertificates,
     checkDomain,
+    ensureDashboardCertificate,
 
     getRoutingCrons: () => [
       {
@@ -1339,12 +1337,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
         fn: () => updateDashboardCertificate({ checkExpire: true }),
         options: { runOnInit: true },
       },
-      {
-        name: 'check-certificate-expiration',
-        time: '0 0 9 * * *', // check on 09:00 every day
-        fn: checkCertificatesExpiration,
-        options: { runOnInit: false },
-      },
       {
         name: 'rotate-log-files',
         time: '5 0 0 * * *', // rotate at 05:00 every day