@abtnode/router-provider 1.6.30 → 1.7.1

package/lib/default/daemon.js

@@ -1,14 +1,11 @@
+/* eslint-disable no-continue */
 const url = require('url');
 const path = require('path');
 const fs = require('fs-extra');
 const get = require('lodash/get');
 const joinUrl = require('url-join');
 const checkDomainMatch = require('@abtnode/util/lib/check-domain-match');
-const {
-  DEFAULT_IP_DNS_DOMAIN_SUFFIX,
-  ROUTING_RULE_TYPES,
-  WELLKNOWN_SERVICE_PATH_PREFIX,
-} = require('@abtnode/constant');
+const { DEFAULT_IP_DOMAIN_SUFFIX, ROUTING_RULE_TYPES, WELLKNOWN_SERVICE_PATH_PREFIX } = require('@abtnode/constant');
 
 const logger = require('@abtnode/logger')('router:default:daemon', { filename: 'engine' });
 
@@ -65,40 +62,39 @@ const createRequestHandler = (id) => (req, res, target) => {
     }
   }
 
-  // FIXME logic related to server gateway should not in provider
-  const rewritePathPrefix = rule.prefix.replace(WELLKNOWN_SERVICE_PATH_PREFIX, '') || '/';
-
   if (rule.type === ROUTING_RULE_TYPES.GENERAL_PROXY) {
-    // we do not need rewrite for internal servers
+    // do not rewrite for internal servers
+  } else if (rule.prefix.includes(WELLKNOWN_SERVICE_PATH_PREFIX)) {
+    // do not rewrite for service requests
   } else {
-    req.url = req.url.substr(rewritePathPrefix.length);
+    req.url = joinUrl(rule.target, req.url.substr(rule.prefix.length));
   }
-
   if (req.url.startsWith('/') === false) {
     req.url = `/${req.url}`;
   }
   logger.debug('transform url', { url: req.url, rule });
 
+  // NOTE: x-* header keys must be lower case to avoid collisions from chained proxy
   if (rule.did) {
-    req.headers['X-Blocklet-Did'] = rule.did;
+    req.headers['x-blocklet-did'] = rule.did;
   }
   if (rule.realDid && rule.did !== rule.realDid) {
-    req.headers['X-Blocklet-Real-Did'] = rule.realDid;
+    req.headers['x-blocklet-real-did'] = rule.realDid;
   }
 
-  req.headers['X-Path-Prefix'] = rewritePathPrefix;
-  req.headers['X-Group-Path-Prefix'] = rule.groupPrefix || '/';
+  req.headers['x-path-prefix'] = rule.prefix.replace(WELLKNOWN_SERVICE_PATH_PREFIX, '') || '/';
+  req.headers['x-group-path-prefix'] = rule.groupPrefix || '/';
 
   if (rule.type === ROUTING_RULE_TYPES.BLOCKLET) {
-    req.headers['X-Blocklet-Url'] = `http://127.0.0.1:${rule.port}`;
+    req.headers['x-blocklet-url'] = `http://127.0.0.1:${rule.port}`;
   }
 
   if (rule.ruleId) {
-    req.headers['X-Routing-Rule-Id'] = rule.ruleId;
+    req.headers['x-routing-rule-id'] = rule.ruleId;
   }
 
   if (rule.target && rule.target !== '/') {
-    req.headers['X-Routing-Rule-Path-Prefix'] = rule.target;
+    req.headers['x-routing-rule-path-prefix'] = rule.target;
   }
 };
 
@@ -249,18 +245,23 @@ const updateConfig = (reload = false) => {
 
     const addedDomains = newDomains.filter((x) => !oldDomains.includes(x));
     for (const domain of addedDomains) {
-      if (isSpecificDomain(domain)) {
-        logger.info('register domain on reload', { domain });
-        const site = sites.find((x) => x.domain === domain);
-        const [rule] = site.rules.filter((x) => ['daemon', 'blocklet'].includes(x.type));
-        const match = findCertificate(config.certs, site.domain);
-        main.register({
-          src: site.domain,
-          target: `http://127.0.0.1:${rule.port}`,
-          ssl: match ? { key: match.privateKey, cert: match.certificate } : null,
-          onRequest: createRequestHandler(rule.id),
-        });
+      if (isSpecificDomain(domain) === false) {
+        continue;
+      }
+      const site = sites.find((x) => x.domain === domain);
+      const [rule] = site.rules.filter((x) => ['daemon', 'blocklet'].includes(x.type));
+      if (!rule) {
+        continue;
       }
+
+      logger.info('register domain on reload', { domain });
+      const match = findCertificate(config.certs, site.domain);
+      main.register({
+        src: site.domain,
+        target: `http://127.0.0.1:${rule.port}`,
+        ssl: match ? { key: match.privateKey, cert: match.certificate } : null,
+        onRequest: createRequestHandler(rule.id),
+      });
     }
 
     const removedDomains = oldDomains.filter((x) => !newDomains.includes(x));
@@ -281,7 +282,7 @@ const updateConfig = (reload = false) => {
 updateConfig();
 
 // create main server
-const defaultCert = config.certs.find((x) => x.domain.endsWith(DEFAULT_IP_DNS_DOMAIN_SUFFIX));
+const defaultCert = config.certs.find((x) => x.domain.endsWith(DEFAULT_IP_DOMAIN_SUFFIX));
 const main = new ProxyServer({
   xfwd: true,
   onError,
@@ -310,8 +311,11 @@ config.sites.forEach((site) => {
   }
 
   const [rule] = site.rules.filter((x) => ['daemon', 'blocklet'].includes(x.type));
-  const match = findCertificate(config.certs, domain);
+  if (!rule) {
+    return;
+  }
 
+  const match = findCertificate(config.certs, domain);
   logger.info('register domain on start', { domain });
   main.register({
     src: domain,

package/lib/default/index.js

@@ -1,5 +1,6 @@
 const fs = require('fs-extra');
 const path = require('path');
+const http = require('http');
 const shelljs = require('shelljs');
 const get = require('lodash/get');
 const omit = require('lodash/omit');
@@ -15,6 +16,7 @@ const BaseProvider = require('../base');
 const {
   decideHttpPort,
   decideHttpsPort,
+  getUsablePorts,
   get404Template,
   get502Template,
   get5xxTemplate,
@@ -70,6 +72,10 @@ class DefaultProvider extends BaseProvider {
   }
 
   async reload() {
+    if (process.env.NODE_ENV === 'development') {
+      return this.start();
+    }
+
     await pm2.connectAsync();
 
     // ensure daemon is live
@@ -218,4 +224,27 @@ DefaultProvider.check = async ({ configDir = '' } = {}) => {
   return result;
 };
 
+DefaultProvider.getUsablePorts = async () =>
+  getUsablePorts(
+    'default',
+    (port) =>
+      new Promise((resolve) => {
+        try {
+          const server = http.createServer();
+
+          server.once('error', () => {
+            server.close();
+            resolve(false);
+          });
+
+          server.listen(port, (err) => {
+            server.close();
+            resolve(!err);
+          });
+        } catch (err) {
+          resolve(false);
+        }
+      })
+  );
+
 module.exports = DefaultProvider;

package/lib/default/proxy.js

@@ -93,6 +93,7 @@ module.exports = class ReverseProxy {
 
     // @link: https://github.com/http-party/node-http-proxy/issues/1401
     this.proxy.on('proxyRes', (proxyRes) => {
+      delete proxyRes.headers['x-powered-by'];
       this.opts.headers.forEach((x) => {
         proxyRes.headers[x.key] = x.value;
       });

package/lib/nginx/index.js

@@ -38,6 +38,7 @@ const {
 const {
   decideHttpPort,
   decideHttpsPort,
+  getUsablePorts,
   get404Template,
   get502Template,
   get5xxTemplate,
@@ -781,7 +782,6 @@ NginxProvider.describe = async ({ configDir = '' } = {}) => {
  * @param {string} param.configDir nginx config directory
  */
 NginxProvider.check = async ({ configDir = '' } = {}) => {
-  logger.info('check nginx provider');
   logger.info('check formal config directory', { configDir });
   const binPath = shelljs.which('nginx');
   const result = {
@@ -823,24 +823,24 @@ NginxProvider.check = async ({ configDir = '' } = {}) => {
     }
   }
 
-  const tempConfigDirectory = path.join(
+  const tmpDir = path.join(
     os.tmpdir(),
-    `test_abtnode_nginx_provider-${Date.now()}-${Math.ceil(Math.random() * 100)}`,
+    `test_nginx_provider-${Date.now()}-${Math.ceil(Math.random() * 10000)}`,
     CONFIG_FOLDER_NAME
   );
-  fs.mkdirSync(tempConfigDirectory, { recursive: true });
+  fs.mkdirSync(tmpDir, { recursive: true });
 
-  const nginxProvider = new NginxProvider({ configDir: tempConfigDirectory, isTest: true });
-  nginxProvider.initialize();
+  const provider = new NginxProvider({ configDir: tmpDir, isTest: true });
+  provider.initialize();
 
-  logger.info('check:addTestServer', { configPath: nginxProvider.configPath });
+  logger.info('check:addTestServer', { configPath: provider.configPath });
   await addTestServer({
-    configPath: nginxProvider.configPath,
+    configPath: provider.configPath,
     port: await getPort(),
     upstreamPort: await getPort(),
   });
 
-  const missingModules = getMissingModules(nginxProvider.readNginxConfigParams());
+  const missingModules = getMissingModules(provider.readNginxConfigParams());
 
   if (missingModules.length > 0) {
     result.available = false;
@@ -850,9 +850,10 @@ NginxProvider.check = async ({ configDir = '' } = {}) => {
     return result;
   }
 
-  await nginxProvider.start();
+  await provider.start();
+  await provider.stop();
 
-  await nginxProvider.stop();
+  fs.rmdirSync(tmpDir, { recursive: true });
 
   return result;
 };
@@ -860,4 +861,29 @@ NginxProvider.check = async ({ configDir = '' } = {}) => {
 NginxProvider.getStatus = getNginxStatus;
 NginxProvider.exists = () => !!shelljs.which('nginx');
 
+NginxProvider.getUsablePorts = async () =>
+  getUsablePorts('nginx', async (port) => {
+    try {
+      const configDir = path.join(
+        os.tmpdir(),
+        `test_nginx_provider-${Date.now()}-${Math.ceil(Math.random() * 10000)}`,
+        CONFIG_FOLDER_NAME
+      );
+      fs.mkdirSync(configDir, { recursive: true });
+
+      const provider = new NginxProvider({ configDir, isTest: true });
+      provider.initialize();
+
+      await addTestServer({ configPath: provider.configPath, port });
+      await provider.start();
+      await provider.stop();
+
+      fs.rmdirSync(configDir, { recursive: true });
+
+      return true;
+    } catch (err) {
+      return false;
+    }
+  });
+
 module.exports = NginxProvider;

package/lib/nginx/util.js

@@ -57,20 +57,22 @@ const addTestServer = ({ configPath, port, upstreamPort }) =>
 
         location / {
           if ($uri = /admin/did-connect) {include includes/cors-strict; include includes/security;}
-          return 200 'Hello AbtNode!';
+          return 200 'Hello Blocklet Server!';
         }
     `
       );
 
-      conf.nginx._addVerbatimBlock(
-        'stream',
-        `
+      if (upstreamPort) {
+        conf.nginx._addVerbatimBlock(
+          'stream',
+          `
         server {
             listen ${upstreamPort} udp;
             proxy_pass 127.0.0.1:${port};
         }
     `
-      );
+        );
+      }
 
       conf.on('flushed', () => resolve());
       conf.live(configPath);

package/lib/util.js

@@ -1,3 +1,8 @@
+/* eslint-disable no-await-in-loop */
+const fs = require('fs-extra');
+const path = require('path');
+const getPort = require('get-port');
+const portUsed = require('port-used');
 const sortBy = require('lodash/sortBy');
 const isValidDomain = require('is-valid-domain');
 const checkDomainMatch = require('@abtnode/util/lib/check-domain-match');
@@ -8,7 +13,7 @@ const {
   IP,
   DEFAULT_HTTP_PORT,
   DEFAULT_HTTPS_PORT,
-  DEFAULT_IP_DNS_DOMAIN_SUFFIX,
+  DEFAULT_IP_DOMAIN_SUFFIX,
   DOMAIN_FOR_DEFAULT_SITE,
   ROUTING_RULE_TYPES,
   SLOT_FOR_IP_DNS_SITE,
@@ -110,7 +115,7 @@ const isSpecificDomain = (domain) => {
     return false;
   }
 
-  if (domain.endsWith(DEFAULT_IP_DNS_DOMAIN_SUFFIX)) {
+  if (domain.endsWith(DEFAULT_IP_DOMAIN_SUFFIX)) {
     return false;
   }
 
@@ -118,7 +123,7 @@ const isSpecificDomain = (domain) => {
 };
 
 const toSlotDomain = (domain) => {
-  if (domain.endsWith(DEFAULT_IP_DNS_DOMAIN_SUFFIX)) {
+  if (domain.endsWith(DEFAULT_IP_DOMAIN_SUFFIX)) {
     const subDomain = domain.split('.').shift();
     const matches = subDomain.match(IP);
     if (matches) {
@@ -148,9 +153,45 @@ const matchRule = (rules, url) => {
   return findRule(rulesWithSuffix, url) || findRule(rulesWithoutSuffix, url);
 };
 
+const isPortOccupied = (port) => portUsed.check(port);
+const getUsablePort = async (preferredPorts, hasPortPermission) => {
+  for (const port of preferredPorts) {
+    const occupied = await isPortOccupied(port);
+    if (!occupied) {
+      const listenable = await hasPortPermission(port);
+      if (listenable) {
+        return port;
+      }
+    }
+  }
+
+  return getPort();
+};
+const getUsablePorts = async (provider, hasPortPermission) => {
+  const file = path.join(process.env.PM2_HOME, `${provider}-preferred-ports.json`);
+  if (fs.existsSync(file)) {
+    const config = fs.readJsonSync(file);
+    if (config.httpPort && config.httpsPort) {
+      return config;
+    }
+  }
+
+  const [httpPort, httpsPort] = await Promise.all([
+    getUsablePort([80, 8080], hasPortPermission),
+    getUsablePort([443, 8443], hasPortPermission),
+  ]);
+
+  const config = { httpPort, httpsPort };
+  fs.writeJsonSync(file, config);
+  return config;
+};
+
 module.exports = {
   decideHttpPort,
   decideHttpsPort,
+  getUsablePorts,
+  getUsablePort,
+  isPortOccupied,
   findCertificate,
   concatPath,
   trimEndSlash,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@abtnode/router-provider",
-  "version": "1.6.30",
+  "version": "1.7.1",
   "description": "Routing engine implementations for abt node",
   "author": "polunzh <polunzh@gmail.com>",
   "homepage": "https://github.com/ArcBlock/blocklet-server#readme",
@@ -32,10 +32,10 @@
     "url": "https://github.com/ArcBlock/blocklet-server/issues"
   },
   "dependencies": {
-    "@abtnode/constant": "1.6.30",
-    "@abtnode/logger": "1.6.30",
-    "@abtnode/router-templates": "1.6.30",
-    "@abtnode/util": "1.6.30",
+    "@abtnode/constant": "1.7.1",
+    "@abtnode/logger": "1.7.1",
+    "@abtnode/router-templates": "1.7.1",
+    "@abtnode/util": "1.7.1",
     "axios": "^0.25.0",
     "debug": "^4.3.3",
     "find-process": "^1.4.3",
@@ -49,6 +49,7 @@
     "moment": "^2.29.1",
     "nginx-conf": "^1.5.0",
     "object-hash": "^3.0.0",
+    "port-used": "^2.0.8",
     "promise-retry": "^2.0.1",
     "shelljs": "^0.8.4",
     "tar": "^6.1.0",
@@ -61,5 +62,5 @@
     "fs-extra": "^10.0.0",
     "needle": "^3.0.0"
   },
-  "gitHead": "d8ea64b807f7852186cd4bf544193496fbb31e7c"
+  "gitHead": "5d7efb21dd09f90206251fb74601ca37b0ef84bf"
 }