@abtnode/router-provider 1.15.17 → 1.16.0-beta-b16cb035

package/lib/nginx/util.js

@@ -8,13 +8,16 @@ const { NginxConfFile } = require('nginx-conf');
 const shelljs = require('shelljs');
 const findProcess = require('find-process');
 const moment = require('moment');
-const { MAX_UPLOAD_FILE_SIZE, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT } = require('@abtnode/constant');
-const checkDomainMatch = require('@abtnode/util/lib/check-domain-match');
-const { template404, template502, template5xx, templateWelcome } = require('@abtnode/router-templates');
-const logger = require('@abtnode/logger')(`${require('../../package.json').name}:provider:nginx:util`);
+const { MAX_UPLOAD_FILE_SIZE, MAX_NGINX_WORKER_CONNECTIONS, ROUTER_CACHE_GROUPS } = require('@abtnode/constant');
+const formatBackSlash = require('@abtnode/util/lib/format-back-slash');
+const { deleteOldLogfiles } = require('@abtnode/logger');
 
+const logger = require('@abtnode/logger')('router:nginx:util');
+
+const MAX_WORKER_CONNECTIONS = process.env.MAX_NGINX_WORKER_CONNECTIONS
+  ? Number(process.env.MAX_NGINX_WORKER_CONNECTIONS)
+  : MAX_NGINX_WORKER_CONNECTIONS;
 const CLIENT_MAX_BODY_SIZE = process.env.MAX_UPLOAD_FILE_SIZE || MAX_UPLOAD_FILE_SIZE;
-const WORKER_CONNECTIONS = 4096;
 
 const formatError = (errStr) => {
   if (!errStr) {
@@ -43,10 +46,10 @@ const formatError = (errStr) => {
   ).join(';');
 };
 
-const addTestServer = ({ configFilePath, port, upstreamPort }) =>
+const addTestServer = ({ configPath, port, upstreamPort }) =>
   new Promise((resolve, reject) => {
     // eslint-disable-next-line consistent-return
-    NginxConfFile.create(configFilePath, async (err, conf) => {
+    NginxConfFile.create(configPath, async (err, conf) => {
       if (err) {
         return reject(err);
       }
@@ -58,23 +61,27 @@ const addTestServer = ({ configFilePath, port, upstreamPort }) =>
 
         location / {
           if ($uri = /admin/did-connect) {include includes/cors-strict; include includes/security;}
-          return 200 'Hello AbtNode!';
+          return 200 'Hello Blocklet Server!';
         }
     `
       );
 
-      conf.nginx._addVerbatimBlock(
-        'stream',
-        `
+      if (upstreamPort && os.platform() !== 'win32') {
+        conf.nginx._addVerbatimBlock(
+          'stream',
+          `
         server {
             listen ${upstreamPort} udp;
+            proxy_responses 1;
+            proxy_timeout 1s;
             proxy_pass 127.0.0.1:${port};
         }
     `
-      );
+        );
+      }
 
       conf.on('flushed', () => resolve());
-      conf.live(configFilePath);
+      conf.live(configPath);
       conf.flush();
     });
   });
@@ -129,9 +136,21 @@ const getUserGroup = (username) => {
   return res.stdout.trim();
 };
 
+const getWorkerConnectionCount = (maxWorkerConnections, workerProcess) => {
+  const { stdout, code } = shelljs.exec('ulimit -n', { silent: true });
+  if (code !== 0) {
+    return maxWorkerConnections;
+  }
+
+  const ulimit = Number(stdout.trim());
+
+  return Math.min(maxWorkerConnections, Math.floor(ulimit / workerProcess));
+};
+
 const getMainTemplate = ({
   logDir,
-  tmpDirectory,
+  tmpDir,
+  cacheDir,
   nginxLoadModules = '',
   workerProcess,
   maxBodySize = CLIENT_MAX_BODY_SIZE,
@@ -144,37 +163,55 @@ user ${os.userInfo().username} ${getUserGroup(os.userInfo().username)};
 pid nginx.pid;
 
 events {
-    worker_connections  ${WORKER_CONNECTIONS};
+    worker_connections  ${getWorkerConnectionCount(MAX_WORKER_CONNECTIONS, workerProcess)};
+    multi_accept on;
 }
 
 http {
     map $http_upgrade $connection_upgrade {
       default upgrade;
-      '' close;
+      '' "";
     }
-    client_body_temp_path ${path.join(tmpDirectory, 'client_body')};
-    proxy_temp_path ${path.join(tmpDirectory, 'proxy')};
-    fastcgi_temp_path ${path.join(tmpDirectory, 'fastcgi')};
-    uwsgi_temp_path ${path.join(tmpDirectory, 'uwsgi')};
-    scgi_temp_path ${path.join(tmpDirectory, 'scgi')};
+    client_body_temp_path ${path.join(tmpDir, 'client_body')};
+    proxy_temp_path ${path.join(tmpDir, 'proxy')};
+    fastcgi_temp_path ${path.join(tmpDir, 'fastcgi')};
+    uwsgi_temp_path ${path.join(tmpDir, 'uwsgi')};
+    scgi_temp_path ${path.join(tmpDir, 'scgi')};
     client_max_body_size ${maxBodySize}m;
+    variables_hash_max_size 2048;
+    variables_hash_bucket_size 128;
+
+    absolute_redirect off;
 
     log_format compression '$remote_addr - $remote_user [$time_local] '
-                          '"$request" $status $body_bytes_sent '
-                          '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+                          '"$host" "$request" $status $body_bytes_sent '
+                          '"$http_referer" "$http_user_agent" "$http_x_forwarded_for" $request_time';
     access_log ${path.join(logDir, 'access.log')} compression;
     error_log ${path.join(logDir, 'error.log')} error;
 
     include includes/params;
     include includes/mime.types;
     include includes/compression;
+
+    ${Object.keys(ROUTER_CACHE_GROUPS)
+      .map((x) => getCachePathDirective(cacheDir, x))
+      .join('\n')}
 }
 `;
 
+const getCachePathDirective = (cacheDir, group) => {
+  if (!ROUTER_CACHE_GROUPS[group]) {
+    return '';
+  }
+
+  const config = ROUTER_CACHE_GROUPS[group];
+  return `proxy_cache_path ${cacheDir}/${group} levels=1:2 keys_zone=${group}:${config.minSize} inactive=${config.period} max_size=${config.maxSize};`;
+};
+
 const getNginxStatus = async (configPath) => {
   const list = await findProcess('name', /nginx:/);
   const result = {
-    ownByABTNode: false,
+    managed: false,
     pid: 0,
     running: false,
   };
@@ -184,7 +221,7 @@ const getNginxStatus = async (configPath) => {
     result.running = true;
 
     if (abtnodeNginxProcess) {
-      result.ownByABTNode = true;
+      result.managed = true;
       result.pid = abtnodeNginxProcess.pid;
     }
 
@@ -194,37 +231,6 @@ const getNginxStatus = async (configPath) => {
   return result;
 };
 
-const decideHttpPort = (port) => port || process.env.ABT_NODE_HTTP_PORT || DEFAULT_HTTP_PORT;
-const decideHttpsPort = (port) => port || process.env.ABT_NODE_HTTPS_PORT || DEFAULT_HTTPS_PORT;
-
-const findCertificate = (certs, domain) => {
-  const results = certs
-    .filter((cert) => [cert.domain, ...(cert.sans || [])].some((d) => checkDomainMatch(d, domain)))
-    .sort((d1) => (d1.domain[0] === '*' ? 1 : -1)); // will first match a.b.com, then *.b.com
-  return results[0];
-};
-
-/**
- * trim tail slash
- * @param {string} str
- * @returns {string}
- */
-const trimEndSlash = (str) => {
-  if (str && str.length > 1 && str[str.length - 1] === '/') {
-    return str.substr(0, str.length - 1);
-  }
-
-  return str;
-};
-
-const concatPath = (prefix = '', suffix = '') => {
-  if (prefix === '/') {
-    return suffix ? `~* .*${suffix}$` : '/';
-  }
-
-  return suffix ? `~* ^${prefix}.*${suffix}$` : prefix;
-};
-
 const compressFile = ({ sourceFile, targetFile, options = {} }) => {
   if (fs.existsSync(targetFile)) {
     fs.unlinkSync(targetFile);
@@ -242,7 +248,7 @@ const compressFile = ({ sourceFile, targetFile, options = {} }) => {
  * rotate log files: access file and access log
  * refer: https://www.nginx.com/resources/wiki/start/topics/examples/logrotation/
  */
-const rotateNginxLogFile = async ({ file, cwd, nginxPid }) => {
+const rotateNginxLogFile = async ({ file, cwd, nginxPid, retain }) => {
   const dateStr = moment().subtract(1, 'days').format('YYYY-MM-DD');
   const uncompressedTargetLogfile = path.join(cwd, `${path.basename(file, '.log')}-${dateStr}.log`);
   const targetLogfile = path.join(cwd, `${path.basename(file, '.log')}-${dateStr}.log.gz`);
@@ -271,6 +277,8 @@ const rotateNginxLogFile = async ({ file, cwd, nginxPid }) => {
       }
     }
   }
+
+  deleteOldLogfiles(file, retain);
 };
 
 const getMissingModules = (configParams) => {
@@ -290,23 +298,25 @@ const getMissingModules = (configParams) => {
   return missingModules;
 };
 
+const getUpstreamName = (port) => `server_${port}`;
+
+const joinNginxPath = (...args) => {
+  const result = path.join(...args);
+  return formatBackSlash(result);
+};
+
 module.exports = {
   addTestServer,
+  formatError,
   getMainTemplate,
   getNginxLoadModuleDirectives,
-  formatError,
   getMissingModules,
   parseNginxConfigArgs,
-  decideHttpPort,
-  decideHttpsPort,
-  concatPath,
-  trimEndSlash,
-  findCertificate,
   getNginxStatus,
   rotateNginxLogFile,
   getUserGroup,
-  get404Template: (nodeInfo) => template404(nodeInfo),
-  get502Template: (nodeInfo) => template502(nodeInfo),
-  get5xxTemplate: (nodeInfo) => template5xx(nodeInfo),
-  getWelcomeTemplate: (nodeInfo) => templateWelcome(nodeInfo),
+  getWorkerConnectionCount,
+  getCachePathDirective,
+  getUpstreamName,
+  joinNginxPath,
 };

package/lib/util.js

@@ -0,0 +1,214 @@
+/* eslint-disable no-await-in-loop */
+const fs = require('fs-extra');
+const path = require('path');
+const getPort = require('get-port');
+const portUsed = require('port-used');
+const sortBy = require('lodash/sortBy');
+const isValidDomain = require('is-valid-domain');
+const checkDomainMatch = require('@abtnode/util/lib/check-domain-match');
+const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+const { template404, template502, template5xx, templateWelcome } = require('@abtnode/router-templates');
+
+const {
+  IP,
+  DEFAULT_HTTP_PORT,
+  DEFAULT_HTTPS_PORT,
+  DEFAULT_IP_DOMAIN_SUFFIX,
+  DOMAIN_FOR_DEFAULT_SITE,
+  ROUTING_RULE_TYPES,
+  SLOT_FOR_IP_DNS_SITE,
+} = require('@abtnode/constant');
+
+const decideHttpPort = (port) => port || process.env.ABT_NODE_HTTP_PORT || DEFAULT_HTTP_PORT;
+const decideHttpsPort = (port) => port || process.env.ABT_NODE_HTTPS_PORT || DEFAULT_HTTPS_PORT;
+
+const findCertificate = (certs, domain) => {
+  const results = certs
+    .filter((cert) => [cert.domain, ...(cert.sans || [])].some((d) => checkDomainMatch(d, domain)))
+    .sort((d1) => (d1.domain[0] === '*' ? 1 : -1)); // will first match a.b.com, then *.b.com
+  return results[0];
+};
+
+const trimEndSlash = (str) => {
+  if (str && str.length > 1 && str[str.length - 1] === '/') {
+    return str.substr(0, str.length - 1);
+  }
+
+  return str;
+};
+
+const concatPath = (prefix = '', suffix = '') => {
+  if (prefix === '/') {
+    return suffix ? `~* .*${suffix}$` : '/';
+  }
+
+  return suffix ? `~* ^${prefix}.*${suffix}$` : prefix;
+};
+
+const formatRoutingTable = (routingTable, onRule) => {
+  const sites = {};
+  const configs = [];
+
+  routingTable.forEach((site) => {
+    let { domain } = site;
+    if (site.domain === DOMAIN_FOR_DEFAULT_SITE) {
+      domain = '_';
+    }
+
+    let corsAllowedOrigins = Array.isArray(site.corsAllowedOrigins) ? site.corsAllowedOrigins : [];
+    if (corsAllowedOrigins.includes('*')) {
+      corsAllowedOrigins = ['*'];
+    }
+    configs.push({ domain, corsAllowedOrigins });
+
+    if (!sites[domain]) {
+      sites[domain] = {
+        domain,
+        blockletDid: site.blockletDid,
+        rules: [],
+        port: site.port,
+        corsAllowedOrigins: site.corsAllowedOrigins,
+      };
+    }
+
+    (site.rules || []).forEach((x) => {
+      const prefix = trimEndSlash(x.from.pathPrefix || '/');
+      const groupPrefix = trimEndSlash(x.from.groupPathPrefix || '/');
+      const suffix = trimEndSlash(x.from.pathSuffix || '');
+
+      const rule = {
+        ruleId: x.id,
+        type: x.to.type,
+        prefix,
+        groupPrefix,
+        suffix,
+        response: x.to.response,
+      };
+      if (x.to.type === ROUTING_RULE_TYPES.REDIRECT) {
+        rule.redirectCode = x.to.redirectCode || 302;
+        rule.url = x.to.url;
+      } else {
+        rule.port = +x.to.port;
+        rule.did = x.to.did;
+        rule.cacheGroup = x.to.cacheGroup;
+        rule.componentId = x.to.componentId;
+        rule.target = trimEndSlash(normalizePathPrefix(x.to.target || '/'));
+        rule.targetPrefix = x.to.targetPrefix || '';
+        rule.services = x.services || [];
+      }
+
+      const tmpPath = concatPath(prefix, suffix);
+      const tmpRules = sites[domain].rules;
+      if (!tmpRules.find(({ prefix: p, suffix: s }) => concatPath(p, s) === tmpPath)) {
+        sites[domain].rules.push(rule);
+        if (typeof onRule === 'function') {
+          onRule(sites[domain].rules, rule);
+        }
+      }
+    });
+
+    const { rules } = sites[domain];
+    const rulesWithoutSuffix = sortBy(rules.filter((x) => !x.suffix), (x) => -x.prefix.length); // prettier-ignore
+    const rulesWithSuffix = sortBy(rules.filter((x) => x.suffix), (x) => -x.prefix.length); // prettier-ignore
+
+    sites[domain].rules = rulesWithoutSuffix.concat(rulesWithSuffix);
+  });
+
+  return { sites: Object.values(sites), configs };
+};
+/**
+ *
+ *
+ * @param {string} domain
+ * @return {boolean}
+ */
+const isSpecificDomain = (domain) => {
+  if (isValidDomain(domain) === false) {
+    return false;
+  }
+
+  return !domain.endsWith(DEFAULT_IP_DOMAIN_SUFFIX);
+};
+
+const toSlotDomain = (domain) => {
+  if (domain.endsWith(DEFAULT_IP_DOMAIN_SUFFIX)) {
+    const subDomain = domain.split('.').shift();
+    const matches = subDomain.match(IP);
+    if (matches) {
+      return domain.replace(matches[0].slice(1), SLOT_FOR_IP_DNS_SITE);
+    }
+  }
+
+  return domain;
+};
+
+const findRule = (rules, url) =>
+  rules.find((x) => {
+    const { prefix, suffix = '' } = x;
+    if (suffix) {
+      return url.startsWith(prefix) && url.endsWith(suffix);
+    }
+
+    if (prefix === '/' && prefix === url) {
+      return true;
+    }
+
+    return url.startsWith(prefix);
+  });
+const matchRule = (rules, url) => {
+  const rulesWithSuffix = rules.filter((x) => x.suffix);
+  const rulesWithoutSuffix = rules.filter((x) => !x.suffix);
+  return findRule(rulesWithSuffix, url) || findRule(rulesWithoutSuffix, url);
+};
+
+const isPortOccupied = (port) => portUsed.check(port);
+const getUsablePort = async (preferredPorts, hasPortPermission) => {
+  for (const port of preferredPorts) {
+    const occupied = await isPortOccupied(port);
+    if (!occupied) {
+      const listenable = await hasPortPermission(port);
+      if (listenable) {
+        return port;
+      }
+    }
+  }
+
+  return getPort();
+};
+const getUsablePorts = async (provider, hasPortPermission) => {
+  const file = path.join(process.env.PM2_HOME, `${provider}-preferred-ports.json`);
+  if (fs.existsSync(file)) {
+    const config = fs.readJsonSync(file);
+    if (config.httpPort && config.httpsPort) {
+      return config;
+    }
+  }
+
+  const [httpPort, httpsPort] = await Promise.all([
+    getUsablePort([80, 8080], hasPortPermission),
+    getUsablePort([443, 8443], hasPortPermission),
+  ]);
+
+  const config = { httpPort, httpsPort };
+  fs.writeJsonSync(file, config);
+  return config;
+};
+
+module.exports = {
+  decideHttpPort,
+  decideHttpsPort,
+  getUsablePorts,
+  getUsablePort,
+  isPortOccupied,
+  findCertificate,
+  concatPath,
+  trimEndSlash,
+  formatRoutingTable,
+  isSpecificDomain,
+  toSlotDomain,
+  matchRule,
+  get404Template: (info) => template404(info),
+  get502Template: (info) => template502(info),
+  get5xxTemplate: (info) => template5xx(info),
+  getWelcomeTemplate: (info) => templateWelcome(info),
+};

package/lib/www/502.html

@@ -22,7 +22,7 @@
       <h2>Bad gateway: Upstream Blocklet or Daemon service is not available!</h2>
       <div>
         Checkout Blocklet Server Documentation:
-        <a href="https://docs.arcblock.io/en/abtnode/">https://docs.arcblock.io/en/abtnode/</a>
+        <a href="https://developer.blocklet.io/docs">https://developer.blocklet.io/docs</a>
       </div>
     </center>
   </body>

package/lib/www/5xx.html

@@ -22,7 +22,7 @@
       <h2>Something went wrong</h2>
       <div>
         Checkout Blocklet Server Documentation:
-        <a href="https://docs.arcblock.io/en/abtnode/">https://docs.arcblock.io/en/abtnode/</a>
+        <a href="https://developer.blocklet.io/docs">https://developer.blocklet.io/docs</a>
       </div>
     </center>
   </body>

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@abtnode/router-provider",
-  "version": "1.15.17",
+  "version": "1.16.0-beta-b16cb035",
   "description": "Routing engine implementations for abt node",
   "author": "polunzh <polunzh@gmail.com>",
-  "homepage": "https://github.com/ArcBlock/abt-node#readme",
+  "homepage": "https://github.com/ArcBlock/blocklet-server#readme",
   "license": "Apache-2.0",
   "main": "lib/index.js",
   "publishConfig": {
@@ -18,7 +18,7 @@
   ],
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/ArcBlock/abt-node.git"
+    "url": "git+https://github.com/ArcBlock/blocklet-server.git"
   },
   "scripts": {
     "lint": "eslint tests lib",
@@ -29,28 +29,38 @@
     "coverage": "npm run test -- --coverage"
   },
   "bugs": {
-    "url": "https://github.com/ArcBlock/abt-node/issues"
+    "url": "https://github.com/ArcBlock/blocklet-server/issues"
   },
   "dependencies": {
-    "@abtnode/constant": "1.15.17",
-    "@abtnode/logger": "1.15.17",
-    "@abtnode/router-templates": "1.15.17",
-    "@abtnode/util": "1.15.17",
-    "axios": "^0.21.4",
-    "debug": "^4.3.2",
-    "find-process": "^1.4.3",
-    "fkill": "^7.0.1",
-    "fs-extra": "^10.0.0",
+    "@abtnode/constant": "1.16.0-beta-b16cb035",
+    "@abtnode/logger": "1.16.0-beta-b16cb035",
+    "@abtnode/router-templates": "1.16.0-beta-b16cb035",
+    "@abtnode/util": "1.16.0-beta-b16cb035",
+    "axios": "^0.27.2",
+    "debug": "^4.3.4",
+    "find-process": "^1.4.7",
+    "fkill": "^7.2.1",
+    "fs-extra": "^10.1.0",
     "get-port": "^5.1.1",
+    "http-proxy": "^1.18.1",
+    "is-valid-domain": "^0.1.6",
     "lodash": "^4.17.21",
-    "moment": "^2.29.1",
-    "nginx-conf": "^1.5.0",
+    "lru-cache": "^7.14.0",
+    "moment": "^2.29.4",
+    "nginx-conf": "^1.7.0",
+    "object-hash": "^3.0.0",
+    "port-used": "^2.0.8",
     "promise-retry": "^2.0.1",
-    "shelljs": "^0.8.4",
-    "tar": "^6.1.0"
+    "shelljs": "^0.8.5",
+    "tar": "^6.1.11",
+    "url-join": "^4.0.1",
+    "uuid": "^8.3.2",
+    "valid-url": "^1.0.9"
   },
   "devDependencies": {
-    "fs-extra": "^10.0.0"
+    "bluebird": "^3.7.2",
+    "fs-extra": "^10.1.0",
+    "needle": "^3.1.0"
   },
-  "gitHead": "22715c3ea74d0230f3413162a17f491614b6735a"
+  "gitHead": "138bd91aee5a35b28c2de4e1e924c44932efaf3a"
 }

package/lib/none/index.js

@@ -1,50 +0,0 @@
-/* eslint-disable no-empty-function */
-const BaseProvider = require('../base');
-
-class NoneProvider extends BaseProvider {
-  constructor() {
-    super('none');
-  }
-
-  async update() {}
-
-  async reload() {}
-
-  async start() {}
-
-  async restart() {}
-
-  async stop() {}
-
-  initialize() {}
-
-  async validateConfig() {}
-
-  async rotateLogs() {}
-}
-
-NoneProvider.describe = () => {
-  const result = NoneProvider.check();
-  return {
-    name: 'none',
-    description: 'This provider does nothing, you must access blocklets through IP + Port',
-    error: result.error,
-    available: result.available,
-    running: result.running,
-  };
-};
-
-NoneProvider.check = () => ({
-  available: true,
-  running: true,
-  error: '',
-});
-
-NoneProvider.getStatus = () => ({
-  pid: 0,
-  running: false,
-});
-
-NoneProvider.exists = () => true;
-
-module.exports = NoneProvider;