@abtnode/router-provider 1.6.23 → 1.6.24

package/README.md

@@ -9,7 +9,7 @@ const { getProvider } = require('router-provider');
 const providerName = 'nginx';
 const Provider = getProvider(providerName);
 
-const provider = new Provider({ configDirectory });
+const provider = new Provider({ configDir });
 provider.start();
 provider.stop();
 // ...

package/lib/base.js

@@ -1,3 +1,5 @@
+const os = require('os');
+
 class BaseProvider {
   constructor(name) {
     this.name = name;
@@ -34,6 +36,18 @@ class BaseProvider {
   rotateLogs() {
     throw new Error('rotateLogs method is not implemented');
   }
+
+  getWorkerProcess(max = +process.env.ABT_NODE_MAX_CLUSTER_SIZE) {
+    if (this.isTest) {
+      return 1;
+    }
+
+    if (process.env.NODE_ENV === 'production') {
+      return Math.min(os.cpus().length, max);
+    }
+
+    return 1;
+  }
 }
 
 module.exports = BaseProvider;

package/lib/default/daemon.js

@@ -0,0 +1,316 @@
+const url = require('url');
+const path = require('path');
+const fs = require('fs-extra');
+const get = require('lodash/get');
+const joinUrl = require('url-join');
+const checkDomainMatch = require('@abtnode/util/lib/check-domain-match');
+const { DEFAULT_IP_DNS_DOMAIN_SUFFIX, ROUTING_RULE_TYPES } = require('@abtnode/constant');
+
+const logger = require('@abtnode/logger')('router:default:daemon', { filename: 'engine' });
+
+const { findCertificate, isSpecificDomain, toSlotDomain, matchRule } = require('../util');
+const ProxyServer = require('./proxy');
+
+const configPath = process.env.ABT_NODE_ROUTER_CONFIG;
+const httpPort = +process.env.ABT_NODE_ROUTER_HTTP_PORT;
+const httpsPort = +process.env.ABT_NODE_ROUTER_HTTPS_PORT;
+const servicePort = +process.env.ABT_NODE_SERVICE_PORT;
+
+if (fs.existsSync(configPath) === false) {
+  throw new Error('Router config file not found');
+}
+
+const wwwDir = path.join(path.dirname(configPath), 'www');
+const fileIndex = fs.readFileSync(path.join(wwwDir, 'index.html')).toString();
+const file404 = fs.readFileSync(path.join(wwwDir, '404.html')).toString();
+const file5xx = fs.readFileSync(path.join(wwwDir, '5xx.html')).toString();
+const file502 = fs.readFileSync(path.join(wwwDir, '502.html')).toString();
+
+process.on('message', (msg) => {
+  if (msg.data === 'reload') {
+    logger.info('update config', msg);
+    updateConfig(true);
+  }
+});
+
+const config = { sites: [], rules: {}, certs: [], headers: [] };
+
+// eslint-disable-next-line
+const createRequestHandler = (id) => (req, res, target) => {
+  const rule = config.rules[id];
+
+  if (rule.type === ROUTING_RULE_TYPES.DIRECT_RESPONSE) {
+    if (rule.response.contentType) {
+      res.writeHead(rule.response.status, { 'Content-Type': rule.response.contentType });
+    }
+
+    res.end(rule.response.body);
+
+    return { abort: true };
+  }
+
+  // Note: redirection required for relative asset loading
+  if (req.method === 'GET' && rule.prefix !== '/') {
+    const parsed = url.parse(req.url);
+    if (parsed.pathname === rule.prefix) {
+      logger.info('redirect url', { url: req.url, rule });
+      parsed.pathname = joinUrl(rule.prefix, '/');
+      res.writeHead(307, { Location: url.format(parsed) });
+      res.end();
+      return { abort: true };
+    }
+  }
+
+  if (rule.type === ROUTING_RULE_TYPES.GENERAL_PROXY) {
+    // we do not need rewrite for internal servers
+  } else {
+    req.url = joinUrl(rule.target, req.url.substr(rule.prefix.length));
+  }
+
+  if (req.url.startsWith('/') === false) {
+    req.url = `/${req.url}`;
+  }
+  logger.debug('transform url', { url: req.url, rule });
+
+  if (rule.did) {
+    req.headers['X-Blocklet-Did'] = rule.did;
+  }
+  if (rule.realDid && rule.did !== rule.realDid) {
+    req.headers['X-Blocklet-Real-Did'] = rule.realDid;
+  }
+
+  req.headers['X-Path-Prefix'] = rule.prefix;
+  req.headers['X-Group-Path-Prefix'] = rule.groupPrefix || '/';
+
+  if (rule.services.length) {
+    req.headers['X-Blocklet-Url'] = `http://127.0.0.1:${rule.port}`;
+  }
+
+  if (rule.ruleId) {
+    req.headers['X-Routing-Rule-Id'] = rule.ruleId;
+  }
+};
+
+// eslint-disable-next-line
+const sharedResolver = (host, url, req) => {
+  // match slot domain
+  const domain = toSlotDomain(host);
+
+  // match specific domain
+  let site = config.sites.find((x) => x.domain === domain);
+
+  // match wildcard domain
+  if (!site) {
+    site = config.sites.find((x) => checkDomainMatch(x.domain, domain));
+  }
+
+  // fallback to default domain
+  if (!site) {
+    site = config.sites.find((x) => x.domain === '_');
+  }
+
+  const rule = matchRule(site.rules, url);
+  if (!rule) {
+    logger.warn('rule not found for request', { host, url, domain });
+    return null;
+  }
+
+  const match = findCertificate(config.certs, domain);
+  const upstream = `http://127.0.0.1:${rule.services.length ? servicePort : rule.port}`;
+  logger.debug('resolved request', { host, url, domain, upstream, rule });
+
+  return {
+    id: rule.id,
+    url: upstream,
+    path: '/',
+    opts: {
+      ssl: match ? { key: match.privateKey, cert: match.certificate } : null,
+      onRequest: createRequestHandler(rule.id),
+    },
+  };
+};
+sharedResolver.priority = 200;
+
+const onError = (err, req, res) => {
+  logger.error('proxy error', { error: err });
+  if (typeof res.writeHead !== 'function') {
+    return;
+  }
+
+  if (err.code === 'ECONNREFUSED') {
+    res.writeHead(502);
+    res.end(file502);
+  } else if (err.code === 'NOTFOUND') {
+    const parsed = url.parse(req.url);
+    if (config.info.enableWelcomePage && parsed.pathname === '/') {
+      res.writeHead(404);
+      res.end(fileIndex);
+    } else {
+      res.writeHead(404);
+      res.end(file404);
+    }
+  } else {
+    res.writeHead(500);
+    res.end(file5xx);
+  }
+};
+
+const corsHandler = (host, req, res) => {
+  if (req.method === 'OPTIONS') {
+    const domain = toSlotDomain(host);
+    const site = config.sites.find((x) => x.domain === domain);
+    if (!site) {
+      return true;
+    }
+
+    const allowedOrigins = site.corsAllowedOrigins;
+    const currentOrigin = req.headers.origin;
+    if (allowedOrigins.includes('*')) {
+      res.writeHead(204, {
+        Vary: 'Origin',
+        'Access-Control-Allow-Origin': '*',
+        'Access-Control-Allow-Credentials': false,
+        'Access-Control-Allow-Methods': 'POST, GET, HEAD, PUT, DELETE, OPTIONS',
+        'Access-Control-Allow-Headers':
+          'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,$http_access_control_request_headers',
+        'Access-Control-Max-Age': 1800,
+      });
+      res.end();
+      return false;
+    }
+
+    if (allowedOrigins.some((x) => checkDomainMatch(x, currentOrigin))) {
+      res.writeHead(204, {
+        Vary: 'Origin',
+        'Access-Control-Allow-Origin': currentOrigin,
+        'Access-Control-Allow-Credentials': false,
+        'Access-Control-Allow-Methods': 'POST, GET, HEAD, PUT, DELETE, OPTIONS',
+        'Access-Control-Allow-Headers':
+          'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,$http_access_control_request_headers',
+        'Access-Control-Max-Age': 1800,
+      });
+      res.end();
+      return false;
+    }
+  }
+
+  return true;
+};
+
+// internal servers
+const internalServers = {};
+const ensureInternalServer = (port) => {
+  if (internalServers[port]) {
+    return;
+  }
+
+  const server = new ProxyServer({ xfwd: false, internal: true, port, headers: config.headers, onError, corsHandler });
+  server.addResolver(sharedResolver);
+  internalServers[port] = server;
+  logger.info('internal server ready on port', { port });
+};
+
+const updateConfig = (reload = false) => {
+  const { sites, certificates: rawCerts, info } = fs.readJsonSync(configPath);
+  config.info = info;
+
+  const obj = get(info, 'routing.headers', {});
+  config.headers = Object.keys(obj).map((key) => ({ key, value: JSON.parse(obj[key]) }));
+
+  // format certificates
+  config.certs = rawCerts.map((x) => {
+    x.privateKey = Buffer.from(x.privateKey);
+    x.certificate = Buffer.from(x.certificate);
+    return x;
+  });
+
+  // format sites
+  sites.forEach((site) => {
+    site.rules.forEach((rule) => {
+      config.rules[rule.id] = rule;
+    });
+  });
+
+  // re-register sites in reload mode
+  if (reload) {
+    const oldDomains = config.sites.map((x) => x.domain);
+    const newDomains = sites.map((x) => x.domain);
+
+    const addedDomains = newDomains.filter((x) => !oldDomains.includes(x));
+    for (const domain of addedDomains) {
+      if (isSpecificDomain(domain)) {
+        logger.info('register domain on reload', { domain });
+        const site = sites.find((x) => x.domain === domain);
+        const [rule] = site.rules.filter((x) => ['daemon', 'blocklet'].includes(x.type));
+        const match = findCertificate(config.certs, site.domain);
+        main.register({
+          src: site.domain,
+          target: `http://127.0.0.1:${rule.port}`,
+          ssl: match ? { key: match.privateKey, cert: match.certificate } : null,
+          onRequest: createRequestHandler(rule.id),
+        });
+      }
+    }
+
+    const removedDomains = oldDomains.filter((x) => !newDomains.includes(x));
+    for (const domain of removedDomains) {
+      main.unregister(domain);
+      logger.info('unregister domain on reload', { domain });
+      delete main.routing[domain];
+      logger.info('remove domain cache on reload', { domain });
+    }
+
+    config.sites.filter((x) => x.port).forEach((x) => ensureInternalServer(x.port));
+  }
+
+  config.sites = sites;
+};
+
+// load initial config
+updateConfig();
+
+// create main server
+const defaultCert = config.certs.find((x) => x.domain.endsWith(DEFAULT_IP_DNS_DOMAIN_SUFFIX));
+const main = new ProxyServer({
+  xfwd: true,
+  onError,
+  corsHandler,
+  port: httpPort,
+  headers: config.headers,
+  ssl: {
+    port: httpsPort,
+    key: defaultCert ? defaultCert.privateKey : null,
+    cert: defaultCert ? defaultCert.certificate : null,
+    opts: {
+      honorCipherOrder: false,
+      maxVersion: 'TLSv1.3',
+      minVersion: 'TLSv1.2',
+    },
+  },
+});
+main.sites = config.sites;
+main.addResolver(sharedResolver);
+
+// register sites: to ensure https certificates are loaded
+config.sites.forEach((site) => {
+  const { domain } = site;
+  if (isSpecificDomain(domain) === false) {
+    return;
+  }
+
+  const [rule] = site.rules.filter((x) => ['daemon', 'blocklet'].includes(x.type));
+  const match = findCertificate(config.certs, domain);
+
+  logger.info('register domain on start', { domain });
+  main.register({
+    src: domain,
+    target: `http://127.0.0.1:${rule.port}`,
+    ssl: match ? { key: match.privateKey, cert: match.certificate } : null,
+    onRequest: createRequestHandler(rule.id),
+  });
+});
+
+// initialize internal servers
+config.sites.filter((x) => x.port).forEach((x) => ensureInternalServer(x.port));
+
+logger.info(`Default routing engine ready on ${httpPort} and ${httpsPort}`);

package/lib/default/index.js

@@ -0,0 +1,221 @@
+const fs = require('fs-extra');
+const path = require('path');
+const shelljs = require('shelljs');
+const get = require('lodash/get');
+const omit = require('lodash/omit');
+const sortBy = require('lodash/sortBy');
+const objectHash = require('object-hash');
+const promiseRetry = require('promise-retry');
+const pm2 = require('@abtnode/util/lib/async-pm2');
+const { PROCESS_NAME_ROUTER, DAEMON_MAX_MEM_LIMIT_IN_MB } = require('@abtnode/constant');
+
+const logger = require('@abtnode/logger')('router:default:controller', { filename: 'engine' });
+
+const BaseProvider = require('../base');
+const {
+  decideHttpPort,
+  decideHttpsPort,
+  get404Template,
+  get502Template,
+  get5xxTemplate,
+  getWelcomeTemplate,
+  formatRoutingTable,
+} = require('../util');
+
+class DefaultProvider extends BaseProvider {
+  constructor({ configDir, httpPort, httpsPort, isTest }) {
+    super('default');
+
+    this.isTest = !!isTest;
+    this.configDir = configDir;
+
+    this.logDir = path.join(this.configDir, 'log');
+    this.wwwDir = path.join(this.configDir, 'www');
+
+    this.engineLog = path.join(this.logDir, 'engine.log'); // should be symbol link created by winston
+    this.errorLog = path.join(this.logDir, 'engine-error.log'); // should be symbol link created by winston
+    this.ctrlLog = path.join(this.logDir, 'controller.log'); // managed by pm2
+
+    this.configPath = path.join(this.configDir, 'config.json');
+
+    this.httpPort = decideHttpPort(httpPort);
+    this.httpsPort = decideHttpsPort(httpsPort);
+
+    // ensure directories
+    [this.configDir, this.logDir, this.wwwDir].forEach((dir) => {
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir);
+      }
+    });
+
+    this.initialize();
+  }
+
+  // Services are not supported by default provider
+  update({ routingTable = [], certificates = [], nodeInfo = {} } = {}) {
+    this.info = nodeInfo;
+
+    logger.info('routing table:', routingTable);
+    this._addWwwFiles(nodeInfo);
+    const { sites: rawSites } = formatRoutingTable(routingTable);
+    const sites = sortBy(rawSites, (x) => -x.domain.length);
+    sites.forEach((site) => {
+      site.rules.forEach((rule) => {
+        // NOTE: we generate uniq hash from rule, because there are duplicate ruleId across different rules
+        rule.id = objectHash(rule);
+      });
+    });
+
+    fs.outputJSONSync(this.configPath, { sites, certificates, info: this.info }, { spaces: 2 });
+  }
+
+  async reload() {
+    await pm2.connectAsync();
+
+    // ensure daemon is live
+    let [info] = await pm2.describeAsync(PROCESS_NAME_ROUTER);
+    if (!info) {
+      await this.start();
+      [info] = await pm2.describeAsync(PROCESS_NAME_ROUTER);
+      logger.info('start daemon because not running', omit(info, ['pm2_env']));
+    } else {
+      logger.info('daemon already running', omit(info, ['pm2_env']));
+    }
+
+    // send signal to daemon
+    return new Promise((resolve) => {
+      pm2.sendDataToProcessId(info.pm2_env.pm_id, { type: 'process:msg', data: 'reload', topic: 'reload' }, (err) => {
+        if (err) {
+          logger.error('failed to reload', { error: err });
+        }
+
+        pm2.disconnect(resolve);
+      });
+    });
+  }
+
+  async start() {
+    await pm2.connectAsync();
+    await pm2.startAsync({
+      namespace: 'daemon',
+      name: PROCESS_NAME_ROUTER,
+      script: path.join(__dirname, 'daemon.js'),
+      max_memory_restart: `${get(this.info, 'runtimeConfig.daemonMaxMemoryLimit', DAEMON_MAX_MEM_LIMIT_IN_MB)}M`,
+      output: this.ctrlLog,
+      error: this.ctrlLog,
+      cwd: __dirname,
+      max_restarts: 1,
+      time: true,
+      mergeLogs: true,
+      execMode: 'fork', // Note: there are some issues with reloading when run in cluster mode
+      env: {
+        ABT_NODE_LOG_DIR: this.logDir, // Note: this tells logger module to write log to router log
+        ABT_NODE_ROUTER_CONFIG: this.configPath,
+        ABT_NODE_ROUTER_HTTP_PORT: this.httpPort,
+        ABT_NODE_ROUTER_HTTPS_PORT: this.httpsPort,
+      },
+    });
+  }
+
+  async restart() {
+    await pm2.connectAsync();
+    await pm2.restartAsync(PROCESS_NAME_ROUTER, { updateEnv: true });
+  }
+
+  async stop() {
+    await pm2.connectAsync();
+    const [info] = await pm2.describeAsync(PROCESS_NAME_ROUTER);
+    if (!info) {
+      return null;
+    }
+
+    const proc = await pm2.deleteAsync(PROCESS_NAME_ROUTER);
+    return proc;
+  }
+
+  initialize() {}
+
+  validateConfig() {}
+
+  rotateLogs() {}
+
+  _addWwwFiles(nodeInfo) {
+    const welcomePage = nodeInfo.enableWelcomePage ? getWelcomeTemplate(nodeInfo) : get404Template(nodeInfo);
+    fs.writeFileSync(`${this.wwwDir}/index.html`, welcomePage); // disable index.html
+    fs.writeFileSync(`${this.wwwDir}/404.html`, get404Template(nodeInfo));
+    fs.writeFileSync(`${this.wwwDir}/502.html`, get502Template(nodeInfo));
+    fs.writeFileSync(`${this.wwwDir}/5xx.html`, get5xxTemplate(nodeInfo));
+  }
+
+  getLogFilesForToday() {
+    return {
+      access: fs.realpathSync(this.engineLog),
+      error: fs.realpathSync(this.errorLog),
+    };
+  }
+}
+
+DefaultProvider.describe = async ({ configDir = '' } = {}) => {
+  const meta = {
+    name: 'default',
+    description: 'Use default to provide a lightweight routing layer for development purpose',
+  };
+
+  try {
+    // Use retry as a workaround for race-conditions between check and normal start
+    const result = await promiseRetry((retry) => DefaultProvider.check({ configDir }).catch(retry), { retries: 3 });
+    return { ...meta, ...result };
+  } catch (err) {
+    return { ...meta, error: err.message, available: false, running: false };
+  }
+};
+
+const getRouterStatus = async (configDir) => {
+  const result = {
+    managed: false,
+    pid: 0,
+    running: false,
+  };
+
+  await pm2.connectAsync();
+
+  const [info] = await pm2.describeAsync(PROCESS_NAME_ROUTER);
+  if (!info) {
+    return result;
+  }
+
+  result.running = true;
+  if (info.pm2_env.ABT_NODE_ROUTER_CONFIG.indexOf(configDir) > -1) {
+    result.managed = true;
+    result.pid = info.pid;
+  }
+
+  return result;
+};
+
+DefaultProvider.exists = () => true;
+DefaultProvider.getStatus = getRouterStatus;
+DefaultProvider.check = async ({ configDir = '' } = {}) => {
+  logger.info('check default provider', { configDir });
+  const binPath = shelljs.which('node').stdout;
+  const result = {
+    binPath,
+    available: true,
+    running: false,
+    managed: false,
+    error: '',
+  };
+
+  const status = await getRouterStatus(configDir);
+  result.managed = status.managed;
+  result.running = status.running;
+
+  if (status.running && !status.managed) {
+    result.available = false;
+    result.error = 'Seems the default routing engine is running, please terminate the process before try again.';
+  }
+
+  return result;
+};
+
+module.exports = DefaultProvider;