@abtnode/core 1.15.17 → 1.16.0-beta-b16cb035

package/lib/util/index.js

@@ -6,40 +6,33 @@ const unzipper = require('unzipper');
 const crypto = require('crypto');
 const shell = require('shelljs');
 const get = require('lodash/get');
-const uniqBy = require('lodash/uniqBy');
 const pickBy = require('lodash/pickBy');
 const { isFromPublicKey } = require('@arcblock/did');
 const joinUrl = require('url-join');
 const { Certificate } = require('@fidm/x509');
 const getPortLib = require('get-port');
-const isIp = require('is-ip');
 const v8 = require('v8');
 const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+const axios = require('@abtnode/util/lib/axios');
+const { encode: encodeBase32 } = require('@abtnode/util/lib/base32');
 const parseBlockletMeta = require('@blocklet/meta/lib/parse');
-const { validateMeta, fixAndValidateService } = require('@blocklet/meta/lib/validate');
-const { BlockletStatus, BLOCKLET_INTERFACE_WELLKNOWN } = require('@blocklet/meta/lib/constants');
+const { BlockletStatus } = require('@blocklet/constant');
+const { replaceSlotToIp } = require('@blocklet/meta/lib/util');
 const {
   StatusCode,
-  ROUTER_PROVIDER_NONE,
   DOMAIN_FOR_DEFAULT_SITE,
   DOMAIN_FOR_IP_SITE,
   DOMAIN_FOR_IP_SITE_REGEXP,
   DEFAULT_HTTP_PORT,
   DEFAULT_HTTPS_PORT,
-  ROUTING_RULE_TYPES,
   SLOT_FOR_IP_DNS_SITE,
-  DEFAULT_IP_DNS_DOMAIN_SUFFIX,
-  BLOCKLET_SITE_GROUP_SUFFIX,
+  NODE_MODES,
 } = require('@abtnode/constant');
-const { BLOCKLET_INTERFACE_TYPE_WEB } = require('@blocklet/meta/lib/constants');
 
 const DEFAULT_WELLKNOWN_PORT = 8088;
 
 const logger = require('@abtnode/logger')('@abtnode/core:util');
 
-const { getServices, getServicesFromBlockletInterface } = require('./service');
-const request = require('./request');
-
 const validateOwner = (owner) => {
   try {
     return owner && owner.did && owner.pk && isFromPublicKey(owner.did, owner.pk);
@@ -54,27 +47,21 @@ const fromStatus = (v) => {
   return match ? match[0] : 'unknown';
 };
 
-const getInterfaceUrl = ({ baseUrl, url, version }) => {
-  if (!url) {
-    return '';
-  }
-
-  if (url.startsWith('https://') || url.startsWith('http://')) {
-    return url;
-  }
-
-  const parsed = new URL(joinUrl(baseUrl, url));
-  const params = parsed.searchParams;
-
-  if (version) {
-    params.set('__bv__', version);
+const replaceDomainSlot = ({ domain, context = {}, nodeIp }) => {
+  let processed = domain;
+  if (processed.includes(SLOT_FOR_IP_DNS_SITE)) {
+    const ipRegex = /\d+[-.]\d+[-.]\d+[-.]\d+/;
+    const match = ipRegex.exec(context.hostname);
+    if (match) {
+      const ip = match[0];
+      processed = replaceSlotToIp(processed, ip);
+    } else if (nodeIp) {
+      processed = replaceSlotToIp(processed, nodeIp);
+    }
   }
-
-  return parsed.href;
+  return processed;
 };
 
-const trimSlash = (str = '') => str.replace(/^\/+/, '').replace(/\/+$/, '');
-
 const getBlockletHost = ({ domain, context, nodeIp }) => {
   const { protocol, port } = context || {};
 
@@ -87,16 +74,7 @@ const getBlockletHost = ({ domain, context, nodeIp }) => {
     tmpDomain = context.hostname || '';
   }
 
-  if (tmpDomain.includes(SLOT_FOR_IP_DNS_SITE)) {
-    const ipRegex = /\d+[-.]\d+[-.]\d+[-.]\d+/;
-    const match = ipRegex.exec(context.hostname);
-    if (match) {
-      const ip = match[0].replace(/\./g, '-');
-      tmpDomain = tmpDomain.replace(SLOT_FOR_IP_DNS_SITE, ip);
-    } else if (nodeIp) {
-      tmpDomain = tmpDomain.replace(SLOT_FOR_IP_DNS_SITE, nodeIp.replace(/\./g, '-'));
-    }
-  }
+  tmpDomain = replaceDomainSlot({ domain: tmpDomain, context, nodeIp });
 
   if (!port) {
     return tmpDomain;
@@ -113,136 +91,7 @@ const getBlockletHost = ({ domain, context, nodeIp }) => {
   return `${tmpDomain}:${port}`;
 };
 
-/**
- * Get config of auth service of blocklet component specified the routing rule
- * @param {*} rule routing rule
- * @param {*} blocklet
- * @returns
- */
-const getAuthConfig = (rule, blocklet) => {
-  if (!blocklet || !rule || rule.to.type !== ROUTING_RULE_TYPES.BLOCKLET || rule.to.did !== blocklet.meta.did) {
-    return null;
-  }
-
-  // find blocklet component
-  const isRootRule = rule.to.did === rule.to.realDid;
-  let _blocklet;
-  if (isRootRule) {
-    _blocklet = blocklet;
-  } else {
-    _blocklet = (blocklet.children || []).find((x) => x.meta.did === rule.to.realDid);
-  }
-  if (!_blocklet) {
-    return null;
-  }
-
-  // find interface in meta
-  const interfaceName = rule.to.realInterfaceName;
-  const { interfaces } = _blocklet.meta;
-  const _interface = interfaces.find((x) => x.type === BLOCKLET_INTERFACE_TYPE_WEB && x.name === interfaceName);
-
-  if (!_interface) {
-    return null;
-  }
-
-  // find auth service
-  const auth = getServicesFromBlockletInterface(_interface, { stringifyConfig: false }).find(
-    (x) => x.name === '@abtnode/auth-service'
-  );
-
-  if (!auth) {
-    return null;
-  }
-
-  return auth.config;
-};
-
-const getBlockletBaseUrls = ({ routingEnabled = false, port, rules = [], context = {}, blocklet, nodeIp }) => {
-  let baseUrls = [];
-
-  if (routingEnabled && Array.isArray(rules) && rules.length > 0) {
-    baseUrls = rules
-      .filter((rule) => !(rule.from.domain || '').endsWith(BLOCKLET_SITE_GROUP_SUFFIX))
-      .map((rule) => {
-        const host = getBlockletHost({ domain: rule.from.domain, context, nodeIp });
-        if (host) {
-          let protocol = 'http'; // TODO: 这里固定为 http, 因为判断 url 是不是 https 和证书相关，这里实现的话比较复杂
-          if (host.includes(DEFAULT_IP_DNS_DOMAIN_SUFFIX)) {
-            protocol = 'https';
-          }
-
-          return {
-            ruleId: rule.id,
-            baseUrl: `${protocol}://${host}/${trimSlash(rule.from.pathPrefix)}`,
-            interfaceName: get(rule, 'to.interfaceName', ''),
-            authConfig: getAuthConfig(rule, blocklet),
-          };
-        }
-
-        return null;
-      })
-      .filter(Boolean);
-  }
-
-  if (!baseUrls.length && isIp(context.hostname)) {
-    baseUrls = [{ baseUrl: `http://${context.hostname}:${port}` }];
-  }
-
-  return baseUrls;
-};
-
-const getBlockletInterfaces = ({ blocklet, context, nodeInfo, routingRules, nodeIp }) => {
-  const interfaces = [];
-  (blocklet.meta.interfaces || []).forEach((x) => {
-    if (x.port && x.port.external) {
-      // If an interface was exposed through external port, path is not appended
-      const baseUrls = getBlockletBaseUrls({
-        routingEnabled: false,
-        port: x.port.external,
-        rules: [],
-        context,
-      });
-      baseUrls.forEach(({ baseUrl }) => interfaces.push({ type: x.type, name: x.name, url: baseUrl }));
-    } else {
-      // Otherwise, the url is composed with routing rules
-      const port = x.port && x.port.internal ? blocklet.ports[x.port.internal] : blocklet.ports[x.port];
-      const baseUrls = getBlockletBaseUrls({
-        routingEnabled: isRoutingEnabled(nodeInfo.routing),
-        port,
-        rules: (routingRules || []).filter((r) => {
-          return (
-            // don't show wellknown interface
-            r.to.type !== ROUTING_RULE_TYPES.GENERAL_PROXY &&
-            // LEGACY don't show wellknown interface
-            r.to.interfaceName !== BLOCKLET_INTERFACE_WELLKNOWN &&
-            // don't show child blocklet interface
-            (!r.from.groupPathPrefix || r.from.pathPrefix === r.from.groupPathPrefix)
-          );
-        }),
-        context,
-        blocklet,
-        nodeIp,
-      });
-
-      baseUrls.forEach(({ baseUrl, ruleId, interfaceName, authConfig }) => {
-        if (interfaceName && interfaceName !== x.name) {
-          return;
-        }
-        interfaces.push({
-          ruleId,
-          type: x.type,
-          name: x.name,
-          url: getInterfaceUrl({ baseUrl, url: '/', version: blocklet.meta.version }),
-          authConfig,
-        });
-      });
-    }
-  });
-
-  return uniqBy(interfaces, 'url');
-};
-
-const isRoutingEnabled = (routing) => !!routing && !!routing.provider && routing.provider !== ROUTER_PROVIDER_NONE;
+const isRoutingEnabled = (routing) => !!routing && !!routing.provider;
 
 const isInProgress = (status) =>
   [
@@ -283,7 +132,7 @@ const asyncExec = (command, options) =>
     });
   });
 
-const getProviderFromNodeInfo = (nodeInfo) => get(nodeInfo, 'routing.provider', null) || ROUTER_PROVIDER_NONE;
+const getProviderFromNodeInfo = (info) => get(info, 'routing.provider', 'default');
 
 const isCLI = () => !process.env.ABT_NODE_SK;
 
@@ -315,7 +164,6 @@ const getHttpsCertInfo = (certificate) => {
     validFrom,
     validTo,
     issuer,
-    serialNumber: info.serialNumber,
     sans: info.dnsNames,
     validityPeriod: info.validTo - info.validFrom,
     fingerprintAlg: 'SHA256',
@@ -332,6 +180,8 @@ const getDataDirs = (dataDir) => ({
   tmp: path.join(dataDir, 'tmp'),
   blocklets: path.join(dataDir, 'blocklets'),
   services: path.join(dataDir, 'services'),
+  modules: path.join(dataDir, 'modules'),
+  certManagerModule: path.join(dataDir, 'modules', 'certificate-manager'),
 });
 
 // Ensure data dir for Blocklet Server exists
@@ -356,39 +206,52 @@ const formatEnvironments = (environments) => Object.keys(environments).map((x) =
 const transformIPToDomain = (ip) => ip.split('.').join('-');
 
 const getBaseUrls = async (node, ips) => {
-  const ipTypes = {
-    internal: 'private',
-    external: 'public',
-  };
   const info = await node.getNodeInfo();
   const { https, httpPort, httpsPort } = info.routing;
-  const ipKeys = Object.keys(ips).filter((x) => ips[x]);
   const getPort = (port, defaultPort) => (port && port !== defaultPort ? `:${port}` : '');
+  const availableIps = ips.filter(Boolean);
 
-  if (info.routing.provider !== ROUTER_PROVIDER_NONE && info.routing.snapshotHash && info.routing.adminPath) {
-    const sites = await node.getSitesFromSnapshot();
+  const getHttpInfo = async (domain) => {
+    const certificate = https ? await node.certManager.getNormalByDomain(domain) : null;
+    const protocol = certificate ? 'https' : 'http';
+    const port = certificate ? getPort(httpsPort, DEFAULT_HTTPS_PORT) : getPort(httpPort, DEFAULT_HTTP_PORT);
 
-    const { dashboardDomain } = info.routing;
+    return { protocol, port };
+  };
+
+  if (info.routing.provider && info.routing.snapshotHash && info.routing.adminPath) {
+    const sites = await node.getSitesFromSnapshot();
+    const { ipWildcardDomain } = info.routing;
     const adminPath = normalizePathPrefix(info.routing.adminPath);
-    const httpUrls = ipKeys.map((x) => {
-      const port = getPort(httpPort, DEFAULT_HTTP_PORT);
+    const tmpHttpPort = getPort(httpPort, DEFAULT_HTTP_PORT);
+
+    const httpUrls = availableIps.map((ip) => {
       return {
-        url: `http://${ips[x]}${port}${adminPath}`,
-        type: ipTypes[x],
+        url: `http://${ip}${tmpHttpPort}${adminPath}`,
       };
     });
 
-    if (dashboardDomain) {
-      const site = sites.find((c) => (c.domainAliases || []).find((item) => item.value === dashboardDomain));
+    if (ipWildcardDomain) {
+      const site = sites.find((c) => (c.domainAliases || []).find((item) => item.value === ipWildcardDomain));
       if (site) {
-        const certificate = https ? await node.findCertificateByDomain(dashboardDomain) : null;
-        const protocol = certificate ? 'https' : 'http';
-        const port = certificate ? getPort(httpsPort, DEFAULT_HTTPS_PORT) : getPort(httpPort, DEFAULT_HTTP_PORT);
-        const httpsUrls = ipKeys.map((x) => ({
-          url: `${protocol}://${transformIPToDomain(ips[x])}.${dashboardDomain.substring(2)}${port}${adminPath}`,
-          type: ipTypes[x],
+        const httpInfo = await getHttpInfo(ipWildcardDomain);
+        const httpsUrls = availableIps.map((ip) => ({
+          url: `${httpInfo.protocol}://${transformIPToDomain(ip)}.${ipWildcardDomain.substring(2)}${
+            httpInfo.port
+          }${adminPath}`,
         }));
 
+        // add did domain access url
+        const didDomainAlias = site.domainAliases.find((item) => item.value.endsWith(info.didDomain));
+        if (didDomainAlias) {
+          const didDomain = didDomainAlias.value;
+          const didDomainHttpInfo = await getHttpInfo(didDomain);
+
+          httpsUrls.push({
+            url: `${didDomainHttpInfo.protocol}://${didDomain}${didDomainHttpInfo.port}${adminPath}`,
+          });
+        }
+
         return httpUrls.concat(httpsUrls);
       }
     }
@@ -397,73 +260,11 @@ const getBaseUrls = async (node, ips) => {
   }
 
   // port urls
-  return ipKeys.map((x) => ({
-    url: `http://${ips[x]}:${info.port}`,
-    type: ipTypes[x],
+  return availableIps.map((ip) => ({
+    url: `http://${ip}:${info.port}`,
   }));
 };
 
-const getBlockletMetaByUrl = async (url) => {
-  const { protocol, pathname } = new URL(url);
-
-  if (protocol.startsWith('file')) {
-    const decoded = decodeURIComponent(pathname);
-    if (!fs.existsSync(decoded)) {
-      throw new Error(`File does not exist: ${decoded}`);
-    }
-    const d = await fs.promises.readFile(decoded);
-    const meta = JSON.parse(d);
-    return meta;
-  }
-
-  if (protocol.startsWith('http')) {
-    try {
-      const { data: meta } = await request({ url, method: 'GET', timeout: 1000 * 20 });
-      if (Object.prototype.toString.call(meta) !== '[object Object]') {
-        throw new Error('Url is not valid');
-      }
-      return meta;
-    } catch (err) {
-      throw new Error(`Failed to get blocklet meta from ${url}: ${err.message}`);
-    }
-  }
-
-  throw new Error(`Invalid url protocol: ${protocol.replace(/:$/, '')}`);
-};
-
-const validateUrl = async (url, expectedHttpResTypes = ['application/json', 'text/plain']) => {
-  const parsed = new URL(url);
-  const { protocol, pathname } = parsed;
-
-  // file
-  if (protocol.startsWith('file')) {
-    const decoded = decodeURIComponent(pathname);
-    if (!fs.existsSync(decoded)) {
-      throw new Error(`File does not exist: ${decoded}`);
-    }
-    return true;
-  }
-
-  // http(s)
-  if (protocol.startsWith('http')) {
-    let res;
-
-    try {
-      res = await request({ url, method: 'HEAD', timeout: 1000 * 10 });
-    } catch (err) {
-      throw new Error(`Cannot get content-type from ${url}: ${err.message}`);
-    }
-
-    if (expectedHttpResTypes.some((x) => res.headers['content-type'].includes(x)) === false) {
-      throw new Error(`Unexpected content-type from ${url}: ${res.headers['content-type']}`);
-    }
-
-    return true;
-  }
-
-  throw new Error(`Invalid url protocol: ${protocol.replace(/:$/, '')}`);
-};
-
 const expandBundle = (bundlePath, destDir) =>
   unzipper.Open.file(bundlePath).then((d) => d.extract({ path: destDir, concurrency: 20 }));
 
@@ -527,12 +328,151 @@ const getQueueConcurrencyByMem = () => {
 
 const getSafeEnv = (inputEnv, processEnv = process.env) => {
   const whiteList = ['ABT_NODE', 'ABT_NODE_DID', 'ABT_NODE_PK', 'ABT_NODE_PORT', 'ABT_NODE_SERVICE_PORT'];
+  const blackList = { ABT_NODE_SK: '' };
   // 此处需要保留 process.env 中的环境变量，只移除和 ABT_NODE 相关的环境变量（否则丢失了 process.env.SHELL 变量可能会造成无法使用 nodejs 的情况）
   const filterProcessEnv = pickBy(processEnv, (value, key) => !key.startsWith('ABT_NODE') || whiteList.includes(key));
-  const mergedEnv = { ...filterProcessEnv, ...inputEnv };
+  const mergedEnv = { ...filterProcessEnv, ...inputEnv, ...blackList };
   return mergedEnv;
 };
 
+// For performance sake, we only support 1 param here, and the cache is flushed every minute
+const memoizeAsync = (fn, flushInterval = 60000) => {
+  const cache = new Map();
+  setInterval(() => cache.clear(), flushInterval);
+
+  return (arg) => {
+    if (cache.has(arg)) {
+      return cache.get(arg);
+    }
+
+    // eslint-disable-next-line prefer-spread
+    const result = fn.apply(null, [arg]);
+    cache.set(arg, result);
+
+    return result;
+  };
+};
+
+const getStateCrons = (states) => [
+  {
+    name: 'cleanup-audit-logs',
+    time: '0 0 6 * * *', // cleanup old logs every day at 6am
+    // time: '0 */5 * * * *', // cleanup every 5 minutes
+    options: { runOnInit: false },
+    fn: async () => {
+      const removed = await states.auditLog.remove({
+        createdAt: { $lt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 90) },
+      });
+      logger.info(`Removed ${removed} audit logs`);
+    },
+  },
+];
+
+const getDelegateState = async (chainHost, address) => {
+  const result = await axios.post(
+    joinUrl(chainHost, '/gql/'),
+    JSON.stringify({
+      query: `{
+        getDelegateState(address: "${address}") {
+          state {
+            address
+            ops {
+              key
+            }
+          }
+        }
+      }`,
+    }),
+    {
+      headers: {
+        'Content-Type': 'application/json',
+        Accept: 'application/json',
+      },
+      timeout: 60 * 1000,
+    }
+  );
+
+  return get(result.data, 'data.getDelegateState.state');
+};
+
+const getNFTState = async (chainHost, nftId) => {
+  const url = joinUrl(new URL(chainHost).origin, '/api/gql/');
+
+  const result = await axios.post(
+    url,
+    JSON.stringify({
+      query: `{
+        getAssetState(address: "${nftId}") {
+          state {
+            address
+            data {
+              typeUrl
+              value
+            }
+            display {
+              type
+              content
+            }
+            issuer
+            owner
+            parent
+            tags
+          }
+        }
+      }`,
+    }),
+    {
+      headers: {
+        'Content-Type': 'application/json',
+        Accept: 'application/json',
+      },
+      timeout: 60 * 1000,
+    }
+  );
+
+  const state = get(result, 'data.data.getAssetState.state');
+  if (state && state.data.typeUrl === 'json') {
+    state.data.value = JSON.parse(state.data.value);
+  }
+
+  return state;
+};
+
+const getServerDidDomain = (nodeInfo) => `${encodeBase32(nodeInfo.did)}.${nodeInfo.didDomain}`;
+
+const prettyURL = (url, isHttps = true) => {
+  if (typeof url !== 'string') {
+    return url;
+  }
+
+  if (url.toLowerCase().startsWith('http')) {
+    return url;
+  }
+
+  return isHttps ? `https://${url}` : `http://${url}`;
+};
+
+const templateReplace = (str, vars = {}) => {
+  if (typeof str === 'string') {
+    return str.replace(/{([.\w]+)}/g, (m, key) => get(vars, key));
+  }
+
+  return str;
+};
+
+const isGatewayCacheEnabled = (info) => {
+  if (info.mode === NODE_MODES.DEBUG) {
+    return false;
+  }
+
+  const cacheEnabled = get(info, 'routing.cacheEnabled');
+  if (typeof cacheEnabled === 'boolean') {
+    return cacheEnabled;
+  }
+
+  return true;
+};
+
 const lib = {
   validateOwner,
   getProviderFromNodeInfo,
@@ -541,30 +481,17 @@ const lib = {
   formatEnvironments,
   isInProgress,
   isBeforeInstalled,
-  getInterfaceUrl,
-  getBlockletBaseUrls,
+  replaceDomainSlot,
   getBlockletHost,
-  getBlockletInterfaces,
   isRoutingEnabled,
   checkDomainDNS,
   asyncExec,
-  trimSlash,
   isCLI,
   getHttpsCertInfo,
   ensureDataDirs,
   getDataDirs,
   getBaseUrls,
-  getBlockletMeta: (dir, opts = {}) =>
-    parseBlockletMeta(dir, {
-      serviceMetas: getServices(),
-      ...opts,
-    }),
-  validateBlockletMeta: (meta, opts = {}) => {
-    fixAndValidateService(meta, getServices(), opts.fix);
-    return validateMeta(meta, opts);
-  },
-  getBlockletMetaByUrl,
-  validateUrl,
+  getBlockletMeta: parseBlockletMeta,
   expandBundle,
   findInterfaceByName,
   findInterfacePortByName,
@@ -573,6 +500,14 @@ const lib = {
   transformIPToDomain,
   getQueueConcurrencyByMem,
   getSafeEnv,
+  memoizeAsync,
+  getStateCrons,
+  getDelegateState,
+  getNFTState,
+  getServerDidDomain,
+  prettyURL,
+  templateReplace,
+  isGatewayCacheEnabled,
 };
 
 module.exports = lib;

package/lib/util/ip.js

@@ -1,6 +1,9 @@
 const getIp = require('@abtnode/util/lib/get-ip');
 const logger = require('@abtnode/logger')('@abtnode/core:ip');
 
+const doRpc = require('./rpc');
+const { memoizeAsync } = require('./index');
+
 let cache = null;
 const fetch = async () => {
   try {
@@ -22,6 +25,9 @@ const cron = {
   fn: fetch,
 };
 
+const lookup = memoizeAsync((ip) => doRpc({ command: 'lookup', ip }));
+
 module.exports.fetch = fetch;
 module.exports.get = get;
 module.exports.cron = cron;
+module.exports.lookup = lookup;