@abtnode/core 1.8.65-beta-5405baf2 → 1.8.65-beta-bfcc12ce

package/lib/router/helper.js

@@ -7,7 +7,7 @@ const get = require('lodash/get');
 const cloneDeep = require('lodash/cloneDeep');
 const isEqual = require('lodash/isEqual');
 const joinUrl = require('url-join');
-const { replaceSlotToIp } = require('@blocklet/meta/lib/util');
+const { replaceSlotToIp, findComponentById, findWebInterface } = require('@blocklet/meta/lib/util');
 const { getProvider } = require('@abtnode/router-provider');
 const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
 const getTmpDir = require('@abtnode/util/lib/get-tmp-directory');
@@ -21,6 +21,7 @@ const {
   DOMAIN_FOR_INTERNAL_SITE,
   WELLKNOWN_PATH_PREFIX,
   WELLKNOWN_SERVICE_PATH_PREFIX,
+  USER_AVATAR_PATH_PREFIX,
   DOMAIN_FOR_IP_SITE,
   NAME_FOR_WELLKNOWN_SITE,
   DEFAULT_HTTP_PORT,
@@ -40,9 +41,11 @@ const {
 const {
   BLOCKLET_DYNAMIC_PATH_PREFIX,
   BLOCKLET_INTERFACE_TYPE_WEB,
+  BLOCKLET_INTERFACE_PUBLIC,
   BLOCKLET_INTERFACE_WELLKNOWN,
   BLOCKLET_INTERFACE_TYPE_WELLKNOWN,
   BlockletEvents,
+  BLOCKLET_MODES,
 } = require('@blocklet/constant');
 
 // eslint-disable-next-line global-require
@@ -205,11 +208,17 @@ const ensureLatestInterfaceInfo = async (sites = []) => {
     }
 
     site.rules = site.rules.map((rule) => {
+      if (rule.dynamic) {
+        return rule;
+      }
       // If a rule already has a target and target is WELLKNOWN_SERVICE_PATH_PREFIX, just return
       // Indicates that the rule id generated by the system for auth service
       if (rule.isProtected && rule.to.target === WELLKNOWN_SERVICE_PATH_PREFIX) {
         return rule;
       }
+      if (rule.isProtected && rule.to.target === joinUrl(WELLKNOWN_SERVICE_PATH_PREFIX, USER_AVATAR_PATH_PREFIX)) {
+        return rule;
+      }
 
       const { did, interfaceName } = rule.to;
       if (interfaces[did] && interfaces[did][interfaceName]) {
@@ -273,7 +282,21 @@ const ensureWellknownRule = async (sites) => {
         if (!site.rules.some((x) => x.from.pathPrefix === servicePathPrefix)) {
           const rule = cloneDeep(rootBlockletRule || blockletRules[0]);
           rule.from.pathPrefix = servicePathPrefix;
+          rule.to.target = servicePathPrefix;
           rule.isProtected = true;
+          rule.dynamic = true; // mark as dynamic to avoid redundant generated rules
+          site.rules.push(rule);
+        }
+
+        // Cache user avatar in nginx
+        const avatarPathPrefix = joinUrl(servicePathPrefix, USER_AVATAR_PATH_PREFIX);
+        if (!site.rules.some((x) => x.from.pathPrefix === avatarPathPrefix)) {
+          const rule = cloneDeep(rootBlockletRule || blockletRules[0]);
+          rule.from.pathPrefix = avatarPathPrefix;
+          rule.to.cacheGroup = 'blockletProxy';
+          rule.to.target = avatarPathPrefix;
+          rule.isProtected = true;
+          rule.dynamic = true; // mark as dynamic to avoid redundant generated rules
           site.rules.push(rule);
         }
       }
@@ -296,7 +319,7 @@ const ensureBlockletDid = async (sites) => {
       return site;
     }
 
-    site.blockletDid = site.domain.replace(BLOCKLET_SITE_GROUP_SUFFIX, '');
+    site.blockletDid = getDidFromDomainGroupName(site.domain);
 
     return site;
   });
@@ -313,25 +336,76 @@ const ensureCorsForWebWallet = async (sites) => {
   return sites;
 };
 
-const filterSitesForRemovedBlocklets = async (sites = []) => {
-  const blocklets = await states.blocklet.getBlocklets();
+const filterSitesForRemovedBlocklets = async (sites = [], blocklets) => {
   return sites.filter((site) => {
     if (!site.domain.endsWith(BLOCKLET_SITE_GROUP_SUFFIX)) {
       return true;
     }
 
-    const did = getDidFromDomainGroupName(site.domain);
-    return blocklets.some((x) => x.meta.did === did);
+    const blocklet = blocklets.find((x) => x.meta.did === site.blockletDid);
+    return !!blocklet;
   });
 };
 
+const ensureBlockletCache = async (sites = [], blocklets) => {
+  return sites
+    .map((site) => {
+      if (!site.domain.endsWith(BLOCKLET_SITE_GROUP_SUFFIX)) {
+        return site;
+      }
+
+      if (site.cacheableGenerated) {
+        return site;
+      }
+
+      // For each rule, get component, check cacheable, clone and push new rule
+      const blocklet = blocklets.find((x) => x.meta.did === site.blockletDid);
+      const cacheRules = [];
+      site.rules
+        .filter(
+          (x) =>
+            x.to.type === ROUTING_RULE_TYPES.BLOCKLET &&
+            x.to.interfaceName === BLOCKLET_INTERFACE_PUBLIC &&
+            x.from.pathPrefix.startsWith(WELLKNOWN_SERVICE_PATH_PREFIX) === false
+        )
+        .forEach((rule) => {
+          const component = findComponentById(blocklet, rule.to.componentId);
+          if (component.mode !== BLOCKLET_MODES.PRODUCTION) {
+            return;
+          }
+          const cacheable = get(findWebInterface(component), 'cacheable', []);
+          cacheable.forEach((cachePrefix) => {
+            const clone = cloneDeep(rule);
+            clone.from.pathPrefix = joinUrl(rule.from.pathPrefix, cachePrefix);
+            clone.to.cacheGroup = 'blockletProxy';
+            clone.to.target = cachePrefix;
+            clone.dynamic = true; // mark as dynamic to avoid redundant generated rules
+            cacheRules.push(clone);
+          });
+        });
+
+      site.rules = site.rules.concat(cacheRules);
+      site.mode = blocklet.mode;
+      site.cacheableGenerated = true;
+
+      return site;
+    })
+    .filter(Boolean);
+};
+
 const ensureLatestInfo = async (sites = [], { withDefaultCors = true } = {}) => {
+  const blocklets = await states.blocklet.getBlocklets();
+
+  // CAUTION: following steps are very important, please do not change the order
   let result = await ensureLatestNodeInfo(sites, { withDefaultCors });
   result = await ensureBlockletDid(result);
+  result = await filterSitesForRemovedBlocklets(sites, blocklets);
+  result = await ensureBlockletCache(result, blocklets);
   result = await ensureWellknownRule(result);
   result = await ensureCorsForWebWallet(result);
+  result = await ensureLatestInterfaceInfo(result);
 
-  return ensureLatestInterfaceInfo(result);
+  return result;
 };
 
 const decompressCertificates = async (source, dest) => {
@@ -1043,7 +1117,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
           try {
             const info = await nodeState.read();
             let { sites } = await readRoutingSites();
-            sites = await filterSitesForRemovedBlocklets(sites);
             sites = await ensureLatestInfo(sites);
             sites = await ensureServiceRule(sites);
             sites = await ensureRootRule(sites);
@@ -1125,7 +1198,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       // Ensure we have system rules for blocklets
       const blocklets = await blockletState.getBlocklets();
       const ensureBlocklet = async (x) => {
-        const blocklet = await blockletManager.ensureBlocklet(x.meta.did);
+        const blocklet = await blockletManager.getBlocklet(x.meta.did);
         return ensureBlockletRouting(blocklet, context);
       };
       const ensureBlockletResults = await Promise.all(blocklets.map((x) => ensureBlocklet(x)));

package/lib/router/index.js

@@ -10,7 +10,7 @@ const {
   BLOCKLET_SITE_GROUP_SUFFIX,
   GATEWAY_REQ_LIMIT,
 } = require('@abtnode/constant');
-const { BLOCKLET_UI_INTERFACES } = require('@blocklet/constant');
+const { BLOCKLET_UI_INTERFACES, BLOCKLET_MODES } = require('@blocklet/constant');
 const logger = require('@abtnode/logger')('@abtnode/core:router');
 
 const expandSites = (sites = []) => {
@@ -192,6 +192,10 @@ Router.formatSites = (sites = []) => {
           return;
         }
 
+        if (rule.dynamic) {
+          return;
+        }
+
         if (daemonRule) {
           // Serve meta js: both prefix and suffix do not contain trailing slash
           // NOTICE: 这里隐含了一个约定
@@ -208,6 +212,7 @@ Router.formatSites = (sites = []) => {
               port: daemonRule.to.port,
               did: rule.to.did,
               componentId: rule.to.componentId,
+              cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletJs' : '',
             },
           });
           site.rules.push({
@@ -221,6 +226,7 @@ Router.formatSites = (sites = []) => {
               port: daemonRule.to.port,
               did: rule.to.did,
               componentId: rule.to.componentId,
+              cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletJs' : '',
             },
           });
 
@@ -233,6 +239,7 @@ Router.formatSites = (sites = []) => {
               did: rule.to.did,
               type: ROUTING_RULE_TYPES.DAEMON,
               target: BLOCKLET_PROXY_PATH_PREFIX,
+              cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletProxy' : '',
             },
           });
         }

package/lib/states/blocklet.js

@@ -53,6 +53,9 @@ const formatBlocklet = (blocklet, phase, dek) => {
       if (!env) {
         return;
       }
+      // salt in blocklet state is different from the salt in blocklet-extra state
+      // in blocklet-extra state, salt is app meta did in each component
+      // in blocklet state, salt is component meta did in each component
       if (phase === 'onUpdate' && isHex(env.value) === true) {
         env.value = security.encrypt(env.value, b.meta.did, dek);
       }
@@ -89,7 +92,7 @@ class BlockletState extends BaseState {
     this.defaultPort = config.blockletPort || 5555;
   }
 
-  getBlocklet(did) {
+  getBlocklet(did, { decryptSk = true } = {}) {
     return new Promise((resolve, reject) => {
       if (!did) {
         resolve(null);
@@ -100,7 +103,7 @@ class BlockletState extends BaseState {
           return reject(err);
         }
 
-        return resolve(doc ? formatBlocklet(doc, 'onRead', this.config.dek) : null);
+        return resolve(doc ? formatBlocklet(doc, 'onRead', decryptSk ? this.config.dek : null) : null);
       });
     });
   }

package/lib/util/blocklet.js

@@ -142,8 +142,9 @@ const PRIVATE_NODE_ENVS = [
  */
 const getComponentDirs = (
   component,
-  { dataDirs, ensure = false, e2eMode = false, validate = true, ancestors = [] } = {}
+  { dataDirs, ensure = false, e2eMode = false, validate = false, ancestors = [] } = {}
 ) => {
+  // FIXME 这个函数做了太多的事
   // get data dirs
 
   const { name: appName } = ancestors.concat(component)[0].meta;
@@ -930,7 +931,7 @@ const verifyIntegrity = async ({ file, integrity: expected }) => {
 
 /**
  * @param {string} installDir
- * @returns {Array<{ key: <[scope/]name/version>, dir: appDir }>}
+ * @returns {Promise<Array<{ key: string, dir: string }>>} key is <[scope/]name/version>, dir is appDir
  */
 const getAppDirs = async (installDir) => {
   const appDirs = [];
@@ -1360,9 +1361,8 @@ const getBlocklet = async ({
   dataDirs,
   states,
   e2eMode = false,
-  validateEnv = true,
   throwOnNotExist = true,
-  ensureDirs = true,
+  ensureIntegrity = false,
 } = {}) => {
   if (!did) {
     throw new Error('Blocklet did does not exist');
@@ -1381,7 +1381,7 @@ const getBlocklet = async ({
 
   const blocklet = await states.blocklet.getBlocklet(did);
   if (!blocklet) {
-    if (throwOnNotExist) {
+    if (throwOnNotExist || ensureIntegrity) {
       throw new Error(`can not find blocklet in database by did ${did}`);
     }
     return null;
@@ -1418,8 +1418,8 @@ const getBlocklet = async ({
       processId: getComponentProcessId(component, ancestors),
       ...getComponentDirs(component, {
         dataDirs,
-        ensure: ensureDirs,
-        validate: validateEnv,
+        ensure: ensureIntegrity,
+        validate: ensureIntegrity,
         ancestors,
         e2eMode: level === 0 ? e2eMode : false,
       }),
@@ -1554,6 +1554,9 @@ const createDataArchive = (dataDir, fileName) => {
 const validateAppConfig = async (config, blockletDid, states) => {
   const x = config;
 
+  // sk should be force secured while other app prop should not be secured
+  config.secure = x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK;
+
   if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK) {
     if (x.value) {
       try {

package/lib/validators/router.js

@@ -1,6 +1,6 @@
 /* eslint-disable newline-per-chained-call */
 const Joi = require('joi');
-const { DOMAIN_FOR_DEFAULT_SITE, ROUTING_RULE_TYPES } = require('@abtnode/constant');
+const { DOMAIN_FOR_DEFAULT_SITE, ROUTING_RULE_TYPES, ROUTER_CACHE_GROUPS } = require('@abtnode/constant');
 const { getMultipleLangParams } = require('./util');
 
 const WILDCARD_DOMAIN_REGEX = /^\*.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]/;
@@ -62,6 +62,12 @@ const ruleSchema = {
         then: Joi.required(),
       }),
     componentId: Joi.string().label('component id'), // component global id
+    // FUTURE: blocklets can register routing rules for provider cache
+    cacheGroup: Joi.string()
+      .label('cache group')
+      .valid(...Object.keys(ROUTER_CACHE_GROUPS))
+      .allow('')
+      .default(''),
   },
 
   // List of services that manipulate the request before the upstream blocklet

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.8.65-beta-5405baf2",
+  "version": "1.8.65-beta-bfcc12ce",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,33 +19,33 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/auth": "1.8.65-beta-5405baf2",
-    "@abtnode/certificate-manager": "1.8.65-beta-5405baf2",
-    "@abtnode/constant": "1.8.65-beta-5405baf2",
-    "@abtnode/cron": "1.8.65-beta-5405baf2",
-    "@abtnode/db": "1.8.65-beta-5405baf2",
-    "@abtnode/logger": "1.8.65-beta-5405baf2",
-    "@abtnode/queue": "1.8.65-beta-5405baf2",
-    "@abtnode/rbac": "1.8.65-beta-5405baf2",
-    "@abtnode/router-provider": "1.8.65-beta-5405baf2",
-    "@abtnode/static-server": "1.8.65-beta-5405baf2",
-    "@abtnode/timemachine": "1.8.65-beta-5405baf2",
-    "@abtnode/util": "1.8.65-beta-5405baf2",
-    "@arcblock/did": "1.18.37",
+    "@abtnode/auth": "1.8.65-beta-bfcc12ce",
+    "@abtnode/certificate-manager": "1.8.65-beta-bfcc12ce",
+    "@abtnode/constant": "1.8.65-beta-bfcc12ce",
+    "@abtnode/cron": "1.8.65-beta-bfcc12ce",
+    "@abtnode/db": "1.8.65-beta-bfcc12ce",
+    "@abtnode/logger": "1.8.65-beta-bfcc12ce",
+    "@abtnode/queue": "1.8.65-beta-bfcc12ce",
+    "@abtnode/rbac": "1.8.65-beta-bfcc12ce",
+    "@abtnode/router-provider": "1.8.65-beta-bfcc12ce",
+    "@abtnode/static-server": "1.8.65-beta-bfcc12ce",
+    "@abtnode/timemachine": "1.8.65-beta-bfcc12ce",
+    "@abtnode/util": "1.8.65-beta-bfcc12ce",
+    "@arcblock/did": "1.18.42",
     "@arcblock/did-motif": "^1.1.10",
-    "@arcblock/did-util": "1.18.37",
-    "@arcblock/event-hub": "1.18.37",
-    "@arcblock/jwt": "^1.18.37",
+    "@arcblock/did-util": "1.18.42",
+    "@arcblock/event-hub": "1.18.42",
+    "@arcblock/jwt": "^1.18.42",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/vc": "1.18.37",
-    "@blocklet/constant": "1.8.65-beta-5405baf2",
-    "@blocklet/meta": "1.8.65-beta-5405baf2",
-    "@blocklet/sdk": "1.8.65-beta-5405baf2",
+    "@arcblock/vc": "1.18.42",
+    "@blocklet/constant": "1.8.65-beta-bfcc12ce",
+    "@blocklet/meta": "1.8.65-beta-bfcc12ce",
+    "@blocklet/sdk": "1.8.65-beta-bfcc12ce",
     "@did-space/client": "^0.1.66",
     "@fidm/x509": "^1.2.1",
-    "@ocap/mcrypto": "1.18.37",
-    "@ocap/util": "1.18.37",
-    "@ocap/wallet": "1.18.37",
+    "@ocap/mcrypto": "1.18.42",
+    "@ocap/util": "1.18.42",
+    "@ocap/wallet": "1.18.42",
     "@slack/webhook": "^5.0.4",
     "archiver": "^5.3.1",
     "axios": "^0.27.2",
@@ -55,9 +55,11 @@
     "deep-diff": "^1.0.2",
     "detect-port": "^1.5.1",
     "escape-string-regexp": "^4.0.0",
+    "fast-glob": "^3.2.12",
     "flat": "^5.0.2",
     "fs-extra": "^11.1.0",
     "get-port": "^5.1.1",
+    "hasha": "^5.2.2",
     "is-base64": "^1.1.0",
     "is-ip": "^3.1.0",
     "is-url": "^1.2.4",
@@ -88,5 +90,5 @@
     "express": "^4.18.2",
     "jest": "^27.5.1"
   },
-  "gitHead": "e5dc838baded7031096118462a604fd70d78ff1c"
+  "gitHead": "f6a56512275a55feecf2f3953542ea98777a08f2"
 }