@abtnode/core 1.8.65-beta-81d3340c → 1.8.65-beta-f7af64a4

package/lib/blocklet/manager/helper/install-from-backup.js

@@ -71,6 +71,8 @@ module.exports = async ({ url, blockletSecretKey, moveDir, context = {}, states,
     const skConfig = extra.configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
     if (skConfig) {
       skConfig.value = blockletSecretKey;
+      skConfig.secure = true;
+      skConfig.shared = false;
     } else {
       extra.configs.push({
         key: BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK,

package/lib/router/helper.js

@@ -21,6 +21,7 @@ const {
   DOMAIN_FOR_INTERNAL_SITE,
   WELLKNOWN_PATH_PREFIX,
   WELLKNOWN_SERVICE_PATH_PREFIX,
+  USER_AVATAR_PATH_PREFIX,
   DOMAIN_FOR_IP_SITE,
   NAME_FOR_WELLKNOWN_SITE,
   DEFAULT_HTTP_PORT,
@@ -210,6 +211,9 @@ const ensureLatestInterfaceInfo = async (sites = []) => {
       if (rule.isProtected && rule.to.target === WELLKNOWN_SERVICE_PATH_PREFIX) {
         return rule;
       }
+      if (rule.isProtected && rule.to.target === joinUrl(WELLKNOWN_SERVICE_PATH_PREFIX, USER_AVATAR_PATH_PREFIX)) {
+        return rule;
+      }
 
       const { did, interfaceName } = rule.to;
       if (interfaces[did] && interfaces[did][interfaceName]) {
@@ -273,6 +277,18 @@ const ensureWellknownRule = async (sites) => {
         if (!site.rules.some((x) => x.from.pathPrefix === servicePathPrefix)) {
           const rule = cloneDeep(rootBlockletRule || blockletRules[0]);
           rule.from.pathPrefix = servicePathPrefix;
+          rule.to.target = servicePathPrefix;
+          rule.isProtected = true;
+          site.rules.push(rule);
+        }
+
+        // Cache user avatar in nginx
+        const avatarPathPrefix = joinUrl(servicePathPrefix, USER_AVATAR_PATH_PREFIX);
+        if (!site.rules.some((x) => x.from.pathPrefix === avatarPathPrefix)) {
+          const rule = cloneDeep(rootBlockletRule || blockletRules[0]);
+          rule.from.pathPrefix = avatarPathPrefix;
+          rule.to.cacheGroup = 'blockletProxy';
+          rule.to.target = avatarPathPrefix;
           rule.isProtected = true;
           site.rules.push(rule);
         }
@@ -321,7 +337,12 @@ const filterSitesForRemovedBlocklets = async (sites = []) => {
     }
 
     const did = getDidFromDomainGroupName(site.domain);
-    return blocklets.some((x) => x.meta.did === did);
+    const blocklet = blocklets.find((x) => x.meta.did === did);
+    if (blocklet) {
+      site.mode = blocklet.mode;
+    }
+
+    return !!blocklet;
   });
 };
 
@@ -728,6 +749,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
             rules: [rule],
           },
           skipCheckDynamicBlacklist: true,
+          skipValidation: true,
         },
         context
       );
@@ -1124,7 +1146,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
       // Ensure we have system rules for blocklets
       const blocklets = await blockletState.getBlocklets();
       const ensureBlocklet = async (x) => {
-        const blocklet = await blockletManager.ensureBlocklet(x.meta.did);
+        const blocklet = await blockletManager.getBlocklet(x.meta.did);
         return ensureBlockletRouting(blocklet, context);
       };
       const ensureBlockletResults = await Promise.all(blocklets.map((x) => ensureBlocklet(x)));

package/lib/router/index.js

@@ -10,7 +10,7 @@ const {
   BLOCKLET_SITE_GROUP_SUFFIX,
   GATEWAY_REQ_LIMIT,
 } = require('@abtnode/constant');
-const { BLOCKLET_UI_INTERFACES } = require('@blocklet/constant');
+const { BLOCKLET_UI_INTERFACES, BLOCKLET_MODES } = require('@blocklet/constant');
 const logger = require('@abtnode/logger')('@abtnode/core:router');
 
 const expandSites = (sites = []) => {
@@ -208,6 +208,7 @@ Router.formatSites = (sites = []) => {
               port: daemonRule.to.port,
               did: rule.to.did,
               componentId: rule.to.componentId,
+              cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletJs' : '',
             },
           });
           site.rules.push({
@@ -221,6 +222,7 @@ Router.formatSites = (sites = []) => {
               port: daemonRule.to.port,
               did: rule.to.did,
               componentId: rule.to.componentId,
+              cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletJs' : '',
             },
           });
 
@@ -233,6 +235,7 @@ Router.formatSites = (sites = []) => {
               did: rule.to.did,
               type: ROUTING_RULE_TYPES.DAEMON,
               target: BLOCKLET_PROXY_PATH_PREFIX,
+              cacheGroup: site.mode === BLOCKLET_MODES.PRODUCTION ? 'blockletProxy' : '',
             },
           });
         }

package/lib/router/manager.js

@@ -8,6 +8,7 @@ const path = require('path');
 const os = require('os');
 const fse = require('fs-extra');
 const get = require('lodash/get');
+const toLower = require('lodash/toLower');
 const { EventEmitter } = require('events');
 const uuid = require('uuid');
 const isUrl = require('is-url');
@@ -154,20 +155,20 @@ class RouterManager extends EventEmitter {
 
   // eslint-disable-next-line no-unused-vars
   async updateRoutingSite(params, context = {}) {
-    await validateUpdateSite(params, context);
-    const site = await states.site.findOne({ _id: params.id });
-    if (!site) {
+    const site = await validateUpdateSite(params, context);
+    const existed = await states.site.findOne({ _id: site.id });
+    if (!existed) {
       throw new Error('Can not update non-existing site');
     }
 
     const updateSet = {};
 
-    if (params.corsAllowedOrigins) {
-      updateSet.corsAllowedOrigins = params.corsAllowedOrigins.filter((x) => x !== '__none__');
+    if (site.corsAllowedOrigins) {
+      updateSet.corsAllowedOrigins = site.corsAllowedOrigins.filter((x) => x !== '__none__');
     }
 
-    if (params.domain) {
-      const newDomain = params.domain;
+    if (site.domain) {
+      const newDomain = site.domain;
       const ruleCountInDB = await states.site.count({
         $or: [{ domain: newDomain }, { domainAliases: newDomain }, { 'domainAliases.value': newDomain }],
       });
@@ -179,18 +180,18 @@ class RouterManager extends EventEmitter {
       updateSet.domain = newDomain;
     }
 
-    const updated = await states.site.update({ _id: params.id }, { $set: updateSet }, { multi: false, upsert: false });
+    const updated = await states.site.update({ _id: site.id }, { $set: updateSet }, { multi: false, upsert: false });
 
-    logger.info('router.site.updated', { params, updated });
-    this.emit('router.site.updated', params.id);
+    logger.info('router.site.updated', { site, updated });
+    this.emit('router.site.updated', site.id);
 
-    const dbSite = await states.site.findOne({ _id: params.id });
+    const dbSite = await states.site.findOne({ _id: site.id });
     await attachRuntimeDomainAliases({ sites: dbSite, context, node: states.node });
     return dbSite;
   }
 
-  async addDomainAlias({ id, domainAlias, force }, context = {}) {
-    await validateAddDomainAlias(domainAlias, context);
+  async addDomainAlias({ id, domainAlias: tmpAlias, force }, context = {}) {
+    const domainAlias = await validateAddDomainAlias(tmpAlias, context);
     const dbSite = await states.site.findOne({ _id: id });
     if (!dbSite) {
       throw new Error(`site ${id} does not exist`);
@@ -222,10 +223,10 @@ class RouterManager extends EventEmitter {
       }
     }
 
-    // let custom domain in front of protected domain
-    const domainAliases = [{ value: domainAlias, isProtected: false }, ...(dbSite.domainAliases || [])];
-
-    const updateResult = await states.site.update({ _id: id }, { $set: { domainAliases } });
+    const updateResult = await states.site.update(
+      { _id: id },
+      { $push: { domainAliases: { value: domainAlias, isProtected: false } } }
+    );
     logger.debug('add domain alias update result', { id, updateResult, domainAlias });
 
     const newSite = await states.site.findOne({ _id: id });
@@ -234,8 +235,8 @@ class RouterManager extends EventEmitter {
     return newSite;
   }
 
-  async deleteDomainAlias({ id, domainAlias }, context = {}) {
-    await validateAddDomainAlias(domainAlias, context);
+  async deleteDomainAlias({ id, domainAlias: tmpAlias }, context = {}) {
+    const domainAlias = await validateAddDomainAlias(tmpAlias, context);
     const dbSite = await states.site.findOne({ _id: id });
     if (!dbSite) {
       throw new Error(`site ${id} does not exist`);
@@ -243,10 +244,10 @@ class RouterManager extends EventEmitter {
 
     dbSite.domainAliases = dbSite.domainAliases.filter((x) => {
       if (typeof x === 'string') {
-        return x !== domainAlias;
+        return toLower(x) !== domainAlias;
       }
 
-      return x.value !== domainAlias;
+      return toLower(x.value) !== domainAlias;
     });
 
     const updateResult = await states.site.update({ _id: id }, { $set: { domainAliases: dbSite.domainAliases } });

package/lib/states/blocklet.js

@@ -53,6 +53,9 @@ const formatBlocklet = (blocklet, phase, dek) => {
       if (!env) {
         return;
       }
+      // salt in blocklet state is different from the salt in blocklet-extra state
+      // in blocklet-extra state, salt is app meta did in each component
+      // in blocklet state, salt is component meta did in each component
       if (phase === 'onUpdate' && isHex(env.value) === true) {
         env.value = security.encrypt(env.value, b.meta.did, dek);
       }

package/lib/states/site.js

@@ -77,6 +77,7 @@ class SiteState extends BaseState {
 
   async findOneByBlocklet(did) {
     const result = await this.findOne({ domain: getBlockletDomainGroupName(did) });
+
     return BaseState.renameIdFiledName(result);
   }
 

package/lib/util/blocklet.js

@@ -142,8 +142,9 @@ const PRIVATE_NODE_ENVS = [
  */
 const getComponentDirs = (
   component,
-  { dataDirs, ensure = false, e2eMode = false, validate = true, ancestors = [] } = {}
+  { dataDirs, ensure = false, e2eMode = false, validate = false, ancestors = [] } = {}
 ) => {
+  // FIXME 这个函数做了太多的事
   // get data dirs
 
   const { name: appName } = ancestors.concat(component)[0].meta;
@@ -1360,9 +1361,8 @@ const getBlocklet = async ({
   dataDirs,
   states,
   e2eMode = false,
-  validateEnv = true,
   throwOnNotExist = true,
-  ensureDirs = true,
+  ensureIntegrity = false,
 } = {}) => {
   if (!did) {
     throw new Error('Blocklet did does not exist');
@@ -1381,7 +1381,7 @@ const getBlocklet = async ({
 
   const blocklet = await states.blocklet.getBlocklet(did);
   if (!blocklet) {
-    if (throwOnNotExist) {
+    if (throwOnNotExist || ensureIntegrity) {
       throw new Error(`can not find blocklet in database by did ${did}`);
     }
     return null;
@@ -1418,8 +1418,8 @@ const getBlocklet = async ({
       processId: getComponentProcessId(component, ancestors),
       ...getComponentDirs(component, {
         dataDirs,
-        ensure: ensureDirs,
-        validate: validateEnv,
+        ensure: ensureIntegrity,
+        validate: ensureIntegrity,
         ancestors,
         e2eMode: level === 0 ? e2eMode : false,
       }),
@@ -1554,6 +1554,9 @@ const createDataArchive = (dataDir, fileName) => {
 const validateAppConfig = async (config, blockletDid, states) => {
   const x = config;
 
+  // sk should be force secured while other app prop should not be secured
+  config.secure = x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK;
+
   if (x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK) {
     if (x.value) {
       try {

package/lib/validators/router.js

@@ -1,10 +1,12 @@
 /* eslint-disable newline-per-chained-call */
 const Joi = require('joi');
-const { DOMAIN_FOR_DEFAULT_SITE, ROUTING_RULE_TYPES } = require('@abtnode/constant');
+const { DOMAIN_FOR_DEFAULT_SITE, ROUTING_RULE_TYPES, ROUTER_CACHE_GROUPS } = require('@abtnode/constant');
 const { getMultipleLangParams } = require('./util');
 
 const WILDCARD_DOMAIN_REGEX = /^\*.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]/;
 
+const DOMAIN_SCHEMA = Joi.string().domain({ minDomainSegments: 1, tlds: false }).lowercase();
+
 const domainMessages = {
   en: {
     'alternatives.match': 'Illegal domain, please enter a domain like arcblock, arcblock.io, *.arcblock.io',
@@ -60,6 +62,12 @@ const ruleSchema = {
         then: Joi.required(),
       }),
     componentId: Joi.string().label('component id'), // component global id
+    // FUTURE: blocklets can register routing rules for provider cache
+    cacheGroup: Joi.string()
+      .label('cache group')
+      .valid(...Object.keys(ROUTER_CACHE_GROUPS))
+      .allow('')
+      .default(''),
   },
 
   // List of services that manipulate the request before the upstream blocklet
@@ -79,11 +87,7 @@ const ruleSchema = {
 };
 
 const corsSchema = Joi.array()
-  .items(
-    Joi.string().domain({ minDomainSegments: 1, tlds: false }),
-    Joi.string().valid(DOMAIN_FOR_DEFAULT_SITE, '__none__'),
-    Joi.string().ip()
-  )
+  .items(DOMAIN_SCHEMA, Joi.string().valid(DOMAIN_FOR_DEFAULT_SITE, '__none__'), Joi.string().ip())
   .min(1)
   .optional();
 
@@ -91,7 +95,7 @@ const ruleJoiSchema = Joi.object(ruleSchema);
 
 const addDomainAlias = Joi.alternatives()
   .try(
-    Joi.string().domain({ minDomainSegments: 1, tlds: false }),
+    DOMAIN_SCHEMA,
     Joi.string().regex(WILDCARD_DOMAIN_REGEX) // 这种其实是一种特殊的 tld
   )
   .required();
@@ -108,7 +112,7 @@ const updateAliases = domainAliases.required();
 const addSiteSchema = Joi.object({
   domain: Joi.alternatives()
     .try(
-      Joi.string().domain({ minDomainSegments: 1, tlds: false }),
+      DOMAIN_SCHEMA,
       Joi.string().valid('', DOMAIN_FOR_DEFAULT_SITE),
       Joi.string().regex(WILDCARD_DOMAIN_REGEX) // 这种其实是一种特殊的 tld
     )
@@ -125,7 +129,7 @@ const updateSite = Joi.object({
   corsAllowedOrigins: corsSchema,
   domain: Joi.alternatives()
     .try(
-      Joi.string().domain({ minDomainSegments: 1, tlds: false }),
+      DOMAIN_SCHEMA,
       Joi.string().regex(WILDCARD_DOMAIN_REGEX) // 这种其实是一种特殊的 tld
     )
     .optional()

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.8.65-beta-81d3340c",
+  "version": "1.8.65-beta-f7af64a4",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,18 +19,18 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/auth": "1.8.65-beta-81d3340c",
-    "@abtnode/certificate-manager": "1.8.65-beta-81d3340c",
-    "@abtnode/constant": "1.8.65-beta-81d3340c",
-    "@abtnode/cron": "1.8.65-beta-81d3340c",
-    "@abtnode/db": "1.8.65-beta-81d3340c",
-    "@abtnode/logger": "1.8.65-beta-81d3340c",
-    "@abtnode/queue": "1.8.65-beta-81d3340c",
-    "@abtnode/rbac": "1.8.65-beta-81d3340c",
-    "@abtnode/router-provider": "1.8.65-beta-81d3340c",
-    "@abtnode/static-server": "1.8.65-beta-81d3340c",
-    "@abtnode/timemachine": "1.8.65-beta-81d3340c",
-    "@abtnode/util": "1.8.65-beta-81d3340c",
+    "@abtnode/auth": "1.8.65-beta-f7af64a4",
+    "@abtnode/certificate-manager": "1.8.65-beta-f7af64a4",
+    "@abtnode/constant": "1.8.65-beta-f7af64a4",
+    "@abtnode/cron": "1.8.65-beta-f7af64a4",
+    "@abtnode/db": "1.8.65-beta-f7af64a4",
+    "@abtnode/logger": "1.8.65-beta-f7af64a4",
+    "@abtnode/queue": "1.8.65-beta-f7af64a4",
+    "@abtnode/rbac": "1.8.65-beta-f7af64a4",
+    "@abtnode/router-provider": "1.8.65-beta-f7af64a4",
+    "@abtnode/static-server": "1.8.65-beta-f7af64a4",
+    "@abtnode/timemachine": "1.8.65-beta-f7af64a4",
+    "@abtnode/util": "1.8.65-beta-f7af64a4",
     "@arcblock/did": "1.18.37",
     "@arcblock/did-motif": "^1.1.10",
     "@arcblock/did-util": "1.18.37",
@@ -38,9 +38,9 @@
     "@arcblock/jwt": "^1.18.37",
     "@arcblock/pm2-events": "^0.0.5",
     "@arcblock/vc": "1.18.37",
-    "@blocklet/constant": "1.8.65-beta-81d3340c",
-    "@blocklet/meta": "1.8.65-beta-81d3340c",
-    "@blocklet/sdk": "1.8.65-beta-81d3340c",
+    "@blocklet/constant": "1.8.65-beta-f7af64a4",
+    "@blocklet/meta": "1.8.65-beta-f7af64a4",
+    "@blocklet/sdk": "1.8.65-beta-f7af64a4",
     "@did-space/client": "^0.1.66",
     "@fidm/x509": "^1.2.1",
     "@ocap/mcrypto": "1.18.37",
@@ -88,5 +88,5 @@
     "express": "^4.18.2",
     "jest": "^27.5.1"
   },
-  "gitHead": "5a904b9192febd1d09a759d929ca4de639a50cd1"
+  "gitHead": "97607d6e12bf8508ac29ab6546110c4ae3b31a82"
 }