@abtnode/core 1.16.0-beta-b16cb035 → 1.16.0-beta-1d6c582e

package/lib/blocklet/manager/disk.js

@@ -4,6 +4,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const flat = require('flat');
 const get = require('lodash/get');
+const uniq = require('lodash/uniq');
 const merge = require('lodash/merge');
 const pick = require('lodash/pick');
 const cloneDeep = require('lodash/cloneDeep');
@@ -989,7 +990,7 @@ class BlockletManager extends BaseBlockletManager {
       throw new Error('configs list is not an array');
     }
 
-    const dids = Array.isArray(did) ? did : [did];
+    const dids = Array.isArray(did) ? uniq(did) : [did];
     const [rootDid, ...childDids] = dids;
     logger.info('config blocklet', { dids });
 

package/lib/blocklet/manager/helper/install-component-from-upload.js

@@ -138,7 +138,7 @@ const diff = async ({ did, hashFiles: clientFiles, rootDid: inputRootDid, states
 
   const rootBlocklet = await states.blocklet.getBlocklet(rootDid);
   if (childDid && !rootBlocklet) {
-    throw new Error('Root blocklet does not exist');
+    throw new Error(`Root blocklet does not exist: ${rootDid}`);
   }
 
   const state = childDid ? await (rootBlocklet.children || []).find((x) => x.meta.did === childDid) : rootBlocklet;

package/lib/blocklet/manager/helper/migrate-application-to-struct-v2.js

@@ -250,7 +250,7 @@ const migrateApplicationToStructV2 = async ({ did, appSk: newAppSk, context = {}
       // add root component to blockletData
       const { source, deployedFrom } = component;
       let bundleSource;
-      if (source === BlockletSource.store) {
+      if (source === BlockletSource.registry && deployedFrom && component.meta.bundleName) {
         bundleSource = {
           store: component.deployedFrom,
           name: component.meta.bundleName,
@@ -325,16 +325,16 @@ const migrateApplicationToStructV2 = async ({ did, appSk: newAppSk, context = {}
       Object.entries(sharedConfigObj).forEach(([key, value]) => {
         if (!extraData.configs.some((x) => x.key === key)) {
           logger.info('add shared config to container configs', { did, key, value });
-          extraData.configs.push({ key, value });
+          extraData.configs.push({ key, value, secure: false, shared: true });
         }
       });
     }
 
     // move component data dir
     const src = component.env.dataDir;
-    const dist = path.join(dataDirSrc, component.meta.bundleName);
-    if (src !== dist) {
-      dataMoveList.push({ src: component.env.dataDir, dist: path.join(dataDirDist, component.meta.bundleName) });
+    const dist = path.join(dataDirDist, component.meta.bundleName);
+    if (fs.existsSync(src)) {
+      dataMoveList.push({ src, dist });
     }
   });
 

package/lib/blocklet/manager/helper/upgrade-components.js

@@ -13,6 +13,7 @@ const {
   checkStructVersion,
   checkVersionCompatibility,
   validateBlocklet,
+  getFixedBundleSource,
 } = require('../../../util/blocklet');
 
 const check = async ({ did, states }) => {
@@ -24,7 +25,10 @@ const check = async ({ did, states }) => {
   const newChildren = [];
 
   for (const child of newBlocklet.children || []) {
-    if (child.bundleSource) {
+    // There may be dirty data without bundleSource but with source and deployedFrom
+    const bundleSource = getFixedBundleSource(child);
+
+    if (bundleSource) {
       const {
         staticComponents: [newChild],
         dynamicComponents,
@@ -32,7 +36,7 @@ const check = async ({ did, states }) => {
         meta: {
           staticComponents: [
             {
-              source: child.bundleSource,
+              source: bundleSource,
               name: child.meta.name,
               title: child.meta.title,
               mountPoint: child.mountPoint,

package/lib/blocklet/storage/backup/blocklet.js

@@ -1,6 +1,12 @@
-const { removeSync, outputJsonSync } = require('fs-extra');
+const { removeSync, outputJsonSync, createWriteStream, createReadStream } = require('fs-extra');
 const { cloneDeep } = require('lodash');
-const { join } = require('path');
+const { join, basename } = require('path');
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+const isEmpty = require('lodash/isEmpty');
+const streamToPromise = require('stream-to-promise');
+const axios = require('@abtnode/util/lib/axios');
+const isUrl = require('is-url');
+const { getLogoUrl } = require('@abtnode/util/lib/logo');
 const { BaseBackup } = require('./base');
 
 class BlockletBackup extends BaseBackup {
@@ -8,6 +14,10 @@ class BlockletBackup extends BaseBackup {
 
   async export() {
     const blocklet = await this.cleanData();
+
+    const targetLogoPath = await this.writeLogoFile();
+    blocklet.meta.appLogo = basename(targetLogoPath);
+
     removeSync(join(this.backupDir, this.filename));
     outputJsonSync(join(this.backupDir, this.filename), blocklet);
   }
@@ -65,6 +75,57 @@ class BlockletBackup extends BaseBackup {
 
     return info;
   }
+
+  /**
+   *
+   *
+   * @param {string} target
+   * @returns {Promise<string>}
+   * @memberof DataBackup
+   */
+  async writeLogoFile() {
+    const customLogoSquareUrl = this.blocklet.environments.find(
+      (e) => e.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_LOGO_SQUARE
+    )?.value;
+    const appDir = this.blocklet.environments.find((e) => e.key === 'BLOCKLET_APP_DIR')?.value;
+    const logo = this.blocklet?.meta?.logo;
+    const defaultLogoPath = join(this.serverDir, 'data', this.blocklet.meta.name, 'logo.svg');
+
+    const logoUrl = await getLogoUrl({
+      customLogoSquareUrl,
+      appDir,
+      logo,
+      defaultLogoPath,
+    });
+
+    const logoStream = await this.getLogoStream(logoUrl);
+    const targetLogoPath = join(this.backupDir, 'data', basename(logoUrl));
+    await streamToPromise(logoStream.pipe(createWriteStream(targetLogoPath)));
+
+    return targetLogoPath;
+  }
+
+  /**
+   *
+   *
+   * @param {string} logoUrl
+   * @returns {Promise<NodeJS.ReadStream>}
+   * @memberof DataBackup
+   */
+  async getLogoStream(logoUrl) {
+    if (isEmpty(logoUrl)) {
+      throw new Error(`logoUrl(${logoUrl}) cannot be empty`);
+    }
+
+    if (isUrl(logoUrl)) {
+      const res = await axios.get(logoUrl, {
+        responseType: 'stream',
+      });
+      return res.data;
+    }
+
+    return createReadStream(logoUrl);
+  }
 }
 
 module.exports = { BlockletBackup };

package/lib/blocklet/storage/backup/spaces.js

@@ -98,7 +98,6 @@ class SpacesBackup {
       new BlockletsBackup(this.input),
       new BlockletExtrasBackup(this.input),
       new RoutingRuleBackup(this.input),
-      new DataBackup(this.input),
     ];
   }
 
@@ -151,12 +150,19 @@ class SpacesBackup {
       progress: 15,
       completed: false,
     });
+
+    // @note: dataBackup 需要先于 blockletBackup 执行，并且 blockletBackup 与其他 backup的执行可以是无序的
+    const dataBackup = new DataBackup(this.input);
+    dataBackup.ensureParams(this);
+    await dataBackup.export();
+
     await Promise.all(
       this.storages.map((storage) => {
         storage.ensureParams(this);
         return storage.export();
       })
     );
+
     this.input.event.emit(BlockletEvents.backupProgress, {
       appDid: this.input.appDid,
       message: 'Data ready, start backup...',
@@ -201,7 +207,7 @@ class SpacesBackup {
           const percent = (data.completed * 100) / data.total;
           this.input.event.emit(BlockletEvents.backupProgress, {
             appDid: this.input.appDid,
-            message: `Uploaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+            message: `Uploading file ${basename(data.key)} (${data.completed}/${data.total})`,
             // 0.8 是因为上传文件到 spaces 占进度的 80%，+ 20 是因为需要累加之前的进度
             progress: +Math.ceil(percent * 0.8).toFixed(2) + 20,
             completed: false,

package/lib/blocklet/storage/restore/spaces.js

@@ -120,7 +120,7 @@ class SpacesRestore {
           logger.info('restore progress', { appDid: this.input.appDid, data });
           this.input.event.emit(BlockletEvents.restoreProgress, {
             appDid: this.input.appDid,
-            message: `Downloaded file ${basename(data.key)} (${data.completed}/${data.total})`,
+            message: `Downloading file ${basename(data.key)} (${data.completed}/${data.total})`,
           });
         },
 

package/lib/router/helper.js

@@ -204,6 +204,7 @@ const ensureLatestNodeInfo = async (sites = [], { withDefaultCors = true } = {})
  * set rule.to.target by interface.path and interface.prefix
  */
 const ensureLatestInterfaceInfo = async (sites = []) => {
+  // FIXME @linchen groupAllInterfaces() may have performance issue
   const interfaces = await states.blocklet.groupAllInterfaces();
   return sites.map((site) => {
     if (!Array.isArray(site.rules)) {
@@ -223,10 +224,10 @@ const ensureLatestInterfaceInfo = async (sites = []) => {
         return rule;
       }
 
-      const { did, interfaceName } = rule.to;
-      if (interfaces[did] && interfaces[did][interfaceName]) {
+      const { componentId, interfaceName } = rule.to;
+      if (interfaces[componentId] && interfaces[componentId][interfaceName]) {
         // eslint-disable-next-line no-shadow
-        const { path, prefix } = interfaces[did][interfaceName];
+        const { path, prefix } = interfaces[componentId][interfaceName];
         if (prefix === BLOCKLET_DYNAMIC_PATH_PREFIX) {
           rule.to.target = path;
         } else {

package/lib/states/blocklet-extras.js

@@ -5,6 +5,7 @@ const logger = require('@abtnode/logger')('state-blocklet-extras');
 const { EXPIRED_BLOCKLET_DATA_RETENTION_DAYS } = require('@abtnode/constant');
 const camelCase = require('lodash/camelCase');
 const get = require('lodash/get');
+const uniq = require('lodash/uniq');
 const dayjs = require('dayjs');
 
 const BaseState = require('./base');
@@ -127,7 +128,7 @@ class BlockletExtrasState extends BaseState {
   generateGetFn(extra) {
     return async (dids, path, defaultValue) => {
       // eslint-disable-next-line no-param-reassign
-      dids = [].concat(dids);
+      dids = uniq([].concat(dids));
       const [rootDid, ...childDids] = dids;
       const { dek } = this.config;
       const { name, afterGet = noop('data') } = extra;
@@ -149,7 +150,7 @@ class BlockletExtrasState extends BaseState {
   generateSetFn(extra) {
     return async (dids, data) => {
       // eslint-disable-next-line no-param-reassign
-      dids = [].concat(dids);
+      dids = uniq([].concat(dids));
       const [rootDid, ...childDids] = dids;
       const { dek } = this.config;
       const { name, beforeSet = noop('cur') } = extra;
@@ -189,7 +190,7 @@ class BlockletExtrasState extends BaseState {
   generateDelFn(extra) {
     return async (dids) => {
       // eslint-disable-next-line no-param-reassign
-      dids = [].concat(dids);
+      dids = uniq([].concat(dids));
       const [rootDid, ...childDids] = dids;
       const { name } = extra;
       const item = await this.findOne({ did: rootDid });

package/lib/states/blocklet.js

@@ -509,17 +509,21 @@ class BlockletState extends BaseState {
    * @return {Object} { <did> : { interfaceName } }
    */
   async groupAllInterfaces() {
-    const blocklets = await this.getBlocklets({}, { meta: 1 });
+    const blocklets = await this.getBlocklets({}, { meta: 1, children: 1 });
     const result = {};
-    blocklets.forEach((blocklet) => {
-      const { did, interfaces } = blocklet.meta;
-      if (typeof result[did] === 'undefined') {
-        result[did] = {};
+    const fillResult = (component, { id }) => {
+      const { interfaces } = component.meta;
+      if (typeof result[id] === 'undefined') {
+        result[id] = {};
       }
 
       (interfaces || []).forEach((x) => {
-        result[did][x.name] = x;
+        result[id][x.name] = x;
       });
+    };
+
+    blocklets.forEach((blocklet) => {
+      forEachBlockletSync(blocklet, fillResult);
     });
 
     return result;

package/lib/util/blocklet.js

@@ -1825,6 +1825,38 @@ const getBlockletKnownAs = (blocklet) => {
   return alsoKnownAs.filter(Boolean).map(toDid);
 };
 
+const getFixedBundleSource = (component) => {
+  if (!component) {
+    return null;
+  }
+
+  if (component.bundleSource) {
+    return component.bundleSource;
+  }
+
+  const { source, deployedFrom, meta: { bundleName } = {} } = component;
+
+  if (!deployedFrom) {
+    return null;
+  }
+
+  if (source === BlockletSource.registry && bundleName) {
+    return {
+      store: deployedFrom,
+      name: bundleName,
+      version: 'latest',
+    };
+  }
+
+  if (source === BlockletSource.url) {
+    return {
+      url: deployedFrom,
+    };
+  }
+
+  return null;
+};
+
 module.exports = {
   consumeServerlessNFT,
   forEachBlocklet,
@@ -1879,4 +1911,5 @@ module.exports = {
   checkVersionCompatibility,
   validateBlockletMeta,
   getBlockletKnownAs,
+  getFixedBundleSource,
 };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.16.0-beta-b16cb035",
+  "version": "1.16.0-beta-1d6c582e",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,34 +19,34 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/auth": "1.16.0-beta-b16cb035",
-    "@abtnode/certificate-manager": "1.16.0-beta-b16cb035",
-    "@abtnode/constant": "1.16.0-beta-b16cb035",
-    "@abtnode/cron": "1.16.0-beta-b16cb035",
-    "@abtnode/db": "1.16.0-beta-b16cb035",
-    "@abtnode/logger": "1.16.0-beta-b16cb035",
-    "@abtnode/queue": "1.16.0-beta-b16cb035",
-    "@abtnode/rbac": "1.16.0-beta-b16cb035",
-    "@abtnode/router-provider": "1.16.0-beta-b16cb035",
-    "@abtnode/static-server": "1.16.0-beta-b16cb035",
-    "@abtnode/timemachine": "1.16.0-beta-b16cb035",
-    "@abtnode/util": "1.16.0-beta-b16cb035",
-    "@arcblock/did": "1.18.59",
+    "@abtnode/auth": "1.16.0-beta-1d6c582e",
+    "@abtnode/certificate-manager": "1.16.0-beta-1d6c582e",
+    "@abtnode/constant": "1.16.0-beta-1d6c582e",
+    "@abtnode/cron": "1.16.0-beta-1d6c582e",
+    "@abtnode/db": "1.16.0-beta-1d6c582e",
+    "@abtnode/logger": "1.16.0-beta-1d6c582e",
+    "@abtnode/queue": "1.16.0-beta-1d6c582e",
+    "@abtnode/rbac": "1.16.0-beta-1d6c582e",
+    "@abtnode/router-provider": "1.16.0-beta-1d6c582e",
+    "@abtnode/static-server": "1.16.0-beta-1d6c582e",
+    "@abtnode/timemachine": "1.16.0-beta-1d6c582e",
+    "@abtnode/util": "1.16.0-beta-1d6c582e",
+    "@arcblock/did": "1.18.62",
     "@arcblock/did-motif": "^1.1.10",
-    "@arcblock/did-util": "1.18.59",
-    "@arcblock/event-hub": "1.18.59",
-    "@arcblock/jwt": "^1.18.59",
+    "@arcblock/did-util": "1.18.62",
+    "@arcblock/event-hub": "1.18.62",
+    "@arcblock/jwt": "^1.18.62",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/vc": "1.18.59",
-    "@blocklet/constant": "1.16.0-beta-b16cb035",
-    "@blocklet/meta": "1.16.0-beta-b16cb035",
-    "@blocklet/sdk": "1.16.0-beta-b16cb035",
-    "@did-space/client": "^0.2.33",
+    "@arcblock/vc": "1.18.62",
+    "@blocklet/constant": "1.16.0-beta-1d6c582e",
+    "@blocklet/meta": "1.16.0-beta-1d6c582e",
+    "@blocklet/sdk": "1.16.0-beta-1d6c582e",
+    "@did-space/client": "^0.2.40",
     "@fidm/x509": "^1.2.1",
-    "@ocap/client": "1.18.59",
-    "@ocap/mcrypto": "1.18.59",
-    "@ocap/util": "1.18.59",
-    "@ocap/wallet": "1.18.59",
+    "@ocap/client": "1.18.62",
+    "@ocap/mcrypto": "1.18.62",
+    "@ocap/util": "1.18.62",
+    "@ocap/wallet": "1.18.62",
     "@slack/webhook": "^5.0.4",
     "archiver": "^5.3.1",
     "axios": "^0.27.2",
@@ -91,5 +91,5 @@
     "express": "^4.18.2",
     "jest": "^27.5.1"
   },
-  "gitHead": "138bd91aee5a35b28c2de4e1e924c44932efaf3a"
+  "gitHead": "209fa1413ae05d8961942b2c21f2d83df66f4b68"
 }