@abtnode/core 1.8.12 → 1.8.15

package/lib/blocklet/manager/disk.js

@@ -217,7 +217,7 @@ class BlockletManager extends BaseBlockletManager {
    * @param {{
    *  url: string;
    *  sync: boolean = false;
-   *  downloadToken: string;
+   *  downloadTokenList: Array<{did: string, token: string};
    * }} params
    * @param {{
    *  [key: string]: any
@@ -228,20 +228,16 @@ class BlockletManager extends BaseBlockletManager {
   async install(params, context = {}) {
     logger.debug('install blocklet', { params, context });
 
-    if (params.downloadToken) {
-      const info = await states.node.read();
+    const info = await states.node.read();
 
-      context.headers = Object.assign(context?.headers || {}, {
-        'x-server-did': info.did,
-        'x-download-token': params.downloadToken,
-        // FIXME: 先保证兼容性,后续删除
-        'x-server-publick-key': info.pk,
-        'x-server-public-key': info.pk,
-        'x-server-signature': sign(info.did, info.sk, {
-          exp: (Date.now() + 5 * 60 * 1000) / 1000,
-        }),
-      });
-    }
+    context.headers = Object.assign(context?.headers || {}, {
+      'x-server-did': info.did,
+      'x-server-public-key': info.pk,
+      'x-server-signature': sign(info.did, info.sk, {
+        exp: (Date.now() + 5 * 60 * 1000) / 1000,
+      }),
+    });
+    context.downloadTokenList = params.downloadTokenList || [];
 
     const source = getSourceFromInstallParams(params);
     if (typeof context.startImmediately === 'undefined') {
@@ -289,7 +285,7 @@ class BlockletManager extends BaseBlockletManager {
    * @param {ConfigEntry} configs pre configs
    */
   async installComponent(
-    { rootDid, mountPoint, url, file, did, diffVersion, deleteSet, title, name, configs, downloadToken },
+    { rootDid, mountPoint, url, file, did, diffVersion, deleteSet, title, name, configs, downloadTokenList },
     context = {}
   ) {
     logger.debug('start install component', { rootDid, mountPoint, url });
@@ -308,7 +304,7 @@ class BlockletManager extends BaseBlockletManager {
         did,
         name,
         configs,
-        downloadToken,
+        downloadTokenList,
       });
     }
 
@@ -533,7 +529,7 @@ class BlockletManager extends BaseBlockletManager {
     logger.info('blocklet reload successfully', { did });
 
     const res = await this.status(did);
-    this.emit(BlockletEvents.statusUpdated, res);
+    this.emit(BlockletEvents.statusChange, res);
     return res;
   }
 
@@ -1370,7 +1366,9 @@ class BlockletManager extends BaseBlockletManager {
         return blocklet;
       }
 
-      return states.blocklet.setBlockletStatus(did, BlockletStatus.stopped);
+      const res = await states.blocklet.setBlockletStatus(did, BlockletStatus.stopped);
+      this.emit(BlockletEvents.statusChange, res);
+      return res;
     };
 
     const blocklet = await this.ensureBlocklet(did);
@@ -1384,22 +1382,22 @@ class BlockletManager extends BaseBlockletManager {
       }
     }
 
+    if (!shouldUpdateStatus) {
+      return blocklet;
+    }
+
     try {
       const status = await getBlockletStatusFromProcess(blocklet);
-      if (shouldUpdateStatus) {
-        if (status === BlockletStatus.stopped) {
-          await states.blocklet.stopBlocklet(did);
-        }
-        return states.blocklet.setBlockletStatus(did, status);
+      if (blocklet.status !== status) {
+        const res = await states.blocklet.setBlockletStatus(did, status);
+        this.emit(BlockletEvents.statusChange, res);
+        return res;
       }
+
+      return blocklet;
     } catch (err) {
-      if (shouldUpdateStatus) {
-        return fastReturnOnForceSync(blocklet);
-      }
-      throw err;
+      return fastReturnOnForceSync(blocklet);
     }
-
-    return blocklet;
   }
 
   async getBlockletInterfaces({ blocklet, nodeInfo, context }) {
@@ -1887,7 +1885,7 @@ class BlockletManager extends BaseBlockletManager {
     name: inputName,
     did: inputDid,
     configs,
-    downloadToken,
+    downloadTokenList,
   }) {
     const blocklet = await this._getBlockletForInstallation(rootDid);
     if (!blocklet) {
@@ -1907,14 +1905,12 @@ class BlockletManager extends BaseBlockletManager {
         ...context,
         headers: {
           'x-server-did': info.did,
-          'x-download-token': downloadToken,
-          // FIXME: 先保证兼容性,后续删除
-          'x-server-publick-key': info.pk,
           'x-server-public-key': info.pk,
           'x-server-signature': sign(info.did, info.sk, {
             exp: (Date.now() + 5 * 60 * 1000) / 1000,
           }),
         },
+        downloadTokenList: downloadTokenList || [],
       };
     }
 
@@ -2301,7 +2297,7 @@ class BlockletManager extends BaseBlockletManager {
   async _syncBlockletStatus() {
     const run = async (blocklet) => {
       try {
-        await this.status(blocklet.meta.did, { forceSync: true });
+        await this.status(blocklet.meta.did);
       } catch (err) {
         logger.error('sync blocklet status failed', { error: err });
       }
@@ -2480,14 +2476,12 @@ class BlockletManager extends BaseBlockletManager {
         ...context,
         headers: {
           'x-server-did': info.did,
-          'x-download-token': blocklet?.tokens?.paidBlockletDownloadToken,
-          // FIXME: 先保证兼容性,后续删除
-          'x-server-publick-key': info.pk,
           'x-server-public-key': info.pk,
           'x-server-signature': sign(info.did, info.sk, {
             exp: (Date.now() + 5 * 60 * 1000) / 1000,
           }),
         },
+        downloadTokenList: blocklet?.tokens?.downloadTokenList || [],
       };
     }
 
@@ -2724,10 +2718,10 @@ class BlockletManager extends BaseBlockletManager {
       // Update dynamic component meta in blocklet settings
       await this._ensureDynamicChildrenInSettings(blocklet);
 
-      if (context?.headers?.['x-download-token']) {
+      if (context?.downloadTokenList?.length) {
         await states.blocklet.updateBlocklet(did, {
           tokens: {
-            paidBlockletDownloadToken: context.headers['x-download-token'],
+            downloadTokenList: context.downloadTokenList,
           },
         });
       }
@@ -2966,7 +2960,13 @@ class BlockletManager extends BaseBlockletManager {
       }
       ctrlStore[rootDid].set(did, cancelCtrl);
 
-      await downloadFile(url, path.join(cwd, tarballName), { cancelCtrl }, context);
+      const headers = context.headers ? cloneDeep(context.headers) : {};
+      const exist = (context.downloadTokenList || []).find((x) => x.did === did);
+      if (exist) {
+        headers['x-download-token'] = exist.token;
+      }
+
+      await downloadFile(url, path.join(cwd, tarballName), { cancelCtrl }, { ...context, headers });
 
       if (ctrlStore[rootDid]) {
         ctrlStore[rootDid].delete(did);

package/lib/states/blocklet.js

@@ -39,7 +39,9 @@ const formatBlocklet = (blocklet, phase, dek) => {
       fixInterfaces(b.meta);
     }
 
-    b.children = b.children || [];
+    if (phase === 'onRead') {
+      b.children = b.children || [];
+    }
 
     if (!b.environments || !b.meta || !dek) {
       return;

package/lib/states/webhook.js

@@ -8,7 +8,7 @@ class WebhookState extends BaseState {
     super(baseDir, { filename: 'webhook.db', ...options });
   }
 
-  async create(info) {
+  async create(info, { mock } = {}) {
     const { type, params } = info;
     const filterParams = params.map((item) => {
       const data = {
@@ -26,6 +26,10 @@ class WebhookState extends BaseState {
 
     await validateWebhook(data);
 
+    if (mock) {
+      return data;
+    }
+
     const webhook = await this.asyncDB.insert(data);
     return webhook;
   }

package/lib/util/blocklet.js

@@ -9,8 +9,6 @@ const streamToPromise = require('stream-to-promise');
 const { Throttle } = require('stream-throttle');
 const ssri = require('ssri');
 const diff = require('deep-diff');
-const any = require('promise.any');
-const joinUrl = require('url-join');
 
 const { toHex } = require('@ocap/util');
 const logger = require('@abtnode/logger')('@abtnode/core:util:blocklet');
@@ -22,7 +20,7 @@ const getFolderSize = require('@abtnode/util/lib/get-folder-size');
 const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
 const hashFiles = require('@abtnode/util/lib/hash-files');
 const isPathPrefixEqual = require('@abtnode/util/lib/is-path-prefix-equal');
-const { BLOCKLET_MAX_MEM_LIMIT_IN_MB, BLOCKLET_STORE_API_BLOCKLET_PREFIX } = require('@abtnode/constant');
+const { BLOCKLET_MAX_MEM_LIMIT_IN_MB } = require('@abtnode/constant');
 
 const {
   BlockletStatus,
@@ -55,19 +53,17 @@ const {
 } = require('@blocklet/meta/lib/util');
 const toBlockletDid = require('@blocklet/meta/lib/did');
 const { titleSchema, descriptionSchema } = require('@blocklet/meta/lib/schema');
+const {
+  getSourceUrlsFromConfig,
+  getBlockletMetaFromUrls,
+  getBlockletMetaFromUrl,
+} = require('@blocklet/meta/lib/util-meta');
 
 const { validate: validateEngine, get: getEngine } = require('../blocklet/manager/engine');
 
 const isRequirementsSatisfied = require('./requirement');
 const { getDidDomainForBlocklet } = require('./get-domain-for-blocklet');
-const {
-  isBeforeInstalled,
-  expandBundle,
-  getBlockletMetaByUrl,
-  validateUrl,
-  findInterfacePortByName,
-  validateBlockletMeta,
-} = require('./index');
+const { isBeforeInstalled, expandBundle, findInterfacePortByName, validateBlockletMeta } = require('./index');
 
 /**
  * get blocklet engine info, default is node
@@ -353,8 +349,8 @@ const getRuntimeEnvironments = (blocklet, nodeEnvironments, ancestors) => {
   for (const x of root.children || []) {
     mountPoints.push({
       title: x.meta.title,
-      did: x.meta.did,
-      name: x.meta.name,
+      did: x.meta.bundleDid,
+      name: x.meta.bundleName,
       mountPoint: x.mountPoint || '',
     });
   }
@@ -366,6 +362,7 @@ const getRuntimeEnvironments = (blocklet, nodeEnvironments, ancestors) => {
     ...devEnvironments,
     BLOCKLET_WEB_PORTS: JSON.stringify(ports),
     BLOCKLET_MOUNT_POINTS: JSON.stringify(mountPoints),
+    BLOCKLET_MODE: blocklet.mode || BLOCKLET_MODES.PRODUCTION,
     ...nodeEnvironments,
     ...safeNodeEnvironments,
   };
@@ -405,45 +402,6 @@ const getHealthyCheckTimeout = (blocklet, { checkHealthImmediately } = {}) => {
   };
 };
 
-const getBlockletMetaFromUrl = async (url) => {
-  const meta = await getBlockletMetaByUrl(url);
-  delete meta.htmlAst;
-
-  validateBlockletMeta(meta, { ensureDist: true });
-
-  try {
-    const { href } = new URL(meta.dist.tarball, url);
-    const tarball = decodeURIComponent(href);
-
-    try {
-      await validateUrl(tarball, ['application/octet-stream', 'application/x-gzip']);
-    } catch (error) {
-      if (!error.message.startsWith('Cannot get content-type')) {
-        throw error;
-      }
-    }
-    logger.info('resolve tarball url base on meta url', { meta: url, tarball });
-
-    meta.dist.tarball = tarball;
-  } catch (err) {
-    const msg = `Invalid blocklet meta: dist.tarball is not a valid url ${err.message}`;
-    logger.error(msg);
-    throw new Error(msg);
-  }
-
-  return meta;
-};
-
-const getBlockletMetaFromUrls = async (urls) => {
-  try {
-    const meta = await any(urls.map(getBlockletMetaFromUrl));
-    return meta;
-  } catch (err) {
-    logger.error('failed get blocklet meta', { urls, error: err });
-    throw new Error('Failed get blocklet meta');
-  }
-};
-
 /**
  * Start all precesses of a blocklet
  * @param {*} blocklet should contain env props
@@ -694,36 +652,6 @@ const parseChildren = (children, parentMeta = {}, { dynamic } = {}) => {
   return children;
 };
 
-/**
- * @param {*} config defined in childrenSchema in blocklet meta schema
- */
-const getSourceUrlsFromConfig = (config) => {
-  if (config.source) {
-    if (config.source.url) {
-      return [config.source.url].flat();
-    }
-
-    const { store, version, name } = config.source;
-    return [store]
-      .flat()
-      .map((x) =>
-        joinUrl(
-          x,
-          BLOCKLET_STORE_API_BLOCKLET_PREFIX,
-          toBlockletDid(name),
-          !version || version === 'latest' ? '' : version,
-          'blocklet.json'
-        )
-      );
-  }
-
-  if (config.resolved) {
-    return [config.resolved];
-  }
-
-  throw new Error('Invalid child config');
-};
-
 /**
  * this function has side effect on children
  */
@@ -751,7 +679,7 @@ const parseChildrenFromMeta = async (src, context = {}) => {
 
     let m;
     try {
-      m = await getBlockletMetaFromUrls(urls);
+      m = await getBlockletMetaFromUrls(urls, { logger });
     } catch {
       throw new Error(`Failed get component meta: ${config.title || config.name}`);
     }
@@ -1297,7 +1225,7 @@ const ensureMeta = (meta, { name, did } = {}) => {
 
 module.exports = {
   forEachBlocklet,
-  getBlockletMetaFromUrl,
+  getBlockletMetaFromUrl: (url) => getBlockletMetaFromUrl(url, { logger }),
   parseChildrenFromMeta,
   parseChildren,
   getComponentDirs,
@@ -1333,5 +1261,4 @@ module.exports = {
   needBlockletDownload,
   findAvailableDid,
   ensureMeta,
-  getSourceUrlsFromConfig,
 };

package/lib/util/index.js

@@ -36,8 +36,6 @@ const DEFAULT_WELLKNOWN_PORT = 8088;
 
 const logger = require('@abtnode/logger')('@abtnode/core:util');
 
-const request = require('./request');
-
 const validateOwner = (owner) => {
   try {
     return owner && owner.did && owner.pk && isFromPublicKey(owner.did, owner.pk);
@@ -351,66 +349,6 @@ const getBaseUrls = async (node, ips) => {
   }));
 };
 
-const getBlockletMetaByUrl = async (url) => {
-  const { protocol, pathname } = new URL(url);
-
-  if (protocol.startsWith('file')) {
-    const decoded = decodeURIComponent(pathname);
-    if (!fs.existsSync(decoded)) {
-      throw new Error(`File does not exist: ${decoded}`);
-    }
-    const d = await fs.promises.readFile(decoded);
-    const meta = JSON.parse(d);
-    return meta;
-  }
-
-  if (protocol.startsWith('http')) {
-    const { data: meta } = await request({ url, method: 'GET', timeout: 1000 * 20 });
-    if (Object.prototype.toString.call(meta) !== '[object Object]') {
-      throw new Error('Url is not valid');
-    }
-    return meta;
-  }
-
-  throw new Error(`Invalid url protocol: ${protocol.replace(/:$/, '')}`);
-};
-
-const validateUrl = async (url, expectedHttpResTypes = ['application/json', 'text/plain']) => {
-  const parsed = new URL(url);
-  const { protocol, pathname } = parsed;
-
-  // file
-  if (protocol.startsWith('file')) {
-    const decoded = decodeURIComponent(pathname);
-    if (!fs.existsSync(decoded)) {
-      throw new Error(`File does not exist: ${decoded}`);
-    }
-    return true;
-  }
-
-  // http(s)
-  if (protocol.startsWith('http')) {
-    let res;
-
-    try {
-      res = await request({ url, method: 'HEAD', timeout: 1000 * 10 });
-    } catch (err) {
-      throw new Error(`Cannot get content-type from ${url}: ${err.message}`);
-    }
-
-    if (
-      res.headers['content-type'] &&
-      expectedHttpResTypes.some((x) => res.headers['content-type'].includes(x)) === false
-    ) {
-      throw new Error(`Unexpected content-type from ${url}: ${res.headers['content-type']}`);
-    }
-
-    return true;
-  }
-
-  throw new Error(`Invalid url protocol: ${protocol.replace(/:$/, '')}`);
-};
-
 const expandBundle = (bundlePath, destDir) =>
   unzipper.Open.file(bundlePath).then((d) => d.extract({ path: destDir, concurrency: 20 }));
 
@@ -567,8 +505,6 @@ const lib = {
     fixAndValidateService(meta);
     return validateMeta(meta, opts);
   },
-  getBlockletMetaByUrl,
-  validateUrl,
   expandBundle,
   findInterfaceByName,
   findInterfacePortByName,

package/lib/webhook/index.js

@@ -134,13 +134,14 @@ module.exports = ({ events, dataDirs, instance }) => {
 
   const createWebhook = async (webhook) => {
     try {
-      const createRes = await webhookState.create(webhook);
-      await sentTextMessage(createRes, `A ${createRes.type} integration is now *successfully added*`);
+      const mockCreateRes = await webhookState.create(webhook, { mock: true });
+      await sentTextMessage(mockCreateRes, `A ${mockCreateRes.type} integration is now *successfully added*`);
 
+      const createRes = await webhookState.create(webhook);
       return createRes;
     } catch (err) {
-      logger.error('create webhook error', { err });
-      throw new Error(err.message);
+      logger.error('Failed to create webhook', { err });
+      throw new Error(`Failed to create webhook: ${err.message}`);
     }
   };
 

package/lib/webhook/sender/slack/index.js

@@ -29,7 +29,7 @@ class SlackSender extends BaseSender {
             type: 'section',
             text: {
               type: 'mrkdwn',
-              text: `*${nodeInfo.name}*`,
+              text: `*${nodeInfo?.name}*`,
             },
           },
           {

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.8.12",
+  "version": "1.8.15",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,32 +19,32 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/certificate-manager": "1.8.12",
-    "@abtnode/constant": "1.8.12",
-    "@abtnode/cron": "1.8.12",
-    "@abtnode/db": "1.8.12",
-    "@abtnode/logger": "1.8.12",
-    "@abtnode/queue": "1.8.12",
-    "@abtnode/rbac": "1.8.12",
-    "@abtnode/router-provider": "1.8.12",
-    "@abtnode/static-server": "1.8.12",
-    "@abtnode/timemachine": "1.8.12",
-    "@abtnode/util": "1.8.12",
-    "@arcblock/did": "1.17.12",
+    "@abtnode/certificate-manager": "1.8.15",
+    "@abtnode/constant": "1.8.15",
+    "@abtnode/cron": "1.8.15",
+    "@abtnode/db": "1.8.15",
+    "@abtnode/logger": "1.8.15",
+    "@abtnode/queue": "1.8.15",
+    "@abtnode/rbac": "1.8.15",
+    "@abtnode/router-provider": "1.8.15",
+    "@abtnode/static-server": "1.8.15",
+    "@abtnode/timemachine": "1.8.15",
+    "@abtnode/util": "1.8.15",
+    "@arcblock/did": "1.17.17",
     "@arcblock/did-motif": "^1.1.10",
-    "@arcblock/did-util": "1.17.12",
-    "@arcblock/event-hub": "1.17.12",
-    "@arcblock/jwt": "^1.17.12",
+    "@arcblock/did-util": "1.17.17",
+    "@arcblock/event-hub": "1.17.17",
+    "@arcblock/jwt": "^1.17.17",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/vc": "1.17.12",
-    "@blocklet/meta": "1.8.12",
-    "@blocklet/sdk": "1.8.12",
+    "@arcblock/vc": "1.17.17",
+    "@blocklet/meta": "1.8.15",
+    "@blocklet/sdk": "1.8.15",
     "@fidm/x509": "^1.2.1",
     "@nedb/core": "^1.3.4",
     "@nedb/multi": "^1.3.4",
-    "@ocap/mcrypto": "1.17.12",
-    "@ocap/util": "1.17.12",
-    "@ocap/wallet": "1.17.12",
+    "@ocap/mcrypto": "1.17.17",
+    "@ocap/util": "1.17.17",
+    "@ocap/wallet": "1.17.17",
     "@slack/webhook": "^5.0.4",
     "axios": "^0.27.2",
     "axon": "^2.0.3",
@@ -62,7 +62,6 @@
     "lodash": "^4.17.21",
     "lru-cache": "^6.0.0",
     "pm2": "^5.2.0",
-    "promise.any": "^2.0.4",
     "semver": "^7.3.7",
     "shelljs": "^0.8.5",
     "slugify": "^1.6.5",
@@ -82,5 +81,5 @@
     "express": "^4.18.1",
     "jest": "^27.5.1"
   },
-  "gitHead": "f4cb25a77f7ee48333b3243918a0a9c81b94ab8c"
+  "gitHead": "57106db1b559b906734df5b8eac7ce87cae72980"
 }