@abtnode/core 1.15.17 → 1.16.0-beta-8ee536d7

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.15.17",
+  "version": "1.16.0-beta-8ee536d7",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,59 +19,77 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/constant": "1.15.17",
-    "@abtnode/cron": "1.15.17",
-    "@abtnode/logger": "1.15.17",
-    "@abtnode/queue": "1.15.17",
-    "@abtnode/rbac": "1.15.17",
-    "@abtnode/router-provider": "1.15.17",
-    "@abtnode/static-server": "1.15.17",
-    "@abtnode/timemachine": "1.15.17",
-    "@abtnode/util": "1.15.17",
-    "@arcblock/did": "^1.13.61",
-    "@arcblock/event-hub": "1.13.61",
+    "@abtnode/auth": "1.16.0-beta-8ee536d7",
+    "@abtnode/certificate-manager": "1.16.0-beta-8ee536d7",
+    "@abtnode/constant": "1.16.0-beta-8ee536d7",
+    "@abtnode/cron": "1.16.0-beta-8ee536d7",
+    "@abtnode/db": "1.16.0-beta-8ee536d7",
+    "@abtnode/logger": "1.16.0-beta-8ee536d7",
+    "@abtnode/queue": "1.16.0-beta-8ee536d7",
+    "@abtnode/rbac": "1.16.0-beta-8ee536d7",
+    "@abtnode/router-provider": "1.16.0-beta-8ee536d7",
+    "@abtnode/static-server": "1.16.0-beta-8ee536d7",
+    "@abtnode/timemachine": "1.16.0-beta-8ee536d7",
+    "@abtnode/util": "1.16.0-beta-8ee536d7",
+    "@arcblock/did": "1.18.62",
+    "@arcblock/did-motif": "^1.1.10",
+    "@arcblock/did-util": "1.18.62",
+    "@arcblock/event-hub": "1.18.62",
+    "@arcblock/jwt": "^1.18.62",
     "@arcblock/pm2-events": "^0.0.5",
-    "@arcblock/vc": "^1.13.61",
-    "@blocklet/meta": "1.15.17",
+    "@arcblock/vc": "1.18.62",
+    "@blocklet/constant": "1.16.0-beta-8ee536d7",
+    "@blocklet/meta": "1.16.0-beta-8ee536d7",
+    "@blocklet/sdk": "1.16.0-beta-8ee536d7",
+    "@did-space/client": "^0.2.33",
     "@fidm/x509": "^1.2.1",
-    "@nedb/core": "^1.2.2",
-    "@nedb/multi": "^1.2.2",
-    "@ocap/mcrypto": "^1.13.61",
-    "@ocap/util": "^1.13.61",
-    "@ocap/wallet": "^1.13.61",
-    "@slack/webhook": "^5.0.3",
-    "axios": "^0.21.4",
+    "@ocap/client": "1.18.62",
+    "@ocap/mcrypto": "1.18.62",
+    "@ocap/util": "1.18.62",
+    "@ocap/wallet": "1.18.62",
+    "@slack/webhook": "^5.0.4",
+    "archiver": "^5.3.1",
+    "axios": "^0.27.2",
     "axon": "^2.0.3",
-    "chalk": "^4.0.0",
+    "chalk": "^4.1.2",
+    "dayjs": "^1.11.7",
     "deep-diff": "^1.0.2",
-    "detect-port": "^1.3.0",
+    "detect-port": "^1.5.1",
+    "escape-string-regexp": "^4.0.0",
+    "fast-glob": "^3.2.12",
     "flat": "^5.0.2",
-    "fs-extra": "^10.0.0",
+    "fs-extra": "^11.1.0",
     "get-port": "^5.1.1",
+    "hasha": "^5.2.2",
     "is-base64": "^1.1.0",
     "is-ip": "^3.1.0",
     "is-url": "^1.2.4",
-    "joi": "^17.4.0",
-    "js-yaml": "^3.14.0",
+    "joi": "17.7.0",
+    "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
     "lru-cache": "^6.0.0",
-    "pm2": "^4.2.3",
-    "semver": "^7.3.2",
-    "shelljs": "^0.8.4",
-    "slugify": "^1.4.6",
-    "ssri": "^8.0.0",
+    "node-stream-zip": "^1.15.0",
+    "p-limit": "^3.1.0",
+    "p-retry": "4.6.1",
+    "pm2": "^5.2.0",
+    "semver": "^7.3.8",
+    "shelljs": "^0.8.5",
+    "ssri": "^8.0.1",
     "stream-throttle": "^0.1.3",
     "stream-to-promise": "^3.0.0",
-    "tar": "^6.1.0",
+    "systeminformation": "^5.12.6",
+    "tar": "^6.1.11",
+    "ua-parser-js": "^1.0.2",
     "unzipper": "^0.10.11",
     "url-join": "^4.0.1",
-    "uuid": "7.0.3"
+    "uuid": "7.0.3",
+    "valid-url": "^1.0.9"
   },
   "devDependencies": {
     "compression": "^1.7.4",
     "expand-tilde": "^2.0.2",
-    "express": "^4.17.1",
-    "jest": "^27.3.1"
+    "express": "^4.18.2",
+    "jest": "^27.5.1"
   },
-  "gitHead": "22715c3ea74d0230f3413162a17f491614b6735a"
+  "gitHead": "57d0c45be311a5fbc1c0fffa2814b62c1a3ee34c"
 }

package/lib/blocklet/registry.js

@@ -1,205 +0,0 @@
-const isBase64 = require('is-base64');
-const { BlockletGroup } = require('@blocklet/meta/lib/constants');
-const joinURL = require('url-join');
-const get = require('lodash/get');
-const pick = require('lodash/pick');
-const { BLOCKLET_STORE_API_PREFIX } = require('@abtnode/constant');
-
-const { name } = require('../../package.json');
-
-const logger = require('@abtnode/logger')(`${name}:blocklet:registry`); // eslint-disable-line
-
-const request = require('../util/request');
-
-const states = require('../states');
-const isRequirementsSatisfied = require('../util/requirement');
-const { fixAndVerifyBlockletMeta } = require('../util/blocklet');
-const { translate } = require('../locales');
-
-const DEFAULT_REFRESH_INTERVAL = 1 * 60 * 1000;
-const MAX_REFRESH_INTERVAL = 1 * 60 * 1000;
-
-class BlockletRegistry {
-  constructor() {
-    this.blocklets = [];
-    this.cacheId = '';
-  }
-
-  getCron(refreshInterval = DEFAULT_REFRESH_INTERVAL) {
-    const interval = Math.min(Math.max(Number(refreshInterval), DEFAULT_REFRESH_INTERVAL), MAX_REFRESH_INTERVAL); // Should between 20 ~ 60 seconds
-    return {
-      name: 'refetch-blocklet-registry',
-      time: `*/${Math.round(interval / 1000)} * * * * *`,
-      fn: this._init.bind(this),
-    };
-  }
-
-  async listBlocklets(input, context) {
-    const registryUrl = await states.node.getBlockletRegistry();
-
-    if (input && input.registryUrl && registryUrl !== input.registryUrl) {
-      this.blocklets = [];
-      this.cacheId = '';
-    }
-
-    if (this.blocklets.length > 0) {
-      return this.blocklets.filter((x) => isRequirementsSatisfied(x.requirements, false));
-    }
-    return this.refreshBlocklets(context);
-  }
-
-  async hasBlocklet(id) {
-    const list = await this.listBlocklets();
-    return list.some((x) => x.did === id || x.name === id);
-  }
-
-  getBlocklet(id, registryUrl) {
-    return this.getBlockletMeta({ did: id, registryUrl });
-  }
-
-  // eslint-disable-next-line no-unused-vars
-  async getBlockletMeta({ did, registryUrl }, context) {
-    const defaultRegistryUrl = await states.node.getBlockletRegistry();
-
-    const url = joinURL(
-      registryUrl || defaultRegistryUrl,
-      BLOCKLET_STORE_API_PREFIX,
-      `/blocklets/${did}/blocklet.json?__t__=${Date.now()}`
-    );
-
-    const { data } = await request.get(url);
-    try {
-      return fixAndVerifyBlockletMeta(data, did);
-    } catch (err) {
-      logger.error('failed to get blocklet meta', { did, data, error: err });
-      throw err;
-    }
-  }
-
-  async resolveTarballURL({ did, tarball = '', registryUrl = '' }) {
-    if (!tarball) {
-      return '';
-    }
-
-    if (tarball.startsWith('file://')) {
-      return decodeURIComponent(tarball);
-    }
-
-    if (tarball.startsWith('http://') || tarball.startsWith('https://')) {
-      return tarball;
-    }
-
-    const defaultRegistryUrl = await states.node.getBlockletRegistry();
-    return joinURL(registryUrl || defaultRegistryUrl, 'api', 'blocklets', did, tarball);
-  }
-
-  clearCache() {
-    this.cacheId = '';
-    this.blocklets = [];
-  }
-
-  async _init() {
-    states.node.onReady(async () => {
-      this.refreshBlocklets().catch(
-        (error) => logger.error('refresh blocklets failed on initialize the registry', { error })
-        // eslint-disable-next-line function-paren-newline
-      );
-    });
-  }
-
-  async refreshBlocklets(context) {
-    const registryUrl = await states.node.getBlockletRegistry();
-    const url = joinURL(registryUrl, BLOCKLET_STORE_API_PREFIX, '/blocklets.json');
-    try {
-      let res = await request.get(url, {
-        validateStatus: (status) => (status >= 200 && status < 300) || status === 304,
-        headers: {
-          'If-None-Match': this.cacheId,
-        },
-      });
-
-      if (res.status === 304 && this.blocklets.length > 0) {
-        logger.debug('use cached data', { etag: res.headers.etag, registryUrl });
-        return this.blocklets;
-      }
-
-      if (res.status === 304) {
-        res = await request.get(url, {
-          headers: {
-            'Cache-Control': 'no-store',
-          },
-        });
-
-        logger.debug('re-fetch from registry', { status: res.status });
-      }
-
-      if (!Array.isArray(res.data)) {
-        logger.error('Blocklet list fetch failed, response data is not a list', { url, data: res.data });
-        throw new Error('Blocklet list fetch failed');
-      }
-
-      const blocklets = res.data.filter((x) => BlockletGroup[x.group]);
-      logger.debug('blocklets loaded', { url, blocklets: blocklets.map((x) => ({ did: x.did, name: x.name })) });
-      this.cacheId = res.headers.etag;
-      this.blocklets = blocklets;
-    } catch (error) {
-      logger.error('refresh blocklet registry failed', { error, registryUrl });
-      this.cacheId = '';
-      throw new Error(translate(get(context, 'query.locale', 'en'), 'registry.getListError', { registryUrl }));
-    }
-
-    return this.blocklets;
-  }
-}
-
-BlockletRegistry.validateRegistryURL = async (registry) => {
-  const url = joinURL(registry, BLOCKLET_STORE_API_PREFIX, `/blocklets.json?__t__=${Date.now()}`);
-  try {
-    const res = await request.get(url);
-    if (Array.isArray(res.data)) {
-      return res.data;
-    }
-
-    logger.error('Blocklet list fetch failed ', { url, data: res.data });
-    throw new Error('blocklet list fetch failed');
-  } catch (error) {
-    logger.error('Blocklet registry refresh failed', { url, error });
-    throw new Error(`Invalid Blocklet Registry URL ${registry}: ${error.message}`);
-  }
-};
-
-BlockletRegistry.getRegistryMeta = async (registry) => {
-  try {
-    const url = joinURL(registry, BLOCKLET_STORE_API_PREFIX, `/registry.json?__t__=${Date.now()}`);
-    const { data } = await request.get(url);
-
-    if (!data) {
-      return {};
-    }
-
-    const requiredFields = ['name', 'description', 'maintainer'];
-
-    const missingFields = requiredFields.filter((x) => !data[x]);
-    if (missingFields.length > 0) {
-      throw new Error(`the registry missing required information: ${missingFields.join(', ')}`);
-    }
-
-    const result = pick(data, ['id', 'name', 'description', 'maintainer', 'cdnUrl', 'chainHost']);
-    const { logoUrl } = data;
-    if (logoUrl) {
-      if (logoUrl.startsWith('http') === true) {
-        result.logoUrl = logoUrl;
-      } else if (isBase64(logoUrl, { allowMime: true })) {
-        result.logoUrl = logoUrl;
-      } else {
-        result.logoUrl = joinURL(registry, logoUrl);
-      }
-    }
-
-    return result;
-  } catch (err) {
-    throw new Error(`Can not get meta info for registry [${registry}]: ${err.message}`);
-  }
-};
-
-module.exports = BlockletRegistry;

package/lib/states/https-cert.js

@@ -1,67 +0,0 @@
-/* eslint-disable no-underscore-dangle */
-
-const logger = require('@abtnode/logger')('@abtnode/core:https-cert');
-const BaseState = require('./base');
-const { getHttpsCertInfo } = require('../util');
-
-const attachHttpsCertInfo = (certificates = []) => {
-  if (!certificates) {
-    return null;
-  }
-
-  if (Array.isArray(certificates)) {
-    return certificates.map((cert) => {
-      const info = cert.certificate ? getHttpsCertInfo(cert.certificate) : {};
-      return { ...cert, ...info };
-    });
-  }
-
-  const info = certificates.certificate ? getHttpsCertInfo(certificates.certificate) : {};
-  return { ...certificates, ...info };
-};
-
-class HttpsCert extends BaseState {
-  constructor(baseDir, options = {}) {
-    super(baseDir, { filename: 'https_cert.db', ...options });
-
-    this.db.ensureIndex({ fieldName: 'domain', unique: true }, (error) => {
-      if (error) {
-        logger.error('ensure unique index failed', { error });
-      }
-    });
-  }
-
-  async upsert(entity) {
-    const updateResult = await this.asyncDB.update(
-      { domain: entity.domain },
-      { $set: entity },
-      {
-        upsert: true,
-      }
-    );
-
-    logger.debug('upsert result', { updateResult });
-
-    const newEntity = await this.findOne({ domain: entity.domain });
-
-    return HttpsCert.renameIdFiledName(newEntity);
-  }
-
-  async find(condition, projection) {
-    const dbEntity = await this.asyncDB.find(condition, { privateKey: 0, ...(projection || {}) });
-
-    return attachHttpsCertInfo(HttpsCert.renameIdFiledName(dbEntity));
-  }
-
-  async findOne(condition, projection) {
-    const dbEntity = await this.asyncDB.findOne(condition, { privateKey: 0, ...(projection || {}) });
-
-    return attachHttpsCertInfo(HttpsCert.renameIdFiledName(dbEntity));
-  }
-
-  async remove(...args) {
-    return this.asyncDB.remove(...args);
-  }
-}
-
-module.exports = HttpsCert;

package/lib/util/get-ip-dns-domain-for-blocklet.js

@@ -1,19 +0,0 @@
-const slugify = require('slugify');
-const { DEFAULT_IP_DNS_DOMAIN_SUFFIX } = require('@abtnode/constant');
-
-const SLOT_FOR_IP_DNS_SITE = '888-888-888-888';
-
-const formatName = (name) => slugify(name.replace(/^[@./-]/, '').replace(/[@./]/g, '-'));
-
-const hiddenInterfaceNames = ['public', 'publicUrl'];
-
-const getIpDnsDomainForBlocklet = (blocklet, blockletInterface) => {
-  const nameSuffix = blocklet.meta.did.slice(-3).toLowerCase();
-  const iName = hiddenInterfaceNames.includes(blockletInterface.name) ? '' : blockletInterface.name.toLowerCase();
-
-  return `${formatName(blocklet.meta.name)}-${nameSuffix}${
-    iName ? '-' : ''
-  }${iName}-${SLOT_FOR_IP_DNS_SITE}.${DEFAULT_IP_DNS_DOMAIN_SUFFIX}`;
-};
-
-module.exports = getIpDnsDomainForBlocklet;

package/lib/util/service.js

@@ -1,66 +0,0 @@
-const fs = require('fs-extra');
-const { NODE_SERVICES } = require('@abtnode/constant');
-
-const servicesNames = Object.values(NODE_SERVICES);
-
-const getServices = ({ stringifySchema = false } = {}) =>
-  servicesNames
-    .map((x) => {
-      const service = {};
-
-      // Read service meta
-      try {
-        const packageFile = require.resolve(`${x}/package.json`);
-        const { name, description, version } = fs.readJSONSync(packageFile);
-        service.name = name;
-        service.description = description || name;
-        service.version = version;
-      } catch (err) {
-        return null;
-      }
-
-      // Read service config: optional
-      try {
-        const schemaFile = require.resolve(`${x}/api/schema.json`);
-        service.schema = fs.readJSONSync(schemaFile);
-      } catch (err) {
-        service.schema = {};
-      }
-
-      if (stringifySchema) {
-        service.schema = JSON.stringify(service.schema);
-      }
-
-      return service;
-    })
-    .filter((x) => x.name && x.version);
-
-const getServicesFromBlockletInterface = (
-  interfaceMeta,
-  { logError, services = getServices(), stringifyConfig = true } = {}
-) => {
-  const list = interfaceMeta.services || [];
-  return list
-    .map((s) => {
-      const service = services.find((t) => t.name === s.name);
-
-      if (!service) {
-        if (typeof logError === 'function') {
-          logError('Service in blocklet meta was not found in node');
-        }
-        return null;
-      }
-
-      const { name, version, description } = service;
-
-      const config = stringifyConfig ? JSON.stringify(s.config) : s.config;
-
-      return { name, version, description, config };
-    })
-    .filter(Boolean);
-};
-
-module.exports = {
-  getServices,
-  getServicesFromBlockletInterface,
-};

package/lib/util/upgrade.js

@@ -1,178 +0,0 @@
-/* eslint-disable no-param-reassign */
-const axon = require('axon');
-const semver = require('semver');
-const sleep = require('@abtnode/util/lib/sleep');
-const { NODE_PACKAGE_NAME, NODE_MODES, NODE_UPGRADE_PROGRESS } = require('@abtnode/constant');
-const listNpmPackageVersion = require('@abtnode/util/lib/list-npm-package-version');
-const Lock = require('@abtnode/util/lib/lock');
-const logger = require('@abtnode/logger')('@abtnode/core:upgrade');
-
-const states = require('../states');
-
-const lock = new Lock('node-upgrade-lock');
-
-// Connect to updater and send RPC call
-const sock = axon.socket('req');
-sock.connect(Number(process.env.ABT_NODE_UPDATER_PORT), '127.0.0.1');
-const waitUpdaterRpc = async (message) =>
-  new Promise((resolve) => {
-    logger.info('send command to updater process', message);
-    sock.send(JSON.stringify(message), (result) => {
-      logger.info('receive response from updater process', result);
-      resolve(result);
-    });
-  });
-
-// eslint-disable-next-line no-unused-vars
-const checkNewVersion = async (params, context) => {
-  try {
-    const info = await states.node.read();
-    if (!info.autoUpgrade) {
-      return '';
-    }
-
-    const versions = await listNpmPackageVersion(NODE_PACKAGE_NAME);
-    if (Array.isArray(versions) && versions.length) {
-      const latestVersion = versions[0].version;
-      if (semver.gt(latestVersion, info.version)) {
-        // if (semver.gte(latestVersion, info.version)) {
-        logger.info('New version found for Blocklet Server', {
-          latestVersion,
-          currentVersion: info.version,
-          nextVersion: info.nextVersion,
-        });
-        await states.node.updateNodeInfo({ nextVersion: latestVersion });
-        await states.notification.create({
-          title: 'Blocklet Server upgrade available',
-          description: 'A new and improved version of blocklet server is now available',
-          entityType: 'node',
-          severity: 'info',
-          sticky: true,
-          action: '/settings/basic',
-        });
-
-        return latestVersion;
-      }
-    }
-
-    return '';
-  } catch (err) {
-    logger.error('Failed to check new version for Blocklet Server', { error: err });
-  }
-
-  return '';
-};
-
-// eslint-disable-next-line no-unused-vars
-const startUpgrade = async (params, context) => {
-  const info = await states.node.read();
-
-  if (!info.nextVersion) {
-    throw new Error('Do nothing since there is no next version to upgrade');
-  }
-
-  const from = info.version;
-  const to = info.nextVersion;
-
-  // 0. ensure we have an upgrading session, we need a session to recover from crash
-  let sessionId = info.upgradeSessionId;
-  if (!sessionId) {
-    const result = await states.session.start({ from, to, stage: NODE_UPGRADE_PROGRESS.SETUP, startedAt: Date.now() });
-    sessionId = result.id;
-    logger.info('generate new session for upgrade', { from, to, sessionId });
-    await states.node.updateNodeInfo({ upgradeSessionId: sessionId });
-  }
-  const session = await states.session.read(sessionId);
-  if (!session) {
-    throw new Error(`Upgrade aborted due to invalid session: ${sessionId}`);
-  }
-
-  process.nextTick(async () => {
-    await doUpgrade(session);
-  });
-
-  return sessionId;
-};
-
-// TODO: error handling and rollback support
-const doUpgrade = async (session) => {
-  await lock.acquire();
-
-  const sessionId = session.id;
-  const info = await states.node.read();
-  const from = info.version;
-  const to = info.nextVersion;
-
-  const goNextState = async (stage) => {
-    session = await states.session.update(sessionId, { stage });
-    // Emit events so client will keep up
-    states.node.emit('node.upgrade.progress', session);
-  };
-
-  // 1. enter maintenance mode
-  if (session.stage === NODE_UPGRADE_PROGRESS.SETUP) {
-    logger.info('enter maintenance mode for upgrading', { from, to, sessionId });
-    await states.node.enterMode(NODE_MODES.MAINTENANCE);
-    await goNextState(NODE_UPGRADE_PROGRESS.INSTALLING);
-  }
-
-  // 2. install specified version
-  if (session.stage === NODE_UPGRADE_PROGRESS.INSTALLING) {
-    await waitUpdaterRpc({ command: 'install', version: to });
-    logger.info('new version installed for upgrading', { from, to, sessionId });
-    await goNextState(NODE_UPGRADE_PROGRESS.RESTARTING);
-  }
-
-  // 3. restarting update version in node config and state
-  if (session.stage === NODE_UPGRADE_PROGRESS.RESTARTING) {
-    logger.info('daemon restarted for upgrading', { from, to, sessionId });
-    await sleep(3000);
-    await goNextState(NODE_UPGRADE_PROGRESS.CLEANUP);
-    await sleep(3000);
-    await waitUpdaterRpc({ command: 'restart', dataDir: process.env.ABT_NODE_DATA_DIR });
-    await lock.release();
-    return; // we should abort here and resume after restart
-  }
-
-  // 4. reset node.nextVersion/upgradeSessionId and exit maintenance mode
-  if (session.stage === NODE_UPGRADE_PROGRESS.CLEANUP) {
-    logger.info('cleanup for upgrading', { from, to, sessionId });
-    await goNextState(NODE_UPGRADE_PROGRESS.COMPLETE);
-    await sleep(5000);
-    await states.node.updateNodeInfo({ nextVersion: '', version: to, upgradeSessionId: '' });
-    try {
-      await states.node.exitMode(NODE_MODES.MAINTENANCE);
-    } catch (error) {
-      logger.error('Failed to exit maintenance mode', { error });
-    }
-  }
-  await lock.release();
-};
-
-const isUpgrading = async () => {
-  const info = await states.node.read();
-  if (!info.nextVersion) {
-    return false;
-  }
-
-  if (!info.upgradeSessionId) {
-    return false;
-  }
-
-  const session = await states.session.read(info.upgradeSessionId);
-  if (!session) {
-    return false;
-  }
-
-  return session;
-};
-
-const getCron = () => ({
-  name: 'check-update',
-  time: '0 0 8 * * *', // check every day
-  // time: '0 */5 * * * *', // check every 5 minutes
-  fn: checkNewVersion,
-  options: { runOnInit: false },
-});
-
-module.exports = { getCron, checkNewVersion, doUpgrade, startUpgrade, isUpgrading };