@abtnode/core 1.15.17 → 1.16.0-beta-b16cb035

package/lib/util/maintain.js

@@ -0,0 +1,233 @@
+/* eslint-disable no-param-reassign */
+const semver = require('semver');
+const sleep = require('@abtnode/util/lib/sleep');
+const SysInfo = require('systeminformation');
+const { NODE_MODES, NODE_MAINTAIN_PROGRESS, EVENTS } = require('@abtnode/constant');
+const listNpmPackageVersion = require('@abtnode/util/lib/list-npm-package-version');
+const Lock = require('@abtnode/util/lib/lock');
+const logger = require('@abtnode/logger')('@abtnode/core:maintain');
+
+const states = require('../states');
+const doRpc = require('./rpc');
+
+const lock = new Lock('node-upgrade-lock');
+
+// eslint-disable-next-line no-unused-vars
+const checkNewVersion = async (params, context) => {
+  try {
+    const info = await states.node.read();
+
+    if (!process.env.ABT_NODE_PACKAGE_NAME) {
+      logger.error('ABT_NODE_PACKAGE_NAME name was not found in environment');
+      return '';
+    }
+
+    const versions = await listNpmPackageVersion(process.env.ABT_NODE_PACKAGE_NAME);
+    if (Array.isArray(versions) && versions.length) {
+      const latestVersion = versions[0].version;
+      if (semver.gt(latestVersion, info.version)) {
+        // if (semver.gte(latestVersion, info.version)) {
+        logger.info('New version found for Blocklet Server', {
+          latestVersion,
+          currentVersion: info.version,
+          nextVersion: info.nextVersion,
+        });
+        await states.node.updateNodeInfo({ nextVersion: latestVersion });
+        await states.notification.create({
+          title: 'Blocklet Server upgrade available',
+          description: 'A new and improved version of blocklet server is now available',
+          entityType: 'node',
+          severity: 'info',
+          sticky: true,
+          action: '/settings/about',
+        });
+
+        return latestVersion;
+      }
+    }
+
+    return '';
+  } catch (err) {
+    logger.error('Failed to check new version for Blocklet Server', { error: err });
+  }
+
+  return '';
+};
+
+const triggerMaintain = async ({ action, next }) => {
+  if (['upgrade', 'restart'].includes(action) === false) {
+    throw new Error(`Unrecognized server maintain action: ${action}`);
+  }
+
+  const info = await states.node.read();
+
+  if (action === 'upgrade') {
+    if (!info.nextVersion) {
+      throw new Error('Do nothing since there is no next version to upgrade');
+    }
+
+    // ensure we have enough memory for upgrading since the upgrading will consume a lot of memory
+    const { mem } = await SysInfo.get({ mem: '*' });
+    const freeMemory = mem.free / 1024 / 1024;
+    const availableMemory = mem.available / 1024 / 1024;
+    if (freeMemory < 200 && availableMemory < 300) {
+      throw new Error('Upgrade aborted because free memory not enough, please stop some blocklets and try again');
+    }
+  }
+
+  const from = info.version;
+  const to = action === 'upgrade' ? info.nextVersion : info.version;
+
+  // ensure we have an maintain session, we need a session to recover from crash
+  let sessionId = info.upgradeSessionId;
+  if (!sessionId) {
+    const result = await states.session.start({
+      action,
+      from,
+      to,
+      stage: NODE_MAINTAIN_PROGRESS.SETUP,
+      history: [],
+      startedAt: Date.now(),
+    });
+    sessionId = result.id;
+    logger.info(`generate new session for ${action}`, { from, to, sessionId });
+    await states.node.updateNodeInfo({ upgradeSessionId: sessionId });
+  }
+  const session = await states.session.read(sessionId);
+  if (!session) {
+    throw new Error(`${action} aborted due to invalid session: ${sessionId}`);
+  }
+
+  process.nextTick(async () => {
+    await next(session);
+  });
+
+  return sessionId;
+};
+
+const resumeMaintain = async (session) => {
+  await lock.acquire();
+
+  const sessionId = session.id;
+  const { action, from, to, history = [] } = session;
+
+  const goNextState = async (stage, previousSucceed) => {
+    session = await states.session.update(sessionId, {
+      stage,
+      history: [...history, { stage: session.stage, succeed: previousSucceed }],
+    });
+    // Emit events so client will keep up
+    states.node.emit(EVENTS.NODE_MAINTAIN_PROGRESS, session);
+  };
+
+  // 1. enter maintenance mode
+  if (session.stage === NODE_MAINTAIN_PROGRESS.SETUP) {
+    logger.info('enter maintenance mode', { from, to, sessionId });
+    await states.node.enterMode(NODE_MODES.MAINTENANCE);
+    if (action === 'upgrade') {
+      await goNextState(NODE_MAINTAIN_PROGRESS.INSTALLING, true);
+    }
+    if (action === 'restart') {
+      await goNextState(NODE_MAINTAIN_PROGRESS.VERIFYING, true);
+    }
+  }
+
+  // 2. install new version
+  if (session.stage === NODE_MAINTAIN_PROGRESS.INSTALLING) {
+    const result = await doRpc({ command: 'install', version: to });
+    logger.info('new version installed', { from, to, sessionId });
+    if (result.code !== 0) {
+      await sleep(3000);
+    }
+    await goNextState(NODE_MAINTAIN_PROGRESS.VERIFYING, result.code === 0);
+  }
+
+  // 2. verify new version
+  if (session.stage === NODE_MAINTAIN_PROGRESS.VERIFYING) {
+    const result = await doRpc({ command: 'verify', version: to });
+    await sleep(3000);
+    if (result.code === 0) {
+      logger.info('new version verified', { from, to, sessionId });
+      await goNextState(NODE_MAINTAIN_PROGRESS.RESTARTING, true);
+    } else {
+      logger.info('new version verify failed', { from, to, sessionId });
+      await goNextState(NODE_MAINTAIN_PROGRESS.ROLLBACK, false);
+    }
+  }
+
+  // 4. reset node.nextVersion/upgradeSessionId and exit maintenance mode
+  if (session.stage === NODE_MAINTAIN_PROGRESS.ROLLBACK) {
+    logger.info('rollback', { from, to, sessionId });
+    await sleep(3000);
+    await goNextState(NODE_MAINTAIN_PROGRESS.COMPLETE, true);
+    await sleep(5000);
+    await states.node.updateNodeInfo({ nextVersion: '', version: from, upgradeSessionId: '' });
+    try {
+      await states.node.exitMode(NODE_MODES.MAINTENANCE);
+    } catch (error) {
+      logger.error('Failed to rollback', { error });
+    }
+    await lock.release();
+    return;
+  }
+
+  // 3. restarting update version in node config and state
+  if (session.stage === NODE_MAINTAIN_PROGRESS.RESTARTING) {
+    logger.info('restart server', { from, to, sessionId });
+    await sleep(3000);
+    await goNextState(NODE_MAINTAIN_PROGRESS.CLEANUP, true);
+    await sleep(3000);
+    await doRpc({ command: 'restart', dataDir: process.env.ABT_NODE_DATA_DIR });
+    await lock.release();
+    return; // we should abort here and resume after restart
+  }
+
+  // 4. reset node.nextVersion/upgradeSessionId and exit maintenance mode
+  if (session.stage === NODE_MAINTAIN_PROGRESS.CLEANUP) {
+    logger.info('cleanup', { from, to, sessionId });
+    await goNextState(NODE_MAINTAIN_PROGRESS.COMPLETE, true);
+    await sleep(8000);
+    await states.node.updateNodeInfo({ nextVersion: '', version: to, upgradeSessionId: '' });
+    try {
+      await states.node.exitMode(NODE_MODES.MAINTENANCE);
+    } catch (error) {
+      logger.error('Failed to exit maintenance mode', { error });
+    }
+  }
+  await lock.release();
+};
+
+const isBeingMaintained = async () => {
+  const info = await states.node.read();
+  if (!info.upgradeSessionId) {
+    return false;
+  }
+
+  const session = await states.session.read(info.upgradeSessionId);
+  if (!session) {
+    return false;
+  }
+
+  if (session.action === 'upgrade' && !info.nextVersion) {
+    return false;
+  }
+
+  return session;
+};
+
+const getCron = () => ({
+  name: 'check-update',
+  time: '0 0 8 * * *', // check every day
+  // time: '0 */5 * * * *', // check every 5 minutes
+  fn: async () => {
+    const info = await states.node.read();
+    if (!info.autoUpgrade) {
+      return;
+    }
+
+    checkNewVersion();
+  },
+  options: { runOnInit: false },
+});
+
+module.exports = { getCron, checkNewVersion, resumeMaintain, triggerMaintain, isBeingMaintained };

package/lib/util/public-to-store.js

@@ -0,0 +1,85 @@
+const { sign } = require('@arcblock/jwt');
+const { BlockletSource } = require('@blocklet/constant');
+const { WHO_CAN_ACCESS } = require('@abtnode/constant');
+const logger = require('@abtnode/logger')('@abtnode/util:public-to-store');
+
+const getWallet = require('@abtnode/util/lib/get-app-wallet');
+const axios = require('@abtnode/util/lib/axios');
+
+const getAppToken = (blocklet) =>
+  sign(blocklet.environmentObj?.BLOCKLET_APP_ID, blocklet.environmentObj?.BLOCKLET_APP_SK);
+
+const getAppId = (blocklet) => blocklet.environmentObj?.BLOCKLET_APP_ID;
+
+const getAppSK = (blocklet) => blocklet.environmentObj?.BLOCKLET_APP_SK;
+
+const getBlockletDid = (blocklet) => blocklet.meta?.bundleDid;
+
+const getAppUrl = (blocklet) => blocklet.environmentObj?.BLOCKLET_APP_URL;
+
+/**
+ * verify manages the permissions of blocklet public instance
+ * @param {*} blocklet
+ * @returns bool
+ */
+const verifyPublicInstance = (blocklet) => {
+  const { whoCanAccess = WHO_CAN_ACCESS.ALL } = blocklet.settings;
+  return blocklet.source === BlockletSource.registry && whoCanAccess === WHO_CAN_ACCESS.ALL;
+};
+
+/**
+ *
+ * @param {*} blocklet
+ * @param {*} {userDid publicToStore}
+ * @returns bool
+ */
+async function handleInstanceInStore(blocklet, { userDid = null, publicToStore = false } = {}) {
+  if (!blocklet) {
+    logger.error('blocklet argument is required');
+    throw new Error('blocklet argument is required');
+  }
+
+  const ownerDid = userDid || blocklet.settings?.owner?.did;
+  if (!ownerDid) {
+    return false;
+  }
+
+  if (!verifyPublicInstance(blocklet)) {
+    logger.error('no permission to set publicInstance');
+    throw new Error('no permission to set publicInstance');
+  }
+
+  const appToken = getAppToken(blocklet);
+  const blockletDid = getBlockletDid(blocklet);
+  const appId = getAppId(blocklet);
+  const appSK = getAppSK(blocklet);
+  const wallet = getWallet(appSK);
+  const body = {
+    appToken,
+    appId,
+    appPK: wallet.publicKey,
+    ownerDid,
+    appUrl: getAppUrl(blocklet),
+    blockletDid,
+  };
+
+  const api = axios.create({ baseURL: blocklet.deployedFrom, timeout: 1000 * 10 });
+  if (!publicToStore) {
+    try {
+      await api.delete(`/api/blocklet-instances/${appId}`, { data: body });
+    } catch (error) {
+      logger.error('failed to delete blocklet instance', { error });
+      throw new Error('failed to delete blocklet instance');
+    }
+  } else {
+    try {
+      await api.put(`/api/blocklet-instances/${appId}`, body);
+    } catch (error) {
+      logger.error('failed to update blocklet instance', { error });
+      throw new Error('failed to delete blocklet instance');
+    }
+  }
+  return true;
+}
+
+module.exports = handleInstanceInStore;

package/lib/util/ready.js

@@ -33,7 +33,7 @@ const createStateReadyHandler =
             console.error('Sharing data dir between Blocklet Server instances may break things!');
             console.error('======================================================\x1b[0m');
             console.log('\nIf you intend to use this dir:');
-            console.log(`  1. Stop Blocklet Server by ${chalk.cyan('abtnode stop --force')}`);
+            console.log(`  1. Stop Blocklet Server by ${chalk.cyan('blocklet server stop --force')}`);
             console.log(`  2. Clear data dir by ${chalk.cyan(`rm -r ${options.dataDir}`)}`);
             console.log('  3. Reinitialize and start Blocklet Server');
             process.exit(1);

package/lib/util/requirement.js

@@ -1,8 +1,10 @@
 const semver = require('semver');
 
-let { version: actualVersion } = require('../../package.json');
+let { version: actualServerVersion } = require('../../package.json');
 
-const defaults = { abtnode: '>= 1.1.0', os: '*', cpu: '*' };
+const { version: actualNodejsVersion } = process;
+
+const defaults = { abtnode: '>= 1.1.0', os: '*', cpu: '*', nodejs: '*' };
 
 const isSatisfied = (requirements, throwOnUnsatisfied = true) => {
   if (!requirements || typeof requirements !== 'object') {
@@ -10,14 +12,23 @@ const isSatisfied = (requirements, throwOnUnsatisfied = true) => {
   }
 
   const { platform: actualOs, arch: actualCpu } = process;
-  const { os: expectedOs, cpu: expectedCpu, abtnode: expectedVersion } = Object.assign(defaults, requirements);
+  const {
+    os: expectedOs,
+    cpu: expectedCpu,
+    abtnode,
+    server,
+    nodejs: expectedNodejsVersion,
+  } = Object.assign(defaults, requirements);
+  const expectedServerVersion = server || abtnode;
 
   const isOsSatisfied =
     expectedOs === '*' || expectedOs === actualOs || (Array.isArray(expectedOs) && expectedOs.includes(actualOs));
   const isCpuSatisfied =
     expectedCpu === '*' || expectedCpu === actualCpu || (Array.isArray(expectedCpu) && expectedCpu.includes(actualCpu));
 
-  const isVersionSatisfied = semver.satisfies(semver.coerce(actualVersion), expectedVersion);
+  const isServerVersionSatisfied = semver.satisfies(semver.coerce(actualServerVersion), expectedServerVersion);
+
+  const isNodejsVersionSatisfied = semver.satisfies(semver.coerce(actualNodejsVersion), expectedNodejsVersion);
 
   if (throwOnUnsatisfied) {
     if (isOsSatisfied === false) {
@@ -26,17 +37,25 @@ const isSatisfied = (requirements, throwOnUnsatisfied = true) => {
     if (isCpuSatisfied === false) {
       throw new Error(`Your blocklet is not supported on architecture ${actualCpu}`);
     }
-    if (isVersionSatisfied === false) {
-      throw new Error(`Your blocklet is not supported on abtnode version v${actualVersion}`);
+    if (isServerVersionSatisfied === false) {
+      throw new Error(
+        `Expected server version for the blocklet is ${expectedServerVersion}, but your server version is ${actualServerVersion}, please upgrade or downgrade your server to continue.`
+      );
+    }
+    if (isNodejsVersionSatisfied === false) {
+      throw new Error(
+        `Expected Nodejs version for the blocklet is ${expectedNodejsVersion}, but your Nodejs version is ${actualNodejsVersion}, please upgrade or downgrade Nodejs and restart your server.`
+      );
     }
   }
 
-  return isOsSatisfied && isCpuSatisfied && isVersionSatisfied;
+  return isOsSatisfied && isCpuSatisfied && isServerVersionSatisfied;
 };
 
 // Just for test
-isSatisfied._setActualVersion = (d) => {
-  actualVersion = d;
+isSatisfied._setActualServerVersion = (d) => {
+  actualServerVersion = d;
 };
+isSatisfied._actualNodejsVersion = actualNodejsVersion;
 
 module.exports = isSatisfied;

package/lib/util/reset-node.js

@@ -52,14 +52,14 @@ const resetSites = async ({ context, routerManager, takeRoutingSnapshot }) => {
 };
 
 /* istanbul ignore next */
-const resetCertificates = async ({ context, routerManager }) => {
-  const certs = await states.certificate.find();
+const resetCertificates = async ({ certManager }) => {
+  const certs = await certManager.getAll();
   for (let i = 0; i < certs.length; i++) {
     const cert = certs[i];
-    if (!cert.sans.includes('*.ip.abtnet.io')) {
+    if (!cert.domain === '*.ip.abtnet.io') {
       try {
         // eslint-disable-next-line no-await-in-loop
-        await routerManager.deleteCertificate({ id: cert.id }, context);
+        await certManager.remove(cert.id);
       } catch (err) {
         // Do nothing
       }
@@ -77,7 +77,7 @@ const resetUsers = async ({ teamManager }) => {
 
   let count = 0;
   for (const user of users) {
-    if (nodeInfo.nodeOwner.did !== user.find) {
+    if (nodeInfo.nodeOwner.did !== user.did) {
       // eslint-disable-next-line no-await-in-loop
       await userState.remove({ did: user.did });
       count++;
@@ -109,7 +109,15 @@ const resetFns = {
   invitations: resetInvitations,
 };
 
-module.exports = async ({ params, context, blockletManager, routerManager, takeRoutingSnapshot, teamManager }) => {
+module.exports = async ({
+  params,
+  context,
+  blockletManager,
+  routerManager,
+  takeRoutingSnapshot,
+  teamManager,
+  certManager,
+}) => {
   if (process.env.NODE_ENV !== 'e2e') {
     throw new Error('Reset node only exists for test purpose');
   }
@@ -140,7 +148,14 @@ module.exports = async ({ params, context, blockletManager, routerManager, takeR
       }
 
       // eslint-disable-next-line no-await-in-loop
-      results[key] = await resetFns[key]({ context, blockletManager, routerManager, takeRoutingSnapshot, teamManager });
+      results[key] = await resetFns[key]({
+        context,
+        blockletManager,
+        routerManager,
+        takeRoutingSnapshot,
+        teamManager,
+        certManager,
+      });
     }
   }
 

package/lib/util/router.js

@@ -0,0 +1,13 @@
+const { BLOCKLET_SITE_GROUP_SUFFIX } = require('@abtnode/constant');
+
+const getBlockletDomainGroupName = (did) => `${did}${BLOCKLET_SITE_GROUP_SUFFIX}`;
+
+const getDidFromDomainGroupName = (name) => {
+  const did = name.replace(BLOCKLET_SITE_GROUP_SUFFIX, '');
+  return did;
+};
+
+module.exports = {
+  getBlockletDomainGroupName,
+  getDidFromDomainGroupName,
+};

package/lib/util/rpc.js

@@ -0,0 +1,16 @@
+const axon = require('axon');
+const logger = require('@abtnode/logger')('@abtnode/core:rpc');
+
+const sock = axon.socket('req');
+sock.connect(Number(process.env.ABT_NODE_UPDATER_PORT || 40405), '127.0.0.1');
+
+const doRpc = async (message) =>
+  new Promise((resolve) => {
+    logger.info('send rpc request', message);
+    sock.send(JSON.stringify(message), (result) => {
+      logger.info('receive rpc response', result);
+      resolve(result);
+    });
+  });
+
+module.exports = doRpc;

package/lib/util/store.js

@@ -0,0 +1,179 @@
+const joinURL = require('url-join');
+const pick = require('lodash/pick');
+const isBase64 = require('is-base64');
+
+const { BLOCKLET_STORE_API_PREFIX, BLOCKLET_STORE_META_PATH } = require('@abtnode/constant');
+const { validateMeta, fixAndValidateService } = require('@blocklet/meta/lib/validate');
+const verifyMultiSig = require('@blocklet/meta/lib/verify-multi-sig');
+
+const { name } = require('../../package.json');
+const logger = require('@abtnode/logger')(`${name}:util:registry`); // eslint-disable-line
+
+const request = require('./request');
+
+const fixAndVerifyMetaFromStore = (meta) => {
+  const sanitized = validateMeta(meta, { ensureDist: false, schemaOptions: { noDefaults: true, stripUnknown: true } });
+
+  if (meta.children) {
+    sanitized.children = meta.children;
+    delete sanitized.components;
+  }
+
+  if (meta.navigation) {
+    meta.navigation.forEach((nav, index) => {
+      if (nav.child) {
+        sanitized.navigation[index].child = nav.child;
+        delete sanitized.navigation[index].component;
+      }
+    });
+  }
+
+  // verify
+
+  try {
+    const result = verifyMultiSig(sanitized);
+    if (!result) {
+      throw new Error('invalid signature from developer or store');
+    }
+  } catch (err) {
+    throw new Error(`Invalid blocklet meta: ${err.message}`);
+  }
+
+  // this step comes last because it has side effects: the meta is changed
+  return fixAndValidateService(meta);
+};
+
+const validateStoreUrl = async (registry) => {
+  const url = joinURL(registry, BLOCKLET_STORE_API_PREFIX, `/blocklets.json?__t__=${Date.now()}`);
+  try {
+    const res = await request.get(url);
+    if (Array.isArray(res.data)) {
+      return res.data;
+    }
+
+    logger.error('Blocklet list fetch failed ', { url, data: res.data });
+    throw new Error('blocklet list fetch failed');
+  } catch (error) {
+    logger.error('Blocklet registry refresh failed', { url, error });
+    throw new Error(`Invalid Blocklet Registry URL ${registry}: ${error.message}`);
+  }
+};
+
+const getStoreMeta = async (registry) => {
+  try {
+    const url = joinURL(registry, BLOCKLET_STORE_META_PATH, `?__t__=${Date.now()}`);
+    const { data } = await request.get(url);
+
+    if (!data) {
+      return {};
+    }
+
+    const requiredFields = ['name', 'description', 'maintainer'];
+
+    const missingFields = requiredFields.filter((x) => !data[x]);
+    if (missingFields.length > 0) {
+      throw new Error(`the registry missing required information: ${missingFields.join(', ')}`);
+    }
+
+    const result = pick(data, ['id', 'name', 'description', 'maintainer', 'cdnUrl', 'chainHost']);
+    const { logoUrl } = data;
+    if (logoUrl) {
+      if (logoUrl.startsWith('http') === true) {
+        result.logoUrl = logoUrl;
+      } else if (isBase64(logoUrl, { allowMime: true })) {
+        result.logoUrl = logoUrl;
+      } else {
+        result.logoUrl = joinURL(registry, logoUrl);
+      }
+    }
+
+    return result;
+  } catch (err) {
+    throw new Error(`Can not get meta info for registry [${registry}]: ${err.message}`);
+  }
+};
+
+const parseSourceUrl = async (url) => {
+  const { origin, pathname } = new URL(url);
+
+  const match = pathname.match(/^\/api\/blocklets\/(\w*)\/blocklet\.json/);
+  if (match) {
+    try {
+      const m = await getStoreMeta(origin);
+      if (m && m.id) {
+        return {
+          inStore: true,
+          registryUrl: origin,
+          blockletDid: match[1],
+        };
+      }
+    } catch {
+      // meat is not in store, do nothing
+    }
+  }
+
+  return {
+    inStore: false,
+  };
+};
+
+const resolveTarballURL = async ({ did, tarball = '', storeUrl = '' }) => {
+  if (!tarball) {
+    return '';
+  }
+
+  if (tarball.startsWith('file://')) {
+    return decodeURIComponent(tarball);
+  }
+
+  if (tarball.startsWith('http://') || tarball.startsWith('https://')) {
+    return tarball;
+  }
+
+  if (!storeUrl) {
+    return '';
+  }
+
+  if (!did) {
+    return '';
+  }
+
+  return joinURL(storeUrl, 'api', 'blocklets', did, tarball);
+};
+
+const getBlockletMetaUrl = ({ did, storeUrl }) =>
+  joinURL(storeUrl, BLOCKLET_STORE_API_PREFIX, `/blocklets/${did}/blocklet.json`);
+
+const getBlockletMeta = async ({ did, storeUrl }) => {
+  const url = joinURL(storeUrl, BLOCKLET_STORE_API_PREFIX, `/blocklets/${did}/blocklet.json?__t__=${Date.now()}`);
+
+  const { data } = await request.get(url);
+  try {
+    if (data.did !== did) {
+      throw new Error('Invalid blocklet meta: did does not match');
+    }
+
+    const meta = fixAndVerifyMetaFromStore(data);
+
+    meta.dist.tarball = await resolveTarballURL({
+      did,
+      tarball: meta.dist.tarball,
+      storeUrl,
+    });
+
+    return meta;
+  } catch (err) {
+    logger.error('failed to get blocklet meta', { did, data, error: err });
+    throw err;
+  }
+};
+
+module.exports = {
+  validateStoreUrl,
+  getStoreMeta,
+  parseSourceUrl,
+  getBlockletMeta,
+  resolveTarballURL,
+  getBlockletMetaUrl,
+  fixAndVerifyMetaFromStore,
+};