@abtnode/core 1.5.13 → 1.15.17

package/lib/blocklet/extras.js

@@ -26,7 +26,7 @@ const mergeConfigs = (oldConfigs, newConfigs = []) => {
   // newConfig 为用户传的,也可以是从环境变量中去读的
   const uniqConfigs = uniqBy(newConfigs, (x) => x.key || x.name);
 
-  // `BLOCKLET_*` and `ABT_NODE_*`  vars can only be set by ABT Node Daemon with only a few exceptions.
+  // `BLOCKLET_*` and `ABT_NODE_*`  vars can only be set by Blocklet Server Daemon with only a few exceptions.
   const newConfig = uniqConfigs.filter((x) => {
     const key = x.key || x.name;
 

package/lib/blocklet/hooks.js

@@ -5,10 +5,13 @@ const camelCase = require('lodash/camelCase');
 // eslint-disable-next-line global-require
 const logger = require('@abtnode/logger')(`${require('../../package.json').name}:blocklet:hooks`);
 
+const { getSafeEnv } = require('../util');
+
 const runAsync = ({ appDir, env, hook, progress = false }) => {
+  const safeEnv = getSafeEnv(env);
   const child = childProcess.exec(hook, {
     cwd: appDir,
-    env: { ...process.env, ...env },
+    env: safeEnv,
     stdio: 'inherit',
   });
 

package/lib/blocklet/manager/disk.js

@@ -221,12 +221,6 @@ class BlockletManager extends BaseBlockletManager {
   async start({ did, checkHealthImmediately = false, throwOnError }, context) {
     logger.info('start blocklet', { did });
     const blocklet = await this.ensureBlocklet(did);
-    try {
-      await runMigrationScripts({ blocklet });
-    } catch (error) {
-      logger.error('Failed to migrate blocklet', { did, error });
-      throw error;
-    }
 
     try {
       // check required config
@@ -961,7 +955,7 @@ class BlockletManager extends BaseBlockletManager {
       return a.from.pathPrefix.length < b.from.pathPrefix ? 1 : -1;
     });
     const nodeIp = await getAccessibleExternalNodeIp(nodeInfo);
-    return getBlockletInterfaces({ blocklet, context, nodeInfo, sites: routingRules, nodeIp });
+    return getBlockletInterfaces({ blocklet, context, nodeInfo, routingRules, nodeIp });
   }
 
   async attachRuntimeInfo({ did, nodeInfo, diskInfo = true, context, cachedBlocklet }) {
@@ -1432,7 +1426,6 @@ class BlockletManager extends BaseBlockletManager {
 
     // download
     await this._downloadBlocklet(blocklet);
-
     return this._installBlocklet({
       did: meta.did,
       context,
@@ -1644,6 +1637,7 @@ class BlockletManager extends BaseBlockletManager {
 
     const oldBlocklet = await this.state.getBlocklet(did);
 
+    // NOTE: 目前的版本移除了降级通道，所以不需要考虑降级通道的情况
     const action = semver.gt(oldBlocklet.meta.version, version) ? 'downgrade' : 'upgrade';
     logger.info(`${action} blocklet`, { did, version });
 
@@ -1915,6 +1909,7 @@ class BlockletManager extends BaseBlockletManager {
     const { meta, source, deployedFrom, children } = newBlocklet;
     const { did, version, name } = meta;
 
+    const oldVersion = oldBlocklet.meta.version;
     const action = semver.gt(oldBlocklet.meta.version, version) ? 'downgrade' : 'upgrade';
     try {
       // delete old process
@@ -1960,6 +1955,33 @@ class BlockletManager extends BaseBlockletManager {
         });
       await forEachBlocklet(blocklet, postInstall, { parallel: true });
 
+      // run migrations
+      const runMigration = (b) => {
+        // BUG: 本身的定义是在执行 upgrade 操作时进行 migration，但是父 blocklet upgrade 时，子 blocklet 可能不需要 upgrade，就会导致子 blocklet 多走了一遍 migration 流程
+        if (b.meta.did === did) {
+          return runMigrationScripts({
+            blocklet: b,
+            oldVersion,
+            newVersion: version,
+            env: getRuntimeEnvironments(b, nodeEnvironments),
+            appDir: b.env.appDir,
+            did: b.meta.did,
+            notification: this.notification,
+            context,
+          });
+        }
+        return Promise.resolve();
+      };
+      logger.info('start migration');
+
+      try {
+        await forEachBlocklet(blocklet, runMigration, { parallel: true });
+      } catch (error) {
+        logger.error('Failed to migrate blocklet', { did, error });
+        throw error;
+      }
+      logger.info('end migration');
+
       logger.info('updated blocklet for upgrading', { did, version, source, name });
 
       // start new process

package/lib/blocklet/migration.js

@@ -1,10 +1,68 @@
 /* eslint-disable no-await-in-loop */
+const childProcess = require('child_process');
 const fs = require('fs-extra');
 const path = require('path');
 const semver = require('semver');
+
 const { getMigrationScripts: getScripts } = require('../migrations');
+const { getSafeEnv } = require('../util');
+const { name } = require('../../package.json');
+const logger = require('@abtnode/logger')(`${name}:blocklet:migration`); // eslint-disable-line
+
+const _runScript = ({ appDir, env, migrationScript, progress = false }) => {
+  const safeEnv = getSafeEnv(env);
+
+  const child = childProcess.exec(`node ${migrationScript}`, {
+    cwd: appDir,
+    env: safeEnv,
+    stdio: 'inherit',
+  });
+  let hasUnhandledRejection = false;
+
+  if (progress) {
+    child.stdout.pipe(process.stdout);
+    child.stderr.pipe(process.stderr);
+  }
+
+  return new Promise((resolve, reject) => {
+    const errorMessages = [];
+
+    child.stderr.on('data', (err) => {
+      // Check if has unhandledRejection in childProcess
+      // https://stackoverflow.com/questions/32784649/gracefully-handle-errors-in-child-processes-in-nodejs
+      if (err.includes('UnhandledPromiseRejectionWarning')) {
+        hasUnhandledRejection = true;
+      }
+      errorMessages.push(err);
+    });
+
+    child.on('exit', (code) => {
+      if (errorMessages.length > 0) {
+        if (code !== 0 || hasUnhandledRejection) {
+          return reject(new Error(errorMessages.join('\r\n')));
+        }
+
+        if (!progress) {
+          errorMessages.forEach((message) => process.stderr.write(message));
+        }
+      }
 
-async function runScripts({ CONFIG_FILE, DB_DIR, BAK_DIR, scriptsDir, printInfo, printSuccess, printError }) {
+      return resolve();
+    });
+  });
+};
+
+async function runScripts({
+  dbDir,
+  backupDir,
+  scriptsDir,
+  printInfo,
+  printSuccess,
+  printError,
+  appDir,
+  env,
+  oldVersion,
+}) {
   let scripts = [];
   try {
     scripts = await getScripts(scriptsDir);
@@ -14,16 +72,7 @@ async function runScripts({ CONFIG_FILE, DB_DIR, BAK_DIR, scriptsDir, printInfo,
     throw err;
   }
 
-  let config;
-  try {
-    config = fs.readJsonSync(CONFIG_FILE);
-  } catch {
-    config = {
-      version: '0.0.0',
-    };
-  }
-
-  const pendingScripts = scripts.filter((x) => semver.gt(x.version, config.version));
+  const pendingScripts = scripts.filter((x) => semver.gt(x.version, oldVersion));
 
   if (!pendingScripts.length) {
     return true;
@@ -31,25 +80,23 @@ async function runScripts({ CONFIG_FILE, DB_DIR, BAK_DIR, scriptsDir, printInfo,
 
   printInfo('pending scripts', pendingScripts);
   try {
-    await doBackup({ DB_DIR, BAK_DIR, printInfo, printSuccess, printError });
+    await doBackup({ dbDir, backupDir, printInfo, printSuccess, printError });
   } catch (err) {
     printError(`Failed to backup state db due to ${err.message}, abort!`);
     throw err;
   }
 
   for (let i = 0; i < pendingScripts.length; i++) {
-    const { path: scriptPath } = pendingScripts[i];
+    const { script: scriptPath } = pendingScripts[i];
     try {
-      // eslint-disable-next-line
-      const scriptFn = require(path.join(folder, scriptPath));
       printInfo(`Migration script started: ${scriptPath}`);
-      await scriptFn({ printInfo, printSuccess, printError });
+      await _runScript({ appDir, env, migrationScript: path.join(scriptsDir, scriptPath) });
       printInfo(`Migration script executed: ${scriptPath}`);
     } catch (migrationErr) {
       printError(`Failed to execute migration script: ${scriptPath}, error: ${migrationErr.message}`);
 
       try {
-        await doRestore({ DB_DIR, BAK_DIR, printInfo, printSuccess, printError });
+        await doRestore({ dbDir, backupDir, printInfo, printSuccess, printError });
       } catch (restoreErr) {
         printError(`Failed to restore state db due to: ${restoreErr.message}`);
       }
@@ -58,51 +105,52 @@ async function runScripts({ CONFIG_FILE, DB_DIR, BAK_DIR, scriptsDir, printInfo,
     }
   }
 
-  fs.writeJsonSync(CONFIG_FILE, {
-    version: pendingScripts[pendingScripts.length - 1].version,
-  });
-
   return true;
 }
 
-async function doBackup({ DB_DIR, BAK_DIR, printInfo, printSuccess }) {
+async function doBackup({ dbDir, backupDir, printInfo, printSuccess }) {
   printInfo('Backing up before migration...');
-  fs.emptyDirSync(BAK_DIR);
-  fs.copySync(DB_DIR, BAK_DIR);
+  fs.emptyDirSync(backupDir);
+  fs.copySync(dbDir, backupDir);
   printSuccess('Backup success');
 }
 
-async function doRestore({ DB_DIR, BAK_DIR, printInfo, printSuccess }) {
+async function doRestore({ dbDir, backupDir, printInfo, printSuccess }) {
   printInfo('Restoring when migration failed...');
-  fs.emptyDirSync(DB_DIR);
-  fs.copySync(BAK_DIR, DB_DIR);
+  fs.emptyDirSync(dbDir);
+  fs.copySync(backupDir, dbDir);
   printSuccess('Restore succeed');
 }
 
 module.exports = async ({
-  blocklet,
-  printInfo = console.info, // eslint-disable-line no-console
-  printSuccess = console.info, // eslint-disable-line no-console
-  printError = console.error,
+  appDir,
+  env,
+  oldVersion,
+  newVersion,
+  printInfo = logger.info,
+  printSuccess = logger.info,
+  printError = logger.error,
 }) => {
-  const { version } = blocklet.meta;
-  const baseDir = blocklet.environmentObj.BLOCKLET_DATA_DIR;
-  const appDir = blocklet.environmentObj.BLOCKLET_APP_DIR;
+  const baseDir = env.BLOCKLET_DATA_DIR;
 
   const scriptsDir = path.join(appDir, 'migration');
-  const DB_DIR = path.join(baseDir, 'db');
-  const BAK_DIR = path.join(baseDir, 'db-bak');
-  const CONFIG_FILE = path.join(baseDir, 'config.json');
+  const dbDir = path.join(baseDir, 'db');
+  const backupDir = path.join(baseDir, 'db-bak');
 
   fs.ensureDirSync(scriptsDir);
-  fs.ensureDirSync(DB_DIR);
-  fs.ensureDirSync(BAK_DIR);
-  fs.ensureFileSync(CONFIG_FILE);
-
-  // 如果是首次安装，没有 CONFIG_FILE
-  if (!fs.existsSync(CONFIG_FILE)) {
-    fs.writeJsonSync(CONFIG_FILE, { version });
-    return;
-  }
-  await runScripts({ DB_DIR, BAK_DIR, CONFIG_FILE, scriptsDir, printError, printInfo, printSuccess });
+  fs.ensureDirSync(dbDir);
+  fs.ensureDirSync(backupDir);
+
+  await runScripts({
+    dbDir,
+    backupDir,
+    scriptsDir,
+    printError,
+    printInfo,
+    printSuccess,
+    appDir,
+    env,
+    oldVersion,
+    newVersion,
+  });
 };

package/lib/blocklet/registry.js

@@ -3,7 +3,7 @@ const { BlockletGroup } = require('@blocklet/meta/lib/constants');
 const joinURL = require('url-join');
 const get = require('lodash/get');
 const pick = require('lodash/pick');
-const { BLOCKLET_REGISTRY_API_PREFIX } = require('@abtnode/constant');
+const { BLOCKLET_STORE_API_PREFIX } = require('@abtnode/constant');
 
 const { name } = require('../../package.json');
 
@@ -63,7 +63,7 @@ class BlockletRegistry {
 
     const url = joinURL(
       registryUrl || defaultRegistryUrl,
-      BLOCKLET_REGISTRY_API_PREFIX,
+      BLOCKLET_STORE_API_PREFIX,
       `/blocklets/${did}/blocklet.json?__t__=${Date.now()}`
     );
 
@@ -109,7 +109,7 @@ class BlockletRegistry {
 
   async refreshBlocklets(context) {
     const registryUrl = await states.node.getBlockletRegistry();
-    const url = joinURL(registryUrl, BLOCKLET_REGISTRY_API_PREFIX, '/blocklets.json');
+    const url = joinURL(registryUrl, BLOCKLET_STORE_API_PREFIX, '/blocklets.json');
     try {
       let res = await request.get(url, {
         validateStatus: (status) => (status >= 200 && status < 300) || status === 304,
@@ -153,7 +153,7 @@ class BlockletRegistry {
 }
 
 BlockletRegistry.validateRegistryURL = async (registry) => {
-  const url = joinURL(registry, BLOCKLET_REGISTRY_API_PREFIX, `/blocklets.json?__t__=${Date.now()}`);
+  const url = joinURL(registry, BLOCKLET_STORE_API_PREFIX, `/blocklets.json?__t__=${Date.now()}`);
   try {
     const res = await request.get(url);
     if (Array.isArray(res.data)) {
@@ -170,7 +170,7 @@ BlockletRegistry.validateRegistryURL = async (registry) => {
 
 BlockletRegistry.getRegistryMeta = async (registry) => {
   try {
-    const url = joinURL(registry, BLOCKLET_REGISTRY_API_PREFIX, `/registry.json?__t__=${Date.now()}`);
+    const url = joinURL(registry, BLOCKLET_STORE_API_PREFIX, `/registry.json?__t__=${Date.now()}`);
     const { data } = await request.get(url);
 
     if (!data) {

package/lib/event.js

@@ -128,8 +128,8 @@ module.exports = ({
   const handleCLIEvent = (eventName) => {
     const [, status] = eventName.split('.');
     onEvent(eventName, {
-      title: `ABT Node ${status}`,
-      description: `ABT Node is ${status} successfully`,
+      title: `Blocklet Server ${status}`,
+      description: `Blocklet Server is ${status} successfully`,
       entityType: 'node',
       status: 'success',
     });

package/lib/migrations/1.5.15-site.js

@@ -0,0 +1,184 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+const normalizePathPrefix = require('@abtnode/util/lib/normalize-path-prefix');
+
+const findBlocklet = (site, blocklets) => {
+  //  prefix = /
+  const rootRule = site.rules.find((x) => x.from.pathPrefix === '/' && x.to.type === 'blocklet');
+  if (rootRule) {
+    const blocklet = blocklets.find((x) => x.meta.did === rootRule.to.did);
+    if (blocklet) {
+      return blocklet;
+    }
+  }
+
+  for (const rule of site.rules) {
+    if (rule.to.type === 'blocklet') {
+      const blocklet = blocklets.find((x) => x.meta.did === rule.to.did);
+      if (blocklet) {
+        return blocklet;
+      }
+    }
+  }
+
+  return null;
+};
+
+const mergeSite = (newSite, oldSite) => {
+  const { domain, domainAliases = [], isProtected, rules, corsAllowedOrigins } = oldSite;
+
+  // merge domain
+  const domains = [
+    {
+      value: domain,
+      isProtected,
+    },
+    ...(domainAliases || []).map((x) => (typeof x === 'string' ? { value: x, isProtected: false } : x)),
+  ];
+  domains.forEach((x) => {
+    if (!newSite.domainAliases.some((y) => y.value === x.value)) {
+      newSite.domainAliases.push(x);
+    }
+  });
+
+  // merge cors
+  (corsAllowedOrigins || []).forEach((x) => {
+    if (!newSite.corsAllowedOrigins.includes(x)) {
+      newSite.corsAllowedOrigins.push(x);
+    }
+  });
+
+  // merge rules
+  (rules || []).forEach((x) => {
+    if (!newSite.rules.some((y) => normalizePathPrefix(y.from.pathPrefix) === normalizePathPrefix(x.from.pathPrefix))) {
+      newSite.rules.push(x);
+    }
+  });
+};
+
+module.exports = async ({ states, node, printInfo }) => {
+  printInfo('Migrate site to 2.0 version for router...\n');
+
+  const sites = await states.site.getSites();
+  const blocklets = await states.blocklet.getBlocklets();
+  printInfo(`Find blocklets: ${blocklets.length}`);
+
+  // blocklet site which ends with 888-888-888-888.ip.abtnet.io
+  const blockletSystemSites = [];
+
+  // custom site added by user
+  const customSites = [];
+
+  sites.forEach((site) => {
+    const { domain } = site;
+
+    // filter 3 default basic site
+    if (domain === '' || domain === '127.0.0.1' || domain === '*') {
+      return;
+    }
+
+    const blocklet = findBlocklet(site, blocklets);
+
+    if (domain.endsWith('888-888-888-888.ip.abtnet.io')) {
+      blockletSystemSites.push({
+        site,
+        blocklet,
+      });
+    } else {
+      customSites.push({
+        site,
+        blocklet,
+      });
+    }
+  });
+
+  printInfo(
+    `Find sites: ${sites.length}; blockletSystemSites: ${blockletSystemSites.length}; customSites: ${customSites.length}\n`
+  );
+  printInfo(`blockletSystemSites: ${blockletSystemSites.map((x) => x.site.domain).join(', ')}\n`);
+  printInfo(`customSites: ${customSites.map((x) => x.site.domain).join(', ')}\n`);
+
+  // generate new blocklet site for every installed blocklet
+  const newBlockletSites = {}; // <blockletDid>: <site>
+  for (const blocklet of blocklets) {
+    const domain = `${blocklet.meta.did}.blocklet-domain-group`;
+    newBlockletSites[blocklet.meta.did] = {
+      domain,
+      domainAliases: [],
+      isProtected: true,
+      rules: [],
+      corsAllowedOrigins: [],
+    };
+  }
+  printInfo(
+    `newBlockletSites: ${Object.values(newBlockletSites)
+      .map((x) => x.domain)
+      .join(', ')}\n`
+  );
+
+  printInfo('\n');
+  printInfo('Start merge blockletSystemSites to new sites');
+  for (const { site: oldSite, blocklet } of blockletSystemSites) {
+    // merge blockletSystemSite to new blocklet site
+    if (blocklet) {
+      const newSite = newBlockletSites[blocklet.meta.did];
+      mergeSite(newSite, oldSite);
+      printInfo(`Merge site from ${oldSite.domain} to ${newSite.domain}`);
+    }
+  }
+
+  printInfo('\n');
+  printInfo('Start merge customSites to new sites');
+  for (const { site: oldSite, blocklet } of customSites) {
+    // reserve custom site which not belong to any blocklet
+    if (!blocklet) {
+      printInfo(`Skip merge custom site: ${oldSite.domain}`);
+      oldSite.skip = true;
+      continue;
+    }
+    // merge custom site to new blocklet site
+    const newSite = newBlockletSites[blocklet.meta.did];
+    mergeSite(newSite, oldSite);
+    printInfo(`Merge site from ${oldSite.domain} to ${newSite.domain}`);
+  }
+
+  printInfo('\n');
+  printInfo('Start delete blockletSystemSites from db');
+  for (const { site: oldSite } of blockletSystemSites) {
+    if (oldSite.skip) {
+      printInfo(`Skip delete site from db: ${oldSite.domain}`);
+      continue;
+    }
+    // delete each blockletSystemSite
+    await states.site.remove({ _id: oldSite.id });
+    printInfo(`Delete site from db: ${oldSite.domain}`);
+  }
+
+  printInfo('\n');
+  printInfo('Start delete customSites from db');
+  for (const { site: oldSite } of customSites) {
+    if (oldSite.skip) {
+      printInfo(`Skip delete site from db: ${oldSite.domain}`);
+      continue;
+    }
+    // delete custom site which bind to a blocklet
+    await states.site.remove({ _id: oldSite.id });
+    printInfo(`Delete site from db: ${oldSite.domain}`);
+  }
+
+  // add new blocklet site to db
+  printInfo('\n');
+  printInfo('Start add new sites to db');
+  for (const site of Object.values(newBlockletSites)) {
+    await states.site.add(site);
+    printInfo(`Add site to db: ${site.domain}`);
+  }
+
+  await node.takeRoutingSnapshot({
+    message: 'Migrate site to 2.0 version for router',
+    dryRun: false,
+    handleRouting: false,
+  });
+  printInfo('\n');
+  printInfo('Take routing snapshot');
+};