@abtnode/util 1.15.17 → 1.16.0-beta-8ee536d7

package/lib/log.js

@@ -0,0 +1,391 @@
+/* eslint-disable no-underscore-dangle */
+const path = require('path');
+const createArchive = require('archiver');
+const fs = require('fs-extra');
+const dayjs = require('dayjs');
+const glob = require('fast-glob');
+const isEqual = require('lodash/isEqual');
+const { Tail } = require('tail');
+const readLastLines = require('read-last-lines');
+const { BLOCKLET_MODES } = require('@blocklet/constant');
+const logger = require('@abtnode/logger')(require('../package.json').name);
+
+class StreamLog {
+  constructor() {
+    this._files = null; // Object { <level>: <filePath> }
+    this._tails = null; // Object { <level>: <Tail> }
+  }
+
+  /**
+   * @param {Object} files Object { <level>: <filePath> }
+   * @return {Boolean} if files change
+   */
+  async setFiles(files) {
+    if (!isEqual(this._files, files)) {
+      this.clearTails();
+      this._files = files;
+      return true;
+    }
+    return false;
+  }
+
+  getRecent(lineNum, callback) {
+    if (!this._files) {
+      callback(new Error('files is empty'));
+    }
+    Object.entries(this._files).forEach(([level, file]) => {
+      if (!file || !fs.existsSync(file)) {
+        return;
+      }
+      readLastLines
+        .read(file, lineNum || 0)
+        .then((data) => {
+          callback(null, level, data);
+        })
+        .catch((error) => {
+          callback(error, level);
+        });
+    });
+  }
+
+  ensureTails() {
+    if (this._tails) {
+      return this._tails;
+    }
+    if (!this._files) {
+      return {};
+    }
+    try {
+      this._tails = {};
+      Object.entries(this._files).forEach(([level, file]) => {
+        if (!file || !fs.existsSync(file)) {
+          return;
+        }
+
+        // 测试发现 Windows 下 fs.watch 没有效果，但是 fs.watch 的性能更好，所以只在 windows 下使用 fs.watchFile
+        this._tails[level] = new Tail(file, {
+          useWatchFile: process.platform === 'win32',
+          fsWatchOptions: { interval: 1500 },
+        });
+      });
+      return this._tails;
+    } catch (error) {
+      this._tails = null;
+      throw error;
+    }
+  }
+
+  clearTails() {
+    if (this._tails) {
+      Object.values(this._tails).forEach((tail) => {
+        tail.unwatch();
+      });
+      this._tails = null;
+    }
+  }
+}
+
+const createFile = (files) => {
+  Object.values(files).forEach((file) => {
+    if (!fs.existsSync(file)) {
+      try {
+        const dir = path.dirname(file);
+        fs.mkdirSync(dir, { recursive: true });
+      } catch (err) {
+        // Do nothing
+      }
+      fs.writeFileSync(file, '', 'utf8');
+    }
+  });
+};
+
+const getLogFiles = async ({ name, node }) => {
+  const dt = new Date();
+  const { yyyy, mm, dd } = {
+    yyyy: dt.getFullYear(),
+    mm: `${dt.getMonth() + 1}`.padStart(2, 0),
+    dd: `${dt.getDate()}`.padStart(2, 0),
+  };
+  const date = `${yyyy}-${mm}-${dd}`;
+
+  if (name === 'abtnode') {
+    const logDir = path.join(node.dataDirs.logs, '_abtnode');
+
+    const info = path.join(logDir, `daemon-${date}.log`);
+    const error = path.join(logDir, `daemon-error-${date}.log`);
+    const access = path.join(logDir, 'access.log');
+    const stdout = path.join(logDir, 'daemon.stdout.log');
+    const stderr = path.join(logDir, 'daemon.stderr.log');
+
+    createFile({
+      info,
+      error,
+      access,
+      stdout,
+      stderr,
+    });
+
+    return {
+      info,
+      error,
+      access,
+      stdout,
+      stderr,
+    };
+  }
+
+  if (name === 'blocklet-services') {
+    const logDir = path.join(node.dataDirs.logs, '_abtnode');
+    const info = path.join(logDir, 'service.log');
+    const error = path.join(logDir, 'service.error.log');
+    createFile({ info, error });
+    return { info, error };
+  }
+
+  if (name.indexOf('blocklet-') === 0) {
+    const did = name.substring('blocklet-'.length);
+    const blocklet = await node.getBlocklet({ did, attachConfig: false });
+    const { name: blockletName } = blocklet.meta;
+
+    const dir = path.join(node.dataDirs.logs, blockletName);
+    const info = path.join(dir, `info-${date}.log`);
+    const error = path.join(dir, `info-error-${date}.log`);
+    const access = path.join(dir, 'access.log');
+    const stdout = path.join(dir, 'output.log');
+    const stderr = path.join(dir, 'error.log');
+
+    createFile({
+      info,
+      error,
+      access,
+      stdout,
+      stderr,
+    });
+
+    return {
+      info,
+      error,
+      access,
+      stdout,
+      stderr,
+      recent: blocklet.mode === BLOCKLET_MODES.DEVELOPMENT ? 0 : 100,
+    };
+  }
+
+  if (name.startsWith('service-gateway-')) {
+    const providerName = name.substring('service-gateway-'.length);
+    const provider = node.getRouterProvider(providerName);
+    if (!provider) {
+      logger.error('router engine is empty', { name, providerName });
+      return {};
+    }
+
+    return provider.getLogFilesForToday();
+  }
+
+  return {};
+};
+
+const getDownloadLogFilesFromServer = async ({ dates, nodeInfo, node } = {}) => {
+  const logDir = path.join(node.dataDirs.logs, '_abtnode');
+
+  const pm2Log = path.join(logDir, 'pm2.log');
+  const pm2LogSrc = path.join(process.env.PM2_HOME, 'pm2.log');
+  if (fs.existsSync(pm2LogSrc)) {
+    await fs.copy(pm2LogSrc, pm2Log);
+  }
+
+  const provider = node.getRouterProvider(nodeInfo.routing.provider);
+  if (!provider) {
+    logger.error('router engine is empty');
+  }
+
+  const list = [];
+
+  dates.forEach((d) => {
+    // log file created by @abtnode/log
+    // log file created by abt-node-log-rotate
+    list.push(path.join(logDir, `*-${d}*`));
+
+    // log file create by router
+    const routerLogDir = provider.getLogDir();
+    if (routerLogDir) {
+      list.push(path.join(routerLogDir, `*-${d}*`));
+    }
+  });
+
+  // abt-node-daemon console & gateway
+  list.push(path.join(logDir, 'daemon.stdout.log*'));
+  list.push(path.join(logDir, 'daemon.stderr.log*'));
+  list.push(path.join(logDir, 'access.log*'));
+
+  // abt-node-service console & gateway
+  list.push(path.join(logDir, 'service.output.log*'));
+  list.push(path.join(logDir, 'service.error.log*'));
+  list.push(path.join(logDir, 'service.log*'));
+
+  // abt-node-db-hub console & backup
+  list.push(path.join(logDir, 'db.output.log*'));
+  list.push(path.join(logDir, 'db.error.log*'));
+
+  // abt-node-event-hub console
+  list.push(path.join(logDir, 'event.output.log*'));
+  list.push(path.join(logDir, 'event.error.log*'));
+
+  // abt-node-log-rotate console
+  list.push(path.join(logDir, 'pm2-logrotate.stdout.log*'));
+  list.push(path.join(logDir, 'pm2-logrotate.stderr.log*'));
+
+  // abt-node-updater console
+  list.push(path.join(logDir, 'updater.error.log*'));
+  list.push(path.join(logDir, 'updater.output.log*'));
+
+  // fallback log
+  list.push(path.join(logDir, 'stderr.log*'));
+  list.push(path.join(logDir, 'stdout.log*'));
+
+  // router
+  list.push(...Object.values(provider.getLogFilesForToday() || {}));
+
+  // pm2 log
+  list.push(pm2Log);
+
+  return glob(list);
+};
+
+const getDownloadLogFilesFromBlocklet = async ({ dates, blocklet } = {}) => {
+  const logDir = path.join(blocklet.env.logsDir);
+
+  const list = [];
+
+  dates.forEach((d) => {
+    // log file created by @abtnode/log
+    // log file created by abt-node-log-rotate
+    list.push(path.join(logDir, `*-${d}*`));
+  });
+
+  list.push(path.join(logDir, 'output.log*'));
+  list.push(path.join(logDir, 'error.log*'));
+  list.push(path.join(logDir, 'access.log*'));
+
+  return glob(list);
+};
+
+const getDownloadLogFiles = async ({ did, node, days = 1, now } = {}) => {
+  const dates = [dayjs(now).format('YYYY-MM-DD')];
+
+  for (let i = 1; i <= days; i++) {
+    dates.unshift(dayjs(now).subtract(i, 'day').format('YYYY-MM-DD'));
+  }
+
+  const nodeInfo = await node.getNodeInfo();
+
+  if (nodeInfo.did === did) {
+    return getDownloadLogFilesFromServer({ dates, node, nodeInfo });
+  }
+
+  const blocklet = await node.getBlocklet({ did, attachRuntimeInfo: false });
+  if (!blocklet) {
+    throw new Error('blocklet not found');
+  }
+
+  return getDownloadLogFilesFromBlocklet({ dates, blocklet });
+};
+
+const createStreamLogManager = ({ onLog, onGetLogFiles }) => {
+  const store = {}; // Object<name: streamLog>
+
+  const ensure = (name) => {
+    if (!store[name]) {
+      store[name] = new StreamLog();
+    }
+    return store[name];
+  };
+
+  const destroy = (name) => {
+    logger.info('log stream: destroy', { name });
+    if (!store[name]) {
+      return;
+    }
+    const log = store[name];
+    try {
+      log.clearTails();
+      delete store[name];
+    } catch (error) {
+      logger.error('log stream: remove ref error ', error);
+      delete store[name];
+    }
+  };
+
+  const add = async (name, topic, firstLogCb) => {
+    logger.info('log stream: add', { name });
+    const log = ensure(name);
+    try {
+      // update files
+      // push recent 100 log
+      const { recent = 100, ...logFiles } = await onGetLogFiles(name);
+      const changed = await log.setFiles(logFiles);
+      logger.info('log stream: added', { name, logFiles });
+      log.getRecent(recent, (error, level, data) => {
+        if (error) {
+          logger.error('log stream error ', error);
+        }
+        if (firstLogCb) {
+          firstLogCb(level, data, logFiles);
+        }
+      });
+      // stream
+      if (changed) {
+        const tails = log.ensureTails();
+        Object.entries(tails).forEach(([level, tail]) => {
+          tail.on('line', (data) => {
+            onLog({ topic, level, logFiles, data });
+          });
+          tail.on('error', (error) => {
+            logger.error('log tail error ', { error });
+            destroy(name);
+          });
+        });
+      }
+    } catch (error) {
+      logger.error('log stream error ', error);
+    }
+  };
+
+  return {
+    add,
+    destroy,
+  };
+};
+
+const createDownloadLogStream = async ({ node, did, days, now }) => {
+  const files = await getDownloadLogFiles({ node, did, days, now });
+
+  if (!files.length) {
+    throw new Error('Log file does not found');
+  }
+
+  const cwd = path.dirname(node.dataDirs.logs);
+
+  const archive = createArchive('zip', { zlib: { level: 9 } });
+  files.forEach((x) => {
+    archive.file(x, {
+      name: x.replace(`${cwd}/logs`, '/logs').replace(`${cwd}`, '/logs').replace('/_abtnode', '/blocklet-server'),
+    });
+  });
+
+  archive.rawPipe = archive.pipe.bind(archive);
+  archive.pipe = (s) => {
+    archive.rawPipe(s);
+    archive.finalize();
+  };
+
+  return archive;
+};
+
+module.exports = {
+  createStreamLogManager,
+  getLogFiles,
+  getDownloadLogFiles,
+  createDownloadLogStream,
+};

package/lib/logo-middleware.js

@@ -0,0 +1,136 @@
+const fs = require('fs');
+const path = require('path');
+const get = require('lodash/get');
+
+const logger = require('@abtnode/logger')('@abtnode/util:logo-middleware');
+
+const ensureBlockletExist = async (req, res, next) => {
+  const { blocklet } = req;
+
+  if (!blocklet || !get(blocklet, 'env.appDir')) {
+    res.sendFallbackLogo();
+    return;
+  }
+
+  next();
+};
+
+const ensureCustomSquareLogo = async (req, res, next) => {
+  const { blocklet, sendOptions } = req;
+
+  const customLogoSquare = get(blocklet, 'environmentObj.BLOCKLET_APP_LOGO_SQUARE');
+
+  if (customLogoSquare) {
+    if (customLogoSquare.startsWith('http')) {
+      res.redirect(customLogoSquare);
+      return;
+    }
+
+    const logoFile = path.join(get(blocklet, 'env.appDir'), customLogoSquare);
+    if (fs.existsSync(logoFile)) {
+      res.sendFile(logoFile, sendOptions);
+      return;
+    }
+  }
+
+  next();
+};
+
+const ensureBundleLogo = async (req, res, next) => {
+  const { blocklet, sendOptions } = req;
+
+  if (blocklet.meta.logo) {
+    const logoFile = path.join(get(blocklet, 'env.appDir'), blocklet.meta.logo);
+
+    if (fs.existsSync(logoFile)) {
+      res.sendFile(logoFile, sendOptions);
+      return;
+    }
+  }
+
+  next();
+};
+
+const fallbackLogo = async (req, res) => {
+  res.sendFallbackLogo();
+};
+
+// eslint-disable-next-line no-unused-vars
+const cacheError = async (err, req, res, next) => {
+  logger.error('failed to send blocklet logo', { url: req.url, error: err });
+  res.sendFallbackLogo();
+};
+
+const ensureDefaultLogo = async (req, res, next) => {
+  const { blocklet, sendOptions } = req;
+
+  if (blocklet && get(blocklet, 'env.dataDir')) {
+    get(blocklet, 'env.dataDir');
+
+    const logoSvgFile = path.join(get(blocklet, 'env.dataDir'), 'logo.svg');
+    if (fs.existsSync(logoSvgFile)) {
+      res.sendFile(logoSvgFile, sendOptions);
+      return;
+    }
+
+    const logoPngFile = path.join(get(blocklet, 'env.dataDir'), 'logo.png');
+    if (fs.existsSync(logoPngFile)) {
+      res.sendFile(logoPngFile, sendOptions);
+      return;
+    }
+  }
+
+  next();
+};
+
+const ensureCustomFavicon = async (req, res, next) => {
+  const { blocklet, sendOptions } = req;
+
+  const customLogoFavicon = get(blocklet, 'environmentObj.BLOCKLET_APP_LOGO_FAVICON');
+  if (customLogoFavicon) {
+    if (customLogoFavicon.startsWith('http')) {
+      res.redirect(customLogoFavicon);
+      return;
+    }
+
+    const logoFile = path.join(get(blocklet, 'env.appDir'), customLogoFavicon);
+    if (fs.existsSync(logoFile)) {
+      res.sendFile(logoFile, sendOptions);
+      return;
+    }
+  }
+
+  next();
+};
+
+const attachSendLogoContext =
+  ({ onSendFallbackLogo, onGetBlocklet }) =>
+  async (req, res, next) => {
+    const sendOptions = { maxAge: '1d' };
+
+    req.sendOptions = sendOptions;
+
+    try {
+      req.blocklet = await onGetBlocklet({ req });
+    } catch (error) {
+      logger.error('failed to get blocklet', { url: req.url, error });
+      req.blocklet = null;
+    }
+
+    res.sendFallbackLogo = () => {
+      onSendFallbackLogo({ res, sendOptions });
+    };
+
+    next();
+  };
+
+module.exports = {
+  attachSendLogoContext,
+  ensureBlockletExist,
+  ensureCustomSquareLogo,
+  ensureBundleLogo,
+  ensureCustomFavicon,
+  ensureDefaultLogo,
+  fallbackLogo,
+  cacheError,
+};

package/lib/nft.js

@@ -0,0 +1,15 @@
+const last = require('lodash/last');
+const get = require('lodash/get');
+
+const getNftExpirationDate = (asset) => last(get(asset, 'data.value.expirationDate', []));
+
+const isNFTExpired = (asset) => {
+  const expirationDate = getNftExpirationDate(asset);
+  return isDateExpired(expirationDate);
+};
+
+const isDateExpired = (expirationDate) => !!expirationDate && new Date(expirationDate).getTime() <= Date.now();
+
+const isNFTConsumed = (asset) => !!asset.data.value.consumedTime;
+
+module.exports = { isNFTExpired, getNftExpirationDate, isNFTConsumed };

package/lib/run-script.js

@@ -0,0 +1,131 @@
+/* eslint-disable prefer-const */
+/* eslint-disable no-nested-ternary */
+const fs = require('fs');
+const which = require('which');
+const stream = require('stream');
+const spawn = require('cross-spawn');
+
+class PrefixTransform extends stream.Transform {
+  constructor(prefix, state) {
+    super();
+
+    this.prefix = prefix;
+    this.state = state;
+  }
+
+  // transform all output from child process with a prefix for better readability
+  _transform(chunk, _encoding, callback) {
+    const { prefix, state } = this;
+    const nPrefix = `\n${prefix}`;
+    const firstPrefix = state.lastIsLineBreak ? prefix : state.lastPrefix !== prefix ? '\n' : '';
+    const prefixed = `${firstPrefix}${chunk}`.replace(/\n/g, nPrefix);
+    const index = prefixed.indexOf(prefix, Math.max(0, prefixed.length - prefix.length));
+
+    state.lastPrefix = prefix;
+    state.lastIsLineBreak = index !== -1;
+
+    callback(null, index !== -1 ? prefixed.slice(0, index) : prefixed);
+  }
+}
+
+const wrapStream = (next, prefix, state) => {
+  const transform = new PrefixTransform(`[${prefix}] `, state);
+  transform.pipe(next);
+  return transform;
+};
+
+/**
+ * Run script with a childProcess.spawn
+ *
+ * @param {string} script
+ * @param {string} label
+ * @param {object} options
+ * @param {object} options.env
+ * @param {string} options.cwd
+ * @param {number} options.timeout
+ * @param {boolean} options.silent
+ * @return {Promise}
+ */
+const runScript = (script, label, options = {}) => {
+  if (!script) {
+    throw new Error('script is required');
+  }
+  if (!label) {
+    throw new Error('label is required');
+  }
+
+  let child = null;
+
+  const cleanup = () => {
+    if (child) {
+      child.kill();
+      child = null;
+    }
+  };
+
+  const promise = new Promise((resolve, reject) => {
+    process.stdout.setMaxListeners(0);
+    process.stderr.setMaxListeners(0);
+
+    let [command, ...args] = script.split(' ');
+    if (fs.existsSync(command) === false) {
+      command = which.sync(command);
+      if (!command) {
+        reject(new Error(`Command not found: ${command}`));
+        return;
+      }
+    }
+
+    const state = {
+      lastPrefix: null,
+      lastIsLineBreak: true,
+    };
+
+    const stdout = wrapStream(process.stdout, label, state);
+    const stderr = wrapStream(process.stderr, label, state);
+
+    child = spawn(command, args, {
+      timeout: 2 * 60 * 1000,
+      detached: false,
+      shell: false,
+      stdio: ['ignore', 'pipe', 'pipe'],
+      ...options,
+    });
+
+    if (!options.silent) {
+      child.stdout.pipe(stdout, { end: false });
+      child.stderr.pipe(stderr, { end: false });
+    }
+
+    let hasUnhandledRejection = false;
+    const errorMessages = [];
+    child.stderr.on('data', (err) => {
+      errorMessages.push(err);
+      if (err.includes('UnhandledPromiseRejectionWarning')) {
+        hasUnhandledRejection = true;
+      }
+    });
+
+    child.on('error', (err) => {
+      console.error(err);
+      errorMessages.push(err.message);
+      return reject(new Error(errorMessages.join('\r\n')));
+    });
+
+    child.on('close', (code) => {
+      if (errorMessages.length > 0) {
+        if (code !== 0 || hasUnhandledRejection) {
+          return reject(new Error(errorMessages.join('\r\n')));
+        }
+      }
+
+      return resolve({ code, script });
+    });
+  });
+
+  promise.abort = cleanup;
+
+  return promise;
+};
+
+module.exports = runScript;

package/lib/security.js

@@ -0,0 +1,45 @@
+const crypto = require('crypto');
+const AES = require('@ocap/mcrypto/lib/crypter/aes-legacy').default;
+
+const encrypt = (m, s, i) => AES.encrypt(m, crypto.pbkdf2Sync(i, s, 256, 32, 'sha512').toString('hex'));
+const decrypt = (m, s, i) => AES.decrypt(m, crypto.pbkdf2Sync(i, s, 256, 32, 'sha512').toString('hex'));
+
+const formatEnv = (raw, stringifyObject = true) => {
+  let value = raw;
+
+  // ensure no
+  if (Array.isArray(value) && value.every((x) => x.originFileObj && x.url)) {
+    value = value.map((x) => x.url);
+    if (value.length === 1) {
+      [value] = value;
+    }
+  }
+
+  // ensure no objects
+  if (stringifyObject) {
+    if (value && typeof value === 'object') {
+      value = JSON.stringify(value);
+    }
+  }
+
+  // ensure no line breaks for environment variables
+  if (value && typeof value === 'string') {
+    value = value.replace(/(\r\n|\n|\r)/gm, ' ');
+  }
+
+  return value;
+};
+
+// https://github.com/joaquimserafim/base64-url/blob/54d9c9ede66a8724f280cf24fd18c38b9a53915f/index.js#L10
+const escape = (str) => str.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
+const encodeEncryptionKey = (key) => escape(Buffer.from(key).toString('base64'));
+const unescape = (str) => (str + '==='.slice((str.length + 3) % 4)).replace(/-/g, '+').replace(/_/g, '/');
+const decodeEncryptionKey = (str) => new Uint8Array(Buffer.from(unescape(str), 'base64'));
+
+module.exports = {
+  encrypt,
+  decrypt,
+  formatEnv,
+  encodeEncryptionKey,
+  decodeEncryptionKey,
+};