@abtnode/core 1.8.65-beta-81d3340c → 1.8.65-beta-5405baf2

package/lib/blocklet/downloader/blocklet-downloader.js

@@ -0,0 +1,205 @@
+const { EventEmitter } = require('events');
+const fs = require('fs-extra');
+const path = require('path');
+const get = require('lodash/get');
+const { toBase58 } = require('@ocap/util');
+
+const defaultLogger = require('@abtnode/logger')('@abtnode/core:blocklet-downloader');
+
+const { forEachBlockletSync, getComponentBundleId } = require('@blocklet/meta/lib/util');
+const validateBlockletEntry = require('@blocklet/meta/lib/entry');
+
+const { BlockletSource, BLOCKLET_META_FILE, BLOCKLET_META_FILE_ALT, BLOCKLET_MODES } = require('@blocklet/constant');
+
+const { getBundleDir } = require('../../util/blocklet');
+const BundleDownloader = require('./bundle-downloader');
+const { CACHE_KEY } = require('./constants');
+
+const isSourceAccessible = (blocklet) =>
+  ![BlockletSource.upload, BlockletSource.local, BlockletSource.custom].includes(blocklet.source);
+
+const isMetaFileExist = (dir) => {
+  const blockletMetaFile = path.join(dir, BLOCKLET_META_FILE);
+  const blockletMetaFileAlt = path.join(dir, BLOCKLET_META_FILE_ALT);
+
+  return fs.existsSync(blockletMetaFile) || fs.existsSync(blockletMetaFileAlt);
+};
+
+/**
+ * @param {blocklet} component
+ * @param {{
+ *   cachedBundles: Array<{bundleId, integrity}>;
+ * }}
+ * @returns {boolean}
+ */
+const needDownload = (component, { installDir, logger = defaultLogger, cachedBundles = [] } = {}) => {
+  if (component.mode === BLOCKLET_MODES.DEVELOPMENT) {
+    return false;
+  }
+
+  if (component.source === BlockletSource.local) {
+    return false;
+  }
+
+  const bundleId = getComponentBundleId(component);
+  const bundleDir = getBundleDir(installDir, component.meta);
+
+  // check bundle exist
+
+  if (!isMetaFileExist(bundleDir)) {
+    return true;
+  }
+
+  // check bundle broken
+  try {
+    validateBlockletEntry(bundleDir, component.meta);
+  } catch (err) {
+    logger.error('bundle is broken', { bundleId, msg: err.message });
+    return true;
+  }
+
+  if (!isSourceAccessible(component)) {
+    return false;
+  }
+
+  // check integrity
+
+  const cachedBundle = cachedBundles.find((x) => x.bundleId === bundleId);
+
+  if (!cachedBundle) {
+    return true;
+  }
+
+  const integrity = get(component, 'meta.dist.integrity');
+  if (toBase58(integrity) === cachedBundle.integrity) {
+    return false;
+  }
+
+  logger.error(`find duplicate bundle with different integrity when downloading ${component.meta.title}`, {
+    bundleId,
+  });
+  return true;
+};
+
+class BlockletDownloader extends EventEmitter {
+  /**
+   * @param {{
+   *   cache: {
+   *     set: (key: string, value: any) => Promise
+   *     get: (key: string) => Promise<any>
+   *   }
+   * }}
+   */
+  constructor({ installDir, downloadDir, cache, logger = defaultLogger }) {
+    super();
+
+    this.installDir = installDir;
+    this.cache = cache;
+    this.logger = logger;
+
+    this.bundleDownloader = new BundleDownloader({
+      installDir,
+      downloadDir,
+      cache,
+    });
+  }
+
+  /**
+   * @param {{}} blocklet
+   * @param {{}} [context={}]
+   * @return {*}
+   */
+  async download(blocklet, context = {}) {
+    const {
+      meta: { name, did },
+    } = blocklet;
+
+    this.logger.info('Download Blocklet', { name, did });
+
+    const { preDownload = () => {}, postDownload = () => {} } = context;
+
+    const { downloadComponentIds, downloadList } = await this.getDownloadList({
+      blocklet,
+    });
+
+    await preDownload({ downloadList, downloadComponentIds });
+
+    try {
+      this.logger.info('Start Download Blocklet', {
+        name,
+        did,
+        bundles: downloadList.map((x) => get(x, 'dist.tarball')),
+      });
+      const tasks = [];
+      for (const meta of downloadList) {
+        const url = meta.dist.tarball;
+        tasks.push(this.bundleDownloader.download(meta, did, url, context));
+      }
+      const results = await Promise.all(tasks);
+
+      await postDownload({ downloadList, downloadComponentIds });
+
+      if (results.find((x) => x.isCancelled)) {
+        return { isCancelled: true };
+      }
+    } catch (error) {
+      this.logger.error('Download blocklet failed', { did, name, error });
+      await this.bundleDownloader.cancelDownload(blocklet.meta.did);
+      throw error;
+    }
+
+    return { isCancelled: false };
+  }
+
+  async cancelDownload(did) {
+    return this.bundleDownloader.cancelDownload(did);
+  }
+
+  /**
+   * @param {{
+   *   blocklet;
+   * }}
+   * @returns {{
+   *   downloadList: Array<blockletMeta>;
+   *   downloadComponentIds: Array<string>;
+   * }}
+   */
+  async getDownloadList({ blocklet }) {
+    const downloadComponentIds = [];
+    const metas = {};
+
+    const cachedBundles = (await this.cache.get(CACHE_KEY)) || [];
+
+    forEachBlockletSync(blocklet, (component, { id: componentId }) => {
+      const bundleId = getComponentBundleId(component);
+
+      if (metas[bundleId]) {
+        this.logger.info(`skip download duplicate bundle ${bundleId}`);
+        return;
+      }
+
+      const needComponentDownload = needDownload(component, { installDir: this.installDir, cachedBundles });
+      if (!needComponentDownload) {
+        this.logger.info(`skip download existed bundle ${bundleId}`, { source: component.source });
+        return;
+      }
+
+      if (!isSourceAccessible(component)) {
+        // should not throw error
+        // broken bundle should not block blocklet installing
+        // broken bundle should only block blocklet starting
+        this.logger.error(`Component bundle is broken and can not be recovered: ${bundleId}`);
+        return;
+      }
+
+      metas[bundleId] = component.meta;
+      downloadComponentIds.push(componentId);
+    });
+
+    const downloadList = Object.values(metas);
+
+    return { downloadList, downloadComponentIds };
+  }
+}
+
+module.exports = BlockletDownloader;

package/lib/blocklet/downloader/bundle-downloader.js

@@ -0,0 +1,267 @@
+const { EventEmitter } = require('events');
+const fs = require('fs-extra');
+const { fileURLToPath } = require('url');
+const path = require('path');
+const cloneDeep = require('lodash/cloneDeep');
+const { toBase58 } = require('@ocap/util');
+
+const defaultLogger = require('@abtnode/logger')('@abtnode/core:bundle-downloader');
+const downloadFile = require('@abtnode/util/lib/download-file');
+const Lock = require('@abtnode/util/lib/lock');
+
+const { getComponentBundleId } = require('@blocklet/meta/lib/util');
+
+const { verifyIntegrity } = require('../../util/blocklet');
+const { resolveDownload } = require('./resolve-download');
+const { CACHE_KEY } = require('./constants');
+
+class BundleDownloader extends EventEmitter {
+  /**
+   * @param {{
+   *   cache: {
+   *     set: (key: string, value: any) => Promise
+   *     get: (key: string) => Promise<any>
+   *   }
+   * }}
+   */
+  constructor({ installDir, downloadDir, cache, logger = defaultLogger }) {
+    super();
+
+    this.installDir = installDir;
+    this.downloadDir = downloadDir;
+    this.cache = cache;
+    this.logger = logger;
+
+    /**
+     * { did: Map({ <childDid>: <downloadFile.cancelCtrl> }) }
+     */
+    this.downloadCtrls = {};
+    /**
+     * { [download-did-version]: Lock }
+     */
+    this.downloadLocks = {};
+
+    this.cacheTarBallLock = new Lock();
+  }
+
+  /**
+   *
+   *
+   * @param {*} meta
+   * @param {*} rootDid
+   * @param {*} url
+   * @param {{}} [context={}]
+   * @return {*}
+   * @memberof BlockletManager
+   */
+  async download(meta, rootDid, url, context = {}) {
+    const { bundleName: name, bundleDid: did, version, dist = {} } = meta;
+    const { tarball, integrity } = dist;
+
+    const lockName = `download-${did}-${version}`;
+    let lock = this.downloadLocks[lockName];
+    if (!lock) {
+      lock = new Lock(lockName);
+      this.downloadLocks[lockName] = lock;
+    }
+
+    try {
+      await lock.acquire();
+      this.logger.info('downloaded blocklet for installing', { name, version, tarball, integrity });
+      const cwd = path.join(this.downloadDir, 'download', name);
+      await fs.ensureDir(cwd);
+      this.logger.info('start download blocklet', { name, version, cwd, tarball, integrity });
+      const tarballPath = await this._downloadTarball({
+        name,
+        did,
+        version,
+        cwd,
+        tarball,
+        integrity,
+        verify: true,
+        ctrlStore: this.downloadCtrls,
+        rootDid,
+        url,
+        context,
+      });
+      this.logger.info('downloaded blocklet tar file', { name, version, tarballPath });
+      if (tarballPath === downloadFile.CANCEL) {
+        lock.release();
+        return { isCancelled: true };
+      }
+
+      // resolve tarball and mv tarball to cache after resolved
+      await resolveDownload(tarballPath, this.installDir, { removeTarFile: false });
+      await this._addCacheTarFile(tarballPath, integrity, getComponentBundleId({ meta }));
+
+      this.logger.info('resolved blocklet tar file to install dir', { name, version });
+      lock.release();
+      return { isCancelled: false };
+    } catch (error) {
+      lock.release();
+      throw error;
+    }
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  async cancelDownload(rootDid) {
+    if (this.downloadCtrls[rootDid]) {
+      for (const cancelCtrl of this.downloadCtrls[rootDid].values()) {
+        cancelCtrl.cancel();
+      }
+    }
+  }
+
+  /**
+   *
+   *
+   * @param {{
+   *   url: string,
+   *   cwd: string,
+   *   tarball: string,
+   *   did: string,
+   *   integrity: string,
+   *   verify: boolean = true,
+   *   ctrlStore: {},
+   *   rootDid: string,
+   *   context: {} = {},
+   * }} { url, cwd, tarball, did, integrity, verify = true, ctrlStore = {}, rootDid, context = {} }
+   * @return {*}
+   * @memberof BlockletManager
+   */
+  async _downloadTarball({ url, cwd, tarball, did, integrity, verify = true, ctrlStore = {}, rootDid, context = {} }) {
+    fs.mkdirSync(cwd, { recursive: true });
+
+    const tarballName = url.split('/').slice(-1)[0];
+
+    const tarballPath = path.join(cwd, tarballName);
+
+    const { protocol } = new URL(url);
+
+    const cachedTarFile = await this._getCachedTarFile(integrity);
+    if (cachedTarFile) {
+      this.logger.info('found cache tarFile', { did, tarballName, integrity });
+
+      await this.cacheTarBallLock.acquire();
+      try {
+        await fs.move(cachedTarFile, tarballPath, { overwrite: true });
+      } catch (error) {
+        await this.cacheTarBallLock.release();
+        this.logger.error('move cache tar file failed', { cachedTarFile, tarballPath, error });
+        throw error;
+      }
+      await this.cacheTarBallLock.release();
+    } else if (protocol.startsWith('file')) {
+      await fs.copy(decodeURIComponent(fileURLToPath(url)), tarballPath);
+    } else {
+      const cancelCtrl = new downloadFile.CancelCtrl();
+
+      if (!ctrlStore[rootDid]) {
+        ctrlStore[rootDid] = new Map();
+      }
+      ctrlStore[rootDid].set(did, cancelCtrl);
+
+      const headers = context.headers ? cloneDeep(context.headers) : {};
+      const exist = (context.downloadTokenList || []).find((x) => x.did === did);
+      if (exist) {
+        headers['x-download-token'] = exist.token;
+      }
+
+      await downloadFile(url, path.join(cwd, tarballName), { cancelCtrl }, { ...context, headers });
+
+      if (ctrlStore[rootDid]) {
+        ctrlStore[rootDid].delete(did);
+        if (!ctrlStore[rootDid].size) {
+          delete ctrlStore[rootDid];
+        }
+      }
+
+      if (cancelCtrl.isCancelled) {
+        return downloadFile.CANCEL;
+      }
+    }
+
+    if (verify) {
+      try {
+        await verifyIntegrity({ file: tarballPath, integrity });
+      } catch (error) {
+        this.logger.error('verify integrity error', { error, tarball, url });
+        throw new Error(`${tarball} integrity check failed.`);
+      }
+    }
+
+    return tarballPath;
+  }
+
+  /**
+   * use LRU algorithm
+   */
+  async _addCacheTarFile(tarballPath, integrity, bundleId) {
+    // eslint-disable-next-line no-param-reassign
+    integrity = toBase58(integrity);
+
+    // move tarball to cache dir
+    const cwd = path.join(this.downloadDir, 'download-cache');
+    const cachePath = path.join(cwd, `${integrity}.tar.gz`);
+    await fs.ensureDir(cwd);
+    await fs.move(tarballPath, cachePath, { overwrite: true });
+
+    const cacheList = (await this.cache.get(CACHE_KEY)) || [];
+    const exist = cacheList.find((x) => x.bundleId === bundleId && x.integrity === integrity);
+
+    // update
+    if (exist) {
+      this.logger.info('update cache tarFile', { base58: integrity });
+      exist.accessAt = Date.now();
+      await this.cache.set(CACHE_KEY, cacheList);
+      return;
+    }
+
+    // add
+    cacheList.push({ integrity, accessAt: Date.now(), bundleId });
+    if (cacheList.length > 50) {
+      // find and remove
+      let minIndex = 0;
+      let min = cacheList[0];
+      cacheList.forEach((x, i) => {
+        if (x.accessAt < min.accessAt) {
+          minIndex = i;
+          min = x;
+        }
+      });
+
+      cacheList.splice(minIndex, 1);
+
+      const removeFile = path.join(cwd, `${min.integrity}.tar.gz`);
+      await this.cacheTarBallLock.acquire();
+      try {
+        await fs.remove(removeFile);
+      } catch (error) {
+        this.logger.error('remove cache tar file failed', { file: removeFile, error });
+      }
+      await this.cacheTarBallLock.release();
+
+      this.logger.info('remove cache tarFile', { base58: min.integrity });
+    }
+    this.logger.info('add cache tarFile', { base58: integrity });
+
+    // update
+    await this.cache.set(CACHE_KEY, cacheList);
+  }
+
+  async _getCachedTarFile(integrity) {
+    // eslint-disable-next-line no-param-reassign
+    integrity = toBase58(integrity);
+
+    const cwd = path.join(this.downloadDir, 'download-cache');
+    const cachePath = path.join(cwd, `${integrity}.tar.gz`);
+
+    if (fs.existsSync(cachePath)) {
+      return cachePath;
+    }
+
+    return null;
+  }
+}
+
+module.exports = BundleDownloader;

package/lib/blocklet/downloader/constants.js

@@ -0,0 +1,3 @@
+module.exports = {
+  CACHE_KEY: 'blocklet:manager:downloadCache',
+};

package/lib/blocklet/downloader/resolve-download.js

@@ -0,0 +1,168 @@
+const path = require('path');
+const fs = require('fs-extra');
+
+const validateBlockletEntry = require('@blocklet/meta/lib/entry');
+const { BLOCKLET_BUNDLE_FOLDER } = require('@blocklet/constant');
+
+const defaultLogger = require('@abtnode/logger')('@abtnode/core:resolve-download');
+
+const { getBlockletMeta } = require('../../util');
+
+const { ensureBlockletExpanded, expandTarball, getBundleDir } = require('../../util/blocklet');
+
+const asyncFs = fs.promises;
+
+/**
+ * decompress file, format dir and move to installDir
+ * @param {string} src file
+ * @param {{
+ *   cwd: string;
+ *   removeTarFile: boolean;
+ *   originalMeta; // for verification
+ *   logger;
+ * }} option
+ */
+const resolveDownload = async (
+  tarFile,
+  dist,
+  { cwd = '/', removeTarFile = true, logger = defaultLogger, originalMeta } = {}
+) => {
+  // eslint-disable-next-line no-param-reassign
+  tarFile = path.join(cwd, tarFile);
+  // eslint-disable-next-line no-param-reassign
+  dist = path.join(cwd, dist);
+
+  const downloadDir = path.join(path.dirname(tarFile), path.basename(tarFile, path.extname(tarFile)));
+  const tmp = `${downloadDir}-tmp`;
+  try {
+    await expandTarball({ source: tarFile, dest: tmp, strip: 0 });
+  } catch (error) {
+    logger.error('expand blocklet tar file error', { error });
+    throw error;
+  } finally {
+    if (removeTarFile) {
+      fs.removeSync(tarFile);
+    }
+  }
+  let installDir;
+  let meta;
+  try {
+    // resolve dir
+    let dir = tmp;
+    const files = await asyncFs.readdir(dir);
+    if (files.includes('package')) {
+      dir = path.join(tmp, 'package');
+    } else if (files.length === 1) {
+      const d = path.join(dir, files[0]);
+      if ((await asyncFs.stat(d)).isDirectory()) {
+        dir = d;
+      }
+    }
+
+    if (fs.existsSync(path.join(dir, BLOCKLET_BUNDLE_FOLDER))) {
+      dir = path.join(dir, BLOCKLET_BUNDLE_FOLDER);
+    }
+
+    logger.info('Move downloadDir to installDir', { downloadDir });
+    await fs.move(dir, downloadDir, { overwrite: true });
+    fs.removeSync(tmp);
+
+    meta = getBlockletMeta(downloadDir, { ensureMain: true });
+    const { did, name, version } = meta;
+
+    // validate
+    if (originalMeta && (originalMeta.did !== did || originalMeta.name !== name || originalMeta.version !== version)) {
+      logger.error('Meta has differences', { originalMeta, meta });
+      throw new Error('There are differences between the meta from tarball file and the original meta');
+    }
+    await validateBlockletEntry(downloadDir, meta);
+
+    await ensureBlockletExpanded(meta, downloadDir);
+
+    installDir = getBundleDir(dist, meta);
+    if (fs.existsSync(installDir)) {
+      fs.removeSync(installDir);
+      logger.info('cleanup blocklet upgrade dir', { name, version, installDir });
+    }
+
+    fs.mkdirSync(installDir, { recursive: true });
+
+    await fs.move(downloadDir, installDir, { overwrite: true });
+  } catch (error) {
+    fs.removeSync(downloadDir);
+    fs.removeSync(tmp);
+    throw error;
+  }
+
+  return { meta, installDir };
+};
+
+const resolveDiffDownload = async (
+  tarFile,
+  dist,
+  { meta: oldMeta, deleteSet, cwd = '/', logger = defaultLogger } = {}
+) => {
+  // eslint-disable-next-line no-param-reassign
+  tarFile = path.join(cwd, tarFile);
+  // eslint-disable-next-line no-param-reassign
+  dist = path.join(cwd, dist);
+
+  logger.info('Resolve diff download', { tarFile, cwd });
+  const downloadDir = path.join(path.dirname(tarFile), path.basename(tarFile, path.extname(tarFile)));
+  const diffDir = `${downloadDir}-diff`;
+  try {
+    await expandTarball({ source: tarFile, dest: diffDir, strip: 0 });
+    fs.removeSync(tarFile);
+  } catch (error) {
+    fs.removeSync(tarFile);
+    logger.error('expand blocklet tar file error', { error });
+    throw error;
+  }
+  logger.info('Copy installDir to downloadDir', { installDir: dist, downloadDir });
+  await fs.copy(getBundleDir(dist, oldMeta), downloadDir);
+  try {
+    // delete
+    logger.info('Delete files from downloadDir', { fileNum: deleteSet.length });
+    // eslint-disable-next-line no-restricted-syntax
+    for (const file of deleteSet) {
+      /* eslint-disable no-await-in-loop */
+      await fs.remove(path.join(downloadDir, file));
+    }
+    // walk & cover
+    logger.info('Move files from diffDir to downloadDir', { diffDir, downloadDir });
+    const walkDiff = async (dir) => {
+      const files = await asyncFs.readdir(dir);
+      // eslint-disable-next-line no-restricted-syntax
+      for (const file of files) {
+        const p = path.join(dir, file);
+        const stat = await asyncFs.stat(p);
+        if (stat.isDirectory()) {
+          await walkDiff(p);
+        } else if (stat.isFile()) {
+          await fs.move(p, path.join(downloadDir, path.relative(diffDir, p)), { overwrite: true });
+        }
+      }
+    };
+    await walkDiff(diffDir);
+    fs.removeSync(diffDir);
+    const meta = getBlockletMeta(downloadDir, { ensureMain: true });
+
+    await ensureBlockletExpanded(meta, downloadDir);
+
+    // move to installDir
+    const bundleDir = getBundleDir(dist, meta);
+    logger.info('Move downloadDir to installDir', { downloadDir, bundleDir });
+    await fs.move(downloadDir, bundleDir, { overwrite: true });
+
+    return { meta, installDir: bundleDir };
+  } catch (error) {
+    fs.removeSync(downloadDir);
+    fs.removeSync(diffDir);
+    throw error;
+  }
+};
+
+module.exports = {
+  resolveDownload,
+  resolveDiffDownload,
+};