@abtnode/core 1.8.69-beta-b0bb2d67 → 1.8.69-beta-76f8a46f

package/lib/blocklet/manager/helper/install-component-from-url.js

@@ -0,0 +1,173 @@
+const capitalize = require('lodash/capitalize');
+const { sign } = require('@arcblock/jwt');
+
+const logger = require('@abtnode/logger')('@abtnode/core:install-component-url');
+
+const { isFreeBlocklet, isComponentBlocklet } = require('@blocklet/meta/lib/util');
+const { titleSchema } = require('@blocklet/meta/lib/schema');
+const hasReservedKey = require('@blocklet/meta/lib/has-reserved-key');
+
+const { BlockletStatus, BlockletEvents, BlockletGroup } = require('@blocklet/constant');
+const {
+  getBlockletMetaFromUrl,
+  parseComponents,
+  filterDuplicateComponents,
+  ensureMeta,
+  checkStructVersion,
+  checkVersionCompatibility,
+  validateBlocklet,
+} = require('../../../util/blocklet');
+const StoreUtil = require('../../../util/store');
+const { formatName } = require('../../../util/get-domain-for-blocklet');
+
+const installComponentFromUrl = async ({
+  rootDid,
+  mountPoint,
+  url,
+  context,
+  title,
+  configs,
+  downloadTokenList,
+  sync,
+  manager,
+  states,
+}) => {
+  const blocklet = await states.blocklet.getBlocklet(rootDid);
+  if (!blocklet) {
+    throw new Error('Root blocklet does not exist');
+  }
+
+  checkStructVersion(blocklet);
+
+  const { inStore } = await StoreUtil.parseSourceUrl(url);
+
+  const meta = await getBlockletMetaFromUrl(url);
+
+  if (meta.group === BlockletGroup.gateway) {
+    throw new Error('Cannot add gateway component');
+  }
+
+  // 如果是一个付费的blocklet,并且url来源为Store, 需要携带token才能下载成功
+  if (!isFreeBlocklet(meta) && inStore) {
+    const info = await states.node.read();
+
+    // eslint-disable-next-line no-param-reassign
+    context = {
+      ...context,
+      headers: {
+        'x-server-did': info.did,
+        'x-server-public-key': info.pk,
+        'x-server-signature': sign(info.did, info.sk, {
+          exp: (Date.now() + 5 * 60 * 1000) / 1000,
+        }),
+      },
+      downloadTokenList: downloadTokenList || [],
+    };
+  }
+
+  if (!isComponentBlocklet(meta)) {
+    throw new Error('The blocklet cannot be a component');
+  }
+
+  if (title) {
+    meta.title = await titleSchema.validateAsync(title);
+  }
+
+  const newChildMeta = ensureMeta(meta);
+
+  // children
+  const newChild = {
+    meta: newChildMeta,
+    mountPoint: mountPoint || formatName(newChildMeta.name),
+    bundleSource: { url },
+  };
+
+  const { dynamicComponents } = await parseComponents(newChild);
+
+  const index = blocklet.children.findIndex((child) => child.meta.did === meta.did);
+  if (index >= 0) {
+    // if upgrade, do not update mountPoint and title
+    newChild.mountPoint = blocklet.children[index].mountPoint;
+    newChild.meta.title = blocklet.children[index].meta.title;
+    blocklet.children.splice(index, 1, newChild);
+  } else {
+    dynamicComponents.unshift(newChild);
+  }
+
+  const newChildren = filterDuplicateComponents(dynamicComponents, blocklet.children);
+
+  blocklet.children.push(...newChildren);
+
+  checkVersionCompatibility(blocklet.children);
+
+  const oldBlocklet = await manager._getBlockletForInstallation(rootDid);
+  const action = 'upgrade';
+  try {
+    // add component to db
+    await states.blocklet.addChildren(rootDid, newChildren);
+
+    // update configs
+    if (Array.isArray(configs)) {
+      if (hasReservedKey(configs)) {
+        throw new Error('Component key of environments can not start with `ABT_NODE_` or `BLOCKLET_`');
+      }
+
+      await states.blockletExtras.setConfigs([blocklet.meta.did, newChild.meta.did], configs);
+    }
+  } catch (err) {
+    logger.error('Add component failed', err);
+    await manager._rollback(action, rootDid, oldBlocklet);
+    throw err;
+  }
+
+  // new blocklet
+  const newBlocklet = await states.blocklet.setBlockletStatus(rootDid, BlockletStatus.waiting);
+
+  newBlocklet.children = blocklet.children;
+  await validateBlocklet(newBlocklet);
+
+  manager.emit(BlockletEvents.statusChange, newBlocklet);
+
+  const downloadParams = {
+    oldBlocklet: { ...oldBlocklet },
+    blocklet: { ...newBlocklet },
+    selectedComponentDids: [newChild.meta.did, ...newChildren.map((x) => x.meta.did)],
+    context,
+    postAction: action,
+  };
+
+  // backup rollback data
+  await manager._rollbackCache.backup({ did: rootDid, action, oldBlocklet });
+
+  if (sync) {
+    await manager._downloadAndInstall({ ...downloadParams, throwOnError: true });
+    return states.blocklet.getBlocklet(rootDid);
+  }
+
+  // add to queue
+  const ticket = manager.installQueue.push(
+    {
+      entity: 'blocklet',
+      action: 'download',
+      id: rootDid,
+      ...downloadParams,
+    },
+    rootDid
+  );
+
+  ticket.on('failed', async (err) => {
+    logger.error('queue failed', { entity: 'blocklet', action, did: rootDid, error: err });
+    await manager._rollback(action, rootDid, oldBlocklet);
+    manager.emit(`blocklet.${action}.failed`, { did: rootDid, err });
+    manager._createNotification(rootDid, {
+      title: `Blocklet ${capitalize(action)} Failed`,
+      description: `Blocklet ${rootDid} ${action} failed with error: ${err.message || 'queue exception'}`,
+      entityType: 'blocklet',
+      entityId: rootDid,
+      severity: 'error',
+    });
+  });
+  return newBlocklet;
+};
+
+module.exports = { installComponentFromUrl };

package/lib/blocklet/manager/helper/migrate-application-to-struct-v2.js

@@ -0,0 +1,377 @@
+const path = require('path');
+const fs = require('fs-extra');
+const pick = require('lodash/pick');
+
+const logger = require('@abtnode/logger')('@abtnode/core:migrate-application-to-struct-v2');
+
+const { forEachBlockletSync, getSharedConfigObj } = require('@blocklet/meta/lib/util');
+const { SLOT_FOR_IP_DNS_SITE } = require('@abtnode/constant');
+
+const {
+  BlockletStatus,
+  BlockletSource,
+  BlockletEvents,
+  BlockletGroup,
+  BLOCKLET_DEFAULT_PORT_NAME,
+  BLOCKLET_INTERFACE_TYPE_WEB,
+  BLOCKLET_INTERFACE_PUBLIC,
+  BLOCKLET_DYNAMIC_PATH_PREFIX,
+  BLOCKLET_INTERFACE_PROTOCOL_HTTP,
+  BLOCKLET_DEFAULT_PATH_REWRITE,
+  BLOCKLET_DEFAULT_VERSION,
+  BLOCKLET_LATEST_SPEC_VERSION,
+  BLOCKLET_CONFIGURABLE_KEY,
+  BLOCKLET_META_FILE,
+  BLOCKLET_UPLOADS_DIR,
+} = require('@blocklet/constant');
+const { update: updateMetaFile } = require('@blocklet/meta/lib/file');
+const { cloneDeep } = require('lodash');
+const { isInProgress } = require('../../../util');
+const { getBlockletDomainGroupName } = require('../../../util/router');
+const { getIpDnsDomainForBlocklet } = require('../../../util/get-domain-for-blocklet');
+const { getBundleDir } = require('../../../util/blocklet');
+
+const sortMoveListBySrc = (list) => {
+  return list.sort((a, b) => (a.src.length > b.src.length ? -1 : 1));
+};
+
+const validateDataMoveList = (list) => {
+  const srcList = list.map((x) => x.src);
+  const distList = list.map((x) => x.dist);
+
+  // find duplicate element in src
+  const duplicateInSrc = srcList.filter((x, i) => srcList.indexOf(x) !== i);
+  if (duplicateInSrc.length) {
+    throw new Error(`Duplicate element in src of dataMoveList: ${duplicateInSrc}`);
+  }
+
+  // find duplicate element in list
+  const duplicateInDist = distList.filter((x, i) => distList.indexOf(x) !== i);
+  if (duplicateInDist.length) {
+    throw new Error(`Duplicate element in dist of dataMoveList: ${duplicateInDist}`);
+  }
+};
+
+const fillBlockletData = (data, app, id) => {
+  Object.assign(data, {
+    meta: {
+      name: id,
+      did: id,
+      bundleDid: id,
+      bundleName: id,
+      title: app.meta.title || '',
+      description: app.meta.description || '',
+      version: BLOCKLET_DEFAULT_VERSION,
+      group: BlockletGroup.gateway,
+      interfaces: [
+        {
+          type: BLOCKLET_INTERFACE_TYPE_WEB,
+          name: BLOCKLET_INTERFACE_PUBLIC,
+          path: BLOCKLET_DEFAULT_PATH_REWRITE,
+          prefix: BLOCKLET_DYNAMIC_PATH_PREFIX,
+          port: BLOCKLET_DEFAULT_PORT_NAME,
+          protocol: BLOCKLET_INTERFACE_PROTOCOL_HTTP,
+        },
+      ],
+      specVersion: BLOCKLET_LATEST_SPEC_VERSION,
+      environments: [],
+    },
+    source: BlockletSource.custom,
+    status: BlockletStatus.stopped,
+  });
+
+  if (app.meta.logo) {
+    data.meta.logo = app.meta.logo;
+  }
+};
+
+const appSystemFiles = ['logo.svg', 'rbac.db', 'session.db', 'user.db', '.assets', BLOCKLET_UPLOADS_DIR];
+
+const migrateApplicationToStructV2 = async ({ did, appSk: newAppSk, context = {}, manager, states }) => {
+  logger.info('Preparing data for migration', { did });
+
+  if (!newAppSk) {
+    throw new Error('appSk is required');
+  }
+
+  const oldBlocklet = await manager.getBlocklet(did, { throwOnNotExist: true, ensureIntegrity: true });
+
+  if (oldBlocklet.structVersion || oldBlocklet.externalSk) {
+    throw new Error('Blocklet already migrated', pick(oldBlocklet, ['structVersion', 'externalSk']));
+  }
+
+  if (oldBlocklet.migratedFrom?.length) {
+    throw new Error('Blocklet already migrated');
+  }
+
+  if (isInProgress(oldBlocklet.status) || oldBlocklet.status === BlockletStatus.running) {
+    throw new Error('Please stop blocklet before migration');
+  }
+
+  const extraData = await states.blockletExtras.findOne({ did: oldBlocklet.meta.did });
+  const siteData = await states.site.findOneByBlocklet(oldBlocklet.meta.did);
+
+  const backupBlocklet = await states.blocklet.findOne({ 'meta.did': oldBlocklet.meta.did });
+  const backupExtra = cloneDeep(extraData);
+  const backupSite = await states.site.findOne({ _id: siteData.id });
+
+  const permanentAppDid = oldBlocklet.appDid;
+  const permanentAppSk = oldBlocklet.environmentObj.BLOCKLET_APP_SK;
+
+  // change index of extraData
+  extraData.did = permanentAppDid;
+  if (extraData.meta) {
+    extraData.meta.did = permanentAppDid;
+    extraData.meta.name = permanentAppDid;
+  }
+  // fork root component's configs to container
+  // FIXME: the configs in container should be managed in dashboard ?
+  // set permanent appSk first and then migrate to new appSk
+  extraData.configs = (extraData.configs || []).filter((x) => x.name !== BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
+  extraData.configs.push({
+    key: BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK,
+    value: permanentAppSk,
+    secure: true,
+    shared: false,
+  });
+  // clean children configs
+  extraData.children = [];
+  // clean dirty data in settings
+  if (extraData.settings) {
+    extraData.settings.children = [];
+    delete extraData.settings.navigation;
+  }
+
+  // delete system generated rules in siteData
+  siteData.rules = siteData.rules.filter((rule) => !rule.isProtected && rule.to.did !== oldBlocklet.meta.did);
+  // change index of siteData
+  siteData.domain = getBlockletDomainGroupName(permanentAppDid);
+
+  const blockletData = {
+    children: [],
+  };
+
+  const dataDirSrc = path.join(oldBlocklet.env.dataDir);
+  const dataDirDist = path.join(manager.dataDirs.data, permanentAppDid);
+  if (fs.existsSync(dataDirDist)) {
+    throw new Error(`blocklet data dir already exist: ${dataDirDist}`);
+  }
+  const dataMoveList = [];
+  let currentMoveIndex = -1; // for rollback
+  appSystemFiles.forEach((file) => {
+    const src = path.join(dataDirSrc, file);
+    const dist = path.join(dataDirDist, file);
+    if (fs.existsSync(src)) {
+      dataMoveList.push({ src, dist });
+    }
+  });
+
+  forEachBlockletSync(oldBlocklet, (component, { level, ancestors }) => {
+    // if have duplicate component, user should fix it manually
+    if (blockletData.children.some((x) => x.meta.bundleDid === component.meta.bundleDid)) {
+      throw new Error(
+        `Find duplicate component ${component.meta.title}, please delete useless components and migrate again`
+      );
+    }
+
+    // If old root component is a container, just skip it
+    if (level === 0 && component.meta.group === BlockletGroup.gateway) {
+      fillBlockletData(blockletData, component, permanentAppDid);
+      return;
+    }
+
+    // If old root component is a blocklet, make it to be level one component
+    if (level === 0 && component.meta.group !== BlockletGroup.gateway) {
+      fillBlockletData(blockletData, component, permanentAppDid);
+
+      // add root component to blockletData
+      const { source, deployedFrom } = component;
+      let bundleSource;
+      if (source === BlockletSource.store) {
+        bundleSource = {
+          store: component.deployedFrom,
+          name: component.meta.bundleName,
+          version: 'latest',
+        };
+      } else if (source === BlockletSource.url) {
+        bundleSource = {
+          url: component.deployedFrom,
+        };
+      }
+
+      blockletData.children.push({
+        ...pick(component, ['mode', 'status']),
+        mountPoint: component.mountPoint || '/',
+        meta: {
+          ...component.meta,
+          // change index of component
+          did: component.meta.bundleDid,
+          name: component.meta.bundleName,
+        },
+        bundleSource,
+        source,
+        deployedFrom,
+        children: [],
+      });
+
+      // add root component to extraData
+      extraData.children.push({
+        // change index of component
+        did: component.meta.bundleDid,
+        // filter application configs in root component
+        configs: (component.configs || []).filter((x) => !x.key.startsWith('BLOCKLET_')),
+      });
+
+      // move root component data by file
+      const componentFolders = (component.children || []).map((x) => x.meta.name.split('/')[0]);
+      const files = fs.readdirSync(dataDirSrc);
+      for (const file of files) {
+        if (!componentFolders.includes(file) && !appSystemFiles.includes(file)) {
+          dataMoveList.push({
+            src: path.join(dataDirSrc, file),
+            dist: path.join(dataDirDist, component.meta.bundleName, file),
+          });
+        }
+      }
+
+      return;
+    }
+
+    // add component to blockletData
+    blockletData.children.push({
+      ...pick(component, ['mode', 'status', 'bundleSource', 'source', 'deployedFrom']),
+      meta: {
+        ...component.meta,
+        // change index of component
+        did: component.meta.bundleDid,
+        name: component.meta.bundleName,
+      },
+      // keep pathPrefix of the component remains the same as before
+      mountPoint: path.join('/', ...ancestors.slice(1).map((x) => x.mountPoint || ''), component.mountPoint || '/'),
+      children: [],
+    });
+
+    // add component to extraData
+    extraData.children.push({
+      did: component.meta.bundleDid,
+      configs: component.configs || [],
+    });
+
+    const sharedConfigObj = getSharedConfigObj(component, ancestors);
+    if (sharedConfigObj) {
+      Object.entries(sharedConfigObj).forEach(([key, value]) => {
+        if (!extraData.configs.some((x) => x.key === key)) {
+          logger.info('add shared config to container configs', { did, key, value });
+          extraData.configs.push({ key, value });
+        }
+      });
+    }
+
+    // move component data dir
+    const src = component.env.dataDir;
+    const dist = path.join(dataDirSrc, component.meta.bundleName);
+    if (src !== dist) {
+      dataMoveList.push({ src: component.env.dataDir, dist: path.join(dataDirDist, component.meta.bundleName) });
+    }
+  });
+
+  // ensure the deepest component is moved first
+  const sortedDataMoveList = sortMoveListBySrc(dataMoveList);
+  validateDataMoveList(sortedDataMoveList);
+
+  // encrypt data in extras
+  states.blockletExtras.encryptSecurityData({ data: extraData });
+
+  // refresh ip dns domain
+  siteData.domainAliases = siteData.domainAliases.filter(
+    (x) => !x.value.includes(SLOT_FOR_IP_DNS_SITE) || !x.isProtected
+  );
+  siteData.domainAliases.push({ value: getIpDnsDomainForBlocklet(blockletData), isProtected: true });
+
+  // update state
+  let newBlocklet;
+  try {
+    logger.info('Start migrate application state', { did, appDid: permanentAppDid });
+    // delete old db in db proxy
+    await manager.teamManager.deleteTeam(oldBlocklet.meta.did);
+
+    // re create blocklet
+    await states.blocklet.remove({ appDid: permanentAppDid });
+    await states.blocklet.addBlocklet(blockletData);
+    await states.blocklet.updateStructV1Did(permanentAppDid, oldBlocklet.meta.did);
+
+    // fake install bundle
+    const bundleDir = getBundleDir(manager.installDir, blockletData.meta);
+    fs.mkdirSync(bundleDir, { recursive: true });
+    updateMetaFile(path.join(bundleDir, BLOCKLET_META_FILE), blockletData.meta);
+
+    if (oldBlocklet.meta.logo) {
+      const fileName = oldBlocklet.meta.logo;
+      const src = path.join(getBundleDir(manager.installDir, oldBlocklet.meta), fileName);
+      const dist = path.join(bundleDir, fileName);
+
+      await fs.copy(src, dist);
+    }
+
+    // update
+    await states.blockletExtras.update({ did: oldBlocklet.meta.did }, extraData);
+
+    // update environment, generate appDid and appPid
+    await manager._updateBlockletEnvironment(permanentAppDid);
+
+    // rotate to newAppSk
+    await manager.config({
+      did: permanentAppDid,
+      configs: [{ key: 'BLOCKLET_APP_SK', value: newAppSk, secure: true }],
+      skipHook: true,
+      skipDidDocument: true,
+    });
+
+    // update routing
+    await states.site.update(
+      { _id: siteData.id },
+      { $set: { domain: siteData.domain, domainAliases: siteData.domainAliases, rules: siteData.rules } }
+    );
+
+    newBlocklet = await manager.getBlocklet(permanentAppDid);
+
+    logger.info('Start migrate application data', { sortedDataMoveList });
+
+    // move data
+    fs.mkdirSync(dataDirDist, { recursive: true });
+    for (let i = 0; i < sortedDataMoveList.length; i++) {
+      const { src, dist } = sortedDataMoveList[i];
+      fs.moveSync(src, dist);
+      currentMoveIndex = i;
+    }
+  } catch (error) {
+    logger.error('Migrate application state failed: ', { did, error });
+
+    await states.blocklet.remove({ _id: backupBlocklet._id });
+    await states.blocklet.remove({ 'meta.did': blockletData.meta.did });
+    await states.blocklet.insert(backupBlocklet);
+    await states.blockletExtras.remove({ _id: backupExtra._id });
+    await states.blockletExtras.insert(backupExtra);
+    await states.site.remove({ _id: backupSite._id });
+    await states.site.insert(backupSite);
+
+    logger.info('Rollback application state');
+
+    // rollback data
+    fs.ensureDirSync(dataDirSrc);
+    for (let i = currentMoveIndex; i >= 0; i--) {
+      const { src, dist } = sortedDataMoveList[i];
+      fs.moveSync(dist, src);
+    }
+    fs.removeSync(dataDirDist);
+
+    logger.info('Rollback application data');
+
+    throw error;
+  }
+
+  // 通过 event 触发 ensureBlockletRouting
+  manager.emit(BlockletEvents.installed, { blocklet: newBlocklet, context });
+};
+
+module.exports = { migrateApplicationToStructV2, sortMoveListBySrc };

package/lib/blocklet/manager/helper/upgrade-components.js

@@ -0,0 +1,152 @@
+/* eslint-disable no-await-in-loop */
+const cloneDeep = require('lodash/cloneDeep');
+const capitalize = require('lodash/capitalize');
+
+const logger = require('@abtnode/logger')('@abtnode/core:upgrade-component');
+
+const { BlockletStatus, BlockletEvents } = require('@blocklet/constant');
+const {
+  getUpdateMetaList,
+  parseComponents,
+  checkDuplicateComponents,
+  filterDuplicateComponents,
+  checkStructVersion,
+  checkVersionCompatibility,
+  validateBlocklet,
+} = require('../../../util/blocklet');
+
+const check = async ({ did, states }) => {
+  const blocklet = await states.blocklet.getBlocklet(did);
+  checkStructVersion(blocklet);
+
+  const newBlocklet = cloneDeep(blocklet);
+
+  const newChildren = [];
+
+  for (const child of newBlocklet.children || []) {
+    if (child.bundleSource) {
+      const {
+        staticComponents: [newChild],
+        dynamicComponents,
+      } = await parseComponents({
+        meta: {
+          staticComponents: [
+            {
+              source: child.bundleSource,
+              name: child.meta.name,
+              title: child.meta.title,
+              mountPoint: child.mountPoint,
+            },
+          ],
+        },
+      });
+      newChild._dynamicComponents = dynamicComponents;
+      newChildren.push(newChild);
+    } else {
+      const { dynamicComponents } = await parseComponents(child);
+      child._dynamicComponents = dynamicComponents;
+      newChildren.push(child);
+    }
+  }
+
+  checkDuplicateComponents(newChildren);
+
+  const updateList = getUpdateMetaList(blocklet, { ...blocklet, children: newChildren });
+
+  if (!updateList.length) {
+    return {};
+  }
+
+  // start session
+  const { id: updateId } = await states.session.start({
+    did,
+    children: newChildren,
+  });
+
+  return {
+    updateId,
+    updateList,
+  };
+};
+
+const upgrade = async ({ updateId, selectedComponentDids, context, states, manager }) => {
+  if (!selectedComponentDids?.length) {
+    throw new Error('At least one component needs to be selected');
+  }
+
+  const sessionData = await states.session.end(updateId);
+  const { did } = sessionData;
+  const oldBlocklet = await manager._getBlockletForInstallation(did);
+  checkStructVersion(oldBlocklet);
+
+  // parse children
+  let dynamicComponents = [];
+  const children = cloneDeep(oldBlocklet.children).map((oldComponent) => {
+    const newComponent = sessionData.children.find((x) => x.meta.did === oldComponent.meta.did);
+    if (newComponent && selectedComponentDids.includes(newComponent.meta.did)) {
+      dynamicComponents.push(...(newComponent._dynamicComponents || []));
+      delete newComponent._dynamicComponents;
+      return newComponent;
+    }
+    return oldComponent;
+  });
+  dynamicComponents = filterDuplicateComponents(dynamicComponents, children);
+  children.push(...dynamicComponents);
+
+  // selectedComponentDids
+  selectedComponentDids.push(...dynamicComponents.map((x) => x.meta.did));
+
+  checkVersionCompatibility(children);
+
+  logger.info('upgrade blocklet children', {
+    did,
+    children: children.map((x) => ({ name: x.meta.name, version: x.meta.version })),
+  });
+
+  // new blocklet
+  const newBlocklet = await states.blocklet.setBlockletStatus(did, BlockletStatus.waiting);
+
+  newBlocklet.children = children;
+  await validateBlocklet(newBlocklet);
+
+  manager.emit(BlockletEvents.statusChange, newBlocklet);
+
+  const action = 'upgrade';
+
+  // backup rollback data
+  await manager._rollbackCache.backup({ did, action, oldBlocklet });
+
+  // add to queue
+  const ticket = manager.installQueue.push(
+    {
+      entity: 'blocklet',
+      action: 'download',
+      id: did,
+      oldBlocklet: { ...oldBlocklet },
+      blocklet: { ...newBlocklet },
+      selectedComponentDids: selectedComponentDids || [],
+      context,
+      postAction: action,
+    },
+    did
+  );
+
+  ticket.on('failed', async (err) => {
+    logger.error('queue failed', { entity: 'blocklet', action, did, error: err });
+    await manager._rollback(action, did, oldBlocklet);
+    manager.emit(`blocklet.${action}.failed`, { did, err });
+    manager._createNotification(did, {
+      title: `Blocklet ${capitalize(action)} Failed`,
+      description: `Blocklet ${did} ${action} failed with error: ${err.message || 'queue exception'}`,
+      entityType: 'blocklet',
+      entityId: did,
+      severity: 'error',
+    });
+  });
+  return newBlocklet;
+};
+
+module.exports = {
+  check,
+  upgrade,
+};