@abtnode/core 1.15.17 → 1.16.0-beta-b16cb035

package/lib/blocklet/manager/helper/install-application-from-backup.js

@@ -0,0 +1,177 @@
+const fs = require('fs-extra');
+const path = require('path');
+const omit = require('lodash/omit');
+
+const { forEachBlockletSync, getBlockletAppIdList } = require('@blocklet/meta/lib/util');
+const getBlockletInfo = require('@blocklet/meta/lib/info');
+
+const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+
+const logger = require('@abtnode/logger')('@abtnode/core:install-app-backup');
+
+const { validateBlocklet, checkDuplicateAppSk, getAppDirs } = require('../../../util/blocklet');
+
+/**
+ * backup 目录结构
+ *  /blocklets/<name1>version>
+ *  /blocklets/<name2>version>
+ *  /blocklets/<name3>version>
+ *  /data
+ *  /blocklet.json
+ *  /blocklet_extras.json
+ * @see Blocklet 端到端备份方案的设计与实现（一期） https://team.arcblock.io/comment/discussions/e62084d5-fafa-489e-91d5-defcd6e93223
+ * @param {{ url: string, appSk: string, moveDir: boolean, manager: import('../disk'), states: import('../../../states/index'), context: Record<string, string> }}
+ * @return {Promise<any>}
+ */
+const installApplicationFromBackup = async ({ url, appSk, moveDir, context = {}, states, manager } = {}) => {
+  // TODO: support more url schema feature (http, did-spaces)
+  if (!url.startsWith('file://')) {
+    throw new Error('url must starts with file://');
+  }
+
+  const dir = url.replace('file://', '');
+
+  if (!dir || !fs.existsSync(dir)) {
+    throw new Error(`dir(${dir}) does not exist`);
+  }
+
+  // parse data from source dir
+  const srcBundleDirs = await getAppDirs(path.join(dir, 'blocklets'));
+
+  const srcDataDir = path.join(dir, 'data');
+
+  /** @type {import('@abtnode/client').BlockletState} */
+  const state = omit(fs.readJSONSync(path.join(dir, 'blocklet.json')), [
+    '_id',
+    'createdAt',
+    'updatedAt',
+    'installedAt',
+    'startedAt',
+  ]);
+
+  if (!state.structVersion) {
+    throw new Error(
+      'Only application of structVersion 2 can be restored on this server. Please migrate you application and re-backup your application on your previous server.'
+    );
+  }
+
+  const { meta } = state;
+  const { did, name: appName } = meta;
+
+  const extra = omit(fs.readJSONSync(path.join(dir, 'blocklet-extras.json')), ['_id', 'createdAt', 'updatedAt']);
+
+  if (state.meta.did !== extra.did) {
+    throw new Error(`did does not match in blocklet.json (${state.meta.did}) and blocklet_extra.json ${extra.did}`);
+  }
+
+  forEachBlockletSync(state, (component) => {
+    delete component.status;
+    delete component.ports;
+    delete component.environments;
+  });
+
+  await validateBlocklet({ meta });
+
+  const existState = await states.blocklet.hasBlocklet(did);
+
+  if (existState) {
+    logger.error('blocklet is already exist', { did });
+    throw new Error(`blocklet is already exist. did: ${did}`);
+  }
+
+  if (appSk) {
+    extra.configs = extra.configs || [];
+    const skConfig = extra.configs.find((x) => x.key === BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK);
+    if (skConfig) {
+      skConfig.value = appSk;
+      skConfig.secure = true;
+      skConfig.shared = false;
+    } else {
+      extra.configs.push({
+        key: BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK,
+        value: appSk,
+        secure: true,
+        shared: false,
+      });
+    }
+  }
+
+  // validate appDid
+  const nodeInfo = await states.node.read();
+  const appIdList = getBlockletAppIdList(state);
+  const { wallet } = getBlockletInfo({ meta: state.meta, configs: extra.configs, environments: [] }, nodeInfo.sk);
+  if (appIdList.includes(wallet.address) === false) {
+    throw new Error('blocklet appDid is different from the previous one');
+  }
+  await checkDuplicateAppSk({ sk: wallet.secretKey, states });
+
+  states.blockletExtras.encryptSecurityData({ data: extra, rootDid: extra.did });
+
+  logger.info('installFromBackup', { srcBundleDirs, srcDataDir });
+
+  try {
+    // copy extra
+    const existExtra = await states.blockletExtras.find({ did });
+    if (existExtra) {
+      // 如果数据存在, 当前视为脏数据, 直接删除
+      logger.error('old extra state exists and will be force removed', { existExtra });
+      await states.blockletExtras.remove({ did });
+    }
+    await states.blockletExtras.insert(extra);
+    logger.info('blocklet extra is copied successfully');
+
+    // add blocklet
+    await states.blocklet.addBlocklet(state);
+    logger.info('blocklet state is added successfully');
+
+    // copy bundle
+    // 假设相同名称的应用，肯定是同一个应用
+    // 假设版本号相同时, 应用不会变更
+    // FIXME: blocklet bundle name/did 不是唯一的. 修改 blocklet bundle name/did 生成方式 使 name/bundle 唯一
+    await Promise.all(
+      srcBundleDirs.map(async ({ key: bundleName, dir: srcDir }) => {
+        const installDir = path.join(manager.dataDirs.blocklets, bundleName);
+        if (fs.existsSync(installDir)) {
+          logger.info(`${bundleName} is already exist`);
+          return;
+        }
+
+        fs.mkdirSync(installDir, { recursive: true });
+        if (moveDir) {
+          await fs.move(srcDir, installDir, { overwrite: true });
+        } else {
+          await fs.copy(srcDir, installDir);
+        }
+        logger.info(`bundle  is ${moveDir ? 'moved' : 'copied'} successfully`, { installDir });
+      })
+    );
+    // 从 store 下载 blocklet
+    await manager.blockletDownloader.download(state, { skipCheckIntegrity: true });
+
+    const dataDir = path.join(manager.dataDirs.data, appName);
+    if (fs.existsSync(dataDir)) {
+      logger.error('old data exists and will be force removed', { dataDir });
+      await fs.remove(dataDir);
+    }
+    fs.mkdirSync(dataDir, { recursive: true });
+    if (fs.existsSync(srcDataDir)) {
+      if (moveDir) {
+        await fs.move(srcDataDir, dataDir, { overwrite: true });
+      } else {
+        await fs.copy(srcDataDir, dataDir);
+      }
+      logger.info(`data is ${moveDir ? 'moved' : 'copied'} successfully`);
+    }
+  } catch (error) {
+    logger.error('installFromBackup failed', { did, error });
+
+    await manager._rollback('install', did);
+
+    throw error;
+  }
+
+  logger.info('start install blocklet', { did });
+  return manager._installBlocklet({ did, context });
+};
+
+module.exports = { installApplicationFromBackup };

package/lib/blocklet/manager/helper/install-application-from-dev.js

@@ -0,0 +1,94 @@
+const path = require('path');
+const fs = require('fs-extra');
+
+const { toSvg: createDidLogo } =
+  process.env.NODE_ENV !== 'test' ? require('@arcblock/did-motif') : require('@arcblock/did-motif/dist/did-motif.cjs');
+const { BlockletStatus, BLOCKLET_MODES, fromBlockletStatus, BlockletSource } = require('@blocklet/constant');
+
+const logger = require('@abtnode/logger')('@abtnode/core:install-app-dev');
+const { ensureMeta } = require('../../../util/blocklet');
+
+/**
+ *
+ * @param {{
+ *  manager: import('../disk'),
+ *  states: import('../../../states/index')
+ * }}
+ * @returns
+ */
+const installApplicationFromDev = async ({ folder, meta, states, manager } = {}) => {
+  const { did, version } = meta;
+
+  const exist = await states.blocklet.getBlocklet(did);
+  if (exist) {
+    if (exist.mode === BLOCKLET_MODES.PRODUCTION) {
+      throw new Error('The blocklet of production mode already exists, please remove it before developing');
+    }
+
+    const status = fromBlockletStatus(exist.status);
+    if (['starting', 'running'].includes(status)) {
+      throw new Error(`The blocklet is already on ${status}, please stop it before developing`);
+    }
+
+    logger.info('remove blocklet for dev', { did, version });
+
+    await manager.delete({ did, keepLogsDir: false });
+  }
+
+  // delete process
+  try {
+    await manager.deleteProcess({ did });
+    logger.info('delete blocklet precess for dev', { did, version });
+  } catch (err) {
+    if (process.env.NODE_ENV !== 'development') {
+      logger.error('failed to delete blocklet process for dev', { error: err });
+    }
+  }
+
+  // create component
+  const component = {
+    meta: ensureMeta(meta),
+    mountPoint: '/',
+    source: BlockletSource.local,
+    deployedFrom: folder,
+    status: BlockletStatus.installed,
+    mode: BLOCKLET_MODES.DEVELOPMENT,
+  };
+
+  // create app
+  const added = await manager._addBlocklet({
+    component,
+    name: meta.name,
+    did: meta.did,
+    mode: BLOCKLET_MODES.DEVELOPMENT,
+  });
+  logger.info('add blocklet for dev', { did, version, meta });
+
+  await manager._downloadBlocklet(added);
+
+  // Add Config
+  await manager._setConfigsFromMeta(did);
+
+  // should ensure blocklet integrity
+  let blocklet = await manager.ensureBlocklet(did);
+
+  // pre install
+  await manager._runPreInstallHook(blocklet);
+
+  // Add environments
+  await manager._updateBlockletEnvironment(did);
+  blocklet = await manager.getBlocklet(did);
+
+  // post install
+  await manager._runPostInstallHook(blocklet);
+
+  await states.blocklet.setBlockletStatus(did, BlockletStatus.installed);
+
+  blocklet = await manager.getBlocklet(did);
+
+  await fs.writeFile(path.join(blocklet.env.dataDir, 'logo.svg'), createDidLogo(blocklet.meta.did));
+
+  return blocklet;
+};
+
+module.exports = { installApplicationFromDev };

package/lib/blocklet/manager/helper/install-application-from-general.js

@@ -0,0 +1,188 @@
+const joi = require('joi');
+
+const { BLOCKLET_INSTALL_TYPE, NODE_MODES } = require('@abtnode/constant');
+const { BlockletStatus, BlockletEvents, BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+
+const logger = require('@abtnode/logger')('@abtnode/core:install-app-general');
+const getApplicationWallet = require('@blocklet/meta/lib/wallet');
+
+const StoreUtil = require('../../../util/store');
+const { getBlockletMetaFromUrl, ensureMeta, validateStore, validateInServerless } = require('../../../util/blocklet');
+
+/**
+ *
+ * @param {{
+ *  manager: import('../disk'),
+ *  states: import('../../../states/index')
+ * }} param0
+ * @returns
+ */
+const installApplicationFromGeneral = async ({
+  type,
+  appSk,
+  sync,
+  delay,
+  controller,
+  title,
+  description,
+  did: bundleDid,
+  storeUrl,
+  url: inputUrl,
+  context = {},
+  states,
+  manager,
+} = {}) => {
+  const nodeInfo = await states.node.read();
+
+  let componentSourceUrl;
+  if (type === BLOCKLET_INSTALL_TYPE.STORE) {
+    if (!storeUrl) {
+      throw new Error('store url should not be empty');
+    }
+    componentSourceUrl = StoreUtil.getBlockletMetaUrl({ did: bundleDid, storeUrl });
+    await validateStore(nodeInfo, componentSourceUrl);
+  } else if (type === BLOCKLET_INSTALL_TYPE.URL) {
+    componentSourceUrl = inputUrl;
+    await validateStore(nodeInfo, componentSourceUrl);
+  } else if (type === BLOCKLET_INSTALL_TYPE.CREATE) {
+    await joi.string().label('title').max(20).required().validateAsync(title);
+    await joi.string().label('description').max(80).required().validateAsync(description);
+  } else {
+    throw new Error(`Should not be here: unknown type ${type}`);
+  }
+
+  // check install count
+  if (controller?.nftId) {
+    // sometimes nedb will throw error if use states.blocklet.count({ ['controller.nftId']: controller.nftId })
+    const installedCount = await states.blockletExtras.count({ 'controller.nftId': controller.nftId });
+
+    if (installedCount >= (controller.appMaxCount || 1)) {
+      throw new Error(
+        `You can only install ${controller.appMaxCount} blocklet with this credential: ${controller.nftId}`
+      );
+    }
+  }
+
+  // app wallet
+  const wallet = getApplicationWallet(appSk);
+  const did = wallet.address;
+  const name = wallet.address;
+
+  // check exist
+  const exist = await states.blocklet.hasBlocklet(did);
+  if (exist) {
+    throw new Error(`blocklet ${did} already exists`);
+  }
+
+  // remove dirty data
+  const oldExtraState = await states.blockletExtras.findOne({ did });
+  if (oldExtraState) {
+    logger.error('find dirty data in blocklet extra', { did });
+    await states.blockletExtras.remove({ did });
+  }
+
+  // create component
+  let component;
+  if (componentSourceUrl) {
+    const meta = await getBlockletMetaFromUrl(componentSourceUrl);
+
+    if (nodeInfo.mode === NODE_MODES.SERVERLESS) {
+      validateInServerless({ blockletMeta: meta });
+    }
+
+    component = {
+      meta: ensureMeta(meta),
+      mountPoint: '/',
+      bundleSource: { url: componentSourceUrl },
+    };
+  }
+
+  // create app
+  const blocklet = await manager._addBlocklet({ component, name, did, title, description });
+  logger.info('blocklet added to database', { did: blocklet.meta.did });
+
+  // create config
+  try {
+    await states.blockletExtras.addMeta({ did, meta: { did, name }, controller });
+    await manager._setConfigsFromMeta(did);
+    await states.blockletExtras.setConfigs(did, [
+      {
+        key: BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK,
+        value: appSk,
+        secure: true,
+      },
+    ]);
+  } catch (err) {
+    await manager._rollback('install', did, {});
+  }
+
+  if (!blocklet.children.length) {
+    return manager._installBlocklet({ did, context });
+  }
+
+  try {
+    const blocklet1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.waiting);
+    manager.emit(BlockletEvents.added, blocklet1);
+
+    const action = 'install';
+    const downloadParams = {
+      blocklet: { ...blocklet1 },
+      context,
+      postAction: action,
+    };
+
+    // backup rollback data
+    await manager._rollbackCache.backup({ did, action });
+
+    if (sync) {
+      await manager._downloadAndInstall({ ...downloadParams, throwOnError: true });
+      return states.blocklet.getBlocklet(did);
+    }
+
+    setTimeout(() => {
+      const ticket = manager.installQueue.push(
+        {
+          entity: 'blocklet',
+          action: 'download',
+          id: did,
+          ...downloadParams,
+        },
+        did
+      );
+      ticket.on('failed', async (err) => {
+        const componentDid = component?.meta?.did;
+        logger.error('failed to install blocklet', { did, componentDid, error: err });
+        try {
+          await manager._rollback('install', did, {});
+        } catch (e) {
+          logger.error('failed to remove blocklet on install error', { did, componentDid, error: e });
+        }
+
+        manager._createNotification(did, {
+          title: 'Blocklet Install Failed',
+          description: `Blocklet ${title || component?.meta?.title} install failed with error: ${
+            err.message || 'queue exception'
+          }`,
+          entityType: 'blocklet',
+          entityId: did,
+          severity: 'error',
+        });
+      });
+    }, delay || 0);
+
+    return blocklet1;
+  } catch (err) {
+    const componentDid = component?.meta?.did;
+    logger.error('failed to install blocklet', { did, componentDid, error: err });
+
+    try {
+      await manager._rollback('install', did, {});
+    } catch (e) {
+      logger.error('failed to remove blocklet on install error', { did, componentDid, error: e });
+    }
+
+    throw err;
+  }
+};
+
+module.exports = { installApplicationFromGeneral };

package/lib/blocklet/manager/helper/install-component-from-dev.js

@@ -0,0 +1,84 @@
+const logger = require('@abtnode/logger')('@abtnode/core:install-component-dev');
+
+const { BlockletStatus, BlockletSource, BLOCKLET_MODES, fromBlockletStatus } = require('@blocklet/constant');
+const {
+  parseComponents,
+  filterDuplicateComponents,
+  ensureMeta,
+  checkVersionCompatibility,
+  validateBlocklet,
+} = require('../../../util/blocklet');
+const { formatName } = require('../../../util/get-domain-for-blocklet');
+
+const installComponentFromDev = async ({ folder, meta, rootDid, mountPoint, manager, states }) => {
+  const { did, version } = meta;
+
+  const existRoot = await states.blocklet.getBlocklet(rootDid);
+  if (!existRoot) {
+    throw new Error('Root blocklet does not exist');
+  }
+
+  const exist = existRoot.children.find((x) => x.meta.did === meta.did);
+  if (exist) {
+    if (exist.mode === BLOCKLET_MODES.PRODUCTION) {
+      throw new Error('The blocklet component of production mode already exists, please remove it before developing');
+    }
+
+    const status = fromBlockletStatus(exist.status);
+    if (['starting', 'running'].includes(status)) {
+      throw new Error(`The blocklet component is already on ${status}, please stop it before developing`);
+    }
+
+    logger.info('remove blocklet component for dev', { did, version });
+
+    await manager.deleteComponent({ did, rootDid });
+  }
+
+  let defaultPath = formatName(meta.name);
+  if (!existRoot.children.map((x) => x.mountPoint).includes('/')) {
+    defaultPath = '';
+  }
+
+  const component = {
+    meta: ensureMeta(meta),
+    mountPoint: mountPoint || `/${defaultPath}`,
+    source: BlockletSource.local,
+    deployedFrom: folder,
+    status: BlockletStatus.installed,
+    mode: BLOCKLET_MODES.DEVELOPMENT,
+  };
+  const { dynamicComponents } = await parseComponents(component);
+  dynamicComponents.unshift(component);
+  const children = filterDuplicateComponents(dynamicComponents, existRoot.children);
+  checkVersionCompatibility([...children, ...existRoot.children]);
+  await states.blocklet.addChildren(rootDid, children);
+
+  const newBlocklet = await states.blocklet.getBlocklet(rootDid);
+  await validateBlocklet(newBlocklet);
+  await manager._downloadBlocklet(newBlocklet);
+  await states.blocklet.setBlockletStatus(rootDid, BlockletStatus.stopped);
+
+  // Add Config
+  await manager._setConfigsFromMeta(rootDid);
+
+  // should ensure blocklet integrity
+  let blocklet = await manager.ensureBlocklet(rootDid);
+
+  // pre install
+  await manager._runPreInstallHook(blocklet);
+
+  // Add environments
+  await manager._updateBlockletEnvironment(rootDid);
+  blocklet = await manager.getBlocklet(rootDid);
+
+  // post install
+  await manager._runPostInstallHook(blocklet);
+
+  logger.info('add blocklet component for dev', { did, version, meta });
+
+  blocklet = await manager.getBlocklet(rootDid);
+
+  return blocklet;
+};
+
+module.exports = { installComponentFromDev };