@abtnode/core 1.8.69-beta-b0bb2d67 → 1.8.69-beta-76f8a46f

package/lib/blocklet/manager/helper/{install-from-backup.js → install-application-from-backup.js}

@@ -7,19 +7,23 @@ const getBlockletInfo = require('@blocklet/meta/lib/info');
 
 const { BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
 
-const logger = require('@abtnode/logger')('@abtnode/core:install-from-backup');
+const logger = require('@abtnode/logger')('@abtnode/core:install-app-backup');
 
 const { validateBlocklet, checkDuplicateAppSk, getAppDirs } = require('../../../util/blocklet');
 
 /**
- *
- * @param {{
- *  manager: import('../disk'),
- *  states: import('../../../states/index')
- * }} param0
- * @returns
+ * backup 目录结构
+ *  /blocklets/<name1>version>
+ *  /blocklets/<name2>version>
+ *  /blocklets/<name3>version>
+ *  /data
+ *  /blocklet.json
+ *  /blocklet_extras.json
+ * @see Blocklet 端到端备份方案的设计与实现（一期） https://team.arcblock.io/comment/discussions/e62084d5-fafa-489e-91d5-defcd6e93223
+ * @param {{ url: string, appSk: string, moveDir: boolean, manager: import('../disk'), states: import('../../../states/index'), context: Record<string, string> }}
+ * @return {Promise<any>}
  */
-module.exports = async ({ url, appSk, moveDir, context = {}, states, manager } = {}) => {
+const installApplicationFromBackup = async ({ url, appSk, moveDir, context = {}, states, manager } = {}) => {
   // TODO: support more url schema feature (http, did-spaces)
   if (!url.startsWith('file://')) {
     throw new Error('url must starts with file://');
@@ -45,10 +49,19 @@ module.exports = async ({ url, appSk, moveDir, context = {}, states, manager } =
     'startedAt',
   ]);
 
+  if (!state.structVersion) {
+    throw new Error(
+      'Only application of structVersion 2 can be restored on this server. Please migrate you application and re-backup your application on your previous server.'
+    );
+  }
+
+  const { meta } = state;
+  const { did, name: appName } = meta;
+
   const extra = omit(fs.readJSONSync(path.join(dir, 'blocklet-extras.json')), ['_id', 'createdAt', 'updatedAt']);
 
   if (state.meta.did !== extra.did) {
-    throw new Error('did does not match in blocklet.json and blocklet_extra.json');
+    throw new Error(`did does not match in blocklet.json (${state.meta.did}) and blocklet_extra.json ${extra.did}`);
   }
 
   forEachBlockletSync(state, (component) => {
@@ -57,18 +70,13 @@ module.exports = async ({ url, appSk, moveDir, context = {}, states, manager } =
     delete component.environments;
   });
 
-  const { meta } = state;
-
   await validateBlocklet({ meta });
 
-  const { did, name: appName } = meta;
-
-  // FIXME: meta.did and meta.name should be dynamic created when installing multiple blocklet is supported
   const existState = await states.blocklet.hasBlocklet(did);
 
   if (existState) {
     logger.error('blocklet is already exist', { did });
-    throw new Error('blocklet is already exist');
+    throw new Error(`blocklet is already exist. did: ${did}`);
   }
 
   if (appSk) {
@@ -106,8 +114,6 @@ module.exports = async ({ url, appSk, moveDir, context = {}, states, manager } =
     const existExtra = await states.blockletExtras.find({ did });
     if (existExtra) {
       // 如果数据存在, 当前视为脏数据, 直接删除
-      // FIXME 另一个人的数据可能被删除. 修复方式: 动态生成 meta.did 或通过 blocklet sk 生成 meta.did
-      // FIXME 简单粗暴的删掉数据可能不是理想的方式，需要考虑旧数据存在时, 如何与新数据 merge
       logger.error('old extra state exists and will be force removed', { existExtra });
       await states.blockletExtras.remove({ did });
     }
@@ -142,7 +148,6 @@ module.exports = async ({ url, appSk, moveDir, context = {}, states, manager } =
     // 从 store 下载 blocklet
     await manager.blockletDownloader.download(state, { skipCheckIntegrity: true });
 
-    // FIXME same as copy extra
     const dataDir = path.join(manager.dataDirs.data, appName);
     if (fs.existsSync(dataDir)) {
       logger.error('old data exists and will be force removed', { dataDir });
@@ -158,7 +163,7 @@ module.exports = async ({ url, appSk, moveDir, context = {}, states, manager } =
       logger.info(`data is ${moveDir ? 'moved' : 'copied'} successfully`);
     }
   } catch (error) {
-    logger.error('installFromBackup failed', { error });
+    logger.error('installFromBackup failed', { did, error });
 
     await manager._rollback('install', did);
 
@@ -168,3 +173,5 @@ module.exports = async ({ url, appSk, moveDir, context = {}, states, manager } =
   logger.info('start install blocklet', { did });
   return manager._installBlocklet({ did, context });
 };
+
+module.exports = { installApplicationFromBackup };

package/lib/blocklet/manager/helper/install-application-from-dev.js

@@ -0,0 +1,94 @@
+const path = require('path');
+const fs = require('fs-extra');
+
+const { toSvg: createDidLogo } =
+  process.env.NODE_ENV !== 'test' ? require('@arcblock/did-motif') : require('@arcblock/did-motif/dist/did-motif.cjs');
+const { BlockletStatus, BLOCKLET_MODES, fromBlockletStatus, BlockletSource } = require('@blocklet/constant');
+
+const logger = require('@abtnode/logger')('@abtnode/core:install-app-dev');
+const { ensureMeta } = require('../../../util/blocklet');
+
+/**
+ *
+ * @param {{
+ *  manager: import('../disk'),
+ *  states: import('../../../states/index')
+ * }}
+ * @returns
+ */
+const installApplicationFromDev = async ({ folder, meta, states, manager } = {}) => {
+  const { did, version } = meta;
+
+  const exist = await states.blocklet.getBlocklet(did);
+  if (exist) {
+    if (exist.mode === BLOCKLET_MODES.PRODUCTION) {
+      throw new Error('The blocklet of production mode already exists, please remove it before developing');
+    }
+
+    const status = fromBlockletStatus(exist.status);
+    if (['starting', 'running'].includes(status)) {
+      throw new Error(`The blocklet is already on ${status}, please stop it before developing`);
+    }
+
+    logger.info('remove blocklet for dev', { did, version });
+
+    await manager.delete({ did, keepLogsDir: false });
+  }
+
+  // delete process
+  try {
+    await manager.deleteProcess({ did });
+    logger.info('delete blocklet precess for dev', { did, version });
+  } catch (err) {
+    if (process.env.NODE_ENV !== 'development') {
+      logger.error('failed to delete blocklet process for dev', { error: err });
+    }
+  }
+
+  // create component
+  const component = {
+    meta: ensureMeta(meta),
+    mountPoint: '/',
+    source: BlockletSource.local,
+    deployedFrom: folder,
+    status: BlockletStatus.installed,
+    mode: BLOCKLET_MODES.DEVELOPMENT,
+  };
+
+  // create app
+  const added = await manager._addBlocklet({
+    component,
+    name: meta.name,
+    did: meta.did,
+    mode: BLOCKLET_MODES.DEVELOPMENT,
+  });
+  logger.info('add blocklet for dev', { did, version, meta });
+
+  await manager._downloadBlocklet(added);
+
+  // Add Config
+  await manager._setConfigsFromMeta(did);
+
+  // should ensure blocklet integrity
+  let blocklet = await manager.ensureBlocklet(did);
+
+  // pre install
+  await manager._runPreInstallHook(blocklet);
+
+  // Add environments
+  await manager._updateBlockletEnvironment(did);
+  blocklet = await manager.getBlocklet(did);
+
+  // post install
+  await manager._runPostInstallHook(blocklet);
+
+  await states.blocklet.setBlockletStatus(did, BlockletStatus.installed);
+
+  blocklet = await manager.getBlocklet(did);
+
+  await fs.writeFile(path.join(blocklet.env.dataDir, 'logo.svg'), createDidLogo(blocklet.meta.did));
+
+  return blocklet;
+};
+
+module.exports = { installApplicationFromDev };

package/lib/blocklet/manager/helper/install-application-from-general.js

@@ -0,0 +1,188 @@
+const joi = require('joi');
+
+const { BLOCKLET_INSTALL_TYPE, NODE_MODES } = require('@abtnode/constant');
+const { BlockletStatus, BlockletEvents, BLOCKLET_CONFIGURABLE_KEY } = require('@blocklet/constant');
+
+const logger = require('@abtnode/logger')('@abtnode/core:install-app-general');
+const getApplicationWallet = require('@blocklet/meta/lib/wallet');
+
+const StoreUtil = require('../../../util/store');
+const { getBlockletMetaFromUrl, ensureMeta, validateStore, validateInServerless } = require('../../../util/blocklet');
+
+/**
+ *
+ * @param {{
+ *  manager: import('../disk'),
+ *  states: import('../../../states/index')
+ * }} param0
+ * @returns
+ */
+const installApplicationFromGeneral = async ({
+  type,
+  appSk,
+  sync,
+  delay,
+  controller,
+  title,
+  description,
+  did: bundleDid,
+  storeUrl,
+  url: inputUrl,
+  context = {},
+  states,
+  manager,
+} = {}) => {
+  const nodeInfo = await states.node.read();
+
+  let componentSourceUrl;
+  if (type === BLOCKLET_INSTALL_TYPE.STORE) {
+    if (!storeUrl) {
+      throw new Error('store url should not be empty');
+    }
+    componentSourceUrl = StoreUtil.getBlockletMetaUrl({ did: bundleDid, storeUrl });
+    await validateStore(nodeInfo, componentSourceUrl);
+  } else if (type === BLOCKLET_INSTALL_TYPE.URL) {
+    componentSourceUrl = inputUrl;
+    await validateStore(nodeInfo, componentSourceUrl);
+  } else if (type === BLOCKLET_INSTALL_TYPE.CREATE) {
+    await joi.string().label('title').max(20).required().validateAsync(title);
+    await joi.string().label('description').max(80).required().validateAsync(description);
+  } else {
+    throw new Error(`Should not be here: unknown type ${type}`);
+  }
+
+  // check install count
+  if (controller?.nftId) {
+    // sometimes nedb will throw error if use states.blocklet.count({ ['controller.nftId']: controller.nftId })
+    const installedCount = await states.blockletExtras.count({ 'controller.nftId': controller.nftId });
+
+    if (installedCount >= (controller.appMaxCount || 1)) {
+      throw new Error(
+        `You can only install ${controller.appMaxCount} blocklet with this credential: ${controller.nftId}`
+      );
+    }
+  }
+
+  // app wallet
+  const wallet = getApplicationWallet(appSk);
+  const did = wallet.address;
+  const name = wallet.address;
+
+  // check exist
+  const exist = await states.blocklet.hasBlocklet(did);
+  if (exist) {
+    throw new Error(`blocklet ${did} already exists`);
+  }
+
+  // remove dirty data
+  const oldExtraState = await states.blockletExtras.findOne({ did });
+  if (oldExtraState) {
+    logger.error('find dirty data in blocklet extra', { did });
+    await states.blockletExtras.remove({ did });
+  }
+
+  // create component
+  let component;
+  if (componentSourceUrl) {
+    const meta = await getBlockletMetaFromUrl(componentSourceUrl);
+
+    if (nodeInfo.mode === NODE_MODES.SERVERLESS) {
+      validateInServerless({ blockletMeta: meta });
+    }
+
+    component = {
+      meta: ensureMeta(meta),
+      mountPoint: '/',
+      bundleSource: { url: componentSourceUrl },
+    };
+  }
+
+  // create app
+  const blocklet = await manager._addBlocklet({ component, name, did, title, description });
+  logger.info('blocklet added to database', { did: blocklet.meta.did });
+
+  // create config
+  try {
+    await states.blockletExtras.addMeta({ did, meta: { did, name }, controller });
+    await manager._setConfigsFromMeta(did);
+    await states.blockletExtras.setConfigs(did, [
+      {
+        key: BLOCKLET_CONFIGURABLE_KEY.BLOCKLET_APP_SK,
+        value: appSk,
+        secure: true,
+      },
+    ]);
+  } catch (err) {
+    await manager._rollback('install', did, {});
+  }
+
+  if (!blocklet.children.length) {
+    return manager._installBlocklet({ did, context });
+  }
+
+  try {
+    const blocklet1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.waiting);
+    manager.emit(BlockletEvents.added, blocklet1);
+
+    const action = 'install';
+    const downloadParams = {
+      blocklet: { ...blocklet1 },
+      context,
+      postAction: action,
+    };
+
+    // backup rollback data
+    await manager._rollbackCache.backup({ did, action });
+
+    if (sync) {
+      await manager._downloadAndInstall({ ...downloadParams, throwOnError: true });
+      return states.blocklet.getBlocklet(did);
+    }
+
+    setTimeout(() => {
+      const ticket = manager.installQueue.push(
+        {
+          entity: 'blocklet',
+          action: 'download',
+          id: did,
+          ...downloadParams,
+        },
+        did
+      );
+      ticket.on('failed', async (err) => {
+        const componentDid = component?.meta?.did;
+        logger.error('failed to install blocklet', { did, componentDid, error: err });
+        try {
+          await manager._rollback('install', did, {});
+        } catch (e) {
+          logger.error('failed to remove blocklet on install error', { did, componentDid, error: e });
+        }
+
+        manager._createNotification(did, {
+          title: 'Blocklet Install Failed',
+          description: `Blocklet ${title || component?.meta?.title} install failed with error: ${
+            err.message || 'queue exception'
+          }`,
+          entityType: 'blocklet',
+          entityId: did,
+          severity: 'error',
+        });
+      });
+    }, delay || 0);
+
+    return blocklet1;
+  } catch (err) {
+    const componentDid = component?.meta?.did;
+    logger.error('failed to install blocklet', { did, componentDid, error: err });
+
+    try {
+      await manager._rollback('install', did, {});
+    } catch (e) {
+      logger.error('failed to remove blocklet on install error', { did, componentDid, error: e });
+    }
+
+    throw err;
+  }
+};
+
+module.exports = { installApplicationFromGeneral };

package/lib/blocklet/manager/helper/install-component-from-dev.js

@@ -0,0 +1,80 @@
+const logger = require('@abtnode/logger')('@abtnode/core:install-component-dev');
+
+const { BlockletStatus, BlockletSource, BLOCKLET_MODES, fromBlockletStatus } = require('@blocklet/constant');
+const {
+  parseComponents,
+  filterDuplicateComponents,
+  ensureMeta,
+  checkVersionCompatibility,
+  validateBlocklet,
+} = require('../../../util/blocklet');
+const { formatName } = require('../../../util/get-domain-for-blocklet');
+
+const installComponentFromDev = async ({ folder, meta, rootDid, mountPoint, manager, states }) => {
+  const { did, version } = meta;
+
+  const existRoot = await states.blocklet.getBlocklet(rootDid);
+  if (!existRoot) {
+    throw new Error('Root blocklet does not exist');
+  }
+
+  const exist = existRoot.children.find((x) => x.meta.did === meta.did);
+  if (exist) {
+    if (exist.mode === BLOCKLET_MODES.PRODUCTION) {
+      throw new Error('The blocklet component of production mode already exists, please remove it before developing');
+    }
+
+    const status = fromBlockletStatus(exist.status);
+    if (['starting', 'running'].includes(status)) {
+      throw new Error(`The blocklet component is already on ${status}, please stop it before developing`);
+    }
+
+    logger.info('remove blocklet component for dev', { did, version });
+
+    await manager.deleteComponent({ did, rootDid });
+  }
+
+  const defaultPath = formatName(meta.name);
+  const component = {
+    meta: ensureMeta(meta),
+    mountPoint: mountPoint || `/${defaultPath}`,
+    source: BlockletSource.local,
+    deployedFrom: folder,
+    status: BlockletStatus.installed,
+    mode: BLOCKLET_MODES.DEVELOPMENT,
+  };
+  const { dynamicComponents } = await parseComponents(component);
+  dynamicComponents.unshift(component);
+  const children = filterDuplicateComponents(dynamicComponents, existRoot.children);
+  checkVersionCompatibility([...children, ...existRoot.children]);
+  await states.blocklet.addChildren(rootDid, children);
+
+  const newBlocklet = await states.blocklet.getBlocklet(rootDid);
+  await validateBlocklet(newBlocklet);
+  await manager._downloadBlocklet(newBlocklet);
+  await states.blocklet.setBlockletStatus(rootDid, BlockletStatus.stopped);
+
+  // Add Config
+  await manager._setConfigsFromMeta(rootDid);
+
+  // should ensure blocklet integrity
+  let blocklet = await manager.ensureBlocklet(rootDid);
+
+  // pre install
+  await manager._runPreInstallHook(blocklet);
+
+  // Add environments
+  await manager._updateBlockletEnvironment(rootDid);
+  blocklet = await manager.getBlocklet(rootDid);
+
+  // post install
+  await manager._runPostInstallHook(blocklet);
+
+  logger.info('add blocklet component for dev', { did, version, meta });
+
+  blocklet = await manager.getBlocklet(rootDid);
+
+  return blocklet;
+};
+
+module.exports = { installComponentFromDev };

package/lib/blocklet/manager/helper/install-component-from-upload.js

@@ -0,0 +1,181 @@
+const path = require('path');
+
+const logger = require('@abtnode/logger')('@abtnode/core:install-component-upload');
+const getComponentProcessId = require('@blocklet/meta/lib/get-component-process-id');
+
+const { BlockletSource, BlockletGroup, fromBlockletStatus } = require('@blocklet/constant');
+const { isInProgress } = require('../../../util');
+const {
+  parseComponents,
+  filterDuplicateComponents,
+  getDiffFiles,
+  getBundleDir,
+  ensureMeta,
+  checkStructVersion,
+  checkVersionCompatibility,
+} = require('../../../util/blocklet');
+const { resolveDownload, resolveDiffDownload, downloadFromUpload } = require('../../downloader/resolve-download');
+
+const installComponentFromUpload = async ({
+  rootDid,
+  mountPoint,
+  file,
+  did,
+  diffVersion,
+  deleteSet,
+  context = {},
+  manager,
+  states,
+}) => {
+  logger.info('install component', { from: 'upload file' });
+
+  const oldBlocklet = await manager._getBlockletForInstallation(rootDid);
+  if (!oldBlocklet) {
+    throw new Error('Root blocklet does not exist');
+  }
+
+  checkStructVersion(oldBlocklet);
+
+  // download
+  const { tarFile } = await downloadFromUpload(file, { downloadDir: path.join(manager.dataDirs.tmp, 'download') });
+
+  if (isInProgress(oldBlocklet.status)) {
+    logger.error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
+    throw new Error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
+  }
+
+  let meta;
+  // diff upload
+  if (did && diffVersion) {
+    const oldChild = oldBlocklet.children.find((x) => x.meta.did === did);
+    if (!oldChild) {
+      throw new Error(`Blocklet ${did} not found when diff deploying`);
+    }
+    if (oldChild.meta.version !== diffVersion) {
+      logger.error('Diff deploy: Blocklet version changed', {
+        preVersion: diffVersion,
+        changedVersion: oldChild.meta.version,
+        name: oldChild.meta.name,
+        did: oldChild.meta.did,
+      });
+      throw new Error('Blocklet version changed when diff deploying');
+    }
+
+    meta = (await resolveDiffDownload(tarFile, manager.installDir, { deleteSet, meta: oldChild.meta })).meta;
+  } else {
+    // full deploy
+    meta = (await resolveDownload(tarFile, manager.installDir)).meta;
+  }
+
+  if (meta.did === rootDid) {
+    // should not be here
+    throw new Error('Cannot add self as a component');
+  }
+
+  if (meta.group === BlockletGroup.gateway) {
+    throw new Error('Cannot add gateway component');
+  }
+
+  const newBlocklet = await states.blocklet.getBlocklet(rootDid);
+
+  const newChild = {
+    meta: ensureMeta(meta),
+    mountPoint,
+    source: BlockletSource.upload,
+    deployedFrom: `Upload by ${context.user.fullName}`,
+    bundleSource: null,
+  };
+  const index = newBlocklet.children.findIndex((child) => child.meta.did === meta.did);
+  if (index >= 0) {
+    // if upgrade, do not update mountPoint and title
+    newChild.mountPoint = newBlocklet.children[index].mountPoint;
+    newChild.meta.title = newBlocklet.children[index].meta.title;
+    newBlocklet.children.splice(index, 1, newChild);
+  } else {
+    newBlocklet.children.push(newChild);
+  }
+
+  const { dynamicComponents } = await parseComponents(newChild);
+  const newChildren = filterDuplicateComponents(dynamicComponents, newBlocklet.children);
+
+  newBlocklet.children.push(...newChildren);
+
+  checkVersionCompatibility(newBlocklet.children);
+
+  // backup rollback data
+  const action = 'upgrade';
+  await manager._rollbackCache.backup({ did: newBlocklet.meta.did, action, oldBlocklet });
+
+  await manager._downloadAndInstall({
+    blocklet: newBlocklet,
+    oldBlocklet,
+    context: { ...context, forceStartProcessIds: [getComponentProcessId(newChild, [newBlocklet])] },
+    throwOnError: true,
+    postAction: action,
+    skipCheckStatusBeforeDownload: true,
+    selectedComponentDids: newChildren.map((x) => x.meta.did),
+  });
+  return manager.getBlocklet(newBlocklet.meta.did);
+};
+
+const diff = async ({ did, hashFiles: clientFiles, rootDid: inputRootDid, states }) => {
+  if (!did) {
+    throw new Error('did is empty');
+  }
+
+  if (!clientFiles || !clientFiles.length) {
+    throw new Error('hashFiles is empty');
+  }
+
+  const rootDid = inputRootDid || did;
+  const childDid = inputRootDid ? did : '';
+
+  if (childDid === rootDid) {
+    throw new Error('Cannot add self as a component');
+  }
+
+  logger.info('Get blocklet diff', { rootDid, childDid, clientFilesNumber: clientFiles.length });
+
+  const rootBlocklet = await states.blocklet.getBlocklet(rootDid);
+  if (childDid && !rootBlocklet) {
+    throw new Error('Root blocklet does not exist');
+  }
+
+  const state = childDid ? await (rootBlocklet.children || []).find((x) => x.meta.did === childDid) : rootBlocklet;
+
+  if (!state) {
+    return {
+      hasBlocklet: false,
+    };
+  }
+
+  if (state.source === BlockletSource.local) {
+    throw new Error(`Blocklet ${state.meta.name} is already deployed from local, can not deployed from remote.`);
+  }
+
+  const { version } = state.meta;
+  const bundleDir = getBundleDir(this.installDir, state.meta);
+  const { addSet, changeSet, deleteSet } = await getDiffFiles(clientFiles, bundleDir);
+
+  logger.info('Diff files', {
+    name: state.meta.name,
+    did: state.meta.did,
+    version: state.meta.version,
+    addNum: addSet.length,
+    changeNum: changeSet.length,
+    deleteNum: deleteSet.length,
+  });
+
+  return {
+    hasBlocklet: true,
+    version,
+    addSet,
+    changeSet,
+    deleteSet,
+  };
+};
+
+module.exports = {
+  installComponentFromUpload,
+  diff,
+};