@abtnode/core 1.15.17 → 1.16.0-beta-b16cb035

package/lib/migrations/1.6.4-security.js

@@ -0,0 +1,59 @@
+/* eslint-disable no-await-in-loop */
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const yaml = require('js-yaml');
+const cloneDeep = require('lodash/cloneDeep');
+const security = require('@abtnode/util/lib/security');
+
+module.exports = async ({ states, configFile, dataDir }) => {
+  if (process.env.CI || process.env.NODE_ENV === 'test') {
+    return;
+  }
+
+  const file = path.join(dataDir, '.sock');
+  if (fs.existsSync(file)) {
+    return;
+  }
+
+  try {
+    fs.writeFileSync(file, crypto.randomBytes(32), { encoding: 'binary', mode: '0600' });
+
+    const config = yaml.load(fs.readFileSync(configFile).toString(), { json: true });
+    config.node.sk = security.encrypt(config.node.sk, config.node.did, fs.readFileSync(file));
+    fs.writeFileSync(configFile, yaml.dump(config));
+    await states.node.updateNodeInfo({ sk: config.node.sk });
+
+    const items = await states.blockletExtras.find();
+    for (const item of items) {
+      const newConfigs = cloneDeep(item.configs || []).map((c) => {
+        if (c.secure) {
+          c.value = security.encrypt(c.value, item.did, fs.readFileSync(file));
+        }
+
+        return c;
+      });
+
+      const newChildren = cloneDeep(item.children || []).map((x) => {
+        return {
+          ...x,
+          configs: cloneDeep(x.configs || []).map((c) => {
+            if (c.secure) {
+              c.value = security.encrypt(c.value, item.did, fs.readFileSync(file));
+            }
+
+            return c;
+          }),
+        };
+      });
+
+      await states.blockletExtras.update({ did: item.did }, { $set: { configs: newConfigs, children: newChildren } });
+    }
+
+    states.node.compactDatafile();
+    states.blockletExtras.compactDatafile();
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
+};

package/lib/migrations/1.6.5-security.js

@@ -0,0 +1,60 @@
+/* eslint-disable no-await-in-loop */
+const fs = require('fs');
+const path = require('path');
+const uniq = require('lodash/uniq');
+const cloneDeep = require('lodash/cloneDeep');
+const security = require('@abtnode/util/lib/security');
+const { forEachBlocklet } = require('@blocklet/meta/lib/util');
+
+module.exports = async ({ states, dataDir }) => {
+  if (process.env.CI || process.env.NODE_ENV === 'test') {
+    return;
+  }
+
+  const file = path.join(dataDir, '.sock');
+  if (fs.existsSync(file) === false) {
+    return;
+  }
+
+  try {
+    const extras = await states.blockletExtras.find();
+    const blocklets = await states.blocklet.find();
+    for (const blocklet of blocklets) {
+      forEachBlocklet(
+        blocklet,
+        (b) => {
+          let configKeys;
+          const extra = extras.find((x) => x.did === blocklet.meta.did);
+          const rootKeys = (extra.configs || []).map((x) => x.key);
+          if (b.meta.did === blocklet.meta.did) {
+            configKeys = rootKeys;
+          } else {
+            const childExtra = (extra.children || []).find((x) => x.did === b.meta.did);
+            const childKeys = childExtra ? childExtra.configs.map((x) => x.key) : [];
+            configKeys = uniq([...rootKeys, ...childKeys]);
+          }
+
+          b.environments = cloneDeep(b.environments || [])
+            .filter((x) => configKeys.includes(x.key) === false)
+            .map((c) => {
+              if (c.key === 'BLOCKLET_APP_SK') {
+                c.value = security.encrypt(c.value, b.meta.did, fs.readFileSync(file));
+              }
+
+              return c;
+            });
+        },
+        { sync: true }
+      );
+
+      await states.blocklet.updateById(blocklet._id, {
+        $set: { environments: blocklet.environments, children: blocklet.children, migratedFrom: '1.6.5' },
+      });
+    }
+
+    states.blocklet.compactDatafile();
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
+};

package/lib/migrations/1.6.9-update-node-info-and-certificate.js

@@ -0,0 +1,38 @@
+/* eslint-disable no-await-in-loop */
+
+const { DEFAULT_DID_REGISTRY, DEFAULT_DID_DOMAIN } = require('@abtnode/constant');
+const { isEmpty, omit } = require('lodash');
+
+const updateNodeInfo = async ({ printInfo, states }) => {
+  // add didRegistry and didDomain
+  const info = await states.node.read();
+  const data = {};
+  if (!info.didRegistry) {
+    data.didRegistry = DEFAULT_DID_REGISTRY;
+  }
+
+  if (!info.didDomain) {
+    data.didDomain = DEFAULT_DID_DOMAIN;
+  }
+
+  if (!isEmpty(data)) {
+    await states.node.update({ _id: info._id }, { $set: data });
+  }
+
+  printInfo('update did registry and did domain successfully');
+};
+
+const updateCertificate = async ({ node, printInfo }) => {
+  const certs = await node.certManager.getAllNormal();
+  for (const cert of certs || []) {
+    await node.certManager.updateWithoutValidations(cert.id, omit(cert, 'id'));
+  }
+
+  printInfo('update certificate successfully');
+};
+
+/* eslint-disable no-underscore-dangle */
+module.exports = async ({ node, printInfo, states }) => {
+  await updateNodeInfo({ printInfo, states });
+  await updateCertificate({ node, printInfo, states });
+};

package/lib/migrations/1.7.1-blocklet-setup.js

@@ -0,0 +1,18 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-underscore-dangle */
+
+module.exports = async ({ states, printInfo }) => {
+  printInfo('Try to update blocklet server to 1.7.1...');
+
+  const blockletExtras = await states.blockletExtras.find({});
+  for (const extra of blockletExtras) {
+    if (!extra) {
+      continue;
+    }
+
+    await states.blockletExtras.setSettings(extra.did, { initialized: true });
+
+    printInfo(`Set initialized: ${extra.did}`);
+  }
+};

package/lib/migrations/1.7.12-blocklet-meta.js

@@ -0,0 +1,51 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-underscore-dangle */
+
+module.exports = async ({ states, printInfo }) => {
+  printInfo('Try to update blocklet server to 1.7.12...');
+
+  const blocklets = await states.blocklet.getBlocklets();
+
+  for (const blocklet of blocklets) {
+    let changed = false;
+
+    if (!blocklet.meta.bundleDid) {
+      blocklet.meta.bundleDid = blocklet.meta.did;
+      blocklet.meta.bundleName = blocklet.meta.name;
+      changed = true;
+    }
+
+    (blocklet.children || []).forEach((child) => {
+      if (!child.meta.bundleDid) {
+        child.meta.bundleDid = child.meta.did;
+        child.meta.bundleName = child.meta.name;
+        changed = true;
+      }
+    });
+
+    if (changed) {
+      await states.blocklet.updateBlocklet(blocklet.meta.did, { meta: blocklet.meta, children: blocklet.children });
+      printInfo(`Blocklet meta in blocklet.db updated: ${blocklet.meta.did}`);
+    }
+  }
+
+  const blockletExtras = await states.blockletExtras.find({});
+
+  for (const extra of blockletExtras) {
+    let changed = false;
+    const children = await states.blockletExtras.getSettings(extra.did, 'children', []);
+    (children || []).forEach((child) => {
+      if (!child.meta.bundleDid) {
+        child.meta.bundleDid = child.meta.did;
+        child.meta.bundleName = child.meta.name;
+        changed = true;
+      }
+    });
+
+    if (changed) {
+      await states.blockletExtras.setSettings(extra.did, { children });
+      printInfo(`Blocklet dynamic component meta in blocklet_extra.db updated: ${extra.did}`);
+    }
+  }
+};

package/lib/migrations/1.7.15-blocklet-bundle-source.js

@@ -0,0 +1,42 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-underscore-dangle */
+
+module.exports = async ({ states, printInfo }) => {
+  printInfo('Try to update blocklet server to 1.7.15...');
+
+  const blocklets = await states.blocklet.getBlocklets();
+
+  for (const blocklet of blocklets) {
+    (blocklet.children || []).forEach((child) => {
+      const { sourceUrl } = child;
+      delete child.sourceUrl;
+      if (sourceUrl && !child.bundleSource) {
+        child.bundleSource = {
+          url: sourceUrl,
+        };
+      }
+    });
+
+    await states.blocklet.updateBlocklet(blocklet.meta.did, { children: blocklet.children });
+    printInfo(`Blocklet children in blocklet.db updated: ${blocklet.meta.did}`);
+  }
+
+  const blockletExtras = await states.blockletExtras.find({});
+
+  for (const extra of blockletExtras) {
+    const children = await states.blockletExtras.getSettings(extra.did, 'children', []);
+    (children || []).forEach((child) => {
+      const { sourceUrl } = child;
+      delete child.sourceUrl;
+      if (sourceUrl && !child.bundleSource) {
+        child.bundleSource = {
+          url: sourceUrl,
+        };
+      }
+    });
+
+    await states.blockletExtras.setSettings(extra.did, { children });
+    printInfo(`Blocklet dynamic component in blocklet_extra.db updated: ${extra.did}`);
+  }
+};

package/lib/migrations/1.7.20-blocklet-component.js

@@ -0,0 +1,41 @@
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-continue */
+
+module.exports = async ({ states, printInfo }) => {
+  printInfo('Try to delete realDid, realInterface and add componentId in db...');
+
+  const sites = await states.site.find({});
+
+  for (const site of sites) {
+    let changed = false;
+    for (const rule of site.rules || []) {
+      if (!rule.to) {
+        continue;
+      }
+
+      if (rule.to.type !== 'blocklet') {
+        continue;
+      }
+
+      if (rule.to.componentId) {
+        continue;
+      }
+
+      if (!rule.to.realDid || rule.to.did === rule.to.realDid) {
+        rule.to.componentId = rule.to.did;
+      } else {
+        rule.to.componentId = `${rule.to.did}/${rule.to.realDid}`;
+      }
+
+      delete rule.to.realDid;
+      delete rule.to.realInterfaceName;
+
+      changed = true;
+    }
+
+    if (changed) {
+      await states.site.update({ _id: site._id }, site);
+      printInfo(`site ${site.domain} has been updated`);
+    }
+  }
+};

package/lib/migrations/1.8.33-blocklet-mem-limit.js

@@ -0,0 +1,20 @@
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-continue */
+
+const { BLOCKLET_MAX_MEM_LIMIT_IN_MB } = require('@abtnode/constant');
+
+module.exports = async ({ states, printInfo }) => {
+  printInfo('Try to update blockletMaxMemoryLimit...');
+
+  const info = await states.node.read();
+  if (
+    info?.runtimeConfig?.blockletMaxMemoryLimit &&
+    info.runtimeConfig.blockletMaxMemoryLimit < BLOCKLET_MAX_MEM_LIMIT_IN_MB
+  ) {
+    info.runtimeConfig.blockletMaxMemoryLimit = BLOCKLET_MAX_MEM_LIMIT_IN_MB;
+    await states.node.updateNodeInfo(info);
+    printInfo(`Update blockletMaxMemoryLimit to ${BLOCKLET_MAX_MEM_LIMIT_IN_MB}MB`);
+  } else {
+    printInfo(`No need to update blockletMaxMemoryLimit (${info?.runtimeConfig?.blockletMaxMemoryLimit}MB)`);
+  }
+};

package/lib/migrations/README.md

@@ -41,7 +41,7 @@ module.exports = async ({ states, config, configFile, printInfo }) => {
   printInfo('Try to update node config to 1.0.21...');
   let changed = false;
 
-  const rawConfig = yaml.safeLoad(fs.readFileSync(configFile).toString());
+  const rawConfig = yaml.load(fs.readFileSync(configFile).toString());
   const info = await states.node.read();
 
   const updates = [

package/lib/migrations/index.js

@@ -132,9 +132,13 @@ const runMigrationScripts = async ({
   for (let i = 0; i < scripts.length; i++) {
     const { script, version } = scripts[i];
     try {
-      const executed = await node.isMigrationExecuted({ script, version });
-      if (executed === false) {
+      if (process.env.NODE_ENV === 'development') {
         pending.push(scripts[i]);
+      } else {
+        const executed = await node.isMigrationExecuted({ script, version });
+        if (executed === false) {
+          pending.push(scripts[i]);
+        }
       }
     } catch (err) {
       printError(`Failed to detect migration script execution status: ${script}, error: ${err.message}`);

package/lib/monitor/blocklet-runtime-monitor.js

@@ -0,0 +1,200 @@
+const EventEmitter = require('events');
+
+const cloneDeep = require('lodash/cloneDeep');
+const pLimit = require('p-limit');
+const { forEachBlocklet } = require('@blocklet/meta/lib/util');
+const getComponentProcessId = require('@blocklet/meta/lib/get-component-process-id');
+const { EVENTS } = require('@abtnode/constant');
+const { BlockletStatus, BlockletGroup } = require('@blocklet/constant');
+const defaultLogger = require('@abtnode/logger')('blocklet-runtime-monitor');
+
+const { getRuntimeInfo } = require('../util/blocklet');
+
+// type BlockletHistoryItem = {
+//   date: number;
+//   cpu: number;
+//   mem: number;
+// };
+
+// type History = Array<BlockletHistoryItem>;
+
+// type RuntimeInfo = {
+//   pid: number;
+//   uptime: number;
+//   memoryUsage: number;
+//   cpuUsage: number;
+//   status: string;
+// };
+
+// type ComponentMonitorData = {
+//   runtimeInfo: RuntimeInfo;
+// }
+
+// type AppMonitorData = ComponentMonitorData & {
+//   history: History;
+// }
+
+// type BlockletMonitorData = {
+//   app: AppMonitorData;
+//   [componentId: string]: ComponentMonitorData;
+// }
+
+// type MonitorData = {
+//   [blockletDid: string]: BlockletMonitorData;
+// }
+
+class BlockletRuntimeMonitor extends EventEmitter {
+  constructor({ historyLength = 100, states, logger = defaultLogger } = {}) {
+    super();
+
+    /**
+     * private
+     */
+    this.historyLength = historyLength;
+    this.states = states;
+    this.data = {};
+    this.logger = logger;
+    this.inProgress = false;
+  }
+
+  getHistory(blockletDid) {
+    if (!this.data[blockletDid]) {
+      return [];
+    }
+
+    return this.data[blockletDid].app.history || [];
+  }
+
+  getRuntimeInfo(blockletDid, componentId = 'app') {
+    if (!this.data[blockletDid]) {
+      return null;
+    }
+
+    if (!this.data[blockletDid][componentId]) {
+      return null;
+    }
+
+    return this.data[blockletDid][componentId].runtimeInfo;
+  }
+
+  async monit(did) {
+    const blocklet = await this.states.blocklet.getBlocklet(did);
+    await this._monit(blocklet);
+  }
+
+  async _monit(blocklet, { addToHistory } = {}) {
+    const {
+      meta: { did: blockletDid },
+      status,
+    } = blocklet;
+
+    if (status !== BlockletStatus.running) {
+      if (this.data[blockletDid]) {
+        Object.keys(this.data[blockletDid]).forEach((key) => {
+          delete this.data[blockletDid][key].runtimeInfo;
+        });
+      }
+      return;
+    }
+
+    this.data[blockletDid] = this.data[blockletDid] || {
+      app: {
+        runtimeInfo: {
+          memoryUsage: 0,
+          cpuUsage: 0,
+          pid: -1,
+          uptime: 0,
+          status: '',
+        },
+        history: [],
+      },
+    };
+
+    let appCpu = 0;
+    let appMem = 0;
+
+    await forEachBlocklet(
+      blocklet,
+      async (component, { id: componentId, ancestors }) => {
+        if (component.meta.group !== BlockletGroup.gateway) {
+          const processId = getComponentProcessId(component, ancestors);
+          try {
+            const runtimeInfo = await getRuntimeInfo(processId);
+
+            this.data[blockletDid][componentId] = { runtimeInfo };
+
+            if (!component.mountPoint || component.mountPoint === '/') {
+              this.data[blockletDid].app.runtimeInfo = cloneDeep(runtimeInfo);
+            }
+
+            appCpu += runtimeInfo.cpuUsage || 0;
+            appMem += runtimeInfo.memoryUsage || 0;
+          } catch (err) {
+            this.logger.error('failed to get blocklet runtime info', { processId, error: err });
+          }
+        }
+      },
+      { parallel: true }
+    );
+
+    this.data[blockletDid].app.runtimeInfo.cpuUsage = appCpu;
+    this.data[blockletDid].app.runtimeInfo.memoryUsage = appMem;
+
+    // push to history
+    if (addToHistory) {
+      this._push(blockletDid, { date: Date.now(), cpu: appCpu, mem: appMem });
+    }
+  }
+
+  async monitAll() {
+    if (this.inProgress) {
+      this.logger.error('monitoring is in progress');
+      return;
+    }
+
+    this.inProgress = true;
+
+    try {
+      const blocklets = await this.states.blocklet.getBlocklets();
+
+      const limit = pLimit(5);
+      await Promise.allSettled(
+        blocklets.map((blocklet) =>
+          limit(async () => {
+            await this._monit(blocklet, { addToHistory: true });
+          })
+        )
+      );
+
+      this.inProgress = false;
+
+      // emit event
+      const eventData = [];
+      Object.entries(this.data).forEach(([did, x]) => {
+        Object.entries(x).forEach(([componentId, { runtimeInfo }]) => {
+          if (componentId !== 'app') {
+            eventData.push({ did, componentId, runtimeInfo });
+          }
+        });
+      });
+      this.emit(EVENTS.BLOCKLETS_RUNTIME_INFO, eventData);
+    } catch (error) {
+      this.inProgress = false;
+      this.logger.error('monit blocklet runtime failed', { error });
+    }
+  }
+
+  delete(blockletDid) {
+    delete this.data[blockletDid];
+  }
+
+  _push(blockletDid, value) {
+    const arr = this.data[blockletDid].app.history;
+    arr.push(value);
+    if (arr.length > this.historyLength) {
+      arr.shift();
+    }
+  }
+}
+
+module.exports.BlockletRuntimeMonitor = BlockletRuntimeMonitor;

package/lib/monitor/get-history-list.js

@@ -0,0 +1,37 @@
+const maxBy = require('lodash/maxBy');
+
+const getHistoryList = ({ history, hours, recordIntervalSec, props = [] }) => {
+  if (hours < 1 || hours > 24) {
+    throw new Error('hours should between 1 to 24');
+  }
+  const intHours = Math.floor(hours);
+
+  const nPerHour = (60 / recordIntervalSec) * 60;
+  const n = nPerHour * intHours;
+  const list = history.slice(-n);
+
+  let i = list.length;
+
+  const res = [];
+
+  while (i > 0 && res.length < nPerHour) {
+    const arr = list.slice(Math.max(0, i - intHours), i);
+
+    // 取一段时间的最大值
+    if (props && props.length) {
+      const obj = {};
+      props.forEach((p) => {
+        obj[p] = (maxBy(arr, (x) => x[p]) || {})[p];
+      });
+      res.unshift(obj);
+    } else {
+      res.unshift(Math.max(...arr));
+    }
+
+    i -= intHours;
+  }
+
+  return res;
+};
+
+module.exports = getHistoryList;