ee-core 2.0.3 → 2.1.0-beta.2

package/jobs/child-pool/index.js

@@ -0,0 +1,205 @@
+const EventEmitter = require('events');
+const LoadBalancer = require('../load-balancer');
+const Loader = require('../../loader');
+const ForkProcess = require('../child/forkProcess');
+const Channel = require('../../const/channel');
+const Helper = require('../../utils/helper');
+
+class ChildPoolJob extends EventEmitter {
+
+  constructor(opt = {}) {
+    super();
+    let options = Object.assign({
+      weights: [],
+    }, opt);
+
+    this.boundMap = new Map();
+    this.children = {};
+    this.min = 3;
+    this.max = 6;
+    this.strategy = 'polling';
+    this.weights = new Array(this.max).fill().map((v, i) => {
+      let w = Helper.validValue(options.weights[i]) ? options.weights[i] : 1
+      return w;
+    });
+
+    let lbOpt = {
+      algorithm: LoadBalancer.Algorithm.polling,
+      targets: [],
+    }
+    this.LB = new LoadBalancer(lbOpt);
+    this._initEvents();
+  }
+
+  /**
+   * 初始化监听
+   */  
+  _initEvents() {
+    this.on(Channel.events.childProcessExit, (data) => {
+      this._removeChild(data.pid);
+    });
+    this.on(Channel.events.childProcessError, (data) => {
+      this._removeChild(data.pid);
+    });
+  }
+
+  /**
+   * 移除对象
+   */  
+  _removeChild(pid) {
+    const length = Object.keys(this.children).length;
+    const lbOpt = {
+      id: pid,
+      weight: this.weights[length - 1],
+    }
+    this.LB.del(lbOpt);
+    delete this.children[pid];
+  }
+
+  /**
+   * 创建一个池子
+   */  
+  async create(number = 3) {
+    if (number < 0 || number > this.max) {
+      throw new Error(`[ee-core] [jobs/child-pool] The number is invalid !`);
+    }
+    let currentNumber = this.children.length;
+    if (currentNumber > this.max) {
+      throw new Error(`[ee-core] [jobs/child-pool] The number of current processes number: ${currentNumber} is greater than the maximum: ${this.max} !`);
+    }
+
+    if (number + currentNumber > this.max) {
+      number = this.max - currentNumber;
+    }
+
+    // args
+    let options = Object.assign({
+      processArgs: {
+        type: 'childPoolJob'
+      }
+    }, {});
+    for (let i = 1; i <= number; i++) {
+      let task = new ForkProcess(this, options);
+      this._childCreated(task);
+    }
+  
+    let pids = Object.keys(this.children);
+
+    return pids;
+  }
+
+  /**
+   * 子进程创建后处理
+   */  
+  _childCreated(childProcess) {
+    let pid = childProcess.pid;
+    this.children[pid] = childProcess;
+
+    const length = Object.keys(this.children).length;
+    let lbTask = {
+      id: pid,
+      weight: this.weights[length - 1],
+    }
+    this.LB.add(lbTask);
+  }
+
+  /**
+   * 执行一个job文件
+   */  
+  run(filepath, params = {}) {
+    const jobPath = Loader.getFullpath(filepath);
+    const childProcess = this.getChild();
+    childProcess.dispatch('run', jobPath, params);
+
+    return childProcess;
+  }
+
+  /**
+   * 异步执行一个job文件
+   */
+  async runPromise(filepath, params = {}) {
+    return this.run(filepath, params);
+  }  
+
+  /**
+   * 获取绑定的进程对象
+   */  
+  getBoundChild(boundId) {
+    let proc;
+    const boundPid = this.boundMap.get(boundId);
+    if (boundPid) {
+      proc = this.children[boundPid];
+      return proc;
+    }
+
+    // 获取进程并绑定
+    proc = this.getChild();
+    this.boundMap.set(boundId, proc.pid);
+
+    return proc;
+  }
+
+  /**
+   * 通过pid获取一个子进程对象
+   */  
+  getChildByPid(pid) {
+    let proc = this.children[pid] || null;
+    return proc;
+  }
+
+  /**
+   * 获取一个子进程对象
+   */  
+  getChild() {
+    let proc;
+    const currentPids = Object.keys(this.children);
+
+    // 没有则创建
+    if (currentPids.length == 0) {
+      let subIds = this.create(1);
+      proc = this.children[subIds[0]];
+    } else {
+      // 从池子中获取一个
+      let onePid = this.LB.pickOne().id;
+      proc = this.children[onePid];
+    }
+    
+    if (!proc) {
+      let errorMessage = `[ee-core] [jobs/child-pool] Failed to obtain the child process !`
+      throw new Error(errorMessage);
+    }
+
+    return proc;
+  }
+
+  /**
+   * 获取当前pids
+   */  
+  getPids() {
+    let pids = Object.keys(this.children);
+    return pids;
+  }   
+
+  /**
+   * kill all 
+   * @param type {String} - 'sequence' | 'parallel'
+   */
+  killAll(type = 'parallel') {
+    let i = 1;
+    Object.keys(this.children).forEach(key => {
+      let proc = this.children[key];
+      if (proc) {
+        if (type == 'sequence') {
+          setTimeout(()=>{
+            proc.kill();
+          }, i * 1000)
+          i++;
+        } else {
+          proc.kill();
+        }
+      }
+    });
+  }
+}
+
+module.exports = ChildPoolJob;

package/jobs/index.js

@@ -1,7 +1,9 @@
 const ChildJob = require('./child');
 const RendererJob = require('./renderer');
+const ChildPoolJob = require('./child-pool');
 
 module.exports = {
   ChildJob,
-  RendererJob
+  RendererJob,
+  ChildPoolJob
 };

package/jobs/load-balancer/algorithm/index.js

@@ -0,0 +1,12 @@
+const Consts = require("../consts");
+
+module.exports = {
+  [Consts.polling]: require('./polling'),
+  [Consts.weights]: require('./weights'),
+  [Consts.random]: require('./random'),
+  [Consts.specify]: require('./specify'),
+  [Consts.minimumConnection]: require('./minimumConnection'),
+  [Consts.weightsPolling]: require('./weightsPolling'),
+  [Consts.weightsRandom]: require('./weightsRandom'),
+  [Consts.weightsMinimumConnection]: require('./weightsMinimumConnection'),
+};

package/jobs/load-balancer/algorithm/minimumConnection.js

@@ -0,0 +1,19 @@
+/**
+ * 最小连接数
+ */
+module.exports = function (tasks, conMap={}) {
+  if (tasks.length < 2) return tasks[0] || null;
+
+  let min = conMap[tasks[0].id];
+  let minIndex = 0;
+
+  for (let i = 1; i < tasks.length; i++) {
+    const con = conMap[tasks[i].id] || 0;
+    if (con <= min) {
+      min = con;
+      minIndex = i;
+    }
+  }
+
+  return tasks[minIndex] || null;
+};

package/jobs/load-balancer/algorithm/polling.js

@@ -0,0 +1,12 @@
+/**
+ * 轮询
+ */
+module.exports = function (tasks, currentIndex, context) {
+  if (!tasks.length) return null;
+
+  const task = tasks[currentIndex];
+  context.currentIndex ++;
+  context.currentIndex %= tasks.length;
+
+  return task || null;
+};

package/jobs/load-balancer/algorithm/random.js

@@ -0,0 +1,10 @@
+/**
+ * 随机
+ */
+module.exports = function (tasks) {
+
+  const length = tasks.length;
+  const target = tasks[Math.floor(Math.random() * length)];
+
+  return target || null;
+};

package/jobs/load-balancer/algorithm/specify.js

@@ -0,0 +1,15 @@
+/**
+ * 声明绑定
+ */
+module.exports = function (tasks, id) {
+  let task;
+
+  for (let i = 0; i < tasks.length; i++) {
+    if (tasks[i].id === id) {
+      task = tasks[i];
+      break;
+    }
+  }
+
+  return task || null;
+};

package/jobs/load-balancer/algorithm/weights.js

@@ -0,0 +1,22 @@
+/**
+ * 权重
+ */
+module.exports = function (tasks, weightTotal, context) {
+
+  if (!tasks.length) return null;
+
+  let max = tasks[0].weight, maxIndex = 0, sum;
+
+  for (let i = 0; i < tasks.length; i++) {
+    sum = (tasks[i].weight || 0) + Math.random() * weightTotal;
+    if (sum >= max) {
+      max = sum;
+      maxIndex = i;
+    }
+  }
+
+  context.weightIndex += 1;
+  context.weightIndex %= (weightTotal + 1);
+
+  return tasks[maxIndex];
+};

package/jobs/load-balancer/algorithm/weightsMinimumConnection.js

@@ -0,0 +1,30 @@
+/**
+ * 权重最小连接数
+ */
+module.exports = function (tasks, weightTotal, connectionsMap, context) {
+
+  if (!tasks.length) return null;
+
+  let min = tasks[0].weight, minIndex = 0, sum;
+
+  const connectionsTotal = tasks.reduce((total, cur) => {
+    total += (connectionsMap[cur.id] || 0);
+    return total;
+  }, 0);
+
+  // algorithm: (weight + connections'weight) + random factor
+  for (let i = 0; i < tasks.length; i++) {
+    sum =
+      (tasks[i].weight || 0) + (Math.random() * weightTotal) +
+      (( (connectionsMap[tasks[i].id] || 0) * weightTotal ) / connectionsTotal);
+    if (sum <= min) {
+      min = sum;
+      minIndex = i;
+    }
+  }
+
+  context.weightIndex += 1;
+  context.weightIndex %= (weightTotal + 1);
+
+  return tasks[minIndex];
+};

package/jobs/load-balancer/algorithm/weightsPolling.js

@@ -0,0 +1,23 @@
+/**
+ * 权重轮询
+ */
+module.exports = function (tasks, weightIndex, weightTotal, context) {
+
+  if (!tasks.length) return null;
+
+  let weight = 0;
+  let task;
+
+  for (let i = 0; i < tasks.length; i++) {
+    weight += tasks[i].weight || 0;
+    if (weight > weightIndex) {
+      task = tasks[i];
+      break;
+    }
+  }
+
+  context.weightIndex += 1;
+  context.weightIndex %= (weightTotal + 1);
+
+  return task;
+};

package/jobs/load-balancer/algorithm/weightsRandom.js

@@ -0,0 +1,17 @@
+/**
+ * 权重随机
+ */
+module.exports = function (tasks, weightTotal) {
+  let task;
+  let weight = Math.ceil(Math.random() * weightTotal);
+
+  for (let i = 0; i < tasks.length; i++) {
+    weight -= tasks[i].weight || 0;
+    if (weight <= 0) {
+      task = tasks[i];
+      break;
+    }
+  }
+
+  return task || null;
+};

package/jobs/load-balancer/consts.js

@@ -0,0 +1,10 @@
+module.exports = {
+  polling: 'polling', // 轮询
+  weights: 'weights', // 权重
+  random: 'random', // 随机
+  specify: 'specify', // 声明绑定
+  weightsPolling: 'weightsPolling', // 权重轮询
+  weightsRandom: 'weightsRandom', // 权重随机
+  minimumConnection: 'minimumConnection', // 最小连接数
+  weightsMinimumConnection: 'weightsMinimumConnection', // 权重最小连接数
+};

package/jobs/load-balancer/index.js

@@ -0,0 +1,202 @@
+const Consts = require("./consts");
+const Scheduler = require("./scheduler");
+
+/**
+ * 负载均衡器
+ * @intro 参考electron-re项目，并做了一些改动
+ * @since 1.0.0
+ */
+class LoadBalancer {
+
+  static Algorithm = Consts;
+
+  /**
+    * @param  {Object} options
+    * @param  {Array } options.targets [ targets for load balancing calculation: [{id: 1, weight: 1}, {id: 2, weight: 2}] ]
+    * @param  {String} options.algorithm 
+    */
+  constructor(options) {
+    this.targets = options.targets;
+    this.algorithm = options.algorithm || Consts.polling;
+    this.params = { // data for algorithm
+      currentIndex: 0, // index
+      weightIndex: 0, // index for weight alogrithm
+      weightTotal: 0, // total weight
+      connectionsMap: {}, // connections of each target
+      cpuOccupancyMap: {}, // cpu occupancy of each target
+      memoryOccupancyMap: {}, // cpu occupancy of each target
+    };
+    this.scheduler = new Scheduler(this.algorithm);
+    this.memoParams = this.memorizedParams();
+    this.calculateWeightIndex();
+  }
+
+  /**
+   * 算法参数
+   */
+  memorizedParams() {
+    return {
+      [Consts.random]: () => [],
+      [Consts.polling]: () => [this.params.currentIndex, this.params],
+      [Consts.weights]: () => [this.params.weightTotal, this.params],
+      [Consts.specify]: (id) => [id],
+      [Consts.weightsRandom]: () => [this.params.weightTotal],
+      [Consts.weightsPolling]: () => [this.params.weightIndex, this.params.weightTotal, this.params],
+      [Consts.minimumConnection]: () => [this.params.connectionsMap],
+      [Consts.weightsMinimumConnection]: () => [this.params.weightTotal, this.params.connectionsMap, this.params],
+    };
+  }
+
+  /**
+   * 刷新参数
+   */
+  refreshParams(pidMap) {
+    const infos = Object.values(pidMap);
+    for (let info of infos) {
+      // this.params.connectionsMap[id] = connections;
+      this.params.cpuOccupancyMap[info.pid] = info.cpu;
+      this.params.memoryOccupancyMap[info.pid] = info.memory;
+    }
+  }
+
+  /**
+   * 选举出一个进程
+   */
+  pickOne(...params) {
+    return this.scheduler.calculate(
+      this.targets, this.memoParams[this.algorithm](...params)
+    );
+  }
+
+  /**
+   * 选举出多个进程
+   */
+  pickMulti(count = 1, ...params) {
+    return new Array(count).fill().map(
+      () => this.pickOne(...params)
+    );
+  }
+
+  /**
+   * 计算权重
+   */
+  calculateWeightIndex() {
+    this.params.weightTotal = this.targets.reduce((total, cur) => total + (cur.weight || 0), 0);
+    if (this.params.weightIndex > this.params.weightTotal) {
+      this.params.weightIndex = this.params.weightTotal;
+    }
+  }
+
+  /**
+   * 计算索引
+   */
+  calculateIndex() {
+    if (this.params.currentIndex >= this.targets.length) {
+      this.params.currentIndex = (this.params.currentIndex - 1 >= 0) ? (this.params.currentIndex - 1) : 0;
+    }
+  }
+
+  /**
+   * 清除data
+   */
+  clean(id) {
+    if (id) {
+      delete this.params.connectionsMap[id];
+      delete this.params.cpuOccupancyMap[id];
+      delete this.params.memoryOccupancyMap[id];
+    } else {
+      this.params = {
+        currentIndex: 0,
+        connectionsMap: {},
+        cpuOccupancyMap: {},
+        memoryOccupancyMap: {},
+      };
+    }
+  }
+
+  /**
+   * 添加一个进程信息
+   */
+  add(task) {
+    if (this.targets.find(target => target.id === task.id)) {
+      return console.warn(`Add Operation: the task ${task.id} already exists.`);
+    }
+    this.targets.push(task);
+    this.calculateWeightIndex();
+  }
+
+  /**
+   * 删除一个进程信息
+   */
+  del(target) {
+    let found = false;
+    for (let i  = 0; i < this.targets.length; i++) {
+      if (this.targets[i].id === target.id) {
+        this.targets.splice(i, 1);
+        this.clean(target.id);
+        this.calculateIndex();
+        found = true;
+        break;
+      }
+    }
+
+    if (found) {
+      this.calculateWeightIndex();
+    } else {
+      console.warn(`Del Operation: the task ${target.id} is not found.`, this.targets);
+    }
+  }
+
+  /**
+   * 擦除
+   */
+  wipe() {
+    this.targets = [];
+    this.calculateWeightIndex();
+    this.clean();
+  }
+
+  /**
+   * 更新计算参数
+   */
+  updateParams(object) {
+    Object.entries(object).map(([key, value]) => {
+      if (key in this.params) {
+        this.params[key] = value;
+      }
+    });
+  }
+
+  /**
+   * 设置targets
+   */
+  setTargets(targets) {
+    const targetsMap = targets.reduce((total, cur) => {
+      total[cur.id] = 1;
+      return total;
+    }, {});
+    this.targets.forEach(target => {
+      if (!(target.id in targetsMap)) {
+        this.clean(target.id);
+        this.calculateIndex();
+      }
+    });
+    this.targets = targets;
+    this.calculateWeightIndex();
+  }
+
+  /**
+   * 设置算法
+   */
+  setAlgorithm = (algorithm) => {
+    if (algorithm in Consts) {
+      this.algorithm = algorithm;
+      this.params.weightIndex = 0;
+      this.scheduler.setAlgorithm(this.algorithm);
+    } else {
+      throw new Error(`Invalid algorithm: ${algorithm}, pick from ${Object.keys(Consts).join('|')}`);
+    }
+  }
+}
+
+module.exports = LoadBalancer;

package/jobs/load-balancer/scheduler.js

@@ -0,0 +1,32 @@
+const Consts = require("./consts");
+const algorithm = require('./algorithm');
+
+/**
+ * 算法调度器
+ */
+class Scheduler {
+  constructor(algorithm) {
+    this.algorithm = algorithm || Consts.polling;
+  }
+
+  /**
+   * 计算
+   */
+  calculate(tasks, params) {
+    const results = algorithm[this.algorithm](tasks, ...params);
+    return results;
+  }
+
+  /**
+   * 设置算法
+   */
+  setAlgorithm = (algorithm) => {
+    if (algorithm in Consts) {
+      this.algorithm = algorithm;
+    } else {
+      throw new Error(`Invalid algorithm: ${algorithm}, pick from ${Object.keys(Consts).join('|')}`);
+    }
+  }
+}
+
+module.exports = Scheduler;

package/loader/index.js

@@ -127,7 +127,7 @@ module.exports = {
     const ret = UtilsCore.loadFile(fullpath);
 
     return ret;
-  },  
+  },   
 
   /**
    * 加载jobs模块(子进程中使用)
@@ -140,7 +140,27 @@ module.exports = {
     const ret = this.requireModule(filepath, 'jobs');
 
     return ret;
-  }, 
+  },
+  
+  /**
+   * 获取electron目录下文件的绝对路径
+   * @param {String} filepath - fullpath
+   */
+  getFullpath(filepath) {
+    let fullpath;
+    const isAbsolute = path.isAbsolute(filepath);
+    if (!isAbsolute) {
+      filepath = path.join(Ps.getBaseDir(), filepath);
+    }
+
+    fullpath = this.resolveModule(filepath);
+    if (!fs.existsSync(fullpath)) {
+      throw new Error(`[ee-core] [loader] getFullpath filepath ${fullpath} not exists`);
+    }
+
+    return fullpath;
+  }
+  
 }