ali-skills 0.0.21 → 0.0.22

package/node_modules/cluster-client/lib/index.js

@@ -0,0 +1,213 @@
+'use strict';
+
+const is = require('is-type-of');
+const symbols = require('./symbol');
+const logger = require('./default_logger');
+const transcode = require('./default_transcode');
+const SingleClient = require('./wrapper/single');
+const ClusterClient = require('./wrapper/cluster');
+const { formatKey } = require('./utils');
+
+const defaultOptions = {
+  port: parseInt(process.env.NODE_CLUSTER_CLIENT_PORT) || 7777,
+  singleMode: process.env.NODE_CLUSTER_CLIENT_SINGLE_MODE === '1',
+  maxWaitTime: 30000,
+  connectTimeout: parseInt(process.env.NODE_CLUSTER_CLIENT_CONNECT_TIMEOUT) || 10000,
+  responseTimeout: 3000,
+  heartbeatInterval: 20000,
+  isCheckHeartbeat: true,
+  autoGenerate: true,
+  isBroadcast: true,
+  logger,
+  transcode,
+  formatKey,
+};
+const autoGenerateMethods = [
+  'subscribe',
+  'unSubscribe',
+  'publish',
+  'close',
+];
+
+class ClientGenerator {
+  /**
+   * Cluster Client Generator
+   *
+   * @param {Function} clientClass - the client class
+   * @param {Object} options
+   *  - {Number} responseTimeout - response timeout, default is 3000
+   *  - {Boolean} autoGenerate - whether generate delegate rule automatically, default is true
+   *  - {Boolean} isBroadcast - whether broadcast subscrption result to all followers or just one, default is true
+   *  - {Logger} logger - log instance
+   *  - {Transcode} [transcode|JSON.stringify/parse]
+   *    - {Function} encode - custom serialize method
+   *    - {Function} decode - custom deserialize method
+   *  - {Boolean} [isLeader|null] - specify whether current instance is leader
+   *  - {Number} [maxWaitTime|30000] - leader startup max time (ONLY effective on isLeader is true)
+   * @class
+   */
+  constructor(clientClass, options) {
+    this._clientClass = clientClass;
+    this._options = Object.assign({
+      name: clientClass.prototype.constructor.name,
+    }, defaultOptions, options);
+
+    // wrapper descptions
+    this._descriptors = new Map();
+  }
+
+  /**
+   * override the property
+   *
+   * @param {String} name - property name
+   * @param {Object} value - property value
+   * @return {ClientGenerator} self
+   */
+  override(name, value) {
+    this._descriptors.set(name, {
+      type: 'override',
+      value,
+    });
+    return this;
+  }
+
+  /**
+   * delegate methods
+   *
+   * @param {String} from - method name
+   * @param {String} to - delegate to subscribe|publish|invoke
+   * @return {ClientGenerator} self
+   */
+  delegate(from, to) {
+    to = to || 'invoke';
+    this._descriptors.set(from, {
+      type: 'delegate',
+      to,
+    });
+    return this;
+  }
+
+  /**
+   * create cluster client instance
+   *
+   * @param {...any} args arguments
+   * @return {Object} instance
+   */
+  create(...args) {
+    const clientClass = this._clientClass;
+    const proto = clientClass.prototype;
+    const descriptors = this._descriptors;
+
+    // auto generate description
+    if (this._options.autoGenerate) {
+      this._generateDescriptors();
+    }
+
+    function createRealClient() {
+      return Reflect.construct(clientClass, args);
+    }
+
+    const ClientWrapper = this._options.singleMode ? SingleClient : ClusterClient;
+    const client = new ClientWrapper(Object.assign({
+      createRealClient,
+      descriptors: this._descriptors,
+    }, this._options));
+
+    for (const name of descriptors.keys()) {
+      let value;
+      const descriptor = descriptors.get(name);
+      switch (descriptor.type) {
+        case 'override':
+          value = descriptor.value;
+          break;
+        case 'delegate':
+          if (/^invoke|invokeOneway$/.test(descriptor.to)) {
+            if (is.generatorFunction(proto[name])) {
+              value = function* (...args) {
+                return yield cb => { client[symbols.invoke](name, args, cb); };
+              };
+            } else if (is.function(proto[name])) {
+              if (descriptor.to === 'invoke') {
+                value = (...args) => {
+                  let cb;
+                  if (is.function(args[args.length - 1])) {
+                    cb = args.pop();
+                  }
+                  // whether callback or promise
+                  if (cb) {
+                    client[symbols.invoke](name, args, cb);
+                  } else {
+                    return new Promise((resolve, reject) => {
+                      client[symbols.invoke](name, args, function(err) {
+                        if (err) {
+                          reject(err);
+                        } else {
+                          resolve.apply(null, Array.from(arguments).slice(1));
+                        }
+                      });
+                    });
+                  }
+                };
+              } else {
+                value = (...args) => {
+                  client[symbols.invoke](name, args);
+                };
+              }
+            } else {
+              throw new Error(`[ClusterClient] api: ${name} not implement in client`);
+            }
+          } else {
+            value = client[Symbol.for(`ClusterClient#${descriptor.to}`)];
+          }
+          break;
+        default:
+          break;
+      }
+      Object.defineProperty(client, name, {
+        value,
+        writable: true,
+        enumerable: true,
+        configurable: true,
+      });
+    }
+
+    return client;
+  }
+
+  _generateDescriptors() {
+    const clientClass = this._clientClass;
+    const proto = clientClass.prototype;
+
+    const needGenerateMethods = new Set(autoGenerateMethods);
+    for (const entry of this._descriptors.entries()) {
+      const key = entry[0];
+      const value = entry[1];
+      if (needGenerateMethods.has(key) ||
+        (value.type === 'delegate' && needGenerateMethods.has(value.to))) {
+        needGenerateMethods.delete(key);
+      }
+    }
+    for (const method of needGenerateMethods.values()) {
+      if (is.function(proto[method])) {
+        this.delegate(method, method);
+      }
+    }
+
+    const keys = Reflect.ownKeys(proto)
+      .filter(key => typeof key !== 'symbol' &&
+        !key.startsWith('_') &&
+        !this._descriptors.has(key));
+
+    for (const key of keys) {
+      const descriptor = Reflect.getOwnPropertyDescriptor(proto, key);
+      if (descriptor.value &&
+        (is.generatorFunction(descriptor.value) || is.asyncFunction(descriptor.value))) {
+        this.delegate(key);
+      }
+    }
+  }
+}
+
+module.exports = function(clientClass, options) {
+  return new ClientGenerator(clientClass, options);
+};

package/node_modules/cluster-client/lib/leader.js

@@ -0,0 +1,437 @@
+'use strict';
+
+const debug = require('util').debuglog('cluster-client#leader');
+const co = require('co');
+const is = require('is-type-of');
+const Base = require('sdk-base');
+const utils = require('./utils');
+const random = require('utility').random;
+const ClusterServer = require('./server');
+const Connection = require('./connection');
+const Request = require('./protocol/request');
+const Response = require('./protocol/response');
+
+class Leader extends Base {
+  /**
+   * The Leader hold the real client
+   *
+   * @param {Object} options
+   *  - {String} name - client name, default is the class name
+   *  - {ClusterServer} server - the cluster server
+   *  - {Boolean} isBroadcast - whether broadcast subscrption result to all followers or just one, default is true
+   *  - {Number} heartbeatInterval - the heartbeat interval
+   *  - {Function} createRealClient - to create the real client
+   *  - {Boolean} isCheckHeartbeat - whether check heartbeat, default is true
+   * @class
+   */
+  constructor(options) {
+    super(options);
+    this._connections = new Map();
+    this._subListeners = new Map(); // subscribe key => listener
+    this._subConnMap = new Map(); // subscribe key => conn key
+    this._subData = new Map();
+    // local socket server
+    this._server = this.options.server;
+    this._transcode = this.options.transcode;
+    this._isReady = false;
+    this._closeByUser = false;
+    // the real client
+    this._realClient = this.options.createRealClient();
+    this._subscribeMethodName = this._findMethodName('subscribe');
+    this._publishMethodName = this._findMethodName('publish');
+
+    // event delegate
+    utils.delegateEvents(this._realClient, this);
+
+    if (is.function(this._realClient.ready)) {
+      this._realClient.ready(err => {
+        if (err) {
+          this.ready(err);
+        } else {
+          this._isReady = true;
+          this.ready(true);
+        }
+      });
+    } else {
+      this._isReady = true;
+      this.ready(true);
+    }
+
+    this._handleConnection = this._handleConnection.bind(this);
+
+    // subscribe its own channel
+    this._server.on(`${this.options.name}_connection`, this._handleConnection);
+    this._server.once('close', () => { this.emit('server_closed'); });
+    this.on('server_closed', () => {
+      this._handleClose().catch(err => { this.emit('error', err); });
+    });
+
+    if (this.options.isCheckHeartbeat) {
+      // maxIdleTime is 3 times of heartbeatInterval
+      const heartbeatInterval = this.options.heartbeatInterval;
+      const maxIdleTime = this.options.heartbeatInterval * 3;
+
+      this._heartbeatTimer = setInterval(() => {
+        const now = Date.now();
+        for (const conn of this._connections.values()) {
+          const dur = now - conn.lastActiveTime;
+          if (dur > maxIdleTime) {
+            const err = new Error(`client no response in ${dur}ms exceeding maxIdleTime ${maxIdleTime}ms, maybe the connection is close on other side.`);
+            err.name = 'ClusterClientNoResponseError';
+            conn.close(err);
+          }
+        }
+      }, heartbeatInterval);
+    }
+  }
+
+  get isLeader() {
+    return true;
+  }
+
+  get logger() {
+    return this.options.logger;
+  }
+
+  formatKey(reg) {
+    return '$$inner$$__' + this.options.formatKey(reg);
+  }
+
+  subscribe(reg, listener) {
+    const transcode = this._transcode;
+    const conn = Object.create(Base.prototype, {
+      isMock: { value: true },
+      key: { value: `${this.options.name}_mock_conn_${utils.nextId()}` },
+      lastActiveTime: {
+        get() {
+          return Date.now();
+        },
+      },
+      listener: {
+        get() {
+          return listener;
+        },
+      },
+      send: {
+        value(req) {
+          const result = transcode.decode(req.data);
+          process.nextTick(() => {
+            listener(result);
+          });
+        },
+      },
+      close: { value() {} },
+    });
+    conn.once('close', () => {
+      this._connections.delete(conn.key);
+      for (const connKeySet of this._subConnMap.values()) {
+        connKeySet.delete(conn.key);
+      }
+    });
+
+    this._connections.set(conn.key, conn);
+    this._doSubscribe(reg, conn);
+  }
+
+  unSubscribe(reg, listener) {
+    const key = this.formatKey(reg);
+    const connKeySet = this._subConnMap.get(key) || new Set();
+    const newConnKeySet = new Set();
+    for (const connKey of connKeySet.values()) {
+      const conn = this._connections.get(connKey);
+      if (!conn) {
+        continue;
+      }
+      if (conn.isMock && (!listener || conn.listener === listener)) {
+        this._connections.delete(connKey);
+        continue;
+      }
+      newConnKeySet.add(connKey);
+    }
+    this._subConnMap.set(key, newConnKeySet);
+  }
+
+  publish(reg) {
+    this._realClient[this._publishMethodName](reg);
+  }
+
+  invoke(methodName, args, callback) {
+    let method = this._realClient[methodName];
+    // compatible with generatorFunction
+    if (is.generatorFunction(method)) {
+      method = co.wrap(method);
+    }
+    args.push(callback);
+    const ret = method.apply(this._realClient, args);
+    if (callback && is.promise(ret)) {
+      ret.then(result => callback(null, result), err => callback(err))
+        // to avoid uncaught exception in callback function, then cause unhandledRejection
+        .catch(err => { this._errorHandler(err); });
+    }
+  }
+
+  _doSubscribe(reg, conn) {
+    const key = this.formatKey(reg);
+    const callback = err => {
+      if (err) {
+        this._errorHandler(err);
+      }
+    };
+    const isBroadcast = this.options.isBroadcast;
+    const timeout = this.options.responseTimeout;
+
+    const connKeySet = this._subConnMap.get(key) || new Set();
+    connKeySet.add(conn.key);
+    this._subConnMap.set(key, connKeySet);
+
+    // only subscribe once in cluster mode, and broadcast to all followers
+    if (!this._subListeners.has(key)) {
+      const listener = result => {
+        const data = this._transcode.encode(result);
+        this._subData.set(key, data);
+
+        const connKeySet = this._subConnMap.get(key);
+        if (!connKeySet) {
+          return;
+        }
+        let keys = Array.from(connKeySet.values());
+        // if isBroadcast equal to false, random pick one to notify
+        if (!isBroadcast) {
+          keys = [ keys[random(keys.length)] ];
+        }
+
+        for (const connKey of keys) {
+          const conn = this._connections.get(connKey);
+          if (conn) {
+            debug('[Leader:%s] push subscribe data to cluster client#%s', this.options.name, connKey);
+            conn.send(new Request({
+              timeout,
+              connObj: {
+                type: 'subscribe_result',
+                key,
+              },
+              data,
+            }), callback);
+          }
+        }
+      };
+      this._subListeners.set(key, listener);
+      this._realClient[this._subscribeMethodName](reg, listener);
+    } else if (this._subData.has(key) && isBroadcast) {
+      conn.send(new Request({
+        timeout,
+        connObj: {
+          type: 'subscribe_result',
+          key,
+        },
+        data: this._subData.get(key),
+      }), callback);
+    }
+  }
+
+  _findMethodName(type) {
+    return utils.findMethodName(this.options.descriptors, type);
+  }
+
+  // handle new socket connect
+  _handleConnection(socket, req) {
+    debug('[Leader:%s] socket connected, port: %d', this.options.name, socket.remotePort);
+
+    const conn = new Connection({
+      socket,
+      name: this.options.name,
+      logger: this.logger,
+      transcode: this.options.transcode,
+      requestTimeout: this.options.requestTimeout,
+    });
+    this._connections.set(conn.key, conn);
+    conn.once('close', () => {
+      this._connections.delete(conn.key);
+      for (const connKeySet of this._subConnMap.values()) {
+        connKeySet.delete(conn.key);
+      }
+    });
+    conn.on('error', err => this._errorHandler(err));
+    conn.on('request', (req, res) => this._handleRequest(req, res, conn));
+
+    // handle register channel request
+    const res = new Response({
+      id: req.id,
+      timeout: req.timeout,
+    });
+    this._handleRequest(req, res, conn);
+  }
+
+  _handleSubscribe(req, conn) {
+    const connObj = req.connObj || {};
+    this._doSubscribe(connObj.reg, conn);
+  }
+
+  _handleUnSubscribe(req, conn) {
+    const connObj = req.connObj || {};
+    const key = this.formatKey(connObj.reg);
+    const connKeySet = this._subConnMap.get(key) || new Set();
+    connKeySet.delete(conn.key);
+    this._subConnMap.set(key, connKeySet);
+  }
+
+  // handle request from followers
+  _handleRequest(req, res, conn) {
+    const connObj = req.connObj || {};
+    // update last active time to make sure not kick out by leader
+    conn.lastActiveTime = Date.now();
+
+    switch (connObj.type) {
+      case 'subscribe':
+        debug('[Leader:%s] received subscribe request from follower, req: %j, conn: %s', this.options.name, req, conn.key);
+        this._handleSubscribe(req, conn);
+        break;
+      case 'unSubscribe':
+        debug('[Leader:%s] received unSubscribe request from follower, req: %j, conn: %s', this.options.name, req, conn.key);
+        this._handleUnSubscribe(req, conn);
+        break;
+      case 'invoke':
+      {
+        debug('[Leader:%s] received invoke request from follower, req: %j, conn: %s', this.options.name, req, conn.key);
+        const argLength = connObj.argLength;
+        const args = [];
+        if (argLength > 0) {
+          const data = req.data;
+          for (let i = 0, offset = 0; i < argLength; ++i) {
+            const len = data.readUInt32BE(offset);
+            const arg = this._transcode.decode(data.slice(offset + 4, offset + 4 + len));
+            args.push(arg);
+            offset += (4 + len);
+          }
+        }
+
+        if (connObj.oneway) {
+          this.invoke(connObj.method, args);
+        } else {
+          const startTime = Date.now();
+          this.invoke(connObj.method, args, (err, result) => {
+            // no response if processing timeout, just record error
+            if (req.timeout && Date.now() - startTime > req.timeout) {
+              const err = new Error(`[Leader:${this.options.name}] invoke method:${connObj.method} timeout for req#${req.id}`);
+              err.name = 'ClusterLeaderTimeoutError';
+              err.method = connObj.method;
+              err.args = args;
+              this._errorHandler(err);
+              return;
+            }
+
+            if (err) {
+              const data = {
+                stack: err.stack,
+                name: err.name,
+                message: err.message,
+                code: err.code,
+              };
+              err.method = connObj.method;
+              err.args = connObj.args;
+              this._errorHandler(err);
+
+              res.connObj = {
+                type: 'invoke_result',
+                success: false,
+              };
+              res.data = this._transcode.encode(data);
+            } else {
+              debug('[Leader:%s] send method:%s result to follower, result: %j', this.options.name, connObj.method, result);
+              const data = this._transcode.encode(result);
+              res.connObj = {
+                type: 'invoke_result',
+                success: true,
+              };
+              res.data = data;
+            }
+            conn.send(res);
+          });
+        }
+        break;
+      }
+      case 'heartbeat':
+        debug('[Leader:%s] received heartbeat request from follower, req: %j, conn: %s', this.options.name, req, conn.key);
+        res.connObj = { type: 'heartbeat_res' };
+        conn.send(res);
+        break;
+      case 'register_channel':
+        // make sure response after leader is ready
+        this.ready(err => {
+          if (err) {
+            res.connObj = { type: 'register_channel_res' };
+            const data = {
+              message: err.message,
+              stack: err.stack,
+              name: err.name,
+              code: err.code,
+            };
+            res.data = this._transcode.encode({
+              success: false,
+              error: data,
+            });
+          } else {
+            res.connObj = { type: 'register_channel_res' };
+            res.data = this._transcode.encode({ success: true });
+          }
+          conn.send(res);
+        });
+        break;
+      default:
+      {
+        const err = new Error(`unsupport data type: ${connObj.type}`);
+        err.name = 'ClusterRequestTypeError';
+        this._errorHandler(err);
+        break;
+      }
+    }
+  }
+
+  // emit error asynchronously
+  _errorHandler(err) {
+    setImmediate(() => {
+      if (!this._closeByUser) {
+        this.emit('error', err);
+      }
+    });
+  }
+
+  async _handleClose() {
+    debug('[Leader:%s] leader server is closed', this.options.name);
+    // close the real client
+    if (this._realClient) {
+      const originClose = this._findMethodName('close');
+      if (originClose) {
+        // support common function, generatorFunction, and function returning a promise
+        await utils.callFn(this._realClient[originClose].bind(this._realClient));
+      }
+    }
+    this._heartbeatTimer && clearInterval(this._heartbeatTimer);
+    this._heartbeatTimer = null;
+    this.emit('close');
+  }
+
+  async close() {
+    this._closeByUser = true;
+    debug('[Leader:%s] try to close leader', this.options.name);
+    // 1. stop listening to server channel
+    this._server.removeListener(`${this.options.name}_connection`, this._handleConnection);
+
+    // 2. close all mock connections
+    for (const conn of this._connections.values()) {
+      if (conn.isMock) {
+        conn.emit('close');
+      }
+    }
+
+    // 3. close server
+    //    CANNOT close server directly by server.close(), other cluster clients may be using it
+    this.removeAllListeners('server_closed');
+    await ClusterServer.close(this.options.name, this._server);
+
+    // 5. close real client
+    await this._handleClose();
+  }
+}
+
+module.exports = Leader;

package/node_modules/cluster-client/lib/protocol/byte_buffer.js

@@ -0,0 +1,8 @@
+'use strict';
+
+const ByteBuffer = require('byte');
+
+// avoid create many buffer
+module.exports = new ByteBuffer({
+  size: 1024 * 1024,
+});