@bigtyphoon/melo 1.7.6

package/lib/connectors/sioconnector.ts

@@ -0,0 +1,161 @@
+import * as util from 'util';
+import { EventEmitter } from 'events';
+import { createServer as createHttpServer } from 'http';
+import { createServer as createHttpsServer } from 'https';
+import { SioSocket } from './siosocket';
+import { IConnector } from '../interfaces/IConnector';
+import { Server, ServerOptions } from 'socket.io';
+
+let PKG_ID_BYTES = 4;
+let PKG_ROUTE_LENGTH_BYTES = 1;
+let PKG_HEAD_BYTES = PKG_ID_BYTES + PKG_ROUTE_LENGTH_BYTES;
+
+let curId = 1;
+export type SIOConnectorOptions = ServerOptions;
+
+/**
+ * Connector that manager low level connection and protocol bewteen server and client.
+ * Develper can provide their own connector to switch the low level prototol, such as tcp or probuf.
+ */
+export class SIOConnector extends EventEmitter implements IConnector {
+    port: number;
+    host: string;
+    opts: SIOConnectorOptions;
+    private server: Server;
+    sshKey?: string;
+    sshCert?: string;
+
+
+    constructor(port: number, host: string, opts: SIOConnectorOptions) {
+        super();
+        this.port = port;
+        this.host = host;
+        this.opts = opts;
+        opts.pingTimeout = opts.pingTimeout || 60;
+        opts.pingInterval = opts.pingInterval || 25;
+        this.sshKey = (opts as any).sshKey;
+        this.sshCert = (opts as any).sshCert;
+    }
+
+
+
+    /**
+     * Start connector to listen the specified port
+     */
+    start(cb: () => void) {
+        let self = this;
+        // issue https://github.com/NetEase/melo-cn/issues/174
+        let opts: SIOConnectorOptions;
+        if (!!this.opts) {
+            opts = this.opts;
+        }
+        else {
+            opts = {
+                transports: [
+                    'websocket', 'polling-xhr', 'polling-jsonp', 'polling'
+                ]
+            } as any;
+        }
+
+        opts.path = '/socket.io';
+        let port = this.port;
+
+        let sio: Server;
+        if (!!this.sshKey) {
+            let httpsServer = createHttpsServer({ key: self.sshKey, cert: self.sshCert }, function (req, res) {
+                // 要是单纯的https连接的话就会返回这个东西
+                console.log('sio https Server listening at port %d', port);
+                res.writeHead(200);
+                res.end('HTTPS Server is up');
+            }).listen(port);
+            sio = new Server(httpsServer, opts);
+        } else {
+            let httpServer = createHttpServer();
+            sio = new Server(httpServer, opts);
+            httpServer.listen(port, function () {
+                console.log('sio http Server listening at port %d', port);
+            });
+        }
+
+        sio.on('connection', (socket) => {
+            // this.wsocket.sockets.on('connection', function (socket) {
+            let siosocket = new SioSocket(curId++, socket);
+            self.emit('connection', siosocket);
+            siosocket.on('closing', function (reason) {
+                siosocket.send({ route: 'onKick', reason: reason });
+            });
+        });
+
+        process.nextTick(cb);
+    }
+
+    /**
+     * Stop connector
+     */
+    stop(force: boolean, cb: () => void) {
+        this.server.close();
+        process.nextTick(cb);
+    }
+
+    encode(reqId: number, route: string, msg: any) {
+        if (reqId) {
+            return composeResponse(reqId, route, msg);
+        } else {
+            return composePush(route, msg);
+        }
+    }
+
+    /**
+     * Decode client message package.
+     *
+     * Package format:
+     *   message id: 4bytes big-endian integer
+     *   route length: 1byte
+     *   route: route length bytes
+     *   body: the rest bytes
+     *
+     * @param  {String} data socket.io package from client
+     * @return {Object}      message object
+     */
+    decode(msg: any) {
+        let index = 0;
+
+        let id = parseIntField(msg, index, PKG_ID_BYTES);
+        index += PKG_ID_BYTES;
+
+        let routeLen = parseIntField(msg, index, PKG_ROUTE_LENGTH_BYTES);
+
+        let route = msg.substr(PKG_HEAD_BYTES, routeLen);
+        let body = msg.substr(PKG_HEAD_BYTES + routeLen);
+
+        return {
+            id: id,
+            route: route,
+            body: JSON.parse(body)
+        };
+    }
+
+}
+
+let composeResponse = function (msgId: number, route: string, msgBody: any) {
+    return {
+        id: msgId,
+        body: msgBody
+    };
+};
+
+let composePush = function (route: string, msgBody: any) {
+    return JSON.stringify({ route: route, body: msgBody });
+};
+
+let parseIntField = function (str: string, offset: number, len: number) {
+    let res = 0;
+    for (let i = 0; i < len; i++) {
+        if (i > 0) {
+            res <<= 8;
+        }
+        res |= str.charCodeAt(offset + i) & 0xff;
+    }
+
+    return res;
+};

package/lib/connectors/siosocket.ts

@@ -0,0 +1,85 @@
+import * as util from 'util';
+import { EventEmitter } from 'events';
+import { ISocket } from '../interfaces/ISocket';
+import { Socket } from 'socket.io';
+
+let ST_INITED = 0;
+let ST_CLOSED = 1;
+
+/**
+ * Socket class that wraps socket.io socket to provide unified interface for up level.
+ */
+export class SioSocket extends EventEmitter implements ISocket {
+    id: number;
+    socket: Socket;
+    remoteAddress: { ip: string };
+    state: number;
+
+    constructor(id: number, socket: Socket) {
+        super();
+        this.id = id;
+        this.socket = socket;
+        this.remoteAddress = {
+            ip: socket.handshake.address
+        };
+
+        let self = this;
+
+        socket.on('disconnect', this.emit.bind(this, 'disconnect'));
+
+        socket.on('error', this.emit.bind(this, 'error'));
+
+        socket.on('message', function (msg) {
+            self.emit('message', msg);
+        });
+
+        this.state = ST_INITED;
+
+        // TODO: any other events?
+    }
+
+
+
+    send(msg: any) {
+        if (this.state !== ST_INITED) {
+            return;
+        }
+        if (typeof msg !== 'string') {
+            msg = JSON.stringify(msg);
+        }
+        this.socket.send(msg);
+    }
+    sendRaw = this.send;
+    disconnect() {
+        if (this.state === ST_CLOSED) {
+            return;
+        }
+
+        this.state = ST_CLOSED;
+        this.socket.disconnect();
+    }
+
+    sendBatch(msgs: any[]) {
+        this.send(encodeBatch(msgs));
+    }
+}
+
+/**
+ * Encode batch msg to client
+ */
+let encodeBatch = function (msgs: any[]) {
+    let res = '[', msg;
+    for (let i = 0, l = msgs.length; i < l; i++) {
+        if (i > 0) {
+            res += ',';
+        }
+        msg = msgs[i];
+        if (typeof msg === 'string') {
+            res += msg;
+        } else {
+            res += JSON.stringify(msg);
+        }
+    }
+    res += ']';
+    return res;
+};

package/lib/connectors/udpconnector.ts

@@ -0,0 +1,113 @@
+import * as net from 'net';
+import * as util from 'util';
+import * as dgram from 'dgram';
+import * as utils from '../util/utils';
+import * as Constants from '../util/constants';
+import {UdpSocket} from './udpsocket';
+import * as Kick from './commands/kick';
+import { HandshakeCommand, HandshakeCommandOptions } from './commands/handshake';
+import { HeartbeatCommand, HeartbeatCommandOptions } from './commands/heartbeat';
+import { Package, Message } from '@bigtyphoon/melo-protocol';
+import * as coder from './common/coder';
+import { EventEmitter } from 'events';
+import { getLogger } from '@bigtyphoon/melo-logger';
+import { SocketType } from 'dgram';
+import { IConnector } from '../interfaces/IConnector';
+import * as path from 'path';
+let logger = getLogger('melo', path.basename(__filename));
+
+
+
+export interface UDPConnectorOptions extends HandshakeCommandOptions , HeartbeatCommandOptions {
+    udpType ?: 'udp4';
+    heartbeat ?: number;
+    timeout ?: number;
+}
+
+
+let curId = 1;
+
+export class UDPConnector extends EventEmitter implements IConnector {
+    opts: any;
+    type: SocketType;
+    handshake: HandshakeCommand;
+    heartbeat: HeartbeatCommand;
+    clients: { [key: string]: any };
+    host: string;
+    port: number;
+    tcpServer: net.Server;
+    socket: dgram.Socket;
+
+    constructor(port: number, host: string, opts: UDPConnectorOptions) {
+        super();
+        this.opts = opts || {};
+        this.type = opts.udpType || 'udp4';
+        this.handshake = new HandshakeCommand(opts);
+        if (!opts.heartbeat) {
+            opts.heartbeat = Constants.TIME.DEFAULT_UDP_HEARTBEAT_TIME;
+            opts.timeout = Constants.TIME.DEFAULT_UDP_HEARTBEAT_TIMEOUT;
+        }
+        this.heartbeat = new HeartbeatCommand(utils.extendsObject(opts, { disconnectOnTimeout: true }));
+        this.clients = {};
+        this.host = host;
+        this.port = port;
+    }
+
+    start(cb: () => void) {
+        let self = this;
+        this.tcpServer = net.createServer();
+        this.socket = dgram.createSocket(this.type, function (msg, peer) {
+            let key = genKey(peer);
+            if (!self.clients[key]) {
+                let udpsocket = new UdpSocket(curId++, self.socket, peer);
+                self.clients[key] = udpsocket;
+
+                udpsocket.on('handshake',
+                    self.handshake.handle.bind(self.handshake, udpsocket));
+
+                udpsocket.on('heartbeat',
+                    self.heartbeat.handle.bind(self.heartbeat, udpsocket));
+
+                udpsocket.on('disconnect',
+                    self.heartbeat.clear.bind(self.heartbeat, udpsocket.id));
+
+                udpsocket.on('disconnect', function () {
+                    delete self.clients[genKey(udpsocket.peer)];
+                });
+
+                udpsocket.on('closing', Kick.handle.bind(null, udpsocket));
+
+                self.emit('connection', udpsocket);
+            }
+        });
+
+        this.socket.on('message', function (data: Buffer, peer: dgram.RemoteInfo) {
+            let socket = self.clients[genKey(peer)];
+            if (!!socket) {
+                socket.emit('package', data);
+            }
+        });
+
+        this.socket.on('error', function (err: Error) {
+            logger.error('udp socket encounters with error: %j', err.stack);
+            return;
+        });
+
+        this.socket.bind(this.port, this.host);
+        this.tcpServer.listen(this.port);
+        process.nextTick(cb);
+    }
+
+    decode = coder.decode;
+
+    encode = coder.encode;
+
+    stop(force: boolean, cb: () => void) {
+        this.socket.close();
+        process.nextTick(cb);
+    }
+}
+
+let genKey = function (peer: dgram.RemoteInfo) {
+    return peer.address + ':' + peer.port;
+};

package/lib/connectors/udpsocket.ts

@@ -0,0 +1,110 @@
+import * as util from 'util';
+import { default as handler } from './common/handler';
+import { Package } from '@bigtyphoon/melo-protocol';
+import {EventEmitter} from 'events';
+import { getLogger } from '@bigtyphoon/melo-logger';
+import { ISocket } from '../interfaces/ISocket';
+import * as dgram from 'dgram';
+import * as path from 'path';
+let logger = getLogger('melo', path.basename(__filename));
+
+
+let ST_INITED = 0;
+let ST_WAIT_ACK = 1;
+let ST_WORKING = 2;
+let ST_CLOSED = 3;
+
+export class UdpSocket extends EventEmitter implements ISocket {
+    id: number;
+    socket: dgram.Socket;
+    peer: dgram.RemoteInfo;
+    host: string;
+    port: number;
+    remoteAddress: { ip: string; port: number };
+    state: number;
+
+    constructor(id: number, socket: dgram.Socket, peer: dgram.RemoteInfo) {
+        super();
+        this.id = id;
+        this.socket = socket;
+        this.peer = peer;
+        this.host = peer.address;
+        this.port = peer.port;
+        this.remoteAddress = {
+            ip: this.host,
+            port: this.port
+        };
+
+        let self = this;
+        this.on('package', function (pkg) {
+            if (!!pkg) {
+                pkg = Package.decode(pkg);
+                handler(self, pkg);
+            }
+        });
+
+        this.state = ST_INITED;
+    }
+
+
+    /**
+     * Send byte data package to client.
+     *
+     * @param  {Buffer} msg byte data
+     */
+    send(msg: any) {
+        if (this.state !== ST_WORKING) {
+            return;
+        }
+        if (msg instanceof String) {
+            msg = Buffer.from(msg as string);
+        } else if (!(msg instanceof Buffer)) {
+            msg = Buffer.from(JSON.stringify(msg));
+        }
+        this.sendRaw(Package.encode(Package.TYPE_DATA, msg));
+    }
+
+    sendRaw(msg: any) {
+        this.socket.send(msg, 0, msg.length, this.port, this.host, function (err, bytes) {
+            if (!!err) {
+                logger.error('send msg to remote with err: %j', err.stack);
+                return;
+            }
+        });
+    }
+
+    sendForce(msg: any) {
+        if (this.state === ST_CLOSED) {
+            return;
+        }
+        this.sendRaw(msg);
+    }
+
+    handshakeResponse(resp: any) {
+        if (this.state !== ST_INITED) {
+            return;
+        }
+        this.sendRaw(resp);
+        this.state = ST_WAIT_ACK;
+    }
+
+    sendBatch(msgs: any[]) {
+        if (this.state !== ST_WORKING) {
+            return;
+        }
+        let rs = [];
+        for (let i = 0; i < msgs.length; i++) {
+            let src = Package.encode(Package.TYPE_DATA, msgs[i]);
+            rs.push(src);
+        }
+        this.sendRaw(Buffer.concat(rs));
+    }
+
+    disconnect() {
+        if (this.state === ST_CLOSED) {
+            return;
+        }
+        this.state = ST_CLOSED;
+        this.emit('disconnect', 'the connection is disconnected.');
+    }
+}

package/lib/filters/handler/serial.ts

@@ -0,0 +1,46 @@
+/**
+ * Filter to keep request sequence.
+ */
+import {getLogger} from '@bigtyphoon/melo-logger';
+import * as taskManager from '../../common/manager/taskManager';
+import {RouteRecord} from '../../util/constants';
+import {IHandlerFilter} from '../../interfaces/IHandlerFilter';
+import {HandlerCallback} from '../../common/service/handlerService';
+import {FrontendOrBackendSession} from '../../server/server';
+import * as path from 'path';
+
+let logger = getLogger('melo', path.basename(__filename));
+
+
+export class SerialFilter implements IHandlerFilter {
+    constructor(private timeout: number, private timeOutResponse?: any) {
+    }
+
+    /**
+     * request serialization after filter
+     */
+    before(routeRecord: RouteRecord, msg: any, session: FrontendOrBackendSession, next: HandlerCallback) {
+        taskManager.addTask(session.id, function (task) {
+            (session as any).__serialTask__ = task;
+            next(null);
+        }, () => {
+            logger.error('[serial filter] msg timeout, msg:' + JSON.stringify(msg) + ' routeRecord:' + JSON.stringify(routeRecord));
+            if (this.timeOutResponse) {
+                next(new Error('msg timeout:' + session.id + ' uid:' + (session.uid ? session.uid : '')), this.timeOutResponse);
+            }
+        }, this.timeout);
+    }
+
+    /**
+     * request serialization after filter
+     */
+    after(err: Error, routeRecord: RouteRecord, msg: any, session: FrontendOrBackendSession, resp: any, next: HandlerCallback) {
+        let task = (session as any).__serialTask__;
+        if (task) {
+            if (!task.done() && !err) {
+                err = new Error('task time out. msg:' + JSON.stringify(msg));
+            }
+        }
+        next(err);
+    }
+}

package/lib/filters/handler/time.ts

@@ -0,0 +1,35 @@
+/**
+ * Filter for statistics.
+ * Record used time for each request.
+ */
+import {getLogger} from '@bigtyphoon/melo-logger';
+
+let conLogger = getLogger('con-log', __filename);
+import * as utils from '../../util/utils';
+import {IHandlerFilter} from '../../interfaces/IHandlerFilter';
+import {RouteRecord} from '../../util/constants';
+import {HandlerCallback} from '../../common/service/handlerService';
+import {FrontendOrBackendSession} from '../../server/server';
+
+
+export class TimeFilter implements IHandlerFilter {
+    before(routeRecord: RouteRecord, msg: any, session: FrontendOrBackendSession, next: HandlerCallback) {
+        (session as any).__startTime__ = Date.now();
+        next(null);
+    }
+
+    after(err: Error, routeRecord: RouteRecord, msg: any, session: FrontendOrBackendSession, resp: any, next: HandlerCallback) {
+        let start = (session as any).__startTime__;
+        if (typeof start === 'number') {
+            let timeUsed = Date.now() - start;
+            let log = {
+                route: routeRecord.route,
+                args: msg,
+                time: utils.format(new Date(start)),
+                timeUsed: timeUsed
+            };
+            conLogger.info(JSON.stringify(log));
+        }
+        next(err);
+    }
+}

package/lib/filters/handler/timeout.ts

@@ -0,0 +1,50 @@
+/**
+ * Filter for timeout.
+ * Print a warn information when request timeout.
+ */
+import {getLogger} from '@bigtyphoon/melo-logger';
+
+let logger = getLogger('melo', __filename);
+import * as  utils from '../../util/utils';
+import {IHandlerFilter} from '../../interfaces/IHandlerFilter';
+import {RouteRecord} from '../../util/constants';
+import {HandlerCallback} from '../../common/service/handlerService';
+import {FrontendOrBackendSession} from '../../server/server';
+
+let DEFAULT_TIMEOUT = 3000;
+let DEFAULT_SIZE = 500;
+
+export class TimeoutFilter implements IHandlerFilter {
+    timeouts: { [id: number]: NodeJS.Timeout } = {};
+    curId = 0;
+    private timeOutCount = 0;
+
+    constructor(private timeout = DEFAULT_TIMEOUT, private maxSize = DEFAULT_SIZE) {
+    }
+
+    before(routeRecord: RouteRecord, msg: any, session: FrontendOrBackendSession, next: HandlerCallback) {
+        if (this.timeOutCount > this.maxSize) {
+            logger.warn('timeout filter is out of range, current size is %s, max size is %s', this.timeOutCount, this.maxSize);
+            next(null);
+            return;
+        }
+        this.curId++;
+        this.timeOutCount++;
+        this.timeouts[this.curId] = setTimeout(function () {
+            logger.error('request %j timeout.', routeRecord ? routeRecord.route : routeRecord);
+        }, this.timeout);
+        (session as any).__timeout__ = this.curId;
+        next(null);
+    }
+
+    after(err: Error, routeRecord: RouteRecord, msg: any, session: FrontendOrBackendSession, resp: any, next: HandlerCallback) {
+        let timeout = this.timeouts[(session as any).__timeout__];
+        if (timeout) {
+            clearTimeout(timeout);
+            this.timeOutCount--;
+            this.timeouts[(session as any).__timeout__] = undefined;
+        }
+        next(err);
+    }
+
+}

package/lib/filters/handler/toobusy.ts

@@ -0,0 +1,37 @@
+/**
+ * Filter for toobusy.
+ * if the process is toobusy, just skip the new request
+ */
+import {getLogger} from '@bigtyphoon/melo-logger';
+import {IHandlerFilter} from '../../interfaces/IHandlerFilter';
+import {RouteRecord} from '../../util/constants';
+import {HandlerCallback} from '../../common/service/handlerService';
+import {FrontendOrBackendSession} from '../../server/server';
+
+let conLogger = getLogger('con-log', __filename);
+let toobusy: any = null;
+let DEFAULT_MAXLAG = 70;
+
+
+export class ToobusyFilter implements IHandlerFilter {
+    constructor(maxLag = DEFAULT_MAXLAG) {
+        try {
+            toobusy = require('toobusy');
+        } catch (e) {
+        }
+        if (!!toobusy) {
+            toobusy.maxLag(maxLag);
+        }
+    }
+
+    before(routeRecord: RouteRecord, msg: any, session: FrontendOrBackendSession, next: HandlerCallback) {
+        if (!!toobusy && toobusy()) {
+            conLogger.warn('[toobusy] reject request msg: ' + msg);
+            let err = new Error('Server toobusy!');
+            (err as any).code = 500;
+            next(err);
+        } else {
+            next(null);
+        }
+    }
+}

package/lib/filters/rpc/rpcLog.ts

@@ -0,0 +1,42 @@
+/**
+ * Filter for rpc log.
+ * Record used time for remote process call.
+ */
+import {getLogger} from '@bigtyphoon/melo-logger';
+
+let rpcLogger = getLogger('rpc-log', __filename);
+
+import * as utils from '../../util/utils';
+import {IRpcFilter} from '@bigtyphoon/melo-rpc';
+
+export class RpcLogFilter implements IRpcFilter {
+    name = 'rpcLog';
+
+    /**
+     * Before filter for rpc
+     */
+    before(serverId: string, msg: any, opts: any, next: (target?: Error | string, message?: any, options?: any) => void) {
+        opts = opts || {};
+        opts.__start_time__ = Date.now();
+        next();
+    }
+
+    /**
+     * After filter for rpc
+     */
+    after(serverId: string, msg: any, opts: any, next: (target?: Error | string, message?: any, options?: any) => void) {
+        if (!!opts && !!opts.__start_time__) {
+            let start = opts.__start_time__;
+            let end = Date.now();
+            let timeUsed = end - start;
+            let log = {
+                route: msg.service,
+                args: msg.args,
+                time: utils.format(new Date(start)),
+                timeUsed: timeUsed
+            };
+            rpcLogger.info(JSON.stringify(log));
+        }
+        next();
+    }
+}