gina 0.0.9-p91b → 0.1.1-alpha.2

package/framework/v0.1.1-alpha.2/lib/logger/src/containers/file/lib/logrotator/index.js

@@ -0,0 +1,274 @@
+const fs      = require('fs');
+const util    = require('util');
+const zlib    = require('zlib');
+const events  = require('events');
+
+/**
+ * Performs scheduled and on demand log rotation on files
+ */
+function Logrotator() {
+  events.EventEmitter.call(this);
+  this.timers = {};
+}
+
+util.inherits(Logrotator, events.EventEmitter);
+
+/**
+ * Schedules a file for rotation. emits a 'rotate' event whenever the file has been rotated.
+ * @param file full file path to rotate
+ * @param options rotation options
+ *  - schedule - how often to check for file rotation conditions. possible values are '1s', '1m', '1h'. default is 5m.
+ *  - size - size of the file to trigger rotation. possible values are '1k', '1m', '1g'. default is 10m.
+ *  - count - number of files to keep. default is 3.
+ *  - compress - whether to gzip rotated files. default is true.
+ *  - format - a function to build the name of a rotated file. the function receives the index of the rotated file.
+ *            default format is the index itself.
+ */
+Logrotator.prototype.register = function(file, options) {
+
+  options = util._extend({schedule: '5m'}, options);
+
+  var match = options.schedule.match(/^([0-9]+)(s|m|h)$/);
+  if (!match) {
+    this.emit('error', 'incorrect schedule format ' + options.schedule);
+    return;
+  }
+
+  if (this.timers[file]) {
+    this.unregister(file);
+  }
+
+  // calculate the schedule
+  var multi = this._timeMultiplier(match[2]);
+  var schedule = parseInt(match[1]) * multi;
+  var _this = this;
+
+  // perform rotation
+  function _doRotate() {
+    _this.rotate(file, options, function(err, rotated) {
+      if (err) {
+        _this.emit('error', err);
+        return;
+      }
+      if (rotated) {
+        _this.emit('rotate', file);
+      }
+    });
+  }
+
+  // register the rotation timer
+  this.timers[file] = setInterval(function() {
+    _doRotate();
+  }, schedule);
+
+  // immediately rotate
+  _doRotate();
+};
+
+/**
+ * Remove the scheduled rotation of a file
+ * @param file the file to stop rotating
+ */
+Logrotator.prototype.unregister = function(file) {
+  if (!this.timers[file]) {
+    return;
+  }
+
+  clearInterval(this.timers[file]);
+  delete this.timers[file];
+};
+
+/**
+ * Stop all schedulers
+ */
+Logrotator.prototype.stop = function() {
+  var _this = this;
+  Object.keys(this.timers).forEach(function(name) {
+    clearInterval(_this.timers[name]);
+  });
+  this.timers = {};
+};
+
+Logrotator.prototype._timeMultiplier = function(multi) {
+  switch (multi) {
+    case 's':
+      return 1000;
+    case 'm':
+      return 60*1000;
+    case 'h':
+      return 60*60*1000;
+  }
+};
+
+Logrotator.prototype._sizeMultiplier = function(multi) {
+  switch (multi) {
+    case 'k':
+      return 1024;
+    case 'm':
+      return 1024*1024;
+    case 'g':
+      return 1024*1024*1024;
+  }
+};
+
+/**
+ * Rotate a file now if size conditions are met.
+ * @param file full file path to rotate
+ * @param options rotation options
+ *  - size - size of the file to trigger rotation. possible values are '1k', '1m', '1g'. default is 10m.
+ *  - count - number of files to keep. default is 3.
+ *  - compress - gzip rotated files. default is true.
+ *  - format - a function to build the name of a rotated file. the function receives the index of the rotated file.
+ *            default format is the index itself.
+ * @param cb - invoked on completion, receives 'err' on error
+ */
+Logrotator.prototype.rotate = function(file, options, cb) {
+
+  if (!cb) {
+    cb = options;
+    options = null;
+  }
+
+  options = util._extend({size: '10m', count: 3, compress: true}, options);
+
+  var match = options.size.match(/^([0-9]+)(k|m|g)$/);
+  if (!match) {
+    cb('incorrect size format ' + options.size);
+    return;
+  }
+
+  var multi = this._sizeMultiplier(match[2]);
+  var size = parseInt(match[1]) * multi;
+
+  // check if the file reached the trigger size
+  var _this = this;
+  fs.stat(file, function(err, stats) {
+    if (err) {
+      var message = null;
+      // if file does not exist, ignore
+      if (err.code !== 'ENOENT') {
+        // other errors
+        message = file + ' stat failed: ' + err.message;
+      }
+      cb(message);
+      return;
+    }
+
+    // this isn't a file
+    if (!stats.isFile()) {
+      cb(file + ' is not a file');
+      return;
+    }
+
+    // check file size to see if rotation is needed
+    if (stats.size >= size) {
+      _this._rotate(file, options.count, options, cb);
+    } else {
+      cb(null, false);
+    }
+  });
+
+};
+
+/**
+ * Get the correct file name based on params
+ * @param file
+ * @param index
+ * @param options
+ * @private
+ */
+Logrotator.prototype._filename = function(file, index, options) {
+  var format = index;
+  if (typeof options.format === 'function') {
+    format = options.format(index);
+  }
+
+  var fileName = file + '.' + format;
+  if (options.compress) {
+    fileName += '.gz';
+  }
+  return fileName;
+};
+
+/**
+ * The log rotation brains
+ * @param file
+ * @param index
+ * @param options
+ * @param cb
+ * @private
+ */
+Logrotator.prototype._rotate = function(file, index, options, cb) {
+
+  // rotate all existing files
+  // 1. delete last file
+  // 2. rename all files to with +1
+  // 3. read + compress current log into 1
+  // 4. truncate file to size 0
+  var _this = this;
+  var fileName = this._filename(file, index, options);
+
+  // delete last file
+  if (index === options.count) {
+    fs.unlink(fileName, function(err) {
+      if (err && err.code !== 'ENOENT') {
+        cb('error deleting file ' + fileName + ': ' + err.message);
+        return;
+      }
+      _this._rotate(file, --index, options, cb);
+    });
+    return;
+  }
+
+  // rename all files to with +1
+  if (index > 0) {
+    var renameTo = this._filename(file, index+1, options);
+    fs.rename(fileName, renameTo, function(err) {
+      if (err && err.code !== 'ENOENT') {
+        cb('error renaming file ' + fileName + ': ' + err.message);
+        return;
+      }
+      _this._rotate(file, --index, options, cb);
+    });
+
+    return;
+  }
+
+  // read (and compress) the file log into index 1
+  var fis = fs.createReadStream(file);
+  var fos = fs.createWriteStream(this._filename(file, 1, options));
+  var pipe;
+  if (options.compress) {
+    pipe = fis.pipe(zlib.createGzip()).pipe(fos);
+  } else {
+    pipe = fis.pipe(fos);
+  }
+
+  var error;
+  pipe.on('finish', function() {
+    if (error) {
+      return;
+    }
+    // truncate log file to size 0
+    fs.truncate(file, 0, function(err) {
+      if (err) {
+        cb && cb('error truncating file ' + file + ': ' + err.message);
+        return;
+      }
+      cb && cb(null, true);
+    })
+  });
+  pipe.on('error', function(err) {
+    error = true;
+    cb('error compressing file ' + file + ': ' + err.message);
+    cb = null;
+  });
+};
+
+// create a new log rotator
+module.exports.create = function() {
+  return new Logrotator();
+};
+
+// global log rotator
+module.exports.rotator = new Logrotator();

package/framework/v0.1.1-alpha.2/lib/logger/src/containers/mq/index.js

@@ -0,0 +1,52 @@
+'use strict';
+// Imports
+const fs          = require('fs');
+const net         = require('net');
+const util        = require('util');
+const promisify   = util.promisify;
+const merge       = require(__dirname +'/../../../../merge');
+
+function MqContainer(opt, loggers) {
+    var self = {
+        // flow or container name/id
+        name: 'mq'
+    };
+
+    // TODO - get options like the `port` from: ~/.gina/user/extensions/logger/{container}/config.json
+
+    var MQSpeaker       = require('./speaker.js');
+    opt = merge(opt, { port: 8125 });
+    var mqSpeaker       = new MQSpeaker(opt, loggers);
+
+    function init() {
+        onPayload()
+
+        // ------------------------------------------------------------------------
+        var level = 'debug';
+        // Init debugging - Logs not in hierarchy will just be ignored
+        if (opt.hierarchies[opt.hierarchy].indexOf( opt.levels[level].code) > -1) {
+            process.emit('logger#'+self.name, JSON.stringify({
+                group       : opt.name,
+                level       : level,
+                // Raw content !
+                content     : '`'+ self.name +'` logger container loaded !'
+            }));
+        }
+        // ------------------------------------------------------------------------
+    }
+
+    function onPayload() {
+        process.on('logger#'+self.name, function onPayload(payload) {
+            if ( !loggers[opt.name]._options.isReporting ) {
+                return;
+            }
+            if (mqSpeaker.write) {
+                mqSpeaker.write( payload +'\r\n' );
+            }
+
+        });
+    }
+
+    init();
+}
+module.exports = MqContainer;

package/framework/v0.1.1-alpha.2/lib/logger/src/containers/mq/listener.js

@@ -0,0 +1,302 @@
+'use strict';
+// Imports
+const fs      = require('fs');
+// const util                = require('util');
+const net     = require('net');
+const uuid    = require('uuid');
+
+function MQListener(opt, cb) {
+    var self = {
+        name: 'MQListener'
+    };
+    var sessions        = {};
+    // tels to the listener when and to whom forward payloads
+    var forwardList     = {}; // for all flows but `speaker` (report)
+    // will be in memory until the framework is stopped
+    // TODO - remove specific `bundle` or `CLI` config when the process is terminated
+    var sharedConfig    = { loggers: {}};
+    var homedir         = getEnvVar('GINA_HOMEDIR');
+    var isLoggedByFile  = false; // by default
+    var fileLogsList    = {};
+
+
+    function init(opt, cb) {
+        setup();
+        return startMQListener(opt.port, cb);
+    }
+
+    function setup() {
+        var mainConfigPath  = _(homedir +'/user/extensions/logger/default/config.json', true);
+        if ( !new _(mainConfigPath, true).existsSync() ) {
+            //throw new Error('Required file not found: '+ mainConfigPath);
+            return;
+        }
+        var mainConfig = requireJSON(mainConfigPath);
+        if ( mainConfig.flows.indexOf('file') > -1 ) {
+            isLoggedByFile = true;
+        }
+    }
+
+    /**
+     * startLogRotator
+     *
+     * @param {string} name - `project name` or `hostname`
+     *
+     * @returns
+     */
+    function startLogRotator(name) {
+
+        if ( typeof(fileLogsList[name]) == 'undefined' ) {
+
+            fileLogsList[name] = {
+                started: false
+            }
+        }
+
+        if (fileLogsList[name].started) {
+            return;
+        }
+
+        //homedir
+        var file = name; // by default, it should be hostname
+        if ( !/^gina$/.test(name) && /\@/.test(name) ) {
+            name     = opt.name.split(/\@/)[1];
+            file = name;
+        }
+        file += '.log';
+        var filename = _(GINA_LOGDIR +'/'+ file);
+        var filenameObj = new _(filename);
+        if ( !filenameObj.existsSync() ) {
+            // create an empty file
+            try {
+                fs.openSync(filename, 'w')
+            } catch (fileErr) {
+                throw fileErr
+            }
+        }
+        // default options
+        var rotatorOptions = {
+            schedule: '5m',
+            size: '10m',
+            compress: true,
+            count: 3
+        };
+        var rotatorOptionsPath = _(homedir +'/user/extensions/logger/file/config.json', true);
+        if ( new _(rotatorOptionsPath).existsSync() ) {
+            try {
+                rotatorOptions = merge( requireJSON(rotatorOptionsPath).logrotator, rotatorOptions );
+            } catch (userConfigErr) {
+                throw userConfigErr
+            }
+        }
+
+
+        var rotator = require(__dirname + './../lib/logrotator').rotator;
+        // check file rotation every 5 minutes, and rotate the file if its size exceeds 10 mb.
+        // keep only 3 rotated files and compress (gzip) them.
+        rotator.register(filename, rotatorOptions);
+
+        rotator.on('error', function(err) {
+            console.error('[MQListener] log rotation failed: '+ err.stack);
+        });
+
+        // 'rotate' event is invoked whenever a registered file gets rotated
+        rotator.on('rotate', function(file) {
+            console.debug('[MQListener] file ' + file + ' was rotated!');
+        });
+
+        fileLogsList[name].started = true;
+    }
+
+    self.report = function(sessionId, payload) {
+        //process.stdout.write(  '[MQListener] sending MQSpeaker to `'+ sessionId +'` '+ JSON.stringify(payload) +'\n' );
+        sessions[sessionId].write( JSON.stringify(payload) +'\r\n');
+        //
+        // dispatching to onctainsers/flows
+        // var i = null , r = sessions[sessionId].request;
+        // if ( typeof(forwardList[r]) == 'undefined' ) {
+        //     forwardList[r] = []
+        // }
+        if ( forwardList && forwardList.count() > 0 ) {
+            for ( let r in forwardList) {
+                let i = -1;
+                while ( i < forwardList[r].length-1) {
+                    i++;
+                    let forwardSessionId = forwardList[r][i];
+                    // filter receipient flow
+                    if (forwardSessionId == sessionId) {
+                        continue;
+                    }
+                    if (sessions[forwardSessionId]) {
+                        if (payload.loggers) {
+                            self.respond(forwardSessionId, sharedConfig);
+                            continue;
+                        }
+                        self.respond(forwardSessionId, payload);
+                    }
+                }
+            }
+        }
+    }
+
+    // self.tail = function(sessionId, payload) {
+    //     //process.stdout.write(  '[MQListener] sending tail to `'+ sessionId +'` '+ JSON.stringify(payload) +'\n' );
+    //     sessions[sessionId].write( JSON.stringify(payload) +'\r\n')
+    // }
+
+    self.respond = function(sessionId, payload) {
+        //process.stdout.write(  '[MQListener] sending `'+ sessionId +'` '+ JSON.stringify(payload) +'\n' );
+        sessions[sessionId].write( JSON.stringify(payload) +'\r\n');
+    }
+
+    function startMQListener(port, cb) {
+        // AbortController is a global object
+        const controller = new AbortController();// jshint ignore:line
+        var server = net.createServer( function(conn) {//'connection' listener
+
+            conn.sessionId = uuid.v4();
+            // conn.request = 'report'; // by default
+            sessions[conn.sessionId] = conn;
+            conn.write(JSON.stringify({ sessionId: conn.sessionId }) +'\r\n' );
+
+            //feedback.
+            var forwardId = null;
+            conn.on('end', function() {
+                delete sessions[this.sessionId];
+                if ( this.request != 'report' ) {
+                    forwardId = forwardList[this.request].indexOf(this.sessionId);
+                    if ( forwardId > -1 ) {
+                        forwardList[this.request].splice(forwardId, 1);
+                    }
+                }
+                console.debug('[MQListener] (end) client disconected');
+                //process.stdout.write('[MQListener] (end) client disconected\n');
+                //conn.end();
+            });
+            // force exit
+            conn.on('exit', function() {
+                delete sessions[this.sessionId];
+                if ( this.request != 'report' ) {
+                    forwardId = forwardList[this.request].indexOf(this.sessionId);
+                    if ( forwardId > -1 ) {
+                        forwardList[this.request].splice(forwardId, 1);
+                    }
+                }
+
+                console.info('[MQListener] (exit) client forced to exit');
+                //process.stdout.write('[MQListener] (exit) client forced to exit\n');
+                //conn.end();
+            });
+
+            //Receiving.
+            var payloads = null, i = null;
+            // do not use console to send data to clients
+            conn.on('data', function(data) {
+                payloads = data.toString();
+
+                // filter payloads
+                if ( /^(\{\"|\[\{\")/.test(payloads) ) {
+                    payloads = payloads.split(/\r\n/g);
+                    //console.log(payloads);
+                    i = -1;
+                    while(i < payloads.length) {
+                        i++;
+                        let payload = payloads[i];
+                        if ( /^\{/.test(payload) && /\}$/.test(payload)) {
+                            let pl = JSON.parse(payload);
+
+
+                            // mostly after client that acknowledeged sessionId
+                            if (!pl.content) {
+                                process.stdout.write(  '[MQListener] (undefined content) '+ JSON.stringify(pl, null) +'\n' );
+                            }
+
+                            if (pl.loggers) {
+                                sharedConfig.loggers = merge(sharedConfig.loggers, pl.loggers);// jshint ignore:line
+                            }
+
+                            if (pl.request && !this.request) {
+                                this.request = pl.request;
+                                // forward to all but `speakers`
+                                if ( this.request != 'report' ) {
+                                    if ( typeof(forwardList[this.request]) == 'undefined' ) {
+                                        forwardList[this.request] = []
+                                    }
+                                    if ( forwardList[this.request].indexOf(this.sessionId) < 0 ) {
+                                        forwardList[this.request].push(this.sessionId)
+                                    }
+                                }
+                            }
+                            else if (!this.request) {
+                                // by default
+                                this.request = 'report';
+                            }
+
+
+
+                            if ( this.request && this.sessionId ) {
+
+                                if ( typeof(self[this.request]) != 'undefined' ) {
+                                    self[this.request](this.sessionId, pl);
+                                    continue;
+                                }
+                                self.respond(this.sessionId, pl);
+                                continue;
+                            }
+
+                            // uncatched sessionId - not supposed to get there
+                            process.stdout.write(  '[MQListener] '+ pl.content +'\n' );
+                        }
+                    }
+
+                    return
+                }
+
+                // regular messages
+                process.stdout.write(  '[MQListener] '+ payloads +'\n'  );
+            });
+
+            conn.on('connect', function(data) {
+                console.info( '[MQListener] connect received "'+ data.toString()  +'"' );
+            });
+        });
+
+
+        server.on('error', function(err) {
+            if (err.code === 'EADDRINUSE') {
+                console.warn('[MQListener] is already running on port [ '+ (err.port || port) +' ]');
+            } else {
+                console.emerg('[MQListener] Could not start', err.stack)
+            }
+
+            if (cb) {
+                cb(err);
+            }
+        });
+
+
+        server.on('close', function() {
+            for ( let i = 0, len = sessions.length; i < len; i++) {
+                //sessions[i].close();
+                sessions[i].destroy()
+            }
+        });
+
+        server.listen({
+            host: '127.0.0.1',
+            port: port,
+            signal: controller.signal
+          }, function() {
+            // Server started.
+            console.info('[MQListener] is waitting for speakers on port `'+ port +'`');
+            if (cb) {
+                cb(false);
+            }
+        });
+
+        return server;
+    }
+
+    return init(opt, cb);
+}
+module.exports = MQListener;