RTALogger 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5cd821ed8beebbda5bfd6aff1088372642f5f20012c7f5d6ca8b2d59cc8dee42
-  data.tar.gz: 8dbf6b01f03b98525acade15eaebece1e500dfbc610f2d9cbda5e760df0e048b
+  metadata.gz: 9d28795fc1ab81da8fe17aea8489d11559cd05d06caa7545a30ac5abcda1e5eb
+  data.tar.gz: 5bd61d89f1d4ec087294845f742c4799c3899590adae9366ab013f67a30924cf
 SHA512:
-  metadata.gz: 7a1e77c33e3fb301780bb6d4a0b254701ab918403ee8705e19bda759b2c0ce56b1e1cf76c18012c195b5c3b28267dea524bbe3db303f7a9fb9dca58662fe9a2f
-  data.tar.gz: fd81ab07bb1a3c79e00e425298068e4cfa357208e4173f157a156e5cdefcb044f456604cfc0c3b9d0f37342e672d47636ffd75ceb4b5ef866a47f8f7db9bc7aa
+  metadata.gz: a0264ccb95941af08eb9ea0be0019a03e7151e7f5eb8a570fec66c421ebc8f031df7e3ac712aec515c603c66996d1c6976d95d1e4a1326be20c4946fb9ebcb22
+  data.tar.gz: 62ad72d8e5443de241cff8c9c8fc30c34006dd8fbae052f5388a275ecfea104b42a6225f5eb358df774a1811c7c64a4b0527b86120a6e9e505f8be0e09a2523c

data/.idea/RTALogger.iml

@@ -10,10 +10,8 @@
     <orderEntry type="library" scope="PROVIDED" name="activesupport (v6.0.3.4, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="bundler (v2.1.4, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="concurrent-ruby (v1.1.7, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="cool.io (v1.7.0, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.4.4, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="fluentd (v1.11.4, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="http_parser.rb (v0.6.0, ruby-2.6.3-p62) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="fluent-logger (v0.9.0, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="i18n (v1.8.5, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="jbuilder (v2.10.1, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="minitest (v5.14.2, ruby-2.6.3-p62) [gem]" level="application" />
@@ -24,13 +22,8 @@
     <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.10.0, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.10.0, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.10.0, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="serverengine (v2.2.2, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="sigdump (v0.2.4, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="strptime (v0.2.5, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="thread_safe (v0.3.6, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="tzinfo (v1.2.7, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="tzinfo-data (v1.2020.4, ruby-2.6.3-p62) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="yajl-ruby (v1.4.1, ruby-2.6.3-p62) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="zeitwerk (v2.4.1, ruby-2.6.3-p62) [gem]" level="application" />
   </component>
 </module>

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    RTALogger (0.1.1)
-      fluentd (~> 1.11, >= 1.11.4)
+    RTALogger (0.1.2)
+      fluent-logger (~> 0.9)
       jbuilder (~> 2.10)
 
 GEM
@@ -15,19 +15,9 @@ GEM
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
     concurrent-ruby (1.1.7)
-    cool.io (1.7.0)
     diff-lcs (1.4.4)
-    fluentd (1.11.4)
-      cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (>= 1.0, < 3.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
+    fluent-logger (0.9.0)
+      msgpack (>= 1.0.0, < 2)
     i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     jbuilder (2.10.1)
@@ -48,16 +38,9 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.0)
-    serverengine (2.2.2)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.5)
     thread_safe (0.3.6)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2020.4)
-      tzinfo (>= 1.0.0)
-    yajl-ruby (1.4.1)
     zeitwerk (2.4.1)
 
 PLATFORMS

data/README.md

@@ -1,8 +1,28 @@
 # RTALogger
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/RTALogger`. To experiment with that code, run `bin/console` for an interactive prompt.
+RTA Log Manager has been designed and implemented to provide standard logging API for developers.
+This prevents chaos in log data format. 
+Also provide multiple extendable log repositories including wrapping existing loggers, like 'Fluentd' or implement completely new custom logger. 
+All log manager's main features are configable through a json config file.
 
-TODO: Delete this and the text above, and describe your gem
+Main purposes of developing this gem are:
+- Creating easy to use logger interface.
+- Apply some rules and restrictions about log structure and data format, which prevents chaos in application log information.
+- No interrupt or wait time for log consumer modules.
+- Utilize multiple log repositories at the same time in background (Console, File, UDP, FluentD, etc.) 
+- Make it possible to implement customized log repositories.
+
+Main Features:
+- Creating multiple log manager instances with different configuration is possible entire application.
+- Each log manager instance could be configured via a json file.
+- Each log manager instance could be config to use multiple log repositories such as Console, File, UDP, Fluentd.
+- Runtime configurations could be applied through log manager APIs.
+- By using multi threading techniques and also buffering techniques, 
+  all logging process will handled in seperated thread.
+  So the log consumer modules should not be wait for log manager to finish the logging task.
+- Multiple standard log severity levels are available through topic APIs (debug, info, warning, error, fatal, unknown)
+- Main features could be set and manipulate through json configuration file.
+- And at the end, it is easy to use for ruby backend developers. 
 
 ## Installation
 
@@ -20,45 +40,213 @@ Or install it yourself as:
 
     $ gem install RTALogger
 
+To add gem to your rails application:
+   
+     $ bundle add RTALogger
+     
 ## Usage
+#### RTA Log Data Structure
+To use log manager APIs, first step is to have a quick review on Log Data Structure
+- Application: The root of each log data record is Application, which specify the log data owner application.
+- Topic: By adding multiple topics to log manager you can categorize log data in logical topics.
+- Context: Under each topic, one or multiple contexts (in one level) could be defined. 
+- As an instance the Application could by 'MyEShopApp', one of Topics could be 'Authentication' and 
+  Contexts could be 'uer_name' which attend in application authorization process.
+- The next step is log severity level, which determines that the log record severity (debug, information, warning, error, fatal, unknown)
+- At last the final element is log message, which contains log message data.
 
-require_relative 'log_factory_manager'
-require_relative 'log_factory_repository'
-
-controller_name = 'test_controller'
-userID = 5
-
-# create log manager instance
-# this could be a global variable declared in application level
-log_manager = RTALogger::LogFactory.log_manager_instance
+### Which Log Severity Levels to use
+- DEBUG = 0 : Low-level information, mostly for developers.
+- INFO = 1 : Generic (useful) information about system operation.
+- WARN = 2 : A warning, which it does NOT cause crashing the process.
+- ERROR = 3 : A handleable error condition.
+- FATAL = 4 : An un-handleable error that results in a program crash.
+- UNKNOWN = 5 : An unknown message that should always be logged.
+    
+### Time for coding
+- create log manager instance:
+```ruby
+    # add required files
+    require 'log_factory_manager'
 
-# set log manage application name (hard code)
-log_manager.app_name = 'myTestApp'
+    # create log manager instance using LogFactory
+    log_manager = RTALogger::LogFactory.log_manager_instance
+```
+- Apply configuration using json config file
+```ruby
+    # the parameter is the json config file
+    log_manager.config_use_json_file('rta_logger_config.json')
+```
+- Add new topic to log manager and get the topic instance
+```ruby
+    # the parameter is the topic name
+    # if add_topic API called multiple times with same parameter,
+    # only one instance will be created for that topic 
+    topic = log_manager.add_topic('Authentication')
+```
+- Finally add log message using topic instance
+```ruby
+    # Assume user 'Tom' is trying to authenticate we will use user_name as log Context_id
+    user_name = 'Tom'
+    topic = log_manager.add_topic('Authentication')
+    topic.debug(user_name, 'use_id is nil for user:', user_name)
+    topic.info(user_name, 'User ', user_name , ' is trying to login.')
+    topic.warning(user_name, 'Authentication failed for user ', user_name)
+    topic.error(user_name, 'Error connecting to data base for user ', user_name)
+    topic.fatal(user_name, 'Authentication service has been stopped working')
+    topic.unknown(user_name, 'An unknown error occured during authentication. user name:', user_name)
+```
+the result will be:
+```
+    {"occurred_at":"2020-11-04 15:56:58:785","app_name":"TestApp","topic_title":"Authentication","context_id":"Tom","severity":0,"message":"user_id is nil for user: Tom"}
+    {"occurred_at":"2020-11-04 15:56:58:785","app_name":"TestApp","topic_title":"Authentication","context_id":"Tom","severity":1,"message":"User Tom is trying to login"}
+    {"occurred_at":"2020-11-04 15:56:58:785","app_name":"TestApp","topic_title":"Authentication","context_id":"Tom","severity":2,"message":"Authentication failed for user Tom"}
+    {"occurred_at":"2020-11-04 15:56:58:785","app_name":"TestApp","topic_title":"Authentication","context_id":"Tom","severity":3,"message":"Error connecting to data base for user Tom"}
+    {"occurred_at":"2020-11-04 15:56:58:785","app_name":"TestApp","topic_title":"Authentication","context_id":"Tom","severity":4,"message":"Authentication service has been stopped working"}
+    {"occurred_at":"2020-11-04 15:56:58:785","app_name":"TestApp","topic_title":"Authentication","context_id":"Tom","severity":5,"message":"An unknown error occured during authentication. user name: Tom"}
+```
+- json config file sample
+```json
+{
+  "RTALogger":
+  {
+    "Default_Manager": "Develop",
+    "Log_Managers":
+    [
+      {
+        "Manager_Name": "Develop",
+        "Enable": true,
+        "App_Name": "TestApp",
+        "Log_Severity": 2,
+        "Buffer_Size": 100,
+        "Flush_Wait_Seconds": 15,
+        "Formatter" : "JSON",
+        "Repos":
+        [
+          {
+            "Enable": true,
+            "Type": "Console"
+          },
+          {
+            "Enable": true,
+            "Type": "UDP",
+            "Host": "localhost",
+            "Port": 8888
+          },
+          {
+            "Enable": false,
+            "Type": "File",
+            "File_Path": "../../log/log.txt",
+            "Roll_Period": "daily",
+            "Roll_Size": "1048576"
+          },
+          {
+            "Enable": true,
+            "Type": "Fluentd",
+            "Host": "localhost",
+            "Port": "24442",
+            "TLS_Options": 
+            {
+              "ca":",/path/to/cacert.pem",
+              "cert":"/path/to/client-cert.pem",
+              "key":"/path/to/client-key.pem",
+              "key_passphrase":"test"
+             }
+          }
+        ]
+      }
+    ]
+  }
+}
+```
+- json config file structure 
+```comment
+  As we described you cap apply RTA log manager using a json config file.
+  
+  log_manager.config_use_json_file('rta_logger_config.json')
+     
+  The file structure:
+  - RTALogger : the root element of RTALogger json configuration.
+    - Default_Manager: the name of default log manager config, when there is 
+      multiple log manager configuration in Log_Managers array.
+    - Log_Managers : the array of LogManagers with different configuration.
+      It is possible to define multiple log manager configurations for differen usages.
+      - Name: the name of log manager. It will be used to define the default log manager.
+      - Enable: (true/false) The value of this property activate or deactivate entire log manager.
+      - App_Name: Application name as the owner of log data.
+      - Log_Severity: Defines which level of log data will be stored in log repositories.
+      - BufferSize: The memory buffer size (number of buffered log objects) to 
+        decread api consumers wait time. when the buffer is full the flush operation will
+        save buffered logs to log repositoies.
+      - Flush_Wait_Seconds: Time in soconds which log managers wait to flush buffered log objects
+        to log repository.
+      - Formatter: (JSON/TEXT) declare log format when it's required to converrt log object to text.
+      - Repos: Array of log repositories. It is possible to define multiple log repositories to
+        store log data. there are variaty of log repositories and it is possible to
+        add new ones. Each item in Repos array will configure a log repository.
+        - Log repository types and config:
+          1- Console: Show log data in text format on standard out put
+             - "Type":"Console"
+             - "Enable": [true/false] this will activate or deactivate log repository.
+          2- File: Store log data in a file.
+             - "Type":"Console"
+             - "Enable": [true/false] this will activate or deactivate log repository.
+             - "File_Path": [file path and file name] the path and the name to store log data.
+             - "Roll_Period": ["daily"/"weekly"/"monthly"] the period to generate new log file.
+             - "Roll_Size": [bytes]  the maximum size of log file to 
+                roll file and create the new log file
+          3- UDP: Send log data over UDP on network. 
+             - "Type":"Console"
+             - "Enable": [true/false] this will activate or deactivate log repository.
+             - "Host": IP of the server to send log data.
+             - "Port": Port of server to send log data.
+          4- Fluentd: send log data to Fluentd Log collector over network using TCP/IP protocol.
+             - "Type":"Console"
+             - "Enable": [true/false] this will activate or deactivate log repository.
+             - "Host": IP of the server to send log data.
+             - "Port": Port of server to send log data.
+             - "TLS_Options": TLS configuration to stablish a secure TCP connection to Fluentd Server.
+ 
+```
+- Some useful features
+```ruby
+    # change log manager app name at run time
+    log_manager.app_name = 'myTestApp'
 
-# load log manager configuration from a json config file
-log_manager.config_use_json_file('rta_logger_config.json')
+    # update specific topic log level if necessary
+    log_manager.update_topic_level(controller_name, RTALogger::LogSeverity::INFO)
 
-# add log repository to log manager
-#log_manager.propagator.add_log_repository(RTALogger::LogFactory.new_log_repository_console)
+    # update all topics log level if necessary
+    log_manager.update_all_topics_log_level(RTALogger::LogSeverity::INFO)
+```
+- Implement and Expand
+  It is possible to implement new log repositories.
+  All repository classes should inherit from 'RTALogger::LogRepository'
+  Here is 'LogRepositoryConsole' implementation:
+```ruby
+require_relative 'log_repository'
+require_relative 'log_factory_log_formatter'
 
-# add new topic to log manager
-# use this api to get a new log topic instance
-# this api could be called in entry point of each service or class initialize method
-topic = log_manager.add_topic(controller_name)
+module RTALogger
+  # show log items on console out put
+  class LogRepositoryConsole < LogRepository
+    def initialize
+      super
 
-# add log information to log topic
-topic.debug(userID, 'Controller Name=', controller_name, 'debug')
-topic.info(userID, 'Controller Name=', controller_name, 'information')
-topic.warning(userID, 'Controller Name=', controller_name, 'warning')
-topic.error(userID, 'Controller Name=', controller_name, 'error')
-topic.fatal(userID, 'Controller Name=', controller_name, 'fatal')
-topic.unknown(userID, 'Controller Name=', controller_name, 'unknown')
+      @formatter = RTALogger::LogFactory.log_formatter_default
+    end
 
-# update specific topic log level if necessary
-# log_manager.update_topic_level(controller_name, RTALogger::LogSeverity::INFO)
+    protected
 
-# update all topics log level if necessary
-# log_manager.update_all_topics_log_level(RTALogger::LogSeverity::INFO)
+    def flush_and_clear
+      semaphore.synchronize do
+        @log_records.each { |log_record| puts @formatter.format(log_record) }
+      end
+      super
+    end
+  end
+end
+```
 
 ## Development
 
@@ -68,7 +256,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/RTALogger.
+Bug reports and pull requests are welcome on GitHub at https://github.com/BBahrainy/RTALogger.
 
 
 ## License

data/RTALogger.gemspec

@@ -27,6 +27,6 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.add_dependency "fluentd", "~> 1.11", ">= 1.11.4"
+  spec.add_dependency "fluent-logger", "~> 0.9"
   spec.add_dependency "jbuilder", "~> 2.10"
 end

data/lib/RTALogger/version.rb

@@ -1,3 +1,3 @@
 module RTALogger
-  VERSION = '0.1.1'.freeze
+  VERSION = '0.1.2'.freeze
 end

data/lib/log_factory_log_formatter.rb

@@ -8,5 +8,13 @@ module RTALogger
       RTALogger::LogFormatterJSON.new
       # RTALogger::LogFormatterText.new
     end
+
+    def self.log_formatter_json
+      RTALogger::LogFormatterJSON.new
+    end
+
+    def self.log_formatter_text
+      RTALogger::LogFormatterText.new
+    end
   end
 end

data/lib/log_factory_repository.rb

@@ -3,6 +3,7 @@
 require_relative 'log_repository_console'
 require_relative 'log_repository_file'
 require_relative 'log_repository_udp'
+require_relative 'log_repository_fluent'
 
 module RTALogger
   # this module generates object instance
@@ -32,11 +33,23 @@ module RTALogger
       ::RTALogger::LogFactory.new_log_repository_udp(host, port)
     end
 
+    def self.new_log_repository_fluent(host = 'localhost', port = '24224', tls_options = nil)
+      LogRepositoryFluent.new(host, port, tls_options)
+    end
+
+    def self.load_log_repository_fluent(config_json)
+      host = config_json['Host'].to_s
+      port = config_json['Port'].to_s
+      tls_options = config_json['TLS_Options']
+      ::RTALogger::LogFactory.new_log_repository_fluent(host, port, tls_options)
+    end
+
     def self.create_repository(type, config_json)
       result = nil
       result = new_log_repository_console if type.to_s.casecmp('Console').zero?
       result = load_log_repository_file(config_json) if type.to_s.casecmp('File').zero?
       result = load_log_repository_udp(config_json) if type.to_s.casecmp('UDP').zero?
+      result = load_log_repository_fluent(config_json) if type.to_s.casecmp('Fluentd').zero?
       result
     end
   end

data/lib/log_repository_fluent.rb

@@ -0,0 +1,32 @@
+require 'fluent-logger'
+require_relative 'log_repository'
+
+module RTALogger
+  class LogRepositoryFluent < LogRepository
+    def initialize(host = 'localhost', port = 24224, tls_options = nil)
+      super()
+      @host = host
+      @port = port
+      @tls_options = tls_options
+      @formatter = RTALogger::LogFactory.log_formatter_json
+      if @tls_options
+        @fluent_logger = ::Fluent::Logger::FluentLogger.new(nil, :host => @host, :port => @port, :use_nonblock => true, :wait_writeable => false)
+      else
+        @fluent_logger = ::Fluent::Logger::FluentLogger.new(nil, :host => @host, :port => @port, :tls_options => @tls_options, :use_nonblock => true, :wait_writeable => false)
+      end
+    end
+
+    def flush_and_clear
+      semaphore.synchronize do
+        @log_records.each do |log_record|
+          fluent_tag = log_record.app_name + '.' + log_record.topic_title
+          log_json_string = @formatter.format(log_record)
+          log_json = JSON.parse(log_json_string)
+
+          @fluent_logger.post(fluent_tag, log_json)
+        end
+      end
+      super
+    end
+  end
+end

data/lib/rta_logger_config.json

@@ -24,6 +24,12 @@
             "File_Path": "../../log/log.txt",
             "Roll_Period": "daily",
             "Roll_Size": "1048576"
+          },
+          {
+            "Enable": true,
+            "Type": "Fluentd",
+            "Host": "localhost",
+            "Port": "8888"
           }
         ]
       }

metadata

@@ -1,35 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: RTALogger
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Babak Bahreini, RTA Backend Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-04 00:00:00.000000000 Z
+date: 2020-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: fluent-logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.11.4
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.11.4
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: jbuilder
   requirement: !ruby/object:Gem::Requirement
@@ -90,6 +84,7 @@ files:
 - lib/log_repository.rb
 - lib/log_repository_console.rb
 - lib/log_repository_file.rb
+- lib/log_repository_fluent.rb
 - lib/log_repository_udp.rb
 - lib/log_severity.rb
 - lib/log_topic.rb