coralogix_fluentd_logger 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ca8727eb2b5cc0c4ea7af0067cb75ce493168c87
+  data.tar.gz: 99d0fb52a2e360ba8e077c17f3630b54e96a5603
+SHA512:
+  metadata.gz: 4681ea0e1fbe83cf63ff7c005281712f718987963674ec1eee59efe86fc1792f885a698756f390167b1e1eabd6a4ec3c55e86f4f28bf7b83f47033a5ea1241fe
+  data.tar.gz: f0b5ea728df44ca089868e5ae9c70bf3ac4d1e07eac7873be356670d5f23d98a5798dc71ca20c122129adc4c79e63f3dd3c33349f82f59089eacd0b659a0520f

data/LICENCE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012-2016 Coralogix, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+
+# Croalogix SDK - Ruby Implementation
+This is an implementation of Coralogix Ruby SDK for Fluentd.

data/lib/constants.rb

@@ -0,0 +1,62 @@
+module Coralogix
+
+    #Maximum log buffer size
+    MAX_LOG_BUFFER_SIZE = 12582912 #12mb
+
+    #Maximum chunk size
+    MAX_LOG_CHUNK_SIZE = 1572864 #1.5 mb
+
+    #Bulk send interval in normal mode.
+    NORMAL_SEND_SPEED_INTERVAL = 500.0 / 1000
+
+    #Bulk send interval in fast mode.
+    FAST_SEND_SPEED_INTERVAL = 100.0 / 1000
+
+    #Corologix severity mapper
+    SEVERITIES = {:debug => 1, :verbose => 2, :info => 3, :warning => 4, :error => 5, :critical => 6}
+
+
+    module Severity
+        DEBUG = 1
+        VERBOSE = 2
+        INFO = 3
+        WARNING = 4
+        ERROR = 5
+        CRITICAL = 6
+    end
+
+
+    #Coralogix logs url
+    CORALOGIX_LOG_URL = "https://api.coralogix.com:443/api/v1/logs"
+
+    #Coralogix time delat url
+    CORALOGIX_TIME_DELTA_URL = "https://api.coralogix.com:443/sdk/v1/time"
+
+    #Default private key
+    FAILED_PRIVATE_KEY = "9626c7dd-8174-5015-a3fe-5572e042b6d9"
+
+    #Default application name
+    NO_APP_NAME = "NO_APP_NAME"  
+
+    #Default subsystem name
+    NO_SUB_SYSTEM = "NO_SUB_NAME"
+
+    #Default log file name
+    LOG_FILE_NAME = "coralogix.sdk.log"
+
+    #Default http timeout
+    HTTP_TIMEOUT = 30
+
+    #Number of attempts to retry http post
+    HTTP_SEND_RETRY_COUNT = 5
+
+    #Interval between failed http post requests
+    HTTP_SEND_RETRY_INTERVAL = 2
+
+    # Coralogix category
+    CORALOGIX_CATEGORY = 'CORALOGIX'
+
+    # Sync time update interval
+    SYNC_TIME_UPDATE_INTERVAL = 5 #minutes
+
+end

data/lib/coralogix_logger.rb

@@ -0,0 +1,86 @@
+require_relative 'manager'
+require_relative 'debug_logger'
+require_relative 'httpsender'
+require_relative 'constants'
+
+module Coralogix
+    class CoralogixLogger
+        # Constructor. 
+        #
+        # @param name    -   logger name.
+        def initialize private_key, app_name, sub_system_name, debug: false
+            self.debug_mode=debug            
+            @category = CORALOGIX_CATEGORY
+            @manager = LoggerManager.new
+            configure private_key, app_name, sub_system_name
+        end
+
+        # A getter for debug_mode. 
+        # Default value is false.
+        # When set to true the coralogix logger will print output messages to a console and a file.
+        #
+        # @return [boolean]    -   true or false. (Default is false)
+        def debug_mode?
+            DebugLogger.debug_mode
+        end
+
+        # A setter for debug_mode.  
+        # Default value is false.
+        # When set to true the coralogix logger will print output messages to a console and a file.
+        #
+        # @param value    -   true or false. (Default is false)
+        def debug_mode=(value)
+            DebugLogger.debug_mode=value
+        end
+
+        # A setter for disable_proxy. 
+        # By default HTTP object will use proxy environment variable if exists. In some cases this migh be an issue
+        # When set to false the HTTP object will ignore any proxy.
+        #
+        # @param value    -   true or false. (Default is false)
+        def disable_proxy=(value)
+            @manager.disable_proxy=value
+        end
+
+        # Configure coralogix logger with customer specific values
+        #
+        # @param private_key    -   private key
+        # @param app_name       -   application name
+        # @param sub_system     -   sub system name
+        # @return [boolean] return a true or false.
+        def configure private_key, app_name, sub_system
+            private_key = (private_key.nil? || private_key.to_s.strip.empty?) ? FAILED_PRIVATE_KEY : private_key
+            app_name = (app_name.nil? || app_name.to_s.strip.empty?) ? NO_APP_NAME : app_name
+            sub_system = (sub_system.nil? || sub_system.to_s.strip.empty?) ? NO_SUB_SYSTEM : sub_system
+            @manager.configure(:privateKey => private_key, :applicationName => app_name, :subsystemName => sub_system) unless @manager.configured
+        end
+
+
+        # Flush all messages in buffer and send them immediately on the current thread.
+        def flush
+            @manager.flush
+        end
+
+        # Log a message. 
+        #
+        # @param severity    -   log severity
+        # @param message     -   log message
+        # @param category    -   log category
+        # @param className   -   log class name
+        # @param methodName  -   log method name
+        # @param threadId    -   log thread id
+        def log severity, message, category: @category, className: "", methodName: "", threadId: Thread.current.object_id.to_s
+            @manager.add_logline message, severity, category, :className => className, :methodName => methodName, :threadId => threadId
+        end 
+
+        # Create log methods for each severity. 
+        # This is a ruby thing. If you are writing in other languages just create a method for each severity.
+        # For instance, for info severity it will create a method:
+        # def info message, category: @category, className: "", methodName: "", threadId: ""
+        SEVERITIES.keys.each do |severity|
+            define_method("#{severity}") do |message, category: @category, className: "", methodName: "", threadId: Thread.current.object_id.to_s|
+                @manager.add_logline message, SEVERITIES["#{__method__}".to_sym], category, :className => className, :methodName => methodName, :threadId => threadId
+            end  
+        end
+    end
+end

data/lib/debug_logger.rb

@@ -0,0 +1,61 @@
+require 'logger'
+
+module Coralogix
+    # @private
+    class DebugLogger
+
+        def self.initialize
+            begin
+                @mutex = Mutex.new
+                @debug = false
+            rescue Exception => e
+                if @debug  
+                    puts e.message  
+                    puts e.backtrace.inspect  
+                end
+            end
+        end 
+
+        def self.debug_mode?
+            @debug
+        end
+
+        def self.debug_mode=(value)
+            begin
+                @debug = value
+                if value
+                    @logger = Logger.new(LOG_FILE_NAME, 1, 10485760)
+                else
+                    @logger.close unless @logger == nil
+                    @logger = nil
+                end
+            rescue Exception => e
+                if @debug  
+                    puts e.message  
+                    puts e.backtrace.inspect  
+                end
+            end
+        end
+
+        Logger::Severity.constants.each do |level|
+            define_singleton_method("#{level.downcase}") do |*args|
+                if @debug
+                    @mutex.synchronize do
+                        begin
+                            puts "#{__method__.upcase}:  #{Time.now.strftime('%H:%M:%S.%L')} - #{args}"
+                            @logger.send("#{__method__}", args)
+                        rescue Exception => e
+                            puts e.message  
+                            puts e.backtrace.inspect  
+                        end
+                    end
+                end
+            end  
+        end
+
+
+        initialize
+    
+    end
+
+end

data/lib/httpsender.rb

@@ -0,0 +1,100 @@
+require 'net/http'
+require 'json'
+require_relative 'constants'
+require_relative 'debug_logger'
+require 'openssl'
+require 'benchmark'
+require 'date'
+require 'time'
+module Coralogix
+    # @private
+    class CoralogixHTTPSender
+        TICKS_IN_SECOND = 10**7
+        def disable_proxy value
+            @disable_proxy = value
+        end
+
+        def disable_proxy=(value)
+            @disable_proxy = value
+        end
+
+        def initialize
+            begin
+                @initialized = false
+                @mutex = Mutex.new
+                @uri = URI(CORALOGIX_LOG_URL)
+                if(@disable_proxy)
+                    @http = Net::HTTP.new(@uri.host, @uri.port, p_addr=nil, p_port=nil)
+                else
+                    @http = Net::HTTP.new(@uri.host, @uri.port)
+                end
+                @http.use_ssl = true
+                @http.keep_alive_timeout = 10
+                @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+                @http.read_timeout = HTTP_TIMEOUT # seconds
+                @http.open_timeout = HTTP_TIMEOUT # seconds
+                @req = Net::HTTP::Post.new(@uri.path, 'Content-Type' => 'application/json')
+            rescue Exception => e  
+                DebugLogger.error e.message  
+                DebugLogger.error e.backtrace.inspect  
+            end
+        end
+
+        # A helper method to post http request
+        #
+        # @param bulk    -   JSON bulk containing the log entries
+        def send_request bulk
+            @mutex.synchronize do
+                attempt = 0
+                while attempt < HTTP_SEND_RETRY_COUNT
+                    begin
+                        DebugLogger.info "About to send bulk to Coralogix server. Attempt number: #{attempt+1}"  
+                        @req.body = bulk.to_json
+                        DebugLogger.debug Benchmark.measure { 
+                            res = @http.request(@req)
+                            DebugLogger.info "Successfully sent bulk to Coralogix server. Result is: #{res.code}"
+                        }.to_s
+                        return true
+                    rescue Exception => e  
+                        DebugLogger.error e.message
+                        DebugLogger.error e.backtrace.inspect  
+                    end
+                    attempt+=1;
+                    DebugLogger.error "Failed to send bulk. Will retry in: #{HTTP_SEND_RETRY_INTERVAL} seconds..."
+                    sleep HTTP_SEND_RETRY_INTERVAL
+                end
+            end
+        end
+
+        # A helper method to get coralogix server current time and calculate the time difference
+        #
+        # @return [float]    -   time delta
+        def get_time_sync
+            @mutex.synchronize do
+                begin
+                    DebugLogger.info "Syncing time with coralogix server"
+                    res = @http.get(CORALOGIX_TIME_DELTA_URL)
+                    
+                    if res.is_a?(Net::HTTPSuccess) && !res.body.to_s.empty?
+                        #Get server ticks from 1970
+                        server_ticks = res.body.to_i.to_s # Relative to 1970
+                        #Take the first 13 digits
+                        server_ticks = server_ticks[0..12]
+                        #Convert the ticks to utc time
+                        server_time = Time.parse(Time.at(server_ticks.to_i / 1000.to_f).strftime('%H:%M:%S.%L')).utc
+                        local_time = Time.now.utc
+                        
+                        time_delta = (server_time - local_time) * 1000.0
+                        DebugLogger.info "Updating time delta to: #{time_delta}"
+                        return true, time_delta
+                    end
+                    return false, 0
+                rescue Exception => e  
+                    DebugLogger.error e.message  
+                    DebugLogger.error e.backtrace.inspect  
+                    return false, 0
+                end
+            end
+        end
+    end
+end

data/lib/manager.rb

@@ -0,0 +1,210 @@
+require_relative 'httpsender'
+require_relative 'constants'
+require_relative 'debug_logger'
+require 'date'
+require 'time'
+
+module Coralogix
+    # @private
+    class LoggerManager
+
+        attr_accessor :configured
+
+        def initialize
+            @bulk_template = {:privateKey => FAILED_PRIVATE_KEY, :applicationName => NO_APP_NAME, :subsystemName => NO_SUB_SYSTEM}
+            @time_delta_last_update = 0
+            @time_delta = 0
+            @http_sender = CoralogixHTTPSender.new
+            @buffer = []
+            @buffer_size = 0
+            @mutex = Mutex.new
+            run
+        end
+
+        # Add a log line to our buffer.
+        #
+        # @param **args    -   Customer parameters:
+        #                           privateKey - Private Key
+        #                           applicationName - Application name
+        #                           subsystemName - Subsystem name
+        # @return [boolean] return true for success or false for failure.
+        def configure **args
+            begin
+                @bulk_template = args.merge({:computerName => `hostname`.strip})
+                DebugLogger.info "Successfully configured Coralogix logger."
+                @configured = true                
+                add_logline "The Application Name #{@bulk_template[:applicationName]} and Subsystem Name #{@bulk_template[:subsystemName]} from the Fluentd SDK, version #{version?} has started to send data.", Severity::INFO, CORALOGIX_CATEGORY 
+            rescue Exception => e  
+                DebugLogger.error e.message  
+                DebugLogger.error e.backtrace.inspect
+                @configured = false
+            end
+            return @configured
+        end
+
+        def version?
+            begin
+                Gem.loaded_specs['coralogix_logger'].version.to_s
+            rescue Exception => e  
+                DebugLogger.error e.message  
+                DebugLogger.error e.backtrace.inspect
+                return '0.0.0'
+            end
+        end
+
+
+        # Add a log line to our buffer.
+        #
+        # @param message    -   The logs message. This is a must parameter.
+        # @param severity   -   The severity of the log message. This is a must parameter.
+        # @param category   -   The category (logger name) of the message. This is a must parameter.
+        # @param **args     -   Optional parameters. It can be:
+        #                           className - The class name where the log message was sent from.
+        #                           methodName - The method name where the log message was sent from.
+        #                           threadId -  The thread id where the log message was sent from.
+        # @return [boolean] return true for success or false for failure.
+        def add_logline message, severity, category, **args
+            begin
+                @mutex.synchronize do
+                    if @buffer_size < MAX_LOG_BUFFER_SIZE
+                        #Validate message
+                        message = (message.nil? || message.to_s.strip.empty?) ? "EMPTY_STRING" : msg2str(message)
+                        #Validate severity
+                        severity = (severity.nil? || severity.to_s < Severity::DEBUG.to_s || severity.to_s > Severity::CRITICAL.to_s) ? Severity::DEBUG : severity
+
+                        #Validate category
+                        category = (category.nil? || category.to_s.strip.empty?) ? CORALOGIX_CATEGORY : category.to_s
+
+                        #Combine a logentry from the must parameters together with the optional one.
+                        new_entry = {:text => message, :timestamp => Time.now.utc.to_f * 1000 + @time_delta, :severity => severity, :category => category}.merge(args)
+                        @buffer << new_entry
+                        #Update the buffer size to reflect the new size.
+                        @buffer_size+=new_entry.to_json.bytesize
+                    end
+                end
+            rescue Exception => e  
+                DebugLogger.error e.message  
+                DebugLogger.error e.backtrace.inspect
+                return false
+            end
+            return true
+        end
+
+        # Convert log message to string
+        # @param msg    -   log message to convert
+        #
+        # @return [String] return log message as string
+        def msg2str(msg)
+            begin
+                case msg
+                    when ::String
+                        msg
+                    when ::Exception
+                        "#{ msg.message } (#{ msg.class })\n" <<
+                            (msg.backtrace || []).join("\n")
+                    else
+                        msg.inspect
+                end
+            rescue Exception => e
+                DebugLogger.error e.message
+                DebugLogger.error e.backtrace.inspect
+                return msg
+            end
+        end
+        
+        # Flush all messages in buffer and send them immediately on the current thread.
+        def flush
+            send_bulk false
+        end
+
+        # Send bulk from the buffer
+        def send_bulk time_sync=true
+            begin
+                update_time_delta_interval if time_sync
+                @mutex.synchronize do
+                    # Total buffer size
+                    size = @buffer.size
+                    return unless size > 0
+
+                    # If the size is bigger than the maximum allowed chunk size then split it by half.
+                    # Keep splitting it until the size is less than MAX_LOG_CHUNK_SIZE
+                    while (@buffer.take(size).join(",").bytesize > MAX_LOG_CHUNK_SIZE) || (size == 0) 
+                        size=size/2;
+                    end
+                    
+                    # We must take at leat one value. If the first message is bigger than MAX_LOG_CHUNK_SIZE
+                    # we need to take it anyway.
+                    size = size > 0 ? size : 1
+
+                    DebugLogger.info "Checking buffer size. Total log entries is: #{size}"
+                    @bulk_template[:logEntries] = @buffer.shift(size)
+
+                    # Extract from the buffer size the total amount of the logs we removed from the buffer
+                    @buffer_size-= (@bulk_template[:logEntries].to_json.bytesize - 2 - size-1)
+
+                    # Make sure we are always positive
+                    @buffer_size = @buffer_size >= 0 ? @buffer_size : 0
+
+                    DebugLogger.info "Bufer size after removal is: #{@buffer.join(",").bytesize}"
+                end
+                @http_sender.send_request(@bulk_template) unless @bulk_template[:logEntries].empty?
+            rescue Exception => e  
+                DebugLogger.error e.message  
+                DebugLogger.error e.backtrace.inspect  
+            end
+        end
+
+        # A setter for disable_proxy. 
+        # By default HTTP object will use proxy environment variable if exists. In some cases this migh be an issue
+        # When set to false the HTTP object will ignore any proxy.
+        #
+        # @param value    -   true or false. (Default is false)
+        def disable_proxy=(value)
+            @http_sender.disable_proxy=value
+        end
+
+        # Sync log timestamps with coralogix server
+        def update_time_delta_interval
+            begin
+                #If more than 5 seconds passed from the last sync update
+                if ((DateTime.now.strftime('%Q').to_i - @time_delta_last_update) / 1000) >= (60 * SYNC_TIME_UPDATE_INTERVAL) #5 minuts
+                    res, _time_delta = @http_sender.get_time_sync
+                    if res
+                        @time_delta = _time_delta
+                        @time_delta_last_update = DateTime.now.strftime('%Q').to_i
+                    end
+                end
+             rescue Exception => e  
+                DebugLogger.error e.message  
+                DebugLogger.error e.backtrace.inspect  
+            end
+        end
+
+
+        # Start timer execution.
+        # The timer should send every X seconds logs from the buffer.
+        def run
+            begin
+                timer_thread = Thread.new do
+                    while true
+                        # Send log bulk
+                        send_bulk
+
+                        # Check when is the next time we should send logs?
+                        # If we already have at least half of the max chunk size then we are working in fast mode
+                        next_check_interval = @buffer_size > (MAX_LOG_CHUNK_SIZE / 2) ? FAST_SEND_SPEED_INTERVAL : NORMAL_SEND_SPEED_INTERVAL
+                        DebugLogger.debug "Next buffer check is scheduled in #{next_check_interval} seconds"
+                        sleep next_check_interval
+                    end
+                end
+
+                #Set thread priority to a high number
+                timer_thread.priority = 100
+            rescue Exception => e  
+                DebugLogger.error e.message  
+                DebugLogger.error e.backtrace.inspect
+                return false
+            end
+        end
+    end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: coralogix_fluentd_logger
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Royee Goldberg
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-17 00:00:00.000000000 Z
+dependencies: []
+description: Coralogix Fluentd sdk to send logs to Coralogix server.
+email: royee@coralogix.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENCE.txt
+- README.md
+- lib/constants.rb
+- lib/coralogix_logger.rb
+- lib/debug_logger.rb
+- lib/httpsender.rb
+- lib/manager.rb
+homepage: http://www.coralogix.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Coralogix Fluentd Logger SDK
+test_files: []