coralogix_fluentd_logger 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: ca8727eb2b5cc0c4ea7af0067cb75ce493168c87
4
+ data.tar.gz: 99d0fb52a2e360ba8e077c17f3630b54e96a5603
5
+ SHA512:
6
+ metadata.gz: 4681ea0e1fbe83cf63ff7c005281712f718987963674ec1eee59efe86fc1792f885a698756f390167b1e1eabd6a4ec3c55e86f4f28bf7b83f47033a5ea1241fe
7
+ data.tar.gz: f0b5ea728df44ca089868e5ae9c70bf3ac4d1e07eac7873be356670d5f23d98a5798dc71ca20c122129adc4c79e63f3dd3c33349f82f59089eacd0b659a0520f
data/LICENCE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012-2016 Coralogix, Inc.
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,3 @@
1
+
2
+ # Croalogix SDK - Ruby Implementation
3
+ This is an implementation of Coralogix Ruby SDK for Fluentd.
data/lib/constants.rb ADDED
@@ -0,0 +1,62 @@
1
+ module Coralogix
2
+
3
+ #Maximum log buffer size
4
+ MAX_LOG_BUFFER_SIZE = 12582912 #12mb
5
+
6
+ #Maximum chunk size
7
+ MAX_LOG_CHUNK_SIZE = 1572864 #1.5 mb
8
+
9
+ #Bulk send interval in normal mode.
10
+ NORMAL_SEND_SPEED_INTERVAL = 500.0 / 1000
11
+
12
+ #Bulk send interval in fast mode.
13
+ FAST_SEND_SPEED_INTERVAL = 100.0 / 1000
14
+
15
+ #Corologix severity mapper
16
+ SEVERITIES = {:debug => 1, :verbose => 2, :info => 3, :warning => 4, :error => 5, :critical => 6}
17
+
18
+
19
+ module Severity
20
+ DEBUG = 1
21
+ VERBOSE = 2
22
+ INFO = 3
23
+ WARNING = 4
24
+ ERROR = 5
25
+ CRITICAL = 6
26
+ end
27
+
28
+
29
+ #Coralogix logs url
30
+ CORALOGIX_LOG_URL = "https://api.coralogix.com:443/api/v1/logs"
31
+
32
+ #Coralogix time delat url
33
+ CORALOGIX_TIME_DELTA_URL = "https://api.coralogix.com:443/sdk/v1/time"
34
+
35
+ #Default private key
36
+ FAILED_PRIVATE_KEY = "9626c7dd-8174-5015-a3fe-5572e042b6d9"
37
+
38
+ #Default application name
39
+ NO_APP_NAME = "NO_APP_NAME"
40
+
41
+ #Default subsystem name
42
+ NO_SUB_SYSTEM = "NO_SUB_NAME"
43
+
44
+ #Default log file name
45
+ LOG_FILE_NAME = "coralogix.sdk.log"
46
+
47
+ #Default http timeout
48
+ HTTP_TIMEOUT = 30
49
+
50
+ #Number of attempts to retry http post
51
+ HTTP_SEND_RETRY_COUNT = 5
52
+
53
+ #Interval between failed http post requests
54
+ HTTP_SEND_RETRY_INTERVAL = 2
55
+
56
+ # Coralogix category
57
+ CORALOGIX_CATEGORY = 'CORALOGIX'
58
+
59
+ # Sync time update interval
60
+ SYNC_TIME_UPDATE_INTERVAL = 5 #minutes
61
+
62
+ end
@@ -0,0 +1,86 @@
1
+ require_relative 'manager'
2
+ require_relative 'debug_logger'
3
+ require_relative 'httpsender'
4
+ require_relative 'constants'
5
+
6
+ module Coralogix
7
+ class CoralogixLogger
8
+ # Constructor.
9
+ #
10
+ # @param name - logger name.
11
+ def initialize private_key, app_name, sub_system_name, debug: false
12
+ self.debug_mode=debug
13
+ @category = CORALOGIX_CATEGORY
14
+ @manager = LoggerManager.new
15
+ configure private_key, app_name, sub_system_name
16
+ end
17
+
18
+ # A getter for debug_mode.
19
+ # Default value is false.
20
+ # When set to true the coralogix logger will print output messages to a console and a file.
21
+ #
22
+ # @return [boolean] - true or false. (Default is false)
23
+ def debug_mode?
24
+ DebugLogger.debug_mode
25
+ end
26
+
27
+ # A setter for debug_mode.
28
+ # Default value is false.
29
+ # When set to true the coralogix logger will print output messages to a console and a file.
30
+ #
31
+ # @param value - true or false. (Default is false)
32
+ def debug_mode=(value)
33
+ DebugLogger.debug_mode=value
34
+ end
35
+
36
+ # A setter for disable_proxy.
37
+ # By default HTTP object will use proxy environment variable if exists. In some cases this migh be an issue
38
+ # When set to false the HTTP object will ignore any proxy.
39
+ #
40
+ # @param value - true or false. (Default is false)
41
+ def disable_proxy=(value)
42
+ @manager.disable_proxy=value
43
+ end
44
+
45
+ # Configure coralogix logger with customer specific values
46
+ #
47
+ # @param private_key - private key
48
+ # @param app_name - application name
49
+ # @param sub_system - sub system name
50
+ # @return [boolean] return a true or false.
51
+ def configure private_key, app_name, sub_system
52
+ private_key = (private_key.nil? || private_key.to_s.strip.empty?) ? FAILED_PRIVATE_KEY : private_key
53
+ app_name = (app_name.nil? || app_name.to_s.strip.empty?) ? NO_APP_NAME : app_name
54
+ sub_system = (sub_system.nil? || sub_system.to_s.strip.empty?) ? NO_SUB_SYSTEM : sub_system
55
+ @manager.configure(:privateKey => private_key, :applicationName => app_name, :subsystemName => sub_system) unless @manager.configured
56
+ end
57
+
58
+
59
+ # Flush all messages in buffer and send them immediately on the current thread.
60
+ def flush
61
+ @manager.flush
62
+ end
63
+
64
+ # Log a message.
65
+ #
66
+ # @param severity - log severity
67
+ # @param message - log message
68
+ # @param category - log category
69
+ # @param className - log class name
70
+ # @param methodName - log method name
71
+ # @param threadId - log thread id
72
+ def log severity, message, category: @category, className: "", methodName: "", threadId: Thread.current.object_id.to_s
73
+ @manager.add_logline message, severity, category, :className => className, :methodName => methodName, :threadId => threadId
74
+ end
75
+
76
+ # Create log methods for each severity.
77
+ # This is a ruby thing. If you are writing in other languages just create a method for each severity.
78
+ # For instance, for info severity it will create a method:
79
+ # def info message, category: @category, className: "", methodName: "", threadId: ""
80
+ SEVERITIES.keys.each do |severity|
81
+ define_method("#{severity}") do |message, category: @category, className: "", methodName: "", threadId: Thread.current.object_id.to_s|
82
+ @manager.add_logline message, SEVERITIES["#{__method__}".to_sym], category, :className => className, :methodName => methodName, :threadId => threadId
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,61 @@
1
+ require 'logger'
2
+
3
+ module Coralogix
4
+ # @private
5
+ class DebugLogger
6
+
7
+ def self.initialize
8
+ begin
9
+ @mutex = Mutex.new
10
+ @debug = false
11
+ rescue Exception => e
12
+ if @debug
13
+ puts e.message
14
+ puts e.backtrace.inspect
15
+ end
16
+ end
17
+ end
18
+
19
+ def self.debug_mode?
20
+ @debug
21
+ end
22
+
23
+ def self.debug_mode=(value)
24
+ begin
25
+ @debug = value
26
+ if value
27
+ @logger = Logger.new(LOG_FILE_NAME, 1, 10485760)
28
+ else
29
+ @logger.close unless @logger == nil
30
+ @logger = nil
31
+ end
32
+ rescue Exception => e
33
+ if @debug
34
+ puts e.message
35
+ puts e.backtrace.inspect
36
+ end
37
+ end
38
+ end
39
+
40
+ Logger::Severity.constants.each do |level|
41
+ define_singleton_method("#{level.downcase}") do |*args|
42
+ if @debug
43
+ @mutex.synchronize do
44
+ begin
45
+ puts "#{__method__.upcase}: #{Time.now.strftime('%H:%M:%S.%L')} - #{args}"
46
+ @logger.send("#{__method__}", args)
47
+ rescue Exception => e
48
+ puts e.message
49
+ puts e.backtrace.inspect
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
55
+
56
+
57
+ initialize
58
+
59
+ end
60
+
61
+ end
data/lib/httpsender.rb ADDED
@@ -0,0 +1,100 @@
1
+ require 'net/http'
2
+ require 'json'
3
+ require_relative 'constants'
4
+ require_relative 'debug_logger'
5
+ require 'openssl'
6
+ require 'benchmark'
7
+ require 'date'
8
+ require 'time'
9
+ module Coralogix
10
+ # @private
11
+ class CoralogixHTTPSender
12
+ TICKS_IN_SECOND = 10**7
13
+ def disable_proxy value
14
+ @disable_proxy = value
15
+ end
16
+
17
+ def disable_proxy=(value)
18
+ @disable_proxy = value
19
+ end
20
+
21
+ def initialize
22
+ begin
23
+ @initialized = false
24
+ @mutex = Mutex.new
25
+ @uri = URI(CORALOGIX_LOG_URL)
26
+ if(@disable_proxy)
27
+ @http = Net::HTTP.new(@uri.host, @uri.port, p_addr=nil, p_port=nil)
28
+ else
29
+ @http = Net::HTTP.new(@uri.host, @uri.port)
30
+ end
31
+ @http.use_ssl = true
32
+ @http.keep_alive_timeout = 10
33
+ @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
34
+ @http.read_timeout = HTTP_TIMEOUT # seconds
35
+ @http.open_timeout = HTTP_TIMEOUT # seconds
36
+ @req = Net::HTTP::Post.new(@uri.path, 'Content-Type' => 'application/json')
37
+ rescue Exception => e
38
+ DebugLogger.error e.message
39
+ DebugLogger.error e.backtrace.inspect
40
+ end
41
+ end
42
+
43
+ # A helper method to post http request
44
+ #
45
+ # @param bulk - JSON bulk containing the log entries
46
+ def send_request bulk
47
+ @mutex.synchronize do
48
+ attempt = 0
49
+ while attempt < HTTP_SEND_RETRY_COUNT
50
+ begin
51
+ DebugLogger.info "About to send bulk to Coralogix server. Attempt number: #{attempt+1}"
52
+ @req.body = bulk.to_json
53
+ DebugLogger.debug Benchmark.measure {
54
+ res = @http.request(@req)
55
+ DebugLogger.info "Successfully sent bulk to Coralogix server. Result is: #{res.code}"
56
+ }.to_s
57
+ return true
58
+ rescue Exception => e
59
+ DebugLogger.error e.message
60
+ DebugLogger.error e.backtrace.inspect
61
+ end
62
+ attempt+=1;
63
+ DebugLogger.error "Failed to send bulk. Will retry in: #{HTTP_SEND_RETRY_INTERVAL} seconds..."
64
+ sleep HTTP_SEND_RETRY_INTERVAL
65
+ end
66
+ end
67
+ end
68
+
69
+ # A helper method to get coralogix server current time and calculate the time difference
70
+ #
71
+ # @return [float] - time delta
72
+ def get_time_sync
73
+ @mutex.synchronize do
74
+ begin
75
+ DebugLogger.info "Syncing time with coralogix server"
76
+ res = @http.get(CORALOGIX_TIME_DELTA_URL)
77
+
78
+ if res.is_a?(Net::HTTPSuccess) && !res.body.to_s.empty?
79
+ #Get server ticks from 1970
80
+ server_ticks = res.body.to_i.to_s # Relative to 1970
81
+ #Take the first 13 digits
82
+ server_ticks = server_ticks[0..12]
83
+ #Convert the ticks to utc time
84
+ server_time = Time.parse(Time.at(server_ticks.to_i / 1000.to_f).strftime('%H:%M:%S.%L')).utc
85
+ local_time = Time.now.utc
86
+
87
+ time_delta = (server_time - local_time) * 1000.0
88
+ DebugLogger.info "Updating time delta to: #{time_delta}"
89
+ return true, time_delta
90
+ end
91
+ return false, 0
92
+ rescue Exception => e
93
+ DebugLogger.error e.message
94
+ DebugLogger.error e.backtrace.inspect
95
+ return false, 0
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
data/lib/manager.rb ADDED
@@ -0,0 +1,210 @@
1
+ require_relative 'httpsender'
2
+ require_relative 'constants'
3
+ require_relative 'debug_logger'
4
+ require 'date'
5
+ require 'time'
6
+
7
+ module Coralogix
8
+ # @private
9
+ class LoggerManager
10
+
11
+ attr_accessor :configured
12
+
13
+ def initialize
14
+ @bulk_template = {:privateKey => FAILED_PRIVATE_KEY, :applicationName => NO_APP_NAME, :subsystemName => NO_SUB_SYSTEM}
15
+ @time_delta_last_update = 0
16
+ @time_delta = 0
17
+ @http_sender = CoralogixHTTPSender.new
18
+ @buffer = []
19
+ @buffer_size = 0
20
+ @mutex = Mutex.new
21
+ run
22
+ end
23
+
24
+ # Add a log line to our buffer.
25
+ #
26
+ # @param **args - Customer parameters:
27
+ # privateKey - Private Key
28
+ # applicationName - Application name
29
+ # subsystemName - Subsystem name
30
+ # @return [boolean] return true for success or false for failure.
31
+ def configure **args
32
+ begin
33
+ @bulk_template = args.merge({:computerName => `hostname`.strip})
34
+ DebugLogger.info "Successfully configured Coralogix logger."
35
+ @configured = true
36
+ add_logline "The Application Name #{@bulk_template[:applicationName]} and Subsystem Name #{@bulk_template[:subsystemName]} from the Fluentd SDK, version #{version?} has started to send data.", Severity::INFO, CORALOGIX_CATEGORY
37
+ rescue Exception => e
38
+ DebugLogger.error e.message
39
+ DebugLogger.error e.backtrace.inspect
40
+ @configured = false
41
+ end
42
+ return @configured
43
+ end
44
+
45
+ def version?
46
+ begin
47
+ Gem.loaded_specs['coralogix_logger'].version.to_s
48
+ rescue Exception => e
49
+ DebugLogger.error e.message
50
+ DebugLogger.error e.backtrace.inspect
51
+ return '0.0.0'
52
+ end
53
+ end
54
+
55
+
56
+ # Add a log line to our buffer.
57
+ #
58
+ # @param message - The logs message. This is a must parameter.
59
+ # @param severity - The severity of the log message. This is a must parameter.
60
+ # @param category - The category (logger name) of the message. This is a must parameter.
61
+ # @param **args - Optional parameters. It can be:
62
+ # className - The class name where the log message was sent from.
63
+ # methodName - The method name where the log message was sent from.
64
+ # threadId - The thread id where the log message was sent from.
65
+ # @return [boolean] return true for success or false for failure.
66
+ def add_logline message, severity, category, **args
67
+ begin
68
+ @mutex.synchronize do
69
+ if @buffer_size < MAX_LOG_BUFFER_SIZE
70
+ #Validate message
71
+ message = (message.nil? || message.to_s.strip.empty?) ? "EMPTY_STRING" : msg2str(message)
72
+ #Validate severity
73
+ severity = (severity.nil? || severity.to_s < Severity::DEBUG.to_s || severity.to_s > Severity::CRITICAL.to_s) ? Severity::DEBUG : severity
74
+
75
+ #Validate category
76
+ category = (category.nil? || category.to_s.strip.empty?) ? CORALOGIX_CATEGORY : category.to_s
77
+
78
+ #Combine a logentry from the must parameters together with the optional one.
79
+ new_entry = {:text => message, :timestamp => Time.now.utc.to_f * 1000 + @time_delta, :severity => severity, :category => category}.merge(args)
80
+ @buffer << new_entry
81
+ #Update the buffer size to reflect the new size.
82
+ @buffer_size+=new_entry.to_json.bytesize
83
+ end
84
+ end
85
+ rescue Exception => e
86
+ DebugLogger.error e.message
87
+ DebugLogger.error e.backtrace.inspect
88
+ return false
89
+ end
90
+ return true
91
+ end
92
+
93
+ # Convert log message to string
94
+ # @param msg - log message to convert
95
+ #
96
+ # @return [String] return log message as string
97
+ def msg2str(msg)
98
+ begin
99
+ case msg
100
+ when ::String
101
+ msg
102
+ when ::Exception
103
+ "#{ msg.message } (#{ msg.class })\n" <<
104
+ (msg.backtrace || []).join("\n")
105
+ else
106
+ msg.inspect
107
+ end
108
+ rescue Exception => e
109
+ DebugLogger.error e.message
110
+ DebugLogger.error e.backtrace.inspect
111
+ return msg
112
+ end
113
+ end
114
+
115
+ # Flush all messages in buffer and send them immediately on the current thread.
116
+ def flush
117
+ send_bulk false
118
+ end
119
+
120
+ # Send bulk from the buffer
121
+ def send_bulk time_sync=true
122
+ begin
123
+ update_time_delta_interval if time_sync
124
+ @mutex.synchronize do
125
+ # Total buffer size
126
+ size = @buffer.size
127
+ return unless size > 0
128
+
129
+ # If the size is bigger than the maximum allowed chunk size then split it by half.
130
+ # Keep splitting it until the size is less than MAX_LOG_CHUNK_SIZE
131
+ while (@buffer.take(size).join(",").bytesize > MAX_LOG_CHUNK_SIZE) || (size == 0)
132
+ size=size/2;
133
+ end
134
+
135
+ # We must take at leat one value. If the first message is bigger than MAX_LOG_CHUNK_SIZE
136
+ # we need to take it anyway.
137
+ size = size > 0 ? size : 1
138
+
139
+ DebugLogger.info "Checking buffer size. Total log entries is: #{size}"
140
+ @bulk_template[:logEntries] = @buffer.shift(size)
141
+
142
+ # Extract from the buffer size the total amount of the logs we removed from the buffer
143
+ @buffer_size-= (@bulk_template[:logEntries].to_json.bytesize - 2 - size-1)
144
+
145
+ # Make sure we are always positive
146
+ @buffer_size = @buffer_size >= 0 ? @buffer_size : 0
147
+
148
+ DebugLogger.info "Bufer size after removal is: #{@buffer.join(",").bytesize}"
149
+ end
150
+ @http_sender.send_request(@bulk_template) unless @bulk_template[:logEntries].empty?
151
+ rescue Exception => e
152
+ DebugLogger.error e.message
153
+ DebugLogger.error e.backtrace.inspect
154
+ end
155
+ end
156
+
157
+ # A setter for disable_proxy.
158
+ # By default HTTP object will use proxy environment variable if exists. In some cases this migh be an issue
159
+ # When set to false the HTTP object will ignore any proxy.
160
+ #
161
+ # @param value - true or false. (Default is false)
162
+ def disable_proxy=(value)
163
+ @http_sender.disable_proxy=value
164
+ end
165
+
166
+ # Sync log timestamps with coralogix server
167
+ def update_time_delta_interval
168
+ begin
169
+ #If more than 5 seconds passed from the last sync update
170
+ if ((DateTime.now.strftime('%Q').to_i - @time_delta_last_update) / 1000) >= (60 * SYNC_TIME_UPDATE_INTERVAL) #5 minuts
171
+ res, _time_delta = @http_sender.get_time_sync
172
+ if res
173
+ @time_delta = _time_delta
174
+ @time_delta_last_update = DateTime.now.strftime('%Q').to_i
175
+ end
176
+ end
177
+ rescue Exception => e
178
+ DebugLogger.error e.message
179
+ DebugLogger.error e.backtrace.inspect
180
+ end
181
+ end
182
+
183
+
184
+ # Start timer execution.
185
+ # The timer should send every X seconds logs from the buffer.
186
+ def run
187
+ begin
188
+ timer_thread = Thread.new do
189
+ while true
190
+ # Send log bulk
191
+ send_bulk
192
+
193
+ # Check when is the next time we should send logs?
194
+ # If we already have at least half of the max chunk size then we are working in fast mode
195
+ next_check_interval = @buffer_size > (MAX_LOG_CHUNK_SIZE / 2) ? FAST_SEND_SPEED_INTERVAL : NORMAL_SEND_SPEED_INTERVAL
196
+ DebugLogger.debug "Next buffer check is scheduled in #{next_check_interval} seconds"
197
+ sleep next_check_interval
198
+ end
199
+ end
200
+
201
+ #Set thread priority to a high number
202
+ timer_thread.priority = 100
203
+ rescue Exception => e
204
+ DebugLogger.error e.message
205
+ DebugLogger.error e.backtrace.inspect
206
+ return false
207
+ end
208
+ end
209
+ end
210
+ end
metadata ADDED
@@ -0,0 +1,50 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: coralogix_fluentd_logger
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Royee Goldberg
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-01-17 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Coralogix Fluentd sdk to send logs to Coralogix server.
14
+ email: royee@coralogix.com
15
+ executables: []
16
+ extensions: []
17
+ extra_rdoc_files: []
18
+ files:
19
+ - LICENCE.txt
20
+ - README.md
21
+ - lib/constants.rb
22
+ - lib/coralogix_logger.rb
23
+ - lib/debug_logger.rb
24
+ - lib/httpsender.rb
25
+ - lib/manager.rb
26
+ homepage: http://www.coralogix.com
27
+ licenses:
28
+ - MIT
29
+ metadata: {}
30
+ post_install_message:
31
+ rdoc_options: []
32
+ require_paths:
33
+ - lib
34
+ required_ruby_version: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ">="
37
+ - !ruby/object:Gem::Version
38
+ version: 2.0.0
39
+ required_rubygems_version: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: '0'
44
+ requirements: []
45
+ rubyforge_project:
46
+ rubygems_version: 2.5.1
47
+ signing_key:
48
+ specification_version: 4
49
+ summary: Coralogix Fluentd Logger SDK
50
+ test_files: []