semantic_logger 4.2.0 → 4.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/Rakefile +1 -1
- data/lib/semantic_logger/ansi_colors.rb +11 -12
- data/lib/semantic_logger/appender.rb +4 -5
- data/lib/semantic_logger/appender/async.rb +24 -16
- data/lib/semantic_logger/appender/async_batch.rb +1 -4
- data/lib/semantic_logger/appender/bugsnag.rb +67 -63
- data/lib/semantic_logger/appender/elasticsearch.rb +154 -157
- data/lib/semantic_logger/appender/elasticsearch_http.rb +59 -55
- data/lib/semantic_logger/appender/file.rb +1 -3
- data/lib/semantic_logger/appender/graylog.rb +114 -110
- data/lib/semantic_logger/appender/honeybadger.rb +54 -51
- data/lib/semantic_logger/appender/http.rb +194 -190
- data/lib/semantic_logger/appender/kafka.rb +152 -149
- data/lib/semantic_logger/appender/mongodb.rb +3 -3
- data/lib/semantic_logger/appender/new_relic.rb +52 -49
- data/lib/semantic_logger/appender/sentry.rb +59 -54
- data/lib/semantic_logger/appender/splunk.rb +108 -103
- data/lib/semantic_logger/appender/splunk_http.rb +82 -79
- data/lib/semantic_logger/appender/syslog.rb +4 -5
- data/lib/semantic_logger/appender/tcp.rb +8 -29
- data/lib/semantic_logger/appender/udp.rb +2 -3
- data/lib/semantic_logger/appender/wrapper.rb +2 -2
- data/lib/semantic_logger/base.rb +18 -16
- data/lib/semantic_logger/concerns/compatibility.rb +0 -1
- data/lib/semantic_logger/core_ext/thread.rb +0 -1
- data/lib/semantic_logger/formatters.rb +3 -5
- data/lib/semantic_logger/formatters/base.rb +2 -3
- data/lib/semantic_logger/formatters/color.rb +29 -12
- data/lib/semantic_logger/formatters/default.rb +10 -10
- data/lib/semantic_logger/formatters/json.rb +0 -2
- data/lib/semantic_logger/formatters/one_line.rb +2 -2
- data/lib/semantic_logger/formatters/raw.rb +7 -10
- data/lib/semantic_logger/formatters/signalfx.rb +3 -5
- data/lib/semantic_logger/formatters/syslog.rb +2 -3
- data/lib/semantic_logger/formatters/syslog_cee.rb +2 -3
- data/lib/semantic_logger/jruby/garbage_collection_logger.rb +8 -5
- data/lib/semantic_logger/log.rb +17 -17
- data/lib/semantic_logger/loggable.rb +6 -9
- data/lib/semantic_logger/logger.rb +0 -1
- data/lib/semantic_logger/metric/new_relic.rb +58 -55
- data/lib/semantic_logger/metric/signalfx.rb +108 -106
- data/lib/semantic_logger/metric/statsd.rb +2 -3
- data/lib/semantic_logger/processor.rb +9 -9
- data/lib/semantic_logger/semantic_logger.rb +50 -30
- data/lib/semantic_logger/subscriber.rb +0 -1
- data/lib/semantic_logger/utils.rb +37 -37
- data/lib/semantic_logger/version.rb +2 -2
- data/test/appender/async_batch_test.rb +0 -1
- data/test/appender/async_test.rb +0 -1
- data/test/appender/bugsnag_test.rb +7 -8
- data/test/appender/elasticsearch_http_test.rb +5 -6
- data/test/appender/elasticsearch_test.rb +14 -10
- data/test/appender/file_test.rb +5 -6
- data/test/appender/graylog_test.rb +8 -8
- data/test/appender/honeybadger_test.rb +6 -7
- data/test/appender/http_test.rb +4 -5
- data/test/appender/kafka_test.rb +5 -6
- data/test/appender/mongodb_test.rb +11 -13
- data/test/appender/new_relic_test.rb +8 -9
- data/test/appender/newrelic_rpm.rb +1 -1
- data/test/appender/sentry_test.rb +7 -8
- data/test/appender/splunk_http_test.rb +4 -4
- data/test/appender/splunk_test.rb +1 -3
- data/test/appender/syslog_test.rb +3 -5
- data/test/appender/tcp_test.rb +4 -5
- data/test/appender/udp_test.rb +4 -5
- data/test/appender/wrapper_test.rb +2 -3
- data/test/concerns/compatibility_test.rb +0 -1
- data/test/debug_as_trace_logger_test.rb +0 -1
- data/test/formatters/color_test.rb +5 -6
- data/test/formatters/default_test.rb +16 -17
- data/test/formatters/one_line_test.rb +1 -2
- data/test/formatters/signalfx_test.rb +8 -11
- data/test/formatters_test.rb +3 -3
- data/test/in_memory_appender.rb +0 -1
- data/test/in_memory_appender_helper.rb +1 -1
- data/test/in_memory_batch_appender.rb +0 -1
- data/test/in_memory_metrics_appender.rb +0 -1
- data/test/loggable_test.rb +2 -3
- data/test/logger_test.rb +11 -14
- data/test/measure_test.rb +13 -15
- data/test/metric/new_relic_test.rb +2 -3
- data/test/metric/signalfx_test.rb +4 -5
- data/test/semantic_logger_test.rb +28 -3
- data/test/test_helper.rb +6 -7
- metadata +34 -34
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 65aba46ee7052dc3653a61a40fad4d69aed81e97
|
4
|
+
data.tar.gz: 2c7b49cb33e90874a966b9cadbbadf99bc4f4291
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a2ef7fb0d78bdb3ecac1ff6d8ef6cef1bb4797a7a0ac55b2cfab4f057d8f7c2c16519b37863a6fc1d8e728277765c678824d8234e3cfdf733386cf7f5ddeb073
|
7
|
+
data.tar.gz: e8df29aebb5e16b47fe87d530daf5f822c436ab18ddba47d682799ddb5c048041ce63841f1c6430987ca89e196f99853c48ecb38c8a9b2a0b65b8e3fbfae1d7a
|
data/README.md
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
#
|
1
|
+
# Semantic Logger
|
2
2
|
[](https://rubygems.org/gems/semantic_logger) [](https://travis-ci.org/rocketjob/semantic_logger) [](https://rubygems.org/gems/semantic_logger) [](http://opensource.org/licenses/Apache-2.0)  [-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
|
3
3
|
|
4
|
-
|
4
|
+
Semantic Logger is a feature rich logging framework, and replacement for existing Ruby & Rails loggers.
|
5
5
|
|
6
|
-
*
|
6
|
+
* https://rocketjob.github.io/semantic_logger/
|
7
7
|
|
8
8
|
## Documentation
|
9
9
|
|
data/Rakefile
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
module SemanticLogger
|
2
2
|
# Formatting & colors used by optional color formatter
|
3
3
|
module AnsiColors
|
4
|
-
CLEAR = "\e[0m"
|
5
|
-
BOLD = "\e[1m"
|
6
|
-
BLACK = "\e[30m"
|
7
|
-
RED = "\e[31m"
|
8
|
-
GREEN = "\e[32m"
|
9
|
-
YELLOW = "\e[33m"
|
10
|
-
BLUE = "\e[34m"
|
11
|
-
MAGENTA = "\e[35m"
|
12
|
-
CYAN = "\e[36m"
|
13
|
-
WHITE = "\e[37m"
|
4
|
+
CLEAR = "\e[0m".freeze
|
5
|
+
BOLD = "\e[1m".freeze
|
6
|
+
BLACK = "\e[30m".freeze
|
7
|
+
RED = "\e[31m".freeze
|
8
|
+
GREEN = "\e[32m".freeze
|
9
|
+
YELLOW = "\e[33m".freeze
|
10
|
+
BLUE = "\e[34m".freeze
|
11
|
+
MAGENTA = "\e[35m".freeze
|
12
|
+
CYAN = "\e[36m".freeze
|
13
|
+
WHITE = "\e[37m".freeze
|
14
14
|
|
15
15
|
# DEPRECATED - NOT USED
|
16
16
|
LEVEL_MAP = {
|
@@ -20,7 +20,6 @@ module SemanticLogger
|
|
20
20
|
warn: BOLD,
|
21
21
|
error: RED,
|
22
22
|
fatal: RED
|
23
|
-
}
|
23
|
+
}.freeze
|
24
24
|
end
|
25
|
-
|
26
25
|
end
|
@@ -61,15 +61,13 @@ module SemanticLogger
|
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
-
|
65
|
-
|
66
|
-
ASYNC_OPTION_KEYS = [:max_queue_size, :lag_threshold_s, :batch_size, :batch_seconds, :lag_check_interval]
|
64
|
+
ASYNC_OPTION_KEYS = %i[max_queue_size lag_threshold_s batch_size batch_seconds lag_check_interval].freeze
|
67
65
|
|
68
66
|
# Returns [Subscriber] instance from the supplied options.
|
69
67
|
def self.build(options, &block)
|
70
68
|
if options[:io] || options[:file_name]
|
71
69
|
SemanticLogger::Appender::File.new(options, &block)
|
72
|
-
elsif appender = options.delete(:appender)
|
70
|
+
elsif (appender = options.delete(:appender))
|
73
71
|
if appender.is_a?(Symbol)
|
74
72
|
SemanticLogger::Utils.constantize_symbol(appender).new(options)
|
75
73
|
elsif appender.is_a?(Subscriber)
|
@@ -77,7 +75,7 @@ module SemanticLogger
|
|
77
75
|
else
|
78
76
|
raise(ArgumentError, "Parameter :appender must be either a Symbol or an object derived from SemanticLogger::Subscriber, not: #{appender.inspect}")
|
79
77
|
end
|
80
|
-
elsif appender = options.delete(:metric)
|
78
|
+
elsif (appender = options.delete(:metric))
|
81
79
|
if appender.is_a?(Symbol)
|
82
80
|
SemanticLogger::Utils.constantize_symbol(appender, 'SemanticLogger::Metric').new(options)
|
83
81
|
elsif appender.is_a?(Subscriber)
|
@@ -90,5 +88,6 @@ module SemanticLogger
|
|
90
88
|
end
|
91
89
|
end
|
92
90
|
|
91
|
+
private_class_method :build
|
93
92
|
end
|
94
93
|
end
|
@@ -22,9 +22,6 @@ module SemanticLogger
|
|
22
22
|
# Appender proxy to allow an existing appender to run asynchronously in a separate thread.
|
23
23
|
#
|
24
24
|
# Parameters:
|
25
|
-
# name: [String]
|
26
|
-
# Name to use for the log thread and the log name when logging any errors from this appender.
|
27
|
-
#
|
28
25
|
# max_queue_size: [Integer]
|
29
26
|
# The maximum number of log messages to hold on the queue before blocking attempts to add to the queue.
|
30
27
|
# -1: The queue size is uncapped and will never block no matter how long the queue is.
|
@@ -38,7 +35,6 @@ module SemanticLogger
|
|
38
35
|
# Number of messages to process before checking for slow logging.
|
39
36
|
# Default: 1,000
|
40
37
|
def initialize(appender:,
|
41
|
-
name: appender.class.name,
|
42
38
|
max_queue_size: 10_000,
|
43
39
|
lag_check_interval: 1_000,
|
44
40
|
lag_threshold_s: 30)
|
@@ -66,13 +62,13 @@ module SemanticLogger
|
|
66
62
|
#
|
67
63
|
# Starts the worker thread if not running.
|
68
64
|
def thread
|
69
|
-
return @thread if @thread
|
65
|
+
return @thread if @thread&.alive?
|
70
66
|
@thread = Thread.new { process }
|
71
67
|
end
|
72
68
|
|
73
69
|
# Returns true if the worker thread is active
|
74
70
|
def active?
|
75
|
-
@thread
|
71
|
+
@thread&.alive?
|
76
72
|
end
|
77
73
|
|
78
74
|
# Add log message for processing.
|
@@ -99,26 +95,38 @@ module SemanticLogger
|
|
99
95
|
# This thread is designed to never go down unless the main thread terminates
|
100
96
|
# or the appender is closed.
|
101
97
|
Thread.current.name = logger.name
|
102
|
-
logger.trace
|
98
|
+
logger.trace 'Async: Appender thread active'
|
103
99
|
begin
|
104
100
|
process_messages
|
105
101
|
rescue StandardError => exception
|
106
102
|
# This block may be called after the file handles have been released by Ruby
|
107
|
-
|
103
|
+
begin
|
104
|
+
logger.error('Async: Restarting due to exception', exception)
|
105
|
+
rescue StandardError
|
106
|
+
nil
|
107
|
+
end
|
108
108
|
retry
|
109
109
|
rescue Exception => exception
|
110
110
|
# This block may be called after the file handles have been released by Ruby
|
111
|
-
|
111
|
+
begin
|
112
|
+
logger.error('Async: Stopping due to fatal exception', exception)
|
113
|
+
rescue StandardError
|
114
|
+
nil
|
115
|
+
end
|
112
116
|
ensure
|
113
117
|
@thread = nil
|
114
118
|
# This block may be called after the file handles have been released by Ruby
|
115
|
-
|
119
|
+
begin
|
120
|
+
logger.trace('Async: Thread has stopped')
|
121
|
+
rescue StandardError
|
122
|
+
nil
|
123
|
+
end
|
116
124
|
end
|
117
125
|
end
|
118
126
|
|
119
127
|
def process_messages
|
120
128
|
count = 0
|
121
|
-
while message = queue.pop
|
129
|
+
while (message = queue.pop)
|
122
130
|
if message.is_a?(Log)
|
123
131
|
appender.log(message)
|
124
132
|
count += 1
|
@@ -150,9 +158,10 @@ module SemanticLogger
|
|
150
158
|
end
|
151
159
|
|
152
160
|
def check_lag(log)
|
153
|
-
|
154
|
-
|
155
|
-
|
161
|
+
diff = Time.now - log.time
|
162
|
+
return unless diff > lag_threshold_s
|
163
|
+
|
164
|
+
logger.warn "Async: Appender thread has fallen behind by #{diff} seconds with #{queue.size} messages queued up. Consider reducing the log level or changing the appenders"
|
156
165
|
end
|
157
166
|
|
158
167
|
# Submit command and wait for reply
|
@@ -165,7 +174,7 @@ module SemanticLogger
|
|
165
174
|
logger.warn msg
|
166
175
|
elsif queue_size > 100
|
167
176
|
logger.info msg
|
168
|
-
elsif queue_size
|
177
|
+
elsif queue_size.positive?
|
169
178
|
logger.trace msg
|
170
179
|
end
|
171
180
|
|
@@ -173,7 +182,6 @@ module SemanticLogger
|
|
173
182
|
queue << {command: command, reply_queue: reply_queue}
|
174
183
|
reply_queue.pop
|
175
184
|
end
|
176
|
-
|
177
185
|
end
|
178
186
|
end
|
179
187
|
end
|
@@ -28,7 +28,6 @@ module SemanticLogger
|
|
28
28
|
# * `lag_check_interval` is not applicable to batches, since the first message of every batch
|
29
29
|
# is the oldest and is always checked to see if the lag interval has been exceeded.
|
30
30
|
def initialize(appender:,
|
31
|
-
name: appender.class.name,
|
32
31
|
max_queue_size: 10_000,
|
33
32
|
lag_threshold_s: 30,
|
34
33
|
batch_size: 300,
|
@@ -39,7 +38,6 @@ module SemanticLogger
|
|
39
38
|
@signal = Concurrent::Event.new
|
40
39
|
super(
|
41
40
|
appender: appender,
|
42
|
-
name: name,
|
43
41
|
max_queue_size: max_queue_size,
|
44
42
|
lag_threshold_s: lag_threshold_s
|
45
43
|
)
|
@@ -79,7 +77,7 @@ module SemanticLogger
|
|
79
77
|
process_message(message)
|
80
78
|
end
|
81
79
|
end
|
82
|
-
appender.batch(logs) if logs.size
|
80
|
+
appender.batch(logs) if logs.size.positive?
|
83
81
|
signal.reset unless queue.size >= batch_size
|
84
82
|
end
|
85
83
|
end
|
@@ -89,7 +87,6 @@ module SemanticLogger
|
|
89
87
|
signal.set
|
90
88
|
super
|
91
89
|
end
|
92
|
-
|
93
90
|
end
|
94
91
|
end
|
95
92
|
end
|
@@ -9,77 +9,81 @@ end
|
|
9
9
|
# Example:
|
10
10
|
# SemanticLogger.add_appender(appender: :bugsnag)
|
11
11
|
#
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
12
|
+
module SemanticLogger
|
13
|
+
module Appender
|
14
|
+
class Bugsnag < SemanticLogger::Subscriber
|
15
|
+
# Create Bugsnag Error / Exception Appender
|
16
|
+
#
|
17
|
+
# Parameters
|
18
|
+
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
19
|
+
# Override the log level for this appender.
|
20
|
+
# Default: :error
|
21
|
+
#
|
22
|
+
# formatter: [Object|Proc]
|
23
|
+
# An instance of a class that implements #call, or a Proc to be used to format
|
24
|
+
# the output from this appender
|
25
|
+
# Default: Use the built-in formatter (See: #call)
|
26
|
+
#
|
27
|
+
# filter: [Regexp|Proc]
|
28
|
+
# RegExp: Only include log messages where the class name matches the supplied.
|
29
|
+
# regular expression. All other messages will be ignored.
|
30
|
+
# Proc: Only include log messages where the supplied Proc returns true
|
31
|
+
# The Proc must return true or false.
|
32
|
+
def initialize(level: :error, formatter: nil, filter: nil, application: nil, host: nil, &block)
|
33
|
+
raise 'Bugsnag only supports :info, :warn, or :error log levels' unless %i[info warn error fatal].include?(level)
|
32
34
|
|
33
|
-
|
34
|
-
|
35
|
+
# Replace the Bugsnag logger so that we can identify its log messages and not forward them to Bugsnag
|
36
|
+
::Bugsnag.configure { |config| config.logger = SemanticLogger[Bugsnag] }
|
35
37
|
|
36
|
-
|
37
|
-
|
38
|
+
super(level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
|
39
|
+
end
|
38
40
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
41
|
+
# Returns [Hash] of parameters to send to Bugsnag.
|
42
|
+
def call(log, logger)
|
43
|
+
h = SemanticLogger::Formatters::Raw.new.call(log, logger)
|
44
|
+
h[:severity] = log_level(log)
|
45
|
+
h.delete(:message) if h[:exception] && (h[:message] == h[:exception][:message])
|
46
|
+
h.delete(:time)
|
47
|
+
h.delete(:exception)
|
48
|
+
h
|
49
|
+
end
|
48
50
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
51
|
+
# Send an error notification to Bugsnag
|
52
|
+
def log(log)
|
53
|
+
# Ignore logs coming from Bugsnag itself
|
54
|
+
return false if log.name == 'Bugsnag'
|
53
55
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
56
|
+
# Send error messages as Runtime exceptions
|
57
|
+
exception =
|
58
|
+
if log.exception
|
59
|
+
# Manually constructed Exception, without a backtrace.
|
60
|
+
log.exception.set_backtrace(log.backtrace) if !log.exception.backtrace && log.backtrace
|
61
|
+
log.exception
|
62
|
+
else
|
63
|
+
error = RuntimeError.new(log.message)
|
64
|
+
error.set_backtrace(log.backtrace) if log.backtrace
|
65
|
+
error
|
66
|
+
end
|
65
67
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
68
|
+
# For more documentation on the Bugsnag.notify method see:
|
69
|
+
# https://bugsnag.com/docs/notifiers/ruby#sending-handled-exceptions
|
70
|
+
::Bugsnag.notify(exception, formatter.call(log, self))
|
71
|
+
true
|
72
|
+
end
|
71
73
|
|
72
|
-
|
74
|
+
private
|
73
75
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
76
|
+
# Bugsnag supports: error, warning or info
|
77
|
+
def log_level(log)
|
78
|
+
case log.level
|
79
|
+
when :error, :fatal
|
80
|
+
'error'
|
81
|
+
when :warn
|
82
|
+
'warning'
|
83
|
+
else
|
84
|
+
'info'
|
85
|
+
end
|
86
|
+
end
|
83
87
|
end
|
84
88
|
end
|
85
89
|
end
|
@@ -14,174 +14,171 @@ require 'date'
|
|
14
14
|
# appender: :elasticsearch,
|
15
15
|
# url: 'http://localhost:9200'
|
16
16
|
# )
|
17
|
-
|
18
|
-
|
17
|
+
module SemanticLogger
|
18
|
+
module Appender
|
19
|
+
class Elasticsearch < SemanticLogger::Subscriber
|
20
|
+
attr_accessor :url, :index, :type, :client, :flush_interval, :timeout_interval, :batch_size, :elasticsearch_args
|
19
21
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
22
|
+
# Create Elasticsearch appender over persistent HTTP(S)
|
23
|
+
#
|
24
|
+
# Parameters:
|
25
|
+
# index: [String]
|
26
|
+
# Prefix of the index to store the logs in Elasticsearch.
|
27
|
+
# The final index appends the date so that indexes are used per day.
|
28
|
+
# I.e. The final index will look like 'semantic_logger-YYYY.MM.DD'
|
29
|
+
# Default: 'semantic_logger'
|
30
|
+
#
|
31
|
+
# type: [String]
|
32
|
+
# Document type to associate with logs when they are written.
|
33
|
+
# Default: 'log'
|
34
|
+
#
|
35
|
+
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
36
|
+
# Override the log level for this appender.
|
37
|
+
# Default: SemanticLogger.default_level
|
38
|
+
#
|
39
|
+
# formatter: [Object|Proc|Symbol|Hash]
|
40
|
+
# An instance of a class that implements #call, or a Proc to be used to format
|
41
|
+
# the output from this appender
|
42
|
+
# Default: :raw_json (See: #call)
|
43
|
+
#
|
44
|
+
# filter: [Regexp|Proc]
|
45
|
+
# RegExp: Only include log messages where the class name matches the supplied.
|
46
|
+
# regular expression. All other messages will be ignored.
|
47
|
+
# Proc: Only include log messages where the supplied Proc returns true
|
48
|
+
# The Proc must return true or false.
|
49
|
+
#
|
50
|
+
# host: [String]
|
51
|
+
# Name of this host to appear in log messages.
|
52
|
+
# Default: SemanticLogger.host
|
53
|
+
#
|
54
|
+
# application: [String]
|
55
|
+
# Name of this application to appear in log messages.
|
56
|
+
# Default: SemanticLogger.application
|
57
|
+
#
|
58
|
+
# Elasticsearch Parameters:
|
59
|
+
# url: [String]
|
60
|
+
# Fully qualified address to the Elasticsearch service.
|
61
|
+
# Default: 'http://localhost:9200'
|
62
|
+
#
|
63
|
+
# hosts: [String|Hash|Array]
|
64
|
+
# Single host passed as a String or Hash, or multiple hosts
|
65
|
+
# passed as an Array; `host` or `url` keys are also valid.
|
66
|
+
# Note:
|
67
|
+
# :url above is ignored when supplying this option.
|
68
|
+
#
|
69
|
+
# resurrect_after [Float]
|
70
|
+
# After how many seconds a dead connection should be tried again.
|
71
|
+
#
|
72
|
+
# reload_connections [true|false|Integer]
|
73
|
+
# Reload connections after X requests.
|
74
|
+
# Default: false
|
75
|
+
#
|
76
|
+
# randomize_hosts [true|false]
|
77
|
+
# Shuffle connections on initialization and reload.
|
78
|
+
# Default: false
|
79
|
+
#
|
80
|
+
# sniffer_timeout [Integer]
|
81
|
+
# Timeout for reloading connections in seconds.
|
82
|
+
# Default: 1
|
83
|
+
#
|
84
|
+
# retry_on_failure [true|false|Integer]
|
85
|
+
# Retry X times when request fails before raising and exception.
|
86
|
+
# Default: false
|
87
|
+
#
|
88
|
+
# retry_on_status [Array<Number>]
|
89
|
+
# Retry when specific status codes are returned.
|
90
|
+
#
|
91
|
+
# reload_on_failure [true|false]
|
92
|
+
# Reload connections after failure.
|
93
|
+
# Default: false
|
94
|
+
#
|
95
|
+
# request_timeout [Integer]
|
96
|
+
# The request timeout to be passed to transport in options.
|
97
|
+
#
|
98
|
+
# adapter [Symbol]
|
99
|
+
# A specific adapter for Faraday (e.g. `:patron`)
|
100
|
+
#
|
101
|
+
# transport_options [Hash]
|
102
|
+
# Options to be passed to the `Faraday::Connection` constructor.
|
103
|
+
#
|
104
|
+
# transport_class [Constant]
|
105
|
+
# A specific transport class to use, will be initialized by
|
106
|
+
# the client and passed hosts and all arguments.
|
107
|
+
#
|
108
|
+
# transport [Object]
|
109
|
+
# A specific transport instance.
|
110
|
+
#
|
111
|
+
# serializer_class [Constant]
|
112
|
+
# A specific serializer class to use, will be initialized by
|
113
|
+
# the transport and passed the transport instance.
|
114
|
+
#
|
115
|
+
# selector [Elasticsearch::Transport::Transport::Connections::Selector::Base]
|
116
|
+
# An instance of selector strategy derived from `Elasticsearch::Transport::Transport::Connections::Selector::Base`.
|
117
|
+
#
|
118
|
+
# send_get_body_as [String]
|
119
|
+
# Specify the HTTP method to use for GET requests with a body.
|
120
|
+
# Default: 'GET'
|
121
|
+
def initialize(url: 'http://localhost:9200',
|
122
|
+
index: 'semantic_logger',
|
123
|
+
type: 'log',
|
124
|
+
level: nil,
|
125
|
+
formatter: nil,
|
126
|
+
filter: nil,
|
127
|
+
application: nil,
|
128
|
+
host: nil,
|
129
|
+
**elasticsearch_args,
|
130
|
+
&block)
|
129
131
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
132
|
+
@url = url
|
133
|
+
@index = index
|
134
|
+
@type = type
|
135
|
+
@elasticsearch_args = elasticsearch_args.dup
|
136
|
+
@elasticsearch_args[:url] = url if url && !elasticsearch_args[:hosts]
|
137
|
+
@elasticsearch_args[:logger] = logger
|
136
138
|
|
137
|
-
|
138
|
-
|
139
|
-
|
139
|
+
super(level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
|
140
|
+
reopen
|
141
|
+
end
|
140
142
|
|
141
|
-
|
142
|
-
|
143
|
-
|
143
|
+
def reopen
|
144
|
+
@client = ::Elasticsearch::Client.new(@elasticsearch_args)
|
145
|
+
end
|
144
146
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
147
|
+
# Log to the index for today
|
148
|
+
def log(log)
|
149
|
+
bulk_payload = formatter.call(log, self)
|
150
|
+
write_to_elasticsearch([bulk_index(log), bulk_payload])
|
151
|
+
true
|
152
|
+
end
|
153
|
+
|
154
|
+
def batch(logs)
|
155
|
+
messages = []
|
156
|
+
logs.each do |log|
|
157
|
+
messages << bulk_index(log) << formatter.call(log, self)
|
158
|
+
end
|
151
159
|
|
152
|
-
|
153
|
-
|
154
|
-
day = nil
|
155
|
-
logs.each do |log|
|
156
|
-
# Only write the bulk index once per day per batch. Supports mixed dates in a batch.
|
157
|
-
if log.time.day != day
|
158
|
-
messages << bulk_index(log)
|
159
|
-
day = log.time.day
|
160
|
+
write_to_elasticsearch(messages)
|
161
|
+
true
|
160
162
|
end
|
161
|
-
messages << formatter.call(log, self)
|
162
|
-
end
|
163
163
|
|
164
|
-
|
165
|
-
true
|
166
|
-
end
|
164
|
+
private
|
167
165
|
|
168
|
-
|
166
|
+
def write_to_elasticsearch(messages)
|
167
|
+
bulk_result = @client.bulk(body: messages)
|
168
|
+
return unless bulk_result['errors']
|
169
169
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
failed = bulk_result["items"].select { |x| x["status"] != 201 }
|
174
|
-
logger.error("ElasticSearch: Write failed. Messages discarded. : #{failed}")
|
175
|
-
end
|
176
|
-
end
|
170
|
+
failed = bulk_result['items'].reject { |x| x['status'] == 201 }
|
171
|
+
logger.error("ElasticSearch: Write failed. Messages discarded. : #{failed}")
|
172
|
+
end
|
177
173
|
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
174
|
+
def bulk_index(log)
|
175
|
+
daily_index = log.time.strftime("#{index}-%Y.%m.%d")
|
176
|
+
{'index' => {'_index' => daily_index, '_type' => type}}
|
177
|
+
end
|
182
178
|
|
183
|
-
|
184
|
-
|
179
|
+
def default_formatter
|
180
|
+
SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: :timestamp)
|
181
|
+
end
|
182
|
+
end
|
185
183
|
end
|
186
|
-
|
187
184
|
end
|