semantic_logger 4.5.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +51 -21
- data/Rakefile +7 -7
- data/lib/semantic_logger/ansi_colors.rb +0 -10
- data/lib/semantic_logger/appender/async.rb +12 -10
- data/lib/semantic_logger/appender/async_batch.rb +7 -3
- data/lib/semantic_logger/appender/bugsnag.rb +43 -30
- data/lib/semantic_logger/appender/elasticsearch.rb +34 -15
- data/lib/semantic_logger/appender/elasticsearch_http.rb +4 -4
- data/lib/semantic_logger/appender/file.rb +249 -67
- data/lib/semantic_logger/appender/graylog.rb +15 -10
- data/lib/semantic_logger/appender/honeybadger.rb +3 -3
- data/lib/semantic_logger/appender/http.rb +41 -20
- data/lib/semantic_logger/appender/io.rb +68 -0
- data/lib/semantic_logger/appender/kafka.rb +46 -31
- data/lib/semantic_logger/appender/mongodb.rb +6 -6
- data/lib/semantic_logger/appender/new_relic.rb +2 -2
- data/lib/semantic_logger/appender/rabbitmq.rb +5 -5
- data/lib/semantic_logger/appender/sentry.rb +7 -7
- data/lib/semantic_logger/appender/sentry_ruby.rb +138 -0
- data/lib/semantic_logger/appender/splunk.rb +7 -5
- data/lib/semantic_logger/appender/splunk_http.rb +6 -5
- data/lib/semantic_logger/appender/syslog.rb +23 -15
- data/lib/semantic_logger/appender/tcp.rb +9 -9
- data/lib/semantic_logger/appender/udp.rb +2 -2
- data/lib/semantic_logger/appender/wrapper.rb +3 -2
- data/lib/semantic_logger/appender.rb +62 -65
- data/lib/semantic_logger/appenders.rb +36 -53
- data/lib/semantic_logger/base.rb +61 -39
- data/lib/semantic_logger/formatters/base.rb +16 -6
- data/lib/semantic_logger/formatters/color.rb +14 -15
- data/lib/semantic_logger/formatters/default.rb +18 -5
- data/lib/semantic_logger/formatters/fluentd.rb +7 -18
- data/lib/semantic_logger/formatters/json.rb +3 -5
- data/lib/semantic_logger/formatters/logfmt.rb +77 -0
- data/lib/semantic_logger/formatters/raw.rb +39 -10
- data/lib/semantic_logger/formatters/signalfx.rb +14 -21
- data/lib/semantic_logger/formatters/syslog.rb +8 -6
- data/lib/semantic_logger/formatters/syslog_cee.rb +9 -7
- data/lib/semantic_logger/formatters.rb +13 -13
- data/lib/semantic_logger/jruby/garbage_collection_logger.rb +4 -2
- data/lib/semantic_logger/levels.rb +9 -7
- data/lib/semantic_logger/log.rb +58 -73
- data/lib/semantic_logger/loggable.rb +8 -1
- data/lib/semantic_logger/logger.rb +19 -11
- data/lib/semantic_logger/metric/new_relic.rb +3 -3
- data/lib/semantic_logger/metric/signalfx.rb +3 -3
- data/lib/semantic_logger/metric/statsd.rb +7 -7
- data/lib/semantic_logger/processor.rb +9 -7
- data/lib/semantic_logger/reporters/minitest.rb +4 -4
- data/lib/semantic_logger/semantic_logger.rb +57 -23
- data/lib/semantic_logger/subscriber.rb +24 -7
- data/lib/semantic_logger/sync.rb +12 -0
- data/lib/semantic_logger/sync_processor.rb +58 -0
- data/lib/semantic_logger/test/capture_log_events.rb +34 -0
- data/lib/semantic_logger/utils.rb +32 -13
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +27 -22
- metadata +15 -10
@@ -1,10 +1,10 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "kafka"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
|
-
require
|
7
|
+
require "date"
|
8
8
|
|
9
9
|
# Forward all log messages to Apache Kafka.
|
10
10
|
#
|
@@ -23,8 +23,8 @@ module SemanticLogger
|
|
23
23
|
module Appender
|
24
24
|
class Kafka < SemanticLogger::Subscriber
|
25
25
|
attr_accessor :seed_brokers, :client_id, :connect_timeout, :socket_timeout,
|
26
|
-
:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key,
|
27
|
-
:delivery_threshold, :delivery_interval,
|
26
|
+
:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :ssl_ca_certs_from_system,
|
27
|
+
:delivery_threshold, :delivery_interval, :required_acks,
|
28
28
|
:topic, :partition, :partition_key, :key
|
29
29
|
|
30
30
|
# Send log messages to Kafka in JSON format.
|
@@ -79,6 +79,9 @@ module SemanticLogger
|
|
79
79
|
# Must be used in combination with ssl_client_cert.
|
80
80
|
# Default: nil
|
81
81
|
#
|
82
|
+
# ssl_ca_certs_from_system: [boolean]
|
83
|
+
# Delegate SSL CA cert to the system certs
|
84
|
+
#
|
82
85
|
# delivery_threshold: [Integer]
|
83
86
|
# Number of messages between triggering a delivery of messages to Apache Kafka.
|
84
87
|
# Default: 100
|
@@ -87,6 +90,10 @@ module SemanticLogger
|
|
87
90
|
# Number of seconds between triggering a delivery of messages to Apache Kafka.
|
88
91
|
# Default: 5
|
89
92
|
#
|
93
|
+
# required_acks: [Integer]
|
94
|
+
# Number of replicas that must acknowledge receipt of each log message to the topic
|
95
|
+
# Default: 1
|
96
|
+
#
|
90
97
|
# Semantic Logger Parameters:
|
91
98
|
#
|
92
99
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
@@ -115,25 +122,27 @@ module SemanticLogger
|
|
115
122
|
# metrics: [Boolean]
|
116
123
|
# Send metrics only events to kafka.
|
117
124
|
# Default: true
|
118
|
-
def initialize(seed_brokers:, client_id:
|
119
|
-
ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
|
120
|
-
topic:
|
121
|
-
delivery_threshold: 100, delivery_interval: 10,
|
125
|
+
def initialize(seed_brokers:, client_id: "semantic-logger", connect_timeout: nil, socket_timeout: nil,
|
126
|
+
ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil, ssl_ca_certs_from_system: false,
|
127
|
+
topic: "log_messages", partition: nil, partition_key: nil, key: nil,
|
128
|
+
delivery_threshold: 100, delivery_interval: 10, required_acks: 1,
|
122
129
|
metrics: true, **args, &block)
|
123
130
|
|
124
|
-
@seed_brokers
|
125
|
-
@client_id
|
126
|
-
@connect_timeout
|
127
|
-
@socket_timeout
|
128
|
-
@ssl_ca_cert
|
129
|
-
@ssl_client_cert
|
130
|
-
@ssl_client_cert_key
|
131
|
-
@
|
132
|
-
@
|
133
|
-
@
|
134
|
-
@
|
135
|
-
@
|
136
|
-
@
|
131
|
+
@seed_brokers = seed_brokers
|
132
|
+
@client_id = client_id
|
133
|
+
@connect_timeout = connect_timeout
|
134
|
+
@socket_timeout = socket_timeout
|
135
|
+
@ssl_ca_cert = ssl_ca_cert
|
136
|
+
@ssl_client_cert = ssl_client_cert
|
137
|
+
@ssl_client_cert_key = ssl_client_cert_key
|
138
|
+
@ssl_ca_certs_from_system = ssl_ca_certs_from_system
|
139
|
+
@topic = topic
|
140
|
+
@partition = partition
|
141
|
+
@partition_key = partition_key
|
142
|
+
@key = key
|
143
|
+
@delivery_threshold = delivery_threshold
|
144
|
+
@delivery_interval = delivery_interval
|
145
|
+
@required_acks = required_acks
|
137
146
|
|
138
147
|
super(metrics: metrics, **args, &block)
|
139
148
|
reopen
|
@@ -141,19 +150,21 @@ module SemanticLogger
|
|
141
150
|
|
142
151
|
def reopen
|
143
152
|
@kafka = ::Kafka.new(
|
144
|
-
seed_brokers:
|
145
|
-
client_id:
|
146
|
-
connect_timeout:
|
147
|
-
socket_timeout:
|
148
|
-
ssl_ca_cert:
|
149
|
-
ssl_client_cert:
|
150
|
-
ssl_client_cert_key:
|
151
|
-
|
153
|
+
seed_brokers: seed_brokers,
|
154
|
+
client_id: client_id,
|
155
|
+
connect_timeout: connect_timeout,
|
156
|
+
socket_timeout: socket_timeout,
|
157
|
+
ssl_ca_cert: ssl_ca_cert,
|
158
|
+
ssl_client_cert: ssl_client_cert,
|
159
|
+
ssl_client_cert_key: ssl_client_cert_key,
|
160
|
+
ssl_ca_certs_from_system: ssl_ca_certs_from_system,
|
161
|
+
logger: logger
|
152
162
|
)
|
153
163
|
|
154
164
|
@producer = @kafka.async_producer(
|
155
165
|
delivery_threshold: delivery_threshold,
|
156
|
-
delivery_interval: delivery_interval
|
166
|
+
delivery_interval: delivery_interval,
|
167
|
+
required_acks: required_acks
|
157
168
|
)
|
158
169
|
end
|
159
170
|
|
@@ -183,6 +194,10 @@ module SemanticLogger
|
|
183
194
|
delivery_interval: delivery_interval
|
184
195
|
)
|
185
196
|
end
|
197
|
+
|
198
|
+
private
|
199
|
+
|
200
|
+
attr_reader :producer
|
186
201
|
end
|
187
202
|
end
|
188
203
|
end
|
@@ -1,8 +1,8 @@
|
|
1
|
-
require
|
1
|
+
require "socket"
|
2
2
|
begin
|
3
|
-
require
|
3
|
+
require "mongo"
|
4
4
|
rescue LoadError
|
5
|
-
raise 'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.'
|
5
|
+
raise LoadError, 'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.'
|
6
6
|
end
|
7
7
|
|
8
8
|
module SemanticLogger
|
@@ -104,9 +104,9 @@ module SemanticLogger
|
|
104
104
|
# Name of this application to appear in log messages.
|
105
105
|
# Default: SemanticLogger.application
|
106
106
|
def initialize(uri:,
|
107
|
-
collection_name:
|
107
|
+
collection_name: "semantic_logger",
|
108
108
|
write_concern: 0,
|
109
|
-
collection_size: 1024
|
109
|
+
collection_size: 1024**3,
|
110
110
|
collection_max: nil,
|
111
111
|
**args,
|
112
112
|
&block)
|
@@ -118,7 +118,7 @@ module SemanticLogger
|
|
118
118
|
size: collection_size,
|
119
119
|
write: {w: write_concern}
|
120
120
|
}
|
121
|
-
@options[:max]
|
121
|
+
@options[:max] = collection_max if collection_max
|
122
122
|
|
123
123
|
reopen
|
124
124
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "newrelic_rpm"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
7
|
# Send log messages to NewRelic
|
@@ -1,7 +1,7 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "bunny"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem bunny is required for logging to RabbitMQ. Please add the gem "bunny" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem bunny is required for logging to RabbitMQ. Please add the gem "bunny" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
7
|
# Forward all log messages to RabbitMQ.
|
@@ -14,7 +14,7 @@ end
|
|
14
14
|
# # Name of the queue in RabbitMQ where to publish the logs. This queue will be bound to "amqp.direct" exchange.
|
15
15
|
# queue: 'semantic_logger',
|
16
16
|
#
|
17
|
-
# # This host will be used for RabbitMQ connection.
|
17
|
+
# # This host will be used for RabbitMQ connection.
|
18
18
|
# # NOTE this is different than :host option which is used by the logger directly.
|
19
19
|
# rabbitmq_host: '127.0.0.1',
|
20
20
|
#
|
@@ -63,7 +63,7 @@ module SemanticLogger
|
|
63
63
|
# RabbitMQ Parameters:
|
64
64
|
#
|
65
65
|
# rabbitmq_host: [String]
|
66
|
-
# Host for AMQP connection. in Bunny this is called :host but here it has
|
66
|
+
# Host for AMQP connection. in Bunny this is called :host but here it has
|
67
67
|
# been remapped to avoid conflicting with SemanticLogger's :host param.
|
68
68
|
# Default: localhost
|
69
69
|
#
|
@@ -76,7 +76,7 @@ module SemanticLogger
|
|
76
76
|
# Default: nil
|
77
77
|
#
|
78
78
|
# more parameters supported by Bunny: http://rubybunny.info/articles/connecting.html
|
79
|
-
def initialize(queue_name:
|
79
|
+
def initialize(queue_name: "semantic_logger", rabbitmq_host: nil, metrics: false, **args, &block)
|
80
80
|
@queue_name = queue_name
|
81
81
|
@rabbitmq_args = args.dup
|
82
82
|
@rabbitmq_args[:host] = rabbitmq_host
|
@@ -1,7 +1,7 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "sentry-raven"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem sentry-raven is required for logging purposes. Please add the gem "sentry-raven" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem sentry-raven is required for logging purposes. Please add the gem "sentry-raven" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
7
|
# Send log messages to sentry
|
@@ -46,12 +46,12 @@ module SemanticLogger
|
|
46
46
|
# Send an error notification to sentry
|
47
47
|
def log(log)
|
48
48
|
# Ignore logs coming from Raven itself
|
49
|
-
return false if log.name ==
|
49
|
+
return false if log.name == "Raven"
|
50
50
|
|
51
|
-
context
|
52
|
-
user
|
53
|
-
tags
|
54
|
-
attrs
|
51
|
+
context = formatter.call(log, self)
|
52
|
+
user = context.delete(:user)
|
53
|
+
tags = context.delete(:tags)
|
54
|
+
attrs = {
|
55
55
|
level: context.delete(:level),
|
56
56
|
extra: context
|
57
57
|
}
|
@@ -0,0 +1,138 @@
|
|
1
|
+
begin
|
2
|
+
require "sentry-ruby"
|
3
|
+
rescue LoadError
|
4
|
+
raise LoadError, 'Gem sentry-ruby is required for logging purposes. Please add the gem "sentry-ruby" to your Gemfile.'
|
5
|
+
end
|
6
|
+
|
7
|
+
# Send log messages to sentry
|
8
|
+
#
|
9
|
+
# Example:
|
10
|
+
# SemanticLogger.add_appender(appender: :sentry_ruby)
|
11
|
+
#
|
12
|
+
module SemanticLogger
|
13
|
+
module Appender
|
14
|
+
class SentryRuby < SemanticLogger::Subscriber
|
15
|
+
# Create Appender
|
16
|
+
#
|
17
|
+
# Parameters
|
18
|
+
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
19
|
+
# Override the log level for this appender.
|
20
|
+
# Default: :error
|
21
|
+
#
|
22
|
+
# formatter: [Object|Proc|Symbol|Hash]
|
23
|
+
# An instance of a class that implements #call, or a Proc to be used to format
|
24
|
+
# the output from this appender
|
25
|
+
# Default: Use the built-in formatter (See: #call)
|
26
|
+
#
|
27
|
+
# filter: [Regexp|Proc]
|
28
|
+
# RegExp: Only include log messages where the class name matches the supplied.
|
29
|
+
# regular expression. All other messages will be ignored.
|
30
|
+
# Proc: Only include log messages where the supplied Proc returns true
|
31
|
+
# The Proc must return true or false.
|
32
|
+
#
|
33
|
+
# host: [String]
|
34
|
+
# Name of this host to appear in log messages.
|
35
|
+
# Default: SemanticLogger.host
|
36
|
+
#
|
37
|
+
# application: [String]
|
38
|
+
# Name of this application to appear in log messages.
|
39
|
+
# Default: SemanticLogger.application
|
40
|
+
def initialize(level: :error, **args, &block)
|
41
|
+
# Replace the Sentry Ruby logger so that we can identify its log
|
42
|
+
# messages and not forward them to Sentry
|
43
|
+
::Sentry.init { |config| config.logger = SemanticLogger[::Sentry] } unless ::Sentry.initialized?
|
44
|
+
super(level: level, **args, &block)
|
45
|
+
end
|
46
|
+
|
47
|
+
# Send an error notification to sentry
|
48
|
+
def log(log)
|
49
|
+
# Ignore logs coming from Sentry itself
|
50
|
+
return false if log.name == "Sentry"
|
51
|
+
|
52
|
+
context = formatter.call(log, self)
|
53
|
+
payload = context.delete(:payload) || {}
|
54
|
+
named_tags = context[:named_tags] || {}
|
55
|
+
transaction_name = named_tags.delete(:transaction_name)
|
56
|
+
|
57
|
+
user = extract_user!(named_tags, payload)
|
58
|
+
tags = extract_tags!(context)
|
59
|
+
|
60
|
+
fingerprint = payload.delete(:fingerprint)
|
61
|
+
|
62
|
+
::Sentry.with_scope do |scope|
|
63
|
+
scope.set_user(user) if user
|
64
|
+
scope.set_level(context.delete(:level)) if context[:level]
|
65
|
+
scope.set_fingerprint(fingerprint) if fingerprint
|
66
|
+
scope.set_transaction_name(transaction_name) if transaction_name
|
67
|
+
scope.set_tags(tags)
|
68
|
+
scope.set_extras(context)
|
69
|
+
scope.set_extras(payload)
|
70
|
+
|
71
|
+
if log.exception
|
72
|
+
::Sentry.capture_exception(log.exception)
|
73
|
+
elsif log.backtrace
|
74
|
+
::Sentry.capture_message(context[:message], backtrace: log.backtrace)
|
75
|
+
else
|
76
|
+
::Sentry.capture_message(context[:message])
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
true
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
# Use Raw Formatter by default
|
86
|
+
def default_formatter
|
87
|
+
SemanticLogger::Formatters::Raw.new
|
88
|
+
end
|
89
|
+
|
90
|
+
# Extract user data from named tags or payload.
|
91
|
+
#
|
92
|
+
# Keys :user_id and :user_email will be used as :id and :email respectively.
|
93
|
+
# Keys :username and :ip_address will be used verbatim.
|
94
|
+
#
|
95
|
+
# Any additional value nested in a :user key will be added, provided any of
|
96
|
+
# the above keys is already present.
|
97
|
+
#
|
98
|
+
def extract_user!(*sources)
|
99
|
+
keys = {user_id: :id, username: :username, user_email: :email, ip_address: :ip_address}
|
100
|
+
|
101
|
+
user = {}
|
102
|
+
|
103
|
+
sources.each do |source|
|
104
|
+
keys.each do |source_key, target_key|
|
105
|
+
value = source.delete(source_key)
|
106
|
+
user[target_key] = value if value
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
return if user.empty?
|
111
|
+
|
112
|
+
sources.each do |source|
|
113
|
+
extras = source.delete(:user)
|
114
|
+
user.merge!(extras) if extras.is_a?(Hash)
|
115
|
+
end
|
116
|
+
|
117
|
+
user
|
118
|
+
end
|
119
|
+
|
120
|
+
# Extract tags.
|
121
|
+
#
|
122
|
+
# Named tags will be stringified (both key and value).
|
123
|
+
# Unnamed tags will be stringified and joined with a comma. Then they will
|
124
|
+
# be used as a "tag" named tag. If such a tag already exists, it is also
|
125
|
+
# joined with a comma.
|
126
|
+
#
|
127
|
+
# Finally, the tag names are limited to 32 characters and the tag values to 256.
|
128
|
+
#
|
129
|
+
def extract_tags!(context)
|
130
|
+
named_tags = context.delete(:named_tags) || {}
|
131
|
+
named_tags = named_tags.map { |k, v| [k.to_s, v.to_s] }.to_h
|
132
|
+
tags = context.delete(:tags)
|
133
|
+
named_tags.merge!("tag" => tags.join(", ")) { |_, v1, v2| "#{v1}, #{v2}" } if tags
|
134
|
+
named_tags.map { |k, v| [k[0...32], v[0...256]] }.to_h
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
@@ -1,7 +1,8 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "splunk-sdk-ruby"
|
3
3
|
rescue LoadError
|
4
|
-
raise
|
4
|
+
raise LoadError,
|
5
|
+
'Gem splunk-sdk-ruby is required for logging to Splunk. Please add the gem "splunk-sdk-ruby" to your Gemfile.'
|
5
6
|
end
|
6
7
|
|
7
8
|
# Splunk log appender.
|
@@ -88,7 +89,7 @@ module SemanticLogger
|
|
88
89
|
# regular expression. All other messages will be ignored.
|
89
90
|
# Proc: Only include log messages where the supplied Proc returns true
|
90
91
|
# The Proc must return true or false.
|
91
|
-
def initialize(index:
|
92
|
+
def initialize(index: "main", source_type: nil, **args, &block)
|
92
93
|
@index = index
|
93
94
|
@source_type = source_type
|
94
95
|
|
@@ -120,14 +121,15 @@ module SemanticLogger
|
|
120
121
|
def call(log, logger)
|
121
122
|
h = SemanticLogger::Formatters::Raw.new.call(log, logger)
|
122
123
|
h.delete(:time)
|
123
|
-
message
|
124
|
+
message = {
|
124
125
|
source: logger.application,
|
125
126
|
host: logger.host,
|
126
127
|
time: log.time.utc.to_f,
|
127
128
|
message: h.delete(:message),
|
128
129
|
event: h
|
129
130
|
}
|
130
|
-
message[:
|
131
|
+
message[:environment] = logger.environment if logger.environment
|
132
|
+
message[:sourcetype] = source_type if source_type
|
131
133
|
message
|
132
134
|
end
|
133
135
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "json"
|
2
2
|
# Splunk log appender using the Splunk HTTP(S) listener.
|
3
3
|
#
|
4
4
|
# Use the newer, faster and more complete JSON over HTTP interface for Splunk.
|
@@ -81,16 +81,17 @@ module SemanticLogger
|
|
81
81
|
super(compress: compress, **args, &block)
|
82
82
|
|
83
83
|
# Put splunk auth token in the header of every HTTP post.
|
84
|
-
@header[
|
84
|
+
@header["Authorization"] = "Splunk #{token}"
|
85
85
|
end
|
86
86
|
|
87
87
|
# Returns [String] JSON to send to Splunk.
|
88
88
|
#
|
89
89
|
# For splunk format requirements see:
|
90
|
-
#
|
90
|
+
# https://docs.splunk.com/Documentation/Splunk/latest/Data/FormateventsforHTTPEventCollector
|
91
91
|
def call(log, logger)
|
92
|
-
h
|
93
|
-
|
92
|
+
h = SemanticLogger::Formatters::Raw.new(time_format: :seconds).call(log, logger)
|
93
|
+
h.delete(:host)
|
94
|
+
message = {
|
94
95
|
source: logger.application,
|
95
96
|
host: logger.host,
|
96
97
|
time: h.delete(:time),
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "syslog"
|
2
|
+
require "uri"
|
3
|
+
require "socket"
|
4
4
|
# Send log messages to local syslog, or remote syslog servers over TCP or UDP.
|
5
5
|
#
|
6
6
|
# Example:
|
@@ -31,7 +31,7 @@ require 'socket'
|
|
31
31
|
module SemanticLogger
|
32
32
|
module Appender
|
33
33
|
class Syslog < SemanticLogger::Subscriber
|
34
|
-
attr_reader :remote_syslog, :url, :server, :port, :protocol, :facility, :options, :level_map
|
34
|
+
attr_reader :remote_syslog, :url, :server, :port, :protocol, :facility, :options, :level_map, :max_size
|
35
35
|
|
36
36
|
# Create a Syslog appender instance.
|
37
37
|
#
|
@@ -57,7 +57,7 @@ module SemanticLogger
|
|
57
57
|
# Only used with the TCP protocol.
|
58
58
|
# Specify custom parameters to pass into Net::TCPClient.new
|
59
59
|
# For a list of options see the net_tcp_client documentation:
|
60
|
-
# https://github.com/
|
60
|
+
# https://github.com/reidmorrison/net_tcp_client/blob/master/lib/net/tcp_client/tcp_client.rb
|
61
61
|
#
|
62
62
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
63
63
|
# Override the log level for this appender.
|
@@ -73,6 +73,10 @@ module SemanticLogger
|
|
73
73
|
# Identity of the program.
|
74
74
|
# Default: SemanticLogger.application
|
75
75
|
#
|
76
|
+
# max_size: [Integer]
|
77
|
+
# Set your own packet size.
|
78
|
+
# Default: 1024 bytes
|
79
|
+
#
|
76
80
|
# options: [Integer]
|
77
81
|
# Default: ::Syslog::LOG_PID | ::Syslog::LOG_CONS
|
78
82
|
# Any of the following (options can be logically OR'd together)
|
@@ -119,8 +123,9 @@ module SemanticLogger
|
|
119
123
|
# Example:
|
120
124
|
# # Change the warn level to LOG_NOTICE level instead of a the default of LOG_WARNING.
|
121
125
|
# SemanticLogger.add_appender(appender: :syslog, level_map: {warn: ::Syslog::LOG_NOTICE})
|
122
|
-
def initialize(url:
|
126
|
+
def initialize(url: "syslog://localhost",
|
123
127
|
facility: ::Syslog::LOG_USER,
|
128
|
+
max_size: 1024,
|
124
129
|
level_map: SemanticLogger::Formatters::Syslog::LevelMap.new,
|
125
130
|
options: ::Syslog::LOG_PID | ::Syslog::LOG_CONS,
|
126
131
|
tcp_client: {},
|
@@ -129,13 +134,14 @@ module SemanticLogger
|
|
129
134
|
|
130
135
|
@options = options
|
131
136
|
@facility = facility
|
137
|
+
@max_size = max_size
|
132
138
|
@level_map = level_map
|
133
139
|
@url = url
|
134
140
|
uri = URI(@url)
|
135
|
-
@server = uri.host ||
|
141
|
+
@server = uri.host || "localhost"
|
136
142
|
@protocol = (uri.scheme || :syslog).to_sym
|
137
143
|
@port = uri.port || 514
|
138
|
-
@server =
|
144
|
+
@server = "localhost" if @protocol == :syslog
|
139
145
|
@tcp_client_options = tcp_client
|
140
146
|
|
141
147
|
raise "Unknown protocol #{@protocol}!" unless %i[syslog tcp udp].include?(@protocol)
|
@@ -143,17 +149,19 @@ module SemanticLogger
|
|
143
149
|
# The syslog_protocol gem is required when logging over TCP or UDP.
|
144
150
|
if %i[tcp udp].include?(@protocol)
|
145
151
|
begin
|
146
|
-
require
|
152
|
+
require "syslog_protocol"
|
147
153
|
rescue LoadError
|
148
|
-
raise
|
154
|
+
raise LoadError,
|
155
|
+
"Missing gem: syslog_protocol. This gem is required when logging over TCP or UDP. To fix this error: gem install syslog_protocol"
|
149
156
|
end
|
150
157
|
|
151
158
|
# The net_tcp_client gem is required when logging over TCP.
|
152
159
|
if protocol == :tcp
|
153
160
|
begin
|
154
|
-
require
|
161
|
+
require "net/tcp_client"
|
155
162
|
rescue LoadError
|
156
|
-
raise
|
163
|
+
raise LoadError,
|
164
|
+
"Missing gem: net_tcp_client. This gem is required when logging over TCP. To fix this error: gem install net_tcp_client"
|
157
165
|
end
|
158
166
|
end
|
159
167
|
end
|
@@ -171,7 +179,7 @@ module SemanticLogger
|
|
171
179
|
::Syslog.send(method, application, options, facility)
|
172
180
|
when :tcp
|
173
181
|
@tcp_client_options[:server] = "#{@server}:#{@port}"
|
174
|
-
@remote_syslog = Net::TCPClient.new(
|
182
|
+
@remote_syslog = Net::TCPClient.new(**@tcp_client_options)
|
175
183
|
# Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
|
176
184
|
@remote_syslog.logger = logger
|
177
185
|
when :udp
|
@@ -186,7 +194,7 @@ module SemanticLogger
|
|
186
194
|
case @protocol
|
187
195
|
when :syslog
|
188
196
|
# Since the Ruby Syslog API supports sprintf format strings, double up all existing '%'
|
189
|
-
message = formatter.call(log, self).gsub
|
197
|
+
message = formatter.call(log, self).gsub "%", "%%"
|
190
198
|
::Syslog.log @level_map[log.level], message
|
191
199
|
when :tcp
|
192
200
|
@remote_syslog.retry_on_connection_failure { @remote_syslog.write("#{formatter.call(log, self)}\r\n") }
|
@@ -209,7 +217,7 @@ module SemanticLogger
|
|
209
217
|
# Format is text output without the time
|
210
218
|
SemanticLogger::Formatters::Default.new(time_format: nil)
|
211
219
|
else
|
212
|
-
SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map)
|
220
|
+
SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map, max_size: max_size)
|
213
221
|
end
|
214
222
|
end
|
215
223
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "net/tcp_client"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem net_tcp_client is required for logging over TCP. Please add the gem "net_tcp_client" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem net_tcp_client is required for logging over TCP. Please add the gem "net_tcp_client" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
|
-
raise
|
7
|
+
raise "Net::TCPClient v2.0 or greater is required to log over TCP" unless Net::TCPClient::VERSION.to_f >= 2.0
|
8
8
|
|
9
9
|
module SemanticLogger
|
10
10
|
module Appender
|
@@ -89,7 +89,7 @@ module SemanticLogger
|
|
89
89
|
# where multiple sends are expected during a single response
|
90
90
|
# Default: true
|
91
91
|
#
|
92
|
-
# :connect_retry_count [
|
92
|
+
# :connect_retry_count [Integer]
|
93
93
|
# Number of times to retry connecting when a connection fails
|
94
94
|
# Default: 10
|
95
95
|
#
|
@@ -97,7 +97,7 @@ module SemanticLogger
|
|
97
97
|
# Number of seconds between connection retry attempts after the first failed attempt
|
98
98
|
# Default: 0.5
|
99
99
|
#
|
100
|
-
# :retry_count [
|
100
|
+
# :retry_count [Integer]
|
101
101
|
# Number of times to retry when calling #retry_on_connection_failure
|
102
102
|
# This is independent of :connect_retry_count which still applies with
|
103
103
|
# connection failures. This retry controls upto how many times to retry the
|
@@ -182,23 +182,23 @@ module SemanticLogger
|
|
182
182
|
# connect_retry_count: 5
|
183
183
|
# )
|
184
184
|
def initialize(separator: "\n",
|
185
|
-
level: nil, formatter: nil, filter: nil, application: nil, host: nil, metrics: false,
|
185
|
+
level: nil, formatter: nil, filter: nil, application: nil, environment: nil, host: nil, metrics: false,
|
186
186
|
**tcp_client_args, &block)
|
187
187
|
@separator = separator
|
188
188
|
@tcp_client_args = tcp_client_args
|
189
189
|
|
190
190
|
# Use the internal logger so that errors with remote logging are only written locally.
|
191
191
|
Net::TCPClient.logger = logger
|
192
|
-
Net::TCPClient.logger.name =
|
192
|
+
Net::TCPClient.logger.name = "Net::TCPClient"
|
193
193
|
|
194
|
-
super(level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
|
194
|
+
super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, &block)
|
195
195
|
reopen
|
196
196
|
end
|
197
197
|
|
198
198
|
# After forking an active process call #reopen to re-open the handles to resources.
|
199
199
|
def reopen
|
200
200
|
close
|
201
|
-
@tcp_client = Net::TCPClient.new(
|
201
|
+
@tcp_client = Net::TCPClient.new(**@tcp_client_args)
|
202
202
|
end
|
203
203
|
|
204
204
|
# Write the log using the specified protocol and server.
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "socket"
|
2
2
|
module SemanticLogger
|
3
3
|
module Appender
|
4
4
|
# UDP log appender.
|
@@ -74,7 +74,7 @@ module SemanticLogger
|
|
74
74
|
def reopen
|
75
75
|
close
|
76
76
|
@socket = UDPSocket.new
|
77
|
-
host, port = server.split(
|
77
|
+
host, port = server.split(":")
|
78
78
|
@socket.connect(host, port.to_i)
|
79
79
|
end
|
80
80
|
|
@@ -32,7 +32,7 @@ module SemanticLogger
|
|
32
32
|
# require 'logger'
|
33
33
|
# require 'semantic_logger'
|
34
34
|
#
|
35
|
-
# ruby_logger = Logger.new(
|
35
|
+
# ruby_logger = Logger.new($stdout)
|
36
36
|
# SemanticLogger.add_appender(logger: ruby_logger)
|
37
37
|
#
|
38
38
|
# logger = SemanticLogger['test']
|
@@ -45,7 +45,8 @@ module SemanticLogger
|
|
45
45
|
# Check if the custom appender responds to all the log levels. For example Ruby ::Logger
|
46
46
|
does_not_implement = LEVELS[1..-1].find { |i| !@logger.respond_to?(i) }
|
47
47
|
if does_not_implement
|
48
|
-
raise(ArgumentError,
|
48
|
+
raise(ArgumentError,
|
49
|
+
"Supplied logger does not implement:#{does_not_implement}. It must implement all of #{LEVELS[1..-1].inspect}")
|
49
50
|
end
|
50
51
|
|
51
52
|
super(**args, &block)
|