semantic_logger 4.5.0 → 4.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +51 -21
- data/Rakefile +7 -7
- data/lib/semantic_logger/ansi_colors.rb +0 -10
- data/lib/semantic_logger/appender/async.rb +12 -10
- data/lib/semantic_logger/appender/async_batch.rb +7 -3
- data/lib/semantic_logger/appender/bugsnag.rb +43 -30
- data/lib/semantic_logger/appender/elasticsearch.rb +34 -15
- data/lib/semantic_logger/appender/elasticsearch_http.rb +4 -4
- data/lib/semantic_logger/appender/file.rb +249 -67
- data/lib/semantic_logger/appender/graylog.rb +15 -10
- data/lib/semantic_logger/appender/honeybadger.rb +3 -3
- data/lib/semantic_logger/appender/http.rb +41 -20
- data/lib/semantic_logger/appender/io.rb +68 -0
- data/lib/semantic_logger/appender/kafka.rb +46 -31
- data/lib/semantic_logger/appender/mongodb.rb +6 -6
- data/lib/semantic_logger/appender/new_relic.rb +2 -2
- data/lib/semantic_logger/appender/rabbitmq.rb +5 -5
- data/lib/semantic_logger/appender/sentry.rb +7 -7
- data/lib/semantic_logger/appender/sentry_ruby.rb +138 -0
- data/lib/semantic_logger/appender/splunk.rb +7 -5
- data/lib/semantic_logger/appender/splunk_http.rb +6 -5
- data/lib/semantic_logger/appender/syslog.rb +23 -15
- data/lib/semantic_logger/appender/tcp.rb +9 -9
- data/lib/semantic_logger/appender/udp.rb +2 -2
- data/lib/semantic_logger/appender/wrapper.rb +3 -2
- data/lib/semantic_logger/appender.rb +62 -65
- data/lib/semantic_logger/appenders.rb +36 -53
- data/lib/semantic_logger/base.rb +61 -39
- data/lib/semantic_logger/formatters/base.rb +16 -6
- data/lib/semantic_logger/formatters/color.rb +14 -15
- data/lib/semantic_logger/formatters/default.rb +18 -5
- data/lib/semantic_logger/formatters/fluentd.rb +7 -18
- data/lib/semantic_logger/formatters/json.rb +3 -5
- data/lib/semantic_logger/formatters/logfmt.rb +77 -0
- data/lib/semantic_logger/formatters/raw.rb +39 -10
- data/lib/semantic_logger/formatters/signalfx.rb +14 -21
- data/lib/semantic_logger/formatters/syslog.rb +8 -6
- data/lib/semantic_logger/formatters/syslog_cee.rb +9 -7
- data/lib/semantic_logger/formatters.rb +13 -13
- data/lib/semantic_logger/jruby/garbage_collection_logger.rb +4 -2
- data/lib/semantic_logger/levels.rb +9 -7
- data/lib/semantic_logger/log.rb +58 -73
- data/lib/semantic_logger/loggable.rb +8 -1
- data/lib/semantic_logger/logger.rb +19 -11
- data/lib/semantic_logger/metric/new_relic.rb +3 -3
- data/lib/semantic_logger/metric/signalfx.rb +3 -3
- data/lib/semantic_logger/metric/statsd.rb +7 -7
- data/lib/semantic_logger/processor.rb +9 -7
- data/lib/semantic_logger/reporters/minitest.rb +4 -4
- data/lib/semantic_logger/semantic_logger.rb +57 -23
- data/lib/semantic_logger/subscriber.rb +24 -7
- data/lib/semantic_logger/sync.rb +12 -0
- data/lib/semantic_logger/sync_processor.rb +58 -0
- data/lib/semantic_logger/test/capture_log_events.rb +34 -0
- data/lib/semantic_logger/utils.rb +32 -13
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +27 -22
- metadata +15 -10
@@ -1,10 +1,10 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "kafka"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
|
-
require
|
7
|
+
require "date"
|
8
8
|
|
9
9
|
# Forward all log messages to Apache Kafka.
|
10
10
|
#
|
@@ -23,8 +23,8 @@ module SemanticLogger
|
|
23
23
|
module Appender
|
24
24
|
class Kafka < SemanticLogger::Subscriber
|
25
25
|
attr_accessor :seed_brokers, :client_id, :connect_timeout, :socket_timeout,
|
26
|
-
:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key,
|
27
|
-
:delivery_threshold, :delivery_interval,
|
26
|
+
:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :ssl_ca_certs_from_system,
|
27
|
+
:delivery_threshold, :delivery_interval, :required_acks,
|
28
28
|
:topic, :partition, :partition_key, :key
|
29
29
|
|
30
30
|
# Send log messages to Kafka in JSON format.
|
@@ -79,6 +79,9 @@ module SemanticLogger
|
|
79
79
|
# Must be used in combination with ssl_client_cert.
|
80
80
|
# Default: nil
|
81
81
|
#
|
82
|
+
# ssl_ca_certs_from_system: [boolean]
|
83
|
+
# Delegate SSL CA cert to the system certs
|
84
|
+
#
|
82
85
|
# delivery_threshold: [Integer]
|
83
86
|
# Number of messages between triggering a delivery of messages to Apache Kafka.
|
84
87
|
# Default: 100
|
@@ -87,6 +90,10 @@ module SemanticLogger
|
|
87
90
|
# Number of seconds between triggering a delivery of messages to Apache Kafka.
|
88
91
|
# Default: 5
|
89
92
|
#
|
93
|
+
# required_acks: [Integer]
|
94
|
+
# Number of replicas that must acknowledge receipt of each log message to the topic
|
95
|
+
# Default: 1
|
96
|
+
#
|
90
97
|
# Semantic Logger Parameters:
|
91
98
|
#
|
92
99
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
@@ -115,25 +122,27 @@ module SemanticLogger
|
|
115
122
|
# metrics: [Boolean]
|
116
123
|
# Send metrics only events to kafka.
|
117
124
|
# Default: true
|
118
|
-
def initialize(seed_brokers:, client_id:
|
119
|
-
ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
|
120
|
-
topic:
|
121
|
-
delivery_threshold: 100, delivery_interval: 10,
|
125
|
+
def initialize(seed_brokers:, client_id: "semantic-logger", connect_timeout: nil, socket_timeout: nil,
|
126
|
+
ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil, ssl_ca_certs_from_system: false,
|
127
|
+
topic: "log_messages", partition: nil, partition_key: nil, key: nil,
|
128
|
+
delivery_threshold: 100, delivery_interval: 10, required_acks: 1,
|
122
129
|
metrics: true, **args, &block)
|
123
130
|
|
124
|
-
@seed_brokers
|
125
|
-
@client_id
|
126
|
-
@connect_timeout
|
127
|
-
@socket_timeout
|
128
|
-
@ssl_ca_cert
|
129
|
-
@ssl_client_cert
|
130
|
-
@ssl_client_cert_key
|
131
|
-
@
|
132
|
-
@
|
133
|
-
@
|
134
|
-
@
|
135
|
-
@
|
136
|
-
@
|
131
|
+
@seed_brokers = seed_brokers
|
132
|
+
@client_id = client_id
|
133
|
+
@connect_timeout = connect_timeout
|
134
|
+
@socket_timeout = socket_timeout
|
135
|
+
@ssl_ca_cert = ssl_ca_cert
|
136
|
+
@ssl_client_cert = ssl_client_cert
|
137
|
+
@ssl_client_cert_key = ssl_client_cert_key
|
138
|
+
@ssl_ca_certs_from_system = ssl_ca_certs_from_system
|
139
|
+
@topic = topic
|
140
|
+
@partition = partition
|
141
|
+
@partition_key = partition_key
|
142
|
+
@key = key
|
143
|
+
@delivery_threshold = delivery_threshold
|
144
|
+
@delivery_interval = delivery_interval
|
145
|
+
@required_acks = required_acks
|
137
146
|
|
138
147
|
super(metrics: metrics, **args, &block)
|
139
148
|
reopen
|
@@ -141,19 +150,21 @@ module SemanticLogger
|
|
141
150
|
|
142
151
|
def reopen
|
143
152
|
@kafka = ::Kafka.new(
|
144
|
-
seed_brokers:
|
145
|
-
client_id:
|
146
|
-
connect_timeout:
|
147
|
-
socket_timeout:
|
148
|
-
ssl_ca_cert:
|
149
|
-
ssl_client_cert:
|
150
|
-
ssl_client_cert_key:
|
151
|
-
|
153
|
+
seed_brokers: seed_brokers,
|
154
|
+
client_id: client_id,
|
155
|
+
connect_timeout: connect_timeout,
|
156
|
+
socket_timeout: socket_timeout,
|
157
|
+
ssl_ca_cert: ssl_ca_cert,
|
158
|
+
ssl_client_cert: ssl_client_cert,
|
159
|
+
ssl_client_cert_key: ssl_client_cert_key,
|
160
|
+
ssl_ca_certs_from_system: ssl_ca_certs_from_system,
|
161
|
+
logger: logger
|
152
162
|
)
|
153
163
|
|
154
164
|
@producer = @kafka.async_producer(
|
155
165
|
delivery_threshold: delivery_threshold,
|
156
|
-
delivery_interval: delivery_interval
|
166
|
+
delivery_interval: delivery_interval,
|
167
|
+
required_acks: required_acks
|
157
168
|
)
|
158
169
|
end
|
159
170
|
|
@@ -183,6 +194,10 @@ module SemanticLogger
|
|
183
194
|
delivery_interval: delivery_interval
|
184
195
|
)
|
185
196
|
end
|
197
|
+
|
198
|
+
private
|
199
|
+
|
200
|
+
attr_reader :producer
|
186
201
|
end
|
187
202
|
end
|
188
203
|
end
|
@@ -1,8 +1,8 @@
|
|
1
|
-
require
|
1
|
+
require "socket"
|
2
2
|
begin
|
3
|
-
require
|
3
|
+
require "mongo"
|
4
4
|
rescue LoadError
|
5
|
-
raise 'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.'
|
5
|
+
raise LoadError, 'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.'
|
6
6
|
end
|
7
7
|
|
8
8
|
module SemanticLogger
|
@@ -104,9 +104,9 @@ module SemanticLogger
|
|
104
104
|
# Name of this application to appear in log messages.
|
105
105
|
# Default: SemanticLogger.application
|
106
106
|
def initialize(uri:,
|
107
|
-
collection_name:
|
107
|
+
collection_name: "semantic_logger",
|
108
108
|
write_concern: 0,
|
109
|
-
collection_size: 1024
|
109
|
+
collection_size: 1024**3,
|
110
110
|
collection_max: nil,
|
111
111
|
**args,
|
112
112
|
&block)
|
@@ -118,7 +118,7 @@ module SemanticLogger
|
|
118
118
|
size: collection_size,
|
119
119
|
write: {w: write_concern}
|
120
120
|
}
|
121
|
-
@options[:max]
|
121
|
+
@options[:max] = collection_max if collection_max
|
122
122
|
|
123
123
|
reopen
|
124
124
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "newrelic_rpm"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
7
|
# Send log messages to NewRelic
|
@@ -1,7 +1,7 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "bunny"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem bunny is required for logging to RabbitMQ. Please add the gem "bunny" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem bunny is required for logging to RabbitMQ. Please add the gem "bunny" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
7
|
# Forward all log messages to RabbitMQ.
|
@@ -14,7 +14,7 @@ end
|
|
14
14
|
# # Name of the queue in RabbitMQ where to publish the logs. This queue will be bound to "amqp.direct" exchange.
|
15
15
|
# queue: 'semantic_logger',
|
16
16
|
#
|
17
|
-
# # This host will be used for RabbitMQ connection.
|
17
|
+
# # This host will be used for RabbitMQ connection.
|
18
18
|
# # NOTE this is different than :host option which is used by the logger directly.
|
19
19
|
# rabbitmq_host: '127.0.0.1',
|
20
20
|
#
|
@@ -63,7 +63,7 @@ module SemanticLogger
|
|
63
63
|
# RabbitMQ Parameters:
|
64
64
|
#
|
65
65
|
# rabbitmq_host: [String]
|
66
|
-
# Host for AMQP connection. in Bunny this is called :host but here it has
|
66
|
+
# Host for AMQP connection. in Bunny this is called :host but here it has
|
67
67
|
# been remapped to avoid conflicting with SemanticLogger's :host param.
|
68
68
|
# Default: localhost
|
69
69
|
#
|
@@ -76,7 +76,7 @@ module SemanticLogger
|
|
76
76
|
# Default: nil
|
77
77
|
#
|
78
78
|
# more parameters supported by Bunny: http://rubybunny.info/articles/connecting.html
|
79
|
-
def initialize(queue_name:
|
79
|
+
def initialize(queue_name: "semantic_logger", rabbitmq_host: nil, metrics: false, **args, &block)
|
80
80
|
@queue_name = queue_name
|
81
81
|
@rabbitmq_args = args.dup
|
82
82
|
@rabbitmq_args[:host] = rabbitmq_host
|
@@ -1,7 +1,7 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "sentry-raven"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem sentry-raven is required for logging purposes. Please add the gem "sentry-raven" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem sentry-raven is required for logging purposes. Please add the gem "sentry-raven" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
7
|
# Send log messages to sentry
|
@@ -46,12 +46,12 @@ module SemanticLogger
|
|
46
46
|
# Send an error notification to sentry
|
47
47
|
def log(log)
|
48
48
|
# Ignore logs coming from Raven itself
|
49
|
-
return false if log.name ==
|
49
|
+
return false if log.name == "Raven"
|
50
50
|
|
51
|
-
context
|
52
|
-
user
|
53
|
-
tags
|
54
|
-
attrs
|
51
|
+
context = formatter.call(log, self)
|
52
|
+
user = context.delete(:user)
|
53
|
+
tags = context.delete(:tags)
|
54
|
+
attrs = {
|
55
55
|
level: context.delete(:level),
|
56
56
|
extra: context
|
57
57
|
}
|
@@ -0,0 +1,138 @@
|
|
1
|
+
begin
|
2
|
+
require "sentry-ruby"
|
3
|
+
rescue LoadError
|
4
|
+
raise LoadError, 'Gem sentry-ruby is required for logging purposes. Please add the gem "sentry-ruby" to your Gemfile.'
|
5
|
+
end
|
6
|
+
|
7
|
+
# Send log messages to sentry
|
8
|
+
#
|
9
|
+
# Example:
|
10
|
+
# SemanticLogger.add_appender(appender: :sentry_ruby)
|
11
|
+
#
|
12
|
+
module SemanticLogger
|
13
|
+
module Appender
|
14
|
+
class SentryRuby < SemanticLogger::Subscriber
|
15
|
+
# Create Appender
|
16
|
+
#
|
17
|
+
# Parameters
|
18
|
+
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
19
|
+
# Override the log level for this appender.
|
20
|
+
# Default: :error
|
21
|
+
#
|
22
|
+
# formatter: [Object|Proc|Symbol|Hash]
|
23
|
+
# An instance of a class that implements #call, or a Proc to be used to format
|
24
|
+
# the output from this appender
|
25
|
+
# Default: Use the built-in formatter (See: #call)
|
26
|
+
#
|
27
|
+
# filter: [Regexp|Proc]
|
28
|
+
# RegExp: Only include log messages where the class name matches the supplied.
|
29
|
+
# regular expression. All other messages will be ignored.
|
30
|
+
# Proc: Only include log messages where the supplied Proc returns true
|
31
|
+
# The Proc must return true or false.
|
32
|
+
#
|
33
|
+
# host: [String]
|
34
|
+
# Name of this host to appear in log messages.
|
35
|
+
# Default: SemanticLogger.host
|
36
|
+
#
|
37
|
+
# application: [String]
|
38
|
+
# Name of this application to appear in log messages.
|
39
|
+
# Default: SemanticLogger.application
|
40
|
+
def initialize(level: :error, **args, &block)
|
41
|
+
# Replace the Sentry Ruby logger so that we can identify its log
|
42
|
+
# messages and not forward them to Sentry
|
43
|
+
::Sentry.init { |config| config.logger = SemanticLogger[::Sentry] } unless ::Sentry.initialized?
|
44
|
+
super(level: level, **args, &block)
|
45
|
+
end
|
46
|
+
|
47
|
+
# Send an error notification to sentry
|
48
|
+
def log(log)
|
49
|
+
# Ignore logs coming from Sentry itself
|
50
|
+
return false if log.name == "Sentry"
|
51
|
+
|
52
|
+
context = formatter.call(log, self)
|
53
|
+
payload = context.delete(:payload) || {}
|
54
|
+
named_tags = context[:named_tags] || {}
|
55
|
+
transaction_name = named_tags.delete(:transaction_name)
|
56
|
+
|
57
|
+
user = extract_user!(named_tags, payload)
|
58
|
+
tags = extract_tags!(context)
|
59
|
+
|
60
|
+
fingerprint = payload.delete(:fingerprint)
|
61
|
+
|
62
|
+
::Sentry.with_scope do |scope|
|
63
|
+
scope.set_user(user) if user
|
64
|
+
scope.set_level(context.delete(:level)) if context[:level]
|
65
|
+
scope.set_fingerprint(fingerprint) if fingerprint
|
66
|
+
scope.set_transaction_name(transaction_name) if transaction_name
|
67
|
+
scope.set_tags(tags)
|
68
|
+
scope.set_extras(context)
|
69
|
+
scope.set_extras(payload)
|
70
|
+
|
71
|
+
if log.exception
|
72
|
+
::Sentry.capture_exception(log.exception)
|
73
|
+
elsif log.backtrace
|
74
|
+
::Sentry.capture_message(context[:message], backtrace: log.backtrace)
|
75
|
+
else
|
76
|
+
::Sentry.capture_message(context[:message])
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
true
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
# Use Raw Formatter by default
|
86
|
+
def default_formatter
|
87
|
+
SemanticLogger::Formatters::Raw.new
|
88
|
+
end
|
89
|
+
|
90
|
+
# Extract user data from named tags or payload.
|
91
|
+
#
|
92
|
+
# Keys :user_id and :user_email will be used as :id and :email respectively.
|
93
|
+
# Keys :username and :ip_address will be used verbatim.
|
94
|
+
#
|
95
|
+
# Any additional value nested in a :user key will be added, provided any of
|
96
|
+
# the above keys is already present.
|
97
|
+
#
|
98
|
+
def extract_user!(*sources)
|
99
|
+
keys = {user_id: :id, username: :username, user_email: :email, ip_address: :ip_address}
|
100
|
+
|
101
|
+
user = {}
|
102
|
+
|
103
|
+
sources.each do |source|
|
104
|
+
keys.each do |source_key, target_key|
|
105
|
+
value = source.delete(source_key)
|
106
|
+
user[target_key] = value if value
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
return if user.empty?
|
111
|
+
|
112
|
+
sources.each do |source|
|
113
|
+
extras = source.delete(:user)
|
114
|
+
user.merge!(extras) if extras.is_a?(Hash)
|
115
|
+
end
|
116
|
+
|
117
|
+
user
|
118
|
+
end
|
119
|
+
|
120
|
+
# Extract tags.
|
121
|
+
#
|
122
|
+
# Named tags will be stringified (both key and value).
|
123
|
+
# Unnamed tags will be stringified and joined with a comma. Then they will
|
124
|
+
# be used as a "tag" named tag. If such a tag already exists, it is also
|
125
|
+
# joined with a comma.
|
126
|
+
#
|
127
|
+
# Finally, the tag names are limited to 32 characters and the tag values to 256.
|
128
|
+
#
|
129
|
+
def extract_tags!(context)
|
130
|
+
named_tags = context.delete(:named_tags) || {}
|
131
|
+
named_tags = named_tags.map { |k, v| [k.to_s, v.to_s] }.to_h
|
132
|
+
tags = context.delete(:tags)
|
133
|
+
named_tags.merge!("tag" => tags.join(", ")) { |_, v1, v2| "#{v1}, #{v2}" } if tags
|
134
|
+
named_tags.map { |k, v| [k[0...32], v[0...256]] }.to_h
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
@@ -1,7 +1,8 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "splunk-sdk-ruby"
|
3
3
|
rescue LoadError
|
4
|
-
raise
|
4
|
+
raise LoadError,
|
5
|
+
'Gem splunk-sdk-ruby is required for logging to Splunk. Please add the gem "splunk-sdk-ruby" to your Gemfile.'
|
5
6
|
end
|
6
7
|
|
7
8
|
# Splunk log appender.
|
@@ -88,7 +89,7 @@ module SemanticLogger
|
|
88
89
|
# regular expression. All other messages will be ignored.
|
89
90
|
# Proc: Only include log messages where the supplied Proc returns true
|
90
91
|
# The Proc must return true or false.
|
91
|
-
def initialize(index:
|
92
|
+
def initialize(index: "main", source_type: nil, **args, &block)
|
92
93
|
@index = index
|
93
94
|
@source_type = source_type
|
94
95
|
|
@@ -120,14 +121,15 @@ module SemanticLogger
|
|
120
121
|
def call(log, logger)
|
121
122
|
h = SemanticLogger::Formatters::Raw.new.call(log, logger)
|
122
123
|
h.delete(:time)
|
123
|
-
message
|
124
|
+
message = {
|
124
125
|
source: logger.application,
|
125
126
|
host: logger.host,
|
126
127
|
time: log.time.utc.to_f,
|
127
128
|
message: h.delete(:message),
|
128
129
|
event: h
|
129
130
|
}
|
130
|
-
message[:
|
131
|
+
message[:environment] = logger.environment if logger.environment
|
132
|
+
message[:sourcetype] = source_type if source_type
|
131
133
|
message
|
132
134
|
end
|
133
135
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "json"
|
2
2
|
# Splunk log appender using the Splunk HTTP(S) listener.
|
3
3
|
#
|
4
4
|
# Use the newer, faster and more complete JSON over HTTP interface for Splunk.
|
@@ -81,16 +81,17 @@ module SemanticLogger
|
|
81
81
|
super(compress: compress, **args, &block)
|
82
82
|
|
83
83
|
# Put splunk auth token in the header of every HTTP post.
|
84
|
-
@header[
|
84
|
+
@header["Authorization"] = "Splunk #{token}"
|
85
85
|
end
|
86
86
|
|
87
87
|
# Returns [String] JSON to send to Splunk.
|
88
88
|
#
|
89
89
|
# For splunk format requirements see:
|
90
|
-
#
|
90
|
+
# https://docs.splunk.com/Documentation/Splunk/latest/Data/FormateventsforHTTPEventCollector
|
91
91
|
def call(log, logger)
|
92
|
-
h
|
93
|
-
|
92
|
+
h = SemanticLogger::Formatters::Raw.new(time_format: :seconds).call(log, logger)
|
93
|
+
h.delete(:host)
|
94
|
+
message = {
|
94
95
|
source: logger.application,
|
95
96
|
host: logger.host,
|
96
97
|
time: h.delete(:time),
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "syslog"
|
2
|
+
require "uri"
|
3
|
+
require "socket"
|
4
4
|
# Send log messages to local syslog, or remote syslog servers over TCP or UDP.
|
5
5
|
#
|
6
6
|
# Example:
|
@@ -31,7 +31,7 @@ require 'socket'
|
|
31
31
|
module SemanticLogger
|
32
32
|
module Appender
|
33
33
|
class Syslog < SemanticLogger::Subscriber
|
34
|
-
attr_reader :remote_syslog, :url, :server, :port, :protocol, :facility, :options, :level_map
|
34
|
+
attr_reader :remote_syslog, :url, :server, :port, :protocol, :facility, :options, :level_map, :max_size
|
35
35
|
|
36
36
|
# Create a Syslog appender instance.
|
37
37
|
#
|
@@ -57,7 +57,7 @@ module SemanticLogger
|
|
57
57
|
# Only used with the TCP protocol.
|
58
58
|
# Specify custom parameters to pass into Net::TCPClient.new
|
59
59
|
# For a list of options see the net_tcp_client documentation:
|
60
|
-
# https://github.com/
|
60
|
+
# https://github.com/reidmorrison/net_tcp_client/blob/master/lib/net/tcp_client/tcp_client.rb
|
61
61
|
#
|
62
62
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
63
63
|
# Override the log level for this appender.
|
@@ -73,6 +73,10 @@ module SemanticLogger
|
|
73
73
|
# Identity of the program.
|
74
74
|
# Default: SemanticLogger.application
|
75
75
|
#
|
76
|
+
# max_size: [Integer]
|
77
|
+
# Set your own packet size.
|
78
|
+
# Default: 1024 bytes
|
79
|
+
#
|
76
80
|
# options: [Integer]
|
77
81
|
# Default: ::Syslog::LOG_PID | ::Syslog::LOG_CONS
|
78
82
|
# Any of the following (options can be logically OR'd together)
|
@@ -119,8 +123,9 @@ module SemanticLogger
|
|
119
123
|
# Example:
|
120
124
|
# # Change the warn level to LOG_NOTICE level instead of a the default of LOG_WARNING.
|
121
125
|
# SemanticLogger.add_appender(appender: :syslog, level_map: {warn: ::Syslog::LOG_NOTICE})
|
122
|
-
def initialize(url:
|
126
|
+
def initialize(url: "syslog://localhost",
|
123
127
|
facility: ::Syslog::LOG_USER,
|
128
|
+
max_size: 1024,
|
124
129
|
level_map: SemanticLogger::Formatters::Syslog::LevelMap.new,
|
125
130
|
options: ::Syslog::LOG_PID | ::Syslog::LOG_CONS,
|
126
131
|
tcp_client: {},
|
@@ -129,13 +134,14 @@ module SemanticLogger
|
|
129
134
|
|
130
135
|
@options = options
|
131
136
|
@facility = facility
|
137
|
+
@max_size = max_size
|
132
138
|
@level_map = level_map
|
133
139
|
@url = url
|
134
140
|
uri = URI(@url)
|
135
|
-
@server = uri.host ||
|
141
|
+
@server = uri.host || "localhost"
|
136
142
|
@protocol = (uri.scheme || :syslog).to_sym
|
137
143
|
@port = uri.port || 514
|
138
|
-
@server =
|
144
|
+
@server = "localhost" if @protocol == :syslog
|
139
145
|
@tcp_client_options = tcp_client
|
140
146
|
|
141
147
|
raise "Unknown protocol #{@protocol}!" unless %i[syslog tcp udp].include?(@protocol)
|
@@ -143,17 +149,19 @@ module SemanticLogger
|
|
143
149
|
# The syslog_protocol gem is required when logging over TCP or UDP.
|
144
150
|
if %i[tcp udp].include?(@protocol)
|
145
151
|
begin
|
146
|
-
require
|
152
|
+
require "syslog_protocol"
|
147
153
|
rescue LoadError
|
148
|
-
raise
|
154
|
+
raise LoadError,
|
155
|
+
"Missing gem: syslog_protocol. This gem is required when logging over TCP or UDP. To fix this error: gem install syslog_protocol"
|
149
156
|
end
|
150
157
|
|
151
158
|
# The net_tcp_client gem is required when logging over TCP.
|
152
159
|
if protocol == :tcp
|
153
160
|
begin
|
154
|
-
require
|
161
|
+
require "net/tcp_client"
|
155
162
|
rescue LoadError
|
156
|
-
raise
|
163
|
+
raise LoadError,
|
164
|
+
"Missing gem: net_tcp_client. This gem is required when logging over TCP. To fix this error: gem install net_tcp_client"
|
157
165
|
end
|
158
166
|
end
|
159
167
|
end
|
@@ -171,7 +179,7 @@ module SemanticLogger
|
|
171
179
|
::Syslog.send(method, application, options, facility)
|
172
180
|
when :tcp
|
173
181
|
@tcp_client_options[:server] = "#{@server}:#{@port}"
|
174
|
-
@remote_syslog = Net::TCPClient.new(
|
182
|
+
@remote_syslog = Net::TCPClient.new(**@tcp_client_options)
|
175
183
|
# Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
|
176
184
|
@remote_syslog.logger = logger
|
177
185
|
when :udp
|
@@ -186,7 +194,7 @@ module SemanticLogger
|
|
186
194
|
case @protocol
|
187
195
|
when :syslog
|
188
196
|
# Since the Ruby Syslog API supports sprintf format strings, double up all existing '%'
|
189
|
-
message = formatter.call(log, self).gsub
|
197
|
+
message = formatter.call(log, self).gsub "%", "%%"
|
190
198
|
::Syslog.log @level_map[log.level], message
|
191
199
|
when :tcp
|
192
200
|
@remote_syslog.retry_on_connection_failure { @remote_syslog.write("#{formatter.call(log, self)}\r\n") }
|
@@ -209,7 +217,7 @@ module SemanticLogger
|
|
209
217
|
# Format is text output without the time
|
210
218
|
SemanticLogger::Formatters::Default.new(time_format: nil)
|
211
219
|
else
|
212
|
-
SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map)
|
220
|
+
SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map, max_size: max_size)
|
213
221
|
end
|
214
222
|
end
|
215
223
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
begin
|
2
|
-
require
|
2
|
+
require "net/tcp_client"
|
3
3
|
rescue LoadError
|
4
|
-
raise 'Gem net_tcp_client is required for logging over TCP. Please add the gem "net_tcp_client" to your Gemfile.'
|
4
|
+
raise LoadError, 'Gem net_tcp_client is required for logging over TCP. Please add the gem "net_tcp_client" to your Gemfile.'
|
5
5
|
end
|
6
6
|
|
7
|
-
raise
|
7
|
+
raise "Net::TCPClient v2.0 or greater is required to log over TCP" unless Net::TCPClient::VERSION.to_f >= 2.0
|
8
8
|
|
9
9
|
module SemanticLogger
|
10
10
|
module Appender
|
@@ -89,7 +89,7 @@ module SemanticLogger
|
|
89
89
|
# where multiple sends are expected during a single response
|
90
90
|
# Default: true
|
91
91
|
#
|
92
|
-
# :connect_retry_count [
|
92
|
+
# :connect_retry_count [Integer]
|
93
93
|
# Number of times to retry connecting when a connection fails
|
94
94
|
# Default: 10
|
95
95
|
#
|
@@ -97,7 +97,7 @@ module SemanticLogger
|
|
97
97
|
# Number of seconds between connection retry attempts after the first failed attempt
|
98
98
|
# Default: 0.5
|
99
99
|
#
|
100
|
-
# :retry_count [
|
100
|
+
# :retry_count [Integer]
|
101
101
|
# Number of times to retry when calling #retry_on_connection_failure
|
102
102
|
# This is independent of :connect_retry_count which still applies with
|
103
103
|
# connection failures. This retry controls upto how many times to retry the
|
@@ -182,23 +182,23 @@ module SemanticLogger
|
|
182
182
|
# connect_retry_count: 5
|
183
183
|
# )
|
184
184
|
def initialize(separator: "\n",
|
185
|
-
level: nil, formatter: nil, filter: nil, application: nil, host: nil, metrics: false,
|
185
|
+
level: nil, formatter: nil, filter: nil, application: nil, environment: nil, host: nil, metrics: false,
|
186
186
|
**tcp_client_args, &block)
|
187
187
|
@separator = separator
|
188
188
|
@tcp_client_args = tcp_client_args
|
189
189
|
|
190
190
|
# Use the internal logger so that errors with remote logging are only written locally.
|
191
191
|
Net::TCPClient.logger = logger
|
192
|
-
Net::TCPClient.logger.name =
|
192
|
+
Net::TCPClient.logger.name = "Net::TCPClient"
|
193
193
|
|
194
|
-
super(level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
|
194
|
+
super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, &block)
|
195
195
|
reopen
|
196
196
|
end
|
197
197
|
|
198
198
|
# After forking an active process call #reopen to re-open the handles to resources.
|
199
199
|
def reopen
|
200
200
|
close
|
201
|
-
@tcp_client = Net::TCPClient.new(
|
201
|
+
@tcp_client = Net::TCPClient.new(**@tcp_client_args)
|
202
202
|
end
|
203
203
|
|
204
204
|
# Write the log using the specified protocol and server.
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "socket"
|
2
2
|
module SemanticLogger
|
3
3
|
module Appender
|
4
4
|
# UDP log appender.
|
@@ -74,7 +74,7 @@ module SemanticLogger
|
|
74
74
|
def reopen
|
75
75
|
close
|
76
76
|
@socket = UDPSocket.new
|
77
|
-
host, port = server.split(
|
77
|
+
host, port = server.split(":")
|
78
78
|
@socket.connect(host, port.to_i)
|
79
79
|
end
|
80
80
|
|
@@ -32,7 +32,7 @@ module SemanticLogger
|
|
32
32
|
# require 'logger'
|
33
33
|
# require 'semantic_logger'
|
34
34
|
#
|
35
|
-
# ruby_logger = Logger.new(
|
35
|
+
# ruby_logger = Logger.new($stdout)
|
36
36
|
# SemanticLogger.add_appender(logger: ruby_logger)
|
37
37
|
#
|
38
38
|
# logger = SemanticLogger['test']
|
@@ -45,7 +45,8 @@ module SemanticLogger
|
|
45
45
|
# Check if the custom appender responds to all the log levels. For example Ruby ::Logger
|
46
46
|
does_not_implement = LEVELS[1..-1].find { |i| !@logger.respond_to?(i) }
|
47
47
|
if does_not_implement
|
48
|
-
raise(ArgumentError,
|
48
|
+
raise(ArgumentError,
|
49
|
+
"Supplied logger does not implement:#{does_not_implement}. It must implement all of #{LEVELS[1..-1].inspect}")
|
49
50
|
end
|
50
51
|
|
51
52
|
super(**args, &block)
|