semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -2,6 +2,10 @@ require "json"
|
|
|
2
2
|
module SemanticLogger
|
|
3
3
|
module Formatters
|
|
4
4
|
class Signalfx < Base
|
|
5
|
+
# Text of a dimension value that is blank (see #blank_value?).
|
|
6
|
+
BLANK_VALUE = /\A[[:space:]]*\z/
|
|
7
|
+
private_constant :BLANK_VALUE
|
|
8
|
+
|
|
5
9
|
attr_accessor :token, :dimensions, :hash, :gauge_name, :counter_name
|
|
6
10
|
|
|
7
11
|
def initialize(token:,
|
|
@@ -21,9 +25,14 @@ module SemanticLogger
|
|
|
21
25
|
# Create SignalFx friendly metric.
|
|
22
26
|
# Strip leading '/'
|
|
23
27
|
# Convert remaining '/' to '.'
|
|
28
|
+
# A name that is not valid UTF-8, such as one with invalid bytes or in UTF-16, which matching
|
|
29
|
+
# it raises for, is converted to valid UTF-8 first (see Utils.encode_utf8_string).
|
|
30
|
+
#
|
|
31
|
+
# A metric with dimensions that are not blank (see #present_dimensions) is sent under its own name.
|
|
32
|
+
# Any other is sent as the gauge or the counter, with the class and action taken from its name.
|
|
24
33
|
def metric
|
|
25
|
-
name = log.metric.to_s.sub(%r{\A/+}, "")
|
|
26
|
-
if
|
|
34
|
+
name = Utils.encode_utf8_string(log.metric.to_s).sub(%r{\A/+}, "")
|
|
35
|
+
if metric_dimensions
|
|
27
36
|
name.tr!("/", ".")
|
|
28
37
|
hash[:metric] = name
|
|
29
38
|
else
|
|
@@ -53,21 +62,19 @@ module SemanticLogger
|
|
|
53
62
|
hash[:value] = log.metric_amount || log.duration || 1
|
|
54
63
|
end
|
|
55
64
|
|
|
56
|
-
# Dimensions for this metric
|
|
65
|
+
# Dimensions for this metric: those of the log entry that are not blank, or when it has none, the
|
|
66
|
+
# named tags listed in the dimensions option whose values are not blank (see #blank_value?).
|
|
57
67
|
def format_dimensions
|
|
58
68
|
h = (hash[:dimensions] ||= {})
|
|
59
|
-
if
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
h[name] = value unless value.empty?
|
|
63
|
-
end
|
|
64
|
-
else
|
|
69
|
+
if metric_dimensions
|
|
70
|
+
h.merge!(metric_dimensions)
|
|
71
|
+
elsif dimensions
|
|
65
72
|
log.named_tags.each_pair do |name, value|
|
|
66
|
-
name
|
|
67
|
-
|
|
68
|
-
next if value.empty?
|
|
73
|
+
name = name.to_sym
|
|
74
|
+
next unless dimensions.include?(name)
|
|
69
75
|
|
|
70
|
-
|
|
76
|
+
value = value.to_s
|
|
77
|
+
h[name] = value unless blank_value?(value)
|
|
71
78
|
end
|
|
72
79
|
end
|
|
73
80
|
h[:host] = logger.host if log_host && logger.host
|
|
@@ -91,7 +98,7 @@ module SemanticLogger
|
|
|
91
98
|
if log.duration
|
|
92
99
|
data[:gauge] = [hash]
|
|
93
100
|
# Also send a count metric whenever it is a gauge so that it can be counted.
|
|
94
|
-
unless
|
|
101
|
+
unless metric_dimensions
|
|
95
102
|
count_hash = hash.dup
|
|
96
103
|
count_hash[:value] = log.metric_amount || 1
|
|
97
104
|
count_hash[:metric] = counter_name
|
|
@@ -125,7 +132,7 @@ module SemanticLogger
|
|
|
125
132
|
add_gauge(gauges, hash)
|
|
126
133
|
|
|
127
134
|
# Also send a count metric whenever it is a gauge so that it can be counted.
|
|
128
|
-
unless
|
|
135
|
+
unless metric_dimensions
|
|
129
136
|
count_hash = hash.dup
|
|
130
137
|
count_hash[:value] = log.metric_amount || 1
|
|
131
138
|
count_hash[:metric] = counter_name
|
|
@@ -141,8 +148,41 @@ module SemanticLogger
|
|
|
141
148
|
Utils.to_json(data)
|
|
142
149
|
end
|
|
143
150
|
|
|
151
|
+
# The log entry to format. Also keeps the dimensions of its metric that are sent (see
|
|
152
|
+
# #present_dimensions), which formatting the entry asks for several times.
|
|
153
|
+
def log=(log)
|
|
154
|
+
super
|
|
155
|
+
@metric_dimensions = log && present_dimensions
|
|
156
|
+
end
|
|
157
|
+
|
|
144
158
|
private
|
|
145
159
|
|
|
160
|
+
# The dimensions of the entry's metric that are sent, or nil when there are none.
|
|
161
|
+
attr_reader :metric_dimensions
|
|
162
|
+
|
|
163
|
+
# Returns [Hash] the dimensions of the entry's metric, as Log#dimensions? tells, with each value as
|
|
164
|
+
# text, leaving out those that are blank (see #blank_value?). Returns nil when there are none, or only
|
|
165
|
+
# blank ones, such as `{state: ""}`, so that the metric is sent as a metric without dimensions is:
|
|
166
|
+
# as the gauge or the counter, with the class and action taken from its name. Log#dimensions?
|
|
167
|
+
# counts such dimensions, which the formatters that write the entry's data write as they are.
|
|
168
|
+
def present_dimensions
|
|
169
|
+
return unless log.dimensions?
|
|
170
|
+
|
|
171
|
+
present = {}
|
|
172
|
+
log.dimensions.each_pair do |name, value|
|
|
173
|
+
value = value.to_s
|
|
174
|
+
present[name] = value unless blank_value?(value)
|
|
175
|
+
end
|
|
176
|
+
present unless present.empty?
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Returns [true|false] whether the text of a dimension value is blank: empty, as for nil, or only
|
|
180
|
+
# whitespace, which includes Unicode whitespace such as a no-break space, but not NUL. The text is
|
|
181
|
+
# converted to valid UTF-8 first (see Utils.encode_utf8_string): matching raises for UTF-8 text
|
|
182
|
+
# with invalid bytes, or text that is not ASCII-compatible, such as UTF-16, and would not find the
|
|
183
|
+
# Unicode whitespace in text in an encoding such as binary or Shift_JIS.
|
|
184
|
+
def blank_value?(text) = BLANK_VALUE.match?(Utils.encode_utf8_string(text))
|
|
185
|
+
|
|
146
186
|
def add_gauge(gauges, metric)
|
|
147
187
|
gauges << metric
|
|
148
188
|
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
module SemanticLogger
|
|
2
2
|
module Formatters
|
|
3
3
|
class Syslog < Default
|
|
4
|
+
include SyslogHeader
|
|
5
|
+
|
|
4
6
|
attr_accessor :max_size
|
|
5
7
|
attr_reader :facility, :level_map
|
|
6
8
|
|
|
@@ -173,9 +175,9 @@ module SemanticLogger
|
|
|
173
175
|
# Create Syslog Packet
|
|
174
176
|
def create_syslog_packet(message)
|
|
175
177
|
packet = SyslogProtocol::Packet.new
|
|
176
|
-
packet.hostname = logger.host
|
|
178
|
+
packet.hostname = syslog_hostname(logger.host)
|
|
177
179
|
packet.facility = @facility_number
|
|
178
|
-
packet.tag = logger.application
|
|
180
|
+
packet.tag = syslog_tag(logger.application)
|
|
179
181
|
packet.content = message
|
|
180
182
|
packet.time = log.time
|
|
181
183
|
packet.severity = level_map[log.level]
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
module SemanticLogger
|
|
2
2
|
module Formatters
|
|
3
3
|
class SyslogCee < Raw
|
|
4
|
+
include SyslogHeader
|
|
5
|
+
|
|
4
6
|
attr_accessor :max_size
|
|
5
7
|
attr_reader :facility, :level_map
|
|
6
8
|
|
|
@@ -20,17 +22,21 @@ module SemanticLogger
|
|
|
20
22
|
# Maximum size of each syslog packet, in bytes.
|
|
21
23
|
# Default: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE (1024)
|
|
22
24
|
#
|
|
25
|
+
# Any other options are those of SemanticLogger::Formatters::Raw, such as `log_host` or
|
|
26
|
+
# `log_exception_details`. Its time options (`time_format`, `time_key`, and `precision`)
|
|
27
|
+
# have no effect, since the time is in the header of the syslog packet.
|
|
28
|
+
#
|
|
23
29
|
# Example:
|
|
24
30
|
# # Log via udp to a remote syslog server on host: `server1` and port `8514`, using the CEE format.
|
|
25
31
|
# SemanticLogger.add_appender(appender: :syslog, formatter: :syslog_cee, url: 'udp://server1:8514')
|
|
26
32
|
def initialize(facility: SemanticLogger::Formatters::Syslog::LOG_USER, level_map: SemanticLogger::Formatters::Syslog::LevelMap.new,
|
|
27
|
-
max_size: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE)
|
|
33
|
+
max_size: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE, **args)
|
|
28
34
|
SemanticLogger::Formatters::Syslog.require_syslog_protocol
|
|
29
35
|
|
|
30
36
|
self.facility = facility
|
|
31
37
|
self.level_map = level_map
|
|
32
38
|
@max_size = max_size
|
|
33
|
-
super()
|
|
39
|
+
super(**args)
|
|
34
40
|
end
|
|
35
41
|
|
|
36
42
|
# Set the facility to one of the ::Syslog::LOG_* facility constants.
|
|
@@ -60,9 +66,9 @@ module SemanticLogger
|
|
|
60
66
|
# Create Syslog Packet
|
|
61
67
|
def create_syslog_packet(message)
|
|
62
68
|
packet = SyslogProtocol::Packet.new
|
|
63
|
-
packet.hostname = logger.host
|
|
69
|
+
packet.hostname = syslog_hostname(logger.host)
|
|
64
70
|
packet.facility = @facility_number
|
|
65
|
-
packet.tag = logger.application
|
|
71
|
+
packet.tag = syslog_tag(logger.application)
|
|
66
72
|
packet.content = message
|
|
67
73
|
packet.time = log.time
|
|
68
74
|
packet.severity = level_map[log.level]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
module Formatters
|
|
3
|
+
# The host name and the application name (the TAG) that the formatters building a remote syslog
|
|
4
|
+
# packet (Syslog and SyslogCee) write in its header.
|
|
5
|
+
#
|
|
6
|
+
# The syslog_protocol gem, following RFC 3164, allows only printable ASCII characters other than
|
|
7
|
+
# the space (33 to 126) in either, and up to 32 in the TAG, and raises for any other, which made
|
|
8
|
+
# the formatter lose every entry, such as for an application named "Café", or with a name over
|
|
9
|
+
# 32 characters. So the other characters are left out, as the spaces always were from the TAG,
|
|
10
|
+
# and the TAG is cut to 32 characters. When no character is left, "localhost" or
|
|
11
|
+
# "SemanticLogger" is written instead.
|
|
12
|
+
#
|
|
13
|
+
# Each is worked out once, and again only when the host or application name changes.
|
|
14
|
+
module SyslogHeader
|
|
15
|
+
# The most characters that the TAG of a syslog header can hold.
|
|
16
|
+
MAX_TAG_SIZE = 32
|
|
17
|
+
|
|
18
|
+
# Written in place of a host name with no character left.
|
|
19
|
+
DEFAULT_HOSTNAME = "localhost".encode(Encoding::US_ASCII).freeze
|
|
20
|
+
|
|
21
|
+
# Written in place of an application name with no character left.
|
|
22
|
+
DEFAULT_TAG = "SemanticLogger".encode(Encoding::US_ASCII).freeze
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
# Returns [String] the host name as the syslog header holds it.
|
|
27
|
+
def syslog_hostname(host) = syslog_header_field(:hostname, host, DEFAULT_HOSTNAME)
|
|
28
|
+
|
|
29
|
+
# Returns [String] the application name as the TAG of the syslog header holds it.
|
|
30
|
+
def syslog_tag(application) = syslog_header_field(:tag, application, DEFAULT_TAG, MAX_TAG_SIZE)
|
|
31
|
+
|
|
32
|
+
# Returns [String] the field cleaned (see SyslogHeader), frozen, from the cache when the text
|
|
33
|
+
# is the one it was worked out from, or holds the same characters, such as an appender's own
|
|
34
|
+
# name that was changed in place, which the cache keeps a frozen copy of. Any other object,
|
|
35
|
+
# other than a Symbol, is read as its #to_s first, so that one changed in place is noticed too.
|
|
36
|
+
def syslog_header_field(field, text, fallback, max_size = nil)
|
|
37
|
+
text = text.to_s unless text.is_a?(String) || text.is_a?(Symbol)
|
|
38
|
+
cached = (@syslog_header_fields ||= {})[field]
|
|
39
|
+
return cached.last if cached && (cached.first.equal?(text) || cached.first == text)
|
|
40
|
+
|
|
41
|
+
value = text.to_s.b.delete("^\x21-\x7E")
|
|
42
|
+
value = value[0, max_size] if max_size
|
|
43
|
+
value = value.empty? ? fallback : value.force_encoding(Encoding::US_ASCII).freeze
|
|
44
|
+
@syslog_header_fields[field] = [text.is_a?(String) && !text.frozen? ? text.dup.freeze : text, value]
|
|
45
|
+
value
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -14,6 +14,7 @@ module SemanticLogger
|
|
|
14
14
|
autoload :Logfmt, "semantic_logger/formatters/logfmt"
|
|
15
15
|
autoload :Pattern, "semantic_logger/formatters/pattern"
|
|
16
16
|
autoload :SyslogCee, "semantic_logger/formatters/syslog_cee"
|
|
17
|
+
autoload :SyslogHeader, "semantic_logger/formatters/syslog_header"
|
|
17
18
|
autoload :NewRelicLogs, "semantic_logger/formatters/new_relic_logs"
|
|
18
19
|
autoload :Loki, "semantic_logger/formatters/loki"
|
|
19
20
|
|