semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -10,15 +10,29 @@ module SemanticLogger
|
|
|
10
10
|
# SemanticLogger.add_appender(
|
|
11
11
|
# appender: :udp,
|
|
12
12
|
# server: 'server:3300',
|
|
13
|
+
# metrics: false
|
|
13
14
|
# )
|
|
14
15
|
class Udp < SemanticLogger::Subscriber
|
|
16
|
+
# The host and port of a server, with an IPv6 address in brackets, as a URL writes it.
|
|
17
|
+
SERVER = /\A(?:\[(?<ipv6>[^\[\]]+)\]|(?<host>[^:\[\]]+)):(?<port>\d+)\z/
|
|
18
|
+
private_constant :SERVER
|
|
19
|
+
|
|
20
|
+
# The codes of a lookup that found no address of the family it asked for, rather than failing, of those that
|
|
21
|
+
# the platform defines.
|
|
22
|
+
NO_ADDRESS = %i[EAI_NONAME EAI_NODATA EAI_ADDRFAMILY].filter_map do |name|
|
|
23
|
+
Socket.const_get(name) if Socket.const_defined?(name)
|
|
24
|
+
end.freeze
|
|
25
|
+
private_constant :NO_ADDRESS
|
|
26
|
+
|
|
15
27
|
attr_accessor :server, :udp_flags
|
|
16
28
|
attr_reader :socket
|
|
17
29
|
|
|
18
30
|
# Create UDP log appender.
|
|
19
31
|
#
|
|
20
32
|
# server: [String]
|
|
21
|
-
#
|
|
33
|
+
# The server to write UDP messages to, as "host:port", such as "localhost:8088", with an IPv6
|
|
34
|
+
# address in brackets, as a URL writes it, such as "[::1]:8088". A host name is sent to over
|
|
35
|
+
# IPv4 when it has an IPv4 address, and otherwise, or when it cannot connect over IPv4, over IPv6.
|
|
22
36
|
#
|
|
23
37
|
# udp_flags: [Integer]
|
|
24
38
|
# Should be a bitwise OR of Socket::MSG_* constants.
|
|
@@ -35,12 +49,13 @@ module SemanticLogger
|
|
|
35
49
|
#
|
|
36
50
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
|
37
51
|
# Override the log level for this appender.
|
|
38
|
-
# Default:
|
|
52
|
+
# Default: none (:trace), so it writes every entry that a logger emits; each logger
|
|
53
|
+
# filters by its own level.
|
|
39
54
|
#
|
|
40
|
-
# formatter: [Object|Proc]
|
|
55
|
+
# formatter: [Object|Proc|Symbol|Hash]
|
|
41
56
|
# An instance of a class that implements #call, or a Proc to be used to format
|
|
42
57
|
# the output from this appender
|
|
43
|
-
# Default:
|
|
58
|
+
# Default: :json
|
|
44
59
|
#
|
|
45
60
|
# filter: [Regexp|Proc]
|
|
46
61
|
# RegExp: Only include log messages where the class name matches the supplied.
|
|
@@ -50,7 +65,7 @@ module SemanticLogger
|
|
|
50
65
|
#
|
|
51
66
|
# metrics: [Boolean]
|
|
52
67
|
# Send metrics only events over udp.
|
|
53
|
-
# Default: true
|
|
68
|
+
# Default: true, with a deprecation warning when it is not supplied, since v6 changes it to false.
|
|
54
69
|
#
|
|
55
70
|
# Limitations:
|
|
56
71
|
# * UDP packet size is limited by the connected network and any routers etc
|
|
@@ -59,39 +74,52 @@ module SemanticLogger
|
|
|
59
74
|
# Example:
|
|
60
75
|
# SemanticLogger.add_appender(
|
|
61
76
|
# appender: :udp,
|
|
62
|
-
# server: 'server:3300'
|
|
77
|
+
# server: 'server:3300',
|
|
78
|
+
# metrics: false
|
|
63
79
|
# )
|
|
64
|
-
def initialize(server:, udp_flags: 0, metrics:
|
|
80
|
+
def initialize(server:, udp_flags: 0, metrics: METRICS_NOT_SUPPLIED, **args, &)
|
|
65
81
|
@server = server
|
|
66
82
|
@udp_flags = udp_flags
|
|
67
83
|
|
|
68
|
-
super(metrics: metrics, **args, &)
|
|
84
|
+
super(metrics: deprecated_metrics_default(metrics, :udp), **args, &)
|
|
69
85
|
reopen
|
|
70
86
|
end
|
|
71
87
|
|
|
72
88
|
# After forking an active process call #reopen to re-open
|
|
73
89
|
# open the handles to resources
|
|
90
|
+
#
|
|
91
|
+
# Connects a new socket to the server before closing the one it replaces. When it cannot, it
|
|
92
|
+
# raises, keeping the socket it has: ArgumentError for a server that is not "host:port", or
|
|
93
|
+
# the error from resolving or connecting, such as SocketError for a host name that does not resolve.
|
|
94
|
+
# An error closing the socket it replaced is reported to the internal logger as a warning, since the
|
|
95
|
+
# reopen itself worked.
|
|
74
96
|
def reopen
|
|
75
|
-
|
|
76
|
-
@socket
|
|
77
|
-
|
|
78
|
-
|
|
97
|
+
socket = connected_socket
|
|
98
|
+
old_socket = @socket
|
|
99
|
+
@socket = socket
|
|
100
|
+
close_socket(old_socket, replaced: true)
|
|
79
101
|
end
|
|
80
102
|
|
|
81
|
-
# Write the log using the specified protocol and server.
|
|
103
|
+
# Write the log using the specified protocol and server. Opens a new socket, connected to the server, first
|
|
104
|
+
# when the appender has none, after #close.
|
|
82
105
|
def log(log)
|
|
83
|
-
@socket.send(format_log(log), udp_flags)
|
|
106
|
+
(@socket ||= connected_socket).send(format_log(log), udp_flags)
|
|
84
107
|
true
|
|
85
108
|
end
|
|
86
109
|
|
|
87
|
-
# Flush is called by the semantic_logger during shutdown.
|
|
110
|
+
# Flush is called by the semantic_logger during shutdown. Does nothing after #close, which left no socket.
|
|
88
111
|
def flush
|
|
89
112
|
@socket&.flush
|
|
90
113
|
end
|
|
91
114
|
|
|
92
|
-
#
|
|
115
|
+
# Closes the socket, such as when the appender is removed, and forgets it, so that the next write opens a new
|
|
116
|
+
# one. An error closing it is reported to the internal logger as a warning, rather than raised.
|
|
117
|
+
#
|
|
118
|
+
# In a forked child, the socket is the child's own copy of the parent's, so closing it leaves the parent's open.
|
|
93
119
|
def close
|
|
94
|
-
@socket
|
|
120
|
+
socket = @socket
|
|
121
|
+
@socket = nil
|
|
122
|
+
close_socket(socket)
|
|
95
123
|
end
|
|
96
124
|
|
|
97
125
|
# A log message too large for a datagram (Errno::EMSGSIZE) cannot be sent by retrying.
|
|
@@ -106,7 +134,91 @@ module SemanticLogger
|
|
|
106
134
|
|
|
107
135
|
private
|
|
108
136
|
|
|
109
|
-
#
|
|
137
|
+
# Closes the given socket, if any: the one that a reopen replaced, when replaced: is true. An error closing
|
|
138
|
+
# it is reported to the internal logger as a warning, rather than raised.
|
|
139
|
+
def close_socket(socket, replaced: false)
|
|
140
|
+
socket&.close
|
|
141
|
+
rescue StandardError => e
|
|
142
|
+
logger.warn("Could not close the socket to the UDP server#{' that a reopen replaced' if replaced}", e)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Returns [UDPSocket] a new socket, connected to the first of the server's addresses that it can connect to, as
|
|
146
|
+
# Ruby's own UDPSocket#connect tries the addresses of its socket's family in turn. A host name's IPv4 addresses
|
|
147
|
+
# come first, so that a name with both is still sent to over IPv4, as when the socket was IPv4 only, whichever
|
|
148
|
+
# address the resolver returns first. Its IPv6 addresses are looked up only when the resolver answers that it has
|
|
149
|
+
# no IPv4 address, or it has none that it can connect to, so that a resolver that does not answer for them does
|
|
150
|
+
# not hold up the others. Each family keeps the resolver's order. When looking up its IPv4 addresses fails for
|
|
151
|
+
# another reason, such as a resolver that does not answer in time, raises that SocketError at once, rather than
|
|
152
|
+
# sending over IPv6 to a name that has IPv4 addresses, or waiting for a resolver that is down a second time.
|
|
153
|
+
#
|
|
154
|
+
# When it cannot connect to any address, raises the error from the first one it tried, the one it prefers. When
|
|
155
|
+
# the host has no address, such as a name that does not resolve, raises the SocketError from looking up its IPv6
|
|
156
|
+
# addresses, with the one from looking up its IPv4 addresses as its cause (see #raise_lookup_error).
|
|
157
|
+
def connected_socket
|
|
158
|
+
host, port, families = server_endpoint
|
|
159
|
+
connect_error = nil
|
|
160
|
+
lookup_errors = []
|
|
161
|
+
families.each do |family|
|
|
162
|
+
Addrinfo.getaddrinfo(host, port, family, :DGRAM).each do |address|
|
|
163
|
+
return connect_to(address)
|
|
164
|
+
rescue SystemCallError => e
|
|
165
|
+
connect_error ||= e
|
|
166
|
+
end
|
|
167
|
+
rescue SocketError => e
|
|
168
|
+
lookup_errors << e
|
|
169
|
+
break unless no_address?(e)
|
|
170
|
+
end
|
|
171
|
+
raise(connect_error) if connect_error
|
|
172
|
+
|
|
173
|
+
raise_lookup_error(host, lookup_errors)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Returns [true|false] whether the SocketError from looking up a host's addresses of one family says that it has
|
|
177
|
+
# none, rather than that the lookup failed. It cannot tell without the error's code, which Socket::ResolutionError
|
|
178
|
+
# has from Ruby 3.3, so on Ruby 3.2, or on JRuby, it takes any SocketError to say so, as before.
|
|
179
|
+
def no_address?(error)
|
|
180
|
+
!error.respond_to?(:error_code) || NO_ADDRESS.include?(error.error_code)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Raises the last of the SocketErrors from looking up the host's addresses, with the one before it, if any, as its
|
|
184
|
+
# cause, since it looked up the later family only because the earlier one had no address. Raises a SocketError
|
|
185
|
+
# of its own, naming the host, when no lookup raised, since JRuby returns no addresses for a family that the host
|
|
186
|
+
# has none in, rather than raising.
|
|
187
|
+
def raise_lookup_error(host, errors)
|
|
188
|
+
last, earlier = errors.reverse
|
|
189
|
+
raise(SocketError, "getaddrinfo: no address for #{host}") unless last
|
|
190
|
+
|
|
191
|
+
earlier ? raise(last, cause: earlier) : raise(last)
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Returns [UDPSocket] a new socket of the address's family, connected to it. Closes the socket when it does not
|
|
195
|
+
# return it, such as when connecting raises, or the thread is interrupted.
|
|
196
|
+
def connect_to(address)
|
|
197
|
+
socket = UDPSocket.new(address.afamily)
|
|
198
|
+
socket.connect(address.ip_address, address.ip_port)
|
|
199
|
+
connected = true
|
|
200
|
+
socket
|
|
201
|
+
ensure
|
|
202
|
+
socket&.close unless connected
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Returns [Array(String, Integer, Array<Symbol>)] the host and port of the server, and the address families to
|
|
206
|
+
# look the host up in, in the order to try them. A host in brackets is an IPv6 address. Any other host is an IPv4
|
|
207
|
+
# address, or a name, which is looked up for its IPv4 addresses first, then for its IPv6 addresses. Whitespace
|
|
208
|
+
# around the server, such as the newline at the end of an environment variable, is ignored.
|
|
209
|
+
def server_endpoint
|
|
210
|
+
match = SERVER.match(server.to_s.strip)
|
|
211
|
+
unless match
|
|
212
|
+
raise(ArgumentError,
|
|
213
|
+
"Invalid :server #{server.inspect}, it must be \"host:port\", such as \"localhost:8088\", " \
|
|
214
|
+
"with an IPv6 address in brackets, such as \"[::1]:8088\".")
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
port = match[:port].to_i
|
|
218
|
+
match[:ipv6] ? [match[:ipv6], port, %i[INET6]] : [match[:host], port, %i[INET INET6]]
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Returns [SemanticLogger::Formatters::Json] formatter default for this Appender
|
|
110
222
|
def default_formatter
|
|
111
223
|
SemanticLogger::Formatters::Json.new
|
|
112
224
|
end
|
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
module SemanticLogger
|
|
5
5
|
module Appender
|
|
6
6
|
class Wrapper < SemanticLogger::Subscriber
|
|
7
|
-
|
|
7
|
+
# The logger this appender writes to. Not #logger, which is the internal logger that this
|
|
8
|
+
# appender reports its own problems to, as every appender does (see Subscriber#logger).
|
|
9
|
+
attr_reader :wrapped_logger
|
|
8
10
|
|
|
9
11
|
# Forward all logging calls to the supplied logging instance.
|
|
10
12
|
#
|
|
@@ -14,7 +16,8 @@ module SemanticLogger
|
|
|
14
16
|
#
|
|
15
17
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
|
16
18
|
# Override the log level for this appender.
|
|
17
|
-
# Default:
|
|
19
|
+
# Default: none (:trace), so it writes every entry that a logger emits; each logger
|
|
20
|
+
# filters by its own level.
|
|
18
21
|
#
|
|
19
22
|
# formatter: [Object|Proc]
|
|
20
23
|
# An instance of a class that implements #call, or a Proc to be used to format
|
|
@@ -39,10 +42,10 @@ module SemanticLogger
|
|
|
39
42
|
#
|
|
40
43
|
# Install the `rails_semantic_logger` gem to replace the Rails logger with Semantic Logger.
|
|
41
44
|
def initialize(logger:, **args, &)
|
|
42
|
-
@
|
|
45
|
+
@wrapped_logger = logger
|
|
43
46
|
|
|
44
47
|
# Check if the custom appender responds to all the log levels. For example Ruby ::Logger
|
|
45
|
-
does_not_implement = LEVELS[1..].find { |i| !@
|
|
48
|
+
does_not_implement = LEVELS[1..].find { |i| !@wrapped_logger.respond_to?(i) }
|
|
46
49
|
if does_not_implement
|
|
47
50
|
raise(ArgumentError,
|
|
48
51
|
"Supplied logger does not implement:#{does_not_implement}. " \
|
|
@@ -57,20 +60,20 @@ module SemanticLogger
|
|
|
57
60
|
# Ruby or Rails Loggers
|
|
58
61
|
def log(log)
|
|
59
62
|
level = log.level == :trace ? :debug : log.level
|
|
60
|
-
@
|
|
63
|
+
@wrapped_logger.send(level, format_log(log))
|
|
61
64
|
true
|
|
62
65
|
end
|
|
63
66
|
|
|
64
67
|
# Flush all pending logs to disk.
|
|
65
68
|
# Waits for all queued log messages to be written to disk.
|
|
66
69
|
def flush
|
|
67
|
-
@
|
|
70
|
+
@wrapped_logger.flush if @wrapped_logger.respond_to?(:flush)
|
|
68
71
|
end
|
|
69
72
|
|
|
70
73
|
# Close underlying log
|
|
71
74
|
# Waits for all queued log messages to be written to disk.
|
|
72
75
|
def close
|
|
73
|
-
@
|
|
76
|
+
@wrapped_logger.close if @wrapped_logger.respond_to?(:close)
|
|
74
77
|
end
|
|
75
78
|
end
|
|
76
79
|
end
|
|
@@ -25,6 +25,7 @@ module SemanticLogger
|
|
|
25
25
|
autoload :SplunkHttp, "semantic_logger/appender/splunk_http"
|
|
26
26
|
autoload :Syslog, "semantic_logger/appender/syslog"
|
|
27
27
|
autoload :Tcp, "semantic_logger/appender/tcp"
|
|
28
|
+
autoload :TextOutput, "semantic_logger/appender/text_output"
|
|
28
29
|
autoload :Udp, "semantic_logger/appender/udp"
|
|
29
30
|
autoload :Wrapper, "semantic_logger/appender/wrapper"
|
|
30
31
|
autoload :SentryRuby, "semantic_logger/appender/sentry_ruby"
|
|
@@ -32,6 +33,10 @@ module SemanticLogger
|
|
|
32
33
|
# @formatter:on
|
|
33
34
|
|
|
34
35
|
# Returns [SemanticLogger::Subscriber] appender for the supplied options
|
|
36
|
+
#
|
|
37
|
+
# Raises ArgumentError when the appender is to be batched, with `batch: true` or by default, see
|
|
38
|
+
# Subscriber#batch_by_default?, and it says why it cannot be, see Subscriber#batch_refusal. When it fails
|
|
39
|
+
# after .build, it first closes the appender that .build says it created, see .close_created.
|
|
35
40
|
def self.factory(async: false, batch: nil,
|
|
36
41
|
max_queue_size: 10_000, lag_check_interval: 1_000, lag_threshold_s: 30,
|
|
37
42
|
batch_size: 300, batch_seconds: 5,
|
|
@@ -39,13 +44,16 @@ module SemanticLogger
|
|
|
39
44
|
async_max_retries: 100,
|
|
40
45
|
**args,
|
|
41
46
|
&)
|
|
42
|
-
appender = build(**args, &)
|
|
47
|
+
appender, created = build(**args, &)
|
|
43
48
|
|
|
44
49
|
# If appender implements #batch, then it should use the batch proxy by default,
|
|
45
50
|
# unless the appender opts out of batching by default (e.g. the HTTP appender).
|
|
46
51
|
batch = true if batch.nil? && appender.respond_to?(:batch) && appender.batch_by_default?
|
|
47
52
|
|
|
48
53
|
if batch == true
|
|
54
|
+
refusal = appender.batch_refusal
|
|
55
|
+
raise(ArgumentError, "add_appender cannot batch #{appender.class}: #{refusal}") if refusal
|
|
56
|
+
|
|
49
57
|
Appender::Async.new(
|
|
50
58
|
appender: appender,
|
|
51
59
|
batch: true,
|
|
@@ -70,21 +78,45 @@ module SemanticLogger
|
|
|
70
78
|
else
|
|
71
79
|
appender
|
|
72
80
|
end
|
|
81
|
+
rescue Exception
|
|
82
|
+
close_created(appender) if created
|
|
83
|
+
raise
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Closes the appender that .build created, since .factory failed after creating it, such as when the appender
|
|
87
|
+
# cannot be batched, and nothing else holds it, yet it may have opened its destination, such as a connection.
|
|
88
|
+
# An error from closing it is ignored, so that .factory raises the one that it failed with.
|
|
89
|
+
def self.close_created(appender)
|
|
90
|
+
appender.close
|
|
91
|
+
rescue StandardError
|
|
92
|
+
nil
|
|
73
93
|
end
|
|
94
|
+
private_class_method :close_created
|
|
74
95
|
|
|
75
|
-
# Returns [Subscriber]
|
|
96
|
+
# Returns [Subscriber, true | false] the appender for the supplied options, and whether .build created both it
|
|
97
|
+
# and its destination, so that nothing else holds them, and .factory closes it when it fails afterwards (see
|
|
98
|
+
# .close_created): true for the appender for `file_name:`, or one named by a Symbol as `appender:` or `metric:`.
|
|
99
|
+
# False for a Subscriber instance that the caller supplied as `appender:` or `metric:`, and for the appender that
|
|
100
|
+
# writes to the caller's own `io:` or `logger:`: closing the one for `logger:` closes that logger, and the stream
|
|
101
|
+
# it writes to, such as $stdout.
|
|
102
|
+
#
|
|
103
|
+
# It adds one destination, the first of file_name:, io:, logger:, appender:, and metric: that is given, in that
|
|
104
|
+
# order, and ignores any other, see .report_ignored_destinations.
|
|
76
105
|
def self.build(io: nil, file_name: nil, appender: nil, metric: nil, logger: nil, **args, &)
|
|
106
|
+
# Named in the order of the if / elsif chain below, which adds the first one given: keep the two in step.
|
|
107
|
+
report_ignored_destinations(file_name: file_name, io: io, logger: logger, appender: appender, metric: metric)
|
|
108
|
+
|
|
77
109
|
if file_name
|
|
78
|
-
SemanticLogger::Appender::File.new(file_name, **args, &)
|
|
110
|
+
[SemanticLogger::Appender::File.new(file_name, **args, &), true]
|
|
79
111
|
elsif io
|
|
80
|
-
SemanticLogger::Appender::IO.new(io, **args, &)
|
|
112
|
+
[SemanticLogger::Appender::IO.new(io, **args, &), false]
|
|
81
113
|
elsif logger
|
|
82
|
-
SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &)
|
|
114
|
+
[SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &), false]
|
|
83
115
|
elsif appender
|
|
84
116
|
if appender.is_a?(Symbol)
|
|
85
|
-
SemanticLogger::Utils.constantize_symbol(appender).new(**args)
|
|
117
|
+
[SemanticLogger::Utils.constantize_symbol(appender).new(**args, &), true]
|
|
86
118
|
elsif appender.is_a?(Subscriber)
|
|
87
|
-
appender
|
|
119
|
+
[subscriber_instance(:appender, appender, **args, &), false]
|
|
88
120
|
else
|
|
89
121
|
raise(ArgumentError,
|
|
90
122
|
"Parameter :appender must be either a Symbol or an object derived from " \
|
|
@@ -92,13 +124,13 @@ module SemanticLogger
|
|
|
92
124
|
end
|
|
93
125
|
elsif metric
|
|
94
126
|
if metric.is_a?(Symbol)
|
|
95
|
-
SemanticLogger::Utils.constantize_symbol(metric, "SemanticLogger::Metric").new(**args)
|
|
127
|
+
[SemanticLogger::Utils.constantize_symbol(metric, "SemanticLogger::Metric").new(**args, &), true]
|
|
96
128
|
elsif metric.is_a?(Subscriber)
|
|
97
|
-
metric
|
|
129
|
+
[subscriber_instance(:metric, metric, **args, &), false]
|
|
98
130
|
else
|
|
99
131
|
raise(ArgumentError,
|
|
100
132
|
"Parameter :metric must be either a Symbol or an object derived from " \
|
|
101
|
-
"SemanticLogger::Subscriber, not: #{
|
|
133
|
+
"SemanticLogger::Subscriber, not: #{metric.inspect}")
|
|
102
134
|
end
|
|
103
135
|
else
|
|
104
136
|
raise(ArgumentError,
|
|
@@ -108,5 +140,78 @@ module SemanticLogger
|
|
|
108
140
|
end
|
|
109
141
|
|
|
110
142
|
private_class_method :build
|
|
143
|
+
|
|
144
|
+
# Returns [SemanticLogger::Base] the supplied logger, ready to be an internal logger: the one that
|
|
145
|
+
# Semantic Logger reports its own problems to (see Processor.logger=).
|
|
146
|
+
#
|
|
147
|
+
# An internal logger is named after whatever reports to it, logs at :trace, and takes an
|
|
148
|
+
# exception after the message, as every Semantic Logger appender does. So an appender, such as the
|
|
149
|
+
# default IO appender, or any other logger that logs at :trace, is returned as it is. Any other
|
|
150
|
+
# logger, such as a Ruby or Rails Logger, does none of that, so it is wrapped in a Wrapper
|
|
151
|
+
# appender, which writes each report to it as text, with the exception. Its own level then decides
|
|
152
|
+
# which reports it writes, apart from the :trace reports, which are below its lowest level, :debug,
|
|
153
|
+
# so are left out.
|
|
154
|
+
#
|
|
155
|
+
# Returns nil for nil or false, which restores the default internal logger.
|
|
156
|
+
#
|
|
157
|
+
# Raises ArgumentError for a SemanticLogger::Logger: it logs back into the processor, so the
|
|
158
|
+
# report of an appender that fails would go to that appender, fail again, and be reported again.
|
|
159
|
+
def self.internal_logger(logger)
|
|
160
|
+
return unless logger
|
|
161
|
+
|
|
162
|
+
if logger.is_a?(SemanticLogger::Logger)
|
|
163
|
+
raise(ArgumentError,
|
|
164
|
+
"A SemanticLogger::Logger cannot be the internal logger, since its reports of a failing " \
|
|
165
|
+
"appender would be logged to that appender. Use an appender, such as " \
|
|
166
|
+
"SemanticLogger::Appender::IO.new($stderr), or a Ruby Logger.")
|
|
167
|
+
end
|
|
168
|
+
return logger if logger.respond_to?(:trace)
|
|
169
|
+
|
|
170
|
+
SemanticLogger::Appender::Wrapper.new(logger: logger, level: :debug)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Returns [Subscriber] the appender or metric subscriber supplied as an instance, with the `option` that named it,
|
|
174
|
+
# `:appender` or `:metric`, as it is, since it was created with its own options. The options and block that .build
|
|
175
|
+
# would hand to an appender that it creates are not used, so reports a deprecation warning that names each one,
|
|
176
|
+
# since v6 raises ArgumentError for them. An option whose value is nil, the default of a Subscriber, sets nothing,
|
|
177
|
+
# so it is not reported, and v6 must not raise for it either, so that a caller can pass on its optional settings.
|
|
178
|
+
# Since an option of .factory that is misspelled arrives here too, the warning names the options that do apply to
|
|
179
|
+
# an instance. Remove in v6, raising ArgumentError in its place.
|
|
180
|
+
def self.subscriber_instance(option, subscriber, **args)
|
|
181
|
+
ignored = args.compact.keys.map { |key| "#{key}:" }
|
|
182
|
+
ignored.unshift("the formatter block") if block_given?
|
|
183
|
+
return subscriber if ignored.empty?
|
|
184
|
+
|
|
185
|
+
pronoun = ignored.one? ? "it" : "them"
|
|
186
|
+
Deprecation.warn(
|
|
187
|
+
"add_appender ignores #{word_list(ignored)} when #{option}: is a Subscriber instance (#{subscriber.class}), " \
|
|
188
|
+
"and v6 raises ArgumentError for #{pronoun}. Set #{pronoun} on the instance itself; " \
|
|
189
|
+
"only async:, batch:, and the queue options of add_appender apply to an instance."
|
|
190
|
+
)
|
|
191
|
+
subscriber
|
|
192
|
+
end
|
|
193
|
+
private_class_method :subscriber_instance
|
|
194
|
+
|
|
195
|
+
# Reports a deprecation warning when .build is given more than one of the destinations, in the order in which it
|
|
196
|
+
# takes the first one, naming the one that it uses and the others, which it ignores, since v6 raises ArgumentError
|
|
197
|
+
# for them. A destination given as nil, or false, is not one, as .build has always taken it, so that a caller can
|
|
198
|
+
# pass on settings of which only one is set, such as `file_name: options[:file_name], io: options[:io]`, and v6
|
|
199
|
+
# must not raise for it either. Remove in v6, raising ArgumentError in its place.
|
|
200
|
+
def self.report_ignored_destinations(**destinations)
|
|
201
|
+
used, *ignored = destinations.filter_map { |name, value| "#{name}:" if value }
|
|
202
|
+
return if ignored.empty?
|
|
203
|
+
|
|
204
|
+
Deprecation.warn(
|
|
205
|
+
"add_appender adds one destination, so it uses #{used} and ignores #{word_list(ignored)}, " \
|
|
206
|
+
"and v6 raises ArgumentError when given more than one. Call add_appender once for each destination."
|
|
207
|
+
)
|
|
208
|
+
end
|
|
209
|
+
private_class_method :report_ignored_destinations
|
|
210
|
+
|
|
211
|
+
# Returns [String] the words as a list in a sentence, such as "a", "a and b", or "a, b, and c".
|
|
212
|
+
def self.word_list(words)
|
|
213
|
+
words.size < 3 ? words.join(" and ") : "#{words[0..-2].join(', ')}, and #{words.last}"
|
|
214
|
+
end
|
|
215
|
+
private_class_method :word_list
|
|
111
216
|
end
|
|
112
217
|
end
|
|
@@ -1,11 +1,26 @@
|
|
|
1
1
|
module SemanticLogger
|
|
2
2
|
# Manage a collection of appenders.
|
|
3
|
+
#
|
|
4
|
+
# While the process exits, it also knows which of its appenders it skips: those without a queue of
|
|
5
|
+
# their own whose destination failed meanwhile, and how many log messages each has not been given
|
|
6
|
+
# since (see #process_exiting=).
|
|
3
7
|
class Appenders < Concurrent::Array
|
|
8
|
+
# An appender skipped while the process exits: the error with which its destination failed, and
|
|
9
|
+
# the number of log messages it has not been given since it was last reported.
|
|
10
|
+
Skipped = Struct.new(:exception, :discarded)
|
|
11
|
+
private_constant :Skipped
|
|
12
|
+
|
|
13
|
+
# No appender is skipped while the process is not exiting.
|
|
14
|
+
NONE_SKIPPED = {}.freeze
|
|
15
|
+
private_constant :NONE_SKIPPED
|
|
16
|
+
|
|
4
17
|
attr_accessor :logger
|
|
5
18
|
|
|
6
19
|
def initialize(logger = Processor.logger.dup)
|
|
7
20
|
@logger = logger
|
|
8
21
|
@logger.name = self.class.name
|
|
22
|
+
@exiting = false
|
|
23
|
+
@skipped = {}.compare_by_identity
|
|
9
24
|
super()
|
|
10
25
|
end
|
|
11
26
|
|
|
@@ -52,21 +67,43 @@ module SemanticLogger
|
|
|
52
67
|
# Write the log message to every appender that should log it. An appender that fails is
|
|
53
68
|
# reported, and the others still write the message.
|
|
54
69
|
#
|
|
70
|
+
# While the process exits, an appender without a queue of its own is skipped once its
|
|
71
|
+
# destination has failed, and the message counted as not given to it (see #process_exiting=).
|
|
72
|
+
#
|
|
73
|
+
# The values of the sensitive keys are first replaced, once, so that no appender writes them
|
|
74
|
+
# (see SemanticLogger.sensitive_keys=).
|
|
75
|
+
#
|
|
55
76
|
# A stack overflow is not a failure of the appender but of the thread, which must not overflow
|
|
56
77
|
# again: on macOS a second stack overflow on the same thread crashes Ruby. It is raised to the
|
|
57
78
|
# processor running this thread instead, and the message goes to no further appenders.
|
|
58
79
|
def log(log)
|
|
80
|
+
redactor = SemanticLogger.redactor
|
|
81
|
+
log.redact(redactor) if redactor
|
|
82
|
+
skipped = skipped_appenders
|
|
59
83
|
each do |appender|
|
|
60
|
-
|
|
84
|
+
writing = appender.should_log?(log)
|
|
85
|
+
next unless writing
|
|
86
|
+
next skipped[appender].discarded += 1 if skipped.key?(appender)
|
|
87
|
+
|
|
88
|
+
appender.log(log)
|
|
61
89
|
rescue SystemStackError
|
|
62
90
|
raise
|
|
63
91
|
rescue Exception => e
|
|
64
92
|
logger.error "Failed to log to appender: #{appender.name}", e
|
|
93
|
+
skip_while_exiting(appender, e) if writing
|
|
65
94
|
end
|
|
66
95
|
end
|
|
67
96
|
|
|
97
|
+
# Flush every appender. An appender that fails is reported, and the others are still flushed.
|
|
98
|
+
#
|
|
99
|
+
# While the process exits, an appender that is skipped (see #process_exiting=) is first reported,
|
|
100
|
+
# with the number of log messages it has not been given since it was last reported, then flushed
|
|
101
|
+
# too, so that it delivers the messages it already accepted, such as those that the producer of a
|
|
102
|
+
# :kafka appender holds.
|
|
68
103
|
def flush
|
|
104
|
+
skipped = skipped_appenders
|
|
69
105
|
each do |appender|
|
|
106
|
+
report_skipped(appender, skipped[appender]) if skipped.key?(appender)
|
|
70
107
|
logger.trace "Flushing appender: #{appender.name}"
|
|
71
108
|
appender.flush
|
|
72
109
|
rescue Exception => e
|
|
@@ -75,6 +112,13 @@ module SemanticLogger
|
|
|
75
112
|
logger.trace "All appenders flushed"
|
|
76
113
|
end
|
|
77
114
|
|
|
115
|
+
# Flush and close every appender, then remove those closed. An appender that fails is reported,
|
|
116
|
+
# and the others are still closed.
|
|
117
|
+
#
|
|
118
|
+
# A stack overflow is raised at once, as #log raises it, since the thread must not carry on: on
|
|
119
|
+
# macOS a second stack overflow on the same thread crashes Ruby. The appenders already closed are
|
|
120
|
+
# still removed. The one that overflowed the stack, and those after it, not yet closed, stay
|
|
121
|
+
# listed, as one whose close fails does.
|
|
78
122
|
def close
|
|
79
123
|
closed_appenders = []
|
|
80
124
|
each do |appender|
|
|
@@ -82,15 +126,21 @@ module SemanticLogger
|
|
|
82
126
|
appender.flush
|
|
83
127
|
appender.close
|
|
84
128
|
closed_appenders << appender
|
|
129
|
+
rescue SystemStackError
|
|
130
|
+
raise
|
|
85
131
|
rescue Exception => e
|
|
86
132
|
logger.error "Failed to close appender: #{appender.name}", e
|
|
87
133
|
end
|
|
88
|
-
# Delete appenders outside the #each above which prevents changes during iteration.
|
|
89
|
-
closed_appenders.each { |appender| delete(appender) }
|
|
90
134
|
logger.trace "All appenders closed and removed from appender list"
|
|
135
|
+
ensure
|
|
136
|
+
# Delete appenders outside the #each above which prevents changes during iteration.
|
|
137
|
+
closed_appenders&.each { |appender| delete(appender) }
|
|
91
138
|
end
|
|
92
139
|
|
|
93
|
-
# After a fork reopen each appender.
|
|
140
|
+
# After a fork, or a log rotation, reopen each appender. Called on the thread that writes to the
|
|
141
|
+
# appenders: the worker thread of the main Processor, or the calling thread with sync!. An
|
|
142
|
+
# appender added with async: true or in batches is asked to reopen, by its own worker thread
|
|
143
|
+
# (see Appender::Async#reopen).
|
|
94
144
|
def reopen
|
|
95
145
|
each do |appender|
|
|
96
146
|
next unless appender.respond_to?(:reopen)
|
|
@@ -102,5 +152,87 @@ module SemanticLogger
|
|
|
102
152
|
end
|
|
103
153
|
logger.trace "All appenders re-opened"
|
|
104
154
|
end
|
|
155
|
+
|
|
156
|
+
# Tell each appender added with async: true or in batches whether the process is exiting: true
|
|
157
|
+
# before the flush at exit, and false once it has returned (see QueueProcessor#process_exiting=).
|
|
158
|
+
# Each one's queue processor is told straight away, rather than through the queue of the thread
|
|
159
|
+
# that writes to every appender, which can be held up handing an entry to the full queue of an
|
|
160
|
+
# appender whose worker thread is backing off during an outage. Takes no lock, so that it can be
|
|
161
|
+
# called from any thread, or a signal trap. An appender that raises is reported, as for #flush,
|
|
162
|
+
# and the others are still told, so that the flush at exit still runs.
|
|
163
|
+
#
|
|
164
|
+
# The appenders without a queue of their own are written to here, one log message after another,
|
|
165
|
+
# by the thread that writes to every appender, so that during an outage the flush at exit would
|
|
166
|
+
# wait while each message queued ahead of it failed in turn, up to a timeout each against a
|
|
167
|
+
# server that does not answer. So while the process exits, once the destination of such an
|
|
168
|
+
# appender fails, other than for the content of a message (see Subscriber#message_error?), it is
|
|
169
|
+
# skipped until the process no longer exits: #log gives it no more messages, counting them, and
|
|
170
|
+
# #flush reports them, then still flushes it, so that it delivers what it already accepted. The
|
|
171
|
+
# other appenders are still given every message.
|
|
172
|
+
#
|
|
173
|
+
# It applies only in the process that exits, not in a child process forked meanwhile.
|
|
174
|
+
def process_exiting=(exiting)
|
|
175
|
+
@skipped = {}.compare_by_identity if exiting
|
|
176
|
+
@exiting = exiting && ::Process.pid
|
|
177
|
+
each do |appender|
|
|
178
|
+
appender.process_exiting = exiting if appender.respond_to?(:process_exiting=)
|
|
179
|
+
rescue Exception => e
|
|
180
|
+
logger.error "Failed to tell appender that the process is exiting: #{appender.name}", e
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# After a fork, replace the queue and worker thread of each appender added with async: true or
|
|
185
|
+
# in batches with the child's own, on the calling thread (see QueueProcessor#renew_after_fork).
|
|
186
|
+
# Takes no lock, so that it can be called from a signal trap.
|
|
187
|
+
def renew_after_fork
|
|
188
|
+
each { |appender| appender.renew_after_fork if appender.respond_to?(:renew_after_fork) }
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Returns [true|false] whether the thread is the worker thread of one of the appenders: one added
|
|
192
|
+
# with async: true or in batches, which writes to it (see QueueProcessor#worker_thread?). The thread
|
|
193
|
+
# that writes to every appender may be waiting for it, so it must not wait for that thread. Takes
|
|
194
|
+
# no lock, so that it can be called from a signal trap.
|
|
195
|
+
def worker_thread?(thread = Thread.current)
|
|
196
|
+
any? { |appender| appender.respond_to?(:worker_thread?) && appender.worker_thread?(thread) }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Returns [true|false] whether the thread is the worker thread of one of the appenders while that
|
|
200
|
+
# appender's queue is full, or a flush or close of it is waiting (see QueueProcessor#holding_up?):
|
|
201
|
+
# the thread that writes to every appender may then be waiting for it, to make room or to run that
|
|
202
|
+
# request. Takes no lock, as #worker_thread? takes none.
|
|
203
|
+
def holding_up?(thread = Thread.current)
|
|
204
|
+
any? { |appender| appender.respond_to?(:holding_up?) && appender.holding_up?(thread) }
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
private
|
|
208
|
+
|
|
209
|
+
# Whether this process is exiting (see #process_exiting=), rather than a child process forked while
|
|
210
|
+
# it was.
|
|
211
|
+
def exiting? = @exiting && @exiting == ::Process.pid
|
|
212
|
+
|
|
213
|
+
# Returns [Hash] the appenders skipped while the process exits, by appender, or none while it does
|
|
214
|
+
# not. The process can stop exiting while the thread that writes to every appender fails to write
|
|
215
|
+
# to one, so one skipped then is left out until the process exits again, which starts afresh.
|
|
216
|
+
def skipped_appenders = exiting? ? @skipped : NONE_SKIPPED
|
|
217
|
+
|
|
218
|
+
# While the process exits, skip an appender without a queue of its own once writing to it failed
|
|
219
|
+
# because of its destination (see #process_exiting=). An appender with a queue of its own is told
|
|
220
|
+
# that the process is exiting, and handles it itself.
|
|
221
|
+
def skip_while_exiting(appender, exception)
|
|
222
|
+
return unless exiting?
|
|
223
|
+
return if appender.respond_to?(:process_exiting=)
|
|
224
|
+
return if appender.respond_to?(:message_error?) && appender.message_error?(exception)
|
|
225
|
+
|
|
226
|
+
@skipped[appender] = Skipped.new(exception, 0)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Report the log messages that a skipped appender has not been given since it was last reported.
|
|
230
|
+
def report_skipped(appender, skipped)
|
|
231
|
+
return if skipped.discarded.zero?
|
|
232
|
+
|
|
233
|
+
logger.error "Discarded #{skipped.discarded} log message(s) during the flush at exit, after the " \
|
|
234
|
+
"destination failed, for appender: #{appender.name}", skipped.exception
|
|
235
|
+
skipped.discarded = 0
|
|
236
|
+
end
|
|
105
237
|
end
|
|
106
238
|
end
|