semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
# Loaded now rather than autoloaded, since #default_formatter builds one, such as for the internal
|
|
2
|
+
# logger that the processor builds on first use, which can be in a signal trap, where Ruby does not
|
|
3
|
+
# allow a require.
|
|
4
|
+
require "semantic_logger/formatters/default"
|
|
5
|
+
|
|
1
6
|
# Abstract Subscriber
|
|
2
7
|
#
|
|
3
8
|
# Abstract base class for all appenders.
|
|
@@ -106,6 +111,23 @@ module SemanticLogger
|
|
|
106
111
|
true
|
|
107
112
|
end
|
|
108
113
|
|
|
114
|
+
# Returns [String] why this appender cannot be batched, and what to do instead, or nil when it
|
|
115
|
+
# can. add_appender asks it before it batches the appender, with `batch: true` or by default, see
|
|
116
|
+
# #batch_by_default?, and raises ArgumentError with it, after "add_appender cannot batch <class>: ",
|
|
117
|
+
# rather than add an appender whose worker thread fails on every batch, and so writes nothing.
|
|
118
|
+
#
|
|
119
|
+
# Nil, since a #batch that writes each log message, formatted by #format_log, works with any
|
|
120
|
+
# formatter. A class whose own #batch cannot write a batch, with some formatters or at all,
|
|
121
|
+
# answers for that #batch, and only while it is the #batch in use (see #batch_defined_by?), so
|
|
122
|
+
# that a subclass that replaces it is batched as before. Appender::Http does, since its #batch
|
|
123
|
+
# formats the whole batch with #format_batch, which needs a formatter that can, see
|
|
124
|
+
# #format_batch_refusal. A subclass whose #batch calls super keeps that need, so it answers too,
|
|
125
|
+
# usually with #format_batch_refusal. One that inherits a #batch that its destination rejects,
|
|
126
|
+
# whatever the formatter, such as Appender::ElasticsearchHttp, says why, and what to use instead.
|
|
127
|
+
def batch_refusal
|
|
128
|
+
nil
|
|
129
|
+
end
|
|
130
|
+
|
|
109
131
|
# Raised by #format_log and #format_batch when the formatter fails, with the formatter's
|
|
110
132
|
# error as its cause. A formatter has no destination, so it can only fail because of the
|
|
111
133
|
# content of the log message, whatever it raises.
|
|
@@ -118,10 +140,14 @@ module SemanticLogger
|
|
|
118
140
|
MESSAGE_ERRORS = [FormatError, JSON::GeneratorError, JSON::NestingError, EncodingError, FloatDomainError].freeze
|
|
119
141
|
|
|
120
142
|
# Whether the supplied exception, raised by #log or #batch, was caused by the content of a
|
|
121
|
-
# log message,
|
|
143
|
+
# log message, or is one that writing the message again cannot fix, such as a request the
|
|
144
|
+
# destination refuses outright, rather than by the destination being unavailable for now.
|
|
122
145
|
#
|
|
123
|
-
#
|
|
124
|
-
#
|
|
146
|
+
# Waiting cannot write such a message, so an asynchronous appender discards it and carries on
|
|
147
|
+
# straight away. On any other exception it backs off before carrying on, since the destination
|
|
148
|
+
# may be briefly unavailable, then writes the message (or the batch) again when #wrote_nothing?
|
|
149
|
+
# says that the write wrote none of it (see QueueProcessor::HeldWrite::MAX_RESENDS), or
|
|
150
|
+
# otherwise carries on with the next one, having discarded it.
|
|
125
151
|
#
|
|
126
152
|
# Every error raised by the formatter is one (see #format_log). An appender whose client
|
|
127
153
|
# raises its own errors for bad content can override this to recognize them.
|
|
@@ -139,6 +165,72 @@ module SemanticLogger
|
|
|
139
165
|
false
|
|
140
166
|
end
|
|
141
167
|
|
|
168
|
+
# Whether the supplied exception, a message error (see #message_error?) that #batch raised,
|
|
169
|
+
# would fail any batch, whatever messages it holds, such as a request the destination refuses
|
|
170
|
+
# for its credentials or its url. Asked only about a message error, and only when
|
|
171
|
+
# #batch_all_or_nothing? is true, or #rejected_entry names a message.
|
|
172
|
+
#
|
|
173
|
+
# When true, the whole batch is discarded at once, rather than split up and each part written
|
|
174
|
+
# again to find the messages at fault, which would be refused in turn, about two writes for each
|
|
175
|
+
# message. False, since an error caused by the content of a message is caused by some of the
|
|
176
|
+
# messages in the batch.
|
|
177
|
+
def fails_any_batch?(_exception)
|
|
178
|
+
false
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Returns [Integer|nil] the position, counted from 0, of the log message in the batch that the
|
|
182
|
+
# destination rejected, when the supplied exception, a message error (see #message_error?) that
|
|
183
|
+
# #batch raised, says so, and the destination stored every message before that one. Nil when it
|
|
184
|
+
# cannot tell, as by default.
|
|
185
|
+
#
|
|
186
|
+
# When an Integer, the messages before that one are counted as written, that one is discarded,
|
|
187
|
+
# or, when the error would fail any batch (see #fails_any_batch?), it and every message after
|
|
188
|
+
# it, and the messages after it are written again as a batch. So none is written twice, and
|
|
189
|
+
# none that the destination did not reject is discarded, without splitting the batch to find
|
|
190
|
+
# the message at fault. Appender::SplunkHttp answers it from the HTTP Event Collector's response.
|
|
191
|
+
def rejected_entry(_exception)
|
|
192
|
+
nil
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Whether #log or #batch, when it raised the supplied exception, wrote none of the log messages
|
|
196
|
+
# it was given, so that writing them again cannot write any of them twice, and writing them
|
|
197
|
+
# again later may succeed. Asked only about an exception that is not a message error (see
|
|
198
|
+
# #message_error?), such as one raised because the destination is unavailable.
|
|
199
|
+
#
|
|
200
|
+
# When true, an asynchronous appender writes the same message, or batch, again once it has
|
|
201
|
+
# backed off. Once the destination has answered it with a failure after it was written again
|
|
202
|
+
# QueueProcessor::HeldWrite::MAX_RESENDS times, the appender discards it and carries on with the
|
|
203
|
+
# next one, unless it gives up after async_max_retries first. A write that never reached the
|
|
204
|
+
# destination (see #not_sent?) is not counted. When false, since the write may have written
|
|
205
|
+
# some or all of them, such as a request whose answer timed out after the destination stored
|
|
206
|
+
# it, or a batch written a message at a time, or since writing them again is bound to fail the
|
|
207
|
+
# same way, they are discarded instead. A flush or close does not wait for them: while one is
|
|
208
|
+
# waiting, they are discarded either way.
|
|
209
|
+
#
|
|
210
|
+
# A #log or #batch that writes the messages in more than one request must return false for an
|
|
211
|
+
# error from any request after the first, which the destination may follow having stored the
|
|
212
|
+
# messages of the earlier ones.
|
|
213
|
+
#
|
|
214
|
+
# False, since only the appender can tell. Appender::Http answers it from the error.
|
|
215
|
+
def wrote_nothing?(_exception)
|
|
216
|
+
false
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Whether #log or #batch, when it raised the supplied exception, sent nothing to the destination,
|
|
220
|
+
# such as when it could not connect to it, so that the destination has not answered the write.
|
|
221
|
+
# Asked only once #wrote_nothing? has returned true for the exception.
|
|
222
|
+
#
|
|
223
|
+
# When true, writing the message, or batch, again after such a failure does not count towards
|
|
224
|
+
# QueueProcessor::HeldWrite::MAX_RESENDS: the limit is for a message the destination answers
|
|
225
|
+
# with a failure every time, such as a request it answers with a 429 that never clears, while
|
|
226
|
+
# one that cannot reach the destination during an outage is still written once it is back, as
|
|
227
|
+
# long as the worker thread keeps trying (async_max_retries). When false, every failure counts.
|
|
228
|
+
#
|
|
229
|
+
# False, since only the appender can tell. Appender::Http answers it from the error.
|
|
230
|
+
def not_sent?(_exception)
|
|
231
|
+
false
|
|
232
|
+
end
|
|
233
|
+
|
|
142
234
|
private
|
|
143
235
|
|
|
144
236
|
# Returns the log message formatted by this appender's formatter.
|
|
@@ -146,6 +238,8 @@ module SemanticLogger
|
|
|
146
238
|
# Any StandardError the formatter raises is raised as a FormatError, with the formatter's
|
|
147
239
|
# error as its cause, so that #message_error? recognizes it: an asynchronous appender then
|
|
148
240
|
# discards the message instead of backing off. Appenders call this rather than formatter.call.
|
|
241
|
+
# So is a ScriptError, such as the NotImplementedError of an abstract method, which would
|
|
242
|
+
# otherwise stop an asynchronous appender's thread, since that rescues only a StandardError.
|
|
149
243
|
#
|
|
150
244
|
# A SystemStackError is raised as it is, so that rescuing StandardError does not swallow it:
|
|
151
245
|
# the thread that overflowed must not keep writing (see Appenders#log).
|
|
@@ -155,33 +249,74 @@ module SemanticLogger
|
|
|
155
249
|
def format_log(log)
|
|
156
250
|
formatted = formatter.call(log, self)
|
|
157
251
|
block_given? ? yield(formatted) : formatted
|
|
158
|
-
rescue StandardError => e
|
|
159
|
-
raise FormatError, "Could not format the log message (#{e
|
|
252
|
+
rescue StandardError, ScriptError => e
|
|
253
|
+
raise FormatError, "Could not format the log message (#{Utils.exception_name(e)})"
|
|
160
254
|
end
|
|
161
255
|
|
|
162
256
|
# Returns the log messages formatted by the formatter's #batch, as for #format_log.
|
|
163
257
|
#
|
|
164
|
-
# A formatter that cannot format a batch is a configuration error
|
|
165
|
-
# it raises ArgumentError instead, which
|
|
166
|
-
# other
|
|
167
|
-
# #
|
|
258
|
+
# A formatter that cannot format a batch, see #format_batch_refusal, is a configuration error
|
|
259
|
+
# rather than bad content, so it raises ArgumentError instead, which backs off and is reported
|
|
260
|
+
# like any other error from the destination. add_appender refuses such a formatter for an
|
|
261
|
+
# appender whose #batch_refusal says so, such as Appender::Http, so for one of those this only
|
|
262
|
+
# happens when #formatter= replaces it afterwards.
|
|
263
|
+
#
|
|
264
|
+
# The formatter is read once, so that one changed by #formatter= on another thread cannot be
|
|
265
|
+
# swapped in between the check and the call.
|
|
168
266
|
def format_batch(logs)
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
267
|
+
batch_formatter = formatter
|
|
268
|
+
refusal = format_batch_refusal(batch_formatter)
|
|
269
|
+
raise(ArgumentError, "Cannot batch #{self.class}: #{refusal}") if refusal
|
|
172
270
|
|
|
173
271
|
begin
|
|
174
|
-
|
|
175
|
-
rescue StandardError => e
|
|
176
|
-
raise FormatError, "Could not format the batch of log messages (#{e
|
|
272
|
+
batch_formatter.batch(logs, self)
|
|
273
|
+
rescue StandardError, ScriptError => e
|
|
274
|
+
raise FormatError, "Could not format the batch of log messages (#{Utils.exception_name(e)})"
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# Returns [String] why #format_batch cannot format a batch with the formatter, and what to do
|
|
279
|
+
# instead, or nil when it can. It needs a formatter that implements #batch(logs, logger), which a
|
|
280
|
+
# formatter block or Proc does not, other than this appender itself, which is its own formatter
|
|
281
|
+
# when it implements #call, and whose #batch is the appender's own. Such an appender sets the
|
|
282
|
+
# format of each entry that its destination takes, so the only advice for it is `batch: false`.
|
|
283
|
+
#
|
|
284
|
+
# #format_batch raises with it, and a #batch_refusal that answers for a #batch that calls
|
|
285
|
+
# #format_batch, such as that of Appender::Http, returns it, so that both give the same answer.
|
|
286
|
+
def format_batch_refusal(batch_formatter = formatter)
|
|
287
|
+
if batch_formatter.equal?(self)
|
|
288
|
+
"it is its own formatter, which cannot format a batch. Add it with batch: false."
|
|
289
|
+
elsif !batch_formatter.respond_to?(:batch)
|
|
290
|
+
"its formatter, #{batch_formatter.class}, cannot format a batch. Give it a formatter that " \
|
|
291
|
+
"implements #batch(logs, logger), or add it with batch: false."
|
|
177
292
|
end
|
|
178
293
|
end
|
|
179
294
|
|
|
295
|
+
# Returns [true|false] whether the #batch in use is the one that `owner` defines, for a
|
|
296
|
+
# #batch_refusal that answers only for its own #batch, such as that of Appender::Http. False for
|
|
297
|
+
# any other, even one that calls super, such as one that a subclass, a prepended module, or the
|
|
298
|
+
# appender itself defines, and when it has none. Kernel#method finds it, since an appender may
|
|
299
|
+
# define a #method of its own, such as for the HTTP method of its requests.
|
|
300
|
+
def batch_defined_by?(owner)
|
|
301
|
+
respond_to?(:batch) && Kernel.instance_method(:method).bind_call(self, :batch).owner.equal?(owner)
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
# Returns [Hash] a copy of the supplied options, with a copy of each Hash among them, such as ssl:, for an
|
|
305
|
+
# appender to add its own settings to, or to give to a client it builds, which may change them:
|
|
306
|
+
# Net::TCPClient takes handshake_timeout: out of the ssl: Hash it is given. An appender never changes
|
|
307
|
+
# the Hashes it is given, which belong to the application: they can be frozen, or shared with another
|
|
308
|
+
# appender, and the next client the appender builds, such as after a reopen, needs them as they were.
|
|
309
|
+
def copy_options(options)
|
|
310
|
+
options.transform_values { |value| value.is_a?(Hash) ? value.dup : value }
|
|
311
|
+
end
|
|
312
|
+
|
|
180
313
|
# Initializer for Abstract Class SemanticLogger::Subscriber
|
|
181
314
|
#
|
|
182
315
|
# Parameters
|
|
183
316
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
|
184
317
|
# Override the log level for this subscriber.
|
|
318
|
+
# Default: none (:trace), so it writes every entry that a logger emits; each logger filters by
|
|
319
|
+
# its own level (see #fallback_level).
|
|
185
320
|
#
|
|
186
321
|
# formatter: [Object|Proc]
|
|
187
322
|
# An instance of a class that implements #call, or a Proc to be used to format
|
|
@@ -231,5 +366,24 @@ module SemanticLogger
|
|
|
231
366
|
def metrics?
|
|
232
367
|
@metrics
|
|
233
368
|
end
|
|
369
|
+
|
|
370
|
+
# The default of the `metrics` option of the appenders that call #deprecated_metrics_default, so that it can tell
|
|
371
|
+
# an option that is not supplied from an explicit nil, which keeps meaning false. Remove in v6, with that method.
|
|
372
|
+
METRICS_NOT_SUPPLIED = Utils.make_shareable(Object.new.freeze)
|
|
373
|
+
private_constant :METRICS_NOT_SUPPLIED
|
|
374
|
+
|
|
375
|
+
# Returns [true|false|nil] the `metrics` option of an appender that sends metric-only entries unless told
|
|
376
|
+
# otherwise: true when it is not supplied, see METRICS_NOT_SUPPLIED, with a deprecation warning, since v6 leaves
|
|
377
|
+
# them out by default, as the other log appenders do, otherwise the option as it was supplied. Remove in v6, with
|
|
378
|
+
# the METRICS_NOT_SUPPLIED default of the appenders that call it.
|
|
379
|
+
def deprecated_metrics_default(metrics, appender)
|
|
380
|
+
return metrics unless METRICS_NOT_SUPPLIED.equal?(metrics)
|
|
381
|
+
|
|
382
|
+
Deprecation.warn(
|
|
383
|
+
"The #{appender.inspect} appender sends metric-only entries by default, which changes in v6. " \
|
|
384
|
+
"Add it with metrics: true to keep sending them, or metrics: false to leave them out."
|
|
385
|
+
)
|
|
386
|
+
true
|
|
387
|
+
end
|
|
234
388
|
end
|
|
235
389
|
end
|
|
@@ -3,18 +3,39 @@ module SemanticLogger
|
|
|
3
3
|
#
|
|
4
4
|
# Appenders are designed to only be used by one thread at a time, so all calls
|
|
5
5
|
# are monitor protected in case SyncProcessor is being used in a multi-threaded environment.
|
|
6
|
+
#
|
|
7
|
+
# Ruby does not allow a Monitor to be entered in a signal trap, so another thread runs a call made
|
|
8
|
+
# in a trap. A log entry is left on a queue of pending calls, which whoever next enters the monitor
|
|
9
|
+
# runs first, in order, and a thread started from the trap enters it to do so when nothing else does
|
|
10
|
+
# (see #defer). Every other call runs on a thread that the trap starts and waits for, at most
|
|
11
|
+
# TRAP_WAIT_SECONDS (see #outside_trap). Neither ever runs on the trap's thread, so that an appender
|
|
12
|
+
# can lock a Mutex of its own as it writes, flushes, closes, or reopens, as a Ruby Logger does.
|
|
6
13
|
class SyncProcessor
|
|
7
|
-
|
|
8
|
-
|
|
14
|
+
# Seconds that a call other than a log call, such as a flush or a reopen, waits in a signal trap
|
|
15
|
+
# for the thread that runs it. It may need a lock that the thread the trap interrupted holds.
|
|
16
|
+
TRAP_WAIT_SECONDS = 1
|
|
17
|
+
|
|
18
|
+
# Returned by #exclusively, which does not run its block, in a signal trap.
|
|
19
|
+
TRAPPED = Object.new.freeze
|
|
20
|
+
private_constant :TRAPPED
|
|
21
|
+
|
|
22
|
+
# Add an appender. In a signal trap, as for #flush, which returns nil instead of the appender when
|
|
23
|
+
# it is not added in time. SemanticLogger.add_appender adds to the appenders directly instead.
|
|
24
|
+
def add(**args, &block)
|
|
25
|
+
synchronize { @appenders.add(**args, &block) }
|
|
9
26
|
end
|
|
10
27
|
|
|
11
28
|
# Logs on the calling thread, which belongs to the application, so a stack overflow while
|
|
12
29
|
# writing (see Appenders#log) is reported rather than raised from the log call.
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
30
|
+
#
|
|
31
|
+
# In a signal trap, the entry is written just after, by another thread, before any entry logged
|
|
32
|
+
# after the trap, and the trap returns without waiting for it (see #defer).
|
|
33
|
+
def log(log)
|
|
34
|
+
logged = exclusively { write(log) }
|
|
35
|
+
return logged unless logged.equal?(TRAPPED)
|
|
36
|
+
|
|
37
|
+
defer { write(log) }
|
|
38
|
+
true
|
|
18
39
|
rescue SystemStackError => e
|
|
19
40
|
@appenders.logger.error("Failed to log a message: the stack overflowed", e)
|
|
20
41
|
false
|
|
@@ -22,40 +43,79 @@ module SemanticLogger
|
|
|
22
43
|
|
|
23
44
|
# Returns [Hash] operational statistics for the logging pipeline.
|
|
24
45
|
#
|
|
25
|
-
# In synchronous mode
|
|
26
|
-
#
|
|
46
|
+
# In synchronous mode messages are written inline on the calling thread, so the queue holds only
|
|
47
|
+
# the entries logged in a signal trap that are still to be written (see #log), and no messages
|
|
48
|
+
# are dropped. In a trap it reads them without entering the monitor.
|
|
27
49
|
def stats
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
queue_size: 0,
|
|
31
|
-
capped: false,
|
|
32
|
-
max_queue_size: nil,
|
|
33
|
-
thread_active: false,
|
|
34
|
-
processed: @processed_count,
|
|
35
|
-
dropped: 0,
|
|
36
|
-
appenders: @appenders.stats
|
|
37
|
-
}
|
|
38
|
-
end
|
|
50
|
+
stats = exclusively { statistics }
|
|
51
|
+
stats.equal?(TRAPPED) ? statistics : stats
|
|
39
52
|
end
|
|
40
53
|
|
|
54
|
+
# Flush the appenders. In a signal trap, a thread that the trap starts flushes them, after the
|
|
55
|
+
# entries logged in the trap before, and the trap waits for it, at most TRAP_WAIT_SECONDS, then
|
|
56
|
+
# returns its result, or raises what it raised. When it has not finished by then, such as when the
|
|
57
|
+
# trap interrupted a call inside this processor, which cannot return before the trap does, it
|
|
58
|
+
# returns nil and the flush runs just after.
|
|
41
59
|
def flush
|
|
42
|
-
|
|
60
|
+
synchronize { @appenders.flush }
|
|
43
61
|
end
|
|
44
62
|
|
|
63
|
+
# Close the appenders, as for #flush in a signal trap.
|
|
64
|
+
#
|
|
65
|
+
# A stack overflow while closing is reported rather than raised, as by #log, since the calling
|
|
66
|
+
# thread belongs to the application, even when Appenders#close raises it, for the asynchronous
|
|
67
|
+
# Processor to move to a new thread.
|
|
45
68
|
def close
|
|
46
|
-
|
|
69
|
+
synchronize do
|
|
70
|
+
@appenders.close
|
|
71
|
+
rescue SystemStackError => e
|
|
72
|
+
logger.error("Failed to close the appenders: the stack overflowed", e)
|
|
73
|
+
nil
|
|
74
|
+
end
|
|
47
75
|
end
|
|
48
76
|
|
|
77
|
+
# Remove the appender from those that this processor writes to, and close it, once any log call
|
|
78
|
+
# writing to it has returned. Closes it after leaving the monitor, so that the other threads
|
|
79
|
+
# can log meanwhile. Returns the result of closing it, or, in a signal trap, as for #flush, nil
|
|
80
|
+
# when it is not closed in time.
|
|
81
|
+
#
|
|
82
|
+
# The appenders are reopened on the calling thread (see #reopen), so none is ever waiting to be
|
|
83
|
+
# reopened when it is removed.
|
|
84
|
+
def remove(appender)
|
|
85
|
+
removed = exclusively { @appenders.flush if @appenders.delete(appender) }
|
|
86
|
+
return outside_trap { remove(appender) } if removed.equal?(TRAPPED)
|
|
87
|
+
|
|
88
|
+
appender.close
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Reopen the appenders on the calling thread, or, in a signal trap, as for #flush, so that an
|
|
92
|
+
# appender that connects or locks a Mutex as it reopens can do so.
|
|
49
93
|
def reopen(*args)
|
|
50
|
-
|
|
94
|
+
synchronize { @appenders.reopen(*args) }
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Returns [Integer] the number of entries logged in a signal trap that are still to be written
|
|
98
|
+
# (see #log), the only ones that wait in synchronous mode. Never enters the monitor.
|
|
99
|
+
def queue_size
|
|
100
|
+
pending_count
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Returns [true|false] whether the calling thread is inside a call to this processor, such as in
|
|
104
|
+
# a signal trap that interrupted a log call. Another thread then waits for that call to return
|
|
105
|
+
# before it can use this processor, so the calling thread must not wait for one that does.
|
|
106
|
+
def current_thread_inside?
|
|
107
|
+
@monitor.mon_owned?
|
|
51
108
|
end
|
|
52
109
|
|
|
53
110
|
# Allow the internal logger to be overridden from its default of $stderr
|
|
54
111
|
# Can be replaced with another Ruby logger or Rails logger, but never to
|
|
55
112
|
# SemanticLogger::Logger itself since it is for reporting problems
|
|
56
|
-
# while trying to log to the various appenders
|
|
57
|
-
|
|
58
|
-
|
|
113
|
+
# while trying to log to the various appenders (raises ArgumentError)
|
|
114
|
+
#
|
|
115
|
+
# A Ruby or Rails logger is wrapped in an appender (see Appender.internal_logger), so .logger
|
|
116
|
+
# returns that appender.
|
|
117
|
+
def self.logger=(logger)
|
|
118
|
+
@logger = Appender.internal_logger(logger)
|
|
59
119
|
end
|
|
60
120
|
|
|
61
121
|
# Internal logger for SemanticLogger
|
|
@@ -82,10 +142,150 @@ module SemanticLogger
|
|
|
82
142
|
@monitor = Monitor.new
|
|
83
143
|
@appenders = appenders || Appenders.new(self.class.logger.dup)
|
|
84
144
|
@processed_count = 0
|
|
145
|
+
@pending = Thread::Queue.new
|
|
146
|
+
@runner = nil
|
|
147
|
+
@pid = Process.pid
|
|
85
148
|
end
|
|
86
149
|
|
|
87
150
|
def start
|
|
88
151
|
# NOP
|
|
89
152
|
end
|
|
153
|
+
|
|
154
|
+
private
|
|
155
|
+
|
|
156
|
+
# Runs the block inside the monitor, after the pending calls, and returns its value, or returns
|
|
157
|
+
# TRAPPED, without running it, in a signal trap. Leaves the monitor even when a pending call
|
|
158
|
+
# raises past its rescue, such as Timeout or Thread#raise, so that no other thread waits for ever.
|
|
159
|
+
#
|
|
160
|
+
# A call made by the thread already inside, such as by an appender while it writes, runs at once,
|
|
161
|
+
# as it always has, without the pending calls, which may write to the appender in the middle of
|
|
162
|
+
# its write. Monitor#enter raises ThreadError in a trap, even when no other thread is inside, so
|
|
163
|
+
# it tells a trap without a check of its own, except when the trap interrupted the thread inside,
|
|
164
|
+
# which the monitor lets in again. The thread inside runs the calls that such a trap left before
|
|
165
|
+
# it leaves the monitor. Monitor#enter also raises ThreadError outside a trap, for a fiber when
|
|
166
|
+
# another fiber of its thread is inside, which is raised again, as before.
|
|
167
|
+
def exclusively
|
|
168
|
+
if @monitor.mon_owned?
|
|
169
|
+
return TRAPPED if TrapSafe.in_trap?
|
|
170
|
+
|
|
171
|
+
return yield
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
begin
|
|
175
|
+
@monitor.enter
|
|
176
|
+
rescue ThreadError
|
|
177
|
+
raise unless TrapSafe.in_trap?
|
|
178
|
+
|
|
179
|
+
return TRAPPED
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
begin
|
|
183
|
+
run_pending
|
|
184
|
+
yield
|
|
185
|
+
ensure
|
|
186
|
+
begin
|
|
187
|
+
run_pending
|
|
188
|
+
ensure
|
|
189
|
+
@monitor.exit
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Runs the block inside the monitor and returns its value, or, in a signal trap, runs it on a
|
|
195
|
+
# thread that the trap starts, as #outside_trap does.
|
|
196
|
+
def synchronize(&block)
|
|
197
|
+
result = exclusively(&block)
|
|
198
|
+
result.equal?(TRAPPED) ? outside_trap { exclusively(&block) } : result
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Leaves the block, given in a signal trap, on the queue of pending calls, and starts a thread to
|
|
202
|
+
# run them, unless one is running (see #start_runner). Never waits.
|
|
203
|
+
def defer(&block)
|
|
204
|
+
forget_parent_calls
|
|
205
|
+
@pending << block
|
|
206
|
+
start_runner
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Runs the block, given in a signal trap, on a thread that the trap starts, and returns its value,
|
|
210
|
+
# waiting at most TRAP_WAIT_SECONDS for it (see TrapSafe.hand_off). Does not wait when the trap
|
|
211
|
+
# interrupted the thread inside the monitor, since that thread cannot leave it before the trap
|
|
212
|
+
# returns. Outside the trap, the block enters the monitor and runs the pending calls first.
|
|
213
|
+
def outside_trap(&)
|
|
214
|
+
TrapSafe.hand_off(@monitor.mon_owned? ? 0 : TRAP_WAIT_SECONDS, &)
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# Starts a thread that enters the monitor and runs the pending calls, unless one is running.
|
|
218
|
+
#
|
|
219
|
+
# Called in a signal trap, where a Mutex cannot be locked, after it left a call on the queue. The
|
|
220
|
+
# runner clears @runner, and only then checks for calls still pending, which it runs before it
|
|
221
|
+
# checks again. So a runner that this finds in @runner has not yet made that check, and runs the
|
|
222
|
+
# call left, and a call left after that check finds no runner and starts one. The runner waits
|
|
223
|
+
# until @runner holds it, so that it never clears it before. Two runners may run at once, rarely,
|
|
224
|
+
# which is harmless.
|
|
225
|
+
def start_runner
|
|
226
|
+
return if @runner&.alive?
|
|
227
|
+
|
|
228
|
+
started = Thread::Queue.new
|
|
229
|
+
@runner = Thread.new do
|
|
230
|
+
started.pop
|
|
231
|
+
loop do
|
|
232
|
+
@monitor.synchronize { run_pending }
|
|
233
|
+
@runner = nil
|
|
234
|
+
break if @pending.empty?
|
|
235
|
+
|
|
236
|
+
@runner = Thread.current
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
started << true
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# Run the calls left by signal traps, in the order they were made. Called inside the monitor, on
|
|
243
|
+
# any thread, so it reports what a call raises instead of raising it.
|
|
244
|
+
def run_pending
|
|
245
|
+
return if @pending.empty?
|
|
246
|
+
|
|
247
|
+
forget_parent_calls
|
|
248
|
+
while (call = @pending.pop(timeout: 0))
|
|
249
|
+
begin
|
|
250
|
+
call.call
|
|
251
|
+
rescue StandardError, SystemStackError => e
|
|
252
|
+
logger.error("Failed to complete a call made in a signal trap", e)
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# In a forked child, forgets the calls left pending in the parent, which the parent runs, and the
|
|
258
|
+
# parent's runner, which the child does not have.
|
|
259
|
+
def forget_parent_calls
|
|
260
|
+
return if @pid == Process.pid
|
|
261
|
+
|
|
262
|
+
@pid = Process.pid
|
|
263
|
+
@runner = nil
|
|
264
|
+
@pending.clear
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def write(log)
|
|
268
|
+
@processed_count += 1
|
|
269
|
+
@appenders.log(log)
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Returns [Integer] the number of calls left pending by signal traps in this process. In a forked
|
|
273
|
+
# child, those of the parent are not counted, before the child forgets them (see
|
|
274
|
+
# #forget_parent_calls). Changes nothing, so that it can run in a trap without the monitor.
|
|
275
|
+
def pending_count
|
|
276
|
+
@pid == Process.pid ? @pending.size : 0
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def statistics
|
|
280
|
+
{
|
|
281
|
+
queue_size: pending_count,
|
|
282
|
+
capped: false,
|
|
283
|
+
max_queue_size: nil,
|
|
284
|
+
thread_active: @runner&.alive? || false,
|
|
285
|
+
processed: @processed_count,
|
|
286
|
+
dropped: 0,
|
|
287
|
+
appenders: @appenders.stats
|
|
288
|
+
}
|
|
289
|
+
end
|
|
90
290
|
end
|
|
91
291
|
end
|
|
@@ -25,11 +25,16 @@ module SemanticLogger
|
|
|
25
25
|
# By default collect all log levels, and collect metric only log events.
|
|
26
26
|
def initialize(level: :trace, metrics: true)
|
|
27
27
|
super
|
|
28
|
-
@events
|
|
28
|
+
@events = []
|
|
29
|
+
@created = ObjectSpace::WeakMap.new
|
|
29
30
|
end
|
|
30
31
|
|
|
32
|
+
# Standing in for a logger, it runs the `on_log` subscribers for the log events it created, as a
|
|
33
|
+
# logger does. Log events from a logger, received as the processor or an appender, already
|
|
34
|
+
# went through them. A log event built elsewhere and passed to `#log` is captured without
|
|
35
|
+
# running them, since it cannot be told apart from one received from a logger.
|
|
31
36
|
def log(log)
|
|
32
|
-
Logger.call_subscribers(log)
|
|
37
|
+
Logger.call_subscribers(log) if @created.key?(log)
|
|
33
38
|
@events << log
|
|
34
39
|
end
|
|
35
40
|
|
|
@@ -42,12 +47,25 @@ module SemanticLogger
|
|
|
42
47
|
@events.clear
|
|
43
48
|
end
|
|
44
49
|
|
|
50
|
+
# Returns [false]: standing in for the processor, it holds no lock, so another thread never
|
|
51
|
+
# waits for the calling thread to return from a call to it.
|
|
52
|
+
def current_thread_inside?
|
|
53
|
+
false
|
|
54
|
+
end
|
|
55
|
+
|
|
45
56
|
def to_h
|
|
46
57
|
events.map(&:to_h)
|
|
47
58
|
end
|
|
48
59
|
|
|
49
60
|
private
|
|
50
61
|
|
|
62
|
+
# Remember each log event created while standing in for a logger.
|
|
63
|
+
def new_log(...)
|
|
64
|
+
log = super
|
|
65
|
+
@created[log] = true
|
|
66
|
+
log
|
|
67
|
+
end
|
|
68
|
+
|
|
51
69
|
# Without a level of its own, follow the global default level the way a logger does,
|
|
52
70
|
# so that `SemanticLogger.silence` applies.
|
|
53
71
|
def fallback_level_index
|
|
@@ -23,7 +23,7 @@ module SemanticLogger
|
|
|
23
23
|
# Verify a single log event has all the required attributes.
|
|
24
24
|
def assert_semantic_logger_event(event, level: nil, name: nil, message: nil, message_includes: nil,
|
|
25
25
|
payload: nil, payload_includes: nil,
|
|
26
|
-
exception: nil, exception_includes: nil, backtrace: nil,
|
|
26
|
+
exception: nil, exception_includes: nil, backtrace: nil, call_site: nil,
|
|
27
27
|
thread_name: nil, tags: nil, named_tags: nil, context: nil,
|
|
28
28
|
level_index: nil, duration: nil, time: nil,
|
|
29
29
|
metric: nil, metric_amount: nil, dimensions: nil)
|
|
@@ -44,6 +44,7 @@ module SemanticLogger
|
|
|
44
44
|
assert_semantic_logger_entry(event, :time, time)
|
|
45
45
|
assert_semantic_logger_entry(event, :exception, exception)
|
|
46
46
|
assert_semantic_logger_entry(event, :backtrace, backtrace)
|
|
47
|
+
assert_semantic_logger_entry(event, :call_site, call_site)
|
|
47
48
|
assert_semantic_logger_entry(event, :payload, payload)
|
|
48
49
|
|
|
49
50
|
if message_includes
|
|
@@ -31,7 +31,7 @@ module SemanticLogger
|
|
|
31
31
|
EVENT_ATTRIBUTES = %i[
|
|
32
32
|
level name message thread_name tags named_tags context
|
|
33
33
|
metric metric_amount dimensions level_index duration time
|
|
34
|
-
exception backtrace payload
|
|
34
|
+
exception backtrace call_site payload
|
|
35
35
|
].freeze
|
|
36
36
|
|
|
37
37
|
# Matches a single SemanticLogger::Log against a set of expectations.
|