semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
class QueueProcessor
|
|
3
|
+
# Internal: what became of a batch of log messages whose write raised a message error (see
|
|
4
|
+
# Subscriber#message_error?), as the appender tells from the error, for the worker thread of a
|
|
5
|
+
# queue processor to carry on with: how many of them the destination stored, those to discard,
|
|
6
|
+
# and the parts of the batch to write again.
|
|
7
|
+
#
|
|
8
|
+
# When the appender says which message the destination rejected, having stored those before it
|
|
9
|
+
# (see Subscriber#rejected_entry), those before it were written, it is discarded, and those after
|
|
10
|
+
# it are written again as a batch. Otherwise, when the appender's #batch is all or nothing (see
|
|
11
|
+
# Subscriber#batch_all_or_nothing?), the batch is split in half, and each half written again, so
|
|
12
|
+
# that only the messages at fault are discarded, unless the error would fail any batch (see
|
|
13
|
+
# Subscriber#fails_any_batch?). Any other batch is discarded, since writing it again could write
|
|
14
|
+
# some of its messages twice.
|
|
15
|
+
class BatchFailure
|
|
16
|
+
# Returns [Integer] how many of the messages, from the first, the destination stored.
|
|
17
|
+
attr_reader :written
|
|
18
|
+
|
|
19
|
+
# Returns [Array<Log>] the messages to discard.
|
|
20
|
+
attr_reader :discarded
|
|
21
|
+
|
|
22
|
+
# Returns [Array<Array<Log>>] the parts of the batch to write again, in order.
|
|
23
|
+
attr_reader :parts
|
|
24
|
+
|
|
25
|
+
def initialize(appender, logs, exception)
|
|
26
|
+
@appender = appender
|
|
27
|
+
@exception = exception
|
|
28
|
+
@written = 0
|
|
29
|
+
@discarded = []
|
|
30
|
+
@parts = []
|
|
31
|
+
rejected = rejected_entry(logs)
|
|
32
|
+
if rejected
|
|
33
|
+
after_rejected(logs, rejected)
|
|
34
|
+
elsif logs.size == 1 || !appender.batch_all_or_nothing? || fails_any_batch?
|
|
35
|
+
@discarded = logs
|
|
36
|
+
else
|
|
37
|
+
@parts = logs.each_slice((logs.size + 1) / 2).to_a
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
attr_reader :appender, :exception
|
|
44
|
+
|
|
45
|
+
# The destination stored the messages before the one at position `rejected`, and rejected that
|
|
46
|
+
# one, along with every one after it when the error would fail any batch.
|
|
47
|
+
def after_rejected(logs, rejected)
|
|
48
|
+
@written = rejected
|
|
49
|
+
@discarded = fails_any_batch? ? logs.drop(rejected) : [logs[rejected]]
|
|
50
|
+
rest = logs.drop(rejected + @discarded.size)
|
|
51
|
+
@parts = [rest] unless rest.empty?
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Returns [Integer|nil] the position in the batch of the message that the destination
|
|
55
|
+
# rejected, when the appender says so, or nil when it does not, or names no message in the
|
|
56
|
+
# batch. An appender that does not implement #rejected_entry cannot say.
|
|
57
|
+
def rejected_entry(logs)
|
|
58
|
+
rejected = appender.respond_to?(:rejected_entry) && appender.rejected_entry(exception)
|
|
59
|
+
rejected if rejected.is_a?(Integer) && rejected.between?(0, logs.size - 1)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# An appender that does not implement #fails_any_batch? has every all or nothing batch split,
|
|
63
|
+
# as before it could say.
|
|
64
|
+
def fails_any_batch?
|
|
65
|
+
appender.respond_to?(:fails_any_batch?) && appender.fails_any_batch?(exception)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
class QueueProcessor
|
|
3
|
+
# Internal: the log messages, one or a batch, whose write the destination of the appender failed
|
|
4
|
+
# without storing any of them, held for the worker thread of a queue processor to write again
|
|
5
|
+
# once it has backed off, ahead of the messages on the queue.
|
|
6
|
+
#
|
|
7
|
+
# It knows the messages held, and how many times in a row they have been held again after the
|
|
8
|
+
# destination answered the write with a failure. It holds them again after such an answer at most
|
|
9
|
+
# MAX_RESENDS times, so that messages the destination fails every time, such as a request it
|
|
10
|
+
# answers with a 429 that never clears, are discarded rather than hold up the messages behind
|
|
11
|
+
# them until the worker thread gives up after async_max_retries. A write that never reached the
|
|
12
|
+
# destination, such as one that could not connect, is not counted: the destination has said
|
|
13
|
+
# nothing about these messages, so an outage loses none of them until the worker thread gives up.
|
|
14
|
+
class HeldWrite
|
|
15
|
+
# The most times the same messages are written again after the destination answered that it
|
|
16
|
+
# failed them. A constant rather than an option: it only limits how long one write that keeps
|
|
17
|
+
# failing holds up the queue, while async_max_retries already sets how long the worker thread
|
|
18
|
+
# keeps trying through an outage.
|
|
19
|
+
MAX_RESENDS = 3
|
|
20
|
+
|
|
21
|
+
def initialize
|
|
22
|
+
@logs = nil
|
|
23
|
+
@resends = 0
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Holds the messages to write again. `answered` says whether the destination answered the
|
|
27
|
+
# write that failed, rather than never received it, which counts towards MAX_RESENDS. Returns
|
|
28
|
+
# false, without holding them, when the destination answered and they have already been held
|
|
29
|
+
# again MAX_RESENDS times after it did.
|
|
30
|
+
def hold(logs, answered:)
|
|
31
|
+
if answered
|
|
32
|
+
return false if @resends >= MAX_RESENDS
|
|
33
|
+
|
|
34
|
+
@resends += 1
|
|
35
|
+
end
|
|
36
|
+
@logs = logs
|
|
37
|
+
true
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Yields the messages held, if any, for the block to write them again. Those the block holds
|
|
41
|
+
# again keep the count; any other messages held later start afresh.
|
|
42
|
+
def resend
|
|
43
|
+
return unless @logs
|
|
44
|
+
|
|
45
|
+
logs = @logs
|
|
46
|
+
@logs = nil
|
|
47
|
+
yield logs
|
|
48
|
+
ensure
|
|
49
|
+
@resends = 0 unless @logs
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Forgets the messages held, if any, and returns them, such as when the worker thread gives up
|
|
53
|
+
# on them, or after a fork, when the parent process writes them.
|
|
54
|
+
def release
|
|
55
|
+
logs = @logs
|
|
56
|
+
@logs = nil
|
|
57
|
+
@resends = 0
|
|
58
|
+
logs
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
class QueueProcessor
|
|
3
|
+
# Internal: the queue on which log messages, and requests such as a flush, wait for the worker
|
|
4
|
+
# thread of a queue processor to take them off.
|
|
5
|
+
#
|
|
6
|
+
# It knows the queue, whether its size is capped, and the process that created it, and adds to
|
|
7
|
+
# it, waiting for room on a full queue or not. After a fork, the child's copy of the queue holds
|
|
8
|
+
# the entries of the parent, which the parent's worker thread writes, so the child replaces it.
|
|
9
|
+
#
|
|
10
|
+
# Also used in a signal trap, where Ruby allows a Queue to wait, but not a Mutex to be locked.
|
|
11
|
+
class Inbox
|
|
12
|
+
# How often, in seconds, a thread waiting for room on the full queue asks whether to keep
|
|
13
|
+
# waiting (see #push), so that it does not wait for ever once nothing makes room.
|
|
14
|
+
WAIT_CHECK_SECONDS = 1
|
|
15
|
+
|
|
16
|
+
# Returns [Queue|SizedQueue] the queue.
|
|
17
|
+
attr_reader :queue
|
|
18
|
+
|
|
19
|
+
# Returns [Integer] the most entries the queue holds, or -1 for no limit.
|
|
20
|
+
attr_reader :max_queue_size
|
|
21
|
+
|
|
22
|
+
def initialize(max_queue_size)
|
|
23
|
+
@max_queue_size = max_queue_size
|
|
24
|
+
create
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Returns [true|false] whether the queue has a capped size.
|
|
28
|
+
def capped? = @capped
|
|
29
|
+
|
|
30
|
+
# Returns [true|false] whether the queue is capped, and holds as many entries as it can.
|
|
31
|
+
def full? = @capped && @queue.size >= max_queue_size
|
|
32
|
+
|
|
33
|
+
# Whether this is a child of the process that created the queue, whose copy of the queue then
|
|
34
|
+
# holds that process's entries.
|
|
35
|
+
def forked? = @pid != ::Process.pid
|
|
36
|
+
|
|
37
|
+
# Closes the queue, and creates an empty one, which belongs to this process.
|
|
38
|
+
def recreate
|
|
39
|
+
@queue.close
|
|
40
|
+
create
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Add to the `target` queue, the current one by default. Returns whether it was added.
|
|
44
|
+
#
|
|
45
|
+
# A queue that is not capped is never full, and its #push takes no timeout. On a full queue,
|
|
46
|
+
# without `wait`, returns false at once. With `wait`, waits for room while the block returns
|
|
47
|
+
# true, which it is asked as soon as the queue is found full, then every WAIT_CHECK_SECONDS, and
|
|
48
|
+
# returns false once it returns false, such as once the worker thread has stopped for good,
|
|
49
|
+
# since then nothing makes room. The block is not asked when the queue has room.
|
|
50
|
+
def push(item, target = queue, wait:)
|
|
51
|
+
return target.push(item) && true unless capped?
|
|
52
|
+
return true if target.push(item, timeout: 0)
|
|
53
|
+
|
|
54
|
+
loop do
|
|
55
|
+
return false unless wait && yield
|
|
56
|
+
return true if target.push(item, timeout: WAIT_CHECK_SECONDS)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Add the items, in order, without making the calling thread wait for room on a full queue,
|
|
61
|
+
# while the block returns true. Once the queue is full, one new thread adds the rest, in order,
|
|
62
|
+
# waiting for room while the block returns true. Once the block returns false, at once or while
|
|
63
|
+
# that thread waits, the items not yet added are passed to `otherwise` instead, on that thread.
|
|
64
|
+
# Returns true, without waiting. Takes no lock, so that it can be called from a signal trap.
|
|
65
|
+
def push_aside(items, otherwise, &adding)
|
|
66
|
+
rest = items.drop_while { |item| adding.call && push(item, wait: false) }
|
|
67
|
+
return true if rest.empty?
|
|
68
|
+
|
|
69
|
+
Thread.new { otherwise.call(rest.drop_while { |item| adding.call && push(item, wait: true, &adding) }) }
|
|
70
|
+
true
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Take the entries of the given class off the queue, and return them, in order. The others are added
|
|
74
|
+
# back, in order, behind any added meanwhile, without waiting for room; yields each one that a thread
|
|
75
|
+
# waiting for room took the place of, so that it is not lost unseen. Takes no lock.
|
|
76
|
+
def take(kind)
|
|
77
|
+
entries = Array.new(@queue.size) { @queue.pop(timeout: 0) }.compact
|
|
78
|
+
taken, others = entries.partition { |entry| entry.is_a?(kind) }
|
|
79
|
+
others.each { |entry| push(entry, wait: false) || (yield(entry) if block_given?) }
|
|
80
|
+
taken
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
def create
|
|
86
|
+
@pid = ::Process.pid
|
|
87
|
+
if max_queue_size == -1
|
|
88
|
+
@queue = Queue.new
|
|
89
|
+
@capped = false
|
|
90
|
+
else
|
|
91
|
+
@queue = SizedQueue.new(max_queue_size)
|
|
92
|
+
@capped = true
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
class QueueProcessor
|
|
3
|
+
# Internal: reopens the appender of a queue processor, on its worker thread, once asked.
|
|
4
|
+
#
|
|
5
|
+
# The worker thread asks it between writes, so the appender is never reopened while it is
|
|
6
|
+
# writing, such as a file being closed under a write, or a connection ended under a request.
|
|
7
|
+
# Nor is it reopened on the thread that asks, often a signal trap after a log rotation, where
|
|
8
|
+
# Ruby does not allow a Mutex to be locked, as connecting with a timeout does.
|
|
9
|
+
#
|
|
10
|
+
# It knows whether a reopen has been asked for, and reports a reopen that fails to the internal
|
|
11
|
+
# logger. Asked from any thread, and from a signal trap, so it records that on a Queue.
|
|
12
|
+
class Reopener
|
|
13
|
+
def initialize(appender, reporter)
|
|
14
|
+
@appender = appender
|
|
15
|
+
@reporter = reporter
|
|
16
|
+
@asked = Queue.new
|
|
17
|
+
@reopening = false
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Ask for the appender to be reopened before the worker thread writes again.
|
|
21
|
+
def ask
|
|
22
|
+
@asked << true
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Returns [true|false] whether a reopen has been asked for and the worker thread has not yet
|
|
26
|
+
# finished it, including one that it is running. Reads only a Queue and a flag, so that it can
|
|
27
|
+
# be called from any thread, or a signal trap. Reads them in the opposite order to the worker
|
|
28
|
+
# thread, which marks the reopen as running before it takes the request, so that a reopen
|
|
29
|
+
# starting between the two reads is still seen.
|
|
30
|
+
def pending? = !@asked.empty? || @reopening
|
|
31
|
+
|
|
32
|
+
# Reopen the appender, if asked to since it was last reopened. Called by the worker thread,
|
|
33
|
+
# between writes. A reopen that fails is reported rather than raised, whatever the exception,
|
|
34
|
+
# as Appenders#reopen reports it, so that one that is not a StandardError, such as a LoadError
|
|
35
|
+
# or NotImplementedError, does not stop the worker thread for good: the appender keeps what it
|
|
36
|
+
# had, and writing to it then fails, or not, as it would have.
|
|
37
|
+
#
|
|
38
|
+
# Returns nil, or the SystemStackError when reopening overflowed the stack, so that the worker
|
|
39
|
+
# thread hands over to a new one, since it must not overflow its stack again: on macOS a second
|
|
40
|
+
# stack overflow on the same thread crashes Ruby.
|
|
41
|
+
def reopen_if_asked
|
|
42
|
+
return if @asked.empty?
|
|
43
|
+
|
|
44
|
+
@reopening = true
|
|
45
|
+
@asked.clear
|
|
46
|
+
@appender.reopen if @appender.respond_to?(:reopen)
|
|
47
|
+
nil
|
|
48
|
+
rescue Exception => e
|
|
49
|
+
@reporter.log(:error, "Async: Failed to reopen the appender", e)
|
|
50
|
+
e if e.is_a?(SystemStackError)
|
|
51
|
+
ensure
|
|
52
|
+
@reopening = false
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
class QueueProcessor
|
|
3
|
+
# Internal: reports what happens on the worker thread of a queue processor, its progress and its
|
|
4
|
+
# problems, to the internal logger of its appender.
|
|
5
|
+
#
|
|
6
|
+
# It knows that logger, which is the appender's to choose, and makes sure that a report neither
|
|
7
|
+
# stops the worker thread nor raises into a calling thread, whatever the logger. It counts the log
|
|
8
|
+
# messages dropped because the queue was full, and those discarded because they could not be
|
|
9
|
+
# written, and reports each count at most once per interval.
|
|
10
|
+
class Reporter
|
|
11
|
+
# The kinds of positional parameter that Method#parameters reports, other than *rest.
|
|
12
|
+
POSITIONAL = %i[req opt].freeze
|
|
13
|
+
private_constant :POSITIONAL
|
|
14
|
+
|
|
15
|
+
def initialize(appender)
|
|
16
|
+
@appender = appender
|
|
17
|
+
@dropped = ThrottledCounter.new
|
|
18
|
+
@failed = ThrottledCounter.new(reported_at: nil)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Returns the internal logger of the appender, which it can replace at any time.
|
|
22
|
+
def logger = @appender.logger
|
|
23
|
+
|
|
24
|
+
# Returns [Integer] the number of log messages dropped because the queue was full.
|
|
25
|
+
def dropped_count = @dropped.total
|
|
26
|
+
|
|
27
|
+
# Returns [Integer] the number of log messages discarded because they could not be written,
|
|
28
|
+
# whether their content or the destination was at fault (see Subscriber#message_error?).
|
|
29
|
+
def failed_count = @failed.total
|
|
30
|
+
|
|
31
|
+
# Log to the internal logger, ignoring any StandardError. These calls may run after Ruby has
|
|
32
|
+
# released file handles during shutdown, and the internal logger may not take them, such as a
|
|
33
|
+
# Ruby Logger, which has no #trace, returned by a custom appender's own #logger.
|
|
34
|
+
#
|
|
35
|
+
# Such a logger also takes only the message. So the exception is passed only when there is one,
|
|
36
|
+
# and is added to the message instead for a logger that does not take it (see #takes_exception?).
|
|
37
|
+
def log(level, message, exception = nil)
|
|
38
|
+
internal = logger
|
|
39
|
+
return internal.public_send(level, message) unless exception
|
|
40
|
+
return internal.public_send(level, message, exception) if takes_exception?(internal, level)
|
|
41
|
+
|
|
42
|
+
internal.public_send(
|
|
43
|
+
level,
|
|
44
|
+
Utils.join_texts(message, " -- ", Utils.exception_name(exception), ": ", Utils.exception_message(exception))
|
|
45
|
+
)
|
|
46
|
+
rescue StandardError
|
|
47
|
+
nil
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Record a log message dropped because the queue was full, reporting the running count at most
|
|
51
|
+
# once every `interval` seconds. The report gives the reason for the latest drop: a queue too
|
|
52
|
+
# small for the volume of messages, or, when `stopped`, a worker thread that has stopped for
|
|
53
|
+
# good, so that nothing empties the queue, however large.
|
|
54
|
+
def dropped(max_queue_size, stopped:, interval:)
|
|
55
|
+
@dropped.record(interval: interval) do |count, seconds|
|
|
56
|
+
advice =
|
|
57
|
+
if stopped
|
|
58
|
+
"The worker thread has stopped after failing to write (see async_max_retries), so nothing " \
|
|
59
|
+
"empties the queue until SemanticLogger.reopen(force: true) starts it again"
|
|
60
|
+
else
|
|
61
|
+
"The queue is full (max_queue_size: #{max_queue_size}). " \
|
|
62
|
+
"Consider reducing the log level, increasing max_queue_size, or changing the appenders"
|
|
63
|
+
end
|
|
64
|
+
log(:warn, "Async: Dropped #{count} log messages in the last #{seconds.round} seconds. #{advice}")
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Report that the worker thread has fallen behind, when the log message it took off the queue
|
|
69
|
+
# was logged more than `threshold` seconds ago, with the number of messages still `queued`.
|
|
70
|
+
def lag(entry, threshold, queued)
|
|
71
|
+
diff = Time.now - entry.time
|
|
72
|
+
return unless diff > threshold
|
|
73
|
+
|
|
74
|
+
log(:warn, "Async: Appender thread has fallen behind by #{diff} seconds with #{queued} messages " \
|
|
75
|
+
"queued up. Consider reducing the log level or changing the appenders")
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Report the number of log messages queued ahead of a command, such as a flush, that is being
|
|
79
|
+
# submitted: at :warn for more than 1,000, at :info for more than 100, otherwise at :trace, and
|
|
80
|
+
# not at all for none.
|
|
81
|
+
def queued(count, command)
|
|
82
|
+
level = (count > 1_000 && :warn) || (count > 100 && :info) || (count.positive? && :trace)
|
|
83
|
+
log(level, "Async: Queued log messages: #{count}, running command: #{command}") if level
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Record log messages discarded because they could not be written, whether their content or the
|
|
87
|
+
# destination was at fault, if any. The first is reported straight away, with the error. After
|
|
88
|
+
# that the count is reported, with the latest error, at most once every `interval` seconds.
|
|
89
|
+
def failed(logs, exception, interval:)
|
|
90
|
+
return if logs.empty?
|
|
91
|
+
|
|
92
|
+
@failed.record(logs.size, interval: interval) do |count, seconds|
|
|
93
|
+
period = seconds ? " in the last #{seconds.round} seconds" : ""
|
|
94
|
+
log(:error, "Async: Discarded #{count} log message(s)#{period} that could not be written, " \
|
|
95
|
+
"most recently from #{logs.first.name}", exception)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
# Whether the logger's method for the level takes the exception after the message, as a Semantic
|
|
102
|
+
# Logger logger does. A Ruby or Rails Logger takes only the message: at most one positional
|
|
103
|
+
# parameter, and no *rest. Decided before the call, rather than by rescuing ArgumentError, which a
|
|
104
|
+
# logger that does take it can raise too, such as a broadcast logger one of whose loggers raises,
|
|
105
|
+
# after the others have written the report.
|
|
106
|
+
def takes_exception?(internal, level)
|
|
107
|
+
parameters = internal.method(level).parameters
|
|
108
|
+
parameters.any? { |type, _name| type == :rest } ||
|
|
109
|
+
parameters.count { |type, _name| POSITIONAL.include?(type) } > 1
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -1,43 +1,81 @@
|
|
|
1
1
|
module SemanticLogger
|
|
2
2
|
class QueueProcessor
|
|
3
3
|
# Internal: a command, such as :flush or :close, submitted on the queue for the worker thread
|
|
4
|
-
# to run, while the submitting thread waits for the reply.
|
|
4
|
+
# to run, while the submitting thread waits for the reply. Or a task, a Proc, handed over for the
|
|
5
|
+
# worker thread to run without waiting for it (see QueueProcessor#hand_over), which nobody waits on.
|
|
5
6
|
#
|
|
6
|
-
# It knows its command and how its reply gets back to the submitting
|
|
7
|
-
# that the submitting thread is never left waiting on a reply that cannot
|
|
7
|
+
# It knows its command, whether it has run, and how its reply gets back to the submitting
|
|
8
|
+
# thread, and makes sure that the submitting thread is never left waiting on a reply that cannot
|
|
9
|
+
# come. It also tells whoever counts the requests waiting, once, when it is no longer waiting.
|
|
8
10
|
class Request
|
|
9
11
|
attr_reader :command
|
|
10
12
|
|
|
11
|
-
|
|
13
|
+
# The block, if any, is called once the request is no longer waiting (see #done).
|
|
14
|
+
def initialize(command, &on_done)
|
|
12
15
|
@command = command
|
|
13
16
|
@replies = Queue.new
|
|
17
|
+
@on_done = on_done
|
|
18
|
+
@ran = false
|
|
19
|
+
# A single token, which the first call to #done takes.
|
|
20
|
+
@waiting = Queue.new
|
|
21
|
+
@waiting << true
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Whether the worker thread has run this request (see #reply).
|
|
25
|
+
def ran? = @ran
|
|
26
|
+
|
|
27
|
+
# Called once the request has run, or will not: one that could not be added to the queue, or
|
|
28
|
+
# that the worker thread took off the queue but skipped, such as one behind a close. Calls the
|
|
29
|
+
# block given to #initialize the first time only, whichever thread calls it, so that the request
|
|
30
|
+
# is counted out once. Also called in a signal trap, where Ruby allows a Queue to be used, but
|
|
31
|
+
# not a Mutex to be locked.
|
|
32
|
+
def done
|
|
33
|
+
@on_done&.call if @waiting.pop(timeout: 0)
|
|
14
34
|
end
|
|
15
35
|
|
|
16
36
|
# Called by the worker thread: run the block for this request, then reply with true, or
|
|
17
37
|
# with false when the block raised. Replies either way.
|
|
18
38
|
def reply
|
|
19
|
-
|
|
39
|
+
succeeded = false
|
|
20
40
|
yield
|
|
21
|
-
|
|
41
|
+
succeeded = true
|
|
22
42
|
ensure
|
|
23
|
-
@
|
|
43
|
+
@ran = true
|
|
44
|
+
done
|
|
45
|
+
@replies.push(succeeded)
|
|
24
46
|
end
|
|
25
47
|
|
|
26
|
-
# Called by the worker thread: run this command on the appender, then reply.
|
|
27
|
-
# once message processing must stop (:close).
|
|
48
|
+
# Called by the worker thread: run this command on the appender, or this task, then reply.
|
|
49
|
+
# Returns false once message processing must stop (:close). A command it does not know is
|
|
50
|
+
# yielded to the block, for the caller to report, and otherwise ignored.
|
|
28
51
|
def run(appender)
|
|
29
52
|
case command
|
|
53
|
+
when Proc
|
|
54
|
+
reply { command.call }
|
|
30
55
|
when :flush
|
|
31
56
|
reply { appender.flush }
|
|
32
57
|
when :close
|
|
33
58
|
reply { appender.close }
|
|
34
59
|
return false
|
|
35
60
|
else
|
|
36
|
-
reply {
|
|
61
|
+
reply { yield(command) if block_given? }
|
|
37
62
|
end
|
|
38
63
|
true
|
|
39
64
|
end
|
|
40
65
|
|
|
66
|
+
# Called once no worker thread will run this request, such as one passed on to the next worker
|
|
67
|
+
# thread when none is running (see QueueProcessor#pass_on). A task runs on a new thread, since
|
|
68
|
+
# nothing then writes to the appender, and returns that thread. A command is not run:
|
|
69
|
+
# it replies false, so that the submitting thread does not wait for it.
|
|
70
|
+
def abandon
|
|
71
|
+
# A handed-over task runs on a thread of its own, so that one that raises, even SystemStackError,
|
|
72
|
+
# never keeps the requests released with it from being released.
|
|
73
|
+
return Thread.new(&command) if command.is_a?(Proc)
|
|
74
|
+
|
|
75
|
+
done
|
|
76
|
+
@replies.push(false)
|
|
77
|
+
end
|
|
78
|
+
|
|
41
79
|
# Called by the submitting thread: wait for the reply from the worker thread.
|
|
42
80
|
#
|
|
43
81
|
# The block returns whether a worker thread that can still reply is running. Returns the
|
|
@@ -6,21 +6,35 @@ module SemanticLogger
|
|
|
6
6
|
# It knows the total number of occurrences, how many there have been since the previous
|
|
7
7
|
# report, and when that report was made. What to report, and where, is up to the caller.
|
|
8
8
|
#
|
|
9
|
-
# Thread-safe: occurrences may be recorded from any thread
|
|
9
|
+
# Thread-safe: occurrences may be recorded from any thread, and from a signal trap, such as one
|
|
10
|
+
# that logs while the queue is full. Ruby does not allow a Mutex to be locked in a trap, so the
|
|
11
|
+
# occurrences recorded there wait on a Queue, which it does allow, until a thread started for
|
|
12
|
+
# them, or the next read or record where the Mutex can be locked, adds them in.
|
|
10
13
|
class ThrottledCounter
|
|
11
|
-
# Returns [
|
|
12
|
-
|
|
14
|
+
# Returns [Float] the seconds on the monotonic clock, which the intervals between reports are
|
|
15
|
+
# measured with, so that a change of the system clock, such as NTP stepping it back, neither
|
|
16
|
+
# holds back a report nor makes one early.
|
|
17
|
+
def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
13
18
|
|
|
14
19
|
# Parameters:
|
|
15
|
-
# reported_at: [
|
|
16
|
-
# When counting starts, so that the first report is due
|
|
20
|
+
# reported_at: [Float|nil]
|
|
21
|
+
# When counting starts, on the monotonic clock (see .now), so that the first report is due
|
|
22
|
+
# one interval later.
|
|
17
23
|
# nil: Report the first occurrence straight away.
|
|
18
|
-
# Default:
|
|
19
|
-
def initialize(reported_at:
|
|
24
|
+
# Default: ThrottledCounter.now
|
|
25
|
+
def initialize(reported_at: ThrottledCounter.now)
|
|
20
26
|
@reported_at = reported_at
|
|
21
27
|
@total = 0
|
|
22
28
|
@unreported = 0
|
|
23
29
|
@mutex = Mutex.new
|
|
30
|
+
@waiting = Queue.new
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Returns [Integer] the total number of occurrences recorded. Read in a signal trap, it leaves
|
|
34
|
+
# out those recorded in a trap that have not been added in yet.
|
|
35
|
+
def total
|
|
36
|
+
TrapSafe.try_synchronize(@mutex) { add(0) }
|
|
37
|
+
@total
|
|
24
38
|
end
|
|
25
39
|
|
|
26
40
|
# Record one or more occurrences.
|
|
@@ -28,18 +42,47 @@ module SemanticLogger
|
|
|
28
42
|
# When the previous report was at least `interval` seconds ago, yields the number of
|
|
29
43
|
# occurrences since then (including these) and the seconds elapsed since that report (nil
|
|
30
44
|
# when there has been no report yet), then starts counting afresh for the next report.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
45
|
+
#
|
|
46
|
+
# In a signal trap the occurrences are only counted, and are added in and reported, when a
|
|
47
|
+
# report is due, by a thread started for them, just after the trap (see #wait).
|
|
48
|
+
def record(count = 1, interval:, &report_block)
|
|
49
|
+
report = nil
|
|
50
|
+
counted = TrapSafe.try_synchronize(@mutex) do
|
|
51
|
+
add(count)
|
|
52
|
+
now = ThrottledCounter.now
|
|
53
|
+
elapsed = @reported_at && (now - @reported_at)
|
|
54
|
+
unless elapsed && elapsed < interval
|
|
55
|
+
report = [@unreported, elapsed]
|
|
56
|
+
@unreported = 0
|
|
57
|
+
@reported_at = now
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
return wait(count, interval, &report_block) unless counted
|
|
61
|
+
|
|
62
|
+
# Reported without holding the lock, so that the report can record occurrences too.
|
|
63
|
+
yield(*report) if report
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
# Leaves occurrences recorded where the lock cannot be taken, such as in a signal trap, on the
|
|
69
|
+
# waiting Queue. The first to wait starts a thread, which can take the lock, to record none:
|
|
70
|
+
# that adds in every occurrence waiting by then, and reports them when a report is due. Any
|
|
71
|
+
# left waiting, such as those that come while it is not due, are added in by the next #record
|
|
72
|
+
# or #total that can lock.
|
|
73
|
+
def wait(count, interval, &report_block)
|
|
74
|
+
@waiting << count
|
|
75
|
+
Thread.new { record(0, interval: interval, &report_block) } if @waiting.size == 1
|
|
76
|
+
nil
|
|
77
|
+
end
|
|
38
78
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
79
|
+
# Adds the occurrences, and those waiting on the queue, to the counts. Holding the lock.
|
|
80
|
+
def add(count)
|
|
81
|
+
while (waiting = @waiting.pop(timeout: 0))
|
|
82
|
+
count += waiting
|
|
42
83
|
end
|
|
84
|
+
@total += count
|
|
85
|
+
@unreported += count
|
|
43
86
|
end
|
|
44
87
|
end
|
|
45
88
|
end
|