semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
require "semantic_logger/queue_processor/request"
|
|
2
2
|
require "semantic_logger/queue_processor/throttled_counter"
|
|
3
|
+
require "semantic_logger/queue_processor/reporter"
|
|
4
|
+
require "semantic_logger/queue_processor/backoff"
|
|
5
|
+
require "semantic_logger/queue_processor/inbox"
|
|
6
|
+
require "semantic_logger/queue_processor/held_write"
|
|
7
|
+
require "semantic_logger/queue_processor/batch_failure"
|
|
8
|
+
require "semantic_logger/queue_processor/reopener"
|
|
3
9
|
|
|
4
10
|
module SemanticLogger
|
|
5
11
|
# Internal class that processes log messages from a queue on a separate thread.
|
|
@@ -14,17 +20,15 @@ module SemanticLogger
|
|
|
14
20
|
# method, either once batch_size messages have accumulated, or
|
|
15
21
|
# batch_seconds have elapsed since the previous batch.
|
|
16
22
|
class QueueProcessor
|
|
23
|
+
# Internal: seconds that #reopen waits for a worker thread that has stopped to end.
|
|
24
|
+
STOPPED_WORKER_WAIT_SECONDS = 0.1
|
|
25
|
+
|
|
17
26
|
attr_accessor :lag_check_interval, :lag_threshold_s, :dropped_message_report_seconds,
|
|
18
27
|
:batch_size, :batch_seconds, :non_blocking
|
|
19
|
-
attr_reader :appender, :
|
|
20
|
-
:processed_count, :async_max_retries, :retry_count
|
|
28
|
+
attr_reader :appender, :signal, :processed_count
|
|
21
29
|
|
|
22
30
|
# Create a new processor and start its worker thread.
|
|
23
|
-
def self.start(**args)
|
|
24
|
-
processor = new(**args)
|
|
25
|
-
processor.thread
|
|
26
|
-
processor
|
|
27
|
-
end
|
|
31
|
+
def self.start(**args) = new(**args).tap(&:thread)
|
|
28
32
|
|
|
29
33
|
# Parameters:
|
|
30
34
|
# appender: [SemanticLogger::Subscriber]
|
|
@@ -65,19 +69,29 @@ module SemanticLogger
|
|
|
65
69
|
# Default: false
|
|
66
70
|
#
|
|
67
71
|
# dropped_message_report_seconds: [Integer]
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
72
|
+
# Log the count of messages dropped because the queue was full, with non_blocking or once
|
|
73
|
+
# the worker thread has stopped (see async_max_retries), to the internal logger at most once
|
|
74
|
+
# every this number of seconds. Also limits how often messages discarded because they could
|
|
75
|
+
# not be written are reported, after the first.
|
|
71
76
|
# Default: 30
|
|
72
77
|
#
|
|
73
78
|
# async_max_retries: [Integer]
|
|
74
79
|
# Maximum number of consecutive times to restart the worker thread after it raises an
|
|
75
80
|
# exception while processing messages. Each restart first sleeps for `retry_count` seconds
|
|
76
|
-
# (1s, then 2s, ...) as a back-off
|
|
77
|
-
#
|
|
78
|
-
#
|
|
81
|
+
# (1s, then 2s, ...) as a back-off, unless a flush or close is waiting (see #flush), or the
|
|
82
|
+
# process is exiting (see #process_exiting=). Once this many consecutive retries are
|
|
83
|
+
# exhausted the thread stops instead of restarting. The counter resets to zero whenever a
|
|
84
|
+
# message is processed successfully. A message or batch that the destination failed is
|
|
85
|
+
# written again after the back-off when the appender says that the write wrote none of it
|
|
86
|
+
# (see Subscriber#wrote_nothing?), up to HeldWrite::MAX_RESENDS times after the destination
|
|
87
|
+
# answered with a failure, and is otherwise discarded.
|
|
79
88
|
# -1: Retry indefinitely and never stop the thread (the pre-v5 behaviour). The back-off
|
|
80
89
|
# still applies, and still resets after any successful message.
|
|
90
|
+
# A worker thread started again by #reopen in the same process carries on with the count, so
|
|
91
|
+
# that one that had used all its retries stops again at its first failure, rather than hold
|
|
92
|
+
# up the threads logging to it with a back-off as long again while the destination is still
|
|
93
|
+
# unavailable; one that stopped before, such as after a fatal exception, has the retries it
|
|
94
|
+
# had left. Only the worker thread of a child process, after a fork, starts counting afresh.
|
|
81
95
|
# Default: 100
|
|
82
96
|
def initialize(appender:,
|
|
83
97
|
max_queue_size: 10_000,
|
|
@@ -90,7 +104,6 @@ module SemanticLogger
|
|
|
90
104
|
dropped_message_report_seconds: 30,
|
|
91
105
|
async_max_retries: 100)
|
|
92
106
|
@appender = appender
|
|
93
|
-
@max_queue_size = max_queue_size
|
|
94
107
|
@lag_check_interval = lag_check_interval
|
|
95
108
|
@lag_threshold_s = lag_threshold_s
|
|
96
109
|
@batch = batch
|
|
@@ -98,15 +111,15 @@ module SemanticLogger
|
|
|
98
111
|
@batch_seconds = batch_seconds
|
|
99
112
|
@non_blocking = non_blocking
|
|
100
113
|
@dropped_message_report_seconds = dropped_message_report_seconds
|
|
101
|
-
@
|
|
102
|
-
@retry_count = 0
|
|
114
|
+
@backoff = Backoff.new(async_max_retries)
|
|
103
115
|
@thread = nil
|
|
104
116
|
# Only batch mode parks the worker on the signal; streaming mode never touches it.
|
|
105
117
|
@signal = Concurrent::Event.new if batch
|
|
106
|
-
@
|
|
107
|
-
@
|
|
118
|
+
@reporter = Reporter.new(appender)
|
|
119
|
+
@reopener = Reopener.new(appender, @reporter)
|
|
108
120
|
@processed_count = 0
|
|
109
|
-
|
|
121
|
+
@inbox = Inbox.new(max_queue_size)
|
|
122
|
+
@held = HeldWrite.new
|
|
110
123
|
|
|
111
124
|
return unless batch? && !appender.respond_to?(:batch)
|
|
112
125
|
|
|
@@ -114,33 +127,52 @@ module SemanticLogger
|
|
|
114
127
|
end
|
|
115
128
|
|
|
116
129
|
# Internal logger used to report problems encountered while processing log messages.
|
|
117
|
-
def logger =
|
|
130
|
+
def logger = @reporter.logger
|
|
131
|
+
|
|
132
|
+
# Returns [Integer] the maximum number of consecutive retries after a failure (see #initialize).
|
|
133
|
+
def async_max_retries = @backoff.max_retries
|
|
134
|
+
|
|
135
|
+
# Returns [Integer] the number of consecutive failures, since a message was last processed.
|
|
136
|
+
def retry_count = @backoff.retry_count
|
|
118
137
|
|
|
119
138
|
# Returns [Thread] the worker thread.
|
|
120
139
|
#
|
|
121
|
-
# Starts the worker thread if it is not currently running.
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
@thread = spawn_worker
|
|
126
|
-
end
|
|
140
|
+
# Starts the worker thread if it is not currently running. It carries on with the count of
|
|
141
|
+
# failures in a row (see async_max_retries).
|
|
142
|
+
def thread = active? ? @thread : spawn_worker
|
|
127
143
|
|
|
128
144
|
# Returns [true|false] whether the worker thread is running.
|
|
129
145
|
def active? = @thread&.alive?
|
|
130
146
|
|
|
147
|
+
# Returns [true|false] whether the thread is the worker thread, the one that writes to the
|
|
148
|
+
# appender. Takes no lock, so that it can be called from a signal trap.
|
|
149
|
+
def worker_thread?(thread = Thread.current) = @thread.equal?(thread)
|
|
150
|
+
|
|
151
|
+
# Returns [true|false] whether the thread is the worker thread while another thread may be waiting
|
|
152
|
+
# for it: the queue is full, so that a thread adding to it waits for it to make room, or a request,
|
|
153
|
+
# such as a flush or close, is waiting for it to run it (see Backoff#request_waiting?), whoever
|
|
154
|
+
# submitted it, so a message it logs that could wait may be dropped (see #waited_on?). Lock free.
|
|
155
|
+
def holding_up?(thread = Thread.current) = worker_thread?(thread) && (@inbox.full? || @backoff.request_waiting?)
|
|
156
|
+
|
|
157
|
+
# Returns [Queue|SizedQueue] the queue of log messages and requests waiting for the worker thread.
|
|
158
|
+
def queue = @inbox.queue
|
|
159
|
+
|
|
160
|
+
# Returns [Integer] the most entries the queue holds, or -1 for no limit.
|
|
161
|
+
def max_queue_size = @inbox.max_queue_size
|
|
162
|
+
|
|
131
163
|
# Returns [true|false] whether the queue has a capped size.
|
|
132
|
-
def capped? = @capped
|
|
164
|
+
def capped? = @inbox.capped?
|
|
133
165
|
|
|
134
166
|
# Returns [true|false] whether messages are dropped instead of blocking when the queue is full.
|
|
135
167
|
# Only a capped queue can drop messages.
|
|
136
168
|
def non_blocking? = @non_blocking && capped?
|
|
137
169
|
|
|
138
170
|
# Returns [Integer] the number of log messages dropped because the queue was full.
|
|
139
|
-
def dropped_count = @
|
|
171
|
+
def dropped_count = @reporter.dropped_count
|
|
140
172
|
|
|
141
|
-
# Returns [Integer] the number of log messages discarded because
|
|
142
|
-
#
|
|
143
|
-
def failed_count = @
|
|
173
|
+
# Returns [Integer] the number of log messages discarded because they could not be written,
|
|
174
|
+
# whether their content or the destination was at fault (see Subscriber#message_error?).
|
|
175
|
+
def failed_count = @reporter.failed_count
|
|
144
176
|
|
|
145
177
|
# Returns [true|false] whether messages are processed in batches.
|
|
146
178
|
def batch? = @batch
|
|
@@ -155,19 +187,16 @@ module SemanticLogger
|
|
|
155
187
|
# calling thread, and the count of dropped messages is reported periodically.
|
|
156
188
|
#
|
|
157
189
|
# Once the worker thread has stopped for good (see async_max_retries), nothing will empty the
|
|
158
|
-
# queue, so messages are also dropped instead of blocking the calling thread forever.
|
|
190
|
+
# queue, so messages are also dropped instead of blocking the calling thread forever. That
|
|
191
|
+
# includes a message already waiting for room on the full queue when the worker thread stops.
|
|
192
|
+
#
|
|
193
|
+
# Nor does a thread that the worker thread may itself be waiting for wait for room on the full
|
|
194
|
+
# queue (see #waited_on?): the message is dropped instead.
|
|
159
195
|
def log(log)
|
|
160
|
-
enqueued =
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
rescue ThreadError
|
|
165
|
-
message_dropped
|
|
166
|
-
enqueued = false
|
|
167
|
-
end
|
|
168
|
-
else
|
|
169
|
-
queue << log
|
|
170
|
-
end
|
|
196
|
+
enqueued = @inbox.push(log, wait: !(non_blocking? || stopped?)) { !stopped? && !waited_on? }
|
|
197
|
+
# Reported at most once every dropped_message_report_seconds, with whether it was dropped
|
|
198
|
+
# because the worker thread has stopped for good.
|
|
199
|
+
message_dropped unless enqueued
|
|
171
200
|
|
|
172
201
|
# For batches wake up the processing thread once the number of queued messages has been
|
|
173
202
|
# exceeded. Also wake it on a drop so it drains the full queue rather than waiting out the
|
|
@@ -179,36 +208,142 @@ module SemanticLogger
|
|
|
179
208
|
|
|
180
209
|
# Flush all queued log entries to the appender.
|
|
181
210
|
# All queued log messages are written and then the appender is flushed.
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
211
|
+
#
|
|
212
|
+
# A flush or close does not wait out back-offs after the destination failed, nor each message
|
|
213
|
+
# ahead of it failing in turn: submitting it ends the back-off under way, and until it has run, a
|
|
214
|
+
# message or batch that the destination fails is discarded, rather than written again after a
|
|
215
|
+
# back-off, and so are the messages queued after that one, ahead of the flush or close, without
|
|
216
|
+
# being written (see Backoff#give_up?).
|
|
217
|
+
def flush = submit_request(:flush)
|
|
185
218
|
|
|
186
219
|
# Flush any outstanding messages and close the appender.
|
|
187
|
-
def close
|
|
188
|
-
|
|
220
|
+
def close = submit_request(:close)
|
|
221
|
+
|
|
222
|
+
# Hand the block over to the worker thread, which runs it between writes, after the entries
|
|
223
|
+
# already queued, without waiting for it to run. The block reports its own errors: one that it
|
|
224
|
+
# raises is handled as a failure of the appender.
|
|
225
|
+
#
|
|
226
|
+
# Never waits, so that it can be called on a thread that the worker thread may itself be
|
|
227
|
+
# waiting for, such as the worker thread of an appender that it hands entries to, whose full
|
|
228
|
+
# queue it waits on. When the queue is full, a new thread adds the block to the queue, waiting for
|
|
229
|
+
# room, rather than the calling thread: once the calling thread returns, the worker thread can
|
|
230
|
+
# make room (see Inbox#push_aside). When the worker thread is not running, or has stopped (see
|
|
231
|
+
# #stopped?), or stops while that thread waits, a new thread runs the block instead, since nothing
|
|
232
|
+
# then writes to the appender. Takes no lock, so that it can be called from a signal trap.
|
|
233
|
+
#
|
|
234
|
+
# A batching worker thread is not woken up, since that would write a partial batch: it runs the
|
|
235
|
+
# block at its next batch, at most batch_seconds later.
|
|
236
|
+
def hand_over(&task) = pass_on([Request.new(task)])
|
|
237
|
+
|
|
238
|
+
# Tell the worker thread whether the process is exiting, before and after the flush at exit
|
|
239
|
+
# (see Backoff#exiting=). While it is, the worker thread acts as though a flush were waiting
|
|
240
|
+
# (see #flush), already before the flush at exit reaches this queue: it stops backing off and
|
|
241
|
+
# writing again what the destination failed, and once the destination fails, discards the
|
|
242
|
+
# messages queued ahead of the next request, such as the flush at exit, so that the process exits
|
|
243
|
+
# promptly during an outage. Takes no lock, so that it can be called from any thread, or a signal
|
|
244
|
+
# trap.
|
|
245
|
+
def process_exiting=(exiting)
|
|
246
|
+
@backoff.exiting = exiting
|
|
189
247
|
end
|
|
190
248
|
|
|
191
|
-
#
|
|
249
|
+
# Reopen the appender, on the worker thread, and the queue and worker thread after a fork (see
|
|
250
|
+
# #renew_after_fork), or start the worker thread again within the same process once it has ended
|
|
251
|
+
# or stopped.
|
|
252
|
+
#
|
|
253
|
+
# The worker thread reopens the appender before it next takes messages off the queue, or writes
|
|
254
|
+
# again the ones it holds (see Reopener), so that it is never reopened while it is writing, and
|
|
255
|
+
# this returns before it has. After a fork, the reopen is asked for before the child's new worker
|
|
256
|
+
# thread starts, so that it reopens the appender before it writes anything. To wake up a
|
|
257
|
+
# streaming worker thread waiting for the next message, :reopen is added to the queue, without
|
|
258
|
+
# waiting for room: one that is not waiting has messages to write, and reopens the appender before
|
|
259
|
+
# them. A batching worker thread is not woken up, since that would write a partial batch, so
|
|
260
|
+
# nothing is added to its queue: it reopens the appender at its next batch, at most batch_seconds
|
|
261
|
+
# later, or when a flush or close wakes it up, before running them.
|
|
262
|
+
#
|
|
263
|
+
# Within the same process, such as after a log rotation, the queue and its entries are kept, and
|
|
264
|
+
# so is the worker thread while it runs (see #thread). It writes the queued entries in order,
|
|
265
|
+
# starting with one it has already taken off the queue, and no other thread writes to the
|
|
266
|
+
# appender meanwhile. A worker thread that has ended is replaced by one that carries on with the
|
|
267
|
+
# queued entries, and with the count of failures in a row (see async_max_retries). So is one that
|
|
268
|
+
# has stopped (see #stopped?), which may still be reporting that it stopped: it is given a moment
|
|
269
|
+
# to end first, so that its report comes before the new thread's. A worker thread that has
|
|
270
|
+
# stopped no longer writes to the appender, nor changes the state of this processor, so none that
|
|
271
|
+
# was replaced can change it once a new one has started.
|
|
272
|
+
#
|
|
273
|
+
# Also called from a signal trap, such as one that reopens the log files after a rotation, where
|
|
274
|
+
# Ruby does not allow a Mutex to be locked, so it takes none: it uses only Queue and Thread
|
|
275
|
+
# methods, which Ruby allows there.
|
|
192
276
|
def reopen
|
|
277
|
+
@reopener.ask
|
|
278
|
+
renew_after_fork
|
|
279
|
+
@inbox.push(:reopen, wait: false) unless batch?
|
|
280
|
+
restart_worker
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Wait until the worker thread has reopened the appender, when a reopen has been asked for (see
|
|
284
|
+
# #reopen) and it has not yet finished it, such as in a forked child whose worker thread has not
|
|
285
|
+
# yet written anything. Returns at once when none is pending, so that it costs nothing otherwise.
|
|
286
|
+
#
|
|
287
|
+
# The worker thread reopens the appender before it takes anything off the queue, so a flush
|
|
288
|
+
# submitted now runs after the reopen, and returns once it has (see #flush). Like #flush, it
|
|
289
|
+
# returns without waiting when called on the worker thread itself, or once that thread has
|
|
290
|
+
# stopped. Takes no lock, as #flush takes none, so that it can be called from a signal trap.
|
|
291
|
+
def await_reopen = !@reopener.pending? || flush
|
|
292
|
+
|
|
293
|
+
# After a fork, the queue holds the entries that the parent process had queued, which the
|
|
294
|
+
# parent's worker thread writes, so the child starts again with an empty queue and a worker
|
|
295
|
+
# thread of its own, rather than write them a second time. Does nothing in the process that
|
|
296
|
+
# created the queue. Asks for no reopen of the appender: #reopen asks before it calls this.
|
|
297
|
+
#
|
|
298
|
+
# Called on the thread that reopens after the fork, which takes no lock (see #reopen). Called
|
|
299
|
+
# by Processor#reopen, through Appenders#renew_after_fork, before the appender is asked to
|
|
300
|
+
# reopen: the new worker thread has nothing to write until the processor's worker thread, which
|
|
301
|
+
# asks first, hands it entries.
|
|
302
|
+
def renew_after_fork
|
|
303
|
+
return unless @inbox.forked?
|
|
304
|
+
|
|
193
305
|
# Workaround CRuby crash on fork by recreating queue on reopen
|
|
194
306
|
# https://github.com/reidmorrison/semantic_logger/issues/103
|
|
195
|
-
@
|
|
196
|
-
|
|
197
|
-
|
|
307
|
+
@inbox.recreate
|
|
308
|
+
# The parent process writes the messages it held to write again.
|
|
309
|
+
@held.release
|
|
198
310
|
@thread&.kill if @thread&.alive?
|
|
199
|
-
|
|
311
|
+
# A worker thread new to this process, which has not failed to write yet.
|
|
312
|
+
@backoff.restart
|
|
313
|
+
spawn_worker
|
|
200
314
|
end
|
|
201
315
|
|
|
202
316
|
private
|
|
203
317
|
|
|
204
|
-
# Start
|
|
318
|
+
# Start a new worker thread, as @thread. It carries on with the count of failures in a row,
|
|
319
|
+
# including one taking over from a thread that overflowed its stack (see #process).
|
|
320
|
+
#
|
|
321
|
+
# The new thread waits until it has been assigned to @thread before it takes anything off the
|
|
322
|
+
# queue. Otherwise it could overflow its stack and hand over to another new thread before then,
|
|
323
|
+
# and the thread that started it would then assign it, after it has ended, leaving the one it
|
|
324
|
+
# handed over to running unseen beside the next one that #thread starts. It waits on a Queue,
|
|
325
|
+
# whose #push Ruby allows in a signal trap (see #reopen).
|
|
205
326
|
def spawn_worker
|
|
206
327
|
@stopped = false
|
|
207
328
|
@stack_overflow = nil
|
|
208
|
-
|
|
329
|
+
assigned = Queue.new
|
|
330
|
+
@thread = Thread.new do
|
|
331
|
+
assigned.pop
|
|
209
332
|
name_worker
|
|
210
333
|
process
|
|
211
334
|
end
|
|
335
|
+
@thread.tap { assigned << true }
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
# Start the worker thread again within the same process, unless it is running (see #reopen).
|
|
339
|
+
#
|
|
340
|
+
# A worker thread that has stopped (see #stopped?) may still be reporting that it stopped, so it
|
|
341
|
+
# is given a moment to end first. Then a new one is started, whether it has ended or not.
|
|
342
|
+
def restart_worker
|
|
343
|
+
return thread unless stopped?
|
|
344
|
+
|
|
345
|
+
@thread&.join(STOPPED_WORKER_WAIT_SECONDS)
|
|
346
|
+
spawn_worker
|
|
212
347
|
end
|
|
213
348
|
|
|
214
349
|
# Name the worker thread after its appender, such as "SL File", short enough for tools that
|
|
@@ -221,105 +356,129 @@ module SemanticLogger
|
|
|
221
356
|
nil
|
|
222
357
|
end
|
|
223
358
|
|
|
224
|
-
def create_queue
|
|
225
|
-
if max_queue_size == -1
|
|
226
|
-
@queue = Queue.new
|
|
227
|
-
@capped = false
|
|
228
|
-
else
|
|
229
|
-
@queue = SizedQueue.new(max_queue_size)
|
|
230
|
-
@capped = true
|
|
231
|
-
end
|
|
232
|
-
end
|
|
233
|
-
|
|
234
359
|
# Separate thread for processing log messages.
|
|
360
|
+
#
|
|
361
|
+
# This thread is designed to never go down unless the main thread terminates or the appender is
|
|
362
|
+
# closed. A StandardError while processing restarts the processing, after a back-off, up to
|
|
363
|
+
# async_max_retries times. Any other exception, even one from the first report, stops this
|
|
364
|
+
# processor, which then drops new messages, rather than ending the thread and leaving the
|
|
365
|
+
# calling threads blocked on a full queue.
|
|
366
|
+
#
|
|
367
|
+
# A flush or close that is waiting (see #flush), or the process exiting (see #process_exiting=),
|
|
368
|
+
# does not wait out a back-off, so processing then restarts straight away, without counting a
|
|
369
|
+
# retry, and discards the messages queued ahead of the next request without writing them (see
|
|
370
|
+
# Backoff#give_up?).
|
|
235
371
|
def process
|
|
236
|
-
|
|
237
|
-
# or the appender is closed.
|
|
238
|
-
logger.trace "Async: Appender thread active"
|
|
372
|
+
safe_log(:trace, "Async: Appender thread active")
|
|
239
373
|
begin
|
|
240
|
-
|
|
374
|
+
resend_held if reopen_appender
|
|
375
|
+
carry_on = @stack_overflow || (batch? ? process_messages_in_batches : process_messages)
|
|
241
376
|
rescue StandardError => e
|
|
242
|
-
if
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
sleep(retry_count)
|
|
377
|
+
retry if @backoff.give_up?(e)
|
|
378
|
+
if @backoff.retry?
|
|
379
|
+
safe_log(:warn, "Async: Restarting due to exception, retry #{retry_count} of #{@backoff.limit}, " \
|
|
380
|
+
"sleeping #{retry_count}s", e)
|
|
381
|
+
@backoff.wait
|
|
248
382
|
retry
|
|
249
|
-
else
|
|
250
|
-
@stopped = true
|
|
251
|
-
safe_log(:error, "Async: Stopping after #{retry_count} failed retries, dropping new messages", e)
|
|
252
383
|
end
|
|
253
|
-
|
|
384
|
+
|
|
254
385
|
@stopped = true
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
# A thread that overflowed its stack hands the queue over to a new one (see #stack_overflowed).
|
|
258
|
-
@thread = carry_on ? spawn_worker : nil
|
|
259
|
-
safe_log(:trace, "Async: Thread has stopped") unless carry_on
|
|
386
|
+
discard_held(e)
|
|
387
|
+
safe_log(:error, "Async: Stopping after #{retry_count} failed retries, dropping new messages", e)
|
|
260
388
|
end
|
|
389
|
+
rescue Exception => e
|
|
390
|
+
@stopped = true
|
|
391
|
+
discard_held(e)
|
|
392
|
+
safe_log(:error, "Async: Stopping due to fatal exception", e)
|
|
393
|
+
ensure
|
|
394
|
+
# A thread that overflowed its stack hands the queue over to a new one (see #stack_overflowed).
|
|
395
|
+
# It is still the worker thread, since #reopen replaces only one that has ended or stopped, and
|
|
396
|
+
# one that overflowed has neither. Any other thread leaves @thread as it is, since one that has
|
|
397
|
+
# ended reads as inactive.
|
|
398
|
+
spawn_worker if carry_on
|
|
399
|
+
safe_log(:trace, "Async: Thread has stopped") unless carry_on
|
|
261
400
|
end
|
|
262
401
|
|
|
263
|
-
#
|
|
264
|
-
#
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
402
|
+
# Returns [true|false] whether the calling thread is one that the worker thread may be waiting
|
|
403
|
+
# for: the worker thread of an appender written to by the appender of this processor, which is
|
|
404
|
+
# then the Appenders of the main Processor, while that appender's own queue is full, or a flush or
|
|
405
|
+
# close submitted to it, such as by the worker thread, waits for it (see Appenders#holding_up?).
|
|
406
|
+
# Otherwise the worker thread, if it waits at all, waits on another thread, which makes room.
|
|
407
|
+
# Asked only once a log message finds the queue full, then before each wait for room.
|
|
408
|
+
def waited_on? = appender.respond_to?(:holding_up?) && appender.holding_up?
|
|
409
|
+
|
|
410
|
+
# Report to the internal logger, ignoring any error (see Reporter#log).
|
|
411
|
+
def safe_log(level, message, exception = nil) = @reporter.log(level, message, exception)
|
|
270
412
|
|
|
271
413
|
# Streaming: write each log message to the appender as it is dequeued.
|
|
272
414
|
#
|
|
273
415
|
# Returns true when processing must carry on in a new thread (see #stack_overflowed).
|
|
274
416
|
def process_messages
|
|
275
417
|
count = 0
|
|
276
|
-
while (message = queue.pop)
|
|
418
|
+
while reopen_appender && (message = queue.pop)
|
|
277
419
|
if message.is_a?(Log)
|
|
420
|
+
gave_up = @backoff.gave_up
|
|
421
|
+
next messages_failed([message], gave_up) if gave_up
|
|
422
|
+
|
|
278
423
|
write_message(message)
|
|
279
424
|
return true if @stack_overflow
|
|
280
425
|
|
|
281
|
-
@
|
|
426
|
+
@backoff.reset
|
|
282
427
|
count += 1
|
|
283
428
|
# Check every few log messages whether this appender thread is falling behind
|
|
284
429
|
if count > lag_check_interval
|
|
285
430
|
check_lag(message)
|
|
286
431
|
count = 0
|
|
287
432
|
end
|
|
288
|
-
|
|
289
|
-
break unless message
|
|
433
|
+
elsif message.is_a?(Request)
|
|
434
|
+
break unless run_command(message)
|
|
290
435
|
end
|
|
436
|
+
# Otherwise :reopen, which #reopen adds to wake up this thread to reopen the appender.
|
|
291
437
|
end
|
|
292
|
-
|
|
293
|
-
|
|
438
|
+
return true if @stack_overflow
|
|
439
|
+
|
|
440
|
+
safe_log(:trace, "Async: Queue Closed")
|
|
441
|
+
release_requests
|
|
294
442
|
end
|
|
295
443
|
|
|
296
444
|
# Batching: group up log messages before writing them via the appender's #batch method.
|
|
297
445
|
#
|
|
446
|
+
# Once the destination failed while the worker thread was waited on (see Backoff#gave_up), the
|
|
447
|
+
# messages taken off the queue ahead of the first request, such as a flush, are discarded without
|
|
448
|
+
# being written, and those taken off the queue after it are written as a batch, as usual.
|
|
449
|
+
#
|
|
298
450
|
# Returns true when processing must carry on in a new thread (see #stack_overflowed).
|
|
299
451
|
def process_messages_in_batches
|
|
300
452
|
loop do
|
|
301
453
|
# Wait for batch interval or number of messages to be exceeded.
|
|
302
454
|
signal.wait(batch_seconds)
|
|
303
455
|
|
|
456
|
+
# Counted before the appender is reopened, so that a flush or close counted here, and so run
|
|
457
|
+
# in this batch, comes after the reopen asked for before it.
|
|
458
|
+
message_count = queue.length
|
|
459
|
+
return true unless reopen_appender
|
|
460
|
+
|
|
304
461
|
logs = []
|
|
305
462
|
commands = []
|
|
306
|
-
|
|
307
|
-
|
|
463
|
+
discarded = []
|
|
464
|
+
gave_up = @backoff.gave_up
|
|
308
465
|
message_count.times do
|
|
309
466
|
# Queue#pop(true) raises an exception when there are no more messages, which is considered expensive.
|
|
310
467
|
message = queue.pop
|
|
311
468
|
if message.is_a?(Log)
|
|
469
|
+
next discarded << message if gave_up && commands.empty?
|
|
470
|
+
|
|
312
471
|
logs << message
|
|
313
|
-
if
|
|
314
|
-
|
|
315
|
-
first = false
|
|
316
|
-
end
|
|
317
|
-
else
|
|
472
|
+
check_lag(message) if logs.size == 1
|
|
473
|
+
elsif message.is_a?(Request)
|
|
318
474
|
commands << message
|
|
319
475
|
end
|
|
320
476
|
end
|
|
321
|
-
|
|
322
|
-
|
|
477
|
+
messages_failed(discarded, gave_up) unless discarded.empty?
|
|
478
|
+
# Stop processing once a command (e.g. :close) signals the thread to terminate, releasing the
|
|
479
|
+
# requests left (see #release_requests), unless the stack overflowed, when a new thread carries
|
|
480
|
+
# on (see #run_command).
|
|
481
|
+
return !@stack_overflow.nil? || release_requests unless write_batch_then_run(logs, commands)
|
|
323
482
|
return true if @stack_overflow
|
|
324
483
|
|
|
325
484
|
signal.reset unless queue.size >= batch_size
|
|
@@ -330,68 +489,136 @@ module SemanticLogger
|
|
|
330
489
|
# queue with them. Returns false once a command (:close) stops processing.
|
|
331
490
|
#
|
|
332
491
|
# When the batch cannot be written, for example because the destination is unavailable, the
|
|
333
|
-
#
|
|
334
|
-
#
|
|
335
|
-
#
|
|
492
|
+
# messages not yet written are held to be written again after the back-off, or discarded and
|
|
493
|
+
# counted as failed (see #not_written). The commands still run before the error propagates to
|
|
494
|
+
# #process to back off. Otherwise they would be lost, and a caller waiting on #flush or #close, such
|
|
495
|
+
# as the flush at exit, would wait forever. Since they are waiting, the messages are then discarded
|
|
496
|
+
# rather than held. A command behind a close is not run, but released (see Request#abandon).
|
|
336
497
|
def write_batch_then_run(logs, commands)
|
|
498
|
+
accounted = processed_count + failed_count
|
|
337
499
|
unless logs.empty?
|
|
338
500
|
write_batch(logs)
|
|
339
|
-
@
|
|
501
|
+
@backoff.reset
|
|
340
502
|
end
|
|
341
|
-
commands.all? { |command| command
|
|
503
|
+
commands.all? { |command| run_command(command) }
|
|
342
504
|
rescue StandardError => e
|
|
505
|
+
# Every message taken off the queue was written or already discarded, except those still
|
|
506
|
+
# to be written when the error was raised: the last ones, since a batch is written in order.
|
|
507
|
+
# The write that raised wrote none of them, if the appender says so, and the rest were not
|
|
508
|
+
# sent at all.
|
|
509
|
+
unwritten = logs.size - (processed_count + failed_count - accounted)
|
|
510
|
+
not_written(logs.last(unwritten), e, hold: commands.empty?) if unwritten.positive?
|
|
511
|
+
# The commands run once the batch is written, until one raises: the last one that ran, if any.
|
|
512
|
+
failed = commands.select(&:ran?).last
|
|
343
513
|
raise e if commands.all? { |command| process_message_after_failure(command) }
|
|
344
514
|
|
|
345
|
-
|
|
515
|
+
message = failed ? "Failed to #{failed.command} the appender" : "Closed without writing #{unwritten} log messages"
|
|
516
|
+
safe_log(:warn, "Async: #{message}", e)
|
|
346
517
|
false
|
|
518
|
+
ensure
|
|
519
|
+
# Once the stack has overflowed, the commands not run are passed on to the next worker thread
|
|
520
|
+
# (see #run_command). Otherwise they were behind a close.
|
|
521
|
+
commands.reject(&:ran?).then { |not_run| @stack_overflow ? pass_on(not_run) : not_run.each(&:abandon) }
|
|
347
522
|
end
|
|
348
523
|
|
|
349
|
-
# Run a command after a batch could not be written
|
|
350
|
-
# stops processing, even when the command
|
|
524
|
+
# Run a command after a batch could not be written, unless it has run already, such as a flush
|
|
525
|
+
# that raised. Returns false once a command (:close) stops processing, even when the command
|
|
526
|
+
# itself fails.
|
|
351
527
|
def process_message_after_failure(command)
|
|
352
|
-
command.
|
|
528
|
+
return command.command != :close if command.ran?
|
|
529
|
+
|
|
530
|
+
run_command(command)
|
|
353
531
|
rescue StandardError => e
|
|
354
532
|
safe_log(:warn, "Async: Failed to #{command.command} the appender", e)
|
|
355
533
|
command.command != :close
|
|
356
534
|
end
|
|
357
535
|
|
|
536
|
+
# Run a command taken off the queue on the appender, reporting one it does not know. Returns
|
|
537
|
+
# false once message processing must stop (:close).
|
|
538
|
+
#
|
|
539
|
+
# A command that overflows the stack, such as a task handed over (see #hand_over), has failed,
|
|
540
|
+
# even a close, and this thread takes nothing more off the queue: a new one carries on, as after
|
|
541
|
+
# a message that overflowed it (see #stack_overflowed). Nor does it run any more commands, since on
|
|
542
|
+
# macOS a second stack overflow on the same thread crashes Ruby: those taken off the queue with it,
|
|
543
|
+
# in the same batch, or after a batch that overflowed it, return true here without running, and
|
|
544
|
+
# are passed on to the new one (see #write_batch_then_run).
|
|
545
|
+
def run_command(request)
|
|
546
|
+
return true if @stack_overflow
|
|
547
|
+
|
|
548
|
+
request.run(appender) do |command|
|
|
549
|
+
safe_log(:warn, "Async: Appender thread: Ignoring unknown command: #{command}")
|
|
550
|
+
end
|
|
551
|
+
rescue SystemStackError => e
|
|
552
|
+
@stack_overflow = e
|
|
553
|
+
end
|
|
554
|
+
|
|
555
|
+
# Add the requests, in order, to the end of the queue, for the worker thread, without waiting for room
|
|
556
|
+
# (see Inbox#push_aside). Once no worker thread runs, those not added are abandoned (see Request#abandon).
|
|
557
|
+
def pass_on(requests) = @inbox.push_aside(requests, ->(rest) { rest.each(&:abandon) }) { active? && !stopped? }
|
|
558
|
+
|
|
559
|
+
# Once the worker thread stops on a close, release the requests still queued (see Request#abandon),
|
|
560
|
+
# which none would run until #reopen; log messages stay queued for it (see Inbox#take). Returns false.
|
|
561
|
+
# A log message that cannot be put back on the full queue is counted as dropped.
|
|
562
|
+
def release_requests = @inbox.take(Request) { message_dropped }.each(&:abandon) && false
|
|
563
|
+
|
|
564
|
+
def message_dropped = @reporter.dropped(max_queue_size, stopped: stopped?, interval: dropped_message_report_seconds)
|
|
565
|
+
|
|
566
|
+
# Reopen the appender, if asked to (see #reopen), before taking more messages off the queue, or
|
|
567
|
+
# writing again the ones held, so that it is never reopened while it is writing, and a stack
|
|
568
|
+
# overflow while it reopens is charged to none of them. Returns false once the stack has
|
|
569
|
+
# overflowed: this thread then takes nothing more off the queue, and a new one carries on (see
|
|
570
|
+
# #stack_overflowed), which writes them.
|
|
571
|
+
def reopen_appender = !(@stack_overflow ||= @reopener.reopen_if_asked)
|
|
572
|
+
|
|
358
573
|
# Write one log message to the appender.
|
|
359
574
|
#
|
|
360
575
|
# A message whose content cannot be written (see #message_error?) is discarded and reported,
|
|
361
|
-
#
|
|
362
|
-
# propagates to #process, which backs off before
|
|
363
|
-
#
|
|
576
|
+
# and the thread carries on with the next message. Any other error, such as the destination
|
|
577
|
+
# being unavailable, propagates to #process, which backs off before writing the message again,
|
|
578
|
+
# or before carrying on with the next one, having discarded it (see #not_written). A stack
|
|
579
|
+
# overflow discards the message too (see #stack_overflowed).
|
|
364
580
|
def write_message(log)
|
|
365
581
|
appender.log(log)
|
|
366
582
|
@processed_count += 1
|
|
367
583
|
rescue SystemStackError => e
|
|
368
584
|
stack_overflowed([log], e)
|
|
369
585
|
rescue StandardError => e
|
|
370
|
-
|
|
586
|
+
return messages_failed([log], e) if message_error?(e)
|
|
371
587
|
|
|
372
|
-
|
|
588
|
+
not_written([log], e)
|
|
589
|
+
raise e
|
|
373
590
|
end
|
|
374
591
|
|
|
375
592
|
# Write the log messages to the appender as a batch.
|
|
376
593
|
#
|
|
377
|
-
# When the content of a message cannot be written (see #message_error?),
|
|
378
|
-
#
|
|
379
|
-
#
|
|
380
|
-
#
|
|
381
|
-
# the
|
|
382
|
-
# batch
|
|
594
|
+
# When the content of a message cannot be written (see #message_error?), the appender says what
|
|
595
|
+
# became of the batch (see BatchFailure): the messages the destination stored are counted as
|
|
596
|
+
# written, those at fault discarded, and the rest written again, as when an all or nothing batch
|
|
597
|
+
# is split in half to find the messages at fault, so that only those are discarded rather than
|
|
598
|
+
# every message in the batch, and none written twice. A stack overflow discards the batch, and the
|
|
599
|
+
# rest of the batch still to be written, without splitting it further (see #stack_overflowed).
|
|
600
|
+
#
|
|
601
|
+
# The parts still to be written are written in turn, in order, each one once the write before it
|
|
602
|
+
# has been rescued, rather than from within its rescue clause. So a destination that rejects one
|
|
603
|
+
# message after another, each time the first of the rest, neither deepens the stack nor chains
|
|
604
|
+
# each error to the one before it (Exception#cause).
|
|
383
605
|
def write_batch(logs)
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
606
|
+
parts = [logs]
|
|
607
|
+
while (part = parts.shift)
|
|
608
|
+
begin
|
|
609
|
+
appender.batch(part)
|
|
610
|
+
@processed_count += part.size
|
|
611
|
+
rescue SystemStackError => e
|
|
612
|
+
return stack_overflowed(part + parts.flatten(1), e)
|
|
613
|
+
rescue StandardError => e
|
|
614
|
+
raise unless message_error?(e)
|
|
615
|
+
|
|
616
|
+
failure = BatchFailure.new(appender, part, e)
|
|
617
|
+
@processed_count += failure.written
|
|
618
|
+
messages_failed(failure.discarded, e)
|
|
619
|
+
parts.unshift(*failure.parts)
|
|
620
|
+
end
|
|
621
|
+
end
|
|
395
622
|
end
|
|
396
623
|
|
|
397
624
|
# Discard log messages whose writing overflowed the stack, for example a value nested too
|
|
@@ -408,61 +635,84 @@ module SemanticLogger
|
|
|
408
635
|
|
|
409
636
|
# Whether the appender says the exception was caused by the content of a log message, rather
|
|
410
637
|
# than by the destination. An appender that cannot tell (it does not implement
|
|
411
|
-
# #message_error?) has every error
|
|
638
|
+
# #message_error?) has every error treated as the destination's, so the thread backs off.
|
|
412
639
|
def message_error?(exception) = appender.respond_to?(:message_error?) && appender.message_error?(exception)
|
|
413
640
|
|
|
414
|
-
#
|
|
415
|
-
#
|
|
416
|
-
#
|
|
417
|
-
def
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
641
|
+
# Whether the appender says that the write that raised the exception wrote none of the log
|
|
642
|
+
# messages it was given. An appender that cannot tell (it does not implement #wrote_nothing?)
|
|
643
|
+
# may have written some of them, so they are not written again.
|
|
644
|
+
def wrote_nothing?(exception) = appender.respond_to?(:wrote_nothing?) && appender.wrote_nothing?(exception)
|
|
645
|
+
|
|
646
|
+
# Whether the appender says that the write that raised the exception sent nothing to the
|
|
647
|
+
# destination, such as when it could not connect, so that the destination has not answered it.
|
|
648
|
+
# An appender that cannot tell (it does not implement #not_sent?) has every failure counted
|
|
649
|
+
# towards HeldWrite::MAX_RESENDS.
|
|
650
|
+
def not_sent?(exception) = appender.respond_to?(:not_sent?) && appender.not_sent?(exception)
|
|
651
|
+
|
|
652
|
+
# Log messages that the destination kept from being written. They are held to be written again
|
|
653
|
+
# once the thread has backed off (see #process), when the appender says that the write wrote
|
|
654
|
+
# none of them, up to HeldWrite::MAX_RESENDS times after the destination answered with a failure
|
|
655
|
+
# (a write it never received, see #not_sent?, is not counted). Otherwise they are discarded and
|
|
656
|
+
# counted as failed, since writing them again could write some of them twice, or has failed every
|
|
657
|
+
# time. So they are while a flush or close is waiting, such as one taken off the queue with them
|
|
658
|
+
# (`hold: false`), which must not wait for them to be written again, or the process is exiting.
|
|
659
|
+
def not_written(logs, exception, hold: true)
|
|
660
|
+
hold &&= wrote_nothing?(exception) && !@backoff.waited_on?
|
|
661
|
+
return if hold && @held.hold(logs, answered: !not_sent?(exception))
|
|
662
|
+
|
|
663
|
+
messages_failed(logs, exception)
|
|
423
664
|
end
|
|
424
665
|
|
|
425
|
-
#
|
|
426
|
-
#
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
666
|
+
# Write the log messages held after the destination failed (see #not_written) again, ahead of
|
|
667
|
+
# those still on the queue, as they were written: as a batch when batching. When that fails as
|
|
668
|
+
# well, they are held again, or discarded once the destination has answered them with a failure
|
|
669
|
+
# after they were written again HeldWrite::MAX_RESENDS times, and the error propagates to
|
|
670
|
+
# #process, which backs off before carrying on.
|
|
671
|
+
def resend_held
|
|
672
|
+
@held.resend do |logs|
|
|
673
|
+
batch? ? write_batch_then_run(logs, []) : write_message(logs.first)
|
|
674
|
+
@backoff.reset
|
|
434
675
|
end
|
|
435
676
|
end
|
|
436
677
|
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
678
|
+
# Discard the log messages held to be written again, if any, once they will not be.
|
|
679
|
+
def discard_held(exception) = @held.release&.then { |logs| messages_failed(logs, exception) }
|
|
680
|
+
|
|
681
|
+
# Record log messages discarded because they could not be written, whether their content or
|
|
682
|
+
# the destination was at fault, if any, reported at most once every dropped_message_report_seconds
|
|
683
|
+
# after the first (see Reporter#failed).
|
|
684
|
+
def messages_failed(logs, exception) = @reporter.failed(logs, exception, interval: dropped_message_report_seconds)
|
|
685
|
+
|
|
686
|
+
# Report when the log message taken off the queue has waited longer than lag_threshold_s.
|
|
687
|
+
def check_lag(log) = @reporter.lag(log, lag_threshold_s, queue.size)
|
|
688
|
+
|
|
689
|
+
# Add the request to the queue, counted as waiting until it has run or will not (see Request#done),
|
|
690
|
+
# so that the worker thread does not wait out back-offs meanwhile (see Backoff). Returns false
|
|
691
|
+
# when it could not be added.
|
|
692
|
+
def enqueue_request(request, submitted_to)
|
|
693
|
+
@backoff.request_submitted
|
|
694
|
+
added = @inbox.push(request, submitted_to, wait: true) { active? && !stopped? }
|
|
695
|
+
ensure
|
|
696
|
+
request.done unless added
|
|
444
697
|
end
|
|
445
698
|
|
|
446
699
|
# Submit a command to the worker thread and wait for the reply.
|
|
700
|
+
#
|
|
701
|
+
# Returns false, without submitting it, when called on the worker thread itself, such as by an
|
|
702
|
+
# appender that, as it writes, removes an appender, which flushes the processor first (see
|
|
703
|
+
# Processor#remove): the worker thread cannot run the command while it waits for
|
|
704
|
+
# the reply, so it would wait forever.
|
|
447
705
|
def submit_request(command)
|
|
448
|
-
return false
|
|
449
|
-
|
|
450
|
-
queue_size = queue.size
|
|
451
|
-
msg = "Async: Queued log messages: #{queue_size}, running command: #{command}"
|
|
452
|
-
if queue_size > 1_000
|
|
453
|
-
logger.warn msg
|
|
454
|
-
elsif queue_size > 100
|
|
455
|
-
logger.info msg
|
|
456
|
-
elsif queue_size.positive?
|
|
457
|
-
logger.trace msg
|
|
458
|
-
end
|
|
706
|
+
return false if !active? || @thread.equal?(Thread.current)
|
|
459
707
|
|
|
708
|
+
@reporter.queued(queue.size, command)
|
|
460
709
|
# Wake up the processing thread to process this command immediately.
|
|
461
710
|
signal.set if batch?
|
|
462
711
|
|
|
463
712
|
submitted_to = queue
|
|
464
|
-
request = Request.new(command)
|
|
465
|
-
|
|
713
|
+
request = Request.new(command) { @backoff.request_done }
|
|
714
|
+
return false unless enqueue_request(request, submitted_to)
|
|
715
|
+
|
|
466
716
|
# A reply can come while the request is on the current queue, which #reopen replaces, and a
|
|
467
717
|
# worker thread is running it, which may be a new one after a stack overflow.
|
|
468
718
|
request.wait { queue.equal?(submitted_to) && active? }
|