semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -1,5 +1,11 @@
1
1
  require "semantic_logger/queue_processor/request"
2
2
  require "semantic_logger/queue_processor/throttled_counter"
3
+ require "semantic_logger/queue_processor/reporter"
4
+ require "semantic_logger/queue_processor/backoff"
5
+ require "semantic_logger/queue_processor/inbox"
6
+ require "semantic_logger/queue_processor/held_write"
7
+ require "semantic_logger/queue_processor/batch_failure"
8
+ require "semantic_logger/queue_processor/reopener"
3
9
 
4
10
  module SemanticLogger
5
11
  # Internal class that processes log messages from a queue on a separate thread.
@@ -14,17 +20,15 @@ module SemanticLogger
14
20
  # method, either once batch_size messages have accumulated, or
15
21
  # batch_seconds have elapsed since the previous batch.
16
22
  class QueueProcessor
23
+ # Internal: seconds that #reopen waits for a worker thread that has stopped to end.
24
+ STOPPED_WORKER_WAIT_SECONDS = 0.1
25
+
17
26
  attr_accessor :lag_check_interval, :lag_threshold_s, :dropped_message_report_seconds,
18
27
  :batch_size, :batch_seconds, :non_blocking
19
- attr_reader :appender, :queue, :max_queue_size, :signal,
20
- :processed_count, :async_max_retries, :retry_count
28
+ attr_reader :appender, :signal, :processed_count
21
29
 
22
30
  # Create a new processor and start its worker thread.
23
- def self.start(**args)
24
- processor = new(**args)
25
- processor.thread
26
- processor
27
- end
31
+ def self.start(**args) = new(**args).tap(&:thread)
28
32
 
29
33
  # Parameters:
30
34
  # appender: [SemanticLogger::Subscriber]
@@ -65,19 +69,29 @@ module SemanticLogger
65
69
  # Default: false
66
70
  #
67
71
  # dropped_message_report_seconds: [Integer]
68
- # When non_blocking is enabled, log the count of dropped messages to the internal logger
69
- # at most once every this number of seconds. Also limits how often messages discarded
70
- # because their content could not be written are reported, after the first.
72
+ # Log the count of messages dropped because the queue was full, with non_blocking or once
73
+ # the worker thread has stopped (see async_max_retries), to the internal logger at most once
74
+ # every this number of seconds. Also limits how often messages discarded because they could
75
+ # not be written are reported, after the first.
71
76
  # Default: 30
72
77
  #
73
78
  # async_max_retries: [Integer]
74
79
  # Maximum number of consecutive times to restart the worker thread after it raises an
75
80
  # exception while processing messages. Each restart first sleeps for `retry_count` seconds
76
- # (1s, then 2s, ...) as a back-off. Once this many consecutive retries are exhausted the
77
- # thread stops instead of restarting. The counter resets to zero whenever a message is
78
- # processed successfully.
81
+ # (1s, then 2s, ...) as a back-off, unless a flush or close is waiting (see #flush), or the
82
+ # process is exiting (see #process_exiting=). Once this many consecutive retries are
83
+ # exhausted the thread stops instead of restarting. The counter resets to zero whenever a
84
+ # message is processed successfully. A message or batch that the destination failed is
85
+ # written again after the back-off when the appender says that the write wrote none of it
86
+ # (see Subscriber#wrote_nothing?), up to HeldWrite::MAX_RESENDS times after the destination
87
+ # answered with a failure, and is otherwise discarded.
79
88
  # -1: Retry indefinitely and never stop the thread (the pre-v5 behaviour). The back-off
80
89
  # still applies, and still resets after any successful message.
90
+ # A worker thread started again by #reopen in the same process carries on with the count, so
91
+ # that one that had used all its retries stops again at its first failure, rather than hold
92
+ # up the threads logging to it with a back-off as long again while the destination is still
93
+ # unavailable; one that stopped before, such as after a fatal exception, has the retries it
94
+ # had left. Only the worker thread of a child process, after a fork, starts counting afresh.
81
95
  # Default: 100
82
96
  def initialize(appender:,
83
97
  max_queue_size: 10_000,
@@ -90,7 +104,6 @@ module SemanticLogger
90
104
  dropped_message_report_seconds: 30,
91
105
  async_max_retries: 100)
92
106
  @appender = appender
93
- @max_queue_size = max_queue_size
94
107
  @lag_check_interval = lag_check_interval
95
108
  @lag_threshold_s = lag_threshold_s
96
109
  @batch = batch
@@ -98,15 +111,15 @@ module SemanticLogger
98
111
  @batch_seconds = batch_seconds
99
112
  @non_blocking = non_blocking
100
113
  @dropped_message_report_seconds = dropped_message_report_seconds
101
- @async_max_retries = async_max_retries
102
- @retry_count = 0
114
+ @backoff = Backoff.new(async_max_retries)
103
115
  @thread = nil
104
116
  # Only batch mode parks the worker on the signal; streaming mode never touches it.
105
117
  @signal = Concurrent::Event.new if batch
106
- @dropped = ThrottledCounter.new
107
- @failed = ThrottledCounter.new(reported_at: nil)
118
+ @reporter = Reporter.new(appender)
119
+ @reopener = Reopener.new(appender, @reporter)
108
120
  @processed_count = 0
109
- create_queue
121
+ @inbox = Inbox.new(max_queue_size)
122
+ @held = HeldWrite.new
110
123
 
111
124
  return unless batch? && !appender.respond_to?(:batch)
112
125
 
@@ -114,33 +127,52 @@ module SemanticLogger
114
127
  end
115
128
 
116
129
  # Internal logger used to report problems encountered while processing log messages.
117
- def logger = appender.logger
130
+ def logger = @reporter.logger
131
+
132
+ # Returns [Integer] the maximum number of consecutive retries after a failure (see #initialize).
133
+ def async_max_retries = @backoff.max_retries
134
+
135
+ # Returns [Integer] the number of consecutive failures, since a message was last processed.
136
+ def retry_count = @backoff.retry_count
118
137
 
119
138
  # Returns [Thread] the worker thread.
120
139
  #
121
- # Starts the worker thread if it is not currently running.
122
- def thread
123
- return @thread if @thread&.alive?
124
-
125
- @thread = spawn_worker
126
- end
140
+ # Starts the worker thread if it is not currently running. It carries on with the count of
141
+ # failures in a row (see async_max_retries).
142
+ def thread = active? ? @thread : spawn_worker
127
143
 
128
144
  # Returns [true|false] whether the worker thread is running.
129
145
  def active? = @thread&.alive?
130
146
 
147
+ # Returns [true|false] whether the thread is the worker thread, the one that writes to the
148
+ # appender. Takes no lock, so that it can be called from a signal trap.
149
+ def worker_thread?(thread = Thread.current) = @thread.equal?(thread)
150
+
151
+ # Returns [true|false] whether the thread is the worker thread while another thread may be waiting
152
+ # for it: the queue is full, so that a thread adding to it waits for it to make room, or a request,
153
+ # such as a flush or close, is waiting for it to run it (see Backoff#request_waiting?), whoever
154
+ # submitted it, so a message it logs that could wait may be dropped (see #waited_on?). Lock free.
155
+ def holding_up?(thread = Thread.current) = worker_thread?(thread) && (@inbox.full? || @backoff.request_waiting?)
156
+
157
+ # Returns [Queue|SizedQueue] the queue of log messages and requests waiting for the worker thread.
158
+ def queue = @inbox.queue
159
+
160
+ # Returns [Integer] the most entries the queue holds, or -1 for no limit.
161
+ def max_queue_size = @inbox.max_queue_size
162
+
131
163
  # Returns [true|false] whether the queue has a capped size.
132
- def capped? = @capped
164
+ def capped? = @inbox.capped?
133
165
 
134
166
  # Returns [true|false] whether messages are dropped instead of blocking when the queue is full.
135
167
  # Only a capped queue can drop messages.
136
168
  def non_blocking? = @non_blocking && capped?
137
169
 
138
170
  # Returns [Integer] the number of log messages dropped because the queue was full.
139
- def dropped_count = @dropped.total
171
+ def dropped_count = @reporter.dropped_count
140
172
 
141
- # Returns [Integer] the number of log messages discarded because their content could not be
142
- # written (see Subscriber#message_error?).
143
- def failed_count = @failed.total
173
+ # Returns [Integer] the number of log messages discarded because they could not be written,
174
+ # whether their content or the destination was at fault (see Subscriber#message_error?).
175
+ def failed_count = @reporter.failed_count
144
176
 
145
177
  # Returns [true|false] whether messages are processed in batches.
146
178
  def batch? = @batch
@@ -155,19 +187,16 @@ module SemanticLogger
155
187
  # calling thread, and the count of dropped messages is reported periodically.
156
188
  #
157
189
  # Once the worker thread has stopped for good (see async_max_retries), nothing will empty the
158
- # queue, so messages are also dropped instead of blocking the calling thread forever.
190
+ # queue, so messages are also dropped instead of blocking the calling thread forever. That
191
+ # includes a message already waiting for room on the full queue when the worker thread stops.
192
+ #
193
+ # Nor does a thread that the worker thread may itself be waiting for wait for room on the full
194
+ # queue (see #waited_on?): the message is dropped instead.
159
195
  def log(log)
160
- enqueued = true
161
- if non_blocking? || stopped?
162
- begin
163
- queue.push(log, true)
164
- rescue ThreadError
165
- message_dropped
166
- enqueued = false
167
- end
168
- else
169
- queue << log
170
- end
196
+ enqueued = @inbox.push(log, wait: !(non_blocking? || stopped?)) { !stopped? && !waited_on? }
197
+ # Reported at most once every dropped_message_report_seconds, with whether it was dropped
198
+ # because the worker thread has stopped for good.
199
+ message_dropped unless enqueued
171
200
 
172
201
  # For batches wake up the processing thread once the number of queued messages has been
173
202
  # exceeded. Also wake it on a drop so it drains the full queue rather than waiting out the
@@ -179,36 +208,142 @@ module SemanticLogger
179
208
 
180
209
  # Flush all queued log entries to the appender.
181
210
  # All queued log messages are written and then the appender is flushed.
182
- def flush
183
- submit_request(:flush)
184
- end
211
+ #
212
+ # A flush or close does not wait out back-offs after the destination failed, nor each message
213
+ # ahead of it failing in turn: submitting it ends the back-off under way, and until it has run, a
214
+ # message or batch that the destination fails is discarded, rather than written again after a
215
+ # back-off, and so are the messages queued after that one, ahead of the flush or close, without
216
+ # being written (see Backoff#give_up?).
217
+ def flush = submit_request(:flush)
185
218
 
186
219
  # Flush any outstanding messages and close the appender.
187
- def close
188
- submit_request(:close)
220
+ def close = submit_request(:close)
221
+
222
+ # Hand the block over to the worker thread, which runs it between writes, after the entries
223
+ # already queued, without waiting for it to run. The block reports its own errors: one that it
224
+ # raises is handled as a failure of the appender.
225
+ #
226
+ # Never waits, so that it can be called on a thread that the worker thread may itself be
227
+ # waiting for, such as the worker thread of an appender that it hands entries to, whose full
228
+ # queue it waits on. When the queue is full, a new thread adds the block to the queue, waiting for
229
+ # room, rather than the calling thread: once the calling thread returns, the worker thread can
230
+ # make room (see Inbox#push_aside). When the worker thread is not running, or has stopped (see
231
+ # #stopped?), or stops while that thread waits, a new thread runs the block instead, since nothing
232
+ # then writes to the appender. Takes no lock, so that it can be called from a signal trap.
233
+ #
234
+ # A batching worker thread is not woken up, since that would write a partial batch: it runs the
235
+ # block at its next batch, at most batch_seconds later.
236
+ def hand_over(&task) = pass_on([Request.new(task)])
237
+
238
+ # Tell the worker thread whether the process is exiting, before and after the flush at exit
239
+ # (see Backoff#exiting=). While it is, the worker thread acts as though a flush were waiting
240
+ # (see #flush), already before the flush at exit reaches this queue: it stops backing off and
241
+ # writing again what the destination failed, and once the destination fails, discards the
242
+ # messages queued ahead of the next request, such as the flush at exit, so that the process exits
243
+ # promptly during an outage. Takes no lock, so that it can be called from any thread, or a signal
244
+ # trap.
245
+ def process_exiting=(exiting)
246
+ @backoff.exiting = exiting
189
247
  end
190
248
 
191
- # Re-open the queue and worker thread after a fork.
249
+ # Reopen the appender, on the worker thread, and the queue and worker thread after a fork (see
250
+ # #renew_after_fork), or start the worker thread again within the same process once it has ended
251
+ # or stopped.
252
+ #
253
+ # The worker thread reopens the appender before it next takes messages off the queue, or writes
254
+ # again the ones it holds (see Reopener), so that it is never reopened while it is writing, and
255
+ # this returns before it has. After a fork, the reopen is asked for before the child's new worker
256
+ # thread starts, so that it reopens the appender before it writes anything. To wake up a
257
+ # streaming worker thread waiting for the next message, :reopen is added to the queue, without
258
+ # waiting for room: one that is not waiting has messages to write, and reopens the appender before
259
+ # them. A batching worker thread is not woken up, since that would write a partial batch, so
260
+ # nothing is added to its queue: it reopens the appender at its next batch, at most batch_seconds
261
+ # later, or when a flush or close wakes it up, before running them.
262
+ #
263
+ # Within the same process, such as after a log rotation, the queue and its entries are kept, and
264
+ # so is the worker thread while it runs (see #thread). It writes the queued entries in order,
265
+ # starting with one it has already taken off the queue, and no other thread writes to the
266
+ # appender meanwhile. A worker thread that has ended is replaced by one that carries on with the
267
+ # queued entries, and with the count of failures in a row (see async_max_retries). So is one that
268
+ # has stopped (see #stopped?), which may still be reporting that it stopped: it is given a moment
269
+ # to end first, so that its report comes before the new thread's. A worker thread that has
270
+ # stopped no longer writes to the appender, nor changes the state of this processor, so none that
271
+ # was replaced can change it once a new one has started.
272
+ #
273
+ # Also called from a signal trap, such as one that reopens the log files after a rotation, where
274
+ # Ruby does not allow a Mutex to be locked, so it takes none: it uses only Queue and Thread
275
+ # methods, which Ruby allows there.
192
276
  def reopen
277
+ @reopener.ask
278
+ renew_after_fork
279
+ @inbox.push(:reopen, wait: false) unless batch?
280
+ restart_worker
281
+ end
282
+
283
+ # Wait until the worker thread has reopened the appender, when a reopen has been asked for (see
284
+ # #reopen) and it has not yet finished it, such as in a forked child whose worker thread has not
285
+ # yet written anything. Returns at once when none is pending, so that it costs nothing otherwise.
286
+ #
287
+ # The worker thread reopens the appender before it takes anything off the queue, so a flush
288
+ # submitted now runs after the reopen, and returns once it has (see #flush). Like #flush, it
289
+ # returns without waiting when called on the worker thread itself, or once that thread has
290
+ # stopped. Takes no lock, as #flush takes none, so that it can be called from a signal trap.
291
+ def await_reopen = !@reopener.pending? || flush
292
+
293
+ # After a fork, the queue holds the entries that the parent process had queued, which the
294
+ # parent's worker thread writes, so the child starts again with an empty queue and a worker
295
+ # thread of its own, rather than write them a second time. Does nothing in the process that
296
+ # created the queue. Asks for no reopen of the appender: #reopen asks before it calls this.
297
+ #
298
+ # Called on the thread that reopens after the fork, which takes no lock (see #reopen). Called
299
+ # by Processor#reopen, through Appenders#renew_after_fork, before the appender is asked to
300
+ # reopen: the new worker thread has nothing to write until the processor's worker thread, which
301
+ # asks first, hands it entries.
302
+ def renew_after_fork
303
+ return unless @inbox.forked?
304
+
193
305
  # Workaround CRuby crash on fork by recreating queue on reopen
194
306
  # https://github.com/reidmorrison/semantic_logger/issues/103
195
- @queue&.close
196
- create_queue
197
-
307
+ @inbox.recreate
308
+ # The parent process writes the messages it held to write again.
309
+ @held.release
198
310
  @thread&.kill if @thread&.alive?
199
- @thread = spawn_worker
311
+ # A worker thread new to this process, which has not failed to write yet.
312
+ @backoff.restart
313
+ spawn_worker
200
314
  end
201
315
 
202
316
  private
203
317
 
204
- # Start the worker thread.
318
+ # Start a new worker thread, as @thread. It carries on with the count of failures in a row,
319
+ # including one taking over from a thread that overflowed its stack (see #process).
320
+ #
321
+ # The new thread waits until it has been assigned to @thread before it takes anything off the
322
+ # queue. Otherwise it could overflow its stack and hand over to another new thread before then,
323
+ # and the thread that started it would then assign it, after it has ended, leaving the one it
324
+ # handed over to running unseen beside the next one that #thread starts. It waits on a Queue,
325
+ # whose #push Ruby allows in a signal trap (see #reopen).
205
326
  def spawn_worker
206
327
  @stopped = false
207
328
  @stack_overflow = nil
208
- Thread.new do
329
+ assigned = Queue.new
330
+ @thread = Thread.new do
331
+ assigned.pop
209
332
  name_worker
210
333
  process
211
334
  end
335
+ @thread.tap { assigned << true }
336
+ end
337
+
338
+ # Start the worker thread again within the same process, unless it is running (see #reopen).
339
+ #
340
+ # A worker thread that has stopped (see #stopped?) may still be reporting that it stopped, so it
341
+ # is given a moment to end first. Then a new one is started, whether it has ended or not.
342
+ def restart_worker
343
+ return thread unless stopped?
344
+
345
+ @thread&.join(STOPPED_WORKER_WAIT_SECONDS)
346
+ spawn_worker
212
347
  end
213
348
 
214
349
  # Name the worker thread after its appender, such as "SL File", short enough for tools that
@@ -221,105 +356,129 @@ module SemanticLogger
221
356
  nil
222
357
  end
223
358
 
224
- def create_queue
225
- if max_queue_size == -1
226
- @queue = Queue.new
227
- @capped = false
228
- else
229
- @queue = SizedQueue.new(max_queue_size)
230
- @capped = true
231
- end
232
- end
233
-
234
359
  # Separate thread for processing log messages.
360
+ #
361
+ # This thread is designed to never go down unless the main thread terminates or the appender is
362
+ # closed. A StandardError while processing restarts the processing, after a back-off, up to
363
+ # async_max_retries times. Any other exception, even one from the first report, stops this
364
+ # processor, which then drops new messages, rather than ending the thread and leaving the
365
+ # calling threads blocked on a full queue.
366
+ #
367
+ # A flush or close that is waiting (see #flush), or the process exiting (see #process_exiting=),
368
+ # does not wait out a back-off, so processing then restarts straight away, without counting a
369
+ # retry, and discards the messages queued ahead of the next request without writing them (see
370
+ # Backoff#give_up?).
235
371
  def process
236
- # This thread is designed to never go down unless the main thread terminates
237
- # or the appender is closed.
238
- logger.trace "Async: Appender thread active"
372
+ safe_log(:trace, "Async: Appender thread active")
239
373
  begin
240
- carry_on = batch? ? process_messages_in_batches : process_messages
374
+ resend_held if reopen_appender
375
+ carry_on = @stack_overflow || (batch? ? process_messages_in_batches : process_messages)
241
376
  rescue StandardError => e
242
- if async_max_retries == -1 || retry_count < async_max_retries
243
- @retry_count += 1
244
- limit = async_max_retries == -1 ? "unlimited" : async_max_retries
245
- safe_log(:warn,
246
- "Async: Restarting due to exception, retry #{retry_count} of #{limit}, sleeping #{retry_count}s", e)
247
- sleep(retry_count)
377
+ retry if @backoff.give_up?(e)
378
+ if @backoff.retry?
379
+ safe_log(:warn, "Async: Restarting due to exception, retry #{retry_count} of #{@backoff.limit}, " \
380
+ "sleeping #{retry_count}s", e)
381
+ @backoff.wait
248
382
  retry
249
- else
250
- @stopped = true
251
- safe_log(:error, "Async: Stopping after #{retry_count} failed retries, dropping new messages", e)
252
383
  end
253
- rescue Exception => e
384
+
254
385
  @stopped = true
255
- safe_log(:error, "Async: Stopping due to fatal exception", e)
256
- ensure
257
- # A thread that overflowed its stack hands the queue over to a new one (see #stack_overflowed).
258
- @thread = carry_on ? spawn_worker : nil
259
- safe_log(:trace, "Async: Thread has stopped") unless carry_on
386
+ discard_held(e)
387
+ safe_log(:error, "Async: Stopping after #{retry_count} failed retries, dropping new messages", e)
260
388
  end
389
+ rescue Exception => e
390
+ @stopped = true
391
+ discard_held(e)
392
+ safe_log(:error, "Async: Stopping due to fatal exception", e)
393
+ ensure
394
+ # A thread that overflowed its stack hands the queue over to a new one (see #stack_overflowed).
395
+ # It is still the worker thread, since #reopen replaces only one that has ended or stopped, and
396
+ # one that overflowed has neither. Any other thread leaves @thread as it is, since one that has
397
+ # ended reads as inactive.
398
+ spawn_worker if carry_on
399
+ safe_log(:trace, "Async: Thread has stopped") unless carry_on
261
400
  end
262
401
 
263
- # Log to the internal logger, ignoring any error.
264
- # These calls may run after Ruby has released file handles during shutdown.
265
- def safe_log(level, message, exception = nil)
266
- logger.public_send(level, message, exception)
267
- rescue StandardError
268
- nil
269
- end
402
+ # Returns [true|false] whether the calling thread is one that the worker thread may be waiting
403
+ # for: the worker thread of an appender written to by the appender of this processor, which is
404
+ # then the Appenders of the main Processor, while that appender's own queue is full, or a flush or
405
+ # close submitted to it, such as by the worker thread, waits for it (see Appenders#holding_up?).
406
+ # Otherwise the worker thread, if it waits at all, waits on another thread, which makes room.
407
+ # Asked only once a log message finds the queue full, then before each wait for room.
408
+ def waited_on? = appender.respond_to?(:holding_up?) && appender.holding_up?
409
+
410
+ # Report to the internal logger, ignoring any error (see Reporter#log).
411
+ def safe_log(level, message, exception = nil) = @reporter.log(level, message, exception)
270
412
 
271
413
  # Streaming: write each log message to the appender as it is dequeued.
272
414
  #
273
415
  # Returns true when processing must carry on in a new thread (see #stack_overflowed).
274
416
  def process_messages
275
417
  count = 0
276
- while (message = queue.pop)
418
+ while reopen_appender && (message = queue.pop)
277
419
  if message.is_a?(Log)
420
+ gave_up = @backoff.gave_up
421
+ next messages_failed([message], gave_up) if gave_up
422
+
278
423
  write_message(message)
279
424
  return true if @stack_overflow
280
425
 
281
- @retry_count = 0 unless retry_count.zero?
426
+ @backoff.reset
282
427
  count += 1
283
428
  # Check every few log messages whether this appender thread is falling behind
284
429
  if count > lag_check_interval
285
430
  check_lag(message)
286
431
  count = 0
287
432
  end
288
- else
289
- break unless message.run(appender)
433
+ elsif message.is_a?(Request)
434
+ break unless run_command(message)
290
435
  end
436
+ # Otherwise :reopen, which #reopen adds to wake up this thread to reopen the appender.
291
437
  end
292
- logger.trace "Async: Queue Closed"
293
- false
438
+ return true if @stack_overflow
439
+
440
+ safe_log(:trace, "Async: Queue Closed")
441
+ release_requests
294
442
  end
295
443
 
296
444
  # Batching: group up log messages before writing them via the appender's #batch method.
297
445
  #
446
+ # Once the destination failed while the worker thread was waited on (see Backoff#gave_up), the
447
+ # messages taken off the queue ahead of the first request, such as a flush, are discarded without
448
+ # being written, and those taken off the queue after it are written as a batch, as usual.
449
+ #
298
450
  # Returns true when processing must carry on in a new thread (see #stack_overflowed).
299
451
  def process_messages_in_batches
300
452
  loop do
301
453
  # Wait for batch interval or number of messages to be exceeded.
302
454
  signal.wait(batch_seconds)
303
455
 
456
+ # Counted before the appender is reopened, so that a flush or close counted here, and so run
457
+ # in this batch, comes after the reopen asked for before it.
458
+ message_count = queue.length
459
+ return true unless reopen_appender
460
+
304
461
  logs = []
305
462
  commands = []
306
- first = true
307
- message_count = queue.length
463
+ discarded = []
464
+ gave_up = @backoff.gave_up
308
465
  message_count.times do
309
466
  # Queue#pop(true) raises an exception when there are no more messages, which is considered expensive.
310
467
  message = queue.pop
311
468
  if message.is_a?(Log)
469
+ next discarded << message if gave_up && commands.empty?
470
+
312
471
  logs << message
313
- if first
314
- check_lag(message)
315
- first = false
316
- end
317
- else
472
+ check_lag(message) if logs.size == 1
473
+ elsif message.is_a?(Request)
318
474
  commands << message
319
475
  end
320
476
  end
321
- # Stop processing once a command (e.g. :close) signals the thread to terminate.
322
- return false unless write_batch_then_run(logs, commands)
477
+ messages_failed(discarded, gave_up) unless discarded.empty?
478
+ # Stop processing once a command (e.g. :close) signals the thread to terminate, releasing the
479
+ # requests left (see #release_requests), unless the stack overflowed, when a new thread carries
480
+ # on (see #run_command).
481
+ return !@stack_overflow.nil? || release_requests unless write_batch_then_run(logs, commands)
323
482
  return true if @stack_overflow
324
483
 
325
484
  signal.reset unless queue.size >= batch_size
@@ -330,68 +489,136 @@ module SemanticLogger
330
489
  # queue with them. Returns false once a command (:close) stops processing.
331
490
  #
332
491
  # When the batch cannot be written, for example because the destination is unavailable, the
333
- # commands still run before the error propagates to #process to back off. Otherwise they
334
- # would be lost, and a caller waiting on #flush or #close, such as the flush at exit, would
335
- # wait forever.
492
+ # messages not yet written are held to be written again after the back-off, or discarded and
493
+ # counted as failed (see #not_written). The commands still run before the error propagates to
494
+ # #process to back off. Otherwise they would be lost, and a caller waiting on #flush or #close, such
495
+ # as the flush at exit, would wait forever. Since they are waiting, the messages are then discarded
496
+ # rather than held. A command behind a close is not run, but released (see Request#abandon).
336
497
  def write_batch_then_run(logs, commands)
498
+ accounted = processed_count + failed_count
337
499
  unless logs.empty?
338
500
  write_batch(logs)
339
- @retry_count = 0 unless retry_count.zero?
501
+ @backoff.reset
340
502
  end
341
- commands.all? { |command| command.run(appender) }
503
+ commands.all? { |command| run_command(command) }
342
504
  rescue StandardError => e
505
+ # Every message taken off the queue was written or already discarded, except those still
506
+ # to be written when the error was raised: the last ones, since a batch is written in order.
507
+ # The write that raised wrote none of them, if the appender says so, and the rest were not
508
+ # sent at all.
509
+ unwritten = logs.size - (processed_count + failed_count - accounted)
510
+ not_written(logs.last(unwritten), e, hold: commands.empty?) if unwritten.positive?
511
+ # The commands run once the batch is written, until one raises: the last one that ran, if any.
512
+ failed = commands.select(&:ran?).last
343
513
  raise e if commands.all? { |command| process_message_after_failure(command) }
344
514
 
345
- safe_log(:warn, "Async: Closed without writing #{logs.size} log messages", e)
515
+ message = failed ? "Failed to #{failed.command} the appender" : "Closed without writing #{unwritten} log messages"
516
+ safe_log(:warn, "Async: #{message}", e)
346
517
  false
518
+ ensure
519
+ # Once the stack has overflowed, the commands not run are passed on to the next worker thread
520
+ # (see #run_command). Otherwise they were behind a close.
521
+ commands.reject(&:ran?).then { |not_run| @stack_overflow ? pass_on(not_run) : not_run.each(&:abandon) }
347
522
  end
348
523
 
349
- # Run a command after a batch could not be written. Returns false once a command (:close)
350
- # stops processing, even when the command itself fails.
524
+ # Run a command after a batch could not be written, unless it has run already, such as a flush
525
+ # that raised. Returns false once a command (:close) stops processing, even when the command
526
+ # itself fails.
351
527
  def process_message_after_failure(command)
352
- command.run(appender)
528
+ return command.command != :close if command.ran?
529
+
530
+ run_command(command)
353
531
  rescue StandardError => e
354
532
  safe_log(:warn, "Async: Failed to #{command.command} the appender", e)
355
533
  command.command != :close
356
534
  end
357
535
 
536
+ # Run a command taken off the queue on the appender, reporting one it does not know. Returns
537
+ # false once message processing must stop (:close).
538
+ #
539
+ # A command that overflows the stack, such as a task handed over (see #hand_over), has failed,
540
+ # even a close, and this thread takes nothing more off the queue: a new one carries on, as after
541
+ # a message that overflowed it (see #stack_overflowed). Nor does it run any more commands, since on
542
+ # macOS a second stack overflow on the same thread crashes Ruby: those taken off the queue with it,
543
+ # in the same batch, or after a batch that overflowed it, return true here without running, and
544
+ # are passed on to the new one (see #write_batch_then_run).
545
+ def run_command(request)
546
+ return true if @stack_overflow
547
+
548
+ request.run(appender) do |command|
549
+ safe_log(:warn, "Async: Appender thread: Ignoring unknown command: #{command}")
550
+ end
551
+ rescue SystemStackError => e
552
+ @stack_overflow = e
553
+ end
554
+
555
+ # Add the requests, in order, to the end of the queue, for the worker thread, without waiting for room
556
+ # (see Inbox#push_aside). Once no worker thread runs, those not added are abandoned (see Request#abandon).
557
+ def pass_on(requests) = @inbox.push_aside(requests, ->(rest) { rest.each(&:abandon) }) { active? && !stopped? }
558
+
559
+ # Once the worker thread stops on a close, release the requests still queued (see Request#abandon),
560
+ # which none would run until #reopen; log messages stay queued for it (see Inbox#take). Returns false.
561
+ # A log message that cannot be put back on the full queue is counted as dropped.
562
+ def release_requests = @inbox.take(Request) { message_dropped }.each(&:abandon) && false
563
+
564
+ def message_dropped = @reporter.dropped(max_queue_size, stopped: stopped?, interval: dropped_message_report_seconds)
565
+
566
+ # Reopen the appender, if asked to (see #reopen), before taking more messages off the queue, or
567
+ # writing again the ones held, so that it is never reopened while it is writing, and a stack
568
+ # overflow while it reopens is charged to none of them. Returns false once the stack has
569
+ # overflowed: this thread then takes nothing more off the queue, and a new one carries on (see
570
+ # #stack_overflowed), which writes them.
571
+ def reopen_appender = !(@stack_overflow ||= @reopener.reopen_if_asked)
572
+
358
573
  # Write one log message to the appender.
359
574
  #
360
575
  # A message whose content cannot be written (see #message_error?) is discarded and reported,
361
- # without restarting the thread. Any other error, such as the destination being unavailable,
362
- # propagates to #process, which backs off before retrying. A stack overflow discards the
363
- # message too (see #stack_overflowed).
576
+ # and the thread carries on with the next message. Any other error, such as the destination
577
+ # being unavailable, propagates to #process, which backs off before writing the message again,
578
+ # or before carrying on with the next one, having discarded it (see #not_written). A stack
579
+ # overflow discards the message too (see #stack_overflowed).
364
580
  def write_message(log)
365
581
  appender.log(log)
366
582
  @processed_count += 1
367
583
  rescue SystemStackError => e
368
584
  stack_overflowed([log], e)
369
585
  rescue StandardError => e
370
- raise unless message_error?(e)
586
+ return messages_failed([log], e) if message_error?(e)
371
587
 
372
- messages_failed([log], e)
588
+ not_written([log], e)
589
+ raise e
373
590
  end
374
591
 
375
592
  # Write the log messages to the appender as a batch.
376
593
  #
377
- # When the content of a message cannot be written (see #message_error?), and the appender's
378
- # #batch is all or nothing, the batch is split in half and each half written separately, so
379
- # that only the offending message is discarded rather than every message in the batch.
380
- # Otherwise the batch is discarded, since writing it again could duplicate the messages that
381
- # the appender had already written. A stack overflow discards the batch, and the rest of the
382
- # batch being split, without splitting it further (see #stack_overflowed).
594
+ # When the content of a message cannot be written (see #message_error?), the appender says what
595
+ # became of the batch (see BatchFailure): the messages the destination stored are counted as
596
+ # written, those at fault discarded, and the rest written again, as when an all or nothing batch
597
+ # is split in half to find the messages at fault, so that only those are discarded rather than
598
+ # every message in the batch, and none written twice. A stack overflow discards the batch, and the
599
+ # rest of the batch still to be written, without splitting it further (see #stack_overflowed).
600
+ #
601
+ # The parts still to be written are written in turn, in order, each one once the write before it
602
+ # has been rescued, rather than from within its rescue clause. So a destination that rejects one
603
+ # message after another, each time the first of the rest, neither deepens the stack nor chains
604
+ # each error to the one before it (Exception#cause).
383
605
  def write_batch(logs)
384
- return messages_failed(logs, @stack_overflow) if @stack_overflow
385
-
386
- appender.batch(logs)
387
- @processed_count += logs.size
388
- rescue SystemStackError => e
389
- stack_overflowed(logs, e)
390
- rescue StandardError => e
391
- raise unless message_error?(e)
392
- return messages_failed(logs, e) if logs.size == 1 || !appender.batch_all_or_nothing?
393
-
394
- logs.each_slice((logs.size + 1) / 2) { |half| write_batch(half) }
606
+ parts = [logs]
607
+ while (part = parts.shift)
608
+ begin
609
+ appender.batch(part)
610
+ @processed_count += part.size
611
+ rescue SystemStackError => e
612
+ return stack_overflowed(part + parts.flatten(1), e)
613
+ rescue StandardError => e
614
+ raise unless message_error?(e)
615
+
616
+ failure = BatchFailure.new(appender, part, e)
617
+ @processed_count += failure.written
618
+ messages_failed(failure.discarded, e)
619
+ parts.unshift(*failure.parts)
620
+ end
621
+ end
395
622
  end
396
623
 
397
624
  # Discard log messages whose writing overflowed the stack, for example a value nested too
@@ -408,61 +635,84 @@ module SemanticLogger
408
635
 
409
636
  # Whether the appender says the exception was caused by the content of a log message, rather
410
637
  # than by the destination. An appender that cannot tell (it does not implement
411
- # #message_error?) has every error retried with a back-off.
638
+ # #message_error?) has every error treated as the destination's, so the thread backs off.
412
639
  def message_error?(exception) = appender.respond_to?(:message_error?) && appender.message_error?(exception)
413
640
 
414
- # Record log messages discarded because their content could not be written. The first is
415
- # reported to the internal logger straight away, with the error. After that the count is
416
- # reported, with the latest error, at most once every dropped_message_report_seconds.
417
- def messages_failed(logs, exception)
418
- @failed.record(logs.size, interval: dropped_message_report_seconds) do |count, seconds|
419
- period = seconds ? " in the last #{seconds.round} seconds" : ""
420
- safe_log(:error, "Async: Discarded #{count} log message(s)#{period} that could not be written, " \
421
- "most recently from #{logs.first.name}", exception)
422
- end
641
+ # Whether the appender says that the write that raised the exception wrote none of the log
642
+ # messages it was given. An appender that cannot tell (it does not implement #wrote_nothing?)
643
+ # may have written some of them, so they are not written again.
644
+ def wrote_nothing?(exception) = appender.respond_to?(:wrote_nothing?) && appender.wrote_nothing?(exception)
645
+
646
+ # Whether the appender says that the write that raised the exception sent nothing to the
647
+ # destination, such as when it could not connect, so that the destination has not answered it.
648
+ # An appender that cannot tell (it does not implement #not_sent?) has every failure counted
649
+ # towards HeldWrite::MAX_RESENDS.
650
+ def not_sent?(exception) = appender.respond_to?(:not_sent?) && appender.not_sent?(exception)
651
+
652
+ # Log messages that the destination kept from being written. They are held to be written again
653
+ # once the thread has backed off (see #process), when the appender says that the write wrote
654
+ # none of them, up to HeldWrite::MAX_RESENDS times after the destination answered with a failure
655
+ # (a write it never received, see #not_sent?, is not counted). Otherwise they are discarded and
656
+ # counted as failed, since writing them again could write some of them twice, or has failed every
657
+ # time. So they are while a flush or close is waiting, such as one taken off the queue with them
658
+ # (`hold: false`), which must not wait for them to be written again, or the process is exiting.
659
+ def not_written(logs, exception, hold: true)
660
+ hold &&= wrote_nothing?(exception) && !@backoff.waited_on?
661
+ return if hold && @held.hold(logs, answered: !not_sent?(exception))
662
+
663
+ messages_failed(logs, exception)
423
664
  end
424
665
 
425
- # Record a dropped message, reporting the running count to the internal logger at most
426
- # once every dropped_message_report_seconds.
427
- def message_dropped
428
- @dropped.record(interval: dropped_message_report_seconds) do |count, seconds|
429
- logger.warn(
430
- "Async: Dropped #{count} log messages in the last #{seconds.round} seconds. " \
431
- "The queue is full (max_queue_size: #{max_queue_size}). " \
432
- "Consider reducing the log level, increasing max_queue_size, or changing the appenders"
433
- )
666
+ # Write the log messages held after the destination failed (see #not_written) again, ahead of
667
+ # those still on the queue, as they were written: as a batch when batching. When that fails as
668
+ # well, they are held again, or discarded once the destination has answered them with a failure
669
+ # after they were written again HeldWrite::MAX_RESENDS times, and the error propagates to
670
+ # #process, which backs off before carrying on.
671
+ def resend_held
672
+ @held.resend do |logs|
673
+ batch? ? write_batch_then_run(logs, []) : write_message(logs.first)
674
+ @backoff.reset
434
675
  end
435
676
  end
436
677
 
437
- def check_lag(log)
438
- diff = Time.now - log.time
439
- return unless diff > lag_threshold_s
440
-
441
- logger.warn "Async: Appender thread has fallen behind by #{diff} seconds with " \
442
- "#{queue.size} messages queued up. Consider reducing the log level or " \
443
- "changing the appenders"
678
+ # Discard the log messages held to be written again, if any, once they will not be.
679
+ def discard_held(exception) = @held.release&.then { |logs| messages_failed(logs, exception) }
680
+
681
+ # Record log messages discarded because they could not be written, whether their content or
682
+ # the destination was at fault, if any, reported at most once every dropped_message_report_seconds
683
+ # after the first (see Reporter#failed).
684
+ def messages_failed(logs, exception) = @reporter.failed(logs, exception, interval: dropped_message_report_seconds)
685
+
686
+ # Report when the log message taken off the queue has waited longer than lag_threshold_s.
687
+ def check_lag(log) = @reporter.lag(log, lag_threshold_s, queue.size)
688
+
689
+ # Add the request to the queue, counted as waiting until it has run or will not (see Request#done),
690
+ # so that the worker thread does not wait out back-offs meanwhile (see Backoff). Returns false
691
+ # when it could not be added.
692
+ def enqueue_request(request, submitted_to)
693
+ @backoff.request_submitted
694
+ added = @inbox.push(request, submitted_to, wait: true) { active? && !stopped? }
695
+ ensure
696
+ request.done unless added
444
697
  end
445
698
 
446
699
  # Submit a command to the worker thread and wait for the reply.
700
+ #
701
+ # Returns false, without submitting it, when called on the worker thread itself, such as by an
702
+ # appender that, as it writes, removes an appender, which flushes the processor first (see
703
+ # Processor#remove): the worker thread cannot run the command while it waits for
704
+ # the reply, so it would wait forever.
447
705
  def submit_request(command)
448
- return false unless active?
449
-
450
- queue_size = queue.size
451
- msg = "Async: Queued log messages: #{queue_size}, running command: #{command}"
452
- if queue_size > 1_000
453
- logger.warn msg
454
- elsif queue_size > 100
455
- logger.info msg
456
- elsif queue_size.positive?
457
- logger.trace msg
458
- end
706
+ return false if !active? || @thread.equal?(Thread.current)
459
707
 
708
+ @reporter.queued(queue.size, command)
460
709
  # Wake up the processing thread to process this command immediately.
461
710
  signal.set if batch?
462
711
 
463
712
  submitted_to = queue
464
- request = Request.new(command)
465
- submitted_to << request
713
+ request = Request.new(command) { @backoff.request_done }
714
+ return false unless enqueue_request(request, submitted_to)
715
+
466
716
  # A reply can come while the request is on the current queue, which #reopen replaces, and a
467
717
  # worker thread is running it, which may be a new one after a stack overflow.
468
718
  request.wait { queue.equal?(submitted_to) && active? }