semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -0,0 +1,69 @@
1
+ module SemanticLogger
2
+ class QueueProcessor
3
+ # Internal: what became of a batch of log messages whose write raised a message error (see
4
+ # Subscriber#message_error?), as the appender tells from the error, for the worker thread of a
5
+ # queue processor to carry on with: how many of them the destination stored, those to discard,
6
+ # and the parts of the batch to write again.
7
+ #
8
+ # When the appender says which message the destination rejected, having stored those before it
9
+ # (see Subscriber#rejected_entry), those before it were written, it is discarded, and those after
10
+ # it are written again as a batch. Otherwise, when the appender's #batch is all or nothing (see
11
+ # Subscriber#batch_all_or_nothing?), the batch is split in half, and each half written again, so
12
+ # that only the messages at fault are discarded, unless the error would fail any batch (see
13
+ # Subscriber#fails_any_batch?). Any other batch is discarded, since writing it again could write
14
+ # some of its messages twice.
15
+ class BatchFailure
16
+ # Returns [Integer] how many of the messages, from the first, the destination stored.
17
+ attr_reader :written
18
+
19
+ # Returns [Array<Log>] the messages to discard.
20
+ attr_reader :discarded
21
+
22
+ # Returns [Array<Array<Log>>] the parts of the batch to write again, in order.
23
+ attr_reader :parts
24
+
25
+ def initialize(appender, logs, exception)
26
+ @appender = appender
27
+ @exception = exception
28
+ @written = 0
29
+ @discarded = []
30
+ @parts = []
31
+ rejected = rejected_entry(logs)
32
+ if rejected
33
+ after_rejected(logs, rejected)
34
+ elsif logs.size == 1 || !appender.batch_all_or_nothing? || fails_any_batch?
35
+ @discarded = logs
36
+ else
37
+ @parts = logs.each_slice((logs.size + 1) / 2).to_a
38
+ end
39
+ end
40
+
41
+ private
42
+
43
+ attr_reader :appender, :exception
44
+
45
+ # The destination stored the messages before the one at position `rejected`, and rejected that
46
+ # one, along with every one after it when the error would fail any batch.
47
+ def after_rejected(logs, rejected)
48
+ @written = rejected
49
+ @discarded = fails_any_batch? ? logs.drop(rejected) : [logs[rejected]]
50
+ rest = logs.drop(rejected + @discarded.size)
51
+ @parts = [rest] unless rest.empty?
52
+ end
53
+
54
+ # Returns [Integer|nil] the position in the batch of the message that the destination
55
+ # rejected, when the appender says so, or nil when it does not, or names no message in the
56
+ # batch. An appender that does not implement #rejected_entry cannot say.
57
+ def rejected_entry(logs)
58
+ rejected = appender.respond_to?(:rejected_entry) && appender.rejected_entry(exception)
59
+ rejected if rejected.is_a?(Integer) && rejected.between?(0, logs.size - 1)
60
+ end
61
+
62
+ # An appender that does not implement #fails_any_batch? has every all or nothing batch split,
63
+ # as before it could say.
64
+ def fails_any_batch?
65
+ appender.respond_to?(:fails_any_batch?) && appender.fails_any_batch?(exception)
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,62 @@
1
+ module SemanticLogger
2
+ class QueueProcessor
3
+ # Internal: the log messages, one or a batch, whose write the destination of the appender failed
4
+ # without storing any of them, held for the worker thread of a queue processor to write again
5
+ # once it has backed off, ahead of the messages on the queue.
6
+ #
7
+ # It knows the messages held, and how many times in a row they have been held again after the
8
+ # destination answered the write with a failure. It holds them again after such an answer at most
9
+ # MAX_RESENDS times, so that messages the destination fails every time, such as a request it
10
+ # answers with a 429 that never clears, are discarded rather than hold up the messages behind
11
+ # them until the worker thread gives up after async_max_retries. A write that never reached the
12
+ # destination, such as one that could not connect, is not counted: the destination has said
13
+ # nothing about these messages, so an outage loses none of them until the worker thread gives up.
14
+ class HeldWrite
15
+ # The most times the same messages are written again after the destination answered that it
16
+ # failed them. A constant rather than an option: it only limits how long one write that keeps
17
+ # failing holds up the queue, while async_max_retries already sets how long the worker thread
18
+ # keeps trying through an outage.
19
+ MAX_RESENDS = 3
20
+
21
+ def initialize
22
+ @logs = nil
23
+ @resends = 0
24
+ end
25
+
26
+ # Holds the messages to write again. `answered` says whether the destination answered the
27
+ # write that failed, rather than never received it, which counts towards MAX_RESENDS. Returns
28
+ # false, without holding them, when the destination answered and they have already been held
29
+ # again MAX_RESENDS times after it did.
30
+ def hold(logs, answered:)
31
+ if answered
32
+ return false if @resends >= MAX_RESENDS
33
+
34
+ @resends += 1
35
+ end
36
+ @logs = logs
37
+ true
38
+ end
39
+
40
+ # Yields the messages held, if any, for the block to write them again. Those the block holds
41
+ # again keep the count; any other messages held later start afresh.
42
+ def resend
43
+ return unless @logs
44
+
45
+ logs = @logs
46
+ @logs = nil
47
+ yield logs
48
+ ensure
49
+ @resends = 0 unless @logs
50
+ end
51
+
52
+ # Forgets the messages held, if any, and returns them, such as when the worker thread gives up
53
+ # on them, or after a fork, when the parent process writes them.
54
+ def release
55
+ logs = @logs
56
+ @logs = nil
57
+ @resends = 0
58
+ logs
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,97 @@
1
+ module SemanticLogger
2
+ class QueueProcessor
3
+ # Internal: the queue on which log messages, and requests such as a flush, wait for the worker
4
+ # thread of a queue processor to take them off.
5
+ #
6
+ # It knows the queue, whether its size is capped, and the process that created it, and adds to
7
+ # it, waiting for room on a full queue or not. After a fork, the child's copy of the queue holds
8
+ # the entries of the parent, which the parent's worker thread writes, so the child replaces it.
9
+ #
10
+ # Also used in a signal trap, where Ruby allows a Queue to wait, but not a Mutex to be locked.
11
+ class Inbox
12
+ # How often, in seconds, a thread waiting for room on the full queue asks whether to keep
13
+ # waiting (see #push), so that it does not wait for ever once nothing makes room.
14
+ WAIT_CHECK_SECONDS = 1
15
+
16
+ # Returns [Queue|SizedQueue] the queue.
17
+ attr_reader :queue
18
+
19
+ # Returns [Integer] the most entries the queue holds, or -1 for no limit.
20
+ attr_reader :max_queue_size
21
+
22
+ def initialize(max_queue_size)
23
+ @max_queue_size = max_queue_size
24
+ create
25
+ end
26
+
27
+ # Returns [true|false] whether the queue has a capped size.
28
+ def capped? = @capped
29
+
30
+ # Returns [true|false] whether the queue is capped, and holds as many entries as it can.
31
+ def full? = @capped && @queue.size >= max_queue_size
32
+
33
+ # Whether this is a child of the process that created the queue, whose copy of the queue then
34
+ # holds that process's entries.
35
+ def forked? = @pid != ::Process.pid
36
+
37
+ # Closes the queue, and creates an empty one, which belongs to this process.
38
+ def recreate
39
+ @queue.close
40
+ create
41
+ end
42
+
43
+ # Add to the `target` queue, the current one by default. Returns whether it was added.
44
+ #
45
+ # A queue that is not capped is never full, and its #push takes no timeout. On a full queue,
46
+ # without `wait`, returns false at once. With `wait`, waits for room while the block returns
47
+ # true, which it is asked as soon as the queue is found full, then every WAIT_CHECK_SECONDS, and
48
+ # returns false once it returns false, such as once the worker thread has stopped for good,
49
+ # since then nothing makes room. The block is not asked when the queue has room.
50
+ def push(item, target = queue, wait:)
51
+ return target.push(item) && true unless capped?
52
+ return true if target.push(item, timeout: 0)
53
+
54
+ loop do
55
+ return false unless wait && yield
56
+ return true if target.push(item, timeout: WAIT_CHECK_SECONDS)
57
+ end
58
+ end
59
+
60
+ # Add the items, in order, without making the calling thread wait for room on a full queue,
61
+ # while the block returns true. Once the queue is full, one new thread adds the rest, in order,
62
+ # waiting for room while the block returns true. Once the block returns false, at once or while
63
+ # that thread waits, the items not yet added are passed to `otherwise` instead, on that thread.
64
+ # Returns true, without waiting. Takes no lock, so that it can be called from a signal trap.
65
+ def push_aside(items, otherwise, &adding)
66
+ rest = items.drop_while { |item| adding.call && push(item, wait: false) }
67
+ return true if rest.empty?
68
+
69
+ Thread.new { otherwise.call(rest.drop_while { |item| adding.call && push(item, wait: true, &adding) }) }
70
+ true
71
+ end
72
+
73
+ # Take the entries of the given class off the queue, and return them, in order. The others are added
74
+ # back, in order, behind any added meanwhile, without waiting for room; yields each one that a thread
75
+ # waiting for room took the place of, so that it is not lost unseen. Takes no lock.
76
+ def take(kind)
77
+ entries = Array.new(@queue.size) { @queue.pop(timeout: 0) }.compact
78
+ taken, others = entries.partition { |entry| entry.is_a?(kind) }
79
+ others.each { |entry| push(entry, wait: false) || (yield(entry) if block_given?) }
80
+ taken
81
+ end
82
+
83
+ private
84
+
85
+ def create
86
+ @pid = ::Process.pid
87
+ if max_queue_size == -1
88
+ @queue = Queue.new
89
+ @capped = false
90
+ else
91
+ @queue = SizedQueue.new(max_queue_size)
92
+ @capped = true
93
+ end
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,56 @@
1
+ module SemanticLogger
2
+ class QueueProcessor
3
+ # Internal: reopens the appender of a queue processor, on its worker thread, once asked.
4
+ #
5
+ # The worker thread asks it between writes, so the appender is never reopened while it is
6
+ # writing, such as a file being closed under a write, or a connection ended under a request.
7
+ # Nor is it reopened on the thread that asks, often a signal trap after a log rotation, where
8
+ # Ruby does not allow a Mutex to be locked, as connecting with a timeout does.
9
+ #
10
+ # It knows whether a reopen has been asked for, and reports a reopen that fails to the internal
11
+ # logger. Asked from any thread, and from a signal trap, so it records that on a Queue.
12
+ class Reopener
13
+ def initialize(appender, reporter)
14
+ @appender = appender
15
+ @reporter = reporter
16
+ @asked = Queue.new
17
+ @reopening = false
18
+ end
19
+
20
+ # Ask for the appender to be reopened before the worker thread writes again.
21
+ def ask
22
+ @asked << true
23
+ end
24
+
25
+ # Returns [true|false] whether a reopen has been asked for and the worker thread has not yet
26
+ # finished it, including one that it is running. Reads only a Queue and a flag, so that it can
27
+ # be called from any thread, or a signal trap. Reads them in the opposite order to the worker
28
+ # thread, which marks the reopen as running before it takes the request, so that a reopen
29
+ # starting between the two reads is still seen.
30
+ def pending? = !@asked.empty? || @reopening
31
+
32
+ # Reopen the appender, if asked to since it was last reopened. Called by the worker thread,
33
+ # between writes. A reopen that fails is reported rather than raised, whatever the exception,
34
+ # as Appenders#reopen reports it, so that one that is not a StandardError, such as a LoadError
35
+ # or NotImplementedError, does not stop the worker thread for good: the appender keeps what it
36
+ # had, and writing to it then fails, or not, as it would have.
37
+ #
38
+ # Returns nil, or the SystemStackError when reopening overflowed the stack, so that the worker
39
+ # thread hands over to a new one, since it must not overflow its stack again: on macOS a second
40
+ # stack overflow on the same thread crashes Ruby.
41
+ def reopen_if_asked
42
+ return if @asked.empty?
43
+
44
+ @reopening = true
45
+ @asked.clear
46
+ @appender.reopen if @appender.respond_to?(:reopen)
47
+ nil
48
+ rescue Exception => e
49
+ @reporter.log(:error, "Async: Failed to reopen the appender", e)
50
+ e if e.is_a?(SystemStackError)
51
+ ensure
52
+ @reopening = false
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,113 @@
1
+ module SemanticLogger
2
+ class QueueProcessor
3
+ # Internal: reports what happens on the worker thread of a queue processor, its progress and its
4
+ # problems, to the internal logger of its appender.
5
+ #
6
+ # It knows that logger, which is the appender's to choose, and makes sure that a report neither
7
+ # stops the worker thread nor raises into a calling thread, whatever the logger. It counts the log
8
+ # messages dropped because the queue was full, and those discarded because they could not be
9
+ # written, and reports each count at most once per interval.
10
+ class Reporter
11
+ # The kinds of positional parameter that Method#parameters reports, other than *rest.
12
+ POSITIONAL = %i[req opt].freeze
13
+ private_constant :POSITIONAL
14
+
15
+ def initialize(appender)
16
+ @appender = appender
17
+ @dropped = ThrottledCounter.new
18
+ @failed = ThrottledCounter.new(reported_at: nil)
19
+ end
20
+
21
+ # Returns the internal logger of the appender, which it can replace at any time.
22
+ def logger = @appender.logger
23
+
24
+ # Returns [Integer] the number of log messages dropped because the queue was full.
25
+ def dropped_count = @dropped.total
26
+
27
+ # Returns [Integer] the number of log messages discarded because they could not be written,
28
+ # whether their content or the destination was at fault (see Subscriber#message_error?).
29
+ def failed_count = @failed.total
30
+
31
+ # Log to the internal logger, ignoring any StandardError. These calls may run after Ruby has
32
+ # released file handles during shutdown, and the internal logger may not take them, such as a
33
+ # Ruby Logger, which has no #trace, returned by a custom appender's own #logger.
34
+ #
35
+ # Such a logger also takes only the message. So the exception is passed only when there is one,
36
+ # and is added to the message instead for a logger that does not take it (see #takes_exception?).
37
+ def log(level, message, exception = nil)
38
+ internal = logger
39
+ return internal.public_send(level, message) unless exception
40
+ return internal.public_send(level, message, exception) if takes_exception?(internal, level)
41
+
42
+ internal.public_send(
43
+ level,
44
+ Utils.join_texts(message, " -- ", Utils.exception_name(exception), ": ", Utils.exception_message(exception))
45
+ )
46
+ rescue StandardError
47
+ nil
48
+ end
49
+
50
+ # Record a log message dropped because the queue was full, reporting the running count at most
51
+ # once every `interval` seconds. The report gives the reason for the latest drop: a queue too
52
+ # small for the volume of messages, or, when `stopped`, a worker thread that has stopped for
53
+ # good, so that nothing empties the queue, however large.
54
+ def dropped(max_queue_size, stopped:, interval:)
55
+ @dropped.record(interval: interval) do |count, seconds|
56
+ advice =
57
+ if stopped
58
+ "The worker thread has stopped after failing to write (see async_max_retries), so nothing " \
59
+ "empties the queue until SemanticLogger.reopen(force: true) starts it again"
60
+ else
61
+ "The queue is full (max_queue_size: #{max_queue_size}). " \
62
+ "Consider reducing the log level, increasing max_queue_size, or changing the appenders"
63
+ end
64
+ log(:warn, "Async: Dropped #{count} log messages in the last #{seconds.round} seconds. #{advice}")
65
+ end
66
+ end
67
+
68
+ # Report that the worker thread has fallen behind, when the log message it took off the queue
69
+ # was logged more than `threshold` seconds ago, with the number of messages still `queued`.
70
+ def lag(entry, threshold, queued)
71
+ diff = Time.now - entry.time
72
+ return unless diff > threshold
73
+
74
+ log(:warn, "Async: Appender thread has fallen behind by #{diff} seconds with #{queued} messages " \
75
+ "queued up. Consider reducing the log level or changing the appenders")
76
+ end
77
+
78
+ # Report the number of log messages queued ahead of a command, such as a flush, that is being
79
+ # submitted: at :warn for more than 1,000, at :info for more than 100, otherwise at :trace, and
80
+ # not at all for none.
81
+ def queued(count, command)
82
+ level = (count > 1_000 && :warn) || (count > 100 && :info) || (count.positive? && :trace)
83
+ log(level, "Async: Queued log messages: #{count}, running command: #{command}") if level
84
+ end
85
+
86
+ # Record log messages discarded because they could not be written, whether their content or the
87
+ # destination was at fault, if any. The first is reported straight away, with the error. After
88
+ # that the count is reported, with the latest error, at most once every `interval` seconds.
89
+ def failed(logs, exception, interval:)
90
+ return if logs.empty?
91
+
92
+ @failed.record(logs.size, interval: interval) do |count, seconds|
93
+ period = seconds ? " in the last #{seconds.round} seconds" : ""
94
+ log(:error, "Async: Discarded #{count} log message(s)#{period} that could not be written, " \
95
+ "most recently from #{logs.first.name}", exception)
96
+ end
97
+ end
98
+
99
+ private
100
+
101
+ # Whether the logger's method for the level takes the exception after the message, as a Semantic
102
+ # Logger logger does. A Ruby or Rails Logger takes only the message: at most one positional
103
+ # parameter, and no *rest. Decided before the call, rather than by rescuing ArgumentError, which a
104
+ # logger that does take it can raise too, such as a broadcast logger one of whose loggers raises,
105
+ # after the others have written the report.
106
+ def takes_exception?(internal, level)
107
+ parameters = internal.method(level).parameters
108
+ parameters.any? { |type, _name| type == :rest } ||
109
+ parameters.count { |type, _name| POSITIONAL.include?(type) } > 1
110
+ end
111
+ end
112
+ end
113
+ end
@@ -1,43 +1,81 @@
1
1
  module SemanticLogger
2
2
  class QueueProcessor
3
3
  # Internal: a command, such as :flush or :close, submitted on the queue for the worker thread
4
- # to run, while the submitting thread waits for the reply.
4
+ # to run, while the submitting thread waits for the reply. Or a task, a Proc, handed over for the
5
+ # worker thread to run without waiting for it (see QueueProcessor#hand_over), which nobody waits on.
5
6
  #
6
- # It knows its command and how its reply gets back to the submitting thread, and makes sure
7
- # that the submitting thread is never left waiting on a reply that cannot come.
7
+ # It knows its command, whether it has run, and how its reply gets back to the submitting
8
+ # thread, and makes sure that the submitting thread is never left waiting on a reply that cannot
9
+ # come. It also tells whoever counts the requests waiting, once, when it is no longer waiting.
8
10
  class Request
9
11
  attr_reader :command
10
12
 
11
- def initialize(command)
13
+ # The block, if any, is called once the request is no longer waiting (see #done).
14
+ def initialize(command, &on_done)
12
15
  @command = command
13
16
  @replies = Queue.new
17
+ @on_done = on_done
18
+ @ran = false
19
+ # A single token, which the first call to #done takes.
20
+ @waiting = Queue.new
21
+ @waiting << true
22
+ end
23
+
24
+ # Whether the worker thread has run this request (see #reply).
25
+ def ran? = @ran
26
+
27
+ # Called once the request has run, or will not: one that could not be added to the queue, or
28
+ # that the worker thread took off the queue but skipped, such as one behind a close. Calls the
29
+ # block given to #initialize the first time only, whichever thread calls it, so that the request
30
+ # is counted out once. Also called in a signal trap, where Ruby allows a Queue to be used, but
31
+ # not a Mutex to be locked.
32
+ def done
33
+ @on_done&.call if @waiting.pop(timeout: 0)
14
34
  end
15
35
 
16
36
  # Called by the worker thread: run the block for this request, then reply with true, or
17
37
  # with false when the block raised. Replies either way.
18
38
  def reply
19
- done = false
39
+ succeeded = false
20
40
  yield
21
- done = true
41
+ succeeded = true
22
42
  ensure
23
- @replies.push(done)
43
+ @ran = true
44
+ done
45
+ @replies.push(succeeded)
24
46
  end
25
47
 
26
- # Called by the worker thread: run this command on the appender, then reply. Returns false
27
- # once message processing must stop (:close).
48
+ # Called by the worker thread: run this command on the appender, or this task, then reply.
49
+ # Returns false once message processing must stop (:close). A command it does not know is
50
+ # yielded to the block, for the caller to report, and otherwise ignored.
28
51
  def run(appender)
29
52
  case command
53
+ when Proc
54
+ reply { command.call }
30
55
  when :flush
31
56
  reply { appender.flush }
32
57
  when :close
33
58
  reply { appender.close }
34
59
  return false
35
60
  else
36
- reply { appender.logger.warn "Async: Appender thread: Ignoring unknown command: #{command}" }
61
+ reply { yield(command) if block_given? }
37
62
  end
38
63
  true
39
64
  end
40
65
 
66
+ # Called once no worker thread will run this request, such as one passed on to the next worker
67
+ # thread when none is running (see QueueProcessor#pass_on). A task runs on a new thread, since
68
+ # nothing then writes to the appender, and returns that thread. A command is not run:
69
+ # it replies false, so that the submitting thread does not wait for it.
70
+ def abandon
71
+ # A handed-over task runs on a thread of its own, so that one that raises, even SystemStackError,
72
+ # never keeps the requests released with it from being released.
73
+ return Thread.new(&command) if command.is_a?(Proc)
74
+
75
+ done
76
+ @replies.push(false)
77
+ end
78
+
41
79
  # Called by the submitting thread: wait for the reply from the worker thread.
42
80
  #
43
81
  # The block returns whether a worker thread that can still reply is running. Returns the
@@ -6,21 +6,35 @@ module SemanticLogger
6
6
  # It knows the total number of occurrences, how many there have been since the previous
7
7
  # report, and when that report was made. What to report, and where, is up to the caller.
8
8
  #
9
- # Thread-safe: occurrences may be recorded from any thread.
9
+ # Thread-safe: occurrences may be recorded from any thread, and from a signal trap, such as one
10
+ # that logs while the queue is full. Ruby does not allow a Mutex to be locked in a trap, so the
11
+ # occurrences recorded there wait on a Queue, which it does allow, until a thread started for
12
+ # them, or the next read or record where the Mutex can be locked, adds them in.
10
13
  class ThrottledCounter
11
- # Returns [Integer] the total number of occurrences recorded.
12
- attr_reader :total
14
+ # Returns [Float] the seconds on the monotonic clock, which the intervals between reports are
15
+ # measured with, so that a change of the system clock, such as NTP stepping it back, neither
16
+ # holds back a report nor makes one early.
17
+ def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
13
18
 
14
19
  # Parameters:
15
- # reported_at: [Time|nil]
16
- # When counting starts, so that the first report is due one interval later.
20
+ # reported_at: [Float|nil]
21
+ # When counting starts, on the monotonic clock (see .now), so that the first report is due
22
+ # one interval later.
17
23
  # nil: Report the first occurrence straight away.
18
- # Default: Time.now
19
- def initialize(reported_at: Time.now)
24
+ # Default: ThrottledCounter.now
25
+ def initialize(reported_at: ThrottledCounter.now)
20
26
  @reported_at = reported_at
21
27
  @total = 0
22
28
  @unreported = 0
23
29
  @mutex = Mutex.new
30
+ @waiting = Queue.new
31
+ end
32
+
33
+ # Returns [Integer] the total number of occurrences recorded. Read in a signal trap, it leaves
34
+ # out those recorded in a trap that have not been added in yet.
35
+ def total
36
+ TrapSafe.try_synchronize(@mutex) { add(0) }
37
+ @total
24
38
  end
25
39
 
26
40
  # Record one or more occurrences.
@@ -28,18 +42,47 @@ module SemanticLogger
28
42
  # When the previous report was at least `interval` seconds ago, yields the number of
29
43
  # occurrences since then (including these) and the seconds elapsed since that report (nil
30
44
  # when there has been no report yet), then starts counting afresh for the next report.
31
- def record(count = 1, interval:)
32
- @mutex.synchronize do
33
- @total += count
34
- @unreported += count
35
- now = Time.now
36
- elapsed = @reported_at && (now - @reported_at)
37
- return if elapsed && elapsed < interval
45
+ #
46
+ # In a signal trap the occurrences are only counted, and are added in and reported, when a
47
+ # report is due, by a thread started for them, just after the trap (see #wait).
48
+ def record(count = 1, interval:, &report_block)
49
+ report = nil
50
+ counted = TrapSafe.try_synchronize(@mutex) do
51
+ add(count)
52
+ now = ThrottledCounter.now
53
+ elapsed = @reported_at && (now - @reported_at)
54
+ unless elapsed && elapsed < interval
55
+ report = [@unreported, elapsed]
56
+ @unreported = 0
57
+ @reported_at = now
58
+ end
59
+ end
60
+ return wait(count, interval, &report_block) unless counted
61
+
62
+ # Reported without holding the lock, so that the report can record occurrences too.
63
+ yield(*report) if report
64
+ end
65
+
66
+ private
67
+
68
+ # Leaves occurrences recorded where the lock cannot be taken, such as in a signal trap, on the
69
+ # waiting Queue. The first to wait starts a thread, which can take the lock, to record none:
70
+ # that adds in every occurrence waiting by then, and reports them when a report is due. Any
71
+ # left waiting, such as those that come while it is not due, are added in by the next #record
72
+ # or #total that can lock.
73
+ def wait(count, interval, &report_block)
74
+ @waiting << count
75
+ Thread.new { record(0, interval: interval, &report_block) } if @waiting.size == 1
76
+ nil
77
+ end
38
78
 
39
- yield(@unreported, elapsed)
40
- @unreported = 0
41
- @reported_at = now
79
+ # Adds the occurrences, and those waiting on the queue, to the counts. Holding the lock.
80
+ def add(count)
81
+ while (waiting = @waiting.pop(timeout: 0))
82
+ count += waiting
42
83
  end
84
+ @total += count
85
+ @unreported += count
43
86
  end
44
87
  end
45
88
  end