semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -12,14 +12,37 @@ module SemanticLogger
12
12
  end
13
13
 
14
14
  subscribers = (@subscribers ||= Concurrent::Array.new)
15
- subscribers << subscriber unless subscribers.include?(subscriber)
15
+ return if subscribers.include?(subscriber)
16
+
17
+ subscribers << subscriber
18
+ # A Ractor other than the main one can only call a shareable subscriber.
19
+ if RactorSupport.enabled? && ::Ractor.shareable?(subscriber)
20
+ @ractor_subscribers = [*@ractor_subscribers, subscriber].freeze
21
+ end
22
+ subscribers
16
23
  end
17
24
 
18
25
  class << self
19
26
  attr_reader :subscribers
20
27
  end
21
28
 
29
+ # Removes the on_log subscriber.
30
+ def self.unsubscribe(subscriber)
31
+ @subscribers&.delete(subscriber)
32
+ @ractor_subscribers = (@ractor_subscribers - [subscriber]).freeze if @ractor_subscribers
33
+ end
34
+
35
+ # Removes every on_log subscriber.
36
+ def self.clear_subscribers
37
+ @subscribers&.clear
38
+ @ractor_subscribers = nil
39
+ end
40
+
41
+ # Returns the processor that writes log entries to the appenders, or in a Ractor other than the
42
+ # main one, the RactorProcessor that sends them to the main Ractor.
22
43
  def self.processor
44
+ return RactorProcessor.current if RactorSupport.other_ractor?
45
+
23
46
  @processor ||= Processor.new
24
47
  end
25
48
 
@@ -74,13 +97,16 @@ module SemanticLogger
74
97
  Logger.processor.log(log)
75
98
  end
76
99
 
77
- @processor = nil
78
- @subscribers = nil
100
+ @processor = nil
101
+ @subscribers = nil
102
+ @ractor_subscribers = nil
79
103
 
104
+ # Calls the on_log subscribers, or in a Ractor other than the main one, the shareable ones.
80
105
  def self.call_subscribers(log)
81
- return unless @subscribers
106
+ subscribers = RactorSupport.other_ractor? ? @ractor_subscribers : @subscribers
107
+ return unless subscribers
82
108
 
83
- @subscribers.each do |subscriber|
109
+ subscribers.each do |subscriber|
84
110
  subscriber.call(log)
85
111
  rescue Exception => e
86
112
  processor.logger.error("Exception calling :on_log subscriber", e)
@@ -7,31 +7,39 @@ end
7
7
 
8
8
  # Send Metrics to NewRelic
9
9
  #
10
- # The :error and :fatal log entries will show up under
11
- # "Applications" > "Application Name" > "Events" > "Errors" in New Relic.
10
+ # Records the metric of each log entry that has one as a New Relic custom metric, through the
11
+ # agent's metric API: with record_metric, in seconds, when the entry has a duration, otherwise with
12
+ # increment_metric, by its metric_amount, or 1. A metric's dimensions are left out, since that API
13
+ # takes none.
12
14
  #
13
15
  # Example:
14
16
  # SemanticLogger.add_appender(metric: :new_relic)
15
17
  module SemanticLogger
16
18
  module Metric
17
19
  class NewRelic < SemanticLogger::Subscriber
18
- attr_accessor :prefix
20
+ attr_reader :prefix
19
21
 
20
- # Create Appender
22
+ # Create the New Relic metrics subscriber
21
23
  #
22
24
  # Parameters
23
- # :prefix [String]
24
- # Prefix to add to every metric before forwarding to NewRelic.
25
+ # prefix: [String]
26
+ # Prefix to add, with a `/`, to every metric name that does not already start with it and
27
+ # a `/`, before forwarding to NewRelic, which charts custom metrics only under `Custom/`.
28
+ # So 'Customers/signup' is recorded as 'Custom/Customers/signup', and 'Custom/Orders/placed'
29
+ # as it is. Another prefix should start with `Custom/` too, such as 'Custom/MyApp'. A
30
+ # trailing `/` is left out, and an empty prefix adds nothing. Only the default formatter,
31
+ # #call, adds it.
25
32
  # Default: 'Custom'
26
33
  #
27
34
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
28
- # Override the log level for this appender.
29
- # Default: :error
35
+ # Override the log level for this subscriber.
36
+ # Default: none (:trace), so it records every metric entry that a logger emits; each
37
+ # logger filters by its own level.
30
38
  #
31
39
  # formatter: [Object|Proc]
32
- # An instance of a class that implements #call, or a Proc to be used to format
33
- # the output from this appender
34
- # Default: Use the built-in formatter (See: #call)
40
+ # An instance of a class that implements #call, or a Proc, that returns the name to
41
+ # record the metric under.
42
+ # Default: #call, which adds the prefix to the name of the entry's metric.
35
43
  #
36
44
  # filter: [Regexp|Proc]
37
45
  # RegExp: Only include log messages where the class name matches the supplied.
@@ -39,16 +47,23 @@ module SemanticLogger
39
47
  # Proc: Only include log messages where the supplied Proc returns true
40
48
  # The Proc must return true or false.
41
49
  def initialize(prefix: "Custom", **args, &)
42
- @prefix = prefix
50
+ self.prefix = prefix
43
51
  super(**args, &)
44
52
  end
45
53
 
54
+ # Sets the prefix that #call adds, without a trailing `/`, since #call adds one. An empty
55
+ # prefix is added as an empty String, which every name already starts with.
56
+ def prefix=(prefix)
57
+ @prefix = prefix.to_s.delete_suffix("/")
58
+ @prefix_with_slash = @prefix.empty? ? "" : "#{@prefix}/"
59
+ end
60
+
46
61
  # Returns metric name to use.
47
62
  def call(log, _logger)
48
63
  metric = log.metric
49
- # Add prefix for NewRelic
50
- metric = "#{prefix}/#{metric}" unless metric.start_with?(prefix)
51
- metric
64
+ return metric if metric.start_with?(@prefix_with_slash)
65
+
66
+ "#{@prefix_with_slash}#{metric}"
52
67
  end
53
68
 
54
69
  def log(log)
@@ -64,8 +79,8 @@ module SemanticLogger
64
79
 
65
80
  # Only forward log entries that contain metrics.
66
81
  def should_log?(log)
67
- # Does not support metrics with dimensions.
68
- log.metric && !log.dimensions && meets_log_level?(log) && !filtered?(log)
82
+ # Records a metric that has dimensions under its name alone, since the agent's metric API has none.
83
+ log.metric && meets_log_level?(log) && !filtered?(log)
69
84
  end
70
85
  end
71
86
  end
@@ -19,10 +19,15 @@ module SemanticLogger
19
19
  # Access Token to use for sending metrics.
20
20
  # Obtain the Signalfx token via the Signalfx Web UI under `Organization` -> `Access Tokens`.
21
21
  #
22
- # dimensions: [Array<String>]
23
- # Dimensions to forward to signalfx when they are present in the named tags of any log message.
24
- # By default `application` and `host` are always included as dimensions in all forwarded metrics.
25
- # Example: [:user_id, :state]
22
+ # dimensions: [Array<Symbol|String>]
23
+ # Dimensions to forward to signalfx when they are present, and not blank, in the named tags of
24
+ # any log message whose metric has no dimensions of its own. A metric logged with `dimensions:`
25
+ # is sent with those instead, so add the named tags it needs to them.
26
+ # By default `application`, `host`, and, when it is set, `environment` are always included as
27
+ # dimensions in all forwarded metrics.
28
+ # Each value of a dimension makes a new metric time series, so list only named tags whose
29
+ # values are bounded, never a user or request id (see docs/metrics.md).
30
+ # Example: [:region, :plan]
26
31
  #
27
32
  # filter: [Regexp|Proc]
28
33
  # RegExp: Only include log messages where the class name matches the supplied
@@ -39,8 +44,7 @@ module SemanticLogger
39
44
  # Default: SemanticLogger.application
40
45
  #
41
46
  # url: [String]
42
- # Override the SignalFx service url.
43
- # For historical data use: https://backfill.signalfx.com/v1/backfill
47
+ # Override the SignalFx service url. The metrics are posted to `v2/datapoint` under its path.
44
48
  # Default: https://ingest.signalfx.com
45
49
  #
46
50
  # Notes:
@@ -51,16 +55,15 @@ module SemanticLogger
51
55
  # Using a `count` of a `gauge` in a chart will significantly under-count the number of occurrences.
52
56
  #
53
57
  # If dimensions are added to the metric, then the metric will be sent as-is and
54
- # the above logic will _not_ be applied.
55
- #
56
- # Example, Gauge metric, supplying the duration in `metric_amount`:
57
- # logger.info(metric: 'Filters.average', metric_amount: 1.2, dimensions: {user: 'jbloggs'})
58
+ # the above logic will _not_ be applied. A dimension whose value is blank (nil, or text that is
59
+ # empty or only whitespace) is left out, and dimensions that are empty, such as `{}`, or all
60
+ # blank, such as `{state: ""}`, count as none.
58
61
  #
59
62
  # Example, Counter metric:
60
- # logger.info(metric: 'Filters.count', dimensions: {user: 'jbloggs'})
63
+ # logger.info(metric: 'Filters.count', dimensions: {plan: 'premium'})
61
64
  #
62
65
  # Example, Counter metric with a count other than 1:
63
- # logger.info(metric: 'Filters.count', metric_amount: 23, dimensions: {user: 'jbloggs'})
66
+ # logger.info(metric: 'Filters.count', metric_amount: 23, dimensions: {plan: 'premium'})
64
67
  #
65
68
  # When a duration is supplied and no dimensions are supplied:
66
69
  # logger.info(metric: 'Common/User/authorize', duration: 1.4)
@@ -104,6 +107,12 @@ module SemanticLogger
104
107
  post(message, full_path)
105
108
  end
106
109
 
110
+ # The #batch above formats the whole batch with #format_batch, so it answers as
111
+ # Appender::Http#batch_refusal does for its own #batch: only while it is the one in use.
112
+ def batch_refusal
113
+ format_batch_refusal if batch_defined_by?(Signalfx)
114
+ end
115
+
107
116
  # Only forward log entries that contain metrics.
108
117
  def should_log?(log)
109
118
  log.metric && meets_log_level?(log) && !filtered?(log)
@@ -14,30 +14,48 @@ module SemanticLogger
14
14
  #
15
15
  # Parameters:
16
16
  # url: [String]
17
- # Valid URL to post to.
17
+ # Url of the Statsd server to send metrics to. Only udp is supported.
18
18
  # Example:
19
19
  # udp://localhost:8125
20
20
  # Example, send all metrics to a particular namespace:
21
21
  # udp://localhost:8125/namespace
22
22
  # Default: udp://localhost:8125
23
23
  #
24
+ # level, filter: See SemanticLogger::Subscriber, as for the other metric subscribers.
25
+ #
24
26
  # Example:
25
27
  # SemanticLogger.add_appender(
26
28
  # metric: :statsd,
27
29
  # url: 'udp://localhost:8125'
28
30
  # )
29
- def initialize(url: "udp://localhost:8125")
31
+ def initialize(url: "udp://localhost:8125", **args, &)
30
32
  @url = url
31
- super()
33
+ super(**args, &)
34
+ reopen
32
35
  end
33
36
 
37
+ # Creates the Statsd client, or reconnects it, such as in a forked process, which closes the socket that the
38
+ # client had, rather than leaving it open. The host is resolved first, so that a failure to resolve it leaves the
39
+ # client connected as it was. Only a url that drops the namespace creates a new client, since a client cannot be
40
+ # given back no namespace, which leaves the old client's socket for garbage collection to close.
34
41
  def reopen
35
42
  uri = Utils.parse_url(@url)
36
43
  raise('Statsd only supports udp. Example: "udp://localhost:8125"') if uri.scheme != "udp"
37
44
 
38
- @statsd = ::Statsd.new(uri.host, uri.port)
39
- path = uri.path.chomp("/")
40
- @statsd.namespace = path.sub("/", "") if path != ""
45
+ # The host without the brackets of an IPv6 address.
46
+ host = uri.hostname
47
+ path = uri.path.chomp("/")
48
+ namespace = path.sub("/", "") unless path == ""
49
+ if @statsd && (namespace || @statsd.namespace.nil?)
50
+ Addrinfo.ip(host)
51
+ @statsd.host = host
52
+ @statsd.port = uri.port
53
+ @statsd.namespace = namespace if namespace
54
+ @statsd.connect
55
+ else
56
+ @statsd = ::Statsd.new(host, uri.port)
57
+ @statsd.namespace = namespace if namespace
58
+ end
41
59
  end
42
60
 
43
61
  def log(log)
@@ -59,8 +77,8 @@ module SemanticLogger
59
77
 
60
78
  # Only forward log entries that contain metrics.
61
79
  def should_log?(log)
62
- # Does not support metrics with dimensions.
63
- log.metric && !log.dimensions && meets_log_level?(log) && !filtered?(log)
80
+ # Records a metric that has dimensions under its name alone, since statsd-ruby has no tags.
81
+ log.metric && meets_log_level?(log) && !filtered?(log)
64
82
  end
65
83
 
66
84
  # The url without any credentials. See Subscriber#inspect.
@@ -1,12 +1,19 @@
1
+ # Loaded now rather than autoloaded, since .logger builds one on first use, which can be the first log
2
+ # call of the process, made in a signal trap, where Ruby does not allow a require.
3
+ require "semantic_logger/appender/io"
4
+
1
5
  module SemanticLogger
2
6
  # Thread that submits and processes log requests
3
7
  class Processor < Appender::Async
4
8
  # Allow the internal logger to be overridden from its default of $stderr
5
9
  # Can be replaced with another Ruby logger or Rails logger, but never to
6
10
  # SemanticLogger::Logger itself since it is for reporting problems
7
- # while trying to log to the various appenders
8
- class << self
9
- attr_writer :logger
11
+ # while trying to log to the various appenders (raises ArgumentError)
12
+ #
13
+ # A Ruby or Rails logger is wrapped in an appender (see Appender.internal_logger), so .logger
14
+ # returns that appender. The appenders added from then on report to it.
15
+ def self.logger=(logger)
16
+ @logger = Appender.internal_logger(logger)
10
17
  end
11
18
 
12
19
  # Internal logger for SemanticLogger
@@ -24,10 +31,124 @@ module SemanticLogger
24
31
  attr_reader :appenders
25
32
 
26
33
  def initialize(max_queue_size: 10_000)
27
- @appenders = Appenders.new(self.class.logger.dup)
34
+ @appenders = Appenders.new(self.class.logger.dup)
35
+ # The appenders whose removal was handed over to the worker thread, and not yet done (see #remove),
36
+ # kept here rather than only on the queue, which a forked child replaces. Takes no lock to change.
37
+ @pending_removals = Concurrent::Array.new
38
+ @pid = ::Process.pid
28
39
  super(appender: @appenders, max_queue_size: max_queue_size)
29
40
  end
30
41
 
42
+ # Reopen the appenders, after a fork or within the same process, each on the thread that writes
43
+ # to it, between writes (see Appender::Async#reopen): the worker thread of this processor reopens
44
+ # the appenders (see Appenders#reopen).
45
+ #
46
+ # After a fork, the queue and worker thread of each appender added with async: true or in batches
47
+ # are first replaced on the calling thread, rather than once this processor's worker thread gets
48
+ # to the reopen, so that they are running when this returns, such as for SemanticLogger.stats.
49
+ # That worker thread still asks each of them to reopen its appender, ahead of the entries it
50
+ # hands them. Takes no lock, so that it can be called from a signal trap.
51
+ #
52
+ # In a forked child, the removals handed over to the worker thread that it had not yet done in the
53
+ # parent (see #remove) were on the parent's queue, which the child replaces. So they are handed
54
+ # over again, to the child's worker thread, after the reopen: each such appender, reopened first,
55
+ # then closes only what it built in the child. They count as pending (see #removal_pending?) just
56
+ # before that, so a #remove of one meanwhile returns true while it is still listed, briefly.
57
+ def reopen
58
+ forked = @pid != ::Process.pid
59
+ @pid = ::Process.pid
60
+ appenders.renew_after_fork
61
+ super
62
+ hand_over_pending_removals if forked
63
+ end
64
+
65
+ # Remove the appender from those that this processor writes to, and close it. Returns the result
66
+ # of closing it, or true when it hands the close over to the worker thread (see below).
67
+ #
68
+ # Before closing an appender that it removed, waits until the log messages already queued are
69
+ # written to the other appenders, and those flushed (see #flush), so that none is still being
70
+ # written to it when it is closed, such as a request on its connection that the worker thread is
71
+ # sending.
72
+ #
73
+ # First, when the appender is one of them, and the appenders are to be reopened (see #reopen) but
74
+ # the worker thread has not yet reopened them, such as in a forked child that has not logged
75
+ # since the fork, waits until it has (see QueueProcessor#await_reopen), so that the appender is
76
+ # reopened before it is removed. Its #close then closes only what it built in this process,
77
+ # rather than what it inherited from the parent, which over TLS would end the parent's connection
78
+ # too, for an appender whose #reopen forgets what the parent built, but whose #close does not. An
79
+ # appender added with async: true or in batches is asked to reopen by that reopen, and its own
80
+ # worker thread reopens it before it runs the close.
81
+ #
82
+ # Called on the worker thread itself, such as by an appender as it writes, it closes the appender
83
+ # at once, without waiting, since nothing else writes to it then.
84
+ #
85
+ # Called where it must not wait for the worker thread (see #current_thread_inside?), such as by an
86
+ # appender added with async: true or in batches as it writes, removing itself or another one, it
87
+ # hands the removal over to the worker thread without waiting (see QueueProcessor#hand_over), and
88
+ # returns true, before the appender is closed. Between writes, never while it writes to the
89
+ # appender, once it has written the log messages queued ahead, the worker thread closes the
90
+ # appender, then removes it (see #close_removed). An appender that removed itself is closed once
91
+ # its current write has returned, after the log messages queued for it.
92
+ #
93
+ # The appender stays in the list until it is closed, so that the thread of an appender that
94
+ # removed itself still counts as one that must not wait while it is being closed, and a close of
95
+ # every appender (see #close) that runs first closes it instead. A removal handed over is never
96
+ # what changes the list while the worker thread goes through it, writing a message or closing
97
+ # every appender, since the worker thread itself takes the appender off the list. Until then,
98
+ # #appenders still lists it.
99
+ #
100
+ # Called on any other thread for an appender whose removal was handed over and is not yet done,
101
+ # it leaves the close to that removal, so that the appender is not closed twice: it waits for the
102
+ # log messages queued ahead to be written (see #flush), which, with the removal queued ahead, takes
103
+ # it off the list, and returns true. Not in a forked child that has not reopened this processor,
104
+ # whose queue still holds none of those removals (see #removal_pending?): it closes the appender.
105
+ #
106
+ # Takes no lock, so that it can be called from a signal trap.
107
+ def remove(appender)
108
+ return hand_over_removal(appender) if current_thread_inside?
109
+ return flush || true if removal_pending?(appender)
110
+
111
+ processor.await_reopen if appenders.include?(appender)
112
+ flush if appenders.delete(appender)
113
+ appender.close
114
+ end
115
+
116
+ # Write the log messages already queued to the appenders, then flush them (see
117
+ # QueueProcessor#flush).
118
+ #
119
+ # Returns false at once on the worker thread of an appender added with async: true or in batches,
120
+ # such as by that appender as it writes, rather than wait forever: the worker thread of this
121
+ # processor waits for that thread to flush the appender. Reported once to the internal logger.
122
+ def flush
123
+ return super unless current_thread_inside?
124
+
125
+ refuse_on_appender_thread(:flush)
126
+ end
127
+
128
+ # Flush and close the appenders, then remove them (see Appenders#close).
129
+ #
130
+ # Returns false at once on the worker thread of an appender added with async: true or in batches,
131
+ # as #flush does.
132
+ def close
133
+ return super unless current_thread_inside?
134
+
135
+ refuse_on_appender_thread(:close)
136
+ end
137
+
138
+ # Returns [true|false] whether the calling thread must not wait for a thread that uses this
139
+ # processor, since that thread may be waiting for it: the calling thread is the worker thread of
140
+ # an appender added with async: true or in batches, for which the worker thread of this processor
141
+ # waits, to flush or close it, or for room on its full queue. A message that such a thread logs
142
+ # while the queue of this processor is full is dropped, rather than wait for room, only while
143
+ # this processor's worker thread can be waiting for it: while that appender's own queue is full,
144
+ # or a flush or close of it is waiting (see QueueProcessor#holding_up?).
145
+ #
146
+ # Otherwise false: holding no lock while it queues an entry, this processor never has another
147
+ # thread wait for the calling thread to return from a call to it, even on its own worker thread.
148
+ def current_thread_inside?
149
+ appenders.worker_thread?
150
+ end
151
+
31
152
  # Start the appender thread
32
153
  def start
33
154
  return false if active?
@@ -36,6 +157,11 @@ module SemanticLogger
36
157
  true
37
158
  end
38
159
 
160
+ # Returns [Integer] the number of log messages waiting on the main pipeline queue.
161
+ def queue_size
162
+ queue.size
163
+ end
164
+
39
165
  # Returns [Hash] operational statistics for the logging pipeline.
40
166
  #
41
167
  # queue_size: [Integer] Number of log messages waiting on the main pipeline queue.
@@ -47,7 +173,7 @@ module SemanticLogger
47
173
  # appenders: [Array<Hash>] Per-appender statistics, see Appenders#stats.
48
174
  def stats
49
175
  {
50
- queue_size: queue.size,
176
+ queue_size: queue_size,
51
177
  capped: capped?,
52
178
  max_queue_size: capped? ? max_queue_size : nil,
53
179
  thread_active: active? || false,
@@ -56,5 +182,68 @@ module SemanticLogger
56
182
  appenders: appenders.stats
57
183
  }
58
184
  end
185
+
186
+ private
187
+
188
+ # Hand the removal of the appender over to the worker thread (see #remove), recording it as
189
+ # pending until it is done.
190
+ def hand_over_removal(appender)
191
+ @pending_removals << appender unless @pending_removals.include?(appender)
192
+ processor.hand_over { close_removed(appender) }
193
+ true
194
+ end
195
+
196
+ # Whether the removal of the appender was handed over to the worker thread (see #remove), and is
197
+ # not yet done. Only in the process that handed it over, or a forked child once #reopen has handed
198
+ # it over again: until then, the queue of a forked child holds the parent's entries, which no
199
+ # worker thread of the child runs.
200
+ def removal_pending?(appender) = @pid == ::Process.pid && @pending_removals.include?(appender)
201
+
202
+ # In a forked child, hand over again the removals pending in the parent (see #reopen), for the
203
+ # appenders still listed.
204
+ def hand_over_pending_removals
205
+ @pending_removals.dup.each do |appender|
206
+ next @pending_removals.delete(appender) unless appenders.include?(appender)
207
+
208
+ processor.hand_over { close_removed(appender) }
209
+ end
210
+ end
211
+
212
+ # Close the appender, then remove it from the list, on the worker thread, unless it is no longer
213
+ # listed: a close of every appender (see #close), or another removal, already closed it. Its
214
+ # removal is then no longer pending.
215
+ #
216
+ # Reports an error rather than raise it to the worker thread, as Appenders#close does, whatever
217
+ # its class, since one that is not a StandardError would stop the worker thread for good. Except
218
+ # a stack overflow, which the worker thread must not carry on after: on macOS a second one on the
219
+ # same thread crashes Ruby. It is raised, and a new worker thread carries on (see
220
+ # QueueProcessor#run_command).
221
+ def close_removed(appender)
222
+ return @pending_removals.delete(appender) unless appenders.include?(appender)
223
+
224
+ begin
225
+ appender.close
226
+ rescue SystemStackError
227
+ raise
228
+ rescue Exception => e
229
+ appenders.logger.error "Failed to close appender: #{appender.name}", e
230
+ ensure
231
+ appenders.delete(appender)
232
+ @pending_removals.delete(appender)
233
+ end
234
+ end
235
+
236
+ # Refuse to flush or close on the worker thread of an appender (see #flush), and report it the first
237
+ # time. Returns false.
238
+ def refuse_on_appender_thread(command)
239
+ unless @refused_on_appender_thread
240
+ @refused_on_appender_thread = true
241
+ appenders.logger.warn "SemanticLogger.#{command} was called on the worker thread of an appender " \
242
+ "added with async: true or in batches, such as by that appender as it writes. " \
243
+ "It returns false there, without waiting, since the processor's thread waits " \
244
+ "for that thread. Reported only once."
245
+ end
246
+ false
247
+ end
59
248
  end
60
249
  end
@@ -0,0 +1,140 @@
1
+ module SemanticLogger
2
+ class QueueProcessor
3
+ # Internal: how long the worker thread of a queue processor waits, after it could not write to
4
+ # the destination of its appender, before it writes again, and when it gives up instead.
5
+ #
6
+ # It knows how many times in a row writing has failed, how many times the worker thread may retry
7
+ # before it gives up (async_max_retries), how many requests, such as a flush or close, are
8
+ # waiting for the worker thread to run them, and whether the process is exiting. Each back-off
9
+ # waits a second longer than the one before, except while the worker thread is waited on, by a
10
+ # request or the exiting process, which must not wait out back-offs: then the worker thread does
11
+ # not wait at all, submitting a request or exiting ends the back-off under way, and once the
12
+ # destination fails, the worker thread gives up on the messages queued ahead of the next request
13
+ # (see #give_up?).
14
+ #
15
+ # Requests are submitted from other threads, and from a signal trap, where Ruby does not allow a
16
+ # Mutex to be locked, so it counts them, and waits, on Queues.
17
+ class Backoff
18
+ # Returns [Integer] the number of times the worker thread retries after a failure, before it
19
+ # gives up, or -1 to retry for ever.
20
+ attr_reader :max_retries
21
+
22
+ # Returns [Integer] the number of failures in a row, which is also how many seconds the next
23
+ # back-off waits.
24
+ attr_reader :retry_count
25
+
26
+ def initialize(max_retries)
27
+ @max_retries = max_retries
28
+ @retry_count = 0
29
+ # One entry for each request waiting.
30
+ @requests = Queue.new
31
+ # Holds an entry once a request has been submitted since the back-off under way began. One is
32
+ # enough to end it, so that submitting many requests without a failure does not add up.
33
+ @interrupts = SizedQueue.new(1)
34
+ @exiting = false
35
+ @gave_up = nil
36
+ end
37
+
38
+ # Start counting afresh, once writing succeeds.
39
+ def reset
40
+ @retry_count = 0
41
+ end
42
+
43
+ # Start afresh for a worker thread new to the process, after a fork: it has not failed yet, the
44
+ # requests counted as waiting were on the queue of the parent process, which the child
45
+ # replaces, and the child is not exiting, even when the parent was, such as when another thread
46
+ # forked while the parent flushed at exit.
47
+ def restart
48
+ reset
49
+ @requests.clear
50
+ @exiting = false
51
+ @gave_up = nil
52
+ end
53
+
54
+ # Count a failure. Returns whether the worker thread retries after it, or false once it has
55
+ # retried max_retries times in a row.
56
+ def retry?
57
+ return false unless max_retries == -1 || retry_count < max_retries
58
+
59
+ @retry_count += 1
60
+ true
61
+ end
62
+
63
+ # Returns [Integer|String] the number of retries the worker thread makes at most, for a report.
64
+ def limit = max_retries == -1 ? "unlimited" : max_retries
65
+
66
+ # Waits retry_count seconds, unless the worker thread is waited on, or until it is.
67
+ #
68
+ # The interruptions from before are cleared before it checks whether it is waited on, so that a
69
+ # request submitted, or the process exiting, in between either is seen or interrupts the wait.
70
+ def wait
71
+ @interrupts.clear
72
+ @interrupts.pop(timeout: retry_count) unless waited_on?
73
+ end
74
+
75
+ # Called before a request is added to the queue: it is waiting until #request_done, and ends the
76
+ # back-off under way. Never waits, since an interruption already queued ends it as well.
77
+ def request_submitted
78
+ @requests << true
79
+ @interrupts.push(true, timeout: 0)
80
+ end
81
+
82
+ # Called once for each request submitted, once it has run or will not (see Request#done). The
83
+ # worker thread no longer gives up on the messages queued after it (see #give_up?).
84
+ def request_done
85
+ @requests.pop(timeout: 0)
86
+ @gave_up = nil
87
+ end
88
+
89
+ # Tell it whether the process is exiting: true before the flush at exit, and false once that
90
+ # flush has returned, since a test framework can run its tests in a later at_exit hook. While
91
+ # the process exits, the worker thread is waited on, as though a request were waiting, and the
92
+ # back-off under way ends. Once it no longer exits, the worker thread gives up no more, unless a
93
+ # request is still waiting (see #gave_up). Called from the exiting thread, which may be in a
94
+ # signal trap, so it takes no lock either.
95
+ def exiting=(exiting)
96
+ @exiting = exiting
97
+ if exiting
98
+ @interrupts.push(true, timeout: 0)
99
+ elsif !waited_on?
100
+ @gave_up = nil
101
+ end
102
+ end
103
+
104
+ # Whether the worker thread is waited on: a request, such as a flush or close, is waiting for
105
+ # it to run it, or the process is exiting. It then neither waits out back-offs nor writes again
106
+ # what the destination failed.
107
+ def waited_on? = @exiting || request_waiting?
108
+
109
+ # Whether a request, such as a flush or close, is waiting for the worker thread to run it. The
110
+ # thread that submitted it may be the worker thread of the queue processor that writes to this
111
+ # one's appender, which then waits for this worker thread (see QueueProcessor#holding_up?).
112
+ def request_waiting? = !@requests.empty?
113
+
114
+ # Called once the destination failed. While the worker thread is waited on, it gives up on the
115
+ # messages queued ahead of the next request, such as a flush, and discards them without writing
116
+ # them (see #gave_up), since each could take as long to fail, such as the open_timeout of a
117
+ # server that does not answer. While the process exits, it does so as well before the flush at
118
+ # exit reaches the queue. Once a request has run (see #request_done), it writes the next
119
+ # message once more, even while the process exits, and gives up again if that fails. So
120
+ # returns true, having recorded the exception, for the worker thread to carry on straight
121
+ # away, without backing off or counting a retry. Otherwise returns false.
122
+ def give_up?(exception)
123
+ return false unless waited_on?
124
+
125
+ @gave_up = exception
126
+ true
127
+ end
128
+
129
+ # Returns the exception with which the destination failed while the worker thread was waited
130
+ # on (see #give_up?), while the messages it takes off the queue are to be discarded: until a
131
+ # request has run, or the worker thread is no longer waited on. Otherwise nil.
132
+ def gave_up
133
+ return unless @gave_up
134
+ return @gave_up if waited_on?
135
+
136
+ @gave_up = nil
137
+ end
138
+ end
139
+ end
140
+ end