semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -10,15 +10,29 @@ module SemanticLogger
10
10
  # SemanticLogger.add_appender(
11
11
  # appender: :udp,
12
12
  # server: 'server:3300',
13
+ # metrics: false
13
14
  # )
14
15
  class Udp < SemanticLogger::Subscriber
16
+ # The host and port of a server, with an IPv6 address in brackets, as a URL writes it.
17
+ SERVER = /\A(?:\[(?<ipv6>[^\[\]]+)\]|(?<host>[^:\[\]]+)):(?<port>\d+)\z/
18
+ private_constant :SERVER
19
+
20
+ # The codes of a lookup that found no address of the family it asked for, rather than failing, of those that
21
+ # the platform defines.
22
+ NO_ADDRESS = %i[EAI_NONAME EAI_NODATA EAI_ADDRFAMILY].filter_map do |name|
23
+ Socket.const_get(name) if Socket.const_defined?(name)
24
+ end.freeze
25
+ private_constant :NO_ADDRESS
26
+
15
27
  attr_accessor :server, :udp_flags
16
28
  attr_reader :socket
17
29
 
18
30
  # Create UDP log appender.
19
31
  #
20
32
  # server: [String]
21
- # URL of the server to write UDP messages to.
33
+ # The server to write UDP messages to, as "host:port", such as "localhost:8088", with an IPv6
34
+ # address in brackets, as a URL writes it, such as "[::1]:8088". A host name is sent to over
35
+ # IPv4 when it has an IPv4 address, and otherwise, or when it cannot connect over IPv4, over IPv6.
22
36
  #
23
37
  # udp_flags: [Integer]
24
38
  # Should be a bitwise OR of Socket::MSG_* constants.
@@ -35,12 +49,13 @@ module SemanticLogger
35
49
  #
36
50
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
37
51
  # Override the log level for this appender.
38
- # Default: SemanticLogger.default_level
52
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
53
+ # filters by its own level.
39
54
  #
40
- # formatter: [Object|Proc]
55
+ # formatter: [Object|Proc|Symbol|Hash]
41
56
  # An instance of a class that implements #call, or a Proc to be used to format
42
57
  # the output from this appender
43
- # Default: Use the built-in formatter (See: #call)
58
+ # Default: :json
44
59
  #
45
60
  # filter: [Regexp|Proc]
46
61
  # RegExp: Only include log messages where the class name matches the supplied.
@@ -50,7 +65,7 @@ module SemanticLogger
50
65
  #
51
66
  # metrics: [Boolean]
52
67
  # Send metrics only events over udp.
53
- # Default: true
68
+ # Default: true, with a deprecation warning when it is not supplied, since v6 changes it to false.
54
69
  #
55
70
  # Limitations:
56
71
  # * UDP packet size is limited by the connected network and any routers etc
@@ -59,39 +74,52 @@ module SemanticLogger
59
74
  # Example:
60
75
  # SemanticLogger.add_appender(
61
76
  # appender: :udp,
62
- # server: 'server:3300'
77
+ # server: 'server:3300',
78
+ # metrics: false
63
79
  # )
64
- def initialize(server:, udp_flags: 0, metrics: true, **args, &)
80
+ def initialize(server:, udp_flags: 0, metrics: METRICS_NOT_SUPPLIED, **args, &)
65
81
  @server = server
66
82
  @udp_flags = udp_flags
67
83
 
68
- super(metrics: metrics, **args, &)
84
+ super(metrics: deprecated_metrics_default(metrics, :udp), **args, &)
69
85
  reopen
70
86
  end
71
87
 
72
88
  # After forking an active process call #reopen to re-open
73
89
  # open the handles to resources
90
+ #
91
+ # Connects a new socket to the server before closing the one it replaces. When it cannot, it
92
+ # raises, keeping the socket it has: ArgumentError for a server that is not "host:port", or
93
+ # the error from resolving or connecting, such as SocketError for a host name that does not resolve.
94
+ # An error closing the socket it replaced is reported to the internal logger as a warning, since the
95
+ # reopen itself worked.
74
96
  def reopen
75
- close
76
- @socket = UDPSocket.new
77
- host, port = server.split(":")
78
- @socket.connect(host, port.to_i)
97
+ socket = connected_socket
98
+ old_socket = @socket
99
+ @socket = socket
100
+ close_socket(old_socket, replaced: true)
79
101
  end
80
102
 
81
- # Write the log using the specified protocol and server.
103
+ # Write the log using the specified protocol and server. Opens a new socket, connected to the server, first
104
+ # when the appender has none, after #close.
82
105
  def log(log)
83
- @socket.send(format_log(log), udp_flags)
106
+ (@socket ||= connected_socket).send(format_log(log), udp_flags)
84
107
  true
85
108
  end
86
109
 
87
- # Flush is called by the semantic_logger during shutdown.
110
+ # Flush is called by the semantic_logger during shutdown. Does nothing after #close, which left no socket.
88
111
  def flush
89
112
  @socket&.flush
90
113
  end
91
114
 
92
- # Close is called during shutdown, or with reopen
115
+ # Closes the socket, such as when the appender is removed, and forgets it, so that the next write opens a new
116
+ # one. An error closing it is reported to the internal logger as a warning, rather than raised.
117
+ #
118
+ # In a forked child, the socket is the child's own copy of the parent's, so closing it leaves the parent's open.
93
119
  def close
94
- @socket&.close
120
+ socket = @socket
121
+ @socket = nil
122
+ close_socket(socket)
95
123
  end
96
124
 
97
125
  # A log message too large for a datagram (Errno::EMSGSIZE) cannot be sent by retrying.
@@ -106,7 +134,91 @@ module SemanticLogger
106
134
 
107
135
  private
108
136
 
109
- # Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
137
+ # Closes the given socket, if any: the one that a reopen replaced, when replaced: is true. An error closing
138
+ # it is reported to the internal logger as a warning, rather than raised.
139
+ def close_socket(socket, replaced: false)
140
+ socket&.close
141
+ rescue StandardError => e
142
+ logger.warn("Could not close the socket to the UDP server#{' that a reopen replaced' if replaced}", e)
143
+ end
144
+
145
+ # Returns [UDPSocket] a new socket, connected to the first of the server's addresses that it can connect to, as
146
+ # Ruby's own UDPSocket#connect tries the addresses of its socket's family in turn. A host name's IPv4 addresses
147
+ # come first, so that a name with both is still sent to over IPv4, as when the socket was IPv4 only, whichever
148
+ # address the resolver returns first. Its IPv6 addresses are looked up only when the resolver answers that it has
149
+ # no IPv4 address, or it has none that it can connect to, so that a resolver that does not answer for them does
150
+ # not hold up the others. Each family keeps the resolver's order. When looking up its IPv4 addresses fails for
151
+ # another reason, such as a resolver that does not answer in time, raises that SocketError at once, rather than
152
+ # sending over IPv6 to a name that has IPv4 addresses, or waiting for a resolver that is down a second time.
153
+ #
154
+ # When it cannot connect to any address, raises the error from the first one it tried, the one it prefers. When
155
+ # the host has no address, such as a name that does not resolve, raises the SocketError from looking up its IPv6
156
+ # addresses, with the one from looking up its IPv4 addresses as its cause (see #raise_lookup_error).
157
+ def connected_socket
158
+ host, port, families = server_endpoint
159
+ connect_error = nil
160
+ lookup_errors = []
161
+ families.each do |family|
162
+ Addrinfo.getaddrinfo(host, port, family, :DGRAM).each do |address|
163
+ return connect_to(address)
164
+ rescue SystemCallError => e
165
+ connect_error ||= e
166
+ end
167
+ rescue SocketError => e
168
+ lookup_errors << e
169
+ break unless no_address?(e)
170
+ end
171
+ raise(connect_error) if connect_error
172
+
173
+ raise_lookup_error(host, lookup_errors)
174
+ end
175
+
176
+ # Returns [true|false] whether the SocketError from looking up a host's addresses of one family says that it has
177
+ # none, rather than that the lookup failed. It cannot tell without the error's code, which Socket::ResolutionError
178
+ # has from Ruby 3.3, so on Ruby 3.2, or on JRuby, it takes any SocketError to say so, as before.
179
+ def no_address?(error)
180
+ !error.respond_to?(:error_code) || NO_ADDRESS.include?(error.error_code)
181
+ end
182
+
183
+ # Raises the last of the SocketErrors from looking up the host's addresses, with the one before it, if any, as its
184
+ # cause, since it looked up the later family only because the earlier one had no address. Raises a SocketError
185
+ # of its own, naming the host, when no lookup raised, since JRuby returns no addresses for a family that the host
186
+ # has none in, rather than raising.
187
+ def raise_lookup_error(host, errors)
188
+ last, earlier = errors.reverse
189
+ raise(SocketError, "getaddrinfo: no address for #{host}") unless last
190
+
191
+ earlier ? raise(last, cause: earlier) : raise(last)
192
+ end
193
+
194
+ # Returns [UDPSocket] a new socket of the address's family, connected to it. Closes the socket when it does not
195
+ # return it, such as when connecting raises, or the thread is interrupted.
196
+ def connect_to(address)
197
+ socket = UDPSocket.new(address.afamily)
198
+ socket.connect(address.ip_address, address.ip_port)
199
+ connected = true
200
+ socket
201
+ ensure
202
+ socket&.close unless connected
203
+ end
204
+
205
+ # Returns [Array(String, Integer, Array<Symbol>)] the host and port of the server, and the address families to
206
+ # look the host up in, in the order to try them. A host in brackets is an IPv6 address. Any other host is an IPv4
207
+ # address, or a name, which is looked up for its IPv4 addresses first, then for its IPv6 addresses. Whitespace
208
+ # around the server, such as the newline at the end of an environment variable, is ignored.
209
+ def server_endpoint
210
+ match = SERVER.match(server.to_s.strip)
211
+ unless match
212
+ raise(ArgumentError,
213
+ "Invalid :server #{server.inspect}, it must be \"host:port\", such as \"localhost:8088\", " \
214
+ "with an IPv6 address in brackets, such as \"[::1]:8088\".")
215
+ end
216
+
217
+ port = match[:port].to_i
218
+ match[:ipv6] ? [match[:ipv6], port, %i[INET6]] : [match[:host], port, %i[INET INET6]]
219
+ end
220
+
221
+ # Returns [SemanticLogger::Formatters::Json] formatter default for this Appender
110
222
  def default_formatter
111
223
  SemanticLogger::Formatters::Json.new
112
224
  end
@@ -4,7 +4,9 @@
4
4
  module SemanticLogger
5
5
  module Appender
6
6
  class Wrapper < SemanticLogger::Subscriber
7
- attr_reader :logger
7
+ # The logger this appender writes to. Not #logger, which is the internal logger that this
8
+ # appender reports its own problems to, as every appender does (see Subscriber#logger).
9
+ attr_reader :wrapped_logger
8
10
 
9
11
  # Forward all logging calls to the supplied logging instance.
10
12
  #
@@ -14,7 +16,8 @@ module SemanticLogger
14
16
  #
15
17
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
16
18
  # Override the log level for this appender.
17
- # Default: SemanticLogger.default_level
19
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
20
+ # filters by its own level.
18
21
  #
19
22
  # formatter: [Object|Proc]
20
23
  # An instance of a class that implements #call, or a Proc to be used to format
@@ -39,10 +42,10 @@ module SemanticLogger
39
42
  #
40
43
  # Install the `rails_semantic_logger` gem to replace the Rails logger with Semantic Logger.
41
44
  def initialize(logger:, **args, &)
42
- @logger = logger
45
+ @wrapped_logger = logger
43
46
 
44
47
  # Check if the custom appender responds to all the log levels. For example Ruby ::Logger
45
- does_not_implement = LEVELS[1..].find { |i| !@logger.respond_to?(i) }
48
+ does_not_implement = LEVELS[1..].find { |i| !@wrapped_logger.respond_to?(i) }
46
49
  if does_not_implement
47
50
  raise(ArgumentError,
48
51
  "Supplied logger does not implement:#{does_not_implement}. " \
@@ -57,20 +60,20 @@ module SemanticLogger
57
60
  # Ruby or Rails Loggers
58
61
  def log(log)
59
62
  level = log.level == :trace ? :debug : log.level
60
- @logger.send(level, format_log(log))
63
+ @wrapped_logger.send(level, format_log(log))
61
64
  true
62
65
  end
63
66
 
64
67
  # Flush all pending logs to disk.
65
68
  # Waits for all queued log messages to be written to disk.
66
69
  def flush
67
- @logger.flush if @logger.respond_to?(:flush)
70
+ @wrapped_logger.flush if @wrapped_logger.respond_to?(:flush)
68
71
  end
69
72
 
70
73
  # Close underlying log
71
74
  # Waits for all queued log messages to be written to disk.
72
75
  def close
73
- @logger.close if @logger.respond_to?(:close)
76
+ @wrapped_logger.close if @wrapped_logger.respond_to?(:close)
74
77
  end
75
78
  end
76
79
  end
@@ -25,6 +25,7 @@ module SemanticLogger
25
25
  autoload :SplunkHttp, "semantic_logger/appender/splunk_http"
26
26
  autoload :Syslog, "semantic_logger/appender/syslog"
27
27
  autoload :Tcp, "semantic_logger/appender/tcp"
28
+ autoload :TextOutput, "semantic_logger/appender/text_output"
28
29
  autoload :Udp, "semantic_logger/appender/udp"
29
30
  autoload :Wrapper, "semantic_logger/appender/wrapper"
30
31
  autoload :SentryRuby, "semantic_logger/appender/sentry_ruby"
@@ -32,6 +33,10 @@ module SemanticLogger
32
33
  # @formatter:on
33
34
 
34
35
  # Returns [SemanticLogger::Subscriber] appender for the supplied options
36
+ #
37
+ # Raises ArgumentError when the appender is to be batched, with `batch: true` or by default, see
38
+ # Subscriber#batch_by_default?, and it says why it cannot be, see Subscriber#batch_refusal. When it fails
39
+ # after .build, it first closes the appender that .build says it created, see .close_created.
35
40
  def self.factory(async: false, batch: nil,
36
41
  max_queue_size: 10_000, lag_check_interval: 1_000, lag_threshold_s: 30,
37
42
  batch_size: 300, batch_seconds: 5,
@@ -39,13 +44,16 @@ module SemanticLogger
39
44
  async_max_retries: 100,
40
45
  **args,
41
46
  &)
42
- appender = build(**args, &)
47
+ appender, created = build(**args, &)
43
48
 
44
49
  # If appender implements #batch, then it should use the batch proxy by default,
45
50
  # unless the appender opts out of batching by default (e.g. the HTTP appender).
46
51
  batch = true if batch.nil? && appender.respond_to?(:batch) && appender.batch_by_default?
47
52
 
48
53
  if batch == true
54
+ refusal = appender.batch_refusal
55
+ raise(ArgumentError, "add_appender cannot batch #{appender.class}: #{refusal}") if refusal
56
+
49
57
  Appender::Async.new(
50
58
  appender: appender,
51
59
  batch: true,
@@ -70,21 +78,45 @@ module SemanticLogger
70
78
  else
71
79
  appender
72
80
  end
81
+ rescue Exception
82
+ close_created(appender) if created
83
+ raise
84
+ end
85
+
86
+ # Closes the appender that .build created, since .factory failed after creating it, such as when the appender
87
+ # cannot be batched, and nothing else holds it, yet it may have opened its destination, such as a connection.
88
+ # An error from closing it is ignored, so that .factory raises the one that it failed with.
89
+ def self.close_created(appender)
90
+ appender.close
91
+ rescue StandardError
92
+ nil
73
93
  end
94
+ private_class_method :close_created
74
95
 
75
- # Returns [Subscriber] instance from the supplied options.
96
+ # Returns [Subscriber, true | false] the appender for the supplied options, and whether .build created both it
97
+ # and its destination, so that nothing else holds them, and .factory closes it when it fails afterwards (see
98
+ # .close_created): true for the appender for `file_name:`, or one named by a Symbol as `appender:` or `metric:`.
99
+ # False for a Subscriber instance that the caller supplied as `appender:` or `metric:`, and for the appender that
100
+ # writes to the caller's own `io:` or `logger:`: closing the one for `logger:` closes that logger, and the stream
101
+ # it writes to, such as $stdout.
102
+ #
103
+ # It adds one destination, the first of file_name:, io:, logger:, appender:, and metric: that is given, in that
104
+ # order, and ignores any other, see .report_ignored_destinations.
76
105
  def self.build(io: nil, file_name: nil, appender: nil, metric: nil, logger: nil, **args, &)
106
+ # Named in the order of the if / elsif chain below, which adds the first one given: keep the two in step.
107
+ report_ignored_destinations(file_name: file_name, io: io, logger: logger, appender: appender, metric: metric)
108
+
77
109
  if file_name
78
- SemanticLogger::Appender::File.new(file_name, **args, &)
110
+ [SemanticLogger::Appender::File.new(file_name, **args, &), true]
79
111
  elsif io
80
- SemanticLogger::Appender::IO.new(io, **args, &)
112
+ [SemanticLogger::Appender::IO.new(io, **args, &), false]
81
113
  elsif logger
82
- SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &)
114
+ [SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &), false]
83
115
  elsif appender
84
116
  if appender.is_a?(Symbol)
85
- SemanticLogger::Utils.constantize_symbol(appender).new(**args)
117
+ [SemanticLogger::Utils.constantize_symbol(appender).new(**args, &), true]
86
118
  elsif appender.is_a?(Subscriber)
87
- appender
119
+ [subscriber_instance(:appender, appender, **args, &), false]
88
120
  else
89
121
  raise(ArgumentError,
90
122
  "Parameter :appender must be either a Symbol or an object derived from " \
@@ -92,13 +124,13 @@ module SemanticLogger
92
124
  end
93
125
  elsif metric
94
126
  if metric.is_a?(Symbol)
95
- SemanticLogger::Utils.constantize_symbol(metric, "SemanticLogger::Metric").new(**args)
127
+ [SemanticLogger::Utils.constantize_symbol(metric, "SemanticLogger::Metric").new(**args, &), true]
96
128
  elsif metric.is_a?(Subscriber)
97
- metric
129
+ [subscriber_instance(:metric, metric, **args, &), false]
98
130
  else
99
131
  raise(ArgumentError,
100
132
  "Parameter :metric must be either a Symbol or an object derived from " \
101
- "SemanticLogger::Subscriber, not: #{appender.inspect}")
133
+ "SemanticLogger::Subscriber, not: #{metric.inspect}")
102
134
  end
103
135
  else
104
136
  raise(ArgumentError,
@@ -108,5 +140,78 @@ module SemanticLogger
108
140
  end
109
141
 
110
142
  private_class_method :build
143
+
144
+ # Returns [SemanticLogger::Base] the supplied logger, ready to be an internal logger: the one that
145
+ # Semantic Logger reports its own problems to (see Processor.logger=).
146
+ #
147
+ # An internal logger is named after whatever reports to it, logs at :trace, and takes an
148
+ # exception after the message, as every Semantic Logger appender does. So an appender, such as the
149
+ # default IO appender, or any other logger that logs at :trace, is returned as it is. Any other
150
+ # logger, such as a Ruby or Rails Logger, does none of that, so it is wrapped in a Wrapper
151
+ # appender, which writes each report to it as text, with the exception. Its own level then decides
152
+ # which reports it writes, apart from the :trace reports, which are below its lowest level, :debug,
153
+ # so are left out.
154
+ #
155
+ # Returns nil for nil or false, which restores the default internal logger.
156
+ #
157
+ # Raises ArgumentError for a SemanticLogger::Logger: it logs back into the processor, so the
158
+ # report of an appender that fails would go to that appender, fail again, and be reported again.
159
+ def self.internal_logger(logger)
160
+ return unless logger
161
+
162
+ if logger.is_a?(SemanticLogger::Logger)
163
+ raise(ArgumentError,
164
+ "A SemanticLogger::Logger cannot be the internal logger, since its reports of a failing " \
165
+ "appender would be logged to that appender. Use an appender, such as " \
166
+ "SemanticLogger::Appender::IO.new($stderr), or a Ruby Logger.")
167
+ end
168
+ return logger if logger.respond_to?(:trace)
169
+
170
+ SemanticLogger::Appender::Wrapper.new(logger: logger, level: :debug)
171
+ end
172
+
173
+ # Returns [Subscriber] the appender or metric subscriber supplied as an instance, with the `option` that named it,
174
+ # `:appender` or `:metric`, as it is, since it was created with its own options. The options and block that .build
175
+ # would hand to an appender that it creates are not used, so reports a deprecation warning that names each one,
176
+ # since v6 raises ArgumentError for them. An option whose value is nil, the default of a Subscriber, sets nothing,
177
+ # so it is not reported, and v6 must not raise for it either, so that a caller can pass on its optional settings.
178
+ # Since an option of .factory that is misspelled arrives here too, the warning names the options that do apply to
179
+ # an instance. Remove in v6, raising ArgumentError in its place.
180
+ def self.subscriber_instance(option, subscriber, **args)
181
+ ignored = args.compact.keys.map { |key| "#{key}:" }
182
+ ignored.unshift("the formatter block") if block_given?
183
+ return subscriber if ignored.empty?
184
+
185
+ pronoun = ignored.one? ? "it" : "them"
186
+ Deprecation.warn(
187
+ "add_appender ignores #{word_list(ignored)} when #{option}: is a Subscriber instance (#{subscriber.class}), " \
188
+ "and v6 raises ArgumentError for #{pronoun}. Set #{pronoun} on the instance itself; " \
189
+ "only async:, batch:, and the queue options of add_appender apply to an instance."
190
+ )
191
+ subscriber
192
+ end
193
+ private_class_method :subscriber_instance
194
+
195
+ # Reports a deprecation warning when .build is given more than one of the destinations, in the order in which it
196
+ # takes the first one, naming the one that it uses and the others, which it ignores, since v6 raises ArgumentError
197
+ # for them. A destination given as nil, or false, is not one, as .build has always taken it, so that a caller can
198
+ # pass on settings of which only one is set, such as `file_name: options[:file_name], io: options[:io]`, and v6
199
+ # must not raise for it either. Remove in v6, raising ArgumentError in its place.
200
+ def self.report_ignored_destinations(**destinations)
201
+ used, *ignored = destinations.filter_map { |name, value| "#{name}:" if value }
202
+ return if ignored.empty?
203
+
204
+ Deprecation.warn(
205
+ "add_appender adds one destination, so it uses #{used} and ignores #{word_list(ignored)}, " \
206
+ "and v6 raises ArgumentError when given more than one. Call add_appender once for each destination."
207
+ )
208
+ end
209
+ private_class_method :report_ignored_destinations
210
+
211
+ # Returns [String] the words as a list in a sentence, such as "a", "a and b", or "a, b, and c".
212
+ def self.word_list(words)
213
+ words.size < 3 ? words.join(" and ") : "#{words[0..-2].join(', ')}, and #{words.last}"
214
+ end
215
+ private_class_method :word_list
111
216
  end
112
217
  end
@@ -1,11 +1,26 @@
1
1
  module SemanticLogger
2
2
  # Manage a collection of appenders.
3
+ #
4
+ # While the process exits, it also knows which of its appenders it skips: those without a queue of
5
+ # their own whose destination failed meanwhile, and how many log messages each has not been given
6
+ # since (see #process_exiting=).
3
7
  class Appenders < Concurrent::Array
8
+ # An appender skipped while the process exits: the error with which its destination failed, and
9
+ # the number of log messages it has not been given since it was last reported.
10
+ Skipped = Struct.new(:exception, :discarded)
11
+ private_constant :Skipped
12
+
13
+ # No appender is skipped while the process is not exiting.
14
+ NONE_SKIPPED = {}.freeze
15
+ private_constant :NONE_SKIPPED
16
+
4
17
  attr_accessor :logger
5
18
 
6
19
  def initialize(logger = Processor.logger.dup)
7
20
  @logger = logger
8
21
  @logger.name = self.class.name
22
+ @exiting = false
23
+ @skipped = {}.compare_by_identity
9
24
  super()
10
25
  end
11
26
 
@@ -52,21 +67,43 @@ module SemanticLogger
52
67
  # Write the log message to every appender that should log it. An appender that fails is
53
68
  # reported, and the others still write the message.
54
69
  #
70
+ # While the process exits, an appender without a queue of its own is skipped once its
71
+ # destination has failed, and the message counted as not given to it (see #process_exiting=).
72
+ #
73
+ # The values of the sensitive keys are first replaced, once, so that no appender writes them
74
+ # (see SemanticLogger.sensitive_keys=).
75
+ #
55
76
  # A stack overflow is not a failure of the appender but of the thread, which must not overflow
56
77
  # again: on macOS a second stack overflow on the same thread crashes Ruby. It is raised to the
57
78
  # processor running this thread instead, and the message goes to no further appenders.
58
79
  def log(log)
80
+ redactor = SemanticLogger.redactor
81
+ log.redact(redactor) if redactor
82
+ skipped = skipped_appenders
59
83
  each do |appender|
60
- appender.log(log) if appender.should_log?(log)
84
+ writing = appender.should_log?(log)
85
+ next unless writing
86
+ next skipped[appender].discarded += 1 if skipped.key?(appender)
87
+
88
+ appender.log(log)
61
89
  rescue SystemStackError
62
90
  raise
63
91
  rescue Exception => e
64
92
  logger.error "Failed to log to appender: #{appender.name}", e
93
+ skip_while_exiting(appender, e) if writing
65
94
  end
66
95
  end
67
96
 
97
+ # Flush every appender. An appender that fails is reported, and the others are still flushed.
98
+ #
99
+ # While the process exits, an appender that is skipped (see #process_exiting=) is first reported,
100
+ # with the number of log messages it has not been given since it was last reported, then flushed
101
+ # too, so that it delivers the messages it already accepted, such as those that the producer of a
102
+ # :kafka appender holds.
68
103
  def flush
104
+ skipped = skipped_appenders
69
105
  each do |appender|
106
+ report_skipped(appender, skipped[appender]) if skipped.key?(appender)
70
107
  logger.trace "Flushing appender: #{appender.name}"
71
108
  appender.flush
72
109
  rescue Exception => e
@@ -75,6 +112,13 @@ module SemanticLogger
75
112
  logger.trace "All appenders flushed"
76
113
  end
77
114
 
115
+ # Flush and close every appender, then remove those closed. An appender that fails is reported,
116
+ # and the others are still closed.
117
+ #
118
+ # A stack overflow is raised at once, as #log raises it, since the thread must not carry on: on
119
+ # macOS a second stack overflow on the same thread crashes Ruby. The appenders already closed are
120
+ # still removed. The one that overflowed the stack, and those after it, not yet closed, stay
121
+ # listed, as one whose close fails does.
78
122
  def close
79
123
  closed_appenders = []
80
124
  each do |appender|
@@ -82,15 +126,21 @@ module SemanticLogger
82
126
  appender.flush
83
127
  appender.close
84
128
  closed_appenders << appender
129
+ rescue SystemStackError
130
+ raise
85
131
  rescue Exception => e
86
132
  logger.error "Failed to close appender: #{appender.name}", e
87
133
  end
88
- # Delete appenders outside the #each above which prevents changes during iteration.
89
- closed_appenders.each { |appender| delete(appender) }
90
134
  logger.trace "All appenders closed and removed from appender list"
135
+ ensure
136
+ # Delete appenders outside the #each above which prevents changes during iteration.
137
+ closed_appenders&.each { |appender| delete(appender) }
91
138
  end
92
139
 
93
- # After a fork reopen each appender.
140
+ # After a fork, or a log rotation, reopen each appender. Called on the thread that writes to the
141
+ # appenders: the worker thread of the main Processor, or the calling thread with sync!. An
142
+ # appender added with async: true or in batches is asked to reopen, by its own worker thread
143
+ # (see Appender::Async#reopen).
94
144
  def reopen
95
145
  each do |appender|
96
146
  next unless appender.respond_to?(:reopen)
@@ -102,5 +152,87 @@ module SemanticLogger
102
152
  end
103
153
  logger.trace "All appenders re-opened"
104
154
  end
155
+
156
+ # Tell each appender added with async: true or in batches whether the process is exiting: true
157
+ # before the flush at exit, and false once it has returned (see QueueProcessor#process_exiting=).
158
+ # Each one's queue processor is told straight away, rather than through the queue of the thread
159
+ # that writes to every appender, which can be held up handing an entry to the full queue of an
160
+ # appender whose worker thread is backing off during an outage. Takes no lock, so that it can be
161
+ # called from any thread, or a signal trap. An appender that raises is reported, as for #flush,
162
+ # and the others are still told, so that the flush at exit still runs.
163
+ #
164
+ # The appenders without a queue of their own are written to here, one log message after another,
165
+ # by the thread that writes to every appender, so that during an outage the flush at exit would
166
+ # wait while each message queued ahead of it failed in turn, up to a timeout each against a
167
+ # server that does not answer. So while the process exits, once the destination of such an
168
+ # appender fails, other than for the content of a message (see Subscriber#message_error?), it is
169
+ # skipped until the process no longer exits: #log gives it no more messages, counting them, and
170
+ # #flush reports them, then still flushes it, so that it delivers what it already accepted. The
171
+ # other appenders are still given every message.
172
+ #
173
+ # It applies only in the process that exits, not in a child process forked meanwhile.
174
+ def process_exiting=(exiting)
175
+ @skipped = {}.compare_by_identity if exiting
176
+ @exiting = exiting && ::Process.pid
177
+ each do |appender|
178
+ appender.process_exiting = exiting if appender.respond_to?(:process_exiting=)
179
+ rescue Exception => e
180
+ logger.error "Failed to tell appender that the process is exiting: #{appender.name}", e
181
+ end
182
+ end
183
+
184
+ # After a fork, replace the queue and worker thread of each appender added with async: true or
185
+ # in batches with the child's own, on the calling thread (see QueueProcessor#renew_after_fork).
186
+ # Takes no lock, so that it can be called from a signal trap.
187
+ def renew_after_fork
188
+ each { |appender| appender.renew_after_fork if appender.respond_to?(:renew_after_fork) }
189
+ end
190
+
191
+ # Returns [true|false] whether the thread is the worker thread of one of the appenders: one added
192
+ # with async: true or in batches, which writes to it (see QueueProcessor#worker_thread?). The thread
193
+ # that writes to every appender may be waiting for it, so it must not wait for that thread. Takes
194
+ # no lock, so that it can be called from a signal trap.
195
+ def worker_thread?(thread = Thread.current)
196
+ any? { |appender| appender.respond_to?(:worker_thread?) && appender.worker_thread?(thread) }
197
+ end
198
+
199
+ # Returns [true|false] whether the thread is the worker thread of one of the appenders while that
200
+ # appender's queue is full, or a flush or close of it is waiting (see QueueProcessor#holding_up?):
201
+ # the thread that writes to every appender may then be waiting for it, to make room or to run that
202
+ # request. Takes no lock, as #worker_thread? takes none.
203
+ def holding_up?(thread = Thread.current)
204
+ any? { |appender| appender.respond_to?(:holding_up?) && appender.holding_up?(thread) }
205
+ end
206
+
207
+ private
208
+
209
+ # Whether this process is exiting (see #process_exiting=), rather than a child process forked while
210
+ # it was.
211
+ def exiting? = @exiting && @exiting == ::Process.pid
212
+
213
+ # Returns [Hash] the appenders skipped while the process exits, by appender, or none while it does
214
+ # not. The process can stop exiting while the thread that writes to every appender fails to write
215
+ # to one, so one skipped then is left out until the process exits again, which starts afresh.
216
+ def skipped_appenders = exiting? ? @skipped : NONE_SKIPPED
217
+
218
+ # While the process exits, skip an appender without a queue of its own once writing to it failed
219
+ # because of its destination (see #process_exiting=). An appender with a queue of its own is told
220
+ # that the process is exiting, and handles it itself.
221
+ def skip_while_exiting(appender, exception)
222
+ return unless exiting?
223
+ return if appender.respond_to?(:process_exiting=)
224
+ return if appender.respond_to?(:message_error?) && appender.message_error?(exception)
225
+
226
+ @skipped[appender] = Skipped.new(exception, 0)
227
+ end
228
+
229
+ # Report the log messages that a skipped appender has not been given since it was last reported.
230
+ def report_skipped(appender, skipped)
231
+ return if skipped.discarded.zero?
232
+
233
+ logger.error "Discarded #{skipped.discarded} log message(s) during the flush at exit, after the " \
234
+ "destination failed, for appender: #{appender.name}", skipped.exception
235
+ skipped.discarded = 0
236
+ end
105
237
  end
106
238
  end