semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -63,7 +63,8 @@ module SemanticLogger
63
63
  #
64
64
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
65
65
  # Override the log level for this appender.
66
- # Default: SemanticLogger.default_level
66
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
67
+ # filters by its own level.
67
68
  #
68
69
  # formatter: [Object|Proc]
69
70
  # An instance of a class that implements #call, or a Proc to be used to format
@@ -99,7 +100,22 @@ module SemanticLogger
99
100
  end
100
101
 
101
102
  # Re-open after process fork
103
+ #
104
+ # Builds a new notifier, which connects to the server over TCP, and then closes the connections of
105
+ # the notifier it replaces, such as after a log rotation. When connecting fails, it raises, and
106
+ # keeps the notifier it has.
107
+ #
108
+ # A child process first forgets the notifier it inherited, without closing it: over TCP, its
109
+ # connection belongs to the parent. So when connecting fails in the child, such as while the server
110
+ # cannot be reached, the appender connects again on its next write, rather than writing through the
111
+ # parent's connection. Over TLS, both processes would then carry on the same TLS session, so that
112
+ # the server could not decrypt what the parent writes next. Ruby closes the child's copy of the
113
+ # socket when it collects the notifier. GELF keeps its sockets to itself, so until then the child
114
+ # holds that copy open, and over plain TCP the server does not see the connection end when the
115
+ # parent closes it.
102
116
  def reopen
117
+ forget_inherited
118
+
103
119
  uri = Utils.parse_url(@url)
104
120
  @server = uri.host
105
121
  @port = uri.port
@@ -109,11 +125,23 @@ module SemanticLogger
109
125
  raise(ArgumentError, "Invalid protocol value: #{@protocol}. Must be :udp or :tcp")
110
126
  end
111
127
 
112
- gelf_options[:protocol] ||= (@protocol == :tcp ? GELF::Protocol::TCP : GELF::Protocol::UDP)
113
- gelf_options[:facility] ||= application
128
+ replaced = @notifier
129
+ build_notifier
130
+ close_notifier(replaced, replaced: true)
131
+ end
114
132
 
115
- @notifier = GELF::Notifier.new(server, port, max_size, gelf_options)
116
- @notifier.collect_file_and_line = false
133
+ # Closes the notifier's TCP connections, such as when the appender is removed, and forgets it, so
134
+ # that the next write builds a new one. An error closing them is reported to the internal logger,
135
+ # rather than raised.
136
+ #
137
+ # In a forked child that has not reopened, such as one that removes the appender before its reopen
138
+ # has run, it forgets the notifier it inherited instead, without closing it: over TCP, its
139
+ # connection belongs to the parent (see #reopen).
140
+ def close
141
+ forget_inherited
142
+ close_notifier(@notifier)
143
+ ensure
144
+ @notifier = nil
117
145
  end
118
146
 
119
147
  # Returns [Hash] of parameters to send
@@ -122,7 +150,7 @@ module SemanticLogger
122
150
 
123
151
  # GELF rejects an empty short_message, such as a message of only ANSI escape codes.
124
152
  h[:short_message] = h.delete(:message).to_s
125
- h[:short_message] = log.exception.message.to_s.strip if h[:short_message].empty? && log.exception
153
+ h[:short_message] = short_message_of(log.exception) if h[:short_message].empty? && log.exception
126
154
  h[:short_message] = "<no-exception-message>" if h[:short_message].empty?
127
155
  h[:level] = logger.level_map[log.level]
128
156
  h[:level_str] = log.level.to_s
@@ -135,8 +163,11 @@ module SemanticLogger
135
163
  # GELF serializes the Hash with #to_json, which raises for Infinity, NaN, invalid UTF-8, or a
136
164
  # structure nested too deeply, and writes a Struct through its #inspect, so such a Hash is
137
165
  # handed over as a copy that JSON can represent (see Utils.json_ready).
166
+ #
167
+ # Builds a notifier first when the appender has none: after #close, or when building one failed as
168
+ # it was reopened after a fork.
138
169
  def log(log)
139
- notifier.notify!(format_log(log) { |hash| Utils.json_ready(hash) })
170
+ (notifier || build_notifier).notify!(format_log(log) { |hash| Utils.json_ready(hash) })
140
171
  true
141
172
  end
142
173
 
@@ -154,9 +185,52 @@ module SemanticLogger
154
185
 
155
186
  private
156
187
 
188
+ # Returns [GELF::Notifier] a new notifier, which connects to the server over TCP, after making it
189
+ # the appender's. Records the process that built it, which owns its connection (see #reopen).
190
+ def build_notifier
191
+ @notifier = GELF::Notifier.new(server, port, max_size, notifier_options)
192
+ @notifier.collect_file_and_line = false
193
+ @pid = ::Process.pid
194
+ @notifier
195
+ end
196
+
197
+ # In a child process, forgets the notifier built in the parent before a fork. See #reopen.
198
+ def forget_inherited
199
+ @notifier = nil unless @pid == ::Process.pid
200
+ end
201
+
202
+ # Closes the TCP connections of the given notifier, if any: the one that a reopen replaced, when
203
+ # replaced: is true. Reports an error to the internal logger as a warning, rather than raising it.
204
+ #
205
+ # GELF::Notifier#close cannot: over TCP it raises NoMethodError, since GELF's TCP transport has no
206
+ # #close, and over UDP it closes the socket that GELF shares among every UDP notifier that sends on
207
+ # the calling thread, where the next one to send fails with IOError. Giving the notifier no address
208
+ # to send to closes the sockets of its TCP transport, and leaves that UDP socket open.
209
+ def close_notifier(notifier, replaced: false)
210
+ notifier&.addresses = []
211
+ rescue StandardError => e
212
+ logger.warn("Could not close the connection to the Graylog server#{' that a reopen replaced' if replaced}", e)
213
+ end
214
+
215
+ # Returns [Hash] a copy of the gelf_options (see Subscriber#copy_options), with the protocol of the url
216
+ # and the application as the facility, unless they name their own, for a new GELF::Notifier.
217
+ def notifier_options
218
+ options = copy_options(gelf_options)
219
+ options[:protocol] ||= (@protocol == :tcp ? GELF::Protocol::TCP : GELF::Protocol::UDP)
220
+ options[:facility] ||= application
221
+ options
222
+ end
223
+
157
224
  def default_formatter
158
225
  SemanticLogger::Formatters::Raw.new(time_format: :seconds, time_key: :timestamp)
159
226
  end
227
+
228
+ # Returns [String] the exception's message, without the whitespace around it, for an entry
229
+ # without a message. Read with Utils.exception_message, so that a #message that raises cannot
230
+ # lose the entry, and converted to valid UTF-8 first, since #strip raises for invalid bytes.
231
+ def short_message_of(exception)
232
+ Utils.encode_utf8_string(Utils.exception_message(exception)).strip
233
+ end
160
234
  end
161
235
  end
162
236
  end
@@ -42,8 +42,14 @@ module SemanticLogger
42
42
  end
43
43
 
44
44
  # Send an error notification to honeybadger
45
+ #
46
+ # Honeybadger replaces each byte of text that is not valid UTF-8 with "?", including binary text
47
+ # that holds UTF-8, such as a response body, so the data is sent as a copy with its text
48
+ # converted to valid UTF-8, as the JSON formatters write it, and a Hash or Array nested too
49
+ # deeply, or that holds itself, cut off there as they cut it off (see Utils.encode_utf8). Any
50
+ # other object is sent as it is, for Honeybadger to clean.
45
51
  def log(log)
46
- context = format_log(log)
52
+ context = format_log(log) { |data| Utils.encode_utf8(data) }
47
53
  if log.exception
48
54
  context.delete(:exception)
49
55
  ::Honeybadger.notify(log.exception, context)
@@ -17,7 +17,7 @@ module SemanticLogger
17
17
  # Parameters
18
18
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
19
19
  # Override the log level for this appender.
20
- # Default: :error
20
+ # Default: :info
21
21
  #
22
22
  # formatter: [Object|Proc|Symbol|Hash]
23
23
  # An instance of a class that implements #call, or a Proc to be used to format
@@ -42,8 +42,14 @@ module SemanticLogger
42
42
  end
43
43
 
44
44
  # Send log to honeybadger events API
45
+ #
46
+ # Honeybadger replaces each byte of text that is not valid UTF-8 with "?", including binary text
47
+ # that holds UTF-8, such as a response body, so the data is sent as a copy with its text
48
+ # converted to valid UTF-8, as the JSON formatters write it, and a Hash or Array nested too
49
+ # deeply, or that holds itself, cut off there as they cut it off (see Utils.encode_utf8). Any
50
+ # other object is sent as it is, for Honeybadger to clean.
45
51
  def log(log)
46
- event = format_log(log).merge(event_type: "log")
52
+ event = format_log(log) { |data| Utils.encode_utf8(data) }.merge(event_type: "log")
47
53
 
48
54
  ::Honeybadger.event(event)
49
55
 
@@ -26,6 +26,18 @@ require "openssl"
26
26
  # url: 'http://localhost:8088/path',
27
27
  # batch: true
28
28
  # )
29
+ #
30
+ # Responses:
31
+ # A response other than 2xx raises ResponseError from #log and #batch. A 4xx other than 408 and
32
+ # 429 rejects the request itself, so #message_error? recognizes it, and an asynchronous appender
33
+ # discards the entries straight away, counted as failed. A batch rejected for what it holds (400,
34
+ # 413, or 422) is first split up and each part sent again, to find the entries at fault; any other
35
+ # such 4xx, such as 401 or 404, would reject any part too, so the whole batch is discarded at once
36
+ # (see #fails_any_batch?). Any other status is a problem with the destination, so an asynchronous
37
+ # appender backs off. After a 408, 429, or 503, which ask to send the request again later, it then
38
+ # sends the entries again, up to QueueProcessor::HeldWrite::MAX_RESENDS times, since the server did
39
+ # not store them (see #wrote_nothing?). After any other, such as a 500, 502, 504, or a redirect, it
40
+ # discards them, counted as failed. An appender that is not asynchronous reports the error.
29
41
  module SemanticLogger
30
42
  module Appender
31
43
  class Http < SemanticLogger::Subscriber
@@ -36,9 +48,68 @@ module SemanticLogger
36
48
  # Maximum number of bytes of an error response body to write to the internal logger.
37
49
  MAX_LOGGED_RESPONSE_BODY = 512
38
50
 
51
+ # Raised when the server answers a request with a status other than 2xx. Its message holds
52
+ # the status code, the redacted url, and the start of the response body, for the internal
53
+ # logger.
54
+ class ResponseError < StandardError
55
+ # The statuses that ask the client to send the request again later, after which the server
56
+ # has not stored it: 408 Request Timeout, 429 Too Many Requests, and 503 Service
57
+ # Unavailable. Not a 500, which often comes again for the same request, or a 502 Bad Gateway
58
+ # or 504 Gateway Timeout, which a proxy can answer once the server behind it has stored it.
59
+ RETRY_LATER_CODES = [408, 429, 503].freeze
60
+
61
+ # The 4xx statuses that reject the request for the entries it holds: 400 Bad Request,
62
+ # 413 Content Too Large, and 422 Unprocessable Content. A server can answer 400 for its own
63
+ # configuration too, such as Splunk's HTTP Event Collector for an index it does not know, which
64
+ # then rejects every part of a batch split up to find the entries at fault. A subclass that
65
+ # can tell such an answer from its body says so from #fails_any_batch?, as
66
+ # Appender::SplunkHttp does.
67
+ CONTENT_CODES = [400, 413, 422].freeze
68
+
69
+ # Returns [Integer] the status code of the response.
70
+ attr_reader :code
71
+
72
+ # Returns [String|nil] the body of the response, for a subclass that reads the server's own
73
+ # reason from it, such as Appender::SplunkHttp. Not in the message beyond its start, see
74
+ # MAX_LOGGED_RESPONSE_BODY.
75
+ attr_reader :body
76
+
77
+ def initialize(message, code:, body: nil)
78
+ super(message)
79
+ @code = code
80
+ @body = body
81
+ end
82
+
83
+ # Whether the server rejected the request itself, with a 4xx status other than 408 and
84
+ # 429, so that sending it again cannot succeed. A 408, 429, or 5xx, and a 3xx, which asks to
85
+ # send it elsewhere, are problems with the destination instead.
86
+ def rejected?
87
+ (400..499).cover?(code) && !retry_later?
88
+ end
89
+
90
+ # Whether the server asks to send the request again later (see RETRY_LATER_CODES).
91
+ def retry_later? = RETRY_LATER_CODES.include?(code)
92
+
93
+ # Whether the server rejected the request for the entries it holds (see CONTENT_CODES), so
94
+ # that it may accept a part of them. It rejects a request with any other 4xx (see
95
+ # #rejected?), such as 401 Unauthorized, 403 Forbidden, or 404 Not Found, whatever it holds.
96
+ def content_rejected?
97
+ CONTENT_CODES.include?(code)
98
+ end
99
+ end
100
+
39
101
  # Describes a valid url in the error raised for an invalid one.
40
102
  URL_FORMAT_HINT = "Should be similar to: 'http://hostname:port/path'".freeze
41
103
 
104
+ # The errors raised when the appender cannot connect to the server, so that the request was
105
+ # never sent: the server's name does not resolve, it cannot be reached, or it does not accept
106
+ # the connection, refusing it or within the open_timeout. Once connected, an error such as a
107
+ # read timeout or a reset connection may come after the server took the request.
108
+ NOT_SENT_ERRORS = [
109
+ SocketError, Net::OpenTimeout, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
110
+ Errno::EADDRNOTAVAIL
111
+ ].freeze
112
+
42
113
  # Create HTTP(S) log appender
43
114
  #
44
115
  # Parameters:
@@ -93,12 +164,13 @@ module SemanticLogger
93
164
  #
94
165
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
95
166
  # Override the log level for this appender.
96
- # Default: SemanticLogger.default_level
167
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
168
+ # filters by its own level.
97
169
  #
98
170
  # formatter: [Object|Proc]
99
171
  # An instance of a class that implements #call, or a Proc to be used to format
100
172
  # the output from this appender
101
- # Default: Use the built-in formatter (See: #call)
173
+ # Default: SemanticLogger::Formatters::Json
102
174
  #
103
175
  # filter: [Regexp|Proc]
104
176
  # RegExp: Only include log messages where the class name matches the supplied.
@@ -128,7 +200,8 @@ module SemanticLogger
128
200
  &)
129
201
  @url = url
130
202
  @proxy_url = proxy_url
131
- @ssl_options = ssl
203
+ # A copy, to which an https url adds use_ssl: and verify_mode: (see Subscriber#copy_options).
204
+ @ssl_options = ssl && copy_options(ssl)
132
205
  @username = username
133
206
  @password = password
134
207
  @compress = compress
@@ -178,33 +251,29 @@ module SemanticLogger
178
251
  reopen
179
252
  end
180
253
 
181
- # Re-open after process fork
254
+ # Closes the connection, if it is open, and starts a new one, such as after a log rotation, or a
255
+ # fork. When starting it fails, such as while the server cannot be reached, it raises, and the
256
+ # next request starts it (see #process_request).
257
+ #
258
+ # After a fork, the child first forgets the connection it inherited, without finishing it: it
259
+ # belongs to the parent. Finishing an https connection sends the server a TLS close_notify, which
260
+ # ends the connection for the parent too. Ruby closes the child's copy of the socket when it
261
+ # collects it, which leaves the parent's open. Net::HTTP keeps its socket to itself, so until then
262
+ # the child holds that copy open, and over plain http the server does not see the connection end
263
+ # when the parent closes it.
182
264
  def reopen
183
- # Close open connection if any
184
- begin
185
- @http&.finish
186
- rescue IOError
187
- nil
188
- end
189
-
190
- @http = if @proxy_uri
191
- Net::HTTP.new(server, port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password)
192
- else
193
- Net::HTTP.new(server, port, @proxy_url)
194
- end
195
-
196
- if @ssl_options
197
- @http.methods.grep(/\A(\w+)=\z/) do |meth|
198
- key = Regexp.last_match(1).to_sym
199
- @ssl_options.key?(key) || next
200
- @http.__send__(meth, @ssl_options[key])
201
- end
202
- end
265
+ finish_connection
266
+ build_http.start
267
+ end
203
268
 
204
- @http.open_timeout = @open_timeout
205
- @http.read_timeout = @read_timeout
206
- @http.continue_timeout = @continue_timeout
207
- @http.start
269
+ # Closes the connection, if it is open, such as when the appender is removed, or add_appender
270
+ # fails to add it. The next request opens it again, see #process_request.
271
+ #
272
+ # In a forked child that has not reopened, such as one that removes the appender before its
273
+ # reopen has run, it forgets the connection it inherited instead, without finishing it, since it
274
+ # belongs to the parent (see #reopen), and the next request starts a new one.
275
+ def close
276
+ finish_connection
208
277
  end
209
278
 
210
279
  # Forward log messages to HTTP Server
@@ -229,11 +298,73 @@ module SemanticLogger
229
298
  false
230
299
  end
231
300
 
232
- # The whole batch is formatted into a single request before anything is sent.
301
+ # The whole batch is formatted into a single request before anything is sent, and the
302
+ # server answers for the request as a whole. So a batch that a format error or a rejected
303
+ # request fails (see #message_error?) is split up and sent again, to find the entries at
304
+ # fault.
233
305
  def batch_all_or_nothing?
234
306
  true
235
307
  end
236
308
 
309
+ # The #batch above formats the whole batch with #format_batch, so it needs a formatter that can
310
+ # format a batch, such as the default one, rather than a formatter block. See
311
+ # Subscriber#batch_refusal.
312
+ #
313
+ # Answers only while that #batch is the one in use (see Subscriber#batch_defined_by?). A
314
+ # subclass that replaces it answers for its own, as Loki and Signalfx do, and is otherwise
315
+ # batched with any formatter, as before, such as one whose #batch formats each log message with
316
+ # #format_log. One whose #batch calls super, or that a prepended module wraps, still formats the
317
+ # whole batch with #format_batch, so it defines its own #batch_refusal, which returns
318
+ # #format_batch_refusal, as the Custom appenders section of docs/config.md shows.
319
+ def batch_refusal
320
+ format_batch_refusal if batch_defined_by?(Http)
321
+ end
322
+
323
+ # Whether the exception, raised by #log or #batch, was caused by the entries sent, rather
324
+ # than by the destination (see Subscriber#message_error?). That includes a request the
325
+ # server rejected (see ResponseError#rejected?), since sending it again cannot succeed.
326
+ def message_error?(exception)
327
+ (exception.is_a?(ResponseError) && exception.rejected?) || super
328
+ end
329
+
330
+ # Whether the exception, a message error, would fail any batch, whatever entries it holds (see
331
+ # Subscriber#fails_any_batch?): the server rejected the request (see ResponseError#rejected?)
332
+ # other than for the entries it holds (see ResponseError#content_rejected?), such as for its
333
+ # credentials or its url, and would reject every part of the batch too. So the batch is
334
+ # discarded at once, rather than split up. A batch that could not be formatted, or that the
335
+ # server rejected for its entries, is split up to find the entries at fault.
336
+ def fails_any_batch?(exception)
337
+ exception.is_a?(ResponseError) && exception.rejected? && !exception.content_rejected?
338
+ end
339
+
340
+ # Whether the request that raised the exception from #log or #batch stored none of its
341
+ # entries, and is worth sending again later (see Subscriber#wrote_nothing?): the server asked
342
+ # to send it again later (see ResponseError#retry_later?), or it was never sent (see
343
+ # #not_sent?). So an asynchronous appender sends the entries again once it has backed off.
344
+ #
345
+ # Decided from the class of the exception alone. Its cause, which Ruby sets to the error being
346
+ # handled when it was raised, such as a format error while a batch is split, says nothing about
347
+ # this request; #process_request raises the error of the request, not one from reconnecting.
348
+ #
349
+ # A subclass whose #log or #batch sends more than one request must override this to return
350
+ # false: a failure of a later request would otherwise have the entries of the earlier ones,
351
+ # which the server stored, sent again.
352
+ def wrote_nothing?(exception)
353
+ return exception.retry_later? if exception.is_a?(ResponseError)
354
+
355
+ not_sent?(exception)
356
+ end
357
+
358
+ # Whether the request that raised the exception from #log or #batch was never sent, because
359
+ # the appender could not connect (see NOT_SENT_ERRORS), so that the server has not answered it
360
+ # (see Subscriber#not_sent?). An asynchronous appender then sends the entries again without
361
+ # counting it towards QueueProcessor::HeldWrite::MAX_RESENDS, so that an outage loses none of
362
+ # them until the worker thread gives up after async_max_retries. Decided from the class of the
363
+ # exception alone, as for #wrote_nothing?.
364
+ def not_sent?(exception)
365
+ NOT_SENT_ERRORS.any? { |error_class| exception.is_a?(error_class) }
366
+ end
367
+
237
368
  # Returns [String] the scheme, host, and any non-default port of the url. Safe to log.
238
369
  # See Utils.redacted_url.
239
370
  def redacted_url
@@ -248,6 +379,47 @@ module SemanticLogger
248
379
 
249
380
  private
250
381
 
382
+ # Returns [Net::HTTP] a new connection, not yet started, after making it the appender's. Records
383
+ # the process that built it, which owns its connection (see #reopen).
384
+ def build_http
385
+ @http = if @proxy_uri
386
+ Net::HTTP.new(server, port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password)
387
+ else
388
+ Net::HTTP.new(server, port, @proxy_url)
389
+ end
390
+
391
+ if @ssl_options
392
+ @http.methods.grep(/\A(\w+)=\z/) do |meth|
393
+ key = Regexp.last_match(1).to_sym
394
+ @ssl_options.key?(key) || next
395
+ @http.__send__(meth, @ssl_options[key])
396
+ end
397
+ end
398
+
399
+ @http.open_timeout = @open_timeout
400
+ @http.read_timeout = @read_timeout
401
+ @http.continue_timeout = @continue_timeout
402
+ @pid = ::Process.pid
403
+ @http
404
+ end
405
+
406
+ # Finishes the connection, if it is open. In a child process, first forgets the connection
407
+ # started in the parent before a fork, without finishing it. See #reopen.
408
+ #
409
+ # Used by #reopen as well as #close, rather than #reopen calling #close, so that a subclass's
410
+ # #close is not run by a reopen.
411
+ def finish_connection
412
+ forget_inherited
413
+ @http&.finish
414
+ rescue IOError
415
+ nil
416
+ end
417
+
418
+ # In a child process, forgets the connection started in the parent before a fork. See #reopen.
419
+ def forget_inherited
420
+ @http = nil unless @pid == ::Process.pid
421
+ end
422
+
251
423
  # Returns [URI] the parsed url, without the url in the error raised for an invalid one.
252
424
  def parse_url(url)
253
425
  Utils.parse_url(url, hint: URL_FORMAT_HINT)
@@ -284,9 +456,16 @@ module SemanticLogger
284
456
  # the server ingests log data, and servers reject it on a maintenance request.
285
457
  # Appender::ElasticsearchHttp#delete_all deletes an index, and Elasticsearch fails
286
458
  # the request with a 400 when it carries unrecognized parameters.
459
+ #
460
+ # A maintenance request is not log data written by the worker thread of an asynchronous
461
+ # appender, so a response other than 2xx is reported to the internal logger, and false
462
+ # returned, rather than raised.
287
463
  def delete(request_uri = path)
288
464
  request = Net::HTTP::Delete.new(request_uri, @header)
289
465
  process_request(request)
466
+ rescue ResponseError => e
467
+ logger.error(e.message)
468
+ false
290
469
  end
291
470
 
292
471
  # Applies the query string of the configured url, if any, to the uri of a request
@@ -306,26 +485,50 @@ module SemanticLogger
306
485
  end
307
486
 
308
487
  # Process HTTP Request
488
+ #
489
+ # Returns true when the server answers with a 2xx status. Raises ResponseError for any other
490
+ # status, so that the entries are counted as failed rather than written. Its message names
491
+ # only the redacted url, since the url can hold credentials, and the start of the body.
492
+ #
493
+ # After a RuntimeError, such as a read timeout, it connects again for the next request, and
494
+ # raises the error of this one even when connecting again fails, since only that error says
495
+ # whether the server may have stored the request (see #wrote_nothing?). Not after a request it
496
+ # could not send (see #not_sent?), such as a Net::OpenTimeout: connecting again straight away
497
+ # would fail the same way, so that a server that cannot be reached would hold up each request
498
+ # for twice its open_timeout.
309
499
  def process_request(request, body = nil)
310
500
  if body
311
501
  request.body = compress ? compress_data(body) : body
312
502
  end
313
503
  request.basic_auth(@username, @password) if @username
314
- response = @http.request(request)
315
- if response.is_a?(Net::HTTPSuccess)
316
- true
317
- else
318
- # Failures are logged to the global semantic logger failsafe logger (Usually stderr or file)
319
- logger.error(
320
- "Bad HTTP response from: #{redacted_url} code: #{response.code}, " \
321
- "#{response.body.to_s[0, MAX_LOGGED_RESPONSE_BODY].inspect}"
322
- )
323
- false
324
- end
504
+ # Net::HTTP sends a request on a connection that is not started, such as one that #close
505
+ # closed, on a new connection that it closes after the response. Starting it keeps the
506
+ # connection for the requests that follow, such as once a removed appender is added again.
507
+ # A new one is built first when the appender has none, since #close forgot the one it
508
+ # inherited in a forked child.
509
+ http = @http || build_http
510
+ http.start unless http.started?
511
+ response = http.request(request)
512
+ return true if response.is_a?(Net::HTTPSuccess)
513
+
514
+ raise ResponseError.new(
515
+ "Bad HTTP response from: #{redacted_url} code: #{response.code}, " \
516
+ "#{response.body.to_s[0, MAX_LOGGED_RESPONSE_BODY].inspect}",
517
+ code: response.code.to_i,
518
+ body: response.body
519
+ )
325
520
  rescue RuntimeError => e
326
- reopen
521
+ reconnect unless not_sent?(e)
327
522
  raise e
328
523
  end
524
+
525
+ # Connects again after a request failed. A failure to connect is reported to the internal
526
+ # logger, and left to the next request, which connects again when the connection is not open.
527
+ def reconnect
528
+ reopen
529
+ rescue StandardError => e
530
+ logger.warn("Could not connect again to #{redacted_url}", e)
531
+ end
329
532
  end
330
533
  end
331
534
  end
@@ -1,3 +1,7 @@
1
+ # Loaded now rather than autoloaded, since #log writes with it on the calling thread when this is the
2
+ # internal logger, which can be in a signal trap, where Ruby does not allow a require.
3
+ require "semantic_logger/appender/text_output"
4
+
1
5
  # File appender
2
6
  #
3
7
  # Writes log messages to a file or open iostream
@@ -13,7 +17,8 @@ module SemanticLogger
13
17
  #
14
18
  # :level [:trace | :debug | :info | :warn | :error | :fatal]
15
19
  # Override the log level for this appender.
16
- # Default: SemanticLogger.default_level
20
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
21
+ # filters by its own level.
17
22
  #
18
23
  # :formatter: [Object|Proc]
19
24
  # An instance of a class that implements #call, or a Proc to be used to format
@@ -29,7 +34,7 @@ module SemanticLogger
29
34
  # Example
30
35
  # require "semantic_logger"
31
36
  #
32
- # # Enable trace level logging
37
+ # # Log :info and above
33
38
  # SemanticLogger.default_level = :info
34
39
  #
35
40
  # # Log to screen
@@ -47,10 +52,10 @@ module SemanticLogger
47
52
  end
48
53
 
49
54
  def log(log)
50
- # Since only one appender thread will be writing to the file at a time
51
- # it is not necessary to protect access to the file with a semaphore
52
- # Allow this logger to filter out log levels lower than it's own
53
- @io.write(format_log(log) << "\n")
55
+ # Since only one appender thread writes to the IO at a time, it is not necessary to protect
56
+ # access to it. The line is made as part of the formatting, so that a formatter's output that
57
+ # cannot be one, such as nil, is taken for the entry's error.
58
+ TextOutput.write(@io, format_log(log) { |text| TextOutput.line(text) })
54
59
  true
55
60
  end
56
61