semantic_logger 5.0.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +20 -10
  3. data/Rakefile +40 -0
  4. data/docs/api.md +506 -0
  5. data/docs/appenders.md +1074 -0
  6. data/docs/config.md +707 -0
  7. data/docs/index.md +223 -0
  8. data/docs/log.md +116 -0
  9. data/docs/metrics.md +171 -0
  10. data/docs/operations.md +446 -0
  11. data/docs/rails.md +890 -0
  12. data/docs/security.md +286 -0
  13. data/docs/testing.md +287 -0
  14. data/docs/upgrading.md +697 -0
  15. data/lib/semantic_logger/appender/async.rb +18 -5
  16. data/lib/semantic_logger/appender/bugsnag.rb +1 -1
  17. data/lib/semantic_logger/appender/cloudwatch_logs.rb +126 -29
  18. data/lib/semantic_logger/appender/elasticsearch.rb +2 -5
  19. data/lib/semantic_logger/appender/elasticsearch_base.rb +48 -23
  20. data/lib/semantic_logger/appender/elasticsearch_http.rb +15 -8
  21. data/lib/semantic_logger/appender/file.rb +141 -28
  22. data/lib/semantic_logger/appender/graylog.rb +22 -6
  23. data/lib/semantic_logger/appender/honeybadger.rb +1 -1
  24. data/lib/semantic_logger/appender/honeybadger_insights.rb +1 -1
  25. data/lib/semantic_logger/appender/http.rb +70 -11
  26. data/lib/semantic_logger/appender/io.rb +1 -1
  27. data/lib/semantic_logger/appender/kafka.rb +26 -12
  28. data/lib/semantic_logger/appender/loki.rb +13 -3
  29. data/lib/semantic_logger/appender/mongodb.rb +20 -1
  30. data/lib/semantic_logger/appender/new_relic.rb +7 -1
  31. data/lib/semantic_logger/appender/new_relic_logs.rb +11 -12
  32. data/lib/semantic_logger/appender/open_telemetry.rb +4 -3
  33. data/lib/semantic_logger/appender/opensearch.rb +2 -1
  34. data/lib/semantic_logger/appender/rabbitmq.rb +10 -4
  35. data/lib/semantic_logger/appender/sentry.rb +6 -1
  36. data/lib/semantic_logger/appender/sentry_ruby.rb +91 -17
  37. data/lib/semantic_logger/appender/splunk.rb +10 -1
  38. data/lib/semantic_logger/appender/syslog.rb +66 -25
  39. data/lib/semantic_logger/appender/tcp.rb +10 -4
  40. data/lib/semantic_logger/appender/udp.rb +11 -1
  41. data/lib/semantic_logger/appender/wrapper.rb +3 -2
  42. data/lib/semantic_logger/appender.rb +6 -3
  43. data/lib/semantic_logger/appenders.rb +11 -2
  44. data/lib/semantic_logger/base.rb +82 -32
  45. data/lib/semantic_logger/exception_details.rb +50 -0
  46. data/lib/semantic_logger/formatters/base.rb +52 -6
  47. data/lib/semantic_logger/formatters/color.rb +22 -3
  48. data/lib/semantic_logger/formatters/default.rb +28 -6
  49. data/lib/semantic_logger/formatters/logfmt.rb +155 -21
  50. data/lib/semantic_logger/formatters/loki.rb +26 -22
  51. data/lib/semantic_logger/formatters/one_line.rb +67 -3
  52. data/lib/semantic_logger/formatters/open_telemetry.rb +16 -17
  53. data/lib/semantic_logger/formatters/pattern.rb +11 -5
  54. data/lib/semantic_logger/formatters/raw.rb +4 -5
  55. data/lib/semantic_logger/formatters/syslog.rb +110 -17
  56. data/lib/semantic_logger/formatters/syslog_cee.rb +29 -15
  57. data/lib/semantic_logger/levels.rb +10 -1
  58. data/lib/semantic_logger/log.rb +183 -37
  59. data/lib/semantic_logger/loggable.rb +26 -3
  60. data/lib/semantic_logger/metric/new_relic.rb +1 -1
  61. data/lib/semantic_logger/metric/signalfx.rb +12 -8
  62. data/lib/semantic_logger/metric/statsd.rb +15 -7
  63. data/lib/semantic_logger/queue_processor/request.rb +55 -0
  64. data/lib/semantic_logger/queue_processor/throttled_counter.rb +46 -0
  65. data/lib/semantic_logger/queue_processor.rb +170 -68
  66. data/lib/semantic_logger/reporters/minitest.rb +4 -4
  67. data/lib/semantic_logger/semantic_logger.rb +15 -2
  68. data/lib/semantic_logger/subscriber.rb +96 -10
  69. data/lib/semantic_logger/sync_processor.rb +11 -0
  70. data/lib/semantic_logger/test/capture_log_events.rb +9 -6
  71. data/lib/semantic_logger/utils.rb +244 -16
  72. data/lib/semantic_logger/version.rb +1 -1
  73. data/lib/semantic_logger.rb +1 -1
  74. metadata +33 -6
  75. data/lib/semantic_logger/core_ext/thread.rb +0 -30
@@ -10,6 +10,15 @@ module SemanticLogger
10
10
  :reopen_period, :reopen_count, :reopen_size, :permissions
11
11
  attr_reader :log_count, :log_size, :current_file_name, :reopen_at
12
12
 
13
+ # With permissions, the log file is opened without following a symbolic link, so that a link
14
+ # planted in the log directory can neither create nor open the file it points at, without
15
+ # blocking, so that a planted FIFO fails to open instead of waiting for a reader, and without
16
+ # becoming the controlling terminal.
17
+ SECURE_OPEN_FLAGS = ::File::NOFOLLOW | ::File::NONBLOCK | ::File::NOCTTY
18
+ # Seconds to wait before opening a refused log file again.
19
+ REFUSED_RETRY_INTERVAL = 5
20
+ private_constant :SECURE_OPEN_FLAGS, :REFUSED_RETRY_INTERVAL
21
+
13
22
  # Create an appender to log to a named file.
14
23
  #
15
24
  # Parameters
@@ -77,7 +86,12 @@ module SemanticLogger
77
86
  # Octal file permissions to apply to the log file.
78
87
  # Log files frequently contain sensitive information, so restrict access
79
88
  # by supplying for example `0o640` (owner read/write, group read).
80
- # Applied both when the file is created and to an existing log file.
89
+ # Applied both when the file is created and to an existing log file, which must be
90
+ # a regular file owned by the process, with a single hard link, that the file_name
91
+ # names directly rather than through a symbolic link. Nothing is written to any
92
+ # other file, or when the file cannot be opened: the appender raises IOError when it
93
+ # is created, and when the file is later reopened, it reports why and writes nothing
94
+ # until the file is fixed, trying again every few seconds.
81
95
  # Default: nil (use the process umask, the standard Ruby behavior)
82
96
  #
83
97
  # :retry_count [Integer]
@@ -150,43 +164,37 @@ module SemanticLogger
150
164
  @reopen_at = nil
151
165
 
152
166
  super(**args, &)
167
+
168
+ # Open the file at creation time so that misconfiguration (bad directory,
169
+ # insufficient permissions, ...) raises immediately in SemanticLogger.add_appender,
170
+ # instead of surfacing later on the appender thread via the internal logger. With
171
+ # permissions, a refused log file raises IOError (see #open_with_permissions).
172
+ raise(IOError, refusal_message, cause: @refusal_cause) unless reopen
153
173
  end
154
174
 
155
175
  # After forking an active process call #reopen to re-open
156
176
  # open the file handles etc to resources.
157
177
  def reopen
158
- begin
159
- @file&.close
160
- rescue StandardError
161
- nil
162
- end
178
+ close_quietly(@file)
163
179
 
164
180
  self.current_file_name = apply_format_directives(file_name)
165
- if ::File.directory?(file_name)
181
+ # With permissions, a directory is refused when it fails to open, like any other file.
182
+ if !permissions && ::File.directory?(current_file_name)
166
183
  raise(ArgumentError, "The supplied log file_name: #{current_file_name} is already a directory.")
167
184
  end
168
185
 
169
186
  self.log_count = 0
170
- if append && reopen_size && ::File.exist?(current_file_name)
171
- self.log_size = ::File.size(current_file_name)
172
- self.log_size = 0 if log_size >= reopen_size
173
- else
174
- self.log_size = 0
175
- end
187
+ # File.size? rather than File.size, since the file can be removed at any time.
188
+ size = ::File.size?(current_file_name) if append && reopen_size
189
+ self.log_size = size && size < reopen_size ? size : 0
176
190
 
177
191
  self.reopen_at = reopen_period ? next_reopen_period(reopen_period) : nil
178
192
 
179
193
  options = ::File::WRONLY | ::File::CREAT
180
194
  options |= ::File::APPEND if append
181
- @file =
182
- if permissions
183
- ::File.open(current_file_name, options, permissions)
184
- else
185
- ::File.open(current_file_name, options)
186
- end
187
- # File.open only applies the permissions when creating the file, so also
188
- # enforce them on an already existing log file.
189
- @file.chmod(permissions) if permissions
195
+ @file = permissions ? open_with_permissions(options) : ::File.open(current_file_name, options)
196
+ return unless @file
197
+
190
198
  # Force all log entries to write immediately without buffering
191
199
  # Allows multiple processes to write to the same log file simultaneously
192
200
  @file.sync = true
@@ -198,18 +206,23 @@ module SemanticLogger
198
206
  # Since only one appender thread will be writing to the file at a time
199
207
  # it is not necessary to protect access to the file with a semaphore.
200
208
  def log(log)
201
- reopen if time_to_reopen?
209
+ # Read once, since #reopen can be called from another thread meanwhile.
210
+ file = time_to_reopen? ? reopen : @file
211
+ return refused unless file
202
212
 
203
- count = 0
213
+ # Formatted once, outside the retry below: reopening the file cannot fix the content.
214
+ message = format_log(log) << "\n"
215
+ count = 0
204
216
  begin
205
- message = formatter.call(log, self) << "\n"
206
- @file.write(message)
217
+ file.write(message)
207
218
  @log_count += 1
208
219
  @log_size += message.size
209
220
  rescue StandardError => e
210
221
  if count < retry_count
211
222
  count += 1
212
- reopen
223
+ file = reopen
224
+ return refused unless file
225
+
213
226
  retry
214
227
  end
215
228
  raise(e)
@@ -223,18 +236,118 @@ module SemanticLogger
223
236
  @file&.flush
224
237
  end
225
238
 
239
+ # See Subscriber#inspect.
240
+ def inspect_details
241
+ "file_name=#{file_name.inspect}"
242
+ end
243
+
226
244
  private
227
245
 
228
246
  attr_writer :log_count, :log_size, :current_file_name, :reopen_at
229
247
 
248
+ # Opens the log file and applies the permissions to it, when it is safe to do so (see
249
+ # #permissions_refusal). Returns the open file, or nil after closing it and recording why it is
250
+ # refused, so that nothing is written to it.
251
+ #
252
+ # Changing the permissions of a file that another name links to could loosen them on an
253
+ # unrelated file, such as a private key linked into a shared log directory. Writing log messages
254
+ # to a file that this process does not own, or that another name links to, would let whoever
255
+ # controls it read them, despite the permissions asked for. So with permissions, the appender
256
+ # fails closed: it raises when it is created (see #initialize), and when the file is later
257
+ # reopened, on a rotation, a fork, or a write retry, it stops writing and reports why (see
258
+ # #refused) until the file is fixed (see #refused_retry_due?).
259
+ def open_with_permissions(options)
260
+ file = ::File.open(current_file_name, options | SECURE_OPEN_FLAGS, permissions)
261
+ reason = permissions_refusal(file)
262
+ if reason
263
+ close_quietly(file)
264
+ return refuse(reason)
265
+ end
266
+
267
+ file.chmod(permissions)
268
+ @refusal = @refusal_cause = @reported_refusal = nil
269
+ file
270
+ rescue SystemCallError => e
271
+ close_quietly(file)
272
+ refuse(open_refusal(e), e)
273
+ end
274
+
275
+ # Records why the log file is refused, and returns nil.
276
+ def refuse(reason, cause = nil)
277
+ @refusal = reason
278
+ @refusal_cause = cause
279
+ @refused_retry_at = monotonic_time + REFUSED_RETRY_INTERVAL
280
+ nil
281
+ end
282
+
283
+ # Reports why log messages are not being written to the log file, once for each file and
284
+ # reason, and returns false.
285
+ def refused
286
+ refusal = [current_file_name, @refusal]
287
+ unless @reported_refusal == refusal
288
+ @reported_refusal = refusal
289
+ logger.error(refusal_message)
290
+ end
291
+ false
292
+ end
293
+
294
+ def refusal_message
295
+ "Not writing to the log file #{current_file_name}: #{@refusal}"
296
+ end
297
+
298
+ # Returns why the log file could not be opened with permissions.
299
+ def open_refusal(error)
300
+ return "it is a symbolic link" if error.is_a?(Errno::ELOOP) && ::File.symlink?(current_file_name)
301
+
302
+ "it could not be opened (#{error.message})"
303
+ end
304
+
305
+ # Returns why the permissions must not be applied to the open log file, nor log messages
306
+ # written to it, or nil when they can.
307
+ #
308
+ # It must be a regular file owned by this process, with a single hard link, since a hard link
309
+ # planted in the log directory is otherwise indistinguishable from the file it links to. The
310
+ # path must also still name it, so that swapping the path after opening cannot redirect the
311
+ # log messages.
312
+ def permissions_refusal(file)
313
+ stat = file.stat
314
+ path_stat = ::File.lstat(current_file_name)
315
+ return "it is not a regular file" unless stat.file?
316
+ return "it is not owned by this process" unless stat.uid == ::Process.euid
317
+ return "it has more than one hard link" unless stat.nlink == 1
318
+ return "it changed after it was opened" unless path_stat.dev == stat.dev && path_stat.ino == stat.ino
319
+
320
+ nil
321
+ rescue SystemCallError => e
322
+ "it could not be checked (#{e.message})"
323
+ end
324
+
325
+ def close_quietly(file)
326
+ file&.close
327
+ rescue StandardError
328
+ nil
329
+ end
330
+
331
+ def monotonic_time
332
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
333
+ end
334
+
230
335
  def time_to_reopen?
231
- return true unless @file
336
+ return refused_retry_due? unless @file
232
337
 
233
338
  (reopen_count.positive? && (log_count >= reopen_count)) ||
234
339
  (reopen_size.positive? && (log_size >= reopen_size)) ||
235
340
  (reopen_at && (Time.now > reopen_at))
236
341
  end
237
342
 
343
+ # A refused log file is opened again every few seconds, and at the next scheduled reopen,
344
+ # rather than for every message.
345
+ def refused_retry_due?
346
+ return true unless @refused_retry_at
347
+
348
+ monotonic_time >= @refused_retry_at || (reopen_at && Time.now > reopen_at)
349
+ end
350
+
238
351
  def apply_format_directives(file_name)
239
352
  return file_name unless file_name.include?("%")
240
353
 
@@ -100,7 +100,7 @@ module SemanticLogger
100
100
 
101
101
  # Re-open after process fork
102
102
  def reopen
103
- uri = URI.parse(@url)
103
+ uri = Utils.parse_url(@url)
104
104
  @server = uri.host
105
105
  @port = uri.port
106
106
  @protocol = uri.scheme.to_sym
@@ -120,10 +120,10 @@ module SemanticLogger
120
120
  def call(log, logger)
121
121
  h = default_formatter.call(log, logger)
122
122
 
123
- h[:short_message] = h.delete(:message)
124
- if h[:short_message].nil?
125
- h[:short_message] = log.exception.nil? ? "<no-exception-message>" : log.exception.message
126
- end
123
+ # GELF rejects an empty short_message, such as a message of only ANSI escape codes.
124
+ h[:short_message] = h.delete(:message).to_s
125
+ h[:short_message] = log.exception.message.to_s.strip if h[:short_message].empty? && log.exception
126
+ h[:short_message] = "<no-exception-message>" if h[:short_message].empty?
127
127
  h[:level] = logger.level_map[log.level]
128
128
  h[:level_str] = log.level.to_s
129
129
  h[:duration_str] = h.delete(:duration)
@@ -131,11 +131,27 @@ module SemanticLogger
131
131
  end
132
132
 
133
133
  # Forward log messages
134
+ #
135
+ # GELF serializes the Hash with #to_json, which raises for Infinity, NaN, invalid UTF-8, or a
136
+ # structure nested too deeply, and writes a Struct through its #inspect, so such a Hash is
137
+ # handed over as a copy that JSON can represent (see Utils.json_ready).
134
138
  def log(log)
135
- notifier.notify!(formatter.call(log, self))
139
+ notifier.notify!(format_log(log) { |hash| Utils.json_ready(hash) })
136
140
  true
137
141
  end
138
142
 
143
+ # GELF raises ArgumentError for a log message it cannot send, such as one too large to fit in
144
+ # the maximum number of chunks, which retrying cannot send either. Unavailable servers raise
145
+ # network errors instead.
146
+ def message_error?(exception)
147
+ exception.is_a?(ArgumentError) || super
148
+ end
149
+
150
+ # The url without any credentials. See Subscriber#inspect.
151
+ def inspect_details
152
+ "url=#{Utils.redacted_url(url).inspect}"
153
+ end
154
+
139
155
  private
140
156
 
141
157
  def default_formatter
@@ -43,7 +43,7 @@ module SemanticLogger
43
43
 
44
44
  # Send an error notification to honeybadger
45
45
  def log(log)
46
- context = formatter.call(log, self)
46
+ context = format_log(log)
47
47
  if log.exception
48
48
  context.delete(:exception)
49
49
  ::Honeybadger.notify(log.exception, context)
@@ -43,7 +43,7 @@ module SemanticLogger
43
43
 
44
44
  # Send log to honeybadger events API
45
45
  def log(log)
46
- event = formatter.call(log, self).merge(event_type: "log")
46
+ event = format_log(log).merge(event_type: "log")
47
47
 
48
48
  ::Honeybadger.event(event)
49
49
 
@@ -31,7 +31,13 @@ module SemanticLogger
31
31
  class Http < SemanticLogger::Subscriber
32
32
  attr_accessor :username, :compress, :header,
33
33
  :open_timeout, :read_timeout, :continue_timeout
34
- attr_reader :http, :url, :server, :port, :path, :ssl_options, :proxy_url
34
+ attr_reader :http, :url, :server, :port, :path, :query, :ssl_options, :proxy_url
35
+
36
+ # Maximum number of bytes of an error response body to write to the internal logger.
37
+ MAX_LOGGED_RESPONSE_BODY = 512
38
+
39
+ # Describes a valid url in the error raised for an invalid one.
40
+ URL_FORMAT_HINT = "Should be similar to: 'http://hostname:port/path'".freeze
35
41
 
36
42
  # Create HTTP(S) log appender
37
43
  #
@@ -42,6 +48,9 @@ module SemanticLogger
42
48
  # To enable SSL include https in the URL.
43
49
  # Example: https://example.com/some_path
44
50
  # verify_mode will default: OpenSSL::SSL::VERIFY_PEER
51
+ # A query string is kept and sent with every request that submits log data,
52
+ # for the log servers that are configured through it.
53
+ # Example: http://example.com/some_path?source=my_app
45
54
  #
46
55
  # application: [String]
47
56
  # Name of this application to appear in log messages.
@@ -136,11 +145,10 @@ module SemanticLogger
136
145
  }.merge(header)
137
146
  @header["Content-Encoding"] = "gzip" if @compress
138
147
 
139
- uri = URI.parse(@url)
148
+ uri = parse_url(@url)
140
149
  @server = uri.host
141
- unless @server
142
- raise(ArgumentError, "Invalid format for :url: #{@url.inspect}. Should be similar to: 'http://hostname:port/path'")
143
- end
150
+ # The url is left out of the message, since it can hold a user and password.
151
+ raise(ArgumentError, "Invalid format for :url, it has no host. #{URL_FORMAT_HINT}") unless @server
144
152
 
145
153
  @port = uri.port
146
154
  @username = uri.user if !@username && uri.user
@@ -148,6 +156,11 @@ module SemanticLogger
148
156
  @path = uri.path
149
157
  # Path cannot be empty
150
158
  @path = "/" if @path == ""
159
+ # Query string of the url, applied to every request, see #uri_with_query.
160
+ # Kept separate from the path, because subclasses build their own request paths
161
+ # from it. For example Appender::ElasticsearchHttp appends the index name.
162
+ @query = uri.query
163
+ @query = nil if @query == ""
151
164
 
152
165
  if uri.scheme == "https"
153
166
  @ssl_options[:use_ssl] = true
@@ -157,7 +170,7 @@ module SemanticLogger
157
170
  @port ||= HTTP.http_default_port
158
171
  end
159
172
 
160
- @proxy_uri = URI.parse(@proxy_url) if @proxy_url && @proxy_url != :ENV
173
+ @proxy_uri = Utils.parse_url(@proxy_url, option: ":proxy_url") if @proxy_url && @proxy_url != :ENV
161
174
 
162
175
  @http = nil
163
176
 
@@ -196,7 +209,7 @@ module SemanticLogger
196
209
 
197
210
  # Forward log messages to HTTP Server
198
211
  def log(log)
199
- message = formatter.call(log, self)
212
+ message = format_log(log)
200
213
  logger.trace(message)
201
214
  post(message)
202
215
  end
@@ -204,7 +217,7 @@ module SemanticLogger
204
217
  # Forward a batch of log messages to the HTTP Server in a single request.
205
218
  # Only used when the appender is created with `batch: true`.
206
219
  def batch(logs)
207
- message = formatter.batch(logs, self)
220
+ message = format_batch(logs)
208
221
  logger.trace(message)
209
222
  post(message)
210
223
  end
@@ -216,8 +229,30 @@ module SemanticLogger
216
229
  false
217
230
  end
218
231
 
232
+ # The whole batch is formatted into a single request before anything is sent.
233
+ def batch_all_or_nothing?
234
+ true
235
+ end
236
+
237
+ # Returns [String] the scheme, host, and any non-default port of the url. Safe to log.
238
+ # See Utils.redacted_url.
239
+ def redacted_url
240
+ @redacted_url ||= Utils.redacted_url(url)
241
+ end
242
+
243
+ # Only the redacted url, since the instance variables hold the password, the authorization
244
+ # headers, and any credential a subclass adds. See Subscriber#inspect.
245
+ def inspect_details
246
+ "url=#{redacted_url.inspect}"
247
+ end
248
+
219
249
  private
220
250
 
251
+ # Returns [URI] the parsed url, without the url in the error raised for an invalid one.
252
+ def parse_url(url)
253
+ Utils.parse_url(url, hint: URL_FORMAT_HINT)
254
+ end
255
+
221
256
  # Use JSON Formatter by default
222
257
  def default_formatter
223
258
  SemanticLogger::Formatters::Json.new
@@ -233,22 +268,43 @@ module SemanticLogger
233
268
 
234
269
  # HTTP Post
235
270
  def post(body, request_uri = path)
236
- request = Net::HTTP::Post.new(request_uri, @header)
271
+ request = Net::HTTP::Post.new(uri_with_query(request_uri), @header)
237
272
  process_request(request, body)
238
273
  end
239
274
 
240
275
  # HTTP Put
241
276
  def put(body, request_uri = path)
242
- request = Net::HTTP::Put.new(request_uri, @header)
277
+ request = Net::HTTP::Put.new(uri_with_query(request_uri), @header)
243
278
  process_request(request, body)
244
279
  end
245
280
 
246
281
  # HTTP Delete
282
+ #
283
+ # The query string of the configured url is _not_ applied here: it configures how
284
+ # the server ingests log data, and servers reject it on a maintenance request.
285
+ # Appender::ElasticsearchHttp#delete_all deletes an index, and Elasticsearch fails
286
+ # the request with a 400 when it carries unrecognized parameters.
247
287
  def delete(request_uri = path)
248
288
  request = Net::HTTP::Delete.new(request_uri, @header)
249
289
  process_request(request)
250
290
  end
251
291
 
292
+ # Applies the query string of the configured url, if any, to the uri of a request
293
+ # that submits log data.
294
+ #
295
+ # Kept out of `path` on purpose: subclasses build their own request paths from it
296
+ # (Appender::ElasticsearchHttp appends the index name and `_doc` to it), and the
297
+ # query has to stay at the end of the request uri regardless.
298
+ #
299
+ # @param request_uri [String] path of the request, `path` by default.
300
+ # @return [String]
301
+ def uri_with_query(request_uri)
302
+ return request_uri unless @query
303
+
304
+ separator = request_uri.include?("?") ? "&" : "?"
305
+ "#{request_uri}#{separator}#{@query}"
306
+ end
307
+
252
308
  # Process HTTP Request
253
309
  def process_request(request, body = nil)
254
310
  if body
@@ -260,7 +316,10 @@ module SemanticLogger
260
316
  true
261
317
  else
262
318
  # Failures are logged to the global semantic logger failsafe logger (Usually stderr or file)
263
- logger.error("Bad HTTP response from: #{url} code: #{response.code}, #{response.body}")
319
+ logger.error(
320
+ "Bad HTTP response from: #{redacted_url} code: #{response.code}, " \
321
+ "#{response.body.to_s[0, MAX_LOGGED_RESPONSE_BODY].inspect}"
322
+ )
264
323
  false
265
324
  end
266
325
  rescue RuntimeError => e
@@ -50,7 +50,7 @@ module SemanticLogger
50
50
  # Since only one appender thread will be writing to the file at a time
51
51
  # it is not necessary to protect access to the file with a semaphore
52
52
  # Allow this logger to filter out log levels lower than it's own
53
- @io.write(formatter.call(log, self) << "\n")
53
+ @io.write(format_log(log) << "\n")
54
54
  true
55
55
  end
56
56
 
@@ -23,6 +23,9 @@ require "date"
23
23
  module SemanticLogger
24
24
  module Appender
25
25
  class Kafka < SemanticLogger::Subscriber
26
+ SEED_BROKERS_HINT = "Each should be similar to: 'hostname:port' or 'kafka://hostname:port'".freeze
27
+ private_constant :SEED_BROKERS_HINT
28
+
26
29
  attr_accessor :seed_brokers, :client_id, :connect_timeout, :socket_timeout,
27
30
  :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :ssl_ca_certs_from_system,
28
31
  :delivery_threshold, :delivery_interval, :required_acks,
@@ -149,17 +152,22 @@ module SemanticLogger
149
152
  end
150
153
 
151
154
  def reopen
152
- @kafka = ::Kafka.new(
153
- seed_brokers: seed_brokers,
154
- client_id: client_id,
155
- connect_timeout: connect_timeout,
156
- socket_timeout: socket_timeout,
157
- ssl_ca_cert: ssl_ca_cert,
158
- ssl_client_cert: ssl_client_cert,
159
- ssl_client_cert_key: ssl_client_cert_key,
160
- ssl_ca_certs_from_system: ssl_ca_certs_from_system,
161
- logger: logger
162
- )
155
+ # ruby-kafka puts the seed broker, which can hold credentials, in the error it raises for an
156
+ # invalid one: URI::InvalidURIError, or Kafka::Error for an unknown scheme.
157
+ errors = [URI::InvalidURIError, ::Kafka::Error]
158
+ @kafka = Utils.without_url_in_errors(option: ":seed_brokers", hint: SEED_BROKERS_HINT, errors: errors) do
159
+ ::Kafka.new(
160
+ seed_brokers: seed_brokers,
161
+ client_id: client_id,
162
+ connect_timeout: connect_timeout,
163
+ socket_timeout: socket_timeout,
164
+ ssl_ca_cert: ssl_ca_cert,
165
+ ssl_client_cert: ssl_client_cert,
166
+ ssl_client_cert_key: ssl_client_cert_key,
167
+ ssl_ca_certs_from_system: ssl_ca_certs_from_system,
168
+ logger: logger
169
+ )
170
+ end
163
171
 
164
172
  @producer = @kafka.async_producer(
165
173
  delivery_threshold: delivery_threshold,
@@ -177,7 +185,7 @@ module SemanticLogger
177
185
 
178
186
  # Forward log messages to Kafka producer thread.
179
187
  def log(log)
180
- json = formatter.call(log, self)
188
+ json = format_log(log)
181
189
  @producer.produce(json, topic: topic, partition: partition, partition_key: partition_key, key: key)
182
190
  end
183
191
 
@@ -195,6 +203,12 @@ module SemanticLogger
195
203
  )
196
204
  end
197
205
 
206
+ # Only the topic: the seed brokers can hold credentials, and the options hold the TLS private
207
+ # key and SASL passwords. See Subscriber#inspect.
208
+ def inspect_details
209
+ "topic=#{@topic.inspect}"
210
+ end
211
+
198
212
  private
199
213
 
200
214
  attr_reader :producer
@@ -33,6 +33,8 @@ module SemanticLogger
33
33
  # url: [String]
34
34
  # Define the loki instance URL.
35
35
  # Example: https://logs-prod-999.grafana.net
36
+ # A query string is kept and sent with every request.
37
+ # Example: https://logs-prod-999.grafana.net?orgid=my_org
36
38
  # Default: nil
37
39
  def initialize(url: nil,
38
40
  formatter: SemanticLogger::Formatters::Loki.new,
@@ -40,18 +42,26 @@ module SemanticLogger
40
42
  path: INGESTION_PATH,
41
43
  **args,
42
44
  &)
43
- super(url: "#{url}/#{path}", formatter: formatter, header: header, **args, &)
45
+ super(url: self.class.ingestion_url(url, path), formatter: formatter, header: header, **args, &)
46
+ end
47
+
48
+ # Appends the ingestion path to the supplied url, keeping any query string at the
49
+ # end of the url where Appender::Http can parse it out.
50
+ def self.ingestion_url(url, path)
51
+ uri = Utils.parse_url(url.to_s)
52
+ uri.path = "#{uri.path.chomp('/')}/#{path.delete_prefix('/')}"
53
+ uri.to_s
44
54
  end
45
55
 
46
56
  def log(log)
47
- message = formatter.call(log, self)
57
+ message = format_log(log)
48
58
  logger.trace(message)
49
59
  post(message)
50
60
  end
51
61
 
52
62
  # Logs in batches
53
63
  def batch(logs)
54
- message = formatter.batch(logs, self)
64
+ message = format_batch(logs)
55
65
  logger.trace(message)
56
66
  post(message)
57
67
  end
@@ -164,10 +164,29 @@ module SemanticLogger
164
164
  # Log the message to MongoDB
165
165
  def log(log)
166
166
  # Insert log entry into Mongo
167
- collection.insert_one(formatter.call(log, self))
167
+ collection.insert_one(format_log(log))
168
168
  true
169
169
  end
170
170
 
171
+ # Errors raised for a document that cannot be stored, which retrying cannot insert either.
172
+ # BSON raises BSON::Error, ArgumentError (a key with a NUL byte), or RangeError (an Integer too
173
+ # large for 64 bits) while the document is serialized, before anything is sent, and the driver
174
+ # raises Mongo::Error::MaxBSONSize for a document larger than the server accepts.
175
+ DOCUMENT_ERRORS = [BSON::Error, ArgumentError, RangeError, Mongo::Error::MaxBSONSize].freeze
176
+
177
+ # Except BSON::Error::BSONDecodeError, which comes from reading the server's reply rather than
178
+ # from the document, so it is retried like any other error from the server.
179
+ def message_error?(exception)
180
+ return false if exception.is_a?(BSON::Error::BSONDecodeError)
181
+
182
+ super || DOCUMENT_ERRORS.any? { |error| exception.is_a?(error) }
183
+ end
184
+
185
+ # Only the collection name: the uri can hold a user and password. See Subscriber#inspect.
186
+ def inspect_details
187
+ "collection=#{@collection_name.inspect}"
188
+ end
189
+
171
190
  private
172
191
 
173
192
  def default_formatter
@@ -33,6 +33,12 @@ module SemanticLogger
33
33
  # Proc: Only include log messages where the supplied Proc returns true
34
34
  # The Proc must return true or false.
35
35
  def initialize(level: :error, **args, &block)
36
+ Kernel.warn(
37
+ "SemanticLogger::Appender::NewRelic (appender: :new_relic) is deprecated and will be removed in v6. " \
38
+ "It reports errors via newrelic_rpm and is superseded by the log-forwarding appender " \
39
+ "appender: :new_relic_logs.",
40
+ category: :deprecated
41
+ )
36
42
  super
37
43
  end
38
44
 
@@ -58,7 +64,7 @@ module SemanticLogger
58
64
  # For more documentation on the NewRelic::Agent.notice_error method see:
59
65
  # http://rubydoc.info/github/newrelic/rpm/NewRelic/Agent#notice_error-instance_method
60
66
  # and https://docs.newrelic.com/docs/ruby/ruby-agent-api
61
- ::NewRelic::Agent.notice_error(exception, formatter.call(log, self))
67
+ ::NewRelic::Agent.notice_error(exception, format_log(log))
62
68
  true
63
69
  end
64
70
  end