semantic_logger 5.1.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +13 -12
  3. data/Rakefile +15 -4
  4. data/docs/api.md +37 -11
  5. data/docs/appenders.md +123 -39
  6. data/docs/config.md +115 -13
  7. data/docs/index.md +31 -5
  8. data/docs/log.md +19 -11
  9. data/docs/metrics.md +5 -2
  10. data/docs/operations.md +63 -17
  11. data/docs/rails.md +77 -3
  12. data/docs/security.md +183 -16
  13. data/docs/testing.md +5 -3
  14. data/docs/upgrading.md +295 -7
  15. data/lib/semantic_logger/appender/async.rb +18 -5
  16. data/lib/semantic_logger/appender/bugsnag.rb +1 -1
  17. data/lib/semantic_logger/appender/cloudwatch_logs.rb +122 -27
  18. data/lib/semantic_logger/appender/elasticsearch_base.rb +31 -6
  19. data/lib/semantic_logger/appender/elasticsearch_http.rb +1 -1
  20. data/lib/semantic_logger/appender/file.rb +138 -30
  21. data/lib/semantic_logger/appender/graylog.rb +22 -6
  22. data/lib/semantic_logger/appender/honeybadger.rb +1 -1
  23. data/lib/semantic_logger/appender/honeybadger_insights.rb +1 -1
  24. data/lib/semantic_logger/appender/http.rb +70 -11
  25. data/lib/semantic_logger/appender/io.rb +1 -1
  26. data/lib/semantic_logger/appender/kafka.rb +26 -12
  27. data/lib/semantic_logger/appender/loki.rb +13 -3
  28. data/lib/semantic_logger/appender/mongodb.rb +20 -1
  29. data/lib/semantic_logger/appender/new_relic.rb +1 -1
  30. data/lib/semantic_logger/appender/new_relic_logs.rb +9 -11
  31. data/lib/semantic_logger/appender/open_telemetry.rb +2 -2
  32. data/lib/semantic_logger/appender/rabbitmq.rb +6 -1
  33. data/lib/semantic_logger/appender/sentry.rb +1 -1
  34. data/lib/semantic_logger/appender/sentry_ruby.rb +91 -17
  35. data/lib/semantic_logger/appender/splunk.rb +10 -1
  36. data/lib/semantic_logger/appender/syslog.rb +64 -26
  37. data/lib/semantic_logger/appender/tcp.rb +6 -1
  38. data/lib/semantic_logger/appender/udp.rb +11 -1
  39. data/lib/semantic_logger/appender/wrapper.rb +1 -1
  40. data/lib/semantic_logger/appenders.rb +9 -1
  41. data/lib/semantic_logger/base.rb +82 -30
  42. data/lib/semantic_logger/exception_details.rb +50 -0
  43. data/lib/semantic_logger/formatters/base.rb +52 -6
  44. data/lib/semantic_logger/formatters/color.rb +18 -2
  45. data/lib/semantic_logger/formatters/default.rb +22 -5
  46. data/lib/semantic_logger/formatters/logfmt.rb +153 -21
  47. data/lib/semantic_logger/formatters/loki.rb +26 -22
  48. data/lib/semantic_logger/formatters/one_line.rb +67 -3
  49. data/lib/semantic_logger/formatters/open_telemetry.rb +16 -17
  50. data/lib/semantic_logger/formatters/pattern.rb +8 -4
  51. data/lib/semantic_logger/formatters/raw.rb +4 -5
  52. data/lib/semantic_logger/formatters/syslog.rb +110 -18
  53. data/lib/semantic_logger/formatters/syslog_cee.rb +29 -16
  54. data/lib/semantic_logger/levels.rb +10 -1
  55. data/lib/semantic_logger/log.rb +183 -37
  56. data/lib/semantic_logger/loggable.rb +26 -3
  57. data/lib/semantic_logger/metric/new_relic.rb +1 -1
  58. data/lib/semantic_logger/metric/signalfx.rb +8 -6
  59. data/lib/semantic_logger/metric/statsd.rb +15 -7
  60. data/lib/semantic_logger/queue_processor/request.rb +55 -0
  61. data/lib/semantic_logger/queue_processor/throttled_counter.rb +46 -0
  62. data/lib/semantic_logger/queue_processor.rb +165 -66
  63. data/lib/semantic_logger/semantic_logger.rb +15 -2
  64. data/lib/semantic_logger/subscriber.rb +96 -10
  65. data/lib/semantic_logger/sync_processor.rb +11 -0
  66. data/lib/semantic_logger/test/capture_log_events.rb +8 -5
  67. data/lib/semantic_logger/utils.rb +244 -16
  68. data/lib/semantic_logger/version.rb +1 -1
  69. data/lib/semantic_logger.rb +1 -1
  70. metadata +21 -5
  71. data/lib/semantic_logger/core_ext/thread.rb +0 -30
@@ -1,4 +1,3 @@
1
- require "syslog"
2
1
  require "uri"
3
2
  require "socket"
4
3
  # Send log messages to local syslog, or remote syslog servers over TCP or UDP.
@@ -78,7 +77,7 @@ module SemanticLogger
78
77
  # Default: 1024 bytes
79
78
  #
80
79
  # options: [Integer]
81
- # Default: ::Syslog::LOG_PID | ::Syslog::LOG_CONS
80
+ # Default: ::Syslog::LOG_PID | ::Syslog::LOG_CONS (when logging to a local syslog daemon)
82
81
  # Any of the following (options can be logically OR'd together)
83
82
  # ::Syslog::LOG_CONS
84
83
  # ::Syslog::LOG_NDELAY
@@ -91,7 +90,7 @@ module SemanticLogger
91
90
  # I.e. When `url: 'syslog://localhost'`
92
91
  #
93
92
  # facility: [Integer]
94
- # Default: ::Syslog::LOG_USER
93
+ # Default: ::Syslog::LOG_USER (the same value as SemanticLogger::Formatters::Syslog::LOG_USER)
95
94
  # Type of program (can be logically OR'd together)
96
95
  # ::Syslog::LOG_AUTH
97
96
  # ::Syslog::LOG_AUTHPRIV
@@ -124,19 +123,19 @@ module SemanticLogger
124
123
  # # Change the warn level to LOG_NOTICE level instead of a the default of LOG_WARNING.
125
124
  # SemanticLogger.add_appender(appender: :syslog, level_map: {warn: ::Syslog::LOG_NOTICE})
126
125
  def initialize(url: "syslog://localhost",
127
- facility: ::Syslog::LOG_USER,
128
- max_size: 1024,
126
+ facility: SemanticLogger::Formatters::Syslog::LOG_USER,
127
+ max_size: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE,
129
128
  level_map: SemanticLogger::Formatters::Syslog::LevelMap.new,
130
- options: ::Syslog::LOG_PID | ::Syslog::LOG_CONS,
129
+ options: nil,
131
130
  tcp_client: {},
132
131
  **args,
133
132
  &)
134
133
  @options = options
135
134
  @facility = facility
136
135
  @max_size = max_size
137
- @level_map = level_map
136
+ @level_map = SemanticLogger::Formatters::Syslog::LevelMap.build(level_map)
138
137
  @url = url
139
- uri = URI(@url)
138
+ uri = Utils.parse_url(@url)
140
139
  @server = uri.host || "localhost"
141
140
  @protocol = (uri.scheme || :syslog).to_sym
142
141
  @port = uri.port || 514
@@ -145,28 +144,36 @@ module SemanticLogger
145
144
 
146
145
  raise "Unknown protocol #{@protocol}!" unless %i[syslog tcp udp].include?(@protocol)
147
146
 
148
- # The syslog_protocol gem is required when logging over TCP or UDP.
149
- if %i[tcp udp].include?(@protocol)
147
+ # The syslog gem, a bundled gem since Ruby 3.4, is only required when logging to a local syslog daemon.
148
+ if @protocol == :syslog
150
149
  begin
151
- require "syslog_protocol"
150
+ require "syslog"
152
151
  rescue LoadError
153
152
  raise LoadError,
154
- "Missing gem: syslog_protocol. This gem is required when logging over TCP or UDP. " \
155
- "To fix this error: gem install syslog_protocol"
153
+ "Missing gem: syslog. This gem is required when logging to a local syslog daemon on Ruby 3.4 " \
154
+ 'and later. Please add the gem "syslog" to your Gemfile.'
156
155
  end
156
+ @options ||= ::Syslog::LOG_PID | ::Syslog::LOG_CONS
157
+ end
157
158
 
158
- # The net_tcp_client gem is required when logging over TCP.
159
- if protocol == :tcp
160
- begin
161
- require "net/tcp_client"
162
- rescue LoadError
163
- raise LoadError,
164
- "Missing gem: net_tcp_client. This gem is required when logging over TCP. " \
165
- "To fix this error: gem install net_tcp_client"
166
- end
159
+ # The net_tcp_client gem is required when logging over TCP.
160
+ # The syslog formatters load the syslog_protocol gem that they use to build remote syslog packets.
161
+ if @protocol == :tcp
162
+ begin
163
+ require "net/tcp_client"
164
+ rescue LoadError
165
+ raise LoadError,
166
+ "Missing gem: net_tcp_client. This gem is required when logging over TCP. " \
167
+ "To fix this error: gem install net_tcp_client"
167
168
  end
168
169
  end
169
170
 
171
+ if %i[tcp udp].include?(@protocol)
172
+ # Reject an invalid facility when the appender is created, even with a formatter that ignores it.
173
+ SemanticLogger::Formatters::Syslog.facility_number(@facility)
174
+ args[:formatter] = with_packet_options(args[:formatter]) if args.key?(:formatter)
175
+ end
176
+
170
177
  super(**args, &)
171
178
  reopen
172
179
  end
@@ -195,12 +202,12 @@ module SemanticLogger
195
202
  case @protocol
196
203
  when :syslog
197
204
  # Since the Ruby Syslog API supports sprintf format strings, double up all existing '%'
198
- message = formatter.call(log, self).gsub "%", "%%"
205
+ message = format_log(log).gsub "%", "%%"
199
206
  ::Syslog.log @level_map[log.level], message
200
207
  when :tcp
201
- @remote_syslog.retry_on_connection_failure { @remote_syslog.write("#{formatter.call(log, self)}\r\n") }
208
+ @remote_syslog.retry_on_connection_failure { @remote_syslog.write("#{format_log(log)}\r\n") }
202
209
  when :udp
203
- @remote_syslog.send(formatter.call(log, self), 0, @server, @port)
210
+ @remote_syslog.send(format_log(log), 0, @server, @port)
204
211
  else
205
212
  raise "Unsupported protocol: #{protocol}"
206
213
  end
@@ -212,17 +219,48 @@ module SemanticLogger
212
219
  @remote_syslog.flush if @remote_syslog.respond_to?(:flush)
213
220
  end
214
221
 
222
+ # Over UDP, a log message too large for a datagram (Errno::EMSGSIZE) cannot be sent by retrying.
223
+ def message_error?(exception)
224
+ (@protocol == :udp && exception.is_a?(Errno::EMSGSIZE)) || super
225
+ end
226
+
215
227
  # Returns [SemanticLogger::Formatters::Base] default formatter for this Appender
216
228
  # depending on the protocol selected
217
229
  def default_formatter
218
230
  if protocol == :syslog
219
231
  # Format is text output without the time.
220
232
  # Escape control characters so untrusted log data cannot forge syslog entries.
221
- SemanticLogger::Formatters::Default.new(time_format: :notime, escape_control_chars: true)
233
+ # Exception details are off, as with the remote syslog formatters, so existing records do not change.
234
+ SemanticLogger::Formatters::Default.new(
235
+ time_format: :notime, escape_control_chars: true, log_exception_details: false
236
+ )
222
237
  else
223
238
  SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map, max_size: max_size)
224
239
  end
225
240
  end
241
+
242
+ # The formatters that build remote syslog packets, and so need this appender's facility, level map,
243
+ # and maximum packet size.
244
+ PACKET_FORMATTERS = %i[syslog syslog_cee].freeze
245
+ private_constant :PACKET_FORMATTERS
246
+
247
+ # The url without any credentials. See Subscriber#inspect.
248
+ def inspect_details
249
+ "url=#{Utils.redacted_url(url).inspect}"
250
+ end
251
+
252
+ private
253
+
254
+ # Returns the supplied `formatter:` argument, with this appender's facility, level map, and
255
+ # maximum packet size added when it names one of the PACKET_FORMATTERS, so that they are not
256
+ # silently ignored. Options in a
257
+ # `{syslog: {...}}` Hash take precedence. A formatter instance keeps its own settings.
258
+ def with_packet_options(formatter)
259
+ name, options = formatter.is_a?(Hash) ? formatter.first : [formatter, {}]
260
+ return formatter unless name.respond_to?(:to_sym) && PACKET_FORMATTERS.include?(name.to_sym)
261
+
262
+ {name.to_sym => {facility: facility, level_map: level_map, max_size: max_size}.merge(options || {})}
263
+ end
226
264
  end
227
265
  end
228
266
  end
@@ -205,7 +205,7 @@ module SemanticLogger
205
205
 
206
206
  # Write the log using the specified protocol and server.
207
207
  def log(log)
208
- message = formatter.call(log, self)
208
+ message = format_log(log)
209
209
  @tcp_client.retry_on_connection_failure do
210
210
  @tcp_client.write("#{message}#{separator}")
211
211
  end
@@ -222,6 +222,11 @@ module SemanticLogger
222
222
  @tcp_client&.close
223
223
  end
224
224
 
225
+ # Only the servers: the other options can hold TLS keys and passwords. See Subscriber#inspect.
226
+ def inspect_details
227
+ "server=#{(@tcp_client_args[:server] || @tcp_client_args[:servers]).inspect}"
228
+ end
229
+
225
230
  private
226
231
 
227
232
  # Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
@@ -80,7 +80,7 @@ module SemanticLogger
80
80
 
81
81
  # Write the log using the specified protocol and server.
82
82
  def log(log)
83
- @socket.send(formatter.call(log, self), udp_flags)
83
+ @socket.send(format_log(log), udp_flags)
84
84
  true
85
85
  end
86
86
 
@@ -94,6 +94,16 @@ module SemanticLogger
94
94
  @socket&.close
95
95
  end
96
96
 
97
+ # A log message too large for a datagram (Errno::EMSGSIZE) cannot be sent by retrying.
98
+ def message_error?(exception)
99
+ exception.is_a?(Errno::EMSGSIZE) || super
100
+ end
101
+
102
+ # See Subscriber#inspect.
103
+ def inspect_details
104
+ "server=#{server.inspect}"
105
+ end
106
+
97
107
  private
98
108
 
99
109
  # Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
@@ -57,7 +57,7 @@ module SemanticLogger
57
57
  # Ruby or Rails Loggers
58
58
  def log(log)
59
59
  level = log.level == :trace ? :debug : log.level
60
- @logger.send(level, formatter.call(log, self))
60
+ @logger.send(level, format_log(log))
61
61
  true
62
62
  end
63
63
 
@@ -37,7 +37,7 @@ module SemanticLogger
37
37
  # Returns [Array<Hash>] operational statistics for each appender.
38
38
  #
39
39
  # Appenders that run asynchronously (see SemanticLogger::Appender::Async#stats) report
40
- # their queue size and processed/dropped counts. Appenders that log inline on the
40
+ # their queue size and processed/dropped/failed counts. Appenders that log inline on the
41
41
  # processor thread report only their name with `async: false`.
42
42
  def stats
43
43
  map do |appender|
@@ -49,9 +49,17 @@ module SemanticLogger
49
49
  end
50
50
  end
51
51
 
52
+ # Write the log message to every appender that should log it. An appender that fails is
53
+ # reported, and the others still write the message.
54
+ #
55
+ # A stack overflow is not a failure of the appender but of the thread, which must not overflow
56
+ # again: on macOS a second stack overflow on the same thread crashes Ruby. It is raised to the
57
+ # processor running this thread instead, and the message goes to no further appenders.
52
58
  def log(log)
53
59
  each do |appender|
54
60
  appender.log(log) if appender.should_log?(log)
61
+ rescue SystemStackError
62
+ raise
55
63
  rescue Exception => e
56
64
  logger.error "Failed to log to appender: #{appender.name}", e
57
65
  end
@@ -28,23 +28,38 @@ module SemanticLogger
28
28
  end
29
29
  end
30
30
 
31
- # Returns the current log level if set, otherwise it returns the global
32
- # default log level
31
+ # Returns the level set by an enclosing `#with_level` block on the current fiber,
32
+ # otherwise the level set for this instance, otherwise the global default level.
33
33
  def level
34
- @level || SemanticLogger.default_level
34
+ index = @with_level_used && with_level_index
35
+ return Levels.level(index) if index
36
+
37
+ @level || fallback_level
35
38
  end
36
39
 
37
- # Set the logging level for this logger during the execution of the given block
40
+ # Change the log level of this instance while the block runs, on the current fiber only.
38
41
  #
39
- # Refer to the documentation for `#level=` for more information about the possible log levels.
40
- def with_level(new_level)
41
- old_level = level
42
-
43
- self.level = new_level
44
-
45
- yield
46
- ensure
47
- self.level = old_level
42
+ # Other threads and fibers that share this instance keep logging at its own level.
43
+ # The level set with `#level=` is left untouched, so an instance that follows the
44
+ # global default level keeps following it after the block. Blocks can be nested.
45
+ #
46
+ # Matches Ruby's `Logger#with_level`, which Sidekiq calls to apply a job's `log_level`.
47
+ # Accepts the same levels as `#level=`. Returns the result of the block.
48
+ def with_level(level)
49
+ index = Levels.index(level)
50
+ overrides = (Thread.current[:semantic_logger_with_level] ||= {}.compare_by_identity)
51
+ previous = overrides[self]
52
+ @with_level_used = true
53
+ overrides[self] = index
54
+ begin
55
+ yield
56
+ ensure
57
+ if previous
58
+ overrides[self] = previous
59
+ else
60
+ overrides.delete(self)
61
+ end
62
+ end
48
63
  end
49
64
 
50
65
  # Implement the log level calls
@@ -150,7 +165,7 @@ module SemanticLogger
150
165
  if thread == Thread.current
151
166
  Utils.extract_backtrace(caller)
152
167
  else
153
- log.thread_name = thread.name
168
+ log.thread_name = Log.thread_name_for(thread)
154
169
  log.tags = (thread[:semantic_logger_tags] || []).clone
155
170
  log.named_tags = (thread[:semantic_logger_named_tags] || {}).clone
156
171
  thread.backtrace
@@ -319,6 +334,10 @@ module SemanticLogger
319
334
  # (empty) instance tags allocate no garbage. Child loggers replace them via #tagged_child.
320
335
  @instance_tags = [].freeze
321
336
  @instance_named_tags = {}.freeze
337
+ # Set the first time `#with_level` is called on this instance, so that instances that
338
+ # never use it skip the fiber-local lookup when checking their level. Only the fiber
339
+ # that sets an override needs to see the flag, and that fiber set it itself.
340
+ @with_level_used = false
322
341
  if level.nil?
323
342
  # Allow the global default level to determine this loggers log level
324
343
  @level_index = nil
@@ -328,11 +347,25 @@ module SemanticLogger
328
347
  end
329
348
  end
330
349
 
331
- # Return the level index for fast comparisons
332
- # Returns the global default level index if the level has not been explicitly
333
- # set for this instance
350
+ # Return the level index for fast comparisons, resolved in the same order as `#level`.
334
351
  def level_index
335
- @level_index || SemanticLogger.default_level_index
352
+ (@with_level_used && with_level_index) || @level_index || fallback_level_index
353
+ end
354
+
355
+ # Level to use when no level has been set for this instance: the global default level.
356
+ def fallback_level
357
+ SemanticLogger.default_level
358
+ end
359
+
360
+ # Level index to use when no level has been set for this instance. Unlike
361
+ # `#fallback_level`, it honors `SemanticLogger.silence`.
362
+ def fallback_level_index
363
+ SemanticLogger.default_level_index
364
+ end
365
+
366
+ # Level index set for this instance by an enclosing `#with_level` block on the current fiber.
367
+ def with_level_index
368
+ Thread.current[:semantic_logger_with_level]&.[](self)
336
369
  end
337
370
 
338
371
  # Whether to log the supplied message based on the current filter if any
@@ -444,6 +477,9 @@ module SemanticLogger
444
477
  end
445
478
 
446
479
  # Measure the supplied block and log the message
480
+ #
481
+ # The options can hold untrusted data, so an option with a value that is not valid for it is
482
+ # ignored (see Log.argument?) and its default is used instead.
447
483
  def measure_internal(level, index, message, params)
448
484
  exception = nil
449
485
  # Single parameter is a hash
@@ -454,9 +490,7 @@ module SemanticLogger
454
490
  start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
455
491
  begin
456
492
  if block_given?
457
- if (silence_level = params[:silence])
458
- # In case someone accidentally sets `silence: true` instead of `silence: :error`
459
- silence_level = :error if silence_level == true
493
+ if (silence_level = measure_silence_level(params))
460
494
  silence(silence_level) { yield(params) }
461
495
  else
462
496
  yield(params)
@@ -467,13 +501,15 @@ module SemanticLogger
467
501
  ensure
468
502
  # Must use ensure block otherwise a `return` in the yield above will skip the log entry
469
503
  log = new_log(level, index)
470
- exception ||= params[:exception]
471
- message = params[:message] if params[:message]
472
- duration =
504
+ # Only an Exception, since it is re-raised below
505
+ exception ||= params[:exception] if params[:exception].is_a?(Exception)
506
+ message = measure_option(params, :message) || message
507
+ duration =
473
508
  if block_given?
474
509
  1_000.0 * (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
475
510
  else
476
- params[:duration] || raise("Mandatory block missing when :duration option is not supplied")
511
+ measure_option(params, :duration) ||
512
+ raise("Mandatory block missing when :duration option is not supplied")
477
513
  end
478
514
 
479
515
  # Extract options after block completes so that block can modify any of the options
@@ -483,13 +519,13 @@ module SemanticLogger
483
519
  should_log = log.assign(
484
520
  message: message,
485
521
  payload: payload,
486
- min_duration: params[:min_duration] || 0.0,
522
+ min_duration: measure_option(params, :min_duration) || 0.0,
487
523
  exception: exception,
488
- metric: params[:metric],
489
- metric_amount: params[:metric_amount],
524
+ metric: measure_option(params, :metric),
525
+ metric_amount: measure_option(params, :metric_amount),
490
526
  duration: duration,
491
- log_exception: params[:log_exception] || :partial,
492
- on_exception_level: params[:on_exception_level]
527
+ log_exception: measure_option(params, :log_exception) || :partial,
528
+ on_exception_level: measure_option(params, :on_exception_level)
493
529
  )
494
530
 
495
531
  # Log level may change during assign due to :on_exception_level
@@ -498,6 +534,22 @@ module SemanticLogger
498
534
  end
499
535
  end
500
536
 
537
+ # Returns the value of the measure option that is also a log argument, or nil when it is not
538
+ # supplied or its value is not valid for that argument.
539
+ def measure_option(params, key)
540
+ value = params[key]
541
+ value if Log.argument?(key, value)
542
+ end
543
+
544
+ # Returns the level to silence other log messages to within a measure block, or nil when none
545
+ # is supplied or it is not a valid level.
546
+ def measure_silence_level(params)
547
+ silence_level = params[:silence]
548
+ # In case someone accidentally sets `silence: true` instead of `silence: :error`
549
+ silence_level = :error if silence_level == true
550
+ silence_level if Levels.valid?(silence_level)
551
+ end
552
+
501
553
  # For measuring methods and logging their duration.
502
554
  def measure_method(index:,
503
555
  level:,
@@ -0,0 +1,50 @@
1
+ module SemanticLogger
2
+ # Separates the detail that Ruby's Exception#detailed_message adds to an exception's message,
3
+ # such as did_you_mean suggestions or error_highlight's source snippet, from the message itself.
4
+ #
5
+ # Internal: Log caches the result for each of its exceptions; see Log#exception_details.
6
+ module ExceptionDetails
7
+ # Ruby's own Exception#detailed_message: the message, decorated with the class name.
8
+ DETAILED_MESSAGE = Exception.instance_method(:detailed_message)
9
+ private_constant :DETAILED_MESSAGE
10
+
11
+ # Returns [String] the detail that #detailed_message adds to the exception's message, without
12
+ # the class name that Ruby also adds, or nil when there is none or it cannot be separated.
13
+ #
14
+ # Never raises, so that an exception's own #detailed_message cannot stop a log entry being
15
+ # written.
16
+ def self.extract(exception, source: false)
17
+ detailed = full_detailed_message(exception, source)
18
+ return unless detailed
19
+
20
+ base = DETAILED_MESSAGE.bind_call(exception, highlight: false)
21
+ # An override that does not build on Ruby's message cannot be separated from it.
22
+ return unless detailed.is_a?(String) && detailed.start_with?(base)
23
+
24
+ details = detailed.delete_prefix(base).sub(/\A\n+/, "").rstrip
25
+ details unless details.empty?
26
+ rescue StandardError
27
+ nil
28
+ end
29
+
30
+ # Returns [String] the exception's #detailed_message, with or without error_highlight's
31
+ # source snippet, or nil when the snippet cannot be left out.
32
+ def self.full_detailed_message(exception, source)
33
+ exception.detailed_message(highlight: false, error_highlight: source)
34
+ rescue ArgumentError
35
+ # An override of #detailed_message that does not accept extra keyword arguments. Calling it
36
+ # without `error_highlight: false` lets error_highlight add its source snippet, so only do so
37
+ # when the snippet was asked for, or error_highlight does not decorate this exception.
38
+ return if !source && error_highlighted?(exception)
39
+
40
+ exception.detailed_message(highlight: false)
41
+ end
42
+ private_class_method :full_detailed_message
43
+
44
+ # Whether error_highlight adds a source snippet to this exception's #detailed_message.
45
+ def self.error_highlighted?(exception)
46
+ defined?(ErrorHighlight::CoreExt) && exception.is_a?(ErrorHighlight::CoreExt)
47
+ end
48
+ private_class_method :error_highlighted?
49
+ end
50
+ end
@@ -3,7 +3,7 @@ module SemanticLogger
3
3
  module Formatters
4
4
  class Base
5
5
  attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision,
6
- :escape_control_chars
6
+ :escape_control_chars, :log_exception_details
7
7
 
8
8
  # Printable escapes for the most common control characters. Any other
9
9
  # control character is escaped to its hexadecimal `\xHH` form by
@@ -50,25 +50,39 @@ module SemanticLogger
50
50
  # Default: PRECISION (6, except on older JRuby, where 3)
51
51
  # escape_control_chars: [Boolean]
52
52
  # Replace control characters (newlines, the ANSI escape, etc.) in
53
- # untrusted log data (message, tags, named tags, and exception
54
- # message) with a printable escaped form, e.g. "\n".
53
+ # untrusted log data (logger name, thread name, message, payload, tags,
54
+ # named tags, and the message and backtrace lines of the exception and
55
+ # of each exception that caused it) with a printable escaped form, e.g. "\n".
55
56
  # This prevents log forging and terminal escape-sequence injection
56
57
  # when logging data that may contain attacker-controlled content.
57
58
  # Note: Has no effect on structured formatters such as :json, which
58
59
  # already escape control characters via JSON encoding.
59
60
  # Default: false (preserve newlines and ANSI colors in text output)
61
+ # log_exception_details: [true|false|:with_source]
62
+ # Whether to log the detail that Ruby's Exception#detailed_message adds to an
63
+ # exception's message, such as did_you_mean suggestions ("Did you mean? upcase") or
64
+ # the detail an exception class supplies by overriding #detailed_message.
65
+ # :with_source also logs error_highlight's snippet of the failing source line. It is
66
+ # about 100 times slower, since it re-parses that source file, and it copies source
67
+ # code into the log.
68
+ # On Ruby 3.2 a NameError or NoMethodError builds its message from receiver.inspect each
69
+ # time it is asked, and logging the details asks for it twice more, so turn this off when
70
+ # inspecting the receivers of logged exceptions is slow.
71
+ # Default: true for the Default, Color, and Pattern text formatters, false otherwise.
60
72
  def initialize(time_format: nil,
61
73
  log_host: true,
62
74
  log_application: true,
63
75
  log_environment: true,
64
76
  precision: PRECISION,
65
- escape_control_chars: false)
77
+ escape_control_chars: false,
78
+ log_exception_details: default_log_exception_details)
66
79
  @time_format = time_format || self.class.build_time_format(precision)
67
80
  @log_host = log_host
68
81
  @log_application = log_application
69
82
  @log_environment = log_environment
70
83
  @precision = precision
71
- @escape_control_chars = escape_control_chars
84
+ @escape_control_chars = escape_control_chars
85
+ @log_exception_details = log_exception_details
72
86
  end
73
87
 
74
88
  # Return default time format string
@@ -101,10 +115,42 @@ module SemanticLogger
101
115
  # Note: This escapes (preserves) control characters, and is opt-in. It is
102
116
  # distinct from Log#cleansed_message, which unconditionally *strips* ANSI
103
117
  # colorization from the message for structured (JSON/Loki) output.
118
+ #
119
+ # A string with invalid byte sequences, for example an exception message that quotes
120
+ # untrusted binary input, or in another encoding, is first converted to valid UTF-8
121
+ # (dropping the invalid bytes), since matching against it would raise and the log entry
122
+ # would be lost.
104
123
  def escape_control_characters(value)
105
124
  return value unless escape_control_chars && value
106
125
 
107
- value.to_s.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
126
+ string = Utils.encode_utf8_string(value.to_s)
127
+ string.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
128
+ end
129
+
130
+ # Whether this formatter logs exception details unless told otherwise. See #initialize.
131
+ def default_log_exception_details
132
+ false
133
+ end
134
+
135
+ # Returns [String] the detail Ruby adds to the message of the supplied exception (see
136
+ # Log#exception_details), or nil when there is none or this formatter does not log it.
137
+ def exception_details_for(exception)
138
+ return unless log_exception_details
139
+
140
+ log.exception_details(exception, source: log_exception_details == :with_source)
141
+ end
142
+
143
+ # Returns [String] the backtrace of the logged exception, followed by each exception
144
+ # that caused it, as laid out by Log#backtrace_to_s.
145
+ #
146
+ # When `escape_control_chars` is enabled, the message of each cause and each backtrace
147
+ # line is passed through #escape_control_characters, the same as the message of the
148
+ # logged exception: causes are often parser or validation errors that quote untrusted
149
+ # input, and a backtrace can be set from untrusted data.
150
+ def exception_backtrace
151
+ return log.backtrace_to_s unless escape_control_chars
152
+
153
+ log.backtrace_to_s { |text| escape_control_characters(text) }
108
154
  end
109
155
 
110
156
  # Return the Time as a formatted string
@@ -71,6 +71,8 @@ module SemanticLogger
71
71
  #
72
72
  # Note: The option :multiline is set to false if not supplied.
73
73
  # Note: Has no effect if Awesome Print is not installed.
74
+ # Note: With `escape_control_chars: true` the payload is always rendered plain
75
+ # (without colors) and on a single line, then escaped.
74
76
  #
75
77
  # color_map: [Hash | SemanticLogger::Formatters::Color::ColorMap]
76
78
  # ColorMaps each of the log levels to a color
@@ -116,7 +118,7 @@ module SemanticLogger
116
118
 
117
119
  if log.payload.respond_to?(:ai)
118
120
  begin
119
- "-- #{log.payload.ai(@ai_options)}"
121
+ "-- #{escape_control_characters(log.payload_to_s { |payload| payload.ai(ai_options) })}"
120
122
  rescue StandardError
121
123
  super
122
124
  end
@@ -128,14 +130,28 @@ module SemanticLogger
128
130
  def exception
129
131
  return unless log.exception
130
132
 
133
+ details = exception_details
131
134
  "-- Exception: #{color}#{log.exception.class}: " \
132
- "#{escape_control_characters(log.exception.message)}#{color_map.clear}\n#{log.backtrace_to_s}"
135
+ "#{escape_control_characters(log.exception.message)}#{color_map.clear}\n" \
136
+ "#{"#{details}\n" if details}#{exception_backtrace}"
133
137
  end
134
138
 
135
139
  def call(log, logger)
136
140
  self.color = color_map[log.level]
137
141
  super
138
142
  end
143
+
144
+ private
145
+
146
+ # Returns [Hash] the Amazing Print options to render the payload with.
147
+ #
148
+ # When `escape_control_chars` is enabled the payload is rendered plain, on one line:
149
+ # the colors and line breaks Amazing Print adds cannot be told apart from control
150
+ # characters in the data (for example an object whose #inspect returns them), and
151
+ # escaping them all leaves no colors or line breaks to keep.
152
+ def ai_options
153
+ escape_control_chars ? @ai_options.merge(plain: true, multiline: false) : @ai_options
154
+ end
139
155
  end
140
156
  end
141
157
  end