semantic_logger 5.0.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +20 -10
  3. data/Rakefile +40 -0
  4. data/docs/api.md +506 -0
  5. data/docs/appenders.md +1074 -0
  6. data/docs/config.md +707 -0
  7. data/docs/index.md +223 -0
  8. data/docs/log.md +116 -0
  9. data/docs/metrics.md +171 -0
  10. data/docs/operations.md +446 -0
  11. data/docs/rails.md +890 -0
  12. data/docs/security.md +286 -0
  13. data/docs/testing.md +287 -0
  14. data/docs/upgrading.md +697 -0
  15. data/lib/semantic_logger/appender/async.rb +18 -5
  16. data/lib/semantic_logger/appender/bugsnag.rb +1 -1
  17. data/lib/semantic_logger/appender/cloudwatch_logs.rb +126 -29
  18. data/lib/semantic_logger/appender/elasticsearch.rb +2 -5
  19. data/lib/semantic_logger/appender/elasticsearch_base.rb +48 -23
  20. data/lib/semantic_logger/appender/elasticsearch_http.rb +15 -8
  21. data/lib/semantic_logger/appender/file.rb +141 -28
  22. data/lib/semantic_logger/appender/graylog.rb +22 -6
  23. data/lib/semantic_logger/appender/honeybadger.rb +1 -1
  24. data/lib/semantic_logger/appender/honeybadger_insights.rb +1 -1
  25. data/lib/semantic_logger/appender/http.rb +70 -11
  26. data/lib/semantic_logger/appender/io.rb +1 -1
  27. data/lib/semantic_logger/appender/kafka.rb +26 -12
  28. data/lib/semantic_logger/appender/loki.rb +13 -3
  29. data/lib/semantic_logger/appender/mongodb.rb +20 -1
  30. data/lib/semantic_logger/appender/new_relic.rb +7 -1
  31. data/lib/semantic_logger/appender/new_relic_logs.rb +11 -12
  32. data/lib/semantic_logger/appender/open_telemetry.rb +4 -3
  33. data/lib/semantic_logger/appender/opensearch.rb +2 -1
  34. data/lib/semantic_logger/appender/rabbitmq.rb +10 -4
  35. data/lib/semantic_logger/appender/sentry.rb +6 -1
  36. data/lib/semantic_logger/appender/sentry_ruby.rb +91 -17
  37. data/lib/semantic_logger/appender/splunk.rb +10 -1
  38. data/lib/semantic_logger/appender/syslog.rb +66 -25
  39. data/lib/semantic_logger/appender/tcp.rb +10 -4
  40. data/lib/semantic_logger/appender/udp.rb +11 -1
  41. data/lib/semantic_logger/appender/wrapper.rb +3 -2
  42. data/lib/semantic_logger/appender.rb +6 -3
  43. data/lib/semantic_logger/appenders.rb +11 -2
  44. data/lib/semantic_logger/base.rb +82 -32
  45. data/lib/semantic_logger/exception_details.rb +50 -0
  46. data/lib/semantic_logger/formatters/base.rb +52 -6
  47. data/lib/semantic_logger/formatters/color.rb +22 -3
  48. data/lib/semantic_logger/formatters/default.rb +28 -6
  49. data/lib/semantic_logger/formatters/logfmt.rb +155 -21
  50. data/lib/semantic_logger/formatters/loki.rb +26 -22
  51. data/lib/semantic_logger/formatters/one_line.rb +67 -3
  52. data/lib/semantic_logger/formatters/open_telemetry.rb +16 -17
  53. data/lib/semantic_logger/formatters/pattern.rb +11 -5
  54. data/lib/semantic_logger/formatters/raw.rb +4 -5
  55. data/lib/semantic_logger/formatters/syslog.rb +110 -17
  56. data/lib/semantic_logger/formatters/syslog_cee.rb +29 -15
  57. data/lib/semantic_logger/levels.rb +10 -1
  58. data/lib/semantic_logger/log.rb +183 -37
  59. data/lib/semantic_logger/loggable.rb +26 -3
  60. data/lib/semantic_logger/metric/new_relic.rb +1 -1
  61. data/lib/semantic_logger/metric/signalfx.rb +12 -8
  62. data/lib/semantic_logger/metric/statsd.rb +15 -7
  63. data/lib/semantic_logger/queue_processor/request.rb +55 -0
  64. data/lib/semantic_logger/queue_processor/throttled_counter.rb +46 -0
  65. data/lib/semantic_logger/queue_processor.rb +170 -68
  66. data/lib/semantic_logger/reporters/minitest.rb +4 -4
  67. data/lib/semantic_logger/semantic_logger.rb +15 -2
  68. data/lib/semantic_logger/subscriber.rb +96 -10
  69. data/lib/semantic_logger/sync_processor.rb +11 -0
  70. data/lib/semantic_logger/test/capture_log_events.rb +9 -6
  71. data/lib/semantic_logger/utils.rb +244 -16
  72. data/lib/semantic_logger/version.rb +1 -1
  73. data/lib/semantic_logger.rb +1 -1
  74. metadata +33 -6
  75. data/lib/semantic_logger/core_ext/thread.rb +0 -30
@@ -1,14 +1,62 @@
1
- begin
2
- require "syslog_protocol"
3
- rescue LoadError
4
- raise LoadError,
5
- 'Gem syslog_protocol is required for remote logging using the Syslog protocol. Please add the gem "syslog_protocol" to your Gemfile.'
6
- end
7
-
8
1
  module SemanticLogger
9
2
  module Formatters
10
3
  class Syslog < Default
11
- attr_accessor :level_map, :facility, :max_size
4
+ attr_accessor :max_size
5
+ attr_reader :facility, :level_map
6
+
7
+ # Syslog severities and facilities, with the same values as the matching `::Syslog` constants.
8
+ # The `syslog` gem is only needed to log to a local syslog daemon, and since Ruby 3.4 it is a
9
+ # bundled gem, so remote syslog must not depend on it for these values.
10
+ LOG_CRIT = 2
11
+ LOG_ERR = 3
12
+ LOG_WARNING = 4
13
+ LOG_NOTICE = 5
14
+ LOG_INFO = 6
15
+ LOG_DEBUG = 7
16
+
17
+ # Syslog facilities, as syslog(3) defines them: the facility number shifted left by 3 bits.
18
+ LOG_KERN = 0 << 3
19
+ LOG_USER = 1 << 3
20
+ LOG_MAIL = 2 << 3
21
+ LOG_DAEMON = 3 << 3
22
+ LOG_AUTH = 4 << 3
23
+ LOG_SYSLOG = 5 << 3
24
+ LOG_LPR = 6 << 3
25
+ LOG_NEWS = 7 << 3
26
+ LOG_UUCP = 8 << 3
27
+ LOG_CRON = 9 << 3
28
+ LOG_AUTHPRIV = 10 << 3
29
+ LOG_FTP = 11 << 3
30
+ LOG_LOCAL0 = 16 << 3
31
+ LOG_LOCAL1 = 17 << 3
32
+ LOG_LOCAL2 = 18 << 3
33
+ LOG_LOCAL3 = 19 << 3
34
+ LOG_LOCAL4 = 20 << 3
35
+ LOG_LOCAL5 = 21 << 3
36
+ LOG_LOCAL6 = 22 << 3
37
+ LOG_LOCAL7 = 23 << 3
38
+
39
+ # Default maximum size of a syslog packet, in bytes, as for the syslog appender.
40
+ DEFAULT_MAX_SIZE = 1024
41
+
42
+ # The highest facility the syslog protocol can encode.
43
+ MAX_FACILITY = LOG_LOCAL7
44
+
45
+ # Returns [Integer] the facility number (0 to 23) that the syslog protocol sends, for a facility
46
+ # given as documented: one of the `::Syslog::LOG_*` facility constants, such as
47
+ # `::Syslog::LOG_LOCAL0`. syslog(3) defines each of those as the facility number shifted left
48
+ # by 3 bits, so `::Syslog::LOG_USER` is 8 and its facility number is 1.
49
+ #
50
+ # Raises ArgumentError for any other value, including a bare facility number.
51
+ def self.facility_number(facility)
52
+ unless facility.is_a?(Integer) && (facility % 8).zero? && facility.between?(0, MAX_FACILITY)
53
+ raise(ArgumentError,
54
+ "Invalid syslog facility: #{facility.inspect}. Must be one of the ::Syslog::LOG_* facility " \
55
+ "constants, for example ::Syslog::LOG_USER or ::Syslog::LOG_LOCAL0.")
56
+ end
57
+
58
+ facility >> 3
59
+ end
12
60
 
13
61
  # Default level map for every log level
14
62
  #
@@ -25,8 +73,14 @@ module SemanticLogger
25
73
  class LevelMap
26
74
  attr_accessor :trace, :debug, :info, :warn, :error, :fatal
27
75
 
28
- def initialize(trace: ::Syslog::LOG_DEBUG, debug: ::Syslog::LOG_INFO, info: ::Syslog::LOG_NOTICE,
29
- warn: ::Syslog::LOG_WARNING, error: ::Syslog::LOG_ERR, fatal: ::Syslog::LOG_CRIT)
76
+ # Returns [LevelMap] the supplied level map, or a new one with a Hash of overrides applied,
77
+ # for example `{warn: LOG_NOTICE}`.
78
+ def self.build(level_map)
79
+ level_map.is_a?(LevelMap) ? level_map : new(**level_map)
80
+ end
81
+
82
+ def initialize(trace: LOG_DEBUG, debug: LOG_INFO, info: LOG_NOTICE,
83
+ warn: LOG_WARNING, error: LOG_ERR, fatal: LOG_CRIT)
30
84
  @trace = trace
31
85
  @debug = debug
32
86
  @info = info
@@ -40,11 +94,23 @@ module SemanticLogger
40
94
  end
41
95
  end
42
96
 
97
+ # Load the syslog_protocol gem, which builds the packets for remote syslog.
98
+ # Called when a syslog formatter is created, so the gem is only needed by applications
99
+ # that log to a remote syslog server.
100
+ def self.require_syslog_protocol
101
+ require "syslog_protocol"
102
+ rescue LoadError
103
+ raise LoadError,
104
+ "Gem syslog_protocol is required for remote logging using the Syslog protocol. " \
105
+ 'Please add the gem "syslog_protocol" to your Gemfile.'
106
+ end
107
+
43
108
  # Create a Syslog Log Formatter
44
109
  #
45
110
  # Parameters:
46
111
  # facility: [Integer]
47
- # Default: ::Syslog::LOG_USER
112
+ # One of the ::Syslog::LOG_* facility constants, for example ::Syslog::LOG_LOCAL0.
113
+ # Default: LOG_USER
48
114
  #
49
115
  # level_map: [Hash | SemanticLogger::Formatters::Syslog::LevelMap]
50
116
  # Supply a custom map of SemanticLogger levels to syslog levels.
@@ -56,27 +122,49 @@ module SemanticLogger
56
122
  # Default: true (unlike other formatters, since syslog frames records
57
123
  # with a separator)
58
124
  #
125
+ # max_size: [Integer]
126
+ # Maximum size of each syslog packet, in bytes.
127
+ # Default: DEFAULT_MAX_SIZE (1024)
128
+ #
59
129
  # Example:
60
130
  # # Change the warn level to LOG_NOTICE level instead of a the default of LOG_WARNING.
61
131
  # SemanticLogger.add_appender(appender: :syslog, level_map: {warn: ::Syslog::LOG_NOTICE})
62
- def initialize(facility: ::Syslog::LOG_USER, level_map: LevelMap.new, max_size: Integer,
132
+ def initialize(facility: LOG_USER, level_map: LevelMap.new, max_size: DEFAULT_MAX_SIZE,
63
133
  escape_control_chars: true, **args)
64
- @facility = facility
65
- @level_map = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(**level_map)
66
- @max_size = max_size
134
+ self.class.require_syslog_protocol
135
+
136
+ self.facility = facility
137
+ self.level_map = level_map
138
+ @max_size = max_size
67
139
  # Syslog frames records with a separator, so embedded newlines or other
68
140
  # control characters in untrusted log data can forge or split records.
69
141
  # Default to escaping them, overridable for backwards compatibility.
70
142
  super(escape_control_chars: escape_control_chars, **args)
71
143
  end
72
144
 
145
+ # Set the facility to one of the ::Syslog::LOG_* facility constants.
146
+ # Raises ArgumentError for any other value, so that a mistake surfaces here rather than on
147
+ # every log entry.
148
+ def facility=(facility)
149
+ @facility_number = Syslog.facility_number(facility)
150
+ @facility = facility
151
+ end
152
+
153
+ # Set the map of log levels to syslog severities, as a LevelMap or a Hash of overrides.
154
+ def level_map=(level_map)
155
+ @level_map = LevelMap.build(level_map)
156
+ end
157
+
73
158
  # Time is part of the syslog packet and is not included in the formatted message.
74
159
  def time
75
160
  nil
76
161
  end
77
162
 
78
163
  def call(log, logger)
79
- message = super
164
+ # The individual fields are already escaped. Also escape the newlines that the
165
+ # exception and its backtrace are laid out with, so that a record never spans more
166
+ # than one line, which a receiver framing records by newline would split.
167
+ message = escape_control_characters(super)
80
168
  create_syslog_packet(message)
81
169
  end
82
170
 
@@ -86,13 +174,18 @@ module SemanticLogger
86
174
  def create_syslog_packet(message)
87
175
  packet = SyslogProtocol::Packet.new
88
176
  packet.hostname = logger.host
89
- packet.facility = facility
177
+ packet.facility = @facility_number
90
178
  packet.tag = logger.application.delete(" ")
91
179
  packet.content = message
92
180
  packet.time = log.time
93
181
  packet.severity = level_map[log.level]
94
182
  packet.assemble(@max_size)
95
183
  end
184
+
185
+ # Off by default, so that existing syslog records do not change.
186
+ def default_log_exception_details
187
+ false
188
+ end
96
189
  end
97
190
  end
98
191
  end
@@ -1,14 +1,8 @@
1
- begin
2
- require "syslog_protocol"
3
- rescue LoadError
4
- raise LoadError,
5
- 'Gem syslog_protocol is required for remote logging using the Syslog protocol. Please add the gem "syslog_protocol" to your Gemfile.'
6
- end
7
-
8
1
  module SemanticLogger
9
2
  module Formatters
10
3
  class SyslogCee < Raw
11
- attr_accessor :level_map, :facility, :max_size
4
+ attr_accessor :max_size
5
+ attr_reader :facility, :level_map
12
6
 
13
7
  # CEE JSON Syslog format
14
8
  # Untested prototype code. Based on documentation only.
@@ -16,22 +10,42 @@ module SemanticLogger
16
10
  #
17
11
  # Parameters:
18
12
  # facility: [Integer]
19
- # Default: ::Syslog::LOG_USER
13
+ # One of the ::Syslog::LOG_* facility constants, for example ::Syslog::LOG_LOCAL0.
14
+ # Default: SemanticLogger::Formatters::Syslog::LOG_USER
20
15
  #
21
16
  # level_map: [Hash | SemanticLogger::Formatters::Syslog::LevelMap]
22
17
  # Supply a custom map of SemanticLogger levels to syslog levels.
23
18
  #
19
+ # max_size: [Integer]
20
+ # Maximum size of each syslog packet, in bytes.
21
+ # Default: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE (1024)
22
+ #
24
23
  # Example:
25
24
  # # Log via udp to a remote syslog server on host: `server1` and port `8514`, using the CEE format.
26
25
  # SemanticLogger.add_appender(appender: :syslog, formatter: :syslog_cee, url: 'udp://server1:8514')
27
- def initialize(facility: ::Syslog::LOG_USER, level_map: SemanticLogger::Formatters::Syslog::LevelMap.new,
28
- max_size: Integer)
29
- @facility = facility
30
- @level_map = level_map.is_a?(SemanticLogger::Formatters::Syslog::LevelMap) ? level_map : SemanticLogger::Formatters::Syslog::LevelMap.new(level_map)
31
- @max_size = max_size
26
+ def initialize(facility: SemanticLogger::Formatters::Syslog::LOG_USER, level_map: SemanticLogger::Formatters::Syslog::LevelMap.new,
27
+ max_size: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE)
28
+ SemanticLogger::Formatters::Syslog.require_syslog_protocol
29
+
30
+ self.facility = facility
31
+ self.level_map = level_map
32
+ @max_size = max_size
32
33
  super()
33
34
  end
34
35
 
36
+ # Set the facility to one of the ::Syslog::LOG_* facility constants.
37
+ # Raises ArgumentError for any other value, so that a mistake surfaces here rather than on
38
+ # every log entry.
39
+ def facility=(facility)
40
+ @facility_number = SemanticLogger::Formatters::Syslog.facility_number(facility)
41
+ @facility = facility
42
+ end
43
+
44
+ # Set the map of log levels to syslog severities, as a LevelMap or a Hash of overrides.
45
+ def level_map=(level_map)
46
+ @level_map = SemanticLogger::Formatters::Syslog::LevelMap.build(level_map)
47
+ end
48
+
35
49
  # Time is part of the syslog packet and is not included in the formatted message.
36
50
  def time
37
51
  end
@@ -47,7 +61,7 @@ module SemanticLogger
47
61
  def create_syslog_packet(message)
48
62
  packet = SyslogProtocol::Packet.new
49
63
  packet.hostname = logger.host
50
- packet.facility = facility
64
+ packet.facility = @facility_number
51
65
  packet.tag = logger.application.delete(" ")
52
66
  packet.content = message
53
67
  packet.time = log.time
@@ -23,11 +23,20 @@ module SemanticLogger
23
23
  when String
24
24
  LEVELS.index(level.downcase.to_sym)
25
25
  when Integer
26
- MAPPED_LEVELS[level]
26
+ # Bounded first, since Array#[] raises RangeError for an Integer too large for a C long.
27
+ MAPPED_LEVELS[level] if level.abs <= MAPPED_LEVELS.size
27
28
  end ||
28
29
  raise(ArgumentError, "Invalid level:#{level.inspect} being requested. Must be one of #{LEVELS.inspect}")
29
30
  end
30
31
 
32
+ # Returns [true|false] whether the supplied value is a level that #index accepts.
33
+ # Never raises, so it can be used to check values that come from untrusted data.
34
+ def self.valid?(level)
35
+ !index(level).nil?
36
+ rescue ArgumentError
37
+ false
38
+ end
39
+
31
40
  # Returns the symbolic level for the supplied level index
32
41
  def self.level(level_index)
33
42
  LEVELS[level_index]
@@ -8,7 +8,8 @@ module SemanticLogger
8
8
  # :trace, :debug, :info, :warn, :error, :fatal
9
9
  #
10
10
  # thread_name
11
- # Name of the thread in which the logging call was called
11
+ # Name of the thread in which the logging call was called,
12
+ # or its object id when the thread has no name
12
13
  #
13
14
  # name
14
15
  # Class name supplied to the logging instance
@@ -53,6 +54,40 @@ module SemanticLogger
53
54
  log_exception on_exception_level
54
55
  metric metric_amount dimensions].freeze
55
56
 
57
+ # Values accepted for the :log_exception argument.
58
+ LOG_EXCEPTION_OPTIONS = %i[full partial none].freeze
59
+ private_constant :LOG_EXCEPTION_OPTIONS
60
+
61
+ # A finite real number, which #assign can compare with another one.
62
+ REAL_NUMBER = ->(value) { value.is_a?(Numeric) && value.real? && value.finite? }
63
+ private_constant :REAL_NUMBER
64
+
65
+ # Checks that a value is valid for its argument, for the arguments that do not accept any value.
66
+ # Any other value would make #assign, or the formatters, raise.
67
+ ARGUMENT_VALIDATORS = {
68
+ backtrace: ->(value) { value.is_a?(Array) && value.all?(String) },
69
+ duration: REAL_NUMBER,
70
+ min_duration: REAL_NUMBER,
71
+ metric_amount: REAL_NUMBER,
72
+ dimensions: ->(value) { value.is_a?(Hash) },
73
+ log_exception: ->(value) { LOG_EXCEPTION_OPTIONS.include?(value) },
74
+ on_exception_level: ->(value) { Levels.valid?(value) }
75
+ }.freeze
76
+ private_constant :ARGUMENT_VALIDATORS
77
+
78
+ # Hash#key?, which unlike HashWithIndifferentAccess#key? only matches the exact Symbol key.
79
+ HASH_KEY = Hash.instance_method(:key?)
80
+ private_constant :HASH_KEY
81
+
82
+ # Whether the value is accepted for the :log_exception argument of #assign.
83
+ def self.log_exception_option?(value)
84
+ value.nil? || LOG_EXCEPTION_OPTIONS.include?(value)
85
+ end
86
+
87
+ # Returns [String] the name that identifies the thread in a log entry: the thread's own name,
88
+ # or its object id when it has none, which Ruby never reuses within a process.
89
+ def self.thread_name_for(thread) = thread.name || thread.object_id.to_s
90
+
56
91
  attr_accessor :level, :level_index, :name, :message, :time, :duration,
57
92
  :payload, :exception, :thread_name, :backtrace,
58
93
  :tags, :named_tags, :context,
@@ -60,7 +95,7 @@ module SemanticLogger
60
95
 
61
96
  def initialize(name, level, index = nil)
62
97
  @level = level
63
- @thread_name = Thread.current.name
98
+ @thread_name = Log.thread_name_for(Thread.current)
64
99
  @name = name
65
100
  @time = Time.now
66
101
  @tags = SemanticLogger.tags
@@ -94,7 +129,7 @@ module SemanticLogger
94
129
 
95
130
  if exception
96
131
  unless exception.is_a?(Exception)
97
- exception = ArgumentError.new("Invalid value for logger exception: #{exception.inspect}")
132
+ exception = ArgumentError.new("Invalid value for logger exception: #{Utils.safe_inspect(exception)}")
98
133
  exception.set_backtrace(Utils.extract_backtrace(caller))
99
134
  end
100
135
 
@@ -111,8 +146,10 @@ module SemanticLogger
111
146
  end
112
147
  # On exception change the log level
113
148
  if on_exception_level
114
- self.level = on_exception_level
115
- self.level_index = Levels.index(level)
149
+ # Normalized through the index, so that a level supplied as a String or an Integer is
150
+ # still stored as its Symbol.
151
+ self.level_index = Levels.index(on_exception_level)
152
+ self.level = Levels.level(level_index)
116
153
  end
117
154
  end
118
155
 
@@ -129,47 +166,74 @@ module SemanticLogger
129
166
  true
130
167
  end
131
168
 
132
- # Keys that #assign_hash may assign directly to a log attribute. Every other
133
- # key is folded into the payload, preventing a supplied hash from overwriting
134
- # sensitive fields such as :level, :name, or :time, and keeping this path
135
- # consistent with #extract_arguments.
136
- ASSIGNABLE_KEYS = (NON_PAYLOAD_KEYS + %i[payload]).freeze
169
+ # Log attributes that a Hash returned by a logging block can set (see #assign_hash). Its other
170
+ # keys go to the payload, so that it cannot set fields such as :level, :name, or :time.
171
+ ASSIGNABLE_KEYS = %i[message exception backtrace duration metric metric_amount dimensions].freeze
137
172
 
138
- # Assign known keys to self, all other keys to the payload.
173
+ # Assign the Hash returned by a logging block, after #assign has applied the log call.
174
+ #
175
+ # A block can return untrusted data, such as a parsed request body, so its keys are classified
176
+ # by #extract_arguments, as for a Hash passed after the message. A log argument is only assigned
177
+ # when this entry does not have it yet (an exception must also be an Exception), and otherwise
178
+ # goes to the payload like any other key. Whatever the key order, the log call's values win,
179
+ # then the block's top-level keys, then those of its :payload. No supplied Hash is modified.
139
180
  def assign_hash(hash)
140
- self.payload ||= {}
141
- hash.each_pair do |key, value|
142
- if ASSIGNABLE_KEYS.include?(key) && respond_to?(:"#{key}=")
143
- public_send(:"#{key}=", value)
144
- else
145
- payload[key] = value
146
- end
147
- end
148
- self.payload = nil if payload.empty?
181
+ args = extract_arguments(hash)
182
+ added = args.delete(:payload)
183
+ assigned, kept = args.partition { |key, value| block_attribute?(key, value) }
184
+ assigned.each { |key, value| public_send(:"#{key}=", value) }
185
+ # An argument this entry already has wins over the block's :payload, like the block's other keys.
186
+ added = merge_payload(kept.to_h, added) unless kept.empty?
187
+ self.payload = merge_payload(payload, added)
149
188
  self
150
189
  end
151
190
 
191
+ # Returns [true|false] whether the value can be used as the log argument named by the key.
192
+ #
193
+ # Never raises, so it can check values from untrusted data before they are passed to #assign.
194
+ # A nil value is never an argument, since it means the argument was not supplied. Arguments
195
+ # without a validator in ARGUMENT_VALIDATORS (such as :message, :metric, or :exception, which
196
+ # #assign logs as an ArgumentError when it is not an Exception) accept any other value.
197
+ def self.argument?(key, value)
198
+ return false if value.nil? || !NON_PAYLOAD_KEYS.include?(key)
199
+
200
+ validator = ARGUMENT_VALIDATORS[key]
201
+ validator.nil? || validator.call(value)
202
+ end
203
+
152
204
  # Extract the arguments from a Hash Payload
205
+ #
206
+ # The hash can hold untrusted data, so this never fails because of its content: keys that are
207
+ # not log attributes, or that hold a value that is not valid for it (see .argument?), are kept
208
+ # in the payload. A log attribute key with a nil value counts as not supplied, so #assign uses
209
+ # its default. When the hash supplies an explicit :payload, any other keys are merged into it,
210
+ # and win over its values of the same name, since the explicit payload is where untrusted data
211
+ # goes (see docs/security.md).
153
212
  def extract_arguments(payload, message = nil)
154
213
  raise(ArgumentError, "payload must be a Hash") unless payload.is_a?(Hash)
155
214
 
156
- message = nil if message == ""
157
- if payload.key?(:payload)
158
- return message ? payload.merge(message: message) : payload
159
- end
160
-
215
+ message = nil if message == ""
216
+ explicit = HASH_KEY.bind_call(payload, :payload)
161
217
  new_payload = {}
162
218
  args = {}
163
219
  payload.each_pair do |key, value|
164
- # Supplied message takes precedence
165
- if (key == :message) && !message.nil?
166
- new_payload[key] = value
220
+ if explicit && key == :payload
221
+ args[:payload] = value
222
+ elsif key == :message && !message.nil?
223
+ # Supplied message takes precedence
224
+ new_payload[key] = value unless explicit
225
+ elsif Log.argument?(key, value)
226
+ args[key] = value
227
+ elsif value.nil? && NON_PAYLOAD_KEYS.include?(key)
228
+ # Not supplied, so #assign uses its default
167
229
  next
230
+ else
231
+ new_payload[key] = value
168
232
  end
169
-
170
- NON_PAYLOAD_KEYS.include?(key) ? args[key] = value : new_payload[key] = value
171
233
  end
172
- args[:payload] = new_payload unless new_payload.empty?
234
+ unless new_payload.empty?
235
+ args[:payload] = explicit ? merge_payload(new_payload, args[:payload]) : new_payload
236
+ end
173
237
  args[:message] = message if message
174
238
  args
175
239
  end
@@ -197,14 +261,52 @@ module SemanticLogger
197
261
  end
198
262
  end
199
263
 
264
+ # Returns [String] the detail that Ruby's Exception#detailed_message adds to the message of the
265
+ # supplied exception, without the class name that Ruby also adds. Since Ruby 3.2 this is where
266
+ # did_you_mean suggestions ("Did you mean? upcase") and the detail an exception class supplies
267
+ # by overriding #detailed_message appear, rather than in #message.
268
+ # Returns nil when there is no detail.
269
+ #
270
+ # Parameters
271
+ # exception: [Exception]
272
+ # This entry's exception, or one of its causes (see #each_exception).
273
+ # source: [true|false]
274
+ # Whether to include error_highlight's snippet of the failing source line, for example
275
+ # "user_name.upcas" with "^^^^^^" under the unknown method. It re-parses that source
276
+ # file on every call, so it is about 100 times slower, and it copies source code into
277
+ # the log.
278
+ # Default: false
279
+ #
280
+ # The result is cached on this entry, since every appender's formatter may ask for it.
281
+ def exception_details(exception, source: false)
282
+ key = [exception, source]
283
+ cache = @exception_details
284
+ return cache[key] if cache&.key?(key)
285
+
286
+ details = ExceptionDetails.extract(exception, source: source)
287
+ # Replace the cache rather than change it, since appenders on other threads may be reading it.
288
+ @exception_details = (cache || {}).merge(key => details).freeze
289
+ details
290
+ end
291
+
200
292
  # Returns [String] the exception backtrace including all of the child / caused by exceptions
201
- def backtrace_to_s
293
+ #
294
+ # When a block is supplied, each cause's message and each backtrace line is passed
295
+ # to it, and the value it returns is written in its place. This lets a caller, for
296
+ # example a formatter, transform that text while this method keeps the layout.
297
+ #
298
+ # Example:
299
+ # log.backtrace_to_s { |text| text.gsub("\n", "\\n") }
300
+ def backtrace_to_s(&block)
202
301
  trace = ""
203
302
  each_exception do |exception, i|
303
+ lines = exception.backtrace || []
304
+ lines = lines.map(&block) if block
204
305
  if i.zero?
205
- trace = (exception.backtrace || []).join("\n")
306
+ trace = lines.join("\n")
206
307
  else
207
- trace << "\nCause: #{exception.class.name}: #{exception.message}\n#{(exception.backtrace || []).join("\n")}"
308
+ message = block ? yield(exception.message) : exception.message
309
+ trace << "\nCause: #{exception.class.name}: #{message}\n#{lines.join("\n")}"
208
310
  end
209
311
  end
210
312
  trace
@@ -281,8 +383,13 @@ module SemanticLogger
281
383
  # terminal escape codes out of structured (JSON/Loki) output. It is distinct
282
384
  # from Formatters::Base#escape_control_characters, which instead *escapes*
283
385
  # (preserves) control characters and is opt-in for the text formatters.
386
+ #
387
+ # A message tagged UTF-8 can hold invalid bytes, for example a request parameter that Rack
388
+ # decoded from "%FF", or be in another encoding, such as a binary request body. Stripping or
389
+ # matching such a String raises, so it is converted to valid UTF-8 first, dropping the
390
+ # invalid bytes, as the JSON formatters do.
284
391
  def cleansed_message
285
- msg = message.to_s
392
+ msg = Utils.encode_utf8_string(message.to_s)
286
393
  return msg.strip unless msg.include?("\e")
287
394
 
288
395
  msg.gsub(/\e\[[\d;]*[mz]?|\e/, "").strip
@@ -290,8 +397,12 @@ module SemanticLogger
290
397
 
291
398
  # Return the payload in text form
292
399
  # Returns nil if payload is missing or empty
293
- def payload_to_s
294
- payload.inspect if payload?
400
+ #
401
+ # Rendered with #inspect, or with the block when given (for example Amazing Print). A payload
402
+ # nested too deeply for either is first cut off at Utils::MAX_JSON_SAFE_DEPTH, rather than
403
+ # raising SystemStackError (see Utils.safe_inspect).
404
+ def payload_to_s(&)
405
+ Utils.safe_inspect(payload, &) if payload?
295
406
  end
296
407
 
297
408
  # Returns [true|false] whether the log entry has a payload
@@ -314,5 +425,40 @@ module SemanticLogger
314
425
  def metric_only?
315
426
  metric && message.nil? && exception.nil?
316
427
  end
428
+
429
+ private
430
+
431
+ # Returns the kept payload with the keys of the added one that it does not have, so that a kept
432
+ # value is never replaced. A String and a Symbol key of the same name, such as "account_id" from
433
+ # JSON.parse and :account_id, count as the same key, since the formatters write both under that
434
+ # name. A payload that is not a Hash is kept as its :payload key, but the kept payload is
435
+ # returned unchanged when there is nothing to add. Neither payload is modified.
436
+ def merge_payload(kept, added)
437
+ added = {payload: added} unless added.nil? || added.is_a?(Hash)
438
+ return kept if added.nil? || added.empty?
439
+
440
+ kept = {payload: kept} unless kept.nil? || kept.is_a?(Hash)
441
+ return added if kept.nil? || kept.empty?
442
+
443
+ merged = kept.merge(added) { |_key, kept_value, _added_value| kept_value }
444
+ # Only the kept keys, usually a few supplied by the application, are checked one at a time.
445
+ kept.each_key do |key|
446
+ next unless key.is_a?(Symbol) || key.is_a?(String)
447
+
448
+ other = key.is_a?(Symbol) ? key.name : key.to_sym
449
+ merged.delete(other) unless kept.key?(other)
450
+ end
451
+ merged
452
+ end
453
+
454
+ # Whether a log argument from the Hash returned by a logging block, already checked by
455
+ # .argument?, is assigned to its attribute: only when this entry does not have it yet. As in
456
+ # #extract_arguments, an empty message counts as none.
457
+ def block_attribute?(key, value)
458
+ return message.nil? || message == "" if key == :message
459
+ return exception.nil? && value.is_a?(Exception) if key == :exception
460
+
461
+ ASSIGNABLE_KEYS.include?(key) && public_send(key).nil?
462
+ end
317
463
  end
318
464
  end
@@ -88,17 +88,40 @@ module SemanticLogger
88
88
  # return false
89
89
  # end
90
90
 
91
- index = Levels.index(level)
91
+ # These values are written into the source of the generated method, so only accept values
92
+ # that are certain to produce valid literals, rather than code.
93
+ #
94
+ # Ruby writes a Symbol that is a valid method name, including operators such as `[]=` and
95
+ # non-ASCII identifiers, without quotes. Any other Symbol is quoted, as in `:"a; b"`.
96
+ # Instance, class, and global variable names are unquoted, but are not method names.
97
+ source_name = method_name.to_s.to_sym.inspect.delete_prefix(":")
98
+ if source_name.start_with?('"', "@", "$")
99
+ raise(ArgumentError, "Invalid method name to measure: #{method_name.inspect}")
100
+ end
101
+ unless Log.log_exception_option?(log_exception)
102
+ raise(ArgumentError, "Invalid value:#{log_exception.inspect} for argument :log_exception")
103
+ end
104
+
105
+ duration = Float(min_duration, exception: false)
106
+ unless duration&.finite?
107
+ raise(ArgumentError, "Invalid value:#{min_duration.inspect} for argument :min_duration")
108
+ end
109
+
110
+ index = Levels.index(level)
111
+ level = Levels.level(index)
112
+ on_exception_level = Levels.level(Levels.index(on_exception_level)) if on_exception_level
113
+ message = message&.to_s
114
+ metric = metric&.to_s
92
115
 
93
116
  logger_measure_module.module_eval(<<~MEASURE_METHOD, __FILE__, __LINE__ + 1)
94
- def #{method_name}(*args, &block)
117
+ def #{source_name}(*args, &block)
95
118
  if logger.send(:level_index) <= #{index}
96
119
  logger.send(
97
120
  :measure_method,
98
121
  index: #{index},
99
122
  level: #{level.inspect},
100
123
  message: #{message.inspect},
101
- min_duration: #{min_duration},
124
+ min_duration: #{duration},
102
125
  metric: #{metric.inspect},
103
126
  log_exception: #{log_exception.inspect},
104
127
  on_exception_level: #{on_exception_level.inspect}
@@ -52,7 +52,7 @@ module SemanticLogger
52
52
  end
53
53
 
54
54
  def log(log)
55
- name = formatter.call(log, self)
55
+ name = format_log(log)
56
56
  if (duration = log.duration)
57
57
  # Convert duration to seconds
58
58
  ::NewRelic::Agent.record_metric(name, duration / 1000.0)