semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -0,0 +1,113 @@
1
+ module SemanticLogger
2
+ # Internal: locks a Mutex only where Ruby allows it, for code that can run in a signal trap, such as
3
+ # a trap that logs, or one that reopens the log files after a rotation.
4
+ #
5
+ # Ruby raises ThreadError for Mutex#lock in a trap. Queue and Thread methods are allowed there, so
6
+ # a caller that cannot lock leaves its work on a Queue, or to a new Thread, instead.
7
+ module TrapSafe
8
+ # Seconds that .compute_if_absent waits in a signal trap for a new thread to add the value.
9
+ ADD_WAIT_SECONDS = 0.1
10
+
11
+ # Runs the block holding the mutex, and returns true. Returns false, without running the block,
12
+ # where the mutex cannot be locked: in a signal trap, or on a thread that already holds it, such
13
+ # as one that a signal trap interrupted while it did. A ThreadError raised by the block itself is
14
+ # raised again.
15
+ def self.try_synchronize(mutex)
16
+ return false if mutex.owned?
17
+
18
+ held = false
19
+ mutex.synchronize do
20
+ held = true
21
+ yield
22
+ end
23
+ true
24
+ rescue ThreadError
25
+ raise if held
26
+
27
+ false
28
+ end
29
+
30
+ # Returns [true|false] whether the calling code runs in a signal trap, for code that writes to a
31
+ # structure that locks a Mutex of its own, such as a Concurrent::Map on CRuby, which it then must
32
+ # not write to. A new Mutex is never held, so only a trap keeps it from being locked.
33
+ def self.in_trap?
34
+ ::Mutex.new.synchronize { false }
35
+ rescue ThreadError
36
+ true
37
+ end
38
+
39
+ # Returns the value that the Concurrent::Map holds for the key, adding the one that the block
40
+ # returns when it holds none, as Concurrent::Map#compute_if_absent does.
41
+ #
42
+ # Adding to it locks a Mutex on CRuby, so in a signal trap a new thread adds the block's value,
43
+ # which the trap waits for, at most ADD_WAIT_SECONDS (see .hand_off), then returns the value that
44
+ # the map holds. The new thread cannot add it while the thread that the trap interrupted is adding
45
+ # to the map, holding its Mutex: the trap then returns the block's value, and the new thread adds
46
+ # it just after the trap, unless another thread has added one for the key by then.
47
+ def self.compute_if_absent(map, key, &)
48
+ value = map[key]
49
+ return value unless value.nil?
50
+ return map.compute_if_absent(key, &) unless in_trap?
51
+
52
+ value = yield
53
+ hand_off(ADD_WAIT_SECONDS) { map.compute_if_absent(key) { value } } || value
54
+ end
55
+
56
+ # Runs the block on a new thread, for code in a signal trap whose work locks a Mutex, and waits for
57
+ # it at most the given seconds, since that thread may need a lock that the thread the trap
58
+ # interrupted holds, which it cannot release before the trap returns.
59
+ #
60
+ # Returns the block's value, or raises what it raised, when it finished in time. Otherwise returns
61
+ # nil, and the block finishes after that, when Ruby reports what it raises, if anything, as for any
62
+ # thread (see Thread#report_on_exception). Either way an exception is raised or reported, never
63
+ # both: the caller and the thread each take the one token of a queue, without waiting, as the
64
+ # caller stops waiting and as the block finishes, and whichever takes it first decides.
65
+ def self.hand_off(seconds)
66
+ started = Thread::Queue.new
67
+ token = Thread::Queue.new
68
+ done = Thread::Queue.new
69
+ token << true
70
+ decided = false
71
+ thread = nil
72
+ begin
73
+ thread = Thread.new do
74
+ started.pop
75
+ yield
76
+ ensure
77
+ # Ruby reports what the block raised only when the caller took the token, as it stopped waiting.
78
+ Thread.current.report_on_exception = token.pop(timeout: 0).nil?
79
+ done << true
80
+ end
81
+ started << true
82
+ # Waits on a queue rather than Thread#join, which would raise what the block raised before the
83
+ # outcome is decided. The thread took the token when it has finished, or is about to, so the
84
+ # caller gets the outcome. Otherwise the caller takes it as it stops waiting, so that Ruby
85
+ # reports what the block raises.
86
+ caller_gets = done.pop(timeout: seconds) || !token.pop(timeout: 0)
87
+ decided = true
88
+ caller_gets ? thread.value : nil
89
+ ensure
90
+ report_outcome(thread, token, started) if thread && !decided
91
+ end
92
+ end
93
+
94
+ # Called when the caller of .hand_off is interrupted, such as by another signal, before it decided who
95
+ # gets the block's outcome: it lets the thread start, then takes the token if the thread has not, so
96
+ # that Ruby reports what the block raises, and otherwise, once the thread has finished, reports it
97
+ # here, as Ruby would have, to standard error, so that it is never lost. An interrupt while it waits
98
+ # for the thread is raised, not reported as the block's.
99
+ def self.report_outcome(thread, token, started)
100
+ started << true
101
+ return if token.pop(timeout: 0)
102
+
103
+ begin
104
+ thread.join
105
+ rescue Exception => e
106
+ raise if thread.alive?
107
+
108
+ $stderr.write("#{thread.inspect} terminated with exception (report_on_exception is true):\n#{e.full_message}")
109
+ end
110
+ end
111
+ private_class_method :report_outcome
112
+ end
113
+ end
@@ -1,5 +1,6 @@
1
1
  require "json"
2
2
  require "uri"
3
+ require "semantic_logger/core_ext/warning"
3
4
 
4
5
  module SemanticLogger
5
6
  # Internal-use only utility functions for Semantic Logger.
@@ -58,6 +59,36 @@ module SemanticLogger
58
59
  "(invalid url)"
59
60
  end
60
61
 
62
+ # Returns the value, deep frozen so that code in any Ractor can read the constant or setting
63
+ # that holds it.
64
+ #
65
+ # copy: false freezes the value itself, so only use it for a value that Semantic Logger owns, such
66
+ # as one of its constants. A value this Ruby cannot share, such as an UnboundMethod before Ruby 4.0,
67
+ # is returned unshareable, and may be left partly frozen. On a Ruby without Ractors (JRuby) the
68
+ # value is returned as it is.
69
+ #
70
+ # copy: true leaves the supplied value as it is, and returns a deep frozen copy of it, for a value
71
+ # that belongs to someone else, such as a setting. Without Ractors, or for a value Ruby cannot
72
+ # copy, the copy is a shallow frozen dup.
73
+ def self.make_shareable(value, copy: false)
74
+ return copy ? frozen_dup(value) : value unless defined?(::Ractor)
75
+
76
+ begin
77
+ ::Ractor.make_shareable(value, copy: copy)
78
+ rescue ::Ractor::Error, TypeError
79
+ copy ? frozen_dup(value) : value
80
+ end
81
+ end
82
+
83
+ # Returns a frozen shallow copy of the value, the value when it is already frozen, or the value
84
+ # itself when it cannot be copied, such as a Thread.
85
+ def self.frozen_dup(value)
86
+ value.frozen? ? value : value.dup.freeze
87
+ rescue TypeError
88
+ value
89
+ end
90
+ private_class_method :frozen_dup
91
+
61
92
  # Borrow from Rails, when not running Rails
62
93
  def self.camelize(term)
63
94
  string = term.to_s
@@ -67,33 +98,18 @@ module SemanticLogger
67
98
  string
68
99
  end
69
100
 
70
- # Returns the visibility for an instance method
71
- def self.method_visibility(mod, method_name)
72
- method_name = method_name.to_sym
73
- if mod.instance_methods.include?(method_name)
74
- :public
75
- elsif mod.private_instance_methods.include?(method_name)
76
- :private
77
- elsif mod.protected_instance_methods.include?(method_name)
78
- :protected
79
- end
80
- end
81
-
82
101
  # Extract the backtrace stripping off the leading semantic logger entries.
83
102
  # Leaves all other system and gem path entries in place.
84
103
  # Returns a new Array, leaving the supplied stack unchanged since it can belong to the caller.
85
- def self.extract_backtrace(stack = caller)
86
- stack.drop_while { |path| extract_path?(path) }
87
- end
104
+ def self.extract_backtrace(stack = caller) = stack.drop_while { |path| extract_path?(path) }
88
105
 
89
- def self.extract_paths
90
- @extract_paths ||= %w[lib/semantic_logger lib/rails_semantic_logger]
91
- end
106
+ # Paths whose entries are stripped from the top of a captured backtrace.
107
+ EXTRACT_PATHS = make_shareable(%w[lib/semantic_logger lib/rails_semantic_logger].freeze)
108
+
109
+ def self.extract_paths = EXTRACT_PATHS
92
110
 
93
111
  # Whether this path should be excluded from any cleansed backtrace
94
- def self.extract_path?(path)
95
- extract_paths.any? { |exclude| path.include?(exclude) }
96
- end
112
+ def self.extract_path?(path) = extract_paths.any? { |exclude| path.include?(exclude) }
97
113
 
98
114
  # Try to strip everything off of the supplied backtrace, until the first application stack entry is at the top.
99
115
  # For example all leading gem paths and built-in ruby code paths are removed from the top.
@@ -108,17 +124,46 @@ module SemanticLogger
108
124
  # Paths to exclude in the stripped backtrace
109
125
  # Includes Gems and built-in Ruby code paths
110
126
  def self.strip_paths
111
- @strip_paths ||=
112
- begin
113
- paths = Gem.path | [Gem.default_dir]
114
- paths << RbConfig::CONFIG["rubylibdir"]
115
- paths
116
- end
127
+ # A copy, since the paths are RubyGems' and RbConfig's own Strings.
128
+ @strip_paths ||= make_shareable(Gem.path | [Gem.default_dir, RbConfig::CONFIG["rubylibdir"]], copy: true)
117
129
  end
118
130
 
119
131
  # Whether this path should be excluded from any cleansed backtrace
120
- def self.strip_path?(path)
121
- strip_paths.any? { |exclude| path.start_with?(exclude) }
132
+ def self.strip_path?(path) = strip_paths.any? { |exclude| path.start_with?(exclude) }
133
+
134
+ # Returns [String] the name of the exception's class as Ruby names it, as in Exception#inspect and
135
+ # #full_message: the constant it was assigned to, such as "Net::ReadTimeout", or, for an
136
+ # anonymous class, which has none, the class as Ruby writes it, such as `#<Class:0x000...>`. It is
137
+ # in UTF-8, whatever the constant's encoding, such as ISO-8859-1, so that it joins with UTF-8 text.
138
+ # Every place that writes an exception's class asks this, so that they all name it alike.
139
+ #
140
+ # Never runs the class's own code, which can override self.name, self.to_s, or self.inspect with
141
+ # code that raises, or that names the class otherwise. Module's own #to_s, bound to the class,
142
+ # returns what Module#name returns when the class has a name, and the class written out when not.
143
+ def self.exception_name(exception) = encode_utf8_string(Module.instance_method(:to_s).bind_call(exception.class))
144
+
145
+ # Returns [String] the exception's message, without raising the error that its #message raises.
146
+ #
147
+ # An exception class can override #message or #to_s with code that raises, such as code that
148
+ # reads a record that is nil, or an abstract #message that raises NotImplementedError. Logging the
149
+ # exception must not replace it with that error, such as the one that a measure_* block re-raises.
150
+ # When #message raises a StandardError or a ScriptError, or does not return a String or an object
151
+ # that converts to one, the message the exception was raised with is returned instead (see
152
+ # .raised_message). Other errors, such as Interrupt, still propagate.
153
+ def self.exception_message(exception)
154
+ String(exception.message)
155
+ rescue StandardError, ScriptError
156
+ raised_message(exception)
157
+ end
158
+
159
+ # Returns [String] the message the exception was raised with, without what its class's own
160
+ # #message or #to_s adds: its class name when it was raised without one. Its class name too when
161
+ # that message cannot be converted to a String, since converting it raised a StandardError or a
162
+ # ScriptError (see .exception_message).
163
+ def self.raised_message(exception)
164
+ Exception.instance_method(:to_s).bind_call(exception)
165
+ rescue StandardError, ScriptError
166
+ exception_name(exception)
122
167
  end
123
168
 
124
169
  # Serializes the value to JSON, repairing the data only when necessary.
@@ -129,6 +174,9 @@ module SemanticLogger
129
174
  # repaired copy when serialization fails because of the data:
130
175
  # * Invalid UTF-8. Depending on the json gem version this raises
131
176
  # Encoding::UndefinedConversionError (an EncodingError) or JSON::GeneratorError.
177
+ # From json 3.1 so does a binary String that holds bytes outside ASCII, even valid UTF-8,
178
+ # which earlier versions write as UTF-8, with a warning that is dropped for log data (see
179
+ # CoreExt::Warning).
132
180
  # * Infinity and NaN, which raise JSON::GeneratorError.
133
181
  # * Structures nested deeper than the json gem allows, which raise
134
182
  # JSON::NestingError (a JSON::ParserError, not a GeneratorError).
@@ -149,15 +197,35 @@ module SemanticLogger
149
197
  #
150
198
  # The retry is attempted only once: if it still fails (for example an object whose
151
199
  # own #to_json raises), the error propagates unchanged rather than being swallowed.
152
- def self.to_json(value)
153
- clean_json(value) || json_safe(value).to_json
154
- end
200
+ def self.to_json(value) = clean_json(value) || json_safe(value).to_json
155
201
 
156
202
  # Returns the value when JSON writes it as it is, otherwise a .json_safe copy, for a writer
157
203
  # that serializes the value itself, such as GELF. The check costs one #to_json; only the
158
204
  # rare value that needs repairing is copied.
205
+ #
206
+ # That includes a value holding a binary String that json 2.8 through 3.0.x warn about, since
207
+ # the writer's own serializing is not Semantic Logger's, so it would print the warning.
159
208
  def self.json_ready(value)
160
- clean_json(value) ? value : json_safe(value)
209
+ CoreExt::Warning.unless_binary_string_warned { clean_json(value) } ? value : json_safe(value)
210
+ end
211
+
212
+ # The methods of this file that serialize log data to JSON (see .serializing_log_data?), as
213
+ # their frames are named in a backtrace. A call one of them makes to a method written in C,
214
+ # such as Hash#to_json, is named after that method but located in this file: "to_json".
215
+ SERIALIZERS = make_shareable(%w[to_json clean_json json_safe_object json_safe_members].freeze)
216
+
217
+ # Returns [true|false] whether the code running now was called by this file to serialize log
218
+ # data, such as the json gem raising a warning (see CoreExt::Warning), or an object's own
219
+ # #to_json in a payload: whether the innermost of this file's frames on the stack is one of
220
+ # SERIALIZERS. When it is another of its methods, such as .exception_message calling
221
+ # an exception's #message, the code is not serializing log data, even if log data is being
222
+ # serialized further out.
223
+ #
224
+ # The stack is walked only as far as that frame, however deep it is, such as below nested
225
+ # objects whose #to_json serializes a Hash, so that the answer does not depend on the depth.
226
+ def self.serializing_log_data?
227
+ Thread.each_caller_location { |frame| return SERIALIZERS.include?(frame.base_label) if frame.path == __FILE__ }
228
+ false
161
229
  end
162
230
 
163
231
  # Returns [String] the value's JSON, or nil when .to_json has to repair the data.
@@ -183,9 +251,10 @@ module SemanticLogger
183
251
  # "#<" unless the Set holds an object. Nil where Set#inspect starts with "#<" too.
184
252
  INSPECTED_SET = (::Set.new.inspect.start_with?("#<") ? nil : "\"Set[").freeze
185
253
 
186
- # Nesting depth after which .json_safe replaces a Hash, Array, Struct, or Data with a
187
- # placeholder. Well below the json gem's default max_nesting of 100, leaving room for
188
- # the levels that a formatter wraps the payload in.
254
+ # Nesting depth after which .json_safe, and any other copy made by .bounded_structure,
255
+ # replaces a Hash, Array, Struct, or Data with a placeholder. Well below the json gem's
256
+ # default max_nesting of 100, leaving room for the levels that a formatter wraps the payload
257
+ # in.
189
258
  MAX_JSON_SAFE_DEPTH = 32
190
259
 
191
260
  # Returns [String] value#inspect, or what the block renders the value as (for example with
@@ -252,7 +321,8 @@ module SemanticLogger
252
321
  private_class_method :open_struct?
253
322
 
254
323
  # Returns a copy of the supplied value that JSON can represent:
255
- # * Every String is converted to valid UTF-8, as by .encode_utf8.
324
+ # * Every String is converted to valid UTF-8, as by .encode_utf8, and so is the name of every
325
+ # Symbol, such as a Hash key, that is not UTF-8 or ASCII (see .encode_utf8_symbol).
256
326
  # * Infinity and NaN floats become the Strings "Infinity", "-Infinity", and "NaN".
257
327
  # * A Struct, Data, or OpenStruct instance becomes a Hash of its members (from #to_h),
258
328
  # and a Set an Array, as ActiveSupport's #as_json does, so that its members follow
@@ -278,20 +348,24 @@ module SemanticLogger
278
348
  case value
279
349
  when String
280
350
  encode_utf8_string(value)
351
+ when Symbol then encode_utf8_symbol(value)
281
352
  when Float
282
353
  value.finite? ? value : value.to_s
283
- when Symbol, Integer, NilClass, TrueClass, FalseClass
354
+ when Integer, NilClass, TrueClass, FalseClass
284
355
  value
285
- when Hash, Array, ::Set, Struct, ::Data
286
- json_safe_structure(value, depth, ancestors)
287
356
  else
288
- open_struct?(value) ? json_safe_structure(value, depth, ancestors) : json_safe_object(value)
357
+ return json_safe_object(value) unless structure?(value)
358
+
359
+ bounded_structure(value, depth, ancestors) { |level, seen| json_safe_members(value, level, seen) }
289
360
  end
290
361
  end
291
362
 
292
- # Returns a copy of a Hash, Array, Set, Struct, Data, or OpenStruct, or the placeholder for
293
- # one nested too deeply or that holds itself.
294
- def self.json_safe_structure(value, depth, ancestors)
363
+ # Returns a bounded copy of a structure, such as a Hash or Array, for .json_safe, .encode_utf8,
364
+ # or an appender that copies log data for a gem that writes it itself: what the block returns for its
365
+ # members, given the next depth and the structures it is inside of, or, for one nested deeper
366
+ # than MAX_JSON_SAFE_DEPTH or that holds itself, a placeholder String naming its class, such
367
+ # as "#<Hash nested too deeply>", in place of the structure.
368
+ def self.bounded_structure(value, depth, ancestors)
295
369
  ancestors ||= []
296
370
  if depth >= MAX_JSON_SAFE_DEPTH || ancestors.any? { |ancestor| ancestor.equal?(value) }
297
371
  return "#<#{value.class} nested too deeply>"
@@ -299,12 +373,11 @@ module SemanticLogger
299
373
 
300
374
  ancestors.push(value)
301
375
  begin
302
- json_safe_members(value, depth + 1, ancestors)
376
+ yield depth + 1, ancestors
303
377
  ensure
304
378
  ancestors.pop
305
379
  end
306
380
  end
307
- private_class_method :json_safe_structure
308
381
 
309
382
  # Returns any other object as it is, when JSON can write it, otherwise its #to_s converted
310
383
  # to valid UTF-8.
@@ -316,72 +389,186 @@ module SemanticLogger
316
389
  end
317
390
  private_class_method :json_safe_object
318
391
 
319
- # Returns a copy of the elements of a Hash, Array, Set, Struct, or Data, each made safe
320
- # by .json_safe at the supplied depth.
392
+ # Returns a copy of the members of a Hash, Array, Set, Struct, Data, or OpenStruct (see
393
+ # .structure_members), each made safe by .json_safe at the supplied depth.
321
394
  def self.json_safe_members(value, depth, ancestors)
322
- case value
323
- when Hash
324
- value.each_with_object({}) do |(key, val), hash|
395
+ members = structure_members(value)
396
+ if members.is_a?(Hash)
397
+ members.each_with_object({}) do |(key, val), hash|
325
398
  key = json_safe(key, depth, ancestors)
326
399
  key = key.to_json if key.is_a?(Hash) || key.is_a?(Array)
327
400
  hash[key] = json_safe(val, depth, ancestors)
328
401
  end
329
- when Array, ::Set
330
- value.map { |element| json_safe(element, depth, ancestors) }
402
+ elsif json_array?(value)
403
+ members.map { |element| json_safe(element, depth, ancestors) }
331
404
  else
332
- # A Struct, Data, or OpenStruct. A #to_h that does not return a Hash is made safe one
333
- # level deeper, so that the recursion still ends.
334
- members = value.to_h
335
- members.is_a?(Hash) ? json_safe_members(members, depth, ancestors) : json_safe(members, depth, ancestors)
405
+ # A #to_h that does not return a Hash is made safe one level deeper, so that the
406
+ # recursion still ends.
407
+ json_safe(members, depth, ancestors)
336
408
  end
337
409
  end
338
410
  private_class_method :json_safe_members
339
411
 
340
- # Returns a copy of the supplied value with every String converted to valid UTF-8.
412
+ # Returns [true|false] whether the value is a structure that the JSON formatters write as an array
413
+ # of its own elements (see .structure_members): an Array or a Set.
414
+ def self.json_array?(value) = value.is_a?(Array) || value.is_a?(::Set)
415
+
416
+ # Returns the members of a structure (see .structure?), as the JSON formatters write them: a
417
+ # Hash or an Array as it is, the elements of a Set as an Array, and the #to_h of a Struct, Data,
418
+ # or OpenStruct, which is a Hash unless that method returns something else.
419
+ def self.structure_members(value)
420
+ case value
421
+ when Hash, Array
422
+ value
423
+ when ::Set
424
+ value.to_a
425
+ else
426
+ value.to_h
427
+ end
428
+ end
429
+
430
+ # Returns a copy of the supplied value with every String converted to valid UTF-8, and every
431
+ # Symbol whose name is not UTF-8 or ASCII converted the same way (see .encode_utf8_symbol).
341
432
  #
342
433
  # Recurses through Hash and Array structures, cleansing both keys and values.
343
434
  # Strings that are already valid UTF-8 are returned unchanged (the common case),
344
- # so the fast path allocates nothing. Any other value (Symbol, Numeric, Time, nil,
345
- # ...) is returned as-is.
435
+ # and so are Symbols that are UTF-8 or ASCII, so the fast path allocates nothing. Any
436
+ # other value (Numeric, Time, nil, a Struct, ...) is returned as-is, without looking
437
+ # inside it.
346
438
  #
347
- # Used by .to_json on the rare failing path, and directly by formatters that
348
- # serialize per value or emit to a non-JSON sink (where a single `.to_json`
349
- # rescue boundary cannot catch an intermediate failure).
350
- def self.encode_utf8(value)
439
+ # A Hash or Array nested deeper than MAX_JSON_SAFE_DEPTH, or that holds itself, is
440
+ # replaced where it is too deep or repeats with the placeholder that .json_safe writes,
441
+ # such as "#<Hash nested too deeply>", so that the copy is bounded, as .json_safe is.
442
+ #
443
+ # Used by formatters that serialize per value or emit to a non-JSON sink (where a
444
+ # single `.to_json` rescue boundary cannot catch an intermediate failure), and by
445
+ # appenders that hand log data to a gem that writes it itself.
446
+ def self.encode_utf8(value, depth = 0, ancestors = nil)
351
447
  case value
352
448
  when String
353
449
  encode_utf8_string(value)
354
- when Hash
355
- value.each_with_object({}) do |(key, val), hash|
356
- hash[encode_utf8(key)] = encode_utf8(val)
357
- end
358
- when Array
359
- value.map { |element| encode_utf8(element) }
450
+ when Symbol then encode_utf8_symbol(value)
451
+ when Hash, Array
452
+ bounded_structure(value, depth, ancestors) { |level, seen| encode_utf8_members(value, level, seen) }
360
453
  else
361
454
  value
362
455
  end
363
456
  end
364
457
 
458
+ # Returns a copy of the members of a Hash or Array, each converted by .encode_utf8.
459
+ def self.encode_utf8_members(value, depth, ancestors)
460
+ return value.map { |element| encode_utf8(element, depth, ancestors) } if value.is_a?(Array)
461
+
462
+ value.each_with_object({}) do |(key, val), hash|
463
+ hash[encode_utf8(key, depth, ancestors)] = encode_utf8(val, depth, ancestors)
464
+ end
465
+ end
466
+ private_class_method :encode_utf8_members
467
+
365
468
  # Options used when transcoding a string to UTF-8.
366
469
  # Invalid byte sequences and characters that cannot be represented in UTF-8 are
367
470
  # dropped rather than substituted, matching the preference in issue #180.
368
471
  ENCODE_UTF8_OPTIONS = {invalid: :replace, undef: :replace, replace: "".freeze}.freeze
369
472
 
370
473
  # Returns the string converted to valid UTF-8, dropping any invalid bytes.
474
+ #
475
+ # Binary text, such as a response body, Base64-decoded text, or a C extension's error message,
476
+ # and US-ASCII text that holds bytes outside ASCII, such as a file read where no locale is set,
477
+ # do not say which characters those bytes are. They are read as the UTF-8 they usually hold,
478
+ # keeping its characters, as the json gem did before 3.1. Converting them would drop every byte
479
+ # outside ASCII, valid UTF-8 or not, since neither encoding defines one. Text in any other
480
+ # encoding, such as ISO-8859-1, is converted, and text tagged UTF-16 or UTF-32 (not UTF-16LE
481
+ # and the like) without a byte order mark is read as big-endian (see .encode_byte_ordered).
371
482
  def self.encode_utf8_string(string)
372
- return string if string.encoding == Encoding::UTF_8 && string.valid_encoding?
373
-
374
- if string.encoding == Encoding::UTF_8
375
- # Correctly tagged as UTF-8 but contains invalid byte sequences.
376
- string.scrub("")
483
+ case string.encoding
484
+ when Encoding::UTF_8
485
+ string.valid_encoding? ? string : string.scrub("")
486
+ when Encoding::BINARY, Encoding::US_ASCII
487
+ read_as_utf8(string)
488
+ when Encoding::UTF_16, Encoding::UTF_32 then encode_byte_ordered(string)
377
489
  else
378
- # Different encoding (e.g. ASCII-8BIT / Latin-1): transcode into UTF-8.
379
490
  string.encode(Encoding::UTF_8, **ENCODE_UTF8_OPTIONS)
380
491
  end
381
492
  rescue EncodingError
382
- # Last resort for encodings without a converter to UTF-8: reinterpret the
383
- # raw bytes as UTF-8 and drop anything invalid. Logging must never raise.
384
- string.dup.force_encoding(Encoding::UTF_8).scrub("")
493
+ # Last resort for encodings without a converter to UTF-8. Logging must never raise.
494
+ read_as_utf8(string)
495
+ end
496
+
497
+ # Returns [Symbol] the Symbol, or, when its name is neither UTF-8 nor ASCII, a Symbol named with
498
+ # that name converted to valid UTF-8 (see .encode_utf8_string).
499
+ #
500
+ # A Symbol is named in the encoding of the text it was made from, such as a binary key from a
501
+ # request turned into a Symbol, which JSON cannot always write once it holds bytes outside
502
+ # ASCII: from json 3.1 not even valid UTF-8, and on CRuby with earlier versions not invalid
503
+ # bytes, nor, from json 2.9, UTF-8 bytes tagged US-ASCII. A UTF-8 Symbol is always valid.
504
+ def self.encode_utf8_symbol(symbol)
505
+ symbol.name.ascii_only? || symbol.encoding == Encoding::UTF_8 ? symbol : encode_utf8_string(symbol.name).to_sym
506
+ end
507
+
508
+ # Returns [String] a copy of the string's bytes read as UTF-8, without its invalid bytes.
509
+ private_class_method def self.read_as_utf8(string) = String.new(string, encoding: Encoding::UTF_8).scrub!("")
510
+
511
+ # Returns [String] UTF-16 or UTF-32 text converted to valid UTF-8. These encodings say their
512
+ # byte order with a byte order mark at the start, and Ruby's converter drops all the text that
513
+ # does not have one. Unicode reads such text as big-endian, and so does this.
514
+ private_class_method def self.encode_byte_ordered(string)
515
+ big_endian = string.encoding == Encoding::UTF_16 ? Encoding::UTF_16BE : Encoding::UTF_32BE
516
+ mark = "\uFEFF".encode(big_endian).b
517
+ string = String.new(string, encoding: big_endian) unless string.b.start_with?(mark, mark.reverse)
518
+ string.encode(Encoding::UTF_8, **ENCODE_UTF8_OPTIONS)
519
+ end
520
+
521
+ # Returns [String] the texts joined into a new String, such as a log message and an exception's
522
+ # message, whatever their encodings.
523
+ #
524
+ # Texts are joined as they are, so that text in an encoding such as ISO-8859-1 can still be
525
+ # joined with the application's other text in that encoding. Two texts are converted to valid
526
+ # UTF-8 first (see .encode_utf8_string) only when joining them would raise, such as for UTF-8
527
+ # and binary texts that both hold bytes outside ASCII, or would make text that is not binary
528
+ # binary, such as an ASCII message joined with a binary exception message.
529
+ #
530
+ # Each text is formatted as interpolation would, rather than added or converted with #to_s, so
531
+ # that a String subclass whose #+ escapes the text added to it, such as
532
+ # ActiveSupport::SafeBuffer, is joined as plain text, and an object whose #to_s does not return a
533
+ # String is still joined.
534
+ #
535
+ # Plain Strings that are all UTF-8, the usual texts, always join as they are, so they are joined
536
+ # without those checks, which take several times as long. With an explicit separator, a new
537
+ # empty String, so that a global `$,` set by another library is not added between them.
538
+ def self.join_texts(*texts)
539
+ return texts.join(+"") if texts.all? { |text| text.instance_of?(String) && text.encoding == Encoding::UTF_8 }
540
+
541
+ texts.map { |text| format("%s", text) }.reduce do |joined, text|
542
+ encoding = Encoding.compatible?(joined, text)
543
+ joinable = encoding && (encoding != Encoding::BINARY || joined.encoding == Encoding::BINARY)
544
+ joinable ? "#{joined}#{text}" : "#{encode_utf8_string(joined)}#{encode_utf8_string(text)}"
545
+ end
546
+ end
547
+
548
+ # Returns [String] the lines, such as those of a backtrace, joined into a new String with a line
549
+ # break between each, whatever their encodings: as they are, as Array#join joins them, or, when
550
+ # that would raise, such as for a UTF-8 line and an ISO-8859-1 one that both hold bytes outside
551
+ # ASCII, with each line formatted as .join_texts formats a text and converted to valid UTF-8
552
+ # (see .encode_utf8_string), which leaves a valid UTF-8 line as it is.
553
+ #
554
+ # Joined text that is binary or US-ASCII and holds bytes outside ASCII is converted to valid UTF-8
555
+ # too, such as for an ASCII line and a binary one, or for Ruby's own backtrace lines, which are
556
+ # US-ASCII where no locale is set, in a directory such as café. Unlike .join_texts, which keeps
557
+ # text binary when it starts with binary text, such as when every text is binary, this converts
558
+ # such text even when every line is binary. Each line of such text is in that encoding or
559
+ # ASCII-only, so it is converted whole, as each line would be. Its encoding is checked first,
560
+ # since checking that it is ASCII-only reads the whole text.
561
+ #
562
+ # Lines are converted only when they cannot be joined as they are, which is rare, since checking
563
+ # each of what can be dozens of lines would double the time it takes to format an exception.
564
+ # Each is then converted once and the lines joined once, rather than pair by pair as .join_texts
565
+ # joins texts, which for a backtrace of thousands of lines takes a time that grows with the
566
+ # square of their number.
567
+ def self.join_lines(lines)
568
+ text = lines.join("\n")
569
+ (text.encoding in Encoding::BINARY | Encoding::US_ASCII) && !text.ascii_only? ? encode_utf8_string(text) : text
570
+ rescue Encoding::CompatibilityError
571
+ lines.map { |line| encode_utf8_string(format("%s", line)) }.join("\n")
385
572
  end
386
573
  end
387
574
  end
@@ -1,3 +1,3 @@
1
1
  module SemanticLogger
2
- VERSION = "5.2.0".freeze
2
+ VERSION = "5.3.0".freeze
3
3
  end
@@ -1,11 +1,18 @@
1
1
  require "concurrent"
2
2
  require "semantic_logger/version"
3
3
  require "semantic_logger/utils"
4
+ require "semantic_logger/trap_safe"
4
5
  require "semantic_logger/ansi_colors"
5
6
  require "semantic_logger/levels"
7
+ require "semantic_logger/ractor_support"
8
+ require "semantic_logger/deprecation"
9
+ require "semantic_logger/context"
6
10
  require "semantic_logger/base"
7
11
  require "semantic_logger/formatters"
8
12
  require "semantic_logger/exception_details"
13
+ require "semantic_logger/backtrace_cleaning"
14
+ require "semantic_logger/redactor"
15
+ require "semantic_logger/call_site"
9
16
  require "semantic_logger/log"
10
17
  require "semantic_logger/subscriber"
11
18
  require "semantic_logger/loggable"
@@ -19,6 +26,13 @@ require "semantic_logger/logger"
19
26
  require "semantic_logger/debug_as_trace_logger"
20
27
  require "semantic_logger/semantic_logger"
21
28
 
29
+ # Logging from Ractors other than the main one (experimental), on Ruby 4.0 and later.
30
+ if SemanticLogger::RactorSupport.enabled?
31
+ require "semantic_logger/ractor_copy"
32
+ require "semantic_logger/ractor_receiver"
33
+ require "semantic_logger/ractor_processor"
34
+ end
35
+
22
36
  # Automatically reopen appenders in the child process after a fork.
23
37
  # Enabled by default; opt out with `SemanticLogger.reopen_on_fork = false`.
24
38
  # Skipped on platforms without `Process._fork` (e.g. JRuby), which cannot fork.
@@ -27,9 +41,28 @@ if Process.respond_to?(:_fork)
27
41
  Process.singleton_class.prepend(SemanticLogger::CoreExt::Process)
28
42
  end
29
43
 
30
- # Flush all appenders at exit, waiting for outstanding messages on the queue
31
- # to be written first.
44
+ # json 2.8.0 through 3.0.x warn about each binary String they write as UTF-8. Drop that warning
45
+ # when it comes from Semantic Logger serializing log data, which it writes the same way with
46
+ # json 3.1 (see SemanticLogger::CoreExt::Warning).
47
+ Warning.singleton_class.prepend(SemanticLogger::CoreExt::Warning) if SemanticLogger::CoreExt::Warning::NEEDED
48
+
49
+ # Flush all appenders at exit, waiting for outstanding messages on the queue to be written first.
50
+ #
51
+ # During an outage of the destination of an appender added with async: true or in batches, that would
52
+ # wait while each queued message failed in turn, which can take hours against a server that does not
53
+ # answer. So the appenders are first told that the process is exiting, each one directly: the thread
54
+ # that writes to every appender can be held up handing a message to that appender's full queue while
55
+ # it backs off, and so would hand it the flush only once it stopped backing off. Each one then stops
56
+ # backing off and writing again what failed, and once its destination fails a write, discards the
57
+ # rest of its queue. An appender without a queue of its own, such as a plain :http, would make that
58
+ # thread wait while each message failed in turn instead, so once its destination fails a write, it is
59
+ # given no more messages (see SemanticLogger::Appenders#process_exiting=). They are told again once
60
+ # the flush has returned, since a test framework can run its tests in a later at_exit hook.
32
61
  at_exit do
62
+ appenders = SemanticLogger.appenders
63
+ appenders.process_exiting = true
33
64
  # Cannot close since test frameworks only run at exit, at which point the logging would already be closed.
34
65
  SemanticLogger.flush
66
+ ensure
67
+ appenders&.process_exiting = false
35
68
  end