semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -0,0 +1,310 @@
1
+ module SemanticLogger
2
+ # Replaces the values of an application's sensitive keys, such as passwords and access tokens, in
3
+ # the keyed data of a log entry: its payload, named tags, context, and metric dimensions (see
4
+ # Log#redact and SemanticLogger.sensitive_keys=).
5
+ #
6
+ # Keys are matched by name, the name of a Symbol key or a String key, both as written and in
7
+ # snake_case, which is lower case, so that "X-Api-Key" and "apiKey" are also asked about as
8
+ # "x_api_key" and "api_key". Keys of any other kind are never sensitive.
9
+ #
10
+ # Internal: built by SemanticLogger.sensitive_keys=, and asked by Log#redact to redact each of
11
+ # an entry's keyed fields, on the thread that writes the entry to the appenders.
12
+ class Redactor
13
+ # The value that replaces a sensitive one, unless the sensitive keys supply their own (see
14
+ # #initialize). The same as Rails uses for its filtered parameters.
15
+ MASK = "[FILTERED]".freeze
16
+
17
+ # Replaces a structure found again inside itself, which cannot be copied whole.
18
+ CIRCULAR = "[FILTERED: circular]".freeze
19
+
20
+ # Replaces a structure nested deeper than the formatters write (see Utils.json_safe), so that
21
+ # walking it can never overflow the stack.
22
+ NESTED_TOO_DEEPLY = "[FILTERED: nested too deeply]".freeze
23
+
24
+ MAX_DEPTH = Utils::MAX_JSON_SAFE_DEPTH
25
+
26
+ # A key name in snake_case already, which is not converted (see #snake_case_forms).
27
+ SNAKE_CASE = /\A[a-z\d_]*\z/
28
+
29
+ # How many keys the answer is remembered for (see #remember). Log entries use the same few keys
30
+ # over and over, but the names of request parameters, for example, are chosen by whoever sends
31
+ # the request, so neither how many nor how long they are is left to them.
32
+ MAX_REMEMBERED_KEYS = 1_000
33
+
34
+ # The longest key name, in characters, that the answer is remembered for (see #remember).
35
+ MAX_REMEMBERED_KEY_LENGTH = 128
36
+
37
+ # No snake_case forms of a key name, for one that is in snake_case already (see #snake_case_forms).
38
+ EMPTY_FORMS = [].freeze
39
+ private_constant :EMPTY_FORMS
40
+
41
+ # Stands for a key not yet asked about in the remembered answers, in which nil means "not sensitive".
42
+ UNKNOWN = Object.new.freeze
43
+ private_constant :UNKNOWN
44
+
45
+ # What the sensitive keys can be, for the error raised for any other value.
46
+ EXPECTED = "an Array of Symbols, Strings, or Regexps, a Regexp, or an object that responds to " \
47
+ "#filter_param (such as ActiveSupport::ParameterFilter) or #call".freeze
48
+ private_constant :EXPECTED
49
+
50
+ # Returns the sensitive keys, as supplied.
51
+ attr_reader :sensitive_keys
52
+
53
+ # Parameters
54
+ # sensitive_keys: [Array|Regexp|Object]
55
+ # Which keys are sensitive, one of:
56
+ # Array: Symbols and Strings, each matching any key name that contains it, ignoring case,
57
+ # as the Symbols in Rails' `config.filter_parameters` do, and Regexps.
58
+ # Regexp: Matches the sensitive key names.
59
+ # An object that responds to `#filter_param(key, value)`, such as
60
+ # ActiveSupport::ParameterFilter: given a key name and nil, it returns nil when the key is
61
+ # not sensitive, and otherwise the value that replaces the key's value.
62
+ # An object that responds to `#call(name)`, such as a Proc: given a key name, it returns
63
+ # whether the key is sensitive.
64
+ #
65
+ # The answer for each key is remembered (see #replacement_for), so the same key name must
66
+ # always get the same answer.
67
+ #
68
+ # Raises ArgumentError for any other value.
69
+ def initialize(sensitive_keys)
70
+ @sensitive_keys = sensitive_keys
71
+ @matcher = matcher_for(sensitive_keys)
72
+ @replacements = Concurrent::Map.new
73
+ end
74
+
75
+ # Returns a copy of the value in which the value of every sensitive key is replaced, at any
76
+ # depth, or the value itself when it has no sensitive key, so that what the formatters write
77
+ # only changes when something is redacted. The exceptions are a structure that holds itself,
78
+ # cut off with CIRCULAR, and one nested deeper than MAX_DEPTH, replaced with NESTED_TOO_DEEPLY.
79
+ #
80
+ # Looks inside the keys and values of the structures that the formatters write as JSON objects
81
+ # and arrays (see Utils.structure?), such as a Hash, Array, Set, Struct, or Data, and inside an
82
+ # object that responds to #to_unsafe_h, such as ActionController::Parameters. A copy is a Hash or
83
+ # an Array, as they write it. Any other value is returned as it is. The value and the structures
84
+ # in it are never changed, since they can be the application's own, or be shared by other log
85
+ # entries, as named tags are.
86
+ #
87
+ # Never raises a StandardError: when the value cannot be redacted, such as when the sensitive
88
+ # keys raise, it is replaced whole, and the error is reported to the internal logger.
89
+ def redact(value)
90
+ return value unless structure?(value)
91
+
92
+ walk(value, {}.compare_by_identity, 0)
93
+ rescue StandardError => e
94
+ report(e)
95
+ {redaction_failed: Utils.exception_name(e)}
96
+ end
97
+
98
+ # Returns the value that replaces the value of a sensitive key, or nil when the key is not
99
+ # sensitive, or is neither a Symbol nor a String.
100
+ #
101
+ # The answer is remembered (see #remember), since asking the sensitive keys costs far more than
102
+ # looking it up.
103
+ def replacement_for(key)
104
+ return unless key.is_a?(Symbol) || key.is_a?(String)
105
+
106
+ replacement = @replacements.fetch(key, UNKNOWN)
107
+ return replacement unless UNKNOWN.equal?(replacement)
108
+
109
+ replacement = ask(key)
110
+ remember(key, replacement)
111
+ replacement
112
+ end
113
+
114
+ private
115
+
116
+ # Returns [Proc] given a key name, the value that replaces a sensitive key's value, or nil.
117
+ # Raises ArgumentError when the sensitive keys are not one of the values #initialize accepts.
118
+ def matcher_for(sensitive_keys)
119
+ case sensitive_keys
120
+ when Array
121
+ regexp_matcher(Regexp.union(sensitive_keys.map { |key| pattern_for(key) }))
122
+ when Regexp
123
+ regexp_matcher(sensitive_keys)
124
+ else
125
+ if sensitive_keys.respond_to?(:filter_param)
126
+ # Or nil, so that a false answer is not taken for the replacement.
127
+ ->(name) { sensitive_keys.filter_param(name, nil) || nil }
128
+ elsif sensitive_keys.respond_to?(:call)
129
+ ->(name) { MASK if sensitive_keys.call(name) }
130
+ else
131
+ raise(ArgumentError, "Expected sensitive keys as #{EXPECTED}, not #{sensitive_keys.inspect}")
132
+ end
133
+ end
134
+ end
135
+
136
+ def regexp_matcher(regexp)
137
+ ->(name) { MASK if regexp.match?(name) }
138
+ end
139
+
140
+ # Returns [Regexp] the pattern for one of an Array of sensitive keys (see #initialize).
141
+ def pattern_for(key)
142
+ case key
143
+ when Regexp
144
+ key
145
+ when Symbol, String
146
+ Regexp.new(Regexp.escape(key.to_s), Regexp::IGNORECASE)
147
+ else
148
+ raise(ArgumentError, "Expected each sensitive key to be a Symbol, String, or Regexp, not #{key.inspect}")
149
+ end
150
+ end
151
+
152
+ # Asks the sensitive keys about the key's name, and when that is not sensitive, about each of its
153
+ # snake_case forms (see #snake_case_forms).
154
+ def ask(key)
155
+ name = Utils.encode_utf8_string(key.is_a?(Symbol) ? key.name : key)
156
+ replacement = @matcher.call(name)
157
+ return replacement unless replacement.nil?
158
+
159
+ snake_case_forms(name).each do |snake|
160
+ replacement = @matcher.call(snake)
161
+ return replacement unless replacement.nil?
162
+ end
163
+ nil
164
+ end
165
+
166
+ # Remembers the answer for the key, unless its name is longer than MAX_REMEMBERED_KEY_LENGTH.
167
+ # When MAX_REMEMBERED_KEYS are already remembered, they are forgotten first, so that the keys
168
+ # being logged now are remembered, rather than the first ones ever seen.
169
+ def remember(key, replacement)
170
+ return if key.length > MAX_REMEMBERED_KEY_LENGTH
171
+
172
+ @replacements.clear if @replacements.size >= MAX_REMEMBERED_KEYS
173
+ # A frozen copy of a String key that is not frozen, as a Hash keeps, so that changing the key
174
+ # afterwards cannot change what it is remembered under.
175
+ key = key.dup.freeze if key.is_a?(String) && !key.frozen?
176
+ @replacements[key] = replacement
177
+ end
178
+
179
+ # Returns the redacted copy of the value, or the value itself when nothing in it is sensitive.
180
+ #
181
+ # seen maps each structure being walked, or already walked, to its result, so that a structure
182
+ # held more than once is walked once, and one that holds itself is cut off rather than copied
183
+ # with the original, unredacted, structure inside.
184
+ def walk(value, seen, depth)
185
+ return value unless structure?(value)
186
+
187
+ if seen.key?(value)
188
+ result = seen[value]
189
+ return result.nil? ? CIRCULAR : result
190
+ end
191
+ return NESTED_TOO_DEEPLY if depth >= MAX_DEPTH
192
+
193
+ seen[value] = nil # Until its result is known.
194
+ seen[value] = walk_members(value, members_of(value), seen, depth)
195
+ end
196
+
197
+ def walk_members(value, members, seen, depth)
198
+ if members.is_a?(Hash)
199
+ walk_hash(value, members, seen, depth)
200
+ elsif Utils.json_array?(value)
201
+ walk_array(value, members, seen, depth)
202
+ else
203
+ # A #to_h, or #to_unsafe_h, that does not return a Hash is walked as a value of its own, one
204
+ # level deeper, the way the formatters write it.
205
+ redacted = walk(members, seen, depth + 1)
206
+ redacted.equal?(members) ? value : redacted
207
+ end
208
+ end
209
+
210
+ # Copies are only made from the first value that changes, since most entries have no sensitive
211
+ # key, and the structures before it are kept.
212
+ #
213
+ # A key that is itself a structure, which the formatters write as its JSON, is redacted too.
214
+ def walk_hash(value, members, seen, depth)
215
+ copy = nil
216
+ index = 0
217
+ members.each_pair do |key, val|
218
+ new_key = key.is_a?(Symbol) || key.is_a?(String) ? key : walk(key, seen, depth + 1)
219
+ replacement = replacement_for(key)
220
+ new_val = replacement.nil? ? walk(val, seen, depth + 1) : replacement
221
+ if copy
222
+ copy[new_key] = new_val
223
+ elsif !(new_val.equal?(val) && new_key.equal?(key))
224
+ copy = hash_head(members, index)
225
+ copy[new_key] = new_val
226
+ end
227
+ index += 1
228
+ end
229
+ copy || value
230
+ end
231
+
232
+ # Returns [Hash] a new Hash with the first count pairs of the members.
233
+ def hash_head(members, count)
234
+ copy = members.compare_by_identity? ? {}.compare_by_identity : {}
235
+ members.each_pair do |key, val|
236
+ break if count.zero?
237
+
238
+ copy[key] = val
239
+ count -= 1
240
+ end
241
+ copy
242
+ end
243
+
244
+ def walk_array(value, members, seen, depth)
245
+ copy = nil
246
+ members.each_with_index do |element, index|
247
+ new_element = walk(element, seen, depth + 1)
248
+ if copy
249
+ copy << new_element
250
+ elsif !new_element.equal?(element)
251
+ copy = members.first(index) << new_element
252
+ end
253
+ end
254
+ copy || value
255
+ end
256
+
257
+ # Whether the value is a structure that #walk looks inside: one that the formatters write as a
258
+ # JSON object or array (see Utils.structure?), or an object that responds to #to_unsafe_h, such
259
+ # as ActionController::Parameters, which they write with all of its parameters.
260
+ def structure?(value)
261
+ case value
262
+ when String, Symbol, Integer, Float, NilClass, TrueClass, FalseClass
263
+ # The most common values, ruled out before the slower checks.
264
+ false
265
+ else
266
+ Utils.structure?(value) || value.respond_to?(:to_unsafe_h)
267
+ end
268
+ end
269
+
270
+ # Returns the members of a structure (see #structure?), usually a Hash or an Array.
271
+ def members_of(value)
272
+ Utils.structure?(value) ? Utils.structure_members(value) : value.to_unsafe_h
273
+ end
274
+
275
+ # Returns [Array<String>] the key name in snake_case, in one or two forms, or none when it is
276
+ # snake_case already: words split where a lower case letter or digit meets a capital, such as
277
+ # "x_api_key" for "X-Api-Key", "api_key" for "apiKey", and "session_ids" for "sessionIDs"; and
278
+ # when it differs, with a run of capitals also split before its last capital, so that an
279
+ # acronym starts a word of its own, such as "api_key" for "APIKey" and "otp_code" for "OTPCode".
280
+ # Asking about both means a plural acronym, which the second form splits into "session_i_ds", is
281
+ # still matched by the first.
282
+ #
283
+ # Each pattern takes time in proportion to the length of the name, on JRuby too, since a key
284
+ # name can be long and chosen by whoever sends a request.
285
+ def snake_case_forms(name)
286
+ return EMPTY_FORMS if SNAKE_CASE.match?(name)
287
+
288
+ words = name.gsub(/([a-z\d])([A-Z])/, '\1_\2')
289
+ acronyms = words.gsub(/([A-Z\d])(?=[A-Z][a-z])/, '\1_')
290
+ forms = [to_snake_case(words)]
291
+ forms << to_snake_case(acronyms) unless acronyms == words
292
+ forms
293
+ end
294
+
295
+ def to_snake_case(words) = words.downcase.gsub(/[^a-z\d]+/, "_")
296
+
297
+ # Reports an error raised while redacting to the internal logger. Only its class and where it
298
+ # was raised: its message, or the exception given to the internal logger, could hold the data
299
+ # being redacted.
300
+ def report(exception)
301
+ where = exception.backtrace&.first
302
+ Logger.processor.logger.warn(
303
+ "Redacting sensitive keys raised #{Utils.exception_name(exception)}" \
304
+ "#{" (#{where})" if where}, so the value was replaced with :redaction_failed"
305
+ )
306
+ rescue StandardError
307
+ nil
308
+ end
309
+ end
310
+ end