semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -0,0 +1,282 @@
1
+ module SemanticLogger
2
+ module Formatters
3
+ class Loki < Base
4
+ # The labels of the stream of one log entry, as Loki accepts them: Loki rejects the whole
5
+ # push, with every entry in it, for a label name it does not accept, or past one of its
6
+ # limits, and keeps only one value of a label name.
7
+ #
8
+ # It knows the labels written so far, a Hash of their names, as Strings, and their values, to
9
+ # which the formatter has written its own labels first (see Loki#build_stream), whether a
10
+ # name may still be a Symbol, the names that the formatter writes to the structured metadata
11
+ # instead of as labels, the labels still to be written under another name, Loki's limits,
12
+ # and how many labels there were when it finished them. It writes each name as a String (see
13
+ # #string_names), adds the named tags and the context to the labels (see #add), and then
14
+ # writes those that need another name and takes out those past Loki's limits (see #finish),
15
+ # which the formatter writes to the structured metadata instead. It names them by the
16
+ # formatter's rules (see Loki.loki_name and Loki.free_name), and converts their values with
17
+ # the converter the formatter gives it.
18
+ class Labels
19
+ # Matches a label name that Loki accepts: an ASCII letter or an underscore, followed by
20
+ # ASCII letters, digits, and underscores. Loki rejects the whole push for any other, such
21
+ # as `request-id`, `1st`, or an empty name, with "couldn't parse labels".
22
+ LABEL_NAME = /\A[A-Za-z_][A-Za-z0-9_]*\z/
23
+
24
+ # The name of a label whose key is empty, with the prefix that Loki puts before a name that
25
+ # starts with a digit.
26
+ EMPTY_LABEL_NAME = "key_".freeze
27
+
28
+ # The label that Loki adds to a stream that does not have it, and does not count toward
29
+ # max_label_names_per_series, even when the stream has it.
30
+ SERVICE_NAME = "service_name".freeze
31
+
32
+ # Parameters
33
+ # labels: [Hash]
34
+ # The labels written so far, which this adds to, and takes the labels past Loki's
35
+ # limits from.
36
+ # loki_value: [#call]
37
+ # Returns a value as Loki stores it, a String: the formatter's #loki_value, which a
38
+ # subclass of the formatter may override.
39
+ # max_names, max_name_length, max_value_length: [Integer]
40
+ # Loki's limits, see Loki#initialize.
41
+ # reserved: [Array<String>]
42
+ # The names that the formatter writes to the structured metadata instead of as labels,
43
+ # such as `thread` (see Loki#initialize). A label of that name is written under another
44
+ # name, since Loki would show the structured metadata as `thread_extracted` beside it.
45
+ # symbols: [true|false]
46
+ # Whether a label may have a name that is not a String, such as a Symbol that a
47
+ # subclass of the formatter written for v5.2 writes its own label under, as v5.2 did.
48
+ # The names are then written as Strings (see #string_names) before the first label is
49
+ # added, and as the labels are finished.
50
+ # Default: false, as the formatter's own methods write every label under a String.
51
+ def initialize(labels, loki_value, max_names, max_name_length, max_value_length, reserved:,
52
+ symbols: false)
53
+ @labels = labels
54
+ @loki_value = loki_value
55
+ @max_names = max_names
56
+ @max_name_length = max_name_length
57
+ @max_value_length = max_value_length
58
+ @reserved = reserved
59
+ @symbols = symbols
60
+ @unchecked = symbols
61
+ @renamed = nil
62
+ @finished_size = nil
63
+ end
64
+
65
+ # Whether these are the labels of the supplied Hash: the same Hash, not an equal one.
66
+ def of?(labels)
67
+ @labels.equal?(labels)
68
+ end
69
+
70
+ # Whether #finish has written and limited these labels, and no label was added since, as a
71
+ # subclass of the formatter may add one after its #build_stream called super, even under
72
+ # the name of another label, such as `:level` beside `"level"`. A label whose value was
73
+ # replaced since is not seen.
74
+ def finished?
75
+ @finished_size == @labels.size
76
+ end
77
+
78
+ # Adds the label of a named tag or a value of the context, with its value as Loki stores it
79
+ # (see #initialize), unless the value is nil, which Loki would store as the text "null".
80
+ #
81
+ # Loki rejects the whole push for a label name it does not accept (see LABEL_NAME), so such
82
+ # a name is replaced (see #label_name_for). Loki keeps one value of a label name, so a label
83
+ # whose name another label already has is written once when that label has the same value,
84
+ # as Loki stores it, and otherwise under another name, as is a label of a reserved name.
85
+ # These are written after the others, by #finish, so that the name they get is not one that
86
+ # a later label has as its own.
87
+ def add(key, value)
88
+ return if value.nil?
89
+
90
+ string_names if @unchecked
91
+ name = key.is_a?(Symbol) ? key.name : key.to_s
92
+ value = @loki_value.call(value)
93
+ if !label_name?(name)
94
+ (@renamed ||= []) << [label_name_for(name), value]
95
+ elsif @labels.key?(name) || @reserved.include?(name)
96
+ (@renamed ||= []) << [name, value]
97
+ else
98
+ @labels[name] = value
99
+ end
100
+ end
101
+
102
+ # Writes the labels whose names are taken, reserved, or were replaced (see #add), after the
103
+ # others, so that a label written under its own name keeps it (see #write_renamed), and
104
+ # then takes out the labels past Loki's limits (see #limit). A label written under a
105
+ # Symbol since, as by a subclass of the formatter, is first written under a String (see
106
+ # #string_names).
107
+ #
108
+ # It can finish them again, after more labels were added (see #finished?).
109
+ #
110
+ # Returns [Array<Array(String, String)>|nil] the labels taken out, each its name and its
111
+ # value as Loki stores it, or nil when there are none.
112
+ def finish
113
+ string_names if @symbols
114
+ write_renamed if @renamed
115
+ moved = limit
116
+ @finished_size = @labels.size
117
+ moved
118
+ end
119
+
120
+ private
121
+
122
+ # Writes each label whose name is not a String under its name as a String, in place and in
123
+ # the same order, such as the `:host` of a subclass's own #host, or the `{pid: pid}` of
124
+ # v5.2's #build_stream. Otherwise #add would not find its name among the labels, and would
125
+ # write a named tag of that name beside it, so the JSON would have the name twice, which
126
+ # json 3 refuses to write. When a label written before it has that name, it is written
127
+ # after the others, as a named tag whose name is taken is (see #write_renamed), with a
128
+ # number unless it has the same value. So #add, #write_renamed, #limit, and #fit? find
129
+ # every label by its name as a String.
130
+ #
131
+ # It runs before the first named tag or context key is added, so that the labels are
132
+ # numbered in the order they are written in, and again as they are finished, for a label
133
+ # written since. Nearly every stream has String names alone, which Hash#any? checks without
134
+ # allocating.
135
+ def string_names
136
+ @unchecked = false
137
+ return unless @labels.any? { |name, _value| !name.is_a?(String) }
138
+
139
+ written = @labels.to_a
140
+ @labels.clear
141
+ written.each do |name, value|
142
+ name = name.is_a?(Symbol) ? name.name : name.to_s
143
+ if @labels.key?(name)
144
+ (@renamed ||= []) << [name, @loki_value.call(value)]
145
+ else
146
+ @labels[name] = value
147
+ end
148
+ end
149
+ end
150
+
151
+ # Writes the labels whose names are taken, reserved, or were replaced (see #add) in two
152
+ # passes: first each whose name is free, and then each of the rest, under the name that
153
+ # Loki.free_name gives it, which is never a reserved name, or not at all when the label of
154
+ # that name, or of a numbered name before it, has the same value (see #same_value?). So the
155
+ # number a label gets is never the name that another of them is written under, such as
156
+ # `host_2` for a named tag `host-2`, whatever the order of their keys. Only Loki.free_name
157
+ # compares the values, so that the value of each label a label is compared with is
158
+ # converted once.
159
+ def write_renamed
160
+ pending = @renamed.reject do |name, value|
161
+ next false if @labels.key?(name) || @reserved.include?(name)
162
+
163
+ @labels[name] = value
164
+ true
165
+ end
166
+ @renamed = nil
167
+ pending.each do |name, value|
168
+ name = Loki.free_name(@labels, name, reserved: @reserved) { |held| same_value?(held, value) }
169
+ @labels[name] = value if name
170
+ end
171
+ end
172
+
173
+ # Whether a label's value, as Loki stores it, is the supplied value. A value that is not a
174
+ # String, such as the `level` Symbol or the `pid` Integer of the formatter's own labels, is
175
+ # compared as Loki stores it.
176
+ def same_value?(held, value)
177
+ @loki_value.call(held) == value
178
+ end
179
+
180
+ # Whether Loki accepts the name as a label name. A name with characters outside ASCII never
181
+ # is, and matching it could raise, when it holds invalid UTF-8.
182
+ def label_name?(name)
183
+ name.ascii_only? && name.match?(LABEL_NAME)
184
+ end
185
+
186
+ # Returns [String] a name that Loki accepts for a label whose name it does not: the name
187
+ # under which Loki stores structured metadata of that name (see Loki.loki_name), such as
188
+ # `request_id` for `request-id`, and `key_1st` for `1st`, or EMPTY_LABEL_NAME for an empty
189
+ # name.
190
+ def label_name_for(name)
191
+ name = Loki.loki_name(name)
192
+ name.empty? ? EMPTY_LABEL_NAME : name
193
+ end
194
+
195
+ # Takes out each label for which Loki would reject the whole push, with every entry in it,
196
+ # so that the push is accepted, and the value is kept in the structured metadata: a label
197
+ # whose name is longer than max_name_length, or whose value, as Loki stores it, is longer
198
+ # than max_value_length, and each label past max_names, in the order they are written (see
199
+ # Loki#build_stream).
200
+ #
201
+ # As Loki 3.7.10 does, a label with an empty value, which Loki leaves out, is neither
202
+ # checked nor counted, and SERVICE_NAME is not counted.
203
+ #
204
+ # Returns [Array<Array(String, String)>|nil] the labels taken out, each its name as a String
205
+ # and its value as Loki stores it, or nil when there are none.
206
+ def limit
207
+ return if @labels.size <= @max_names && fit?
208
+
209
+ moved = nil
210
+ count = 0
211
+ @labels.delete_if do |name, value|
212
+ next false if empty_value?(value) ||
213
+ (label_fits?(name, value) && (name == SERVICE_NAME || (count += 1) <= @max_names))
214
+
215
+ (moved ||= []) << [name, @loki_value.call(value)]
216
+ end
217
+ moved
218
+ end
219
+
220
+ # Whether Loki accepts the length of each label's name and value (see #label_fits?). A
221
+ # label with an empty value fits, whatever the length of its name, since Loki leaves it out.
222
+ #
223
+ # Every entry is checked, so an ASCII String, as most values are, is checked here, without
224
+ # another method call.
225
+ def fit?
226
+ max_name = @max_name_length
227
+ max_value = @max_value_length
228
+ @labels.each_pair do |name, value|
229
+ next if value.is_a?(String) && value.ascii_only? && value.bytesize <= max_value &&
230
+ (name.bytesize <= max_name || value.empty?)
231
+ return false unless label_fits?(name, value) || empty_value?(value)
232
+ end
233
+ true
234
+ end
235
+
236
+ # Whether the value is empty, which Loki leaves out. A value that is neither a String nor a
237
+ # Symbol, such as the `pid` Integer or the tags, is never written as an empty value.
238
+ def empty_value?(value)
239
+ (value.is_a?(String) || value.is_a?(Symbol)) && value.empty?
240
+ end
241
+
242
+ # Whether Loki accepts the length of the label's name, a String, and of its value, as Loki
243
+ # stores it, in bytes, in UTF-8 without invalid bytes, as Utils.to_json writes them.
244
+ #
245
+ # The `level` Symbol and the `pid` Integer of every entry are read as text without
246
+ # converting them, which first checks whether a value is a structure, and the tags of
247
+ # nearly every entry that has them are not written as JSON only to measure them (see
248
+ # #short_strings?).
249
+ def label_fits?(name, value)
250
+ return false if name.bytesize > @max_name_length
251
+ return true if short_strings?(value)
252
+
253
+ text =
254
+ case value
255
+ when String then value
256
+ when Symbol then value.name
257
+ when Integer then value.to_s
258
+ else @loki_value.call(value)
259
+ end
260
+ text = Utils.encode_utf8_string(text) unless text.ascii_only?
261
+ text.bytesize <= @max_value_length
262
+ end
263
+
264
+ # Whether the value is an Array of Strings, such as the tags, whose JSON cannot be longer
265
+ # than max_value_length: JSON writes each byte of a String as at most 6 bytes, as for a
266
+ # control character, such as `\u0001`, with 2 quotes around it and a comma after it, all
267
+ # within brackets.
268
+ def short_strings?(value)
269
+ return false unless value.is_a?(Array)
270
+
271
+ size = 2
272
+ value.each do |item|
273
+ return false unless item.is_a?(String)
274
+
275
+ size += (item.bytesize * 6) + 3
276
+ end
277
+ size <= @max_value_length
278
+ end
279
+ end
280
+ end
281
+ end
282
+ end