semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
module Formatters
|
|
3
|
+
class Loki < Base
|
|
4
|
+
# The labels of the stream of one log entry, as Loki accepts them: Loki rejects the whole
|
|
5
|
+
# push, with every entry in it, for a label name it does not accept, or past one of its
|
|
6
|
+
# limits, and keeps only one value of a label name.
|
|
7
|
+
#
|
|
8
|
+
# It knows the labels written so far, a Hash of their names, as Strings, and their values, to
|
|
9
|
+
# which the formatter has written its own labels first (see Loki#build_stream), whether a
|
|
10
|
+
# name may still be a Symbol, the names that the formatter writes to the structured metadata
|
|
11
|
+
# instead of as labels, the labels still to be written under another name, Loki's limits,
|
|
12
|
+
# and how many labels there were when it finished them. It writes each name as a String (see
|
|
13
|
+
# #string_names), adds the named tags and the context to the labels (see #add), and then
|
|
14
|
+
# writes those that need another name and takes out those past Loki's limits (see #finish),
|
|
15
|
+
# which the formatter writes to the structured metadata instead. It names them by the
|
|
16
|
+
# formatter's rules (see Loki.loki_name and Loki.free_name), and converts their values with
|
|
17
|
+
# the converter the formatter gives it.
|
|
18
|
+
class Labels
|
|
19
|
+
# Matches a label name that Loki accepts: an ASCII letter or an underscore, followed by
|
|
20
|
+
# ASCII letters, digits, and underscores. Loki rejects the whole push for any other, such
|
|
21
|
+
# as `request-id`, `1st`, or an empty name, with "couldn't parse labels".
|
|
22
|
+
LABEL_NAME = /\A[A-Za-z_][A-Za-z0-9_]*\z/
|
|
23
|
+
|
|
24
|
+
# The name of a label whose key is empty, with the prefix that Loki puts before a name that
|
|
25
|
+
# starts with a digit.
|
|
26
|
+
EMPTY_LABEL_NAME = "key_".freeze
|
|
27
|
+
|
|
28
|
+
# The label that Loki adds to a stream that does not have it, and does not count toward
|
|
29
|
+
# max_label_names_per_series, even when the stream has it.
|
|
30
|
+
SERVICE_NAME = "service_name".freeze
|
|
31
|
+
|
|
32
|
+
# Parameters
|
|
33
|
+
# labels: [Hash]
|
|
34
|
+
# The labels written so far, which this adds to, and takes the labels past Loki's
|
|
35
|
+
# limits from.
|
|
36
|
+
# loki_value: [#call]
|
|
37
|
+
# Returns a value as Loki stores it, a String: the formatter's #loki_value, which a
|
|
38
|
+
# subclass of the formatter may override.
|
|
39
|
+
# max_names, max_name_length, max_value_length: [Integer]
|
|
40
|
+
# Loki's limits, see Loki#initialize.
|
|
41
|
+
# reserved: [Array<String>]
|
|
42
|
+
# The names that the formatter writes to the structured metadata instead of as labels,
|
|
43
|
+
# such as `thread` (see Loki#initialize). A label of that name is written under another
|
|
44
|
+
# name, since Loki would show the structured metadata as `thread_extracted` beside it.
|
|
45
|
+
# symbols: [true|false]
|
|
46
|
+
# Whether a label may have a name that is not a String, such as a Symbol that a
|
|
47
|
+
# subclass of the formatter written for v5.2 writes its own label under, as v5.2 did.
|
|
48
|
+
# The names are then written as Strings (see #string_names) before the first label is
|
|
49
|
+
# added, and as the labels are finished.
|
|
50
|
+
# Default: false, as the formatter's own methods write every label under a String.
|
|
51
|
+
def initialize(labels, loki_value, max_names, max_name_length, max_value_length, reserved:,
|
|
52
|
+
symbols: false)
|
|
53
|
+
@labels = labels
|
|
54
|
+
@loki_value = loki_value
|
|
55
|
+
@max_names = max_names
|
|
56
|
+
@max_name_length = max_name_length
|
|
57
|
+
@max_value_length = max_value_length
|
|
58
|
+
@reserved = reserved
|
|
59
|
+
@symbols = symbols
|
|
60
|
+
@unchecked = symbols
|
|
61
|
+
@renamed = nil
|
|
62
|
+
@finished_size = nil
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Whether these are the labels of the supplied Hash: the same Hash, not an equal one.
|
|
66
|
+
def of?(labels)
|
|
67
|
+
@labels.equal?(labels)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Whether #finish has written and limited these labels, and no label was added since, as a
|
|
71
|
+
# subclass of the formatter may add one after its #build_stream called super, even under
|
|
72
|
+
# the name of another label, such as `:level` beside `"level"`. A label whose value was
|
|
73
|
+
# replaced since is not seen.
|
|
74
|
+
def finished?
|
|
75
|
+
@finished_size == @labels.size
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Adds the label of a named tag or a value of the context, with its value as Loki stores it
|
|
79
|
+
# (see #initialize), unless the value is nil, which Loki would store as the text "null".
|
|
80
|
+
#
|
|
81
|
+
# Loki rejects the whole push for a label name it does not accept (see LABEL_NAME), so such
|
|
82
|
+
# a name is replaced (see #label_name_for). Loki keeps one value of a label name, so a label
|
|
83
|
+
# whose name another label already has is written once when that label has the same value,
|
|
84
|
+
# as Loki stores it, and otherwise under another name, as is a label of a reserved name.
|
|
85
|
+
# These are written after the others, by #finish, so that the name they get is not one that
|
|
86
|
+
# a later label has as its own.
|
|
87
|
+
def add(key, value)
|
|
88
|
+
return if value.nil?
|
|
89
|
+
|
|
90
|
+
string_names if @unchecked
|
|
91
|
+
name = key.is_a?(Symbol) ? key.name : key.to_s
|
|
92
|
+
value = @loki_value.call(value)
|
|
93
|
+
if !label_name?(name)
|
|
94
|
+
(@renamed ||= []) << [label_name_for(name), value]
|
|
95
|
+
elsif @labels.key?(name) || @reserved.include?(name)
|
|
96
|
+
(@renamed ||= []) << [name, value]
|
|
97
|
+
else
|
|
98
|
+
@labels[name] = value
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Writes the labels whose names are taken, reserved, or were replaced (see #add), after the
|
|
103
|
+
# others, so that a label written under its own name keeps it (see #write_renamed), and
|
|
104
|
+
# then takes out the labels past Loki's limits (see #limit). A label written under a
|
|
105
|
+
# Symbol since, as by a subclass of the formatter, is first written under a String (see
|
|
106
|
+
# #string_names).
|
|
107
|
+
#
|
|
108
|
+
# It can finish them again, after more labels were added (see #finished?).
|
|
109
|
+
#
|
|
110
|
+
# Returns [Array<Array(String, String)>|nil] the labels taken out, each its name and its
|
|
111
|
+
# value as Loki stores it, or nil when there are none.
|
|
112
|
+
def finish
|
|
113
|
+
string_names if @symbols
|
|
114
|
+
write_renamed if @renamed
|
|
115
|
+
moved = limit
|
|
116
|
+
@finished_size = @labels.size
|
|
117
|
+
moved
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
private
|
|
121
|
+
|
|
122
|
+
# Writes each label whose name is not a String under its name as a String, in place and in
|
|
123
|
+
# the same order, such as the `:host` of a subclass's own #host, or the `{pid: pid}` of
|
|
124
|
+
# v5.2's #build_stream. Otherwise #add would not find its name among the labels, and would
|
|
125
|
+
# write a named tag of that name beside it, so the JSON would have the name twice, which
|
|
126
|
+
# json 3 refuses to write. When a label written before it has that name, it is written
|
|
127
|
+
# after the others, as a named tag whose name is taken is (see #write_renamed), with a
|
|
128
|
+
# number unless it has the same value. So #add, #write_renamed, #limit, and #fit? find
|
|
129
|
+
# every label by its name as a String.
|
|
130
|
+
#
|
|
131
|
+
# It runs before the first named tag or context key is added, so that the labels are
|
|
132
|
+
# numbered in the order they are written in, and again as they are finished, for a label
|
|
133
|
+
# written since. Nearly every stream has String names alone, which Hash#any? checks without
|
|
134
|
+
# allocating.
|
|
135
|
+
def string_names
|
|
136
|
+
@unchecked = false
|
|
137
|
+
return unless @labels.any? { |name, _value| !name.is_a?(String) }
|
|
138
|
+
|
|
139
|
+
written = @labels.to_a
|
|
140
|
+
@labels.clear
|
|
141
|
+
written.each do |name, value|
|
|
142
|
+
name = name.is_a?(Symbol) ? name.name : name.to_s
|
|
143
|
+
if @labels.key?(name)
|
|
144
|
+
(@renamed ||= []) << [name, @loki_value.call(value)]
|
|
145
|
+
else
|
|
146
|
+
@labels[name] = value
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Writes the labels whose names are taken, reserved, or were replaced (see #add) in two
|
|
152
|
+
# passes: first each whose name is free, and then each of the rest, under the name that
|
|
153
|
+
# Loki.free_name gives it, which is never a reserved name, or not at all when the label of
|
|
154
|
+
# that name, or of a numbered name before it, has the same value (see #same_value?). So the
|
|
155
|
+
# number a label gets is never the name that another of them is written under, such as
|
|
156
|
+
# `host_2` for a named tag `host-2`, whatever the order of their keys. Only Loki.free_name
|
|
157
|
+
# compares the values, so that the value of each label a label is compared with is
|
|
158
|
+
# converted once.
|
|
159
|
+
def write_renamed
|
|
160
|
+
pending = @renamed.reject do |name, value|
|
|
161
|
+
next false if @labels.key?(name) || @reserved.include?(name)
|
|
162
|
+
|
|
163
|
+
@labels[name] = value
|
|
164
|
+
true
|
|
165
|
+
end
|
|
166
|
+
@renamed = nil
|
|
167
|
+
pending.each do |name, value|
|
|
168
|
+
name = Loki.free_name(@labels, name, reserved: @reserved) { |held| same_value?(held, value) }
|
|
169
|
+
@labels[name] = value if name
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Whether a label's value, as Loki stores it, is the supplied value. A value that is not a
|
|
174
|
+
# String, such as the `level` Symbol or the `pid` Integer of the formatter's own labels, is
|
|
175
|
+
# compared as Loki stores it.
|
|
176
|
+
def same_value?(held, value)
|
|
177
|
+
@loki_value.call(held) == value
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Whether Loki accepts the name as a label name. A name with characters outside ASCII never
|
|
181
|
+
# is, and matching it could raise, when it holds invalid UTF-8.
|
|
182
|
+
def label_name?(name)
|
|
183
|
+
name.ascii_only? && name.match?(LABEL_NAME)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Returns [String] a name that Loki accepts for a label whose name it does not: the name
|
|
187
|
+
# under which Loki stores structured metadata of that name (see Loki.loki_name), such as
|
|
188
|
+
# `request_id` for `request-id`, and `key_1st` for `1st`, or EMPTY_LABEL_NAME for an empty
|
|
189
|
+
# name.
|
|
190
|
+
def label_name_for(name)
|
|
191
|
+
name = Loki.loki_name(name)
|
|
192
|
+
name.empty? ? EMPTY_LABEL_NAME : name
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Takes out each label for which Loki would reject the whole push, with every entry in it,
|
|
196
|
+
# so that the push is accepted, and the value is kept in the structured metadata: a label
|
|
197
|
+
# whose name is longer than max_name_length, or whose value, as Loki stores it, is longer
|
|
198
|
+
# than max_value_length, and each label past max_names, in the order they are written (see
|
|
199
|
+
# Loki#build_stream).
|
|
200
|
+
#
|
|
201
|
+
# As Loki 3.7.10 does, a label with an empty value, which Loki leaves out, is neither
|
|
202
|
+
# checked nor counted, and SERVICE_NAME is not counted.
|
|
203
|
+
#
|
|
204
|
+
# Returns [Array<Array(String, String)>|nil] the labels taken out, each its name as a String
|
|
205
|
+
# and its value as Loki stores it, or nil when there are none.
|
|
206
|
+
def limit
|
|
207
|
+
return if @labels.size <= @max_names && fit?
|
|
208
|
+
|
|
209
|
+
moved = nil
|
|
210
|
+
count = 0
|
|
211
|
+
@labels.delete_if do |name, value|
|
|
212
|
+
next false if empty_value?(value) ||
|
|
213
|
+
(label_fits?(name, value) && (name == SERVICE_NAME || (count += 1) <= @max_names))
|
|
214
|
+
|
|
215
|
+
(moved ||= []) << [name, @loki_value.call(value)]
|
|
216
|
+
end
|
|
217
|
+
moved
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Whether Loki accepts the length of each label's name and value (see #label_fits?). A
|
|
221
|
+
# label with an empty value fits, whatever the length of its name, since Loki leaves it out.
|
|
222
|
+
#
|
|
223
|
+
# Every entry is checked, so an ASCII String, as most values are, is checked here, without
|
|
224
|
+
# another method call.
|
|
225
|
+
def fit?
|
|
226
|
+
max_name = @max_name_length
|
|
227
|
+
max_value = @max_value_length
|
|
228
|
+
@labels.each_pair do |name, value|
|
|
229
|
+
next if value.is_a?(String) && value.ascii_only? && value.bytesize <= max_value &&
|
|
230
|
+
(name.bytesize <= max_name || value.empty?)
|
|
231
|
+
return false unless label_fits?(name, value) || empty_value?(value)
|
|
232
|
+
end
|
|
233
|
+
true
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Whether the value is empty, which Loki leaves out. A value that is neither a String nor a
|
|
237
|
+
# Symbol, such as the `pid` Integer or the tags, is never written as an empty value.
|
|
238
|
+
def empty_value?(value)
|
|
239
|
+
(value.is_a?(String) || value.is_a?(Symbol)) && value.empty?
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# Whether Loki accepts the length of the label's name, a String, and of its value, as Loki
|
|
243
|
+
# stores it, in bytes, in UTF-8 without invalid bytes, as Utils.to_json writes them.
|
|
244
|
+
#
|
|
245
|
+
# The `level` Symbol and the `pid` Integer of every entry are read as text without
|
|
246
|
+
# converting them, which first checks whether a value is a structure, and the tags of
|
|
247
|
+
# nearly every entry that has them are not written as JSON only to measure them (see
|
|
248
|
+
# #short_strings?).
|
|
249
|
+
def label_fits?(name, value)
|
|
250
|
+
return false if name.bytesize > @max_name_length
|
|
251
|
+
return true if short_strings?(value)
|
|
252
|
+
|
|
253
|
+
text =
|
|
254
|
+
case value
|
|
255
|
+
when String then value
|
|
256
|
+
when Symbol then value.name
|
|
257
|
+
when Integer then value.to_s
|
|
258
|
+
else @loki_value.call(value)
|
|
259
|
+
end
|
|
260
|
+
text = Utils.encode_utf8_string(text) unless text.ascii_only?
|
|
261
|
+
text.bytesize <= @max_value_length
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Whether the value is an Array of Strings, such as the tags, whose JSON cannot be longer
|
|
265
|
+
# than max_value_length: JSON writes each byte of a String as at most 6 bytes, as for a
|
|
266
|
+
# control character, such as `\u0001`, with 2 quotes around it and a comma after it, all
|
|
267
|
+
# within brackets.
|
|
268
|
+
def short_strings?(value)
|
|
269
|
+
return false unless value.is_a?(Array)
|
|
270
|
+
|
|
271
|
+
size = 2
|
|
272
|
+
value.each do |item|
|
|
273
|
+
return false unless item.is_a?(String)
|
|
274
|
+
|
|
275
|
+
size += (item.bytesize * 6) + 3
|
|
276
|
+
end
|
|
277
|
+
size <= @max_value_length
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|