semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -1,16 +1,166 @@
|
|
|
1
1
|
require "json"
|
|
2
|
+
require "semantic_logger/formatters/loki/labels"
|
|
2
3
|
|
|
3
4
|
module SemanticLogger
|
|
4
5
|
module Formatters
|
|
5
6
|
class Loki < Base
|
|
6
7
|
attr_accessor :stream, :payload_value
|
|
7
8
|
|
|
9
|
+
# The names of the structured metadata that this formatter writes itself for the fields of an
|
|
10
|
+
# entry, to which the labels named by `structured_metadata:` are added (see #initialize). A
|
|
11
|
+
# payload key that Loki would store under one of this formatter's own names is written with
|
|
12
|
+
# PAYLOAD_PREFIX, as in `payload_duration`, whether or not the entry has that field, so that
|
|
13
|
+
# each name always holds this formatter's value. Loki keeps one value of a name: the last of
|
|
14
|
+
# a name written twice, but of names that it stores alike, such as `metric.value` and
|
|
15
|
+
# `metric_value` (see .own_name), the one it had to change. So the payload's value would
|
|
16
|
+
# otherwise be lost, or replace this formatter's.
|
|
17
|
+
OWN_KEYS = %w[
|
|
18
|
+
metric metric_value dimensions duration duration_human exception_name exception_message
|
|
19
|
+
stack_trace
|
|
20
|
+
].freeze
|
|
21
|
+
|
|
22
|
+
# The labels that `structured_metadata:` can write to the structured metadata instead (see
|
|
23
|
+
# #initialize).
|
|
24
|
+
STRUCTURED_METADATA = %i[pid thread].freeze
|
|
25
|
+
|
|
26
|
+
# A run of characters other than ASCII letters and digits, which Loki replaces with one
|
|
27
|
+
# underscore in the name of structured metadata. So Loki stores `metric.value`,
|
|
28
|
+
# `metric__value`, and `metric` followed by a zero width space and `value` all as
|
|
29
|
+
# `metric_value`.
|
|
30
|
+
LOKI_SEPARATOR = /[^A-Za-z0-9]+/
|
|
31
|
+
|
|
32
|
+
# Returns [Regexp] a pattern that matches a name that Loki stores as one of the supplied
|
|
33
|
+
# names: each underscore of a name stands for a LOKI_SEPARATOR. Matching it allocates
|
|
34
|
+
# nothing, unlike comparing #loki_name with each name, and nearly every payload key is
|
|
35
|
+
# checked against it.
|
|
36
|
+
def self.own_name(names)
|
|
37
|
+
/\A(?:#{names.map { |name| name.gsub('_', LOKI_SEPARATOR.source) }.join('|')})\z/
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Matches a name that Loki stores as one of OWN_KEYS.
|
|
41
|
+
OWN_NAME = own_name(OWN_KEYS)
|
|
42
|
+
|
|
43
|
+
# Matches a name that holds an ASCII letter or digit. Loki rejects the whole push for a
|
|
44
|
+
# structured metadata name without one, such as an empty name, `-`, or `_`, since it would
|
|
45
|
+
# store it as an empty name or as `_`.
|
|
46
|
+
LETTER_OR_DIGIT = /[A-Za-z0-9]/
|
|
47
|
+
|
|
48
|
+
# The prefix of a payload key named like one of this formatter's own structured metadata
|
|
49
|
+
# names, or without a letter or digit. The :logfmt and :open_telemetry formatters write
|
|
50
|
+
# `payload.`, which Loki would store as `payload_`.
|
|
51
|
+
PAYLOAD_PREFIX = "payload_".freeze
|
|
52
|
+
|
|
53
|
+
# A name that starts with a digit, which Loki stores with `key_` before it.
|
|
54
|
+
LEADING_DIGIT = /\A[0-9]/
|
|
55
|
+
|
|
56
|
+
# The most labels that Loki accepts for a stream, its `max_label_names_per_series`, the
|
|
57
|
+
# longest label name, its `max_label_name_length`, and the longest label value, its
|
|
58
|
+
# `max_label_value_length`, in bytes, unless a Loki tenant's `limits_config` raises them. See
|
|
59
|
+
# #initialize.
|
|
60
|
+
attr_reader :max_label_names_per_series, :max_label_name_length, :max_label_value_length
|
|
61
|
+
|
|
62
|
+
# The labels that are written to the structured metadata instead (see #initialize): a frozen
|
|
63
|
+
# Array of :pid, :thread, both, or neither.
|
|
64
|
+
attr_reader :structured_metadata
|
|
65
|
+
|
|
66
|
+
# Parameters
|
|
67
|
+
# structured_metadata: [Array<Symbol>]
|
|
68
|
+
# The labels to write to the structured metadata instead, as Strings: `:pid`, the process
|
|
69
|
+
# id, `:thread`, the thread name, both, or neither. Loki advises against labels like these,
|
|
70
|
+
# whose values are many and change: each process, and each of its threads, adds a stream,
|
|
71
|
+
# which Loki indexes and stores apart. As structured metadata, a query can still filter on
|
|
72
|
+
# them, as in `{application="shop"} | thread="worker-1"`. Their names are then among this
|
|
73
|
+
# formatter's own: a payload key of that name is written with PAYLOAD_PREFIX, as a key
|
|
74
|
+
# named like one of its other fields is, and a named tag or context key of that name is
|
|
75
|
+
# written as a label with a number, as in `thread_2`, since Loki would show the structured
|
|
76
|
+
# metadata as `thread_extracted` beside a label of its name.
|
|
77
|
+
# Structured metadata needs a Loki that accepts it, with schema v13 and
|
|
78
|
+
# `allow_structured_metadata`, which is on by default since Loki 3.0. Any other Loki
|
|
79
|
+
# rejects each entry that has structured metadata, which with `:pid` is every entry.
|
|
80
|
+
# Default: neither, so that both are labels, with a deprecation warning, since Semantic
|
|
81
|
+
# Logger v6 writes both to the structured metadata by default. Supply [] to keep both as
|
|
82
|
+
# labels without it.
|
|
83
|
+
#
|
|
84
|
+
# max_label_names_per_series: [Integer]
|
|
85
|
+
# The most labels that Loki accepts for a stream. Loki rejects the whole push, with every
|
|
86
|
+
# entry in it, for a stream with more, so a label past it, in the order they are written
|
|
87
|
+
# (see #build_stream), is written to the structured metadata instead.
|
|
88
|
+
# Default: 15, as Loki's `max_label_names_per_series`
|
|
89
|
+
# max_label_name_length: [Integer]
|
|
90
|
+
# The longest label name, in bytes, that Loki accepts. A label with a longer name is
|
|
91
|
+
# written to the structured metadata instead.
|
|
92
|
+
# Default: 1024, as Loki's `max_label_name_length`
|
|
93
|
+
# max_label_value_length: [Integer]
|
|
94
|
+
# The longest label value, in bytes, that Loki accepts. A label with a longer value, such as
|
|
95
|
+
# many tags, is written to the structured metadata instead.
|
|
96
|
+
# Default: 2048, as Loki's `max_label_value_length`
|
|
97
|
+
#
|
|
98
|
+
# Set each limit, an Integer of 1 or more, to the limit of the same name in the
|
|
99
|
+
# `limits_config` of a Loki tenant that raises it.
|
|
100
|
+
#
|
|
101
|
+
# See Formatters::Base for the other options.
|
|
102
|
+
#
|
|
103
|
+
# The instance variables that are not options are named for Loki, as in @loki_pid_label, so
|
|
104
|
+
# that they are not names that a subclass may already use for its own.
|
|
105
|
+
def initialize(structured_metadata: nil, max_label_names_per_series: 15, max_label_name_length: 1024,
|
|
106
|
+
max_label_value_length: 2048, **args)
|
|
107
|
+
super(**args)
|
|
108
|
+
@structured_metadata = structured_metadata_option(structured_metadata)
|
|
109
|
+
@loki_metadata_labels = @structured_metadata.map(&:name).freeze
|
|
110
|
+
@loki_own_keys = (OWN_KEYS + @loki_metadata_labels).freeze
|
|
111
|
+
@loki_own_name = @loki_metadata_labels.empty? ? OWN_NAME : Loki.own_name(@loki_own_keys)
|
|
112
|
+
@loki_pid_label = !@structured_metadata.include?(:pid)
|
|
113
|
+
@loki_thread_label = !@structured_metadata.include?(:thread)
|
|
114
|
+
@max_label_names_per_series = label_limit(:max_label_names_per_series, max_label_names_per_series)
|
|
115
|
+
@max_label_name_length = label_limit(:max_label_name_length, max_label_name_length)
|
|
116
|
+
@max_label_value_length = label_limit(:max_label_value_length, max_label_value_length)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Returns [String|nil] the name under which to write a value, among names, a Hash of the
|
|
120
|
+
# names already written and their values: the name itself when it is free; when another value
|
|
121
|
+
# has it, the first name with a number after it that is free, as in `request_id_2`, so that
|
|
122
|
+
# both values are kept, since Loki keeps one value of a name; or nil when the block returns
|
|
123
|
+
# true for the value of that name, or of a numbered name before it, as the same value, so
|
|
124
|
+
# that it is written once. Both the labels (see Labels) and the structured metadata (see
|
|
125
|
+
# #write_moved_labels) are named so.
|
|
126
|
+
#
|
|
127
|
+
# The number is added to the stem, which is the name unless given. For structured metadata
|
|
128
|
+
# it is the name without a trailing underscore, so that `payload_` is numbered `payload_2`,
|
|
129
|
+
# as Loki would store `payload__2`. A name in reserved, which a value outside names has, is
|
|
130
|
+
# never free, nor the same, such as the label `thread` while the thread is structured
|
|
131
|
+
# metadata (see Labels).
|
|
132
|
+
def self.free_name(names, name, stem = name, reserved: nil)
|
|
133
|
+
candidate = name
|
|
134
|
+
number = 1
|
|
135
|
+
while names.key?(candidate) || reserved&.include?(candidate)
|
|
136
|
+
return if names.key?(candidate) && yield(names[candidate])
|
|
137
|
+
|
|
138
|
+
number += 1
|
|
139
|
+
candidate = "#{stem}_#{number}"
|
|
140
|
+
end
|
|
141
|
+
candidate
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Returns [String] the name that Loki stores structured metadata of the supplied name under:
|
|
145
|
+
# in UTF-8 without invalid bytes, as Utils.to_json writes it, with each run of characters
|
|
146
|
+
# other than ASCII letters and digits replaced with one underscore, and `key_` before a name
|
|
147
|
+
# that starts with a digit.
|
|
148
|
+
#
|
|
149
|
+
# The `key_` cannot change which name a prefixed key gets: that name starts with
|
|
150
|
+
# PAYLOAD_PREFIX, and only the other keys it is checked against can start with a digit. It is
|
|
151
|
+
# there so that this returns the name Loki stores for any name, not only for the names it is
|
|
152
|
+
# used for here.
|
|
153
|
+
def self.loki_name(name)
|
|
154
|
+
name = Utils.encode_utf8_string(name).gsub(LOKI_SEPARATOR, "_")
|
|
155
|
+
name.match?(LEADING_DIGIT) ? "key_#{name}" : name
|
|
156
|
+
end
|
|
157
|
+
|
|
8
158
|
# Returns [String] a single JSON log
|
|
9
159
|
def call(log, logger)
|
|
10
160
|
self.logger = logger
|
|
11
161
|
self.log = log
|
|
12
162
|
|
|
13
|
-
Utils.to_json({streams: [
|
|
163
|
+
Utils.to_json({streams: [loki_stream]})
|
|
14
164
|
end
|
|
15
165
|
|
|
16
166
|
# Returns [String] a JSON batch of logs
|
|
@@ -19,7 +169,7 @@ module SemanticLogger
|
|
|
19
169
|
|
|
20
170
|
streams = logs.map do |log|
|
|
21
171
|
self.log = log
|
|
22
|
-
|
|
172
|
+
loki_stream
|
|
23
173
|
end
|
|
24
174
|
|
|
25
175
|
Utils.to_json({streams: streams})
|
|
@@ -27,8 +177,53 @@ module SemanticLogger
|
|
|
27
177
|
|
|
28
178
|
private
|
|
29
179
|
|
|
180
|
+
# Returns [Hash] the stream of the current log entry, which #build_stream builds.
|
|
181
|
+
#
|
|
182
|
+
# A subclass that overrides #build_stream without calling super, as one written for v5.2
|
|
183
|
+
# may, does not finish the stream's labels, so they are finished here (see
|
|
184
|
+
# #finish_loki_stream) when the stream it returns has the current stream's labels. So it
|
|
185
|
+
# still writes each named tag and context key whose name is taken, reserved, or replaced,
|
|
186
|
+
# writes the process id and the thread name to the structured metadata when
|
|
187
|
+
# structured_metadata names them, and moves the labels past Loki's limits. The labels are
|
|
188
|
+
# finished again when a subclass added one after its #build_stream called super (see
|
|
189
|
+
# Labels#finished?), such as one under a Symbol whose name another label has. The structured
|
|
190
|
+
# metadata written is that of the entry it returns (see #entry_metadata). A formatter of
|
|
191
|
+
# this class itself finishes them in #build_stream, and so skips this.
|
|
192
|
+
def loki_stream
|
|
193
|
+
return build_stream if instance_of?(Loki)
|
|
194
|
+
|
|
195
|
+
built = build_stream
|
|
196
|
+
finish_loki_stream(entry_metadata(built)) if current_labels?(built) && !loki_labels.finished?
|
|
197
|
+
built
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Whether the stream that #build_stream returned has the current stream's labels, a Hash,
|
|
201
|
+
# which a subclass's own stream may not, such as one under String keys.
|
|
202
|
+
def current_labels?(built)
|
|
203
|
+
labels = built[:stream] if built.is_a?(Hash)
|
|
204
|
+
labels.is_a?(Hash) && stream.is_a?(Hash) && labels.equal?(stream[:stream])
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Returns [Hash|nil] the structured metadata of the entry of the stream that #build_stream
|
|
208
|
+
# returned, the third of its values: payload_value, unless a subclass wrote a copy of it, as
|
|
209
|
+
# in `payload_value.merge(...)`, or nil when it wrote none.
|
|
210
|
+
def entry_metadata(built)
|
|
211
|
+
values = built[:values]
|
|
212
|
+
entry = values[0] if values.is_a?(Array)
|
|
213
|
+
entry[2] if entry.is_a?(Array)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# The labels are written in this order: this formatter's own (pid, application, environment,
|
|
217
|
+
# host, level, thread, and tags, without those that structured_metadata names), each named
|
|
218
|
+
# tag, and each value of the context, under their own names, and then each named tag and
|
|
219
|
+
# context key whose name another label has, or that Loki does not accept, or that
|
|
220
|
+
# structured_metadata names, under another name, or not at all (see Labels#add). So every
|
|
221
|
+
# label written under its own name keeps it. Their names are Strings, so that a named tag or
|
|
222
|
+
# context key, whether a Symbol or a String, is found among them by name, as Loki finds it.
|
|
223
|
+
# The labels past Loki's limits are then taken out and written to the structured metadata
|
|
224
|
+
# (see #finish_loki_stream).
|
|
30
225
|
def build_stream
|
|
31
|
-
self.stream = {stream: {pid
|
|
226
|
+
self.stream = {stream: @loki_pid_label ? {"pid" => pid} : {}, values: [[]]}
|
|
32
227
|
|
|
33
228
|
application
|
|
34
229
|
environment
|
|
@@ -44,45 +239,116 @@ module SemanticLogger
|
|
|
44
239
|
metric
|
|
45
240
|
duration
|
|
46
241
|
exception
|
|
242
|
+
finish_loki_stream
|
|
47
243
|
|
|
48
244
|
stream[:values][0] << payload_value
|
|
49
245
|
stream
|
|
50
246
|
end
|
|
51
247
|
|
|
52
248
|
def host
|
|
53
|
-
stream[:stream][
|
|
249
|
+
stream[:stream]["host"] = logger.host if log_host && logger&.host
|
|
54
250
|
end
|
|
55
251
|
|
|
56
252
|
def application
|
|
57
|
-
stream[:stream][
|
|
253
|
+
stream[:stream]["application"] = logger.application if log_application && logger&.application
|
|
58
254
|
end
|
|
59
255
|
|
|
60
256
|
def environment
|
|
61
|
-
stream[:stream][
|
|
257
|
+
stream[:stream]["environment"] = logger.environment if log_environment && logger&.environment
|
|
62
258
|
end
|
|
63
259
|
|
|
64
260
|
def level
|
|
65
|
-
stream[:stream][
|
|
261
|
+
stream[:stream]["level"] = log.level
|
|
66
262
|
end
|
|
67
263
|
|
|
68
264
|
def thread
|
|
69
|
-
stream[:stream][
|
|
265
|
+
stream[:stream]["thread"] = log.thread_name if @loki_thread_label && log.thread_name
|
|
70
266
|
end
|
|
71
267
|
|
|
72
268
|
def tags
|
|
73
|
-
stream[:stream][
|
|
269
|
+
stream[:stream]["tags"] = log.tags if log.tags.respond_to?(:empty?) && !log.tags.empty?
|
|
74
270
|
end
|
|
75
271
|
|
|
76
272
|
def named_tags
|
|
77
|
-
|
|
273
|
+
return unless log.named_tags.respond_to?(:empty?) && !log.named_tags.empty?
|
|
274
|
+
|
|
275
|
+
stream_labels = loki_labels
|
|
276
|
+
log.named_tags.each_pair { |key, value| stream_labels.add(key, value) }
|
|
78
277
|
end
|
|
79
278
|
|
|
80
279
|
def context
|
|
81
280
|
return unless log.context && !log.context.empty?
|
|
82
281
|
|
|
83
|
-
|
|
84
|
-
|
|
282
|
+
stream_labels = loki_labels
|
|
283
|
+
log.context.each { |key, value| stream_labels.add(key, value) }
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Returns [Labels] the labels of the current stream, to which the named tags and the context
|
|
287
|
+
# are added, created for it unless they are already its labels. So a subclass whose own
|
|
288
|
+
# #build_stream does not create them, as in v5.2, still adds the named tags and the context.
|
|
289
|
+
# Each label value is converted by #loki_value, so a subclass that overrides it converts the
|
|
290
|
+
# labels' values too. This formatter's own methods write every label under a String, while a
|
|
291
|
+
# subclass's may write one under a Symbol, as v5.2 did, so only a subclass's labels are told
|
|
292
|
+
# that they may have a Symbol name (see Labels#initialize).
|
|
293
|
+
#
|
|
294
|
+
# This method and its instance variables are named for Loki, so that they are not names that
|
|
295
|
+
# a subclass may already use for its own, such as #labels or @labels.
|
|
296
|
+
def loki_labels
|
|
297
|
+
labels = stream[:stream]
|
|
298
|
+
return @loki_labels if @loki_labels&.of?(labels)
|
|
299
|
+
|
|
300
|
+
@loki_value_of ||= method(:loki_value)
|
|
301
|
+
@loki_labels = Labels.new(labels, @loki_value_of, max_label_names_per_series, max_label_name_length,
|
|
302
|
+
max_label_value_length, reserved: @loki_metadata_labels, symbols: !instance_of?(Loki))
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# Finishes the current stream: finishes its labels (see Labels#finish), which writes the
|
|
306
|
+
# named tags and context keys whose names are taken, reserved, or replaced, and takes out the
|
|
307
|
+
# labels past Loki's limits; then writes the process id and the thread name to the structured
|
|
308
|
+
# metadata, the supplied Hash, when structured_metadata names them (see
|
|
309
|
+
# #write_pid_and_thread), and after them, the labels taken out (see #write_moved_labels).
|
|
310
|
+
#
|
|
311
|
+
# When a subclass's own #build_stream writes no structured metadata, so that there is none
|
|
312
|
+
# (see #loki_stream), there is nowhere to write them, so the labels past the limits, and the
|
|
313
|
+
# process id and thread name, are left out, rather than having Loki reject the push.
|
|
314
|
+
def finish_loki_stream(metadata = payload_value)
|
|
315
|
+
moved = loki_labels.finish
|
|
316
|
+
return unless metadata.is_a?(Hash)
|
|
317
|
+
|
|
318
|
+
write_pid_and_thread(metadata)
|
|
319
|
+
write_moved_labels(moved, metadata) if moved
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
# Adds the labels past Loki's limits (see #finish_loki_stream) to the structured metadata,
|
|
323
|
+
# after the others, each under its name as Loki stores it, such as `a_b` for the label `a__b`
|
|
324
|
+
# (see #loki_name), and with PAYLOAD_PREFIX when a payload key with that name would have it,
|
|
325
|
+
# as for `duration` or `_` (see #prefixed_key?). When the structured metadata already has
|
|
326
|
+
# that name, as Loki stores it, the label is written with a number after it, as a label is
|
|
327
|
+
# (see .free_name), rather than with the prefix again, as a payload key is (see
|
|
328
|
+
# #add_prefixed), or not at all when the only key that Loki stores under that name has the
|
|
329
|
+
# same value (see #metadata_names).
|
|
330
|
+
def write_moved_labels(moved, metadata)
|
|
331
|
+
names = metadata_names(metadata)
|
|
332
|
+
moved.each do |name, value|
|
|
333
|
+
name = prefixed_key?(name) ? prefixed_name(name) : loki_name(name)
|
|
334
|
+
name = Loki.free_name(names, name, name.delete_suffix("_")) do |held|
|
|
335
|
+
!held.nil? && loki_value(held) == value
|
|
336
|
+
end
|
|
337
|
+
names[name] = metadata[name] = value if name
|
|
338
|
+
end
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Returns [Hash] each name of the structured metadata, as Loki stores it (see #loki_name),
|
|
342
|
+
# with its value when one key alone has that name, such as `a.b` for `a_b`, or else with nil,
|
|
343
|
+
# which no value is the same as (see #write_moved_labels): of the keys that it stores alike,
|
|
344
|
+
# Loki keeps the value of one whose name it changed, such as `a.b` over `a_b`.
|
|
345
|
+
def metadata_names(metadata)
|
|
346
|
+
names = {}
|
|
347
|
+
metadata.each_pair do |key, value|
|
|
348
|
+
name = loki_name(key.to_s)
|
|
349
|
+
names[name] = names.key?(name) ? nil : value
|
|
85
350
|
end
|
|
351
|
+
names
|
|
86
352
|
end
|
|
87
353
|
|
|
88
354
|
def time
|
|
@@ -101,7 +367,7 @@ module SemanticLogger
|
|
|
101
367
|
self.payload_value = if !log.payload?
|
|
102
368
|
{}
|
|
103
369
|
elsif log.payload.is_a?(Hash)
|
|
104
|
-
|
|
370
|
+
payload_metadata(log.payload)
|
|
105
371
|
else
|
|
106
372
|
# A log call can supply any value as the payload, such as a String
|
|
107
373
|
# parsed from a request body.
|
|
@@ -109,38 +375,127 @@ module SemanticLogger
|
|
|
109
375
|
end
|
|
110
376
|
end
|
|
111
377
|
|
|
378
|
+
# Every value is a String, since structured metadata only holds Strings: Loki rejects the
|
|
379
|
+
# whole push, with every entry in it, when one holds a number or null. The dimensions are a
|
|
380
|
+
# single value, as JSON.
|
|
112
381
|
def metric
|
|
113
382
|
return unless log.metric
|
|
114
383
|
|
|
115
|
-
payload_value[:metric]
|
|
116
|
-
payload_value[:metric_value] = log.metric_amount
|
|
384
|
+
payload_value[:metric] = loki_value(log.metric)
|
|
385
|
+
payload_value[:metric_value] = number_text(log.metric_amount) if log.metric_amount
|
|
386
|
+
payload_value[:dimensions] = loki_value(log.dimensions) if log.dimensions?
|
|
117
387
|
end
|
|
118
388
|
|
|
119
389
|
def duration
|
|
120
390
|
return unless log.duration
|
|
121
391
|
|
|
122
|
-
payload_value[:duration] = log.duration
|
|
392
|
+
payload_value[:duration] = number_text(log.duration)
|
|
123
393
|
payload_value[:duration_human] = log.duration_human
|
|
124
394
|
end
|
|
125
395
|
|
|
396
|
+
# Returns [String] the number as LogQL reads it, to compare it or `unwrap` it: an Integer as
|
|
397
|
+
# it is, and any other number as a Float, which is how LogQL holds it. A Rational's own text,
|
|
398
|
+
# such as `3/2`, would make every LogQL query that unwraps it fail, and a BigDecimal's, such
|
|
399
|
+
# as `0.25e1`, is not how anyone writes a number.
|
|
400
|
+
def number_text(number)
|
|
401
|
+
number.is_a?(Integer) ? number.to_s : number.to_f.to_s
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
# The stack trace is one line per frame, followed by each exception that caused it, as the
|
|
405
|
+
# text formatters write it.
|
|
406
|
+
#
|
|
407
|
+
# The exception is named as Utils.exception_name names it, as every formatter does: as Ruby
|
|
408
|
+
# names its class, without asking the class, such as `#<Class:0x000...>` for an anonymous class,
|
|
409
|
+
# which has no name. Its message is read with Utils.exception_message, so that a #message that
|
|
410
|
+
# raises, or does not return a String, cannot lose the entry.
|
|
126
411
|
def exception
|
|
127
412
|
return unless log.exception
|
|
128
413
|
|
|
129
414
|
payload_value.merge!(
|
|
130
|
-
exception_name: log.exception
|
|
131
|
-
exception_message: log.exception
|
|
132
|
-
stack_trace:
|
|
415
|
+
exception_name: Utils.exception_name(log.exception),
|
|
416
|
+
exception_message: Utils.exception_message(log.exception),
|
|
417
|
+
stack_trace: exception_backtrace
|
|
133
418
|
)
|
|
134
419
|
end
|
|
135
420
|
|
|
136
|
-
#
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
421
|
+
# Writes the process id and the thread name to the structured metadata, the supplied Hash, as
|
|
422
|
+
# Strings, when structured_metadata names them (see #initialize).
|
|
423
|
+
def write_pid_and_thread(metadata)
|
|
424
|
+
metadata[:pid] = pid.to_s unless @loki_pid_label
|
|
425
|
+
metadata[:thread] = log.thread_name if !@loki_thread_label && log.thread_name
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
# Returns [Hash] the payload as structured metadata, with String keys and values, since Loki
|
|
429
|
+
# only accepts Strings.
|
|
430
|
+
#
|
|
431
|
+
# A key that Loki would store as one of this formatter's own names (see #prefixed_key?), or
|
|
432
|
+
# that it would reject (see LETTER_OR_DIGIT), is written with PAYLOAD_PREFIX, after the other
|
|
433
|
+
# keys (see #add_prefixed).
|
|
434
|
+
def payload_metadata(hash)
|
|
435
|
+
metadata = {}
|
|
436
|
+
prefixed = nil
|
|
437
|
+
hash.each_pair do |key, value|
|
|
438
|
+
name = key.to_s
|
|
439
|
+
if prefixed_key?(name)
|
|
440
|
+
(prefixed ||= []) << [name, value]
|
|
441
|
+
else
|
|
442
|
+
metadata[name] = loki_value(value)
|
|
443
|
+
end
|
|
140
444
|
end
|
|
445
|
+
add_prefixed(metadata, prefixed) if prefixed
|
|
446
|
+
metadata
|
|
141
447
|
end
|
|
142
448
|
|
|
143
|
-
#
|
|
449
|
+
# Adds each of the payload's keys named like this formatter's own, or without an ASCII letter
|
|
450
|
+
# or digit, with PAYLOAD_PREFIX, under the name Loki stores it as (see #loki_name), such as
|
|
451
|
+
# `payload_metric_value` for `metric.value`, and `payload_` for an empty key or `-`, and with
|
|
452
|
+
# the prefix again while another key has that name once Loki has stored it. Loki keeps only
|
|
453
|
+
# one value of the names that it stores alike, so this keeps the value of each of these keys.
|
|
454
|
+
# Other payload keys that Loki stores alike, such as `a.b` and `a_b`, still keep only one.
|
|
455
|
+
#
|
|
456
|
+
# A key that is exactly one of this formatter's own names, OWN_KEYS and those that
|
|
457
|
+
# structured_metadata names, is named first, so that `payload_metric_value` holds the
|
|
458
|
+
# payload's `metric_value`, whatever the order of the keys, and `metric.value` is prefixed
|
|
459
|
+
# again.
|
|
460
|
+
def add_prefixed(metadata, prefixed)
|
|
461
|
+
taken = Set.new(metadata.each_key) { |name| loki_name(name) }
|
|
462
|
+
exact, lookalikes = prefixed.partition { |name, _value| @loki_own_keys.include?(name) }
|
|
463
|
+
(exact + lookalikes).each do |name, value|
|
|
464
|
+
name = prefixed_name(name)
|
|
465
|
+
name = PAYLOAD_PREFIX + name while taken.include?(name)
|
|
466
|
+
taken << name
|
|
467
|
+
metadata[name] = loki_value(value)
|
|
468
|
+
end
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
# Returns [String] the name with PAYLOAD_PREFIX, as Loki stores it.
|
|
472
|
+
#
|
|
473
|
+
# Joining the prefix with the name as Loki stores it gives the name Loki stores the joined
|
|
474
|
+
# name as: a name like one of this formatter's own starts with a letter, and one without a
|
|
475
|
+
# letter or digit is stored as an empty name or as `_`, which Loki would write as one
|
|
476
|
+
# underscore with the prefix's, so that underscore is left out.
|
|
477
|
+
def prefixed_name(name)
|
|
478
|
+
PAYLOAD_PREFIX + loki_name(name).delete_prefix("_")
|
|
479
|
+
end
|
|
480
|
+
|
|
481
|
+
# Returns [String] the name that Loki stores structured metadata of the supplied name under
|
|
482
|
+
# (see .loki_name).
|
|
483
|
+
def loki_name(name) = Loki.loki_name(name)
|
|
484
|
+
|
|
485
|
+
# Whether the payload key is written with PAYLOAD_PREFIX: Loki would store it as one of this
|
|
486
|
+
# formatter's own names, OWN_KEYS and those that structured_metadata names (see .own_name),
|
|
487
|
+
# or it has no ASCII letter or digit (see LETTER_OR_DIGIT).
|
|
488
|
+
#
|
|
489
|
+
# A name with characters outside ASCII is matched as Utils.to_json writes it: in UTF-8,
|
|
490
|
+
# without invalid bytes, which matching it as it is could raise for. So a key of invalid
|
|
491
|
+
# bytes alone is prefixed, since it is written as an empty name.
|
|
492
|
+
def prefixed_key?(name)
|
|
493
|
+
name = Utils.encode_utf8_string(name) unless name.ascii_only?
|
|
494
|
+
name.match?(@loki_own_name) || !name.match?(LETTER_OR_DIGIT)
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
# Returns [String] the value as Loki accepts it, as both the labels (see #loki_labels) and
|
|
498
|
+
# the structured metadata write it.
|
|
144
499
|
#
|
|
145
500
|
# A structure is written once, as JSON, depth-limited and repaired by Utils.to_json.
|
|
146
501
|
# Encoding each nested level into a String that the next level encodes again doubled the
|
|
@@ -156,6 +511,39 @@ module SemanticLogger
|
|
|
156
511
|
value.to_s
|
|
157
512
|
end
|
|
158
513
|
end
|
|
514
|
+
|
|
515
|
+
# Returns [Array<Symbol>] the labels to write to the structured metadata (see #initialize):
|
|
516
|
+
# none when the option is not supplied, with a deprecation warning, since v6 writes both
|
|
517
|
+
# there by default.
|
|
518
|
+
# Raises ArgumentError for anything but a list of STRUCTURED_METADATA, so that a mistake
|
|
519
|
+
# surfaces here rather than as a label left where it was.
|
|
520
|
+
def structured_metadata_option(labels)
|
|
521
|
+
if labels.nil?
|
|
522
|
+
Deprecation.warn(
|
|
523
|
+
"The :loki formatter writes pid and thread as stream labels by default, which changes in v6. " \
|
|
524
|
+
"Give it structured_metadata: [] to keep them as labels, or structured_metadata: [:pid, :thread] to " \
|
|
525
|
+
"write them as structured metadata, which needs a Loki that accepts it (schema v13 with " \
|
|
526
|
+
"allow_structured_metadata, the default since Loki 3.0), as in " \
|
|
527
|
+
"add_appender(appender: :loki, formatter: {loki: {structured_metadata: [:pid, :thread]}}). See " \
|
|
528
|
+
"https://logger.reidmorrison.com/appenders.html#pid-and-thread-as-structured-metadata"
|
|
529
|
+
)
|
|
530
|
+
return [].freeze
|
|
531
|
+
end
|
|
532
|
+
return labels.uniq.freeze if labels.is_a?(Array) && (labels - STRUCTURED_METADATA).empty?
|
|
533
|
+
|
|
534
|
+
raise(ArgumentError, "structured_metadata must be a list of :pid and :thread, not #{labels.inspect}")
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
# Returns [Integer] the supplied limit (see #initialize).
|
|
538
|
+
# Raises ArgumentError for any value other than an Integer of 1 or more, so that a mistake
|
|
539
|
+
# surfaces here rather than on every log entry. A limit of 0 would leave every stream without a
|
|
540
|
+
# label, which Loki stores as the one stream `service_name="unknown_service"`, or rejects when
|
|
541
|
+
# it does not add `service_name`.
|
|
542
|
+
def label_limit(option, value)
|
|
543
|
+
return value if value.is_a?(Integer) && value.positive?
|
|
544
|
+
|
|
545
|
+
raise(ArgumentError, "#{option} must be an Integer of 1 or more, not #{value.inspect}")
|
|
546
|
+
end
|
|
159
547
|
end
|
|
160
548
|
end
|
|
161
549
|
end
|
|
@@ -52,6 +52,7 @@ module SemanticLogger
|
|
|
52
52
|
tags: hash[:tags],
|
|
53
53
|
metric: hash[:metric],
|
|
54
54
|
metric_amount: hash[:metric_amount],
|
|
55
|
+
dimensions: hash[:dimensions],
|
|
55
56
|
environment: hash[:environment],
|
|
56
57
|
application: hash[:application],
|
|
57
58
|
payload: hash[:payload],
|
|
@@ -75,21 +76,12 @@ module SemanticLogger
|
|
|
75
76
|
result[:error] = {
|
|
76
77
|
message: hash[:exception][:message],
|
|
77
78
|
class: hash[:exception][:name],
|
|
78
|
-
|
|
79
|
+
# An exception that was never raised, such as one built to be logged, has no backtrace.
|
|
80
|
+
stack: stack_trace(Array(hash[:exception][:stack_trace]))
|
|
79
81
|
}
|
|
80
82
|
end
|
|
81
83
|
|
|
82
|
-
|
|
83
|
-
result[:file] = {
|
|
84
|
-
name: hash[:file]
|
|
85
|
-
}
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
if hash[:line]
|
|
89
|
-
result[:line] = {
|
|
90
|
-
number: hash[:line].to_s
|
|
91
|
-
}
|
|
92
|
-
end
|
|
84
|
+
add_location(result, hash)
|
|
93
85
|
|
|
94
86
|
# NOTE: Any named tags are merged directly into the result
|
|
95
87
|
# unless there are conflicts with other keys. In that
|
|
@@ -115,9 +107,19 @@ module SemanticLogger
|
|
|
115
107
|
|
|
116
108
|
private
|
|
117
109
|
|
|
110
|
+
# Where the entry was logged: the file name, line number, and method, when it has them.
|
|
111
|
+
#
|
|
112
|
+
# The method is a flat "method.name" key, which New Relic reads as the same attribute as a nested
|
|
113
|
+
# one, so that it does not replace a named tag called "method", such as a request's HTTP method.
|
|
114
|
+
def add_location(result, hash)
|
|
115
|
+
result[:file] = {name: hash[:file]} if hash[:file]
|
|
116
|
+
result[:line] = {number: hash[:line].to_s} if hash[:line]
|
|
117
|
+
result[:"method.name"] = hash[:method] if hash[:method]
|
|
118
|
+
end
|
|
119
|
+
|
|
118
120
|
# returns "entity.name", "trace.id" etc from NewRelic::Agent.linking_metadata
|
|
119
121
|
def linking_metadata(log)
|
|
120
|
-
|
|
122
|
+
log.captured(:new_relic_metadata) || {}
|
|
121
123
|
end
|
|
122
124
|
end
|
|
123
125
|
end
|