semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -1,16 +1,166 @@
1
1
  require "json"
2
+ require "semantic_logger/formatters/loki/labels"
2
3
 
3
4
  module SemanticLogger
4
5
  module Formatters
5
6
  class Loki < Base
6
7
  attr_accessor :stream, :payload_value
7
8
 
9
+ # The names of the structured metadata that this formatter writes itself for the fields of an
10
+ # entry, to which the labels named by `structured_metadata:` are added (see #initialize). A
11
+ # payload key that Loki would store under one of this formatter's own names is written with
12
+ # PAYLOAD_PREFIX, as in `payload_duration`, whether or not the entry has that field, so that
13
+ # each name always holds this formatter's value. Loki keeps one value of a name: the last of
14
+ # a name written twice, but of names that it stores alike, such as `metric.value` and
15
+ # `metric_value` (see .own_name), the one it had to change. So the payload's value would
16
+ # otherwise be lost, or replace this formatter's.
17
+ OWN_KEYS = %w[
18
+ metric metric_value dimensions duration duration_human exception_name exception_message
19
+ stack_trace
20
+ ].freeze
21
+
22
+ # The labels that `structured_metadata:` can write to the structured metadata instead (see
23
+ # #initialize).
24
+ STRUCTURED_METADATA = %i[pid thread].freeze
25
+
26
+ # A run of characters other than ASCII letters and digits, which Loki replaces with one
27
+ # underscore in the name of structured metadata. So Loki stores `metric.value`,
28
+ # `metric__value`, and `metric` followed by a zero width space and `value` all as
29
+ # `metric_value`.
30
+ LOKI_SEPARATOR = /[^A-Za-z0-9]+/
31
+
32
+ # Returns [Regexp] a pattern that matches a name that Loki stores as one of the supplied
33
+ # names: each underscore of a name stands for a LOKI_SEPARATOR. Matching it allocates
34
+ # nothing, unlike comparing #loki_name with each name, and nearly every payload key is
35
+ # checked against it.
36
+ def self.own_name(names)
37
+ /\A(?:#{names.map { |name| name.gsub('_', LOKI_SEPARATOR.source) }.join('|')})\z/
38
+ end
39
+
40
+ # Matches a name that Loki stores as one of OWN_KEYS.
41
+ OWN_NAME = own_name(OWN_KEYS)
42
+
43
+ # Matches a name that holds an ASCII letter or digit. Loki rejects the whole push for a
44
+ # structured metadata name without one, such as an empty name, `-`, or `_`, since it would
45
+ # store it as an empty name or as `_`.
46
+ LETTER_OR_DIGIT = /[A-Za-z0-9]/
47
+
48
+ # The prefix of a payload key named like one of this formatter's own structured metadata
49
+ # names, or without a letter or digit. The :logfmt and :open_telemetry formatters write
50
+ # `payload.`, which Loki would store as `payload_`.
51
+ PAYLOAD_PREFIX = "payload_".freeze
52
+
53
+ # A name that starts with a digit, which Loki stores with `key_` before it.
54
+ LEADING_DIGIT = /\A[0-9]/
55
+
56
+ # The most labels that Loki accepts for a stream, its `max_label_names_per_series`, the
57
+ # longest label name, its `max_label_name_length`, and the longest label value, its
58
+ # `max_label_value_length`, in bytes, unless a Loki tenant's `limits_config` raises them. See
59
+ # #initialize.
60
+ attr_reader :max_label_names_per_series, :max_label_name_length, :max_label_value_length
61
+
62
+ # The labels that are written to the structured metadata instead (see #initialize): a frozen
63
+ # Array of :pid, :thread, both, or neither.
64
+ attr_reader :structured_metadata
65
+
66
+ # Parameters
67
+ # structured_metadata: [Array<Symbol>]
68
+ # The labels to write to the structured metadata instead, as Strings: `:pid`, the process
69
+ # id, `:thread`, the thread name, both, or neither. Loki advises against labels like these,
70
+ # whose values are many and change: each process, and each of its threads, adds a stream,
71
+ # which Loki indexes and stores apart. As structured metadata, a query can still filter on
72
+ # them, as in `{application="shop"} | thread="worker-1"`. Their names are then among this
73
+ # formatter's own: a payload key of that name is written with PAYLOAD_PREFIX, as a key
74
+ # named like one of its other fields is, and a named tag or context key of that name is
75
+ # written as a label with a number, as in `thread_2`, since Loki would show the structured
76
+ # metadata as `thread_extracted` beside a label of its name.
77
+ # Structured metadata needs a Loki that accepts it, with schema v13 and
78
+ # `allow_structured_metadata`, which is on by default since Loki 3.0. Any other Loki
79
+ # rejects each entry that has structured metadata, which with `:pid` is every entry.
80
+ # Default: neither, so that both are labels, with a deprecation warning, since Semantic
81
+ # Logger v6 writes both to the structured metadata by default. Supply [] to keep both as
82
+ # labels without it.
83
+ #
84
+ # max_label_names_per_series: [Integer]
85
+ # The most labels that Loki accepts for a stream. Loki rejects the whole push, with every
86
+ # entry in it, for a stream with more, so a label past it, in the order they are written
87
+ # (see #build_stream), is written to the structured metadata instead.
88
+ # Default: 15, as Loki's `max_label_names_per_series`
89
+ # max_label_name_length: [Integer]
90
+ # The longest label name, in bytes, that Loki accepts. A label with a longer name is
91
+ # written to the structured metadata instead.
92
+ # Default: 1024, as Loki's `max_label_name_length`
93
+ # max_label_value_length: [Integer]
94
+ # The longest label value, in bytes, that Loki accepts. A label with a longer value, such as
95
+ # many tags, is written to the structured metadata instead.
96
+ # Default: 2048, as Loki's `max_label_value_length`
97
+ #
98
+ # Set each limit, an Integer of 1 or more, to the limit of the same name in the
99
+ # `limits_config` of a Loki tenant that raises it.
100
+ #
101
+ # See Formatters::Base for the other options.
102
+ #
103
+ # The instance variables that are not options are named for Loki, as in @loki_pid_label, so
104
+ # that they are not names that a subclass may already use for its own.
105
+ def initialize(structured_metadata: nil, max_label_names_per_series: 15, max_label_name_length: 1024,
106
+ max_label_value_length: 2048, **args)
107
+ super(**args)
108
+ @structured_metadata = structured_metadata_option(structured_metadata)
109
+ @loki_metadata_labels = @structured_metadata.map(&:name).freeze
110
+ @loki_own_keys = (OWN_KEYS + @loki_metadata_labels).freeze
111
+ @loki_own_name = @loki_metadata_labels.empty? ? OWN_NAME : Loki.own_name(@loki_own_keys)
112
+ @loki_pid_label = !@structured_metadata.include?(:pid)
113
+ @loki_thread_label = !@structured_metadata.include?(:thread)
114
+ @max_label_names_per_series = label_limit(:max_label_names_per_series, max_label_names_per_series)
115
+ @max_label_name_length = label_limit(:max_label_name_length, max_label_name_length)
116
+ @max_label_value_length = label_limit(:max_label_value_length, max_label_value_length)
117
+ end
118
+
119
+ # Returns [String|nil] the name under which to write a value, among names, a Hash of the
120
+ # names already written and their values: the name itself when it is free; when another value
121
+ # has it, the first name with a number after it that is free, as in `request_id_2`, so that
122
+ # both values are kept, since Loki keeps one value of a name; or nil when the block returns
123
+ # true for the value of that name, or of a numbered name before it, as the same value, so
124
+ # that it is written once. Both the labels (see Labels) and the structured metadata (see
125
+ # #write_moved_labels) are named so.
126
+ #
127
+ # The number is added to the stem, which is the name unless given. For structured metadata
128
+ # it is the name without a trailing underscore, so that `payload_` is numbered `payload_2`,
129
+ # as Loki would store `payload__2`. A name in reserved, which a value outside names has, is
130
+ # never free, nor the same, such as the label `thread` while the thread is structured
131
+ # metadata (see Labels).
132
+ def self.free_name(names, name, stem = name, reserved: nil)
133
+ candidate = name
134
+ number = 1
135
+ while names.key?(candidate) || reserved&.include?(candidate)
136
+ return if names.key?(candidate) && yield(names[candidate])
137
+
138
+ number += 1
139
+ candidate = "#{stem}_#{number}"
140
+ end
141
+ candidate
142
+ end
143
+
144
+ # Returns [String] the name that Loki stores structured metadata of the supplied name under:
145
+ # in UTF-8 without invalid bytes, as Utils.to_json writes it, with each run of characters
146
+ # other than ASCII letters and digits replaced with one underscore, and `key_` before a name
147
+ # that starts with a digit.
148
+ #
149
+ # The `key_` cannot change which name a prefixed key gets: that name starts with
150
+ # PAYLOAD_PREFIX, and only the other keys it is checked against can start with a digit. It is
151
+ # there so that this returns the name Loki stores for any name, not only for the names it is
152
+ # used for here.
153
+ def self.loki_name(name)
154
+ name = Utils.encode_utf8_string(name).gsub(LOKI_SEPARATOR, "_")
155
+ name.match?(LEADING_DIGIT) ? "key_#{name}" : name
156
+ end
157
+
8
158
  # Returns [String] a single JSON log
9
159
  def call(log, logger)
10
160
  self.logger = logger
11
161
  self.log = log
12
162
 
13
- Utils.to_json({streams: [build_stream]})
163
+ Utils.to_json({streams: [loki_stream]})
14
164
  end
15
165
 
16
166
  # Returns [String] a JSON batch of logs
@@ -19,7 +169,7 @@ module SemanticLogger
19
169
 
20
170
  streams = logs.map do |log|
21
171
  self.log = log
22
- build_stream
172
+ loki_stream
23
173
  end
24
174
 
25
175
  Utils.to_json({streams: streams})
@@ -27,8 +177,53 @@ module SemanticLogger
27
177
 
28
178
  private
29
179
 
180
+ # Returns [Hash] the stream of the current log entry, which #build_stream builds.
181
+ #
182
+ # A subclass that overrides #build_stream without calling super, as one written for v5.2
183
+ # may, does not finish the stream's labels, so they are finished here (see
184
+ # #finish_loki_stream) when the stream it returns has the current stream's labels. So it
185
+ # still writes each named tag and context key whose name is taken, reserved, or replaced,
186
+ # writes the process id and the thread name to the structured metadata when
187
+ # structured_metadata names them, and moves the labels past Loki's limits. The labels are
188
+ # finished again when a subclass added one after its #build_stream called super (see
189
+ # Labels#finished?), such as one under a Symbol whose name another label has. The structured
190
+ # metadata written is that of the entry it returns (see #entry_metadata). A formatter of
191
+ # this class itself finishes them in #build_stream, and so skips this.
192
+ def loki_stream
193
+ return build_stream if instance_of?(Loki)
194
+
195
+ built = build_stream
196
+ finish_loki_stream(entry_metadata(built)) if current_labels?(built) && !loki_labels.finished?
197
+ built
198
+ end
199
+
200
+ # Whether the stream that #build_stream returned has the current stream's labels, a Hash,
201
+ # which a subclass's own stream may not, such as one under String keys.
202
+ def current_labels?(built)
203
+ labels = built[:stream] if built.is_a?(Hash)
204
+ labels.is_a?(Hash) && stream.is_a?(Hash) && labels.equal?(stream[:stream])
205
+ end
206
+
207
+ # Returns [Hash|nil] the structured metadata of the entry of the stream that #build_stream
208
+ # returned, the third of its values: payload_value, unless a subclass wrote a copy of it, as
209
+ # in `payload_value.merge(...)`, or nil when it wrote none.
210
+ def entry_metadata(built)
211
+ values = built[:values]
212
+ entry = values[0] if values.is_a?(Array)
213
+ entry[2] if entry.is_a?(Array)
214
+ end
215
+
216
+ # The labels are written in this order: this formatter's own (pid, application, environment,
217
+ # host, level, thread, and tags, without those that structured_metadata names), each named
218
+ # tag, and each value of the context, under their own names, and then each named tag and
219
+ # context key whose name another label has, or that Loki does not accept, or that
220
+ # structured_metadata names, under another name, or not at all (see Labels#add). So every
221
+ # label written under its own name keeps it. Their names are Strings, so that a named tag or
222
+ # context key, whether a Symbol or a String, is found among them by name, as Loki finds it.
223
+ # The labels past Loki's limits are then taken out and written to the structured metadata
224
+ # (see #finish_loki_stream).
30
225
  def build_stream
31
- self.stream = {stream: {pid: pid}, values: [[]]}
226
+ self.stream = {stream: @loki_pid_label ? {"pid" => pid} : {}, values: [[]]}
32
227
 
33
228
  application
34
229
  environment
@@ -44,45 +239,116 @@ module SemanticLogger
44
239
  metric
45
240
  duration
46
241
  exception
242
+ finish_loki_stream
47
243
 
48
244
  stream[:values][0] << payload_value
49
245
  stream
50
246
  end
51
247
 
52
248
  def host
53
- stream[:stream][:host] = logger.host if log_host && logger.host.to_s
249
+ stream[:stream]["host"] = logger.host if log_host && logger&.host
54
250
  end
55
251
 
56
252
  def application
57
- stream[:stream][:application] = logger.application if log_application && logger&.application
253
+ stream[:stream]["application"] = logger.application if log_application && logger&.application
58
254
  end
59
255
 
60
256
  def environment
61
- stream[:stream][:environment] = logger.environment if log_environment && logger&.environment
257
+ stream[:stream]["environment"] = logger.environment if log_environment && logger&.environment
62
258
  end
63
259
 
64
260
  def level
65
- stream[:stream][:level] = log.level
261
+ stream[:stream]["level"] = log.level
66
262
  end
67
263
 
68
264
  def thread
69
- stream[:stream][:thread] = log.thread_name if log.thread_name
265
+ stream[:stream]["thread"] = log.thread_name if @loki_thread_label && log.thread_name
70
266
  end
71
267
 
72
268
  def tags
73
- stream[:stream][:tags] = log.tags if log.tags.respond_to?(:empty?) && !log.tags.empty?
269
+ stream[:stream]["tags"] = log.tags if log.tags.respond_to?(:empty?) && !log.tags.empty?
74
270
  end
75
271
 
76
272
  def named_tags
77
- stream[:stream].merge!(log.named_tags) if log.named_tags.respond_to?(:empty?) && !log.named_tags.empty?
273
+ return unless log.named_tags.respond_to?(:empty?) && !log.named_tags.empty?
274
+
275
+ stream_labels = loki_labels
276
+ log.named_tags.each_pair { |key, value| stream_labels.add(key, value) }
78
277
  end
79
278
 
80
279
  def context
81
280
  return unless log.context && !log.context.empty?
82
281
 
83
- log.context.each do |key, value|
84
- stream[:stream].merge!(key.to_s => loki_value(value))
282
+ stream_labels = loki_labels
283
+ log.context.each { |key, value| stream_labels.add(key, value) }
284
+ end
285
+
286
+ # Returns [Labels] the labels of the current stream, to which the named tags and the context
287
+ # are added, created for it unless they are already its labels. So a subclass whose own
288
+ # #build_stream does not create them, as in v5.2, still adds the named tags and the context.
289
+ # Each label value is converted by #loki_value, so a subclass that overrides it converts the
290
+ # labels' values too. This formatter's own methods write every label under a String, while a
291
+ # subclass's may write one under a Symbol, as v5.2 did, so only a subclass's labels are told
292
+ # that they may have a Symbol name (see Labels#initialize).
293
+ #
294
+ # This method and its instance variables are named for Loki, so that they are not names that
295
+ # a subclass may already use for its own, such as #labels or @labels.
296
+ def loki_labels
297
+ labels = stream[:stream]
298
+ return @loki_labels if @loki_labels&.of?(labels)
299
+
300
+ @loki_value_of ||= method(:loki_value)
301
+ @loki_labels = Labels.new(labels, @loki_value_of, max_label_names_per_series, max_label_name_length,
302
+ max_label_value_length, reserved: @loki_metadata_labels, symbols: !instance_of?(Loki))
303
+ end
304
+
305
+ # Finishes the current stream: finishes its labels (see Labels#finish), which writes the
306
+ # named tags and context keys whose names are taken, reserved, or replaced, and takes out the
307
+ # labels past Loki's limits; then writes the process id and the thread name to the structured
308
+ # metadata, the supplied Hash, when structured_metadata names them (see
309
+ # #write_pid_and_thread), and after them, the labels taken out (see #write_moved_labels).
310
+ #
311
+ # When a subclass's own #build_stream writes no structured metadata, so that there is none
312
+ # (see #loki_stream), there is nowhere to write them, so the labels past the limits, and the
313
+ # process id and thread name, are left out, rather than having Loki reject the push.
314
+ def finish_loki_stream(metadata = payload_value)
315
+ moved = loki_labels.finish
316
+ return unless metadata.is_a?(Hash)
317
+
318
+ write_pid_and_thread(metadata)
319
+ write_moved_labels(moved, metadata) if moved
320
+ end
321
+
322
+ # Adds the labels past Loki's limits (see #finish_loki_stream) to the structured metadata,
323
+ # after the others, each under its name as Loki stores it, such as `a_b` for the label `a__b`
324
+ # (see #loki_name), and with PAYLOAD_PREFIX when a payload key with that name would have it,
325
+ # as for `duration` or `_` (see #prefixed_key?). When the structured metadata already has
326
+ # that name, as Loki stores it, the label is written with a number after it, as a label is
327
+ # (see .free_name), rather than with the prefix again, as a payload key is (see
328
+ # #add_prefixed), or not at all when the only key that Loki stores under that name has the
329
+ # same value (see #metadata_names).
330
+ def write_moved_labels(moved, metadata)
331
+ names = metadata_names(metadata)
332
+ moved.each do |name, value|
333
+ name = prefixed_key?(name) ? prefixed_name(name) : loki_name(name)
334
+ name = Loki.free_name(names, name, name.delete_suffix("_")) do |held|
335
+ !held.nil? && loki_value(held) == value
336
+ end
337
+ names[name] = metadata[name] = value if name
338
+ end
339
+ end
340
+
341
+ # Returns [Hash] each name of the structured metadata, as Loki stores it (see #loki_name),
342
+ # with its value when one key alone has that name, such as `a.b` for `a_b`, or else with nil,
343
+ # which no value is the same as (see #write_moved_labels): of the keys that it stores alike,
344
+ # Loki keeps the value of one whose name it changed, such as `a.b` over `a_b`.
345
+ def metadata_names(metadata)
346
+ names = {}
347
+ metadata.each_pair do |key, value|
348
+ name = loki_name(key.to_s)
349
+ names[name] = names.key?(name) ? nil : value
85
350
  end
351
+ names
86
352
  end
87
353
 
88
354
  def time
@@ -101,7 +367,7 @@ module SemanticLogger
101
367
  self.payload_value = if !log.payload?
102
368
  {}
103
369
  elsif log.payload.is_a?(Hash)
104
- stringify_hash(log.payload)
370
+ payload_metadata(log.payload)
105
371
  else
106
372
  # A log call can supply any value as the payload, such as a String
107
373
  # parsed from a request body.
@@ -109,38 +375,127 @@ module SemanticLogger
109
375
  end
110
376
  end
111
377
 
378
+ # Every value is a String, since structured metadata only holds Strings: Loki rejects the
379
+ # whole push, with every entry in it, when one holds a number or null. The dimensions are a
380
+ # single value, as JSON.
112
381
  def metric
113
382
  return unless log.metric
114
383
 
115
- payload_value[:metric] = log.metric
116
- payload_value[:metric_value] = log.metric_amount
384
+ payload_value[:metric] = loki_value(log.metric)
385
+ payload_value[:metric_value] = number_text(log.metric_amount) if log.metric_amount
386
+ payload_value[:dimensions] = loki_value(log.dimensions) if log.dimensions?
117
387
  end
118
388
 
119
389
  def duration
120
390
  return unless log.duration
121
391
 
122
- payload_value[:duration] = log.duration.to_s
392
+ payload_value[:duration] = number_text(log.duration)
123
393
  payload_value[:duration_human] = log.duration_human
124
394
  end
125
395
 
396
+ # Returns [String] the number as LogQL reads it, to compare it or `unwrap` it: an Integer as
397
+ # it is, and any other number as a Float, which is how LogQL holds it. A Rational's own text,
398
+ # such as `3/2`, would make every LogQL query that unwraps it fail, and a BigDecimal's, such
399
+ # as `0.25e1`, is not how anyone writes a number.
400
+ def number_text(number)
401
+ number.is_a?(Integer) ? number.to_s : number.to_f.to_s
402
+ end
403
+
404
+ # The stack trace is one line per frame, followed by each exception that caused it, as the
405
+ # text formatters write it.
406
+ #
407
+ # The exception is named as Utils.exception_name names it, as every formatter does: as Ruby
408
+ # names its class, without asking the class, such as `#<Class:0x000...>` for an anonymous class,
409
+ # which has no name. Its message is read with Utils.exception_message, so that a #message that
410
+ # raises, or does not return a String, cannot lose the entry.
126
411
  def exception
127
412
  return unless log.exception
128
413
 
129
414
  payload_value.merge!(
130
- exception_name: log.exception.class.name,
131
- exception_message: log.exception.message,
132
- stack_trace: log.exception.backtrace.to_s
415
+ exception_name: Utils.exception_name(log.exception),
416
+ exception_message: Utils.exception_message(log.exception),
417
+ stack_trace: exception_backtrace
133
418
  )
134
419
  end
135
420
 
136
- # Loki only accepts Strings as keys and values.
137
- def stringify_hash(hash)
138
- hash.each_with_object({}) do |(key, value), result|
139
- result[key.to_s] = loki_value(value)
421
+ # Writes the process id and the thread name to the structured metadata, the supplied Hash, as
422
+ # Strings, when structured_metadata names them (see #initialize).
423
+ def write_pid_and_thread(metadata)
424
+ metadata[:pid] = pid.to_s unless @loki_pid_label
425
+ metadata[:thread] = log.thread_name if !@loki_thread_label && log.thread_name
426
+ end
427
+
428
+ # Returns [Hash] the payload as structured metadata, with String keys and values, since Loki
429
+ # only accepts Strings.
430
+ #
431
+ # A key that Loki would store as one of this formatter's own names (see #prefixed_key?), or
432
+ # that it would reject (see LETTER_OR_DIGIT), is written with PAYLOAD_PREFIX, after the other
433
+ # keys (see #add_prefixed).
434
+ def payload_metadata(hash)
435
+ metadata = {}
436
+ prefixed = nil
437
+ hash.each_pair do |key, value|
438
+ name = key.to_s
439
+ if prefixed_key?(name)
440
+ (prefixed ||= []) << [name, value]
441
+ else
442
+ metadata[name] = loki_value(value)
443
+ end
140
444
  end
445
+ add_prefixed(metadata, prefixed) if prefixed
446
+ metadata
141
447
  end
142
448
 
143
- # Returns [String] the value as Loki accepts it.
449
+ # Adds each of the payload's keys named like this formatter's own, or without an ASCII letter
450
+ # or digit, with PAYLOAD_PREFIX, under the name Loki stores it as (see #loki_name), such as
451
+ # `payload_metric_value` for `metric.value`, and `payload_` for an empty key or `-`, and with
452
+ # the prefix again while another key has that name once Loki has stored it. Loki keeps only
453
+ # one value of the names that it stores alike, so this keeps the value of each of these keys.
454
+ # Other payload keys that Loki stores alike, such as `a.b` and `a_b`, still keep only one.
455
+ #
456
+ # A key that is exactly one of this formatter's own names, OWN_KEYS and those that
457
+ # structured_metadata names, is named first, so that `payload_metric_value` holds the
458
+ # payload's `metric_value`, whatever the order of the keys, and `metric.value` is prefixed
459
+ # again.
460
+ def add_prefixed(metadata, prefixed)
461
+ taken = Set.new(metadata.each_key) { |name| loki_name(name) }
462
+ exact, lookalikes = prefixed.partition { |name, _value| @loki_own_keys.include?(name) }
463
+ (exact + lookalikes).each do |name, value|
464
+ name = prefixed_name(name)
465
+ name = PAYLOAD_PREFIX + name while taken.include?(name)
466
+ taken << name
467
+ metadata[name] = loki_value(value)
468
+ end
469
+ end
470
+
471
+ # Returns [String] the name with PAYLOAD_PREFIX, as Loki stores it.
472
+ #
473
+ # Joining the prefix with the name as Loki stores it gives the name Loki stores the joined
474
+ # name as: a name like one of this formatter's own starts with a letter, and one without a
475
+ # letter or digit is stored as an empty name or as `_`, which Loki would write as one
476
+ # underscore with the prefix's, so that underscore is left out.
477
+ def prefixed_name(name)
478
+ PAYLOAD_PREFIX + loki_name(name).delete_prefix("_")
479
+ end
480
+
481
+ # Returns [String] the name that Loki stores structured metadata of the supplied name under
482
+ # (see .loki_name).
483
+ def loki_name(name) = Loki.loki_name(name)
484
+
485
+ # Whether the payload key is written with PAYLOAD_PREFIX: Loki would store it as one of this
486
+ # formatter's own names, OWN_KEYS and those that structured_metadata names (see .own_name),
487
+ # or it has no ASCII letter or digit (see LETTER_OR_DIGIT).
488
+ #
489
+ # A name with characters outside ASCII is matched as Utils.to_json writes it: in UTF-8,
490
+ # without invalid bytes, which matching it as it is could raise for. So a key of invalid
491
+ # bytes alone is prefixed, since it is written as an empty name.
492
+ def prefixed_key?(name)
493
+ name = Utils.encode_utf8_string(name) unless name.ascii_only?
494
+ name.match?(@loki_own_name) || !name.match?(LETTER_OR_DIGIT)
495
+ end
496
+
497
+ # Returns [String] the value as Loki accepts it, as both the labels (see #loki_labels) and
498
+ # the structured metadata write it.
144
499
  #
145
500
  # A structure is written once, as JSON, depth-limited and repaired by Utils.to_json.
146
501
  # Encoding each nested level into a String that the next level encodes again doubled the
@@ -156,6 +511,39 @@ module SemanticLogger
156
511
  value.to_s
157
512
  end
158
513
  end
514
+
515
+ # Returns [Array<Symbol>] the labels to write to the structured metadata (see #initialize):
516
+ # none when the option is not supplied, with a deprecation warning, since v6 writes both
517
+ # there by default.
518
+ # Raises ArgumentError for anything but a list of STRUCTURED_METADATA, so that a mistake
519
+ # surfaces here rather than as a label left where it was.
520
+ def structured_metadata_option(labels)
521
+ if labels.nil?
522
+ Deprecation.warn(
523
+ "The :loki formatter writes pid and thread as stream labels by default, which changes in v6. " \
524
+ "Give it structured_metadata: [] to keep them as labels, or structured_metadata: [:pid, :thread] to " \
525
+ "write them as structured metadata, which needs a Loki that accepts it (schema v13 with " \
526
+ "allow_structured_metadata, the default since Loki 3.0), as in " \
527
+ "add_appender(appender: :loki, formatter: {loki: {structured_metadata: [:pid, :thread]}}). See " \
528
+ "https://logger.reidmorrison.com/appenders.html#pid-and-thread-as-structured-metadata"
529
+ )
530
+ return [].freeze
531
+ end
532
+ return labels.uniq.freeze if labels.is_a?(Array) && (labels - STRUCTURED_METADATA).empty?
533
+
534
+ raise(ArgumentError, "structured_metadata must be a list of :pid and :thread, not #{labels.inspect}")
535
+ end
536
+
537
+ # Returns [Integer] the supplied limit (see #initialize).
538
+ # Raises ArgumentError for any value other than an Integer of 1 or more, so that a mistake
539
+ # surfaces here rather than on every log entry. A limit of 0 would leave every stream without a
540
+ # label, which Loki stores as the one stream `service_name="unknown_service"`, or rejects when
541
+ # it does not add `service_name`.
542
+ def label_limit(option, value)
543
+ return value if value.is_a?(Integer) && value.positive?
544
+
545
+ raise(ArgumentError, "#{option} must be an Integer of 1 or more, not #{value.inspect}")
546
+ end
159
547
  end
160
548
  end
161
549
  end
@@ -52,6 +52,7 @@ module SemanticLogger
52
52
  tags: hash[:tags],
53
53
  metric: hash[:metric],
54
54
  metric_amount: hash[:metric_amount],
55
+ dimensions: hash[:dimensions],
55
56
  environment: hash[:environment],
56
57
  application: hash[:application],
57
58
  payload: hash[:payload],
@@ -75,21 +76,12 @@ module SemanticLogger
75
76
  result[:error] = {
76
77
  message: hash[:exception][:message],
77
78
  class: hash[:exception][:name],
78
- stack: hash[:exception][:stack_trace].join("\n")
79
+ # An exception that was never raised, such as one built to be logged, has no backtrace.
80
+ stack: stack_trace(Array(hash[:exception][:stack_trace]))
79
81
  }
80
82
  end
81
83
 
82
- if hash[:file]
83
- result[:file] = {
84
- name: hash[:file]
85
- }
86
- end
87
-
88
- if hash[:line]
89
- result[:line] = {
90
- number: hash[:line].to_s
91
- }
92
- end
84
+ add_location(result, hash)
93
85
 
94
86
  # NOTE: Any named tags are merged directly into the result
95
87
  # unless there are conflicts with other keys. In that
@@ -115,9 +107,19 @@ module SemanticLogger
115
107
 
116
108
  private
117
109
 
110
+ # Where the entry was logged: the file name, line number, and method, when it has them.
111
+ #
112
+ # The method is a flat "method.name" key, which New Relic reads as the same attribute as a nested
113
+ # one, so that it does not replace a named tag called "method", such as a request's HTTP method.
114
+ def add_location(result, hash)
115
+ result[:file] = {name: hash[:file]} if hash[:file]
116
+ result[:line] = {number: hash[:line].to_s} if hash[:line]
117
+ result[:"method.name"] = hash[:method] if hash[:method]
118
+ end
119
+
118
120
  # returns "entity.name", "trace.id" etc from NewRelic::Agent.linking_metadata
119
121
  def linking_metadata(log)
120
- (log.context && log.context[:new_relic_metadata]) || {}
122
+ log.captured(:new_relic_metadata) || {}
121
123
  end
122
124
  end
123
125
  end