semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -2,7 +2,7 @@ begin
2
2
  require "kafka"
3
3
  rescue LoadError
4
4
  raise LoadError,
5
- 'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
5
+ 'Gem ruby-kafka is required for logging to Kafka. Please add the gem "ruby-kafka" to your Gemfile.'
6
6
  end
7
7
 
8
8
  require "date"
@@ -19,6 +19,9 @@ require "date"
19
19
  #
20
20
  # # Set an optional client id in order to identify the client to Kafka:
21
21
  # client_id: "my-application",
22
+ #
23
+ # # Leave out metric-only entries, as v6 does by default:
24
+ # metrics: false
22
25
  # )
23
26
  module SemanticLogger
24
27
  module Appender
@@ -38,8 +41,9 @@ module SemanticLogger
38
41
  # seed_brokers: [Array<String>, String]
39
42
  # The list of brokers used to initialize the client. Either an Array of connections,
40
43
  # or a comma separated string of connections.
41
- # Connections can either be a string of "port:protocol" or a full URI with a scheme.
42
- # If there's a scheme it's ignored and only host/port are used.
44
+ # Connections can either be a string of "hostname:port" or a full URI with a scheme.
45
+ # The scheme must be kafka, kafka+ssl, plaintext, or ssl, or ruby-kafka raises
46
+ # Kafka::Error. Otherwise only the host and port are used, the port defaulting to 9092.
43
47
  #
44
48
  # client_id: [String]
45
49
  # The identifier for this application.
@@ -78,7 +82,7 @@ module SemanticLogger
78
82
  # Must be used in combination with ssl_client_cert_key.
79
83
  # Default: nil
80
84
  #
81
- # ssl_client_cert_key [String]
85
+ # ssl_client_cert_key: [String]
82
86
  # A PEM encoded client cert key to use with a SSL connection.
83
87
  # Must be used in combination with ssl_client_cert.
84
88
  # Default: nil
@@ -86,28 +90,29 @@ module SemanticLogger
86
90
  # ssl_ca_certs_from_system: [boolean]
87
91
  # Delegate SSL CA cert to the system certs
88
92
  #
89
- # delivery_threshold: [Integer]
90
- # Number of messages between triggering a delivery of messages to Apache Kafka.
91
- # Default: 100
93
+ # delivery_threshold: [Integer]
94
+ # Number of messages between triggering a delivery of messages to Apache Kafka.
95
+ # Default: 100
92
96
  #
93
- # delivery_interval: [Integer]
94
- # Number of seconds between triggering a delivery of messages to Apache Kafka.
95
- # Default: 5
97
+ # delivery_interval: [Integer]
98
+ # Number of seconds between triggering a delivery of messages to Apache Kafka.
99
+ # Default: 10
96
100
  #
97
- # required_acks: [Integer]
98
- # Number of replicas that must acknowledge receipt of each log message to the topic
99
- # Default: 1
101
+ # required_acks: [Integer]
102
+ # Number of replicas that must acknowledge receipt of each log message to the topic
103
+ # Default: 1
100
104
  #
101
105
  # Semantic Logger Parameters:
102
106
  #
103
107
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
104
108
  # Override the log level for this appender.
105
- # Default: SemanticLogger.default_level
109
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
110
+ # filters by its own level.
106
111
  #
107
112
  # formatter: [Object|Proc|Symbol|Hash]
108
113
  # An instance of a class that implements #call, or a Proc to be used to format
109
114
  # the output from this appender
110
- # Default: :raw_json (See: #call)
115
+ # Default: :json
111
116
  #
112
117
  # filter: [Regexp|Proc]
113
118
  # RegExp: Only include log messages where the class name matches the supplied.
@@ -125,12 +130,12 @@ module SemanticLogger
125
130
  #
126
131
  # metrics: [Boolean]
127
132
  # Send metrics only events to kafka.
128
- # Default: true
133
+ # Default: true, with a deprecation warning when it is not supplied, since v6 changes it to false.
129
134
  def initialize(seed_brokers:, client_id: "semantic-logger", connect_timeout: nil, socket_timeout: nil,
130
135
  ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil, ssl_ca_certs_from_system: false,
131
136
  topic: "log_messages", partition: nil, partition_key: nil, key: nil,
132
137
  delivery_threshold: 100, delivery_interval: 10, required_acks: 1,
133
- metrics: true, **args, &)
138
+ metrics: METRICS_NOT_SUPPLIED, **args, &)
134
139
  @seed_brokers = seed_brokers
135
140
  @client_id = client_id
136
141
  @connect_timeout = connect_timeout
@@ -146,16 +151,120 @@ module SemanticLogger
146
151
  @delivery_threshold = delivery_threshold
147
152
  @delivery_interval = delivery_interval
148
153
  @required_acks = required_acks
154
+ @retired = Thread::Queue.new
155
+ @retiring = []
156
+ @generation = 0
149
157
 
150
- super(metrics: metrics, **args, &)
158
+ super(metrics: deprecated_metrics_default(metrics, :kafka), **args, &)
151
159
  reopen
152
160
  end
153
161
 
162
+ # Replaces the client and its producer with new ones. When building the new ones fails, the
163
+ # appender keeps the client and producer it has.
164
+ #
165
+ # It does not wait for the producer it replaced to deliver the messages it still holds: the next
166
+ # #log starts a thread that shuts that producer down, which delivers them, and closes the client it
167
+ # replaced, and #flush and #close wait for that thread, or shut them down themselves (see
168
+ # #start_retiring and #shut_down_retired). An application can reopen the appender in a signal trap,
169
+ # such as one that reopens the log files after a rotation, where ruby-kafka cannot shut down a
170
+ # producer whose threads have not started, since Ruby does not allow it to lock its Mutex there.
171
+ #
172
+ # It replaces the producer before it hands the old one on, so a #flush on another thread that runs
173
+ # entirely between the two returns before the old producer has delivered the messages it holds:
174
+ # the next #log, #flush, or #close shuts that producer down, which still delivers them.
175
+ #
176
+ # After a fork, the child first forgets the client and producer it inherited, and any that the
177
+ # parent had replaced but not shut down yet, without shutting them down or closing them: they
178
+ # belong to the parent. Shutting a producer down in the child would start its threads again there,
179
+ # and deliver another copy of each message the parent had not delivered yet, over the connections
180
+ # that the parent is still using. Ruby closes the child's copies of those connections when it
181
+ # collects them, which leaves the parent's open. When building the new ones fails in the child, the
182
+ # appender has no producer, and #log raises IOError, until a reopen builds one.
154
183
  def reopen
184
+ forget_inherited
185
+ kafka = build_client
186
+ producer = build_producer(kafka)
187
+ old_kafka = @kafka
188
+ old_producer = @producer
189
+ @kafka = kafka
190
+ @producer = producer
191
+ @pid = ::Process.pid
192
+ retire(old_kafka, old_producer)
193
+ @generation += 1
194
+ end
195
+
196
+ # Shuts down the producer, which delivers the messages it holds, and closes the client, and those
197
+ # that a reopen replaced, such as when the appender is removed.
198
+ #
199
+ # In a forked child that has not reopened, such as one that removes the appender before its reopen
200
+ # has run, it first forgets the clients and producers it inherited, without shutting them down or
201
+ # closing them, since they belong to the parent (see #reopen).
202
+ def close
203
+ forget_inherited
204
+ shut_down_retired
205
+ @producer&.shutdown
206
+ @producer = nil
207
+ @kafka&.close
208
+ @kafka = nil
209
+ end
210
+
211
+ # Forward log messages to Kafka producer thread. Raises IOError when the appender has no producer:
212
+ # after #close, or when a reopen in a forked process could not build one (see #reopen).
213
+ def log(log)
214
+ start_retiring
215
+ producer = @producer || raise(IOError, "The Kafka appender has no producer, since it was closed, " \
216
+ "or reopening it after a fork could not build one")
217
+ json = format_log(log)
218
+ producer.produce(json, topic: topic, partition: partition, partition_key: partition_key, key: key)
219
+ end
220
+
221
+ # Use JSON Formatter by default.
222
+ def default_formatter
223
+ SemanticLogger::Formatters::Json.new
224
+ end
225
+
226
+ # Shuts down the producer, which waits for it to deliver the messages it holds, since ruby-kafka has
227
+ # no other way to flush one, after replacing it with a new one. When building the new one fails, the
228
+ # appender keeps the producer it has, still running. First shuts down each producer that a reopen
229
+ # replaced, or waits for the thread that shuts it down (see #reopen). Does nothing after #close.
230
+ #
231
+ # A reopen that runs while it builds the new producer, such as in a signal trap, has already
232
+ # replaced the producer, and handed the old one to #shut_down_retired. The flush then drops the
233
+ # producer it built, which has neither threads nor connections, rather than putting it in place of
234
+ # the reopen's, and calls #shut_down_retired again, rather than shutting the old producer down
235
+ # itself, a second time.
236
+ def flush
237
+ shut_down_retired
238
+ return unless @producer
239
+
240
+ generation = @generation
241
+ old_producer = @producer
242
+ producer = build_producer(@kafka)
243
+ if generation == @generation
244
+ @producer = producer
245
+ old_producer.shutdown
246
+ else
247
+ shut_down_retired
248
+ end
249
+ end
250
+
251
+ # Only the topic: the seed brokers can hold credentials, and the options hold the TLS private
252
+ # key and SASL passwords. See Subscriber#inspect.
253
+ def inspect_details
254
+ "topic=#{@topic.inspect}"
255
+ end
256
+
257
+ private
258
+
259
+ attr_reader :producer
260
+
261
+ # Returns [Kafka::Client] a new client with the connection options of this appender. ruby-kafka connects to the
262
+ # brokers only when a producer first delivers messages, so this raises only for invalid options.
263
+ def build_client
155
264
  # ruby-kafka puts the seed broker, which can hold credentials, in the error it raises for an
156
265
  # invalid one: URI::InvalidURIError, or Kafka::Error for an unknown scheme.
157
266
  errors = [URI::InvalidURIError, ::Kafka::Error]
158
- @kafka = Utils.without_url_in_errors(option: ":seed_brokers", hint: SEED_BROKERS_HINT, errors: errors) do
267
+ Utils.without_url_in_errors(option: ":seed_brokers", hint: SEED_BROKERS_HINT, errors: errors) do
159
268
  ::Kafka.new(
160
269
  seed_brokers: seed_brokers,
161
270
  client_id: client_id,
@@ -168,50 +277,87 @@ module SemanticLogger
168
277
  logger: logger
169
278
  )
170
279
  end
280
+ end
171
281
 
172
- @producer = @kafka.async_producer(
282
+ # Returns [Kafka::AsyncProducer] a producer for the given Kafka client, with the delivery and acknowledgement
283
+ # options of this appender. #reopen and #flush both build their producer here, so that a flush keeps them, rather
284
+ # than leaving out an option, which ruby-kafka then replaces with its own default.
285
+ def build_producer(kafka)
286
+ kafka.async_producer(
173
287
  delivery_threshold: delivery_threshold,
174
288
  delivery_interval: delivery_interval,
175
289
  required_acks: required_acks
176
290
  )
177
291
  end
178
292
 
179
- def close
180
- @producer&.shutdown
181
- @producer = nil
182
- @kafka&.close
183
- @kafka = nil
293
+ # Keeps the client and producer that #reopen replaced for #start_retiring or #shut_down_retired,
294
+ # since #reopen can run in a signal trap. A Thread::Queue, since Ruby allows pushing to one there.
295
+ def retire(kafka, producer)
296
+ @retired << [kafka, producer] if producer
184
297
  end
185
298
 
186
- # Forward log messages to Kafka producer thread.
187
- def log(log)
188
- json = format_log(log)
189
- @producer.produce(json, topic: topic, partition: partition, partition_key: partition_key, key: key)
190
- end
299
+ # Starts a thread for each producer that #reopen replaced, to shut that producer down, which
300
+ # delivers the messages it still holds, and close its client. #log calls it first, rather than
301
+ # shutting them down itself, since it runs on the thread that writes to every appender: while
302
+ # Kafka cannot be reached, shutting a producer down can take half a minute, and once the
303
+ # producer's buffer is full, does not end until Kafka can be reached again, since ruby-kafka
304
+ # retries without a limit. Forgets the threads that have ended.
305
+ #
306
+ # A message that a #log already running gave a replaced producer is delivered too, since the next
307
+ # call starts that producer's shutdown only after it. Since the new producer delivers meanwhile,
308
+ # the messages of a replaced one can reach Kafka after newer ones. Only the thread that writes to
309
+ # the appenders calls #log, #flush, and #close, one at a time, so only that thread uses the list of
310
+ # threads.
311
+ def start_retiring
312
+ return if @retired.empty?
191
313
 
192
- # Use JSON Formatter by default.
193
- def default_formatter
194
- SemanticLogger::Formatters::Json.new
314
+ @retiring.select!(&:alive?)
315
+ while (retired = @retired.pop(timeout: 0))
316
+ @retiring << Thread.new(*retired) do |kafka, producer|
317
+ Thread.current.name = "SL Kafka retire"
318
+ shut_down(kafka, producer)
319
+ end
320
+ end
195
321
  end
196
322
 
197
- # Restart producer thread since there is no other way to flush.
198
- def flush
199
- @producer.shutdown
200
- @producer = @kafka.async_producer(
201
- delivery_threshold: delivery_threshold,
202
- delivery_interval: delivery_interval
203
- )
323
+ # Waits for each thread that #start_retiring started, then shuts down each producer that #reopen
324
+ # replaced since, and closes its client, oldest first. #flush and #close call it first, so that
325
+ # each producer that the appender replaced has delivered the messages it held when they return.
326
+ # Forgets the threads before waiting for them, so that a thread that ended with an exception
327
+ # raises it from one call at most.
328
+ def shut_down_retired
329
+ retiring = @retiring
330
+ @retiring = []
331
+ retiring.each(&:join)
332
+ while (retired = @retired.pop(timeout: 0))
333
+ shut_down(*retired)
334
+ end
204
335
  end
205
336
 
206
- # Only the topic: the seed brokers can hold credentials, and the options hold the TLS private
207
- # key and SASL passwords. See Subscriber#inspect.
208
- def inspect_details
209
- "topic=#{@topic.inspect}"
337
+ # Shuts down a producer that #reopen replaced, which delivers the messages it holds, and closes its
338
+ # client, even when the shutdown raises. Reports an error to the internal logger rather than
339
+ # raising it, as ruby-kafka does when it cannot deliver them, so that a producer the appender no
340
+ # longer uses cannot keep #flush or #close from shutting down the one it does use.
341
+ def shut_down(kafka, producer)
342
+ begin
343
+ producer.shutdown
344
+ ensure
345
+ kafka.close
346
+ end
347
+ rescue StandardError => e
348
+ logger.error("Could not shut down a Kafka producer that a reopen replaced", e)
210
349
  end
211
350
 
212
- private
351
+ # In a child process, forgets the client and producer built in the parent before a fork, and those
352
+ # the parent replaced but has not shut down yet. See #reopen.
353
+ def forget_inherited
354
+ return if @pid == ::Process.pid
213
355
 
214
- attr_reader :producer
356
+ @kafka = nil
357
+ @producer = nil
358
+ @retired = Thread::Queue.new
359
+ @retiring = []
360
+ end
215
361
  end
216
362
  end
217
363
  end
@@ -1,4 +1,4 @@
1
- # Forward application metrics to a Loki instance using HTTP push API
1
+ # Forward log messages to a Grafana Loki instance using its HTTP push API
2
2
  #
3
3
  # Example:
4
4
  # SemanticLogger.add_appender(
@@ -6,8 +6,12 @@
6
6
  # url: "https://logs-prod-001.grafana.net",
7
7
  # username: "grafana_username",
8
8
  # password: "grafana_token_here",
9
- # compress: true
9
+ # compress: true,
10
+ # formatter: {loki: {structured_metadata: [:pid, :thread]}}
10
11
  # )
12
+ #
13
+ # structured_metadata: needs a Loki that accepts structured metadata (see
14
+ # SemanticLogger::Formatters::Loki#initialize).
11
15
  module SemanticLogger
12
16
  module Appender
13
17
  class Loki < SemanticLogger::Appender::Http
@@ -23,11 +27,11 @@ module SemanticLogger
23
27
  # The Proc must return true or false.
24
28
  #
25
29
  # host: [String]
26
- # Name of this host to send as a dimension.
30
+ # Name of this host to send as a stream label.
27
31
  # Default: SemanticLogger.host
28
32
  #
29
33
  # application: [String]
30
- # Name of this application to send as a dimension.
34
+ # Name of this application to send as a stream label.
31
35
  # Default: SemanticLogger.application
32
36
  #
33
37
  # url: [String]
@@ -35,9 +39,28 @@ module SemanticLogger
35
39
  # Example: https://logs-prod-999.grafana.net
36
40
  # A query string is kept and sent with every request.
37
41
  # Example: https://logs-prod-999.grafana.net?orgid=my_org
38
- # Default: nil
42
+ # Required.
43
+ #
44
+ # path: [String]
45
+ # Path of the push API, appended to the path of the url.
46
+ # Default: 'loki/api/v1/push'
47
+ #
48
+ # header: [Hash]
49
+ # HTTP headers to send with every request, such as X-Scope-OrgID, merged over the
50
+ # headers of the HTTP appender, whose Content-Type is already application/json.
51
+ # Default: {'Content-Type' => 'application/json'}
52
+ #
53
+ # formatter: [Object|Proc|Symbol|Hash]
54
+ # An instance of a class that implements #call and #batch, or a Proc, to be used to
55
+ # format the output from this appender.
56
+ # A Hash sets the options of the Loki formatter, such as the labels to write as structured
57
+ # metadata, or the label limits of a Loki that raises them (see
58
+ # SemanticLogger::Formatters::Loki#initialize).
59
+ # Example: {loki: {structured_metadata: [:pid, :thread]}}
60
+ # Default: SemanticLogger::Formatters::Loki, which reports a deprecation warning without
61
+ # structured_metadata:, since v6 writes pid and thread as structured metadata by default.
39
62
  def initialize(url: nil,
40
- formatter: SemanticLogger::Formatters::Loki.new,
63
+ formatter: nil,
41
64
  header: {"Content-Type" => "application/json"},
42
65
  path: INGESTION_PATH,
43
66
  **args,
@@ -65,6 +88,19 @@ module SemanticLogger
65
88
  logger.trace(message)
66
89
  post(message)
67
90
  end
91
+
92
+ # The #batch above formats the whole batch with #format_batch, so it answers as
93
+ # Appender::Http#batch_refusal does for its own #batch: only while it is the one in use.
94
+ def batch_refusal
95
+ format_batch_refusal if batch_defined_by?(Loki)
96
+ end
97
+
98
+ private
99
+
100
+ # Use the Loki formatter by default
101
+ def default_formatter
102
+ SemanticLogger::Formatters::Loki.new
103
+ end
68
104
  end
69
105
  end
70
106
  end
@@ -81,7 +81,8 @@ module SemanticLogger
81
81
  #
82
82
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
83
83
  # Override the log level for this appender.
84
- # Default: SemanticLogger.default_level
84
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
85
+ # filters by its own level.
85
86
  #
86
87
  # formatter: [Object|Proc|Symbol]
87
88
  # An instance of a class that implements #call, or a Proc to be used to format
@@ -108,7 +109,7 @@ module SemanticLogger
108
109
  collection_max: nil,
109
110
  **args,
110
111
  &)
111
- @client = Mongo::Client.new(uri, logger: logger)
112
+ @uri = uri
112
113
  @collection_name = collection_name
113
114
  @options = {
114
115
  capped: true,
@@ -127,8 +128,28 @@ module SemanticLogger
127
128
 
128
129
  # After forking an active process call #reopen to re-open
129
130
  # open the handles to resources
131
+ #
132
+ # Opens the collection on the client again. Connects a new client first when the appender has none:
133
+ # when it is created, and after #close. After a fork, the child keeps the client it inherited, whose
134
+ # connections the MongoDB driver replaces itself.
130
135
  def reopen
131
- @collection = client[@collection_name, @options]
136
+ connect unless @client
137
+ @collection = @client[@collection_name, @options]
138
+ end
139
+
140
+ # Closes the client's connections, and stops its monitoring threads, such as when the appender is
141
+ # removed, and forgets it, so that a reopen connects a new one. An error closing it is reported to
142
+ # the internal logger, rather than raised.
143
+ #
144
+ # In a forked child, it forgets the client built in the parent instead, without closing it, whether
145
+ # or not the child reopened: closing it would send the server an endSessions command for the
146
+ # parent's sessions that the client holds.
147
+ def close
148
+ closing = @client
149
+ @client = nil
150
+ closing.close if closing && @pid == ::Process.pid
151
+ rescue StandardError => e
152
+ logger.warn("Could not close the MongoDB client", e)
132
153
  end
133
154
 
134
155
  # Create the required capped collection.
@@ -162,20 +183,38 @@ module SemanticLogger
162
183
  end
163
184
 
164
185
  # Log the message to MongoDB
186
+ #
187
+ # BSON stores only valid UTF-8 text, and keys without NUL bytes. A document that holds other
188
+ # text, such as a binary response body in the payload, or an exception message that quotes
189
+ # invalid input, is stored as a copy that BSON accepts (see #storable). Only such a document is
190
+ # copied: BSON checks the text as it serializes the document, before anything is sent.
165
191
  def log(log)
166
- # Insert log entry into Mongo
167
- collection.insert_one(format_log(log))
192
+ document = format_log(log)
193
+ begin
194
+ collection.insert_one(document)
195
+ rescue *TEXT_ERRORS
196
+ collection.insert_one(storable(document))
197
+ end
168
198
  true
169
199
  end
170
200
 
171
201
  # Errors raised for a document that cannot be stored, which retrying cannot insert either.
172
- # BSON raises BSON::Error, ArgumentError (a key with a NUL byte), or RangeError (an Integer too
173
- # large for 64 bits) while the document is serialized, before anything is sent, and the driver
174
- # raises Mongo::Error::MaxBSONSize for a document larger than the server accepts.
202
+ # BSON raises BSON::Error, ArgumentError (a NUL byte in text that it writes as a C string, such
203
+ # as a Regexp's source; the copy #log stores removes them from keys), or RangeError (an Integer too large
204
+ # for 64 bits) while the document is serialized, before anything is sent, and the driver raises
205
+ # Mongo::Error::MaxBSONSize for a document larger than the server accepts. A document whose text
206
+ # is at fault (TEXT_ERRORS) is first tried again as a copy that BSON accepts.
175
207
  DOCUMENT_ERRORS = [BSON::Error, ArgumentError, RangeError, Mongo::Error::MaxBSONSize].freeze
176
208
 
209
+ # Errors BSON raises for text it cannot store: EncodingError for text that is not valid UTF-8 or
210
+ # cannot be converted to it, and ArgumentError for a NUL byte in text that it writes as a C
211
+ # string, such as a key. The copy cannot fix every such text, such as a Regexp's source, which
212
+ # then raises again.
213
+ TEXT_ERRORS = [EncodingError, ArgumentError].freeze
214
+
177
215
  # Except BSON::Error::BSONDecodeError, which comes from reading the server's reply rather than
178
- # from the document, so it is retried like any other error from the server.
216
+ # from the document, so it is treated like any other error from the server: the thread backs
217
+ # off.
179
218
  def message_error?(exception)
180
219
  return false if exception.is_a?(BSON::Error::BSONDecodeError)
181
220
 
@@ -189,9 +228,55 @@ module SemanticLogger
189
228
 
190
229
  private
191
230
 
231
+ # Returns [Mongo::Client] a new client for the uri, after making it the appender's. Records the process
232
+ # that built it, which owns its sessions (see #close).
233
+ def connect
234
+ @client = Mongo::Client.new(@uri, logger: logger)
235
+ @pid = ::Process.pid
236
+ @client
237
+ end
238
+
192
239
  def default_formatter
193
240
  SemanticLogger::Formatters::Raw.new
194
241
  end
242
+
243
+ # Returns a copy of the document that BSON can store: every String and Symbol converted to
244
+ # valid UTF-8 (see Utils.encode_utf8), and the NUL bytes removed from every key, which BSON
245
+ # writes as a C string. Binary text is read as the UTF-8 it usually holds, and invalid bytes
246
+ # are dropped, as the JSON formatters write them. Any other value is kept as it is, for BSON
247
+ # to store in its own type.
248
+ #
249
+ # So that the copy is bounded, a Hash or Array nested too deeply, or that holds itself, is
250
+ # replaced where it is too deep or repeats, as the JSON formatters replace it (see
251
+ # Utils.bounded_structure). Only a document whose text BSON refuses before it reaches a
252
+ # structure that holds itself gets here: BSON's own serializer overflows the stack at one.
253
+ def storable(value, depth = 0, ancestors = nil)
254
+ return Utils.encode_utf8(value) unless value.is_a?(Hash) || value.is_a?(Array)
255
+
256
+ Utils.bounded_structure(value, depth, ancestors) { |level, seen| storable_members(value, level, seen) }
257
+ end
258
+
259
+ # Returns a copy of the members of a Hash or Array, each made storable (see #storable).
260
+ def storable_members(value, depth, ancestors)
261
+ if value.is_a?(Hash)
262
+ value.each_with_object({}) { |(key, val), hash| hash[storable_key(key)] = storable(val, depth, ancestors) }
263
+ else
264
+ value.map { |element| storable(element, depth, ancestors) }
265
+ end
266
+ end
267
+
268
+ # Returns the key, converted to valid UTF-8 and without NUL bytes (see #storable). A key other
269
+ # than a String or Symbol is kept as it is, for BSON to write as it writes the original key,
270
+ # such as an Integer as its digits, or to refuse, such as a Hash.
271
+ def storable_key(key)
272
+ return key unless key.is_a?(String) || key.is_a?(Symbol)
273
+
274
+ key = Utils.encode_utf8(key)
275
+ text = key.is_a?(Symbol) ? key.name : key
276
+ return key unless text.include?("\0")
277
+
278
+ key.is_a?(Symbol) ? text.delete("\0").to_sym : text.delete("\0")
279
+ end
195
280
  end
196
281
  end
197
282
  end
@@ -33,11 +33,10 @@ module SemanticLogger
33
33
  # Proc: Only include log messages where the supplied Proc returns true
34
34
  # The Proc must return true or false.
35
35
  def initialize(level: :error, **args, &block)
36
- Kernel.warn(
36
+ Deprecation.warn(
37
37
  "SemanticLogger::Appender::NewRelic (appender: :new_relic) is deprecated and will be removed in v6. " \
38
38
  "It reports errors via newrelic_rpm and is superseded by the log-forwarding appender " \
39
- "appender: :new_relic_logs.",
40
- category: :deprecated
39
+ "appender: :new_relic_logs."
41
40
  )
42
41
  super
43
42
  end
@@ -25,7 +25,7 @@ module SemanticLogger
25
25
  class NewRelicLogs < SemanticLogger::Subscriber
26
26
  CAPTURE_CONTEXT = lambda do |log|
27
27
  meta = ::NewRelic::Agent.linking_metadata
28
- log.set_context(:new_relic_metadata, meta) unless meta.empty?
28
+ log.capture(:new_relic_metadata, meta) unless meta.empty?
29
29
  end
30
30
 
31
31
  # Create Appender
@@ -33,7 +33,8 @@ module SemanticLogger
33
33
  # Parameters
34
34
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
35
35
  # Override the log level for this appender.
36
- # Default: SemanticLogger.default_level
36
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
37
+ # filters by its own level.
37
38
  #
38
39
  # formatter: [Object|Proc]
39
40
  # An instance of a class that implements #call, or a Proc to be used to format
@@ -45,7 +46,7 @@ module SemanticLogger
45
46
  # regular expression. All other messages will be ignored.
46
47
  # Proc: Only include log messages where the supplied Proc returns true
47
48
  # The Proc must return true or false.
48
- def initialize(formatter: SemanticLogger::Formatters::NewRelicLogs.new, **args, &block)
49
+ def initialize(formatter: nil, **args, &block)
49
50
  super
50
51
 
51
52
  # Record NewRelic's "trace.id"/"entity.name"/"hostname"/etc, so we can include them
@@ -55,23 +56,44 @@ module SemanticLogger
55
56
  SemanticLogger.on_log(CAPTURE_CONTEXT)
56
57
  end
57
58
 
58
- # Send an error notification to New Relic
59
+ # Send the log entry to the New Relic agent, which forwards it to New Relic Logs.
60
+ #
61
+ # Raises any error, as other appenders do, so that the asynchronous appender or the appenders
62
+ # that called it can count and report it (see #message_error?).
59
63
  def log(log)
60
64
  message = format_log(log) # Generate the structured log
61
65
  json_message = Utils.to_json(message) # Convert the log to JSON, repairing any non UTF-8 data
62
66
  level = log.level.to_s.upcase # Determine the log level
63
67
  self.class.log_newrelic(json_message, level)
64
68
  true
65
- rescue StandardError => e
66
- # Reported to the internal logger: this appender's own #warn would log the error through
67
- # this method again, recursing until SystemStackError.
68
- logger.warn("Could not log to New Relic", e)
69
+ end
70
+
71
+ # Every error that #log raises is one that writing the entry again cannot fix, so an
72
+ # asynchronous appender discards the entry, counts it and reports it, rather than backing off.
73
+ #
74
+ # The agent's LogEventAggregator#record only adds the entry to the agent's own buffer in this
75
+ # process, and rescues any StandardError of its own; the agent sends that buffer to New Relic
76
+ # later, on its own thread. So nothing here waits on New Relic, and an error comes either from
77
+ # the entry, when it cannot be formatted or converted to JSON, or from the agent not being
78
+ # available, before newrelic_rpm has been initialized, when NewRelic::Agent.agent is nil, such as
79
+ # during Rails boot before its initializer runs. An agent that is initialized but not started, or
80
+ # disabled, is not nil, and drops the entry itself without raising. Backing off for
81
+ # the latter would hold up every entry, and stop the worker thread after async_max_retries,
82
+ # dropping every entry after it, even once the agent has started.
83
+ def message_error?(_exception)
69
84
  true
70
85
  end
71
86
 
72
87
  def self.log_newrelic(json_message, level)
73
88
  ::NewRelic::Agent.agent.log_event_aggregator.record(json_message, level)
74
89
  end
90
+
91
+ private
92
+
93
+ # Use the New Relic Logs formatter by default
94
+ def default_formatter
95
+ SemanticLogger::Formatters::NewRelicLogs.new
96
+ end
75
97
  end
76
98
  end
77
99
  end