semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -12,9 +12,38 @@ require "json"
12
12
  # url: 'http://localhost:8080',
13
13
  # token: '70CA900C-3D7E-42A4-9C79-7975D1C422A8'
14
14
  # )
15
+ #
16
+ # Batching:
17
+ # Add it with `batch: true` to send many entries in each request, as event objects stacked one after
18
+ # the other, each one what the appender sends for that entry on its own (see #batch).
19
+ #
20
+ # Responses:
21
+ # The HTTP Event Collector answers 400 Bad Request both for the entries a request holds, and for its
22
+ # own configuration, such as an index it does not know. Its JSON body says which, with a numeric
23
+ # `code` and a `text`. A batch it rejects for its configuration is discarded at once, rather than
24
+ # split up to find the entries at fault, since every part would be rejected too (see
25
+ # #fails_any_batch?). For a batch it rejects at one of its events, it says which one, having indexed
26
+ # those before it (see #rejected_entry).
15
27
  module SemanticLogger
16
28
  module Appender
17
29
  class SplunkHttp < SemanticLogger::Appender::Http
30
+ # The status codes in the body of a 400 Bad Request from the HTTP Event Collector that reject
31
+ # the request for the collector's configuration or the appender's, whatever entries it holds:
32
+ # 7 Incorrect index
33
+ # 10 Data channel is missing
34
+ # 11 Invalid data channel
35
+ # 14 ACK is disabled
36
+ # 16 Query string authorization is not enabled
37
+ # 21 Invalid token
38
+ # 22 Token disabled
39
+ # The others it answers with a 400 are caused by the entries, so a batch is split up to find
40
+ # them: 5 No data, 6 Invalid data format, 12 Event field is required, 13 Event field cannot be
41
+ # blank, and 15 Error in handling indexed fields.
42
+ #
43
+ # See "Troubleshoot HTTP Event Collector", Possible error codes:
44
+ # https://help.splunk.com/en/splunk-enterprise/get-started/get-data-in/10.4/get-data-with-http-event-collector/troubleshoot-http-event-collector
45
+ CONFIGURATION_CODES = [7, 10, 11, 14, 16, 21, 22].freeze
46
+
18
47
  attr_accessor :source_type, :index
19
48
 
20
49
  # Create Splunk appender over persistent HTTP(S)
@@ -56,7 +85,8 @@ module SemanticLogger
56
85
  #
57
86
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
58
87
  # Override the log level for this appender.
59
- # Default: SemanticLogger.default_level
88
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
89
+ # filters by its own level.
60
90
  #
61
91
  # formatter: [Object|Proc]
62
92
  # An instance of a class that implements #call, or a Proc to be used to format
@@ -100,6 +130,67 @@ module SemanticLogger
100
130
  message[:index] = index if index
101
131
  Utils.to_json(message)
102
132
  end
133
+
134
+ # Whether the exception would fail any batch, whatever entries it holds (see
135
+ # Http#fails_any_batch?), including a 400 Bad Request whose body holds one of the collector's
136
+ # CONFIGURATION_CODES, such as for a wrong `index:`. So the batch is discarded at once, and
137
+ # reported once, with the collector's text, rather than split up and each entry sent on its own.
138
+ def fails_any_batch?(exception)
139
+ super || (exception.is_a?(ResponseError) && exception.rejected? &&
140
+ CONFIGURATION_CODES.include?(collector_response(exception.body)["code"]))
141
+ end
142
+
143
+ # Send a batch of log messages to the HTTP Event Collector in a single request, as event objects
144
+ # stacked one after the other, the batch format of "Format events for HTTP Event Collector":
145
+ # https://help.splunk.com/en/splunk-enterprise/get-started/get-data-in/10.6/get-data-with-http-event-collector/format-events-for-http-event-collector
146
+ #
147
+ # Each event is what #log sends for that entry, formatted by #format_log. The formatter must
148
+ # format each entry as exactly one event object, as the default one, the appender itself (see
149
+ # #call), does, since the collector names a rejected event by its position in the request (see
150
+ # #rejected_entry).
151
+ # Every entry is formatted before the request is sent, so a format error sends nothing, and the
152
+ # batch is split up to find the entry at fault (see Http#batch_all_or_nothing?).
153
+ def batch(logs)
154
+ message = logs.map { |log| format_log(log) }.join
155
+ logger.trace(message)
156
+ post(message)
157
+ end
158
+
159
+ # Returns [Integer|nil] the position in the batch of the event that the HTTP Event Collector
160
+ # rejected, from the `invalid-event-number` in the body of its 400 Bad Request, such as
161
+ # {"text":"Invalid data format","code":6,"invalid-event-number":2}, or nil when it holds none
162
+ # (see Subscriber#rejected_entry).
163
+ #
164
+ # The collector reads the events of a batch in order, and indexes those before the one it
165
+ # rejects. Splunk's documentation does not say so: "Troubleshoot HTTP Event Collector" and
166
+ # "Format events for HTTP Event Collector" describe neither `invalid-event-number` nor what
167
+ # happens to the rest of a batch. Users report it, for example
168
+ # https://github.com/moby/moby/issues/35862, where a batch of 10 whose last event is malformed,
169
+ # sent again and again, had "the first 1-9 ... accepted by Splunk over and over again".
170
+ #
171
+ # So splitting such a batch to find the event at fault would index the events before it twice.
172
+ # Instead the events before it are counted as written, it is discarded, and those after it,
173
+ # which the collector never read, are sent again. A 400 without the number, such as a proxy's,
174
+ # is split as for Http, and so is any other status, which the collector does not answer with an
175
+ # event number. The number counts events, so a formatter must format each entry as one event
176
+ # object, as the default one does.
177
+ def rejected_entry(exception)
178
+ return unless exception.is_a?(ResponseError) && exception.code == 400
179
+
180
+ collector_response(exception.body)["invalid-event-number"]
181
+ end
182
+
183
+ private
184
+
185
+ # Returns [Hash] the body of a response from the HTTP Event Collector, such as
186
+ # {"text":"Incorrect index","code":7}, or an empty Hash for a body that is not a JSON object,
187
+ # such as a response from a proxy in front of it.
188
+ def collector_response(body)
189
+ parsed = JSON.parse(body.to_s)
190
+ parsed.is_a?(Hash) ? parsed : {}
191
+ rescue JSON::ParserError
192
+ {}
193
+ end
103
194
  end
104
195
  end
105
196
  end
@@ -60,7 +60,8 @@ module SemanticLogger
60
60
  #
61
61
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
62
62
  # Override the log level for this appender.
63
- # Default: SemanticLogger.default_level
63
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
64
+ # filters by its own level.
64
65
  #
65
66
  # filter: [Regexp|Proc]
66
67
  # RegExp: Only include log messages where the class name matches the supplied.
@@ -171,33 +172,72 @@ module SemanticLogger
171
172
  if %i[tcp udp].include?(@protocol)
172
173
  # Reject an invalid facility when the appender is created, even with a formatter that ignores it.
173
174
  SemanticLogger::Formatters::Syslog.facility_number(@facility)
174
- args[:formatter] = with_packet_options(args[:formatter]) if args.key?(:formatter)
175
175
  end
176
176
 
177
177
  super(**args, &)
178
178
  reopen
179
179
  end
180
180
 
181
+ # Set the formatter from Symbol|Hash|Block, as for any appender.
182
+ #
183
+ # When sending over TCP or UDP, a formatter named as one of the PACKET_FORMATTERS also gets
184
+ # this appender's facility, level map, and maximum packet size, whether it is supplied when
185
+ # the appender is created or set later.
186
+ def formatter=(formatter)
187
+ super(%i[tcp udp].include?(@protocol) ? with_packet_options(formatter) : formatter)
188
+ end
189
+
181
190
  # After forking an active process call #reopen to re-open
182
191
  # open the handles to resources
192
+ #
193
+ # Over TCP, it connects again, and then closes the connection it replaces, such as after a log rotation.
194
+ # When connecting fails, it raises, and keeps the connection it has. Over UDP, it closes the socket it
195
+ # replaces, which in a forked child is only the child's copy.
196
+ #
197
+ # Over TCP, a child process first forgets the client it inherited, without closing its connection, which
198
+ # belongs to the parent. The child closes only its own copy of the socket's file descriptor, which sends
199
+ # the server nothing, so that the server sees the connection end once the parent closes it, rather than
200
+ # once the child exits, or collects the client. When connecting fails in the child, such as while the
201
+ # server cannot be reached, the appender connects again on its next write, rather than writing through
202
+ # the parent's connection. Over TLS, both processes would then carry on the same TLS session, so that the
203
+ # server could not decrypt what the parent writes next.
183
204
  def reopen
184
205
  case @protocol
185
206
  when :syslog
186
207
  method = ::Syslog.opened? ? :reopen : :open
187
208
  ::Syslog.send(method, application, options, facility)
188
209
  when :tcp
189
- @tcp_client_options[:server] = "#{@server}:#{@port}"
190
- @remote_syslog = Net::TCPClient.new(**@tcp_client_options)
191
- # Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
192
- @remote_syslog.logger = logger
210
+ forget_inherited
211
+ replaced = @remote_syslog
212
+ connect_tcp
213
+ close_remote_syslog(replaced, replaced: true)
193
214
  when :udp
215
+ replaced = @remote_syslog
194
216
  @remote_syslog = UDPSocket.new
217
+ close_remote_syslog(replaced, replaced: true)
195
218
  else
196
219
  raise "Unsupported protocol: #{@protocol}"
197
220
  end
198
221
  end
199
222
 
200
- # Write the log using the specified protocol and server.
223
+ # Closes the connection to a remote syslog server, such as when the appender is removed, and forgets it, so
224
+ # that the next write connects again over TCP, or opens a new socket over UDP. The local syslog is left
225
+ # open, since ::Syslog is shared by the whole process. An error closing it is reported to the internal
226
+ # logger, rather than raised.
227
+ #
228
+ # Over TCP, in a forked child that has not reopened, such as one that removes the appender before its
229
+ # reopen has run, it forgets the client it inherited instead, without closing its connection, which
230
+ # belongs to the parent (see #reopen).
231
+ def close
232
+ forget_inherited if @protocol == :tcp
233
+ close_remote_syslog(@remote_syslog)
234
+ ensure
235
+ @remote_syslog = nil
236
+ end
237
+
238
+ # Write the log using the specified protocol and server. Connects over TCP, or opens a socket over UDP,
239
+ # first when the appender has none: after #close, or over TCP when connecting failed as it was reopened
240
+ # after a fork.
201
241
  def log(log)
202
242
  case @protocol
203
243
  when :syslog
@@ -205,9 +245,10 @@ module SemanticLogger
205
245
  message = format_log(log).gsub "%", "%%"
206
246
  ::Syslog.log @level_map[log.level], message
207
247
  when :tcp
208
- @remote_syslog.retry_on_connection_failure { @remote_syslog.write("#{format_log(log)}\r\n") }
248
+ client = @remote_syslog || connect_tcp
249
+ client.retry_on_connection_failure { client.write("#{format_log(log)}\r\n") }
209
250
  when :udp
210
- @remote_syslog.send(format_log(log), 0, @server, @port)
251
+ (@remote_syslog ||= UDPSocket.new).send(format_log(log), 0, @server, @port)
211
252
  else
212
253
  raise "Unsupported protocol: #{protocol}"
213
254
  end
@@ -251,7 +292,45 @@ module SemanticLogger
251
292
 
252
293
  private
253
294
 
254
- # Returns the supplied `formatter:` argument, with this appender's facility, level map, and
295
+ # Returns [Net::TCPClient] a new client, connected to the server, after making it the appender's. Records
296
+ # the process that built it, which owns its connection (see #reopen).
297
+ def connect_tcp
298
+ @remote_syslog = Net::TCPClient.new(**tcp_client_options)
299
+ # Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
300
+ @remote_syslog.logger = logger
301
+ @pid = ::Process.pid
302
+ @remote_syslog
303
+ end
304
+
305
+ # In a child process, forgets the TCP client built in the parent before a fork, and then closes the child's
306
+ # copy of its socket's file descriptor, without a TLS shutdown: SSLSocket#to_io is the TCPSocket under it.
307
+ # See #reopen.
308
+ def forget_inherited
309
+ return if @pid == ::Process.pid
310
+
311
+ inherited = @remote_syslog
312
+ @remote_syslog = nil
313
+ socket = inherited&.socket&.to_io
314
+ socket.close unless socket.nil? || socket.closed?
315
+ end
316
+
317
+ # Closes the given TCP client or UDP socket, if any: the one that a reopen replaced, when replaced: is true.
318
+ # Reports an error to the internal logger as a warning, rather than raising it.
319
+ def close_remote_syslog(remote_syslog, replaced: false)
320
+ remote_syslog&.close
321
+ rescue StandardError => e
322
+ logger.warn("Could not close the connection to the syslog server#{' that a reopen replaced' if replaced}", e)
323
+ end
324
+
325
+ # Returns [Hash] a copy of the options given as tcp_client: (see Subscriber#copy_options), with the server
326
+ # from the url, for a new Net::TCPClient, since it takes handshake_timeout: out of the ssl: Hash it is
327
+ # given: each client gets a copy of its own, which keeps the timeout for the next one, such as after a
328
+ # reopen.
329
+ def tcp_client_options
330
+ copy_options(@tcp_client_options).merge(server: "#{@server}:#{@port}")
331
+ end
332
+
333
+ # Returns the supplied formatter, with this appender's facility, level map, and
255
334
  # maximum packet size added when it names one of the PACKET_FORMATTERS, so that they are not
256
335
  # silently ignored. Options in a
257
336
  # `{syslog: {...}}` Hash take precedence. A formatter instance keeps its own settings.
@@ -47,98 +47,19 @@ module SemanticLogger
47
47
  # Create TCP log appender.
48
48
  #
49
49
  # Net::TCPClient Parameters:
50
+ # Every keyword argument other than the appender parameters below goes to Net::TCPClient.new,
51
+ # which raises ArgumentError for a keyword it does not take. See Net::TCPClient#initialize for
52
+ # them and their defaults, such as connect_timeout:, write_timeout:, connect_retry_count:,
53
+ # retry_count:, policy:, and ssl:
54
+ # https://github.com/reidmorrison/net_tcp_client/blob/master/lib/net/tcp_client/tcp_client.rb
55
+ #
50
56
  # :server [String]
51
- # URL of the server to connect to with port number
52
- # 'localhost:2000'
53
- # '192.168.1.10:80'
57
+ # The server to connect to, as "host:port", such as 'localhost:2000' or '192.168.1.10:80'.
58
+ # Net::TCPClient connects over IPv4 only, so the host must be an IPv4 address or a host name
59
+ # that has one.
54
60
  #
55
61
  # :servers [Array of String]
56
- # Array of URL's of servers to connect to with port numbers
57
- # ['server1:2000', 'server2:2000']
58
- #
59
- # The second server will only be attempted once the first server
60
- # cannot be connected to or has timed out on connect
61
- # A read failure or timeout will not result in switching to the second
62
- # server, only a connection failure or during an automatic reconnect
63
- #
64
- # :connect_timeout [Float]
65
- # Time in seconds to timeout when trying to connect to the server
66
- # A value of -1 will cause the connect wait time to be infinite
67
- # Default: Half of the :read_timeout ( 30 seconds )
68
- #
69
- # :read_timeout [Float]
70
- # Time in seconds to timeout on read
71
- # Can be overridden by supplying a timeout in the read call
72
- # Default: 60
73
- #
74
- # :write_timeout [Float]
75
- # Time in seconds to timeout on write
76
- # Can be overridden by supplying a timeout in the write call
77
- # Default: 60
78
- #
79
- # :log_level [Symbol]
80
- # Optional: Set the logging level for the TCPClient
81
- # Any valid SemanticLogger log level:
82
- # :trace, :debug, :info, :warn, :error, :fatal
83
- # Default: SemanticLogger.default_level
84
- #
85
- # :buffered [Boolean]
86
- # Whether to use Nagle's Buffering algorithm (http://en.wikipedia.org/wiki/Nagle's_algorithm)
87
- # Recommend disabling for RPC style invocations where we don't want to wait for an
88
- # ACK from the server before sending the last partial segment
89
- # Buffering is recommended in a browser or file transfer style environment
90
- # where multiple sends are expected during a single response
91
- # Default: true
92
- #
93
- # :connect_retry_count [Integer]
94
- # Number of times to retry connecting when a connection fails
95
- # Default: 10
96
- #
97
- # :connect_retry_interval [Float]
98
- # Number of seconds between connection retry attempts after the first failed attempt
99
- # Default: 0.5
100
- #
101
- # :retry_count [Integer]
102
- # Number of times to retry when calling #retry_on_connection_failure
103
- # This is independent of :connect_retry_count which still applies with
104
- # connection failures. This retry controls upto how many times to retry the
105
- # supplied block should a connection failure occurr during the block
106
- # Default: 3
107
- #
108
- # :on_connect [Proc]
109
- # Directly after a connection is established and before it is made available
110
- # for use this Block is invoked.
111
- # Typical Use Cases:
112
- # - Initialize per connection session sequence numbers
113
- # - Pass any authentication information to the server
114
- # - Perform a handshake with the server
115
- #
116
- # :policy [Symbol|Proc]
117
- # Specify the policy to use when connecting to servers.
118
- # :ordered
119
- # Select a server in the order supplied in the array, with the first
120
- # having the highest priority. The second server will only be connected
121
- # to if the first server is unreachable
122
- # :random
123
- # Randomly select a server from the list every time a connection
124
- # is established, including during automatic connection recovery.
125
- # :ping_time
126
- # FUTURE - Not implemented yet - Pull request anyone?
127
- # The server with the lowest ping time will be tried first
128
- # Proc:
129
- # When a Proc is supplied, it will be called passing in the list
130
- # of servers. The Proc must return one server name
131
- # Example:
132
- # :policy => Proc.new do |servers|
133
- # servers.last
134
- # end
135
- # Default: :ordered
136
- #
137
- # :close_on_error [True|False]
138
- # To prevent the connection from going into an inconsistent state
139
- # automatically close the connection if an error occurs
140
- # This includes a Read Timeout
141
- # Default: true
62
+ # Instead of :server, several servers, each as "host:port".
142
63
  #
143
64
  # Appender Parameters:
144
65
  # separator: [String]
@@ -157,18 +78,24 @@ module SemanticLogger
157
78
  #
158
79
  # level: [:trace | :debug | :info | :warn | :error | :fatal]
159
80
  # Override the log level for this appender.
160
- # Default: SemanticLogger.default_level
81
+ # Default: none (:trace), so it writes every entry that a logger emits; each logger
82
+ # filters by its own level.
161
83
  #
162
- # formatter: [Object|Proc]
84
+ # formatter: [Object|Proc|Symbol|Hash]
163
85
  # An instance of a class that implements #call, or a Proc to be used to format
164
86
  # the output from this appender
165
- # Default: Use the built-in formatter (See: #call)
87
+ # Default: :json
166
88
  #
167
89
  # filter: [Regexp|Proc]
168
90
  # RegExp: Only include log messages where the class name matches the supplied.
169
91
  # regular expression. All other messages will be ignored.
170
92
  # Proc: Only include log messages where the supplied Proc returns true
171
93
  # The Proc must return true or false.
94
+ #
95
+ # metrics: [Boolean]
96
+ # Also send metric-only entries: a metric without a message or an exception.
97
+ # Default: false
98
+ #
172
99
  # Example:
173
100
  # SemanticLogger.add_appender(
174
101
  # appender: :tcp,
@@ -197,17 +124,28 @@ module SemanticLogger
197
124
  reopen
198
125
  end
199
126
 
200
- # After forking an active process call #reopen to re-open the handles to resources.
127
+ # Closes the connection, and connects to the server again, such as after a log rotation, or a fork.
128
+ #
129
+ # After a fork, the child first forgets the client it inherited, without closing its connection, which
130
+ # belongs to the parent: closing a TLS connection sends the server a close_notify, which ends it for the
131
+ # parent too. The child closes only its own copy of the socket's file descriptor, which sends the server
132
+ # nothing, so that the server sees the connection end once the parent closes it, rather than once the
133
+ # child exits, or collects the client. When connecting fails in the child, the appender has no client
134
+ # until the next write connects again (see #log), rather than writing through the parent's connection.
201
135
  def reopen
136
+ forget_inherited
202
137
  close
203
- @tcp_client = Net::TCPClient.new(**@tcp_client_args)
138
+ connect
204
139
  end
205
140
 
206
- # Write the log using the specified protocol and server.
141
+ # Write the log using the specified protocol and server. Connects first when the appender has no
142
+ # client, since connecting failed when it was reopened after a fork, or it was closed in a forked
143
+ # child.
207
144
  def log(log)
208
- message = format_log(log)
209
- @tcp_client.retry_on_connection_failure do
210
- @tcp_client.write("#{message}#{separator}")
145
+ message = format_log(log)
146
+ tcp_client = @tcp_client || connect
147
+ tcp_client.retry_on_connection_failure do
148
+ tcp_client.write("#{message}#{separator}")
211
149
  end
212
150
  true
213
151
  end
@@ -218,7 +156,12 @@ module SemanticLogger
218
156
  end
219
157
 
220
158
  # Close is called during shutdown, or with reopen
159
+ #
160
+ # In a forked child that has not reopened, such as one that removes the appender before its reopen has
161
+ # run, it forgets the client it inherited instead, without closing its connection, which belongs to the
162
+ # parent, and the next write connects again (see #reopen).
221
163
  def close
164
+ forget_inherited
222
165
  @tcp_client&.close
223
166
  end
224
167
 
@@ -229,6 +172,33 @@ module SemanticLogger
229
172
 
230
173
  private
231
174
 
175
+ # Returns [Net::TCPClient] a new client, connected to the server, after making it the appender's.
176
+ # Records the process that built it, which owns its connection (see #reopen).
177
+ def connect
178
+ @tcp_client = Net::TCPClient.new(**tcp_client_options)
179
+ @pid = ::Process.pid
180
+ @tcp_client
181
+ end
182
+
183
+ # In a child process, forgets the client built in the parent before a fork, and then closes the child's
184
+ # copy of its socket's file descriptor, without a TLS shutdown: SSLSocket#to_io is the TCPSocket under
185
+ # it. See #reopen.
186
+ def forget_inherited
187
+ return if @pid == ::Process.pid
188
+
189
+ inherited = @tcp_client
190
+ @tcp_client = nil
191
+ socket = inherited&.socket&.to_io
192
+ socket.close unless socket.nil? || socket.closed?
193
+ end
194
+
195
+ # Returns [Hash] a copy of the options for a new Net::TCPClient (see Subscriber#copy_options), since it
196
+ # takes handshake_timeout: out of the ssl: Hash it is given: each client gets a copy of its own, which
197
+ # keeps the timeout for the next one, such as after a reopen.
198
+ def tcp_client_options
199
+ copy_options(@tcp_client_args)
200
+ end
201
+
232
202
  # Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
233
203
  def default_formatter
234
204
  SemanticLogger::Formatters::Json.new
@@ -0,0 +1,65 @@
1
+ module SemanticLogger
2
+ module Appender
3
+ # Writes the formatted text of log entries to an IO, such as a log file or standard output, for
4
+ # the appenders that write to one (Appender::File and Appender::IO).
5
+ #
6
+ # An IO with an external encoding converts what it writes to that encoding, and raises an
7
+ # EncodingError for text that it cannot convert, which would lose the entry. A file opened with an
8
+ # `encoding:` such as UTF-16LE refuses invalid UTF-8, binary text that holds bytes outside ASCII,
9
+ # which a custom formatter can return, and a character that its encoding does not have. An IO
10
+ # that converts to UTF-8, such as one opened after Rails set `Encoding.default_internal`, refuses
11
+ # only such binary text: it writes text tagged UTF-8 that holds invalid bytes as it is.
12
+ module TextOutput
13
+ # Writes the text to the IO, or, when the IO cannot convert it to its encoding, the text
14
+ # converted first (see .writable_text). Only text that the IO refuses is converted.
15
+ #
16
+ # Only an IO that converts the whole text before it writes any of it is written to again (see
17
+ # .converts_whole_text?). Another object that responds to #write, such as one that writes to
18
+ # two IOs, could have written part of the text before it raised, which would then be written
19
+ # twice, so its error is raised as before.
20
+ #
21
+ # Returns what IO#write returns: the number of bytes written, in the IO's encoding.
22
+ def self.write(io, text)
23
+ io.write(text)
24
+ rescue EncodingError
25
+ raise unless converts_whole_text?(io)
26
+
27
+ io.write(writable_text(text, io))
28
+ end
29
+
30
+ # Returns [String] the text, converted to valid UTF-8 as the JSON formatters convert it (see
31
+ # Utils.encode_utf8_string), and then, for an IO with an external encoding other than UTF-8,
32
+ # converted to that encoding, with "?" (or U+FFFD for a Unicode encoding) in place of each
33
+ # character that it does not have.
34
+ def self.writable_text(text, io)
35
+ text = Utils.encode_utf8_string(text)
36
+ encoding = io.external_encoding if io.respond_to?(:external_encoding)
37
+ return text if encoding.nil? || encoding == Encoding::UTF_8 || encoding == Encoding::BINARY
38
+
39
+ text.encode(encoding, invalid: :replace, undef: :replace)
40
+ end
41
+ private_class_method :writable_text
42
+
43
+ # Whether the IO converts the whole text before it writes any of it: an ::IO, a Tempfile, which
44
+ # writes with File#write, or a StringIO.
45
+ def self.converts_whole_text?(io)
46
+ io.is_a?(::IO) || (defined?(::Tempfile) && io.is_a?(::Tempfile)) ||
47
+ (defined?(::StringIO) && io.is_a?(::StringIO))
48
+ end
49
+ private_class_method :converts_whole_text?
50
+
51
+ # Returns [String] the formatted text of a log entry as the line that .write writes: a new
52
+ # String of it followed by a line break. A new one, since the formatter's String can be one it
53
+ # does not own, such as a custom formatter's log.message, the application's own, or frozen. It
54
+ # keeps the text's encoding. An object that converts to a String implicitly (#to_str) is read
55
+ # as that String, and anything else, such as nil, a Hash, or a Pathname, raises TypeError.
56
+ # Interpolation would write any object, and tag US-ASCII text UTF-8.
57
+ def self.line(text)
58
+ string = String.try_convert(text)
59
+ raise(TypeError, "Formatter returned a #{text.class}, not a String") unless string
60
+
61
+ string + "\n" # rubocop:disable Style/StringConcatenation
62
+ end
63
+ end
64
+ end
65
+ end