semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -12,9 +12,38 @@ require "json"
|
|
|
12
12
|
# url: 'http://localhost:8080',
|
|
13
13
|
# token: '70CA900C-3D7E-42A4-9C79-7975D1C422A8'
|
|
14
14
|
# )
|
|
15
|
+
#
|
|
16
|
+
# Batching:
|
|
17
|
+
# Add it with `batch: true` to send many entries in each request, as event objects stacked one after
|
|
18
|
+
# the other, each one what the appender sends for that entry on its own (see #batch).
|
|
19
|
+
#
|
|
20
|
+
# Responses:
|
|
21
|
+
# The HTTP Event Collector answers 400 Bad Request both for the entries a request holds, and for its
|
|
22
|
+
# own configuration, such as an index it does not know. Its JSON body says which, with a numeric
|
|
23
|
+
# `code` and a `text`. A batch it rejects for its configuration is discarded at once, rather than
|
|
24
|
+
# split up to find the entries at fault, since every part would be rejected too (see
|
|
25
|
+
# #fails_any_batch?). For a batch it rejects at one of its events, it says which one, having indexed
|
|
26
|
+
# those before it (see #rejected_entry).
|
|
15
27
|
module SemanticLogger
|
|
16
28
|
module Appender
|
|
17
29
|
class SplunkHttp < SemanticLogger::Appender::Http
|
|
30
|
+
# The status codes in the body of a 400 Bad Request from the HTTP Event Collector that reject
|
|
31
|
+
# the request for the collector's configuration or the appender's, whatever entries it holds:
|
|
32
|
+
# 7 Incorrect index
|
|
33
|
+
# 10 Data channel is missing
|
|
34
|
+
# 11 Invalid data channel
|
|
35
|
+
# 14 ACK is disabled
|
|
36
|
+
# 16 Query string authorization is not enabled
|
|
37
|
+
# 21 Invalid token
|
|
38
|
+
# 22 Token disabled
|
|
39
|
+
# The others it answers with a 400 are caused by the entries, so a batch is split up to find
|
|
40
|
+
# them: 5 No data, 6 Invalid data format, 12 Event field is required, 13 Event field cannot be
|
|
41
|
+
# blank, and 15 Error in handling indexed fields.
|
|
42
|
+
#
|
|
43
|
+
# See "Troubleshoot HTTP Event Collector", Possible error codes:
|
|
44
|
+
# https://help.splunk.com/en/splunk-enterprise/get-started/get-data-in/10.4/get-data-with-http-event-collector/troubleshoot-http-event-collector
|
|
45
|
+
CONFIGURATION_CODES = [7, 10, 11, 14, 16, 21, 22].freeze
|
|
46
|
+
|
|
18
47
|
attr_accessor :source_type, :index
|
|
19
48
|
|
|
20
49
|
# Create Splunk appender over persistent HTTP(S)
|
|
@@ -56,7 +85,8 @@ module SemanticLogger
|
|
|
56
85
|
#
|
|
57
86
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
|
58
87
|
# Override the log level for this appender.
|
|
59
|
-
# Default:
|
|
88
|
+
# Default: none (:trace), so it writes every entry that a logger emits; each logger
|
|
89
|
+
# filters by its own level.
|
|
60
90
|
#
|
|
61
91
|
# formatter: [Object|Proc]
|
|
62
92
|
# An instance of a class that implements #call, or a Proc to be used to format
|
|
@@ -100,6 +130,67 @@ module SemanticLogger
|
|
|
100
130
|
message[:index] = index if index
|
|
101
131
|
Utils.to_json(message)
|
|
102
132
|
end
|
|
133
|
+
|
|
134
|
+
# Whether the exception would fail any batch, whatever entries it holds (see
|
|
135
|
+
# Http#fails_any_batch?), including a 400 Bad Request whose body holds one of the collector's
|
|
136
|
+
# CONFIGURATION_CODES, such as for a wrong `index:`. So the batch is discarded at once, and
|
|
137
|
+
# reported once, with the collector's text, rather than split up and each entry sent on its own.
|
|
138
|
+
def fails_any_batch?(exception)
|
|
139
|
+
super || (exception.is_a?(ResponseError) && exception.rejected? &&
|
|
140
|
+
CONFIGURATION_CODES.include?(collector_response(exception.body)["code"]))
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Send a batch of log messages to the HTTP Event Collector in a single request, as event objects
|
|
144
|
+
# stacked one after the other, the batch format of "Format events for HTTP Event Collector":
|
|
145
|
+
# https://help.splunk.com/en/splunk-enterprise/get-started/get-data-in/10.6/get-data-with-http-event-collector/format-events-for-http-event-collector
|
|
146
|
+
#
|
|
147
|
+
# Each event is what #log sends for that entry, formatted by #format_log. The formatter must
|
|
148
|
+
# format each entry as exactly one event object, as the default one, the appender itself (see
|
|
149
|
+
# #call), does, since the collector names a rejected event by its position in the request (see
|
|
150
|
+
# #rejected_entry).
|
|
151
|
+
# Every entry is formatted before the request is sent, so a format error sends nothing, and the
|
|
152
|
+
# batch is split up to find the entry at fault (see Http#batch_all_or_nothing?).
|
|
153
|
+
def batch(logs)
|
|
154
|
+
message = logs.map { |log| format_log(log) }.join
|
|
155
|
+
logger.trace(message)
|
|
156
|
+
post(message)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Returns [Integer|nil] the position in the batch of the event that the HTTP Event Collector
|
|
160
|
+
# rejected, from the `invalid-event-number` in the body of its 400 Bad Request, such as
|
|
161
|
+
# {"text":"Invalid data format","code":6,"invalid-event-number":2}, or nil when it holds none
|
|
162
|
+
# (see Subscriber#rejected_entry).
|
|
163
|
+
#
|
|
164
|
+
# The collector reads the events of a batch in order, and indexes those before the one it
|
|
165
|
+
# rejects. Splunk's documentation does not say so: "Troubleshoot HTTP Event Collector" and
|
|
166
|
+
# "Format events for HTTP Event Collector" describe neither `invalid-event-number` nor what
|
|
167
|
+
# happens to the rest of a batch. Users report it, for example
|
|
168
|
+
# https://github.com/moby/moby/issues/35862, where a batch of 10 whose last event is malformed,
|
|
169
|
+
# sent again and again, had "the first 1-9 ... accepted by Splunk over and over again".
|
|
170
|
+
#
|
|
171
|
+
# So splitting such a batch to find the event at fault would index the events before it twice.
|
|
172
|
+
# Instead the events before it are counted as written, it is discarded, and those after it,
|
|
173
|
+
# which the collector never read, are sent again. A 400 without the number, such as a proxy's,
|
|
174
|
+
# is split as for Http, and so is any other status, which the collector does not answer with an
|
|
175
|
+
# event number. The number counts events, so a formatter must format each entry as one event
|
|
176
|
+
# object, as the default one does.
|
|
177
|
+
def rejected_entry(exception)
|
|
178
|
+
return unless exception.is_a?(ResponseError) && exception.code == 400
|
|
179
|
+
|
|
180
|
+
collector_response(exception.body)["invalid-event-number"]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
private
|
|
184
|
+
|
|
185
|
+
# Returns [Hash] the body of a response from the HTTP Event Collector, such as
|
|
186
|
+
# {"text":"Incorrect index","code":7}, or an empty Hash for a body that is not a JSON object,
|
|
187
|
+
# such as a response from a proxy in front of it.
|
|
188
|
+
def collector_response(body)
|
|
189
|
+
parsed = JSON.parse(body.to_s)
|
|
190
|
+
parsed.is_a?(Hash) ? parsed : {}
|
|
191
|
+
rescue JSON::ParserError
|
|
192
|
+
{}
|
|
193
|
+
end
|
|
103
194
|
end
|
|
104
195
|
end
|
|
105
196
|
end
|
|
@@ -60,7 +60,8 @@ module SemanticLogger
|
|
|
60
60
|
#
|
|
61
61
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
|
62
62
|
# Override the log level for this appender.
|
|
63
|
-
# Default:
|
|
63
|
+
# Default: none (:trace), so it writes every entry that a logger emits; each logger
|
|
64
|
+
# filters by its own level.
|
|
64
65
|
#
|
|
65
66
|
# filter: [Regexp|Proc]
|
|
66
67
|
# RegExp: Only include log messages where the class name matches the supplied.
|
|
@@ -171,33 +172,72 @@ module SemanticLogger
|
|
|
171
172
|
if %i[tcp udp].include?(@protocol)
|
|
172
173
|
# Reject an invalid facility when the appender is created, even with a formatter that ignores it.
|
|
173
174
|
SemanticLogger::Formatters::Syslog.facility_number(@facility)
|
|
174
|
-
args[:formatter] = with_packet_options(args[:formatter]) if args.key?(:formatter)
|
|
175
175
|
end
|
|
176
176
|
|
|
177
177
|
super(**args, &)
|
|
178
178
|
reopen
|
|
179
179
|
end
|
|
180
180
|
|
|
181
|
+
# Set the formatter from Symbol|Hash|Block, as for any appender.
|
|
182
|
+
#
|
|
183
|
+
# When sending over TCP or UDP, a formatter named as one of the PACKET_FORMATTERS also gets
|
|
184
|
+
# this appender's facility, level map, and maximum packet size, whether it is supplied when
|
|
185
|
+
# the appender is created or set later.
|
|
186
|
+
def formatter=(formatter)
|
|
187
|
+
super(%i[tcp udp].include?(@protocol) ? with_packet_options(formatter) : formatter)
|
|
188
|
+
end
|
|
189
|
+
|
|
181
190
|
# After forking an active process call #reopen to re-open
|
|
182
191
|
# open the handles to resources
|
|
192
|
+
#
|
|
193
|
+
# Over TCP, it connects again, and then closes the connection it replaces, such as after a log rotation.
|
|
194
|
+
# When connecting fails, it raises, and keeps the connection it has. Over UDP, it closes the socket it
|
|
195
|
+
# replaces, which in a forked child is only the child's copy.
|
|
196
|
+
#
|
|
197
|
+
# Over TCP, a child process first forgets the client it inherited, without closing its connection, which
|
|
198
|
+
# belongs to the parent. The child closes only its own copy of the socket's file descriptor, which sends
|
|
199
|
+
# the server nothing, so that the server sees the connection end once the parent closes it, rather than
|
|
200
|
+
# once the child exits, or collects the client. When connecting fails in the child, such as while the
|
|
201
|
+
# server cannot be reached, the appender connects again on its next write, rather than writing through
|
|
202
|
+
# the parent's connection. Over TLS, both processes would then carry on the same TLS session, so that the
|
|
203
|
+
# server could not decrypt what the parent writes next.
|
|
183
204
|
def reopen
|
|
184
205
|
case @protocol
|
|
185
206
|
when :syslog
|
|
186
207
|
method = ::Syslog.opened? ? :reopen : :open
|
|
187
208
|
::Syslog.send(method, application, options, facility)
|
|
188
209
|
when :tcp
|
|
189
|
-
|
|
190
|
-
@remote_syslog
|
|
191
|
-
|
|
192
|
-
|
|
210
|
+
forget_inherited
|
|
211
|
+
replaced = @remote_syslog
|
|
212
|
+
connect_tcp
|
|
213
|
+
close_remote_syslog(replaced, replaced: true)
|
|
193
214
|
when :udp
|
|
215
|
+
replaced = @remote_syslog
|
|
194
216
|
@remote_syslog = UDPSocket.new
|
|
217
|
+
close_remote_syslog(replaced, replaced: true)
|
|
195
218
|
else
|
|
196
219
|
raise "Unsupported protocol: #{@protocol}"
|
|
197
220
|
end
|
|
198
221
|
end
|
|
199
222
|
|
|
200
|
-
#
|
|
223
|
+
# Closes the connection to a remote syslog server, such as when the appender is removed, and forgets it, so
|
|
224
|
+
# that the next write connects again over TCP, or opens a new socket over UDP. The local syslog is left
|
|
225
|
+
# open, since ::Syslog is shared by the whole process. An error closing it is reported to the internal
|
|
226
|
+
# logger, rather than raised.
|
|
227
|
+
#
|
|
228
|
+
# Over TCP, in a forked child that has not reopened, such as one that removes the appender before its
|
|
229
|
+
# reopen has run, it forgets the client it inherited instead, without closing its connection, which
|
|
230
|
+
# belongs to the parent (see #reopen).
|
|
231
|
+
def close
|
|
232
|
+
forget_inherited if @protocol == :tcp
|
|
233
|
+
close_remote_syslog(@remote_syslog)
|
|
234
|
+
ensure
|
|
235
|
+
@remote_syslog = nil
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Write the log using the specified protocol and server. Connects over TCP, or opens a socket over UDP,
|
|
239
|
+
# first when the appender has none: after #close, or over TCP when connecting failed as it was reopened
|
|
240
|
+
# after a fork.
|
|
201
241
|
def log(log)
|
|
202
242
|
case @protocol
|
|
203
243
|
when :syslog
|
|
@@ -205,9 +245,10 @@ module SemanticLogger
|
|
|
205
245
|
message = format_log(log).gsub "%", "%%"
|
|
206
246
|
::Syslog.log @level_map[log.level], message
|
|
207
247
|
when :tcp
|
|
208
|
-
|
|
248
|
+
client = @remote_syslog || connect_tcp
|
|
249
|
+
client.retry_on_connection_failure { client.write("#{format_log(log)}\r\n") }
|
|
209
250
|
when :udp
|
|
210
|
-
@remote_syslog.send(format_log(log), 0, @server, @port)
|
|
251
|
+
(@remote_syslog ||= UDPSocket.new).send(format_log(log), 0, @server, @port)
|
|
211
252
|
else
|
|
212
253
|
raise "Unsupported protocol: #{protocol}"
|
|
213
254
|
end
|
|
@@ -251,7 +292,45 @@ module SemanticLogger
|
|
|
251
292
|
|
|
252
293
|
private
|
|
253
294
|
|
|
254
|
-
# Returns
|
|
295
|
+
# Returns [Net::TCPClient] a new client, connected to the server, after making it the appender's. Records
|
|
296
|
+
# the process that built it, which owns its connection (see #reopen).
|
|
297
|
+
def connect_tcp
|
|
298
|
+
@remote_syslog = Net::TCPClient.new(**tcp_client_options)
|
|
299
|
+
# Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
|
|
300
|
+
@remote_syslog.logger = logger
|
|
301
|
+
@pid = ::Process.pid
|
|
302
|
+
@remote_syslog
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# In a child process, forgets the TCP client built in the parent before a fork, and then closes the child's
|
|
306
|
+
# copy of its socket's file descriptor, without a TLS shutdown: SSLSocket#to_io is the TCPSocket under it.
|
|
307
|
+
# See #reopen.
|
|
308
|
+
def forget_inherited
|
|
309
|
+
return if @pid == ::Process.pid
|
|
310
|
+
|
|
311
|
+
inherited = @remote_syslog
|
|
312
|
+
@remote_syslog = nil
|
|
313
|
+
socket = inherited&.socket&.to_io
|
|
314
|
+
socket.close unless socket.nil? || socket.closed?
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Closes the given TCP client or UDP socket, if any: the one that a reopen replaced, when replaced: is true.
|
|
318
|
+
# Reports an error to the internal logger as a warning, rather than raising it.
|
|
319
|
+
def close_remote_syslog(remote_syslog, replaced: false)
|
|
320
|
+
remote_syslog&.close
|
|
321
|
+
rescue StandardError => e
|
|
322
|
+
logger.warn("Could not close the connection to the syslog server#{' that a reopen replaced' if replaced}", e)
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Returns [Hash] a copy of the options given as tcp_client: (see Subscriber#copy_options), with the server
|
|
326
|
+
# from the url, for a new Net::TCPClient, since it takes handshake_timeout: out of the ssl: Hash it is
|
|
327
|
+
# given: each client gets a copy of its own, which keeps the timeout for the next one, such as after a
|
|
328
|
+
# reopen.
|
|
329
|
+
def tcp_client_options
|
|
330
|
+
copy_options(@tcp_client_options).merge(server: "#{@server}:#{@port}")
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# Returns the supplied formatter, with this appender's facility, level map, and
|
|
255
334
|
# maximum packet size added when it names one of the PACKET_FORMATTERS, so that they are not
|
|
256
335
|
# silently ignored. Options in a
|
|
257
336
|
# `{syslog: {...}}` Hash take precedence. A formatter instance keeps its own settings.
|
|
@@ -47,98 +47,19 @@ module SemanticLogger
|
|
|
47
47
|
# Create TCP log appender.
|
|
48
48
|
#
|
|
49
49
|
# Net::TCPClient Parameters:
|
|
50
|
+
# Every keyword argument other than the appender parameters below goes to Net::TCPClient.new,
|
|
51
|
+
# which raises ArgumentError for a keyword it does not take. See Net::TCPClient#initialize for
|
|
52
|
+
# them and their defaults, such as connect_timeout:, write_timeout:, connect_retry_count:,
|
|
53
|
+
# retry_count:, policy:, and ssl:
|
|
54
|
+
# https://github.com/reidmorrison/net_tcp_client/blob/master/lib/net/tcp_client/tcp_client.rb
|
|
55
|
+
#
|
|
50
56
|
# :server [String]
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
57
|
+
# The server to connect to, as "host:port", such as 'localhost:2000' or '192.168.1.10:80'.
|
|
58
|
+
# Net::TCPClient connects over IPv4 only, so the host must be an IPv4 address or a host name
|
|
59
|
+
# that has one.
|
|
54
60
|
#
|
|
55
61
|
# :servers [Array of String]
|
|
56
|
-
#
|
|
57
|
-
# ['server1:2000', 'server2:2000']
|
|
58
|
-
#
|
|
59
|
-
# The second server will only be attempted once the first server
|
|
60
|
-
# cannot be connected to or has timed out on connect
|
|
61
|
-
# A read failure or timeout will not result in switching to the second
|
|
62
|
-
# server, only a connection failure or during an automatic reconnect
|
|
63
|
-
#
|
|
64
|
-
# :connect_timeout [Float]
|
|
65
|
-
# Time in seconds to timeout when trying to connect to the server
|
|
66
|
-
# A value of -1 will cause the connect wait time to be infinite
|
|
67
|
-
# Default: Half of the :read_timeout ( 30 seconds )
|
|
68
|
-
#
|
|
69
|
-
# :read_timeout [Float]
|
|
70
|
-
# Time in seconds to timeout on read
|
|
71
|
-
# Can be overridden by supplying a timeout in the read call
|
|
72
|
-
# Default: 60
|
|
73
|
-
#
|
|
74
|
-
# :write_timeout [Float]
|
|
75
|
-
# Time in seconds to timeout on write
|
|
76
|
-
# Can be overridden by supplying a timeout in the write call
|
|
77
|
-
# Default: 60
|
|
78
|
-
#
|
|
79
|
-
# :log_level [Symbol]
|
|
80
|
-
# Optional: Set the logging level for the TCPClient
|
|
81
|
-
# Any valid SemanticLogger log level:
|
|
82
|
-
# :trace, :debug, :info, :warn, :error, :fatal
|
|
83
|
-
# Default: SemanticLogger.default_level
|
|
84
|
-
#
|
|
85
|
-
# :buffered [Boolean]
|
|
86
|
-
# Whether to use Nagle's Buffering algorithm (http://en.wikipedia.org/wiki/Nagle's_algorithm)
|
|
87
|
-
# Recommend disabling for RPC style invocations where we don't want to wait for an
|
|
88
|
-
# ACK from the server before sending the last partial segment
|
|
89
|
-
# Buffering is recommended in a browser or file transfer style environment
|
|
90
|
-
# where multiple sends are expected during a single response
|
|
91
|
-
# Default: true
|
|
92
|
-
#
|
|
93
|
-
# :connect_retry_count [Integer]
|
|
94
|
-
# Number of times to retry connecting when a connection fails
|
|
95
|
-
# Default: 10
|
|
96
|
-
#
|
|
97
|
-
# :connect_retry_interval [Float]
|
|
98
|
-
# Number of seconds between connection retry attempts after the first failed attempt
|
|
99
|
-
# Default: 0.5
|
|
100
|
-
#
|
|
101
|
-
# :retry_count [Integer]
|
|
102
|
-
# Number of times to retry when calling #retry_on_connection_failure
|
|
103
|
-
# This is independent of :connect_retry_count which still applies with
|
|
104
|
-
# connection failures. This retry controls upto how many times to retry the
|
|
105
|
-
# supplied block should a connection failure occurr during the block
|
|
106
|
-
# Default: 3
|
|
107
|
-
#
|
|
108
|
-
# :on_connect [Proc]
|
|
109
|
-
# Directly after a connection is established and before it is made available
|
|
110
|
-
# for use this Block is invoked.
|
|
111
|
-
# Typical Use Cases:
|
|
112
|
-
# - Initialize per connection session sequence numbers
|
|
113
|
-
# - Pass any authentication information to the server
|
|
114
|
-
# - Perform a handshake with the server
|
|
115
|
-
#
|
|
116
|
-
# :policy [Symbol|Proc]
|
|
117
|
-
# Specify the policy to use when connecting to servers.
|
|
118
|
-
# :ordered
|
|
119
|
-
# Select a server in the order supplied in the array, with the first
|
|
120
|
-
# having the highest priority. The second server will only be connected
|
|
121
|
-
# to if the first server is unreachable
|
|
122
|
-
# :random
|
|
123
|
-
# Randomly select a server from the list every time a connection
|
|
124
|
-
# is established, including during automatic connection recovery.
|
|
125
|
-
# :ping_time
|
|
126
|
-
# FUTURE - Not implemented yet - Pull request anyone?
|
|
127
|
-
# The server with the lowest ping time will be tried first
|
|
128
|
-
# Proc:
|
|
129
|
-
# When a Proc is supplied, it will be called passing in the list
|
|
130
|
-
# of servers. The Proc must return one server name
|
|
131
|
-
# Example:
|
|
132
|
-
# :policy => Proc.new do |servers|
|
|
133
|
-
# servers.last
|
|
134
|
-
# end
|
|
135
|
-
# Default: :ordered
|
|
136
|
-
#
|
|
137
|
-
# :close_on_error [True|False]
|
|
138
|
-
# To prevent the connection from going into an inconsistent state
|
|
139
|
-
# automatically close the connection if an error occurs
|
|
140
|
-
# This includes a Read Timeout
|
|
141
|
-
# Default: true
|
|
62
|
+
# Instead of :server, several servers, each as "host:port".
|
|
142
63
|
#
|
|
143
64
|
# Appender Parameters:
|
|
144
65
|
# separator: [String]
|
|
@@ -157,18 +78,24 @@ module SemanticLogger
|
|
|
157
78
|
#
|
|
158
79
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
|
159
80
|
# Override the log level for this appender.
|
|
160
|
-
# Default:
|
|
81
|
+
# Default: none (:trace), so it writes every entry that a logger emits; each logger
|
|
82
|
+
# filters by its own level.
|
|
161
83
|
#
|
|
162
|
-
# formatter: [Object|Proc]
|
|
84
|
+
# formatter: [Object|Proc|Symbol|Hash]
|
|
163
85
|
# An instance of a class that implements #call, or a Proc to be used to format
|
|
164
86
|
# the output from this appender
|
|
165
|
-
# Default:
|
|
87
|
+
# Default: :json
|
|
166
88
|
#
|
|
167
89
|
# filter: [Regexp|Proc]
|
|
168
90
|
# RegExp: Only include log messages where the class name matches the supplied.
|
|
169
91
|
# regular expression. All other messages will be ignored.
|
|
170
92
|
# Proc: Only include log messages where the supplied Proc returns true
|
|
171
93
|
# The Proc must return true or false.
|
|
94
|
+
#
|
|
95
|
+
# metrics: [Boolean]
|
|
96
|
+
# Also send metric-only entries: a metric without a message or an exception.
|
|
97
|
+
# Default: false
|
|
98
|
+
#
|
|
172
99
|
# Example:
|
|
173
100
|
# SemanticLogger.add_appender(
|
|
174
101
|
# appender: :tcp,
|
|
@@ -197,17 +124,28 @@ module SemanticLogger
|
|
|
197
124
|
reopen
|
|
198
125
|
end
|
|
199
126
|
|
|
200
|
-
#
|
|
127
|
+
# Closes the connection, and connects to the server again, such as after a log rotation, or a fork.
|
|
128
|
+
#
|
|
129
|
+
# After a fork, the child first forgets the client it inherited, without closing its connection, which
|
|
130
|
+
# belongs to the parent: closing a TLS connection sends the server a close_notify, which ends it for the
|
|
131
|
+
# parent too. The child closes only its own copy of the socket's file descriptor, which sends the server
|
|
132
|
+
# nothing, so that the server sees the connection end once the parent closes it, rather than once the
|
|
133
|
+
# child exits, or collects the client. When connecting fails in the child, the appender has no client
|
|
134
|
+
# until the next write connects again (see #log), rather than writing through the parent's connection.
|
|
201
135
|
def reopen
|
|
136
|
+
forget_inherited
|
|
202
137
|
close
|
|
203
|
-
|
|
138
|
+
connect
|
|
204
139
|
end
|
|
205
140
|
|
|
206
|
-
# Write the log using the specified protocol and server.
|
|
141
|
+
# Write the log using the specified protocol and server. Connects first when the appender has no
|
|
142
|
+
# client, since connecting failed when it was reopened after a fork, or it was closed in a forked
|
|
143
|
+
# child.
|
|
207
144
|
def log(log)
|
|
208
|
-
message
|
|
209
|
-
@tcp_client
|
|
210
|
-
|
|
145
|
+
message = format_log(log)
|
|
146
|
+
tcp_client = @tcp_client || connect
|
|
147
|
+
tcp_client.retry_on_connection_failure do
|
|
148
|
+
tcp_client.write("#{message}#{separator}")
|
|
211
149
|
end
|
|
212
150
|
true
|
|
213
151
|
end
|
|
@@ -218,7 +156,12 @@ module SemanticLogger
|
|
|
218
156
|
end
|
|
219
157
|
|
|
220
158
|
# Close is called during shutdown, or with reopen
|
|
159
|
+
#
|
|
160
|
+
# In a forked child that has not reopened, such as one that removes the appender before its reopen has
|
|
161
|
+
# run, it forgets the client it inherited instead, without closing its connection, which belongs to the
|
|
162
|
+
# parent, and the next write connects again (see #reopen).
|
|
221
163
|
def close
|
|
164
|
+
forget_inherited
|
|
222
165
|
@tcp_client&.close
|
|
223
166
|
end
|
|
224
167
|
|
|
@@ -229,6 +172,33 @@ module SemanticLogger
|
|
|
229
172
|
|
|
230
173
|
private
|
|
231
174
|
|
|
175
|
+
# Returns [Net::TCPClient] a new client, connected to the server, after making it the appender's.
|
|
176
|
+
# Records the process that built it, which owns its connection (see #reopen).
|
|
177
|
+
def connect
|
|
178
|
+
@tcp_client = Net::TCPClient.new(**tcp_client_options)
|
|
179
|
+
@pid = ::Process.pid
|
|
180
|
+
@tcp_client
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# In a child process, forgets the client built in the parent before a fork, and then closes the child's
|
|
184
|
+
# copy of its socket's file descriptor, without a TLS shutdown: SSLSocket#to_io is the TCPSocket under
|
|
185
|
+
# it. See #reopen.
|
|
186
|
+
def forget_inherited
|
|
187
|
+
return if @pid == ::Process.pid
|
|
188
|
+
|
|
189
|
+
inherited = @tcp_client
|
|
190
|
+
@tcp_client = nil
|
|
191
|
+
socket = inherited&.socket&.to_io
|
|
192
|
+
socket.close unless socket.nil? || socket.closed?
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Returns [Hash] a copy of the options for a new Net::TCPClient (see Subscriber#copy_options), since it
|
|
196
|
+
# takes handshake_timeout: out of the ssl: Hash it is given: each client gets a copy of its own, which
|
|
197
|
+
# keeps the timeout for the next one, such as after a reopen.
|
|
198
|
+
def tcp_client_options
|
|
199
|
+
copy_options(@tcp_client_args)
|
|
200
|
+
end
|
|
201
|
+
|
|
232
202
|
# Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
|
|
233
203
|
def default_formatter
|
|
234
204
|
SemanticLogger::Formatters::Json.new
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
module Appender
|
|
3
|
+
# Writes the formatted text of log entries to an IO, such as a log file or standard output, for
|
|
4
|
+
# the appenders that write to one (Appender::File and Appender::IO).
|
|
5
|
+
#
|
|
6
|
+
# An IO with an external encoding converts what it writes to that encoding, and raises an
|
|
7
|
+
# EncodingError for text that it cannot convert, which would lose the entry. A file opened with an
|
|
8
|
+
# `encoding:` such as UTF-16LE refuses invalid UTF-8, binary text that holds bytes outside ASCII,
|
|
9
|
+
# which a custom formatter can return, and a character that its encoding does not have. An IO
|
|
10
|
+
# that converts to UTF-8, such as one opened after Rails set `Encoding.default_internal`, refuses
|
|
11
|
+
# only such binary text: it writes text tagged UTF-8 that holds invalid bytes as it is.
|
|
12
|
+
module TextOutput
|
|
13
|
+
# Writes the text to the IO, or, when the IO cannot convert it to its encoding, the text
|
|
14
|
+
# converted first (see .writable_text). Only text that the IO refuses is converted.
|
|
15
|
+
#
|
|
16
|
+
# Only an IO that converts the whole text before it writes any of it is written to again (see
|
|
17
|
+
# .converts_whole_text?). Another object that responds to #write, such as one that writes to
|
|
18
|
+
# two IOs, could have written part of the text before it raised, which would then be written
|
|
19
|
+
# twice, so its error is raised as before.
|
|
20
|
+
#
|
|
21
|
+
# Returns what IO#write returns: the number of bytes written, in the IO's encoding.
|
|
22
|
+
def self.write(io, text)
|
|
23
|
+
io.write(text)
|
|
24
|
+
rescue EncodingError
|
|
25
|
+
raise unless converts_whole_text?(io)
|
|
26
|
+
|
|
27
|
+
io.write(writable_text(text, io))
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Returns [String] the text, converted to valid UTF-8 as the JSON formatters convert it (see
|
|
31
|
+
# Utils.encode_utf8_string), and then, for an IO with an external encoding other than UTF-8,
|
|
32
|
+
# converted to that encoding, with "?" (or U+FFFD for a Unicode encoding) in place of each
|
|
33
|
+
# character that it does not have.
|
|
34
|
+
def self.writable_text(text, io)
|
|
35
|
+
text = Utils.encode_utf8_string(text)
|
|
36
|
+
encoding = io.external_encoding if io.respond_to?(:external_encoding)
|
|
37
|
+
return text if encoding.nil? || encoding == Encoding::UTF_8 || encoding == Encoding::BINARY
|
|
38
|
+
|
|
39
|
+
text.encode(encoding, invalid: :replace, undef: :replace)
|
|
40
|
+
end
|
|
41
|
+
private_class_method :writable_text
|
|
42
|
+
|
|
43
|
+
# Whether the IO converts the whole text before it writes any of it: an ::IO, a Tempfile, which
|
|
44
|
+
# writes with File#write, or a StringIO.
|
|
45
|
+
def self.converts_whole_text?(io)
|
|
46
|
+
io.is_a?(::IO) || (defined?(::Tempfile) && io.is_a?(::Tempfile)) ||
|
|
47
|
+
(defined?(::StringIO) && io.is_a?(::StringIO))
|
|
48
|
+
end
|
|
49
|
+
private_class_method :converts_whole_text?
|
|
50
|
+
|
|
51
|
+
# Returns [String] the formatted text of a log entry as the line that .write writes: a new
|
|
52
|
+
# String of it followed by a line break. A new one, since the formatter's String can be one it
|
|
53
|
+
# does not own, such as a custom formatter's log.message, the application's own, or frozen. It
|
|
54
|
+
# keeps the text's encoding. An object that converts to a String implicitly (#to_str) is read
|
|
55
|
+
# as that String, and anything else, such as nil, a Hash, or a Pathname, raises TypeError.
|
|
56
|
+
# Interpolation would write any object, and tag US-ASCII text UTF-8.
|
|
57
|
+
def self.line(text)
|
|
58
|
+
string = String.try_convert(text)
|
|
59
|
+
raise(TypeError, "Formatter returned a #{text.class}, not a String") unless string
|
|
60
|
+
|
|
61
|
+
string + "\n" # rubocop:disable Style/StringConcatenation
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|