semantic_logger 5.0.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +20 -10
- data/Rakefile +40 -0
- data/docs/api.md +506 -0
- data/docs/appenders.md +1074 -0
- data/docs/config.md +707 -0
- data/docs/index.md +223 -0
- data/docs/log.md +116 -0
- data/docs/metrics.md +171 -0
- data/docs/operations.md +446 -0
- data/docs/rails.md +890 -0
- data/docs/security.md +286 -0
- data/docs/testing.md +287 -0
- data/docs/upgrading.md +697 -0
- data/lib/semantic_logger/appender/async.rb +18 -5
- data/lib/semantic_logger/appender/bugsnag.rb +1 -1
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +126 -29
- data/lib/semantic_logger/appender/elasticsearch.rb +2 -5
- data/lib/semantic_logger/appender/elasticsearch_base.rb +48 -23
- data/lib/semantic_logger/appender/elasticsearch_http.rb +15 -8
- data/lib/semantic_logger/appender/file.rb +141 -28
- data/lib/semantic_logger/appender/graylog.rb +22 -6
- data/lib/semantic_logger/appender/honeybadger.rb +1 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +1 -1
- data/lib/semantic_logger/appender/http.rb +70 -11
- data/lib/semantic_logger/appender/io.rb +1 -1
- data/lib/semantic_logger/appender/kafka.rb +26 -12
- data/lib/semantic_logger/appender/loki.rb +13 -3
- data/lib/semantic_logger/appender/mongodb.rb +20 -1
- data/lib/semantic_logger/appender/new_relic.rb +7 -1
- data/lib/semantic_logger/appender/new_relic_logs.rb +11 -12
- data/lib/semantic_logger/appender/open_telemetry.rb +4 -3
- data/lib/semantic_logger/appender/opensearch.rb +2 -1
- data/lib/semantic_logger/appender/rabbitmq.rb +10 -4
- data/lib/semantic_logger/appender/sentry.rb +6 -1
- data/lib/semantic_logger/appender/sentry_ruby.rb +91 -17
- data/lib/semantic_logger/appender/splunk.rb +10 -1
- data/lib/semantic_logger/appender/syslog.rb +66 -25
- data/lib/semantic_logger/appender/tcp.rb +10 -4
- data/lib/semantic_logger/appender/udp.rb +11 -1
- data/lib/semantic_logger/appender/wrapper.rb +3 -2
- data/lib/semantic_logger/appender.rb +6 -3
- data/lib/semantic_logger/appenders.rb +11 -2
- data/lib/semantic_logger/base.rb +82 -32
- data/lib/semantic_logger/exception_details.rb +50 -0
- data/lib/semantic_logger/formatters/base.rb +52 -6
- data/lib/semantic_logger/formatters/color.rb +22 -3
- data/lib/semantic_logger/formatters/default.rb +28 -6
- data/lib/semantic_logger/formatters/logfmt.rb +155 -21
- data/lib/semantic_logger/formatters/loki.rb +26 -22
- data/lib/semantic_logger/formatters/one_line.rb +67 -3
- data/lib/semantic_logger/formatters/open_telemetry.rb +16 -17
- data/lib/semantic_logger/formatters/pattern.rb +11 -5
- data/lib/semantic_logger/formatters/raw.rb +4 -5
- data/lib/semantic_logger/formatters/syslog.rb +110 -17
- data/lib/semantic_logger/formatters/syslog_cee.rb +29 -15
- data/lib/semantic_logger/levels.rb +10 -1
- data/lib/semantic_logger/log.rb +183 -37
- data/lib/semantic_logger/loggable.rb +26 -3
- data/lib/semantic_logger/metric/new_relic.rb +1 -1
- data/lib/semantic_logger/metric/signalfx.rb +12 -8
- data/lib/semantic_logger/metric/statsd.rb +15 -7
- data/lib/semantic_logger/queue_processor/request.rb +55 -0
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +46 -0
- data/lib/semantic_logger/queue_processor.rb +170 -68
- data/lib/semantic_logger/reporters/minitest.rb +4 -4
- data/lib/semantic_logger/semantic_logger.rb +15 -2
- data/lib/semantic_logger/subscriber.rb +96 -10
- data/lib/semantic_logger/sync_processor.rb +11 -0
- data/lib/semantic_logger/test/capture_log_events.rb +9 -6
- data/lib/semantic_logger/utils.rb +244 -16
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +1 -1
- metadata +33 -6
- data/lib/semantic_logger/core_ext/thread.rb +0 -30
|
@@ -10,6 +10,15 @@ module SemanticLogger
|
|
|
10
10
|
:reopen_period, :reopen_count, :reopen_size, :permissions
|
|
11
11
|
attr_reader :log_count, :log_size, :current_file_name, :reopen_at
|
|
12
12
|
|
|
13
|
+
# With permissions, the log file is opened without following a symbolic link, so that a link
|
|
14
|
+
# planted in the log directory can neither create nor open the file it points at, without
|
|
15
|
+
# blocking, so that a planted FIFO fails to open instead of waiting for a reader, and without
|
|
16
|
+
# becoming the controlling terminal.
|
|
17
|
+
SECURE_OPEN_FLAGS = ::File::NOFOLLOW | ::File::NONBLOCK | ::File::NOCTTY
|
|
18
|
+
# Seconds to wait before opening a refused log file again.
|
|
19
|
+
REFUSED_RETRY_INTERVAL = 5
|
|
20
|
+
private_constant :SECURE_OPEN_FLAGS, :REFUSED_RETRY_INTERVAL
|
|
21
|
+
|
|
13
22
|
# Create an appender to log to a named file.
|
|
14
23
|
#
|
|
15
24
|
# Parameters
|
|
@@ -77,7 +86,12 @@ module SemanticLogger
|
|
|
77
86
|
# Octal file permissions to apply to the log file.
|
|
78
87
|
# Log files frequently contain sensitive information, so restrict access
|
|
79
88
|
# by supplying for example `0o640` (owner read/write, group read).
|
|
80
|
-
# Applied both when the file is created and to an existing log file
|
|
89
|
+
# Applied both when the file is created and to an existing log file, which must be
|
|
90
|
+
# a regular file owned by the process, with a single hard link, that the file_name
|
|
91
|
+
# names directly rather than through a symbolic link. Nothing is written to any
|
|
92
|
+
# other file, or when the file cannot be opened: the appender raises IOError when it
|
|
93
|
+
# is created, and when the file is later reopened, it reports why and writes nothing
|
|
94
|
+
# until the file is fixed, trying again every few seconds.
|
|
81
95
|
# Default: nil (use the process umask, the standard Ruby behavior)
|
|
82
96
|
#
|
|
83
97
|
# :retry_count [Integer]
|
|
@@ -150,43 +164,37 @@ module SemanticLogger
|
|
|
150
164
|
@reopen_at = nil
|
|
151
165
|
|
|
152
166
|
super(**args, &)
|
|
167
|
+
|
|
168
|
+
# Open the file at creation time so that misconfiguration (bad directory,
|
|
169
|
+
# insufficient permissions, ...) raises immediately in SemanticLogger.add_appender,
|
|
170
|
+
# instead of surfacing later on the appender thread via the internal logger. With
|
|
171
|
+
# permissions, a refused log file raises IOError (see #open_with_permissions).
|
|
172
|
+
raise(IOError, refusal_message, cause: @refusal_cause) unless reopen
|
|
153
173
|
end
|
|
154
174
|
|
|
155
175
|
# After forking an active process call #reopen to re-open
|
|
156
176
|
# open the file handles etc to resources.
|
|
157
177
|
def reopen
|
|
158
|
-
|
|
159
|
-
@file&.close
|
|
160
|
-
rescue StandardError
|
|
161
|
-
nil
|
|
162
|
-
end
|
|
178
|
+
close_quietly(@file)
|
|
163
179
|
|
|
164
180
|
self.current_file_name = apply_format_directives(file_name)
|
|
165
|
-
|
|
181
|
+
# With permissions, a directory is refused when it fails to open, like any other file.
|
|
182
|
+
if !permissions && ::File.directory?(current_file_name)
|
|
166
183
|
raise(ArgumentError, "The supplied log file_name: #{current_file_name} is already a directory.")
|
|
167
184
|
end
|
|
168
185
|
|
|
169
186
|
self.log_count = 0
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
else
|
|
174
|
-
self.log_size = 0
|
|
175
|
-
end
|
|
187
|
+
# File.size? rather than File.size, since the file can be removed at any time.
|
|
188
|
+
size = ::File.size?(current_file_name) if append && reopen_size
|
|
189
|
+
self.log_size = size && size < reopen_size ? size : 0
|
|
176
190
|
|
|
177
191
|
self.reopen_at = reopen_period ? next_reopen_period(reopen_period) : nil
|
|
178
192
|
|
|
179
193
|
options = ::File::WRONLY | ::File::CREAT
|
|
180
194
|
options |= ::File::APPEND if append
|
|
181
|
-
@file =
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
else
|
|
185
|
-
::File.open(current_file_name, options)
|
|
186
|
-
end
|
|
187
|
-
# File.open only applies the permissions when creating the file, so also
|
|
188
|
-
# enforce them on an already existing log file.
|
|
189
|
-
@file.chmod(permissions) if permissions
|
|
195
|
+
@file = permissions ? open_with_permissions(options) : ::File.open(current_file_name, options)
|
|
196
|
+
return unless @file
|
|
197
|
+
|
|
190
198
|
# Force all log entries to write immediately without buffering
|
|
191
199
|
# Allows multiple processes to write to the same log file simultaneously
|
|
192
200
|
@file.sync = true
|
|
@@ -198,18 +206,23 @@ module SemanticLogger
|
|
|
198
206
|
# Since only one appender thread will be writing to the file at a time
|
|
199
207
|
# it is not necessary to protect access to the file with a semaphore.
|
|
200
208
|
def log(log)
|
|
201
|
-
reopen
|
|
209
|
+
# Read once, since #reopen can be called from another thread meanwhile.
|
|
210
|
+
file = time_to_reopen? ? reopen : @file
|
|
211
|
+
return refused unless file
|
|
202
212
|
|
|
203
|
-
|
|
213
|
+
# Formatted once, outside the retry below: reopening the file cannot fix the content.
|
|
214
|
+
message = format_log(log) << "\n"
|
|
215
|
+
count = 0
|
|
204
216
|
begin
|
|
205
|
-
|
|
206
|
-
@file.write(message)
|
|
217
|
+
file.write(message)
|
|
207
218
|
@log_count += 1
|
|
208
219
|
@log_size += message.size
|
|
209
220
|
rescue StandardError => e
|
|
210
221
|
if count < retry_count
|
|
211
222
|
count += 1
|
|
212
|
-
reopen
|
|
223
|
+
file = reopen
|
|
224
|
+
return refused unless file
|
|
225
|
+
|
|
213
226
|
retry
|
|
214
227
|
end
|
|
215
228
|
raise(e)
|
|
@@ -223,18 +236,118 @@ module SemanticLogger
|
|
|
223
236
|
@file&.flush
|
|
224
237
|
end
|
|
225
238
|
|
|
239
|
+
# See Subscriber#inspect.
|
|
240
|
+
def inspect_details
|
|
241
|
+
"file_name=#{file_name.inspect}"
|
|
242
|
+
end
|
|
243
|
+
|
|
226
244
|
private
|
|
227
245
|
|
|
228
246
|
attr_writer :log_count, :log_size, :current_file_name, :reopen_at
|
|
229
247
|
|
|
248
|
+
# Opens the log file and applies the permissions to it, when it is safe to do so (see
|
|
249
|
+
# #permissions_refusal). Returns the open file, or nil after closing it and recording why it is
|
|
250
|
+
# refused, so that nothing is written to it.
|
|
251
|
+
#
|
|
252
|
+
# Changing the permissions of a file that another name links to could loosen them on an
|
|
253
|
+
# unrelated file, such as a private key linked into a shared log directory. Writing log messages
|
|
254
|
+
# to a file that this process does not own, or that another name links to, would let whoever
|
|
255
|
+
# controls it read them, despite the permissions asked for. So with permissions, the appender
|
|
256
|
+
# fails closed: it raises when it is created (see #initialize), and when the file is later
|
|
257
|
+
# reopened, on a rotation, a fork, or a write retry, it stops writing and reports why (see
|
|
258
|
+
# #refused) until the file is fixed (see #refused_retry_due?).
|
|
259
|
+
def open_with_permissions(options)
|
|
260
|
+
file = ::File.open(current_file_name, options | SECURE_OPEN_FLAGS, permissions)
|
|
261
|
+
reason = permissions_refusal(file)
|
|
262
|
+
if reason
|
|
263
|
+
close_quietly(file)
|
|
264
|
+
return refuse(reason)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
file.chmod(permissions)
|
|
268
|
+
@refusal = @refusal_cause = @reported_refusal = nil
|
|
269
|
+
file
|
|
270
|
+
rescue SystemCallError => e
|
|
271
|
+
close_quietly(file)
|
|
272
|
+
refuse(open_refusal(e), e)
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Records why the log file is refused, and returns nil.
|
|
276
|
+
def refuse(reason, cause = nil)
|
|
277
|
+
@refusal = reason
|
|
278
|
+
@refusal_cause = cause
|
|
279
|
+
@refused_retry_at = monotonic_time + REFUSED_RETRY_INTERVAL
|
|
280
|
+
nil
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Reports why log messages are not being written to the log file, once for each file and
|
|
284
|
+
# reason, and returns false.
|
|
285
|
+
def refused
|
|
286
|
+
refusal = [current_file_name, @refusal]
|
|
287
|
+
unless @reported_refusal == refusal
|
|
288
|
+
@reported_refusal = refusal
|
|
289
|
+
logger.error(refusal_message)
|
|
290
|
+
end
|
|
291
|
+
false
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def refusal_message
|
|
295
|
+
"Not writing to the log file #{current_file_name}: #{@refusal}"
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# Returns why the log file could not be opened with permissions.
|
|
299
|
+
def open_refusal(error)
|
|
300
|
+
return "it is a symbolic link" if error.is_a?(Errno::ELOOP) && ::File.symlink?(current_file_name)
|
|
301
|
+
|
|
302
|
+
"it could not be opened (#{error.message})"
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# Returns why the permissions must not be applied to the open log file, nor log messages
|
|
306
|
+
# written to it, or nil when they can.
|
|
307
|
+
#
|
|
308
|
+
# It must be a regular file owned by this process, with a single hard link, since a hard link
|
|
309
|
+
# planted in the log directory is otherwise indistinguishable from the file it links to. The
|
|
310
|
+
# path must also still name it, so that swapping the path after opening cannot redirect the
|
|
311
|
+
# log messages.
|
|
312
|
+
def permissions_refusal(file)
|
|
313
|
+
stat = file.stat
|
|
314
|
+
path_stat = ::File.lstat(current_file_name)
|
|
315
|
+
return "it is not a regular file" unless stat.file?
|
|
316
|
+
return "it is not owned by this process" unless stat.uid == ::Process.euid
|
|
317
|
+
return "it has more than one hard link" unless stat.nlink == 1
|
|
318
|
+
return "it changed after it was opened" unless path_stat.dev == stat.dev && path_stat.ino == stat.ino
|
|
319
|
+
|
|
320
|
+
nil
|
|
321
|
+
rescue SystemCallError => e
|
|
322
|
+
"it could not be checked (#{e.message})"
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def close_quietly(file)
|
|
326
|
+
file&.close
|
|
327
|
+
rescue StandardError
|
|
328
|
+
nil
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
def monotonic_time
|
|
332
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
333
|
+
end
|
|
334
|
+
|
|
230
335
|
def time_to_reopen?
|
|
231
|
-
return
|
|
336
|
+
return refused_retry_due? unless @file
|
|
232
337
|
|
|
233
338
|
(reopen_count.positive? && (log_count >= reopen_count)) ||
|
|
234
339
|
(reopen_size.positive? && (log_size >= reopen_size)) ||
|
|
235
340
|
(reopen_at && (Time.now > reopen_at))
|
|
236
341
|
end
|
|
237
342
|
|
|
343
|
+
# A refused log file is opened again every few seconds, and at the next scheduled reopen,
|
|
344
|
+
# rather than for every message.
|
|
345
|
+
def refused_retry_due?
|
|
346
|
+
return true unless @refused_retry_at
|
|
347
|
+
|
|
348
|
+
monotonic_time >= @refused_retry_at || (reopen_at && Time.now > reopen_at)
|
|
349
|
+
end
|
|
350
|
+
|
|
238
351
|
def apply_format_directives(file_name)
|
|
239
352
|
return file_name unless file_name.include?("%")
|
|
240
353
|
|
|
@@ -100,7 +100,7 @@ module SemanticLogger
|
|
|
100
100
|
|
|
101
101
|
# Re-open after process fork
|
|
102
102
|
def reopen
|
|
103
|
-
uri =
|
|
103
|
+
uri = Utils.parse_url(@url)
|
|
104
104
|
@server = uri.host
|
|
105
105
|
@port = uri.port
|
|
106
106
|
@protocol = uri.scheme.to_sym
|
|
@@ -120,10 +120,10 @@ module SemanticLogger
|
|
|
120
120
|
def call(log, logger)
|
|
121
121
|
h = default_formatter.call(log, logger)
|
|
122
122
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
123
|
+
# GELF rejects an empty short_message, such as a message of only ANSI escape codes.
|
|
124
|
+
h[:short_message] = h.delete(:message).to_s
|
|
125
|
+
h[:short_message] = log.exception.message.to_s.strip if h[:short_message].empty? && log.exception
|
|
126
|
+
h[:short_message] = "<no-exception-message>" if h[:short_message].empty?
|
|
127
127
|
h[:level] = logger.level_map[log.level]
|
|
128
128
|
h[:level_str] = log.level.to_s
|
|
129
129
|
h[:duration_str] = h.delete(:duration)
|
|
@@ -131,11 +131,27 @@ module SemanticLogger
|
|
|
131
131
|
end
|
|
132
132
|
|
|
133
133
|
# Forward log messages
|
|
134
|
+
#
|
|
135
|
+
# GELF serializes the Hash with #to_json, which raises for Infinity, NaN, invalid UTF-8, or a
|
|
136
|
+
# structure nested too deeply, and writes a Struct through its #inspect, so such a Hash is
|
|
137
|
+
# handed over as a copy that JSON can represent (see Utils.json_ready).
|
|
134
138
|
def log(log)
|
|
135
|
-
notifier.notify!(
|
|
139
|
+
notifier.notify!(format_log(log) { |hash| Utils.json_ready(hash) })
|
|
136
140
|
true
|
|
137
141
|
end
|
|
138
142
|
|
|
143
|
+
# GELF raises ArgumentError for a log message it cannot send, such as one too large to fit in
|
|
144
|
+
# the maximum number of chunks, which retrying cannot send either. Unavailable servers raise
|
|
145
|
+
# network errors instead.
|
|
146
|
+
def message_error?(exception)
|
|
147
|
+
exception.is_a?(ArgumentError) || super
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# The url without any credentials. See Subscriber#inspect.
|
|
151
|
+
def inspect_details
|
|
152
|
+
"url=#{Utils.redacted_url(url).inspect}"
|
|
153
|
+
end
|
|
154
|
+
|
|
139
155
|
private
|
|
140
156
|
|
|
141
157
|
def default_formatter
|
|
@@ -31,7 +31,13 @@ module SemanticLogger
|
|
|
31
31
|
class Http < SemanticLogger::Subscriber
|
|
32
32
|
attr_accessor :username, :compress, :header,
|
|
33
33
|
:open_timeout, :read_timeout, :continue_timeout
|
|
34
|
-
attr_reader :http, :url, :server, :port, :path, :ssl_options, :proxy_url
|
|
34
|
+
attr_reader :http, :url, :server, :port, :path, :query, :ssl_options, :proxy_url
|
|
35
|
+
|
|
36
|
+
# Maximum number of bytes of an error response body to write to the internal logger.
|
|
37
|
+
MAX_LOGGED_RESPONSE_BODY = 512
|
|
38
|
+
|
|
39
|
+
# Describes a valid url in the error raised for an invalid one.
|
|
40
|
+
URL_FORMAT_HINT = "Should be similar to: 'http://hostname:port/path'".freeze
|
|
35
41
|
|
|
36
42
|
# Create HTTP(S) log appender
|
|
37
43
|
#
|
|
@@ -42,6 +48,9 @@ module SemanticLogger
|
|
|
42
48
|
# To enable SSL include https in the URL.
|
|
43
49
|
# Example: https://example.com/some_path
|
|
44
50
|
# verify_mode will default: OpenSSL::SSL::VERIFY_PEER
|
|
51
|
+
# A query string is kept and sent with every request that submits log data,
|
|
52
|
+
# for the log servers that are configured through it.
|
|
53
|
+
# Example: http://example.com/some_path?source=my_app
|
|
45
54
|
#
|
|
46
55
|
# application: [String]
|
|
47
56
|
# Name of this application to appear in log messages.
|
|
@@ -136,11 +145,10 @@ module SemanticLogger
|
|
|
136
145
|
}.merge(header)
|
|
137
146
|
@header["Content-Encoding"] = "gzip" if @compress
|
|
138
147
|
|
|
139
|
-
uri =
|
|
148
|
+
uri = parse_url(@url)
|
|
140
149
|
@server = uri.host
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
end
|
|
150
|
+
# The url is left out of the message, since it can hold a user and password.
|
|
151
|
+
raise(ArgumentError, "Invalid format for :url, it has no host. #{URL_FORMAT_HINT}") unless @server
|
|
144
152
|
|
|
145
153
|
@port = uri.port
|
|
146
154
|
@username = uri.user if !@username && uri.user
|
|
@@ -148,6 +156,11 @@ module SemanticLogger
|
|
|
148
156
|
@path = uri.path
|
|
149
157
|
# Path cannot be empty
|
|
150
158
|
@path = "/" if @path == ""
|
|
159
|
+
# Query string of the url, applied to every request, see #uri_with_query.
|
|
160
|
+
# Kept separate from the path, because subclasses build their own request paths
|
|
161
|
+
# from it. For example Appender::ElasticsearchHttp appends the index name.
|
|
162
|
+
@query = uri.query
|
|
163
|
+
@query = nil if @query == ""
|
|
151
164
|
|
|
152
165
|
if uri.scheme == "https"
|
|
153
166
|
@ssl_options[:use_ssl] = true
|
|
@@ -157,7 +170,7 @@ module SemanticLogger
|
|
|
157
170
|
@port ||= HTTP.http_default_port
|
|
158
171
|
end
|
|
159
172
|
|
|
160
|
-
@proxy_uri =
|
|
173
|
+
@proxy_uri = Utils.parse_url(@proxy_url, option: ":proxy_url") if @proxy_url && @proxy_url != :ENV
|
|
161
174
|
|
|
162
175
|
@http = nil
|
|
163
176
|
|
|
@@ -196,7 +209,7 @@ module SemanticLogger
|
|
|
196
209
|
|
|
197
210
|
# Forward log messages to HTTP Server
|
|
198
211
|
def log(log)
|
|
199
|
-
message =
|
|
212
|
+
message = format_log(log)
|
|
200
213
|
logger.trace(message)
|
|
201
214
|
post(message)
|
|
202
215
|
end
|
|
@@ -204,7 +217,7 @@ module SemanticLogger
|
|
|
204
217
|
# Forward a batch of log messages to the HTTP Server in a single request.
|
|
205
218
|
# Only used when the appender is created with `batch: true`.
|
|
206
219
|
def batch(logs)
|
|
207
|
-
message =
|
|
220
|
+
message = format_batch(logs)
|
|
208
221
|
logger.trace(message)
|
|
209
222
|
post(message)
|
|
210
223
|
end
|
|
@@ -216,8 +229,30 @@ module SemanticLogger
|
|
|
216
229
|
false
|
|
217
230
|
end
|
|
218
231
|
|
|
232
|
+
# The whole batch is formatted into a single request before anything is sent.
|
|
233
|
+
def batch_all_or_nothing?
|
|
234
|
+
true
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# Returns [String] the scheme, host, and any non-default port of the url. Safe to log.
|
|
238
|
+
# See Utils.redacted_url.
|
|
239
|
+
def redacted_url
|
|
240
|
+
@redacted_url ||= Utils.redacted_url(url)
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Only the redacted url, since the instance variables hold the password, the authorization
|
|
244
|
+
# headers, and any credential a subclass adds. See Subscriber#inspect.
|
|
245
|
+
def inspect_details
|
|
246
|
+
"url=#{redacted_url.inspect}"
|
|
247
|
+
end
|
|
248
|
+
|
|
219
249
|
private
|
|
220
250
|
|
|
251
|
+
# Returns [URI] the parsed url, without the url in the error raised for an invalid one.
|
|
252
|
+
def parse_url(url)
|
|
253
|
+
Utils.parse_url(url, hint: URL_FORMAT_HINT)
|
|
254
|
+
end
|
|
255
|
+
|
|
221
256
|
# Use JSON Formatter by default
|
|
222
257
|
def default_formatter
|
|
223
258
|
SemanticLogger::Formatters::Json.new
|
|
@@ -233,22 +268,43 @@ module SemanticLogger
|
|
|
233
268
|
|
|
234
269
|
# HTTP Post
|
|
235
270
|
def post(body, request_uri = path)
|
|
236
|
-
request = Net::HTTP::Post.new(request_uri, @header)
|
|
271
|
+
request = Net::HTTP::Post.new(uri_with_query(request_uri), @header)
|
|
237
272
|
process_request(request, body)
|
|
238
273
|
end
|
|
239
274
|
|
|
240
275
|
# HTTP Put
|
|
241
276
|
def put(body, request_uri = path)
|
|
242
|
-
request = Net::HTTP::Put.new(request_uri, @header)
|
|
277
|
+
request = Net::HTTP::Put.new(uri_with_query(request_uri), @header)
|
|
243
278
|
process_request(request, body)
|
|
244
279
|
end
|
|
245
280
|
|
|
246
281
|
# HTTP Delete
|
|
282
|
+
#
|
|
283
|
+
# The query string of the configured url is _not_ applied here: it configures how
|
|
284
|
+
# the server ingests log data, and servers reject it on a maintenance request.
|
|
285
|
+
# Appender::ElasticsearchHttp#delete_all deletes an index, and Elasticsearch fails
|
|
286
|
+
# the request with a 400 when it carries unrecognized parameters.
|
|
247
287
|
def delete(request_uri = path)
|
|
248
288
|
request = Net::HTTP::Delete.new(request_uri, @header)
|
|
249
289
|
process_request(request)
|
|
250
290
|
end
|
|
251
291
|
|
|
292
|
+
# Applies the query string of the configured url, if any, to the uri of a request
|
|
293
|
+
# that submits log data.
|
|
294
|
+
#
|
|
295
|
+
# Kept out of `path` on purpose: subclasses build their own request paths from it
|
|
296
|
+
# (Appender::ElasticsearchHttp appends the index name and `_doc` to it), and the
|
|
297
|
+
# query has to stay at the end of the request uri regardless.
|
|
298
|
+
#
|
|
299
|
+
# @param request_uri [String] path of the request, `path` by default.
|
|
300
|
+
# @return [String]
|
|
301
|
+
def uri_with_query(request_uri)
|
|
302
|
+
return request_uri unless @query
|
|
303
|
+
|
|
304
|
+
separator = request_uri.include?("?") ? "&" : "?"
|
|
305
|
+
"#{request_uri}#{separator}#{@query}"
|
|
306
|
+
end
|
|
307
|
+
|
|
252
308
|
# Process HTTP Request
|
|
253
309
|
def process_request(request, body = nil)
|
|
254
310
|
if body
|
|
@@ -260,7 +316,10 @@ module SemanticLogger
|
|
|
260
316
|
true
|
|
261
317
|
else
|
|
262
318
|
# Failures are logged to the global semantic logger failsafe logger (Usually stderr or file)
|
|
263
|
-
logger.error(
|
|
319
|
+
logger.error(
|
|
320
|
+
"Bad HTTP response from: #{redacted_url} code: #{response.code}, " \
|
|
321
|
+
"#{response.body.to_s[0, MAX_LOGGED_RESPONSE_BODY].inspect}"
|
|
322
|
+
)
|
|
264
323
|
false
|
|
265
324
|
end
|
|
266
325
|
rescue RuntimeError => e
|
|
@@ -50,7 +50,7 @@ module SemanticLogger
|
|
|
50
50
|
# Since only one appender thread will be writing to the file at a time
|
|
51
51
|
# it is not necessary to protect access to the file with a semaphore
|
|
52
52
|
# Allow this logger to filter out log levels lower than it's own
|
|
53
|
-
@io.write(
|
|
53
|
+
@io.write(format_log(log) << "\n")
|
|
54
54
|
true
|
|
55
55
|
end
|
|
56
56
|
|
|
@@ -23,6 +23,9 @@ require "date"
|
|
|
23
23
|
module SemanticLogger
|
|
24
24
|
module Appender
|
|
25
25
|
class Kafka < SemanticLogger::Subscriber
|
|
26
|
+
SEED_BROKERS_HINT = "Each should be similar to: 'hostname:port' or 'kafka://hostname:port'".freeze
|
|
27
|
+
private_constant :SEED_BROKERS_HINT
|
|
28
|
+
|
|
26
29
|
attr_accessor :seed_brokers, :client_id, :connect_timeout, :socket_timeout,
|
|
27
30
|
:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :ssl_ca_certs_from_system,
|
|
28
31
|
:delivery_threshold, :delivery_interval, :required_acks,
|
|
@@ -149,17 +152,22 @@ module SemanticLogger
|
|
|
149
152
|
end
|
|
150
153
|
|
|
151
154
|
def reopen
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
155
|
+
# ruby-kafka puts the seed broker, which can hold credentials, in the error it raises for an
|
|
156
|
+
# invalid one: URI::InvalidURIError, or Kafka::Error for an unknown scheme.
|
|
157
|
+
errors = [URI::InvalidURIError, ::Kafka::Error]
|
|
158
|
+
@kafka = Utils.without_url_in_errors(option: ":seed_brokers", hint: SEED_BROKERS_HINT, errors: errors) do
|
|
159
|
+
::Kafka.new(
|
|
160
|
+
seed_brokers: seed_brokers,
|
|
161
|
+
client_id: client_id,
|
|
162
|
+
connect_timeout: connect_timeout,
|
|
163
|
+
socket_timeout: socket_timeout,
|
|
164
|
+
ssl_ca_cert: ssl_ca_cert,
|
|
165
|
+
ssl_client_cert: ssl_client_cert,
|
|
166
|
+
ssl_client_cert_key: ssl_client_cert_key,
|
|
167
|
+
ssl_ca_certs_from_system: ssl_ca_certs_from_system,
|
|
168
|
+
logger: logger
|
|
169
|
+
)
|
|
170
|
+
end
|
|
163
171
|
|
|
164
172
|
@producer = @kafka.async_producer(
|
|
165
173
|
delivery_threshold: delivery_threshold,
|
|
@@ -177,7 +185,7 @@ module SemanticLogger
|
|
|
177
185
|
|
|
178
186
|
# Forward log messages to Kafka producer thread.
|
|
179
187
|
def log(log)
|
|
180
|
-
json =
|
|
188
|
+
json = format_log(log)
|
|
181
189
|
@producer.produce(json, topic: topic, partition: partition, partition_key: partition_key, key: key)
|
|
182
190
|
end
|
|
183
191
|
|
|
@@ -195,6 +203,12 @@ module SemanticLogger
|
|
|
195
203
|
)
|
|
196
204
|
end
|
|
197
205
|
|
|
206
|
+
# Only the topic: the seed brokers can hold credentials, and the options hold the TLS private
|
|
207
|
+
# key and SASL passwords. See Subscriber#inspect.
|
|
208
|
+
def inspect_details
|
|
209
|
+
"topic=#{@topic.inspect}"
|
|
210
|
+
end
|
|
211
|
+
|
|
198
212
|
private
|
|
199
213
|
|
|
200
214
|
attr_reader :producer
|
|
@@ -33,6 +33,8 @@ module SemanticLogger
|
|
|
33
33
|
# url: [String]
|
|
34
34
|
# Define the loki instance URL.
|
|
35
35
|
# Example: https://logs-prod-999.grafana.net
|
|
36
|
+
# A query string is kept and sent with every request.
|
|
37
|
+
# Example: https://logs-prod-999.grafana.net?orgid=my_org
|
|
36
38
|
# Default: nil
|
|
37
39
|
def initialize(url: nil,
|
|
38
40
|
formatter: SemanticLogger::Formatters::Loki.new,
|
|
@@ -40,18 +42,26 @@ module SemanticLogger
|
|
|
40
42
|
path: INGESTION_PATH,
|
|
41
43
|
**args,
|
|
42
44
|
&)
|
|
43
|
-
super(url:
|
|
45
|
+
super(url: self.class.ingestion_url(url, path), formatter: formatter, header: header, **args, &)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Appends the ingestion path to the supplied url, keeping any query string at the
|
|
49
|
+
# end of the url where Appender::Http can parse it out.
|
|
50
|
+
def self.ingestion_url(url, path)
|
|
51
|
+
uri = Utils.parse_url(url.to_s)
|
|
52
|
+
uri.path = "#{uri.path.chomp('/')}/#{path.delete_prefix('/')}"
|
|
53
|
+
uri.to_s
|
|
44
54
|
end
|
|
45
55
|
|
|
46
56
|
def log(log)
|
|
47
|
-
message =
|
|
57
|
+
message = format_log(log)
|
|
48
58
|
logger.trace(message)
|
|
49
59
|
post(message)
|
|
50
60
|
end
|
|
51
61
|
|
|
52
62
|
# Logs in batches
|
|
53
63
|
def batch(logs)
|
|
54
|
-
message =
|
|
64
|
+
message = format_batch(logs)
|
|
55
65
|
logger.trace(message)
|
|
56
66
|
post(message)
|
|
57
67
|
end
|
|
@@ -164,10 +164,29 @@ module SemanticLogger
|
|
|
164
164
|
# Log the message to MongoDB
|
|
165
165
|
def log(log)
|
|
166
166
|
# Insert log entry into Mongo
|
|
167
|
-
collection.insert_one(
|
|
167
|
+
collection.insert_one(format_log(log))
|
|
168
168
|
true
|
|
169
169
|
end
|
|
170
170
|
|
|
171
|
+
# Errors raised for a document that cannot be stored, which retrying cannot insert either.
|
|
172
|
+
# BSON raises BSON::Error, ArgumentError (a key with a NUL byte), or RangeError (an Integer too
|
|
173
|
+
# large for 64 bits) while the document is serialized, before anything is sent, and the driver
|
|
174
|
+
# raises Mongo::Error::MaxBSONSize for a document larger than the server accepts.
|
|
175
|
+
DOCUMENT_ERRORS = [BSON::Error, ArgumentError, RangeError, Mongo::Error::MaxBSONSize].freeze
|
|
176
|
+
|
|
177
|
+
# Except BSON::Error::BSONDecodeError, which comes from reading the server's reply rather than
|
|
178
|
+
# from the document, so it is retried like any other error from the server.
|
|
179
|
+
def message_error?(exception)
|
|
180
|
+
return false if exception.is_a?(BSON::Error::BSONDecodeError)
|
|
181
|
+
|
|
182
|
+
super || DOCUMENT_ERRORS.any? { |error| exception.is_a?(error) }
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Only the collection name: the uri can hold a user and password. See Subscriber#inspect.
|
|
186
|
+
def inspect_details
|
|
187
|
+
"collection=#{@collection_name.inspect}"
|
|
188
|
+
end
|
|
189
|
+
|
|
171
190
|
private
|
|
172
191
|
|
|
173
192
|
def default_formatter
|
|
@@ -33,6 +33,12 @@ module SemanticLogger
|
|
|
33
33
|
# Proc: Only include log messages where the supplied Proc returns true
|
|
34
34
|
# The Proc must return true or false.
|
|
35
35
|
def initialize(level: :error, **args, &block)
|
|
36
|
+
Kernel.warn(
|
|
37
|
+
"SemanticLogger::Appender::NewRelic (appender: :new_relic) is deprecated and will be removed in v6. " \
|
|
38
|
+
"It reports errors via newrelic_rpm and is superseded by the log-forwarding appender " \
|
|
39
|
+
"appender: :new_relic_logs.",
|
|
40
|
+
category: :deprecated
|
|
41
|
+
)
|
|
36
42
|
super
|
|
37
43
|
end
|
|
38
44
|
|
|
@@ -58,7 +64,7 @@ module SemanticLogger
|
|
|
58
64
|
# For more documentation on the NewRelic::Agent.notice_error method see:
|
|
59
65
|
# http://rubydoc.info/github/newrelic/rpm/NewRelic/Agent#notice_error-instance_method
|
|
60
66
|
# and https://docs.newrelic.com/docs/ruby/ruby-agent-api
|
|
61
|
-
::NewRelic::Agent.notice_error(exception,
|
|
67
|
+
::NewRelic::Agent.notice_error(exception, format_log(log))
|
|
62
68
|
true
|
|
63
69
|
end
|
|
64
70
|
end
|