semantic_logger 4.18.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +51 -81
- data/Rakefile +39 -3
- data/docs/api.md +480 -0
- data/docs/appenders.md +990 -0
- data/docs/config.md +605 -0
- data/docs/index.md +197 -0
- data/docs/log.md +108 -0
- data/docs/metrics.md +168 -0
- data/docs/operations.md +400 -0
- data/docs/rails.md +816 -0
- data/docs/security.md +119 -0
- data/docs/testing.md +285 -0
- data/docs/upgrading.md +409 -0
- data/lib/semantic_logger/appender/async.rb +86 -173
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +8 -6
- data/lib/semantic_logger/appender/elasticsearch.rb +8 -187
- data/lib/semantic_logger/appender/elasticsearch_base.rb +212 -0
- data/lib/semantic_logger/appender/elasticsearch_http.rb +16 -9
- data/lib/semantic_logger/appender/file.rb +25 -4
- data/lib/semantic_logger/appender/graylog.rb +2 -2
- data/lib/semantic_logger/appender/http.rb +27 -2
- data/lib/semantic_logger/appender/io.rb +8 -4
- data/lib/semantic_logger/appender/kafka.rb +2 -2
- data/lib/semantic_logger/appender/loki.rb +2 -4
- data/lib/semantic_logger/appender/mongodb.rb +3 -6
- data/lib/semantic_logger/appender/new_relic.rb +6 -0
- data/lib/semantic_logger/appender/new_relic_logs.rb +4 -3
- data/lib/semantic_logger/appender/open_telemetry.rb +10 -7
- data/lib/semantic_logger/appender/opensearch.rb +36 -0
- data/lib/semantic_logger/appender/rabbitmq.rb +5 -4
- data/lib/semantic_logger/appender/sentry.rb +5 -0
- data/lib/semantic_logger/appender/sentry_ruby.rb +2 -2
- data/lib/semantic_logger/appender/splunk.rb +2 -2
- data/lib/semantic_logger/appender/splunk_http.rb +3 -3
- data/lib/semantic_logger/appender/syslog.rb +11 -7
- data/lib/semantic_logger/appender/tcp.rb +4 -3
- data/lib/semantic_logger/appender/udp.rb +2 -2
- data/lib/semantic_logger/appender/wrapper.rb +5 -4
- data/lib/semantic_logger/appender.rb +36 -22
- data/lib/semantic_logger/appenders.rb +27 -5
- data/lib/semantic_logger/base.rb +100 -23
- data/lib/semantic_logger/concerns/compatibility.rb +2 -2
- data/lib/semantic_logger/core_ext/process.rb +34 -0
- data/lib/semantic_logger/formatters/base.rb +46 -7
- data/lib/semantic_logger/formatters/color.rb +9 -3
- data/lib/semantic_logger/formatters/default.rb +11 -4
- data/lib/semantic_logger/formatters/ecs.rb +151 -0
- data/lib/semantic_logger/formatters/fluentd.rb +15 -4
- data/lib/semantic_logger/formatters/json.rb +6 -1
- data/lib/semantic_logger/formatters/logfmt.rb +6 -4
- data/lib/semantic_logger/formatters/loki.rb +4 -4
- data/lib/semantic_logger/formatters/open_telemetry.rb +15 -5
- data/lib/semantic_logger/formatters/pattern.rb +237 -0
- data/lib/semantic_logger/formatters/raw.rb +2 -2
- data/lib/semantic_logger/formatters/signalfx.rb +2 -2
- data/lib/semantic_logger/formatters/syslog.rb +16 -4
- data/lib/semantic_logger/formatters/syslog_cee.rb +3 -2
- data/lib/semantic_logger/formatters.rb +2 -0
- data/lib/semantic_logger/log.rb +18 -4
- data/lib/semantic_logger/logger.rb +2 -2
- data/lib/semantic_logger/metric/new_relic.rb +2 -2
- data/lib/semantic_logger/metric/signalfx.rb +6 -4
- data/lib/semantic_logger/metric/statsd.rb +2 -2
- data/lib/semantic_logger/processor.rb +21 -0
- data/lib/semantic_logger/queue_processor.rb +372 -0
- data/lib/semantic_logger/reporters/minitest.rb +8 -8
- data/lib/semantic_logger/semantic_logger.rb +103 -11
- data/lib/semantic_logger/subscriber.rb +15 -2
- data/lib/semantic_logger/sync_processor.rb +25 -3
- data/lib/semantic_logger/test/capture_log_events.rb +3 -3
- data/lib/semantic_logger/test/minitest.rb +8 -4
- data/lib/semantic_logger/test/rspec.rb +249 -0
- data/lib/semantic_logger/utils.rb +83 -4
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +9 -0
- metadata +28 -8
- data/lib/semantic_logger/appender/async_batch.rb +0 -93
data/lib/semantic_logger/base.rb
CHANGED
|
@@ -155,8 +155,6 @@ module SemanticLogger
|
|
|
155
155
|
log.named_tags = (thread[:semantic_logger_named_tags] || {}).clone
|
|
156
156
|
thread.backtrace
|
|
157
157
|
end
|
|
158
|
-
# TODO: Keep backtrace instead of transforming into a text message at this point
|
|
159
|
-
# Maybe log_backtrace: true
|
|
160
158
|
if backtrace
|
|
161
159
|
message += "\n"
|
|
162
160
|
message << backtrace.join("\n")
|
|
@@ -198,7 +196,18 @@ module SemanticLogger
|
|
|
198
196
|
# to:
|
|
199
197
|
# `logger.tagged('first', 'more', 'other')`
|
|
200
198
|
# - For better performance with clean tags, see `SemanticLogger.tagged`.
|
|
199
|
+
#
|
|
200
|
+
# Child logger:
|
|
201
|
+
# - When called *without* a block, returns a new logger instance that permanently
|
|
202
|
+
# carries the supplied tags ("instance tags"). Those tags are added to every log
|
|
203
|
+
# entry emitted by the returned logger, and _only_ that logger, even across threads.
|
|
204
|
+
# Unlike the block form they are not pushed onto the thread, so other loggers are
|
|
205
|
+
# unaffected. This is the recommended way to bind a logger to an object's identity:
|
|
206
|
+
# logger = SemanticLogger['Cart'].tagged(cart_id: cart.id)
|
|
201
207
|
def tagged(*tags)
|
|
208
|
+
# No block: build a child logger that carries these tags on every entry.
|
|
209
|
+
return tagged_child(tags) unless block_given?
|
|
210
|
+
|
|
202
211
|
block = -> { yield(self) }
|
|
203
212
|
# Allow named tags to be passed into the logger
|
|
204
213
|
# Rails::Rack::Logger passes logs as an array with a single argument
|
|
@@ -226,6 +235,10 @@ module SemanticLogger
|
|
|
226
235
|
SemanticLogger.named_tags
|
|
227
236
|
end
|
|
228
237
|
|
|
238
|
+
# Tags permanently bound to this logger instance via `#tagged` (without a block).
|
|
239
|
+
# Empty for a regular (non-child) logger.
|
|
240
|
+
attr_reader :instance_tags, :instance_named_tags
|
|
241
|
+
|
|
229
242
|
# Returns the list of tags pushed after flattening them out and removing blanks
|
|
230
243
|
#
|
|
231
244
|
# Note:
|
|
@@ -244,13 +257,13 @@ module SemanticLogger
|
|
|
244
257
|
end
|
|
245
258
|
|
|
246
259
|
# :nodoc:
|
|
247
|
-
def silence(new_level = :error, &
|
|
248
|
-
SemanticLogger.silence(new_level, &
|
|
260
|
+
def silence(new_level = :error, &)
|
|
261
|
+
SemanticLogger.silence(new_level, &)
|
|
249
262
|
end
|
|
250
263
|
|
|
251
264
|
# :nodoc:
|
|
252
|
-
def fast_tag(tag, &
|
|
253
|
-
SemanticLogger.fast_tag(tag, &
|
|
265
|
+
def fast_tag(tag, &)
|
|
266
|
+
SemanticLogger.fast_tag(tag, &)
|
|
254
267
|
end
|
|
255
268
|
|
|
256
269
|
# Write log data to underlying data storage
|
|
@@ -263,6 +276,11 @@ module SemanticLogger
|
|
|
263
276
|
meets_log_level?(log) && !filtered?(log)
|
|
264
277
|
end
|
|
265
278
|
|
|
279
|
+
protected
|
|
280
|
+
|
|
281
|
+
# Only assigned to when building a child logger via `#tagged` (see #tagged_child).
|
|
282
|
+
attr_writer :instance_tags, :instance_named_tags
|
|
283
|
+
|
|
266
284
|
private
|
|
267
285
|
|
|
268
286
|
# Initializer for Abstract Class SemanticLogger::Base
|
|
@@ -294,8 +312,13 @@ module SemanticLogger
|
|
|
294
312
|
raise ":filter must be a Regexp, Proc, or implement :call"
|
|
295
313
|
end
|
|
296
314
|
|
|
297
|
-
@filter
|
|
298
|
-
@name
|
|
315
|
+
@filter = filter.is_a?(Regexp) ? filter.freeze : filter
|
|
316
|
+
@name = klass.is_a?(String) ? klass : klass.name
|
|
317
|
+
# `[].freeze` / `{}.freeze` compile to `opt_ary_freeze` / `opt_hash_freeze`, which return
|
|
318
|
+
# a shared frozen singleton. Every logger points at the same two objects, so these default
|
|
319
|
+
# (empty) instance tags allocate no garbage. Child loggers replace them via #tagged_child.
|
|
320
|
+
@instance_tags = [].freeze
|
|
321
|
+
@instance_named_tags = {}.freeze
|
|
299
322
|
if level.nil?
|
|
300
323
|
# Allow the global default level to determine this loggers log level
|
|
301
324
|
@level_index = nil
|
|
@@ -324,13 +347,59 @@ module SemanticLogger
|
|
|
324
347
|
(level_index <= (log.level_index || 0))
|
|
325
348
|
end
|
|
326
349
|
|
|
350
|
+
# Build a new Log entry, layering this logger's instance tags on top of the
|
|
351
|
+
# current thread context.
|
|
352
|
+
def new_log(level, index = nil)
|
|
353
|
+
apply_instance_tags(Log.new(name, level, index))
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
# Merge this logger's instance tags into the supplied Log entry.
|
|
357
|
+
#
|
|
358
|
+
# Composition (mirrors the per-logger merge rule proposed in PR #301):
|
|
359
|
+
# - Positional: thread tags first, then this logger's instance tags appended.
|
|
360
|
+
# - Named: instance named tags merged on top of the thread named tags, so on a
|
|
361
|
+
# key conflict the logger's own identity wins (it is the more specific value).
|
|
362
|
+
# Only this logger's entries are affected; the thread context is left untouched.
|
|
363
|
+
def apply_instance_tags(log)
|
|
364
|
+
unless @instance_tags.empty?
|
|
365
|
+
log.tags = log.tags.empty? ? @instance_tags.dup : log.tags + @instance_tags
|
|
366
|
+
end
|
|
367
|
+
unless @instance_named_tags.empty?
|
|
368
|
+
log.named_tags = log.named_tags.empty? ? @instance_named_tags.dup : log.named_tags.merge(@instance_named_tags)
|
|
369
|
+
end
|
|
370
|
+
log
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
# Returns a copy of this logger that permanently carries the supplied tags.
|
|
374
|
+
# Positional (string) and named (Hash) tags may be mixed; they are merged on
|
|
375
|
+
# top of any instance tags this logger already carries.
|
|
376
|
+
def tagged_child(tags)
|
|
377
|
+
positional = @instance_tags.dup
|
|
378
|
+
named = @instance_named_tags.dup
|
|
379
|
+
tags.flatten.each do |tag|
|
|
380
|
+
case tag
|
|
381
|
+
when Hash
|
|
382
|
+
named = named.merge(tag)
|
|
383
|
+
when nil, ""
|
|
384
|
+
# Ignore blanks for Rails compatibility, matching the block form.
|
|
385
|
+
else
|
|
386
|
+
positional << tag.to_s
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
child = clone
|
|
391
|
+
child.instance_tags = positional.freeze
|
|
392
|
+
child.instance_named_tags = named.freeze
|
|
393
|
+
child
|
|
394
|
+
end
|
|
395
|
+
|
|
327
396
|
# Log message at the specified level
|
|
328
|
-
def log_internal(level, index, message = nil, payload = nil, exception = nil)
|
|
397
|
+
def log_internal(level, index, message = nil, payload = nil, exception = nil, &block)
|
|
329
398
|
# Handle variable number of arguments by detecting exception object and payload hash.
|
|
330
399
|
if exception.nil? && payload.nil? && message.respond_to?(:backtrace) && message.respond_to?(:message)
|
|
331
400
|
exception = message
|
|
332
401
|
message = nil
|
|
333
|
-
elsif exception.nil? && payload
|
|
402
|
+
elsif exception.nil? && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
|
|
334
403
|
exception = payload
|
|
335
404
|
payload = nil
|
|
336
405
|
elsif payload && !payload.is_a?(Hash)
|
|
@@ -338,12 +407,12 @@ module SemanticLogger
|
|
|
338
407
|
payload = nil
|
|
339
408
|
end
|
|
340
409
|
|
|
341
|
-
log =
|
|
410
|
+
log = new_log(level, index)
|
|
342
411
|
should_log =
|
|
343
412
|
if exception.nil? && payload.nil? && message.is_a?(Hash)
|
|
344
413
|
# All arguments as a hash in the message.
|
|
345
414
|
log.assign(**log.extract_arguments(message))
|
|
346
|
-
elsif exception.nil? && message && payload
|
|
415
|
+
elsif exception.nil? && message && payload.is_a?(Hash)
|
|
347
416
|
# Message supplied along with a hash with the remaining arguments.
|
|
348
417
|
log.assign(**log.extract_arguments(payload, message))
|
|
349
418
|
else
|
|
@@ -352,20 +421,28 @@ module SemanticLogger
|
|
|
352
421
|
end
|
|
353
422
|
|
|
354
423
|
# Add result of block to message or payload if not nil
|
|
355
|
-
if
|
|
356
|
-
result = yield(log)
|
|
357
|
-
case result
|
|
358
|
-
when String
|
|
359
|
-
log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
|
|
360
|
-
when Hash
|
|
361
|
-
log.assign_hash(result)
|
|
362
|
-
end
|
|
363
|
-
end
|
|
424
|
+
apply_block_result(log, &block) if block
|
|
364
425
|
|
|
365
426
|
# Log level may change during assign due to :on_exception_level
|
|
366
427
|
self.log(log) if should_log && should_log?(log)
|
|
367
428
|
end
|
|
368
429
|
|
|
430
|
+
# Merge the result of a logging block into the log entry.
|
|
431
|
+
#
|
|
432
|
+
# Zero-arity blocks (e.g. `logger.info { "msg" }` or a zero-arity lambda) are called without
|
|
433
|
+
# arguments. Lambdas enforce their arity, so yielding the log to a zero-arity lambda would
|
|
434
|
+
# raise ArgumentError. Blocks that accept an argument still receive the log so they can read
|
|
435
|
+
# or mutate it.
|
|
436
|
+
def apply_block_result(log, &block)
|
|
437
|
+
result = block.arity.zero? ? block.call : block.call(log)
|
|
438
|
+
case result
|
|
439
|
+
when String
|
|
440
|
+
log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
|
|
441
|
+
when Hash
|
|
442
|
+
log.assign_hash(result)
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
|
|
369
446
|
# Measure the supplied block and log the message
|
|
370
447
|
def measure_internal(level, index, message, params)
|
|
371
448
|
exception = nil
|
|
@@ -389,7 +466,7 @@ module SemanticLogger
|
|
|
389
466
|
exception = e
|
|
390
467
|
ensure
|
|
391
468
|
# Must use ensure block otherwise a `return` in the yield above will skip the log entry
|
|
392
|
-
log =
|
|
469
|
+
log = new_log(level, index)
|
|
393
470
|
exception ||= params[:exception]
|
|
394
471
|
message = params[:message] if params[:message]
|
|
395
472
|
duration =
|
|
@@ -437,7 +514,7 @@ module SemanticLogger
|
|
|
437
514
|
rescue Exception => e
|
|
438
515
|
exception = e
|
|
439
516
|
ensure
|
|
440
|
-
log =
|
|
517
|
+
log = new_log(level, index)
|
|
441
518
|
# May return false due to elastic logging
|
|
442
519
|
should_log = log.assign(
|
|
443
520
|
message: message,
|
|
@@ -37,11 +37,11 @@ module SemanticLogger
|
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
# :nodoc:
|
|
40
|
-
def add(severity, message = nil, progname = nil, &
|
|
40
|
+
def add(severity, message = nil, progname = nil, &)
|
|
41
41
|
index = Levels.index(severity)
|
|
42
42
|
if level_index <= index
|
|
43
43
|
level = Levels.level(index)
|
|
44
|
-
log_internal(level, index, message, progname, &
|
|
44
|
+
log_internal(level, index, message, progname, &)
|
|
45
45
|
true
|
|
46
46
|
else
|
|
47
47
|
false
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
module CoreExt
|
|
3
|
+
# Reopen all appenders in the child process after a fork.
|
|
4
|
+
#
|
|
5
|
+
# Prepended onto `Process.singleton_class` when Semantic Logger is loaded.
|
|
6
|
+
# Enabled by default; opt out with `SemanticLogger.reopen_on_fork = false`.
|
|
7
|
+
#
|
|
8
|
+
# `Process._fork` (Ruby 3.1+) is the single method that every fork path routes
|
|
9
|
+
# through (`Kernel#fork`, `Process.fork`, `IO.popen`, `Kernel#system`, and
|
|
10
|
+
# backticks), so overriding it covers them all with one hook.
|
|
11
|
+
#
|
|
12
|
+
# Note: both `_fork` and `daemon` must be overridden to reliably restart the
|
|
13
|
+
# appender thread, since `Process.daemon` does not route through `_fork`
|
|
14
|
+
# (https://bugs.ruby-lang.org/issues/18911). Do not collapse this down to only
|
|
15
|
+
# `_fork`.
|
|
16
|
+
module Process
|
|
17
|
+
# `_fork` runs in both the parent and the child. It returns 0 in the child
|
|
18
|
+
# and the child's pid in the parent, so only reopen in the child. Reopening
|
|
19
|
+
# in the parent would needlessly recreate its queue and worker thread,
|
|
20
|
+
# risking the loss of messages still on the queue.
|
|
21
|
+
def _fork
|
|
22
|
+
child_pid = super
|
|
23
|
+
SemanticLogger.reopen if child_pid.zero? && SemanticLogger.reopen_on_fork?
|
|
24
|
+
child_pid
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# `Process.daemon` does not route through `_fork`, so reopen explicitly. Once
|
|
28
|
+
# it returns, the caller is running as the daemon (child) process.
|
|
29
|
+
def daemon(...)
|
|
30
|
+
super.tap { SemanticLogger.reopen if SemanticLogger.reopen_on_fork? }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -2,7 +2,21 @@ require "time"
|
|
|
2
2
|
module SemanticLogger
|
|
3
3
|
module Formatters
|
|
4
4
|
class Base
|
|
5
|
-
attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision
|
|
5
|
+
attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision,
|
|
6
|
+
:escape_control_chars
|
|
7
|
+
|
|
8
|
+
# Printable escapes for the most common control characters. Any other
|
|
9
|
+
# control character is escaped to its hexadecimal `\xHH` form by
|
|
10
|
+
# #escape_control_characters.
|
|
11
|
+
CONTROL_CHAR_ESCAPES = {
|
|
12
|
+
"\n" => "\\n",
|
|
13
|
+
"\r" => "\\r",
|
|
14
|
+
"\t" => "\\t",
|
|
15
|
+
"\e" => "\\e"
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
# Matches C0 control characters (including newlines and the ANSI escape) and DEL.
|
|
19
|
+
CONTROL_CHARS = /[\x00-\x1f\x7f]/
|
|
6
20
|
|
|
7
21
|
# Time precision varies by Ruby interpreter
|
|
8
22
|
# JRuby 9.1.8.0 supports microseconds
|
|
@@ -34,16 +48,27 @@ module SemanticLogger
|
|
|
34
48
|
# precision: [Integer]
|
|
35
49
|
# How many fractional digits to log times with.
|
|
36
50
|
# Default: PRECISION (6, except on older JRuby, where 3)
|
|
51
|
+
# escape_control_chars: [Boolean]
|
|
52
|
+
# Replace control characters (newlines, the ANSI escape, etc.) in
|
|
53
|
+
# untrusted log data (message, tags, named tags, and exception
|
|
54
|
+
# message) with a printable escaped form, e.g. "\n".
|
|
55
|
+
# This prevents log forging and terminal escape-sequence injection
|
|
56
|
+
# when logging data that may contain attacker-controlled content.
|
|
57
|
+
# Note: Has no effect on structured formatters such as :json, which
|
|
58
|
+
# already escape control characters via JSON encoding.
|
|
59
|
+
# Default: false (preserve newlines and ANSI colors in text output)
|
|
37
60
|
def initialize(time_format: nil,
|
|
38
61
|
log_host: true,
|
|
39
62
|
log_application: true,
|
|
40
63
|
log_environment: true,
|
|
41
|
-
precision: PRECISION
|
|
42
|
-
|
|
43
|
-
@
|
|
44
|
-
@
|
|
45
|
-
@
|
|
46
|
-
@
|
|
64
|
+
precision: PRECISION,
|
|
65
|
+
escape_control_chars: false)
|
|
66
|
+
@time_format = time_format || self.class.build_time_format(precision)
|
|
67
|
+
@log_host = log_host
|
|
68
|
+
@log_application = log_application
|
|
69
|
+
@log_environment = log_environment
|
|
70
|
+
@precision = precision
|
|
71
|
+
@escape_control_chars = escape_control_chars
|
|
47
72
|
end
|
|
48
73
|
|
|
49
74
|
# Return default time format string
|
|
@@ -68,6 +93,20 @@ module SemanticLogger
|
|
|
68
93
|
|
|
69
94
|
private
|
|
70
95
|
|
|
96
|
+
# When `escape_control_chars` is enabled, return a copy of the supplied
|
|
97
|
+
# value with any control characters replaced by a printable escaped form
|
|
98
|
+
# so that untrusted log data cannot forge log entries or inject terminal
|
|
99
|
+
# escape sequences. Otherwise the value is returned unchanged.
|
|
100
|
+
#
|
|
101
|
+
# Note: This escapes (preserves) control characters, and is opt-in. It is
|
|
102
|
+
# distinct from Log#cleansed_message, which unconditionally *strips* ANSI
|
|
103
|
+
# colorization from the message for structured (JSON/Loki) output.
|
|
104
|
+
def escape_control_characters(value)
|
|
105
|
+
return value unless escape_control_chars && value
|
|
106
|
+
|
|
107
|
+
value.to_s.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
|
|
108
|
+
end
|
|
109
|
+
|
|
71
110
|
# Return the Time as a formatted string
|
|
72
111
|
def format_time(time)
|
|
73
112
|
time = time.dup
|
|
@@ -85,7 +85,10 @@ module SemanticLogger
|
|
|
85
85
|
end
|
|
86
86
|
|
|
87
87
|
def tags
|
|
88
|
-
|
|
88
|
+
return if log.tags.nil? || log.tags.empty?
|
|
89
|
+
|
|
90
|
+
tags = log.tags.map { |tag| escape_control_characters(tag) }
|
|
91
|
+
"[#{color}#{tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]"
|
|
89
92
|
end
|
|
90
93
|
|
|
91
94
|
# Named Tags
|
|
@@ -94,7 +97,9 @@ module SemanticLogger
|
|
|
94
97
|
return if named_tags.nil? || named_tags.empty?
|
|
95
98
|
|
|
96
99
|
list = []
|
|
97
|
-
named_tags.each_pair
|
|
100
|
+
named_tags.each_pair do |name, value|
|
|
101
|
+
list << "#{color}#{escape_control_characters(name)}: #{escape_control_characters(value)}#{color_map.clear}"
|
|
102
|
+
end
|
|
98
103
|
"{#{list.join(', ')}}"
|
|
99
104
|
end
|
|
100
105
|
|
|
@@ -123,7 +128,8 @@ module SemanticLogger
|
|
|
123
128
|
def exception
|
|
124
129
|
return unless log.exception
|
|
125
130
|
|
|
126
|
-
"-- Exception: #{color}#{log.exception.class}:
|
|
131
|
+
"-- Exception: #{color}#{log.exception.class}: " \
|
|
132
|
+
"#{escape_control_characters(log.exception.message)}#{color_map.clear}\n#{log.backtrace_to_s}"
|
|
127
133
|
end
|
|
128
134
|
|
|
129
135
|
def call(log, logger)
|
|
@@ -32,7 +32,9 @@ module SemanticLogger
|
|
|
32
32
|
|
|
33
33
|
# Tags
|
|
34
34
|
def tags
|
|
35
|
-
|
|
35
|
+
return if log.tags.nil? || log.tags.empty?
|
|
36
|
+
|
|
37
|
+
"[#{log.tags.map { |tag| escape_control_characters(tag) }.join('] [')}]"
|
|
36
38
|
end
|
|
37
39
|
|
|
38
40
|
# Named Tags
|
|
@@ -41,7 +43,9 @@ module SemanticLogger
|
|
|
41
43
|
return if named_tags.nil? || named_tags.empty?
|
|
42
44
|
|
|
43
45
|
list = []
|
|
44
|
-
named_tags.each_pair
|
|
46
|
+
named_tags.each_pair do |name, value|
|
|
47
|
+
list << "#{escape_control_characters(name)}: #{escape_control_characters(value)}"
|
|
48
|
+
end
|
|
45
49
|
"{#{list.join(', ')}}"
|
|
46
50
|
end
|
|
47
51
|
|
|
@@ -57,7 +61,7 @@ module SemanticLogger
|
|
|
57
61
|
|
|
58
62
|
# Log message
|
|
59
63
|
def message
|
|
60
|
-
"-- #{log.message}" if log.message
|
|
64
|
+
"-- #{escape_control_characters(log.message)}" if log.message
|
|
61
65
|
end
|
|
62
66
|
|
|
63
67
|
# Payload
|
|
@@ -70,7 +74,10 @@ module SemanticLogger
|
|
|
70
74
|
|
|
71
75
|
# Exception
|
|
72
76
|
def exception
|
|
73
|
-
|
|
77
|
+
return unless log.exception
|
|
78
|
+
|
|
79
|
+
"-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n" \
|
|
80
|
+
"#{log.backtrace_to_s}"
|
|
74
81
|
end
|
|
75
82
|
|
|
76
83
|
# Default text log format
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
module SemanticLogger
|
|
3
|
+
module Formatters
|
|
4
|
+
# Formatter conforming to the Elastic Common Schema (ECS).
|
|
5
|
+
#
|
|
6
|
+
# Emits log events using the nested field names defined by ECS so that they
|
|
7
|
+
# integrate cleanly with Filebeat and the Elastic stack (Elasticsearch,
|
|
8
|
+
# Kibana) without requiring an ingest pipeline to rename fields.
|
|
9
|
+
#
|
|
10
|
+
# Usage:
|
|
11
|
+
# SemanticLogger.add_appender(io: $stdout, formatter: :ecs)
|
|
12
|
+
#
|
|
13
|
+
# # Route the payload, metric, and other SemanticLogger-specific data into
|
|
14
|
+
# # a custom top-level namespace (default "semantic_logger"):
|
|
15
|
+
# SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: "my_app"}})
|
|
16
|
+
#
|
|
17
|
+
# # Or merge the payload directly into ECS `labels` instead of a namespace:
|
|
18
|
+
# SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: nil}})
|
|
19
|
+
#
|
|
20
|
+
# == Field mapping (SemanticLogger -> ECS 8.x)
|
|
21
|
+
# time -> @timestamp (ISO-8601)
|
|
22
|
+
# level -> log.level
|
|
23
|
+
# name -> log.logger
|
|
24
|
+
# file_name / line -> log.origin.file.name / log.origin.file.line
|
|
25
|
+
# message -> message
|
|
26
|
+
# thread_name -> process.thread.name
|
|
27
|
+
# pid -> process.pid
|
|
28
|
+
# host -> host.hostname
|
|
29
|
+
# application -> service.name
|
|
30
|
+
# environment -> service.environment
|
|
31
|
+
# exception -> error.type / error.message / error.stack_trace
|
|
32
|
+
# duration -> event.duration (nanoseconds, as required by ECS)
|
|
33
|
+
# tags -> tags (ECS top-level array)
|
|
34
|
+
# named_tags -> labels.* (scalar key/value pairs)
|
|
35
|
+
# payload -> <namespace>.* (or labels.* when namespace is nil)
|
|
36
|
+
# metric/metric_amount -> <namespace>.metric / <namespace>.metric_amount
|
|
37
|
+
#
|
|
38
|
+
# == Reference
|
|
39
|
+
# * https://www.elastic.co/docs/reference/ecs
|
|
40
|
+
# * https://www.elastic.co/docs/reference/ecs/ecs-custom-fields-in-ecs
|
|
41
|
+
class Ecs < Raw
|
|
42
|
+
# ECS version this formatter targets.
|
|
43
|
+
ECS_VERSION = "8.11.0".freeze
|
|
44
|
+
|
|
45
|
+
# namespace: [String|Symbol|nil]
|
|
46
|
+
# Top-level field set used to hold SemanticLogger-specific data that has
|
|
47
|
+
# no native ECS home (payload, metric, metric_amount). A proper-noun
|
|
48
|
+
# namespace is guaranteed never to collide with a current or future ECS
|
|
49
|
+
# field. Set to nil to merge the payload into ECS `labels` instead.
|
|
50
|
+
# Default: "semantic_logger"
|
|
51
|
+
attr_reader :namespace
|
|
52
|
+
|
|
53
|
+
def initialize(namespace: "semantic_logger", time_format: :iso_8601, time_key: :timestamp, **args)
|
|
54
|
+
@namespace = namespace&.to_sym
|
|
55
|
+
|
|
56
|
+
super(time_format: time_format, time_key: time_key, **args)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Returns the log event as a single line of ECS-formatted JSON, so it can
|
|
60
|
+
# be written to stdout / a file and shipped by Filebeat or Elastic Agent.
|
|
61
|
+
def call(log, logger)
|
|
62
|
+
Utils.to_json(ecs_hash(super))
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Returns a batch of log events as a single JSON array.
|
|
66
|
+
def batch(logs, logger)
|
|
67
|
+
"[#{logs.map { |log| call(log, logger) }.join(',')}]"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
# Remap the flat hash built by Raw#call into the nested ECS field layout.
|
|
73
|
+
def ecs_hash(hash)
|
|
74
|
+
result = base(hash)
|
|
75
|
+
result[:process] = process(hash)
|
|
76
|
+
result[:host] = {hostname: hash[:host]} if hash[:host]
|
|
77
|
+
add_service(result, hash)
|
|
78
|
+
add_origin(result, hash)
|
|
79
|
+
add_event(result, hash)
|
|
80
|
+
result[:tags] = hash[:tags] if hash[:tags]
|
|
81
|
+
add_error(result, hash)
|
|
82
|
+
add_extras(result, hash)
|
|
83
|
+
result
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def base(hash)
|
|
87
|
+
log = {level: hash[:level].to_s}
|
|
88
|
+
log[:logger] = hash[:name] if hash[:name]
|
|
89
|
+
{
|
|
90
|
+
"@timestamp": hash[:timestamp],
|
|
91
|
+
message: hash[:message],
|
|
92
|
+
ecs: {version: ECS_VERSION},
|
|
93
|
+
log: log
|
|
94
|
+
}
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def process(hash)
|
|
98
|
+
result = {pid: hash[:pid]}
|
|
99
|
+
result[:thread] = {name: hash[:thread].to_s} if hash[:thread]
|
|
100
|
+
result
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def add_service(result, hash)
|
|
104
|
+
service = {}
|
|
105
|
+
service[:name] = hash[:application] if hash[:application]
|
|
106
|
+
service[:environment] = hash[:environment] if hash[:environment]
|
|
107
|
+
result[:service] = service unless service.empty?
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def add_origin(result, hash)
|
|
111
|
+
return unless hash[:file]
|
|
112
|
+
|
|
113
|
+
result[:log][:origin] = {file: {name: hash[:file], line: hash[:line]}.compact}
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# ECS event.duration is measured in nanoseconds.
|
|
117
|
+
def add_event(result, hash)
|
|
118
|
+
return unless hash[:duration_ms]
|
|
119
|
+
|
|
120
|
+
result[:event] = {duration: (hash[:duration_ms] * 1_000_000).round}
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def add_error(result, hash)
|
|
124
|
+
return unless hash[:exception]
|
|
125
|
+
|
|
126
|
+
result[:error] = {
|
|
127
|
+
type: hash[:exception][:name],
|
|
128
|
+
message: hash[:exception][:message],
|
|
129
|
+
stack_trace: Array(hash[:exception][:stack_trace]).join("\n")
|
|
130
|
+
}
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Place SemanticLogger-specific data (payload, metric) that has no native
|
|
134
|
+
# ECS home into the configured namespace, plus named_tags into labels.
|
|
135
|
+
def add_extras(result, hash)
|
|
136
|
+
labels = hash[:named_tags].is_a?(Hash) ? hash[:named_tags].dup : {}
|
|
137
|
+
|
|
138
|
+
extra = {}
|
|
139
|
+
extra[:payload] = hash[:payload] if hash[:payload]
|
|
140
|
+
extra[:metric] = hash[:metric] if hash[:metric]
|
|
141
|
+
extra[:metric_amount] = hash[:metric_amount] if hash[:metric_amount]
|
|
142
|
+
|
|
143
|
+
unless extra.empty?
|
|
144
|
+
namespace ? result[namespace] = extra : labels.merge!(extra)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
result[:labels] = labels unless labels.empty?
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
@@ -7,9 +7,9 @@ module SemanticLogger
|
|
|
7
7
|
class Fluentd < Json
|
|
8
8
|
attr_reader :need_process_info
|
|
9
9
|
|
|
10
|
-
def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, **args)
|
|
10
|
+
def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, log_host: false, **args)
|
|
11
11
|
@need_process_info = need_process_info
|
|
12
|
-
super(time_format: time_format, time_key: time_key, **args)
|
|
12
|
+
super(time_format: time_format, time_key: time_key, log_host: log_host, **args)
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
def level
|
|
@@ -17,8 +17,19 @@ module SemanticLogger
|
|
|
17
17
|
hash["severity_index"] = log.level_index
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
# Ignore process fields: pid, thread, file and line by default.
|
|
21
|
+
# These are rarely useful under Fluentd (e.g. containerized processes
|
|
22
|
+
# usually have pid 1), so they are only included when explicitly requested
|
|
23
|
+
# via `need_process_info: true`.
|
|
24
|
+
def pid
|
|
25
|
+
super if need_process_info
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def thread_name
|
|
29
|
+
super if need_process_info
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def file_name_and_line
|
|
22
33
|
super if need_process_info
|
|
23
34
|
end
|
|
24
35
|
end
|
|
@@ -9,7 +9,12 @@ module SemanticLogger
|
|
|
9
9
|
|
|
10
10
|
# Returns log messages in JSON format
|
|
11
11
|
def call(log, logger)
|
|
12
|
-
|
|
12
|
+
Utils.to_json(super)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Returns a batch of log messages as a single JSON array.
|
|
16
|
+
def batch(logs, logger)
|
|
17
|
+
"[#{logs.map { |log| call(log, logger) }.join(',')}]"
|
|
13
18
|
end
|
|
14
19
|
end
|
|
15
20
|
end
|
|
@@ -8,11 +8,13 @@ module SemanticLogger
|
|
|
8
8
|
# :timestamp, :level, :name, :message, :duration, :tags, :named_tags
|
|
9
9
|
#
|
|
10
10
|
# E.g.
|
|
11
|
-
# timestamp="2020-07-20T08:32:05.375276Z" level=info name="DefaultTest" base="breakfast"
|
|
11
|
+
# timestamp="2020-07-20T08:32:05.375276Z" level=info name="DefaultTest" base="breakfast"
|
|
12
|
+
# spaces="second breakfast" double_quotes="\"elevensies\"" single_quotes="'lunch'" tag="success"
|
|
12
13
|
#
|
|
13
14
|
# All timestamps are ISO8601 formatteed
|
|
14
15
|
# All user supplied values are escaped and surrounded by double quotes to avoid ambiguious message delimeters
|
|
15
|
-
# `tags` are treated as keys with boolean values. Tag names are not formatted or validated
|
|
16
|
+
# `tags` are treated as keys with boolean values. Tag names are not formatted or validated;
|
|
17
|
+
# ensure you use valid logfmt format for tag names.
|
|
16
18
|
# `named_tags` are flattened are merged into the top level message field. Any conflicting fields are overridden.
|
|
17
19
|
# `payload` values take precedence over `tags` and `named_tags`. Any conflicting fields are overridden.
|
|
18
20
|
#
|
|
@@ -64,9 +66,9 @@ module SemanticLogger
|
|
|
64
66
|
flattened = @parsed.map do |key, value|
|
|
65
67
|
case value
|
|
66
68
|
when Hash, Array
|
|
67
|
-
"#{key}=#{value.to_s
|
|
69
|
+
"#{key}=#{Utils.to_json(value.to_s)}"
|
|
68
70
|
else
|
|
69
|
-
"#{key}=#{
|
|
71
|
+
"#{key}=#{Utils.to_json(value)}"
|
|
70
72
|
end
|
|
71
73
|
end
|
|
72
74
|
|